@spooky-sync/core 0.0.1-canary.333 → 0.0.1-canary.335

package/src/state/lifecycle.ts

@@ -15,6 +15,17 @@ import type { QueryStatus } from '../types';
  * behind `status: 'fetching'`, `notified` says whether subscribers received
  * at least one materialization this registration.
  *
+ * `answered` says subscribers have been handed the rows of a known membership
+ * (the server's, or the durable one of an earlier session) at least once:
+ * a materialization ran past `cold`, with no body of a server set still to
+ * download, and the membership it drew was still the one in state when it
+ * committed. Only a re-seed (a new entry, a bucket switch) clears it, so a
+ * reconnect or a lost view does not send a painted query back to loading.
+ * It is what the bindings' `isAuthoritative()` mirrors (see `isAnswered`):
+ * a `cached` seed or a freshly applied server set alone is not enough,
+ * because `records` is still empty (or stale) until `materialize` paints, and
+ * "known and empty" read in that beat made apps create rows that existed.
+ *
  * `backgroundDepth` is the part of `fetchDepth` that re-verifies rows the
  * query already shows: a registration of a query that is not `cold`. It still
  * holds `status: 'fetching'` (results are not yet confirmed), but it is not
@@ -29,6 +40,7 @@ export interface QueryLifecycle {
   readonly fetchDepth: number;
   readonly backgroundDepth: number;
   readonly notified: boolean;
+  readonly answered: boolean;
 }
 
 export type LifecycleEvent =
@@ -41,7 +53,7 @@ export type LifecycleEvent =
   | { type: 'remote-dropped' }
   | { type: 'fetch-begin'; background?: boolean }
   | { type: 'fetch-end'; background?: boolean }
-  | { type: 'notified' }
+  | { type: 'notified'; answered?: boolean }
   | { type: 'bucket-switch'; resolvedBefore: boolean };
 
 export class LifecycleError extends Error {
@@ -52,7 +64,14 @@ export class LifecycleError extends Error {
 }
 
 export function seedLifecycle(resolvedBefore: boolean): QueryLifecycle {
-  return { phase: resolvedBefore ? 'cached' : 'cold', remote: 'unregistered', fetchDepth: 0, backgroundDepth: 0, notified: false };
+  return {
+    phase: resolvedBefore ? 'cached' : 'cold',
+    remote: 'unregistered',
+    fetchDepth: 0,
+    backgroundDepth: 0,
+    notified: false,
+    answered: false,
+  };
 }
 
 /** Total on every (lifecycle, event) pair the sagas can produce; throws on the rest. */
@@ -84,14 +103,23 @@ export function transition(l: QueryLifecycle, ev: LifecycleEvent): QueryLifecycl
       const backgroundDepth = Math.min(fetchDepth, l.backgroundDepth - (ev.background && l.backgroundDepth > 0 ? 1 : 0));
       return { ...l, fetchDepth, backgroundDepth };
     }
-    case 'notified':
-      return l.notified ? l : { ...l, notified: true };
+    case 'notified': {
+      // A cold render is a guess from the local store, never an answer.
+      const answered = l.answered || (ev.answered === true && l.phase !== 'cold');
+      return l.notified && answered === l.answered ? l : { ...l, notified: true, answered };
+    }
     case 'bucket-switch':
       return { ...seedLifecycle(ev.resolvedBefore), fetchDepth: 0 };
   }
 }
 
+/** Membership is known (server set or durable seed); its rows may not be painted yet. */
 export const isAuthoritative = (l: QueryLifecycle): boolean => l.phase !== 'cold';
+/**
+ * Subscribers hold the rows of a known membership: what the authority channel
+ * (`subscribeQueryAuthority`, the bindings' `isAuthoritative()`) reports.
+ */
+export const isAnswered = (l: QueryLifecycle): boolean => l.answered;
 export const hasServerMembership = (l: QueryLifecycle): boolean =>
   l.phase === 'live' || l.phase === 'view-lost';
 export const deriveStatus = (l: QueryLifecycle): QueryStatus => (l.fetchDepth > 0 ? 'fetching' : 'idle');

package/src/state/reducers.test.ts

@@ -79,6 +79,14 @@ describe('query reducers', () => {
     expect(s2.queries.get('a')!.telemetry.phaseSamples.localFetch).toEqual([12]);
   });
 
+  it('setRecords answers a query only when told to, and never a cold one', () => {
+    const cold = buildState([e('a')]);
+    expect(R.setRecords('a', [], false, null, true)(cold).queries.get('a')!.lifecycle.answered).toBe(false);
+    const cached = buildState([e('a', { lifecycle: { phase: 'cached' } })]);
+    expect(R.setRecords('a', [], false, null)(cached).queries.get('a')!.lifecycle).toMatchObject({ notified: true, answered: false });
+    expect(R.setRecords('a', [], false, null, true)(cached).queries.get('a')!.lifecycle).toMatchObject({ notified: true, answered: true });
+  });
+
   it('sample windows are capped', () => {
     let s = buildState([e('a')]);
     for (let i = 0; i < 105; i++) s = R.recordPhase('a', 'localFetch', i)(s);
@@ -200,6 +208,9 @@ describe('outbox', () => {
     const s3 = R.outboxBumpAttempts('m1')(s2);
     expect(s3.outbox[0].attempts).toBe(1);
     expect(R.outboxBumpAttempts('nope')(s3)).toBe(s3);
+    const doubt = R.outboxMarkInDoubt(['m1', 'nope'])(s3);
+    expect(doubt.outbox[0]).toMatchObject({ id: 'm1', inDoubt: true, attempts: 1 });
+    expect(R.outboxMarkInDoubt(['m1'])(doubt)).toBe(doubt);
     const s4 = R.outboxRemove('m1')(s3);
     expect(s4.outbox).toEqual([]);
     expect(s4.dirty.has('a')).toBe(true);
@@ -249,12 +260,14 @@ describe('bucket switch reducers', () => {
       R.outboxReplace([buildOutboxItem()])
     );
     const id = { table: '_00_query', id: 'new' } as any;
-    const s1 = R.rebindQuery('a', { id, lifecycle: { phase: 'cached', remote: 'unregistered', fetchDepth: 0, backgroundDepth: 0, notified: false }, remoteArray: [['t:2', 1]], localArray: [] })({ ...s0, dirty: new Set() });
+    const s1 = R.rebindQuery('a', { id, lifecycle: { phase: 'cached', remote: 'unregistered', fetchDepth: 0, backgroundDepth: 0, notified: false, answered: false }, remoteArray: [['t:2', 1]], localArray: [] })({ ...s0, dirty: new Set() });
     const en = s1.queries.get('a')!;
     expect(en.def.id).toBe(id);
     expect(en.lifecycle.phase).toBe('cached');
     expect(en.remoteArray).toEqual([['t:2', 1]]);
-    expect(en.records).toEqual([]);
+    // What subscribers still hold: the next render diffs against it, so a new
+    // store that renders `[]` is a change they hear about.
+    expect(en.records).toEqual([{ id: 't:1' }]);
     expect(en.serverState).toBeNull();
     expect(en.registerAttempts).toBe(0);
     expect(s1.dirty.has('a')).toBe(true);

package/src/state/reducers.ts

@@ -153,8 +153,13 @@ export const setSubqueryRemoteArray =
     return withView(next, entry.def.viewKey, vouchedIds(entry.remoteArray, subqueryRemoteArray));
   };
 
+/**
+ * Commit a materialization. `answered`: it drew a known membership with no
+ * body still to download, and that membership is still current (see
+ * `QueryLifecycle`).
+ */
 export const setRecords =
-  (hash: QueryHash, records: ReadonlyArray<Row>, changed: boolean, materializeMs: number | null): Reducer =>
+  (hash: QueryHash, records: ReadonlyArray<Row>, changed: boolean, materializeMs: number | null, answered = false): Reducer =>
   (s) => {
     const next = withEntry(s, hash, (e) => {
       const prev = e.telemetry.phaseSamples.localFetch ?? [];
@@ -171,7 +176,7 @@ export const setRecords =
         ...e,
         records: changed ? records : e.records,
         telemetry,
-        lifecycle: transition(e.lifecycle, { type: 'notified' }),
+        lifecycle: transition(e.lifecycle, { type: 'notified', answered }),
       };
     });
     return next === s ? s : { ...next, dirty: withoutAll(next.dirty, [hash]) };
@@ -403,6 +408,15 @@ export const outboxBumpAttempts =
     return { ...s, outbox };
   };
 
+/** Mark items whose last push may have committed although no answer came back. */
+export const outboxMarkInDoubt =
+  (ids: readonly string[]): Reducer =>
+  (s) => {
+    const marked = new Set(ids);
+    if (!s.outbox.some((i) => marked.has(i.id) && !i.inDoubt)) return s;
+    return { ...s, outbox: s.outbox.map((i) => (marked.has(i.id) && !i.inDoubt ? { ...i, inDoubt: true } : i)) };
+  };
+
 /** Drop acked items older than `graceMs` (membership never named them). */
 export const outboxPruneAcked =
   (now: number, graceMs: number): Reducer =>
@@ -454,7 +468,13 @@ export const setIdentity =
   ): Reducer =>
   (s) => ({ ...s, ...patch });
 
-/** Re-home a query after a bucket switch: new store, same hash, fresh sync state. */
+/**
+ * Re-home a query after a bucket switch: new store, same hash, fresh sync state.
+ * `records` is kept: it is what subscribers were last handed, and the next
+ * render diffs against it. Blanking it here (without telling anyone) let a
+ * new store that renders `[]` read as unchanged, so the previous principal's
+ * rows stayed on screen under an authority flip that vouched for them.
+ */
 export const rebindQuery =
   (
     hash: QueryHash,
@@ -468,7 +488,6 @@ export const rebindQuery =
       remoteArray: next.remoteArray,
       localArray: next.localArray,
       subqueryRemoteArray: [],
-      records: [],
       serverState: null,
       lastHeartbeatAt: null,
       lastPolledAt: null,

package/src/state/selectors.ts

@@ -217,6 +217,7 @@ export function toQueryState(e: QueryEntry): QueryState {
     viewLost: l.phase === 'view-lost',
     serverState: e.serverState,
     syncNotified: l.notified,
+    answered: l.answered,
     ttlTimer: null,
     ttlDurationMs: e.def.ttlMs,
     lastHeartbeatAt: e.lastHeartbeatAt ?? undefined,

package/src/types.ts

@@ -706,6 +706,12 @@ export interface QueryState {
    * always emits at least once even when its records are unchanged — otherwise
    * an empty re-registered window would never notify and stay "loading". */
   syncNotified?: boolean;
+  /**
+   * Subscribers hold the rows of a known membership (see `QueryLifecycle`'s
+   * `answered`): what the authority channel and the bindings'
+   * `isAuthoritative()` report. Ephemeral.
+   */
+  answered?: boolean;
   /** Timer for TTL expiration. */
   ttlTimer: NodeJS.Timeout | null;
   /** TTL duration in milliseconds. */
@@ -798,9 +804,12 @@ export interface QueryTimings {
 export type QueryUpdateCallback = (records: Record<string, any>[]) => void;
 export type QueryStatusCallback = (status: QueryStatus) => void;
 /**
- * Observer for a query's authority flips: `true` once server membership is
- * known for it (registration, poll, durable seed), `false` when it is reset
- * (bucket switch). What the bindings' `isAuthoritative()` mirrors.
+ * Observer for a query's authority flips: `true` once subscribers hold the
+ * rendered rows of a known membership (a server answer whose bodies are
+ * local, or the durable membership of an earlier session), delivered after
+ * the rows that vouch for it; `false` when it is reset (bucket switch). Knowing the set is
+ * not enough: a `cached` re-seed or a just-applied server set has no rows
+ * painted yet. What the bindings' `isAuthoritative()` mirrors.
  */
 export type QueryAuthorityCallback = (known: boolean) => void;
 export type MutationCallback = (mutations: MutationEvent[]) => void;

package/src/utils/error-classification.test.ts

@@ -39,10 +39,43 @@ describe('classifySyncError', () => {
     expect(classifySyncError(new Error(message))).toBe('network');
   });
 
+  // Verbatim from SurrealDB 3.1.5 and 3.3.4: the write was refused because
+  // sp00ky's own ingest event could not reach the scheduler / SSP. The outbox
+  // must keep these queued, never roll them back (see ingest-outage.integration.test.ts).
+  it.each([
+    // Refused connection and a post past the event TIMEOUT read the same.
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: error sending request for url (http://scheduler:9667/ingest)',
+    'Error while processing event _00_item_delete: There was an error processing a remote HTTP request: error sending request for url (http://host.docker.internal:50466/ingest)',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 500 Internal Server Error',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 502 Bad Gateway',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 503 Service Unavailable',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 408 Request Timeout',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 429 Too Many Requests',
+    'Error while processing event _00_item_mutation: failed to lookup address information: Name or service not known',
+    // An app event that wrote to a synced table carries the inner event.
+    'Error while processing event app_fanout: Error while processing event _00_b_mutation: There was an error processing a remote HTTP request: 503 Service Unavailable',
+  ])('classifies a failed sp00ky ingest post %j as network (retry, never rollback)', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
   it.each([
     'Permission denied',
+    // Verbatim from SurrealDB 3.3.4: a record user running a statement it may not.
+    'IAM error: Not enough permissions to perform this action',
     'There was a problem with the database: record already exists',
     'Parse error: unexpected token',
+    "Couldn't coerce value for field `title` of `item:1`: Expected `string` but found `1`",
+    // An app's own event: its endpoint, its THROW, its business.
+    'Error while processing event notify_crm: There was an error processing a remote HTTP request: 500 Internal Server Error',
+    'Error while processing event notify_crm: There was an error processing a remote HTTP request: error sending request for url (http://crm.example/hook)',
+    'Error while processing event cf_guard: An error occurred: conformance guard: write refused',
+    // A THROW inside a sp00ky event is not a failed post.
+    'Error while processing event _00_item_mutation: An error occurred: inner',
+    // A 4xx from /ingest no retry fixes: payload rejected, too large, secret mismatch.
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 400 Bad Request',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 401 Unauthorized',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 413 Payload Too Large',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 422 Unprocessable Entity',
   ])('classifies %j as application', (message) => {
     expect(classifySyncError(new Error(message))).toBe('application');
   });

package/src/utils/error-classification.ts

@@ -39,10 +39,37 @@ const NETWORK_ERROR_PATTERNS = [
   'specify a database',
 ];
 
+/**
+ * A write SurrealDB refused because sp00ky's OWN ingest post failed: every
+ * synced table's `_00_<table>_mutation` / `_00_<table>_delete` event posts to
+ * the scheduler or SSP inside the write's transaction (`sp00ky.rs::ingest_post`),
+ * so a scheduler that is down, slow past the event TIMEOUT, or answering 5xx
+ * aborts the write. That is an outage, not the mutation's fault, and it clears
+ * on its own; classified `application` it was rolled back and discarded, so
+ * every write queued during a scheduler restart or a cold SSP was lost.
+ *
+ * Wording on SurrealDB 3.1.5 and 3.3.4 (the event name may be nested inside an
+ * app event that wrote to a synced table):
+ *   "Error while processing event _00_item_mutation: There was an error
+ *    processing a remote HTTP request: error sending request for url (...)"
+ *    (refused connection or TIMEOUT), "...: 503 Service Unavailable" (5xx),
+ *   "Error while processing event _00_item_mutation: failed to lookup address
+ *    information: ..." (DNS).
+ *
+ * Only `_00_` events count: an app's own event posting to its own endpoint
+ * stays `application`, as does a 4xx from `/ingest` other than 408/429 (a
+ * malformed or oversized payload, a secret mismatch), which no retry fixes and
+ * which would otherwise hold the outbox's serial lane forever.
+ */
+const SP00KY_EVENT_POST_FAILED =
+  /error while processing event _00_[^:\s]*: (?:there was an error processing a remote http request: (?:error sending request|5\d\d\b|408\b|429\b)|failed to lookup address information)/;
+
 export function classifySyncError(error: unknown): 'network' | 'application' {
   const message =
     error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
 
+  if (SP00KY_EVENT_POST_FAILED.test(message)) return 'network';
+
   for (const pattern of NETWORK_ERROR_PATTERNS) {
     if (message.includes(pattern)) {
       return 'network';