@spooky-sync/core 0.0.1-canary.334 → 0.0.1-canary.335

package/src/mutation/rows.test.ts

@@ -60,6 +60,8 @@ describe('parsePendingRow', () => {
       status: 'pending',
       ackedAt: null,
       attempts: 0,
+      // A row read back from the store may have been sent by a session that died.
+      inDoubt: true,
     });
     expect(rows.loadPendingRows()).toBe('SELECT * FROM _00_pending_mutations ORDER BY id ASC');
     expect(rows.loadFailedRows()).toBe('SELECT * FROM _00_failed_mutations ORDER BY failedAt ASC');
@@ -143,7 +145,14 @@ describe('remoteBatch', () => {
       { id: 'm3', mutationType: 'delete', recordId: 'thing:3', tableName: 'thing', createdAt: 0, v: 2 },
       { id: 'm4', mutationType: 'update', recordId: 'thing:4', tableName: 'thing', createdAt: 0, v: 2 },
     ]);
-    expect(sql).toBe('CREATE ONLY $id0 SET a = $d0_a;\nUPDATE $id1 MERGE $data1;\nDELETE $id2;\nUPDATE $id3 MERGE $data3');
+    // Chained: each statement runs only if the one before it committed.
+    const skip = `IF !$ok0 { THROW "${rows.BATCH_SKIPPED}" }; `;
+    expect(sql.split(';\n')).toEqual([
+      'LET $ok0 = { CREATE ONLY $id0 SET a = $d0_a; true }',
+      `LET $ok1 = { ${skip}UPDATE $id1 MERGE $data1; true }`,
+      `LET $ok2 = { ${skip.replace('$ok0', '$ok1')}DELETE $id2; true }`,
+      `LET $ok3 = { ${skip.replace('$ok0', '$ok2')}UPDATE $id3 MERGE $data3; true }`,
+    ]);
     expect(vars).toEqual({
       id0: rid('thing', '1'),
       d0_a: 1,
@@ -153,7 +162,45 @@ describe('remoteBatch', () => {
       id3: rid('thing', '4'),
       data3: {},
     });
-    expect(rows.remoteBatch([{ id: 'm', mutationType: 'create', recordId: 'thing:9', tableName: 'thing', createdAt: 0, v: 2 }]).sql).toBe('CREATE ONLY $id0 SET ');
+    expect(rows.remoteBatch([{ id: 'm', mutationType: 'create', recordId: 'thing:9', tableName: 'thing', createdAt: 0, v: 2 }]).sql).toBe('LET $ok0 = { CREATE ONLY $id0 SET ; true }');
+  });
+
+  it('an in-doubt CREATE is a no-op when the server row matches every field no later in-doubt row of it may have changed', () => {
+    const row = (id: string, mutationType: rows.PendingMutationRow['mutationType'], recordId: string, data?: Record<string, unknown>): rows.PendingMutationRow => ({
+      id,
+      mutationType,
+      recordId,
+      tableName: recordId.split(':')[0],
+      data,
+      createdAt: 0,
+      v: 2,
+    });
+    const was = (i: number, ...keys: string[]) =>
+      `LET $was${i} = (SELECT * FROM ONLY $id${i}); IF ${[`($was${i} = NONE)`, ...keys.map((k) => `($was${i}.${k} != $d${i}_${k})`)].join(' OR ')} { CREATE ONLY $id${i} SET ${['a', 'b']
+        .map((k) => `${k} = $d${i}_${k}`)
+        .join(', ')} }`;
+    const statement = (sql: string, i: number) => sql.split(';\n')[i].replace(/^LET \$ok\d+ = \{ (IF !\$ok\d+ \{ THROW "[^"]+" \}; )?/, '').replace(/; true \}$/, '');
+    const batch = [
+      row('m1', 'create', 'thing:1', { a: 1, b: 1 }),
+      row('m2', 'update', 'thing:1', { b: 2 }),
+      row('m3', 'create', 'thing:2', { a: 1, b: 1 }),
+      row('m4', 'update', 'thing:2', { a: 2 }), // not in doubt: it never ran
+      row('m5', 'create', 'thing:3', { a: 1, b: 1 }),
+      row('m6', 'delete', 'thing:3'),
+      row('m7', 'create', 'thing:4', { a: 1, b: 1 }),
+      row('m8', 'update', 'thing:9', { a: 2 }), // another record
+      row('m9', 'update', 'thing:1'), // no payload: changes nothing
+    ];
+    const { sql } = rows.remoteBatch(batch, new Set(['m1', 'm2', 'm3', 'm5', 'm6', 'm8', 'm9']));
+    expect(statement(sql, 0)).toBe(was(0, 'a')); // b may be m2's
+    expect(statement(sql, 2)).toBe(was(2, 'a', 'b'));
+    expect(statement(sql, 4)).toBe(was(4)); // m6 may have deleted it and something else re-created it
+    expect(statement(sql, 6)).toBe('CREATE ONLY $id6 SET a = $d6_a, b = $d6_b'); // not in doubt: strict
+    // A CRDT log row's id is fresh and the row immutable: existence is proof, in doubt or not.
+    expect(statement(rows.remoteBatch([row('l', 'create', 'doc__crdt_log:1', { a: 1, b: 1 })]).sql, 0)).toBe(was(0));
+    expect(statement(rows.remoteBatch([row('e', 'create', 'thing:5')], new Set(['e'])).sql, 0)).toBe(
+      'LET $was0 = (SELECT * FROM ONLY $id0); IF ($was0 = NONE) { CREATE ONLY $id0 SET  }'
+    );
   });
 });
 

package/src/mutation/rows.ts

@@ -109,6 +109,11 @@ export function hydrateRowData(row: PendingMutationRow, columns: Record<string,
   }
 }
 
+/**
+ * A row read back from the store at boot, bucket switch or leader takeover.
+ * The session that queued it may have sent it and died before the answer (a
+ * closed tab, a killed app, a lost leader), so its outcome is in doubt.
+ */
 export const toOutboxItem = (row: PendingMutationRow): OutboxItem => ({
   id: row.id,
   type: row.mutationType,
@@ -117,6 +122,7 @@ export const toOutboxItem = (row: PendingMutationRow): OutboxItem => ({
   status: 'pending',
   ackedAt: null,
   attempts: 0,
+  inDoubt: true,
 });
 
 /**
@@ -286,29 +292,95 @@ export interface RemoteBatch {
 }
 
 /**
- * Many outbox rows as ONE multi-statement request. Statement i answers row i.
- * Each statement is its own transaction on the server, as before.
+ * What a statement of a `remoteBatch` throws instead of running, because an
+ * earlier statement in the same request failed.
  */
-export function remoteBatch(rows: ReadonlyArray<PendingMutationRow>): RemoteBatch {
-  const vars: Vars = {};
-  const stmts = rows.map((row, i) => {
-    const rid = parseRecordIdString(row.recordId);
-    vars[`id${i}`] = rid;
-    switch (row.mutationType) {
-      case 'create': {
-        const data = row.data ?? {};
-        const sets = Object.keys(data).map((key) => {
-          vars[`d${i}_${key}`] = data[key];
-          return `${key} = $d${i}_${key}`;
-        });
-        return `CREATE ONLY $id${i} SET ${sets.join(', ')}`;
-      }
-      case 'update':
-        vars[`data${i}`] = row.data ?? {};
-        return `UPDATE $id${i} MERGE $data${i}`;
-      case 'delete':
-        return `DELETE $id${i}`;
+export const BATCH_SKIPPED = 'sp00ky: not run, an earlier statement in this batch failed';
+
+/** A CRDT edit-log row (`<table>__crdt_log`): written once under a fresh id, never changed. */
+const isLogTable = (table: string): boolean => table.endsWith('__crdt_log');
+
+/**
+ * Which fields of the row already on the server must equal the payload of the
+ * CREATE at `i` for that row to count as this CREATE's own earlier commit, or
+ * `null` when the CREATE must not tolerate an existing row at all.
+ *
+ * Only a CREATE whose outcome is in doubt tolerates one: elsewhere "already
+ * exists" means someone else holds the id, a rejection as before. The check
+ * is every payload field except those a later in-doubt row of the same
+ * record, which may have committed in the same lost round, could have changed
+ * since; those rows run again after it and set them anyway. A later in-doubt
+ * DELETE or CREATE of the record may have replaced the row, so then existence
+ * is all that is checked. Rows that are not in doubt never ran.
+ *
+ * A CRDT log row's id is fresh and the row immutable, so its existence is
+ * proof enough, in doubt or not.
+ */
+function matchFields(rows: ReadonlyArray<PendingMutationRow>, i: number, inDoubt: ReadonlySet<string>): string[] | null {
+  const row = rows[i];
+  if (isLogTable(row.tableName)) return [];
+  if (!inDoubt.has(row.id)) return null;
+  const changed = new Set<string>();
+  for (const later of rows.slice(i + 1)) {
+    if (later.recordId !== row.recordId || !inDoubt.has(later.id)) continue;
+    if (later.mutationType !== 'update') return [];
+    for (const key of Object.keys(later.data ?? {})) changed.add(key);
+  }
+  return Object.keys(row.data ?? {}).filter((key) => !changed.has(key));
+}
+
+function remoteStatement(rows: ReadonlyArray<PendingMutationRow>, i: number, vars: Vars, inDoubt: ReadonlySet<string>): string {
+  const row = rows[i];
+  vars[`id${i}`] = parseRecordIdString(row.recordId);
+  switch (row.mutationType) {
+    case 'create': {
+      const data = row.data ?? {};
+      const sets = Object.keys(data).map((key) => {
+        vars[`d${i}_${key}`] = data[key];
+        return `${key} = $d${i}_${key}`;
+      });
+      const create = `CREATE ONLY $id${i} SET ${sets.join(', ')}`;
+      const match = matchFields(rows, i, inDoubt);
+      if (!match) return create;
+      // The CREATE runs unless the row is there and matches; on a mismatch it
+      // fails with SurrealDB's own "already exists", a rejection as before.
+      const differs = [`($was${i} = NONE)`, ...match.map((key) => `($was${i}.${key} != $d${i}_${key})`)];
+      return `LET $was${i} = (SELECT * FROM ONLY $id${i}); IF ${differs.join(' OR ')} { ${create} }`;
     }
+    case 'update':
+      vars[`data${i}`] = row.data ?? {};
+      return `UPDATE $id${i} MERGE $data${i}`;
+    case 'delete':
+      return `DELETE $id${i}`;
+  }
+}
+
+/**
+ * Many outbox rows as ONE multi-statement request. Statement i answers row i
+ * and is its own transaction on the server, as before, but the statements are
+ * chained: statement i binds `$ok<i>` only when it commits, and statement i+1
+ * throws `BATCH_SKIPPED` without touching anything unless `$ok<i>` is bound
+ * (a LET lives for one request only). So the server stops at the first
+ * failure: everything before it is committed, nothing after it ran.
+ *
+ * Without the chain, statements after a failed one ran anyway. A retryable
+ * failure at i left the rows after i queued although they had committed; the
+ * re-sent CREATE then read "already exists", was taken for a rejection and
+ * rolled back locally while the server kept the row. Acking them instead
+ * would have let a later write land before an earlier one it may depend on.
+ *
+ * The chain cannot help when the answer itself is lost: the request may have
+ * committed any prefix of the batch. Rows named in `inDoubt` (mutation ids)
+ * are such rows. An UPDATE MERGE or a DELETE is safe to run twice, a CREATE
+ * is not, so an in-doubt CREATE checks the row already there first (see
+ * `matchFields`) and is a no-op when it is its own: the batch goes on in
+ * order, with no "already exists" to cut it.
+ */
+export function remoteBatch(rows: ReadonlyArray<PendingMutationRow>, inDoubt: ReadonlySet<string> = new Set()): RemoteBatch {
+  const vars: Vars = {};
+  const stmts = rows.map((_, i) => {
+    const guard = i === 0 ? '' : `IF !$ok${i - 1} { THROW "${BATCH_SKIPPED}" }; `;
+    return `LET $ok${i} = { ${guard}${remoteStatement(rows, i, vars, inDoubt)}; true }`;
   });
   return { sql: stmts.join(';\n'), vars };
 }

package/src/state/client-state.ts

@@ -79,6 +79,14 @@ export interface OutboxItem {
   readonly status: OutboxStatus;
   readonly ackedAt: number | null;
   readonly attempts: number;
+  /**
+   * A request carrying this row may have committed without the answer
+   * reaching this client: it threw a transport error, or the row was loaded
+   * from the store, so a session that died mid-push may have sent it. Its
+   * CREATE then accepts the row already on the server when that row holds
+   * what it would write (see `remoteBatch`). Absent means false.
+   */
+  readonly inDoubt?: boolean;
 }
 
 export type TabRole = 'solo' | 'leader' | 'follower';

package/src/state/reducers.test.ts

@@ -208,6 +208,9 @@ describe('outbox', () => {
     const s3 = R.outboxBumpAttempts('m1')(s2);
     expect(s3.outbox[0].attempts).toBe(1);
     expect(R.outboxBumpAttempts('nope')(s3)).toBe(s3);
+    const doubt = R.outboxMarkInDoubt(['m1', 'nope'])(s3);
+    expect(doubt.outbox[0]).toMatchObject({ id: 'm1', inDoubt: true, attempts: 1 });
+    expect(R.outboxMarkInDoubt(['m1'])(doubt)).toBe(doubt);
     const s4 = R.outboxRemove('m1')(s3);
     expect(s4.outbox).toEqual([]);
     expect(s4.dirty.has('a')).toBe(true);

package/src/state/reducers.ts

@@ -408,6 +408,15 @@ export const outboxBumpAttempts =
     return { ...s, outbox };
   };
 
+/** Mark items whose last push may have committed although no answer came back. */
+export const outboxMarkInDoubt =
+  (ids: readonly string[]): Reducer =>
+  (s) => {
+    const marked = new Set(ids);
+    if (!s.outbox.some((i) => marked.has(i.id) && !i.inDoubt)) return s;
+    return { ...s, outbox: s.outbox.map((i) => (marked.has(i.id) && !i.inDoubt ? { ...i, inDoubt: true } : i)) };
+  };
+
 /** Drop acked items older than `graceMs` (membership never named them). */
 export const outboxPruneAcked =
   (now: number, graceMs: number): Reducer =>

package/src/utils/error-classification.test.ts

@@ -39,10 +39,43 @@ describe('classifySyncError', () => {
     expect(classifySyncError(new Error(message))).toBe('network');
   });
 
+  // Verbatim from SurrealDB 3.1.5 and 3.3.4: the write was refused because
+  // sp00ky's own ingest event could not reach the scheduler / SSP. The outbox
+  // must keep these queued, never roll them back (see ingest-outage.integration.test.ts).
+  it.each([
+    // Refused connection and a post past the event TIMEOUT read the same.
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: error sending request for url (http://scheduler:9667/ingest)',
+    'Error while processing event _00_item_delete: There was an error processing a remote HTTP request: error sending request for url (http://host.docker.internal:50466/ingest)',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 500 Internal Server Error',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 502 Bad Gateway',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 503 Service Unavailable',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 408 Request Timeout',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 429 Too Many Requests',
+    'Error while processing event _00_item_mutation: failed to lookup address information: Name or service not known',
+    // An app event that wrote to a synced table carries the inner event.
+    'Error while processing event app_fanout: Error while processing event _00_b_mutation: There was an error processing a remote HTTP request: 503 Service Unavailable',
+  ])('classifies a failed sp00ky ingest post %j as network (retry, never rollback)', (message) => {
+    expect(classifySyncError(new Error(message))).toBe('network');
+  });
+
   it.each([
     'Permission denied',
+    // Verbatim from SurrealDB 3.3.4: a record user running a statement it may not.
+    'IAM error: Not enough permissions to perform this action',
     'There was a problem with the database: record already exists',
     'Parse error: unexpected token',
+    "Couldn't coerce value for field `title` of `item:1`: Expected `string` but found `1`",
+    // An app's own event: its endpoint, its THROW, its business.
+    'Error while processing event notify_crm: There was an error processing a remote HTTP request: 500 Internal Server Error',
+    'Error while processing event notify_crm: There was an error processing a remote HTTP request: error sending request for url (http://crm.example/hook)',
+    'Error while processing event cf_guard: An error occurred: conformance guard: write refused',
+    // A THROW inside a sp00ky event is not a failed post.
+    'Error while processing event _00_item_mutation: An error occurred: inner',
+    // A 4xx from /ingest no retry fixes: payload rejected, too large, secret mismatch.
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 400 Bad Request',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 401 Unauthorized',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 413 Payload Too Large',
+    'Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: 422 Unprocessable Entity',
   ])('classifies %j as application', (message) => {
     expect(classifySyncError(new Error(message))).toBe('application');
   });

package/src/utils/error-classification.ts

@@ -39,10 +39,37 @@ const NETWORK_ERROR_PATTERNS = [
   'specify a database',
 ];
 
+/**
+ * A write SurrealDB refused because sp00ky's OWN ingest post failed: every
+ * synced table's `_00_<table>_mutation` / `_00_<table>_delete` event posts to
+ * the scheduler or SSP inside the write's transaction (`sp00ky.rs::ingest_post`),
+ * so a scheduler that is down, slow past the event TIMEOUT, or answering 5xx
+ * aborts the write. That is an outage, not the mutation's fault, and it clears
+ * on its own; classified `application` it was rolled back and discarded, so
+ * every write queued during a scheduler restart or a cold SSP was lost.
+ *
+ * Wording on SurrealDB 3.1.5 and 3.3.4 (the event name may be nested inside an
+ * app event that wrote to a synced table):
+ *   "Error while processing event _00_item_mutation: There was an error
+ *    processing a remote HTTP request: error sending request for url (...)"
+ *    (refused connection or TIMEOUT), "...: 503 Service Unavailable" (5xx),
+ *   "Error while processing event _00_item_mutation: failed to lookup address
+ *    information: ..." (DNS).
+ *
+ * Only `_00_` events count: an app's own event posting to its own endpoint
+ * stays `application`, as does a 4xx from `/ingest` other than 408/429 (a
+ * malformed or oversized payload, a secret mismatch), which no retry fixes and
+ * which would otherwise hold the outbox's serial lane forever.
+ */
+const SP00KY_EVENT_POST_FAILED =
+  /error while processing event _00_[^:\s]*: (?:there was an error processing a remote http request: (?:error sending request|5\d\d\b|408\b|429\b)|failed to lookup address information)/;
+
 export function classifySyncError(error: unknown): 'network' | 'application' {
   const message =
     error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
 
+  if (SP00KY_EVENT_POST_FAILED.test(message)) return 'network';
+
   for (const pattern of NETWORK_ERROR_PATTERNS) {
     if (message.includes(pattern)) {
       return 'network';