@spooky-sync/core 0.0.1-canary.38 → 0.0.1-canary.381

package/src/mutation/push.saga.ts

@@ -0,0 +1,245 @@
+import type { Saga } from '../kernel/saga';
+import type { StatementResult } from '../kernel/effects';
+import { fx } from '../kernel/effects';
+import { ACK_GRACE_MS, backoffMs } from '../kernel/constants';
+import type { ClientState, OutboxItem } from '../state/client-state';
+import * as R from '../state/reducers';
+import { classifySyncError } from '../utils/error-classification';
+import type { SagaEnv } from '../query/env';
+import { columnsFor } from '../query/env';
+import { mutationOwnerTabId } from './mutation-id';
+import {
+  buildFailedRow,
+  deletePendingRow,
+  hydrateRowData,
+  loadPendingRows,
+  moveToFailedTx,
+  parsePendingRow,
+  parseStoredRecordId,
+  planRevert,
+  remoteBatch,
+  toOutboxItem,
+  BATCH_SKIPPED,
+  FAILED_TABLE,
+  type PendingMutationRow,
+} from './rows';
+
+/**
+ * Boot / bucket switch: mirror the outbox table into state. Rows that can
+ * never be replayed (a create without its payload) go straight to the tray.
+ * Debounced patches a dead page left in `_00_pending_writes` are adopted next.
+ */
+export function* loadOutbox(env: SagaEnv): Saga<void> {
+  const res = (yield fx.local.query(loadPendingRows())) as unknown[];
+  const raw = Array.isArray(res?.[0]) ? (res[0] as unknown[]) : [];
+  const items: OutboxItem[] = [];
+  const unreplayable: Array<Record<string, unknown>> = [];
+  for (const row of raw) {
+    const parsed = parsePendingRow(row);
+    if (parsed) items.push(toOutboxItem(parsed));
+    else if (row && typeof row === 'object') unreplayable.push(row as Record<string, unknown>);
+  }
+  yield fx.state.update(R.outboxReplace(items));
+  for (const row of unreplayable) {
+    const stub: PendingMutationRow = {
+      id: String(row.id ?? ''),
+      mutationType: (row.mutationType as PendingMutationRow['mutationType']) ?? 'create',
+      recordId: String(row.recordId ?? ''),
+      tableName: String(row.tableName ?? String(row.recordId ?? '').split(':')[0]),
+      createdAt: 0,
+      v: 1,
+    };
+    yield* rollback(env, stub, { message: 'pending mutation is not replayable', kind: 'unreplayable' });
+  }
+  yield* refreshFailedCount();
+  if (items.length > 0) yield fx.dispatch({ type: 'Drain' });
+  yield fx.dispatch({ type: 'AdoptPendingWrites' });
+}
+
+export function* refreshFailedCount(): Saga<void> {
+  try {
+    const res = (yield fx.local.query(`SELECT count() FROM ${FAILED_TABLE} GROUP ALL`)) as unknown[];
+    const row = Array.isArray(res?.[0]) ? (res[0] as Array<{ count?: number }>)[0] : undefined;
+    yield fx.state.update(R.setFailedCount(typeof row?.count === 'number' ? row.count : 0));
+  } catch {
+    // The tray table may not exist yet on a fresh store; the count stays as is.
+  }
+}
+
+const ERR = (r: StatementResult | undefined): string | null => (r && r.status === 'ERR' ? r.error : null);
+
+/**
+ * A CRDT edit-log row (`<table>__crdt_log`) is written once and never changes,
+ * so a CREATE retried after its first attempt committed (and only the answer
+ * was lost) finds it already there: that is a success, not a rejection.
+ * `remoteBatch` already makes such a CREATE a no-op on the server; this covers
+ * a row the user may create but not read back, where that check cannot see it.
+ */
+const isRepeatedLogCreate = (row: { mutationType: string; tableName: string }, err: string): boolean =>
+  row.mutationType === 'create' && row.tableName.endsWith('__crdt_log') && /already exists/i.test(err);
+
+/**
+ * Drain the outbox: one serial lane, FIFO, up to `outboxBatchSize` statements
+ * per request, per-statement outcome. Accepted rows are acked (they stay in
+ * the overlay until membership names them), rejected rows are rolled back,
+ * a transport failure leaves the unsent tail queued behind a backoff timer.
+ * The server stops a batch at its first failed statement (`remoteBatch`), so
+ * the rows after a rejection are sent again at once, and a re-sent row behind
+ * a retryable failure never ran before, so it cannot read "already exists".
+ *
+ * A request that throws a transport error (the push timeout, a dropped
+ * socket) may still have committed on the server, and so may one sent by a
+ * session that died before its answer. Those rows are marked in doubt
+ * (`OutboxItem.inDoubt`, every row loaded from the store is), and their
+ * re-sent CREATE accepts the row already on the server when it holds what the
+ * CREATE would write, instead of failing "already exists" and rolling back a
+ * write the server kept.
+ */
+export function* drain(env: SagaEnv): Saga<void> {
+  for (;;) {
+    const state = (yield fx.state.read((s) => s)) as ClientState;
+    if (state.tabRole === 'follower') return;
+    const batch = state.outbox.filter((i) => i.status === 'pending').slice(0, env.outboxBatchSize);
+    if (batch.length === 0) return;
+    const rowsRes = (yield fx.local.query('SELECT * FROM $ids', { ids: batch.map((i) => parseStoredRecordId(i.id)) })) as unknown[];
+    const rawRows = Array.isArray(rowsRes?.[0]) ? (rowsRes[0] as unknown[]) : [];
+    const byId = new Map<string, PendingMutationRow>();
+    for (const raw of rawRows) {
+      const parsed = parsePendingRow(raw);
+      if (parsed) byId.set(parsed.id, parsed);
+    }
+    const rows: PendingMutationRow[] = [];
+    for (const item of batch) {
+      const row = byId.get(item.id);
+      if (row) rows.push(hydrateRowData(row, columnsFor(env, row.tableName)));
+      else yield fx.state.update(R.outboxRemove(item.id)); // row already gone from the store
+    }
+    if (rows.length === 0) continue;
+
+    const req = remoteBatch(rows, new Set(batch.filter((i) => i.inDoubt).map((i) => i.id)));
+    let results: StatementResult[];
+    try {
+      const raw = (yield fx.remote.query(req.sql, req.vars, env.pushTimeoutMs)) as unknown;
+      // A response without statement results is a transport fault: every
+      // row stays queued, in doubt (handled as `!r` below).
+      results = Array.isArray(raw) ? (raw as StatementResult[]) : [];
+    } catch (error) {
+      if (classifySyncError(error) === 'application') {
+        // The whole request was rejected before any statement ran: treat the
+        // head as rejected, keep the rest for the next round.
+        yield* rollback(env, rows[0], { message: error instanceof Error ? error.message : String(error), kind: 'application' });
+        yield fx.dispatch({ type: 'SyncOutcome', ok: false, error });
+        continue;
+      }
+      // No answer: the server may have committed any prefix of the batch.
+      const attempts = batch.find((i) => i.id === rows[0].id)!.attempts;
+      yield fx.state.update(R.compose(R.outboxMarkInDoubt(rows.map((row) => row.id)), R.outboxBumpAttempts(rows[0].id)));
+      yield fx.dispatch({ type: 'SyncOutcome', ok: false, error });
+      yield fx.timer.set('outbox', backoffMs(attempts), { type: 'Drain' });
+      return;
+    }
+
+    const now = (yield fx.now()) as number;
+    let cutAt: number | null = null;
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i];
+      const r = results[i];
+      if (!r) {
+        // Statements after a transport-level cut: no answer says whether
+        // they ran, so they are in doubt.
+        yield fx.state.update(R.outboxMarkInDoubt(rows.slice(i).map((unanswered) => unanswered.id)));
+        cutAt = i;
+        break;
+      }
+      const err = ERR(r);
+      if (err === null || isRepeatedLogCreate(row, err)) {
+        const del = deletePendingRow(row.id);
+        try {
+          yield fx.local.query(del.sql, del.vars);
+        } catch (error) {
+          yield fx.emit({ type: 'log', level: 'error', message: 'outbox row delete failed after a successful push', data: { id: row.id, error } });
+        }
+        yield fx.state.update(R.outboxAck(row.id, now));
+        yield fx.emit({ type: 'mutation:settled', mutationId: row.id, recordId: row.recordId, eventType: row.mutationType });
+        yield fx.emit({ type: 'tabs:broadcast', message: { type: 'mutation-settled', mutationId: row.id, recordId: row.recordId, eventType: row.mutationType } });
+        continue;
+      }
+      // An earlier statement in this batch was rejected (or was a repeated
+      // log row), so this one never ran: the next round sends it again.
+      if (err.includes(BATCH_SKIPPED)) break;
+      if (classifySyncError(new Error(err)) === 'network') {
+        cutAt = i;
+        break;
+      }
+      yield* rollback(env, row, { message: err, kind: 'application' });
+    }
+    yield fx.timer.set('ack-prune', ACK_GRACE_MS, { type: 'AckPrune' });
+    yield fx.dispatch({ type: 'SyncOutcome', ok: cutAt === null, error: cutAt === null ? undefined : 'push interrupted' });
+    if (cutAt !== null) {
+      const cut = rows[cutAt];
+      const attempts = batch.find((i) => i.id === cut.id)!.attempts;
+      yield fx.state.update(R.outboxBumpAttempts(cut.id));
+      yield fx.timer.set('outbox', backoffMs(attempts), { type: 'Drain' });
+      return;
+    }
+  }
+}
+
+/**
+ * Undo one rejected mutation. Order matters: the tray row is written and the
+ * pending row deleted in ONE transaction before any local revert, so a crash
+ * mid-way leaves the mutation in the tray, never silently re-applied.
+ */
+export function* rollback(
+  env: SagaEnv,
+  row: PendingMutationRow,
+  error: { message: string; kind: 'application' | 'unreplayable' }
+): Saga<void> {
+  const attempts = ((yield fx.state.read((s) => s.outbox.find((i) => i.id === row.id)?.attempts)) as number | undefined) ?? 0;
+  const now = (yield fx.now()) as number;
+  let before: Record<string, unknown> | null = row.beforeRecord ?? null;
+  if (row.beforeRecord === undefined && row.mutationType !== 'create') {
+    // Legacy row: nothing captured locally, ask the server for the row.
+    try {
+      const res = (yield fx.remote.query('SELECT * FROM ONLY $id', { id: parseStoredRecordId(row.recordId) }, env.remoteTimeoutMs)) as StatementResult[];
+      const r = res?.[0];
+      before = r && r.status === 'OK' && r.result && typeof r.result === 'object' ? (r.result as Record<string, unknown>) : null;
+    } catch {
+      before = null;
+    }
+  }
+  const plan = planRevert(row, before);
+  const failed = buildFailedRow(row, error, before, attempts, now, plan.revert);
+  const move = moveToFailedTx(failed);
+  try {
+    yield fx.local.execute(move.query, move.vars);
+  } catch (err) {
+    yield fx.emit({ type: 'log', level: 'error', message: 'failed to move a rejected mutation to the tray', data: { id: row.id, err } });
+  }
+  yield fx.state.update(R.compose(R.outboxRemove(row.id), R.setFailedCount((yield fx.state.read((s) => s.failedCount)) as number + 1)));
+  if (plan.tx) {
+    try {
+      yield fx.local.execute(plan.tx.query, plan.tx.vars);
+    } catch (err) {
+      yield fx.emit({ type: 'log', level: 'error', message: 'local revert failed', data: { id: row.id, err } });
+    }
+  }
+  try {
+    yield fx.ssp.ingest([plan.circuit]);
+  } catch (err) {
+    yield fx.emit({ type: 'log', level: 'warn', message: 'circuit revert failed', data: { id: row.id, err } });
+  }
+  const roleNow = (yield fx.state.read((s) => s.tabRole)) as ClientState['tabRole'];
+  if (roleNow !== 'solo') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records: [plan.circuit] } });
+  if (plan.circuit.op === 'DELETE') yield fx.state.update(R.deleteVersions([row.recordId]));
+  else if (before && typeof before._00_rv === 'number') yield fx.state.update(R.setVersions([[row.recordId, before._00_rv]]));
+  yield fx.state.update(R.markTableDirty(row.tableName));
+  const rolledBack = { type: 'mutation-rolled-back', mutationId: row.id, recordId: row.recordId, eventType: row.mutationType, error: error.message };
+  yield fx.emit({ type: 'mutation:rolled-back', mutationId: row.id, recordId: row.recordId, eventType: row.mutationType, error: error.message });
+  const [tabId, role] = (yield fx.state.read((s) => [s.tabId, s.tabRole])) as [string, ClientState['tabRole']];
+  const owner = mutationOwnerTabId(row.id);
+  if (role === 'leader' && owner && owner !== tabId) yield fx.emit({ type: 'tabs:sendTo', tabId: owner, message: rolledBack });
+  const count = (yield fx.state.read((s) => s.failedCount)) as number;
+  yield fx.emit({ type: 'tray:changed', count });
+  if (role === 'leader') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'failed-mutations-changed', count } });
+}

package/src/mutation/rows.test.ts

@@ -0,0 +1,278 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import * as rows from './rows';
+
+const rid = (t: string, i: string) => new RecordId(t, i);
+
+describe('stored ids', () => {
+  it('strips ⟨⟩ escaping and reads the timestamp prefix', () => {
+    expect(rows.parseStoredRecordId('_00_pending_mutations:⟨1700000000000_0001_tab⟩')).toEqual(
+      rid('_00_pending_mutations', '1700000000000_0001_tab')
+    );
+    expect(rows.storedIdString(rid('_00_pending_mutations', 'x'))).toBe('_00_pending_mutations:x');
+    expect(rows.storedIdString('_00_pending_mutations:⟨x⟩')).toBe('_00_pending_mutations:x');
+    expect(rows.createdAtFromId('_00_pending_mutations:⟨1700000000000_0001_tab⟩')).toBe(1700000000000);
+    expect(rows.createdAtFromId('1700000000000_0001_tab')).toBe(1700000000000);
+    expect(rows.createdAtFromId('_00_pending_mutations:abc')).toBeNull();
+  });
+});
+
+describe('parsePendingRow', () => {
+  it('reads v2 rows', () => {
+    const row = rows.parsePendingRow({
+      id: '_00_pending_mutations:⟨1700000000000_0001_tab⟩',
+      mutationType: 'update',
+      recordId: 'thing:1',
+      tableName: 'thing',
+      data: { a: 1 },
+      beforeRecord: { id: 'thing:1', a: 0 },
+      createdAt: 5,
+      v: 2,
+    });
+    expect(row).toEqual({
+      id: '_00_pending_mutations:1700000000000_0001_tab',
+      mutationType: 'update',
+      recordId: 'thing:1',
+      tableName: 'thing',
+      data: { a: 1 },
+      beforeRecord: { id: 'thing:1', a: 0 },
+      createdAt: 5,
+      v: 2,
+    });
+  });
+  it('tolerates legacy rows and rejects unreplayable ones', () => {
+    const legacy = rows.parsePendingRow({
+      id: '_00_pending_mutations:1700000000000_0001_tab',
+      mutationType: 'delete',
+      recordId: rid('thing', '1'),
+    });
+    expect(legacy).toMatchObject({ tableName: 'thing', createdAt: 1700000000000, v: 1, beforeRecord: undefined });
+    expect(rows.parsePendingRow({ id: '_00_pending_mutations:1', mutationType: 'create', recordId: 'thing:1' })).toBeNull();
+    expect(rows.parsePendingRow({ id: '_00_pending_mutations:abc', mutationType: 'delete', recordId: 'thing:1' })!.createdAt).toBe(0);
+    expect(rows.parsePendingRow(null)).toBeNull();
+    expect(rows.parsePendingRow({ mutationType: 'bogus', recordId: 'x' })).toBeNull();
+    expect(rows.parsePendingRow({ mutationType: 'delete', recordId: 5 })).toBeNull();
+    expect(rows.toOutboxItem(legacy!)).toEqual({
+      id: legacy!.id,
+      type: 'delete',
+      recordId: 'thing:1',
+      table: 'thing',
+      status: 'pending',
+      ackedAt: null,
+      attempts: 0,
+      // A row read back from the store may have been sent by a session that died.
+      inDoubt: true,
+    });
+    expect(rows.loadPendingRows()).toBe('SELECT * FROM _00_pending_mutations ORDER BY id ASC');
+    expect(rows.loadFailedRows()).toBe('SELECT * FROM _00_failed_mutations ORDER BY failedAt ASC');
+  });
+});
+
+describe('local write transactions', () => {
+  const base = { recordId: rid('thing', '1'), mutationId: rid('_00_pending_mutations', 'm'), table: 'thing', now: 7 };
+  it('create: row + v2 outbox entry, result index 0', () => {
+    const { query, vars } = rows.planCreateTx({ ...base, data: { a: 1, b: 'x' } });
+    expect(query.sql).toBe(
+      "BEGIN TRANSACTION;\nCREATE ONLY $id SET a = $data_a, b = $data_b;CREATE ONLY $mid SET mutationType = 'create', recordId = $id, tableName = $table, createdAt = $createdAt, v = 2, data = $data;\nCOMMIT TRANSACTION;"
+    );
+    expect(vars).toEqual({ id: base.recordId, mid: base.mutationId, table: 'thing', createdAt: 7, data: { a: 1, b: 'x' }, data_a: 1, data_b: 'x' });
+    expect(query.extract([null, 'row', 'outbox'])).toBe('row');
+    expect(rows.planCreateTx(base).vars.data).toEqual({});
+  });
+  it('update: rv bump, merge, outbox with beforeRecord, RETURN target', () => {
+    const { query, vars } = rows.planUpdateTx({ ...base, data: { a: 2 }, before: { id: 'thing:1', a: 1 } });
+    expect(query.sql).toContain('UPDATE $id SET _00_rv += 1;LET $updated = (UPDATE ONLY $id MERGE $data);');
+    expect(query.sql).toContain("mutationType = 'update', recordId = $id, tableName = $table, createdAt = $createdAt, v = 2, data = $data, beforeRecord = $before;RETURN {target: $updated}");
+    expect(vars.before).toEqual({ id: 'thing:1', a: 1 });
+    expect(rows.planUpdateTx(base).vars).toMatchObject({ data: {}, before: null });
+  });
+  it('delete: DELETE + outbox with beforeRecord', () => {
+    const { query, vars } = rows.planDeleteTx({ ...base, before: { id: 'thing:1' } });
+    expect(query.sql).toBe(
+      "BEGIN TRANSACTION;\nDELETE $id;CREATE ONLY $mid SET mutationType = 'delete', recordId = $id, tableName = $table, createdAt = $createdAt, v = 2, beforeRecord = $before;\nCOMMIT TRANSACTION;"
+    );
+    expect(vars.before).toEqual({ id: 'thing:1' });
+    expect(rows.planDeleteTx(base).vars.before).toBeNull();
+  });
+});
+
+describe('debounced update transactions', () => {
+  it('local-only update and deferred outbox row', () => {
+    const local = rows.planLocalOnlyUpdateTx({ recordId: rid('thing', '1'), data: { a: 1 } });
+    expect(local.query.sql).toBe('BEGIN TRANSACTION;\nUPDATE $id SET _00_rv += 1;LET $updated = (UPDATE ONLY $id MERGE $data);RETURN {target: $updated};\nCOMMIT TRANSACTION;');
+    expect(local.query.extract([null, 1, 2, { target: 'x' }])).toEqual({ target: 'x' });
+    const deferred = rows.planDeferredOutboxRowTx({ recordId: rid('thing', '1'), mutationId: rid('_00_pending_mutations', 'm'), table: 'thing', data: { a: 1 }, before: { a: 0 }, now: 3 });
+    expect(deferred.query.sql).toBe("BEGIN TRANSACTION;\nCREATE ONLY $mid SET mutationType = 'update', recordId = $id, tableName = $table, createdAt = $createdAt, v = 2, data = $data, beforeRecord = $before;\nCOMMIT TRANSACTION;");
+    expect(rows.planDeferredOutboxRowTx({ recordId: rid('thing', '1'), mutationId: rid('_00_pending_mutations', 'm'), table: 'thing', now: 3 }).vars).toMatchObject({ data: {}, before: null });
+  });
+
+  it('with a mirror: the patch is upserted beside the row, and the flush moves it in one transaction', () => {
+    const wrow = { recordId: rid('thing', '1'), tableName: 'thing', data: { a: 1, b: 2 }, beforeRecord: null, flushBy: 9 };
+    const local = rows.planLocalOnlyUpdateTx({ recordId: rid('thing', '1'), data: { b: 2 }, mirror: { id: rid('_00_pending_writes', 'w'), row: wrow } });
+    expect(local.query.sql).toBe(
+      'BEGIN TRANSACTION;\nUPDATE $id SET _00_rv += 1;LET $updated = (UPDATE ONLY $id MERGE $data);UPSERT ONLY $wid REPLACE $wrow;RETURN {target: $updated};\nCOMMIT TRANSACTION;'
+    );
+    expect(local.vars).toEqual({ id: rid('thing', '1'), data: { b: 2 }, wid: rid('_00_pending_writes', 'w'), wrow });
+    expect(local.query.extract([null, 1, 2, 3, { target: 'x' }])).toEqual({ target: 'x' });
+    const moved = rows.planDeferredOutboxRowTx({ recordId: rid('thing', '1'), mutationId: rid('_00_pending_mutations', 'm'), table: 'thing', data: { a: 1 }, now: 3, mirrorId: rid('_00_pending_writes', 'w') });
+    expect(moved.query.sql).toContain('beforeRecord = $before;DELETE $wid;\nCOMMIT');
+    expect(moved.vars.wid).toEqual(rid('_00_pending_writes', 'w'));
+  });
+
+  it('pending write ids and rows', () => {
+    expect(rows.pendingWriteId('_00_pending_mutations:0000000000001_0001_tab')).toBe('_00_pending_writes:0000000000001_0001_tab');
+    expect(rows.pendingWriteId('mutation-1')).toBe('_00_pending_writes:mutation-1');
+    expect(rows.parsePendingWriteRow({ id: '_00_pending_writes:⟨w-1⟩', recordId: rid('thing', '1'), data: { a: 1 }, beforeRecord: { a: 0 }, flushBy: 5 })).toEqual({
+      id: '_00_pending_writes:w-1',
+      recordId: 'thing:1',
+      tableName: 'thing',
+      data: { a: 1 },
+      beforeRecord: { a: 0 },
+      flushBy: 5,
+    });
+    expect(rows.parsePendingWriteRow({ id: 'w', recordId: 'thing:1', tableName: 'thing', data: {} })).toMatchObject({ beforeRecord: null, flushBy: 0 });
+    expect(rows.parsePendingWriteRow({ id: 'w', recordId: 'thing:1' })).toBeNull();
+    expect(rows.parsePendingWriteRow({ id: 'w', data: {} })).toBeNull();
+    expect(rows.parsePendingWriteRow('junk')).toBeNull();
+  });
+});
+
+describe('remoteBatch', () => {
+  it('one statement per row, indexed vars, mixed types', () => {
+    const { sql, vars } = rows.remoteBatch([
+      { id: 'm1', mutationType: 'create', recordId: 'thing:1', tableName: 'thing', data: { a: 1 }, createdAt: 0, v: 2 },
+      { id: 'm2', mutationType: 'update', recordId: 'thing:2', tableName: 'thing', data: { b: 2 }, createdAt: 0, v: 2 },
+      { id: 'm3', mutationType: 'delete', recordId: 'thing:3', tableName: 'thing', createdAt: 0, v: 2 },
+      { id: 'm4', mutationType: 'update', recordId: 'thing:4', tableName: 'thing', createdAt: 0, v: 2 },
+    ]);
+    // Chained: each statement runs only if the one before it committed.
+    const skip = `IF !$ok0 { THROW "${rows.BATCH_SKIPPED}" }; `;
+    expect(sql.split(';\n')).toEqual([
+      'LET $ok0 = { CREATE ONLY $id0 SET a = $d0_a; true }',
+      `LET $ok1 = { ${skip}UPDATE $id1 MERGE $data1; true }`,
+      `LET $ok2 = { ${skip.replace('$ok0', '$ok1')}DELETE $id2; true }`,
+      `LET $ok3 = { ${skip.replace('$ok0', '$ok2')}UPDATE $id3 MERGE $data3; true }`,
+    ]);
+    expect(vars).toEqual({
+      id0: rid('thing', '1'),
+      d0_a: 1,
+      id1: rid('thing', '2'),
+      data1: { b: 2 },
+      id2: rid('thing', '3'),
+      id3: rid('thing', '4'),
+      data3: {},
+    });
+    expect(rows.remoteBatch([{ id: 'm', mutationType: 'create', recordId: 'thing:9', tableName: 'thing', createdAt: 0, v: 2 }]).sql).toBe('LET $ok0 = { CREATE ONLY $id0 SET ; true }');
+  });
+
+  it('an in-doubt CREATE is a no-op when the server row matches every field no later in-doubt row of it may have changed', () => {
+    const row = (id: string, mutationType: rows.PendingMutationRow['mutationType'], recordId: string, data?: Record<string, unknown>): rows.PendingMutationRow => ({
+      id,
+      mutationType,
+      recordId,
+      tableName: recordId.split(':')[0],
+      data,
+      createdAt: 0,
+      v: 2,
+    });
+    const was = (i: number, ...keys: string[]) =>
+      `LET $was${i} = (SELECT * FROM ONLY $id${i}); IF ${[`($was${i} = NONE)`, ...keys.map((k) => `($was${i}.${k} != $d${i}_${k})`)].join(' OR ')} { CREATE ONLY $id${i} SET ${['a', 'b']
+        .map((k) => `${k} = $d${i}_${k}`)
+        .join(', ')} }`;
+    const statement = (sql: string, i: number) => sql.split(';\n')[i].replace(/^LET \$ok\d+ = \{ (IF !\$ok\d+ \{ THROW "[^"]+" \}; )?/, '').replace(/; true \}$/, '');
+    const batch = [
+      row('m1', 'create', 'thing:1', { a: 1, b: 1 }),
+      row('m2', 'update', 'thing:1', { b: 2 }),
+      row('m3', 'create', 'thing:2', { a: 1, b: 1 }),
+      row('m4', 'update', 'thing:2', { a: 2 }), // not in doubt: it never ran
+      row('m5', 'create', 'thing:3', { a: 1, b: 1 }),
+      row('m6', 'delete', 'thing:3'),
+      row('m7', 'create', 'thing:4', { a: 1, b: 1 }),
+      row('m8', 'update', 'thing:9', { a: 2 }), // another record
+      row('m9', 'update', 'thing:1'), // no payload: changes nothing
+    ];
+    const { sql } = rows.remoteBatch(batch, new Set(['m1', 'm2', 'm3', 'm5', 'm6', 'm8', 'm9']));
+    expect(statement(sql, 0)).toBe(was(0, 'a')); // b may be m2's
+    expect(statement(sql, 2)).toBe(was(2, 'a', 'b'));
+    expect(statement(sql, 4)).toBe(was(4)); // m6 may have deleted it and something else re-created it
+    expect(statement(sql, 6)).toBe('CREATE ONLY $id6 SET a = $d6_a, b = $d6_b'); // not in doubt: strict
+    // A CRDT log row's id is fresh and the row immutable: existence is proof, in doubt or not.
+    expect(statement(rows.remoteBatch([row('l', 'create', 'doc__crdt_log:1', { a: 1, b: 1 })]).sql, 0)).toBe(was(0));
+    expect(statement(rows.remoteBatch([row('e', 'create', 'thing:5')], new Set(['e'])).sql, 0)).toBe(
+      'LET $was0 = (SELECT * FROM ONLY $id0); IF ($was0 = NONE) { CREATE ONLY $id0 SET  }'
+    );
+  });
+});
+
+describe('rollback plans', () => {
+  const pending = (type: rows.PendingMutationRow['mutationType']): rows.PendingMutationRow => ({
+    id: '_00_pending_mutations:m',
+    mutationType: type,
+    recordId: 'thing:1',
+    tableName: 'thing',
+    data: { a: 1 },
+    createdAt: 1,
+    v: 2,
+  });
+  it('create reverts by delete', () => {
+    const plan = rows.planRevert(pending('create'), null);
+    expect(plan.revert).toBe('full');
+    expect(plan.tx!.query.sql).toBe('BEGIN TRANSACTION;\nDELETE $id;\nCOMMIT TRANSACTION;');
+    expect(plan.circuit).toEqual({ table: 'thing', op: 'DELETE', id: 'thing:1', record: { a: 1 } });
+    expect(rows.planRevert({ ...pending('create'), data: undefined }, null).circuit.record).toEqual({});
+  });
+  it('update / delete restore beforeRecord, or are partial without it', () => {
+    const before = { id: 'thing:1', a: 0, _00_rv: 3 };
+    const upd = rows.planRevert(pending('update'), before);
+    expect(upd.tx!.query.sql).toBe('BEGIN TRANSACTION;\nUPSERT ONLY $id REPLACE $content;\nCOMMIT TRANSACTION;');
+    expect(upd.tx!.vars).toEqual({ id: rid('thing', '1'), content: { a: 0, _00_rv: 3 } });
+    expect(upd.circuit).toMatchObject({ op: 'UPDATE', record: { a: 0, _00_rv: 3, id: rid('thing', '1') } });
+    const del = rows.planRevert(pending('delete'), before);
+    expect(del.circuit.op).toBe('CREATE');
+    const partial = rows.planRevert(pending('update'), null);
+    expect(partial).toEqual({ tx: null, circuit: { table: 'thing', op: 'UPDATE', id: 'thing:1', record: {} }, revert: 'partial' });
+  });
+  it('failed rows: build, move tx, delete statements, parse', () => {
+    const failed = rows.buildFailedRow(pending('update'), { message: 'denied', kind: 'application' }, { id: 'thing:1' }, 2, 99, 'full');
+    expect(failed).toMatchObject({ id: '_00_pending_mutations:m', attempts: 2, failedAt: 99, revert: 'full', beforeRecord: { id: 'thing:1' } });
+    const { query, vars } = rows.moveToFailedTx(failed);
+    expect(query.sql).toBe('BEGIN TRANSACTION;\nCREATE ONLY $fid CONTENT $failed;DELETE $mid;\nCOMMIT TRANSACTION;');
+    expect(vars.fid).toEqual(rid('_00_failed_mutations', 'm'));
+    expect(vars.mid).toEqual(rid('_00_pending_mutations', 'm'));
+    expect((vars.failed as Record<string, unknown>).id).toBeUndefined();
+    expect(rows.deletePendingRow('_00_pending_mutations:m')).toEqual({ sql: 'DELETE $mid', vars: { mid: rid('_00_pending_mutations', 'm') } });
+    expect(rows.deleteFailedRow('_00_pending_mutations:m')).toEqual({ sql: 'DELETE $fid', vars: { fid: rid('_00_failed_mutations', 'm') } });
+    const parsed = rows.parseFailedRow({ ...failed, id: '_00_failed_mutations:⟨m⟩' });
+    expect(parsed).toMatchObject({ id: '_00_pending_mutations:m', mutationType: 'update', error: { message: 'denied', kind: 'application' }, revert: 'full' });
+    expect(rows.parseFailedRow({ id: '_00_failed_mutations:m', mutationType: 'delete', recordId: 'thing:2', error: { kind: 'unreplayable' }, revert: 'partial' })).toMatchObject({
+      tableName: 'thing',
+      error: { message: 'unknown', kind: 'unreplayable' },
+      attempts: 0,
+      createdAt: 0,
+      failedAt: 0,
+      revert: 'partial',
+      beforeRecord: null,
+      data: undefined,
+    });
+    expect(rows.parseFailedRow(null)).toBeNull();
+    expect(rows.parseFailedRow({ id: '_00_failed_mutations:m', mutationType: 'create', recordId: 'thing:1' })!.error).toEqual({ message: 'unknown', kind: 'application' });
+    expect(rows.parseFailedRow({ mutationType: 'nope', recordId: 'x' })).toBeNull();
+    expect(rows.parseFailedRow({ mutationType: 'delete', recordId: 1 })).toBeNull();
+  });
+});
+
+describe('hydrateRowData', () => {
+  const columns = { owner: { recordId: true }, at: { dateTime: true }, n: {} } as any;
+  const base: rows.PendingMutationRow = { id: 'm', mutationType: 'create', recordId: 'thing:1', tableName: 'thing', createdAt: 0, v: 2 };
+  it('re-types record and datetime fields from the schema, keeps unknown keys, leaves rows without data or schema alone', () => {
+    const out = rows.hydrateRowData({ ...base, data: { owner: 'user:u1', at: '2026-01-02T00:00:00.000Z', n: 1, extra: 'x' } }, columns);
+    expect(out.data).toEqual({ owner: rid('user', 'u1'), at: new Date('2026-01-02T00:00:00.000Z'), n: 1, extra: 'x' });
+    expect(rows.hydrateRowData(base, columns)).toBe(base);
+    const noSchema = { ...base, data: { owner: 'user:u1' } };
+    expect(rows.hydrateRowData(noSchema, null)).toBe(noSchema);
+  });
+  it('sends a payload the schema rejects as it is', () => {
+    const bad = { ...base, data: { owner: 42 } };
+    expect(rows.hydrateRowData(bad, columns)).toBe(bad);
+  });
+});