@spooky-sync/core 0.0.1-canary.333 → 0.0.1-canary.335

package/src/client/runtime.test.ts

@@ -6,6 +6,7 @@ import { fx } from '../kernel/effects';
 import type { Saga } from '../kernel/saga';
 import * as R from '../state/reducers';
 import { Runtime } from './runtime';
+import { rebindQueries } from '../boot/bucket-switch.saga';
 
 const env = defaultEnv({ tables: [] } as any, { materializeDebounceMs: 5 });
 const noop = () => {};
@@ -88,6 +89,45 @@ describe('Runtime state hooks', () => {
     expect(seen).toEqual([[{ id: 'thing:1' }]]);
     expect(rt.state.dirty.has('q')).toBe(false);
   });
+  it('the bucket-switch window: a re-seeded query never reads "known and idle" before its rows are painted', async () => {
+    // A sign-in: the query was registered in the anon bucket (cold, nothing
+    // painted), then the switch re-seeds it from the user's durable view.
+    // Before the fix the re-seed announced authority with `records` still
+    // empty, so a binding's `isSettled() && !row` held for a beat and the app
+    // created a row the user already had.
+    const { a, rt } = make(
+      {
+        local: {
+          getById: async () => ({ ids: [['thing:1', 1]], confirmed: true }),
+          select: async (plan: any) => (plan.ids ?? []).map((id: any) => ({ id: `${id.table}:${id.id}` })),
+        },
+      },
+      { ...buildState([buildEntry({ def: { hash: 'q', plan: { table: 'thing' } as any } })]), sessionId: 's' }
+    );
+    // What a binding derives `isSettled()` and `data()` from.
+    let known = false;
+    let idle = false;
+    let rows: unknown[] = [];
+    const verdicts: string[] = [];
+    const look = () => {
+      if (known && idle) verdicts.push(rows.length > 0 ? 'settled with rows' : 'settled empty');
+    };
+    rt.subscribe('q', (r) => ((rows = r), look()), { immediate: true });
+    rt.subscribeAuthority('q', (k) => ((known = k), look()), { immediate: true });
+    rt.subscribeStatus('q', (st) => ((idle = st === 'idle'), look()), { immediate: true });
+
+    await rt.run(rebindQueries());
+    expect(rt.state.queries.get('q')!.lifecycle).toMatchObject({ phase: 'cached', fetchDepth: 0, answered: false });
+    expect(idle).toBe(true);
+    expect(known).toBe(false);
+
+    a.timers.fire('mat:q');
+    await tick();
+    await tick();
+    expect(rows).toEqual([{ id: 'thing:1' }]);
+    expect(known).toBe(true);
+    expect(verdicts).toEqual(['settled with rows']);
+  });
   it('status and authority subscriptions fire on transitions, immediate delivers the current value', async () => {
     const { rt } = make({}, buildState([buildEntry({ def: { hash: 'q' } })]));
     const statuses: string[] = [];

package/src/client/runtime.ts

@@ -7,7 +7,7 @@ import type { Lane, Saga } from '../kernel/saga';
 import { runSaga } from '../kernel/saga';
 import type { ClientState } from '../state/client-state';
 import { emptyState } from '../state/client-state';
-import { isAuthoritative } from '../state/lifecycle';
+import { isAnswered } from '../state/lifecycle';
 import * as R from '../state/reducers';
 import { fetchingQueryCount, pendingMutationCount, queryStatus } from '../state/selectors';
 import type { SagaEnv } from '../query/env';
@@ -274,7 +274,7 @@ export class Runtime {
     this.authoritySubs.set(hash, set);
     if (options.immediate) {
       const entry = this.stateValue.queries.get(hash);
-      if (entry) this.safely(() => cb(isAuthoritative(entry.lifecycle)));
+      if (entry) this.safely(() => cb(isAnswered(entry.lifecycle)));
     }
     return () => {
       set.delete(cb);

package/src/modules/devtools/index.ts

@@ -318,7 +318,9 @@ export class DevToolsService implements StreamUpdateReceiver {
         // the panel: is the server's set known, has a non-empty one been seen
         // this session, how many empty reads were ignored.
         membershipKnown: q.config.membershipKnown === true,
-        authoritative: q.config.membershipKnown === true,
+        // What the bindings' `isAuthoritative()` reads: the known set's rows
+        // are painted, not just the set read.
+        authoritative: q.answered === true,
         viewLost: q.viewLost === true,
         serverState: q.serverState ?? null,
         // Detailed per-phase processing-time breakdown (SSP sub-phases, local/

package/src/mutation/ingest-outage.integration.test.ts

@@ -0,0 +1,400 @@
+import { createServer, type Server } from 'node:http';
+import { Socket } from 'node:net';
+import { readFile } from 'node:fs/promises';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import { RecordId, Surreal } from 'surrealdb';
+import { liveEndpoint } from '../live/index';
+import { defaultEnv } from '../query/env';
+import { RemoteDatabaseService } from '../services/database/remote';
+import { buildState } from '../testing/build';
+import { runPure } from '../testing/run-pure';
+import type { ClientState } from '../state/client-state';
+import type { OutEvent } from '../kernel/events';
+import type { StatementResult } from '../kernel/effects';
+import { mintMutationId } from './mutation-id';
+import { drain, loadOutbox } from './push.saga';
+import { BATCH_SKIPPED } from './rows';
+import { write } from './write.saga';
+
+/**
+ * Integration: a queued write that SurrealDB refuses because sp00ky's OWN
+ * ingest post failed (scheduler/SSP down, slow past the event TIMEOUT, or
+ * answering 5xx) must stay in the outbox and land once `/ingest` is back. It
+ * used to read as an application error, so the client rolled the write back
+ * and discarded it: silent data loss on every scheduler outage.
+ *
+ * Runs the real `write` and `drain` sagas, the client's real remote adapter,
+ * and the `http::post` line `spky migrate` generates (read from the Rust
+ * source, so the two cannot drift) against a real SurrealDB, with an HTTP sink
+ * in this process playing the scheduler. Boot one first, e.g.:
+ *   docker run -d --name spooky-surreal-it -p 18011:8000 \
+ *     surrealdb/surrealdb:v3.1.5 start --user root --pass root --allow-all memory
+ *
+ * Override the endpoint with SURREAL_IT_ENDPOINT and the host SurrealDB reaches
+ * this process at with SURREAL_IT_SINK_HOST (default `host.docker.internal`;
+ * `127.0.0.1` for a SurrealDB running on the host). Skips if unreachable.
+ */
+const endpoint = liveEndpoint(process.env.SURREAL_IT_ENDPOINT ?? 'ws://127.0.0.1:18011');
+const sinkHost = process.env.SURREAL_IT_SINK_HOST ?? 'host.docker.internal';
+const NS = 'it';
+/** Short so the hang case stays quick; the shipped value is 10s. */
+const EVENT_TIMEOUT_SECS = 2;
+
+const cliSource = (file: string) =>
+  readFileSync(fileURLToPath(new URL(`../../../../apps/cli/src/${file}`, import.meta.url)), 'utf8');
+
+/** The `/ingest` post every generated event makes (`sp00ky.rs::ingest_post`). */
+function ingestPost(): string {
+  const src = cliSource('sp00ky.rs');
+  const m = /fn ingest_post\(\) -> String \{\s*format!\(\s*"(.+?)\\n",/s.exec(src);
+  if (!m) throw new Error('ingest_post format string not found');
+  if (!src.includes('"DEFINE EVENT OVERWRITE _00_{}_mutation ON TABLE {}\\n"')) {
+    throw new Error('the mutation event is no longer named _00_<table>_mutation');
+  }
+  return m[1]
+    .replaceAll('\\"', '"')
+    .replaceAll('{{', '{')
+    .replaceAll('}}', '}')
+    .replace('{}', String(EVENT_TIMEOUT_SECS));
+}
+
+/** `flap` answers the next post 503, then switches itself back to `ok`. */
+type SinkMode = 'ok' | '500' | '503' | 'hang' | 'flap';
+
+/** Plays the scheduler's `/ingest` (and an app's own third-party endpoint). */
+function startSink(): Promise<{ server: Server; port: number; mode: { current: SinkMode }; hits: string[] }> {
+  const mode = { current: 'ok' as SinkMode };
+  const hits: string[] = [];
+  const server = createServer((req, res) => {
+    req.resume();
+    hits.push(`${req.method} ${req.url} ${mode.current}`);
+    if (mode.current === 'hang') return; // never answers; SurrealDB's TIMEOUT fires
+    if (mode.current === 'flap') {
+      mode.current = 'ok';
+      res.writeHead(503);
+      res.end();
+      return;
+    }
+    if (mode.current === 'ok') {
+      res.writeHead(200, { 'content-type': 'application/json' });
+      res.end('{}');
+      return;
+    }
+    res.writeHead(Number(mode.current));
+    res.end();
+  });
+  return new Promise((resolve) =>
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      resolve({ server, port: typeof address === 'object' && address ? address.port : 0, mode, hits });
+    })
+  );
+}
+
+/** A port nothing listens on: a refused connection. */
+async function closedPort(): Promise<number> {
+  const server = createServer();
+  await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+  const address = server.address();
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+  return typeof address === 'object' && address ? address.port : 1;
+}
+
+const silent: any = { debug() {}, info() {}, warn() {}, error() {}, trace() {} };
+silent.child = () => silent;
+
+const env = defaultEnv({
+  tables: [
+    { name: 'item', columns: { title: { type: 'string', optional: true } } },
+    { name: 'lead', columns: { title: { type: 'string', optional: true } } },
+  ],
+} as any);
+
+describe('a write refused by a failed ingest post is retried, not rolled back (real SurrealDB)', () => {
+  let reachable = false;
+  let local: Surreal;
+  let remote: RemoteDatabaseService;
+  let sink: Awaited<ReturnType<typeof startSink>>;
+  let sinkUrl = '';
+  let database = '';
+  /** Every per-statement error the remote answered, in order. */
+  const statementErrors: string[] = [];
+
+  beforeAll(async () => {
+    const url = new URL(endpoint);
+    reachable = await new Promise<boolean>((resolve) => {
+      const socket = new Socket();
+      const done = (ok: boolean) => {
+        socket.destroy();
+        resolve(ok);
+      };
+      socket.setTimeout(1_000, () => done(false));
+      socket.once('error', () => done(false));
+      socket.connect(Number(url.port || 80), url.hostname, () => done(true));
+    });
+    if (!reachable) return;
+
+    sink = await startSink();
+    sinkUrl = `http://${sinkHost}:${sink.port}`;
+
+    // The wasm engine loads its binary with `fetch(new URL(..., import.meta.url))`,
+    // and Node's fetch has no `file:` scheme: serve that one read from disk.
+    const realFetch = globalThis.fetch;
+    vi.stubGlobal('fetch', async (input: unknown, init?: RequestInit) => {
+      const target = input instanceof URL ? input : new URL(String(input));
+      if (target.protocol === 'file:') return new Response(await readFile(fileURLToPath(target)));
+      return realFetch(input as RequestInfo, init);
+    });
+    const { createWasmEngines } = await import('@surrealdb/wasm');
+    local = new Surreal({ engines: createWasmEngines() });
+    await local.connect('mem://');
+    await local.use({ namespace: 'n', database: 'd' });
+    await local.query('DEFINE TABLE item SCHEMALESS; DEFINE TABLE lead SCHEMALESS; DEFINE TABLE _00_pending_mutations SCHEMALESS;');
+
+    database = `ingest_outage_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    remote = new RemoteDatabaseService({ endpoint, namespace: NS, database } as any, silent);
+    await remote.connect();
+    await remote.signin({ username: 'root', password: 'root' });
+    await remote.query(`
+      DEFINE PARAM OVERWRITE $sp00ky_endpoint VALUE '${sinkUrl}' PERMISSIONS NONE;
+      DEFINE PARAM OVERWRITE $sp00ky_secret VALUE 'it-secret' PERMISSIONS NONE;
+      DEFINE TABLE item SCHEMALESS;
+      DEFINE EVENT OVERWRITE _00_item_mutation ON TABLE item
+      WHEN $before != $after AND $event != "DELETE"
+      THEN {
+          LET $payload = { table: 'item', op: $event, id: <string>$after.id, record: $after, hash: "" };
+      ${ingestPost()}
+      };
+      DEFINE TABLE lead SCHEMALESS;
+    `);
+  }, 60_000);
+
+  afterAll(async () => {
+    if (!reachable) return;
+    sink.mode.current = 'ok';
+    await remote?.query(`REMOVE DATABASE ${database}`).catch(() => undefined);
+    await remote?.close().catch(() => undefined);
+    await local?.close().catch(() => undefined);
+    sink?.server.closeAllConnections();
+    await new Promise<void>((resolve) => sink?.server.close(() => resolve()));
+    vi.unstubAllGlobals();
+  });
+
+  /**
+   * With `loseAnswer` the push reaches the server and commits, then the client
+   * sees the push timeout instead of the answer (a slow server, a socket that
+   * dropped on the way back).
+   */
+  const run = (saga: Generator<any, any, any>, state: ClientState, opts: { loseAnswer?: boolean } = {}) =>
+    runPure(saga, {
+      state,
+      now: Date.now(),
+      handlers: {
+        'local.query': (e: any) => local.query(e.sql, e.vars) as unknown as Promise<unknown>,
+        'local.execute': async (e: any) => e.query.extract((await local.query(e.query.sql, e.vars)) as unknown[]),
+        'remote.query': async (e: any) => {
+          const results = (await remote.queryResponses(e.sql, e.vars)) as StatementResult[];
+          for (const r of results) if (r.status === 'ERR') statementErrors.push(r.error);
+          if (opts.loseAnswer) throw new Error(`Remote request timed out after ${e.timeoutMs}ms`);
+          return results;
+        },
+        'ssp.ingest': () => undefined,
+        id: () => mintMutationId(state.tabId),
+      },
+    });
+
+  const remoteRow = async (id: RecordId<string>) =>
+    ((await remote.query('SELECT * FROM ONLY $id', { id })) as [Record<string, unknown> | null])[0] ?? null;
+  const localRow = async (id: RecordId<string>) =>
+    ((await local.query('SELECT * FROM ONLY $id', { id })) as [Record<string, unknown> | null])[0] ?? null;
+  const rolledBack = (emitted: OutEvent[]) => emitted.filter((e) => e.type === 'mutation:rolled-back');
+  const rid = (recordId: string) => new RecordId('item', recordId.slice('item:'.length));
+  const setEndpoint = (url: string) =>
+    remote.query(`DEFINE PARAM OVERWRITE $sp00ky_endpoint VALUE '${url}' PERMISSIONS NONE`);
+
+  /** Queue creates locally, the way `db.create` does, and return the state with them in the outbox. */
+  async function queueCreate(...recordIds: string[]): Promise<ClientState> {
+    let state = buildState();
+    for (const recordId of recordIds) {
+      state = (await run(write(env, { kind: 'create', recordId, data: { title: recordId } }), state)).state;
+    }
+    expect(state.outbox).toHaveLength(recordIds.length);
+    return state;
+  }
+
+  /**
+   * The outbox of `state`, loaded again from the local store the way a
+   * restarted client does (without what other tests left in the store).
+   */
+  async function reloaded(state: ClientState): Promise<ClientState> {
+    const ids = new Set(state.outbox.map((i) => i.id));
+    const loaded = (await run(loadOutbox(env), { ...buildState(), tabId: state.tabId })).state;
+    return { ...loaded, outbox: loaded.outbox.filter((i) => ids.has(i.id)) };
+  }
+
+  it.for([
+    ['a refused connection', 'refused'],
+    ['a post that hangs past the event TIMEOUT', 'hang'],
+    ['a 500 from /ingest', '500'],
+    ['a 503 from a cold SSP', '503'],
+  ] as const)('%s: the write stays queued, then lands once /ingest answers', { timeout: 30_000 }, async ([, failure], ctx) => {
+    if (!reachable) ctx.skip();
+    const id = new RecordId('item', `outage_${failure}`);
+    const state = await queueCreate(`item:outage_${failure}`);
+    const errorsBefore = statementErrors.length;
+
+    await setEndpoint(failure === 'refused' ? `http://${sinkHost}:${await closedPort()}` : sinkUrl);
+    sink.mode.current = failure === 'refused' ? 'ok' : failure;
+    const failed = await run(drain(env), state);
+
+    const seen = statementErrors.slice(errorsBefore);
+    expect(seen).toHaveLength(1);
+    expect(seen[0]).toMatch(
+      /Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: /
+    );
+    expect(rolledBack(failed.emitted)).toEqual([]);
+    expect(failed.state.outbox).toEqual([expect.objectContaining({ recordId: `item:outage_${failure}`, attempts: 1 })]);
+    expect(failed.timers.get('outbox')).toEqual({ ms: expect.any(Number), event: { type: 'Drain' } });
+    expect(await localRow(id)).toMatchObject({ title: `item:outage_${failure}` });
+    expect(await remoteRow(id)).toBeNull();
+
+    // The scheduler is back: the retry lands the write exactly once.
+    await setEndpoint(sinkUrl);
+    sink.mode.current = 'ok';
+    const landed = await run(drain(env), failed.state);
+    expect(rolledBack(landed.emitted)).toEqual([]);
+    expect(landed.emitted).toContainEqual(expect.objectContaining({ type: 'mutation:settled', recordId: `item:outage_${failure}` }));
+    expect(landed.state.outbox.filter((i) => i.status === 'pending')).toEqual([]);
+    expect(await remoteRow(id)).toMatchObject({ title: `item:outage_${failure}` });
+  });
+
+  it('an ingest post that fails once mid-batch: the batch stops there, then every write lands once', { timeout: 30_000 }, async (ctx) => {
+    if (!reachable) ctx.skip();
+    const tag = Math.random().toString(36).slice(2, 8);
+    const [a, b] = [`item:flap_a_${tag}`, `item:flap_b_${tag}`];
+    const state = await queueCreate(a, b);
+    const errorsBefore = statementErrors.length;
+
+    // The first post (item a's) answers 503, every later one 200. Statements
+    // used to run independently, so b committed while a was refused, stayed
+    // queued behind a's cut, and its re-sent CREATE then failed "already
+    // exists" and was rolled back locally while the server kept the row.
+    await setEndpoint(sinkUrl);
+    sink.mode.current = 'flap';
+    const cut = await run(drain(env), state);
+    const seen = statementErrors.slice(errorsBefore);
+    expect(seen).toHaveLength(2);
+    expect(seen[0]).toMatch(/Error while processing event _00_item_mutation: .*503 Service Unavailable/);
+    expect(seen[1]).toContain(BATCH_SKIPPED);
+    expect(rolledBack(cut.emitted)).toEqual([]);
+    expect(cut.state.outbox.map((i) => [i.recordId, i.status])).toEqual([
+      [a, 'pending'],
+      [b, 'pending'],
+    ]);
+    expect(await remoteRow(rid(a))).toBeNull();
+    expect(await remoteRow(rid(b))).toBeNull();
+
+    const landed = await run(drain(env), cut.state);
+    expect(statementErrors.slice(errorsBefore + 2)).toEqual([]);
+    expect(rolledBack(landed.emitted)).toEqual([]);
+    expect(landed.emitted.filter((e) => e.type === 'mutation:settled').map((e: any) => e.recordId)).toEqual([a, b]);
+    expect(landed.state.outbox.filter((i) => i.status === 'pending')).toEqual([]);
+    expect(landed.state.failedCount).toBe(0);
+    for (const id of [a, b]) {
+      expect(await remoteRow(rid(id))).toMatchObject({ title: id });
+      expect(await localRow(rid(id))).toMatchObject({ title: id });
+    }
+  });
+
+  it.for([['in the same session'], ['after a restart']] as const)(
+    'a push whose answer is lost after the server committed, re-sent %s: nothing rolls back, every write lands once',
+    { timeout: 30_000 },
+    async ([when], ctx) => {
+      if (!reachable) ctx.skip();
+      await setEndpoint(sinkUrl);
+      sink.mode.current = 'ok';
+      const tag = Math.random().toString(36).slice(2, 8);
+      const [a, b] = [`item:lost_a_${tag}`, `item:lost_b_${tag}`];
+      let state = await queueCreate(a, b);
+      // Edited before the push: the batch carries a's CREATE and then its UPDATE.
+      state = (await run(write(env, { kind: 'update', recordId: a, data: { title: 'edited' } }), state)).state;
+      const [errorsBefore, postsBefore] = [statementErrors.length, sink.hits.length];
+
+      const lost = await run(drain(env), state, { loseAnswer: true });
+      expect(statementErrors.slice(errorsBefore)).toEqual([]);
+      expect(sink.hits.length - postsBefore).toBe(3); // the server ran CREATE a, CREATE b, UPDATE a
+      expect(await remoteRow(rid(a))).toMatchObject({ title: 'edited' });
+      expect(await remoteRow(rid(b))).toMatchObject({ title: b });
+      expect(rolledBack(lost.emitted)).toEqual([]);
+      expect(lost.state.outbox.map((i) => [i.recordId, i.status, i.inDoubt])).toEqual([
+        [a, 'pending', true],
+        [b, 'pending', true],
+        [a, 'pending', true],
+      ]);
+
+      // The re-sent CREATEs used to fail "already exists" and roll back: the
+      // local rows deleted, two false entries in the tray, the server keeping both.
+      const retry = when === 'after a restart' ? await reloaded(lost.state) : lost.state;
+      const landed = await run(drain(env), retry);
+      expect(statementErrors.slice(errorsBefore)).toEqual([]);
+      expect(rolledBack(landed.emitted)).toEqual([]);
+      expect(landed.state.failedCount).toBe(retry.failedCount);
+      expect(landed.emitted.filter((e) => e.type === 'mutation:settled').map((e: any) => e.recordId)).toEqual([a, b, a]);
+      expect(landed.state.outbox.filter((i) => i.status === 'pending')).toEqual([]);
+      // Once: the re-send changed nothing on the server, so no event fired again.
+      expect(sink.hits.length - postsBefore).toBe(3);
+      expect(await remoteRow(rid(a))).toMatchObject({ title: 'edited' });
+      expect(await remoteRow(rid(b))).toMatchObject({ title: b });
+      expect(await localRow(rid(a))).toMatchObject({ title: 'edited' });
+      expect(await localRow(rid(b))).toMatchObject({ title: b });
+    }
+  );
+
+  it('an in-doubt CREATE whose id someone else holds with other content still rolls back', { timeout: 30_000 }, async (ctx) => {
+    if (!reachable) ctx.skip();
+    await setEndpoint(sinkUrl);
+    sink.mode.current = 'ok';
+    const recordId = `item:taken_${Math.random().toString(36).slice(2, 8)}`;
+    await remote.query('CREATE ONLY $id SET title = "theirs"', { id: rid(recordId) });
+    const state = await queueCreate(recordId);
+    const errorsBefore = statementErrors.length;
+
+    const lost = await run(drain(env), state, { loseAnswer: true });
+    expect(lost.state.outbox).toEqual([expect.objectContaining({ recordId, inDoubt: true })]);
+    const res = await run(drain(env), lost.state);
+    expect(statementErrors.slice(errorsBefore)).toEqual([
+      expect.stringMatching(/already exists/),
+      expect.stringMatching(/already exists/),
+    ]);
+    expect(rolledBack(res.emitted)).toEqual([expect.objectContaining({ recordId, error: expect.stringMatching(/already exists/) })]);
+    expect(res.state.outbox).toEqual([]);
+    expect(await remoteRow(rid(recordId))).toMatchObject({ title: 'theirs' });
+    expect(await localRow(rid(recordId))).toBeNull();
+  });
+
+  it.for([
+    [
+      "a failing http::post in the app's own event",
+      `DEFINE EVENT OVERWRITE notify_crm ON TABLE lead THEN { SELECT * FROM http::post('${'SINK'}/crm', {}) TIMEOUT 2s; };`,
+      /Error while processing event notify_crm: There was an error processing a remote HTTP request: 500/,
+    ],
+    [
+      "a THROW from the app's own event",
+      `DEFINE EVENT OVERWRITE notify_crm ON TABLE lead THEN { THROW 'lead refused'; };`,
+      /lead refused/,
+    ],
+  ] as const)('%s still rolls the write back', { timeout: 30_000 }, async ([, ddl, expected], ctx) => {
+    if (!reachable) ctx.skip();
+    await setEndpoint(sinkUrl);
+    await remote.query(ddl.replace('SINK', sinkUrl));
+    sink.mode.current = '500';
+    const recordId = `lead:${Math.random().toString(36).slice(2, 8)}`;
+    const state = await queueCreate(recordId);
+    const res = await run(drain(env), state);
+    sink.mode.current = 'ok';
+    expect(rolledBack(res.emitted)).toEqual([expect.objectContaining({ recordId, error: expect.stringMatching(expected) })]);
+    expect(res.state.outbox).toEqual([]);
+  });
+});

package/src/mutation/push.saga.test.ts

@@ -6,11 +6,18 @@ import * as R from '../state/reducers';
 import { defaultEnv } from '../query/env';
 import { drain, loadOutbox, refreshFailedCount, rollback } from './push.saga';
 import type { StatementResult } from '../kernel/effects';
-import type { PendingMutationRow } from './rows';
+import { BATCH_SKIPPED, type PendingMutationRow } from './rows';
 
 const env = defaultEnv({ tables: [{ name: 'thing', columns: {} }] } as any, { outboxBatchSize: 2 });
 const ok = (result: unknown = null): StatementResult => ({ status: 'OK', result });
 const err = (error: string): StatementResult => ({ status: 'ERR', error });
+/** What SurrealDB answers for a chained statement whose predecessor failed. */
+const skipped = (): StatementResult => err(`An error occurred: ${BATCH_SKIPPED}`);
+/** The row numbers a pushed batch carries, from its `$id<i>` vars. */
+const pushedRows = (vars: Record<string, unknown>) =>
+  Object.keys(vars)
+    .filter((k) => /^id\d+$/.test(k))
+    .map((k) => String((vars[k] as RecordId).id));
 const pendingRow = (n: number, type: PendingMutationRow['mutationType'] = 'create', extra: Partial<PendingMutationRow> = {}) => ({
   id: `_00_pending_mutations:m${n}`,
   mutationType: type,
@@ -56,7 +63,10 @@ describe('drain', () => {
         },
       },
     });
-    expect(pushed).toEqual(['CREATE ONLY $id0 SET a = $d0_a;\nCREATE ONLY $id1 SET a = $d1_a', 'CREATE ONLY $id0 SET a = $d0_a']);
+    expect(pushed).toEqual([
+      `LET $ok0 = { CREATE ONLY $id0 SET a = $d0_a; true };\nLET $ok1 = { IF !$ok0 { THROW "${BATCH_SKIPPED}" }; CREATE ONLY $id1 SET a = $d1_a; true }`,
+      'LET $ok0 = { CREATE ONLY $id0 SET a = $d0_a; true }',
+    ]);
     expect(out.state.outbox.every((i) => i.status === 'acked' && i.ackedAt === 500)).toBe(true);
     expect(out.log.filter((e) => e.kind === 'local.query' && (e as any).sql === 'DELETE $mid')).toHaveLength(3);
     expect(out.emitted.filter((e) => e.type === 'mutation:settled')).toHaveLength(3);
@@ -64,34 +74,49 @@ describe('drain', () => {
     expect(out.dispatched.filter((d) => d.type === 'SyncOutcome')).toEqual([{ type: 'SyncOutcome', ok: true, error: undefined }, { type: 'SyncOutcome', ok: true, error: undefined }]);
   });
 
-  it('a rejected statement rolls back only that row; the rest of the batch is acked', async () => {
+  it('a rejected statement rolls back only that row; the rows the server skipped after it go out again at once', async () => {
+    const pushed: string[][] = [];
     const out = await runPure(drain(env), {
-      state: stateWith([1, 2]),
+      state: stateWith([1, 2, 3]),
       handlers: {
-        'local.query': (e: any) => (e.sql === 'SELECT * FROM $ids' ? [[pendingRow(1), pendingRow(2)]] : []),
-        'remote.query': () => [err('permission denied'), ok()],
+        'local.query': (e: any) =>
+          e.sql === 'SELECT * FROM $ids' ? [e.vars.ids.map((id: RecordId) => pendingRow(Number(String(id.id).slice(1))))] : [],
+        'remote.query': (e: any) => {
+          pushed.push(pushedRows(e.vars));
+          return pushed.length === 1 ? [err('permission denied'), skipped()] : e.sql.split(';\n').map(() => ok());
+        },
         'local.execute': () => undefined,
         'ssp.ingest': () => undefined,
       },
     });
-    expect(out.state.outbox.map((i) => [i.id, i.status])).toEqual([['_00_pending_mutations:m2', 'acked']]);
+    expect(pushed).toEqual([['1', '2'], ['2', '3']]);
+    expect(out.state.outbox.map((i) => [i.id, i.status, i.attempts])).toEqual([
+      ['_00_pending_mutations:m2', 'acked', 0],
+      ['_00_pending_mutations:m3', 'acked', 0],
+    ]);
     expect(out.state.failedCount).toBe(1);
-    expect(out.emitted).toContainEqual(expect.objectContaining({ type: 'mutation:rolled-back', mutationId: '_00_pending_mutations:m1', error: 'permission denied' }));
-    expect(out.dispatched.at(-1)).toEqual({ type: 'SyncOutcome', ok: true, error: undefined });
+    expect(out.emitted.filter((e) => e.type === 'mutation:rolled-back')).toEqual([
+      expect.objectContaining({ mutationId: '_00_pending_mutations:m1', error: 'permission denied' }),
+    ]);
+    expect(out.timers.has('outbox')).toBe(false);
+    expect(out.dispatched.filter((d) => d.type === 'SyncOutcome').every((d: any) => d.ok)).toBe(true);
   });
 
   it('a CRDT edit-log CREATE that finds its row already there is acked, any other duplicate rolls back', async () => {
     const logEnv = defaultEnv({ tables: [{ name: 'thing', columns: {} }, { name: 'doc__crdt_log', columns: {} }] } as any, { outboxBatchSize: 2 });
+    const logRow = pendingRow(1, 'create', { tableName: 'doc__crdt_log', recordId: 'doc__crdt_log:1' });
+    let calls = 0;
     const out = await runPure(drain(logEnv), {
       state: stateWith([1, 2]),
       handlers: {
         'local.query': (e: any) =>
-          e.sql === 'SELECT * FROM $ids'
-            ? [[pendingRow(1, 'create', { tableName: 'doc__crdt_log', recordId: 'doc__crdt_log:1' }), pendingRow(2)]]
-            : [],
-        'remote.query': () => [err('Database record `doc__crdt_log:1` already exists'), err('Database record `thing:2` already exists')],
+          e.sql === 'SELECT * FROM $ids' ? [[logRow, pendingRow(2)].filter((r) => e.vars.ids.some((id: RecordId) => r.id.endsWith(String(id.id))))] : [],
+        // The duplicate log row still stops the chain on the server: row 2 goes again on its own.
+        'remote.query': () =>
+          ++calls === 1 ? [err('Database record `doc__crdt_log:1` already exists'), skipped()] : [err('Database record `thing:2` already exists')],
       },
     });
+    expect(calls).toBe(2);
     expect(out.state.outbox.find((i) => i.id === '_00_pending_mutations:m1')?.status).toBe('acked');
     expect(out.state.outbox.some((i) => i.id === '_00_pending_mutations:m2')).toBe(false);
   });
@@ -108,6 +133,8 @@ describe('drain', () => {
     });
     expect(thrown.state.outbox.map((i) => i.status)).toEqual(['pending', 'pending']);
     expect(thrown.state.outbox[0].attempts).toBe(1);
+    // No answer: the server may have committed any of them.
+    expect(thrown.state.outbox.map((i) => i.inDoubt)).toEqual([true, true]);
     expect(thrown.timers.get('outbox')).toEqual({ ms: 500, event: { type: 'Drain' } });
     expect(thrown.dispatched).toEqual([{ type: 'SyncOutcome', ok: false, error: new Error('socket closed') }]);
     const mid = await runPure(drain(env), {
@@ -115,12 +142,14 @@ describe('drain', () => {
       handlers: { 'local.query': () => [[pendingRow(1), pendingRow(2)]], 'remote.query': () => [ok(), err('connection reset')] },
     });
     expect(mid.state.outbox.map((i) => [i.status, i.attempts])).toEqual([['acked', 0], ['pending', 1]]);
+    // A failed statement did not commit, so its row is not in doubt.
+    expect(mid.state.outbox[1].inDoubt).toBeUndefined();
     expect(mid.timers.get('outbox')!.ms).toBe(500);
     const short = await runPure(drain(env), {
       state: stateWith([1, 2]),
       handlers: { 'local.query': () => [[pendingRow(1), pendingRow(2)]], 'remote.query': () => [ok()] },
     });
-    expect(short.state.outbox.map((i) => i.status)).toEqual(['acked', 'pending']);
+    expect(short.state.outbox.map((i) => [i.status, i.inDoubt])).toEqual([['acked', undefined], ['pending', true]]);
     const undefinedRows = await runPure(drain(env), { state: stateWith([1]), handlers: { 'local.query': () => undefined } });
     expect(undefinedRows.state.outbox).toEqual([]);
     const notArray = await runPure(drain(env), {
@@ -130,6 +159,37 @@ describe('drain', () => {
     expect(notArray.timers.has('outbox')).toBe(true);
   });
 
+  it('a push whose answer was lost goes out again with CREATEs that accept their own earlier commit', async () => {
+    const pushed: string[] = [];
+    const rowsOf = (e: any) => [e.vars.ids.map((id: RecordId) => pendingRow(Number(String(id.id).slice(1))))];
+    const out = await runPure(drain(env), {
+      state: stateWith([1, 2]),
+      handlers: {
+        'local.query': (e: any) => (e.sql === 'SELECT * FROM $ids' ? rowsOf(e) : []),
+        'remote.query': (e: any) => {
+          pushed.push(e.sql);
+          // The server ran the batch; the answer never came back.
+          if (pushed.length === 1) throw new Error('Remote request timed out after 15000ms');
+          return [ok(), ok()];
+        },
+      },
+    });
+    expect(out.timers.get('outbox')).toEqual({ ms: 500, event: { type: 'Drain' } });
+    const again = await runPure(drain(env), {
+      state: out.state,
+      handlers: {
+        'local.query': (e: any) => (e.sql === 'SELECT * FROM $ids' ? rowsOf(e) : []),
+        'remote.query': (e: any) => (pushed.push(e.sql), [ok(), ok()]),
+      },
+    });
+    const tolerant = (i: number) =>
+      `LET $was${i} = (SELECT * FROM ONLY $id${i}); IF ($was${i} = NONE) OR ($was${i}.a != $d${i}_a) { CREATE ONLY $id${i} SET a = $d${i}_a }`;
+    expect(pushed[0]).not.toContain('$was');
+    expect(pushed[1]).toBe(`LET $ok0 = { ${tolerant(0)}; true };\nLET $ok1 = { IF !$ok0 { THROW "${BATCH_SKIPPED}" }; ${tolerant(1)}; true }`);
+    expect(again.emitted.filter((e) => e.type === 'mutation:rolled-back')).toEqual([]);
+    expect(again.state.outbox.map((i) => i.status)).toEqual(['acked', 'acked']);
+  });
+
   it('a request rejected as a whole with an application error rolls back the head and continues', async () => {
     let calls = 0;
     const out = await runPure(drain(env), {