@spooky-sync/core 0.0.1-canary.38 → 0.0.1-canary.381

package/src/mutation/ingest-outage.integration.test.ts

@@ -0,0 +1,400 @@
+import { createServer, type Server } from 'node:http';
+import { Socket } from 'node:net';
+import { readFile } from 'node:fs/promises';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import { RecordId, Surreal } from 'surrealdb';
+import { liveEndpoint } from '../live/index';
+import { defaultEnv } from '../query/env';
+import { RemoteDatabaseService } from '../services/database/remote';
+import { buildState } from '../testing/build';
+import { runPure } from '../testing/run-pure';
+import type { ClientState } from '../state/client-state';
+import type { OutEvent } from '../kernel/events';
+import type { StatementResult } from '../kernel/effects';
+import { mintMutationId } from './mutation-id';
+import { drain, loadOutbox } from './push.saga';
+import { BATCH_SKIPPED } from './rows';
+import { write } from './write.saga';
+
+/**
+ * Integration: a queued write that SurrealDB refuses because sp00ky's OWN
+ * ingest post failed (scheduler/SSP down, slow past the event TIMEOUT, or
+ * answering 5xx) must stay in the outbox and land once `/ingest` is back. It
+ * used to read as an application error, so the client rolled the write back
+ * and discarded it: silent data loss on every scheduler outage.
+ *
+ * Runs the real `write` and `drain` sagas, the client's real remote adapter,
+ * and the `http::post` line `spky migrate` generates (read from the Rust
+ * source, so the two cannot drift) against a real SurrealDB, with an HTTP sink
+ * in this process playing the scheduler. Boot one first, e.g.:
+ *   docker run -d --name spooky-surreal-it -p 18011:8000 \
+ *     surrealdb/surrealdb:v3.1.5 start --user root --pass root --allow-all memory
+ *
+ * Override the endpoint with SURREAL_IT_ENDPOINT and the host SurrealDB reaches
+ * this process at with SURREAL_IT_SINK_HOST (default `host.docker.internal`;
+ * `127.0.0.1` for a SurrealDB running on the host). Skips if unreachable.
+ */
+const endpoint = liveEndpoint(process.env.SURREAL_IT_ENDPOINT ?? 'ws://127.0.0.1:18011');
+const sinkHost = process.env.SURREAL_IT_SINK_HOST ?? 'host.docker.internal';
+const NS = 'it';
+/** Short so the hang case stays quick; the shipped value is 10s. */
+const EVENT_TIMEOUT_SECS = 2;
+
+const cliSource = (file: string) =>
+  readFileSync(fileURLToPath(new URL(`../../../../apps/cli/src/${file}`, import.meta.url)), 'utf8');
+
+/** The `/ingest` post every generated event makes (`sp00ky.rs::ingest_post`). */
+function ingestPost(): string {
+  const src = cliSource('sp00ky.rs');
+  const m = /fn ingest_post\(\) -> String \{\s*format!\(\s*"(.+?)\\n",/s.exec(src);
+  if (!m) throw new Error('ingest_post format string not found');
+  if (!src.includes('"DEFINE EVENT OVERWRITE _00_{}_mutation ON TABLE {}\\n"')) {
+    throw new Error('the mutation event is no longer named _00_<table>_mutation');
+  }
+  return m[1]
+    .replaceAll('\\"', '"')
+    .replaceAll('{{', '{')
+    .replaceAll('}}', '}')
+    .replace('{}', String(EVENT_TIMEOUT_SECS));
+}
+
+/** `flap` answers the next post 503, then switches itself back to `ok`. */
+type SinkMode = 'ok' | '500' | '503' | 'hang' | 'flap';
+
+/** Plays the scheduler's `/ingest` (and an app's own third-party endpoint). */
+function startSink(): Promise<{ server: Server; port: number; mode: { current: SinkMode }; hits: string[] }> {
+  const mode = { current: 'ok' as SinkMode };
+  const hits: string[] = [];
+  const server = createServer((req, res) => {
+    req.resume();
+    hits.push(`${req.method} ${req.url} ${mode.current}`);
+    if (mode.current === 'hang') return; // never answers; SurrealDB's TIMEOUT fires
+    if (mode.current === 'flap') {
+      mode.current = 'ok';
+      res.writeHead(503);
+      res.end();
+      return;
+    }
+    if (mode.current === 'ok') {
+      res.writeHead(200, { 'content-type': 'application/json' });
+      res.end('{}');
+      return;
+    }
+    res.writeHead(Number(mode.current));
+    res.end();
+  });
+  return new Promise((resolve) =>
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      resolve({ server, port: typeof address === 'object' && address ? address.port : 0, mode, hits });
+    })
+  );
+}
+
+/** A port nothing listens on: a refused connection. */
+async function closedPort(): Promise<number> {
+  const server = createServer();
+  await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+  const address = server.address();
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+  return typeof address === 'object' && address ? address.port : 1;
+}
+
+const silent: any = { debug() {}, info() {}, warn() {}, error() {}, trace() {} };
+silent.child = () => silent;
+
+const env = defaultEnv({
+  tables: [
+    { name: 'item', columns: { title: { type: 'string', optional: true } } },
+    { name: 'lead', columns: { title: { type: 'string', optional: true } } },
+  ],
+} as any);
+
+describe('a write refused by a failed ingest post is retried, not rolled back (real SurrealDB)', () => {
+  let reachable = false;
+  let local: Surreal;
+  let remote: RemoteDatabaseService;
+  let sink: Awaited<ReturnType<typeof startSink>>;
+  let sinkUrl = '';
+  let database = '';
+  /** Every per-statement error the remote answered, in order. */
+  const statementErrors: string[] = [];
+
+  beforeAll(async () => {
+    const url = new URL(endpoint);
+    reachable = await new Promise<boolean>((resolve) => {
+      const socket = new Socket();
+      const done = (ok: boolean) => {
+        socket.destroy();
+        resolve(ok);
+      };
+      socket.setTimeout(1_000, () => done(false));
+      socket.once('error', () => done(false));
+      socket.connect(Number(url.port || 80), url.hostname, () => done(true));
+    });
+    if (!reachable) return;
+
+    sink = await startSink();
+    sinkUrl = `http://${sinkHost}:${sink.port}`;
+
+    // The wasm engine loads its binary with `fetch(new URL(..., import.meta.url))`,
+    // and Node's fetch has no `file:` scheme: serve that one read from disk.
+    const realFetch = globalThis.fetch;
+    vi.stubGlobal('fetch', async (input: unknown, init?: RequestInit) => {
+      const target = input instanceof URL ? input : new URL(String(input));
+      if (target.protocol === 'file:') return new Response(await readFile(fileURLToPath(target)));
+      return realFetch(input as RequestInfo, init);
+    });
+    const { createWasmEngines } = await import('@surrealdb/wasm');
+    local = new Surreal({ engines: createWasmEngines() });
+    await local.connect('mem://');
+    await local.use({ namespace: 'n', database: 'd' });
+    await local.query('DEFINE TABLE item SCHEMALESS; DEFINE TABLE lead SCHEMALESS; DEFINE TABLE _00_pending_mutations SCHEMALESS;');
+
+    database = `ingest_outage_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    remote = new RemoteDatabaseService({ endpoint, namespace: NS, database } as any, silent);
+    await remote.connect();
+    await remote.signin({ username: 'root', password: 'root' });
+    await remote.query(`
+      DEFINE PARAM OVERWRITE $sp00ky_endpoint VALUE '${sinkUrl}' PERMISSIONS NONE;
+      DEFINE PARAM OVERWRITE $sp00ky_secret VALUE 'it-secret' PERMISSIONS NONE;
+      DEFINE TABLE item SCHEMALESS;
+      DEFINE EVENT OVERWRITE _00_item_mutation ON TABLE item
+      WHEN $before != $after AND $event != "DELETE"
+      THEN {
+          LET $payload = { table: 'item', op: $event, id: <string>$after.id, record: $after, hash: "" };
+      ${ingestPost()}
+      };
+      DEFINE TABLE lead SCHEMALESS;
+    `);
+  }, 60_000);
+
+  afterAll(async () => {
+    if (!reachable) return;
+    sink.mode.current = 'ok';
+    await remote?.query(`REMOVE DATABASE ${database}`).catch(() => undefined);
+    await remote?.close().catch(() => undefined);
+    await local?.close().catch(() => undefined);
+    sink?.server.closeAllConnections();
+    await new Promise<void>((resolve) => sink?.server.close(() => resolve()));
+    vi.unstubAllGlobals();
+  });
+
+  /**
+   * With `loseAnswer` the push reaches the server and commits, then the client
+   * sees the push timeout instead of the answer (a slow server, a socket that
+   * dropped on the way back).
+   */
+  const run = (saga: Generator<any, any, any>, state: ClientState, opts: { loseAnswer?: boolean } = {}) =>
+    runPure(saga, {
+      state,
+      now: Date.now(),
+      handlers: {
+        'local.query': (e: any) => local.query(e.sql, e.vars) as unknown as Promise<unknown>,
+        'local.execute': async (e: any) => e.query.extract((await local.query(e.query.sql, e.vars)) as unknown[]),
+        'remote.query': async (e: any) => {
+          const results = (await remote.queryResponses(e.sql, e.vars)) as StatementResult[];
+          for (const r of results) if (r.status === 'ERR') statementErrors.push(r.error);
+          if (opts.loseAnswer) throw new Error(`Remote request timed out after ${e.timeoutMs}ms`);
+          return results;
+        },
+        'ssp.ingest': () => undefined,
+        id: () => mintMutationId(state.tabId),
+      },
+    });
+
+  const remoteRow = async (id: RecordId<string>) =>
+    ((await remote.query('SELECT * FROM ONLY $id', { id })) as [Record<string, unknown> | null])[0] ?? null;
+  const localRow = async (id: RecordId<string>) =>
+    ((await local.query('SELECT * FROM ONLY $id', { id })) as [Record<string, unknown> | null])[0] ?? null;
+  const rolledBack = (emitted: OutEvent[]) => emitted.filter((e) => e.type === 'mutation:rolled-back');
+  const rid = (recordId: string) => new RecordId('item', recordId.slice('item:'.length));
+  const setEndpoint = (url: string) =>
+    remote.query(`DEFINE PARAM OVERWRITE $sp00ky_endpoint VALUE '${url}' PERMISSIONS NONE`);
+
+  /** Queue creates locally, the way `db.create` does, and return the state with them in the outbox. */
+  async function queueCreate(...recordIds: string[]): Promise<ClientState> {
+    let state = buildState();
+    for (const recordId of recordIds) {
+      state = (await run(write(env, { kind: 'create', recordId, data: { title: recordId } }), state)).state;
+    }
+    expect(state.outbox).toHaveLength(recordIds.length);
+    return state;
+  }
+
+  /**
+   * The outbox of `state`, loaded again from the local store the way a
+   * restarted client does (without what other tests left in the store).
+   */
+  async function reloaded(state: ClientState): Promise<ClientState> {
+    const ids = new Set(state.outbox.map((i) => i.id));
+    const loaded = (await run(loadOutbox(env), { ...buildState(), tabId: state.tabId })).state;
+    return { ...loaded, outbox: loaded.outbox.filter((i) => ids.has(i.id)) };
+  }
+
+  it.for([
+    ['a refused connection', 'refused'],
+    ['a post that hangs past the event TIMEOUT', 'hang'],
+    ['a 500 from /ingest', '500'],
+    ['a 503 from a cold SSP', '503'],
+  ] as const)('%s: the write stays queued, then lands once /ingest answers', { timeout: 30_000 }, async ([, failure], ctx) => {
+    if (!reachable) ctx.skip();
+    const id = new RecordId('item', `outage_${failure}`);
+    const state = await queueCreate(`item:outage_${failure}`);
+    const errorsBefore = statementErrors.length;
+
+    await setEndpoint(failure === 'refused' ? `http://${sinkHost}:${await closedPort()}` : sinkUrl);
+    sink.mode.current = failure === 'refused' ? 'ok' : failure;
+    const failed = await run(drain(env), state);
+
+    const seen = statementErrors.slice(errorsBefore);
+    expect(seen).toHaveLength(1);
+    expect(seen[0]).toMatch(
+      /Error while processing event _00_item_mutation: There was an error processing a remote HTTP request: /
+    );
+    expect(rolledBack(failed.emitted)).toEqual([]);
+    expect(failed.state.outbox).toEqual([expect.objectContaining({ recordId: `item:outage_${failure}`, attempts: 1 })]);
+    expect(failed.timers.get('outbox')).toEqual({ ms: expect.any(Number), event: { type: 'Drain' } });
+    expect(await localRow(id)).toMatchObject({ title: `item:outage_${failure}` });
+    expect(await remoteRow(id)).toBeNull();
+
+    // The scheduler is back: the retry lands the write exactly once.
+    await setEndpoint(sinkUrl);
+    sink.mode.current = 'ok';
+    const landed = await run(drain(env), failed.state);
+    expect(rolledBack(landed.emitted)).toEqual([]);
+    expect(landed.emitted).toContainEqual(expect.objectContaining({ type: 'mutation:settled', recordId: `item:outage_${failure}` }));
+    expect(landed.state.outbox.filter((i) => i.status === 'pending')).toEqual([]);
+    expect(await remoteRow(id)).toMatchObject({ title: `item:outage_${failure}` });
+  });
+
+  it('an ingest post that fails once mid-batch: the batch stops there, then every write lands once', { timeout: 30_000 }, async (ctx) => {
+    if (!reachable) ctx.skip();
+    const tag = Math.random().toString(36).slice(2, 8);
+    const [a, b] = [`item:flap_a_${tag}`, `item:flap_b_${tag}`];
+    const state = await queueCreate(a, b);
+    const errorsBefore = statementErrors.length;
+
+    // The first post (item a's) answers 503, every later one 200. Statements
+    // used to run independently, so b committed while a was refused, stayed
+    // queued behind a's cut, and its re-sent CREATE then failed "already
+    // exists" and was rolled back locally while the server kept the row.
+    await setEndpoint(sinkUrl);
+    sink.mode.current = 'flap';
+    const cut = await run(drain(env), state);
+    const seen = statementErrors.slice(errorsBefore);
+    expect(seen).toHaveLength(2);
+    expect(seen[0]).toMatch(/Error while processing event _00_item_mutation: .*503 Service Unavailable/);
+    expect(seen[1]).toContain(BATCH_SKIPPED);
+    expect(rolledBack(cut.emitted)).toEqual([]);
+    expect(cut.state.outbox.map((i) => [i.recordId, i.status])).toEqual([
+      [a, 'pending'],
+      [b, 'pending'],
+    ]);
+    expect(await remoteRow(rid(a))).toBeNull();
+    expect(await remoteRow(rid(b))).toBeNull();
+
+    const landed = await run(drain(env), cut.state);
+    expect(statementErrors.slice(errorsBefore + 2)).toEqual([]);
+    expect(rolledBack(landed.emitted)).toEqual([]);
+    expect(landed.emitted.filter((e) => e.type === 'mutation:settled').map((e: any) => e.recordId)).toEqual([a, b]);
+    expect(landed.state.outbox.filter((i) => i.status === 'pending')).toEqual([]);
+    expect(landed.state.failedCount).toBe(0);
+    for (const id of [a, b]) {
+      expect(await remoteRow(rid(id))).toMatchObject({ title: id });
+      expect(await localRow(rid(id))).toMatchObject({ title: id });
+    }
+  });
+
+  it.for([['in the same session'], ['after a restart']] as const)(
+    'a push whose answer is lost after the server committed, re-sent %s: nothing rolls back, every write lands once',
+    { timeout: 30_000 },
+    async ([when], ctx) => {
+      if (!reachable) ctx.skip();
+      await setEndpoint(sinkUrl);
+      sink.mode.current = 'ok';
+      const tag = Math.random().toString(36).slice(2, 8);
+      const [a, b] = [`item:lost_a_${tag}`, `item:lost_b_${tag}`];
+      let state = await queueCreate(a, b);
+      // Edited before the push: the batch carries a's CREATE and then its UPDATE.
+      state = (await run(write(env, { kind: 'update', recordId: a, data: { title: 'edited' } }), state)).state;
+      const [errorsBefore, postsBefore] = [statementErrors.length, sink.hits.length];
+
+      const lost = await run(drain(env), state, { loseAnswer: true });
+      expect(statementErrors.slice(errorsBefore)).toEqual([]);
+      expect(sink.hits.length - postsBefore).toBe(3); // the server ran CREATE a, CREATE b, UPDATE a
+      expect(await remoteRow(rid(a))).toMatchObject({ title: 'edited' });
+      expect(await remoteRow(rid(b))).toMatchObject({ title: b });
+      expect(rolledBack(lost.emitted)).toEqual([]);
+      expect(lost.state.outbox.map((i) => [i.recordId, i.status, i.inDoubt])).toEqual([
+        [a, 'pending', true],
+        [b, 'pending', true],
+        [a, 'pending', true],
+      ]);
+
+      // The re-sent CREATEs used to fail "already exists" and roll back: the
+      // local rows deleted, two false entries in the tray, the server keeping both.
+      const retry = when === 'after a restart' ? await reloaded(lost.state) : lost.state;
+      const landed = await run(drain(env), retry);
+      expect(statementErrors.slice(errorsBefore)).toEqual([]);
+      expect(rolledBack(landed.emitted)).toEqual([]);
+      expect(landed.state.failedCount).toBe(retry.failedCount);
+      expect(landed.emitted.filter((e) => e.type === 'mutation:settled').map((e: any) => e.recordId)).toEqual([a, b, a]);
+      expect(landed.state.outbox.filter((i) => i.status === 'pending')).toEqual([]);
+      // Once: the re-send changed nothing on the server, so no event fired again.
+      expect(sink.hits.length - postsBefore).toBe(3);
+      expect(await remoteRow(rid(a))).toMatchObject({ title: 'edited' });
+      expect(await remoteRow(rid(b))).toMatchObject({ title: b });
+      expect(await localRow(rid(a))).toMatchObject({ title: 'edited' });
+      expect(await localRow(rid(b))).toMatchObject({ title: b });
+    }
+  );
+
+  it('an in-doubt CREATE whose id someone else holds with other content still rolls back', { timeout: 30_000 }, async (ctx) => {
+    if (!reachable) ctx.skip();
+    await setEndpoint(sinkUrl);
+    sink.mode.current = 'ok';
+    const recordId = `item:taken_${Math.random().toString(36).slice(2, 8)}`;
+    await remote.query('CREATE ONLY $id SET title = "theirs"', { id: rid(recordId) });
+    const state = await queueCreate(recordId);
+    const errorsBefore = statementErrors.length;
+
+    const lost = await run(drain(env), state, { loseAnswer: true });
+    expect(lost.state.outbox).toEqual([expect.objectContaining({ recordId, inDoubt: true })]);
+    const res = await run(drain(env), lost.state);
+    expect(statementErrors.slice(errorsBefore)).toEqual([
+      expect.stringMatching(/already exists/),
+      expect.stringMatching(/already exists/),
+    ]);
+    expect(rolledBack(res.emitted)).toEqual([expect.objectContaining({ recordId, error: expect.stringMatching(/already exists/) })]);
+    expect(res.state.outbox).toEqual([]);
+    expect(await remoteRow(rid(recordId))).toMatchObject({ title: 'theirs' });
+    expect(await localRow(rid(recordId))).toBeNull();
+  });
+
+  it.for([
+    [
+      "a failing http::post in the app's own event",
+      `DEFINE EVENT OVERWRITE notify_crm ON TABLE lead THEN { SELECT * FROM http::post('${'SINK'}/crm', {}) TIMEOUT 2s; };`,
+      /Error while processing event notify_crm: There was an error processing a remote HTTP request: 500/,
+    ],
+    [
+      "a THROW from the app's own event",
+      `DEFINE EVENT OVERWRITE notify_crm ON TABLE lead THEN { THROW 'lead refused'; };`,
+      /lead refused/,
+    ],
+  ] as const)('%s still rolls the write back', { timeout: 30_000 }, async ([, ddl, expected], ctx) => {
+    if (!reachable) ctx.skip();
+    await setEndpoint(sinkUrl);
+    await remote.query(ddl.replace('SINK', sinkUrl));
+    sink.mode.current = '500';
+    const recordId = `lead:${Math.random().toString(36).slice(2, 8)}`;
+    const state = await queueCreate(recordId);
+    const res = await run(drain(env), state);
+    sink.mode.current = 'ok';
+    expect(rolledBack(res.emitted)).toEqual([expect.objectContaining({ recordId, error: expect.stringMatching(expected) })]);
+    expect(res.state.outbox).toEqual([]);
+  });
+});

package/src/mutation/jobs.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest';
+import { buildJobRecord } from './jobs';
+
+const schema = {
+  tables: [],
+  backends: {
+    api: { outboxTable: '_00_job_api', routes: { hello: { args: { name: { optional: false }, tone: { optional: true } } } } },
+    bare: { routes: { x: { args: {} } } },
+  },
+} as any;
+
+describe('buildJobRecord', () => {
+  it('builds a pending job row with the route payload and option overrides', () => {
+    const { tableName, record } = buildJobRecord(schema, 'api', 'hello', { name: 'a', extra: 1 }, { max_retries: 5, retry_strategy: 'exponential', timeout: 9, delay: 100, assignedTo: 'w1' });
+    expect(tableName).toBe('_00_job_api');
+    expect(record).toEqual({ path: 'hello', payload: JSON.stringify({ name: 'a', tone: undefined }), status: 'pending', max_retries: 5, retry_strategy: 'exponential', timeout: 9, delay: 100, assigned_to: 'w1' });
+    expect(buildJobRecord(schema, 'api', 'hello', { name: 'a' }).record).toMatchObject({ max_retries: 3, retry_strategy: 'linear' });
+  });
+  it('rejects unknown routes, missing outbox tables and missing required args', () => {
+    expect(() => buildJobRecord(schema, 'api', 'nope', {})).toThrow('Route api.nope not found');
+    expect(() => buildJobRecord(schema, 'bare', 'x', {})).toThrow('Outbox table for backend bare not found');
+    expect(() => buildJobRecord(schema, 'api', 'hello', {})).toThrow('Missing required argument name');
+    expect(() => buildJobRecord({ tables: [] } as any, 'api', 'hello', {})).toThrow('Route api.hello not found');
+  });
+});

package/src/mutation/jobs.ts

@@ -0,0 +1,41 @@
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import type { RunOptions } from '../types';
+
+/**
+ * Build the outbox job record for a backend route and resolve its table.
+ * Every job is a single execution; recurring work is declared server-side.
+ */
+export function buildJobRecord(
+  schema: SchemaStructure,
+  backend: string,
+  path: string,
+  data: Record<string, unknown>,
+  options?: RunOptions
+): { tableName: string; record: Record<string, unknown> } {
+  const backends = (schema as { backends?: Record<string, { outboxTable?: string; routes?: Record<string, { args: Record<string, { optional: boolean }> }> }> }).backends;
+  const route = backends?.[backend]?.routes?.[path];
+  if (!route) throw new Error(`Route ${backend}.${path} not found`);
+  const tableName = backends?.[backend]?.outboxTable;
+  if (!tableName) throw new Error(`Outbox table for backend ${backend} not found`);
+
+  const payload: Record<string, unknown> = {};
+  for (const argName of Object.keys(route.args)) {
+    const arg = route.args[argName];
+    if (data[argName] === undefined && arg.optional === false) throw new Error(`Missing required argument ${argName}`);
+    payload[argName] = data[argName];
+  }
+
+  const record: Record<string, unknown> = {
+    path,
+    payload: JSON.stringify(payload),
+    // Explicit: the schema's server-side DEFAULT does not apply to an
+    // optimistic local create, and in-flight indicators key on `pending`.
+    status: 'pending',
+    max_retries: options?.max_retries ?? 3,
+    retry_strategy: options?.retry_strategy ?? 'linear',
+  };
+  if (options?.timeout != null) record.timeout = options.timeout;
+  if (options?.delay != null) record.delay = options.delay;
+  if (options?.assignedTo) record.assigned_to = options.assignedTo;
+  return { tableName, record };
+}

package/src/mutation/mutation-id.test.ts

@@ -0,0 +1,37 @@
+import { describe, it, expect } from 'vitest';
+import { mintMutationId, mutationOwnerTabId } from './mutation-id';
+
+describe('mutationOwnerTabId (raw ids)', () => {
+  it('mints with the fallback tab id when none is given', () => {
+    const id = mintMutationId();
+    expect(id.startsWith('_00_pending_mutations:')).toBe(true);
+    expect(mutationOwnerTabId(id)).toMatch(/^[a-z0-9]+$/);
+  });
+  it('accepts an id without the table prefix', () => {
+    expect(mutationOwnerTabId('1700000000000_0001_tab9')).toBe('tab9');
+    expect(mutationOwnerTabId('1700000000000')).toBeNull();
+  });
+});
+
+// The old `_00_pending_mutations:${Date.now()}` collided within a millisecond
+// (guaranteed once multiple tabs share a store) and never actually ordered the
+// drain. These pin the new id's three properties: unique, sortable, routable.
+describe('mutation ids', () => {
+  it('never collides within a burst', () => {
+    const ids = new Set(Array.from({ length: 1000 }, () => mintMutationId('tabx')));
+    expect(ids.size).toBe(1000);
+  });
+
+  it('sorts lexicographically in mint order (ORDER BY id is chronological)', () => {
+    const ids = Array.from({ length: 200 }, () => mintMutationId('tabx'));
+    expect([...ids].sort()).toEqual(ids);
+  });
+
+  it('routes back to the owning tab', () => {
+    expect(mutationOwnerTabId(mintMutationId('tab-abc'))).toBe('tab-abc');
+  });
+
+  it('returns null for a legacy numeric id instead of misrouting', () => {
+    expect(mutationOwnerTabId('_00_pending_mutations:1753875000000')).toBeNull();
+  });
+});

package/src/mutation/mutation-id.ts

@@ -0,0 +1,35 @@
+/**
+ * Outbox mutation ids. Previously `_00_pending_mutations:${Date.now()}`, which
+ * (a) collides when two mutations land in the same millisecond, a real case
+ * once multiple tabs share one store, and (b) never actually ordered the
+ * drain, because `loadFromDatabase` sorted by a `created_at` field that does
+ * not exist on the table. The new shape is sortable AND collision-free:
+ *
+ *   <13-digit zero-padded ms timestamp>_<4-digit base36 seq>_<tabId>
+ *
+ * Lexicographic order == chronological order (per tab; cross-tab within-ms
+ * ordering is arbitrary, which matches reality), so `ORDER BY id` in the
+ * drain is now meaningful. The tabId suffix also routes rollbacks back to the
+ * tab that owns the mutation in shared-tabs mode.
+ */
+
+let seq = 0;
+
+/** Tab identity for mutation ids in SOLO mode, minted once per session. In
+ *  shared-tabs mode the coordinator's tabId is passed in instead. */
+const FALLBACK_TAB_ID = Math.random().toString(36).slice(2, 8);
+
+export function mintMutationId(tabId?: string): string {
+  const ts = Date.now().toString().padStart(13, '0');
+  const n = (seq = (seq + 1) % 36 ** 4).toString(36).padStart(4, '0');
+  return `_00_pending_mutations:${ts}_${n}_${tabId ?? FALLBACK_TAB_ID}`;
+}
+
+/** The tab that created the mutation, or null for a legacy numeric id. */
+export function mutationOwnerTabId(mutationId: string): string | null {
+  const raw = mutationId.startsWith('_00_pending_mutations:')
+    ? mutationId.slice('_00_pending_mutations:'.length)
+    : mutationId;
+  const parts = raw.split('_');
+  return parts.length >= 3 ? parts[parts.length - 1] : null;
+}