@spooky-sync/core 0.0.1-canary.288 → 0.0.1-canary.289

package/src/mutation/write.saga.ts

@@ -8,7 +8,19 @@ import { parseParams } from '../utils/parser';
 import { extractTablePart, parseRecordIdString } from '../utils/index';
 import type { SagaEnv } from '../query/env';
 import { columnsFor } from '../query/env';
-import { planCreateTx, planDeferredOutboxRowTx, planDeleteTx, planLocalOnlyUpdateTx, planUpdateTx, type LocalTx } from './rows';
+import { PENDING_WRITE_ORPHAN_MS } from '../kernel/constants';
+import {
+  loadPendingWriteRows,
+  parsePendingWriteRow,
+  pendingWriteId,
+  planCreateTx,
+  planDeferredOutboxRowTx,
+  planDeleteTx,
+  planLocalOnlyUpdateTx,
+  planUpdateTx,
+  type LocalTx,
+  type PendingWriteRow,
+} from './rows';
 
 export interface WriteInput {
   kind: MutationEventType;
@@ -35,6 +47,10 @@ const debounceDelay = (input: WriteInput): number => {
   const d = input.options?.debounced;
   return d && d !== true && typeof d.delay === 'number' ? d.delay : 300;
 };
+const flushOnHide = (input: WriteInput): boolean => {
+  const d = input.options?.debounced;
+  return !(d && d !== true && d.flushOnHide === false);
+};
 
 /**
  * One optimistic write: read the previous row (update/delete), run the local
@@ -107,7 +123,12 @@ function* commitWrite(kind: MutationEventType, tx: LocalTx, ctx: CommitCtx): Sag
   return { mutationId: ctx.mutationId, record };
 }
 
-/** Apply locally now, remember the merged patch, write the outbox row on flush. */
+/**
+ * Apply locally now, remember the merged patch, write the outbox row on flush.
+ * With `flushOnHide` (the default) the merged patch is also mirrored to
+ * `_00_pending_writes` in the same transaction, so a reload or a closed tab
+ * inside the delay cannot lose it.
+ */
 function* debouncedUpdate(
   input: WriteInput,
   key: string,
@@ -118,10 +139,15 @@ function* debouncedUpdate(
   const existing = (yield fx.state.read((s) => s.pendingWrites.get(key))) as PendingWrite | undefined;
   const now = (yield fx.now()) as number;
   const before = existing ? existing.before : yield* readBefore(rid);
-  const local = planLocalOnlyUpdateTx({ recordId: rid, data: params });
+  const mirrorId = existing?.mirrorId ?? (flushOnHide(input) ? pendingWriteId((yield fx.id('mutation')) as string) : null);
+  const delay = debounceDelay(input);
+  const mirror = mirrorId
+    ? { id: parseRecordIdString(mirrorId), row: { recordId: rid, tableName: table, data: { ...existing?.data, ...params }, beforeRecord: before, flushBy: now + delay } }
+    : undefined;
+  const local = planLocalOnlyUpdateTx({ recordId: rid, data: params, mirror });
   const res = (yield fx.local.execute(local.query, local.vars)) as { target?: Record<string, unknown> } | undefined;
   const record = res?.target ?? null;
-  yield fx.state.update(R.mergePendingWrite({ key, table, recordId: input.recordId, data: params, before, firstAt: existing?.firstAt ?? now }));
+  yield fx.state.update(R.mergePendingWrite({ key, table, recordId: input.recordId, data: params, before, firstAt: existing?.firstAt ?? now, mirrorId }));
   yield fx.state.update(R.markTableDirty(table));
   const version = typeof record?._00_rv === 'number' ? (record._00_rv as number) : 0;
   const tuple = { table, op: 'UPDATE' as const, id: input.recordId, record: { ...record, _00_rv: version } };
@@ -133,7 +159,7 @@ function* debouncedUpdate(
   }
   const role = (yield fx.state.read((s) => s.tabRole)) as ClientState['tabRole'];
   if (role !== 'solo') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records: [tuple] } });
-  yield fx.timer.set(`debounce:${key}`, debounceDelay(input), { type: 'FlushWrite', key });
+  yield fx.timer.set(`debounce:${key}`, delay, { type: 'FlushWrite', key });
   return { mutationId: '', record };
 }
 
@@ -152,8 +178,15 @@ export function* flushWrite(env: SagaEnv, key: string): Saga<void> {
     data: { ...pending.data },
     before: pending.before,
     now,
+    mirrorId: pending.mirrorId ? parseRecordIdString(pending.mirrorId) : undefined,
   });
-  yield fx.local.execute(insert.query, insert.vars);
+  try {
+    yield fx.local.execute(insert.query, insert.vars);
+  } catch (error) {
+    // The patch is no longer in memory; a mirrored one is still in the store.
+    if (pending.mirrorId) yield fx.timer.set('adopt-writes', PENDING_WRITE_ORPHAN_MS, { type: 'AdoptPendingWrites' });
+    throw error;
+  }
   yield fx.state.update(R.outboxPush({ id: mutationId, type: 'update', recordId: pending.recordId, table: pending.table, status: 'pending', ackedAt: null, attempts: 0 }));
   yield fx.emit({
     type: 'mutation:event',
@@ -164,3 +197,72 @@ export function* flushWrite(env: SagaEnv, key: string): Saga<void> {
   else yield fx.dispatch({ type: 'Drain' });
 }
 
+/**
+ * The page is being hidden (tab switch, reload, close, backgrounded): flush
+ * every mirrored pending write now rather than when its timer fires. On a
+ * reload or close this races the teardown; whatever does not make it stays
+ * in `_00_pending_writes` for the next boot.
+ */
+export function* flushPendingWrites(): Saga<void> {
+  const keys = (yield fx.state.read((s) => [...s.pendingWrites.values()].filter((w) => w.mirrorId !== null).map((w) => w.key))) as string[];
+  for (const key of keys) {
+    yield fx.timer.clear(`debounce:${key}`);
+    yield fx.dispatch({ type: 'FlushWrite', key });
+  }
+}
+
+/**
+ * Move abandoned `_00_pending_writes` rows into the outbox: rows no pending
+ * write of this tab holds, `PENDING_WRITE_ORPHAN_MS` past their due flush. That
+ * is a tab that reloaded or closed inside the delay, or a flush of ours that
+ * failed. A row not yet overdue may still be live in another tab and is looked
+ * at again once it would be. A follower only nudges the leader, so a single
+ * tab adopts and a patch is never queued twice.
+ */
+export function* adoptPendingWrites(): Saga<void> {
+  const state = (yield fx.state.read((s) => s)) as ClientState;
+  let raw: unknown[] = [];
+  try {
+    const res = (yield fx.local.query(loadPendingWriteRows())) as unknown[];
+    raw = Array.isArray(res?.[0]) ? (res[0] as unknown[]) : [];
+  } catch {
+    return; // An older store may not have the table yet: nothing to adopt.
+  }
+  const held = new Set([...state.pendingWrites.values()].map((w) => w.mirrorId));
+  const rows = raw.map(parsePendingWriteRow).filter((r): r is PendingWriteRow => r !== null && !held.has(r.id));
+  if (rows.length === 0) return;
+  if (state.tabRole === 'follower') {
+    yield fx.emit({ type: 'tabs:broadcast', message: { type: 'outbox-changed', mutationId: '' } });
+    return;
+  }
+  const now = (yield fx.now()) as number;
+  let recheckAt = Infinity;
+  let adopted = 0;
+  for (const row of rows) {
+    const due = row.flushBy + PENDING_WRITE_ORPHAN_MS;
+    if (now < due) {
+      recheckAt = Math.min(recheckAt, due);
+      continue;
+    }
+    const mutationId = (yield fx.id('mutation')) as string;
+    const move = planDeferredOutboxRowTx({
+      recordId: parseRecordIdString(row.recordId),
+      mutationId: parseRecordIdString(mutationId),
+      table: row.tableName,
+      data: row.data,
+      before: row.beforeRecord,
+      now,
+      mirrorId: parseRecordIdString(row.id),
+    });
+    try {
+      yield fx.local.execute(move.query, move.vars);
+    } catch (error) {
+      yield fx.emit({ type: 'log', level: 'warn', message: 'could not move an abandoned debounced write into the outbox', data: { id: row.id, error } });
+      continue;
+    }
+    yield fx.state.update(R.outboxPush({ id: mutationId, type: 'update', recordId: row.recordId, table: row.tableName, status: 'pending', ackedAt: null, attempts: 0 }));
+    adopted += 1;
+  }
+  if (adopted > 0) yield fx.dispatch({ type: 'Drain' });
+  if (recheckAt !== Infinity) yield fx.timer.set('adopt-writes', recheckAt - now, { type: 'AdoptPendingWrites' });
+}

package/src/services/database/local-migrator.integration.test.ts

@@ -0,0 +1,110 @@
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
+import { readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import { RecordId, Surreal } from 'surrealdb';
+import { LocalMigrator } from './local-migrator';
+import * as rows from '../../mutation/rows';
+
+/**
+ * Integration: the client schema the CLI emits, provisioned by the real
+ * LocalMigrator into a real SurrealDB-WASM store, then every local write the
+ * outbox makes. The meta tables are read from the CLI's source file, so this
+ * cannot drift from what `spky generate` ships; the per-table `_00_rv` field
+ * is appended the way the codegen does it.
+ *
+ * `_00_pending_mutations` used to be SCHEMAFULL there, without the fields the
+ * client writes (`tableName`, `createdAt`, `v`, `beforeRecord`): SurrealDB
+ * failed every create, update and delete transaction on this engine.
+ */
+
+const META = new URL('../../../../../apps/cli/src/meta_tables_client.surql', import.meta.url);
+
+async function clientSchema(userSchema: string): Promise<string> {
+  const schema = `${userSchema}\n${await readFile(fileURLToPath(META), 'utf8')}`;
+  const tables = [...schema.matchAll(/DEFINE TABLE (?:IF NOT EXISTS )?(\w+)/g)].map((m) => m[1]);
+  return schema + tables.map((t) => `\nDEFINE FIELD _00_rv ON TABLE ${t} TYPE int DEFAULT 0 PERMISSIONS FOR select, create, update WHERE true;`).join('');
+}
+
+const DOC_V1 = `DEFINE TABLE doc SCHEMAFULL PERMISSIONS FOR select, create, update, delete WHERE true;
+DEFINE FIELD title ON TABLE doc TYPE option<string>;`;
+const DOC_V2 = `${DOC_V1}
+DEFINE FIELD body ON TABLE doc TYPE option<string>;`;
+
+const doc = new RecordId('doc', 'd1');
+const mid = (n: string) => new RecordId('_00_pending_mutations', n);
+const noop = () => {};
+
+describe('client schema on the SurrealDB-WASM engine', () => {
+  let db: Surreal;
+  let migrator: LocalMigrator;
+
+  beforeAll(async () => {
+    // The wasm engine loads its binary with `fetch(new URL(..., import.meta.url))`,
+    // and Node's fetch has no `file:` scheme: serve that one read from disk.
+    const realFetch = globalThis.fetch;
+    vi.stubGlobal('fetch', async (input: unknown, init?: RequestInit) => {
+      const url = input instanceof URL ? input : new URL(String(input));
+      if (url.protocol === 'file:') return new Response(await readFile(fileURLToPath(url)));
+      return realFetch(input as RequestInfo, init);
+    });
+    const { createWasmEngines } = await import('@surrealdb/wasm');
+    db = new Surreal({ engines: createWasmEngines() });
+    await db.connect('mem://');
+    await db.use({ namespace: 'n', database: 'd' });
+    const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+    logger.child = () => logger;
+    const store: any = { queryUngated: (sql: string, vars?: Record<string, unknown>) => db.query(sql, vars), getConfig: () => ({ database: 'd' }) };
+    migrator = new LocalMigrator(store, logger);
+    await migrator.provision(await clientSchema(DOC_V1));
+  }, 60_000);
+
+  afterAll(async () => {
+    await db?.close();
+    vi.unstubAllGlobals();
+  });
+
+  const exec = (tx: rows.LocalTx) => db.query(tx.query.sql, tx.vars);
+  const outbox = async () => ((await db.query(rows.loadPendingRows())) as unknown[][])[0].map(rows.parsePendingRow);
+
+  it('every outbox write commits', async () => {
+    await exec(rows.planCreateTx({ recordId: doc, mutationId: mid('1'), table: 'doc', data: { title: 'a' }, now: 1 }));
+    await exec(rows.planUpdateTx({ recordId: doc, mutationId: mid('2'), table: 'doc', data: { title: 'b' }, before: { id: doc, title: 'a' }, now: 2 }));
+    const wid = new RecordId('_00_pending_writes', 'w1');
+    await exec(rows.planLocalOnlyUpdateTx({ recordId: doc, data: { title: 'c' }, mirror: { id: wid, row: { recordId: doc, tableName: 'doc', data: { title: 'c' }, beforeRecord: null, flushBy: 9 } } }));
+    await exec(rows.planDeferredOutboxRowTx({ recordId: doc, mutationId: mid('3'), table: 'doc', data: { title: 'c' }, before: null, now: 3, mirrorId: wid }));
+    await exec(rows.planDeleteTx({ recordId: doc, mutationId: mid('4'), table: 'doc', before: { id: doc, title: 'c' }, now: 4 }));
+
+    const pending = await outbox();
+    expect(pending.map((r) => [r?.mutationType, r?.tableName, r?.v])).toEqual([
+      ['create', 'doc', 2],
+      ['update', 'doc', 2],
+      ['update', 'doc', 2],
+      ['delete', 'doc', 2],
+    ]);
+    expect(pending[1]?.beforeRecord).toEqual({ id: doc, title: 'a' });
+
+    // A rejected one moves to the tray; an accepted one is deleted.
+    await exec(rows.moveToFailedTx(rows.buildFailedRow(pending[1]!, { message: 'denied', kind: 'application' }, pending[1]!.beforeRecord ?? null, 1, 5, 'full')));
+    const del = rows.deletePendingRow(pending[0]!.id);
+    await db.query(del.sql, del.vars);
+    expect((await outbox()).map((r) => r?.mutationType)).toEqual(['update', 'delete']);
+    const [tray] = (await db.query(rows.loadFailedRows())) as [Array<Record<string, unknown>>];
+    expect(tray).toEqual([expect.objectContaining({ mutationType: 'update', tableName: 'doc', error: { message: 'denied', kind: 'application' } })]);
+  });
+
+  it('a schema change keeps the outbox, the tray and pending writes, and drops the cache', async () => {
+    await exec(rows.planLocalOnlyUpdateTx({ recordId: doc, data: { title: 'd' }, mirror: { id: new RecordId('_00_pending_writes', 'w2'), row: { recordId: doc, tableName: 'doc', data: { title: 'd' }, beforeRecord: null, flushBy: 9 } } }));
+    const ids = async () =>
+      ((await db.query('SELECT VALUE id FROM _00_pending_mutations; SELECT VALUE id FROM _00_failed_mutations; SELECT VALUE id FROM _00_pending_writes')) as RecordId[][]).map((r) =>
+        r.map(String).sort()
+      );
+    const before = await ids();
+    expect(before.map((r) => r.length)).toEqual([2, 1, 1]);
+
+    await migrator.provision(await clientSchema(DOC_V2));
+    expect(await ids()).toEqual(before);
+    expect(((await db.query('SELECT * FROM doc')) as unknown[][])[0]).toEqual([]);
+    await exec(rows.planCreateTx({ recordId: new RecordId('doc', 'd2'), mutationId: mid('5'), table: 'doc', data: { title: 'e', body: 'x' }, now: 6 }));
+    expect(await outbox()).toHaveLength(3);
+  });
+});

package/src/services/database/local-migrator.ts

@@ -15,6 +15,13 @@ export const sha1 = async (str: string): Promise<string> => {
     .join('');
 };
 
+/**
+ * Local-only state a schema change must not throw away: the outbox, the
+ * failed-mutation tray and debounced patches not yet in the outbox. Cached
+ * rows refill from the server; these exist nowhere else.
+ */
+const KEPT_TABLES = ['_00_pending_mutations', '_00_failed_mutations', '_00_pending_writes'] as const;
+
 export class LocalMigrator {
   private logger: Logger;
 
@@ -40,6 +47,7 @@ export class LocalMigrator {
       return;
     }
 
+    const kept = await this.readKeptRows();
     await this.recreateDatabase(database);
 
     const systemSchema = `
@@ -49,6 +57,7 @@ export class LocalMigrator {
       DEFINE TABLE IF NOT EXISTS _00_failed_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_pending_writes SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_blob SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
     `;
     const fullSchema = schemaSurql + '\n' + systemSchema;
@@ -88,9 +97,40 @@ export class LocalMigrator {
       }
     }
 
+    await this.restoreKeptRows(kept);
     await this.createHashRecord(hash);
   }
 
+  private async readKeptRows(): Promise<Array<[string, unknown[]]>> {
+    const kept: Array<[string, unknown[]]> = [];
+    for (const table of KEPT_TABLES) {
+      try {
+        const [rows] = await this.localDb.queryUngated<[unknown[]]>(`SELECT * FROM ${table}`);
+        if (Array.isArray(rows) && rows.length > 0) kept.push([table, rows]);
+      } catch {
+        // A fresh store has no such table yet: nothing to keep.
+      }
+    }
+    return kept;
+  }
+
+  private async restoreKeptRows(kept: Array<[string, unknown[]]>): Promise<void> {
+    for (const [table, rows] of kept) {
+      try {
+        await this.localDb.queryUngated(`INSERT INTO ${table} $rows`, { rows });
+        this.logger.info(
+          { table, count: rows.length, Category: 'sp00ky-client::LocalMigrator::provision' },
+          '[Provisioning] Kept local rows across the schema change'
+        );
+      } catch (error) {
+        this.logger.error(
+          { table, count: rows.length, error, Category: 'sp00ky-client::LocalMigrator::provision' },
+          '[Provisioning] Could not restore local rows after the schema change; they are lost'
+        );
+      }
+    }
+  }
+
   private async isSchemaUpToDate(hash: string): Promise<boolean> {
     try {
       const [lastSchemaRecord] = await this.localDb.queryUngated<any>(

package/src/services/database/sqlite-cache-engine.ts

@@ -66,6 +66,8 @@ const SYSTEM_TABLES = [
   '_00_failed_mutations',
   '_00_schema',
   '_00_pending_mutations',
+  // Debounced patches still inside their delay; read on every boot.
+  '_00_pending_writes',
   // Blob cache manifest. Read before its first write on every boot (reconcile
   // asks for the ids it found in OPFS), so it has to exist up front like the
   // rest — otherwise the very first reconcile throws and the cache runs cold.

package/src/state/client-state.ts

@@ -91,6 +91,8 @@ export interface PendingWrite {
   readonly data: Readonly<Record<string, unknown>>;
   readonly before: Readonly<Record<string, unknown>> | null;
   readonly firstAt: number;
+  /** Its durable copy in `_00_pending_writes`; null when `flushOnHide` is off. */
+  readonly mirrorId: string | null;
 }
 
 export interface SyncSlice {

package/src/state/reducers.test.ts

@@ -221,10 +221,12 @@ describe('outbox', () => {
     expect(R.outboxPruneAcked(100, 50)(s1)).toBe(s1);
   });
   it('pending writes merge per key and clear', () => {
-    const w = { key: 'k', table: 'thing', recordId: 'thing:1', data: { a: 1 }, before: { a: 0 }, firstAt: 1 };
+    const w = { key: 'k', table: 'thing', recordId: 'thing:1', data: { a: 1 }, before: { a: 0 }, firstAt: 1, mirrorId: null };
     let s = R.mergePendingWrite(w)(buildState());
-    s = R.mergePendingWrite({ ...w, data: { b: 2 }, before: null, firstAt: 9 })(s);
-    expect(s.pendingWrites.get('k')).toEqual({ ...w, data: { a: 1, b: 2 } });
+    s = R.mergePendingWrite({ ...w, data: { b: 2 }, before: null, firstAt: 9, mirrorId: '_00_pending_writes:m' })(s);
+    expect(s.pendingWrites.get('k')).toEqual({ ...w, data: { a: 1, b: 2 }, mirrorId: '_00_pending_writes:m' });
+    s = R.mergePendingWrite({ ...w, mirrorId: '_00_pending_writes:other' })(s);
+    expect(s.pendingWrites.get('k')!.mirrorId).toBe('_00_pending_writes:m');
     s = R.clearPendingWrite('k')(s);
     expect(s.pendingWrites.size).toBe(0);
     expect(R.clearPendingWrite('k')(s)).toBe(s);

package/src/state/reducers.ts

@@ -420,7 +420,7 @@ export const mergePendingWrite =
     const pendingWrites = new Map(s.pendingWrites);
     pendingWrites.set(
       write.key,
-      prev ? { ...prev, data: { ...prev.data, ...write.data } } : write
+      prev ? { ...prev, data: { ...prev.data, ...write.data }, mirrorId: prev.mirrorId ?? write.mirrorId } : write
     );
     return { ...s, pendingWrites };
   };

package/src/state/selectors.test.ts

@@ -158,7 +158,7 @@ describe('view index selectors', () => {
     const s = R.compose(
       R.reloadViews(new Map([['v', ['t:vouched']]])),
       R.outboxReplace([buildOutboxItem({ recordId: 't:outbox' })]),
-      (st) => ({ ...st, pendingWrites: new Map([['k', { key: 'k', table: 't', recordId: 't:debounced', data: {}, before: null, firstAt: 0 }]]) })
+      (st) => ({ ...st, pendingWrites: new Map([['k', { key: 'k', table: 't', recordId: 't:debounced', data: {}, before: null, firstAt: 0, mirrorId: null }]]) })
     )(buildState([e('a', { remoteArray: [['t:member', 1]], subqueryRemoteArray: [['t:child', 1]] })]));
     const keep = S.retained(s);
     for (const id of ['t:vouched', 't:member', 't:child', 't:outbox', 't:debounced', '_00_view:x']) expect(keep(id)).toBe(true);

package/src/sync/tabs.saga.test.ts

@@ -11,7 +11,7 @@ describe('setRole', () => {
   it('leader: loads the outbox and starts live/poll/registrations; follower: stops its timers; same role is a no-op', async () => {
     const leader = await runPure(setRole(env, 'leader'), { state: buildState(), handlers: { 'local.query': () => [] } });
     expect(leader.state.tabRole).toBe('leader');
-    expect(leader.dispatched).toEqual([{ type: 'LiveStart' }, { type: 'PollTick' }, { type: 'EnsureRegistered' }]);
+    expect(leader.dispatched).toEqual([{ type: 'AdoptPendingWrites' }, { type: 'LiveStart' }, { type: 'PollTick' }, { type: 'EnsureRegistered' }]);
     const follower = await runPure(setRole(env, 'follower'), { state: R.patchSync({ liveUuid: 'u' })(buildState()) });
     expect(follower.state.tabRole).toBe('follower');
     expect(follower.state.sync.liveUuid).toBeNull();

package/src/types.ts

@@ -839,4 +839,18 @@ export interface DebounceOptions {
   key?: 'recordId' | 'recordId_x_fields';
   /** The debounce delay in milliseconds. */
   delay?: number;
+  /**
+   * Keep the pending patch across a reload or a closed tab. Default `true`.
+   *
+   * Every call also writes the merged patch to `_00_pending_writes`, in the
+   * same local transaction as the row itself. When the page is hidden (tab
+   * switch, reload, close, app sent to the background) every pending patch is
+   * moved into the outbox and pushed at once instead of waiting out `delay`.
+   * A page that dies before that finishes leaves the patch in the store, and
+   * the next boot moves it into the outbox.
+   *
+   * `false` keeps the patch in memory only, so whatever is still inside its
+   * delay is lost on reload. Meant for throwaway state like a cursor position.
+   */
+  flushOnHide?: boolean;
 }