@spooky-sync/core 0.0.1-canary.208 → 0.0.1-canary.209

package/dist/sqlite-worker.js

@@ -1,6 +1,26 @@
 import { a as renderOrderSql, i as projectedDataSql, l as resolveRelations, n as openDb, o as renderWhereSql, r as project, s as reviveRow, u as stableKey } from "./sqlite-open.js";
 import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
 
+//#region src/services/database/sqlite-lock-verify.ts
+const LOCK_VERIFY_TIMEOUT_MS = 2e3;
+async function verifyLockStillHeld$1(locks, name, fence, timeoutMs = LOCK_VERIFY_TIMEOUT_MS, warn = () => {}) {
+	if (!locks || !name || typeof locks.query !== "function") return;
+	let timer;
+	try {
+		const state = await Promise.race([locks.query(), new Promise((resolve) => {
+			timer = setTimeout(() => resolve(null), timeoutMs);
+		})]);
+		if (state === null) {
+			warn(`lock verification did not answer within ${timeoutMs}ms; trusting the steal callback`);
+			return;
+		}
+		if (!(state.held ?? []).some((l) => l.name === name)) await fence("lock missing after suspected freeze");
+	} catch {} finally {
+		if (timer) clearTimeout(timer);
+	}
+}
+
+//#endregion
 //#region src/services/database/sqlite-select.ts
 function ensureTable(db, table) {
 	if (db.knownTables.has(table)) return;
@@ -230,13 +250,8 @@ function noteTick() {
 	});
 }
 setInterval(noteTick, 1e3);
-async function verifyLockStillHeld() {
-	const locks = workerLocks();
-	const name = currentLock?.name;
-	if (!locks || !name || typeof locks.query !== "function") return;
-	try {
-		if (!((await locks.query()).held ?? []).some((l) => l.name === name)) await fence("lock missing after suspected freeze");
-	} catch {}
+function verifyLockStillHeld() {
+	return verifyLockStillHeld$1(workerLocks(), currentLock?.name, (reason) => fence(reason), void 0, (msg) => console.warn(`[sqlite-worker] ${msg}`));
 }
 async function open(dbName, useOpfs, systemTables = [], workerLockName, openOptions) {
 	if (workerLockName) {

package/dist/types.d.ts

@@ -504,6 +504,16 @@ interface Sp00kyConfig<S extends SchemaStructure> {
      * failure the queue retries. `0` disables. Defaults to `60_000`.
      */
     queryTimeoutMs?: number;
+    /**
+     * Deadline (ms) for every LOCAL store operation (a SQLite worker round trip,
+     * or a query on the in-process surrealdb engine). Local ops are serialized
+     * too, and one that never answered - a worker starved behind a long select,
+     * a lock verification with no clock - left every `db.create`/`db.update`
+     * promise pending for the tab's lifetime. On expiry the call rejects with
+     * `LocalOpTimeoutError` (the op itself keeps running in the engine and is
+     * not retried). `0` disables. Defaults to `30_000`.
+     */
+    localOpTimeoutMs?: number;
   };
   /** The schema definition. */
   schema: S;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.208",
+  "version": "0.0.1-canary.209",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.208",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.208",
+    "@spooky-sync/query-builder": "0.0.1-canary.209",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.209",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "blurhash": "^2.0.5",

package/src/index.ts

@@ -23,6 +23,7 @@ export type {
   BlobUrlLease,
 } from './services/blobs/index';
 export { semverGt } from './utils/semver';
+export { LocalOpTimeoutError, DEFAULT_LOCAL_OP_TIMEOUT_MS } from './services/database/errors';
 export { fileToUint8Array, textToHtml } from './utils/index';
 export {
   blurhashSidecarPath,

package/src/modules/data/data.pending-ids.test.ts

@@ -123,3 +123,77 @@ describe('DataModule pending-id cache', () => {
     expect([...deletes]).toEqual(['game:b']);
   });
 });
+
+describe('DataModule pending-id cache: generations', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  // The outbox read is a round trip down the local op queue, so it can be
+  // queued BEFORE a create's outbox row commits and resolve AFTER
+  // `invalidatePendingIds()` ran for it. Cached as-is, that pre-create set
+  // hid the new row from every materialization that joined the read - the one
+  // stream update a CREATE gets, so the row was invisible until reload.
+  function makeDeferredModule() {
+    const pendingReads: Array<(rows: any[]) => void> = [];
+    const state = { reads: 0 };
+    const local = {
+      query: vi.fn(
+        () =>
+          new Promise<any[]>((resolve) => {
+            state.reads++;
+            pendingReads.push((rows) => resolve([rows]));
+          })
+      ),
+    };
+    const dm = new DataModule({} as any, local as any, { tables: [] } as any, makeLogger(), 100);
+    return { dm, state, pendingReads };
+  }
+  // Let the awaiting loop observe a resolved read and issue the next one.
+  const settle = async () => {
+    for (let i = 0; i < 8; i++) await Promise.resolve();
+  };
+
+  it('does not cache a read that was in flight when the outbox changed', async () => {
+    const { dm, state, pendingReads } = makeDeferredModule();
+    const first = dm.getPendingRecordIds();
+    expect(state.reads).toBe(1);
+    // The create commits and invalidates while the read is still out.
+    (dm as any).invalidatePendingIds();
+    // The stale read answers with the pre-create outbox...
+    pendingReads[0]!([]);
+    await settle();
+    // ...so the joiner re-read under the new generation and gets the fresh one.
+    expect(state.reads).toBe(2);
+    pendingReads[1]!([pending('new', 'create')]);
+    expect([...(await first).writes]).toEqual(['game:new']);
+  });
+
+  it('a stale in-flight result never repopulates the cache', async () => {
+    const { dm, state, pendingReads } = makeDeferredModule();
+    const first = dm.getPendingRecordIds();
+    (dm as any).invalidatePendingIds();
+    pendingReads[0]!([]);
+    await settle();
+    pendingReads[1]!([pending('new', 'create')]);
+    await first;
+    // A fresh call must be served from the cache the CURRENT generation's
+    // read wrote, i.e. see the new row without another read.
+    const again = await dm.getPendingRecordIds();
+    expect(state.reads).toBe(2);
+    expect([...again.writes]).toEqual(['game:new']);
+  });
+
+  it('bounds the re-read when the outbox changes on every tick', async () => {
+    const { dm, state, pendingReads } = makeDeferredModule();
+    const call = dm.getPendingRecordIds();
+    for (let i = 0; i < 5; i++) {
+      (dm as any).invalidatePendingIds();
+      pendingReads[i]?.([pending(`r${i}`)]);
+      await settle();
+    }
+    const result = await call;
+    // Three reads at most, and the caller still gets an answer.
+    expect(state.reads).toBeLessThanOrEqual(3);
+    expect(result.writes.size).toBe(1);
+  });
+});

package/src/modules/data/data.rematerialize.test.ts

@@ -0,0 +1,114 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import type { QueryState, RecordVersionArray } from '../../types';
+
+/**
+ * A row this client wrote itself reaches server membership at the very
+ * version the local CREATE memoized, so the sync engine rightly fetches
+ * nothing when the membership lands - and then nothing re-materialized the
+ * query: `remoteArray` gained the id and no subscriber heard about it until a
+ * reload rebuilt the view from membership. `scheduleRematerialize` closes that
+ * gap through the same per-query debounce a real stream update uses.
+ */
+
+const noop = () => {};
+function makeLogger(): any {
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+const schema = { tables: [{ name: 'message', columns: {} }] } as any;
+const plan: QueryPlan = { table: 'message', where: [['conversation', '=', 'conversation:c1']] } as any;
+
+function makeQueryState(hash: string, remoteArray: RecordVersionArray, localArray: RecordVersionArray): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM message WHERE conversation = $conversation;',
+      plan,
+      params: { conversation: 'conversation:c1' },
+      localArray,
+      remoteArray,
+      membershipKnown: true,
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'message',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  } as QueryState;
+}
+
+function setup(remoteArray: RecordVersionArray, localArray: RecordVersionArray) {
+  const body = { id: new RecordId('message', 'm1'), text: 'hi' };
+  const local: any = {
+    epoch: 1,
+    select: vi.fn(async (p: QueryPlan) => {
+      const ids = ((p as any).ids as RecordId[] | undefined) ?? [];
+      return ids.some((id) => id.toString() === 'message:m1') ? [body] : [];
+    }),
+    query: vi.fn(async (sql: string) => {
+      if (sql.includes('_00_pending_mutations')) return [[]];
+      return [[]];
+    }),
+  };
+  const dm = new DataModule({} as any, local, schema, makeLogger(), 10);
+  const hash = 'q1';
+  (dm as any).activeQueries.set(hash, makeQueryState(hash, remoteArray, localArray));
+  const subscriber = vi.fn();
+  (dm as any).subscriptions.set(hash, new Set([subscriber]));
+  return { dm, local, hash, subscriber };
+}
+
+describe('DataModule.scheduleRematerialize', () => {
+  afterEach(() => vi.useRealTimers());
+
+  it('notifies subscribers with the row once membership holds it', async () => {
+    vi.useFakeTimers();
+    const { dm, local, hash, subscriber } = setup([['message:m1', 1]], [['message:m1', 1]]);
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    expect(subscriber).toHaveBeenCalledTimes(1);
+    expect(subscriber.mock.calls[0]![0].map((r: any) => r.id.toString())).toEqual(['message:m1']);
+    // A synthetic update describes no ingest: nothing is persisted to _00_query.
+    expect(local.query.mock.calls.some(([sql]: [string]) => sql.includes('localArray'))).toBe(false);
+  });
+
+  it('is a no-op behind a pending real stream update', async () => {
+    vi.useFakeTimers();
+    const { dm, hash, subscriber } = setup([['message:m1', 1]], [['message:m1', 1]]);
+    await dm.onStreamUpdate({ queryHash: hash, localArray: [['message:m1', 1]], op: 'CREATE' });
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    // Exactly one notify: the real update's, not two.
+    expect(subscriber).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not notify when the records are unchanged', async () => {
+    vi.useFakeTimers();
+    const { dm, hash, subscriber } = setup([['message:m1', 1]], [['message:m1', 1]]);
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    expect(subscriber).toHaveBeenCalledTimes(1);
+  });
+
+  it('ignores an unknown query', () => {
+    const { dm } = setup([], []);
+    expect(() => dm.scheduleRematerialize('nope')).not.toThrow();
+  });
+});

package/src/modules/data/index.ts

@@ -428,6 +428,12 @@ export class DataModule<S extends SchemaStructure> {
       return;
     }
 
+    this.queueStreamUpdate(update);
+  }
+
+  /** Coalesce `update` onto the query's trailing timer (see onStreamUpdate). */
+  private queueStreamUpdate(update: StreamUpdate): void {
+    const { queryHash } = update;
     // Clear existing timer if any
     if (this.debounceTimers.has(queryHash)) {
       // oxlint-disable-next-line no-non-null-assertion -- guarded by .has() check above
@@ -445,6 +451,31 @@ export class DataModule<S extends SchemaStructure> {
     this.debounceTimers.set(queryHash, timer);
   }
 
+  /**
+   * Re-materialize + notify a query whose MEMBERSHIP changed without any row
+   * needing to be fetched, i.e. without the SSP stream update that normally
+   * carries the notify. That is every row this client wrote itself: the local
+   * CREATE memoized it at `_00_rv = 1`, the server publishes it at 1, so the
+   * sync engine rightly fetches nothing - and then nobody told the subscribers
+   * that `remoteArray` now holds the id. The row appeared on reload only.
+   *
+   * Routed through the same per-query debounce as a real stream update, so it
+   * cannot race one: a pending real update already materializes against the
+   * current `remoteArray` and wins. The synthetic update re-uses the circuit's
+   * last `localArray` and skips the persist/metrics that describe an ingest.
+   */
+  scheduleRematerialize(queryHash: string): void {
+    if (this.debounceTimers.has(queryHash)) return;
+    const queryState = this.activeQueries.get(queryHash);
+    if (!queryState) return;
+    this.queueStreamUpdate({
+      queryHash,
+      localArray: queryState.config.localArray ?? [],
+      op: 'UPDATE',
+      synthetic: true,
+    });
+  }
+
   /**
    * Process a query's pending (debounced) stream update NOW instead of on the
    * trailing edge. Called by the sync engine before it flips a query back to
@@ -614,11 +645,24 @@ export class DataModule<S extends SchemaStructure> {
   private pendingIdsAt = 0;
   private pendingIdsInflight: Promise<{ writes: Set<string>; deletes: Set<string> }> | null = null;
   private static readonly PENDING_IDS_TTL_MS = 250;
+  // Bumped by every invalidation. A read carries the generation it started
+  // under, and a result from an older generation is neither cached nor
+  // returned to a caller that asked after the invalidation: it was taken
+  // before the outbox row it needs existed. Without this, a materialization
+  // that joined a read already in flight when `create()` invalidated saw the
+  // pre-create set, found "no change", and skipped its notify - the one
+  // stream update a CREATE gets, so the row stayed invisible until reload.
+  private pendingIdsGen = 0;
+  private static readonly PENDING_IDS_MAX_REREADS = 3;
 
   /** Drop the cached outbox ids. Cheap; call it on anything that could change
    *  `_00_pending_mutations`. */
   private invalidatePendingIds(): void {
     this.pendingIds = null;
+    this.pendingIdsGen++;
+    // The in-flight read (if any) predates this invalidation; whoever joins
+    // next starts a fresh one. Its own joiners re-read via the generation check.
+    this.pendingIdsInflight = null;
   }
 
   /**
@@ -770,7 +814,7 @@ export class DataModule<S extends SchemaStructure> {
       // normal re-query path.
       const newRecords = await this.materializeRecords(queryState, localArray);
       if (epoch !== this.local.epoch) return;
-      queryState.config.localArray = localArray;
+      if (!update.synthetic) queryState.config.localArray = localArray;
 
       const prevJson = JSON.stringify(queryState.records);
       const newJson = JSON.stringify(newRecords);
@@ -785,30 +829,34 @@ export class DataModule<S extends SchemaStructure> {
         queryState.lastUpdatedAt = Date.now();
       }
 
-      await this.local.query(
-        surql.seal(
-          surql.updateSet('id', [
-            'localArray',
-            'rowCount',
-            'updateCount',
-            'lastIngestLatency',
-            'materializationP55',
-            'materializationP90',
-            'materializationP99',
-          ])
-        ),
-        {
-          id: queryState.config.id,
-          localArray,
-          rowCount: localArray.length,
-          updateCount: queryState.updateCount,
-          lastIngestLatency: queryState.lastIngestLatencyMs,
-          materializationP55: percentiles.p55,
-          materializationP90: percentiles.p90,
-          materializationP99: percentiles.p99,
-        },
-        { epoch }
-      );
+      // A synthetic re-materialize did not come from an ingest: nothing about
+      // the circuit's view changed, so there is nothing to persist for it.
+      if (!update.synthetic) {
+        await this.local.query(
+          surql.seal(
+            surql.updateSet('id', [
+              'localArray',
+              'rowCount',
+              'updateCount',
+              'lastIngestLatency',
+              'materializationP55',
+              'materializationP90',
+              'materializationP99',
+            ])
+          ),
+          {
+            id: queryState.config.id,
+            localArray,
+            rowCount: localArray.length,
+            updateCount: queryState.updateCount,
+            lastIngestLatency: queryState.lastIngestLatencyMs,
+            materializationP55: percentiles.p55,
+            materializationP90: percentiles.p90,
+            materializationP99: percentiles.p99,
+          },
+          { epoch }
+        );
+      }
 
       if (!recordsChanged) {
         this.logger.debug(
@@ -1249,18 +1297,32 @@ export class DataModule<S extends SchemaStructure> {
     }
     // Single-flight: one ingest fans out to many queries materializing at once,
     // and without this they all queue their own identical read behind each other.
-    if (!this.pendingIdsInflight) {
-      this.pendingIdsInflight = this.readPendingRecordIds().finally(() => {
-        this.pendingIdsInflight = null;
-      });
+    // Bounded re-read: a result from a generation older than the one current
+    // by the time it lands was taken before some outbox row existed, so it is
+    // re-issued rather than returned. Bounded, because a write on every tick
+    // must not spin this forever - the last read is returned then.
+    let fresh: { writes: Set<string>; deletes: Set<string> } | null = null;
+    for (let attempt = 0; attempt < DataModule.PENDING_IDS_MAX_REREADS; attempt++) {
+      const gen = this.pendingIdsGen;
+      if (!this.pendingIdsInflight) {
+        const read = this.readPendingRecordIds(gen).finally(() => {
+          // Only clear the slot this read still owns; an invalidation in the
+          // meantime has replaced it (with null) for a newer read to fill.
+          if (this.pendingIdsInflight === read) this.pendingIdsInflight = null;
+        });
+        this.pendingIdsInflight = read;
+      }
+      fresh = await this.pendingIdsInflight;
+      if (gen === this.pendingIdsGen) break;
     }
-    const fresh = await this.pendingIdsInflight;
-    return { writes: new Set(fresh.writes), deletes: new Set(fresh.deletes) };
+    // oxlint-disable-next-line no-non-null-assertion -- the loop runs at least once
+    return { writes: new Set(fresh!.writes), deletes: new Set(fresh!.deletes) };
   }
 
   /** The uncached read. Also the reload path after an invalidation, so the ids
-   *  still survive a reload exactly as before. */
-  private async readPendingRecordIds(): Promise<{
+   *  still survive a reload exactly as before. `gen` is the generation the read
+   *  was issued under; the result is cached only if it is still current. */
+  private async readPendingRecordIds(gen: number): Promise<{
     writes: Set<string>;
     deletes: Set<string>;
   }> {
@@ -1285,8 +1347,10 @@ export class DataModule<S extends SchemaStructure> {
       // not a statement that the outbox is empty.
       return { writes, deletes };
     }
-    this.pendingIds = { writes, deletes };
-    this.pendingIdsAt = Date.now();
+    if (gen === this.pendingIdsGen) {
+      this.pendingIds = { writes, deletes };
+      this.pendingIdsAt = Date.now();
+    }
     return { writes, deletes };
   }
 
@@ -1757,18 +1821,36 @@ export class DataModule<S extends SchemaStructure> {
       })
     );
 
+    // The local tx has committed: the row and its outbox entry exist. The
+    // cached pending-id sets no longer describe the outbox, so drop them HERE,
+    // before the ingest below fans out to every view's materialization (which
+    // is exactly when they are read).
+    this.invalidatePendingIds();
+
     const parsedRecord = parseParams(tableSchema.columns, target) as RecordWithId;
 
-    // Save to cache (which handles DBSP ingestion)
-    await this.cache.save(
-      {
-        table: tableName,
-        op: 'CREATE',
-        record: parsedRecord,
-        version: 1,
-      },
-      true
-    );
+    // Save to cache (which handles DBSP ingestion). Best-effort, like
+    // `delete()`: the local write is durable and the outbox row exists, so a
+    // throw from the in-browser ingest must not abort the rest of this method -
+    // it used to, and then the mutation was never handed to sync, the row sat
+    // in the outbox until the next leader promotion re-scanned it, and the
+    // caller's promise rejected after the write had already happened.
+    try {
+      await this.cache.save(
+        {
+          table: tableName,
+          op: 'CREATE',
+          record: parsedRecord,
+          version: 1,
+        },
+        true
+      );
+    } catch (err) {
+      this.logger.error(
+        { err, id, Category: 'sp00ky-client::DataModule::create' },
+        'SSP create-ingest failed; the row is written and queued, but views only see it once membership arrives'
+      );
+    }
 
     // Emit mutation event for sync
     const mutationEvent: CreateEvent = {
@@ -1780,8 +1862,6 @@ export class DataModule<S extends SchemaStructure> {
       tableName,
     };
 
-    // A new outbox row: the cached pending-id sets no longer describe it.
-    this.invalidatePendingIds();
     for (const callback of this.mutationCallbacks) {
       callback([mutationEvent]);
     }
@@ -1849,18 +1929,28 @@ export class DataModule<S extends SchemaStructure> {
     }
     this.replaceRecordInQueries(updatedFields);
 
+    // Committed: see create() for why this precedes the ingest.
+    this.invalidatePendingIds();
+
     const parsedRecord = parseParams(tableSchema.columns, target) as RecordWithId;
 
-    // Save to cache
-    await this.cache.save(
-      {
-        table: table,
-        op: 'UPDATE',
-        record: parsedRecord,
-        version: target._00_rv as number,
-      },
-      true
-    );
+    // Save to cache. Best-effort for the same reason as in create().
+    try {
+      await this.cache.save(
+        {
+          table: table,
+          op: 'UPDATE',
+          record: parsedRecord,
+          version: target._00_rv as number,
+        },
+        true
+      );
+    } catch (err) {
+      this.logger.error(
+        { err, id, Category: 'sp00ky-client::DataModule::update' },
+        'SSP update-ingest failed; the row is written and queued'
+      );
+    }
 
     const pushEventOptions = parseUpdateOptions(id, data, options);
 
@@ -1875,8 +1965,6 @@ export class DataModule<S extends SchemaStructure> {
       options: pushEventOptions,
     };
 
-    // A new outbox row: the cached pending-id sets no longer describe it.
-    this.invalidatePendingIds();
     for (const callback of this.mutationCallbacks) {
       callback([mutationEvent]);
     }
@@ -1912,6 +2000,10 @@ export class DataModule<S extends SchemaStructure> {
 
     await withRetry(this.logger, () => this.local.execute(query, { id: rid, mid: mutationId }));
 
+    // Committed: the outbox row exists, drop the cached pending-id sets before
+    // the re-materialize below reads them (see create()).
+    this.invalidatePendingIds();
+
     // The local DELETE has now committed. Everything below must reflect that in
     // active live queries — so the deleted row disappears optimistically without
     // a reload — even if the optimistic SSP-view ingest below fails. Previously a
@@ -1940,8 +2032,6 @@ export class DataModule<S extends SchemaStructure> {
       record_id: rid,
     };
 
-    // A new outbox row: the cached pending-id sets no longer describe it.
-    this.invalidatePendingIds();
     for (const callback of this.mutationCallbacks) {
       callback([mutationEvent]);
     }