@spooky-sync/core 0.0.1-canary.204 → 0.0.1-canary.206

package/src/modules/sync/engine.ts

@@ -8,6 +8,9 @@ import { SyncEventTypes, createSyncEventSystem } from './events/index';
 import { encodeRecordId } from '../../utils/index';
 import { cleanRecord } from '../../utils/parser';
 
+/** Ids per remote `SELECT * FROM $ids` round trip. */
+const FETCH_CHUNK = 500;
+
 /**
  * SyncEngine handles the core sync operations: fetching remote records,
  * caching them locally, and ingesting into DBSP.
@@ -55,10 +58,16 @@ export class SyncEngine {
       stillRemoteIds = await this.handleRemovedRecords(removed);
     }
 
-    // Fetch added/updated records from remote
-    const toFetch = [...added, ...updated];
-    const idsToFetch = toFetch.map((x) => x.id);
-    if (idsToFetch.length === 0) {
+    // Fetch added/updated records from remote. Skip ids whose body the local
+    // store already holds at this version (the circuit prime seeded the memo
+    // from `_00_rv`): after a reload every id in the server's list_ref used to
+    // classify as `added` against an empty circuit, and this fetched the
+    // whole working set again.
+    const toFetch = [...added, ...updated].filter((item) => {
+      const localVersion = this.cache.lookup(encodeRecordId(item.id));
+      return !(localVersion && item.version <= localVersion);
+    });
+    if (toFetch.length === 0) {
       return { remoteFetchMs: 0, stillRemoteIds };
     }
 
@@ -67,66 +76,75 @@ export class SyncEngine {
     for (const item of toFetch) {
       versionMap.set(encodeRecordId(item.id), item.version);
     }
+    const addedIds = new Set(added.map((item) => encodeRecordId(item.id)));
 
     // Fetch records from remote — avoid SELECT *, <subquery> FROM $param
     // pattern which drops the * fields in SurrealDB v3 (known bug).
     // Versions are already known from the diff's list_ref data.
-    const remoteFetchStart = performance.now();
-    const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
-      'SELECT * FROM $idsToFetch',
-      { idsToFetch }
-    );
-    const remoteFetchMs = performance.now() - remoteFetchStart;
-
-    // Prepare batch for cache (which handles both DB and DBSP)
-    const cacheBatch: CacheRecord[] = [];
-
-    for (const record of remoteResults) {
-      if (!record?.id) {
-        this.logger.warn(
-          {
-            record,
-            idsToFetch,
-            Category: 'sp00ky-client::SyncEngine::syncRecords',
-          },
-          'Remote record has no id (possibly deleted). Skipping record'
-        );
-        continue;
-      }
-      const fullId = encodeRecordId(record.id);
-      const table = record.id.table.toString();
-      const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
-      const version = versionMap.get(fullId) ?? 0;
-
-      const localVersion = this.cache.lookup(fullId);
-      if (localVersion && version <= localVersion) {
-        this.logger.info(
-          {
-            recordId: fullId,
-            version,
-            localVersion,
-            Category: 'sp00ky-client::SyncEngine::syncRecords',
-          },
-          'Local version is higher than remote version. Skipping record'
-        );
-        continue;
+    //
+    // Chunked so a genuinely cold load lands progressively: one response
+    // holding thousands of bodies, then one transaction MERGEing all of them,
+    // was a single main-thread stall and a memory spike on top of it.
+    let remoteFetchMs = 0;
+    const remoteResults: RecordWithId[] = [];
+    for (let i = 0; i < toFetch.length; i += FETCH_CHUNK) {
+      const idsToFetch = toFetch.slice(i, i + FETCH_CHUNK).map((x) => x.id);
+      const remoteFetchStart = performance.now();
+      const [chunkResults] = await this.remote.query<[RecordWithId[]]>(
+        'SELECT * FROM $idsToFetch',
+        { idsToFetch }
+      );
+      remoteFetchMs += performance.now() - remoteFetchStart;
+
+      // Prepare batch for cache (which handles both DB and DBSP)
+      const cacheBatch: CacheRecord[] = [];
+      for (const record of chunkResults ?? []) {
+        if (!record?.id) {
+          this.logger.warn(
+            {
+              record,
+              idsToFetch,
+              Category: 'sp00ky-client::SyncEngine::syncRecords',
+            },
+            'Remote record has no id (possibly deleted). Skipping record'
+          );
+          continue;
+        }
+        remoteResults.push(record);
+        const fullId = encodeRecordId(record.id);
+        const table = record.id.table.toString();
+        const version = versionMap.get(fullId) ?? 0;
+
+        const localVersion = this.cache.lookup(fullId);
+        if (localVersion && version <= localVersion) {
+          this.logger.info(
+            {
+              recordId: fullId,
+              version,
+              localVersion,
+              Category: 'sp00ky-client::SyncEngine::syncRecords',
+            },
+            'Local version is higher than remote version. Skipping record'
+          );
+          continue;
+        }
+        const tableSchema = this.schema.tables.find((t) => t.name === table);
+        const cleanedRecord = tableSchema
+          ? cleanRecord(tableSchema.columns, record)
+          : record;
+
+        cacheBatch.push({
+          table,
+          op: addedIds.has(fullId) ? 'CREATE' : 'UPDATE',
+          record: cleanedRecord as RecordWithId,
+          version,
+        });
       }
-      const tableSchema = this.schema.tables.find((t) => t.name === table);
-      const cleanedRecord = tableSchema
-        ? cleanRecord(tableSchema.columns, record)
-        : record;
-
-      cacheBatch.push({
-        table,
-        op: isAdded ? 'CREATE' : 'UPDATE',
-        record: cleanedRecord as RecordWithId,
-        version,
-      });
-    }
 
-    // Use CacheModule to handle both local DB and DBSP ingestion
-    if (cacheBatch.length > 0) {
-      await this.cache.saveBatch(cacheBatch);
+      // Use CacheModule to handle both local DB and DBSP ingestion
+      if (cacheBatch.length > 0) {
+        await this.cache.saveBatch(cacheBatch);
+      }
     }
 
     this.events.emit(SyncEventTypes.RemoteDataIngested, {

package/src/modules/sync/sync.ts

@@ -527,14 +527,40 @@ export class Sp00kySync<S extends SchemaStructure> {
     this.tabId = tabId;
   }
 
-  /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
-   *  relay LIVE events and rollbacks to followers via `hub`. Idempotent for a
-   *  boot-time leader; a runtime promotion (failover) reloads the outbox,
-   *  which now holds EVERY tab's rows, and restarts LIVE under this session. */
-  public async promoteToLeader(hub: LeaderSyncHub): Promise<void> {
+  /** In-flight {@link resumeLeaderDuties}, so a second call joins the first
+   *  instead of double-draining the outbox. */
+  private leaderDutiesInFlight: Promise<void> | null = null;
+
+  /** Resolves once the in-browser circuit has been primed from the local
+   *  store. Every sync diff waits on it: diffing against an empty circuit
+   *  classifies the whole working set as missing and re-downloads it. */
+  private primeGate: () => Promise<void> = () => Promise.resolve();
+  /** The prime we last waited on. `whenPrimed` hands out one promise per
+   *  prime, so a new identity means a new prime (boot, bucket switch) ran. */
+  private settledPrime: Promise<void> | null = null;
+
+  setPrimeGate(gate: () => Promise<void>): void {
+    this.primeGate = gate;
+    this.settledPrime = null;
+  }
+
+  /**
+   * Leader WIRING only, and deliberately synchronous.
+   *
+   * The coordinator publishes the leader role and tells the broker
+   * `leader-ready` the moment the store is adopted, and the broker can mint a
+   * follower's ports on the very next tick. So the follower-message handler
+   * has to be live before this returns, or a mutation forwarded in that window
+   * is dropped. Everything that can block (outbox reload, LIVE restart) moved
+   * to {@link resumeLeaderDuties}: a promotion that waits on the network holds
+   * `leader-ready` back, and a broker whose leader never reports ready serves
+   * no follower ports and re-elects no one, which wedges the whole namespace.
+   */
+  public promoteToLeader(hub: LeaderSyncHub): void {
     this.tabRole = 'leader';
     this.hub = hub;
     this.forwarder = null;
+    this.leaderDutiesInFlight = null;
     hub.onFollowerMessage = (tabId, msg) => {
       void tabId;
       switch (msg.type) {
@@ -548,19 +574,34 @@ export class Sp00kySync<S extends SchemaStructure> {
           break;
       }
     };
-    if (this.isInit) {
+  }
+
+  /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
+   *  relay LIVE events and rollbacks to followers. Idempotent for a boot-time
+   *  leader; a runtime promotion (failover) reloads the outbox, which now
+   *  holds EVERY tab's rows, and restarts LIVE under this session. Runs in the
+   *  background off the promotion path, so however long it takes (or if it
+   *  never finishes) the tab is already a working leader. */
+  public resumeLeaderDuties(): Promise<void> {
+    if (this.leaderDutiesInFlight) return this.leaderDutiesInFlight;
+    const run = (async () => {
+      if (!this.isInit || this.tabRole !== 'leader') return;
       await this.upQueue.loadFromDatabase();
       void this.scheduler.syncUp();
       if (this.currentUserId || this.anonLiveEnabled) {
         this.startListRefPoll();
         await this.restartRefLiveQuery().catch((err) => {
           this.logger.warn(
-            { err, Category: 'sp00ky-client::Sp00kySync::promoteToLeader' },
+            { err, Category: 'sp00ky-client::Sp00kySync::resumeLeaderDuties' },
             'LIVE restart failed on promotion; poll fallback covers it'
           );
         });
       }
-    }
+    })();
+    this.leaderDutiesInFlight = run;
+    return run.finally(() => {
+      if (this.leaderDutiesInFlight === run) this.leaderDutiesInFlight = null;
+    });
   }
 
   /** Follower duties: no outbox drain, no LIVE. Mutations forward to the
@@ -570,6 +611,9 @@ export class Sp00kySync<S extends SchemaStructure> {
     this.tabRole = 'follower';
     this.hub = null;
     this.forwarder = forwarder;
+    // A still-running resumeLeaderDuties self-cancels on its `tabRole` guard;
+    // drop the handle so a later re-promotion starts a fresh drain.
+    this.leaderDutiesInFlight = null;
     void this.killRefLiveQuery();
     forwarder.onLeaderMessage = (msg) => {
       switch (msg.type) {
@@ -1531,6 +1575,19 @@ export class Sp00kySync<S extends SchemaStructure> {
    * means the single resulting UI update lands after this completes.
    */
   private async runSyncForQuery(hash: string, diff: RecordVersionDiff): Promise<void> {
+    // The diff was computed against whatever the circuit held at call time; if
+    // the boot prime is still filling it, wait and recompute from the primed
+    // `localArray` so the delta is real rather than "everything".
+    const prime = this.primeGate();
+    if (prime !== this.settledPrime) {
+      await prime;
+      this.settledPrime = prime;
+      const fresh = this.dataModule.getQueryByHash(hash);
+      if (!fresh) return;
+      const recomputed = new ArraySyncer(fresh.config.localArray, fresh.config.remoteArray).nextSet();
+      if (!recomputed) return;
+      diff = recomputed;
+    }
     // Don't let sync re-add a record the user just deleted locally. The remote
     // delete is queued in the outbox, so until it's processed the server's
     // `_00_list_ref` still lists the record — the diff then classifies it as

package/src/services/database/cache-engine.ts

@@ -138,6 +138,39 @@ export interface LocalStore extends LocalCacheEngine {
   /** Fires immediately with the current snapshot, then on every change.
    *  Returns an unsubscribe function. */
   subscribeToStorageHealth?(cb: (health: StorageHealth) => void): () => void;
+  /**
+   * Every cached row's `(id, _00_rv)` per table, ids in stable `table:id`
+   * form. The in-browser circuit primes and reconciles itself from this on
+   * boot instead of re-downloading the working set. OPTIONAL: an engine
+   * without it boots the circuit empty, as before.
+   */
+  scanVersions?(tables: string[]): Promise<Record<string, [string, number][]>>;
+  /**
+   * Circuit snapshot storage, keyed. Lives in the same durable store as the
+   * rows (OPFS SQLite), so it is per-bucket by construction, atomic, and
+   * readable by follower tabs over the port transport. OPTIONAL: an engine
+   * without it primes from `scanVersions` alone.
+   */
+  getSnapshot?(key: string): Promise<StoredSnapshot | null>;
+  putSnapshot?(key: string, bytes: Uint8Array, meta: SnapshotMeta): Promise<void>;
+  deleteSnapshot?(key: string): Promise<void>;
+}
+
+/** Describes a stored circuit snapshot; what decides whether it is usable. */
+export interface SnapshotMeta {
+  /** Bump when the wire shape the circuit reads changes. */
+  formatVersion: number;
+  /** Hash of the schema the rows were projected under. */
+  schemaHash: string;
+  savedAt: number;
+  /** Highest `_00_rv` per table at save time (diagnostics). */
+  maxRv?: Record<string, number>;
+  [key: string]: unknown;
+}
+
+export interface StoredSnapshot {
+  bytes: Uint8Array;
+  meta: SnapshotMeta;
 }
 
 /** Selected local cache backend. Mirrors the `persistenceClient` config pattern. */

package/src/services/database/sqlite-cache-engine.test.ts

@@ -556,3 +556,61 @@ describe('create() tx result shaping (fast path parity)', () => {
     expect(outboxRow.data).toEqual(payload);
   });
 });
+
+
+// The circuit snapshot + version scan the boot-time prime reads through this
+// engine. Bytes and meta must round-trip as ONE batch (a reader must never see
+// new bytes with old meta) and a table that does not exist yet scans as empty.
+describe('SqliteCacheEngine circuit snapshot + scanVersions', () => {
+  it('stores bytes and meta in one batch and reads them back', async () => {
+    const stored = new Map<string, unknown>();
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
+    const types: string[] = [];
+    stubTransport(engine, (type: string, payload: any) => {
+      types.push(type);
+      if (type === 'open') return { persisted: true };
+      if (type === 'batch') {
+        for (const stmt of payload as { sql: string; bind: unknown[] }[]) {
+          stored.set(stmt.bind[0] as string, stmt.bind[1]);
+        }
+        return {};
+      }
+      if (type === 'exec') {
+        const ids = (payload.bind ?? []) as string[];
+        return { rows: ids.filter((id) => stored.has(id)).map((id) => ({ id, data: stored.get(id) })) };
+      }
+      return {};
+    });
+    await engine.connect('anon');
+
+    expect(await engine.getSnapshot('circuit')).toBeNull();
+    const bytes = new Uint8Array([1, 2, 3]);
+    await engine.putSnapshot('circuit', bytes, { formatVersion: 1, schemaHash: 'h', savedAt: 5 });
+    expect(types.filter((t) => t === 'batch')).toHaveLength(1);
+    const snap = await engine.getSnapshot('circuit');
+    expect(snap?.bytes).toEqual(bytes);
+    expect(snap?.meta).toEqual({ formatVersion: 1, schemaHash: 'h', savedAt: 5 });
+  });
+
+  it('scans (id, rv) per table and reads a missing table as empty', async () => {
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
+    stubTransport(engine, (type: string, payload: any) => {
+      if (type === 'open') return { persisted: true };
+      if (type === 'exec') {
+        if (String(payload.sql).includes('"thing"')) {
+          return { rows: [{ id: 'thing:a', rv: 3 }, { id: 'thing:b', rv: null }] };
+        }
+        throw new Error('no such table');
+      }
+      return {};
+    });
+    await engine.connect('anon');
+    expect(await engine.scanVersions(['thing', 'ghost'])).toEqual({
+      thing: [
+        ['thing:a', 3],
+        ['thing:b', 0],
+      ],
+      ghost: [],
+    });
+  });
+});

package/src/services/database/sqlite-cache-engine.ts

@@ -10,6 +10,7 @@ import {
 } from './sqlite-plan-sql';
 import type { Logger } from '../logger/index';
 import type { Sp00kyConfig, StorageHealth } from '../../types';
+import type { SnapshotMeta, StoredSnapshot } from './cache-engine';
 import type { SealedQuery } from '../../utils/surql';
 import { resolveRelations, stableKey } from './relation-resolver';
 import {
@@ -50,8 +51,13 @@ import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-in
  * on a fresh bucket throws "no such table". Keep in sync with the systemSchema
  * block in `local-migrator.ts`.
  */
+const SNAPSHOT_TABLE = '_00_circuit_snapshot';
+
 const SYSTEM_TABLES = [
   '_00_stream_processor_state',
+  // In-browser circuit snapshot: one BLOB row (`circuit`) + a JSON meta row.
+  // Excluded from DevTools' table listing, since its data column is not JSON.
+  '_00_circuit_snapshot',
   '_00_query',
   '_00_preload',
   '_00_window',
@@ -197,7 +203,7 @@ export class SqliteCacheEngine implements LocalStore {
         const { rows: tables } = await this.call<{ rows: { name: string }[] }>('exec', {
           sql: "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name",
         });
-        const names = (tables ?? []).map((r) => r.name);
+        const names = (tables ?? []).map((r) => r.name).filter((n) => n !== SNAPSHOT_TABLE);
         if (names.length) {
           // Names come from sqlite_master itself; double-quoting is enough.
           const sql = names
@@ -837,6 +843,70 @@ export class SqliteCacheEngine implements LocalStore {
     return rows[0] ?? null;
   }
 
+  /**
+   * `(id, _00_rv)` of every row per table, straight off the worker: no body
+   * parse, one round trip per table. A table that does not exist yet reads as
+   * empty rather than failing the whole scan.
+   */
+  async scanVersions(tables: string[]): Promise<Record<string, [string, number][]>> {
+    const out: Record<string, [string, number][]> = {};
+    for (const table of tables) {
+      try {
+        const { rows } = await this.call<{ rows: { id: string; rv: unknown }[] }>('exec', {
+          sql: `SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table.replace(/"/g, '""')}"`,
+        });
+        out[table] = (rows ?? []).map((r) => [r.id, Number(r.rv) || 0]);
+      } catch {
+        out[table] = [];
+      }
+    }
+    return out;
+  }
+
+  async getSnapshot(key: string): Promise<StoredSnapshot | null> {
+    const { rows } = await this.call<{ rows: { id: string; data: unknown }[] }>('exec', {
+      sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
+      bind: [key, `${key}:meta`],
+    });
+    let bytes: Uint8Array | null = null;
+    let meta: SnapshotMeta | null = null;
+    for (const r of rows ?? []) {
+      if (r.id === key) {
+        if (r.data instanceof Uint8Array) bytes = r.data;
+        else if (r.data instanceof ArrayBuffer) bytes = new Uint8Array(r.data);
+        else if (typeof r.data === 'string') bytes = new TextEncoder().encode(r.data);
+      } else if (typeof r.data === 'string') {
+        try {
+          meta = JSON.parse(r.data) as SnapshotMeta;
+        } catch {
+          meta = null;
+        }
+      }
+    }
+    return bytes && meta ? { bytes, meta } : null;
+  }
+
+  async putSnapshot(key: string, bytes: Uint8Array, meta: SnapshotMeta): Promise<void> {
+    // One transaction, so a reader never sees new bytes with old meta.
+    await this.call('batch', [
+      {
+        sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
+        bind: [key, bytes],
+      },
+      {
+        sql: `INSERT OR REPLACE INTO "${SNAPSHOT_TABLE}" (id, data) VALUES (?, ?)`,
+        bind: [`${key}:meta`, JSON.stringify(meta)],
+      },
+    ]);
+  }
+
+  async deleteSnapshot(key: string): Promise<void> {
+    await this.call('run', {
+      sql: `DELETE FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
+      bind: [key, `${key}:meta`],
+    });
+  }
+
   // ---- writes --------------------------------------------------------------
 
   async upsert(table: string, id: Id, data: Row, mode: 'replace' | 'merge'): Promise<void> {

package/src/services/database/surreal-cache-engine.ts

@@ -99,6 +99,26 @@ export class SurrealCacheEngine extends LocalDatabaseService implements LocalCac
     return new RecordId(table, raw);
   }
 
+  /**
+   * `(id, _00_rv)` of every row per table. Runs on the main thread against the
+   * embedded engine (no worker here), which is why the circuit prime caps how
+   * much it is willing to read through this path.
+   */
+  async scanVersions(tables: string[]): Promise<Record<string, [string, number][]>> {
+    const out: Record<string, [string, number][]> = {};
+    for (const table of tables) {
+      try {
+        const [rows] = await this.query<[{ id: Id; _00_rv?: unknown }[]]>(
+          `SELECT id, _00_rv FROM ${table};`
+        );
+        out[table] = (rows ?? []).map((r) => [stableKey(r.id), Number(r._00_rv) || 0]);
+      } catch {
+        out[table] = [];
+      }
+    }
+    return out;
+  }
+
   async getById(table: string, id: Id): Promise<Row | null> {
     const [row] = await this.query<[Row | null]>('SELECT * FROM ONLY $__id;', {
       __id: this.toRecordId(table, id),