@spooky-sync/core 0.0.1-canary.204 → 0.0.1-canary.205

package/src/services/tabs/coordinator.ts

@@ -53,7 +53,12 @@ export interface CoordinatorHooks {
   exposeClientPort(clientId: string, port: MessagePort): Promise<void>;
   removeClientPort(clientId: string): Promise<void>;
   /** Sync layer role changes (implemented in the sync module). */
-  becomeSyncLeader(hub: LeaderSyncHub): Promise<void>;
+  /** Synchronous leader WIRING. Must not block: the role is published and
+   *  `leader-ready` is sent right after it returns. */
+  becomeSyncLeader(hub: LeaderSyncHub): void;
+  /** The blocking half of the leader handover (outbox drain, LIVE restart),
+   *  run in the background once the tab is already a working leader. */
+  resumeSyncLeaderDuties(): Promise<void>;
   becomeSyncFollower(forwarder: SyncForwarder): void;
   becomeSyncSolo(): void;
   /** Current storage health, for db-ready sent to late-joining followers. */
@@ -208,6 +213,10 @@ export class TabsCoordinator {
   leadershipId = 0;
   leaderTabId: TabId | null = null;
   brokerRole: 'pending' | TabRole = 'pending';
+  /** Wall time from `become-leader` to `leader-ready` on the last promotion.
+   *  Reported in the DevTools role-change event: a slow store adopt (OPFS pool
+   *  contention after the previous leader died) shows up here as seconds. */
+  lastPromotionMs: number | null = null;
   private fingerprint: string;
   private bucketId: string;
   private broker: TabBrokerClient;
@@ -285,7 +294,12 @@ export class TabsCoordinator {
     if (this.role !== role) {
       this.role = role;
       this.deps.logger.info(
-        { role, leadershipId: this.leadershipId, Category: 'sp00ky-client::TabsCoordinator' },
+        {
+          role,
+          leadershipId: this.leadershipId,
+          promotionMs: role === 'leader' ? this.lastPromotionMs : undefined,
+          Category: 'sp00ky-client::TabsCoordinator',
+        },
         'Tab role changed'
       );
       for (const cb of this.roleListeners) cb(role);
@@ -379,6 +393,7 @@ export class TabsCoordinator {
     if (this.closed) return;
     if (msg.leadershipId <= this.leadershipId && !msg.resumeHeld) return;
     const previousRole = this.role;
+    const startedAt = Date.now();
     try {
       if (previousRole === 'follower') this.teardownFollower('promoted');
       this.leadershipId = msg.leadershipId;
@@ -408,7 +423,16 @@ export class TabsCoordinator {
       });
       void health;
       this.hub = new LeaderSyncHub(msg.leadershipId, this.deps.logger);
-      await this.deps.hooks.becomeSyncLeader(this.hub);
+      // Owning the store IS being the leader: it unparks this tab's ops and is
+      // everything a follower's ports need. Publish the role and tell the
+      // broker BEFORE any sync work. Waiting on the outbox drain and the LIVE
+      // restart here once left a tab that owned the OPFS pool but never sent
+      // `leader-ready`, so the broker minted no follower ports and re-elected
+      // no one (`assignFollowerPorts` and `electIfNeeded` both bail on that
+      // state) and the whole namespace stayed leaderless until a reload. Those
+      // duties are best-effort anyway: the list_ref poll covers a missing LIVE.
+      this.deps.hooks.becomeSyncLeader(this.hub);
+      this.lastPromotionMs = Date.now() - startedAt;
       this.setRole('leader');
       this.broker.send({
         type: 'leader-ready',
@@ -416,6 +440,14 @@ export class TabsCoordinator {
         bucketId: this.bucketId,
         leadershipId: msg.leadershipId,
       });
+      // Off the promotion chain on purpose: a hang here must not block the
+      // next role transition (demote, bucket switch) for the life of the tab.
+      void this.deps.hooks.resumeSyncLeaderDuties().catch((e) => {
+        this.deps.logger.error(
+          { err: e, Category: 'sp00ky-client::TabsCoordinator' },
+          'Leader duties failed after promotion'
+        );
+      });
     } catch (e) {
       const reason = e instanceof Error ? e.message : String(e);
       this.deps.logger.error(

package/src/services/tabs/tabs-broker-worker.ts

@@ -33,6 +33,13 @@ import type {
 // ---- constants duplicated from protocol.ts (keep in sync) -------------------
 const PING_INTERVAL_MS = 5000;
 const PONG_TIMEOUT_MS = 15_000;
+/** How long a tab has to answer `become-leader` with `leader-ready`. A tab
+ *  that hangs mid-promotion sends neither `leader-ready` nor `leader-failed`,
+ *  and keeps answering pings, so nothing else here would notice: the
+ *  namespace would keep a leader that is not `ready`, mint no follower ports,
+ *  and refuse to re-elect (`electIfNeeded` bails while `ns.leader` is set)
+ *  until every tab was reloaded. Treat the silence as a failed promotion. */
+const PROMOTION_TIMEOUT_MS = 20_000;
 const FORCE_TAKEOVER_TIMEOUT_MS = 1000;
 const LEADER_FAILURE_BACKOFF_MS = 1000;
 const FAILED_CYCLES_BEFORE_MEMORY = 3;
@@ -72,6 +79,8 @@ interface Namespace {
    *  lock (granted = the leader tab died; locks release instantly on death,
    *  unlike the pong timeout). */
   tabLockMonitor: AbortController | null;
+  /** Deadline for the pending `become-leader` to report `leader-ready`. */
+  promotionTimer: ReturnType<typeof setTimeout> | null;
 }
 
 const brokerInstanceId =
@@ -116,6 +125,7 @@ function getNamespace(fingerprint: string, bucketId: string): Namespace {
       electing: false,
       attachRetry: new Map(),
       tabLockMonitor: null,
+      promotionTimer: null,
     };
     namespaces.set(key, ns);
   }
@@ -165,6 +175,7 @@ function clearLeader(
   const leader = ns.leader;
   if (!leader) return null;
   ns.leader = null;
+  clearPromotionTimer(ns);
   ns.tabLockMonitor?.abort();
   ns.tabLockMonitor = null;
   const tab = ns.tabs.get(leader.tabId);
@@ -188,6 +199,54 @@ function clearLeader(
   return { tabId: leader.tabId, leadershipId: leader.leadershipId, workerLock };
 }
 
+function clearPromotionTimer(ns: Namespace): void {
+  if (ns.promotionTimer) clearTimeout(ns.promotionTimer);
+  ns.promotionTimer = null;
+}
+
+/**
+ * A promotion did not work out: back the tab off, clear the leadership, and
+ * re-elect. Shared by the tab's own `leader-failed` report and by the
+ * {@link PROMOTION_TIMEOUT_MS} deadline, so a tab that hangs mid-promotion
+ * costs the namespace the same 1s backoff as one that fails loudly, instead of
+ * owning it forever.
+ *
+ * Every reason counts toward `failedCycles`. Gating that on 'opfs-unavailable'
+ * left any other recurring failure (e.g. a tab lock nobody frees) looping at
+ * the backoff interval forever, with allowMemoryFallback never granted, so
+ * every tab timed out in start() and booted solo - each one then contending
+ * for the OPFS pool on its own, which is exactly what shared-tabs exists to
+ * stop.
+ */
+function failPromotion(
+  ns: Namespace,
+  tabId: string,
+  leadershipId: number,
+  opts: { timedOut: boolean }
+): void {
+  if (ns.leader?.tabId !== tabId || ns.leader.leadershipId !== leadershipId) return;
+  ns.failedCycles += 1;
+  ns.failedUntil.set(tabId, Date.now() + LEADER_FAILURE_BACKOFF_MS);
+  const previous = clearLeader(ns, { demote: false, removeTab: false });
+  const tab = ns.tabs.get(tabId);
+  if (tab) tab.heldLeadership = null;
+  // A tab that reported `leader-failed` has already given back its tab lock and
+  // any worker it opened, so there is nothing for the next candidate to wait
+  // on. A tab that went silent may have opened the OPFS pool and never told us,
+  // so hand the next election the lock name it would be holding: the probe in
+  // `waitForPreviousLeaderLocks` returns immediately when it is free, and
+  // steals it when it is not. `heldLeadership` is deliberately NOT used for
+  // this - it means "confirmed leader" and drives the broker-restart resume.
+  const stranded =
+    opts.timedOut && previous
+      ? {
+          ...previous,
+          workerLock: `sp00ky-tabs:${ns.fingerprint}:${ns.bucketId}:worker:${leadershipId}`,
+        }
+      : previous;
+  electIfNeeded(ns, stranded);
+}
+
 /**
  * Fast crash detection: queue a Web Lock request on the leader's tab lock.
  * Locks release the instant their holder's tab dies, so being GRANTED means
@@ -277,6 +336,11 @@ function electIfNeeded(ns: Namespace, previous: ClearedLeader | null = null): vo
         allowMemoryFallback: ns.failedCycles >= FAILED_CYCLES_BEFORE_MEMORY,
         resumeHeld,
       });
+      clearPromotionTimer(ns);
+      ns.promotionTimer = setTimeout(() => {
+        ns.promotionTimer = null;
+        failPromotion(ns, candidate.tabId, leadershipId, { timedOut: true });
+      }, PROMOTION_TIMEOUT_MS);
     } finally {
       ns.electing = false;
     }
@@ -489,6 +553,7 @@ function handleTabMessage(port: MessagePort, msg: TabToBrokerMessage, ports: rea
       }
       ns.leader.ready = true;
       ns.failedCycles = 0;
+      clearPromotionTimer(ns);
       startTabLockMonitor(ns, msg.leadershipId);
       const tab = ns.tabs.get(msg.tabId);
       if (tab) {
@@ -511,18 +576,7 @@ function handleTabMessage(port: MessagePort, msg: TabToBrokerMessage, ports: rea
       break;
     }
     case 'leader-failed': {
-      if (ns.leader?.tabId !== msg.tabId || ns.leader.leadershipId !== msg.leadershipId) break;
-      // Every reason counts. Gating this on 'opfs-unavailable' left any other
-      // recurring failure (e.g. a tab lock nobody frees) looping at the backoff
-      // interval forever, with allowMemoryFallback never granted, so every tab
-      // timed out in start() and booted solo — each one then contending for the
-      // OPFS pool on its own, which is exactly what shared-tabs exists to stop.
-      ns.failedCycles += 1;
-      ns.failedUntil.set(msg.tabId, Date.now() + LEADER_FAILURE_BACKOFF_MS);
-      const previous = clearLeader(ns, { demote: false, removeTab: false });
-      const tab = ns.tabs.get(msg.tabId);
-      if (tab) tab.heldLeadership = null;
-      electIfNeeded(ns, previous);
+      failPromotion(ns, msg.tabId, msg.leadershipId, { timedOut: false });
       break;
     }
     case 'follower-port-attached': {

package/src/sp00ky.ts

@@ -492,15 +492,16 @@ export class Sp00kyClient<S extends SchemaStructure> {
     this.streamProcessor = new StreamProcessorService(
       new EventSystem(['stream_update']),
       this.local,
-      this.persistenceClient,
       logger
     );
-    // Circuit snapshots are opt-in and checkpointed, never per-ingest. See
-    // `persistCircuit` in types.ts for the measurements behind that default.
+    // Circuit snapshots are checkpointed, never per-ingest, and on by default
+    // where the store can hold them (the OPFS SQLite engine). See
+    // `persistCircuit` / `circuitProjection` in types.ts.
     this.streamProcessor.configureCircuitPersistence(
-      config.persistCircuit ?? false,
+      config.persistCircuit ?? config.localEngine === 'sqlite',
       config.circuitCheckpointMs
     );
+    this.streamProcessor.configureProjection(config.circuitProjection ?? true);
     this.migrator = new LocalMigrator(this.local, logger);
 
     this.cache = new CacheModule(
@@ -560,6 +561,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
         connectionSupervisor: this.connectionSupervisor,
       }
     );
+    this.sync.setPrimeGate(() => this.streamProcessor.whenPrimed());
 
     // Initialize feature flags. Reuses the down-queue to register SSP plans
     // on `_00_user_feature` and the auth subscription to re-register handles
@@ -661,13 +663,14 @@ export class Sp00kyClient<S extends SchemaStructure> {
       onLeaderLost: (reason) => engine.onLeaderLost(reason),
       exposeClientPort: (clientId, port) => engine.exposeClientPort(clientId, port),
       removeClientPort: (clientId) => engine.removeClientPort(clientId),
-      becomeSyncLeader: async (hub) => {
+      becomeSyncLeader: (hub) => {
         this.streamProcessor.setPersistenceEnabled(true);
         this.cache.setIngestRelay((tuples) => hub.relayIngest(tuples));
         this.sync.setTabContext('leader', tabId);
         this.dataModule.setTabId(tabId);
-        await this.sync.promoteToLeader(hub);
+        this.sync.promoteToLeader(hub);
       },
+      resumeSyncLeaderDuties: () => this.sync.resumeLeaderDuties(),
       becomeSyncFollower: (forwarder) => {
         this.streamProcessor.setPersistenceEnabled(false);
         this.cache.setIngestRelay(null);
@@ -812,6 +815,16 @@ export class Sp00kyClient<S extends SchemaStructure> {
         // the current one instead of frozen at whatever boot saw.
         this.tabsCoordinator.onRoleChange((role) => {
           this.sharedActive = role !== 'solo';
+          // Surface the transition in DevTools. Reconstructing a failover from
+          // the `database.tabs` snapshot alone is guesswork after the fact; a
+          // discrete event gives `get_events` the leader handover directly.
+          this.devTools.logEvent('TABS_ROLE_CHANGED', {
+            role,
+            tabId: this.tabsCoordinator?.tabId,
+            leadershipId: this.tabsCoordinator?.leadershipId,
+            leaderTabId: this.tabsCoordinator?.leaderTabId,
+            promotionMs: this.tabsCoordinator?.lastPromotionMs,
+          });
         });
         try {
           const role = await this.tabsCoordinator.start(bootBucket);
@@ -862,11 +875,14 @@ export class Sp00kyClient<S extends SchemaStructure> {
         }
       })();
 
-      this.streamProcessor.setStateKeySuffix(bootBucket);
       await this.streamProcessor.init();
       // Seed table `select` permissions from the schema before any query is
       // registered — otherwise the SSP default-denies every non-`_00_` table.
       this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
+      // Fill the circuit from the local store in the background (snapshot +
+      // reconcile, or every cached row). Not awaited: first paint reads the
+      // store directly; sync waits on `whenPrimed` before its first diff.
+      void this.primeCircuit();
       this.logger.debug(
         { Category: 'sp00ky-client::Sp00kyClient::init' },
         'StreamProcessor initialized'
@@ -1179,10 +1195,10 @@ export class Sp00kyClient<S extends SchemaStructure> {
           'Blob cache bucket switch failed'
         );
       }
-      this.streamProcessor.setStateKeySuffix(target);
       await this.streamProcessor.reset();
       this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
       this.cache.clearVersionLookups();
+      void this.primeCircuit();
       // Preload dedup is per-bucket: the `_00_preload` markers + cached rows it
       // guards live in the local store we just swapped away from. Keeping the
       // hashes would make `preload()` skip warming the NEW bucket (its store is
@@ -1226,6 +1242,8 @@ export class Sp00kyClient<S extends SchemaStructure> {
     // Flush the blob manifest while the local store is still open — the flush
     // writes `_00_blob` rows through it — and revoke every object URL.
     await this.blobs.close().catch(() => {});
+    // Last chance to persist the circuit while the store is still ours.
+    await this.streamProcessor.checkpoint('close').catch(() => {});
     // Leaving the broker first hands leadership to another tab (and releases
     // the OPFS handles via the worker shutdown) before the store closes.
     if (this.tabsCoordinator) await this.tabsCoordinator.stop();
@@ -1633,6 +1651,34 @@ export class Sp00kyClient<S extends SchemaStructure> {
    * the former to `encodeRecordId` reads `.table` off a string and throws
    * during boot.
    */
+  /**
+   * Prime the in-browser circuit from the local store. Builds the context the
+   * stream processor needs: every synced table (the app schema plus the
+   * server-written meta tables that sync down), a schema hash so a snapshot
+   * projected under another schema is not trusted, and the ids whose local
+   * `_00_rv` was bumped by an unsettled mutation.
+   */
+  private async primeCircuit(): Promise<void> {
+    const tables = [
+      ...this.config.schema.tables.map((t) => t.name),
+      '_00_user_feature',
+      '_00_app_release',
+    ];
+    let pendingIds = new Set<string>();
+    try {
+      const pending = await this.dataModule.getPendingRecordIds();
+      pendingIds = new Set([...pending.writes, ...pending.deletes]);
+    } catch {
+      /* no outbox yet: nothing is pending */
+    }
+    await this.streamProcessor.primeFromLocal({
+      tables,
+      schemaHash: String(hash53(this.config.schemaSurql)),
+      pendingIds,
+      onVersions: (_table, entries) => this.cache.primeVersions(entries),
+    });
+  }
+
   private sessionAuthId(): string | null {
     const id = this.auth.currentUser?.id;
     if (!id) return null;

package/src/types.ts

@@ -205,21 +205,19 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
    */
   sharedTabs?: boolean;
   /**
-   * Persist the in-browser SSP circuit (store + view caches) as a snapshot so a
-   * reload can restore it instead of re-materializing. Default `false`, and
-   * that default is deliberate.
+   * Persist the in-browser SSP circuit's store as a snapshot in the local
+   * store, so a reload restores it and only steps in what changed since.
+   * Defaults to `true` under `localEngine: 'sqlite'` (the OPFS-backed store
+   * can hold the bytes) and `false` otherwise.
    *
-   * The circuit is DERIVED state: the durable local store (OPFS SQLite) is the
-   * source of truth, and every first paint already reads row bodies from it
-   * (`DataManager.createNewQuery` / `materializeRecords`) using the circuit only
-   * for row identity and ordering. A snapshot buys nothing on reload while
-   * costing a full deep clone of every row of every ingested table plus a JSON
-   * encode of the result, `Circuit::save` in the Rust core, mirroring the
-   * server's rule in `ssp-node`: *never per-ingest*.
+   * Without a snapshot the circuit is primed by reading every cached row back
+   * out of the local store; either way a reload never re-downloads the working
+   * set, which is what an empty circuit used to force: every id in the
+   * server's list_ref classified as missing.
    *
-   * When enabled, snapshots are written on a checkpoint interval
-   * ({@link circuitCheckpointMs}) and on `pagehide`, never per ingest or per
-   * query registration. Enable only for a workload that has measured a win.
+   * Snapshots are store-only (views are re-registered per session) and written
+   * on a checkpoint interval ({@link circuitCheckpointMs}) and when the page
+   * goes hidden, never per ingest or per query registration.
    */
   persistCircuit?: boolean;
   /**
@@ -227,6 +225,16 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
    * 30000. Ignored when `persistCircuit` is off.
    */
   circuitCheckpointMs?: number;
+  /**
+   * Keep only the fields registered queries evaluate (filter predicates, join
+   * keys, sort keys, plus `id`/`_00_rv`) per row in the in-browser circuit.
+   * Default `true`. The circuit only ever reads those fields; bodies are
+   * rendered from the local store. Measured on 7700 rows with 20 KB bodies:
+   * the wasm heap peak went from ~500 MB to ~24 MB and the snapshot from
+   * 156 MB to 1.2 MB. A query that evaluates a field earlier rows were kept
+   * without has that field merged in from the local store on registration.
+   */
+  circuitProjection?: boolean;
   /** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
   otelTransmit?: PinoTransmit;
   /**