@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210

package/src/services/stream-processor/stream-processor.reset.test.ts

@@ -0,0 +1,226 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// `StreamProcessorService.reset()` is the SSP half of a local-bucket switch:
+// the old circuit holds the previous user's rows and views registered with the
+// previous `$auth`, so reset must swap in a FRESH processor (no state load),
+// and a checkpoint racing the reset must not persist the old circuit into the
+// new bucket's store.
+
+const processorInstances: any[] = [];
+
+vi.mock('@spooky-sync/ssp-wasm', () => ({
+  default: vi.fn(async () => {}),
+  Sp00kyProcessor: vi.fn(() => {
+    const instance = {
+      ingest: vi.fn(() => []),
+      register_view: vi.fn(() => ({ query_id: 'q1', result_data: [] })),
+      unregister_view: vi.fn(),
+      set_permissions: vi.fn(),
+      set_projection: vi.fn(),
+      save_store_state: vi.fn(() => new Uint8Array([1, 2, 3])),
+      load_store_state: vi.fn(() => []),
+      reconcile: vi.fn(() => ({ fetch: [], deleted: 0, updates: [] })),
+      free: vi.fn(),
+    };
+    processorInstances.push(instance);
+    return instance;
+  }),
+}));
+
+import { StreamProcessorService } from './index';
+
+const silentLogger = {
+  child: () => silentLogger,
+  trace: () => {},
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+function makeService() {
+  const snapshots: Record<string, { bytes: Uint8Array; meta: any }> = {};
+  const db = {
+    epoch: 0,
+    engineKind: 'sqlite' as const,
+    scanVersions: vi.fn(async () => ({})),
+    selectByIds: vi.fn(async () => []),
+    getSnapshot: vi.fn(async (key: string) => snapshots[key] ?? null),
+    putSnapshot: vi.fn(async (key: string, bytes: Uint8Array, meta: any) => {
+      snapshots[key] = { bytes, meta };
+    }),
+  };
+  const service = new StreamProcessorService({} as any, db as any, silentLogger);
+  return { service, db, snapshots };
+}
+
+const primeCtx = { tables: [], schemaHash: 'schema-1', pendingIds: new Set<string>() };
+
+beforeEach(() => {
+  processorInstances.length = 0;
+});
+
+describe('StreamProcessorService.reset', () => {
+  it('swaps in a fresh processor without loading persisted state', async () => {
+    const { service } = makeService();
+    await service.init();
+    expect(processorInstances).toHaveLength(1);
+    const first = processorInstances[0];
+
+    await service.reset();
+    expect(processorInstances).toHaveLength(2);
+    const second = processorInstances[1];
+    // Fresh circuit: nothing loaded into it (a persisted snapshot references
+    // views under a dead sessionId salt and the previous user's data).
+    expect(second.load_store_state).not.toHaveBeenCalled();
+
+    // New registrations land on the fresh processor, not the old circuit.
+    service.registerQueryPlan({
+      queryHash: 'q1',
+      surql: 'SELECT * FROM thing;',
+      params: {},
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      localArray: [],
+      remoteArray: [],
+      meta: { tableName: 'thing' },
+    } as any);
+    expect(second.register_view).toHaveBeenCalled();
+    expect(first.register_view).not.toHaveBeenCalled();
+  });
+
+  it('writes a store-only snapshot through the engine on checkpoint', async () => {
+    const { service, db, snapshots } = makeService();
+    service.configureCircuitPersistence(true);
+    await service.init();
+    await service.primeFromLocal(primeCtx);
+    await service.checkpoint('test');
+    expect(processorInstances[0].save_store_state).toHaveBeenCalledTimes(1);
+    expect(db.putSnapshot).toHaveBeenCalledTimes(1);
+    expect(snapshots.circuit.bytes).toEqual(new Uint8Array([1, 2, 3]));
+    expect(snapshots.circuit.meta.schemaHash).toBe('schema-1');
+    expect(snapshots.circuit.meta.formatVersion).toBe(1);
+  });
+
+  it('drops a checkpoint that raced a reset (old circuit never persists into the new bucket)', async () => {
+    const { service, db } = makeService();
+    service.configureCircuitPersistence(true);
+    await service.init();
+    await service.primeFromLocal(primeCtx);
+    // The bytes are taken, then a reset lands before the write step,
+    // simulated by resetting from inside save_store_state().
+    processorInstances[0].save_store_state.mockImplementation(() => {
+      void service.reset();
+      return new Uint8Array([9]);
+    });
+    await service.checkpoint('test');
+    expect(db.putSnapshot).not.toHaveBeenCalled();
+  });
+
+  it('does not write a snapshot from a follower tab', async () => {
+    const { service, db } = makeService();
+    service.configureCircuitPersistence(true);
+    await service.init();
+    await service.primeFromLocal(primeCtx);
+    service.setPersistenceEnabled(false);
+    await service.checkpoint('test');
+    expect(db.putSnapshot).not.toHaveBeenCalled();
+  });
+
+  it('frees the replaced wasm circuit on reset', async () => {
+    const { service } = makeService();
+    await service.init();
+    const first = processorInstances[0];
+    await service.reset();
+    // V8 cannot see wasm-internal bytes, so the FinalizationRegistry may never
+    // run. The old circuit must be freed explicitly or its whole store stays
+    // resident for the rest of the session.
+    expect(first.free).toHaveBeenCalledTimes(1);
+  });
+});
+
+// The renderer-OOM guardrail. A snapshot walks every row of the store, so it
+// must never run inline with an ingest or a query register/unregister: on a
+// large windowed list that meant hundreds of whole-store serializations per
+// scroll. Snapshots are checkpointed on a timer, and only once enough rows
+// changed to be worth the write.
+describe('StreamProcessorService circuit snapshots', () => {
+  const plan = {
+    queryHash: 'q1',
+    surql: 'SELECT * FROM thing;',
+    params: {},
+    ttl: '10m',
+    lastActiveAt: new Date(),
+    localArray: [],
+    remoteArray: [],
+    meta: { tableName: 'thing' },
+  } as any;
+
+  it('never snapshots on ingest or register/unregister when persistence is off', async () => {
+    const { service, db } = makeService();
+    service.configureCircuitPersistence(false);
+    await service.init();
+    const processor = processorInstances[0];
+
+    service.registerQueryPlan(plan);
+    service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
+    service.ingestMany([
+      { table: 'thing', op: 'CREATE', id: 'thing:b', record: { id: 'thing:b' } },
+      { table: 'thing', op: 'CREATE', id: 'thing:c', record: { id: 'thing:c' } },
+    ]);
+    service.unregisterQueryPlan('q1');
+
+    expect(processor.save_store_state).not.toHaveBeenCalled();
+    expect(db.putSnapshot).not.toHaveBeenCalled();
+  });
+
+  it('still never snapshots inline when persistence is on', async () => {
+    const { service, db } = makeService();
+    service.configureCircuitPersistence(true, 30_000);
+    await service.init();
+    await service.primeFromLocal(primeCtx);
+    const processor = processorInstances[0];
+
+    service.registerQueryPlan(plan);
+    service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
+    service.unregisterQueryPlan('q1');
+
+    // Marked dirty, but the write waits for the checkpoint interval.
+    expect(processor.save_store_state).not.toHaveBeenCalled();
+    expect(db.putSnapshot).not.toHaveBeenCalled();
+    service.stopCheckpoints();
+  });
+
+  it('writes at most one snapshot per checkpoint tick, and none for a handful of rows', async () => {
+    vi.useFakeTimers();
+    try {
+      const { service, db } = makeService();
+      service.configureCircuitPersistence(true, 1000);
+      await service.init();
+      await service.primeFromLocal(primeCtx);
+      const processor = processorInstances[0];
+
+      // Below the row threshold: the tick stays quiet.
+      for (let i = 0; i < 10; i++) {
+        service.ingest('thing', 'CREATE', `thing:${i}`, { id: `thing:${i}` });
+      }
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(processor.save_store_state).not.toHaveBeenCalled();
+
+      for (let i = 10; i < 60; i++) {
+        service.ingest('thing', 'CREATE', `thing:${i}`, { id: `thing:${i}` });
+      }
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(processor.save_store_state).toHaveBeenCalledTimes(1);
+      expect(db.putSnapshot).toHaveBeenCalledTimes(1);
+
+      // Idle interval: nothing changed, so nothing is serialized.
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(processor.save_store_state).toHaveBeenCalledTimes(1);
+
+      service.stopCheckpoints();
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+});

package/src/services/stream-processor/stream-processor.test.ts

@@ -127,7 +127,7 @@ describe('StreamProcessor Ingest Behavior', () => {
     const params = { id: new MockRecordId('user', '2dng4ngbicbl0scod87i') };
     const normalizedParams = normalizeValue(params);
 
-    const ingestedRecord = {
+    const _ingestedRecord = {
       id: 'user:2dng4ngbicbl0scod87i',
       username: 'sara',
     };

package/src/services/stream-processor/wasm-types.ts

@@ -1,9 +1,17 @@
-import { RecordVersionArray } from '../../types';
+import type { RecordVersionArray } from '../../types';
 
 export interface WasmStreamUpdate {
   query_id: string;
   result_hash: string;
   result_data: RecordVersionArray; // Match Rust 'result_data' field
+  // Per-phase SSP processing time (ms). Ingest path: store_apply/circuit_step/
+  // transform. Register path: parse/plan/snapshot. Unused side is 0.
+  timing_store_apply_ms?: number;
+  timing_circuit_step_ms?: number;
+  timing_transform_ms?: number;
+  timing_parse_ms?: number;
+  timing_plan_ms?: number;
+  timing_snapshot_ms?: number;
 }
 
 export interface WasmQueryConfig {
@@ -23,9 +31,57 @@ export interface WasmIngestItem {
   version?: number;
 }
 
-// Interface matching the SpookyProcessor class from WASM
+/** `register_view`'s result: the initial update plus, under projection, the
+ *  fields this plan evaluates that already-stored rows were kept without. */
+export interface WasmRegistration extends WasmStreamUpdate {
+  missing_fields?: Record<string, string[]>;
+}
+
+/** `reconcile`'s result. */
+export interface WasmReconciled {
+  /** Ids (caller's spelling) whose body the store lacks or holds stale. */
+  fetch: string[];
+  /** Rows deleted because the caller's list did not have them. */
+  deleted: number;
+  /** View updates produced by those deletes. */
+  updates: WasmStreamUpdate[];
+}
+
+// Interface matching the Sp00kyProcessor class from WASM
 export interface WasmProcessor {
   ingest(table: string, op: string, id: string, record: any): WasmStreamUpdate[];
-  register_view(config: WasmQueryConfig): WasmStreamUpdate | undefined;
+  // Bulk ingest: ONE circuit step for the whole array, returning the coalesced
+  // updates. Optional because a stale WASM build won't have it — callers guard
+  // with `typeof x === 'function'` and fall back to a loop over `ingest`.
+  ingest_many?(items: WasmIngestItem[]): WasmStreamUpdate[];
+  register_view(config: WasmQueryConfig): WasmRegistration | undefined;
   unregister_view(id: string): void;
+  // Seed per-table `select` permission predicates ({ [table]: whereText }) so
+  // register_view can inject them instead of default-denying the table.
+  set_permissions(permissions: Record<string, string>): void;
+  // Persistence hooks: present on current WASM builds, absent on stale ones.
+  // Always guarded with `typeof x === 'function'` before calling so an older
+  // build degrades gracefully instead of throwing.
+  load_state?(state: string): void;
+  save_state?(): string;
+  // Store-only snapshot as bytes (views are re-registered per session, so
+  // they are never persisted). `load_store_state` installs it UNDER the views
+  // already registered and returns their re-primed results.
+  save_store_state?(): Uint8Array;
+  load_store_state?(bytes: Uint8Array): WasmStreamUpdate[];
+  // Client-side `_00_rv` catch-up against the durable store's `[id, rv][]`.
+  reconcile?(table: string, entries: [string, number][]): WasmReconciled;
+  max_row_versions?(): Record<string, number>;
+  // Row-arena hygiene. `compact` decodes every row, so checkpoint-time only.
+  compact?(): number;
+  dead_bytes?(): number;
+  live_bytes?(): number;
+  // Keep only the fields registered plans evaluate per stored row.
+  set_projection?(enabled: boolean): void;
+  size_report?(): unknown;
+  // wasm-bindgen destructor. Releases the circuit (store + every view cache)
+  // inside wasm linear memory. Without it the bytes only come back when V8
+  // happens to GC the JS wrapper, which it has no reason to hurry since it
+  // cannot see how much wasm memory the wrapper is holding.
+  free?(): void;
 }

package/src/services/tabs/broker-client.ts

@@ -0,0 +1,283 @@
+/**
+ * Per-tab connection to the SharedWorker broker. Owns: the hello handshake,
+ * pong replies + the ping watchdog (a silent broker means the browser killed
+ * the SharedWorker; reconnect with a fresh one), visibility reporting, and the
+ * pagehide/freeze shutdown notifications. Role decisions and port handling are
+ * the coordinator's job; this class only surfaces them as events.
+ */
+import type { Logger } from '../logger/index';
+import {
+  PING_INTERVAL_MS,
+  PONG_TIMEOUT_MS,
+  type BrokerToTabMessage,
+  type HeldLeadership,
+  type TabId,
+  type TabToBrokerMessage,
+  type TabVisibility,
+} from './protocol';
+
+export interface BrokerClientEvents {
+  onBecomeLeader(msg: {
+    leadershipId: number;
+    forceTakeover: boolean;
+    allowMemoryFallback: boolean;
+    resumeHeld: boolean;
+  }): void;
+  onDemote(leadershipId: number): void;
+  onLeaderReady(leadershipId: number, leaderTabId: TabId): void;
+  /** Leader side: the broker minted ports for a follower. */
+  onAttachFollowerPorts(
+    followerTabId: TabId,
+    leadershipId: number,
+    dbPort: MessagePort,
+    syncPort: MessagePort
+  ): void;
+  /** Follower side: our ports into the current leader. */
+  onUseFollowerPorts(
+    leaderTabId: TabId,
+    leadershipId: number,
+    dbPort: MessagePort,
+    syncPort: MessagePort
+  ): void;
+  onCloseFollowerPorts(leadershipId: number): void;
+  onUnsupported(reason: string): void;
+  /** The broker restarted (instance id changed or pings stopped); the client
+   *  already re-helloed. Roles survive only via become-leader/use-ports. */
+  onBrokerRestarted(): void;
+}
+
+/** What the tab reports about itself on (re)connect. */
+export interface BrokerHello {
+  fingerprint: string;
+  bucketId: string;
+  heldLeadership: () => HeldLeadership | null;
+}
+
+export class TabBrokerClient {
+  private worker: SharedWorker | null = null;
+  private brokerInstanceId: string | null = null;
+  private watchdog: ReturnType<typeof setInterval> | null = null;
+  private lastPingAt = 0;
+  private closed = false;
+  private helloState: BrokerHello | null = null;
+
+  constructor(
+    private workerUrl: URL,
+    readonly tabId: TabId,
+    private events: BrokerClientEvents,
+    private logger: Logger
+  ) {}
+
+  /** Connect and hello. Resolves once broker-hello arrives (or rejects on
+   *  timeout / SharedWorker error, in which case the caller goes solo). */
+  connect(hello: BrokerHello): Promise<void> {
+    this.helloState = hello;
+    this.installLifecycleListeners();
+    return this.openWorker();
+  }
+
+  private openWorker(): Promise<void> {
+    if (this.closed || !this.helloState) return Promise.reject(new Error('broker client closed'));
+    return new Promise<void>((resolve, reject) => {
+      let settled = false;
+      const settle = (fn: () => void) => {
+        if (!settled) {
+          settled = true;
+          fn();
+        }
+      };
+      let worker: SharedWorker;
+      try {
+        worker = new SharedWorker(this.workerUrl, { type: 'module', name: 'sp00ky-tabs-broker' });
+      } catch (e) {
+        reject(e);
+        return;
+      }
+      this.worker = worker;
+      const timeout = setTimeout(() => settle(() => reject(new Error('broker hello timeout'))), 5000);
+      worker.onerror = () => {
+        clearTimeout(timeout);
+        settle(() => reject(new Error('SharedWorker failed to start')));
+      };
+      worker.port.onmessage = (ev: MessageEvent) => {
+        const msg = ev.data as BrokerToTabMessage;
+        if (!msg || typeof msg !== 'object') return;
+        if (msg.type === 'broker-hello') {
+          clearTimeout(timeout);
+          this.noteInstance(msg.brokerInstanceId);
+          this.lastPingAt = Date.now();
+          this.startWatchdog();
+          settle(resolve);
+          return;
+        }
+        this.handleMessage(msg, (ev.ports ?? []) as readonly MessagePort[]);
+      };
+      worker.port.onmessageerror = () => this.reconnect('messageerror');
+      worker.port.start?.();
+      this.send({
+        type: 'hello',
+        tabId: this.tabId,
+        fingerprint: this.helloState!.fingerprint,
+        bucketId: this.helloState!.bucketId,
+        visibility: currentVisibility(),
+        heldLeadership: this.helloState!.heldLeadership(),
+      });
+    });
+  }
+
+  private noteInstance(id: string): void {
+    if (this.brokerInstanceId !== null && this.brokerInstanceId !== id) {
+      this.events.onBrokerRestarted();
+    }
+    this.brokerInstanceId = id;
+  }
+
+  private handleMessage(msg: BrokerToTabMessage, ports: readonly MessagePort[]): void {
+    // Any message from a NEW instance implies a restart happened while our
+    // watchdog had not fired yet.
+    if ('brokerInstanceId' in msg) this.noteInstance(msg.brokerInstanceId);
+    switch (msg.type) {
+      case 'ping':
+        this.lastPingAt = Date.now();
+        this.send({ type: 'pong', tabId: this.tabId });
+        break;
+      case 'become-leader':
+        this.events.onBecomeLeader({
+          leadershipId: msg.leadershipId,
+          forceTakeover: msg.forceTakeover,
+          allowMemoryFallback: msg.allowMemoryFallback,
+          resumeHeld: msg.resumeHeld,
+        });
+        break;
+      case 'demote':
+        this.events.onDemote(msg.leadershipId);
+        break;
+      case 'leader-ready':
+        this.events.onLeaderReady(msg.leadershipId, msg.leaderTabId);
+        break;
+      case 'attach-follower-ports':
+        if (ports.length >= 2) {
+          this.events.onAttachFollowerPorts(msg.followerTabId, msg.leadershipId, ports[0], ports[1]);
+        }
+        break;
+      case 'use-follower-ports':
+        if (ports.length >= 2) {
+          this.events.onUseFollowerPorts(msg.leaderTabId, msg.leadershipId, ports[0], ports[1]);
+        }
+        break;
+      case 'close-follower-ports':
+        this.events.onCloseFollowerPorts(msg.leadershipId);
+        break;
+      case 'unsupported':
+        this.logger.warn(
+          { reason: msg.reason, Category: 'sp00ky-client::BrokerClient' },
+          'Broker rejected this tab; running solo'
+        );
+        this.events.onUnsupported(msg.reason);
+        this.close();
+        break;
+    }
+  }
+
+  send(msg: TabToBrokerMessage): void {
+    try {
+      this.worker?.port.postMessage(msg);
+    } catch {
+      /* dead port; watchdog reconnects */
+    }
+  }
+
+  /** The tab moved to a different bucket: re-hello under the new namespace.
+   *  The broker treats a re-hello of a known tabId as leave + rejoin. */
+  rehello(hello: BrokerHello): Promise<void> {
+    this.helloState = hello;
+    if (!this.worker) return this.openWorker();
+    this.send({
+      type: 'hello',
+      tabId: this.tabId,
+      fingerprint: hello.fingerprint,
+      bucketId: hello.bucketId,
+      visibility: currentVisibility(),
+      heldLeadership: hello.heldLeadership(),
+    });
+    return Promise.resolve();
+  }
+
+  private startWatchdog(): void {
+    if (this.watchdog) return;
+    this.watchdog = setInterval(() => {
+      if (Date.now() - this.lastPingAt > PING_INTERVAL_MS + PONG_TIMEOUT_MS) {
+        this.reconnect('broker pings stopped');
+      }
+    }, PING_INTERVAL_MS);
+  }
+
+  private reconnect(reason: string): void {
+    if (this.closed) return;
+    this.logger.info(
+      { reason, Category: 'sp00ky-client::BrokerClient' },
+      'Reconnecting to the tabs broker'
+    );
+    try {
+      this.worker?.port.close();
+    } catch {
+      /* ignore */
+    }
+    this.worker = null;
+    this.brokerInstanceId = null;
+    this.events.onBrokerRestarted();
+    void this.openWorker().catch((e) => {
+      this.logger.warn(
+        { err: e, Category: 'sp00ky-client::BrokerClient' },
+        'Broker reconnect failed; staying detached until the next watchdog tick'
+      );
+    });
+  }
+
+  private installLifecycleListeners(): void {
+    if (typeof window === 'undefined' || typeof document === 'undefined') return;
+    document.addEventListener('visibilitychange', () => {
+      if (!this.helloState) return;
+      this.send({
+        type: 'visibility',
+        tabId: this.tabId,
+        bucketId: this.helloState.bucketId,
+        visibility: currentVisibility(),
+      });
+    });
+    // pagehide fires for both close and bfcache entry; treat both as a full
+    // departure (a bfcache restore re-hellos via pageshow below). freeze is
+    // Chromium's battery-saver signal; leaving proactively hands leadership
+    // over in ~0s instead of waiting out the pong timeout.
+    const leave = () => {
+      if (this.helloState) {
+        this.send({ type: 'shutdown', tabId: this.tabId, bucketId: this.helloState.bucketId });
+      }
+    };
+    window.addEventListener('pagehide', leave);
+    document.addEventListener('freeze', leave);
+    window.addEventListener('pageshow', (e: PageTransitionEvent) => {
+      if (e.persisted && this.helloState) void this.rehello(this.helloState);
+    });
+    document.addEventListener('resume', () => {
+      if (this.helloState) void this.rehello(this.helloState);
+    });
+  }
+
+  close(): void {
+    this.closed = true;
+    if (this.watchdog) clearInterval(this.watchdog);
+    this.watchdog = null;
+    try {
+      this.worker?.port.close();
+    } catch {
+      /* ignore */
+    }
+    this.worker = null;
+  }
+}
+
+function currentVisibility(): TabVisibility {
+  if (typeof document === 'undefined') return 'visible';
+  return document.visibilityState === 'hidden' ? 'hidden' : 'visible';
+}