@spooky-sync/core 0.0.1-canary.16 → 0.0.1-canary.161

package/src/services/stream-processor/permissions.test.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect } from 'vitest';
+import { extractSelectPermissions } from './permissions';
+
+describe('extractSelectPermissions', () => {
+  it('maps a permissive multi-action table to true', () => {
+    const surql = `DEFINE TABLE game SCHEMAFULL
+      PERMISSIONS FOR select, create, update, delete WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ game: 'true' });
+  });
+
+  it('handles FULL and NONE', () => {
+    const surql = `
+      DEFINE TABLE a PERMISSIONS FULL;
+      DEFINE TABLE b PERMISSIONS NONE;`;
+    expect(extractSelectPermissions(surql)).toEqual({ a: 'true', b: 'false' });
+  });
+
+  it('extracts the select predicate from multiple FOR groups', () => {
+    const surql = `DEFINE TABLE doc SCHEMAFULL
+      PERMISSIONS
+        FOR create, update, delete WHERE owner = $auth.id
+        FOR select WHERE owner = $auth.id OR public = true;`;
+    expect(extractSelectPermissions(surql)).toEqual({
+      doc: 'owner = $auth.id OR public = true',
+    });
+  });
+
+  it('returns false when permissions exist but none grant select', () => {
+    const surql = `DEFINE TABLE log PERMISSIONS FOR create WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ log: 'false' });
+  });
+
+  it('treats a missing PERMISSIONS clause as default-deny', () => {
+    const surql = `DEFINE TABLE bare SCHEMALESS;`;
+    expect(extractSelectPermissions(surql)).toEqual({ bare: 'false' });
+  });
+
+  it('ignores commented-out clauses and handles OVERWRITE / IF NOT EXISTS', () => {
+    const surql = `
+      -- DEFINE TABLE ghost PERMISSIONS FULL;
+      DEFINE TABLE OVERWRITE u PERMISSIONS FOR select WHERE true;
+      DEFINE TABLE IF NOT EXISTS v PERMISSIONS FULL;`;
+    const perms = extractSelectPermissions(surql);
+    expect(perms).toEqual({ u: 'true', v: 'true' });
+    expect(perms.ghost).toBeUndefined();
+  });
+});

package/src/services/stream-processor/permissions.ts

@@ -0,0 +1,53 @@
+/**
+ * Extract per-table `select` permission predicates from a raw `.surql` schema.
+ *
+ * The in-browser SSP default-denies any non-`_00_` table that has no permission
+ * predicate registered (see `permission_inject::build_predicate`). The client
+ * already holds the full schema text (`config.schemaSurql`), so we parse each
+ * `DEFINE TABLE … PERMISSIONS …` clause and hand the `select` predicate to the
+ * processor via `set_permissions` at boot — mirroring the native path that
+ * seeds the same map from `INFO FOR DB`.
+ *
+ * Returns `{ [table]: whereText }` where `whereText` is the raw SurrealQL
+ * expression (`'true'` for `FULL`, `'false'` for `NONE` or a table with no
+ * `select` permission).
+ */
+export function extractSelectPermissions(schemaSurql: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  if (!schemaSurql) return out;
+
+  // Drop line comments so a `--` comment can't contain a stray PERMISSIONS/FOR.
+  const cleaned = schemaSurql.replace(/--[^\n]*/g, '');
+
+  // One entry per `DEFINE TABLE … ;` statement.
+  const tableStmt = /DEFINE\s+TABLE\s+(?:OVERWRITE\s+|IF\s+NOT\s+EXISTS\s+)?([A-Za-z_][\w]*)\b([^;]*);/gi;
+  let m: RegExpExecArray | null;
+  while ((m = tableStmt.exec(cleaned)) !== null) {
+    const table = m[1];
+    const body = m[2];
+    out[table] = selectPredicateFromBody(body);
+  }
+  return out;
+}
+
+function selectPredicateFromBody(body: string): string {
+  const permIdx = body.search(/\bPERMISSIONS\b/i);
+  if (permIdx === -1) return 'false'; // no clause → SurrealDB default-deny
+  const perms = body.slice(permIdx + 'PERMISSIONS'.length).trim();
+
+  if (/^FULL\b/i.test(perms)) return 'true';
+  if (/^NONE\b/i.test(perms)) return 'false';
+
+  // `FOR <actions> WHERE <expr>` groups, possibly several. Split on the FOR
+  // boundary; the leading empty segment (before the first FOR) is ignored.
+  const groups = perms.split(/\bFOR\b/i).map((g) => g.trim()).filter(Boolean);
+  for (const group of groups) {
+    const where = group.search(/\bWHERE\b/i);
+    if (where === -1) continue;
+    const actions = group.slice(0, where).toLowerCase();
+    if (/\bselect\b/.test(actions)) {
+      return group.slice(where + 'WHERE'.length).trim();
+    }
+  }
+  return 'false'; // permissions present but none grant select
+}

package/src/services/stream-processor/stream-processor.batch.test.ts

@@ -0,0 +1,136 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { StreamProcessorService } from './index';
+import type { StreamUpdate, StreamUpdateReceiver } from './index';
+import type { WasmProcessor, WasmStreamUpdate } from './wasm-types';
+
+/**
+ * Tests for `ingestMany` bulk insert on StreamProcessorService.
+ *
+ * A batched ingest (e.g. sync fetching N missing records) used to emit one
+ * stream update per record, making the UI render row-by-row. ingestMany
+ * collapses those into a single coalesced update per affected query.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = {
+    debug: noop,
+    info: noop,
+    warn: noop,
+    error: noop,
+    trace: noop,
+  };
+  logger.child = () => logger;
+  return logger;
+}
+
+/** Records every dispatched update so we can count notifications. */
+class RecordingReceiver implements StreamUpdateReceiver {
+  public received: StreamUpdate[] = [];
+  onStreamUpdate(update: StreamUpdate): void {
+    this.received.push(update);
+  }
+}
+
+/**
+ * Build a service with a stubbed WASM processor. The processor accumulates
+ * ingested ids and returns the *full* materialized array on every call (as the
+ * real WASM does), so coalescing's last-write-wins must yield the final array.
+ */
+function makeService(queryHashesFor: (id: string) => string[]) {
+  const svc = new StreamProcessorService(
+    {} as any,
+    {} as any,
+    { get: async () => undefined, set: async () => {} } as any,
+    makeLogger()
+  );
+
+  const ingestedByQuery = new Map<string, Array<[string, number]>>();
+  const mockProcessor: Partial<WasmProcessor> = {
+    ingest: (_table, _op, id, record: any): WasmStreamUpdate[] => {
+      const version = record?._00_rv ?? 1;
+      const updates: WasmStreamUpdate[] = [];
+      for (const queryHash of queryHashesFor(id)) {
+        const arr = ingestedByQuery.get(queryHash) ?? [];
+        arr.push([id, version]);
+        ingestedByQuery.set(queryHash, arr);
+        updates.push({ query_id: queryHash, result_data: [...arr] });
+      }
+      return updates;
+    },
+  };
+  // processor is private and normally set by init() via WASM; inject the stub.
+  (svc as any).processor = mockProcessor;
+  return svc;
+}
+
+describe('StreamProcessor ingestMany bulk insert', () => {
+  let receiver: RecordingReceiver;
+
+  beforeEach(() => {
+    receiver = new RecordingReceiver();
+  });
+
+  it('emits one update per record when ingesting one-by-one (baseline)', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingest('user', 'CREATE', 'user:1', { id: 'user:1', _00_rv: 1 });
+    svc.ingest('user', 'CREATE', 'user:2', { id: 'user:2', _00_rv: 1 });
+    svc.ingest('user', 'CREATE', 'user:3', { id: 'user:3', _00_rv: 1 });
+
+    expect(receiver.received).toHaveLength(3);
+  });
+
+  it('coalesces a bulk insert into a single update per query with the final array', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([
+      { table: 'user', op: 'CREATE', id: 'user:1', record: { id: 'user:1', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:2', record: { id: 'user:2', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:3', record: { id: 'user:3', _00_rv: 1 } },
+    ]);
+
+    // Nothing dispatched until the whole batch is ingested, then exactly one
+    // coalesced update.
+    expect(receiver.received).toHaveLength(1);
+    const update = receiver.received[0];
+    expect(update.queryHash).toBe('q1');
+    // Last-write-wins carries the full materialized array (all 3 records).
+    expect(update.localArray).toHaveLength(3);
+    // Coalesced updates take DataModule's immediate (non-debounced) path.
+    expect(update.op).toBe('CREATE');
+    expect(typeof update.materializationTimeMs).toBe('number');
+  });
+
+  it('coalesces independently per affected query', () => {
+    // user:1 -> q1, user:2 -> q1 & q2, user:3 -> q2
+    const svc = makeService((id) =>
+      id === 'user:1' ? ['q1'] : id === 'user:2' ? ['q1', 'q2'] : ['q2']
+    );
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([
+      { table: 'user', op: 'CREATE', id: 'user:1', record: { id: 'user:1', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:2', record: { id: 'user:2', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:3', record: { id: 'user:3', _00_rv: 1 } },
+    ]);
+
+    expect(receiver.received).toHaveLength(2);
+    const byHash = Object.fromEntries(receiver.received.map((u) => [u.queryHash, u]));
+    expect(Object.keys(byHash).sort()).toEqual(['q1', 'q2']);
+  });
+
+  it('is a no-op for an empty batch and leaves the window closed', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([]);
+
+    expect(receiver.received).toHaveLength(0);
+    // A subsequent single ingest dispatches normally (window never opened).
+    svc.ingest('user', 'CREATE', 'user:1', { id: 'user:1', _00_rv: 1 });
+    expect(receiver.received).toHaveLength(1);
+  });
+});

package/src/services/stream-processor/stream-processor.reset.test.ts

@@ -0,0 +1,216 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// `StreamProcessorService.reset()` is the SSP half of a local-bucket switch:
+// the old circuit holds the previous user's rows and views registered with the
+// previous `$auth`, so reset must swap in a FRESH processor (no state load),
+// and a `saveState` racing the reset must not persist the old circuit into the
+// new bucket's key.
+
+const processorInstances: any[] = [];
+
+vi.mock('@spooky-sync/ssp-wasm', () => ({
+  default: vi.fn(async () => {}),
+  Sp00kyProcessor: vi.fn(() => {
+    const instance = {
+      ingest: vi.fn(() => []),
+      register_view: vi.fn(() => ({ query_id: 'q1', result_data: [] })),
+      unregister_view: vi.fn(),
+      set_permissions: vi.fn(),
+      save_state: vi.fn(() => 'state-bytes'),
+      load_state: vi.fn(),
+      free: vi.fn(),
+    };
+    processorInstances.push(instance);
+    return instance;
+  }),
+}));
+
+import { StreamProcessorService } from './index';
+
+const silentLogger = {
+  child: () => silentLogger,
+  trace: () => {},
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+function makeService() {
+  const persisted: Record<string, unknown> = {};
+  const persistence = {
+    set: vi.fn(async (key: string, value: unknown) => {
+      persisted[key] = value;
+    }),
+    get: vi.fn(async () => null),
+    remove: vi.fn(async () => {}),
+  };
+  const service = new StreamProcessorService(
+    {} as any,
+    {} as any,
+    persistence as any,
+    silentLogger
+  );
+  return { service, persistence, persisted };
+}
+
+beforeEach(() => {
+  processorInstances.length = 0;
+});
+
+describe('StreamProcessorService.reset', () => {
+  it('swaps in a fresh processor without loading persisted state', async () => {
+    const { service } = makeService();
+    await service.init();
+    expect(processorInstances).toHaveLength(1);
+    const first = processorInstances[0];
+
+    await service.reset();
+    expect(processorInstances).toHaveLength(2);
+    const second = processorInstances[1];
+    // Fresh circuit: nothing loaded into it (a persisted snapshot references
+    // views under a dead sessionId salt and the previous user's data).
+    expect(second.load_state).not.toHaveBeenCalled();
+
+    // New registrations land on the fresh processor, not the old circuit.
+    service.registerQueryPlan({
+      queryHash: 'q1',
+      surql: 'SELECT * FROM thing;',
+      params: {},
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      localArray: [],
+      remoteArray: [],
+      meta: { tableName: 'thing' },
+    } as any);
+    expect(second.register_view).toHaveBeenCalled();
+    expect(first.register_view).not.toHaveBeenCalled();
+  });
+
+  it('routes persisted state to the per-bucket key', async () => {
+    const { service, persistence } = makeService();
+    service.configureCircuitPersistence(true);
+    await service.init();
+    service.setStateKeySuffix('u1');
+    await service.saveState();
+    expect(persistence.set).toHaveBeenCalledWith('_00_stream_processor_state:u1', 'state-bytes');
+  });
+
+  it('drops a saveState that raced a reset (old circuit never persists into the new key)', async () => {
+    const { service, persistence } = makeService();
+    service.configureCircuitPersistence(true);
+    await service.init();
+    // The snapshot is taken, then a reset lands before the persist step,
+    // simulated by bumping the generation from inside save_state().
+    processorInstances[0].save_state.mockImplementation(() => {
+      void service.reset();
+      return 'stale-circuit';
+    });
+    await service.saveState();
+    expect(persistence.set).not.toHaveBeenCalled();
+  });
+
+  it('frees the replaced wasm circuit on reset', async () => {
+    const { service } = makeService();
+    await service.init();
+    const first = processorInstances[0];
+    await service.reset();
+    // V8 cannot see wasm-internal bytes, so the FinalizationRegistry may never
+    // run. The old circuit must be freed explicitly or its whole store stays
+    // resident for the rest of the session.
+    expect(first.free).toHaveBeenCalledTimes(1);
+  });
+});
+
+// The renderer-OOM guardrail. `Circuit::save` deep-clones the entire store (every
+// row of every ingested table) and JSON-encodes it. It used to run once per
+// ingest batch and once per query register/unregister, which on a large windowed
+// list meant hundreds of whole-store serializations per scroll, for a snapshot
+// the browser could never read back. Snapshots are now opt-in and checkpointed.
+describe('StreamProcessorService circuit snapshots', () => {
+  const plan = {
+    queryHash: 'q1',
+    surql: 'SELECT * FROM thing;',
+    params: {},
+    ttl: '10m',
+    lastActiveAt: new Date(),
+    localArray: [],
+    remoteArray: [],
+    meta: { tableName: 'thing' },
+  } as any;
+
+  it('never snapshots on ingest or register/unregister by default', async () => {
+    const { service, persistence } = makeService();
+    await service.init();
+    const processor = processorInstances[0];
+
+    service.registerQueryPlan(plan);
+    service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
+    service.ingestMany([
+      { table: 'thing', op: 'CREATE', id: 'thing:b', record: { id: 'thing:b' } },
+      { table: 'thing', op: 'CREATE', id: 'thing:c', record: { id: 'thing:c' } },
+    ]);
+    service.unregisterQueryPlan('q1');
+
+    expect(processor.save_state).not.toHaveBeenCalled();
+    expect(persistence.set).not.toHaveBeenCalled();
+  });
+
+  it('still never snapshots inline when persistence is opted in', async () => {
+    const { service, persistence } = makeService();
+    service.configureCircuitPersistence(true, 30_000);
+    await service.init();
+    const processor = processorInstances[0];
+
+    service.registerQueryPlan(plan);
+    service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
+    service.unregisterQueryPlan('q1');
+
+    // Marked dirty, but the write waits for the checkpoint interval.
+    expect(processor.save_state).not.toHaveBeenCalled();
+    expect(persistence.set).not.toHaveBeenCalled();
+    service.stopCheckpoints();
+  });
+
+  it('writes at most one snapshot per checkpoint tick', async () => {
+    vi.useFakeTimers();
+    try {
+      const { service } = makeService();
+      service.configureCircuitPersistence(true, 1000);
+      await service.init();
+      const processor = processorInstances[0];
+
+      for (let i = 0; i < 50; i++) {
+        service.ingest('thing', 'CREATE', `thing:${i}`, { id: `thing:${i}` });
+      }
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(processor.save_state).toHaveBeenCalledTimes(1);
+
+      // Idle interval: nothing changed, so nothing is serialized.
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(processor.save_state).toHaveBeenCalledTimes(1);
+
+      service.stopCheckpoints();
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it('restores a snapshot the shipped persistence clients actually return', async () => {
+    const { service, persistence } = makeService();
+    // Both clients round-trip `save_state`'s output as a bare string. The old
+    // shape check looked for a raw SurrealDB result, so it never matched and the
+    // browser never restored anything it wrote.
+    persistence.get.mockResolvedValue('state-bytes' as any);
+    service.configureCircuitPersistence(true);
+    await service.init();
+    expect(processorInstances[0].load_state).toHaveBeenCalledWith('state-bytes');
+  });
+
+  it('does not restore a snapshot when persistence is off', async () => {
+    const { service, persistence } = makeService();
+    persistence.get.mockResolvedValue('state-bytes' as any);
+    await service.init();
+    expect(processorInstances[0].load_state).not.toHaveBeenCalled();
+  });
+});

package/src/services/stream-processor/stream-processor.test.ts

@@ -127,7 +127,7 @@ describe('StreamProcessor Ingest Behavior', () => {
     const params = { id: new MockRecordId('user', '2dng4ngbicbl0scod87i') };
     const normalizedParams = normalizeValue(params);
 
-    const ingestedRecord = {
+    const _ingestedRecord = {
       id: 'user:2dng4ngbicbl0scod87i',
       username: 'sara',
     };

package/src/services/stream-processor/wasm-types.ts

@@ -1,9 +1,17 @@
-import { RecordVersionArray } from '../../types';
+import type { RecordVersionArray } from '../../types';
 
 export interface WasmStreamUpdate {
   query_id: string;
   result_hash: string;
   result_data: RecordVersionArray; // Match Rust 'result_data' field
+  // Per-phase SSP processing time (ms). Ingest path: store_apply/circuit_step/
+  // transform. Register path: parse/plan/snapshot. Unused side is 0.
+  timing_store_apply_ms?: number;
+  timing_circuit_step_ms?: number;
+  timing_transform_ms?: number;
+  timing_parse_ms?: number;
+  timing_plan_ms?: number;
+  timing_snapshot_ms?: number;
 }
 
 export interface WasmQueryConfig {
@@ -23,9 +31,22 @@ export interface WasmIngestItem {
   version?: number;
 }
 
-// Interface matching the SpookyProcessor class from WASM
+// Interface matching the Sp00kyProcessor class from WASM
 export interface WasmProcessor {
   ingest(table: string, op: string, id: string, record: any): WasmStreamUpdate[];
   register_view(config: WasmQueryConfig): WasmStreamUpdate | undefined;
   unregister_view(id: string): void;
+  // Seed per-table `select` permission predicates ({ [table]: whereText }) so
+  // register_view can inject them instead of default-denying the table.
+  set_permissions(permissions: Record<string, string>): void;
+  // Persistence hooks: present on current WASM builds, absent on stale ones.
+  // Always guarded with `typeof x === 'function'` before calling so an older
+  // build degrades gracefully instead of throwing.
+  load_state?(state: string): void;
+  save_state?(): string;
+  // wasm-bindgen destructor. Releases the circuit (store + every view cache)
+  // inside wasm linear memory. Without it the bytes only come back when V8
+  // happens to GC the JS wrapper, which it has no reason to hurry since it
+  // cannot see how much wasm memory the wrapper is holding.
+  free?(): void;
 }