@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

package/src/services/stream-processor/permissions.test.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect } from 'vitest';
+import { extractSelectPermissions } from './permissions';
+
+describe('extractSelectPermissions', () => {
+  it('maps a permissive multi-action table to true', () => {
+    const surql = `DEFINE TABLE game SCHEMAFULL
+      PERMISSIONS FOR select, create, update, delete WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ game: 'true' });
+  });
+
+  it('handles FULL and NONE', () => {
+    const surql = `
+      DEFINE TABLE a PERMISSIONS FULL;
+      DEFINE TABLE b PERMISSIONS NONE;`;
+    expect(extractSelectPermissions(surql)).toEqual({ a: 'true', b: 'false' });
+  });
+
+  it('extracts the select predicate from multiple FOR groups', () => {
+    const surql = `DEFINE TABLE doc SCHEMAFULL
+      PERMISSIONS
+        FOR create, update, delete WHERE owner = $auth.id
+        FOR select WHERE owner = $auth.id OR public = true;`;
+    expect(extractSelectPermissions(surql)).toEqual({
+      doc: 'owner = $auth.id OR public = true',
+    });
+  });
+
+  it('returns false when permissions exist but none grant select', () => {
+    const surql = `DEFINE TABLE log PERMISSIONS FOR create WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ log: 'false' });
+  });
+
+  it('treats a missing PERMISSIONS clause as default-deny', () => {
+    const surql = `DEFINE TABLE bare SCHEMALESS;`;
+    expect(extractSelectPermissions(surql)).toEqual({ bare: 'false' });
+  });
+
+  it('ignores commented-out clauses and handles OVERWRITE / IF NOT EXISTS', () => {
+    const surql = `
+      -- DEFINE TABLE ghost PERMISSIONS FULL;
+      DEFINE TABLE OVERWRITE u PERMISSIONS FOR select WHERE true;
+      DEFINE TABLE IF NOT EXISTS v PERMISSIONS FULL;`;
+    const perms = extractSelectPermissions(surql);
+    expect(perms).toEqual({ u: 'true', v: 'true' });
+    expect(perms.ghost).toBeUndefined();
+  });
+});

package/src/services/stream-processor/permissions.ts

@@ -0,0 +1,53 @@
+/**
+ * Extract per-table `select` permission predicates from a raw `.surql` schema.
+ *
+ * The in-browser SSP default-denies any non-`_00_` table that has no permission
+ * predicate registered (see `permission_inject::build_predicate`). The client
+ * already holds the full schema text (`config.schemaSurql`), so we parse each
+ * `DEFINE TABLE … PERMISSIONS …` clause and hand the `select` predicate to the
+ * processor via `set_permissions` at boot — mirroring the native path that
+ * seeds the same map from `INFO FOR DB`.
+ *
+ * Returns `{ [table]: whereText }` where `whereText` is the raw SurrealQL
+ * expression (`'true'` for `FULL`, `'false'` for `NONE` or a table with no
+ * `select` permission).
+ */
+export function extractSelectPermissions(schemaSurql: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  if (!schemaSurql) return out;
+
+  // Drop line comments so a `--` comment can't contain a stray PERMISSIONS/FOR.
+  const cleaned = schemaSurql.replace(/--[^\n]*/g, '');
+
+  // One entry per `DEFINE TABLE … ;` statement.
+  const tableStmt = /DEFINE\s+TABLE\s+(?:OVERWRITE\s+|IF\s+NOT\s+EXISTS\s+)?([A-Za-z_][\w]*)\b([^;]*);/gi;
+  let m: RegExpExecArray | null;
+  while ((m = tableStmt.exec(cleaned)) !== null) {
+    const table = m[1];
+    const body = m[2];
+    out[table] = selectPredicateFromBody(body);
+  }
+  return out;
+}
+
+function selectPredicateFromBody(body: string): string {
+  const permIdx = body.search(/\bPERMISSIONS\b/i);
+  if (permIdx === -1) return 'false'; // no clause → SurrealDB default-deny
+  const perms = body.slice(permIdx + 'PERMISSIONS'.length).trim();
+
+  if (/^FULL\b/i.test(perms)) return 'true';
+  if (/^NONE\b/i.test(perms)) return 'false';
+
+  // `FOR <actions> WHERE <expr>` groups, possibly several. Split on the FOR
+  // boundary; the leading empty segment (before the first FOR) is ignored.
+  const groups = perms.split(/\bFOR\b/i).map((g) => g.trim()).filter(Boolean);
+  for (const group of groups) {
+    const where = group.search(/\bWHERE\b/i);
+    if (where === -1) continue;
+    const actions = group.slice(0, where).toLowerCase();
+    if (/\bselect\b/.test(actions)) {
+      return group.slice(where + 'WHERE'.length).trim();
+    }
+  }
+  return 'false'; // permissions present but none grant select
+}

package/src/services/stream-processor/stream-processor.batch.test.ts

@@ -0,0 +1,186 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { StreamProcessorService } from './index';
+import type { StreamUpdate, StreamUpdateReceiver } from './index';
+import type { WasmProcessor, WasmStreamUpdate } from './wasm-types';
+
+/**
+ * Tests for `ingestMany` bulk insert on StreamProcessorService.
+ *
+ * A batched ingest (e.g. sync fetching N missing records) used to emit one
+ * stream update per record, making the UI render row-by-row. ingestMany
+ * collapses those into a single coalesced update per affected query.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = {
+    debug: noop,
+    info: noop,
+    warn: noop,
+    error: noop,
+    trace: noop,
+  };
+  logger.child = () => logger;
+  return logger;
+}
+
+/** Records every dispatched update so we can count notifications. */
+class RecordingReceiver implements StreamUpdateReceiver {
+  public received: StreamUpdate[] = [];
+  onStreamUpdate(update: StreamUpdate): void {
+    this.received.push(update);
+  }
+}
+
+/**
+ * Build a service with a stubbed WASM processor. The processor accumulates
+ * ingested ids and returns the *full* materialized array on every call (as the
+ * real WASM does), so coalescing's last-write-wins must yield the final array.
+ */
+function makeService(queryHashesFor: (id: string) => string[]) {
+  const svc = new StreamProcessorService({} as any, {} as any, makeLogger());
+
+  const ingestedByQuery = new Map<string, Array<[string, number]>>();
+  const mockProcessor: Partial<WasmProcessor> = {
+    ingest: (_table, _op, id, record: any): WasmStreamUpdate[] => {
+      const version = record?._00_rv ?? 1;
+      const updates: WasmStreamUpdate[] = [];
+      for (const queryHash of queryHashesFor(id)) {
+        const arr = ingestedByQuery.get(queryHash) ?? [];
+        arr.push([id, version]);
+        ingestedByQuery.set(queryHash, arr);
+        updates.push({ query_id: queryHash, result_hash: '', result_data: [...arr] });
+      }
+      return updates;
+    },
+  };
+  // processor is private and normally set by init() via WASM; inject the stub.
+  (svc as any).processor = mockProcessor;
+  return svc;
+}
+
+describe('StreamProcessor ingestMany bulk insert', () => {
+  let receiver: RecordingReceiver;
+
+  beforeEach(() => {
+    receiver = new RecordingReceiver();
+  });
+
+  it('emits one update per record when ingesting one-by-one (baseline)', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingest('user', 'CREATE', 'user:1', { id: 'user:1', _00_rv: 1 });
+    svc.ingest('user', 'CREATE', 'user:2', { id: 'user:2', _00_rv: 1 });
+    svc.ingest('user', 'CREATE', 'user:3', { id: 'user:3', _00_rv: 1 });
+
+    expect(receiver.received).toHaveLength(3);
+  });
+
+  it('coalesces a bulk insert into a single update per query with the final array', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([
+      { table: 'user', op: 'CREATE', id: 'user:1', record: { id: 'user:1', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:2', record: { id: 'user:2', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:3', record: { id: 'user:3', _00_rv: 1 } },
+    ]);
+
+    // Nothing dispatched until the whole batch is ingested, then exactly one
+    // coalesced update.
+    expect(receiver.received).toHaveLength(1);
+    const update = receiver.received[0];
+    expect(update.queryHash).toBe('q1');
+    // Last-write-wins carries the full materialized array (all 3 records).
+    expect(update.localArray).toHaveLength(3);
+    // Coalesced updates take DataModule's immediate (non-debounced) path.
+    expect(update.op).toBe('CREATE');
+    expect(typeof update.materializationTimeMs).toBe('number');
+  });
+
+  it('coalesces independently per affected query', () => {
+    // user:1 -> q1, user:2 -> q1 & q2, user:3 -> q2
+    const svc = makeService((id) =>
+      id === 'user:1' ? ['q1'] : id === 'user:2' ? ['q1', 'q2'] : ['q2']
+    );
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([
+      { table: 'user', op: 'CREATE', id: 'user:1', record: { id: 'user:1', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:2', record: { id: 'user:2', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:3', record: { id: 'user:3', _00_rv: 1 } },
+    ]);
+
+    expect(receiver.received).toHaveLength(2);
+    const byHash = Object.fromEntries(receiver.received.map((u) => [u.queryHash, u]));
+    expect(Object.keys(byHash).sort()).toEqual(['q1', 'q2']);
+  });
+
+  it('is a no-op for an empty batch and leaves the window closed', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([]);
+
+    expect(receiver.received).toHaveLength(0);
+    // A subsequent single ingest dispatches normally (window never opened).
+    svc.ingest('user', 'CREATE', 'user:1', { id: 'user:1', _00_rv: 1 });
+    expect(receiver.received).toHaveLength(1);
+  });
+
+  // A WASM build that exposes `ingest_many` takes one circuit step for the whole
+  // batch. The per-record loop above stays as the fallback for older builds.
+  describe('with a WASM build that supports bulk ingest', () => {
+    function makeBulkService() {
+      const svc = new StreamProcessorService({} as any, {} as any, makeLogger());
+      const calls: { items: any[] }[] = [];
+      const mockProcessor: Partial<WasmProcessor> = {
+        ingest: () => {
+          throw new Error('per-record ingest must not be used when ingest_many exists');
+        },
+        ingest_many: (items): WasmStreamUpdate[] => {
+          calls.push({ items });
+          return [
+            {
+              query_id: 'q1',
+              result_data: items.map((i) => [i.id, 1] as [string, number]),
+              timing_circuit_step_ms: 4,
+            } as WasmStreamUpdate,
+          ];
+        },
+      };
+      (svc as any).processor = mockProcessor;
+      return { svc, calls };
+    }
+
+    it('takes a single bulk call and dispatches one update per affected query', () => {
+      const { svc, calls } = makeBulkService();
+      svc.addReceiver(receiver);
+
+      svc.ingestMany([
+        { table: 'user', op: 'CREATE', id: 'user:1', record: { id: 'user:1', _00_rv: 1 } },
+        { table: 'user', op: 'CREATE', id: 'user:2', record: { id: 'user:2', _00_rv: 1 } },
+        { table: 'user', op: 'CREATE', id: 'user:3', record: { id: 'user:3', _00_rv: 1 } },
+      ]);
+
+      expect(calls).toHaveLength(1);
+      expect(calls[0].items.map((i) => i.id)).toEqual(['user:1', 'user:2', 'user:3']);
+      expect(receiver.received).toHaveLength(1);
+      expect(receiver.received[0].queryHash).toBe('q1');
+      expect(receiver.received[0].localArray).toHaveLength(3);
+      expect(receiver.received[0].op).toBe('CREATE');
+      expect(receiver.received[0].circuitStepMs).toBe(4);
+    });
+
+    it('never calls the bulk path for an empty batch', () => {
+      const { svc, calls } = makeBulkService();
+      svc.addReceiver(receiver);
+
+      svc.ingestMany([]);
+
+      expect(calls).toHaveLength(0);
+      expect(receiver.received).toHaveLength(0);
+    });
+  });
+});

package/src/services/stream-processor/stream-processor.prime.test.ts

@@ -0,0 +1,198 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// The boot-time prime: the circuit fills from the LOCAL store (a snapshot plus
+// a reconcile against `(id, rv)`, or every cached row) instead of from the
+// network. Before this, a reload left the circuit empty, every id in the
+// server's list_ref classified as `added`, and the whole working set was
+// re-downloaded, re-written and re-ingested.
+
+const processorInstances: any[] = [];
+
+vi.mock('@spooky-sync/ssp-wasm', () => ({
+  default: vi.fn(async () => {}),
+  Sp00kyProcessor: vi.fn(() => {
+    const instance = {
+      ingest: vi.fn(() => []),
+      ingest_many: vi.fn(() => []),
+      register_view: vi.fn(() => ({ query_id: 'q1', result_hash: '', result_data: [] })),
+      unregister_view: vi.fn(),
+      set_permissions: vi.fn(),
+      set_projection: vi.fn(),
+      save_store_state: vi.fn(() => new Uint8Array([1])),
+      load_store_state: vi.fn(() => [{ query_id: 'q1', result_hash: '', result_data: [['thing:a', 1]] }]),
+      reconcile: vi.fn(() => ({ fetch: [], deleted: 0, updates: [] })),
+      free: vi.fn(),
+    };
+    processorInstances.push(instance);
+    return instance;
+  }),
+}));
+
+import { StreamProcessorService, CIRCUIT_SNAPSHOT_FORMAT } from './index';
+import type { StreamUpdate } from './index';
+
+const silentLogger = {
+  child: () => silentLogger,
+  trace: () => {},
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+function row(id: string, rv: number) {
+  return { id, _00_rv: rv, title: id };
+}
+
+function makeService(opts: { snapshot?: { bytes: Uint8Array; meta: any } | null; engineKind?: string } = {}) {
+  const versions: Record<string, [string, number][]> = {
+    thing: [
+      ['thing:a', 1],
+      ['thing:b', 2],
+    ],
+    other: [['other:x', 5]],
+  };
+  const db = {
+    epoch: 0,
+    engineKind: opts.engineKind ?? 'sqlite',
+    scanVersions: vi.fn(async (tables: string[]) =>
+      Object.fromEntries(tables.map((t) => [t, versions[t] ?? []]))
+    ),
+    selectByIds: vi.fn(async (table: string, ids: string[]) =>
+      ids.map((id) => row(id, versions[table]?.find(([i]) => i === id)?.[1] ?? 0))
+    ),
+    getSnapshot: vi.fn(async () => opts.snapshot ?? null),
+    putSnapshot: vi.fn(async () => {}),
+  };
+  const service = new StreamProcessorService({} as any, db as any, silentLogger);
+  service.configureCircuitPersistence(true);
+  const received: StreamUpdate[] = [];
+  service.addReceiver({ onStreamUpdate: (u) => received.push(u) });
+  return { service, db, received };
+}
+
+const goodMeta = { formatVersion: CIRCUIT_SNAPSHOT_FORMAT, schemaHash: 'h', savedAt: 1 };
+const ctx = () => ({
+  tables: ['thing', 'other'],
+  schemaHash: 'h',
+  pendingIds: new Set<string>(),
+  onVersions: vi.fn(),
+});
+
+beforeEach(() => {
+  processorInstances.length = 0;
+});
+
+describe('StreamProcessorService.primeFromLocal', () => {
+  it('without a snapshot, ingests every cached row and reports their versions', async () => {
+    const { service, db } = makeService();
+    await service.init();
+    const c = ctx();
+    await service.primeFromLocal(c);
+    const p = processorInstances[0];
+    expect(p.load_store_state).not.toHaveBeenCalled();
+    expect(db.selectByIds).toHaveBeenCalledWith('thing', ['thing:a', 'thing:b']);
+    expect(db.selectByIds).toHaveBeenCalledWith('other', ['other:x']);
+    const items = p.ingest_many.mock.calls.flatMap((call: any[]) => call[0]);
+    expect(items.map((i: any) => [i.table, i.op, i.id])).toEqual([
+      ['thing', 'CREATE', 'thing:a'],
+      ['thing', 'CREATE', 'thing:b'],
+      ['other', 'CREATE', 'other:x'],
+    ]);
+    expect(c.onVersions).toHaveBeenCalledWith('thing', [
+      ['thing:a', 1],
+      ['thing:b', 2],
+    ]);
+    await expect(service.whenPrimed()).resolves.toBeUndefined();
+  });
+
+  it('with a snapshot, restores it under the registered views and only fetches what reconcile asks for', async () => {
+    const { service, db, received } = makeService({ snapshot: { bytes: new Uint8Array([7]), meta: goodMeta } });
+    await service.init();
+    const p = processorInstances[0];
+    p.reconcile.mockImplementation((table: string) =>
+      table === 'thing'
+        ? { fetch: ['thing:b'], deleted: 1, updates: [{ query_id: 'q1', result_hash: '', result_data: [] }] }
+        : { fetch: [], deleted: 0, updates: [] }
+    );
+    await service.primeFromLocal(ctx());
+    expect(p.load_store_state).toHaveBeenCalledWith(new Uint8Array([7]));
+    // The restore's re-primed view result and reconcile's retraction reached
+    // the receivers.
+    expect(received.map((u) => u.queryHash)).toEqual(['q1', 'q1']);
+    expect(p.reconcile).toHaveBeenCalledWith('thing', [
+      ['thing:a', 1],
+      ['thing:b', 2],
+    ]);
+    expect(db.selectByIds).toHaveBeenCalledTimes(1);
+    expect(db.selectByIds).toHaveBeenCalledWith('thing', ['thing:b']);
+    const items = p.ingest_many.mock.calls.flatMap((call: any[]) => call[0]);
+    expect(items.map((i: any) => i.id)).toEqual(['thing:b']);
+  });
+
+  it('ignores a snapshot written under another schema or format', async () => {
+    const { service } = makeService({
+      snapshot: { bytes: new Uint8Array([7]), meta: { ...goodMeta, schemaHash: 'other' } },
+    });
+    await service.init();
+    await service.primeFromLocal(ctx());
+    expect(processorInstances[0].load_store_state).not.toHaveBeenCalled();
+    expect(processorInstances[0].ingest_many).toHaveBeenCalled();
+  });
+
+  it('leaves out ids with a pending local mutation when reporting versions', async () => {
+    const { service } = makeService();
+    await service.init();
+    const c = ctx();
+    c.pendingIds.add('thing:b');
+    await service.primeFromLocal(c);
+    expect(c.onVersions).toHaveBeenCalledWith('thing', [['thing:a', 1]]);
+  });
+
+  it('aborts when the store epoch moves under it (bucket switch)', async () => {
+    const { service, db } = makeService();
+    await service.init();
+    db.scanVersions.mockImplementation(async () => {
+      db.epoch++;
+      return { thing: [['thing:a', 1]], other: [] };
+    });
+    const c = ctx();
+    await service.primeFromLocal(c);
+    expect(db.selectByIds).not.toHaveBeenCalled();
+    expect(processorInstances[0].ingest_many).not.toHaveBeenCalled();
+    expect(c.onVersions).not.toHaveBeenCalled();
+  });
+
+  it('is a no-op on an engine without scanVersions', async () => {
+    const service = new StreamProcessorService({} as any, {} as any, silentLogger);
+    await service.init();
+    await service.primeFromLocal(ctx());
+    expect(processorInstances[0].ingest_many).not.toHaveBeenCalled();
+  });
+
+  it('widens projected rows when a new view evaluates fields they lack', async () => {
+    const { service, db } = makeService();
+    await service.init();
+    const p = processorInstances[0];
+    p.register_view.mockReturnValue({
+      query_id: 'q2',
+      result_hash: '',
+      result_data: [],
+      missing_fields: { thing: ['score'] },
+    });
+    service.registerQueryPlan({
+      queryHash: 'q2',
+      surql: 'SELECT * FROM thing ORDER BY score;',
+      params: {},
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      localArray: [],
+      remoteArray: [],
+      meta: { tableName: 'thing' },
+    } as any);
+    await vi.waitFor(() => expect(p.ingest_many).toHaveBeenCalled());
+    expect(db.selectByIds).toHaveBeenCalledWith('thing', ['thing:a', 'thing:b'], { select: ['score'] });
+    const items = p.ingest_many.mock.calls.flatMap((call: any[]) => call[0]);
+    expect(items.map((i: any) => i.op)).toEqual(['MERGE', 'MERGE']);
+  });
+});