@spooky-sync/core 0.0.1-canary.7 → 0.0.1-canary.71

package/src/services/persistence/surrealdb.ts

@@ -1,7 +1,7 @@
-import { PersistenceClient } from '../../types';
+import type { PersistenceClient } from '../../types';
 import { parseRecordIdString, surql } from '../../utils/index';
-import { Logger } from 'pino';
-import { AbstractDatabaseService } from '../database/database';
+import type { Logger } from 'pino';
+import type { AbstractDatabaseService } from '../database/database';
 
 export class SurrealDBPersistenceClient implements PersistenceClient {
   private logger: Logger;
@@ -15,11 +15,11 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
 
   async set<T>(key: string, val: T) {
     try {
-      const id = parseRecordIdString(`_spooky_kv:${key}`);
+      const id = parseRecordIdString(`_00_kv:${key}`);
       await this.db.query(surql.seal(surql.upsert('id', 'data')), { id, data: { val } });
     } catch (error) {
       this.logger.error(
-        { error, Category: 'spooky-client::SurrealDBPersistenceClient::set' },
+        { error, Category: 'sp00ky-client::SurrealDBPersistenceClient::set' },
         'Failed to set KV'
       );
       throw error;
@@ -28,7 +28,7 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
 
   async get<T>(key: string) {
     try {
-      const id = parseRecordIdString(`_spooky_kv:${key}`);
+      const id = parseRecordIdString(`_00_kv:${key}`);
       const [result] = await this.db.query<[{ val: T }]>(
         surql.seal(surql.selectById('id', ['val'])),
         {
@@ -41,7 +41,7 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
       return result.val;
     } catch (error) {
       this.logger.warn(
-        { error, Category: 'spooky-client::SurrealDBPersistenceClient::get' },
+        { error, Category: 'sp00ky-client::SurrealDBPersistenceClient::get' },
         'Failed to get KV'
       );
       return null;
@@ -50,11 +50,11 @@ export class SurrealDBPersistenceClient implements PersistenceClient {
 
   async remove(key: string) {
     try {
-      const id = parseRecordIdString(`_spooky_kv:${key}`);
+      const id = parseRecordIdString(`_00_kv:${key}`);
       await this.db.query(surql.seal(surql.delete('id')), { id });
     } catch (err) {
       this.logger.info(
-        { err, Category: 'spooky-client::SurrealDBPersistenceClient::remove' },
+        { err, Category: 'sp00ky-client::SurrealDBPersistenceClient::remove' },
         'Failed to delete KV'
       );
     }

package/src/services/stream-processor/index.ts

@@ -1,10 +1,11 @@
-import init, { SpookyProcessor } from '@spooky-sync/ssp-wasm';
-import { EventDefinition, EventSystem } from '../../events/index';
-import { Logger } from 'pino';
-import { LocalDatabaseService } from '../database/index';
-import { WasmProcessor, WasmStreamUpdate } from './wasm-types';
-import { Duration } from 'surrealdb';
-import { PersistenceClient, QueryTimeToLive, RecordVersionArray } from '../../types';
+// oxlint-disable-next-line no-named-as-default -- WASM module default export convention
+import init, { Sp00kyProcessor } from '@spooky-sync/ssp-wasm';
+import type { EventDefinition, EventSystem } from '../../events/index';
+import type { Logger } from 'pino';
+import type { LocalDatabaseService } from '../database/index';
+import type { WasmProcessor, WasmStreamUpdate } from './wasm-types';
+import type { Duration } from 'surrealdb';
+import type { PersistenceClient, QueryTimeToLive, RecordVersionArray } from '../../types';
 
 // Simple interface for query plan registration (replaces Incantation class)
 interface QueryPlanConfig {
@@ -27,6 +28,21 @@ export interface StreamUpdate {
   queryHash: string;
   localArray: RecordVersionArray;
   op?: 'CREATE' | 'UPDATE' | 'DELETE'; // Operation type for conditional debouncing
+  /**
+   * End-to-end ingest latency for the WASM call that produced this update,
+   * in milliseconds. Populated by StreamProcessorService.ingest. Undefined
+   * for the initial register_view snapshot.
+   */
+  materializationTimeMs?: number;
+  /** SSP internal sub-phase timings (ms) for this ingest, from the WASM binding. */
+  storeApplyMs?: number;
+  circuitStepMs?: number;
+  transformMs?: number;
+  /**
+   * One-shot registration timings (ms). Only set on the StreamUpdate returned
+   * by `registerQueryPlan` (the register_view snapshot), not on ingest updates.
+   */
+  registration?: { parseMs: number; planMs: number; snapshotMs: number };
 }
 
 // Define events map (kept for DevTools compatibility)
@@ -46,6 +62,12 @@ export class StreamProcessorService {
   private processor: WasmProcessor | undefined;
   private isInitialized = false;
   private receivers: StreamUpdateReceiver[] = [];
+  // When true, `notifyUpdates` coalesces updates into `batchBuffer` (keyed by
+  // queryHash) instead of dispatching them. Used to collapse the per-record
+  // stream updates produced by a batched ingest into a single notification per
+  // query, so the UI updates once after the whole batch rather than row-by-row.
+  private batching = false;
+  private batchBuffer: Map<string, StreamUpdate> = new Map();
 
   constructor(
     public events: EventSystem<StreamProcessorEvents>,
@@ -65,6 +87,32 @@ export class StreamProcessorService {
   }
 
   private notifyUpdates(updates: StreamUpdate[]) {
+    if (this.batching) {
+      // Coalesce by queryHash instead of dispatching. The WASM `result_data`
+      // (localArray) is the full materialized array, so last-write-wins
+      // already reflects every prior ingest in the batch. We sum the
+      // materialization times so the single recorded sample reflects the
+      // batch's total work, and emit `op: 'CREATE'` on flush so the coalesced
+      // update takes DataModule's immediate (non-debounced) path.
+      for (const update of updates) {
+        const prev = this.batchBuffer.get(update.queryHash);
+        const sum = (a?: number, b?: number) => (a ?? 0) + (b ?? 0);
+        this.batchBuffer.set(update.queryHash, {
+          ...update,
+          op: 'CREATE',
+          materializationTimeMs: sum(prev?.materializationTimeMs, update.materializationTimeMs),
+          storeApplyMs: sum(prev?.storeApplyMs, update.storeApplyMs),
+          circuitStepMs: sum(prev?.circuitStepMs, update.circuitStepMs),
+          transformMs: sum(prev?.transformMs, update.transformMs),
+        });
+      }
+      return;
+    }
+
+    this.dispatchUpdates(updates);
+  }
+
+  private dispatchUpdates(updates: StreamUpdate[]) {
     for (const update of updates) {
       for (const receiver of this.receivers) {
         receiver.onStreamUpdate(update);
@@ -72,6 +120,72 @@ export class StreamProcessorService {
     }
   }
 
+  /**
+   * Ingest a batch of record changes as a single bulk operation, firing only
+   * one coalesced `StreamUpdate` per affected query once every record has been
+   * ingested (instead of one update per record). Use this whenever multiple
+   * records land at once — e.g. sync fetching N missing rows — so a list query
+   * re-runs and the UI re-renders once for the whole batch rather than
+   * row-by-row.
+   *
+   * Internally opens a coalescing window, ingests each record, then flushes;
+   * processor state is persisted once for the whole batch. No-op for an empty
+   * batch.
+   */
+  ingestMany(
+    records: Array<{
+      table: string;
+      op: 'CREATE' | 'UPDATE' | 'DELETE';
+      id: string;
+      record: any;
+    }>
+  ): void {
+    if (records.length === 0) return;
+
+    this.beginCoalescing();
+    try {
+      for (const record of records) {
+        this.ingest(record.table, record.op, record.id, record.record);
+      }
+    } finally {
+      this.flushCoalescing();
+    }
+  }
+
+  /**
+   * Open a coalescing window. While open, the per-record stream updates
+   * emitted by `ingest` are buffered (one entry per queryHash) instead of
+   * dispatched. Always paired with `flushCoalescing()` in a try/finally by
+   * `ingestMany` so the window always closes — otherwise the processor stays
+   * stuck buffering forever.
+   *
+   * No-op if a window is already open (nested batches aren't expected here).
+   */
+  private beginCoalescing() {
+    if (this.batching) return;
+    this.batching = true;
+    this.batchBuffer.clear();
+  }
+
+  /**
+   * Close the coalescing window and flush: dispatch one coalesced
+   * `StreamUpdate` per buffered queryHash, then persist processor state once
+   * for the whole batch (instead of once per ingest).
+   */
+  private flushCoalescing() {
+    if (!this.batching) return;
+    this.batching = false;
+    const buffered = Array.from(this.batchBuffer.values());
+    this.batchBuffer.clear();
+    if (buffered.length > 0) {
+      this.dispatchUpdates(buffered);
+    }
+    // The processor state after the last ingest is cumulative, so a single
+    // snapshot covers the whole batch. Kept fire-and-forget like the per-ingest
+    // call it replaces.
+    this.saveState();
+  }
+
   /**
    * Initialize the WASM module and processor.
    * This must be called before using other methods.
@@ -80,25 +194,25 @@ export class StreamProcessorService {
     if (this.isInitialized) return;
 
     this.logger.info(
-      { Category: 'spooky-client::StreamProcessorService::init' },
+      { Category: 'sp00ky-client::StreamProcessorService::init' },
       'Initializing WASM...'
     );
     try {
       await init(); // Initialize the WASM module (web target)
-      // We cast the generated SpookyProcessor to our interface which is safer
-      this.processor = new SpookyProcessor() as unknown as WasmProcessor;
+      // We cast the generated Sp00kyProcessor to our interface which is safer
+      this.processor = new Sp00kyProcessor() as unknown as WasmProcessor;
 
       // Try to load state
       await this.loadState();
 
       this.isInitialized = true;
       this.logger.info(
-        { Category: 'spooky-client::StreamProcessorService::init' },
+        { Category: 'sp00ky-client::StreamProcessorService::init' },
         'Initialized successfully'
       );
     } catch (e) {
       this.logger.error(
-        { error: e, Category: 'spooky-client::StreamProcessorService::init' },
+        { error: e, Category: 'sp00ky-client::StreamProcessorService::init' },
         'Failed to initialize'
       );
       throw e;
@@ -108,7 +222,7 @@ export class StreamProcessorService {
   async loadState() {
     if (!this.processor) return;
     try {
-      const result = await this.persistenceClient.get('_spooky_stream_processor_state');
+      const result = await this.persistenceClient.get('_00_stream_processor_state');
 
       // Check if we have a valid result from the query
       if (
@@ -122,51 +236,75 @@ export class StreamProcessorService {
         this.logger.info(
           {
             stateLength: state.length,
-            Category: 'spooky-client::StreamProcessorService::loadState',
+            Category: 'sp00ky-client::StreamProcessorService::loadState',
           },
           'Loading state from DB'
         );
         // Assuming processor has a load_state method matching the save_state behavior
         // If not, we might need to adjust based on the actual WASM API
-        if (typeof (this.processor as any).load_state === 'function') {
-          (this.processor as any).load_state(state);
+        if (typeof this.processor.load_state === 'function') {
+          this.processor.load_state(state);
         } else {
           this.logger.warn(
-            { Category: 'spooky-client::StreamProcessorService::loadState' },
+            { Category: 'sp00ky-client::StreamProcessorService::loadState' },
             'load_state method not found on processor'
           );
         }
       } else {
         this.logger.info(
-          { Category: 'spooky-client::StreamProcessorService::loadState' },
+          { Category: 'sp00ky-client::StreamProcessorService::loadState' },
           'No saved state found'
         );
       }
     } catch (e) {
       this.logger.error(
-        { error: e, Category: 'spooky-client::StreamProcessorService::loadState' },
+        { error: e, Category: 'sp00ky-client::StreamProcessorService::loadState' },
         'Failed to load state'
       );
     }
   }
 
+  /**
+   * Seed per-table `select` permission predicates ({ [table]: whereText }).
+   * Must run after the processor exists and before any `register_view`, else
+   * non-`_00_` tables are default-denied and registration fails.
+   */
+  setPermissions(permissions: Record<string, string>) {
+    if (!this.processor) return;
+    if (typeof this.processor.set_permissions !== 'function') {
+      this.logger.warn(
+        { Category: 'sp00ky-client::StreamProcessorService::setPermissions' },
+        'set_permissions not found on processor (stale WASM build?)'
+      );
+      return;
+    }
+    this.processor.set_permissions(permissions);
+    this.logger.info(
+      {
+        tables: Object.keys(permissions).length,
+        Category: 'sp00ky-client::StreamProcessorService::setPermissions',
+      },
+      'Seeded table permissions'
+    );
+  }
+
   async saveState() {
     if (!this.processor) return;
     try {
       // Assuming processor has a save_state method that returns the state string/bytes
-      if (typeof (this.processor as any).save_state === 'function') {
-        const state = (this.processor as any).save_state();
+      if (typeof this.processor.save_state === 'function') {
+        const state = this.processor.save_state();
         if (state) {
-          await this.persistenceClient.set('_spooky_stream_processor_state', state);
+          await this.persistenceClient.set('_00_stream_processor_state', state);
           this.logger.trace(
-            { Category: 'spooky-client::StreamProcessorService::saveState' },
+            { Category: 'sp00ky-client::StreamProcessorService::saveState' },
             'State saved'
           );
         }
       }
     } catch (e) {
       this.logger.error(
-        { error: e, Category: 'spooky-client::StreamProcessorService::saveState' },
+        { error: e, Category: 'sp00ky-client::StreamProcessorService::saveState' },
         'Failed to save state'
       );
     }
@@ -188,14 +326,14 @@ export class StreamProcessorService {
         table,
         op,
         id,
-        Category: 'spooky-client::StreamProcessorService::ingest',
+        Category: 'sp00ky-client::StreamProcessorService::ingest',
       },
       'Ingesting into ssp'
     );
 
     if (!this.processor) {
       this.logger.warn(
-        { Category: 'spooky-client::StreamProcessorService::ingest' },
+        { Category: 'sp00ky-client::StreamProcessorService::ingest' },
         'Not initialized, skipping ingest'
       );
       return [];
@@ -204,14 +342,17 @@ export class StreamProcessorService {
     try {
       const normalizedRecord = this.normalizeValue(record);
 
+      const t0 = performance.now();
       const rawUpdates = this.processor.ingest(table, op, id, normalizedRecord);
+      const materializationTimeMs = performance.now() - t0;
       this.logger.debug(
         {
           table,
           op,
           id,
           rawUpdates: rawUpdates.length,
-          Category: 'spooky-client::StreamProcessorService::ingest',
+          materializationTimeMs,
+          Category: 'sp00ky-client::StreamProcessorService::ingest',
         },
         'Ingesting into ssp done'
       );
@@ -221,15 +362,23 @@ export class StreamProcessorService {
           queryHash: u.query_id,
           localArray: u.result_data,
           op: op,
+          materializationTimeMs,
+          storeApplyMs: u.timing_store_apply_ms,
+          circuitStepMs: u.timing_circuit_step_ms,
+          transformMs: u.timing_transform_ms,
         }));
         // Direct handler call instead of event
         this.notifyUpdates(updates);
       }
-      this.saveState();
+      // While batching (inside `ingestMany`), `flushCoalescing` persists once
+      // for the whole batch — skip the redundant per-record snapshot here.
+      if (!this.batching) {
+        this.saveState();
+      }
       return rawUpdates;
     } catch (e) {
       this.logger.error(
-        { error: e, Category: 'spooky-client::StreamProcessorService::ingest' },
+        { error: e, Category: 'sp00ky-client::StreamProcessorService::ingest' },
         'Ingesting into ssp failed'
       );
     }
@@ -243,7 +392,7 @@ export class StreamProcessorService {
   registerQueryPlan(queryPlan: QueryPlanConfig) {
     if (!this.processor) {
       this.logger.warn(
-        { Category: 'spooky-client::StreamProcessorService::registerQueryPlan' },
+        { Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan' },
         'Not initialized, skipping registration'
       );
       return;
@@ -254,7 +403,7 @@ export class StreamProcessorService {
         queryHash: queryPlan.queryHash,
         surql: queryPlan.surql,
         params: queryPlan.params,
-        Category: 'spooky-client::StreamProcessorService::registerQueryPlan',
+        Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan',
       },
       'Registering query plan'
     );
@@ -272,7 +421,7 @@ export class StreamProcessorService {
       });
 
       this.logger.debug(
-        { initialUpdate, Category: 'spooky-client::StreamProcessorService::registerQueryPlan' },
+        { initialUpdate, Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan' },
         'register_view result'
       );
 
@@ -282,6 +431,11 @@ export class StreamProcessorService {
       const update: StreamUpdate = {
         queryHash: initialUpdate.query_id,
         localArray: initialUpdate.result_data,
+        registration: {
+          parseMs: initialUpdate.timing_parse_ms ?? 0,
+          planMs: initialUpdate.timing_plan_ms ?? 0,
+          snapshotMs: initialUpdate.timing_snapshot_ms ?? 0,
+        },
       };
       this.saveState();
       this.logger.debug(
@@ -289,14 +443,14 @@ export class StreamProcessorService {
           queryHash: queryPlan.queryHash,
           surql: queryPlan.surql,
           params: queryPlan.params,
-          Category: 'spooky-client::StreamProcessorService::registerQueryPlan',
+          Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan',
         },
         'Registered query plan'
       );
       return update;
     } catch (e) {
       this.logger.error(
-        { error: e, Category: 'spooky-client::StreamProcessorService::registerQueryPlan' },
+        { error: e, Category: 'sp00ky-client::StreamProcessorService::registerQueryPlan' },
         'Error registering query plan'
       );
       throw e;
@@ -313,7 +467,7 @@ export class StreamProcessorService {
       this.saveState();
     } catch (e) {
       this.logger.error(
-        { error: e, Category: 'spooky-client::StreamProcessorService::unregisterQueryPlan' },
+        { error: e, Category: 'sp00ky-client::StreamProcessorService::unregisterQueryPlan' },
         'Error unregistering query plan'
       );
     }
@@ -323,6 +477,21 @@ export class StreamProcessorService {
     if (value === null || value === undefined) return value;
 
     if (typeof value === 'object') {
+      // CRDT snapshots arrive as `Uint8Array` (or `ArrayBuffer` /
+      // typed-array views). `serde_wasm_bindgen::from_value` rejects
+      // those when deserializing into `serde_json::Value` (JSON has no
+      // binary variant), and the SSP can't filter on opaque bytes
+      // anyway. Replace with `null` so the row still flows through the
+      // ingest path with its other columns intact, and downstream
+      // predicates referencing the bytes column simply don't match.
+      if (
+        value instanceof Uint8Array ||
+        value instanceof ArrayBuffer ||
+        ArrayBuffer.isView(value)
+      ) {
+        return null;
+      }
+
       // RecordId detection using duck typing (constructor.name may be minified)
       // SurrealDB's RecordId has: table (getter returning Table), id, and toString()
       // Check for table getter that has its own toString AND id property
@@ -334,7 +503,7 @@ export class StreamProcessorService {
       if (hasTable && hasId && hasToString && isNotPlainObject) {
         const result = value.toString();
         this.logger.trace(
-          { result, Category: 'spooky-client::StreamProcessorService::normalizeValue' },
+          { result, Category: 'sp00ky-client::StreamProcessorService::normalizeValue' },
           'RecordId detected'
         );
         return result;

package/src/services/stream-processor/permissions.test.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect } from 'vitest';
+import { extractSelectPermissions } from './permissions';
+
+describe('extractSelectPermissions', () => {
+  it('maps a permissive multi-action table to true', () => {
+    const surql = `DEFINE TABLE game SCHEMAFULL
+      PERMISSIONS FOR select, create, update, delete WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ game: 'true' });
+  });
+
+  it('handles FULL and NONE', () => {
+    const surql = `
+      DEFINE TABLE a PERMISSIONS FULL;
+      DEFINE TABLE b PERMISSIONS NONE;`;
+    expect(extractSelectPermissions(surql)).toEqual({ a: 'true', b: 'false' });
+  });
+
+  it('extracts the select predicate from multiple FOR groups', () => {
+    const surql = `DEFINE TABLE doc SCHEMAFULL
+      PERMISSIONS
+        FOR create, update, delete WHERE owner = $auth.id
+        FOR select WHERE owner = $auth.id OR public = true;`;
+    expect(extractSelectPermissions(surql)).toEqual({
+      doc: 'owner = $auth.id OR public = true',
+    });
+  });
+
+  it('returns false when permissions exist but none grant select', () => {
+    const surql = `DEFINE TABLE log PERMISSIONS FOR create WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ log: 'false' });
+  });
+
+  it('treats a missing PERMISSIONS clause as default-deny', () => {
+    const surql = `DEFINE TABLE bare SCHEMALESS;`;
+    expect(extractSelectPermissions(surql)).toEqual({ bare: 'false' });
+  });
+
+  it('ignores commented-out clauses and handles OVERWRITE / IF NOT EXISTS', () => {
+    const surql = `
+      -- DEFINE TABLE ghost PERMISSIONS FULL;
+      DEFINE TABLE OVERWRITE u PERMISSIONS FOR select WHERE true;
+      DEFINE TABLE IF NOT EXISTS v PERMISSIONS FULL;`;
+    const perms = extractSelectPermissions(surql);
+    expect(perms).toEqual({ u: 'true', v: 'true' });
+    expect(perms.ghost).toBeUndefined();
+  });
+});

package/src/services/stream-processor/permissions.ts

@@ -0,0 +1,53 @@
+/**
+ * Extract per-table `select` permission predicates from a raw `.surql` schema.
+ *
+ * The in-browser SSP default-denies any non-`_00_` table that has no permission
+ * predicate registered (see `permission_inject::build_predicate`). The client
+ * already holds the full schema text (`config.schemaSurql`), so we parse each
+ * `DEFINE TABLE … PERMISSIONS …` clause and hand the `select` predicate to the
+ * processor via `set_permissions` at boot — mirroring the native path that
+ * seeds the same map from `INFO FOR DB`.
+ *
+ * Returns `{ [table]: whereText }` where `whereText` is the raw SurrealQL
+ * expression (`'true'` for `FULL`, `'false'` for `NONE` or a table with no
+ * `select` permission).
+ */
+export function extractSelectPermissions(schemaSurql: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  if (!schemaSurql) return out;
+
+  // Drop line comments so a `--` comment can't contain a stray PERMISSIONS/FOR.
+  const cleaned = schemaSurql.replace(/--[^\n]*/g, '');
+
+  // One entry per `DEFINE TABLE … ;` statement.
+  const tableStmt = /DEFINE\s+TABLE\s+(?:OVERWRITE\s+|IF\s+NOT\s+EXISTS\s+)?([A-Za-z_][\w]*)\b([^;]*);/gi;
+  let m: RegExpExecArray | null;
+  while ((m = tableStmt.exec(cleaned)) !== null) {
+    const table = m[1];
+    const body = m[2];
+    out[table] = selectPredicateFromBody(body);
+  }
+  return out;
+}
+
+function selectPredicateFromBody(body: string): string {
+  const permIdx = body.search(/\bPERMISSIONS\b/i);
+  if (permIdx === -1) return 'false'; // no clause → SurrealDB default-deny
+  const perms = body.slice(permIdx + 'PERMISSIONS'.length).trim();
+
+  if (/^FULL\b/i.test(perms)) return 'true';
+  if (/^NONE\b/i.test(perms)) return 'false';
+
+  // `FOR <actions> WHERE <expr>` groups, possibly several. Split on the FOR
+  // boundary; the leading empty segment (before the first FOR) is ignored.
+  const groups = perms.split(/\bFOR\b/i).map((g) => g.trim()).filter(Boolean);
+  for (const group of groups) {
+    const where = group.search(/\bWHERE\b/i);
+    if (where === -1) continue;
+    const actions = group.slice(0, where).toLowerCase();
+    if (/\bselect\b/.test(actions)) {
+      return group.slice(where + 'WHERE'.length).trim();
+    }
+  }
+  return 'false'; // permissions present but none grant select
+}