@spooky-sync/core 0.0.1-canary.69 → 0.0.1-canary.70

package/src/services/database/local.ts

@@ -59,42 +59,201 @@ export class LocalDatabaseService extends AbstractDatabaseService {
 
   async connect(): Promise<void> {
     const { namespace, database } = this.getConfig();
+    const store = this.getConfig().store ?? 'memory';
+    const storeUrl = store === 'memory' ? 'mem://' : 'indxdb://sp00ky';
     this.logger.info(
-      { namespace, database, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+      { namespace, database, storeUrl, Category: 'sp00ky-client::LocalDatabaseService::connect' },
       'Connecting to local database'
     );
+
+    this.registerUnloadClose();
+
     try {
-      const store = this.getConfig().store ?? 'memory';
-      const storeUrl = store === 'memory' ? 'mem://' : 'indxdb://sp00ky';
-      this.logger.debug(
-        { storeUrl, Category: 'sp00ky-client::LocalDatabaseService::connect' },
-        '[LocalDatabaseService] Calling client.connect'
+      await this.openStore(storeUrl, namespace, database);
+      this.logger.info(
+        { Category: 'sp00ky-client::LocalDatabaseService::connect' },
+        'Connected to local database'
       );
-      await this.client.connect(storeUrl, {});
-      this.logger.debug(
-        { namespace, database, Category: 'sp00ky-client::LocalDatabaseService::connect' },
-        '[LocalDatabaseService] client.connect returned. Calling client.use'
+      return;
+    } catch (err) {
+      // A persistent (IndexedDB) local store can fail to open if it was left
+      // corrupt or version-incompatible by a prior session/crash/engine bump.
+      // The local store is only a cache (everything re-syncs from the server),
+      // so recover by dropping it and reconnecting rather than bricking startup.
+      if (store === 'memory' || !isLocalStoreOpenError(err)) {
+        this.logger.error(
+          { err, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+          'Failed to connect to local database'
+        );
+        throw err;
+      }
+      this.logger.warn(
+        { err, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+        'Local IndexedDB store failed to open; retrying before clearing'
       );
+    }
 
-      await this.client.use({
-        namespace,
-        database,
-      });
-      this.logger.debug(
-        { Category: 'sp00ky-client::LocalDatabaseService::connect' },
-        '[LocalDatabaseService] client.use returned'
-      );
+    // Tier 1 — RETRY the SAME store WITHOUT dropping. The idb open/`use` failure
+    // is often transient (a not-yet-released handle from the previous page, or a
+    // first-open WAL-recovery race), not real corruption. Closing and reopening
+    // frequently succeeds — and crucially PRESERVES the cache, so a warm load
+    // stays warm. Dropping the store every time (the old behavior) silently wiped
+    // the cache on every reload, making warm loads as slow as cold ones.
+    for (let attempt = 1; attempt <= 2; attempt++) {
+      try {
+        await this.client.close();
+      } catch {
+        /* ignore */
+      }
+      await delay(150 * attempt);
+      try {
+        await this.openStore(storeUrl, namespace, database);
+        this.logger.info(
+          { attempt, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+          'Connected to local database on retry (cache preserved)'
+        );
+        return;
+      } catch (retryErr) {
+        this.logger.warn(
+          { err: retryErr, attempt, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+          'Local store retry failed'
+        );
+      }
+    }
+
+    // Tier 2 — the store is genuinely unopenable; drop it and reconnect fresh.
+    // This loses the cache (re-syncs from the server), so it's the last resort
+    // before in-memory.
+    try {
+      await this.client.close();
+    } catch {
+      /* ignore — closing a half-open connection is best-effort */
+    }
+    await dropLocalIndexedDbStores(this.logger);
 
+    try {
+      await this.openStore(storeUrl, namespace, database);
       this.logger.info(
         { Category: 'sp00ky-client::LocalDatabaseService::connect' },
-        'Connected to local database'
+        'Reconnected to local database after clearing the corrupt store'
       );
-    } catch (err) {
+    } catch (retryErr) {
+      // Last resort: run in-memory so the app still loads. No local persistence
+      // this session; the freshly-dropped IndexedDB is recreated cleanly next
+      // load, and all data re-syncs from the server regardless.
       this.logger.error(
-        { err, Category: 'sp00ky-client::LocalDatabaseService::connect' },
-        'Failed to connect to local database'
+        { err: retryErr, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+        'Local store still failing after clear; falling back to in-memory'
+      );
+      try {
+        await this.client.close();
+      } catch {
+        /* ignore */
+      }
+      await this.openStore('mem://', namespace, database);
+      this.logger.warn(
+        { Category: 'sp00ky-client::LocalDatabaseService::connect' },
+        'Connected to local database (in-memory fallback)'
       );
-      throw err;
     }
   }
+
+  private unloadCloseRegistered = false;
+
+  /**
+   * Close the local DB on page unload so the SurrealDB-WASM worker releases its
+   * IndexedDB connection cleanly. Without this, the previous page's connection
+   * lingers; the next load's `client.connect` opens the store but the first
+   * write transaction in `client.use` hits an "IndexedDB error" — which then
+   * (mis)triggered the corrupt-store recovery and WIPED the cache on every
+   * reload, making warm loads as slow as cold ones. `pagehide` is the reliable
+   * unload signal (fires on bfcache + normal navigation); `close()` is async but
+   * the WASM worker initiates the IndexedDB connection teardown synchronously.
+   */
+  private registerUnloadClose(): void {
+    if (this.unloadCloseRegistered || typeof window === 'undefined') return;
+    this.unloadCloseRegistered = true;
+    const close = () => {
+      try {
+        void this.client.close();
+      } catch {
+        /* best-effort */
+      }
+    };
+    window.addEventListener('pagehide', close);
+    window.addEventListener('beforeunload', close);
+  }
+
+  private async openStore(storeUrl: string, namespace: string, database: string): Promise<void> {
+    this.logger.debug(
+      { storeUrl, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+      '[LocalDatabaseService] Calling client.connect'
+    );
+    await this.client.connect(storeUrl, {});
+    this.logger.debug(
+      { namespace, database, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+      '[LocalDatabaseService] client.connect returned. Calling client.use'
+    );
+    await this.client.use({ namespace, database });
+    this.logger.debug(
+      { Category: 'sp00ky-client::LocalDatabaseService::connect' },
+      '[LocalDatabaseService] client.use returned'
+    );
+  }
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** True for the SurrealDB-WASM error raised when its IndexedDB-backed key-value
+ *  store can't be opened (corrupt / version-incompatible / blocked). Exported
+ *  for unit testing the error-message match. */
+export function isLocalStoreOpenError(err: unknown): boolean {
+  const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+  return (
+    msg.includes('indexeddb') ||
+    msg.includes('idb error') ||
+    msg.includes('key-value store')
+  );
+}
+
+/** Best-effort delete of this client's IndexedDB store(s). The persistent local
+ *  DB lives at `indxdb://sp00ky`; SurrealDB-WASM backs it with one or more
+ *  IndexedDB databases whose names include `sp00ky`. Resolves even on
+ *  error/blocked so startup can proceed. No-op outside a browser. */
+async function dropLocalIndexedDbStores(logger: Logger): Promise<void> {
+  if (typeof indexedDB === 'undefined') return;
+  const remove = (name: string): Promise<void> =>
+    new Promise((resolve) => {
+      try {
+        const req = indexedDB.deleteDatabase(name);
+        req.onsuccess = () => resolve();
+        req.onerror = () => resolve();
+        req.onblocked = () => resolve();
+      } catch {
+        resolve();
+      }
+    });
+  try {
+    let names: string[] = [];
+    if (typeof indexedDB.databases === 'function') {
+      const dbs = await indexedDB.databases();
+      names = dbs
+        .map((d) => d.name)
+        .filter((n): n is string => !!n && n.toLowerCase().includes('sp00ky'));
+    }
+    // Fall back to the known store name if enumeration is unavailable/empty.
+    if (names.length === 0) names = ['sp00ky'];
+    await Promise.all(names.map(remove));
+    logger.info(
+      { names, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+      'Cleared local IndexedDB store(s)'
+    );
+  } catch (e) {
+    logger.warn(
+      { err: e, Category: 'sp00ky-client::LocalDatabaseService::connect' },
+      'Failed to enumerate/clear IndexedDB; proceeding anyway'
+    );
+  }
 }

package/src/services/persistence/resilient.ts

@@ -23,7 +23,14 @@ export class ResilientPersistenceClient implements PersistenceClient {
         { key, error: e, Category: 'sp00ky-client::ResilientPersistenceClient::get' },
         'Persistence read failed, dropping key'
       );
-      await this.inner.remove(key).catch(() => {});
+      // Best-effort cleanup of the corrupt key; if removal also fails, the key
+      // just gets dropped again on the next failed read, so the failure is safe.
+      await this.inner.remove(key).catch((removeErr) => {
+        this.logger.debug(
+          { key, error: removeErr, Category: 'sp00ky-client::ResilientPersistenceClient::get' },
+          'Failed to drop corrupt persistence key'
+        );
+      });
       return null;
     }
   }

package/src/services/stream-processor/index.ts

@@ -34,6 +34,15 @@ export interface StreamUpdate {
    * for the initial register_view snapshot.
    */
   materializationTimeMs?: number;
+  /** SSP internal sub-phase timings (ms) for this ingest, from the WASM binding. */
+  storeApplyMs?: number;
+  circuitStepMs?: number;
+  transformMs?: number;
+  /**
+   * One-shot registration timings (ms). Only set on the StreamUpdate returned
+   * by `registerQueryPlan` (the register_view snapshot), not on ingest updates.
+   */
+  registration?: { parseMs: number; planMs: number; snapshotMs: number };
 }
 
 // Define events map (kept for DevTools compatibility)
@@ -53,6 +62,12 @@ export class StreamProcessorService {
   private processor: WasmProcessor | undefined;
   private isInitialized = false;
   private receivers: StreamUpdateReceiver[] = [];
+  // When true, `notifyUpdates` coalesces updates into `batchBuffer` (keyed by
+  // queryHash) instead of dispatching them. Used to collapse the per-record
+  // stream updates produced by a batched ingest into a single notification per
+  // query, so the UI updates once after the whole batch rather than row-by-row.
+  private batching = false;
+  private batchBuffer: Map<string, StreamUpdate> = new Map();
 
   constructor(
     public events: EventSystem<StreamProcessorEvents>,
@@ -72,6 +87,32 @@ export class StreamProcessorService {
   }
 
   private notifyUpdates(updates: StreamUpdate[]) {
+    if (this.batching) {
+      // Coalesce by queryHash instead of dispatching. The WASM `result_data`
+      // (localArray) is the full materialized array, so last-write-wins
+      // already reflects every prior ingest in the batch. We sum the
+      // materialization times so the single recorded sample reflects the
+      // batch's total work, and emit `op: 'CREATE'` on flush so the coalesced
+      // update takes DataModule's immediate (non-debounced) path.
+      for (const update of updates) {
+        const prev = this.batchBuffer.get(update.queryHash);
+        const sum = (a?: number, b?: number) => (a ?? 0) + (b ?? 0);
+        this.batchBuffer.set(update.queryHash, {
+          ...update,
+          op: 'CREATE',
+          materializationTimeMs: sum(prev?.materializationTimeMs, update.materializationTimeMs),
+          storeApplyMs: sum(prev?.storeApplyMs, update.storeApplyMs),
+          circuitStepMs: sum(prev?.circuitStepMs, update.circuitStepMs),
+          transformMs: sum(prev?.transformMs, update.transformMs),
+        });
+      }
+      return;
+    }
+
+    this.dispatchUpdates(updates);
+  }
+
+  private dispatchUpdates(updates: StreamUpdate[]) {
     for (const update of updates) {
       for (const receiver of this.receivers) {
         receiver.onStreamUpdate(update);
@@ -79,6 +120,72 @@ export class StreamProcessorService {
     }
   }
 
+  /**
+   * Ingest a batch of record changes as a single bulk operation, firing only
+   * one coalesced `StreamUpdate` per affected query once every record has been
+   * ingested (instead of one update per record). Use this whenever multiple
+   * records land at once — e.g. sync fetching N missing rows — so a list query
+   * re-runs and the UI re-renders once for the whole batch rather than
+   * row-by-row.
+   *
+   * Internally opens a coalescing window, ingests each record, then flushes;
+   * processor state is persisted once for the whole batch. No-op for an empty
+   * batch.
+   */
+  ingestMany(
+    records: Array<{
+      table: string;
+      op: 'CREATE' | 'UPDATE' | 'DELETE';
+      id: string;
+      record: any;
+    }>
+  ): void {
+    if (records.length === 0) return;
+
+    this.beginCoalescing();
+    try {
+      for (const record of records) {
+        this.ingest(record.table, record.op, record.id, record.record);
+      }
+    } finally {
+      this.flushCoalescing();
+    }
+  }
+
+  /**
+   * Open a coalescing window. While open, the per-record stream updates
+   * emitted by `ingest` are buffered (one entry per queryHash) instead of
+   * dispatched. Always paired with `flushCoalescing()` in a try/finally by
+   * `ingestMany` so the window always closes — otherwise the processor stays
+   * stuck buffering forever.
+   *
+   * No-op if a window is already open (nested batches aren't expected here).
+   */
+  private beginCoalescing() {
+    if (this.batching) return;
+    this.batching = true;
+    this.batchBuffer.clear();
+  }
+
+  /**
+   * Close the coalescing window and flush: dispatch one coalesced
+   * `StreamUpdate` per buffered queryHash, then persist processor state once
+   * for the whole batch (instead of once per ingest).
+   */
+  private flushCoalescing() {
+    if (!this.batching) return;
+    this.batching = false;
+    const buffered = Array.from(this.batchBuffer.values());
+    this.batchBuffer.clear();
+    if (buffered.length > 0) {
+      this.dispatchUpdates(buffered);
+    }
+    // The processor state after the last ingest is cumulative, so a single
+    // snapshot covers the whole batch. Kept fire-and-forget like the per-ingest
+    // call it replaces.
+    this.saveState();
+  }
+
   /**
    * Initialize the WASM module and processor.
    * This must be called before using other methods.
@@ -135,8 +242,8 @@ export class StreamProcessorService {
         );
         // Assuming processor has a load_state method matching the save_state behavior
         // If not, we might need to adjust based on the actual WASM API
-        if (typeof (this.processor as any).load_state === 'function') {
-          (this.processor as any).load_state(state);
+        if (typeof this.processor.load_state === 'function') {
+          this.processor.load_state(state);
         } else {
           this.logger.warn(
             { Category: 'sp00ky-client::StreamProcessorService::loadState' },
@@ -157,12 +264,36 @@ export class StreamProcessorService {
     }
   }
 
+  /**
+   * Seed per-table `select` permission predicates ({ [table]: whereText }).
+   * Must run after the processor exists and before any `register_view`, else
+   * non-`_00_` tables are default-denied and registration fails.
+   */
+  setPermissions(permissions: Record<string, string>) {
+    if (!this.processor) return;
+    if (typeof this.processor.set_permissions !== 'function') {
+      this.logger.warn(
+        { Category: 'sp00ky-client::StreamProcessorService::setPermissions' },
+        'set_permissions not found on processor (stale WASM build?)'
+      );
+      return;
+    }
+    this.processor.set_permissions(permissions);
+    this.logger.info(
+      {
+        tables: Object.keys(permissions).length,
+        Category: 'sp00ky-client::StreamProcessorService::setPermissions',
+      },
+      'Seeded table permissions'
+    );
+  }
+
   async saveState() {
     if (!this.processor) return;
     try {
       // Assuming processor has a save_state method that returns the state string/bytes
-      if (typeof (this.processor as any).save_state === 'function') {
-        const state = (this.processor as any).save_state();
+      if (typeof this.processor.save_state === 'function') {
+        const state = this.processor.save_state();
         if (state) {
           await this.persistenceClient.set('_00_stream_processor_state', state);
           this.logger.trace(
@@ -232,11 +363,18 @@ export class StreamProcessorService {
           localArray: u.result_data,
           op: op,
           materializationTimeMs,
+          storeApplyMs: u.timing_store_apply_ms,
+          circuitStepMs: u.timing_circuit_step_ms,
+          transformMs: u.timing_transform_ms,
         }));
         // Direct handler call instead of event
         this.notifyUpdates(updates);
       }
-      this.saveState();
+      // While batching (inside `ingestMany`), `flushCoalescing` persists once
+      // for the whole batch — skip the redundant per-record snapshot here.
+      if (!this.batching) {
+        this.saveState();
+      }
       return rawUpdates;
     } catch (e) {
       this.logger.error(
@@ -293,6 +431,11 @@ export class StreamProcessorService {
       const update: StreamUpdate = {
         queryHash: initialUpdate.query_id,
         localArray: initialUpdate.result_data,
+        registration: {
+          parseMs: initialUpdate.timing_parse_ms ?? 0,
+          planMs: initialUpdate.timing_plan_ms ?? 0,
+          snapshotMs: initialUpdate.timing_snapshot_ms ?? 0,
+        },
       };
       this.saveState();
       this.logger.debug(

package/src/services/stream-processor/permissions.test.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect } from 'vitest';
+import { extractSelectPermissions } from './permissions';
+
+describe('extractSelectPermissions', () => {
+  it('maps a permissive multi-action table to true', () => {
+    const surql = `DEFINE TABLE game SCHEMAFULL
+      PERMISSIONS FOR select, create, update, delete WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ game: 'true' });
+  });
+
+  it('handles FULL and NONE', () => {
+    const surql = `
+      DEFINE TABLE a PERMISSIONS FULL;
+      DEFINE TABLE b PERMISSIONS NONE;`;
+    expect(extractSelectPermissions(surql)).toEqual({ a: 'true', b: 'false' });
+  });
+
+  it('extracts the select predicate from multiple FOR groups', () => {
+    const surql = `DEFINE TABLE doc SCHEMAFULL
+      PERMISSIONS
+        FOR create, update, delete WHERE owner = $auth.id
+        FOR select WHERE owner = $auth.id OR public = true;`;
+    expect(extractSelectPermissions(surql)).toEqual({
+      doc: 'owner = $auth.id OR public = true',
+    });
+  });
+
+  it('returns false when permissions exist but none grant select', () => {
+    const surql = `DEFINE TABLE log PERMISSIONS FOR create WHERE true;`;
+    expect(extractSelectPermissions(surql)).toEqual({ log: 'false' });
+  });
+
+  it('treats a missing PERMISSIONS clause as default-deny', () => {
+    const surql = `DEFINE TABLE bare SCHEMALESS;`;
+    expect(extractSelectPermissions(surql)).toEqual({ bare: 'false' });
+  });
+
+  it('ignores commented-out clauses and handles OVERWRITE / IF NOT EXISTS', () => {
+    const surql = `
+      -- DEFINE TABLE ghost PERMISSIONS FULL;
+      DEFINE TABLE OVERWRITE u PERMISSIONS FOR select WHERE true;
+      DEFINE TABLE IF NOT EXISTS v PERMISSIONS FULL;`;
+    const perms = extractSelectPermissions(surql);
+    expect(perms).toEqual({ u: 'true', v: 'true' });
+    expect(perms.ghost).toBeUndefined();
+  });
+});

package/src/services/stream-processor/permissions.ts

@@ -0,0 +1,53 @@
+/**
+ * Extract per-table `select` permission predicates from a raw `.surql` schema.
+ *
+ * The in-browser SSP default-denies any non-`_00_` table that has no permission
+ * predicate registered (see `permission_inject::build_predicate`). The client
+ * already holds the full schema text (`config.schemaSurql`), so we parse each
+ * `DEFINE TABLE … PERMISSIONS …` clause and hand the `select` predicate to the
+ * processor via `set_permissions` at boot — mirroring the native path that
+ * seeds the same map from `INFO FOR DB`.
+ *
+ * Returns `{ [table]: whereText }` where `whereText` is the raw SurrealQL
+ * expression (`'true'` for `FULL`, `'false'` for `NONE` or a table with no
+ * `select` permission).
+ */
+export function extractSelectPermissions(schemaSurql: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  if (!schemaSurql) return out;
+
+  // Drop line comments so a `--` comment can't contain a stray PERMISSIONS/FOR.
+  const cleaned = schemaSurql.replace(/--[^\n]*/g, '');
+
+  // One entry per `DEFINE TABLE … ;` statement.
+  const tableStmt = /DEFINE\s+TABLE\s+(?:OVERWRITE\s+|IF\s+NOT\s+EXISTS\s+)?([A-Za-z_][\w]*)\b([^;]*);/gi;
+  let m: RegExpExecArray | null;
+  while ((m = tableStmt.exec(cleaned)) !== null) {
+    const table = m[1];
+    const body = m[2];
+    out[table] = selectPredicateFromBody(body);
+  }
+  return out;
+}
+
+function selectPredicateFromBody(body: string): string {
+  const permIdx = body.search(/\bPERMISSIONS\b/i);
+  if (permIdx === -1) return 'false'; // no clause → SurrealDB default-deny
+  const perms = body.slice(permIdx + 'PERMISSIONS'.length).trim();
+
+  if (/^FULL\b/i.test(perms)) return 'true';
+  if (/^NONE\b/i.test(perms)) return 'false';
+
+  // `FOR <actions> WHERE <expr>` groups, possibly several. Split on the FOR
+  // boundary; the leading empty segment (before the first FOR) is ignored.
+  const groups = perms.split(/\bFOR\b/i).map((g) => g.trim()).filter(Boolean);
+  for (const group of groups) {
+    const where = group.search(/\bWHERE\b/i);
+    if (where === -1) continue;
+    const actions = group.slice(0, where).toLowerCase();
+    if (/\bselect\b/.test(actions)) {
+      return group.slice(where + 'WHERE'.length).trim();
+    }
+  }
+  return 'false'; // permissions present but none grant select
+}