@spooky-sync/core 0.0.1-canary.66 → 0.0.1-canary.67

package/src/modules/sync/utils.test.ts

@@ -5,6 +5,10 @@ import {
   applyRecordVersionDiff,
   createDiffFromDbOp,
   ArraySyncer,
+  resolveListRefPollInterval,
+  DEFAULT_LIST_REF_POLL_INTERVAL_MS,
+  buildListRefSelect,
+  nextPollDelayMs,
 } from './utils';
 import type { RecordVersionArray, RecordVersionDiff } from '../../types';
 import { encodeRecordId } from '../../utils/index';
@@ -309,3 +313,156 @@ describe('ArraySyncer', () => {
     expect(removedIds).toEqual(sorted);
   });
 });
+
+describe('resolveListRefPollInterval', () => {
+  it('returns the default when no option is provided', () => {
+    expect(resolveListRefPollInterval()).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
+    expect(resolveListRefPollInterval(undefined)).toBe(
+      DEFAULT_LIST_REF_POLL_INTERVAL_MS
+    );
+  });
+
+  it('respects a positive override', () => {
+    expect(resolveListRefPollInterval(1000)).toBe(1000);
+    expect(resolveListRefPollInterval(5000)).toBe(5000);
+  });
+
+  it('falls back to default for zero, negative, NaN, or Infinity', () => {
+    // Accepting these would either silently disable polling or busy-loop
+    // the event loop, so they fall back to the default.
+    expect(resolveListRefPollInterval(0)).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
+    expect(resolveListRefPollInterval(-500)).toBe(
+      DEFAULT_LIST_REF_POLL_INTERVAL_MS
+    );
+    expect(resolveListRefPollInterval(NaN)).toBe(
+      DEFAULT_LIST_REF_POLL_INTERVAL_MS
+    );
+    expect(resolveListRefPollInterval(Infinity)).toBe(
+      DEFAULT_LIST_REF_POLL_INTERVAL_MS
+    );
+  });
+
+  it('locks the default at 500ms so accidental tuning trips this test', () => {
+    expect(DEFAULT_LIST_REF_POLL_INTERVAL_MS).toBe(500);
+  });
+});
+
+describe('buildListRefSelect', () => {
+  it('substitutes the table name', () => {
+    expect(buildListRefSelect('_00_list_ref')).toContain('FROM _00_list_ref ');
+    expect(buildListRefSelect('_00_list_ref_user_abc')).toContain(
+      'FROM _00_list_ref_user_abc '
+    );
+  });
+
+  it('filters by the bound query id', () => {
+    expect(buildListRefSelect('_00_list_ref')).toContain('WHERE in = $in');
+  });
+
+  it('excludes subquery rows via parent IS NONE', () => {
+    // Without this predicate the poll would surface child records (the
+    // subquery-projection rows the SSP also writes into list_ref) as
+    // spurious "added" diffs against `localArray` every tick.
+    expect(buildListRefSelect('_00_list_ref')).toContain('parent IS NONE');
+  });
+
+  it('selects only `out` and `version`', () => {
+    // The diff only needs the record id and its version; pulling more
+    // fields would bloat the per-tick query traffic.
+    const sql = buildListRefSelect('_00_list_ref');
+    expect(sql.startsWith('SELECT out, version FROM ')).toBe(true);
+  });
+});
+
+describe('nextPollDelayMs', () => {
+  it('returns the base interval when LIVE has never delivered', () => {
+    expect(
+      nextPollDelayMs({
+        now: 10_000,
+        lastLiveEventAt: null,
+        baseIntervalMs: 500,
+      })
+    ).toBe(500);
+  });
+
+  it('returns the healthy interval when a LIVE event fired within the cooldown', () => {
+    expect(
+      nextPollDelayMs({
+        now: 10_000,
+        lastLiveEventAt: 8_000, // 2s ago, well within 5s cooldown
+        baseIntervalMs: 500,
+      })
+    ).toBe(5_000);
+  });
+
+  it('snaps back to the base interval after the cooldown elapses', () => {
+    expect(
+      nextPollDelayMs({
+        now: 20_000,
+        lastLiveEventAt: 10_000, // 10s ago, well past cooldown
+        baseIntervalMs: 500,
+      })
+    ).toBe(500);
+  });
+
+  it('treats the cooldown boundary as expired (>= cooldownMs returns base)', () => {
+    // Exactly at the boundary, the LIVE feed is considered cold so we
+    // resume the aggressive cadence. Picks the same direction as
+    // `if (sinceLive >= cooldownMs)`.
+    expect(
+      nextPollDelayMs({
+        now: 15_000,
+        lastLiveEventAt: 10_000, // exactly 5s ago
+        baseIntervalMs: 500,
+        cooldownMs: 5_000,
+      })
+    ).toBe(500);
+  });
+
+  it('clamps a clock-skew negative sinceLive to the base interval', () => {
+    // If a stale timestamp puts the LIVE event in the future, we
+    // should not interpret that as "LIVE healthy" — treat it as
+    // unknown and use the conservative aggressive cadence.
+    expect(
+      nextPollDelayMs({
+        now: 10_000,
+        lastLiveEventAt: 20_000, // future timestamp
+        baseIntervalMs: 500,
+      })
+    ).toBe(500);
+  });
+
+  it('never widens below an aggressively-configured base interval', () => {
+    // If the caller picked a faster base (e.g. 100ms) on purpose,
+    // the healthy path should not silently slow things back to 5s.
+    // The healthy interval is clamped to at least base.
+    expect(
+      nextPollDelayMs({
+        now: 10_000,
+        lastLiveEventAt: 9_500,
+        baseIntervalMs: 8_000,
+      })
+    ).toBe(8_000);
+  });
+
+  it('respects custom cooldown and healthy-interval values', () => {
+    expect(
+      nextPollDelayMs({
+        now: 10_000,
+        lastLiveEventAt: 9_500, // 0.5s ago
+        baseIntervalMs: 250,
+        cooldownMs: 1_000,
+        healthyIntervalMs: 2_000,
+      })
+    ).toBe(2_000);
+    expect(
+      nextPollDelayMs({
+        now: 10_000,
+        lastLiveEventAt: 8_500, // 1.5s ago > cooldown
+        baseIntervalMs: 250,
+        cooldownMs: 1_000,
+        healthyIntervalMs: 2_000,
+      })
+    ).toBe(250);
+  });
+});

package/src/modules/sync/utils.ts

@@ -167,3 +167,82 @@ export function createDiffFromDbOp(
     };
   }
 }
+
+/**
+ * Default cadence for the `_00_list_ref_user_<id>` poll fallback. The
+ * poll is the safety net for SurrealDB v3's occasionally-dropped LIVE
+ * deliveries; 500ms is aggressive enough to feel real-time on the
+ * happy path while keeping the per-session query load bounded.
+ */
+export const DEFAULT_LIST_REF_POLL_INTERVAL_MS = 500;
+
+/**
+ * Build the SurrealQL select that powers both the initial-fetch and
+ * the periodic poll of `_00_list_ref[_user_<id>]`. The `parent IS NONE`
+ * predicate excludes subquery entries (rows with `parent_rel` set)
+ * because the client's `RecordVersionArray` only tracks primary rows;
+ * including subquery rows would surface them as spurious "added"
+ * diffs every tick.
+ */
+export function buildListRefSelect(table: string): string {
+  return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
+}
+
+/**
+ * Resolve the effective list-ref poll interval. Negative or zero
+ * values fall back to the default — accepting them would either
+ * disable polling silently or busy-loop the event loop.
+ */
+export function resolveListRefPollInterval(opt?: number): number {
+  if (typeof opt !== 'number' || !Number.isFinite(opt) || opt <= 0) {
+    return DEFAULT_LIST_REF_POLL_INTERVAL_MS;
+  }
+  return opt;
+}
+
+/**
+ * When the LIVE feed has delivered an event within this window, treat
+ * the LIVE subscription as healthy and back the poll off to
+ * {@link LIVE_HEALTHY_POLL_INTERVAL_MS}. As soon as LIVE quiets for
+ * longer than this, the poll snaps back to the aggressive default.
+ */
+export const LIVE_HEALTHY_COOLDOWN_MS = 5_000;
+
+/**
+ * Poll interval while LIVE is delivering events. The aggressive
+ * default 500ms costs ~120 queries / minute / session — wasted work
+ * when LIVE is already covering us. 5s keeps the safety net in place
+ * at 1/10th the load.
+ */
+export const LIVE_HEALTHY_POLL_INTERVAL_MS = 5_000;
+
+/**
+ * Pick the next poll delay based on whether LIVE has been healthy
+ * recently. If a LIVE event fired within `cooldownMs`, use the slow
+ * (`healthyIntervalMs`) cadence; otherwise the fast (`baseIntervalMs`)
+ * cadence. Pure so it's unit-testable; `Sp00kySync.startListRefPoll`
+ * calls it from a self-rescheduling timer.
+ *
+ * The healthy interval is clamped to at least `baseIntervalMs` so
+ * configuring an aggressive base (e.g. 100ms) never gets implicitly
+ * widened by this helper.
+ */
+export function nextPollDelayMs(args: {
+  now: number;
+  lastLiveEventAt: number | null;
+  baseIntervalMs: number;
+  cooldownMs?: number;
+  healthyIntervalMs?: number;
+}): number {
+  const {
+    now,
+    lastLiveEventAt,
+    baseIntervalMs,
+    cooldownMs = LIVE_HEALTHY_COOLDOWN_MS,
+    healthyIntervalMs = LIVE_HEALTHY_POLL_INTERVAL_MS,
+  } = args;
+  if (lastLiveEventAt === null) return baseIntervalMs;
+  const sinceLive = now - lastLiveEventAt;
+  if (sinceLive < 0 || sinceLive >= cooldownMs) return baseIntervalMs;
+  return Math.max(healthyIntervalMs, baseIntervalMs);
+}

package/src/services/stream-processor/index.ts

@@ -28,6 +28,12 @@ export interface StreamUpdate {
   queryHash: string;
   localArray: RecordVersionArray;
   op?: 'CREATE' | 'UPDATE' | 'DELETE'; // Operation type for conditional debouncing
+  /**
+   * End-to-end ingest latency for the WASM call that produced this update,
+   * in milliseconds. Populated by StreamProcessorService.ingest. Undefined
+   * for the initial register_view snapshot.
+   */
+  materializationTimeMs?: number;
 }
 
 // Define events map (kept for DevTools compatibility)
@@ -205,13 +211,16 @@ export class StreamProcessorService {
     try {
       const normalizedRecord = this.normalizeValue(record);
 
+      const t0 = performance.now();
       const rawUpdates = this.processor.ingest(table, op, id, normalizedRecord);
+      const materializationTimeMs = performance.now() - t0;
       this.logger.debug(
         {
           table,
           op,
           id,
           rawUpdates: rawUpdates.length,
+          materializationTimeMs,
           Category: 'sp00ky-client::StreamProcessorService::ingest',
         },
         'Ingesting into ssp done'
@@ -222,6 +231,7 @@ export class StreamProcessorService {
           queryHash: u.query_id,
           localArray: u.result_data,
           op: op,
+          materializationTimeMs,
         }));
         // Direct handler call instead of event
         this.notifyUpdates(updates);
@@ -324,6 +334,21 @@ export class StreamProcessorService {
     if (value === null || value === undefined) return value;
 
     if (typeof value === 'object') {
+      // CRDT snapshots arrive as `Uint8Array` (or `ArrayBuffer` /
+      // typed-array views). `serde_wasm_bindgen::from_value` rejects
+      // those when deserializing into `serde_json::Value` (JSON has no
+      // binary variant), and the SSP can't filter on opaque bytes
+      // anyway. Replace with `null` so the row still flows through the
+      // ingest path with its other columns intact, and downstream
+      // predicates referencing the bytes column simply don't match.
+      if (
+        value instanceof Uint8Array ||
+        value instanceof ArrayBuffer ||
+        ArrayBuffer.isView(value)
+      ) {
+        return null;
+      }
+
       // RecordId detection using duck typing (constructor.name may be minified)
       // SurrealDB's RecordId has: table (getter returning Table), id, and toString()
       // Check for table getter that has its own toString AND id property

package/src/sp00ky.auth-order.test.ts

@@ -0,0 +1,65 @@
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+// Regression guard for the auth-callback ordering bug we fixed in
+// `Sp00kyClient.init()`. The contract:
+//
+//   this.auth.subscribe(async (userId) => {
+//     this.dataModule.setCurrentUserId(userId);   // ← must be first
+//     // ... awaits and other work ...
+//   });
+//
+// If any future refactor moves `setCurrentUserId` after the first
+// `await`, the AuthProvider's sibling subscriber gets to run with a
+// stale `dataModule.currentUserId` and registers queries against the
+// wrong `_00_query[_user_*]` / `_00_list_ref_user_<id>` tables.
+//
+// A runtime mock test for this is heavy — Sp00kyClient drags in
+// SurrealDB, the WASM SSP, the persistence client, etc. A
+// structural regex test is enough: it catches the only failure mode
+// (someone moves the synchronous setCurrentUserId call) without
+// needing the full graph.
+
+describe('Sp00kyClient.auth.subscribe ordering invariant', () => {
+  const sourcePath = resolve(__dirname, 'sp00ky.ts');
+  const source = readFileSync(sourcePath, 'utf-8');
+
+  it('source file is readable', () => {
+    expect(source.length).toBeGreaterThan(0);
+  });
+
+  it('auth.subscribe sets currentUserId before any await', () => {
+    // Find the auth.subscribe arrow body. Match the contents of the
+    // outermost `{}` after `this.auth.subscribe(async (userId) => `.
+    const match = source.match(
+      /this\.auth\.subscribe\(\s*async\s*\(\s*userId\s*[^)]*\)\s*=>\s*\{([\s\S]*?)\n {6}\}\s*\)/
+    );
+    expect(
+      match,
+      'expected an auth.subscribe(async (userId) => { ... }) block in sp00ky.ts'
+    ).not.toBeNull();
+
+    const body = match![1];
+
+    // Strip line comments so a stray `//` doesn't trick the regex.
+    const stripped = body
+      .split('\n')
+      .map((line) => line.replace(/\/\/.*$/, ''))
+      .join('\n');
+
+    const setUserIdIdx = stripped.indexOf('this.dataModule.setCurrentUserId(userId)');
+    expect(
+      setUserIdIdx,
+      'dataModule.setCurrentUserId(userId) must appear in the auth.subscribe body'
+    ).toBeGreaterThanOrEqual(0);
+
+    const firstAwaitIdx = stripped.search(/\bawait\b/);
+    if (firstAwaitIdx >= 0) {
+      expect(
+        setUserIdIdx,
+        `dataModule.setCurrentUserId(userId) (idx ${setUserIdIdx}) must come BEFORE the first \`await\` (idx ${firstAwaitIdx}) so the AuthProvider's sibling subscriber sees a fresh user id when it fires synchronously after our callback. Moving the call past an \`await\` reintroduces the stale-user-id race documented in docs/surrealdb-bugs/ if a related gap is found, and the prior session's retrospective.`
+      ).toBeLessThan(firstAwaitIdx);
+    }
+  });
+});

package/src/sp00ky.ts

@@ -36,7 +36,7 @@ import { EventSystem } from './events/index';
 import { CacheModule } from './modules/cache/index';
 import { CrdtManager, CrdtField } from './modules/crdt/index';
 import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
-import { generateId, parseParams } from './utils/index';
+import { parseParams } from './utils/index';
 import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
 import { ResilientPersistenceClient } from './services/persistence/resilient';
 
@@ -109,6 +109,15 @@ export class Sp00kyClient<S extends SchemaStructure> {
     return this.sync.pendingMutationCount;
   }
 
+  /** Number of times the initial list_ref LIVE subscription retried on
+   *  the most recent `setCurrentUserId` call. 0 when the SSP's
+   *  pre-emptive user-table creation got there first; >0 when LIVE
+   *  registration hit a "table not found" race. Exposed so the e2e
+   *  suite can guard the pre-emptive path against regression. */
+  get liveRetryCount(): number {
+    return this.sync.liveRetryCount;
+  }
+
   subscribeToPendingMutations(cb: (count: number) => void): () => void {
     return this.sync.subscribeToPendingMutations(cb);
   }
@@ -156,8 +165,18 @@ export class Sp00kyClient<S extends SchemaStructure> {
       logger
     );
 
-    // Initialize CRDT Manager (needs remote for _00_crdt LIVE SELECT)
-    this.crdtManager = new CrdtManager(this.config.schema, this.remote, logger);
+    // Initialize CRDT Manager. `local` is used to read the initial
+    // `_00_crdt` snapshot when a field opens AND to mirror every local
+    // edit (so reload/offline see the freshest state); `remote` is used
+    // for the debounced outgoing UPSERTs and the parent-table LIVE feed.
+    // The debounce window is configurable via `crdtDebounceMs`.
+    this.crdtManager = new CrdtManager(
+      this.config.schema,
+      this.local,
+      this.remote,
+      logger,
+      config.crdtDebounceMs ?? 500,
+    );
 
     this.dataModule = new DataModule(
       this.cache,
@@ -171,7 +190,15 @@ export class Sp00kyClient<S extends SchemaStructure> {
     this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
 
     // Initialize Sync
-    this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger);
+    this.sync = new Sp00kySync(
+      this.local,
+      this.remote,
+      this.cache,
+      this.dataModule,
+      this.config.schema,
+      this.logger,
+      { refSyncIntervalMs: this.config.refSyncIntervalMs }
+    );
 
     // Initialize DevTools
     this.devTools = new DevToolsService(
@@ -210,6 +237,32 @@ export class Sp00kyClient<S extends SchemaStructure> {
       this.devTools.logEvent('SYNC_QUERY_UPDATED', event.payload);
     });
 
+    // Hand list_ref-driven row ingests to the CrdtManager so CRDT body
+    // / cursor updates reach the receiver even when the cross-session
+    // LIVE on the parent table is filtered out by the SurrealDB
+    // permission-LIVE gap. Same-user clients receive these rows via
+    // CrdtManager's own `LIVE SELECT * FROM <table>`; this hook is the
+    // redundant path that fires when only the list_ref bumped.
+    this.sync.engineEvents.subscribe('SYNC_REMOTE_DATA_INGESTED', (event: any) => {
+      try {
+        const records: Array<Record<string, any>> = event.payload?.records ?? [];
+        for (const row of records) {
+          const id = row?.id;
+          const table =
+            id && typeof id === 'object' && id.table !== undefined
+              ? String(id.table)
+              : undefined;
+          if (!table) continue;
+          this.crdtManager.applyRow(table, row);
+        }
+      } catch (err) {
+        this.logger.debug(
+          { err, Category: 'sp00ky-client::engineEvents::ingested' },
+          'applyRow forwarding from sync ingest failed'
+        );
+      }
+    });
+
     // Database events for DevTools
     this.local.getEvents().subscribe('DATABASE_LOCAL_QUERY', (event: any) => {
       this.devTools.logEvent('LOCAL_QUERY', event.payload);
@@ -226,13 +279,6 @@ export class Sp00kyClient<S extends SchemaStructure> {
       'Sp00kyClient initialization started'
     );
     try {
-      const clientId = this.config.clientId ?? (await this.loadOrGenerateClientId());
-      this.persistClientId(clientId);
-      this.logger.debug(
-        { clientId, Category: 'sp00ky-client::Sp00kyClient::init' },
-        'Client ID loaded'
-      );
-
       await this.local.connect();
       this.logger.debug(
         { Category: 'sp00ky-client::Sp00kyClient::init' },
@@ -257,13 +303,51 @@ export class Sp00kyClient<S extends SchemaStructure> {
       await this.auth.init();
       this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Auth initialized');
 
-      await this.dataModule.init();
+      // Salt query-id hashing with the SurrealDB session id so two browsers
+      // for the same user don't collide on shared `_00_query` rows. The same
+      // session id is the `session_id` key in `_00_cursor` rows, so the
+      // CrdtManager needs it too.
+      const sessionId = await this.fetchSessionId();
+      await this.dataModule.init(sessionId);
+      this.crdtManager.setSessionId(sessionId);
       this.logger.debug(
-        { Category: 'sp00ky-client::Sp00kyClient::init' },
+        { sessionId, Category: 'sp00ky-client::Sp00kyClient::init' },
         'DataModule initialized'
       );
 
-      await this.sync.init(clientId);
+      // Refresh the salt whenever auth state flips (sign-in, sign-out).
+      // session::id() changes per WebSocket session, and a sign-in spawns
+      // a new authenticated session, so the salt must follow. Also
+      // forward the user id into `DataModule` and `Sp00kySync` so they
+      // can route to per-user `_00_query_user_<id>` /
+      // `_00_list_ref_user_<id>` tables in `RefMode.Dedicated` — the
+      // LIVE subscription on `_00_list_ref_user_<id>` is restarted
+      // under the new auth context inside `Sp00kySync.setCurrentUserId`
+      // since SurrealDB binds the LIVE permission at registration time.
+      //
+      // Sync prefix BEFORE the first `await`: setting `currentUserId`
+      // synchronously here is critical because the AuthProvider's own
+      // subscribe callback runs right after ours and immediately enables
+      // queries that depend on the user id. Any `await` before
+      // `setCurrentUserId` would let those queries register against the
+      // stale (null) user id and hit the wrong `_00_query[_user_*]`
+      // table.
+      this.auth.subscribe(async (userId) => {
+        this.dataModule.setCurrentUserId(userId);
+        const next = await this.fetchSessionId();
+        this.dataModule.setSessionId(next);
+        this.crdtManager.setSessionId(next);
+        try {
+          await this.sync.setCurrentUserId(userId);
+        } catch (e) {
+          this.logger.error(
+            { error: e, Category: 'sp00ky-client::Sp00kyClient::authChange' },
+            'sync.setCurrentUserId failed'
+          );
+        }
+      });
+
+      await this.sync.init();
       this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Sync initialized');
 
       this.logger.info(
@@ -292,7 +376,8 @@ export class Sp00kyClient<S extends SchemaStructure> {
   /**
    * Open a CRDT field for collaborative editing.
    * Returns a CrdtField with a LoroDoc that can be bound to any editor.
-   * Also starts a LIVE SELECT on _00_crdt for real-time sync.
+   * Also starts a LIVE SELECT on the parent table for real-time sync;
+   * incoming events trigger a subquery fetch of `_00_crdt` / `_00_cursor`.
    */
   async openCrdtField(
     table: string,
@@ -396,26 +481,22 @@ export class Sp00kyClient<S extends SchemaStructure> {
     return fn(this.remote.getClient());
   }
 
-  private persistClientId(id: string) {
+  /**
+   * Fetch SurrealDB's `session::id()` as a string. Used as a salt for
+   * query-id hashing so two sessions for the same user get distinct
+   * `_00_query` rows. Returns empty string if the query fails (we still
+   * boot, just without session scoping for IDs).
+   */
+  private async fetchSessionId(): Promise<string> {
     try {
-      this.persistenceClient.set('sp00ky_client_id', id);
+      const [sid] = await this.remote.query<[string]>('RETURN <string>session::id()');
+      return typeof sid === 'string' ? sid : '';
     } catch (e) {
       this.logger.warn(
-        { error: e, Category: 'sp00ky-client::Sp00kyClient::persistClientId' },
-        'Failed to persist client ID'
+        { error: e, Category: 'sp00ky-client::Sp00kyClient::fetchSessionId' },
+        'Failed to fetch session::id() — proceeding with empty salt'
       );
+      return '';
     }
   }
-
-  private async loadOrGenerateClientId(): Promise<string> {
-    const clientId = await this.persistenceClient.get<string>('sp00ky_client_id');
-
-    if (clientId) {
-      return clientId;
-    }
-
-    const newId = generateId();
-    await this.persistClientId(newId);
-    return newId;
-  }
 }

package/src/types.ts

@@ -99,8 +99,6 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
     /** Authentication token. */
     token?: string;
   };
-  /** Unique client identifier. If not provided, one will be generated. */
-  clientId?: string;
   /** The schema definition. */
   schema: S;
   /** The compiled SURQL schema string. */
@@ -119,6 +117,22 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
    * Defaults to 100ms.
    */
   streamDebounceTime?: number;
+  /**
+   * Debounce time in milliseconds for syncing collaborative (CRDT) field
+   * changes to the remote database. Local writes happen immediately on
+   * every keystroke (so reload/offline works), but the remote UPSERT is
+   * coalesced over this window. Lower = snappier remote propagation +
+   * more network traffic; higher = less traffic + more lag for other
+   * collaborators. Defaults to 500ms.
+   */
+  crdtDebounceMs?: number;
+  /**
+   * Cadence (ms) for the `_00_list_ref` poll that catches cross-session
+   * UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
+   * convergence + more query load; higher = the inverse. Non-positive
+   * values fall back to the default (500ms).
+   */
+  refSyncIntervalMs?: number;
 }
 
 export type QueryHash = string;
@@ -178,8 +192,21 @@ export interface QueryState {
   ttlDurationMs: number;
   /** Number of times the query has been updated. */
   updateCount: number;
+  /**
+   * Rolling window of the most recent materialization-step latencies (ms).
+   * Capped at MATERIALIZATION_SAMPLE_WINDOW; used to recompute p55/p90/p99
+   * before each persist to `_00_query`. Samples themselves are not persisted.
+   */
+  materializationSamples: number[];
+  /** Most recent end-to-end ingest latency in ms, or null until the first ingest. */
+  lastIngestLatencyMs: number | null;
+  /** Cumulative count of ingest/materialization errors observed for this query. */
+  errorCount: number;
 }
 
+/** Cap on the rolling materialization-sample window kept per query in memory. */
+export const MATERIALIZATION_SAMPLE_WINDOW = 100;
+
 // Callback types
 export type QueryUpdateCallback = (records: Record<string, any>[]) => void;
 export type MutationCallback = (mutations: UpEvent[]) => void;

package/src/utils/surql.ts

@@ -34,6 +34,7 @@ export interface SurqlHelper {
     keyDataVars: ({ key: string; variable: string } | { statement: string } | string)[]
   ): string;
   upsert(idVar: string, dataVar: string): string;
+  upsertMerge(idVar: string, dataVar: string): string;
   updateMerge(idVar: string, dataVar: string): string;
   updateSet(
     idVar: string,
@@ -127,6 +128,14 @@ export const surql: SurqlHelper = {
     return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
   },
 
+  // Use this for sync-down ingestion: the remote payload omits local-only
+  // fields injected by the CLI's local-schema setup (`_00_crdt`,
+  // `_00_cursor`), and REPLACE would wipe them on every round-trip and
+  // break offline reload of CRDT state.
+  upsertMerge(idVar: string, dataVar: string) {
+    return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
+  },
+
   updateMerge(idVar: string, dataVar: string) {
     return `UPDATE ONLY $${idVar} MERGE $${dataVar}`;
   },