@spooky-sync/core 0.0.1-canary.65 → 0.0.1-canary.67

package/dist/otel/index.d.ts

@@ -1,4 +1,4 @@
-import { c as PinoTransmit } from "../types.js";
+import { l as PinoTransmit } from "../types.js";
 import { Level } from "pino";
 
 //#region src/otel/index.d.ts

package/dist/types.d.ts

@@ -221,8 +221,6 @@ interface Sp00kyConfig<S extends SchemaStructure> {
     /** Authentication token. */
     token?: string;
   };
-  /** Unique client identifier. If not provided, one will be generated. */
-  clientId?: string;
   /** The schema definition. */
   schema: S;
   /** The compiled SURQL schema string. */
@@ -241,6 +239,22 @@ interface Sp00kyConfig<S extends SchemaStructure> {
    * Defaults to 100ms.
    */
   streamDebounceTime?: number;
+  /**
+   * Debounce time in milliseconds for syncing collaborative (CRDT) field
+   * changes to the remote database. Local writes happen immediately on
+   * every keystroke (so reload/offline works), but the remote UPSERT is
+   * coalesced over this window. Lower = snappier remote propagation +
+   * more network traffic; higher = less traffic + more lag for other
+   * collaborators. Defaults to 500ms.
+   */
+  crdtDebounceMs?: number;
+  /**
+   * Cadence (ms) for the `_00_list_ref` poll that catches cross-session
+   * UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
+   * convergence + more query load; higher = the inverse. Non-positive
+   * values fall back to the default (500ms).
+   */
+  refSyncIntervalMs?: number;
 }
 type QueryHash = string;
 type RecordVersionArray = Array<[string, number]>;
@@ -301,7 +315,19 @@ interface QueryState {
   ttlDurationMs: number;
   /** Number of times the query has been updated. */
   updateCount: number;
+  /**
+   * Rolling window of the most recent materialization-step latencies (ms).
+   * Capped at MATERIALIZATION_SAMPLE_WINDOW; used to recompute p55/p90/p99
+   * before each persist to `_00_query`. Samples themselves are not persisted.
+   */
+  materializationSamples: number[];
+  /** Most recent end-to-end ingest latency in ms, or null until the first ingest. */
+  lastIngestLatencyMs: number | null;
+  /** Cumulative count of ingest/materialization errors observed for this query. */
+  errorCount: number;
 }
+/** Cap on the rolling materialization-sample window kept per query in memory. */
+declare const MATERIALIZATION_SAMPLE_WINDOW = 100;
 type QueryUpdateCallback = (records: Record<string, any>[]) => void;
 type MutationCallback = (mutations: UpEvent[]) => void;
 type MutationEventType = 'create' | 'update' | 'delete';
@@ -358,4 +384,4 @@ interface DebounceOptions {
   delay?: number;
 }
 //#endregion
-export { Logger$1 as C, UpdateOptions as S, EventSystem as T, RunOptions as _, MutationEvent as a, Sp00kyQueryResultPromise as b, PinoTransmit as c, QueryHash as d, QueryState as f, RecordVersionDiff as g, RecordVersionArray as h, MutationCallback as i, QueryConfig as l, QueryUpdateCallback as m, EventSubscriptionOptions as n, MutationEventType as o, QueryTimeToLive as p, Level$1 as r, PersistenceClient as s, DebounceOptions as t, QueryConfigRecord as u, Sp00kyConfig as v, EventDefinition as w, StoreType as x, Sp00kyQueryResult as y };
+export { UpdateOptions as C, EventSystem as E, StoreType as S, EventDefinition as T, RecordVersionDiff as _, MutationCallback as a, Sp00kyQueryResult as b, PersistenceClient as c, QueryConfigRecord as d, QueryHash as f, RecordVersionArray as g, QueryUpdateCallback as h, MATERIALIZATION_SAMPLE_WINDOW as i, PinoTransmit as l, QueryTimeToLive as m, EventSubscriptionOptions as n, MutationEvent as o, QueryState as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, QueryConfig as u, RunOptions as v, Logger$1 as w, Sp00kyQueryResultPromise as x, Sp00kyConfig as y };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.65",
+  "version": "0.0.1-canary.67",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -65,7 +65,7 @@
     "fast-json-patch": "^3.1.1",
     "loro-crdt": "^1.5.6",
     "pino": "^10.1.0",
-    "surrealdb": "2.0.0"
+    "surrealdb": "2.0.3"
   },
   "devDependencies": {
     "@types/node": "^22.5.2",

package/src/modules/cache/index.ts

@@ -94,7 +94,12 @@ export class CacheModule implements StreamUpdateReceiver {
         const query = surql.seal<void>(
           surql.tx(
             populatedRecords.map((_, i) => {
-              return surql.upsert(`id${i}`, `content${i}`);
+              // MERGE, not REPLACE: the remote payload omits local-only
+              // fields (`_00_crdt`, `_00_cursor`) injected by the CLI's
+              // local schema, so REPLACE would wipe the persisted CRDT
+              // snapshot on every sync-down round-trip and break offline
+              // reload of formatted text.
+              return surql.upsertMerge(`id${i}`, `content${i}`);
             })
           )
         );

package/src/modules/crdt/crdt-field.ts

@@ -1,5 +1,5 @@
 import { LoroDoc } from 'loro-crdt';
-import type { RemoteDatabaseService } from '../../services/database/index';
+import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
 import type { Logger } from '../../services/logger/index';
 import { parseRecordIdString } from '../../utils/index';
 
@@ -23,14 +23,21 @@ export function cursorColorFromName(name: string): string {
 export class CrdtField {
   private doc: LoroDoc;
   private pushTimer: ReturnType<typeof setTimeout> | null = null;
+  private local: LocalDatabaseService | null = null;
   private remote: RemoteDatabaseService | null = null;
   private recordId: string | null = null;
+  private sessionId: string = '';
   private unsubscribe: (() => void) | null = null;
   private lastPushTime = 0;
   private lastCursorPushTime = 0;
   private loadedFromCrdt = false;
   private pushRetryCount = 0;
   private logger: Logger | null;
+  private cursorsEnabled: boolean;
+  /** Remote-push debounce. Local writes happen immediately on every Loro
+   *  update; the remote UPSERT is coalesced over this window. Configured
+   *  via `Sp00kyConfig.crdtDebounceMs`, default 500. */
+  private remoteDebounceMs: number = 500;
 
   private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
   private pendingCursorUpdate: Uint8Array | null = null;
@@ -54,14 +61,36 @@ export class CrdtField {
 
   constructor(
     private fieldName: string,
-    initialState?: string,
+    cursorsEnabled: boolean,
+    initialState?: Uint8Array,
     logger?: Logger | null,
   ) {
+    if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) {
+      throw new Error(
+        `CrdtField: refusing unsafe field identifier '${fieldName}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
+      );
+    }
     this.logger = logger ?? null;
+    this.cursorsEnabled = cursorsEnabled;
     this.doc = new LoroDoc();
-    if (initialState) {
-      this.doc.import(decodeBase64(initialState));
-      this.loadedFromCrdt = true;
+    if (initialState && initialState.length > 0) {
+      // Tolerance: catch bad-snapshot data (corrupt blob, stale legacy
+      // value left over from a pre-`bytes` migration) so the editor still
+      // mounts. Without this guard the rejection bubbles through
+      // `useCrdtField` → permanent fallback `<p>`, with no cursor.
+      try {
+        this.doc.import(initialState);
+        this.loadedFromCrdt = true;
+      } catch (e) {
+        this.logger?.warn(
+          {
+            error: e,
+            fieldName,
+            Category: 'sp00ky-client::CrdtField::constructor',
+          },
+          'Initial CRDT state is not a valid LoroDoc snapshot — starting empty and will seed from fallback text'
+        );
+      }
     }
   }
 
@@ -72,11 +101,26 @@ export class CrdtField {
     return this.loadedFromCrdt;
   }
 
-  startSync(remote: RemoteDatabaseService, recordId: string): void {
+  startSync(
+    local: LocalDatabaseService,
+    remote: RemoteDatabaseService,
+    recordId: string,
+    sessionId: string,
+    debounceMs: number,
+  ): void {
+    this.local = local;
     this.remote = remote;
     this.recordId = recordId;
+    this.sessionId = sessionId;
+    this.remoteDebounceMs = debounceMs;
+    // Every local Loro update writes the snapshot to the local cache
+    // *immediately* (so reload/offline see the latest text), then
+    // schedules a debounced push to remote. The local UPSERT is cheap —
+    // it's an in-memory SurrealKV write — but errors are swallowed so a
+    // bad write never blocks user input.
     this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
-      this.schedulePush();
+      void this.persistLocal();
+      this.scheduleRemotePush();
     });
   }
 
@@ -86,11 +130,18 @@ export class CrdtField {
     if (this.remote && this.recordId) { void this.pushToRemote(); }
   }
 
-  importRemote(base64State: string): void {
-    // Suppress echo: skip imports within 500ms of our own push
-    if (Date.now() - this.lastPushTime < 500) return;
+  importRemote(state: Uint8Array): void {
+    // Echo suppression: skip imports within `remoteDebounceMs + 200` of
+    // our own push. The +200 guards against round-trip jitter where our
+    // own write echoes back from the LIVE feed before the debounce
+    // window closes.
+    if (Date.now() - this.lastPushTime < this.remoteDebounceMs + 200) return;
     try {
-      this.doc.import(decodeBase64(base64State));
+      this.doc.import(state);
+      // Persist the merged snapshot locally so the next reload/offline
+      // open sees the freshest converged state without waiting for the
+      // LIVE feed.
+      void this.persistLocal();
     } catch (e) {
       this.logger?.warn(
         { error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
@@ -99,20 +150,29 @@ export class CrdtField {
     }
   }
 
-  exportSnapshot(): string {
-    return encodeBase64(this.doc.export({ mode: 'snapshot' }));
+  exportSnapshot(): Uint8Array {
+    return this.doc.export({ mode: 'snapshot' });
   }
 
-  /** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
+  /** Push this session's cursor blob into the parent row at
+   *  `<field>.cursors[$sid]`. No-op when cursors aren't enabled on this
+   *  field — the editor still calls this method optimistically, but
+   *  without `@cursor` on the schema there's nowhere to store the blob.
+   *  The UPDATE itself fires the parent table's LIVE feed, so other
+   *  browsers receive the cursor change without a separate `_00_rv` bump. */
   async pushCursorState(encoded: Uint8Array): Promise<void> {
     if (!this.remote || !this.recordId) return;
+    if (!this.cursorsEnabled) return;
     this.lastCursorPushTime = Date.now();
     try {
       const state = encodeBase64(encoded);
       await this.remote.query(
-        `INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
-         ON DUPLICATE KEY UPDATE state = $state`,
-        { rid: parseRecordIdString(this.recordId), field: `_cursor_${this.fieldName}`, state }
+        `UPDATE $id SET ${this.fieldName}.cursors[$sid] = $state RETURN NONE;`,
+        {
+          id: parseRecordIdString(this.recordId),
+          sid: this.sessionId,
+          state,
+        }
       );
     } catch (e) {
       this.logger?.warn(
@@ -141,19 +201,48 @@ export class CrdtField {
     }
   }
 
-  private schedulePush(): void {
+  private scheduleRemotePush(): void {
     if (this.pushTimer) clearTimeout(this.pushTimer);
-    this.pushTimer = setTimeout(() => void this.pushToRemote(), 300);
+    this.pushTimer = setTimeout(() => void this.pushToRemote(), this.remoteDebounceMs);
+  }
+
+  /** SET path inside a parent row for the current snapshot. `@crdt`-only
+   *  fields hold the snapshot directly (`<field>`); `@crdt @cursor`
+   *  fields hold a `{ state, cursors }` object so the snapshot lives at
+   *  `<field>.state` next to per-session cursor blobs. */
+  private statePath(): string {
+    return this.cursorsEnabled ? `${this.fieldName}.state` : this.fieldName;
+  }
+
+  /** Mirror the LoroDoc snapshot into the parent row locally. Runs on
+   *  every local update and every remote import so reloads (online or
+   *  offline) see the freshest content immediately. Failures are
+   *  swallowed — a stale local write must never block user input. */
+  private async persistLocal(): Promise<void> {
+    if (!this.local || !this.recordId) return;
+    try {
+      await this.local.query(
+        `UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
+        { id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
+      );
+    } catch (e) {
+      this.logger?.debug(
+        { error: e, Category: 'sp00ky-client::CrdtField::persistLocal' },
+        'Local CRDT persist failed (best-effort)'
+      );
+    }
   }
 
   private async pushToRemote(): Promise<void> {
     if (!this.remote || !this.recordId) return;
     this.lastPushTime = Date.now();
     try {
+      // The UPDATE on the parent fires the parent table's LIVE feed,
+      // so cross-browser receivers see this change directly in the LIVE
+      // payload — no separate `_00_rv` bump or sidecar UPSERT.
       await this.remote.query(
-        `INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
-         ON DUPLICATE KEY UPDATE state = $state`,
-        { rid: parseRecordIdString(this.recordId), field: this.fieldName, state: this.exportSnapshot() }
+        `UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
+        { id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
       );
       this.pushRetryCount = 0;
     } catch (e) {
@@ -161,9 +250,12 @@ export class CrdtField {
         { error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
         'Failed to push CRDT state to remote'
       );
+      // Bounded retry. Offline first-loads will exhaust this and stop
+      // hammering; the next user keystroke (or the next time a remote
+      // event lands once we're back online) will kick off another push.
       if (this.pushRetryCount < 2) {
         this.pushRetryCount++;
-        this.schedulePush();
+        this.scheduleRemotePush();
       }
     }
   }

package/src/modules/crdt/crdt-hydration.test.ts

@@ -0,0 +1,206 @@
+import { describe, it, expect, vi } from 'vitest';
+import { LoroDoc } from 'loro-crdt';
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import { CrdtManager } from './index';
+import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
+import type { Logger } from '../../services/logger/index';
+
+// Regression test for the offline-reload formatting-loss class of bug.
+//
+// Repro pattern: app uses `store: 'memory'` for the local SurrealDB (or
+// any scenario where the local row exists without a CRDT snapshot —
+// fresh device, post-sync-down with stale meta, …). On reload, the
+// editor mounts but stays empty until some unrelated edit fires the
+// parent LIVE feed. From the user's POV, formatting is gone.
+//
+// Fix: when `CrdtManager.open` finds no local snapshot, it must fetch
+// the parent row from remote (the snapshot lives inline on the row in
+// the consolidated design) and dispatch its CRDT field. This test pins
+// that behavior across both shapes — `@crdt`-only fields where the
+// snapshot IS the field value, and `@crdt @cursor` fields where the
+// snapshot lives at `<field>.state`.
+
+function silentLogger(): Logger {
+  const noop = () => {};
+  const fake: any = {};
+  fake.info = noop;
+  fake.warn = noop;
+  fake.debug = noop;
+  fake.error = noop;
+  fake.trace = noop;
+  fake.child = () => fake;
+  return fake as Logger;
+}
+
+function singleTableSchema(opts: { cursor?: boolean } = {}): SchemaStructure {
+  return {
+    tables: [
+      {
+        name: 'thread',
+        columns: {
+          body: {
+            type: 'string',
+            optional: false,
+            crdt: 'text',
+            ...(opts.cursor ? { cursor: true } : {}),
+          },
+        },
+        primaryKey: ['id'],
+      },
+    ],
+    relationships: [],
+    backends: {},
+  };
+}
+
+function buildSnapshot(text: string): Uint8Array {
+  const doc = new LoroDoc();
+  doc.getText('text').insert(0, text);
+  return doc.export({ mode: 'snapshot' });
+}
+
+function makeRemote(handlers: {
+  selectRow?: () => unknown;
+}): RemoteDatabaseService {
+  const remote = {
+    query: vi.fn().mockImplementation(async (sql: string) => {
+      if (sql.includes('LIVE SELECT')) return ['live-uuid'];
+      if (sql.includes('SELECT * FROM ONLY')) {
+        return [handlers.selectRow ? handlers.selectRow() : null];
+      }
+      return [];
+    }),
+    getClient: () =>
+      ({
+        liveOf: async () => ({
+          subscribe: () => () => {},
+        }),
+      }) as any,
+  } satisfies Partial<RemoteDatabaseService> as unknown as RemoteDatabaseService;
+  return remote;
+}
+
+describe('CrdtManager.open hydration', () => {
+  it('@crdt-only: fetches the parent row from remote when local is empty', async () => {
+    const remoteSnapshot = buildSnapshot('hello world');
+
+    const local = {
+      query: vi.fn().mockImplementation(async (sql: string) => {
+        if (sql.includes('SELECT VALUE body FROM ONLY')) return [null];
+        return [];
+      }),
+    } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
+
+    const remote = makeRemote({
+      selectRow: () => ({ id: 'thread:abc', body: remoteSnapshot }),
+    });
+
+    const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:abc', 'body');
+
+    await vi.waitFor(() => {
+      expect(field.getDoc().getText('text').toString()).toBe('hello world');
+    });
+
+    const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
+      (c) => c[0] as string
+    );
+    expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(true);
+  });
+
+  it('@crdt @cursor: extracts the snapshot from the object shape', async () => {
+    const remoteSnapshot = buildSnapshot('object shape');
+
+    const local = {
+      query: vi.fn().mockImplementation(async () => [null]),
+    } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
+
+    // Cursor-enabled field stores `{ state, cursors }`. The remote row
+    // returns this shape; the manager must drill into `.state` for the
+    // snapshot bytes.
+    const remote = makeRemote({
+      selectRow: () => ({
+        id: 'thread:abc',
+        body: { state: remoteSnapshot, cursors: {} },
+      }),
+    });
+
+    const manager = new CrdtManager(
+      singleTableSchema({ cursor: true }),
+      local,
+      remote,
+      silentLogger()
+    );
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:abc', 'body');
+
+    await vi.waitFor(() => {
+      expect(field.getDoc().getText('text').toString()).toBe('object shape');
+    });
+  });
+
+  it('skips the remote fetch when a local snapshot already exists', async () => {
+    const localSnapshot = buildSnapshot('cached');
+
+    const local = {
+      query: vi.fn().mockImplementation(async (sql: string) => {
+        if (sql.includes('SELECT VALUE body FROM ONLY')) return [localSnapshot];
+        return [];
+      }),
+    } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
+
+    const remote = makeRemote({});
+
+    const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:def', 'body');
+
+    expect(field.getDoc().getText('text').toString()).toBe('cached');
+
+    // Give any stray async work a tick to land before asserting absence.
+    await new Promise((r) => setTimeout(r, 20));
+
+    const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
+      (c) => c[0] as string
+    );
+    expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(false);
+  });
+
+  // Regression: SurrealDB returns bytes nested inside a FLEXIBLE object as a
+  // plain `number[]` from the local WASM engine (e.g. `state: [108, 111, 114,
+  // 111, ...]` — the literal LoroDoc magic header bytes). The cursor-enabled
+  // extractSnapshot path used to only accept Uint8Array and would silently
+  // return undefined for the array form, leaving every editor empty and
+  // killing realtime updates for `@crdt @cursor` fields.
+  it('@crdt @cursor: accepts number[] as the state shape (local WASM transport)', async () => {
+    const remoteSnapshot = buildSnapshot('from local wasm');
+    const remoteAsArray = Array.from(remoteSnapshot) as unknown as number[];
+
+    const local = {
+      query: vi.fn().mockImplementation(async (sql: string) => {
+        if (sql.includes('SELECT VALUE body FROM ONLY')) {
+          return [{ state: remoteAsArray, cursors: {} }];
+        }
+        return [];
+      }),
+    } satisfies Partial<LocalDatabaseService> as unknown as LocalDatabaseService;
+
+    const remote = makeRemote({});
+
+    const manager = new CrdtManager(
+      singleTableSchema({ cursor: true }),
+      local,
+      remote,
+      silentLogger()
+    );
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:wasm', 'body');
+
+    expect(field.getDoc().getText('text').toString()).toBe('from local wasm');
+  });
+});