@spooky-sync/core 0.0.1-canary.13 → 0.0.1-canary.131

package/src/modules/crdt/crdt-field.ts

@@ -0,0 +1,288 @@
+import { LoroDoc } from 'loro-crdt';
+import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
+import type { Logger } from '../../services/logger/index';
+import { parseRecordIdString } from '../../utils/index';
+
+// ==================== CURSOR UTILITIES ====================
+
+export const CURSOR_COLORS = [
+  '#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
+  '#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
+];
+
+export function cursorColorFromName(name: string): string {
+  let hash = 0;
+  for (let i = 0; i < name.length; i++) {
+    hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
+  }
+  return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
+}
+
+// ==================== CRDT FIELD ====================
+
+export class CrdtField {
+  private doc: LoroDoc;
+  private pushTimer: ReturnType<typeof setTimeout> | null = null;
+  private local: LocalStore | null = null;
+  private remote: RemoteDatabaseService | null = null;
+  private recordId: string | null = null;
+  private sessionId: string = '';
+  private unsubscribe: (() => void) | null = null;
+  private lastPushTime = 0;
+  private lastCursorPushTime = 0;
+  private loadedFromCrdt = false;
+  private pushRetryCount = 0;
+  private logger: Logger | null;
+  private cursorsEnabled: boolean;
+  /** Remote-push debounce. Local writes happen immediately on every Loro
+   *  update; the remote UPSERT is coalesced over this window. Configured
+   *  via `Sp00kyConfig.crdtDebounceMs`, default 500. */
+  private remoteDebounceMs: number = 500;
+
+  private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
+  private pendingCursorUpdate: Uint8Array | null = null;
+
+  /** Callback set by the editor to receive remote cursor updates.
+   *  Any cursor data that arrived before this callback was set will be replayed. */
+  set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
+    this._onCursorUpdate = cb;
+    if (cb && this.pendingCursorUpdate) {
+      try { cb(this.pendingCursorUpdate); } catch (e) {
+        this.logger?.warn(
+          { error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
+          'Failed to replay pending cursor update'
+        );
+      }
+      this.pendingCursorUpdate = null;
+    }
+  }
+
+  get onCursorUpdate() { return this._onCursorUpdate; }
+
+  constructor(
+    private fieldName: string,
+    cursorsEnabled: boolean,
+    initialState?: Uint8Array,
+    logger?: Logger | null,
+  ) {
+    if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) {
+      throw new Error(
+        `CrdtField: refusing unsafe field identifier '${fieldName}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
+      );
+    }
+    this.logger = logger ?? null;
+    this.cursorsEnabled = cursorsEnabled;
+    this.doc = new LoroDoc();
+    if (initialState && initialState.length > 0) {
+      // Tolerance: catch bad-snapshot data (corrupt blob, stale legacy
+      // value left over from a pre-`bytes` migration) so the editor still
+      // mounts. Without this guard the rejection bubbles through
+      // `useCrdtField` → permanent fallback `<p>`, with no cursor.
+      try {
+        this.doc.import(initialState);
+        this.loadedFromCrdt = true;
+      } catch (e) {
+        this.logger?.warn(
+          {
+            error: e,
+            fieldName,
+            Category: 'sp00ky-client::CrdtField::constructor',
+          },
+          'Initial CRDT state is not a valid LoroDoc snapshot — starting empty and will seed from fallback text'
+        );
+      }
+    }
+  }
+
+  getDoc(): LoroDoc { return this.doc; }
+
+  /** Whether the LoroDoc was loaded from saved CRDT state */
+  hasContent(): boolean {
+    return this.loadedFromCrdt;
+  }
+
+  startSync(
+    local: LocalStore,
+    remote: RemoteDatabaseService,
+    recordId: string,
+    sessionId: string,
+    debounceMs: number,
+  ): void {
+    this.local = local;
+    this.remote = remote;
+    this.recordId = recordId;
+    this.sessionId = sessionId;
+    this.remoteDebounceMs = debounceMs;
+    // Every local Loro update writes the snapshot to the local cache
+    // *immediately* (so reload/offline see the latest text), then
+    // schedules a debounced push to remote. The local UPSERT is cheap —
+    // it's an in-memory SurrealKV write — but errors are swallowed so a
+    // bad write never blocks user input.
+    this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
+      void this.persistLocal();
+      this.scheduleRemotePush();
+    });
+  }
+
+  /**
+   * Stop syncing this field. Flushes one final remote push by default so the
+   * last keystrokes aren't lost. Pass `{ flush: false }` on a bucket switch —
+   * the remote session already belongs to the NEXT user, and pushing this
+   * (previous user's) snapshot under it would clobber the record remotely.
+   */
+  stopSync(options: { flush?: boolean } = {}): void {
+    const flush = options.flush ?? true;
+    if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
+    if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
+    if (flush && this.remote && this.recordId) { void this.pushToRemote(); }
+  }
+
+  importRemote(state: Uint8Array): void {
+    // Echo suppression: skip imports within `remoteDebounceMs + 200` of
+    // our own push. The +200 guards against round-trip jitter where our
+    // own write echoes back from the LIVE feed before the debounce
+    // window closes.
+    if (Date.now() - this.lastPushTime < this.remoteDebounceMs + 200) return;
+    try {
+      this.doc.import(state);
+      // Persist the merged snapshot locally so the next reload/offline
+      // open sees the freshest converged state without waiting for the
+      // LIVE feed.
+      void this.persistLocal();
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
+        'Failed to import remote CRDT state'
+      );
+    }
+  }
+
+  exportSnapshot(): Uint8Array {
+    return this.doc.export({ mode: 'snapshot' });
+  }
+
+  /** Push this session's cursor blob into the parent row at
+   *  `<field>.cursors[$sid]`. No-op when cursors aren't enabled on this
+   *  field — the editor still calls this method optimistically, but
+   *  without `@cursor` on the schema there's nowhere to store the blob.
+   *  The UPDATE itself fires the parent table's LIVE feed, so other
+   *  browsers receive the cursor change without a separate `_00_rv` bump. */
+  async pushCursorState(encoded: Uint8Array): Promise<void> {
+    if (!this.remote || !this.recordId) return;
+    if (!this.cursorsEnabled) return;
+    this.lastCursorPushTime = Date.now();
+    try {
+      const state = encodeBase64(encoded);
+      await this.remote.query(
+        `UPDATE $id SET ${this.fieldName}.cursors[$sid] = $state RETURN NONE;`,
+        {
+          id: parseRecordIdString(this.recordId),
+          sid: this.sessionId,
+          state,
+        }
+      );
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
+        'Failed to push cursor state'
+      );
+    }
+  }
+
+  /** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
+  importRemoteCursor(base64State: string): void {
+    if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
+    try {
+      const data = decodeBase64(base64State);
+      if (this._onCursorUpdate) {
+        this._onCursorUpdate(data);
+      } else {
+        // Only keep the latest cursor state — older positions are useless
+        this.pendingCursorUpdate = data;
+      }
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
+        'Failed to apply remote cursor data'
+      );
+    }
+  }
+
+  private scheduleRemotePush(): void {
+    if (this.pushTimer) clearTimeout(this.pushTimer);
+    this.pushTimer = setTimeout(() => void this.pushToRemote(), this.remoteDebounceMs);
+  }
+
+  /** SET path inside a parent row for the current snapshot. `@crdt`-only
+   *  fields hold the snapshot directly (`<field>`); `@crdt @cursor`
+   *  fields hold a `{ state, cursors }` object so the snapshot lives at
+   *  `<field>.state` next to per-session cursor blobs. */
+  private statePath(): string {
+    return this.cursorsEnabled ? `${this.fieldName}.state` : this.fieldName;
+  }
+
+  /** Mirror the LoroDoc snapshot into the parent row locally. Runs on
+   *  every local update and every remote import so reloads (online or
+   *  offline) see the freshest content immediately. Failures are
+   *  swallowed — a stale local write must never block user input. */
+  private async persistLocal(): Promise<void> {
+    if (!this.local || !this.recordId) return;
+    try {
+      await this.local.query(
+        `UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
+        { id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
+      );
+    } catch (e) {
+      this.logger?.debug(
+        { error: e, Category: 'sp00ky-client::CrdtField::persistLocal' },
+        'Local CRDT persist failed (best-effort)'
+      );
+    }
+  }
+
+  private async pushToRemote(): Promise<void> {
+    if (!this.remote || !this.recordId) return;
+    this.lastPushTime = Date.now();
+    try {
+      // The UPDATE on the parent fires the parent table's LIVE feed,
+      // so cross-browser receivers see this change directly in the LIVE
+      // payload — no separate `_00_rv` bump or sidecar UPSERT.
+      await this.remote.query(
+        `UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
+        { id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
+      );
+      this.pushRetryCount = 0;
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
+        'Failed to push CRDT state to remote'
+      );
+      // Bounded retry. Offline first-loads will exhaust this and stop
+      // hammering; the next user keystroke (or the next time a remote
+      // event lands once we're back online) will kick off another push.
+      if (this.pushRetryCount < 2) {
+        this.pushRetryCount++;
+        this.scheduleRemotePush();
+      }
+    }
+  }
+}
+
+export function decodeBase64(b64: string): Uint8Array {
+  if (typeof atob === 'function') {
+    const binary = atob(b64);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+    return bytes;
+  }
+  return new Uint8Array(Buffer.from(b64, 'base64'));
+}
+
+export function encodeBase64(bytes: Uint8Array): string {
+  if (typeof btoa === 'function') {
+    let binary = '';
+    for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+    return btoa(binary);
+  }
+  return Buffer.from(bytes).toString('base64');
+}

package/src/modules/crdt/crdt-hydration.test.ts

@@ -0,0 +1,206 @@
+import { describe, it, expect, vi } from 'vitest';
+import { LoroDoc } from 'loro-crdt';
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import { CrdtManager } from './index';
+import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
+import type { Logger } from '../../services/logger/index';
+
+// Regression test for the offline-reload formatting-loss class of bug.
+//
+// Repro pattern: app uses `store: 'memory'` for the local SurrealDB (or
+// any scenario where the local row exists without a CRDT snapshot —
+// fresh device, post-sync-down with stale meta, …). On reload, the
+// editor mounts but stays empty until some unrelated edit fires the
+// parent LIVE feed. From the user's POV, formatting is gone.
+//
+// Fix: when `CrdtManager.open` finds no local snapshot, it must fetch
+// the parent row from remote (the snapshot lives inline on the row in
+// the consolidated design) and dispatch its CRDT field. This test pins
+// that behavior across both shapes — `@crdt`-only fields where the
+// snapshot IS the field value, and `@crdt @cursor` fields where the
+// snapshot lives at `<field>.state`.
+
+function silentLogger(): Logger {
+  const noop = () => {};
+  const fake: any = {};
+  fake.info = noop;
+  fake.warn = noop;
+  fake.debug = noop;
+  fake.error = noop;
+  fake.trace = noop;
+  fake.child = () => fake;
+  return fake as Logger;
+}
+
+function singleTableSchema(opts: { cursor?: boolean } = {}): SchemaStructure {
+  return {
+    tables: [
+      {
+        name: 'thread',
+        columns: {
+          body: {
+            type: 'string',
+            optional: false,
+            crdt: 'text',
+            ...(opts.cursor ? { cursor: true } : {}),
+          },
+        },
+        primaryKey: ['id'],
+      },
+    ],
+    relationships: [],
+    backends: {},
+  };
+}
+
+function buildSnapshot(text: string): Uint8Array {
+  const doc = new LoroDoc();
+  doc.getText('text').insert(0, text);
+  return doc.export({ mode: 'snapshot' });
+}
+
+function makeRemote(handlers: {
+  selectRow?: () => unknown;
+}): RemoteDatabaseService {
+  const remote = {
+    query: vi.fn().mockImplementation(async (sql: string) => {
+      if (sql.includes('LIVE SELECT')) return ['live-uuid'];
+      if (sql.includes('SELECT * FROM ONLY')) {
+        return [handlers.selectRow ? handlers.selectRow() : null];
+      }
+      return [];
+    }),
+    getClient: () =>
+      ({
+        liveOf: async () => ({
+          subscribe: () => () => {},
+        }),
+      }) as any,
+  } satisfies Partial<RemoteDatabaseService> as unknown as RemoteDatabaseService;
+  return remote;
+}
+
+describe('CrdtManager.open hydration', () => {
+  it('@crdt-only: fetches the parent row from remote when local is empty', async () => {
+    const remoteSnapshot = buildSnapshot('hello world');
+
+    const local = {
+      query: vi.fn().mockImplementation(async (sql: string) => {
+        if (sql.includes('SELECT VALUE body FROM ONLY')) return [null];
+        return [];
+      }),
+    } satisfies Partial<LocalStore> as unknown as LocalStore;
+
+    const remote = makeRemote({
+      selectRow: () => ({ id: 'thread:abc', body: remoteSnapshot }),
+    });
+
+    const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:abc', 'body');
+
+    await vi.waitFor(() => {
+      expect(field.getDoc().getText('text').toString()).toBe('hello world');
+    });
+
+    const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
+      (c) => c[0] as string
+    );
+    expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(true);
+  });
+
+  it('@crdt @cursor: extracts the snapshot from the object shape', async () => {
+    const remoteSnapshot = buildSnapshot('object shape');
+
+    const local = {
+      query: vi.fn().mockImplementation(async () => [null]),
+    } satisfies Partial<LocalStore> as unknown as LocalStore;
+
+    // Cursor-enabled field stores `{ state, cursors }`. The remote row
+    // returns this shape; the manager must drill into `.state` for the
+    // snapshot bytes.
+    const remote = makeRemote({
+      selectRow: () => ({
+        id: 'thread:abc',
+        body: { state: remoteSnapshot, cursors: {} },
+      }),
+    });
+
+    const manager = new CrdtManager(
+      singleTableSchema({ cursor: true }),
+      local,
+      remote,
+      silentLogger()
+    );
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:abc', 'body');
+
+    await vi.waitFor(() => {
+      expect(field.getDoc().getText('text').toString()).toBe('object shape');
+    });
+  });
+
+  it('skips the remote fetch when a local snapshot already exists', async () => {
+    const localSnapshot = buildSnapshot('cached');
+
+    const local = {
+      query: vi.fn().mockImplementation(async (sql: string) => {
+        if (sql.includes('SELECT VALUE body FROM ONLY')) return [localSnapshot];
+        return [];
+      }),
+    } satisfies Partial<LocalStore> as unknown as LocalStore;
+
+    const remote = makeRemote({});
+
+    const manager = new CrdtManager(singleTableSchema(), local, remote, silentLogger());
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:def', 'body');
+
+    expect(field.getDoc().getText('text').toString()).toBe('cached');
+
+    // Give any stray async work a tick to land before asserting absence.
+    await new Promise((r) => setTimeout(r, 20));
+
+    const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map(
+      (c) => c[0] as string
+    );
+    expect(sqls.some((s) => s.includes('SELECT * FROM ONLY'))).toBe(false);
+  });
+
+  // Regression: SurrealDB returns bytes nested inside a FLEXIBLE object as a
+  // plain `number[]` from the local WASM engine (e.g. `state: [108, 111, 114,
+  // 111, ...]` — the literal LoroDoc magic header bytes). The cursor-enabled
+  // extractSnapshot path used to only accept Uint8Array and would silently
+  // return undefined for the array form, leaving every editor empty and
+  // killing realtime updates for `@crdt @cursor` fields.
+  it('@crdt @cursor: accepts number[] as the state shape (local WASM transport)', async () => {
+    const remoteSnapshot = buildSnapshot('from local wasm');
+    const remoteAsArray = Array.from(remoteSnapshot) as unknown as number[];
+
+    const local = {
+      query: vi.fn().mockImplementation(async (sql: string) => {
+        if (sql.includes('SELECT VALUE body FROM ONLY')) {
+          return [{ state: remoteAsArray, cursors: {} }];
+        }
+        return [];
+      }),
+    } satisfies Partial<LocalStore> as unknown as LocalStore;
+
+    const remote = makeRemote({});
+
+    const manager = new CrdtManager(
+      singleTableSchema({ cursor: true }),
+      local,
+      remote,
+      silentLogger()
+    );
+    manager.setSessionId('session-under-test');
+
+    const field = await manager.open('thread', 'thread:wasm', 'body');
+
+    expect(field.getDoc().getText('text').toString()).toBe('from local wasm');
+  });
+});