@spooky-sync/core 0.0.1-canary.33 → 0.0.1-canary.331

package/src/services/database/sqlite-open.test.ts

@@ -0,0 +1,150 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { openDb } from './sqlite-open';
+
+class FakeDb {
+  constructor(public arg: unknown) {}
+  exec() {
+    return [];
+  }
+  close() {}
+}
+
+/** Minimal stand-in for the initialized sqlite-wasm module. `install` is the
+ *  `installOpfsSAHPoolVfs` behavior under test; omit it to model a build that
+ *  lacks the SAHPool VFS entirely. */
+function makeSqlite3(install?: (opts: any) => Promise<unknown>) {
+  const sqlite3: any = { oo1: { DB: FakeDb } };
+  if (install) sqlite3.installOpfsSAHPoolVfs = vi.fn(install);
+  return sqlite3;
+}
+
+const pool = { OpfsSAHPoolDb: FakeDb };
+/** What a pool locked by another tab of the app actually throws. */
+function lockedError(): Error {
+  const e = new Error('Access Handles cannot be acquired');
+  e.name = 'NoModificationAllowedError';
+  return e;
+}
+
+const noSleep = () => Promise.resolve();
+
+let errSpy: ReturnType<typeof vi.spyOn>;
+beforeEach(() => {
+  errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+});
+afterEach(() => {
+  errSpy.mockRestore();
+});
+
+// The bare `catch {}` this replaces turned every OPFS failure into a silent
+// full-RAM database: no log, no reason, no way for the app to know its writes
+// die on reload. Each case below pins one half of the fix: keep trying when
+// retrying can plausibly work, and when it can't, say so loudly and hand the
+// reason back.
+describe('openDb', () => {
+  it('opens the OPFS pool on the first attempt', async () => {
+    const sqlite3 = makeSqlite3(async () => pool);
+    const res = await openDb(sqlite3, 'user:abc', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(true);
+    expect(res.opfsError).toBeUndefined();
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
+    // The pool is named per bucket, and the first try must NOT force a re-init
+    // (that would throw away a pool another attempt is legitimately using).
+    expect(sqlite3.installOpfsSAHPoolVfs.mock.calls[0][0]).toEqual({ name: 'sp00ky-user:abc' });
+    expect(errSpy).not.toHaveBeenCalled();
+  });
+
+  // The tab-closing race: the old tab still holds the sync access handles when
+  // the new one boots. Without a retry that tab is stuck in RAM for its whole
+  // lifetime, even though the lock frees milliseconds later.
+  it('retries a locked pool and succeeds, forcing re-init after the first failure', async () => {
+    let calls = 0;
+    const sqlite3 = makeSqlite3(async () => {
+      if (++calls < 3) throw lockedError();
+      return pool;
+    });
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(true);
+    expect(res.opfsError).toBeUndefined();
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
+    // sqlite-wasm caches the first rejection against the VFS name, so retries
+    // that don't ask for a real re-init just replay it.
+    const [first, second, third] = sqlite3.installOpfsSAHPoolVfs.mock.calls.map((c: any[]) => c[0]);
+    expect(first.forceReinitIfPreviouslyFailed).toBeUndefined();
+    expect(second.forceReinitIfPreviouslyFailed).toBe(true);
+    expect(third.forceReinitIfPreviouslyFailed).toBe(true);
+    expect(errSpy).not.toHaveBeenCalled();
+  });
+
+  it('falls back loudly after exhausting the retries, keeping the reason', async () => {
+    const sqlite3 = makeSqlite3(async () => {
+      throw lockedError();
+    });
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    // The DOMException name is the diagnostic part: it names the lock.
+    expect(res.opfsError).toContain('NoModificationAllowedError');
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
+    expect(errSpy).toHaveBeenCalledTimes(1);
+    expect(String(errSpy.mock.calls[0][0])).toContain('IN MEMORY');
+    // Still a usable handle: losing durability must not break the app.
+    expect(res.db).toBeDefined();
+  });
+
+  // An insecure context has no sync access handles at all, so retrying just
+  // adds boot latency to a foregone conclusion.
+  it('does not retry when the OPFS APIs are missing entirely', async () => {
+    const sqlite3 = makeSqlite3(async () => {
+      throw new Error('Missing required OPFS APIs.');
+    });
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    expect(res.opfsError).toContain('Missing required OPFS APIs');
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
+    expect(errSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('reports a build without the SAHPool VFS without calling anything', async () => {
+    const sqlite3 = makeSqlite3();
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    expect(res.opfsError).toContain('installOpfsSAHPoolVfs');
+    expect(errSpy).toHaveBeenCalledTimes(1);
+  });
+
+  // `store: 'memory'` is a configuration choice, not a degradation, so it must
+  // stay silent and carry no error for the UI to warn about.
+  it('opens in memory quietly when persistence was not requested', async () => {
+    const sqlite3 = makeSqlite3(async () => pool);
+    const res = await openDb(sqlite3, 'main', false, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    expect(res.opfsError).toBeUndefined();
+    expect(sqlite3.installOpfsSAHPoolVfs).not.toHaveBeenCalled();
+    expect(errSpy).not.toHaveBeenCalled();
+  });
+
+  it('honors maxAttempts and waits the configured backoff between tries', async () => {
+    const slept: number[] = [];
+    const sqlite3 = makeSqlite3(async () => {
+      throw lockedError();
+    });
+    const res = await openDb(sqlite3, 'main', true, {
+      maxAttempts: 4,
+      backoffMs: [10, 20],
+      sleep: async (ms) => {
+        slept.push(ms);
+      },
+    });
+
+    expect(res.persisted).toBe(false);
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(4);
+    // One sleep per gap (never after the last attempt), last delay repeating.
+    expect(slept).toEqual([10, 20, 20]);
+  });
+});

package/src/services/database/sqlite-open.ts

@@ -0,0 +1,164 @@
+/**
+ * Opening the worker's SQLite handle: the OPFS SAHPool VFS when durable
+ * storage was asked for, an in-memory DB only as a last resort. Extracted from
+ * `sqlite-worker.ts` (which imports the wasm module at module scope and so
+ * can't be loaded in a unit test) to keep the retry/fallback policy testable
+ * off-worker, the same split as `sqlite-select.ts`.
+ *
+ * Why retry: SAHPool holds an EXCLUSIVE sync access handle on every file in
+ * its pool, so only one client per pool name can have it open. A second tab of
+ * the same app therefore fails init, and `installOpfsSAHPoolVfs` CACHES that
+ * rejection per VFS name, so a later call only gets a real second chance when
+ * it passes `forceReinitIfPreviouslyFailed`. Retrying with that flag turns the
+ * common "the other tab is still closing" race into a success instead of a
+ * permanent in-memory session.
+ *
+ * Why the noise: `:memory:` holds the whole dataset in RAM (the
+ * OOM-on-wasm-heavy-pages failure mode the OPFS store exists to avoid) and
+ * drops every local write on reload. Host apps run pino at their own level,
+ * some at `fatal`, so the fallback ALSO writes to `console.error` from inside
+ * the worker, and the reason travels back to the engine as `opfsError` for the
+ * app to surface.
+ */
+
+/** The DB surface the worker uses (a `sqlite3.oo1.DB` or an `OpfsSAHPoolDb`). */
+export interface SqliteDbHandle {
+  exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
+  close: () => void;
+}
+
+/** The slice of the OpfsSAHPoolUtil the worker needs for teardown. */
+export interface SqlitePoolHandle {
+  /** Unregisters the VFS and releases every sync access handle, leaving the
+   *  files intact, so another worker can open the pool without waiting for
+   *  this worker to be garbage collected. Throws while files are open. */
+  pauseVfs?: () => unknown;
+}
+
+export interface OpenDbResult {
+  db: SqliteDbHandle;
+  /** True only when the handle is backed by OPFS and survives a reload. */
+  persisted: boolean;
+  /** Why persistence failed. Set only when OPFS was requested and fell back. */
+  opfsError?: string;
+  /** Pool util, present only for an OPFS-backed handle. */
+  pool?: SqlitePoolHandle;
+}
+
+export interface OpenDbOptions {
+  /** Total OPFS init attempts, including the first. Default 3. */
+  maxAttempts?: number;
+  /** Delay before each retry; the last entry repeats. Default [250, 500]. */
+  backoffMs?: number[];
+  /**
+   * Throw (`opfs-unavailable: <reason>`) instead of falling back to memory.
+   * Used by shared-tabs leader promotion: a silently-in-memory LEADER would
+   * put every tab's data in RAM, so promotion prefers failing the election
+   * (the broker retries, possibly on another tab) over degrading. The broker
+   * grants an explicit memory fallback only after repeated failed cycles.
+   */
+  disallowMemoryFallback?: boolean;
+  /** Injectable for tests. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+const DEFAULT_MAX_ATTEMPTS = 3;
+/** Bounded on purpose: this runs on the boot path, before the first query. */
+const DEFAULT_BACKOFF_MS = [250, 500];
+
+/**
+ * Leader-promotion profile (~5.8s worst case). A dead leader's sync access
+ * handles release when the browser garbage-collects its worker, typically
+ * well under a second but not synchronously with the Web Lock release the
+ * election observed, so promotion retries longer than a cold boot.
+ */
+export const PROMOTION_OPEN_OPTIONS: Pick<OpenDbOptions, 'maxAttempts' | 'backoffMs'> = {
+  maxAttempts: 10,
+  backoffMs: [50, 100, 200, 400, 800, 1000],
+};
+
+/** Failures no retry can fix: the APIs aren't there at all (insecure context,
+ *  or a browser without sync access handles). Fall back immediately. */
+const UNRETRYABLE = ['Missing required OPFS APIs'];
+
+const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+/** Keep the DOMException name (e.g. `NoModificationAllowedError` for a pool
+ *  locked by another tab): it is the most diagnostic part of the failure. */
+function errMessage(e: unknown): string {
+  if (e instanceof Error) return e.name && e.name !== 'Error' ? `${e.name}: ${e.message}` : e.message;
+  return String(e);
+}
+
+function fallbackToMemory(
+  sqlite3: any,
+  dbName: string,
+  reason: string,
+  attempts: number
+): OpenDbResult {
+  const tried = attempts > 0 ? ` after ${attempts} attempt${attempts === 1 ? '' : 's'}` : '';
+  // Deliberately console, not the logger: host apps configure pino's level (some
+  // run `fatal`), and losing durability must never be filtered into silence.
+  // oxlint-disable-next-line no-console
+  console.error(
+    `[sp00ky] OPFS persistence unavailable for "${dbName}"${tried}: ${reason}. The local SQLite ` +
+      'cache is running IN MEMORY, which keeps the whole dataset in RAM and loses every local ' +
+      'write on reload. The usual cause is another tab of this app holding the storage lock, so ' +
+      'closing the other tabs and reloading restores persistence.'
+  );
+  return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false, opfsError: reason };
+}
+
+/**
+ * Open `dbName`'s handle. Never throws for a storage problem: a caller that
+ * asked for persistence and can't have it gets a working in-memory handle plus
+ * `persisted: false` and an `opfsError` to report.
+ */
+export async function openDb(
+  sqlite3: any,
+  dbName: string,
+  useOpfs: boolean,
+  opts: OpenDbOptions = {}
+): Promise<OpenDbResult> {
+  // Memory was the configured choice (`store: 'memory'`), not a failure, so no
+  // error and no noise.
+  if (!useOpfs) return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false };
+
+  if (!sqlite3.installOpfsSAHPoolVfs) {
+    if (opts.disallowMemoryFallback) {
+      throw new Error('opfs-unavailable: sqlite-wasm build has no installOpfsSAHPoolVfs');
+    }
+    return fallbackToMemory(sqlite3, dbName, 'sqlite-wasm build has no installOpfsSAHPoolVfs', 0);
+  }
+
+  const maxAttempts = Math.max(1, opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
+  const backoffMs = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
+  const sleep = opts.sleep ?? defaultSleep;
+
+  let lastError = 'unknown error';
+  let attempts = 0;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    attempts = attempt;
+    try {
+      // `initialCapacity` stays at the sqlite-wasm default (6 files): one pool
+      // per bucket holds a single DB plus its journals, so preallocating more
+      // OPFS files would only be waste. A "SAH pool is full" error still
+      // reaches the caller verbatim via `opfsError`.
+      const pool = await sqlite3.installOpfsSAHPoolVfs({
+        name: `sp00ky-${dbName}`,
+        // The first failure is cached against the VFS name, so a retry that
+        // doesn't ask for a real re-init just replays the same rejection.
+        ...(attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}),
+      });
+      return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true, pool };
+    } catch (e) {
+      lastError = errMessage(e);
+      if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
+      await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
+    }
+  }
+  if (opts.disallowMemoryFallback) {
+    throw new Error(`opfs-unavailable: ${lastError} (after ${attempts} attempts)`);
+  }
+  return fallbackToMemory(sqlite3, dbName, lastError, attempts);
+}

package/src/services/database/sqlite-plan-sql.test.ts

@@ -0,0 +1,164 @@
+import { describe, it, expect } from 'vitest';
+import { RecordId } from 'surrealdb';
+import type { WhereNode } from '@spooky-sync/query-builder';
+import {
+  comparisonSql,
+  renderWhereSql,
+  renderOrderSql,
+  scalar,
+  serializeRow,
+  serializeMergeRow,
+  reviveRow,
+  project,
+} from './sqlite-plan-sql';
+
+describe('comparisonSql param slaving (aa4af79b)', () => {
+  it('prefers params[paramRef] over the baked literal', () => {
+    const bind: unknown[] = [];
+    const sql = comparisonSql(
+      { field: 'id', op: '=', value: 'thread:A', paramRef: 'id' },
+      bind,
+      { id: 'thread:B' }
+    );
+    expect(sql).toBe('id = ?');
+    expect(bind).toEqual(['thread:B']);
+  });
+
+  it('falls back to the baked literal when the param key is absent', () => {
+    const bind: unknown[] = [];
+    comparisonSql({ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }, bind, {});
+    expect(bind).toEqual(['thread:A']);
+  });
+
+  it('a pure $-ref (no baked value) always reads the param', () => {
+    const bind: unknown[] = [];
+    comparisonSql({ field: 'owner', op: '=', value: undefined, paramRef: 'auth' }, bind, {
+      auth: 'user:1',
+    });
+    expect(bind).toEqual(['user:1']);
+  });
+
+  it('renders non-id fields via json_extract and honors swap', () => {
+    const bind: unknown[] = [];
+    const sql = comparisonSql({ field: 'votes', op: '<', value: 5, swap: true }, bind, {});
+    expect(sql).toBe(`? < json_extract(data, '$.votes')`);
+    expect(bind).toEqual([5]);
+  });
+});
+
+describe('renderWhereSql', () => {
+  it('joins top-level nodes with AND and parenthesizes OR groups', () => {
+    const nodes: WhereNode[] = [
+      { field: 'kind', op: '=', value: 'a' },
+      {
+        or: [
+          { field: 'votes', op: '>', value: 1 },
+          { field: 'votes', op: '=', value: 0 },
+        ],
+      },
+    ];
+    const bind: unknown[] = [];
+    const sql = renderWhereSql(nodes, bind, {});
+    expect(sql).toBe(
+      `json_extract(data, '$.kind') = ? AND (json_extract(data, '$.votes') > ? OR json_extract(data, '$.votes') = ?)`
+    );
+    expect(bind).toEqual(['a', 1, 0]);
+  });
+});
+
+describe('renderOrderSql', () => {
+  it('renders multi-key ordering', () => {
+    expect(renderOrderSql([['a', 'desc'], ['b', 'asc']])).toBe(
+      ` ORDER BY json_extract(data, '$.a') DESC, json_extract(data, '$.b') ASC`
+    );
+  });
+});
+
+describe('scalar', () => {
+  it('binds RecordId-shaped objects as table:id strings', () => {
+    expect(scalar({ tb: 'user', id: '1' })).toBe('user:1');
+    expect(scalar('plain')).toBe('plain');
+    expect(scalar(3)).toBe(3);
+    expect(scalar(null)).toBeNull();
+    expect(scalar(undefined)).toBeNull();
+  });
+});
+
+describe('serializeRow / reviveRow round-trip', () => {
+  it('tags Uint8Array as {__u8} and revives it', () => {
+    const json = serializeRow({ id: 's:1', blob: new Uint8Array([0, 128, 255]) });
+    expect(json).toContain('"__u8"');
+    const back = reviveRow(json);
+    expect(back.blob).toBeInstanceOf(Uint8Array);
+    expect([...(back.blob as Uint8Array)]).toEqual([0, 128, 255]);
+  });
+
+  // The surrealdb SDK decodes a `bytes` column as an ArrayBuffer, which
+  // JSON.stringify writes as `{}`: synced CRDT log rows (and presence states)
+  // landed with no data, so every other client's edit was refused as empty.
+  it('keeps the bytes of every binary shape: an SDK ArrayBuffer, a view into a larger buffer', () => {
+    const backing = new Uint8Array([9, 9, 1, 2, 250, 9]);
+    const json = serializeRow({
+      id: 'thread__crdt_log:1',
+      data: backing.slice(2, 5).buffer,
+      window: backing.subarray(2, 5),
+      dv: new DataView(backing.buffer, 2, 3),
+      nested: { state: [new Int8Array([1, 2, -6]).buffer] },
+    });
+    const back = reviveRow(json) as any;
+    for (const bytes of [back.data, back.window, back.dv, back.nested.state[0]]) {
+      expect(bytes).toBeInstanceOf(Uint8Array);
+      expect([...bytes]).toEqual([1, 2, 250]);
+    }
+  });
+
+  it('serializes RecordId-shaped links to strings and takes the fast parse path otherwise', () => {
+    const json = serializeRow({ id: 't:1', author: { tb: 'user', id: 'u1' }, n: 2 });
+    expect(reviveRow(json)).toEqual({ id: 't:1', author: 'user:u1', n: 2 });
+  });
+
+  it('stores an SDK RecordId link raw, not as its escaped toJSON', () => {
+    const json = serializeRow({ id: 'game:1', white: new RecordId('player_name', '3abc'), tags: [new RecordId('t', '9z')] });
+    expect(reviveRow(json)).toEqual({ id: 'game:1', white: 'player_name:3abc', tags: ['t:9z'] });
+  });
+});
+
+describe('project', () => {
+  it('keeps id plus the listed fields only, skipping absent ones', () => {
+    expect(project({ id: 'a', x: 1, y: 2 }, ['x', 'missing'])).toEqual({ id: 'a', x: 1 });
+  });
+});
+
+describe('serializeMergeRow', () => {
+  it('reuses one string for insert and patch when nothing is cleared', () => {
+    const row = { id: 't:1', a: 1, b: null };
+    const { insert, patch } = serializeMergeRow(row);
+    expect(insert).toBe(serializeRow(row));
+    expect(patch).toBe(insert);
+  });
+
+  // json_patch (RFC 7396) removes a member whose patch value is null; JSON drops
+  // an undefined one, which is how a cleared field used to survive the MERGE.
+  it('carries a top-level undefined as a null member in the patch only', () => {
+    const { insert, patch } = serializeMergeRow({ id: 't:1', a: 1, club: undefined, 'we"ird': undefined });
+    expect(JSON.parse(insert)).toStrictEqual({ id: 't:1', a: 1 });
+    expect(JSON.parse(patch)).toStrictEqual({ id: 't:1', a: 1, club: null, 'we"ird': null });
+  });
+
+  it('handles a row with nothing but cleared keys', () => {
+    const { insert, patch } = serializeMergeRow({ club: undefined });
+    expect(insert).toBe('{}');
+    expect(JSON.parse(patch)).toStrictEqual({ club: null });
+  });
+
+  it('leaves nested undefined dropped (only top-level keys clear)', () => {
+    const { patch } = serializeMergeRow({ id: 't:1', meta: { x: undefined, y: 1 } });
+    expect(JSON.parse(patch)).toStrictEqual({ id: 't:1', meta: { y: 1 } });
+  });
+
+  it('does not append to a root the replacer collapsed to a scalar', () => {
+    const { insert, patch } = serializeMergeRow({ tb: 'user', id: 'u1', gone: undefined });
+    expect(insert).toBe('"user:u1"');
+    expect(patch).toBe(insert);
+  });
+});

package/src/services/database/sqlite-plan-sql.ts

@@ -0,0 +1,196 @@
+import type { WhereComparison, WhereNode } from '@spooky-sync/query-builder';
+import { recordIdKey, stableKey } from './relation-resolver';
+import type { OrderBy, Row } from './cache-engine';
+
+/**
+ * SQL rendering + value (de)serialization for the SQLite cache backend.
+ * Extracted from `sqlite-cache-engine.ts` so BOTH sides of the worker boundary
+ * can use it: the engine (main thread) for the legacy/shim paths, and
+ * `sqlite-worker.ts` for worker-side plan execution (`select`). Pure module —
+ * no DOM, no worker, no engine imports — mirroring `plan-render.ts`.
+ */
+
+export function renderOrderSql(orderBy: OrderBy): string {
+  return ` ORDER BY ${orderBy
+    .map(([f, d]) => `json_extract(data, '$.${f}') ${d === 'desc' ? 'DESC' : 'ASC'}`)
+    .join(', ')}`;
+}
+
+export function comparisonSql(
+  c: WhereComparison,
+  bind: unknown[],
+  params: Record<string, unknown>
+): string {
+  const lhs = c.field === 'id' ? 'id' : `json_extract(data, '$.${c.field}')`;
+  // Prefer the query's own param so a filter materializes from `params` (the
+  // query's identity), not a baked literal. A pure `$`-ref has no `value`; a
+  // slave-mode node keeps `value` as a fallback for a param absent from params.
+  const useParam =
+    c.paramRef !== undefined &&
+    (c.value === undefined || Object.prototype.hasOwnProperty.call(params, c.paramRef));
+  const value = useParam ? params[c.paramRef!] : c.value;
+  bind.push(scalar(value));
+  const op = c.op === '!=' ? '!=' : c.op;
+  return c.swap ? `? ${op} ${lhs}` : `${lhs} ${op} ?`;
+}
+
+export function renderWhereSql(
+  nodes: WhereNode[],
+  bind: unknown[],
+  params: Record<string, unknown>
+): string {
+  return nodes
+    .map((node) => {
+      if ('or' in node) {
+        return `(${node.or.map((c) => comparisonSql(c, bind, params)).join(' OR ')})`;
+      }
+      return comparisonSql(node, bind, params);
+    })
+    .join(' AND ');
+}
+
+// ==================== value (de)serialization ====================
+
+/** A comparable scalar for SQL binding: record links → `table:id`, everything
+ *  else passed through (numbers/strings/bools). */
+export function scalar(value: unknown): unknown {
+  if (value == null) return null;
+  if (typeof value === 'object') return stableKey(value);
+  return value;
+}
+
+export function serializeRow(row: Row): string {
+  return JSON.stringify(row, function (this: Record<string, unknown>, k, v) {
+    // `v` has already been through `toJSON()`, and a RecordId's is escaped
+    // SurrealQL (`game:⟨476g…⟩`), so key the original value instead.
+    const raw = this[k];
+    if (raw && typeof raw === 'object') {
+      const bytes = bytesOf(raw);
+      if (bytes) return { __u8: toBase64(bytes) };
+      const key = recordIdKey(raw);
+      if (key !== null) return key;
+    }
+    return v;
+  });
+}
+
+/**
+ * The bytes of a binary value, whatever shape it arrived in. Rows from the
+ * server carry a `bytes` column as an `ArrayBuffer` (the surrealdb SDK's CBOR
+ * decode), not a `Uint8Array`, and `JSON.stringify` turns an `ArrayBuffer`
+ * into `{}`: every synced CRDT log row and presence state was stored empty.
+ */
+function bytesOf(value: object): Uint8Array | null {
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  if (ArrayBuffer.isView(value)) return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+  return null;
+}
+
+/**
+ * Serialize a MERGE row ONCE for both halves of the merge upsert: the row a
+ * fresh INSERT stores and the RFC 7396 delta `json_patch` applies on conflict.
+ *
+ * A top-level own key holding `undefined` means "clear this field", which is
+ * what SurrealDB's MERGE does with it (the SDK sends it as NONE). JSON drops
+ * such a key, so a patch built by {@link serializeRow} alone never mentioned
+ * it and the stored value survived every merge: a field cleared on the server
+ * stayed set locally. The patch therefore carries each one as a `null` member,
+ * which `json_patch` removes; the insert row simply lacks it, exactly as
+ * before. Appended to the already-serialized text, so the common case costs no
+ * second stringify. Nested `undefined` stays dropped (never a clear here).
+ */
+export function serializeMergeRow(row: Row): { insert: string; patch: string } {
+  const insert = serializeRow(row);
+  const cleared: string[] = [];
+  for (const key of Object.keys(row)) {
+    if (row[key] === undefined) cleared.push(`${JSON.stringify(key)}:null`);
+  }
+  // Not an object text (the replacer collapsed the root): nothing to append to.
+  if (cleared.length === 0 || insert[0] !== '{') return { insert, patch: insert };
+  const members = cleared.join(',');
+  const patch = insert === '{}' ? `{${members}}` : `${insert.slice(0, -1)},${members}}`;
+  return { insert, patch };
+}
+
+export function reviveRow(json: string): Row {
+  // Fast path: the per-key reviver is only needed to rebuild `Uint8Array`s from
+  // `{__u8}` tags. Most rows (e.g. game bodies) have none — a plain parse avoids
+  // invoking a JS callback for every key of every row on the read hot path.
+  if (json.indexOf('"__u8"') === -1) return JSON.parse(json);
+  return JSON.parse(json, (_k, v) => {
+    if (v && typeof v === 'object' && typeof (v as any).__u8 === 'string') {
+      return fromBase64((v as any).__u8);
+    }
+    return v;
+  });
+}
+
+/**
+ * SELECT-clause expression yielding the row's `data` narrowed to `fields` (plus
+ * `id`), so SQLite never returns — and neither side ever parses — the fields the
+ * caller did not ask for. On a game list that is every row's `pgn`, which was
+ * being read, parsed and thrown away 50 rows at a time for eight rendered fields.
+ *
+ * Byte-identical to running {@link project} over the fully-parsed row, which is
+ * what both paths did before, and verified so against real SQLite:
+ * `json_each` walks the keys the row ACTUALLY has, so an ABSENT key stays absent
+ * rather than becoming an explicit null — the difference `json_object`/
+ * `json_extract` would have introduced. A stored null, nested objects and
+ * arrays, and `{__u8}` blob tags all round-trip unchanged, because
+ * `json_group_object` understands `json_each`'s `value` column as JSON rather
+ * than as text. `COALESCE` covers a row sharing none of the requested keys,
+ * where the subquery yields NULL and `reviveRow` would throw.
+ *
+ * The one value `je.value` does NOT carry back is a TOP-LEVEL boolean. SQLite
+ * has no boolean type, so `json_each` hands `true`/`false` out as the integers
+ * 1/0 and `json_group_object` writes integers back. A projected row therefore
+ * read `published: 1` where the same row read whole read `published: true`,
+ * and every `=== true` in an app turned false the moment a screen painted from
+ * the local cache. (A boolean NESTED in an object or array survives: those come
+ * back as whole JSON documents, untouched.) `je.type` still knows what it was,
+ * so the two boolean types are rebuilt as JSON literals on the way out.
+ *
+ * Deliberately unaliased, so the emitted statement keeps the exact shape the
+ * callers already produce (`FROM "t" WHERE id IN (…)`).
+ *
+ * Binds one parameter per key, pushed onto `bind` — these land in the SELECT
+ * clause, so they must be bound BEFORE any WHERE parameters.
+ */
+export function projectedDataSql(fields: string[], bind: unknown[]): string {
+  const keys = ['id', ...fields];
+  for (const k of keys) bind.push(k);
+  const placeholders = keys.map(() => '?').join(', ');
+  // `json('true')` / `json('false')` return JSON-subtyped values, so
+  // `json_group_object` embeds them as the literals `true` / `false` rather
+  // than as quoted text.
+  const value =
+    `CASE je.type WHEN 'true' THEN json('true') ` +
+    `WHEN 'false' THEN json('false') ELSE je.value END`;
+  return (
+    `COALESCE((SELECT json_group_object(je.key, ${value}) ` +
+    `FROM json_each(data) je WHERE je.key IN (${placeholders})), '{}') AS data`
+  );
+}
+
+export function project(row: Row, fields: string[]): Row {
+  const out: Row = {};
+  for (const f of ['id', ...fields]) if (f in row) out[f] = row[f];
+  return out;
+}
+
+export function toBase64(bytes: Uint8Array): string {
+  let bin = '';
+  for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
+  return typeof btoa !== 'undefined' ? btoa(bin) : Buffer.from(bytes).toString('base64');
+}
+
+export function fromBase64(b64: string): Uint8Array {
+  if (typeof atob !== 'undefined') {
+    const bin = atob(b64);
+    const out = new Uint8Array(bin.length);
+    for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
+    return out;
+  }
+  return new Uint8Array(Buffer.from(b64, 'base64'));
+}