@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210

package/src/services/database/sqlite-open.ts

@@ -0,0 +1,164 @@
+/**
+ * Opening the worker's SQLite handle: the OPFS SAHPool VFS when durable
+ * storage was asked for, an in-memory DB only as a last resort. Extracted from
+ * `sqlite-worker.ts` (which imports the wasm module at module scope and so
+ * can't be loaded in a unit test) to keep the retry/fallback policy testable
+ * off-worker, the same split as `sqlite-select.ts`.
+ *
+ * Why retry: SAHPool holds an EXCLUSIVE sync access handle on every file in
+ * its pool, so only one client per pool name can have it open. A second tab of
+ * the same app therefore fails init, and `installOpfsSAHPoolVfs` CACHES that
+ * rejection per VFS name, so a later call only gets a real second chance when
+ * it passes `forceReinitIfPreviouslyFailed`. Retrying with that flag turns the
+ * common "the other tab is still closing" race into a success instead of a
+ * permanent in-memory session.
+ *
+ * Why the noise: `:memory:` holds the whole dataset in RAM (the
+ * OOM-on-wasm-heavy-pages failure mode the OPFS store exists to avoid) and
+ * drops every local write on reload. Host apps run pino at their own level,
+ * some at `fatal`, so the fallback ALSO writes to `console.error` from inside
+ * the worker, and the reason travels back to the engine as `opfsError` for the
+ * app to surface.
+ */
+
+/** The DB surface the worker uses (a `sqlite3.oo1.DB` or an `OpfsSAHPoolDb`). */
+export interface SqliteDbHandle {
+  exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
+  close: () => void;
+}
+
+/** The slice of the OpfsSAHPoolUtil the worker needs for teardown. */
+export interface SqlitePoolHandle {
+  /** Unregisters the VFS and releases every sync access handle, leaving the
+   *  files intact, so another worker can open the pool without waiting for
+   *  this worker to be garbage collected. Throws while files are open. */
+  pauseVfs?: () => unknown;
+}
+
+export interface OpenDbResult {
+  db: SqliteDbHandle;
+  /** True only when the handle is backed by OPFS and survives a reload. */
+  persisted: boolean;
+  /** Why persistence failed. Set only when OPFS was requested and fell back. */
+  opfsError?: string;
+  /** Pool util, present only for an OPFS-backed handle. */
+  pool?: SqlitePoolHandle;
+}
+
+export interface OpenDbOptions {
+  /** Total OPFS init attempts, including the first. Default 3. */
+  maxAttempts?: number;
+  /** Delay before each retry; the last entry repeats. Default [250, 500]. */
+  backoffMs?: number[];
+  /**
+   * Throw (`opfs-unavailable: <reason>`) instead of falling back to memory.
+   * Used by shared-tabs leader promotion: a silently-in-memory LEADER would
+   * put every tab's data in RAM, so promotion prefers failing the election
+   * (the broker retries, possibly on another tab) over degrading. The broker
+   * grants an explicit memory fallback only after repeated failed cycles.
+   */
+  disallowMemoryFallback?: boolean;
+  /** Injectable for tests. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+const DEFAULT_MAX_ATTEMPTS = 3;
+/** Bounded on purpose: this runs on the boot path, before the first query. */
+const DEFAULT_BACKOFF_MS = [250, 500];
+
+/**
+ * Leader-promotion profile (~5.8s worst case). A dead leader's sync access
+ * handles release when the browser garbage-collects its worker, typically
+ * well under a second but not synchronously with the Web Lock release the
+ * election observed, so promotion retries longer than a cold boot.
+ */
+export const PROMOTION_OPEN_OPTIONS: Pick<OpenDbOptions, 'maxAttempts' | 'backoffMs'> = {
+  maxAttempts: 10,
+  backoffMs: [50, 100, 200, 400, 800, 1000],
+};
+
+/** Failures no retry can fix: the APIs aren't there at all (insecure context,
+ *  or a browser without sync access handles). Fall back immediately. */
+const UNRETRYABLE = ['Missing required OPFS APIs'];
+
+const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+/** Keep the DOMException name (e.g. `NoModificationAllowedError` for a pool
+ *  locked by another tab): it is the most diagnostic part of the failure. */
+function errMessage(e: unknown): string {
+  if (e instanceof Error) return e.name && e.name !== 'Error' ? `${e.name}: ${e.message}` : e.message;
+  return String(e);
+}
+
+function fallbackToMemory(
+  sqlite3: any,
+  dbName: string,
+  reason: string,
+  attempts: number
+): OpenDbResult {
+  const tried = attempts > 0 ? ` after ${attempts} attempt${attempts === 1 ? '' : 's'}` : '';
+  // Deliberately console, not the logger: host apps configure pino's level (some
+  // run `fatal`), and losing durability must never be filtered into silence.
+  // oxlint-disable-next-line no-console
+  console.error(
+    `[sp00ky] OPFS persistence unavailable for "${dbName}"${tried}: ${reason}. The local SQLite ` +
+      'cache is running IN MEMORY, which keeps the whole dataset in RAM and loses every local ' +
+      'write on reload. The usual cause is another tab of this app holding the storage lock, so ' +
+      'closing the other tabs and reloading restores persistence.'
+  );
+  return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false, opfsError: reason };
+}
+
+/**
+ * Open `dbName`'s handle. Never throws for a storage problem: a caller that
+ * asked for persistence and can't have it gets a working in-memory handle plus
+ * `persisted: false` and an `opfsError` to report.
+ */
+export async function openDb(
+  sqlite3: any,
+  dbName: string,
+  useOpfs: boolean,
+  opts: OpenDbOptions = {}
+): Promise<OpenDbResult> {
+  // Memory was the configured choice (`store: 'memory'`), not a failure, so no
+  // error and no noise.
+  if (!useOpfs) return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false };
+
+  if (!sqlite3.installOpfsSAHPoolVfs) {
+    if (opts.disallowMemoryFallback) {
+      throw new Error('opfs-unavailable: sqlite-wasm build has no installOpfsSAHPoolVfs');
+    }
+    return fallbackToMemory(sqlite3, dbName, 'sqlite-wasm build has no installOpfsSAHPoolVfs', 0);
+  }
+
+  const maxAttempts = Math.max(1, opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
+  const backoffMs = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
+  const sleep = opts.sleep ?? defaultSleep;
+
+  let lastError = 'unknown error';
+  let attempts = 0;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    attempts = attempt;
+    try {
+      // `initialCapacity` stays at the sqlite-wasm default (6 files): one pool
+      // per bucket holds a single DB plus its journals, so preallocating more
+      // OPFS files would only be waste. A "SAH pool is full" error still
+      // reaches the caller verbatim via `opfsError`.
+      const pool = await sqlite3.installOpfsSAHPoolVfs({
+        name: `sp00ky-${dbName}`,
+        // The first failure is cached against the VFS name, so a retry that
+        // doesn't ask for a real re-init just replays the same rejection.
+        ...(attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}),
+      });
+      return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true, pool };
+    } catch (e) {
+      lastError = errMessage(e);
+      if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
+      await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
+    }
+  }
+  if (opts.disallowMemoryFallback) {
+    throw new Error(`opfs-unavailable: ${lastError} (after ${attempts} attempts)`);
+  }
+  return fallbackToMemory(sqlite3, dbName, lastError, attempts);
+}

package/src/services/database/sqlite-plan-sql.test.ts

@@ -0,0 +1,104 @@
+import { describe, it, expect } from 'vitest';
+import type { WhereNode } from '@spooky-sync/query-builder';
+import {
+  comparisonSql,
+  renderWhereSql,
+  renderOrderSql,
+  scalar,
+  serializeRow,
+  reviveRow,
+  project,
+} from './sqlite-plan-sql';
+
+describe('comparisonSql param slaving (aa4af79b)', () => {
+  it('prefers params[paramRef] over the baked literal', () => {
+    const bind: unknown[] = [];
+    const sql = comparisonSql(
+      { field: 'id', op: '=', value: 'thread:A', paramRef: 'id' },
+      bind,
+      { id: 'thread:B' }
+    );
+    expect(sql).toBe('id = ?');
+    expect(bind).toEqual(['thread:B']);
+  });
+
+  it('falls back to the baked literal when the param key is absent', () => {
+    const bind: unknown[] = [];
+    comparisonSql({ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }, bind, {});
+    expect(bind).toEqual(['thread:A']);
+  });
+
+  it('a pure $-ref (no baked value) always reads the param', () => {
+    const bind: unknown[] = [];
+    comparisonSql({ field: 'owner', op: '=', value: undefined, paramRef: 'auth' }, bind, {
+      auth: 'user:1',
+    });
+    expect(bind).toEqual(['user:1']);
+  });
+
+  it('renders non-id fields via json_extract and honors swap', () => {
+    const bind: unknown[] = [];
+    const sql = comparisonSql({ field: 'votes', op: '<', value: 5, swap: true }, bind, {});
+    expect(sql).toBe(`? < json_extract(data, '$.votes')`);
+    expect(bind).toEqual([5]);
+  });
+});
+
+describe('renderWhereSql', () => {
+  it('joins top-level nodes with AND and parenthesizes OR groups', () => {
+    const nodes: WhereNode[] = [
+      { field: 'kind', op: '=', value: 'a' },
+      {
+        or: [
+          { field: 'votes', op: '>', value: 1 },
+          { field: 'votes', op: '=', value: 0 },
+        ],
+      },
+    ];
+    const bind: unknown[] = [];
+    const sql = renderWhereSql(nodes, bind, {});
+    expect(sql).toBe(
+      `json_extract(data, '$.kind') = ? AND (json_extract(data, '$.votes') > ? OR json_extract(data, '$.votes') = ?)`
+    );
+    expect(bind).toEqual(['a', 1, 0]);
+  });
+});
+
+describe('renderOrderSql', () => {
+  it('renders multi-key ordering', () => {
+    expect(renderOrderSql([['a', 'desc'], ['b', 'asc']])).toBe(
+      ` ORDER BY json_extract(data, '$.a') DESC, json_extract(data, '$.b') ASC`
+    );
+  });
+});
+
+describe('scalar', () => {
+  it('binds RecordId-shaped objects as table:id strings', () => {
+    expect(scalar({ tb: 'user', id: '1' })).toBe('user:1');
+    expect(scalar('plain')).toBe('plain');
+    expect(scalar(3)).toBe(3);
+    expect(scalar(null)).toBeNull();
+    expect(scalar(undefined)).toBeNull();
+  });
+});
+
+describe('serializeRow / reviveRow round-trip', () => {
+  it('tags Uint8Array as {__u8} and revives it', () => {
+    const json = serializeRow({ id: 's:1', blob: new Uint8Array([0, 128, 255]) });
+    expect(json).toContain('"__u8"');
+    const back = reviveRow(json);
+    expect(back.blob).toBeInstanceOf(Uint8Array);
+    expect([...(back.blob as Uint8Array)]).toEqual([0, 128, 255]);
+  });
+
+  it('serializes RecordId-shaped links to strings and takes the fast parse path otherwise', () => {
+    const json = serializeRow({ id: 't:1', author: { tb: 'user', id: 'u1' }, n: 2 });
+    expect(reviveRow(json)).toEqual({ id: 't:1', author: 'user:u1', n: 2 });
+  });
+});
+
+describe('project', () => {
+  it('keeps id plus the listed fields only, skipping absent ones', () => {
+    expect(project({ id: 'a', x: 1, y: 2 }, ['x', 'missing'])).toEqual({ id: 'a', x: 1 });
+  });
+});

package/src/services/database/sqlite-plan-sql.ts

@@ -0,0 +1,138 @@
+import type { WhereComparison, WhereNode } from '@spooky-sync/query-builder';
+import { stableKey } from './relation-resolver';
+import type { OrderBy, Row } from './cache-engine';
+
+/**
+ * SQL rendering + value (de)serialization for the SQLite cache backend.
+ * Extracted from `sqlite-cache-engine.ts` so BOTH sides of the worker boundary
+ * can use it: the engine (main thread) for the legacy/shim paths, and
+ * `sqlite-worker.ts` for worker-side plan execution (`select`). Pure module —
+ * no DOM, no worker, no engine imports — mirroring `plan-render.ts`.
+ */
+
+export function renderOrderSql(orderBy: OrderBy): string {
+  return ` ORDER BY ${orderBy
+    .map(([f, d]) => `json_extract(data, '$.${f}') ${d === 'desc' ? 'DESC' : 'ASC'}`)
+    .join(', ')}`;
+}
+
+export function comparisonSql(
+  c: WhereComparison,
+  bind: unknown[],
+  params: Record<string, unknown>
+): string {
+  const lhs = c.field === 'id' ? 'id' : `json_extract(data, '$.${c.field}')`;
+  // Prefer the query's own param so a filter materializes from `params` (the
+  // query's identity), not a baked literal. A pure `$`-ref has no `value`; a
+  // slave-mode node keeps `value` as a fallback for a param absent from params.
+  const useParam =
+    c.paramRef !== undefined &&
+    (c.value === undefined || Object.prototype.hasOwnProperty.call(params, c.paramRef));
+  const value = useParam ? params[c.paramRef!] : c.value;
+  bind.push(scalar(value));
+  const op = c.op === '!=' ? '!=' : c.op;
+  return c.swap ? `? ${op} ${lhs}` : `${lhs} ${op} ?`;
+}
+
+export function renderWhereSql(
+  nodes: WhereNode[],
+  bind: unknown[],
+  params: Record<string, unknown>
+): string {
+  return nodes
+    .map((node) => {
+      if ('or' in node) {
+        return `(${node.or.map((c) => comparisonSql(c, bind, params)).join(' OR ')})`;
+      }
+      return comparisonSql(node, bind, params);
+    })
+    .join(' AND ');
+}
+
+// ==================== value (de)serialization ====================
+
+/** A comparable scalar for SQL binding: record links → `table:id`, everything
+ *  else passed through (numbers/strings/bools). */
+export function scalar(value: unknown): unknown {
+  if (value == null) return null;
+  if (typeof value === 'object') return stableKey(value);
+  return value;
+}
+
+export function serializeRow(row: Row): string {
+  return JSON.stringify(row, (_k, v) => {
+    if (v instanceof Uint8Array) return { __u8: toBase64(v) };
+    if (v && typeof v === 'object') {
+      const rid = v as { tb?: unknown; id?: unknown };
+      if (rid.tb !== undefined && rid.id !== undefined) return stableKey(v);
+    }
+    return v;
+  });
+}
+
+export function reviveRow(json: string): Row {
+  // Fast path: the per-key reviver is only needed to rebuild `Uint8Array`s from
+  // `{__u8}` tags. Most rows (e.g. game bodies) have none — a plain parse avoids
+  // invoking a JS callback for every key of every row on the read hot path.
+  if (json.indexOf('"__u8"') === -1) return JSON.parse(json);
+  return JSON.parse(json, (_k, v) => {
+    if (v && typeof v === 'object' && typeof (v as any).__u8 === 'string') {
+      return fromBase64((v as any).__u8);
+    }
+    return v;
+  });
+}
+
+/**
+ * SELECT-clause expression yielding the row's `data` narrowed to `fields` (plus
+ * `id`), so SQLite never returns — and neither side ever parses — the fields the
+ * caller did not ask for. On a game list that is every row's `pgn`, which was
+ * being read, parsed and thrown away 50 rows at a time for eight rendered fields.
+ *
+ * Byte-identical to running {@link project} over the fully-parsed row, which is
+ * what both paths did before, and verified so against real SQLite:
+ * `json_each` walks the keys the row ACTUALLY has, so an ABSENT key stays absent
+ * rather than becoming an explicit null — the difference `json_object`/
+ * `json_extract` would have introduced. A stored null, nested objects and
+ * arrays, and `{__u8}` blob tags all round-trip unchanged, because
+ * `json_group_object` understands `json_each`'s `value` column as JSON rather
+ * than as text. `COALESCE` covers a row sharing none of the requested keys,
+ * where the subquery yields NULL and `reviveRow` would throw.
+ *
+ * Deliberately unaliased, so the emitted statement keeps the exact shape the
+ * callers already produce (`FROM "t" WHERE id IN (…)`).
+ *
+ * Binds one parameter per key, pushed onto `bind` — these land in the SELECT
+ * clause, so they must be bound BEFORE any WHERE parameters.
+ */
+export function projectedDataSql(fields: string[], bind: unknown[]): string {
+  const keys = ['id', ...fields];
+  for (const k of keys) bind.push(k);
+  const placeholders = keys.map(() => '?').join(', ');
+  return (
+    `COALESCE((SELECT json_group_object(je.key, je.value) ` +
+    `FROM json_each(data) je WHERE je.key IN (${placeholders})), '{}') AS data`
+  );
+}
+
+export function project(row: Row, fields: string[]): Row {
+  const out: Row = {};
+  for (const f of ['id', ...fields]) if (f in row) out[f] = row[f];
+  return out;
+}
+
+export function toBase64(bytes: Uint8Array): string {
+  let bin = '';
+  for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
+  return typeof btoa !== 'undefined' ? btoa(bin) : Buffer.from(bytes).toString('base64');
+}
+
+export function fromBase64(b64: string): Uint8Array {
+  if (typeof atob !== 'undefined') {
+    const bin = atob(b64);
+    const out = new Uint8Array(bin.length);
+    for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
+    return out;
+  }
+  return new Uint8Array(Buffer.from(b64, 'base64'));
+}

package/src/services/database/sqlite-projection.test.ts

@@ -0,0 +1,99 @@
+import { describe, it, expect } from 'vitest';
+import { DatabaseSync } from 'node:sqlite';
+import { projectedDataSql, project, reviveRow, serializeRow } from './sqlite-plan-sql';
+import type { Row } from './cache-engine';
+
+/**
+ * `projectedDataSql` is the one change in this area that alters what SQLite
+ * actually returns, and every row read goes through it — so it is verified
+ * against a REAL SQLite, not the scripted responder the parity tests use.
+ *
+ * The contract: byte-identical to parsing the whole row and running `project`
+ * over it, which is what both engine paths did before. The traps it has to
+ * clear are an ABSENT key (must stay absent, not become an explicit null — the
+ * difference `json_object`/`json_extract` would have introduced), a STORED
+ * null, nested objects/arrays, and the `{__u8}` tag that carries binary.
+ */
+
+function makeDb(rows: Row[]) {
+  const db = new DatabaseSync(':memory:');
+  db.exec('CREATE TABLE "game" (id TEXT PRIMARY KEY, data TEXT NOT NULL)');
+  const ins = db.prepare('INSERT INTO "game" (id,data) VALUES (?,?)');
+  for (const r of rows) ins.run(String(r.id), serializeRow(r));
+  return db;
+}
+
+/** What the code did before: read the whole row, parse it, then project. */
+function referenceRows(db: DatabaseSync, ids: string[], fields: string[]): Row[] {
+  const ph = ids.map(() => '?').join(', ');
+  return db
+    .prepare(`SELECT data FROM "game" WHERE id IN (${ph}) ORDER BY id ASC`)
+    .all(...ids)
+    .map((r) => project(reviveRow((r as { data: string }).data), fields));
+}
+
+/** What the code does now: let SQLite narrow the row before it is ever parsed. */
+function projectedRows(db: DatabaseSync, ids: string[], fields: string[]): Row[] {
+  const bind: unknown[] = [];
+  const col = projectedDataSql(fields, bind);
+  bind.push(...ids);
+  const ph = ids.map(() => '?').join(', ');
+  return db
+    .prepare(`SELECT ${col} FROM "game" WHERE id IN (${ph}) ORDER BY id ASC`)
+    .all(...(bind as string[]))
+    .map((r) => reviveRow((r as { data: string }).data));
+}
+
+const FIXTURE: Row[] = [
+  { id: 'game:1', white: 'player_name:PN_a', pgn: '[Event "x"]\n1. e4 e5', sort_index: -5 },
+  { id: 'game:2', white: null, pgn: 'heavy', sort_index: -3 }, // STORED null
+  { id: 'game:3', pgn: 'heavy', sort_index: -4 }, // 'white' ABSENT
+  { id: 'game:4', white: 'w', pgn: 'heavy', sort_index: -1, meta: { a: 1, b: [2, 3] } },
+];
+const IDS = ['game:1', 'game:2', 'game:3', 'game:4'];
+const FIELDS = ['white', 'sort_index', 'meta'];
+
+describe('projectedDataSql against real SQLite', () => {
+  it('is byte-identical to parse-then-project', () => {
+    const db = makeDb(FIXTURE);
+    expect(projectedRows(db, IDS, FIELDS)).toEqual(referenceRows(db, IDS, FIELDS));
+  });
+
+  it('never returns the fields that were not asked for', () => {
+    // The point of the change: `pgn` must not leave SQLite, so it is never
+    // parsed, never crosses the worker boundary, never reaches the store.
+    const db = makeDb(FIXTURE);
+    for (const row of projectedRows(db, IDS, FIELDS)) expect('pgn' in row).toBe(false);
+  });
+
+  it('keeps an absent key absent and a stored null null', () => {
+    const db = makeDb(FIXTURE);
+    const byId = new Map(projectedRows(db, IDS, FIELDS).map((r) => [r.id, r]));
+    expect('white' in byId.get('game:3')!).toBe(false); // absent stays absent
+    expect('white' in byId.get('game:2')!).toBe(true); // stored null is a real key
+    expect(byId.get('game:2')!.white).toBeNull();
+  });
+
+  it('round-trips nested objects and arrays as JSON, not as strings', () => {
+    const db = makeDb(FIXTURE);
+    const row = projectedRows(db, IDS, FIELDS).find((r) => r.id === 'game:4')!;
+    expect(row.meta).toEqual({ a: 1, b: [2, 3] });
+  });
+
+  it('round-trips binary through the {__u8} tag', () => {
+    const db = makeDb([{ id: 'game:9', blob: new Uint8Array([0, 1, 2]), pgn: 'heavy' }]);
+    const [row] = projectedRows(db, ['game:9'], ['blob']);
+    expect(row!.blob).toEqual(new Uint8Array([0, 1, 2]));
+  });
+
+  it('always includes id, even when it is not in the field list', () => {
+    const db = makeDb(FIXTURE);
+    for (const row of projectedRows(db, IDS, ['sort_index'])) expect(row.id).toBeTruthy();
+  });
+
+  it('yields an empty object, not NULL, for a row sharing none of the keys', () => {
+    // Without COALESCE the subquery returns NULL here and reviveRow throws.
+    const db = makeDb([{ id: 'game:8', pgn: 'heavy' }]);
+    expect(projectedRows(db, ['game:8'], ['nothing_matches'])).toEqual([{ id: 'game:8' }]);
+  });
+});