@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

package/src/services/database/sqlite-devtools-queries.integration.test.ts

@@ -0,0 +1,143 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
+import { SqliteCacheEngine } from './sqlite-cache-engine';
+import { stubTransport } from './sqlite-transport.fixture';
+import { serializeRow } from './sqlite-plan-sql';
+import type { Row } from './cache-engine';
+
+/**
+ * Integration: the DevTools Database explorer's statements run end-to-end
+ * (translate → engine SQL → REAL in-memory SQLite, the same
+ * @sqlite.org/sqlite-wasm build the worker loads). The unit test asserts the
+ * translation; this asserts the SQL actually executes and returns the right
+ * rows — the paging window in particular is easy to render into invalid SQLite
+ * (there is no bare OFFSET).
+ */
+
+let db: any;
+
+function makeLogger(): any {
+  const noop = () => {};
+  const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  l.child = () => l;
+  return l;
+}
+
+/** An engine whose transport is a real SQLite database. */
+function realEngine(): SqliteCacheEngine {
+  const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
+  stubTransport(engine, (type, payload: any) => {
+    switch (type) {
+      case 'open':
+        for (const t of payload.systemTables ?? []) {
+          db.exec({
+            sql: `CREATE TABLE IF NOT EXISTS "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`,
+          });
+        }
+        return { persisted: true };
+      case 'exec':
+        return {
+          rows: db.exec({
+            sql: payload.sql,
+            bind: payload.bind,
+            rowMode: 'object',
+            returnValue: 'resultRows',
+          }),
+        };
+      case 'run':
+        db.exec({ sql: payload.sql, bind: payload.bind });
+        return {};
+      case 'batch':
+        for (const stmt of payload as { sql: string; bind?: unknown[] }[]) {
+          db.exec({ sql: stmt.sql, bind: stmt.bind });
+        }
+        return {};
+      default:
+        return {};
+    }
+  });
+  return engine;
+}
+
+function seed(table: string, rows: Row[]): void {
+  db.exec({
+    sql: `CREATE TABLE IF NOT EXISTS "${table}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`,
+  });
+  for (const row of rows) {
+    db.exec({
+      sql: `INSERT INTO "${table}"(id, data) VALUES(?, ?)`,
+      bind: [row.id, serializeRow(row)],
+    });
+  }
+}
+
+beforeAll(async () => {
+  const sqlite3: any = await sqlite3InitModule();
+  db = new sqlite3.oo1.DB(':memory:', 'c');
+  seed(
+    'game_insight',
+    Array.from({ length: 5 }, (_, i) => ({
+      id: `game_insight:${i}`,
+      n: i,
+      tag: i % 2 ? 'odd' : 'even',
+    }))
+  );
+});
+
+describe('DevTools explorer against real SQLite', () => {
+  it('pages a table with LIMIT/START', async () => {
+    const engine = realEngine();
+    await engine.connect('anon');
+
+    const [page1] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 2 START 0');
+    const [page2] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 2 START 2');
+    const [tail] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 20 START 4');
+
+    expect(page1.map((r) => r.id)).toEqual(['game_insight:0', 'game_insight:1']);
+    expect(page2.map((r) => r.id)).toEqual(['game_insight:2', 'game_insight:3']);
+    expect(tail.map((r) => r.id)).toEqual(['game_insight:4']);
+  });
+
+  it('counts rows, with and without a filter', async () => {
+    const engine = realEngine();
+    await engine.connect('anon');
+
+    const [all] = await engine.query<[{ count: number }[]]>(
+      'SELECT count() FROM game_insight GROUP ALL'
+    );
+    const [odd] = await engine.query<[{ count: number }[]]>(
+      "SELECT count() FROM game_insight WHERE tag = 'odd' GROUP ALL"
+    );
+
+    expect(all).toEqual([{ count: 5 }]);
+    expect(odd).toEqual([{ count: 2 }]);
+  });
+
+  it('lists tables via INFO FOR DB', async () => {
+    const engine = realEngine();
+    await engine.connect('anon');
+
+    const [info] = await engine.query<[{ tables: Record<string, string> }]>('INFO FOR DB');
+
+    expect(Object.keys(info.tables)).toContain('game_insight');
+    // Seeded system tables show up too — the panel's "internal" toggle filters
+    // them, the engine must not pre-filter.
+    expect(Object.keys(info.tables)).toContain('_00_query');
+    expect(Object.keys(info.tables).some((t) => t.startsWith('sqlite_'))).toBe(false);
+  });
+
+  it('edits and deletes a row by literal record id', async () => {
+    const engine = realEngine();
+    await engine.connect('anon');
+
+    await engine.query('UPDATE game_insight:1 MERGE $updates', { updates: { tag: 'edited' } });
+    const [edited] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 20 START 1');
+    expect(edited[0]).toMatchObject({ id: 'game_insight:1', n: 1, tag: 'edited' });
+
+    await engine.query('DELETE game_insight:1');
+    const [after] = await engine.query<[{ count: number }[]]>(
+      'SELECT count() FROM game_insight GROUP ALL'
+    );
+    expect(after).toEqual([{ count: 4 }]);
+  });
+});

package/src/services/database/sqlite-devtools-queries.test.ts

@@ -0,0 +1,154 @@
+import { describe, it, expect } from 'vitest';
+import { SqliteCacheEngine } from './sqlite-cache-engine';
+import { stubTransport } from './sqlite-transport.fixture';
+import { translateSurql } from './surql-translate';
+
+/**
+ * The statements the DevTools Database explorer emits against the LOCAL store.
+ * None of them come from `surql`, so they only ever exercised the SurrealDB
+ * engine: on SQLite the paging read threw "unsupported SurrealQL for
+ * translation: SELECT * FROM game_insight LIMIT 20 START 0", the table list came
+ * back empty (`INFO FOR DB` lowered to a noop) and row edit/delete never
+ * matched (they inline a literal record id).
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  l.child = () => l;
+  return l;
+}
+
+/** Records every exec/run SQL so the paging window can be asserted. */
+function recordingEngine(rows: Record<string, unknown[]> = {}) {
+  const sql: string[] = [];
+  const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
+  stubTransport(engine, (type, payload: any) => {
+    if (type === 'open') return { persisted: true };
+    if (type === 'exec' || type === 'run') {
+      sql.push(payload.sql);
+      for (const [needle, out] of Object.entries(rows)) {
+        if (payload.sql.includes(needle)) return { rows: out };
+      }
+      return { rows: [] };
+    }
+    return {};
+  });
+  return { engine, sql };
+}
+
+describe('DevTools table paging (LIMIT/START)', () => {
+  it('translates the window instead of throwing', () => {
+    const { ops } = translateSurql('SELECT * FROM game_insight LIMIT 20 START 0', {});
+    expect(ops).toEqual([
+      {
+        kind: 'selectTable',
+        table: 'game_insight',
+        where: undefined,
+        orderBy: undefined,
+        select: undefined,
+        value: undefined,
+        limit: 20,
+        start: 0,
+      },
+    ]);
+  });
+
+  it('renders LIMIT/OFFSET in SQL, page 3 of 20', async () => {
+    const { engine, sql } = recordingEngine();
+    await engine.connect('anon');
+    await engine.query('SELECT * FROM game_insight LIMIT 20 START 40');
+    expect(sql.some((s) => /SELECT data FROM "game_insight" LIMIT 20 OFFSET 40$/.test(s))).toBe(
+      true
+    );
+  });
+
+  it('keeps WHERE and ORDER BY alongside the window', () => {
+    const { ops } = translateSurql(
+      'SELECT * FROM game WHERE done = true ORDER BY date desc LIMIT 10 START 30',
+      {}
+    );
+    expect(ops[0]).toMatchObject({
+      kind: 'selectTable',
+      table: 'game',
+      where: [{ field: 'done', op: '=', value: true }],
+      orderBy: [['date', 'desc']],
+      limit: 10,
+      start: 30,
+    });
+  });
+
+  it('leaves a LIMIT that is part of a string literal alone', () => {
+    const { ops } = translateSurql("SELECT * FROM game WHERE note = 'LIMIT 5'", {});
+    expect(ops[0]).toMatchObject({
+      kind: 'selectTable',
+      where: [{ field: 'note', op: '=', value: 'LIMIT 5' }],
+      limit: undefined,
+      start: undefined,
+    });
+  });
+
+  it('a START with no LIMIT still renders valid SQLite', async () => {
+    const { engine, sql } = recordingEngine();
+    await engine.connect('anon');
+    await engine.query('SELECT * FROM game START 5');
+    // SQLite has no bare OFFSET — `LIMIT -1` is its "everything" sentinel.
+    expect(sql.some((s) => s.endsWith('LIMIT -1 OFFSET 5'))).toBe(true);
+  });
+});
+
+describe('DevTools row count', () => {
+  it("answers `SELECT count() FROM t GROUP ALL` in SurrealDB's shape", async () => {
+    const { engine } = recordingEngine({ 'COUNT(*)': [{ n: 137 }] });
+    await engine.connect('anon');
+    const res = await engine.query<[{ count: number }[]]>(
+      'SELECT count() FROM game_insight GROUP ALL'
+    );
+    expect(res[0]).toEqual([{ count: 137 }]);
+  });
+
+  it('counts zero rows as 0, not undefined', async () => {
+    const { engine } = recordingEngine();
+    await engine.connect('anon');
+    const res = await engine.query<[{ count: number }[]]>('SELECT count() FROM empty GROUP ALL');
+    expect(res[0]).toEqual([{ count: 0 }]);
+  });
+});
+
+describe('DevTools table list', () => {
+  it('answers INFO FOR DB from sqlite_master, minus SQLite internals', async () => {
+    const { engine } = recordingEngine({
+      sqlite_master: [{ name: 'game' }, { name: '_00_query' }],
+    });
+    await engine.connect('anon');
+    const [info] = await engine.query<[{ tables: Record<string, string> }]>('INFO FOR DB');
+    expect(Object.keys(info.tables)).toEqual(['game', '_00_query']);
+  });
+});
+
+describe('DevTools row edit / delete (literal record ids)', () => {
+  it('UPDATE <table>:<id> MERGE $updates writes that row', async () => {
+    const { engine, sql } = recordingEngine();
+    await engine.connect('anon');
+    await engine.query('UPDATE game:abc MERGE $updates', { updates: { white: 'hikaru' } });
+    const write = sql.find((s) => s.startsWith('INSERT INTO "game"'));
+    expect(write).toBeDefined();
+    expect(write).toContain('json_patch');
+  });
+
+  it('DELETE <table>:<id> deletes one row, DELETE <table> still clears the table', () => {
+    expect(translateSurql('DELETE game:abc', {}).ops[0]).toEqual({
+      kind: 'delete',
+      id: 'game:abc',
+    });
+    expect(translateSurql('DELETE game', {}).ops[0]).toEqual({ kind: 'deleteAll', table: 'game' });
+  });
+
+  it('refuses a table-wide MERGE rather than writing a row named after the table', () => {
+    // `UPDATE game MERGE $x` means every row of `game` in SurrealQL. Unsupported
+    // here — and it must SAY so, not silently create `game:undefined`.
+    expect(() => translateSurql('UPDATE game MERGE $x', { x: {} })).toThrow(
+      /unsupported SurrealQL/
+    );
+  });
+});

package/src/services/database/sqlite-lock-verify.test.ts

@@ -0,0 +1,33 @@
+import { describe, it, expect, vi } from 'vitest';
+import { verifyLockStillHeld } from './sqlite-lock-verify';
+
+describe('verifyLockStillHeld', () => {
+  it('fences when the lock is no longer held', async () => {
+    const fence = vi.fn(async () => {});
+    await verifyLockStillHeld({ query: async () => ({ held: [] }) }, 'lock-a', fence, 50);
+    expect(fence).toHaveBeenCalledWith('lock missing after suspected freeze');
+  });
+
+  it('does nothing when the lock is still held', async () => {
+    const fence = vi.fn(async () => {});
+    await verifyLockStillHeld({ query: async () => ({ held: [{ name: 'lock-a' }] }) }, 'lock-a', fence, 50);
+    expect(fence).not.toHaveBeenCalled();
+  });
+
+  it('resolves within the bound when the lock query never answers, without fencing', async () => {
+    const fence = vi.fn(async () => {});
+    const warn = vi.fn();
+    const started = Date.now();
+    await verifyLockStillHeld({ query: () => new Promise(() => {}) }, 'lock-a', fence, 30, warn);
+    expect(Date.now() - started).toBeLessThan(1_000);
+    expect(fence).not.toHaveBeenCalled();
+    expect(warn).toHaveBeenCalledTimes(1);
+  });
+
+  it('is a no-op without a lock or a query API', async () => {
+    const fence = vi.fn(async () => {});
+    await verifyLockStillHeld(null, 'x', fence);
+    await verifyLockStillHeld({ query: async () => ({}) }, undefined, fence);
+    expect(fence).not.toHaveBeenCalled();
+  });
+});

package/src/services/database/sqlite-lock-verify.ts

@@ -0,0 +1,45 @@
+/**
+ * "Do we still hold the OPFS lock?" - asked by the SQLite worker after a gap
+ * in its ticks long enough that the tab may have been frozen and the broker
+ * may have handed the lock to another tab.
+ *
+ * Bounded. `navigator.locks.query()` has no deadline of its own, and every op
+ * on the worker's chain awaits this verification: one query that never
+ * resolves stopped every reply to every client for good. On timeout we resolve
+ * WITHOUT fencing - the steal callback registered on the lock request remains
+ * the backstop that fences a genuinely stolen lock - and say so.
+ */
+export interface LockQueryLike {
+  query(): Promise<{ held?: { name?: string }[] }>;
+}
+
+export const LOCK_VERIFY_TIMEOUT_MS = 2_000;
+
+export async function verifyLockStillHeld(
+  locks: LockQueryLike | null | undefined,
+  name: string | undefined,
+  fence: (reason: string) => Promise<void> | void,
+  timeoutMs = LOCK_VERIFY_TIMEOUT_MS,
+  warn: (msg: string) => void = () => {}
+): Promise<void> {
+  if (!locks || !name || typeof locks.query !== 'function') return;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    const state = await Promise.race([
+      locks.query(),
+      new Promise<null>((resolve) => {
+        timer = setTimeout(() => resolve(null), timeoutMs);
+      }),
+    ]);
+    if (state === null) {
+      warn(`lock verification did not answer within ${timeoutMs}ms; trusting the steal callback`);
+      return;
+    }
+    const stillHeld = (state.held ?? []).some((l) => l.name === name);
+    if (!stillHeld) await fence('lock missing after suspected freeze');
+  } catch {
+    /* query unavailable: fall back to the steal callback */
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}

package/src/services/database/sqlite-open.test.ts

@@ -0,0 +1,150 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { openDb } from './sqlite-open';
+
+class FakeDb {
+  constructor(public arg: unknown) {}
+  exec() {
+    return [];
+  }
+  close() {}
+}
+
+/** Minimal stand-in for the initialized sqlite-wasm module. `install` is the
+ *  `installOpfsSAHPoolVfs` behavior under test; omit it to model a build that
+ *  lacks the SAHPool VFS entirely. */
+function makeSqlite3(install?: (opts: any) => Promise<unknown>) {
+  const sqlite3: any = { oo1: { DB: FakeDb } };
+  if (install) sqlite3.installOpfsSAHPoolVfs = vi.fn(install);
+  return sqlite3;
+}
+
+const pool = { OpfsSAHPoolDb: FakeDb };
+/** What a pool locked by another tab of the app actually throws. */
+function lockedError(): Error {
+  const e = new Error('Access Handles cannot be acquired');
+  e.name = 'NoModificationAllowedError';
+  return e;
+}
+
+const noSleep = () => Promise.resolve();
+
+let errSpy: ReturnType<typeof vi.spyOn>;
+beforeEach(() => {
+  errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+});
+afterEach(() => {
+  errSpy.mockRestore();
+});
+
+// The bare `catch {}` this replaces turned every OPFS failure into a silent
+// full-RAM database: no log, no reason, no way for the app to know its writes
+// die on reload. Each case below pins one half of the fix: keep trying when
+// retrying can plausibly work, and when it can't, say so loudly and hand the
+// reason back.
+describe('openDb', () => {
+  it('opens the OPFS pool on the first attempt', async () => {
+    const sqlite3 = makeSqlite3(async () => pool);
+    const res = await openDb(sqlite3, 'user:abc', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(true);
+    expect(res.opfsError).toBeUndefined();
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
+    // The pool is named per bucket, and the first try must NOT force a re-init
+    // (that would throw away a pool another attempt is legitimately using).
+    expect(sqlite3.installOpfsSAHPoolVfs.mock.calls[0][0]).toEqual({ name: 'sp00ky-user:abc' });
+    expect(errSpy).not.toHaveBeenCalled();
+  });
+
+  // The tab-closing race: the old tab still holds the sync access handles when
+  // the new one boots. Without a retry that tab is stuck in RAM for its whole
+  // lifetime, even though the lock frees milliseconds later.
+  it('retries a locked pool and succeeds, forcing re-init after the first failure', async () => {
+    let calls = 0;
+    const sqlite3 = makeSqlite3(async () => {
+      if (++calls < 3) throw lockedError();
+      return pool;
+    });
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(true);
+    expect(res.opfsError).toBeUndefined();
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
+    // sqlite-wasm caches the first rejection against the VFS name, so retries
+    // that don't ask for a real re-init just replay it.
+    const [first, second, third] = sqlite3.installOpfsSAHPoolVfs.mock.calls.map((c: any[]) => c[0]);
+    expect(first.forceReinitIfPreviouslyFailed).toBeUndefined();
+    expect(second.forceReinitIfPreviouslyFailed).toBe(true);
+    expect(third.forceReinitIfPreviouslyFailed).toBe(true);
+    expect(errSpy).not.toHaveBeenCalled();
+  });
+
+  it('falls back loudly after exhausting the retries, keeping the reason', async () => {
+    const sqlite3 = makeSqlite3(async () => {
+      throw lockedError();
+    });
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    // The DOMException name is the diagnostic part: it names the lock.
+    expect(res.opfsError).toContain('NoModificationAllowedError');
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
+    expect(errSpy).toHaveBeenCalledTimes(1);
+    expect(String(errSpy.mock.calls[0][0])).toContain('IN MEMORY');
+    // Still a usable handle: losing durability must not break the app.
+    expect(res.db).toBeDefined();
+  });
+
+  // An insecure context has no sync access handles at all, so retrying just
+  // adds boot latency to a foregone conclusion.
+  it('does not retry when the OPFS APIs are missing entirely', async () => {
+    const sqlite3 = makeSqlite3(async () => {
+      throw new Error('Missing required OPFS APIs.');
+    });
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    expect(res.opfsError).toContain('Missing required OPFS APIs');
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
+    expect(errSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('reports a build without the SAHPool VFS without calling anything', async () => {
+    const sqlite3 = makeSqlite3();
+    const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    expect(res.opfsError).toContain('installOpfsSAHPoolVfs');
+    expect(errSpy).toHaveBeenCalledTimes(1);
+  });
+
+  // `store: 'memory'` is a configuration choice, not a degradation, so it must
+  // stay silent and carry no error for the UI to warn about.
+  it('opens in memory quietly when persistence was not requested', async () => {
+    const sqlite3 = makeSqlite3(async () => pool);
+    const res = await openDb(sqlite3, 'main', false, { sleep: noSleep });
+
+    expect(res.persisted).toBe(false);
+    expect(res.opfsError).toBeUndefined();
+    expect(sqlite3.installOpfsSAHPoolVfs).not.toHaveBeenCalled();
+    expect(errSpy).not.toHaveBeenCalled();
+  });
+
+  it('honors maxAttempts and waits the configured backoff between tries', async () => {
+    const slept: number[] = [];
+    const sqlite3 = makeSqlite3(async () => {
+      throw lockedError();
+    });
+    const res = await openDb(sqlite3, 'main', true, {
+      maxAttempts: 4,
+      backoffMs: [10, 20],
+      sleep: async (ms) => {
+        slept.push(ms);
+      },
+    });
+
+    expect(res.persisted).toBe(false);
+    expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(4);
+    // One sleep per gap (never after the last attempt), last delay repeating.
+    expect(slept).toEqual([10, 20, 20]);
+  });
+});