@spooky-sync/core 0.0.1-canary.20 → 0.0.1-canary.201

package/src/services/database/sqlite-select.ts

@@ -0,0 +1,121 @@
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import { resolveRelations, stableKey } from './relation-resolver';
+import { renderOrderSql, renderWhereSql, reviveRow, project } from './sqlite-plan-sql';
+import type { OrderBy, RelationFetch, Row, RowFetcher } from './cache-engine';
+
+/**
+ * Worker-side execution of a whole {@link QueryPlan} — table creation, base
+ * select (either the `ids` window or where/order/limit), projection, and the
+ * full `.related()` tree via the SHARED `resolveRelations` — against an
+ * injected DB handle. Runs inside `sqlite-worker.ts` so the engine pays ONE
+ * postMessage round-trip per select instead of one per table/relation level;
+ * extracted into its own module so the logic is unit-testable off-worker
+ * (parity with the engine's legacy multi-hop path).
+ *
+ * Semantics mirror `SqliteCacheEngine.selectLegacy` exactly: same SQL, same
+ * ordering rules, same projection, same resolver.
+ */
+
+/** The slice of the worker's DB surface `executeSelect` needs. */
+export interface SelectDb {
+  /** Run a row-returning statement; rows come back as `{ data: <json> }`. */
+  exec(sql: string, bind?: unknown[]): { data: string }[];
+  /** Run a statement for effect only (CREATE TABLE). */
+  run(sql: string, bind?: unknown[]): void;
+  /** Tables already CREATEd on this handle (caller owns the lifecycle). */
+  knownTables: Set<string>;
+}
+
+function ensureTable(db: SelectDb, table: string): void {
+  if (db.knownTables.has(table)) return;
+  db.run(`CREATE TABLE IF NOT EXISTS "${table}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`);
+  db.knownTables.add(table);
+}
+
+function execRows(db: SelectDb, sql: string, bind: unknown[]): Row[] {
+  return db.exec(sql, bind).map((r) => reviveRow(r.data));
+}
+
+/** Mirrors the engine's `selectByIds`: fetch by primary id, preserving `ids`
+ *  order unless an ORDER BY overrides it. */
+function selectByIds(
+  db: SelectDb,
+  table: string,
+  ids: unknown[],
+  opts?: { select?: string[]; orderBy?: OrderBy }
+): Row[] {
+  if (ids.length === 0) return [];
+  ensureTable(db, table);
+  const keys = ids.map(stableKey);
+  const placeholders = keys.map(() => '?').join(', ');
+  let sql = `SELECT data FROM "${table}" WHERE id IN (${placeholders})`;
+  if (opts?.orderBy && opts.orderBy.length > 0) sql += renderOrderSql(opts.orderBy);
+  let rows = execRows(db, sql, keys);
+  if (!opts?.orderBy || opts.orderBy.length === 0) {
+    const pos = new Map(keys.map((k, i) => [k, i]));
+    rows = rows.sort((a, b) => (pos.get(stableKey(a.id)) ?? 0) - (pos.get(stableKey(b.id)) ?? 0));
+  }
+  return opts?.select ? rows.map((r) => project(r, opts.select!)) : rows;
+}
+
+/** Mirrors the engine's `fetchRelation` SQL exactly. */
+function fetchRelation(db: SelectDb, req: RelationFetch): Row[] {
+  ensureTable(db, req.table);
+  const keys = req.keys.map(stableKey);
+  const placeholders = keys.map(() => '?').join(', ');
+  const bind: unknown[] = [...keys];
+  const lhs = req.matchField === 'id' ? 'id' : `json_extract(data, '$.${req.matchField}')`;
+  let sql = `SELECT data FROM "${req.table}" WHERE ${lhs} IN (${placeholders})`;
+  if (req.where && req.where.length > 0) {
+    sql += ` AND ${renderWhereSql(req.where, bind, {})}`;
+  }
+  if (req.orderBy && req.orderBy.length > 0) sql += renderOrderSql(req.orderBy);
+  const rows = execRows(db, sql, bind);
+  return req.select ? rows.map((r) => project(r, req.select!)) : rows;
+}
+
+export async function executeSelect(
+  plan: QueryPlan,
+  params: Record<string, unknown>,
+  db: SelectDb
+): Promise<{ rows: Row[]; relationFetches: number }> {
+  // Per-call fetch counter (the engine folds it into `__sqliteStats`).
+  const counter = { n: 0 };
+  const fetcher: RowFetcher = {
+    fetchRelation: (req) => {
+      counter.n++;
+      return Promise.resolve(fetchRelation(db, req));
+    },
+  };
+  // Window materialization: base rows are exactly `plan.ids`, ordered.
+  if (plan.ids) {
+    const rows = selectByIds(db, plan.table, plan.ids, {
+      select: plan.select,
+      orderBy: plan.orderBy,
+    });
+    await resolveRelations(rows, plan.relations, fetcher);
+    return { rows, relationFetches: counter.n };
+  }
+  ensureTable(db, plan.table);
+  const bind: unknown[] = [];
+  let sql = `SELECT data FROM "${plan.table}"`;
+  if (plan.where && plan.where.length > 0) {
+    sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
+  }
+  if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderSql(plan.orderBy);
+  // A query with no ORDER BY still has to render in SOME order, and "whatever
+  // SQLite hands back" is insertion order — which disagrees with the order the
+  // same query gets once it renders from server membership, and disagrees with
+  // SurrealDB, whose natural order is by id. That mismatch is visible: the
+  // first paint comes from this scan and the second from membership, so an
+  // unordered list visibly reshuffled about a second after load. Ordering by
+  // id here makes the two agree and makes the result stable across reloads.
+  else sql += ` ORDER BY id`;
+  if (plan.limit !== undefined) sql += ` LIMIT ${Number(plan.limit)}`;
+  if (plan.offset !== undefined) sql += ` OFFSET ${Number(plan.offset)}`;
+  const rows = execRows(db, sql, bind);
+  // Optional projection trimming to match `SELECT <fields>`.
+  const projected = plan.select ? rows.map((r) => project(r, plan.select!)) : rows;
+  await resolveRelations(projected, plan.relations, fetcher);
+  return { rows: projected, relationFetches: counter.n };
+}

package/src/services/database/sqlite-transport.fixture.ts

@@ -0,0 +1,30 @@
+/**
+ * Test-only helper: stubs `SqliteCacheEngine.createTransport` with a fake
+ * transport driven by a plain handler function, replacing the old pattern of
+ * faking a whole Worker plus the engine's pending-map wiring. Not exported
+ * from the package; imported only by *.test.ts files.
+ */
+import type { SqliteTransport } from './sqlite-transport';
+
+export type FakeTransportHandler = (type: string, payload: any) => unknown | Promise<unknown>;
+
+/** Replace the engine's transport factory. Each open spawns a fresh fake, like
+ *  the real factory spawns a fresh Worker. The handler returns the reply rest
+ *  (without id/ok/wt) or throws to produce an error reply. */
+export function stubTransport(engine: unknown, handler: FakeTransportHandler): void {
+  (engine as { createTransport: () => SqliteTransport }).createTransport = () => {
+    let closed = false;
+    return {
+      kind: 'worker',
+      get connected() {
+        return !closed;
+      },
+      call: <T>(type: string, payload?: unknown) =>
+        Promise.resolve().then(() => handler(type, payload)) as Promise<T>,
+      failAll() {},
+      close() {
+        closed = true;
+      },
+    } as SqliteTransport;
+  };
+}

package/src/services/database/sqlite-transport.ts

@@ -0,0 +1,221 @@
+/**
+ * The wire between `SqliteCacheEngine` and the SQLite worker, extracted so the
+ * engine can swap it at runtime without touching any SQL logic. Two shapes:
+ *
+ * - {@link WorkerSqliteTransport}: owns a dedicated Worker (solo mode, and the
+ *   leader tab in shared-tabs mode). Controls the worker's lifecycle and can
+ *   forward follower MessagePorts into it (`add-client`).
+ * - {@link PortSqliteTransport}: speaks the same request/response protocol over
+ *   a MessagePort handed out by the leader (follower tabs). No lifecycle
+ *   control; the port dying surfaces through `onPortDead`.
+ *
+ * Both keep the pending-map semantics the engine had inline before: requests
+ * keyed by a locally-minted numeric id, one resolve/reject per id, `failAll`
+ * rejects everything in flight (worker crash, port loss, role change).
+ */
+import type { Logger } from '../logger/index';
+
+export interface SqliteTransport {
+  readonly kind: 'worker' | 'port';
+  /** True while the transport can carry a request. */
+  readonly connected: boolean;
+  call<T = unknown>(type: string, payload?: unknown): Promise<T>;
+  /** Reject every pending request with `reason`. Safe to call repeatedly. */
+  failAll(reason: string): void;
+  /** failAll + release the underlying channel. Terminal. `err` overrides the
+   *  transport's own error shape — used for a deliberate teardown (role change)
+   *  so callers see a retryable "transport lost", not "the worker crashed". */
+  close(reason?: string, err?: Error): void;
+}
+
+/** Thrown into pending follower calls when the leader (or its port) goes away.
+ *  Callers treat it like a transient error: the op may be retried once a new
+ *  leader is attached; nothing was necessarily executed. */
+export class BrokerPortClosedError extends Error {
+  readonly retryable = true;
+  constructor(reason: string) {
+    super(`sqlite transport lost: ${reason}`);
+    this.name = 'BrokerPortClosedError';
+  }
+}
+
+interface Pending {
+  resolve: (v: any) => void;
+  reject: (e: unknown) => void;
+}
+
+/** Shared request/response bookkeeping over any postMessage-shaped channel. */
+abstract class BaseTransport implements SqliteTransport {
+  abstract readonly kind: 'worker' | 'port';
+  protected pending = new Map<number, Pending>();
+  protected seq = 0;
+  protected closed = false;
+
+  constructor(protected logger: Logger) {}
+
+  get connected(): boolean {
+    return !this.closed;
+  }
+
+  protected abstract post(msg: unknown, transfer?: Transferable[]): void;
+  protected abstract makeError(reason: string): Error;
+
+  protected handleMessage(data: any): void {
+    const { id, ok, error, ...rest } = data ?? {};
+    const p = this.pending.get(id);
+    if (!p) return;
+    this.pending.delete(id);
+    if (ok) p.resolve(rest);
+    else p.reject(new Error(error));
+  }
+
+  call<T = unknown>(type: string, payload?: unknown): Promise<T> {
+    if (this.closed) return Promise.reject(this.makeError('transport closed'));
+    const id = ++this.seq;
+    return new Promise<T>((resolve, reject) => {
+      this.pending.set(id, { resolve, reject });
+      try {
+        this.post({ id, type, payload });
+      } catch (e) {
+        this.pending.delete(id);
+        reject(e);
+      }
+    });
+  }
+
+  failAll(reason: string, err?: Error): void {
+    if (this.pending.size === 0) return;
+    const e = err ?? this.makeError(reason);
+    for (const [, p] of this.pending) p.reject(e);
+    this.pending.clear();
+  }
+
+  close(reason = 'closed', err?: Error): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.failAll(reason, err);
+  }
+}
+
+export class WorkerSqliteTransport extends BaseTransport {
+  readonly kind = 'worker' as const;
+  private worker: Worker;
+  /** Fired when the worker reports it fenced itself after a leadership steal
+   *  (see sqlite-worker.ts). The engine must treat the store as gone. */
+  onLockLost: ((reason: string) => void) | null = null;
+
+  constructor(logger: Logger) {
+    super(logger);
+    // Source references the `.ts` so the monorepo's src-bundling consumers
+    // (e.g. the example app, which aliases `@spooky-sync/core` to `src`)
+    // resolve it — Vite handles `.ts` workers. For the published package, the
+    // tsdown build rewrites this to `./sqlite-worker.js` (the top-level
+    // emitted entry; see tsdown.config.ts), which the flat `dist/index.js`
+    // resolves. The worker (+ `@sqlite.org/sqlite-wasm`) still loads lazily —
+    // only when `localEngine: 'sqlite'` is used.
+    this.worker = new Worker(new URL('./sqlite-worker.ts', import.meta.url), { type: 'module' });
+    this.worker.onmessage = (ev: MessageEvent) => {
+      // Unsolicited worker-initiated notification, not a reply.
+      if (ev.data?.type === 'lock-lost') {
+        this.logger.error(
+          { reason: ev.data.reason, Category: 'sp00ky-client::SqliteCacheEngine::worker' },
+          'SQLite worker fenced after leadership loss'
+        );
+        this.failAll('worker fenced');
+        this.onLockLost?.(String(ev.data.reason ?? 'lock-lost'));
+        return;
+      }
+      this.handleMessage(ev.data);
+    };
+    // Surface a worker crash (wasm abort / OOM) instead of leaving every
+    // pending call hung forever — reject them all with a clear error.
+    const crash = (msg: string) => {
+      this.logger.error(
+        { err: this.makeError(msg), Category: 'sp00ky-client::SqliteCacheEngine::worker' },
+        'Worker error'
+      );
+      this.failAll(msg);
+    };
+    this.worker.onerror = (e: ErrorEvent) => crash(e.message || 'onerror');
+    this.worker.onmessageerror = () => crash('messageerror');
+  }
+
+  protected post(msg: unknown, transfer?: Transferable[]): void {
+    if (transfer) this.worker.postMessage(msg, transfer);
+    else this.worker.postMessage(msg);
+  }
+
+  protected makeError(reason: string): Error {
+    return new Error(`SQLite worker crashed: ${reason}`);
+  }
+
+  /** Forward a follower's MessagePort into the worker as an extra client. */
+  addClientPort(clientId: string, port: MessagePort): Promise<void> {
+    if (this.closed) return Promise.reject(this.makeError('transport closed'));
+    const id = ++this.seq;
+    return new Promise<void>((resolve, reject) => {
+      this.pending.set(id, { resolve: () => resolve(), reject });
+      this.worker.postMessage({ id, type: 'add-client', payload: { clientId } }, [port]);
+    });
+  }
+
+  removeClientPort(clientId: string): Promise<void> {
+    return this.call('remove-client', { clientId }).then(() => undefined);
+  }
+
+  /** Ask the worker to close + pauseVfs + self-close (graceful pagehide). */
+  shutdown(): Promise<void> {
+    return this.call('shutdown').then(() => undefined);
+  }
+
+  close(reason = 'closed', err?: Error): void {
+    if (this.closed) return;
+    super.close(reason, err);
+    this.worker.terminate();
+  }
+}
+
+export class PortSqliteTransport extends BaseTransport {
+  readonly kind = 'port' as const;
+
+  constructor(
+    private port: MessagePort,
+    private onPortDead: (reason: string) => void,
+    logger: Logger
+  ) {
+    super(logger);
+    port.onmessage = (ev: MessageEvent) => this.handleMessage(ev.data);
+    port.onmessageerror = () => this.dead('messageerror');
+    port.start?.();
+  }
+
+  /** The broker/coordinator learned the leader is gone; the port itself has no
+   *  close event, so the coordinator calls this explicitly. */
+  markDead(reason: string): void {
+    this.dead(reason);
+  }
+
+  private dead(reason: string, err?: Error): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.failAll(reason, err);
+    try {
+      this.port.close();
+    } catch {
+      /* ignore */
+    }
+    this.onPortDead(reason);
+  }
+
+  protected post(msg: unknown): void {
+    this.port.postMessage(msg);
+  }
+
+  protected makeError(reason: string): Error {
+    return new BrokerPortClosedError(reason);
+  }
+
+  close(reason = 'closed', err?: Error): void {
+    this.dead(reason, err);
+  }
+}