@spooky-sync/core 0.0.1-canary.33 → 0.0.1-canary.331

package/src/services/database/database.ts

@@ -1,16 +1,31 @@
-import { Surreal, SurrealTransaction } from 'surrealdb';
-import { createLogger, Logger } from '../logger/index';
-import {
+import type { Surreal, SurrealTransaction } from 'surrealdb';
+import type { Logger } from '../logger/index';
+import type {
   DatabaseEventSystem,
-  DatabaseEventTypes,
-  DatabaseQueryEventPayload,
-} from './events/index';
-import { SealedQuery } from '../../utils/surql';
+  DatabaseEventTypes} from './events/index';
+import type { SealedQuery } from '../../utils/surql';
+import { withTimeout } from '../../utils/index';
 
 export abstract class AbstractDatabaseService {
   protected client: Surreal;
   protected logger: Logger;
   protected events: DatabaseEventSystem;
+  /**
+   * Per-query deadline in ms; `0` disables. The remote service sets it from
+   * `queryTimeoutMs` (see `RemoteDatabaseService`), the local one from
+   * `localOpTimeoutMs` (see `LocalDatabaseService`): a local query can be
+   * legitimately slow, but it must never be endless - every query waits on the
+   * previous link of {@link query}'s chain, and one that never settled wedged
+   * every later local op behind it.
+   */
+  protected queryTimeoutMs = 0;
+
+  /** The error a deadline expiry rejects with; the local service substitutes
+   *  its typed `LocalOpTimeoutError`. "timed out" in the message is
+   *  load-bearing either way: `classifySyncError` keys off it. */
+  protected timeoutError(_query: string): Error {
+    return new Error(`Remote query timed out after ${this.queryTimeoutMs}ms`);
+  }
   protected abstract eventType:
     | typeof DatabaseEventTypes.LocalQuery
     | typeof DatabaseEventTypes.RemoteQuery;
@@ -35,67 +50,147 @@ export abstract class AbstractDatabaseService {
     return this.client.beginTransaction();
   }
 
-  private queryQueue: Promise<void> = Promise.resolve();
+  /**
+   * How many statements may be in flight at once.
+   *
+   * `1` serializes every statement behind the previous one. The local engines
+   * need that: overlapping ops trip the WASM / SQLite transaction layer. The
+   * remote service raises it (`RemoteDatabaseService`): a SurrealDB socket
+   * multiplexes requests by id, and serializing them capped the client at one
+   * statement per round trip (~5/s at 170ms RTT), so the `_00_list_ref` poll
+   * alone could hold every registration behind it and one slow one-shot read
+   * stalled the whole client. The scheduler's `MAX_CONCURRENT_DOWN` only means
+   * something when this is above 1.
+   */
+  protected maxConcurrentQueries = 1;
+  private inFlightQueries = 0;
+  private queryWaiters: Array<() => void> = [];
+
+  /** FIFO slot acquisition, so `maxConcurrentQueries = 1` keeps strict order. */
+  private acquireQuerySlot(): Promise<void> {
+    if (this.inFlightQueries < this.maxConcurrentQueries) {
+      this.inFlightQueries++;
+      return Promise.resolve();
+    }
+    return new Promise<void>((resolve) => {
+      this.queryWaiters.push(() => {
+        this.inFlightQueries++;
+        resolve();
+      });
+    });
+  }
+
+  private releaseQuerySlot(): void {
+    this.inFlightQueries--;
+    const next = this.queryWaiters.shift();
+    if (next) next();
+  }
+
+  /**
+   * Resolve once every statement issued before this call has settled: take
+   * every slot (FIFO, so earlier statements go first), then hand them back.
+   * Bounded by {@link queryTimeoutMs}, like the statements it waits on.
+   */
+  protected async drainStatements(): Promise<void> {
+    for (let i = 0; i < this.maxConcurrentQueries; i++) await this.acquireQuerySlot();
+    for (let i = 0; i < this.maxConcurrentQueries; i++) this.releaseQuerySlot();
+  }
 
   /**
-   * Execute a query with serialized execution to prevent WASM transaction issues.
+   * Hook run right before a statement is sent, inside its slot. The remote
+   * service uses it to hold statements while a connect is in flight, so a
+   * query never runs on a socket that is open but has not had its namespace
+   * and token applied yet. The default does nothing.
+   */
+  protected async beforeQuery(): Promise<void> {}
+
+  /**
+   * Execute a statement under {@link maxConcurrentQueries}.
+   *
+   * A statement that never settles would hold its slot forever, and with a
+   * limit of 1 that blocks every later statement behind it. {@link
+   * queryTimeoutMs} bounds each one: on expiry this promise rejects and the
+   * slot is released, even though the underlying RPC is still parked in the
+   * SDK's pending map.
    */
   async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
-    return new Promise((resolve, reject) => {
-      this.queryQueue = this.queryQueue
-        .then(async () => {
-          const startTime = performance.now();
-          try {
-            this.logger.debug(
-              { query, vars, Category: 'sp00ky-client::Database::query' },
-              'Executing query'
-            );
-            const pending = this.client.query(query, vars);
-            // In SurrealDB 2.0, .query() collects results by default.
-            // We cast to T directly as proper typing depends on the caller knowing the return structure.
-            const result = (await pending) as unknown as T;
-            const duration = performance.now() - startTime;
-
-            // Emit query event
-            this.events.emit(this.eventType, {
-              query,
-              vars,
-              duration,
-              success: true,
-              timestamp: Date.now(),
-            });
-
-            resolve(result);
-            this.logger.trace(
-              { query, result, Category: 'sp00ky-client::Database::query' },
-              'Query executed successfully'
-            );
-          } catch (err) {
-            const duration = performance.now() - startTime;
-
-            // Emit query event with error
-            this.events.emit(this.eventType, {
-              query,
-              vars,
-              duration,
-              success: false,
-              error: err instanceof Error ? err.message : String(err),
-              timestamp: Date.now(),
-            });
-
-            this.logger.error(
-              { query, vars, err, Category: 'sp00ky-client::Database::query' },
-              'Query execution failed'
-            );
-            reject(err);
-          }
-        })
-        .catch(() => {
-          // Ignore queue errors to keep the chain alive; the specific promise was rejected above.
-        });
+    return this.runStatement<T>(query, vars, () => this.client.query(query, vars) as unknown as Promise<T>);
+  }
+
+  /**
+   * Like {@link query} but never rejects on a statement error: every
+   * statement answers on its own as `{ status: 'OK', result }` or
+   * `{ status: 'ERR', error }`. This is what lets a batched outbox push judge
+   * each mutation separately.
+   */
+  async queryResponses(
+    query: string,
+    vars?: Record<string, unknown>
+  ): Promise<Array<{ status: 'OK'; result: unknown } | { status: 'ERR'; error: string }>> {
+    return this.runStatement(query, vars, async () => {
+      const responses = await this.client.query(query, vars).responses();
+      return responses.map((r) =>
+        r.success
+          ? { status: 'OK' as const, result: r.result }
+          : { status: 'ERR' as const, error: r.error instanceof Error ? r.error.message : String(r.error) }
+      );
     });
   }
 
+  private async runStatement<T>(query: string, vars: Record<string, unknown> | undefined, dispatch: () => Promise<T>): Promise<T> {
+    await this.acquireQuerySlot();
+    try {
+      await this.beforeQuery();
+      const startTime = performance.now();
+      try {
+        // Trace: the finished statement is logged at debug by the client,
+        // from the query event below. This line is for a statement that hangs.
+        this.logger.trace(
+          { query, vars, Category: 'sp00ky-client::Database::query' },
+          'Executing query'
+        );
+        // "timed out" in the message is load-bearing: `classifySyncError`
+        // keys off it to classify this as `network` so the sync queues
+        // retry rather than rolling the mutation back.
+        const result = await withTimeout(dispatch(), this.queryTimeoutMs, () => this.timeoutError(query));
+        const duration = performance.now() - startTime;
+
+        this.events.emit(this.eventType, {
+          query,
+          vars,
+          duration,
+          success: true,
+          timestamp: Date.now(),
+        });
+
+        this.logger.trace(
+          { query, result, Category: 'sp00ky-client::Database::query' },
+          'Query executed successfully'
+        );
+        return result;
+      } catch (err) {
+        const duration = performance.now() - startTime;
+
+        this.events.emit(this.eventType, {
+          query,
+          vars,
+          duration,
+          success: false,
+          error: err instanceof Error ? err.message : String(err),
+          timestamp: Date.now(),
+        });
+
+        this.logger.error(
+          { query, vars, err, Category: 'sp00ky-client::Database::query' },
+          'Query execution failed'
+        );
+        throw err;
+      }
+    } finally {
+      this.releaseQuerySlot();
+    }
+  }
+
   async execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>): Promise<T> {
     const raw = await this.query<unknown[]>(query.sql, vars);
     return query.extract(raw);

package/src/services/database/engine-factory.ts

@@ -0,0 +1,33 @@
+import type { Sp00kyConfig } from '../../types';
+import type { Logger } from '../logger/index';
+import type { LocalEngineChoice, LocalStore } from './cache-engine';
+import { SurrealCacheEngine } from './surreal-cache-engine';
+import { SqliteCacheEngine } from './sqlite-cache-engine';
+
+/**
+ * Build the local cache engine for the given `localEngine` config choice.
+ *
+ * - `'surrealdb'` / unset → {@link SurrealCacheEngine} (subclass of
+ *   `LocalDatabaseService`; the historical behavior, verbatim).
+ * - `'sqlite'` → {@link SqliteCacheEngine} (SQLite-WASM Worker + OPFS). Backs
+ *   `this.local` through the SurrealQL-vocabulary shim + verb surface.
+ * - a custom object → used as-is (must satisfy {@link LocalStore}).
+ */
+export function createLocalEngine(
+  choice: LocalEngineChoice | undefined,
+  config: Sp00kyConfig<any>['database'],
+  logger: Logger,
+  opts: { shared?: boolean } = {}
+): LocalStore {
+  if (choice === undefined || choice === 'surrealdb') {
+    return new SurrealCacheEngine(config, logger);
+  }
+  if (choice === 'sqlite') {
+    // Mirror the SurrealDB engine's `store` semantics: 'memory' → in-memory
+    // SQLite (transient), 'indexeddb' → OPFS-backed (durable).
+    const useOpfs = (config.store ?? 'memory') !== 'memory';
+    return new SqliteCacheEngine(config, logger, { useOpfs, shared: opts.shared });
+  }
+  // Custom engine instance — trust it to satisfy LocalStore.
+  return choice as unknown as LocalStore;
+}

package/src/services/database/errors.ts

@@ -0,0 +1,34 @@
+/**
+ * A local-store operation that did not answer within its deadline.
+ *
+ * The local write path (`db.create` / `db.update` / `db.delete`, every local
+ * query behind them) used to have no deadline anywhere: the SQLite worker
+ * transport parks a call until the worker replies, the surrealdb engine's
+ * query chain waits on the previous link, and `withRetry` retries without a
+ * clock. One op that never settled (a worker starved behind a long select, a
+ * lock verification awaiting `navigator.locks.query()` forever) left the
+ * caller's promise pending for the tab's lifetime - a chat composer that never
+ * re-enabled, a call that never got past "Connecting".
+ *
+ * The message says "timed out" on purpose: `classifySyncError` keys off it and
+ * treats the failure as transient (re-queue), never as an application error
+ * that rolls the mutation back. `retryable: false` keeps `withRetry` from
+ * spinning on it: the op is still running in the engine, retrying queues a
+ * second copy behind it.
+ */
+export class LocalOpTimeoutError extends Error {
+  override readonly name = 'LocalOpTimeoutError';
+  readonly retryable = false;
+  readonly op: string;
+  readonly timeoutMs: number;
+
+  constructor(op: string, timeoutMs: number) {
+    super(`Local database operation timed out after ${timeoutMs}ms (${op})`);
+    this.op = op;
+    this.timeoutMs = timeoutMs;
+  }
+}
+
+/** Default deadline for one local-store operation. Generous: a cold 4k-row
+ *  select on a throttled tab is seconds, not tens of seconds. */
+export const DEFAULT_LOCAL_OP_TIMEOUT_MS = 30_000;

package/src/services/database/events/index.ts

@@ -1,4 +1,5 @@
-import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
+import type { EventDefinition, EventSystem } from '../../../events/index';
+import { createEventSystem } from '../../../events/index';
 
 export const DatabaseEventTypes = {
   LocalQuery: 'DATABASE_LOCAL_QUERY',

package/src/services/database/index.ts

@@ -1,5 +1,12 @@
 export * from './database';
 export * from './local';
 export * from './remote';
+export * from './connection-supervisor';
 export * from './local-migrator';
 export * from './events/index';
+export * from './cache-engine';
+export * from './surreal-cache-engine';
+export * from './sqlite-cache-engine';
+export * from './relation-resolver';
+export * from './plan-render';
+export { createLocalEngine } from './engine-factory';

package/src/services/database/landed-body-clear.integration.test.ts

@@ -0,0 +1,236 @@
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
+import { readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
+import { RecordId, Surreal } from 'surrealdb';
+import { landChunk } from '../../query/fetch.saga';
+import { defaultEnv } from '../../query/env';
+import { runPure } from '../../testing/run-pure';
+import { buildState } from '../../testing/build';
+import type { Vars } from '../../kernel/effects';
+import type { SealedQuery } from '../../utils/surql';
+import { SqliteCacheEngine } from './sqlite-cache-engine';
+import { stubTransport } from './sqlite-transport.fixture';
+import type { Row } from './cache-engine';
+
+/**
+ * Integration: a field cleared on the server (`UPDATE broadcast:x UNSET club`)
+ * is ABSENT from the body the server returns, and the body is landed with
+ * MERGE, which keeps every key it does not name. These land real bodies
+ * through `landChunk` and run the exact transaction it emits on BOTH local
+ * engines: a real in-memory SQLite (the worker's @sqlite.org/sqlite-wasm
+ * build) behind the SqliteCacheEngine, and a real SurrealDB-WASM `mem://`
+ * store provisioned with a typed schema, as the local migrator does.
+ *
+ * On each: the cleared optional field is gone, local `_00_*` bookkeeping
+ * survives (still a MERGE, not a REPLACE), and a required field the body
+ * lacks is kept rather than cleared.
+ */
+
+const env = defaultEnv({
+  tables: [
+    {
+      name: 'broadcast',
+      columns: {
+        title: { type: 'string', optional: false },
+        secret: { type: 'string', optional: false },
+        club: { type: 'string', recordId: true, optional: true },
+        note: { type: 'string', optional: true },
+      },
+    },
+  ],
+} as any);
+
+type Exec = (query: SealedQuery<unknown>, vars: Vars) => Promise<unknown>;
+
+/** Land `rows` at `version` through the real saga, executing its write with `exec`. */
+async function land(
+  exec: Exec,
+  rows: Array<Record<string, unknown>>,
+  version: number
+): Promise<void> {
+  const ids = rows.map((r) => String(r.id));
+  const out = await runPure(
+    landChunk(env, ids, rows, new Map(ids.map((id) => [id, version] as const)), buildState(), 0),
+    {
+      handlers: {
+        'local.execute': (e: any) => exec(e.query, e.vars),
+        'ssp.ingest': () => undefined,
+      },
+    }
+  );
+  expect(out.emitted).not.toContainEqual(expect.objectContaining({ message: 'body write failed' }));
+  expect(out.result).toBe(true);
+}
+
+const x = new RecordId('broadcast', 'x');
+const fresh = new RecordId('broadcast', 'fresh');
+const club = new RecordId('club', 'c1');
+
+const noop = () => {};
+
+// ==================== SQLite ====================
+
+describe('landed body clears an absent optional field: SQLite engine', () => {
+  let db: any;
+
+  function realEngine(): SqliteCacheEngine {
+    const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+    logger.child = () => logger;
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, logger);
+    stubTransport(engine, (type, payload: any) => {
+      switch (type) {
+        case 'open':
+          return { persisted: true };
+        case 'exec':
+          return {
+            rows: db.exec({
+              sql: payload.sql,
+              bind: payload.bind,
+              rowMode: 'object',
+              returnValue: 'resultRows',
+            }),
+          };
+        case 'run':
+          db.exec({ sql: payload.sql, bind: payload.bind });
+          return {};
+        case 'batch':
+          for (const stmt of payload as { sql: string; bind?: unknown[] }[])
+            db.exec({ sql: stmt.sql, bind: stmt.bind });
+          return {};
+        default:
+          return {};
+      }
+    });
+    return engine;
+  }
+
+  beforeAll(async () => {
+    const sqlite3: any = await sqlite3InitModule();
+    db = new sqlite3.oo1.DB(':memory:', 'c');
+  });
+
+  afterAll(() => db?.close());
+
+  it('removes the cleared key on the batched fast path, keeps _00_* and required fields', async () => {
+    const engine = realEngine();
+    await engine.connect('anon');
+    const exec: Exec = (q, v) => engine.execute(q, v);
+
+    await land(exec, [{ id: x, title: 't1', secret: 's', club, note: 'n' }], 1);
+    await engine.upsert('broadcast', x, { _00_local: 'keep' }, 'merge');
+    const before = (await engine.getById('broadcast', x)) as Row;
+    expect(before.club).toBeDefined();
+
+    // The server unset `club`; `secret` is hidden from this reader (required,
+    // so its absence is not a clear).
+    await land(exec, [{ id: x, title: 't2', note: 'n' }], 2);
+    const after = (await engine.getById('broadcast', x)) as Row;
+    expect('club' in after).toBe(false);
+    expect(after).toMatchObject({
+      id: 'broadcast:x',
+      title: 't2',
+      note: 'n',
+      secret: 's',
+      _00_local: 'keep',
+      _00_rv: 2,
+    });
+
+    // A fresh row stores no placeholder for the cleared key.
+    await land(exec, [{ id: fresh, title: 'f', secret: 's' }], 1);
+    const inserted = (await engine.getById('broadcast', fresh)) as Row;
+    expect(Object.keys(inserted).toSorted()).toEqual(['_00_rv', 'id', 'secret', 'title']);
+  });
+
+  it('removes the cleared key on the single-statement merge path too', async () => {
+    const engine = realEngine();
+    await engine.connect('anon');
+
+    const y = new RecordId('broadcast', 'y');
+    await engine.query('UPSERT ONLY $id MERGE $c', {
+      id: y,
+      c: { club, note: 'n2', _00_local: 'keep' },
+    });
+    expect((await engine.getById('broadcast', y))?.club).toBeDefined();
+    await engine.query('UPSERT ONLY $id MERGE $c', { id: y, c: { club: undefined, note: 'n3' } });
+    const row = (await engine.getById('broadcast', y)) as Row;
+    expect('club' in row).toBe(false);
+    expect(row.note).toBe('n3');
+    expect(row._00_local).toBe('keep');
+  });
+});
+
+// ==================== SurrealDB-WASM ====================
+
+describe('landed body clears an absent optional field: SurrealDB-WASM engine', () => {
+  let db: Surreal;
+
+  beforeAll(async () => {
+    // The wasm engine loads its binary with `fetch(new URL(..., import.meta.url))`,
+    // and Node's fetch has no `file:` scheme: serve that one read from disk.
+    const realFetch = globalThis.fetch;
+    vi.stubGlobal('fetch', async (input: unknown, init?: RequestInit) => {
+      const url = input instanceof URL ? input : new URL(String(input));
+      if (url.protocol === 'file:') return new Response(await readFile(fileURLToPath(url)));
+      return realFetch(input as RequestInfo, init);
+    });
+    const { createWasmEngines } = await import('@surrealdb/wasm');
+    db = new Surreal({ engines: createWasmEngines() });
+    await db.connect('mem://');
+    await db.use({ namespace: 'n', database: 'd' });
+    await db.query(`
+      DEFINE TABLE club SCHEMAFULL;
+      DEFINE TABLE broadcast SCHEMAFULL;
+      DEFINE FIELD title ON broadcast TYPE string;
+      DEFINE FIELD secret ON broadcast TYPE string;
+      DEFINE FIELD club ON broadcast TYPE option<record<club>>;
+      DEFINE FIELD note ON broadcast TYPE option<string>;
+      DEFINE FIELD _00_rv ON broadcast TYPE option<int>;
+      DEFINE FIELD _00_local ON broadcast TYPE option<string>;
+    `);
+  }, 60_000);
+
+  afterAll(async () => {
+    await db?.close();
+    vi.unstubAllGlobals();
+  });
+
+  const exec: Exec = (q, v) => db.query(q.sql, v) as unknown as Promise<unknown>;
+  const read = async (id: RecordId<string>): Promise<Record<string, unknown>> => {
+    const [row] = (await db.query('SELECT * FROM ONLY $id', { id })) as [Record<string, unknown>];
+    return row;
+  };
+
+  it('the undefined clear reaches SurrealDB as NONE and the MERGE removes the field', async () => {
+    await land(exec, [{ id: x, title: 't1', secret: 's', club, note: 'n' }], 1);
+    await db.query("UPDATE $id SET _00_local = 'keep'", { id: x });
+    expect(String((await read(x)).club)).toBe('club:c1');
+
+    // Would fail the whole transaction if the clear were NULL (a coercion error
+    // on `option<record<club>>`) or named the required `secret` as NONE.
+    await land(exec, [{ id: x, title: 't2', note: 'n' }], 2);
+    const after = await read(x);
+    expect('club' in after).toBe(false);
+    expect(after).toMatchObject({
+      title: 't2',
+      note: 'n',
+      secret: 's',
+      _00_local: 'keep',
+      _00_rv: 2,
+    });
+
+    await land(exec, [{ id: fresh, title: 'f', secret: 's' }], 1);
+    expect(Object.keys(await read(fresh)).toSorted()).toEqual(['_00_rv', 'id', 'secret', 'title']);
+  });
+
+  it('why the clear is NONE on optional columns only: NULL and a required NONE both fail', async () => {
+    const z = new RecordId('broadcast', 'z');
+    await db.query('CREATE $id CONTENT $c', { id: z, c: { title: 't', secret: 's', club } });
+    await expect(
+      db.query('UPSERT ONLY $id MERGE $c', { id: z, c: { club: null } })
+    ).rejects.toThrow(/club/);
+    await expect(
+      db.query('UPSERT ONLY $id MERGE $c', { id: z, c: { secret: undefined } })
+    ).rejects.toThrow(/secret/);
+  });
+});