@spooky-sync/core 0.0.1-canary.25 → 0.0.1-canary.255

package/src/services/database/database.ts

@@ -1,16 +1,31 @@
-import { Surreal, SurrealTransaction } from 'surrealdb';
-import { createLogger, Logger } from '../logger/index';
-import {
+import type { Surreal, SurrealTransaction } from 'surrealdb';
+import type { Logger } from '../logger/index';
+import type {
   DatabaseEventSystem,
-  DatabaseEventTypes,
-  DatabaseQueryEventPayload,
-} from './events/index';
-import { SealedQuery } from '../../utils/surql';
+  DatabaseEventTypes} from './events/index';
+import type { SealedQuery } from '../../utils/surql';
+import { withTimeout } from '../../utils/index';
 
 export abstract class AbstractDatabaseService {
   protected client: Surreal;
   protected logger: Logger;
   protected events: DatabaseEventSystem;
+  /**
+   * Per-query deadline in ms; `0` disables. The remote service sets it from
+   * `queryTimeoutMs` (see `RemoteDatabaseService`), the local one from
+   * `localOpTimeoutMs` (see `LocalDatabaseService`): a local query can be
+   * legitimately slow, but it must never be endless - every query waits on the
+   * previous link of {@link query}'s chain, and one that never settled wedged
+   * every later local op behind it.
+   */
+  protected queryTimeoutMs = 0;
+
+  /** The error a deadline expiry rejects with; the local service substitutes
+   *  its typed `LocalOpTimeoutError`. "timed out" in the message is
+   *  load-bearing either way: `classifySyncError` keys off it. */
+  protected timeoutError(_query: string): Error {
+    return new Error(`Remote query timed out after ${this.queryTimeoutMs}ms`);
+  }
   protected abstract eventType:
     | typeof DatabaseEventTypes.LocalQuery
     | typeof DatabaseEventTypes.RemoteQuery;
@@ -35,74 +50,142 @@ export abstract class AbstractDatabaseService {
     return this.client.beginTransaction();
   }
 
-  private queryQueue: Promise<void> = Promise.resolve();
+  /**
+   * How many statements may be in flight at once.
+   *
+   * `1` serializes every statement behind the previous one. The local engines
+   * need that: overlapping ops trip the WASM / SQLite transaction layer. The
+   * remote service raises it (`RemoteDatabaseService`): a SurrealDB socket
+   * multiplexes requests by id, and serializing them capped the client at one
+   * statement per round trip (~5/s at 170ms RTT), so the `_00_list_ref` poll
+   * alone could hold every registration behind it and one slow one-shot read
+   * stalled the whole client. The scheduler's `MAX_CONCURRENT_DOWN` only means
+   * something when this is above 1.
+   */
+  protected maxConcurrentQueries = 1;
+  private inFlightQueries = 0;
+  private queryWaiters: Array<() => void> = [];
+
+  /** FIFO slot acquisition, so `maxConcurrentQueries = 1` keeps strict order. */
+  private acquireQuerySlot(): Promise<void> {
+    if (this.inFlightQueries < this.maxConcurrentQueries) {
+      this.inFlightQueries++;
+      return Promise.resolve();
+    }
+    return new Promise<void>((resolve) => {
+      this.queryWaiters.push(() => {
+        this.inFlightQueries++;
+        resolve();
+      });
+    });
+  }
+
+  private releaseQuerySlot(): void {
+    this.inFlightQueries--;
+    const next = this.queryWaiters.shift();
+    if (next) next();
+  }
+
+  /**
+   * Hook run right before a statement is sent, inside its slot. The remote
+   * service uses it to hold statements while a connect is in flight, so a
+   * query never runs on a socket that is open but has not had its namespace
+   * and token applied yet. The default does nothing.
+   */
+  protected async beforeQuery(): Promise<void> {}
 
   /**
-   * Execute a query with serialized execution to prevent WASM transaction issues.
+   * Execute a statement under {@link maxConcurrentQueries}.
+   *
+   * A statement that never settles would hold its slot forever, and with a
+   * limit of 1 that blocks every later statement behind it. {@link
+   * queryTimeoutMs} bounds each one: on expiry this promise rejects and the
+   * slot is released, even though the underlying RPC is still parked in the
+   * SDK's pending map.
    */
   async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
-    return new Promise((resolve, reject) => {
-      this.queryQueue = this.queryQueue
-        .then(async () => {
-          const startTime = performance.now();
-          try {
-            this.logger.debug(
-              { query, vars, Category: 'spooky-client::Database::query' },
-              'Executing query'
-            );
-            const pending = this.client.query(query, vars);
-            // In SurrealDB 2.0, .query() collects results by default.
-            // We cast to T directly as proper typing depends on the caller knowing the return structure.
-            const result = (await pending) as unknown as T;
-            const duration = performance.now() - startTime;
-
-            // Emit query event
-            this.events.emit(this.eventType, {
-              query,
-              vars,
-              duration,
-              success: true,
-              timestamp: Date.now(),
-            });
-
-            resolve(result);
-            this.logger.trace(
-              { query, result, Category: 'spooky-client::Database::query' },
-              'Query executed successfully'
-            );
-          } catch (err) {
-            const duration = performance.now() - startTime;
-
-            // Emit query event with error
-            this.events.emit(this.eventType, {
-              query,
-              vars,
-              duration,
-              success: false,
-              error: err instanceof Error ? err.message : String(err),
-              timestamp: Date.now(),
-            });
-
-            this.logger.error(
-              { query, vars, err, Category: 'spooky-client::Database::query' },
-              'Query execution failed'
-            );
-            reject(err);
-          }
-        })
-        .catch(() => {
-          // Ignore queue errors to keep the chain alive; the specific promise was rejected above.
-        });
+    return this.runStatement<T>(query, vars, () => this.client.query(query, vars) as unknown as Promise<T>);
+  }
+
+  /**
+   * Like {@link query} but never rejects on a statement error: every
+   * statement answers on its own as `{ status: 'OK', result }` or
+   * `{ status: 'ERR', error }`. This is what lets a batched outbox push judge
+   * each mutation separately.
+   */
+  async queryResponses(
+    query: string,
+    vars?: Record<string, unknown>
+  ): Promise<Array<{ status: 'OK'; result: unknown } | { status: 'ERR'; error: string }>> {
+    return this.runStatement(query, vars, async () => {
+      const responses = await this.client.query(query, vars).responses();
+      return responses.map((r) =>
+        r.success
+          ? { status: 'OK' as const, result: r.result }
+          : { status: 'ERR' as const, error: r.error instanceof Error ? r.error.message : String(r.error) }
+      );
     });
   }
 
+  private async runStatement<T>(query: string, vars: Record<string, unknown> | undefined, dispatch: () => Promise<T>): Promise<T> {
+    await this.acquireQuerySlot();
+    try {
+      await this.beforeQuery();
+      const startTime = performance.now();
+      try {
+        this.logger.debug(
+          { query, vars, Category: 'sp00ky-client::Database::query' },
+          'Executing query'
+        );
+        // "timed out" in the message is load-bearing: `classifySyncError`
+        // keys off it to classify this as `network` so the sync queues
+        // retry rather than rolling the mutation back.
+        const result = await withTimeout(dispatch(), this.queryTimeoutMs, () => this.timeoutError(query));
+        const duration = performance.now() - startTime;
+
+        this.events.emit(this.eventType, {
+          query,
+          vars,
+          duration,
+          success: true,
+          timestamp: Date.now(),
+        });
+
+        this.logger.trace(
+          { query, result, Category: 'sp00ky-client::Database::query' },
+          'Query executed successfully'
+        );
+        return result;
+      } catch (err) {
+        const duration = performance.now() - startTime;
+
+        this.events.emit(this.eventType, {
+          query,
+          vars,
+          duration,
+          success: false,
+          error: err instanceof Error ? err.message : String(err),
+          timestamp: Date.now(),
+        });
+
+        this.logger.error(
+          { query, vars, err, Category: 'sp00ky-client::Database::query' },
+          'Query execution failed'
+        );
+        throw err;
+      }
+    } finally {
+      this.releaseQuerySlot();
+    }
+  }
+
   async execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>): Promise<T> {
     const raw = await this.query<unknown[]>(query.sql, vars);
     return query.extract(raw);
   }
 
   async close(): Promise<void> {
-    this.logger.info({ Category: 'spooky-client::Database::close' }, 'Closing database connection');
+    this.logger.info({ Category: 'sp00ky-client::Database::close' }, 'Closing database connection');
     await this.client.close();
   }
 }

package/src/services/database/engine-factory.ts

@@ -0,0 +1,33 @@
+import type { Sp00kyConfig } from '../../types';
+import type { Logger } from '../logger/index';
+import type { LocalEngineChoice, LocalStore } from './cache-engine';
+import { SurrealCacheEngine } from './surreal-cache-engine';
+import { SqliteCacheEngine } from './sqlite-cache-engine';
+
+/**
+ * Build the local cache engine for the given `localEngine` config choice.
+ *
+ * - `'surrealdb'` / unset → {@link SurrealCacheEngine} (subclass of
+ *   `LocalDatabaseService`; the historical behavior, verbatim).
+ * - `'sqlite'` → {@link SqliteCacheEngine} (SQLite-WASM Worker + OPFS). Backs
+ *   `this.local` through the SurrealQL-vocabulary shim + verb surface.
+ * - a custom object → used as-is (must satisfy {@link LocalStore}).
+ */
+export function createLocalEngine(
+  choice: LocalEngineChoice | undefined,
+  config: Sp00kyConfig<any>['database'],
+  logger: Logger,
+  opts: { shared?: boolean } = {}
+): LocalStore {
+  if (choice === undefined || choice === 'surrealdb') {
+    return new SurrealCacheEngine(config, logger);
+  }
+  if (choice === 'sqlite') {
+    // Mirror the SurrealDB engine's `store` semantics: 'memory' → in-memory
+    // SQLite (transient), 'indexeddb' → OPFS-backed (durable).
+    const useOpfs = (config.store ?? 'memory') !== 'memory';
+    return new SqliteCacheEngine(config, logger, { useOpfs, shared: opts.shared });
+  }
+  // Custom engine instance — trust it to satisfy LocalStore.
+  return choice as unknown as LocalStore;
+}

package/src/services/database/errors.ts

@@ -0,0 +1,34 @@
+/**
+ * A local-store operation that did not answer within its deadline.
+ *
+ * The local write path (`db.create` / `db.update` / `db.delete`, every local
+ * query behind them) used to have no deadline anywhere: the SQLite worker
+ * transport parks a call until the worker replies, the surrealdb engine's
+ * query chain waits on the previous link, and `withRetry` retries without a
+ * clock. One op that never settled (a worker starved behind a long select, a
+ * lock verification awaiting `navigator.locks.query()` forever) left the
+ * caller's promise pending for the tab's lifetime - a chat composer that never
+ * re-enabled, a call that never got past "Connecting".
+ *
+ * The message says "timed out" on purpose: `classifySyncError` keys off it and
+ * treats the failure as transient (re-queue), never as an application error
+ * that rolls the mutation back. `retryable: false` keeps `withRetry` from
+ * spinning on it: the op is still running in the engine, retrying queues a
+ * second copy behind it.
+ */
+export class LocalOpTimeoutError extends Error {
+  override readonly name = 'LocalOpTimeoutError';
+  readonly retryable = false;
+  readonly op: string;
+  readonly timeoutMs: number;
+
+  constructor(op: string, timeoutMs: number) {
+    super(`Local database operation timed out after ${timeoutMs}ms (${op})`);
+    this.op = op;
+    this.timeoutMs = timeoutMs;
+  }
+}
+
+/** Default deadline for one local-store operation. Generous: a cold 4k-row
+ *  select on a throttled tab is seconds, not tens of seconds. */
+export const DEFAULT_LOCAL_OP_TIMEOUT_MS = 30_000;

package/src/services/database/events/index.ts

@@ -1,4 +1,5 @@
-import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
+import type { EventDefinition, EventSystem } from '../../../events/index';
+import { createEventSystem } from '../../../events/index';
 
 export const DatabaseEventTypes = {
   LocalQuery: 'DATABASE_LOCAL_QUERY',

package/src/services/database/index.ts

@@ -1,5 +1,12 @@
 export * from './database';
 export * from './local';
 export * from './remote';
+export * from './connection-supervisor';
 export * from './local-migrator';
 export * from './events/index';
+export * from './cache-engine';
+export * from './surreal-cache-engine';
+export * from './sqlite-cache-engine';
+export * from './relation-resolver';
+export * from './plan-render';
+export { createLocalEngine } from './engine-factory';

package/src/services/database/local-migrator.ts

@@ -1,6 +1,6 @@
-import type { Surreal } from 'surrealdb';
-import { Logger, createLogger } from '../logger/index';
-import { LocalDatabaseService } from './local';
+import type { Logger} from '../logger/index';
+import { createLogger } from '../logger/index';
+import type { LocalStore } from './cache-engine';
 
 export interface SchemaRecord {
   hash: string;
@@ -19,12 +19,12 @@ export class LocalMigrator {
   private logger: Logger;
 
   constructor(
-    private localDb: LocalDatabaseService,
+    private localDb: LocalStore,
     logger: Logger
   ) {
     this.logger = logger.child({ service: 'LocalMigrator' });
-    logger?.child({ service: 'LocalMigrator' }) ??
-      createLogger('info').child({ service: 'LocalMigrator' });
+    void (logger?.child({ service: 'LocalMigrator' }) ??
+      createLogger('info').child({ service: 'LocalMigrator' }));
   }
 
   async provision(schemaSurql: string): Promise<void> {
@@ -34,7 +34,7 @@ export class LocalMigrator {
 
     if (await this.isSchemaUpToDate(hash)) {
       this.logger.info(
-        { Category: 'spooky-client::LocalMigrator::provision' },
+        { Category: 'sp00ky-client::LocalMigrator::provision' },
         '[Provisioning] Schema is up to date, skipping migration'
       );
       return;
@@ -43,10 +43,13 @@ export class LocalMigrator {
     await this.recreateDatabase(database);
 
     const systemSchema = `
-      DEFINE TABLE IF NOT EXISTS _spooky_stream_processor_state SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _spooky_query SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _spooky_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _spooky_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_stream_processor_state SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_view SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_window SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_failed_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_blob SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
     `;
     const fullSchema = schemaSurql + '\n' + systemSchema;
 
@@ -60,7 +63,7 @@ export class LocalMigrator {
       // SKIP INDEXES: WASM engine hangs on DEFINE INDEX (confirmed)
       if (cleanStatement.toUpperCase().startsWith('DEFINE INDEX')) {
         this.logger.warn(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] Skipping index definition (WASM hang avoidance): ${cleanStatement.substring(0, 50)}...`
         );
         continue;
@@ -68,17 +71,17 @@ export class LocalMigrator {
 
       try {
         this.logger.info(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] (${i + 1}/${statements.length}) Executing: ${statement.substring(0, 50)}...`
         );
-        await this.localDb.query(statement);
+        await this.localDb.queryUngated(statement);
         this.logger.info(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] (${i + 1}/${statements.length}) Done`
         );
       } catch (e) {
         this.logger.error(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] (${i + 1}/${statements.length}) Error executing statement: ${statement}`
         );
         throw e;
@@ -90,32 +93,32 @@ export class LocalMigrator {
 
   private async isSchemaUpToDate(hash: string): Promise<boolean> {
     try {
-      const [lastSchemaRecord] = await this.localDb.query<any>(
-        `SELECT hash, created_at FROM ONLY _spooky_schema ORDER BY created_at DESC LIMIT 1;`
+      const [lastSchemaRecord] = await this.localDb.queryUngated<any>(
+        `SELECT hash, created_at FROM ONLY _00_schema ORDER BY created_at DESC LIMIT 1;`
       );
       return lastSchemaRecord?.hash === hash;
-    } catch (error) {
+    } catch (_error) {
       return false;
     }
   }
 
   private async recreateDatabase(database: string) {
     try {
-      await this.localDb.query(`DEFINE DATABASE _spooky_temp;`);
-    } catch (e) {
+      await this.localDb.queryUngated(`DEFINE DATABASE _00_temp;`);
+    } catch (_e) {
       // Ignore if exists
     }
 
     try {
-      await this.localDb.query(`
-        USE DB _spooky_temp;
+      await this.localDb.queryUngated(`
+        USE DB _00_temp;
         REMOVE DATABASE ${database};
       `);
-    } catch (e) {
+    } catch (_e) {
       // Ignore error if database doesn't exist
     }
 
-    await this.localDb.query(`
+    await this.localDb.queryUngated(`
       DEFINE DATABASE ${database};
       USE DB ${database};
     `);
@@ -195,8 +198,8 @@ export class LocalMigrator {
   }
 
   private async createHashRecord(hash: string) {
-    await this.localDb.query(
-      `UPSERT _spooky_schema SET hash = $hash, created_at = time::now() WHERE hash = $hash;`,
+    await this.localDb.queryUngated(
+      `UPSERT _00_schema SET hash = $hash, created_at = time::now() WHERE hash = $hash;`,
       { hash }
     );
   }

package/src/services/database/local.test.ts

@@ -0,0 +1,64 @@
+import { describe, it, expect } from 'vitest';
+import { isLocalStoreOpenError } from './local';
+
+// `LocalDatabaseService.connect` recovers (drops the store + reconnects, falling
+// back to in-memory) only when the failure is the SurrealDB-WASM IndexedDB
+// open/transaction error — NOT for unrelated errors. This pins the message match
+// against the real error the engine throws so a recovery path doesn't silently
+// stop triggering (or start swallowing unrelated failures).
+describe('isLocalStoreOpenError', () => {
+  it('matches the real SurrealDB-WASM IndexedDB key-value store error', () => {
+    const real = new Error(
+      'There was a problem with the key-value store: There was a problem with a ' +
+        'transaction: An IndexedDB error occured: idb error'
+    );
+    expect(isLocalStoreOpenError(real)).toBe(true);
+  });
+
+  it('matches on the individual signals (indexeddb / idb error / key-value store)', () => {
+    expect(isLocalStoreOpenError(new Error('IndexedDB is not available'))).toBe(true);
+    expect(isLocalStoreOpenError(new Error('idb error'))).toBe(true);
+    expect(isLocalStoreOpenError(new Error('problem with the key-value store'))).toBe(true);
+    // Non-Error inputs are stringified.
+    expect(isLocalStoreOpenError('An IndexedDB error occured')).toBe(true);
+  });
+
+  it('does NOT match unrelated errors (so we never clear the store spuriously)', () => {
+    expect(isLocalStoreOpenError(new Error('WebSocket connection refused'))).toBe(false);
+    expect(isLocalStoreOpenError(new Error('Parse error: unexpected token'))).toBe(false);
+    expect(isLocalStoreOpenError(new Error('permission denied'))).toBe(false);
+    expect(isLocalStoreOpenError(null)).toBe(false);
+    expect(isLocalStoreOpenError(undefined)).toBe(false);
+  });
+});
+
+// Per-user local buckets: URL/name derivation and the SCOPE of the tier-2
+// corruption drop. The drop must only ever match the failing bucket's store —
+// a substring wipe would take every user's cache AND their un-pushed mutation
+// outboxes down with one corrupt store.
+import { bucketStoreName, bucketStoreUrl, matchesBucketStore } from './local';
+
+describe('bucket store naming', () => {
+  it('derives the store url/name from the bucket id', () => {
+    expect(bucketStoreUrl('abc')).toBe('indxdb://sp00ky-abc');
+    expect(bucketStoreName('abc')).toBe('sp00ky-abc');
+    expect(bucketStoreUrl('anon')).toBe('indxdb://sp00ky-anon');
+  });
+});
+
+describe('matchesBucketStore', () => {
+  it('matches the exact store and derived names', () => {
+    expect(matchesBucketStore('sp00ky-abc', 'sp00ky-abc')).toBe(true);
+    expect(matchesBucketStore('sp00ky-abc-wal', 'sp00ky-abc')).toBe(true);
+    expect(matchesBucketStore('surrealdb/sp00ky-abc', 'sp00ky-abc')).toBe(true);
+    expect(matchesBucketStore('SP00KY-ABC', 'sp00ky-abc')).toBe(true);
+  });
+
+  it("never matches another bucket's store", () => {
+    expect(matchesBucketStore('sp00ky-abcdef', 'sp00ky-abc')).toBe(false);
+    expect(matchesBucketStore('sp00ky-anon', 'sp00ky-abc')).toBe(false);
+    expect(matchesBucketStore('sp00ky', 'sp00ky-abc')).toBe(false);
+    // The legacy shared store is not any bucket's store.
+    expect(matchesBucketStore('sp00ky', 'sp00ky-anon')).toBe(false);
+  });
+});