@spooky-sync/core 0.0.1-canary.16 → 0.0.1-canary.161

package/src/services/database/cache-engine.ts

@@ -0,0 +1,160 @@
+import type { QueryPlan, RelationPlan, WhereNode } from '@spooky-sync/query-builder';
+import type { SealedQuery } from '../../utils/surql';
+import type { DatabaseEventSystem } from './events/index';
+import type { Sp00kyConfig, StorageHealth } from '../../types';
+import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-info';
+
+/**
+ * A materialized row. Keys are field names; values are already decoded to the
+ * client's runtime shapes (RecordId stays a RecordId, bytes a Uint8Array, …) so
+ * every backend hands `DataModule` the same shape SurrealDB does today.
+ */
+export type Row = Record<string, unknown>;
+
+/** A record identifier — a `RecordId` or its stable string form (`table:id`). */
+export type Id = unknown;
+
+/** How an order clause is expressed everywhere in the engine layer. */
+export type OrderBy = [field: string, direction: 'asc' | 'desc'][];
+
+/**
+ * Batched relation fetch: "give me every row of `table` whose `matchField` is
+ * one of `keys`, filtered by `where`, ordered by `orderBy`". This is the single
+ * primitive relation decomposition (§3) leans on — implemented as
+ * `SELECT … WHERE <matchField> IN (…)` on SQLite, `SELECT … FROM $keys` /
+ * `WHERE <matchField> IN $keys` on SurrealDB, or an index scan elsewhere. Order
+ * here is a hint; the resolver re-applies order+limit PER PARENT after grouping.
+ */
+export interface RelationFetch {
+  table: string;
+  matchField: string;
+  keys: Id[];
+  where?: WhereNode[];
+  orderBy?: OrderBy;
+  select?: string[];
+}
+
+/**
+ * The read side of an engine, minus relation resolution — the surface a
+ * {@link RelationResolver} needs. Kept separate so the resolver can be unit
+ * tested against an in-memory fake without a full engine.
+ */
+export interface RowFetcher {
+  /** Batched fan-out fetch. See {@link RelationFetch}. */
+  fetchRelation(req: RelationFetch): Promise<Row[]>;
+}
+
+/** A transaction handle — the same verbs as the engine, but atomic. */
+export interface EngineTx {
+  upsert(table: string, id: Id, data: Row, mode: 'replace' | 'merge'): Promise<void>;
+  patch(table: string, id: Id, patches: unknown[]): Promise<void>;
+  delete(table: string, id: Id): Promise<void>;
+}
+
+/**
+ * A pluggable local cache backend. SurrealDB (the default) and SQLite both
+ * implement this; the rest of the client talks verbs, never SurrealQL.
+ *
+ * Reactivity is NOT part of this contract: the local cache is passive. The SSP
+ * (remote) drives change; `DataModule` writes rows here and re-reads them. The
+ * `epoch` field preserves the existing bucket-switch fencing (see
+ * `LocalDatabaseService.epoch`): an async chain captures it at start and its
+ * write is dropped if the epoch moved (a bucket switch) in between.
+ */
+export interface LocalCacheEngine extends RowFetcher {
+  /** Monotonic store generation; bumped on every bucket switch. */
+  readonly epoch: number;
+
+  connect(bucketId: string): Promise<void>;
+  switchBucket(bucketId: string): Promise<void>;
+  close(): Promise<void>;
+
+  /** Run `fn` inside a single atomic transaction. */
+  transaction<T>(fn: (tx: EngineTx) => Promise<T>): Promise<T>;
+
+  /**
+   * Materialize a query, including its `.related()` tree (via §3
+   * decomposition). Params bind `where` `paramRef`s and any windowing id-set.
+   */
+  select(plan: QueryPlan, params?: Record<string, unknown>): Promise<Row[]>;
+
+  /** Fetch rows by primary id, preserving `ids` order; missing ids are skipped. */
+  selectByIds(table: string, ids: Id[], opts?: { select?: string[]; orderBy?: OrderBy }): Promise<Row[]>;
+
+  /** Single-record read by primary id, or `null`. */
+  getById(table: string, id: Id): Promise<Row | null>;
+
+  upsert(table: string, id: Id, data: Row, mode: 'replace' | 'merge'): Promise<void>;
+  patch(table: string, id: Id, patches: unknown[]): Promise<void>;
+  delete(table: string, id: Id): Promise<void>;
+}
+
+/**
+ * The full surface the client's `this.local` field depends on: the
+ * engine-neutral {@link LocalCacheEngine} verbs PLUS the legacy
+ * SurrealQL/lifecycle methods the not-yet-migrated call sites still use.
+ * `SurrealCacheEngine` (subclass of `LocalDatabaseService`) and
+ * `SqliteCacheEngine` (via a SurrealQL-vocabulary shim) both satisfy this, so
+ * either can back `this.local`.
+ *
+ * `getClient()` returns the underlying SurrealDB `Surreal` handle where one
+ * exists (SurrealDB backend); backends without one (SQLite) throw — it is only
+ * used by advanced/DevTools paths, never on the hot path.
+ */
+export interface LocalStore extends LocalCacheEngine {
+  /**
+   * Whether this engine needs SurrealQL schema provisioning (`DEFINE TABLE`,
+   * `DEFINE FIELD`, …) run against it at init / bucket switch. SurrealDB → true;
+   * schemaless engines (SQLite creates tables lazily) → false, so the client
+   * skips the `LocalMigrator` entirely for them.
+   */
+  readonly usesSurqlSchema: boolean;
+  query<T extends unknown[]>(
+    query: string,
+    vars?: Record<string, unknown>,
+    opts?: { epoch?: number }
+  ): Promise<T>;
+  execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>, opts?: { epoch?: number }): Promise<T>;
+  queryUngated<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T>;
+  switchStore(bucketId: string): Promise<void>;
+  beginSwitch(): () => void;
+  getEvents(): DatabaseEventSystem;
+  getClient(): unknown;
+  getConfig(): Sp00kyConfig<any>['database'];
+  readonly currentBucketId: string;
+  /** Which built-in backend this is. OPTIONAL: absent (custom engines) is
+   *  reported as `'custom'` by DevTools. More robust than `instanceof` for
+   *  engines constructed outside this package. */
+  readonly engineKind?: 'surrealdb' | 'sqlite';
+  /** Engine-specific storage numbers for DevTools (DB file size, per-table
+   *  row counts). OPTIONAL: only engines with something to report implement it. */
+  getStorageDiagnostics?(opts?: { tableCounts?: boolean }): Promise<EngineStorageDiagnostics>;
+  /**
+   * Durability of this engine's local store. OPTIONAL: engines that don't
+   * report it (SurrealDB, custom engines) are treated as `'unknown'` by the
+   * client facade, so adding this needs no change on their side.
+   */
+  readonly storageHealth?: StorageHealth;
+  /** Fires immediately with the current snapshot, then on every change.
+   *  Returns an unsubscribe function. */
+  subscribeToStorageHealth?(cb: (health: StorageHealth) => void): () => void;
+}
+
+/** Selected local cache backend. Mirrors the `persistenceClient` config pattern. */
+export type LocalEngineChoice = 'surrealdb' | 'sqlite' | LocalStore;
+
+/** Thrown when relation decomposition nests past {@link MAX_RELATION_DEPTH} —
+ *  a guard against a cyclic schema producing unbounded fan-out. */
+export class RelationCycleError extends Error {
+  constructor(path: string[]) {
+    super(`Relation nesting exceeded safe depth; possible cyclic schema: ${path.join(' -> ')}`);
+    this.name = 'RelationCycleError';
+  }
+}
+
+/** Defensive ceiling on relation nesting depth. A finite plan tree never hits
+ *  this in practice; it exists so a malformed/cyclic plan fails loudly instead
+ *  of running away. */
+export const MAX_RELATION_DEPTH = 12;
+
+export type { QueryPlan, RelationPlan, WhereNode };

package/src/services/database/database.ts

@@ -1,11 +1,9 @@
-import { Surreal, SurrealTransaction } from 'surrealdb';
-import { createLogger, Logger } from '../logger/index';
-import {
+import type { Surreal, SurrealTransaction } from 'surrealdb';
+import type { Logger } from '../logger/index';
+import type {
   DatabaseEventSystem,
-  DatabaseEventTypes,
-  DatabaseQueryEventPayload,
-} from './events/index';
-import { SealedQuery } from '../../utils/surql';
+  DatabaseEventTypes} from './events/index';
+import type { SealedQuery } from '../../utils/surql';
 
 export abstract class AbstractDatabaseService {
   protected client: Surreal;
@@ -43,11 +41,12 @@ export abstract class AbstractDatabaseService {
   async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
     return new Promise((resolve, reject) => {
       this.queryQueue = this.queryQueue
+        // oxlint-disable-next-line promise/always-return
         .then(async () => {
           const startTime = performance.now();
           try {
             this.logger.debug(
-              { query, vars, Category: 'spooky-client::Database::query' },
+              { query, vars, Category: 'sp00ky-client::Database::query' },
               'Executing query'
             );
             const pending = this.client.query(query, vars);
@@ -67,7 +66,7 @@ export abstract class AbstractDatabaseService {
 
             resolve(result);
             this.logger.trace(
-              { query, result, Category: 'spooky-client::Database::query' },
+              { query, result, Category: 'sp00ky-client::Database::query' },
               'Query executed successfully'
             );
           } catch (err) {
@@ -84,9 +83,10 @@ export abstract class AbstractDatabaseService {
             });
 
             this.logger.error(
-              { query, vars, err, Category: 'spooky-client::Database::query' },
+              { query, vars, err, Category: 'sp00ky-client::Database::query' },
               'Query execution failed'
             );
+            // oxlint-disable-next-line no-multiple-resolved -- resolve/reject are in try/catch, mutually exclusive
             reject(err);
           }
         })
@@ -102,7 +102,7 @@ export abstract class AbstractDatabaseService {
   }
 
   async close(): Promise<void> {
-    this.logger.info({ Category: 'spooky-client::Database::close' }, 'Closing database connection');
+    this.logger.info({ Category: 'sp00ky-client::Database::close' }, 'Closing database connection');
     await this.client.close();
   }
 }

package/src/services/database/engine-factory.ts

@@ -0,0 +1,33 @@
+import type { Sp00kyConfig } from '../../types';
+import type { Logger } from '../logger/index';
+import type { LocalEngineChoice, LocalStore } from './cache-engine';
+import { SurrealCacheEngine } from './surreal-cache-engine';
+import { SqliteCacheEngine } from './sqlite-cache-engine';
+
+/**
+ * Build the local cache engine for the given `localEngine` config choice.
+ *
+ * - `'surrealdb'` / unset → {@link SurrealCacheEngine} (subclass of
+ *   `LocalDatabaseService`; the historical behavior, verbatim).
+ * - `'sqlite'` → {@link SqliteCacheEngine} (SQLite-WASM Worker + OPFS). Backs
+ *   `this.local` through the SurrealQL-vocabulary shim + verb surface.
+ * - a custom object → used as-is (must satisfy {@link LocalStore}).
+ */
+export function createLocalEngine(
+  choice: LocalEngineChoice | undefined,
+  config: Sp00kyConfig<any>['database'],
+  logger: Logger,
+  opts: { shared?: boolean } = {}
+): LocalStore {
+  if (choice === undefined || choice === 'surrealdb') {
+    return new SurrealCacheEngine(config, logger);
+  }
+  if (choice === 'sqlite') {
+    // Mirror the SurrealDB engine's `store` semantics: 'memory' → in-memory
+    // SQLite (transient), 'indexeddb' → OPFS-backed (durable).
+    const useOpfs = (config.store ?? 'memory') !== 'memory';
+    return new SqliteCacheEngine(config, logger, { useOpfs, shared: opts.shared });
+  }
+  // Custom engine instance — trust it to satisfy LocalStore.
+  return choice as unknown as LocalStore;
+}

package/src/services/database/events/index.ts

@@ -1,4 +1,5 @@
-import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
+import type { EventDefinition, EventSystem } from '../../../events/index';
+import { createEventSystem } from '../../../events/index';
 
 export const DatabaseEventTypes = {
   LocalQuery: 'DATABASE_LOCAL_QUERY',

package/src/services/database/index.ts

@@ -3,3 +3,9 @@ export * from './local';
 export * from './remote';
 export * from './local-migrator';
 export * from './events/index';
+export * from './cache-engine';
+export * from './surreal-cache-engine';
+export * from './sqlite-cache-engine';
+export * from './relation-resolver';
+export * from './plan-render';
+export { createLocalEngine } from './engine-factory';

package/src/services/database/local-migrator.ts

@@ -1,6 +1,6 @@
-import type { Surreal } from 'surrealdb';
-import { Logger, createLogger } from '../logger/index';
-import { LocalDatabaseService } from './local';
+import type { Logger} from '../logger/index';
+import { createLogger } from '../logger/index';
+import type { LocalStore } from './cache-engine';
 
 export interface SchemaRecord {
   hash: string;
@@ -19,12 +19,12 @@ export class LocalMigrator {
   private logger: Logger;
 
   constructor(
-    private localDb: LocalDatabaseService,
+    private localDb: LocalStore,
     logger: Logger
   ) {
     this.logger = logger.child({ service: 'LocalMigrator' });
-    logger?.child({ service: 'LocalMigrator' }) ??
-      createLogger('info').child({ service: 'LocalMigrator' });
+    void (logger?.child({ service: 'LocalMigrator' }) ??
+      createLogger('info').child({ service: 'LocalMigrator' }));
   }
 
   async provision(schemaSurql: string): Promise<void> {
@@ -34,7 +34,7 @@ export class LocalMigrator {
 
     if (await this.isSchemaUpToDate(hash)) {
       this.logger.info(
-        { Category: 'spooky-client::LocalMigrator::provision' },
+        { Category: 'sp00ky-client::LocalMigrator::provision' },
         '[Provisioning] Schema is up to date, skipping migration'
       );
       return;
@@ -43,10 +43,11 @@ export class LocalMigrator {
     await this.recreateDatabase(database);
 
     const systemSchema = `
-      DEFINE TABLE IF NOT EXISTS _spooky_stream_processor_state SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _spooky_query SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _spooky_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _spooky_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_stream_processor_state SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_query SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_preload SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
     `;
     const fullSchema = schemaSurql + '\n' + systemSchema;
 
@@ -60,7 +61,7 @@ export class LocalMigrator {
       // SKIP INDEXES: WASM engine hangs on DEFINE INDEX (confirmed)
       if (cleanStatement.toUpperCase().startsWith('DEFINE INDEX')) {
         this.logger.warn(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] Skipping index definition (WASM hang avoidance): ${cleanStatement.substring(0, 50)}...`
         );
         continue;
@@ -68,17 +69,17 @@ export class LocalMigrator {
 
       try {
         this.logger.info(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] (${i + 1}/${statements.length}) Executing: ${statement.substring(0, 50)}...`
         );
-        await this.localDb.query(statement);
+        await this.localDb.queryUngated(statement);
         this.logger.info(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] (${i + 1}/${statements.length}) Done`
         );
       } catch (e) {
         this.logger.error(
-          { Category: 'spooky-client::LocalMigrator::provision' },
+          { Category: 'sp00ky-client::LocalMigrator::provision' },
           `[Provisioning] (${i + 1}/${statements.length}) Error executing statement: ${statement}`
         );
         throw e;
@@ -90,32 +91,32 @@ export class LocalMigrator {
 
   private async isSchemaUpToDate(hash: string): Promise<boolean> {
     try {
-      const [lastSchemaRecord] = await this.localDb.query<any>(
-        `SELECT hash, created_at FROM ONLY _spooky_schema ORDER BY created_at DESC LIMIT 1;`
+      const [lastSchemaRecord] = await this.localDb.queryUngated<any>(
+        `SELECT hash, created_at FROM ONLY _00_schema ORDER BY created_at DESC LIMIT 1;`
       );
       return lastSchemaRecord?.hash === hash;
-    } catch (error) {
+    } catch (_error) {
       return false;
     }
   }
 
   private async recreateDatabase(database: string) {
     try {
-      await this.localDb.query(`DEFINE DATABASE _spooky_temp;`);
-    } catch (e) {
+      await this.localDb.queryUngated(`DEFINE DATABASE _00_temp;`);
+    } catch (_e) {
       // Ignore if exists
     }
 
     try {
-      await this.localDb.query(`
-        USE DB _spooky_temp;
+      await this.localDb.queryUngated(`
+        USE DB _00_temp;
         REMOVE DATABASE ${database};
       `);
-    } catch (e) {
+    } catch (_e) {
       // Ignore error if database doesn't exist
     }
 
-    await this.localDb.query(`
+    await this.localDb.queryUngated(`
       DEFINE DATABASE ${database};
       USE DB ${database};
     `);
@@ -195,8 +196,8 @@ export class LocalMigrator {
   }
 
   private async createHashRecord(hash: string) {
-    await this.localDb.query(
-      `UPSERT _spooky_schema SET hash = $hash, created_at = time::now() WHERE hash = $hash;`,
+    await this.localDb.queryUngated(
+      `UPSERT _00_schema SET hash = $hash, created_at = time::now() WHERE hash = $hash;`,
       { hash }
     );
   }

package/src/services/database/local.test.ts

@@ -0,0 +1,64 @@
+import { describe, it, expect } from 'vitest';
+import { isLocalStoreOpenError } from './local';
+
+// `LocalDatabaseService.connect` recovers (drops the store + reconnects, falling
+// back to in-memory) only when the failure is the SurrealDB-WASM IndexedDB
+// open/transaction error — NOT for unrelated errors. This pins the message match
+// against the real error the engine throws so a recovery path doesn't silently
+// stop triggering (or start swallowing unrelated failures).
+describe('isLocalStoreOpenError', () => {
+  it('matches the real SurrealDB-WASM IndexedDB key-value store error', () => {
+    const real = new Error(
+      'There was a problem with the key-value store: There was a problem with a ' +
+        'transaction: An IndexedDB error occured: idb error'
+    );
+    expect(isLocalStoreOpenError(real)).toBe(true);
+  });
+
+  it('matches on the individual signals (indexeddb / idb error / key-value store)', () => {
+    expect(isLocalStoreOpenError(new Error('IndexedDB is not available'))).toBe(true);
+    expect(isLocalStoreOpenError(new Error('idb error'))).toBe(true);
+    expect(isLocalStoreOpenError(new Error('problem with the key-value store'))).toBe(true);
+    // Non-Error inputs are stringified.
+    expect(isLocalStoreOpenError('An IndexedDB error occured')).toBe(true);
+  });
+
+  it('does NOT match unrelated errors (so we never clear the store spuriously)', () => {
+    expect(isLocalStoreOpenError(new Error('WebSocket connection refused'))).toBe(false);
+    expect(isLocalStoreOpenError(new Error('Parse error: unexpected token'))).toBe(false);
+    expect(isLocalStoreOpenError(new Error('permission denied'))).toBe(false);
+    expect(isLocalStoreOpenError(null)).toBe(false);
+    expect(isLocalStoreOpenError(undefined)).toBe(false);
+  });
+});
+
+// Per-user local buckets: URL/name derivation and the SCOPE of the tier-2
+// corruption drop. The drop must only ever match the failing bucket's store —
+// a substring wipe would take every user's cache AND their un-pushed mutation
+// outboxes down with one corrupt store.
+import { bucketStoreName, bucketStoreUrl, matchesBucketStore } from './local';
+
+describe('bucket store naming', () => {
+  it('derives the store url/name from the bucket id', () => {
+    expect(bucketStoreUrl('abc')).toBe('indxdb://sp00ky-abc');
+    expect(bucketStoreName('abc')).toBe('sp00ky-abc');
+    expect(bucketStoreUrl('anon')).toBe('indxdb://sp00ky-anon');
+  });
+});
+
+describe('matchesBucketStore', () => {
+  it('matches the exact store and derived names', () => {
+    expect(matchesBucketStore('sp00ky-abc', 'sp00ky-abc')).toBe(true);
+    expect(matchesBucketStore('sp00ky-abc-wal', 'sp00ky-abc')).toBe(true);
+    expect(matchesBucketStore('surrealdb/sp00ky-abc', 'sp00ky-abc')).toBe(true);
+    expect(matchesBucketStore('SP00KY-ABC', 'sp00ky-abc')).toBe(true);
+  });
+
+  it("never matches another bucket's store", () => {
+    expect(matchesBucketStore('sp00ky-abcdef', 'sp00ky-abc')).toBe(false);
+    expect(matchesBucketStore('sp00ky-anon', 'sp00ky-abc')).toBe(false);
+    expect(matchesBucketStore('sp00ky', 'sp00ky-abc')).toBe(false);
+    // The legacy shared store is not any bucket's store.
+    expect(matchesBucketStore('sp00ky', 'sp00ky-anon')).toBe(false);
+  });
+});