@spooky-sync/core 0.0.1-canary.27 → 0.0.1-canary.270

package/src/utils/surql.ts

@@ -1,4 +1,4 @@
-import { MutationEventType } from '../types';
+import type { MutationEventType } from '../types';
 
 // ==================== TYPES ====================
 
@@ -34,6 +34,7 @@ export interface SurqlHelper {
     keyDataVars: ({ key: string; variable: string } | { statement: string } | string)[]
   ): string;
   upsert(idVar: string, dataVar: string): string;
+  upsertMerge(idVar: string, dataVar: string): string;
   updateMerge(idVar: string, dataVar: string): string;
   updateSet(
     idVar: string,
@@ -90,17 +91,9 @@ export const surql: SurqlHelper = {
     returnValues: ({ field: string; alias: string } | string)[]
   ) {
     return `SELECT ${returnValues
-      .map((returnValues) =>
-        typeof returnValues === 'string'
-          ? returnValues
-          : `${returnValues.field} as ${returnValues.alias}`
-      )
+      .map((rv) => (typeof rv === 'string' ? rv : `${rv.field} as ${rv.alias}`))
       .join(',')} FROM ${table} WHERE ${whereVar
-      .map((whereVar) =>
-        typeof whereVar === 'string'
-          ? `${whereVar} = $${whereVar}`
-          : `${whereVar.field} = $${whereVar.variable}`
-      )
+      .map((wv) => (typeof wv === 'string' ? `${wv} = $${wv}` : `${wv.field} = $${wv.variable}`))
       .join(' AND ')}`;
   },
 
@@ -127,6 +120,14 @@ export const surql: SurqlHelper = {
     return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
   },
 
+  // Use this for sync-down ingestion: the remote payload omits local-only
+  // fields injected by the CLI's local-schema setup (`_00_crdt`,
+  // `_00_cursor`), and REPLACE would wipe them on every round-trip and
+  // break offline reload of CRDT state.
+  upsertMerge(idVar: string, dataVar: string) {
+    return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
+  },
+
   updateMerge(idVar: string, dataVar: string) {
     return `UPDATE ONLY $${idVar} MERGE $${dataVar}`;
   },
@@ -136,12 +137,12 @@ export const surql: SurqlHelper = {
     keyDataVar: ({ key: string; variable: string } | { statement: string } | string)[]
   ) {
     return `UPDATE $${idVar} SET ${keyDataVar
-      .map((keyDataVar) =>
-        typeof keyDataVar === 'string'
-          ? `${keyDataVar} = $${keyDataVar}`
-          : 'statement' in keyDataVar
-            ? keyDataVar.statement
-            : `${keyDataVar.key} = $${keyDataVar.variable}`
+      .map((kdv) =>
+        typeof kdv === 'string'
+          ? `${kdv} = $${kdv}`
+          : 'statement' in kdv
+            ? kdv.statement
+            : `${kdv.key} = $${kdv.variable}`
       )
       .join(', ')}`;
   },
@@ -163,7 +164,18 @@ export const surql: SurqlHelper = {
   ) {
     switch (t) {
       case 'create':
-        return `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
+        // `data` is REQUIRED here, not decorative. The outbox row is the only
+        // copy of a pending create once the in-memory UpEvent is gone (reload,
+        // or a shared-tabs follower whose row is replayed by the leader). This
+        // used to drop `dataVar` on the floor, so every replayed create arrived
+        // with `data: undefined` and `processUpEvent` threw on
+        // `Object.keys(event.data)` before it ever reached the network: the
+        // create was unsendable AND it sat at the head of the queue blocking
+        // every later mutation. The payload was simply never persisted, so
+        // those creates were unrecoverable.
+        return dataVar
+          ? `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}, data = $${dataVar}`
+          : `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
       case 'update': {
         let stmt = `CREATE ONLY $${mutationIdVar} SET mutationType = 'update', recordId = $${recordIdVar}, data = $${dataVar}`;
         if (beforeRecordVar) {

package/src/utils/utils.test.ts

@@ -9,6 +9,7 @@ import {
   generateId,
   parseDuration,
 } from './index';
+import type { QueryTimeToLive } from '../types';
 
 describe('compareRecordIds', () => {
   it('returns true for equal strings', () => {
@@ -123,7 +124,7 @@ describe('generateId', () => {
 
 describe('parseDuration', () => {
   it('parses seconds', () => {
-    expect(parseDuration('30s')).toBe(30000);
+    expect(parseDuration('30s' as QueryTimeToLive)).toBe(30000);
   });
 
   it('parses minutes', () => {

package/src/utils/withRetry.test.ts

@@ -145,7 +145,7 @@ describe('withRetry', () => {
         attempt: 1,
         retries: 3,
         error: 'Can not open transaction',
-        Category: 'spooky-client::utils::withRetry',
+        Category: 'sp00ky-client::utils::withRetry',
       }),
       'Retrying DB operation'
     );

package/tsdown.config.ts

@@ -1,9 +1,94 @@
 import { defineConfig } from 'tsdown';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+// Resolve real package versions at build time so the DevTools can report the
+// actual bundled frontend versions (used to detect frontend/backend drift).
+const corePkg = require('./package.json');
+const coreVersion: string = corePkg.version;
+// ssp-wasm has no `exports` map; resolve its package.json directly, falling
+// back to the relative workspace path if module resolution can't find it.
+let wasmVersion: string;
+try {
+  wasmVersion = require('@spooky-sync/ssp-wasm/package.json').version;
+} catch {
+  wasmVersion = require('../ssp-wasm/package.json').version;
+}
+// Report the SurrealDB WASM ENGINE version (`@surrealdb/wasm`) — the engine the
+// in-browser local DB actually runs — NOT the `surrealdb` JS client library
+// version. The WASM engine is the one that must stay compatible with the server
+// engine, so it's the meaningful number to compare in DevTools. Prefer the
+// actually-installed version; fall back to our pinned range if its `exports`
+// map blocks importing its package.json.
+let surrealVersion: string;
+try {
+  surrealVersion = require('@surrealdb/wasm/package.json').version;
+} catch {
+  surrealVersion = String(corePkg.dependencies?.['@surrealdb/wasm'] ?? 'unknown').replace(
+    /^[\^~]/,
+    ''
+  );
+}
+
+// tsdown 0.12.x forwards a top-level `define` to rolldown's inputOptions, which
+// rejects it — so we do the substitution ourselves with a tiny transform
+// plugin. Only our own modules reference the `__SP00KY_*__` identifiers, and
+// the early `includes` check skips everything else.
+const replacements: Record<string, string> = {
+  __SP00KY_CORE_VERSION__: JSON.stringify(coreVersion),
+  __SP00KY_WASM_VERSION__: JSON.stringify(wasmVersion),
+  __SP00KY_SURREAL_VERSION__: JSON.stringify(surrealVersion),
+};
+
+const versionDefinePlugin = {
+  name: 'sp00ky-version-define',
+  transform(code: string) {
+    if (!code.includes('__SP00KY_')) return null;
+    let out = code;
+    for (const [token, value] of Object.entries(replacements)) {
+      out = out.split(token).join(value);
+    }
+    return out;
+  },
+};
+
+// Worker URLs are written as `./<name>.ts` in source so Vite's src-bundling
+// consumers (the example app aliases core to `src`) resolve them. In the
+// published build each worker is emitted at `dist/<name>.js`, so rewrite the
+// references to `.js` here — otherwise the flat `dist/index.js` carries
+// dangling `.ts` URLs that a consumer's bundler can't resolve.
+const workerUrlPlugin = {
+  name: 'sp00ky-worker-url',
+  transform(code: string) {
+    if (!code.includes('sqlite-worker.ts') && !code.includes('tabs-broker-worker.ts')) return null;
+    return code
+      .split('sqlite-worker.ts')
+      .join('sqlite-worker.js')
+      .split('tabs-broker-worker.ts')
+      .join('tabs-broker-worker.js');
+  },
+};
 
 export default defineConfig({
-  entry: ['src/index.ts', 'src/otel/index.ts'],
+  // `sqlite-worker` is emitted at `dist/sqlite-worker.js` (top level, NOT under
+  // services/database) so the `new URL('./sqlite-worker.js', import.meta.url)`
+  // in `SqliteCacheEngine` — which gets bundled into the flat `dist/index.js` —
+  // resolves correctly. It (and `@sqlite.org/sqlite-wasm`) load lazily in the
+  // Worker; never pulled into the main bundle unless `localEngine: 'sqlite'`.
+  entry: {
+    index: 'src/index.ts',
+    'otel/index': 'src/otel/index.ts',
+    'sqlite-worker': 'src/services/database/sqlite-worker.ts',
+    // The shared-tabs broker MUST stay a self-contained emit (no chunk
+    // imports): bundlers cannot trace `new SharedWorker(url)` graphs, so a
+    // relative chunk import would 404 in consumers. The source keeps zero
+    // runtime imports and scripts/check-broker-bundle.mjs enforces it.
+    'tabs-broker-worker': 'src/services/tabs/tabs-broker-worker.ts',
+  },
   format: ['esm'],
   dts: true,
   clean: true,
   hash: false,
+  plugins: [versionDefinePlugin, workerUrlPlugin],
 });

package/vitest.config.ts

@@ -0,0 +1,48 @@
+import { defineConfig } from 'vitest/config';
+
+// Thresholds start at the numbers the suite hits today and are ratcheted per
+// directory as the saga core lands (see the plan's commit 10). Exclusions:
+//   *.fixture.ts / src/testing/**   test-only harness code
+//   sqlite-worker.ts                runs inside a Worker (sqlite-wasm + OPFS +
+//                                   postMessage globals); covered by the sqlite
+//                                   integration tests and the build check
+//   client/services.ts              constructs the real adapters (SurrealDB
+//                                   WASM store, socket, SharedWorker broker,
+//                                   OPFS blob cache); exercised by the example
+//                                   app and the WhitePawn A/B, not by vitest
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['src/**/*.test.ts'],
+    coverage: {
+      provider: 'v8',
+      all: true,
+      reporter: ['text-summary', 'lcov'],
+      include: ['src/**/*.ts'],
+      exclude: [
+        'src/**/*.test.ts',
+        'src/**/*.d.ts',
+        'src/**/*.fixture.ts',
+        'src/testing/**',
+        'src/services/database/sqlite-worker.ts',
+        'src/client/services.ts',
+      ],
+      thresholds: {
+        // Global numbers are held back by the legacy adapters (devtools,
+        // sqlite engine, tabs broker, blobs, crdt); the saga core below is 100%.
+        lines: 75,
+        functions: 82,
+        branches: 87,
+        statements: 75,
+        'src/kernel/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/state/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/query/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/mutation/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/sync/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/boot/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        // The facade's shared-tabs host callbacks need a SharedWorker broker to fire.
+        'src/client/**': { lines: 93, functions: 92, branches: 89, statements: 93 },
+      },
+    },
+  },
+});

package/src/modules/cache/index.ts

@@ -1,241 +0,0 @@
-import { LocalDatabaseService } from '../../services/database/index';
-import {
-  StreamProcessorService,
-  StreamUpdate,
-  StreamUpdateReceiver,
-} from '../../services/stream-processor/index';
-import { Logger } from '../../services/logger/index';
-import { parseRecordIdString, encodeRecordId, surql } from '../../utils/index';
-import { CacheRecord, QueryConfig } from './types';
-import { RecordVersionArray } from '../../types';
-
-export * from './types';
-
-/**
- * CacheModule - Centralized storage and DBSP ingestion
- *
- * Single responsibility: Handle all local storage operations and DBSP ingestion.
- * This module acts as the bridge between data operations and persistence.
- */
-export class CacheModule implements StreamUpdateReceiver {
-  private logger: Logger;
-  private streamUpdateCallback: (update: StreamUpdate) => void;
-  private versionLookups: Record<string, number> = {};
-
-  constructor(
-    private local: LocalDatabaseService,
-    private streamProcessor: StreamProcessorService,
-    streamUpdateCallback: (update: StreamUpdate) => void,
-    logger: Logger
-  ) {
-    this.logger = logger.child({ service: 'CacheModule' });
-    this.streamUpdateCallback = streamUpdateCallback;
-    // Register as receiver for DBSP stream updates
-    this.streamProcessor.addReceiver(this);
-  }
-
-  /**
-   * Implements StreamUpdateReceiver interface
-   * Called directly by StreamProcessor when views change
-   */
-  onStreamUpdate(update: StreamUpdate): void {
-    this.logger.debug(
-      {
-        queryHash: update.queryHash,
-        arrayLength: update.localArray?.length,
-        Category: 'spooky-client::CacheModule::onStreamUpdate',
-      },
-      'Stream update received'
-    );
-    this.streamUpdateCallback(update);
-  }
-
-  public lookup(recordId: string): number {
-    return this.versionLookups[recordId] ?? 0;
-  }
-
-  /**
-   * Save a single record to local DB and ingest into DBSP
-   * Used by mutations (create/update)
-   */
-  async save(cacheRecord: CacheRecord, skipDbInsert: boolean = false): Promise<void> {
-    return this.saveBatch([cacheRecord], skipDbInsert);
-  }
-
-  /**
-   * Save multiple records in a batch
-   * More efficient than calling save() multiple times
-   * Used by sync operations
-   */
-  async saveBatch(records: CacheRecord[], skipDbInsert: boolean = false): Promise<void> {
-    if (records.length === 0) return;
-
-    this.logger.debug(
-      {
-        count: records.length,
-        Category: 'spooky-client::CacheModule::saveBatch',
-      },
-      'Saving record batch'
-    );
-
-    try {
-      const populatedRecords = records.map((record) => {
-        if (!record.version) throw new Error('Record version is required');
-        return {
-          ...record,
-          record: {
-            ...record.record,
-            spooky_rv: record.version,
-          },
-        };
-      });
-
-      if (!skipDbInsert) {
-        const query = surql.seal<void>(
-          surql.tx(
-            populatedRecords.map((_, i) => {
-              return surql.upsert(`id${i}`, `content${i}`);
-            })
-          )
-        );
-
-        const params = populatedRecords.reduce(
-          (acc, record, i) => {
-            const { id, ...content } = record.record;
-            return {
-              ...acc,
-              [`id${i}`]: id,
-              [`content${i}`]: content,
-            };
-          },
-          {} as Record<string, any>
-        );
-
-        await this.local.execute(query, params);
-      }
-
-      // 2. Batch ingest into DBSP (use populatedRecords which has spooky_rv set)
-      for (const record of populatedRecords) {
-        const recordId = encodeRecordId(record.record.id);
-        this.versionLookups[recordId] = record.version;
-        this.streamProcessor.ingest(record.table, record.op, recordId, record.record);
-      }
-
-      this.logger.debug(
-        { count: records.length, Category: 'spooky-client::CacheModule::saveBatch' },
-        'Batch saved successfully'
-      );
-    } catch (err) {
-      this.logger.error(
-        { err, count: records.length, Category: 'spooky-client::CacheModule::saveBatch' },
-        'Failed to save batch'
-      );
-      throw err;
-    }
-  }
-
-  /**
-   * Delete a record from local DB and ingest deletion into DBSP
-   */
-  async delete(table: string, id: string, skipDbDelete: boolean = false, recordData: Record<string, any> = {}): Promise<void> {
-    this.logger.debug(
-      { table, id, Category: 'spooky-client::CacheModule::delete' },
-      'Deleting record'
-    );
-
-    try {
-      // 1. Delete from local database
-      if (!skipDbDelete) {
-        await this.local.query('DELETE $id', { id: parseRecordIdString(id) });
-      }
-
-      // 2. Ingest deletion into DBSP (pass record data so predicates can be matched)
-      delete this.versionLookups[id];
-      this.streamProcessor.ingest(table, 'DELETE', id, recordData);
-
-      this.logger.debug(
-        { table, id, Category: 'spooky-client::CacheModule::delete' },
-        'Record deleted successfully'
-      );
-    } catch (err) {
-      this.logger.error(
-        { err, table, id, Category: 'spooky-client::CacheModule::delete' },
-        'Failed to delete record'
-      );
-      throw err;
-    }
-  }
-
-  /**
-   * Register a query with DBSP to create a materialized view
-   * Returns the initial result array
-   */
-  registerQuery(config: QueryConfig): { localArray: RecordVersionArray } {
-    this.logger.debug(
-      {
-        queryHash: config.queryHash,
-        surql: config.surql,
-        Category: 'spooky-client::CacheModule::registerQuery',
-      },
-      'Registering query'
-    );
-
-    try {
-      const update = this.streamProcessor.registerQueryPlan({
-        queryHash: config.queryHash,
-        surql: config.surql,
-        params: config.params,
-        ttl: config.ttl,
-        lastActiveAt: config.lastActiveAt,
-        localArray: [],
-        remoteArray: [],
-        meta: {
-          tableName: '',
-        },
-      });
-
-      if (!update) {
-        throw new Error('Failed to register query with DBSP');
-      }
-
-      this.logger.debug(
-        {
-          queryHash: config.queryHash,
-          arrayLength: update.localArray?.length,
-          Category: 'spooky-client::CacheModule::registerQuery',
-        },
-        'Query registered successfully'
-      );
-
-      return { localArray: update.localArray };
-    } catch (err) {
-      this.logger.error(
-        { err, queryHash: config.queryHash, Category: 'spooky-client::CacheModule::registerQuery' },
-        'Failed to register query'
-      );
-      throw err;
-    }
-  }
-
-  /**
-   * Unregister a query from DBSP
-   */
-  unregisterQuery(queryHash: string): void {
-    this.logger.debug(
-      { queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
-      'Unregistering query'
-    );
-    try {
-      this.streamProcessor.unregisterQueryPlan(queryHash);
-      this.logger.debug(
-        { queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
-        'Query unregistered successfully'
-      );
-    } catch (err) {
-      this.logger.error(
-        { err, queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
-        'Failed to unregister query'
-      );
-    }
-  }
-}

package/src/modules/cache/types.ts

@@ -1,19 +0,0 @@
-import { RecordId, Duration } from 'surrealdb';
-import { QueryTimeToLive, RecordVersionArray } from '../../types';
-
-export type RecordWithId = Record<string, any> & { id: RecordId<string> };
-
-export interface QueryConfig {
-  queryHash: string;
-  surql: string;
-  params: Record<string, any>;
-  ttl: QueryTimeToLive | Duration;
-  lastActiveAt: Date;
-}
-
-export interface CacheRecord {
-  table: string;
-  op: 'CREATE' | 'UPDATE' | 'DELETE';
-  record: RecordWithId;
-  version: number;
-}

package/src/modules/data/data.test.ts

@@ -1,58 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { parseUpdateOptions } from './index';
-
-describe('parseUpdateOptions', () => {
-  it('returns empty options when no debounce', () => {
-    const result = parseUpdateOptions('user:1', { name: 'Alice' });
-    expect(result).toEqual({});
-  });
-
-  it('returns empty options when options is undefined', () => {
-    const result = parseUpdateOptions('user:1', { name: 'Alice' }, undefined);
-    expect(result).toEqual({});
-  });
-
-  it('returns empty options when debounced is false', () => {
-    const result = parseUpdateOptions('user:1', { name: 'Alice' }, { debounced: false });
-    expect(result).toEqual({});
-  });
-
-  it('debounced: true uses default delay (200) and key = id', () => {
-    const result = parseUpdateOptions('user:1', { name: 'Alice' }, { debounced: true });
-    expect(result).toEqual({
-      debounced: {
-        delay: 200,
-        key: 'user:1',
-      },
-    });
-  });
-
-  it('supports custom delay', () => {
-    const result = parseUpdateOptions('user:1', { name: 'Alice' }, {
-      debounced: { delay: 500 },
-    });
-    expect(result.debounced?.delay).toBe(500);
-  });
-
-  it('supports custom key type recordId', () => {
-    const result = parseUpdateOptions('user:1', { name: 'Alice', age: 30 }, {
-      debounced: { key: 'recordId' },
-    });
-    expect(result.debounced?.key).toBe('user:1');
-  });
-
-  it('key: recordId_x_fields generates composite key from id + sorted field names', () => {
-    const result = parseUpdateOptions(
-      'user:1',
-      { name: 'Alice', age: 30, email: 'a@b.com' },
-      { debounced: { key: 'recordId_x_fields' } }
-    );
-    expect(result.debounced?.key).toBe('user:1::age#email#name');
-  });
-
-  it('uses defaults when debounced is object with no key or delay', () => {
-    const result = parseUpdateOptions('user:1', { name: 'Alice' }, { debounced: {} });
-    expect(result.debounced?.delay).toBe(200);
-    expect(result.debounced?.key).toBe('user:1');
-  });
-});