@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

package/src/utils/surql.ts

@@ -1,4 +1,4 @@
-import { MutationEventType } from '../types';
+import type { MutationEventType } from '../types';
 
 // ==================== TYPES ====================
 
@@ -34,6 +34,7 @@ export interface SurqlHelper {
     keyDataVars: ({ key: string; variable: string } | { statement: string } | string)[]
   ): string;
   upsert(idVar: string, dataVar: string): string;
+  upsertMerge(idVar: string, dataVar: string): string;
   updateMerge(idVar: string, dataVar: string): string;
   updateSet(
     idVar: string,
@@ -90,17 +91,9 @@ export const surql: SurqlHelper = {
     returnValues: ({ field: string; alias: string } | string)[]
   ) {
     return `SELECT ${returnValues
-      .map((returnValues) =>
-        typeof returnValues === 'string'
-          ? returnValues
-          : `${returnValues.field} as ${returnValues.alias}`
-      )
+      .map((rv) => (typeof rv === 'string' ? rv : `${rv.field} as ${rv.alias}`))
       .join(',')} FROM ${table} WHERE ${whereVar
-      .map((whereVar) =>
-        typeof whereVar === 'string'
-          ? `${whereVar} = $${whereVar}`
-          : `${whereVar.field} = $${whereVar.variable}`
-      )
+      .map((wv) => (typeof wv === 'string' ? `${wv} = $${wv}` : `${wv.field} = $${wv.variable}`))
       .join(' AND ')}`;
   },
 
@@ -127,6 +120,14 @@ export const surql: SurqlHelper = {
     return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
   },
 
+  // Use this for sync-down ingestion: the remote payload omits local-only
+  // fields injected by the CLI's local-schema setup (`_00_crdt`,
+  // `_00_cursor`), and REPLACE would wipe them on every round-trip and
+  // break offline reload of CRDT state.
+  upsertMerge(idVar: string, dataVar: string) {
+    return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
+  },
+
   updateMerge(idVar: string, dataVar: string) {
     return `UPDATE ONLY $${idVar} MERGE $${dataVar}`;
   },
@@ -136,12 +137,12 @@ export const surql: SurqlHelper = {
     keyDataVar: ({ key: string; variable: string } | { statement: string } | string)[]
   ) {
     return `UPDATE $${idVar} SET ${keyDataVar
-      .map((keyDataVar) =>
-        typeof keyDataVar === 'string'
-          ? `${keyDataVar} = $${keyDataVar}`
-          : 'statement' in keyDataVar
-            ? keyDataVar.statement
-            : `${keyDataVar.key} = $${keyDataVar.variable}`
+      .map((kdv) =>
+        typeof kdv === 'string'
+          ? `${kdv} = $${kdv}`
+          : 'statement' in kdv
+            ? kdv.statement
+            : `${kdv.key} = $${kdv.variable}`
       )
       .join(', ')}`;
   },
@@ -163,7 +164,18 @@ export const surql: SurqlHelper = {
   ) {
     switch (t) {
       case 'create':
-        return `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
+        // `data` is REQUIRED here, not decorative. The outbox row is the only
+        // copy of a pending create once the in-memory UpEvent is gone (reload,
+        // or a shared-tabs follower whose row is replayed by the leader). This
+        // used to drop `dataVar` on the floor, so every replayed create arrived
+        // with `data: undefined` and `processUpEvent` threw on
+        // `Object.keys(event.data)` before it ever reached the network: the
+        // create was unsendable AND it sat at the head of the queue blocking
+        // every later mutation. The payload was simply never persisted, so
+        // those creates were unrecoverable.
+        return dataVar
+          ? `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}, data = $${dataVar}`
+          : `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
       case 'update': {
         let stmt = `CREATE ONLY $${mutationIdVar} SET mutationType = 'update', recordId = $${recordIdVar}, data = $${dataVar}`;
         if (beforeRecordVar) {

package/src/utils/withRetry.test.ts

@@ -145,7 +145,7 @@ describe('withRetry', () => {
         attempt: 1,
         retries: 3,
         error: 'Can not open transaction',
-        Category: 'spooky-client::utils::withRetry',
+        Category: 'sp00ky-client::utils::withRetry',
       }),
       'Retrying DB operation'
     );

package/tsdown.config.ts

@@ -1,9 +1,94 @@
 import { defineConfig } from 'tsdown';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+// Resolve real package versions at build time so the DevTools can report the
+// actual bundled frontend versions (used to detect frontend/backend drift).
+const corePkg = require('./package.json');
+const coreVersion: string = corePkg.version;
+// ssp-wasm has no `exports` map; resolve its package.json directly, falling
+// back to the relative workspace path if module resolution can't find it.
+let wasmVersion: string;
+try {
+  wasmVersion = require('@spooky-sync/ssp-wasm/package.json').version;
+} catch {
+  wasmVersion = require('../ssp-wasm/package.json').version;
+}
+// Report the SurrealDB WASM ENGINE version (`@surrealdb/wasm`) — the engine the
+// in-browser local DB actually runs — NOT the `surrealdb` JS client library
+// version. The WASM engine is the one that must stay compatible with the server
+// engine, so it's the meaningful number to compare in DevTools. Prefer the
+// actually-installed version; fall back to our pinned range if its `exports`
+// map blocks importing its package.json.
+let surrealVersion: string;
+try {
+  surrealVersion = require('@surrealdb/wasm/package.json').version;
+} catch {
+  surrealVersion = String(corePkg.dependencies?.['@surrealdb/wasm'] ?? 'unknown').replace(
+    /^[\^~]/,
+    ''
+  );
+}
+
+// tsdown 0.12.x forwards a top-level `define` to rolldown's inputOptions, which
+// rejects it — so we do the substitution ourselves with a tiny transform
+// plugin. Only our own modules reference the `__SP00KY_*__` identifiers, and
+// the early `includes` check skips everything else.
+const replacements: Record<string, string> = {
+  __SP00KY_CORE_VERSION__: JSON.stringify(coreVersion),
+  __SP00KY_WASM_VERSION__: JSON.stringify(wasmVersion),
+  __SP00KY_SURREAL_VERSION__: JSON.stringify(surrealVersion),
+};
+
+const versionDefinePlugin = {
+  name: 'sp00ky-version-define',
+  transform(code: string) {
+    if (!code.includes('__SP00KY_')) return null;
+    let out = code;
+    for (const [token, value] of Object.entries(replacements)) {
+      out = out.split(token).join(value);
+    }
+    return out;
+  },
+};
+
+// Worker URLs are written as `./<name>.ts` in source so Vite's src-bundling
+// consumers (the example app aliases core to `src`) resolve them. In the
+// published build each worker is emitted at `dist/<name>.js`, so rewrite the
+// references to `.js` here — otherwise the flat `dist/index.js` carries
+// dangling `.ts` URLs that a consumer's bundler can't resolve.
+const workerUrlPlugin = {
+  name: 'sp00ky-worker-url',
+  transform(code: string) {
+    if (!code.includes('sqlite-worker.ts') && !code.includes('tabs-broker-worker.ts')) return null;
+    return code
+      .split('sqlite-worker.ts')
+      .join('sqlite-worker.js')
+      .split('tabs-broker-worker.ts')
+      .join('tabs-broker-worker.js');
+  },
+};
 
 export default defineConfig({
-  entry: ['src/index.ts', 'src/otel/index.ts'],
+  // `sqlite-worker` is emitted at `dist/sqlite-worker.js` (top level, NOT under
+  // services/database) so the `new URL('./sqlite-worker.js', import.meta.url)`
+  // in `SqliteCacheEngine` — which gets bundled into the flat `dist/index.js` —
+  // resolves correctly. It (and `@sqlite.org/sqlite-wasm`) load lazily in the
+  // Worker; never pulled into the main bundle unless `localEngine: 'sqlite'`.
+  entry: {
+    index: 'src/index.ts',
+    'otel/index': 'src/otel/index.ts',
+    'sqlite-worker': 'src/services/database/sqlite-worker.ts',
+    // The shared-tabs broker MUST stay a self-contained emit (no chunk
+    // imports): bundlers cannot trace `new SharedWorker(url)` graphs, so a
+    // relative chunk import would 404 in consumers. The source keeps zero
+    // runtime imports and scripts/check-broker-bundle.mjs enforces it.
+    'tabs-broker-worker': 'src/services/tabs/tabs-broker-worker.ts',
+  },
   format: ['esm'],
   dts: true,
   clean: true,
   hash: false,
+  plugins: [versionDefinePlugin, workerUrlPlugin],
 });

package/src/spooky.ts

@@ -1,395 +0,0 @@
-import { DataModule } from './modules/data/index';
-import {
-  SpookyConfig,
-  QueryTimeToLive,
-  SpookyQueryResultPromise,
-  PersistenceClient,
-  MutationEvent,
-  UpdateOptions,
-  RunOptions,
-} from './types';
-import {
-  LocalDatabaseService,
-  LocalMigrator,
-  RemoteDatabaseService,
-} from './services/database/index';
-import { Surreal } from 'surrealdb';
-import { SpookySync, UpEvent } from './modules/sync/index';
-import {
-  GetTable,
-  InnerQuery,
-  QueryBuilder,
-  QueryOptions,
-  SchemaStructure,
-  TableModel,
-  TableNames,
-  BucketNames,
-  BackendNames,
-  BackendRoutes,
-  RoutePayload,
-} from '@spooky-sync/query-builder';
-
-import { DevToolsService } from './modules/devtools/index';
-import { createLogger } from './services/logger/index';
-import { AuthService } from './modules/auth/index';
-import { StreamProcessorService } from './services/stream-processor/index';
-import { EventSystem } from './events/index';
-import { CacheModule } from './modules/cache/index';
-import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
-import { generateId, parseParams } from './utils/index';
-import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
-import { ResilientPersistenceClient } from './services/persistence/resilient';
-
-export class BucketHandle {
-  constructor(private bucketName: string, private remote: RemoteDatabaseService) {}
-
-  async put(path: string, content: string | Uint8Array | Blob): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${path}".put($content);`, { content });
-  }
-
-  async get(path: string): Promise<unknown> {
-    const [result] = await this.remote.query<[unknown]>(`RETURN f"${this.bucketName}:/${path}".get();`);
-    return result;
-  }
-
-  async delete(path: string): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${path}".delete();`);
-  }
-
-  async exists(path: string): Promise<boolean> {
-    const [result] = await this.remote.query<[boolean]>(`RETURN f"${this.bucketName}:/${path}".exists();`);
-    return result;
-  }
-
-  async head(path: string): Promise<Record<string, unknown>> {
-    const [result] = await this.remote.query<[Record<string, unknown>]>(`RETURN f"${this.bucketName}:/${path}".head();`);
-    return result;
-  }
-
-  async copy(sourcePath: string, targetPath: string): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".copy($target);`, { target: targetPath });
-  }
-
-  async rename(sourcePath: string, targetPath: string): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".rename($target);`, { target: targetPath });
-  }
-
-  async list(prefix?: string): Promise<string[]> {
-    const p = prefix ?? '';
-    const [result] = await this.remote.query<[string[]]>(`RETURN f"${this.bucketName}:/${p}".list();`);
-    return result;
-  }
-}
-
-export class SpookyClient<S extends SchemaStructure> {
-  private local: LocalDatabaseService;
-  private remote: RemoteDatabaseService;
-  private persistenceClient: PersistenceClient;
-
-  private migrator: LocalMigrator;
-  private cache: CacheModule;
-  private dataModule: DataModule<S>;
-  private sync: SpookySync<S>;
-  private devTools: DevToolsService;
-
-  private logger: ReturnType<typeof createLogger>;
-  public auth: AuthService<S>;
-  public streamProcessor: StreamProcessorService;
-
-  get remoteClient() {
-    return this.remote.getClient();
-  }
-
-  get localClient() {
-    return this.local.getClient();
-  }
-
-  get pendingMutationCount(): number {
-    return this.sync.pendingMutationCount;
-  }
-
-  subscribeToPendingMutations(cb: (count: number) => void): () => void {
-    return this.sync.subscribeToPendingMutations(cb);
-  }
-
-  constructor(private config: SpookyConfig<S>) {
-    const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
-    this.logger = logger.child({ service: 'SpookyClient' });
-
-    this.logger.info(
-      {
-        config: { ...config, schema: '[SchemaStructure]' },
-        Category: 'spooky-client::SpookyClient::constructor',
-      },
-      'SpookyClient initialized'
-    );
-
-    this.local = new LocalDatabaseService(this.config.database, logger);
-    this.remote = new RemoteDatabaseService(this.config.database, logger);
-
-    if (config.persistenceClient === 'surrealdb') {
-      this.persistenceClient = new SurrealDBPersistenceClient(this.local, logger);
-    } else if (config.persistenceClient === 'localstorage' || !config.persistenceClient) {
-      this.persistenceClient = new LocalStoragePersistenceClient(logger);
-    } else {
-      this.persistenceClient = config.persistenceClient;
-    }
-
-    this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
-
-    this.streamProcessor = new StreamProcessorService(
-      new EventSystem(['stream_update']),
-      this.local,
-      this.persistenceClient,
-      logger
-    );
-    this.migrator = new LocalMigrator(this.local, logger);
-
-    this.cache = new CacheModule(
-      this.local,
-      this.streamProcessor,
-      (update) => {
-        // Direct callback from cache to data module
-        this.dataModule.onStreamUpdate(update);
-      },
-      logger
-    );
-
-    this.dataModule = new DataModule(
-      this.cache,
-      this.local,
-      this.config.schema,
-      logger,
-      this.config.streamDebounceTime
-    );
-
-    // Initialize Auth
-    this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
-
-    // Initialize Sync
-    this.sync = new SpookySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger);
-
-    // Initialize DevTools
-    this.devTools = new DevToolsService(
-      this.local,
-      this.remote,
-      logger,
-      this.config.schema,
-      this.auth,
-      this.dataModule
-    );
-
-    // Register DevTools as a receiver for stream updates
-    this.streamProcessor.addReceiver(this.devTools);
-
-    // Wire up callbacks instead of events
-    this.setupCallbacks();
-  }
-
-  /**
-   * Setup direct callbacks instead of event subscriptions
-   */
-  private setupCallbacks() {
-    // Mutation callback for sync
-    this.dataModule.onMutation((mutations: UpEvent[]) => {
-      // Notify DevTools
-      this.devTools.onMutation(mutations);
-
-      // Enqueue in Sync
-      if (mutations.length > 0) {
-        this.sync.enqueueMutation(mutations);
-      }
-    });
-
-    // Sync events for incoming updates
-    this.sync.events.subscribe('SYNC_QUERY_UPDATED', (event: any) => {
-      this.devTools.logEvent('SYNC_QUERY_UPDATED', event.payload);
-    });
-
-    // Database events for DevTools
-    this.local.getEvents().subscribe('DATABASE_LOCAL_QUERY', (event: any) => {
-      this.devTools.logEvent('LOCAL_QUERY', event.payload);
-    });
-
-    this.remote.getEvents().subscribe('DATABASE_REMOTE_QUERY', (event: any) => {
-      this.devTools.logEvent('REMOTE_QUERY', event.payload);
-    });
-  }
-
-  async init() {
-    this.logger.info(
-      { Category: 'spooky-client::SpookyClient::init' },
-      'SpookyClient initialization started'
-    );
-    try {
-      const clientId = this.config.clientId ?? (await this.loadOrGenerateClientId());
-      this.persistClientId(clientId);
-      this.logger.debug(
-        { clientId, Category: 'spooky-client::SpookyClient::init' },
-        'Client ID loaded'
-      );
-
-      await this.local.connect();
-      this.logger.debug(
-        { Category: 'spooky-client::SpookyClient::init' },
-        'Local database connected'
-      );
-
-      await this.migrator.provision(this.config.schemaSurql);
-      this.logger.debug({ Category: 'spooky-client::SpookyClient::init' }, 'Schema provisioned');
-
-      await this.remote.connect();
-      this.logger.debug(
-        { Category: 'spooky-client::SpookyClient::init' },
-        'Remote database connected'
-      );
-
-      await this.streamProcessor.init();
-      this.logger.debug(
-        { Category: 'spooky-client::SpookyClient::init' },
-        'StreamProcessor initialized'
-      );
-
-      await this.auth.init();
-      this.logger.debug({ Category: 'spooky-client::SpookyClient::init' }, 'Auth initialized');
-
-      await this.dataModule.init();
-      this.logger.debug(
-        { Category: 'spooky-client::SpookyClient::init' },
-        'DataModule initialized'
-      );
-
-      await this.sync.init(clientId);
-      this.logger.debug({ Category: 'spooky-client::SpookyClient::init' }, 'Sync initialized');
-
-      this.logger.info(
-        { Category: 'spooky-client::SpookyClient::init' },
-        'SpookyClient initialization completed successfully'
-      );
-    } catch (e) {
-      this.logger.error(
-        { error: e, Category: 'spooky-client::SpookyClient::init' },
-        'SpookyClient initialization failed'
-      );
-      throw e;
-    }
-  }
-
-  async close() {
-    await this.local.close();
-    await this.remote.close();
-  }
-
-  authenticate(token: string) {
-    return this.remote.getClient().authenticate(token);
-  }
-
-  deauthenticate() {
-    return this.remote.getClient().invalidate();
-  }
-
-  query<Table extends TableNames<S>>(
-    table: Table,
-    options: QueryOptions<TableModel<GetTable<S, Table>>, false>,
-    ttl: QueryTimeToLive = '10m'
-  ): QueryBuilder<S, Table, SpookyQueryResultPromise> {
-    return new QueryBuilder<S, Table, SpookyQueryResultPromise>(
-      this.config.schema,
-      table,
-      async (q) => ({
-        hash: await this.initQuery(table, q, ttl),
-      }),
-      options
-    );
-  }
-
-  private async initQuery<Table extends TableNames<S>>(
-    table: Table,
-    q: InnerQuery<any, any, any>,
-    ttl: QueryTimeToLive
-  ) {
-    const tableSchema = this.config.schema.tables.find((t) => t.name === table);
-    if (!tableSchema) {
-      throw new Error(`Table ${table} not found`);
-    }
-
-    const hash = await this.dataModule.query(
-      table,
-      q.selectQuery.query,
-      parseParams(tableSchema.columns, q.selectQuery.vars ?? {}),
-      ttl
-    );
-
-    await this.sync.enqueueDownEvent({
-      type: 'register',
-      payload: {
-        hash,
-      },
-    });
-
-    return hash;
-  }
-
-  async queryRaw(sql: string, params: Record<string, any>, ttl: QueryTimeToLive) {
-    const tableName = sql.split('FROM ')[1].split(' ')[0];
-    return this.dataModule.query(tableName, sql, params, ttl);
-  }
-
-  async subscribe(
-    queryHash: string,
-    callback: (records: Record<string, any>[]) => void,
-    options?: { immediate?: boolean }
-  ): Promise<() => void> {
-    return this.dataModule.subscribe(queryHash, callback, options);
-  }
-
-  run<
-    B extends BackendNames<S>,
-    R extends BackendRoutes<S, B>,
-  >(backend: B, path: R, payload: RoutePayload<S, B, R>, options?: RunOptions) {
-    return this.dataModule.run(backend, path, payload, options);
-  }
-
-  bucket<B extends BucketNames<S>>(name: B): BucketHandle {
-    return new BucketHandle(name, this.remote);
-  }
-
-  create(id: string, data: Record<string, unknown>) {
-    return this.dataModule.create(id, data);
-  }
-
-  update(table: string, id: string, data: Record<string, unknown>, options?: UpdateOptions) {
-    return this.dataModule.update(table, id, data, options);
-  }
-
-  delete(table: string, id: string) {
-    return this.dataModule.delete(table, id);
-  }
-
-  async useRemote<T>(fn: (client: Surreal) => Promise<T> | T): Promise<T> {
-    return fn(this.remote.getClient());
-  }
-
-  private persistClientId(id: string) {
-    try {
-      this.persistenceClient.set('spooky_client_id', id);
-    } catch (e) {
-      this.logger.warn(
-        { error: e, Category: 'spooky-client::SpookyClient::persistClientId' },
-        'Failed to persist client ID'
-      );
-    }
-  }
-
-  private async loadOrGenerateClientId(): Promise<string> {
-    const clientId = await this.persistenceClient.get<string>('spooky_client_id');
-
-    if (clientId) {
-      return clientId;
-    }
-
-    const newId = generateId();
-    await this.persistClientId(newId);
-    return newId;
-  }
-}