@spooky-sync/core 0.0.1-canary.17 → 0.0.1-canary.170

package/src/utils/index.ts

@@ -1,7 +1,7 @@
-import { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
+import type { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
 import { Uuid, RecordId, Duration } from 'surrealdb';
-import { Logger } from '../services/logger/index';
-import { QueryTimeToLive } from '../types';
+import type { Logger } from '../services/logger/index';
+import type { QueryTimeToLive } from '../types';
 
 export * from './surql';
 export * from './parser';
@@ -59,7 +59,7 @@ export function generateNewTableId<S extends SchemaStructure, T extends TableNam
 
 // ==================== SCHEMA ENCODING/DECODING ====================
 
-export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames<S>>(
+export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames<S>>(
   schema: S,
   tableName: T,
   record: TableModel<GetTable<S, T>>
@@ -74,19 +74,19 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
   for (const field of Object.keys(table.columns)) {
     const column = table.columns[field] as any;
     const relation = schema.relationships.find((r) => r.from === tableName && r.field === field);
-    if ((column.recordId || relation) && encoded[field] != null) {
+    if ((column.recordId || relation) && encoded[field] !== null && encoded[field] !== undefined) {
       if (encoded[field] instanceof RecordId) {
         encoded[field] = `${encoded[field].table.toString()}:${encoded[field].id}`;
       } else if (
         relation &&
-        (encoded[field] instanceof Object || encoded[field] instanceof Array)
+        (encoded[field] instanceof Object || Array.isArray(encoded[field]))
       ) {
         if (Array.isArray(encoded[field])) {
           encoded[field] = encoded[field].map((item) =>
-            decodeFromSpooky(schema, relation.to, item)
+            decodeFromSp00ky(schema, relation.to, item)
           );
         } else {
-          encoded[field] = decodeFromSpooky(schema, relation.to, encoded[field]);
+          encoded[field] = decodeFromSp00ky(schema, relation.to, encoded[field]);
         }
       }
     }
@@ -97,15 +97,25 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
 
 // ==================== TIME/DURATION UTILITIES ====================
 
+/**
+ * Read the millisecond count off a surrealdb `Duration`. Different `surrealdb`
+ * releases expose it under `milliseconds` (current) or the older private
+ * `_milliseconds` field, so read whichever is set; returns 0 when neither is.
+ */
+function durationMillis(duration: Duration): number {
+  const d = duration as { milliseconds?: number | bigint; _milliseconds?: number | bigint };
+  return Number(d.milliseconds || d._milliseconds || 0);
+}
+
 /**
  * Parse duration string or Duration object to milliseconds
  */
 export function parseDuration(duration: QueryTimeToLive | Duration): number {
   if (duration instanceof Duration) {
-    const ms = (duration as any).milliseconds || (duration as any)._milliseconds;
-    if (ms) return Number(ms);
+    const ms = durationMillis(duration);
+    if (ms) return ms;
     const str = duration.toString();
-    if (str !== '[object Object]') return parseDuration(str as any);
+    if (str !== '[object Object]') return parseDuration(str as QueryTimeToLive);
     return 600000;
   }
 
@@ -117,7 +127,7 @@ export function parseDuration(duration: QueryTimeToLive | Duration): number {
 
   const match = duration.match(/^(\d+)([smh])$/);
   if (!match) return 600000;
-  const val = parseInt(match[1], 10);
+  const val = Number.parseInt(match[1], 10);
   const unit = match[2];
   switch (unit) {
     case 's':
@@ -137,6 +147,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
   return new Uint8Array(buffer);
 }
 
+// ==================== TEXT UTILITIES ====================
+
+/**
+ * Convert plain text to simple HTML paragraphs.
+ * Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
+ */
+export function textToHtml(text: string): string {
+  return text
+    .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+    .split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
+}
+
 // ==================== DATABASE UTILITIES ====================
 
 /**
@@ -165,7 +187,7 @@ export async function withRetry<T>(
             attempt: i + 1,
             retries,
             error: msg,
-            Category: 'spooky-client::utils::withRetry',
+            Category: 'sp00ky-client::utils::withRetry',
           },
           'Retrying DB operation'
         );
@@ -177,3 +199,41 @@ export async function withRetry<T>(
   }
   throw lastError;
 }
+
+/**
+ * Reject after `timeoutMs` if `promise` hasn't settled.
+ *
+ * Exists because a WebSocket RPC has no deadline of its own: the SurrealDB SDK
+ * parks each call in a pending map and only rejects it when the socket reports
+ * a close. On a half-open socket (peer gone, no `close` event, `readyState`
+ * still OPEN) the call never settles at all, which wedges anything serialized
+ * behind it.
+ *
+ * `message` MUST contain "timed out" so `classifySyncError` treats the
+ * rejection as `network` — that's what makes the sync queues retry the
+ * operation instead of rolling the mutation back as an application error.
+ *
+ * Non-positive `timeoutMs` returns `promise` unchanged. The underlying promise
+ * is NOT cancelled (nothing can cancel an in-flight RPC); its eventual
+ * settlement is absorbed, so attach no expectations to it after a timeout.
+ */
+export function withTimeout<T>(
+  promise: Promise<T>,
+  timeoutMs: number,
+  message: string
+): Promise<T> {
+  if (!(timeoutMs > 0)) return promise;
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (err) => {
+        clearTimeout(timer);
+        reject(err);
+      }
+    );
+  });
+}

package/src/utils/parser.ts

@@ -1,4 +1,5 @@
-import { ColumnSchema, RecordId } from '@spooky-sync/query-builder';
+import type { ColumnSchema} from '@spooky-sync/query-builder';
+import { RecordId } from '@spooky-sync/query-builder';
 import { parseRecordIdString } from './index';
 import { DateTime } from 'surrealdb';
 
@@ -8,7 +9,7 @@ export function cleanRecord(
 ): Record<string, any> {
   const cleaned: Record<string, any> = {};
   for (const [key, value] of Object.entries(record)) {
-    if (key === 'id' || key in tableSchema) {
+    if (key === 'id' || key.startsWith('_00_') || key in tableSchema) {
       cleaned[key] = value;
     }
   }

package/src/utils/semver.test.ts

@@ -0,0 +1,32 @@
+import { describe, it, expect } from 'vitest';
+import { semverGt } from './semver';
+
+describe('semverGt', () => {
+  it('compares strictly greater', () => {
+    expect(semverGt('1.0.1', '1.0.0')).toBe(true);
+    expect(semverGt('1.1.0', '1.0.9')).toBe(true);
+    expect(semverGt('2.0.0', '1.9.9')).toBe(true);
+    expect(semverGt('1.10.0', '1.9.0')).toBe(true); // numeric, not lexicographic
+  });
+
+  it('equal and lower are not greater', () => {
+    expect(semverGt('1.0.0', '1.0.0')).toBe(false);
+    expect(semverGt('1.0.0', '1.0.1')).toBe(false);
+    expect(semverGt('0.9.9', '1.0.0')).toBe(false);
+  });
+
+  it('missing parts default to zero', () => {
+    expect(semverGt('1.2', '1.2.0')).toBe(false);
+    expect(semverGt('1.2.1', '1.2')).toBe(true);
+    expect(semverGt('2', '1.9.9')).toBe(true);
+  });
+
+  it('junk never compares greater', () => {
+    expect(semverGt('abc', '1.0.0')).toBe(false);
+    expect(semverGt('1.0.0', 'abc')).toBe(false);
+    expect(semverGt('', '0.0.0')).toBe(false);
+    expect(semverGt(null, undefined)).toBe(false);
+    expect(semverGt('1.0.0-beta', '0.9.0')).toBe(false); // prerelease unsupported
+    expect(semverGt('1.2.3.4', '1.0.0')).toBe(false);
+  });
+});

package/src/utils/semver.ts

@@ -0,0 +1,30 @@
+// Minimal semver comparison for app-release version checks. "X.Y.Z" numeric
+// with missing parts read as 0 ("1.2" == "1.2.0"); any malformed input never
+// compares greater, so a bad release row can never nag (or force-reload)
+// every client.
+
+function parse(v: unknown): [number, number, number] | null {
+  const parts = String(v ?? '')
+    .trim()
+    .split('.');
+  if (parts.length === 0 || parts.length > 3 || parts[0] === '') return null;
+  const nums: number[] = [];
+  for (let i = 0; i < 3; i++) {
+    const raw = parts[i] ?? '0';
+    if (!/^\d+$/.test(raw)) return null;
+    nums.push(parseInt(raw, 10));
+  }
+  return nums as [number, number, number];
+}
+
+/** True when `a` is a valid version strictly greater than valid version `b`. */
+export function semverGt(a: unknown, b: unknown): boolean {
+  const pa = parse(a);
+  const pb = parse(b);
+  if (!pa || !pb) return false;
+  for (let i = 0; i < 3; i++) {
+    if (pa[i] > pb[i]) return true;
+    if (pa[i] < pb[i]) return false;
+  }
+  return false;
+}

package/src/utils/surql.ts

@@ -1,4 +1,4 @@
-import { MutationEventType } from '../types';
+import type { MutationEventType } from '../types';
 
 // ==================== TYPES ====================
 
@@ -34,6 +34,7 @@ export interface SurqlHelper {
     keyDataVars: ({ key: string; variable: string } | { statement: string } | string)[]
   ): string;
   upsert(idVar: string, dataVar: string): string;
+  upsertMerge(idVar: string, dataVar: string): string;
   updateMerge(idVar: string, dataVar: string): string;
   updateSet(
     idVar: string,
@@ -90,17 +91,9 @@ export const surql: SurqlHelper = {
     returnValues: ({ field: string; alias: string } | string)[]
   ) {
     return `SELECT ${returnValues
-      .map((returnValues) =>
-        typeof returnValues === 'string'
-          ? returnValues
-          : `${returnValues.field} as ${returnValues.alias}`
-      )
+      .map((rv) => (typeof rv === 'string' ? rv : `${rv.field} as ${rv.alias}`))
       .join(',')} FROM ${table} WHERE ${whereVar
-      .map((whereVar) =>
-        typeof whereVar === 'string'
-          ? `${whereVar} = $${whereVar}`
-          : `${whereVar.field} = $${whereVar.variable}`
-      )
+      .map((wv) => (typeof wv === 'string' ? `${wv} = $${wv}` : `${wv.field} = $${wv.variable}`))
       .join(' AND ')}`;
   },
 
@@ -127,6 +120,14 @@ export const surql: SurqlHelper = {
     return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
   },
 
+  // Use this for sync-down ingestion: the remote payload omits local-only
+  // fields injected by the CLI's local-schema setup (`_00_crdt`,
+  // `_00_cursor`), and REPLACE would wipe them on every round-trip and
+  // break offline reload of CRDT state.
+  upsertMerge(idVar: string, dataVar: string) {
+    return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
+  },
+
   updateMerge(idVar: string, dataVar: string) {
     return `UPDATE ONLY $${idVar} MERGE $${dataVar}`;
   },
@@ -136,12 +137,12 @@ export const surql: SurqlHelper = {
     keyDataVar: ({ key: string; variable: string } | { statement: string } | string)[]
   ) {
     return `UPDATE $${idVar} SET ${keyDataVar
-      .map((keyDataVar) =>
-        typeof keyDataVar === 'string'
-          ? `${keyDataVar} = $${keyDataVar}`
-          : 'statement' in keyDataVar
-            ? keyDataVar.statement
-            : `${keyDataVar.key} = $${keyDataVar.variable}`
+      .map((kdv) =>
+        typeof kdv === 'string'
+          ? `${kdv} = $${kdv}`
+          : 'statement' in kdv
+            ? kdv.statement
+            : `${kdv.key} = $${kdv.variable}`
       )
       .join(', ')}`;
   },
@@ -163,7 +164,18 @@ export const surql: SurqlHelper = {
   ) {
     switch (t) {
       case 'create':
-        return `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
+        // `data` is REQUIRED here, not decorative. The outbox row is the only
+        // copy of a pending create once the in-memory UpEvent is gone (reload,
+        // or a shared-tabs follower whose row is replayed by the leader). This
+        // used to drop `dataVar` on the floor, so every replayed create arrived
+        // with `data: undefined` and `processUpEvent` threw on
+        // `Object.keys(event.data)` before it ever reached the network: the
+        // create was unsendable AND it sat at the head of the queue blocking
+        // every later mutation. The payload was simply never persisted, so
+        // those creates were unrecoverable.
+        return dataVar
+          ? `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}, data = $${dataVar}`
+          : `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
       case 'update': {
         let stmt = `CREATE ONLY $${mutationIdVar} SET mutationType = 'update', recordId = $${recordIdVar}, data = $${dataVar}`;
         if (beforeRecordVar) {

package/src/utils/withRetry.test.ts

@@ -145,7 +145,7 @@ describe('withRetry', () => {
         attempt: 1,
         retries: 3,
         error: 'Can not open transaction',
-        Category: 'spooky-client::utils::withRetry',
+        Category: 'sp00ky-client::utils::withRetry',
       }),
       'Retrying DB operation'
     );

package/tsdown.config.ts

@@ -1,9 +1,94 @@
 import { defineConfig } from 'tsdown';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+// Resolve real package versions at build time so the DevTools can report the
+// actual bundled frontend versions (used to detect frontend/backend drift).
+const corePkg = require('./package.json');
+const coreVersion: string = corePkg.version;
+// ssp-wasm has no `exports` map; resolve its package.json directly, falling
+// back to the relative workspace path if module resolution can't find it.
+let wasmVersion: string;
+try {
+  wasmVersion = require('@spooky-sync/ssp-wasm/package.json').version;
+} catch {
+  wasmVersion = require('../ssp-wasm/package.json').version;
+}
+// Report the SurrealDB WASM ENGINE version (`@surrealdb/wasm`) — the engine the
+// in-browser local DB actually runs — NOT the `surrealdb` JS client library
+// version. The WASM engine is the one that must stay compatible with the server
+// engine, so it's the meaningful number to compare in DevTools. Prefer the
+// actually-installed version; fall back to our pinned range if its `exports`
+// map blocks importing its package.json.
+let surrealVersion: string;
+try {
+  surrealVersion = require('@surrealdb/wasm/package.json').version;
+} catch {
+  surrealVersion = String(corePkg.dependencies?.['@surrealdb/wasm'] ?? 'unknown').replace(
+    /^[\^~]/,
+    ''
+  );
+}
+
+// tsdown 0.12.x forwards a top-level `define` to rolldown's inputOptions, which
+// rejects it — so we do the substitution ourselves with a tiny transform
+// plugin. Only our own modules reference the `__SP00KY_*__` identifiers, and
+// the early `includes` check skips everything else.
+const replacements: Record<string, string> = {
+  __SP00KY_CORE_VERSION__: JSON.stringify(coreVersion),
+  __SP00KY_WASM_VERSION__: JSON.stringify(wasmVersion),
+  __SP00KY_SURREAL_VERSION__: JSON.stringify(surrealVersion),
+};
+
+const versionDefinePlugin = {
+  name: 'sp00ky-version-define',
+  transform(code: string) {
+    if (!code.includes('__SP00KY_')) return null;
+    let out = code;
+    for (const [token, value] of Object.entries(replacements)) {
+      out = out.split(token).join(value);
+    }
+    return out;
+  },
+};
+
+// Worker URLs are written as `./<name>.ts` in source so Vite's src-bundling
+// consumers (the example app aliases core to `src`) resolve them. In the
+// published build each worker is emitted at `dist/<name>.js`, so rewrite the
+// references to `.js` here — otherwise the flat `dist/index.js` carries
+// dangling `.ts` URLs that a consumer's bundler can't resolve.
+const workerUrlPlugin = {
+  name: 'sp00ky-worker-url',
+  transform(code: string) {
+    if (!code.includes('sqlite-worker.ts') && !code.includes('tabs-broker-worker.ts')) return null;
+    return code
+      .split('sqlite-worker.ts')
+      .join('sqlite-worker.js')
+      .split('tabs-broker-worker.ts')
+      .join('tabs-broker-worker.js');
+  },
+};
 
 export default defineConfig({
-  entry: ['src/index.ts'],
+  // `sqlite-worker` is emitted at `dist/sqlite-worker.js` (top level, NOT under
+  // services/database) so the `new URL('./sqlite-worker.js', import.meta.url)`
+  // in `SqliteCacheEngine` — which gets bundled into the flat `dist/index.js` —
+  // resolves correctly. It (and `@sqlite.org/sqlite-wasm`) load lazily in the
+  // Worker; never pulled into the main bundle unless `localEngine: 'sqlite'`.
+  entry: {
+    index: 'src/index.ts',
+    'otel/index': 'src/otel/index.ts',
+    'sqlite-worker': 'src/services/database/sqlite-worker.ts',
+    // The shared-tabs broker MUST stay a self-contained emit (no chunk
+    // imports): bundlers cannot trace `new SharedWorker(url)` graphs, so a
+    // relative chunk import would 404 in consumers. The source keeps zero
+    // runtime imports and scripts/check-broker-bundle.mjs enforces it.
+    'tabs-broker-worker': 'src/services/tabs/tabs-broker-worker.ts',
+  },
   format: ['esm'],
   dts: true,
   clean: true,
   hash: false,
+  plugins: [versionDefinePlugin, workerUrlPlugin],
 });