@spooky-sync/core 0.0.1-canary.33 → 0.0.1-canary.331

package/src/utils/parser.ts

@@ -1,4 +1,5 @@
-import { ColumnSchema, RecordId } from '@spooky-sync/query-builder';
+import type { ColumnSchema} from '@spooky-sync/query-builder';
+import { RecordId, baseFieldOfParam } from '@spooky-sync/query-builder';
 import { parseRecordIdString } from './index';
 import { DateTime } from 'surrealdb';
 
@@ -8,13 +9,49 @@ export function cleanRecord(
 ): Record<string, any> {
   const cleaned: Record<string, any> = {};
   for (const [key, value] of Object.entries(record)) {
-    if (key === 'id' || key in tableSchema) {
+    if (key === 'id' || key.startsWith('_00_') || key in tableSchema) {
       cleaned[key] = value;
     }
   }
   return cleaned;
 }
 
+/**
+ * The OPTIONAL schema columns a full server row does not carry, each mapped to
+ * `undefined`, for spreading into a body that is landed with MERGE.
+ *
+ * The server omits an optional field that is unset (`UPDATE t UNSET club`
+ * leaves no `club` key in `SELECT *` or the SSP body), and MERGE keeps every
+ * key the patch does not mention, so without this the local store held on to
+ * a cleared value forever. `undefined` is the one representation both engines
+ * read as "remove": the SurrealDB SDK encodes it as NONE, which MERGE removes
+ * (NULL would be a type error on an `option<record<...>>` field), and the
+ * SQLite engine turns it into the `null` member that `json_patch` removes (see
+ * `serializeMergeRow`).
+ *
+ * Required columns are left alone. A SCHEMAFULL server cannot hold NONE in
+ * one, so its absence means the row was not readable in full (a field
+ * permission), not that it was cleared, and NONE in a typed local store is a
+ * coercion error that would fail the whole chunk's transaction. `id` and the
+ * local-only `_00_*` bookkeeping are never touched.
+ */
+export function absentOptionalColumns(
+  tableSchema: Record<string, ColumnSchema> | undefined,
+  record: Record<string, unknown>
+): Record<string, undefined> {
+  const cleared: Record<string, undefined> = {};
+  if (!tableSchema) return cleared;
+  for (const [key, column] of Object.entries(tableSchema)) {
+    if (key === 'id' || key.startsWith('_00_') || column?.optional !== true) continue;
+    if (record[key] === undefined) cleared[key] = undefined;
+  }
+  return cleared;
+}
+
+/**
+ * Parse a RECORD's fields against the table schema. Anything the schema does not
+ * know is dropped, which is what keeps a stray field out of a write.
+ */
 export function parseParams(
   tableSchema: Record<string, ColumnSchema>,
   params: Record<string, any>
@@ -30,6 +67,31 @@ export function parseParams(
   return parsedParams;
 }
 
+/**
+ * Parse a QUERY's params. Unlike a record's fields, a param name is not always a
+ * column: an `_or` branch binds under a synthetic `white__or0` (see
+ * `orParamName` in @spooky-sync/query-builder), and the surql that references it
+ * is already written. So resolve the column through the synthetic name, and keep
+ * a param we cannot type rather than dropping it.
+ *
+ * Dropping was the bug: `parseParams` kept only column-named params, so every
+ * `_or` query registered with `$or0`/`$or1` unbound and matched no rows at all,
+ * silently (rowCount 0, errorCount 0).
+ */
+export function parseQueryParams(
+  tableSchema: Record<string, ColumnSchema>,
+  params: Record<string, any>
+) {
+  const parsedParams: Record<string, any> = {};
+  for (const [key, value] of Object.entries(params)) {
+    if (value === undefined) continue;
+    const column = tableSchema[key] ?? tableSchema[baseFieldOfParam(key)];
+    parsedParams[key] = column ? parseValue(key, column, value) : value;
+  }
+
+  return parsedParams;
+}
+
 function parseValue(name: string, column: ColumnSchema, value: any) {
   if (column.recordId) {
     if (value instanceof RecordId) return value;

package/src/utils/semver.test.ts

@@ -0,0 +1,32 @@
+import { describe, it, expect } from 'vitest';
+import { semverGt } from './semver';
+
+describe('semverGt', () => {
+  it('compares strictly greater', () => {
+    expect(semverGt('1.0.1', '1.0.0')).toBe(true);
+    expect(semverGt('1.1.0', '1.0.9')).toBe(true);
+    expect(semverGt('2.0.0', '1.9.9')).toBe(true);
+    expect(semverGt('1.10.0', '1.9.0')).toBe(true); // numeric, not lexicographic
+  });
+
+  it('equal and lower are not greater', () => {
+    expect(semverGt('1.0.0', '1.0.0')).toBe(false);
+    expect(semverGt('1.0.0', '1.0.1')).toBe(false);
+    expect(semverGt('0.9.9', '1.0.0')).toBe(false);
+  });
+
+  it('missing parts default to zero', () => {
+    expect(semverGt('1.2', '1.2.0')).toBe(false);
+    expect(semverGt('1.2.1', '1.2')).toBe(true);
+    expect(semverGt('2', '1.9.9')).toBe(true);
+  });
+
+  it('junk never compares greater', () => {
+    expect(semverGt('abc', '1.0.0')).toBe(false);
+    expect(semverGt('1.0.0', 'abc')).toBe(false);
+    expect(semverGt('', '0.0.0')).toBe(false);
+    expect(semverGt(null, undefined)).toBe(false);
+    expect(semverGt('1.0.0-beta', '0.9.0')).toBe(false); // prerelease unsupported
+    expect(semverGt('1.2.3.4', '1.0.0')).toBe(false);
+  });
+});

package/src/utils/semver.ts

@@ -0,0 +1,30 @@
+// Minimal semver comparison for app-release version checks. "X.Y.Z" numeric
+// with missing parts read as 0 ("1.2" == "1.2.0"); any malformed input never
+// compares greater, so a bad release row can never nag (or force-reload)
+// every client.
+
+function parse(v: unknown): [number, number, number] | null {
+  const parts = String(v ?? '')
+    .trim()
+    .split('.');
+  if (parts.length === 0 || parts.length > 3 || parts[0] === '') return null;
+  const nums: number[] = [];
+  for (let i = 0; i < 3; i++) {
+    const raw = parts[i] ?? '0';
+    if (!/^\d+$/.test(raw)) return null;
+    nums.push(parseInt(raw, 10));
+  }
+  return nums as [number, number, number];
+}
+
+/** True when `a` is a valid version strictly greater than valid version `b`. */
+export function semverGt(a: unknown, b: unknown): boolean {
+  const pa = parse(a);
+  const pb = parse(b);
+  if (!pa || !pb) return false;
+  for (let i = 0; i < 3; i++) {
+    if (pa[i] > pb[i]) return true;
+    if (pa[i] < pb[i]) return false;
+  }
+  return false;
+}

package/src/utils/sha256.ts

@@ -0,0 +1,15 @@
+/**
+ * Lowercase hex SHA-256 of a UTF-8 string, through WebCrypto.
+ *
+ * Pure and dependency free so it can live in `@spooky-sync/core/pure` and run
+ * in a service worker, a dedicated worker or Node (`globalThis.crypto` is
+ * there on Node 19+). Query keys are `sha256Hex(JSON.stringify(..))`, see
+ * `query/hash.ts`.
+ */
+export async function sha256Hex(input: string): Promise<string> {
+  const bytes = new TextEncoder().encode(input);
+  const digest = await crypto.subtle.digest('SHA-256', bytes);
+  return Array.from(new Uint8Array(digest))
+    .map((b) => b.toString(16).padStart(2, '0'))
+    .join('');
+}

package/src/utils/surql.ts

@@ -1,4 +1,4 @@
-import { MutationEventType } from '../types';
+import type { MutationEventType } from '../types';
 
 // ==================== TYPES ====================
 
@@ -34,6 +34,7 @@ export interface SurqlHelper {
     keyDataVars: ({ key: string; variable: string } | { statement: string } | string)[]
   ): string;
   upsert(idVar: string, dataVar: string): string;
+  upsertMerge(idVar: string, dataVar: string): string;
   updateMerge(idVar: string, dataVar: string): string;
   updateSet(
     idVar: string,
@@ -90,17 +91,9 @@ export const surql: SurqlHelper = {
     returnValues: ({ field: string; alias: string } | string)[]
   ) {
     return `SELECT ${returnValues
-      .map((returnValues) =>
-        typeof returnValues === 'string'
-          ? returnValues
-          : `${returnValues.field} as ${returnValues.alias}`
-      )
+      .map((rv) => (typeof rv === 'string' ? rv : `${rv.field} as ${rv.alias}`))
       .join(',')} FROM ${table} WHERE ${whereVar
-      .map((whereVar) =>
-        typeof whereVar === 'string'
-          ? `${whereVar} = $${whereVar}`
-          : `${whereVar.field} = $${whereVar.variable}`
-      )
+      .map((wv) => (typeof wv === 'string' ? `${wv} = $${wv}` : `${wv.field} = $${wv.variable}`))
       .join(' AND ')}`;
   },
 
@@ -127,6 +120,13 @@ export const surql: SurqlHelper = {
     return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
   },
 
+  // Use this for sync-down ingestion: the remote payload can omit local-only
+  // fields injected by the CLI's local-schema setup, and REPLACE would wipe
+  // them on every round-trip.
+  upsertMerge(idVar: string, dataVar: string) {
+    return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
+  },
+
   updateMerge(idVar: string, dataVar: string) {
     return `UPDATE ONLY $${idVar} MERGE $${dataVar}`;
   },
@@ -136,12 +136,12 @@ export const surql: SurqlHelper = {
     keyDataVar: ({ key: string; variable: string } | { statement: string } | string)[]
   ) {
     return `UPDATE $${idVar} SET ${keyDataVar
-      .map((keyDataVar) =>
-        typeof keyDataVar === 'string'
-          ? `${keyDataVar} = $${keyDataVar}`
-          : 'statement' in keyDataVar
-            ? keyDataVar.statement
-            : `${keyDataVar.key} = $${keyDataVar.variable}`
+      .map((kdv) =>
+        typeof kdv === 'string'
+          ? `${kdv} = $${kdv}`
+          : 'statement' in kdv
+            ? kdv.statement
+            : `${kdv.key} = $${kdv.variable}`
       )
       .join(', ')}`;
   },
@@ -163,7 +163,18 @@ export const surql: SurqlHelper = {
   ) {
     switch (t) {
       case 'create':
-        return `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
+        // `data` is REQUIRED here, not decorative. The outbox row is the only
+        // copy of a pending create once the in-memory UpEvent is gone (reload,
+        // or a shared-tabs follower whose row is replayed by the leader). This
+        // used to drop `dataVar` on the floor, so every replayed create arrived
+        // with `data: undefined` and `processUpEvent` threw on
+        // `Object.keys(event.data)` before it ever reached the network: the
+        // create was unsendable AND it sat at the head of the queue blocking
+        // every later mutation. The payload was simply never persisted, so
+        // those creates were unrecoverable.
+        return dataVar
+          ? `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}, data = $${dataVar}`
+          : `CREATE ONLY $${mutationIdVar} SET mutationType = 'create', recordId = $${recordIdVar}`;
       case 'update': {
         let stmt = `CREATE ONLY $${mutationIdVar} SET mutationType = 'update', recordId = $${recordIdVar}, data = $${dataVar}`;
         if (beforeRecordVar) {

package/src/utils/utils.test.ts

@@ -9,6 +9,7 @@ import {
   generateId,
   parseDuration,
 } from './index';
+import type { QueryTimeToLive } from '../types';
 
 describe('compareRecordIds', () => {
   it('returns true for equal strings', () => {
@@ -104,7 +105,7 @@ describe('parseRecordIdString', () => {
 
   it('parses id containing colons', () => {
     const rid = parseRecordIdString('table:part1:part2');
-    expect(rid.table.toString()).toBe('table');
+    expect(rid.table.name).toBe('table');
     expect(rid.id).toBe('part1:part2');
   });
 });
@@ -123,7 +124,7 @@ describe('generateId', () => {
 
 describe('parseDuration', () => {
   it('parses seconds', () => {
-    expect(parseDuration('30s')).toBe(30000);
+    expect(parseDuration('30s' as QueryTimeToLive)).toBe(30000);
   });
 
   it('parses minutes', () => {

package/tsdown.config.ts

@@ -1,9 +1,123 @@
 import { defineConfig } from 'tsdown';
+import { createRequire } from 'node:module';
 
-export default defineConfig({
-  entry: ['src/index.ts', 'src/otel/index.ts'],
+const require = createRequire(import.meta.url);
+
+// Resolve real package versions at build time so the DevTools can report the
+// actual bundled frontend versions (used to detect frontend/backend drift).
+const corePkg = require('./package.json');
+const coreVersion: string = corePkg.version;
+// ssp-wasm has no `exports` map; resolve its package.json directly, falling
+// back to the relative workspace path if module resolution can't find it.
+let wasmVersion: string;
+try {
+  wasmVersion = require('@spooky-sync/ssp-wasm/package.json').version;
+} catch {
+  wasmVersion = require('../ssp-wasm/package.json').version;
+}
+// Report the SurrealDB WASM ENGINE version (`@surrealdb/wasm`) — the engine the
+// in-browser local DB actually runs — NOT the `surrealdb` JS client library
+// version. The WASM engine is the one that must stay compatible with the server
+// engine, so it's the meaningful number to compare in DevTools. Prefer the
+// actually-installed version; fall back to our pinned range if its `exports`
+// map blocks importing its package.json.
+let surrealVersion: string;
+try {
+  surrealVersion = require('@surrealdb/wasm/package.json').version;
+} catch {
+  surrealVersion = String(corePkg.dependencies?.['@surrealdb/wasm'] ?? 'unknown').replace(
+    /^[\^~]/,
+    ''
+  );
+}
+
+// tsdown 0.12.x forwards a top-level `define` to rolldown's inputOptions, which
+// rejects it — so we do the substitution ourselves with a tiny transform
+// plugin. Only our own modules reference the `__SP00KY_*__` identifiers, and
+// the early `includes` check skips everything else.
+const replacements: Record<string, string> = {
+  __SP00KY_CORE_VERSION__: JSON.stringify(coreVersion),
+  __SP00KY_WASM_VERSION__: JSON.stringify(wasmVersion),
+  __SP00KY_SURREAL_VERSION__: JSON.stringify(surrealVersion),
+};
+
+const versionDefinePlugin = {
+  name: 'sp00ky-version-define',
+  transform(code: string) {
+    if (!code.includes('__SP00KY_')) return null;
+    let out = code;
+    for (const [token, value] of Object.entries(replacements)) {
+      out = out.split(token).join(value);
+    }
+    return out;
+  },
+};
+
+// Worker URLs are written as `./<name>.ts` in source so Vite's src-bundling
+// consumers (the example app aliases core to `src`) resolve them. In the
+// published build each worker is emitted at `dist/<name>.js`, so rewrite the
+// references to `.js` here — otherwise the flat `dist/index.js` carries
+// dangling `.ts` URLs that a consumer's bundler can't resolve.
+const workerUrlPlugin = {
+  name: 'sp00ky-worker-url',
+  transform(code: string) {
+    if (!code.includes('sqlite-worker.ts') && !code.includes('tabs-broker-worker.ts')) return null;
+    return code
+      .split('sqlite-worker.ts')
+      .join('sqlite-worker.js')
+      .split('tabs-broker-worker.ts')
+      .join('tabs-broker-worker.js');
+  },
+};
+
+export default defineConfig([
+  {
+  // `sqlite-worker` is emitted at `dist/sqlite-worker.js` (top level, NOT under
+  // services/database) so the `new URL('./sqlite-worker.js', import.meta.url)`
+  // in `SqliteCacheEngine` — which gets bundled into the flat `dist/index.js` —
+  // resolves correctly. It (and `@sqlite.org/sqlite-wasm`) load lazily in the
+  // Worker; never pulled into the main bundle unless `localEngine: 'sqlite'`.
+  entry: {
+    index: 'src/index.ts',
+    'otel/index': 'src/otel/index.ts',
+    'sqlite-worker': 'src/services/database/sqlite-worker.ts',
+    // The shared-tabs broker MUST stay a self-contained emit (no chunk
+    // imports): bundlers cannot trace `new SharedWorker(url)` graphs, so a
+    // relative chunk import would 404 in consumers. The source keeps zero
+    // runtime imports and scripts/check-broker-bundle.mjs enforces it.
+    'tabs-broker-worker': 'src/services/tabs/tabs-broker-worker.ts',
+  },
   format: ['esm'],
   dts: true,
   clean: true,
   hash: false,
-});
+  plugins: [versionDefinePlugin, workerUrlPlugin],
+  },
+  // The wasm-free entries a service worker, a dedicated worker or Node can
+  // import: `/pure` (protocol builders), `/live` (the live feed over the
+  // pure-JS `surrealdb` SDK) and `/sw` (push handlers). Built separately so
+  // they never share a chunk with `index` (whose graph holds the wasm engines)
+  // and so `dist/index.js` stays one flat file. They may share chunks among
+  // themselves; scripts/check-worker-entries.mjs walks those too and fails
+  // the build on any wasm/pino import or top-level window/document access.
+  {
+    entry: {
+      pure: 'src/pure.ts',
+      live: 'src/live/index.ts',
+      sw: 'src/sw/index.ts',
+    },
+    format: ['esm'],
+    dts: true,
+    clean: false,
+    hash: false,
+    // Own prefix so a chunk of this build never overwrites one of the first
+    // (both write `dist/`; `types.d.ts` collided).
+    // The entries' own declaration files arrive as chunks named `<entry>.d`
+    // and must keep their plain name.
+    outputOptions: {
+      chunkFileNames: (chunk: { name: string }) =>
+        ['pure.d', 'live.d', 'sw.d'].includes(chunk.name) ? '[name].js' : 'worker-[name].js',
+    },
+    plugins: [versionDefinePlugin],
+  },
+]);

package/vitest.config.ts

@@ -0,0 +1,48 @@
+import { defineConfig } from 'vitest/config';
+
+// Thresholds start at the numbers the suite hits today and are ratcheted per
+// directory as the saga core lands (see the plan's commit 10). Exclusions:
+//   *.fixture.ts / src/testing/**   test-only harness code
+//   sqlite-worker.ts                runs inside a Worker (sqlite-wasm + OPFS +
+//                                   postMessage globals); covered by the sqlite
+//                                   integration tests and the build check
+//   client/services.ts              constructs the real adapters (SurrealDB
+//                                   WASM store, socket, SharedWorker broker,
+//                                   OPFS blob cache); exercised by the example
+//                                   app and the WhitePawn A/B, not by vitest
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['src/**/*.test.ts'],
+    coverage: {
+      provider: 'v8',
+      all: true,
+      reporter: ['text-summary', 'lcov'],
+      include: ['src/**/*.ts'],
+      exclude: [
+        'src/**/*.test.ts',
+        'src/**/*.d.ts',
+        'src/**/*.fixture.ts',
+        'src/testing/**',
+        'src/services/database/sqlite-worker.ts',
+        'src/client/services.ts',
+      ],
+      thresholds: {
+        // Global numbers are held back by the legacy adapters (devtools,
+        // sqlite engine, tabs broker, blobs, crdt); the saga core below is 100%.
+        lines: 75,
+        functions: 82,
+        branches: 87,
+        statements: 75,
+        'src/kernel/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/state/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/query/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/mutation/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/sync/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        'src/boot/**': { lines: 100, functions: 100, branches: 100, statements: 100 },
+        // The facade's shared-tabs host callbacks need a SharedWorker broker to fire.
+        'src/client/**': { lines: 93, functions: 92, branches: 89, statements: 93 },
+      },
+    },
+  },
+});