@spooky-sync/core 0.0.1-canary.7 → 0.0.1-canary.71

package/src/types.ts

@@ -1,9 +1,14 @@
-import { RecordId, SchemaStructure } from '@spooky-sync/query-builder';
-import { Level } from 'pino';
-import { PushEventOptions } from './events/index';
-import { UpEvent } from './modules/sync/index';
+import type { RecordId, SchemaStructure } from '@spooky-sync/query-builder';
+import type { Level, LoggerOptions } from 'pino';
+import type { PushEventOptions } from './events/index';
+import type { UpEvent } from './modules/sync/index';
 
-export type { Level } from 'pino';
+export type { Level };
+
+/**
+ * A pino browser transmit object for forwarding logs to an external sink (e.g. OpenTelemetry).
+ */
+export type PinoTransmit = NonNullable<NonNullable<LoggerOptions['browser']>['transmit']>;
 
 /**
  * The type of storage backend to use for the local database.
@@ -65,22 +70,22 @@ export type QueryTimeToLive =
 /**
  * Result object returned when a query is registered or executed.
  */
-export interface SpookyQueryResult {
+export interface Sp00kyQueryResult {
   /** The unique hash identifier for the query. */
   hash: string;
 }
 
-export type SpookyQueryResultPromise = Promise<SpookyQueryResult>;
+export type Sp00kyQueryResultPromise = Promise<Sp00kyQueryResult>;
 
 export interface EventSubscriptionOptions {
   priority?: number;
 }
 
 /**
- * Configuration options for the Spooky client.
+ * Configuration options for the Sp00ky client.
  * @template S The schema structure type.
  */
-export interface SpookyConfig<S extends SchemaStructure> {
+export interface Sp00kyConfig<S extends SchemaStructure> {
   /** Database connection configuration. */
   database: {
     /** The SurrealDB endpoint URL. */
@@ -94,8 +99,6 @@ export interface SpookyConfig<S extends SchemaStructure> {
     /** Authentication token. */
     token?: string;
   };
-  /** Unique client identifier. If not provided, one will be generated. */
-  clientId?: string;
   /** The schema definition. */
   schema: S;
   /** The compiled SURQL schema string. */
@@ -107,13 +110,39 @@ export interface SpookyConfig<S extends SchemaStructure> {
    * Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
    */
   persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
-  /** OpenTelemetry collector endpoint for telemetry data. */
-  otelEndpoint?: string;
+  /** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
+  otelTransmit?: PinoTransmit;
   /**
    * Debounce time in milliseconds for stream updates.
    * Defaults to 100ms.
    */
   streamDebounceTime?: number;
+  /**
+   * Debounce time in milliseconds for syncing collaborative (CRDT) field
+   * changes to the remote database. Local writes happen immediately on
+   * every keystroke (so reload/offline works), but the remote UPSERT is
+   * coalesced over this window. Lower = snappier remote propagation +
+   * more network traffic; higher = less traffic + more lag for other
+   * collaborators. Defaults to 500ms.
+   */
+  crdtDebounceMs?: number;
+  /**
+   * Cadence (ms) for the `_00_list_ref` poll that catches cross-session
+   * UPDATEs the SurrealDB v3 LIVE-permission gap drops. Lower = faster
+   * convergence + more query load; higher = the inverse. Non-positive
+   * values fall back to the default (500ms).
+   */
+  refSyncIntervalMs?: number;
+  /**
+   * Instant-hydrate cold queries: when a query is registered with no local
+   * data yet, first run its surql directly on the remote (one-shot) and display
+   * the result immediately, THEN do the full realtime registration in the
+   * background. The hydrated rows are ingested with their versions so the
+   * registration's `syncRecords` skips re-pulling unchanged bodies. Cuts cold
+   * first-paint from ~one full registration round-trip to ~one query.
+   * Defaults to `true`; set `false` to keep the old wait-for-registration path.
+   */
+  instantHydrate?: boolean;
 }
 
 export type QueryHash = string;
@@ -159,6 +188,15 @@ export interface QueryConfig {
 
 export type QueryConfigRecord = QueryConfig & { id: string };
 
+/**
+ * Runtime fetch status of a live query.
+ * - `idle`: not currently fetching missing records.
+ * - `fetching`: the sync engine is fetching/ingesting missing records for this
+ *   query. UI notifications are coalesced so the result lands as a single
+ *   update once fetching completes.
+ */
+export type QueryStatus = 'idle' | 'fetching';
+
 /**
  * Internal state of a live query.
  */
@@ -167,16 +205,95 @@ export interface QueryState {
   config: QueryConfig;
   /** The current cached records for this query. */
   records: Record<string, any>[];
+  /** Set once `applyHydration` has run for this query, so the cold instant-hydrate
+   * path fires at most once per query (see DataModule.isCold/applyHydration). */
+  hydrated?: boolean;
   /** Timer for TTL expiration. */
   ttlTimer: NodeJS.Timeout | null;
   /** TTL duration in milliseconds. */
   ttlDurationMs: number;
   /** Number of times the query has been updated. */
   updateCount: number;
+  /**
+   * Rolling window of the most recent materialization-step latencies (ms).
+   * Capped at MATERIALIZATION_SAMPLE_WINDOW; used to recompute p55/p90/p99
+   * before each persist to `_00_query`. Samples themselves are not persisted.
+   */
+  materializationSamples: number[];
+  /** Most recent end-to-end ingest latency in ms, or null until the first ingest. */
+  lastIngestLatencyMs: number | null;
+  /** Cumulative count of ingest/materialization errors observed for this query. */
+  errorCount: number;
+  /**
+   * Ephemeral runtime fetch status. Not persisted to `_00_query`; observable
+   * via DevTools and the `useQuery` hook. `fetching` while the sync engine is
+   * pulling missing records for this query, otherwise `idle`.
+   */
+  status: QueryStatus;
+  /**
+   * Rolling per-phase timing samples (ms), in addition to `materializationSamples`
+   * (which holds the SSP whole-ingest wall time). Keyed by `TimingPhase` minus
+   * `ssp`. Not persisted — surfaced live to DevTools + MCP via `phaseTimings`.
+   */
+  phaseSamples: Record<string, number[]>;
+  /** Most recent sample (ms) per phase, or null. */
+  phaseLast: Record<string, number | null>;
+  /** One-shot SSP registration timings (ms). */
+  registrationTimings: RegistrationTimings;
+}
+
+/** Cap on the rolling materialization-sample window kept per query in memory. */
+export const MATERIALIZATION_SAMPLE_WINDOW = 100;
+
+/** Timed processing phases surfaced per query. `ssp` is the WASM-ingest wall
+ *  time; the `ssp*` phases are its internal breakdown from the SSP binding. */
+export type TimingPhase =
+  | 'ssp'
+  | 'sspStoreApply'
+  | 'sspCircuitStep'
+  | 'sspTransform'
+  | 'localFetch'
+  | 'remoteFetch'
+  | 'frontend';
+
+/** One-shot registration timings (ms), captured once when a query registers. */
+export interface RegistrationTimings {
+  /** SSP surql→plan parse + permission injection. */
+  parseMs: number | null;
+  /** SSP operator-DAG build. */
+  planMs: number | null;
+  /** SSP initial snapshot evaluation. */
+  snapshotMs: number | null;
+  /** Wall time of `cache.registerQuery` (register_view round-trip). */
+  wallMs: number | null;
+}
+
+/** Percentile summary for one timed phase, surfaced to DevTools + MCP. */
+export interface PhaseStat {
+  lastMs: number | null;
+  p50: number | null;
+  p90: number | null;
+  p99: number | null;
+  count: number;
+}
+
+/** Per-query processing-time breakdown surfaced via DevTools panel + MCP. */
+export interface QueryTimings {
+  ssp: PhaseStat;
+  sspStoreApply: PhaseStat;
+  sspCircuitStep: PhaseStat;
+  sspTransform: PhaseStat;
+  localFetch: PhaseStat;
+  remoteFetch: PhaseStat;
+  frontend: PhaseStat;
+  registration: RegistrationTimings;
+  updateCount: number;
+  errorCount: number;
 }
 
 // Callback types
 export type QueryUpdateCallback = (records: Record<string, any>[]) => void;
+export type QueryStatusCallback = (status: QueryStatus) => void;
 export type MutationCallback = (mutations: UpEvent[]) => void;
 
 export type MutationEventType = 'create' | 'update' | 'delete';
@@ -209,6 +326,8 @@ export interface RunOptions {
   assignedTo?: string;
   max_retries?: number;
   retry_strategy?: 'linear' | 'exponential';
+  /** Timeout in seconds for the backend HTTP call. Only used if the backend allows timeout override. */
+  timeout?: number;
 }
 
 /**

package/src/utils/index.ts

@@ -1,7 +1,7 @@
-import { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
+import type { GetTable, SchemaStructure, TableModel, TableNames } from '@spooky-sync/query-builder';
 import { Uuid, RecordId, Duration } from 'surrealdb';
-import { Logger } from '../services/logger/index';
-import { QueryTimeToLive } from '../types';
+import type { Logger } from '../services/logger/index';
+import type { QueryTimeToLive } from '../types';
 
 export * from './surql';
 export * from './parser';
@@ -59,7 +59,7 @@ export function generateNewTableId<S extends SchemaStructure, T extends TableNam
 
 // ==================== SCHEMA ENCODING/DECODING ====================
 
-export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames<S>>(
+export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames<S>>(
   schema: S,
   tableName: T,
   record: TableModel<GetTable<S, T>>
@@ -74,19 +74,19 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
   for (const field of Object.keys(table.columns)) {
     const column = table.columns[field] as any;
     const relation = schema.relationships.find((r) => r.from === tableName && r.field === field);
-    if ((column.recordId || relation) && encoded[field] != null) {
+    if ((column.recordId || relation) && encoded[field] !== null && encoded[field] !== undefined) {
       if (encoded[field] instanceof RecordId) {
         encoded[field] = `${encoded[field].table.toString()}:${encoded[field].id}`;
       } else if (
         relation &&
-        (encoded[field] instanceof Object || encoded[field] instanceof Array)
+        (encoded[field] instanceof Object || Array.isArray(encoded[field]))
       ) {
         if (Array.isArray(encoded[field])) {
           encoded[field] = encoded[field].map((item) =>
-            decodeFromSpooky(schema, relation.to, item)
+            decodeFromSp00ky(schema, relation.to, item)
           );
         } else {
-          encoded[field] = decodeFromSpooky(schema, relation.to, encoded[field]);
+          encoded[field] = decodeFromSp00ky(schema, relation.to, encoded[field]);
         }
       }
     }
@@ -97,15 +97,25 @@ export function decodeFromSpooky<S extends SchemaStructure, T extends TableNames
 
 // ==================== TIME/DURATION UTILITIES ====================
 
+/**
+ * Read the millisecond count off a surrealdb `Duration`. Different `surrealdb`
+ * releases expose it under `milliseconds` (current) or the older private
+ * `_milliseconds` field, so read whichever is set; returns 0 when neither is.
+ */
+function durationMillis(duration: Duration): number {
+  const d = duration as { milliseconds?: number | bigint; _milliseconds?: number | bigint };
+  return Number(d.milliseconds || d._milliseconds || 0);
+}
+
 /**
  * Parse duration string or Duration object to milliseconds
  */
 export function parseDuration(duration: QueryTimeToLive | Duration): number {
   if (duration instanceof Duration) {
-    const ms = (duration as any).milliseconds || (duration as any)._milliseconds;
-    if (ms) return Number(ms);
+    const ms = durationMillis(duration);
+    if (ms) return ms;
     const str = duration.toString();
-    if (str !== '[object Object]') return parseDuration(str as any);
+    if (str !== '[object Object]') return parseDuration(str as QueryTimeToLive);
     return 600000;
   }
 
@@ -117,7 +127,7 @@ export function parseDuration(duration: QueryTimeToLive | Duration): number {
 
   const match = duration.match(/^(\d+)([smh])$/);
   if (!match) return 600000;
-  const val = parseInt(match[1], 10);
+  const val = Number.parseInt(match[1], 10);
   const unit = match[2];
   switch (unit) {
     case 's':
@@ -137,6 +147,18 @@ export async function fileToUint8Array(file: File | Blob): Promise<Uint8Array> {
   return new Uint8Array(buffer);
 }
 
+// ==================== TEXT UTILITIES ====================
+
+/**
+ * Convert plain text to simple HTML paragraphs.
+ * Useful for seeding a rich-text editor (e.g. TipTap/ProseMirror) with fallback content.
+ */
+export function textToHtml(text: string): string {
+  return text
+    .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+    .split('\n').map((l) => `<p>${l || '<br>'}</p>`).join('');
+}
+
 // ==================== DATABASE UTILITIES ====================
 
 /**
@@ -165,7 +187,7 @@ export async function withRetry<T>(
             attempt: i + 1,
             retries,
             error: msg,
-            Category: 'spooky-client::utils::withRetry',
+            Category: 'sp00ky-client::utils::withRetry',
           },
           'Retrying DB operation'
         );

package/src/utils/parser.ts

@@ -1,4 +1,5 @@
-import { ColumnSchema, RecordId } from '@spooky-sync/query-builder';
+import type { ColumnSchema} from '@spooky-sync/query-builder';
+import { RecordId } from '@spooky-sync/query-builder';
 import { parseRecordIdString } from './index';
 import { DateTime } from 'surrealdb';
 
@@ -8,7 +9,7 @@ export function cleanRecord(
 ): Record<string, any> {
   const cleaned: Record<string, any> = {};
   for (const [key, value] of Object.entries(record)) {
-    if (key === 'id' || key in tableSchema) {
+    if (key === 'id' || key.startsWith('_00_') || key in tableSchema) {
       cleaned[key] = value;
     }
   }

package/src/utils/surql.ts

@@ -1,4 +1,4 @@
-import { MutationEventType } from '../types';
+import type { MutationEventType } from '../types';
 
 // ==================== TYPES ====================
 
@@ -34,6 +34,7 @@ export interface SurqlHelper {
     keyDataVars: ({ key: string; variable: string } | { statement: string } | string)[]
   ): string;
   upsert(idVar: string, dataVar: string): string;
+  upsertMerge(idVar: string, dataVar: string): string;
   updateMerge(idVar: string, dataVar: string): string;
   updateSet(
     idVar: string,
@@ -90,16 +91,16 @@ export const surql: SurqlHelper = {
     returnValues: ({ field: string; alias: string } | string)[]
   ) {
     return `SELECT ${returnValues
-      .map((returnValues) =>
-        typeof returnValues === 'string'
-          ? returnValues
-          : `${returnValues.field} as ${returnValues.alias}`
+      .map((rv) =>
+        typeof rv === 'string'
+          ? rv
+          : `${rv.field} as ${rv.alias}`
       )
       .join(',')} FROM ${table} WHERE ${whereVar
-      .map((whereVar) =>
-        typeof whereVar === 'string'
-          ? `${whereVar} = $${whereVar}`
-          : `${whereVar.field} = $${whereVar.variable}`
+      .map((wv) =>
+        typeof wv === 'string'
+          ? `${wv} = $${wv}`
+          : `${wv.field} = $${wv.variable}`
       )
       .join(' AND ')}`;
   },
@@ -127,6 +128,14 @@ export const surql: SurqlHelper = {
     return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
   },
 
+  // Use this for sync-down ingestion: the remote payload omits local-only
+  // fields injected by the CLI's local-schema setup (`_00_crdt`,
+  // `_00_cursor`), and REPLACE would wipe them on every round-trip and
+  // break offline reload of CRDT state.
+  upsertMerge(idVar: string, dataVar: string) {
+    return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
+  },
+
   updateMerge(idVar: string, dataVar: string) {
     return `UPDATE ONLY $${idVar} MERGE $${dataVar}`;
   },
@@ -136,12 +145,12 @@ export const surql: SurqlHelper = {
     keyDataVar: ({ key: string; variable: string } | { statement: string } | string)[]
   ) {
     return `UPDATE $${idVar} SET ${keyDataVar
-      .map((keyDataVar) =>
-        typeof keyDataVar === 'string'
-          ? `${keyDataVar} = $${keyDataVar}`
-          : 'statement' in keyDataVar
-            ? keyDataVar.statement
-            : `${keyDataVar.key} = $${keyDataVar.variable}`
+      .map((kdv) =>
+        typeof kdv === 'string'
+          ? `${kdv} = $${kdv}`
+          : 'statement' in kdv
+            ? kdv.statement
+            : `${kdv.key} = $${kdv.variable}`
       )
       .join(', ')}`;
   },

package/src/utils/withRetry.test.ts

@@ -145,7 +145,7 @@ describe('withRetry', () => {
         attempt: 1,
         retries: 3,
         error: 'Can not open transaction',
-        Category: 'spooky-client::utils::withRetry',
+        Category: 'sp00ky-client::utils::withRetry',
       }),
       'Retrying DB operation'
     );

package/tsdown.config.ts

@@ -1,9 +1,63 @@
 import { defineConfig } from 'tsdown';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+// Resolve real package versions at build time so the DevTools can report the
+// actual bundled frontend versions (used to detect frontend/backend drift).
+const corePkg = require('./package.json');
+const coreVersion: string = corePkg.version;
+// ssp-wasm has no `exports` map; resolve its package.json directly, falling
+// back to the relative workspace path if module resolution can't find it.
+let wasmVersion: string;
+try {
+  wasmVersion = require('@spooky-sync/ssp-wasm/package.json').version;
+} catch {
+  wasmVersion = require('../ssp-wasm/package.json').version;
+}
+// Report the SurrealDB WASM ENGINE version (`@surrealdb/wasm`) — the engine the
+// in-browser local DB actually runs — NOT the `surrealdb` JS client library
+// version. The WASM engine is the one that must stay compatible with the server
+// engine, so it's the meaningful number to compare in DevTools. Prefer the
+// actually-installed version; fall back to our pinned range if its `exports`
+// map blocks importing its package.json.
+let surrealVersion: string;
+try {
+  surrealVersion = require('@surrealdb/wasm/package.json').version;
+} catch {
+  surrealVersion = String(corePkg.dependencies?.['@surrealdb/wasm'] ?? 'unknown').replace(
+    /^[\^~]/,
+    ''
+  );
+}
+
+// tsdown 0.12.x forwards a top-level `define` to rolldown's inputOptions, which
+// rejects it — so we do the substitution ourselves with a tiny transform
+// plugin. Only our own modules reference the `__SP00KY_*__` identifiers, and
+// the early `includes` check skips everything else.
+const replacements: Record<string, string> = {
+  __SP00KY_CORE_VERSION__: JSON.stringify(coreVersion),
+  __SP00KY_WASM_VERSION__: JSON.stringify(wasmVersion),
+  __SP00KY_SURREAL_VERSION__: JSON.stringify(surrealVersion),
+};
+
+const versionDefinePlugin = {
+  name: 'sp00ky-version-define',
+  transform(code: string) {
+    if (!code.includes('__SP00KY_')) return null;
+    let out = code;
+    for (const [token, value] of Object.entries(replacements)) {
+      out = out.split(token).join(value);
+    }
+    return out;
+  },
+};
 
 export default defineConfig({
-  entry: ['src/index.ts'],
+  entry: ['src/index.ts', 'src/otel/index.ts'],
   format: ['esm'],
   dts: true,
   clean: true,
   hash: false,
+  plugins: [versionDefinePlugin],
 });