@spooky-sync/core 0.0.1-canary.69 → 0.0.1-canary.70

package/src/services/stream-processor/stream-processor.batch.test.ts

@@ -0,0 +1,136 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { StreamProcessorService } from './index';
+import type { StreamUpdate, StreamUpdateReceiver } from './index';
+import type { WasmProcessor, WasmStreamUpdate } from './wasm-types';
+
+/**
+ * Tests for `ingestMany` bulk insert on StreamProcessorService.
+ *
+ * A batched ingest (e.g. sync fetching N missing records) used to emit one
+ * stream update per record, making the UI render row-by-row. ingestMany
+ * collapses those into a single coalesced update per affected query.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = {
+    debug: noop,
+    info: noop,
+    warn: noop,
+    error: noop,
+    trace: noop,
+  };
+  logger.child = () => logger;
+  return logger;
+}
+
+/** Records every dispatched update so we can count notifications. */
+class RecordingReceiver implements StreamUpdateReceiver {
+  public received: StreamUpdate[] = [];
+  onStreamUpdate(update: StreamUpdate): void {
+    this.received.push(update);
+  }
+}
+
+/**
+ * Build a service with a stubbed WASM processor. The processor accumulates
+ * ingested ids and returns the *full* materialized array on every call (as the
+ * real WASM does), so coalescing's last-write-wins must yield the final array.
+ */
+function makeService(queryHashesFor: (id: string) => string[]) {
+  const svc = new StreamProcessorService(
+    {} as any,
+    {} as any,
+    { get: async () => undefined, set: async () => {} } as any,
+    makeLogger()
+  );
+
+  const ingestedByQuery = new Map<string, Array<[string, number]>>();
+  const mockProcessor: Partial<WasmProcessor> = {
+    ingest: (_table, _op, id, record: any): WasmStreamUpdate[] => {
+      const version = record?._00_rv ?? 1;
+      const updates: WasmStreamUpdate[] = [];
+      for (const queryHash of queryHashesFor(id)) {
+        const arr = ingestedByQuery.get(queryHash) ?? [];
+        arr.push([id, version]);
+        ingestedByQuery.set(queryHash, arr);
+        updates.push({ query_id: queryHash, result_data: [...arr] });
+      }
+      return updates;
+    },
+  };
+  // processor is private and normally set by init() via WASM; inject the stub.
+  (svc as any).processor = mockProcessor;
+  return svc;
+}
+
+describe('StreamProcessor ingestMany bulk insert', () => {
+  let receiver: RecordingReceiver;
+
+  beforeEach(() => {
+    receiver = new RecordingReceiver();
+  });
+
+  it('emits one update per record when ingesting one-by-one (baseline)', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingest('user', 'CREATE', 'user:1', { id: 'user:1', _00_rv: 1 });
+    svc.ingest('user', 'CREATE', 'user:2', { id: 'user:2', _00_rv: 1 });
+    svc.ingest('user', 'CREATE', 'user:3', { id: 'user:3', _00_rv: 1 });
+
+    expect(receiver.received).toHaveLength(3);
+  });
+
+  it('coalesces a bulk insert into a single update per query with the final array', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([
+      { table: 'user', op: 'CREATE', id: 'user:1', record: { id: 'user:1', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:2', record: { id: 'user:2', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:3', record: { id: 'user:3', _00_rv: 1 } },
+    ]);
+
+    // Nothing dispatched until the whole batch is ingested, then exactly one
+    // coalesced update.
+    expect(receiver.received).toHaveLength(1);
+    const update = receiver.received[0];
+    expect(update.queryHash).toBe('q1');
+    // Last-write-wins carries the full materialized array (all 3 records).
+    expect(update.localArray).toHaveLength(3);
+    // Coalesced updates take DataModule's immediate (non-debounced) path.
+    expect(update.op).toBe('CREATE');
+    expect(typeof update.materializationTimeMs).toBe('number');
+  });
+
+  it('coalesces independently per affected query', () => {
+    // user:1 -> q1, user:2 -> q1 & q2, user:3 -> q2
+    const svc = makeService((id) =>
+      id === 'user:1' ? ['q1'] : id === 'user:2' ? ['q1', 'q2'] : ['q2']
+    );
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([
+      { table: 'user', op: 'CREATE', id: 'user:1', record: { id: 'user:1', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:2', record: { id: 'user:2', _00_rv: 1 } },
+      { table: 'user', op: 'CREATE', id: 'user:3', record: { id: 'user:3', _00_rv: 1 } },
+    ]);
+
+    expect(receiver.received).toHaveLength(2);
+    const byHash = Object.fromEntries(receiver.received.map((u) => [u.queryHash, u]));
+    expect(Object.keys(byHash).sort()).toEqual(['q1', 'q2']);
+  });
+
+  it('is a no-op for an empty batch and leaves the window closed', () => {
+    const svc = makeService(() => ['q1']);
+    svc.addReceiver(receiver);
+
+    svc.ingestMany([]);
+
+    expect(receiver.received).toHaveLength(0);
+    // A subsequent single ingest dispatches normally (window never opened).
+    svc.ingest('user', 'CREATE', 'user:1', { id: 'user:1', _00_rv: 1 });
+    expect(receiver.received).toHaveLength(1);
+  });
+});

package/src/services/stream-processor/wasm-types.ts

@@ -4,6 +4,14 @@ export interface WasmStreamUpdate {
   query_id: string;
   result_hash: string;
   result_data: RecordVersionArray; // Match Rust 'result_data' field
+  // Per-phase SSP processing time (ms). Ingest path: store_apply/circuit_step/
+  // transform. Register path: parse/plan/snapshot. Unused side is 0.
+  timing_store_apply_ms?: number;
+  timing_circuit_step_ms?: number;
+  timing_transform_ms?: number;
+  timing_parse_ms?: number;
+  timing_plan_ms?: number;
+  timing_snapshot_ms?: number;
 }
 
 export interface WasmQueryConfig {
@@ -28,4 +36,12 @@ export interface WasmProcessor {
   ingest(table: string, op: string, id: string, record: any): WasmStreamUpdate[];
   register_view(config: WasmQueryConfig): WasmStreamUpdate | undefined;
   unregister_view(id: string): void;
+  // Seed per-table `select` permission predicates ({ [table]: whereText }) so
+  // register_view can inject them instead of default-denying the table.
+  set_permissions(permissions: Record<string, string>): void;
+  // Persistence hooks: present on current WASM builds, absent on stale ones.
+  // Always guarded with `typeof x === 'function'` before calling so an older
+  // build degrades gracefully instead of throwing.
+  load_state?(state: string): void;
+  save_state?(): string;
 }

package/src/sp00ky.ts

@@ -2,6 +2,7 @@ import { DataModule } from './modules/data/index';
 import type {
   Sp00kyConfig,
   QueryTimeToLive,
+  QueryStatusCallback,
   Sp00kyQueryResultPromise,
   PersistenceClient,
   UpdateOptions,
@@ -32,8 +33,10 @@ import { DevToolsService } from './modules/devtools/index';
 import { createLogger } from './services/logger/index';
 import { AuthService } from './modules/auth/index';
 import { StreamProcessorService } from './services/stream-processor/index';
+import { extractSelectPermissions } from './services/stream-processor/permissions';
 import { EventSystem } from './events/index';
 import { CacheModule } from './modules/cache/index';
+import type { RecordWithId } from './modules/cache/index';
 import { CrdtManager, CrdtField } from './modules/crdt/index';
 import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
 import { parseParams } from './utils/index';
@@ -221,6 +224,32 @@ export class Sp00kyClient<S extends SchemaStructure> {
    * Setup direct callbacks instead of event subscriptions
    */
   private setupCallbacks() {
+    // Surface query fetch-status changes (idle/fetching) in DevTools. Logs a
+    // discrete event and triggers a state push so the active-queries panel
+    // reflects the flip immediately.
+    this.dataModule.onQueryStatusChange = (queryHash, status) => {
+      this.devTools.logEvent('QUERY_STATUS_CHANGED', { queryHash, status });
+    };
+
+    // Keep an actively-watched query's remote `_00_query.lastActiveAt` fresh so
+    // the server TTL sweep doesn't expire it out from under live subscribers.
+    // DataModule fires this only while the query still has ≥1 subscriber.
+    this.dataModule.onHeartbeat = (queryHash) => {
+      void this.sync.heartbeatQuery(queryHash).catch((err) => {
+        this.logger.warn(
+          { err, queryHash, Category: 'sp00ky-client::Sp00kyClient::onHeartbeat' },
+          'TTL heartbeat failed'
+        );
+      });
+    };
+
+    // Eager teardown of an opt-in deregistered query: enqueue a `cleanup`
+    // down-event so it's serialized after any in-flight register/sync for the
+    // same query (avoids out-of-order delete-before-create).
+    this.dataModule.onDeregister = (queryHash) => {
+      this.sync.enqueueDownEvent({ type: 'cleanup', payload: { hash: queryHash } });
+    };
+
     // Mutation callback for sync
     this.dataModule.onMutation((mutations: UpEvent[]) => {
       // Notify DevTools
@@ -295,6 +324,11 @@ export class Sp00kyClient<S extends SchemaStructure> {
       );
 
       await this.streamProcessor.init();
+      // Seed table `select` permissions from the schema before any query is
+      // registered — otherwise the SSP default-denies every non-`_00_` table.
+      this.streamProcessor.setPermissions(
+        extractSelectPermissions(this.config.schemaSurql)
+      );
       this.logger.debug(
         { Category: 'sp00ky-client::Sp00kyClient::init' },
         'StreamProcessor initialized'
@@ -424,12 +458,25 @@ export class Sp00kyClient<S extends SchemaStructure> {
       throw new Error(`Table ${table} not found`);
     }
 
-    const hash = await this.dataModule.query(
-      table,
-      q.selectQuery.query,
-      parseParams(tableSchema.columns, q.selectQuery.vars ?? {}),
-      ttl
-    );
+    const params = parseParams(tableSchema.columns, q.selectQuery.vars ?? {});
+    const hash = await this.dataModule.query(table, q.selectQuery.query, params, ttl);
+
+    // Instant-hydrate: for a cold query, fetch its rows one-shot from the remote
+    // (its own surql, run directly like useRemote) and display them NOW; the full
+    // realtime registration proceeds below. Hydrated rows carry their `_00_rv`
+    // versions so the registration's syncRecords skips re-pulling unchanged bodies.
+    // Best-effort: any failure (e.g. offline) just falls through to registration.
+    if (this.config.instantHydrate !== false && this.dataModule.isCold(hash)) {
+      try {
+        const [rows] = await this.remote.query<[RecordWithId[]]>(q.selectQuery.query, params);
+        await this.dataModule.applyHydration(hash, rows ?? []);
+      } catch (err) {
+        this.logger.warn(
+          { err, hash, Category: 'sp00ky-client::Sp00kyClient::instantHydrate' },
+          'Instant hydrate failed; proceeding with registration'
+        );
+      }
+    }
 
     await this.sync.enqueueDownEvent({
       type: 'register',
@@ -454,6 +501,39 @@ export class Sp00kyClient<S extends SchemaStructure> {
     return this.dataModule.subscribe(queryHash, callback, options);
   }
 
+  /**
+   * Opt-in eager teardown for a query whose last subscriber has gone away
+   * (e.g. a viewport-windowed list cancelling an off-screen window). No-op
+   * while any subscriber remains. Tears down the remote `_00_query` view +
+   * local WASM view instead of waiting for the TTL sweep. Default behavior
+   * (no call here) keeps the view resident for cheap re-subscription.
+   */
+  deregisterQuery(queryHash: string): void {
+    this.dataModule.deregisterQuery(queryHash);
+  }
+
+  /**
+   * Subscribe to a query's fetch-status changes (idle/fetching). With
+   * `{ immediate: true }` the callback fires synchronously with the current
+   * status. Powers the `useQuery` hook's `isFetching()` accessor.
+   */
+  subscribeQueryStatus(
+    queryHash: string,
+    callback: QueryStatusCallback,
+    options?: { immediate?: boolean }
+  ): () => void {
+    return this.dataModule.subscribeStatus(queryHash, callback, options);
+  }
+
+  /**
+   * Report the frontend processing time (ms) a client framework spent applying
+   * an update for a query (e.g. `useQuery`'s `reconcile()`), so DevTools/MCP can
+   * surface the "frontend" phase of the per-query timing breakdown.
+   */
+  reportFrontendTiming(queryHash: string, ms: number): void {
+    this.dataModule.recordFrontendTiming(queryHash, ms);
+  }
+
   run<
     B extends BackendNames<S>,
     R extends BackendRoutes<S, B>,

package/src/types.ts

@@ -133,6 +133,16 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
    * values fall back to the default (500ms).
    */
   refSyncIntervalMs?: number;
+  /**
+   * Instant-hydrate cold queries: when a query is registered with no local
+   * data yet, first run its surql directly on the remote (one-shot) and display
+   * the result immediately, THEN do the full realtime registration in the
+   * background. The hydrated rows are ingested with their versions so the
+   * registration's `syncRecords` skips re-pulling unchanged bodies. Cuts cold
+   * first-paint from ~one full registration round-trip to ~one query.
+   * Defaults to `true`; set `false` to keep the old wait-for-registration path.
+   */
+  instantHydrate?: boolean;
 }
 
 export type QueryHash = string;
@@ -178,6 +188,15 @@ export interface QueryConfig {
 
 export type QueryConfigRecord = QueryConfig & { id: string };
 
+/**
+ * Runtime fetch status of a live query.
+ * - `idle`: not currently fetching missing records.
+ * - `fetching`: the sync engine is fetching/ingesting missing records for this
+ *   query. UI notifications are coalesced so the result lands as a single
+ *   update once fetching completes.
+ */
+export type QueryStatus = 'idle' | 'fetching';
+
 /**
  * Internal state of a live query.
  */
@@ -186,6 +205,9 @@ export interface QueryState {
   config: QueryConfig;
   /** The current cached records for this query. */
   records: Record<string, any>[];
+  /** Set once `applyHydration` has run for this query, so the cold instant-hydrate
+   * path fires at most once per query (see DataModule.isCold/applyHydration). */
+  hydrated?: boolean;
   /** Timer for TTL expiration. */
   ttlTimer: NodeJS.Timeout | null;
   /** TTL duration in milliseconds. */
@@ -202,13 +224,76 @@ export interface QueryState {
   lastIngestLatencyMs: number | null;
   /** Cumulative count of ingest/materialization errors observed for this query. */
   errorCount: number;
+  /**
+   * Ephemeral runtime fetch status. Not persisted to `_00_query`; observable
+   * via DevTools and the `useQuery` hook. `fetching` while the sync engine is
+   * pulling missing records for this query, otherwise `idle`.
+   */
+  status: QueryStatus;
+  /**
+   * Rolling per-phase timing samples (ms), in addition to `materializationSamples`
+   * (which holds the SSP whole-ingest wall time). Keyed by `TimingPhase` minus
+   * `ssp`. Not persisted — surfaced live to DevTools + MCP via `phaseTimings`.
+   */
+  phaseSamples: Record<string, number[]>;
+  /** Most recent sample (ms) per phase, or null. */
+  phaseLast: Record<string, number | null>;
+  /** One-shot SSP registration timings (ms). */
+  registrationTimings: RegistrationTimings;
 }
 
 /** Cap on the rolling materialization-sample window kept per query in memory. */
 export const MATERIALIZATION_SAMPLE_WINDOW = 100;
 
+/** Timed processing phases surfaced per query. `ssp` is the WASM-ingest wall
+ *  time; the `ssp*` phases are its internal breakdown from the SSP binding. */
+export type TimingPhase =
+  | 'ssp'
+  | 'sspStoreApply'
+  | 'sspCircuitStep'
+  | 'sspTransform'
+  | 'localFetch'
+  | 'remoteFetch'
+  | 'frontend';
+
+/** One-shot registration timings (ms), captured once when a query registers. */
+export interface RegistrationTimings {
+  /** SSP surql→plan parse + permission injection. */
+  parseMs: number | null;
+  /** SSP operator-DAG build. */
+  planMs: number | null;
+  /** SSP initial snapshot evaluation. */
+  snapshotMs: number | null;
+  /** Wall time of `cache.registerQuery` (register_view round-trip). */
+  wallMs: number | null;
+}
+
+/** Percentile summary for one timed phase, surfaced to DevTools + MCP. */
+export interface PhaseStat {
+  lastMs: number | null;
+  p50: number | null;
+  p90: number | null;
+  p99: number | null;
+  count: number;
+}
+
+/** Per-query processing-time breakdown surfaced via DevTools panel + MCP. */
+export interface QueryTimings {
+  ssp: PhaseStat;
+  sspStoreApply: PhaseStat;
+  sspCircuitStep: PhaseStat;
+  sspTransform: PhaseStat;
+  localFetch: PhaseStat;
+  remoteFetch: PhaseStat;
+  frontend: PhaseStat;
+  registration: RegistrationTimings;
+  updateCount: number;
+  errorCount: number;
+}
+
 // Callback types
 export type QueryUpdateCallback = (records: Record<string, any>[]) => void;
+export type QueryStatusCallback = (status: QueryStatus) => void;
 export type MutationCallback = (mutations: UpEvent[]) => void;
 
 export type MutationEventType = 'create' | 'update' | 'delete';

package/src/utils/index.ts

@@ -97,15 +97,25 @@ export function decodeFromSp00ky<S extends SchemaStructure, T extends TableNames
 
 // ==================== TIME/DURATION UTILITIES ====================
 
+/**
+ * Read the millisecond count off a surrealdb `Duration`. Different `surrealdb`
+ * releases expose it under `milliseconds` (current) or the older private
+ * `_milliseconds` field, so read whichever is set; returns 0 when neither is.
+ */
+function durationMillis(duration: Duration): number {
+  const d = duration as { milliseconds?: number | bigint; _milliseconds?: number | bigint };
+  return Number(d.milliseconds || d._milliseconds || 0);
+}
+
 /**
  * Parse duration string or Duration object to milliseconds
  */
 export function parseDuration(duration: QueryTimeToLive | Duration): number {
   if (duration instanceof Duration) {
-    const ms = (duration as any).milliseconds || (duration as any)._milliseconds;
-    if (ms) return Number(ms);
+    const ms = durationMillis(duration);
+    if (ms) return ms;
     const str = duration.toString();
-    if (str !== '[object Object]') return parseDuration(str as any);
+    if (str !== '[object Object]') return parseDuration(str as QueryTimeToLive);
     return 600000;
   }
 

package/tsdown.config.ts

@@ -1,4 +1,57 @@
 import { defineConfig } from 'tsdown';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+
+// Resolve real package versions at build time so the DevTools can report the
+// actual bundled frontend versions (used to detect frontend/backend drift).
+const corePkg = require('./package.json');
+const coreVersion: string = corePkg.version;
+// ssp-wasm has no `exports` map; resolve its package.json directly, falling
+// back to the relative workspace path if module resolution can't find it.
+let wasmVersion: string;
+try {
+  wasmVersion = require('@spooky-sync/ssp-wasm/package.json').version;
+} catch {
+  wasmVersion = require('../ssp-wasm/package.json').version;
+}
+// Report the SurrealDB WASM ENGINE version (`@surrealdb/wasm`) — the engine the
+// in-browser local DB actually runs — NOT the `surrealdb` JS client library
+// version. The WASM engine is the one that must stay compatible with the server
+// engine, so it's the meaningful number to compare in DevTools. Prefer the
+// actually-installed version; fall back to our pinned range if its `exports`
+// map blocks importing its package.json.
+let surrealVersion: string;
+try {
+  surrealVersion = require('@surrealdb/wasm/package.json').version;
+} catch {
+  surrealVersion = String(corePkg.dependencies?.['@surrealdb/wasm'] ?? 'unknown').replace(
+    /^[\^~]/,
+    ''
+  );
+}
+
+// tsdown 0.12.x forwards a top-level `define` to rolldown's inputOptions, which
+// rejects it — so we do the substitution ourselves with a tiny transform
+// plugin. Only our own modules reference the `__SP00KY_*__` identifiers, and
+// the early `includes` check skips everything else.
+const replacements: Record<string, string> = {
+  __SP00KY_CORE_VERSION__: JSON.stringify(coreVersion),
+  __SP00KY_WASM_VERSION__: JSON.stringify(wasmVersion),
+  __SP00KY_SURREAL_VERSION__: JSON.stringify(surrealVersion),
+};
+
+const versionDefinePlugin = {
+  name: 'sp00ky-version-define',
+  transform(code: string) {
+    if (!code.includes('__SP00KY_')) return null;
+    let out = code;
+    for (const [token, value] of Object.entries(replacements)) {
+      out = out.split(token).join(value);
+    }
+    return out;
+  },
+};
 
 export default defineConfig({
   entry: ['src/index.ts', 'src/otel/index.ts'],
@@ -6,4 +59,5 @@ export default defineConfig({
   dts: true,
   clean: true,
   hash: false,
+  plugins: [versionDefinePlugin],
 });