@spooky-sync/core 0.0.1-canary.240 → 0.0.1-canary.241

package/src/query/render-set.ts

@@ -0,0 +1,60 @@
+import type { RecordVersionArray } from '../types';
+import type { QueryPhase } from '../state/lifecycle';
+import type { Overlay } from '../state/selectors';
+
+export interface RenderInput {
+  phase: QueryPhase;
+  remoteArray: RecordVersionArray;
+  localArray: RecordVersionArray;
+  isWindow: boolean;
+}
+
+/**
+ * The id-set a query renders from, or `null` when it must fall back to a
+ * local predicate scan (a cold query that was never resolved on this device).
+ * A cold window has no usable scan (re-applying `START m` against the shared
+ * store returns the wrong rows), so it renders the SSP's local window instead.
+ */
+export function resolveMembership(input: RenderInput): RecordVersionArray | null {
+  if (input.phase !== 'cold') return input.remoteArray;
+  if (input.isWindow) return input.localArray;
+  return null;
+}
+
+export interface RenderOptions {
+  hasExplicitOrder: boolean;
+  isWindow: boolean;
+}
+
+/**
+ *   render = (membership ∪ (writes ∩ localView)) − deletes
+ *
+ * `writes`/`deletes` are the outbox overlay (pending and acked items). The
+ * middle term keeps optimistic writes visible without re-admitting stale rows:
+ * `localView` (the SSP's local id-set) says whether the written row matches
+ * the query per local truth. Sorted unless the query orders itself or is a
+ * window (whose id order is the slice order).
+ */
+export function buildRenderIds(
+  membership: RecordVersionArray,
+  localView: RecordVersionArray,
+  overlay: Overlay,
+  opts: RenderOptions
+): string[] {
+  const ordered: string[] = [];
+  const seen = new Set<string>();
+  for (const [id] of membership) {
+    if (overlay.deletes.has(id) || seen.has(id)) continue;
+    seen.add(id);
+    ordered.push(id);
+  }
+  if (overlay.writes.size > 0) {
+    for (const [id] of localView) {
+      if (!overlay.writes.has(id) || overlay.deletes.has(id) || seen.has(id)) continue;
+      seen.add(id);
+      ordered.push(id);
+    }
+  }
+  if (!opts.hasExplicitOrder && !opts.isWindow) ordered.sort();
+  return ordered;
+}

package/src/query/sql.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import * as sql from './sql';
+import {
+  buildListRefBatchSelect,
+  buildListRefSelect,
+  buildQueryRowCountBatchSelect,
+  buildQueryRowCountSelect,
+  buildSubqueryListRefSelect,
+} from '../sync/policy';
+
+describe('query sql builders (golden against the previous builders)', () => {
+  it('edge and meta selects are byte-identical to the old helpers', () => {
+    expect(sql.listRefSelect('_00_list_ref_user_x')).toBe(buildListRefSelect('_00_list_ref_user_x'));
+    expect(sql.subqueryListRefSelect('t')).toBe(buildSubqueryListRefSelect('t'));
+    expect(sql.queryRowCountSelect()).toBe(buildQueryRowCountSelect());
+    expect(sql.listRefBatchSelect('t')).toBe(buildListRefBatchSelect('t'));
+    expect(sql.queryRowCountBatchSelect()).toBe(buildQueryRowCountBatchSelect());
+  });
+  it('composes the single, batch and register requests', () => {
+    expect(sql.singleSnapshotSelect('t').split(';\n')).toEqual([
+      sql.listRefSelect('t'),
+      sql.queryRowCountSelect(),
+      sql.subqueryListRefSelect('t'),
+    ]);
+    expect(sql.batchSnapshotSelect('t')).toBe(`${sql.listRefBatchSelect('t')};\n${sql.queryRowCountBatchSelect()}`);
+    expect(sql.registerSelect('t').startsWith('fn::query::register($config);\n')).toBe(true);
+    const id = new RecordId('_00_query', 'h');
+    expect(sql.registerVars({ id, surql: 's', params: { a: 1 }, ttl: '10m' })).toEqual({
+      config: { id, surql: 's', params: { a: 1 }, ttl: '10m' },
+      in: id,
+    });
+  });
+  it('view rows and ids', () => {
+    expect(sql.viewRecordId('k')).toEqual(new RecordId('_00_view', 'k'));
+    expect(sql.viewRow([['t:1', 1]], true, 5)).toEqual({ ids: [['t:1', 1]], confirmed: true, updatedAt: 5 });
+    expect(sql.readLegacyViewRows()).toBe('SELECT * FROM _00_window');
+    expect(sql.countViewRows()).toContain('_00_view');
+    expect(sql.bodySelect()).toBe('SELECT * FROM $ids');
+  });
+  it('heartbeat batch answers per index and detects reclaimed rows', () => {
+    const a = new RecordId('_00_query', 'a');
+    const b = new RecordId('_00_query', 'b');
+    expect(sql.heartbeatBatch([a, b])).toEqual({
+      sql: 'fn::query::heartbeat($id0);\nfn::query::heartbeat($id1)',
+      vars: { id0: a, id1: b },
+    });
+    expect(sql.heartbeatRowGone([])).toBe(true);
+    expect(sql.heartbeatRowGone([{ id: a }])).toBe(false);
+    expect(sql.heartbeatRowGone(undefined)).toBe(false);
+  });
+  it('upsertBodiesTx MERGEs each body inside one transaction', () => {
+    const { query, vars } = sql.upsertBodiesTx([
+      { id: 'thing:1', content: { a: 1 } },
+      { id: 'thing:2', content: { b: 2 } },
+    ]);
+    expect(query.sql).toBe(
+      'BEGIN TRANSACTION;\nUPSERT ONLY $id0 MERGE $content0;UPSERT ONLY $id1 MERGE $content1;\nCOMMIT TRANSACTION;'
+    );
+    expect(vars).toEqual({ id0: 'thing:1', content0: { a: 1 }, id1: 'thing:2', content1: { b: 2 } });
+  });
+});

package/src/query/sql.ts

@@ -0,0 +1,79 @@
+import { RecordId } from 'surrealdb';
+import type { RecordVersionArray } from '../types';
+import type { Vars } from '../kernel/effects';
+import { surql } from '../utils/surql';
+import type { SealedQuery } from '../utils/surql';
+
+/** Every SurrealQL statement the query side sends, as pure builders. */
+
+export const VIEW_TABLE = '_00_view';
+export const LEGACY_VIEW_TABLE = '_00_window';
+
+export const viewRecordId = (viewKey: string): RecordId<string> => new RecordId(VIEW_TABLE, viewKey);
+
+export const viewRow = (ids: RecordVersionArray, confirmed: boolean, now: number): Record<string, unknown> => ({
+  ids,
+  confirmed,
+  updatedAt: now,
+});
+
+/** Copy every legacy `_00_window` row into `_00_view` once. */
+export const readLegacyViewRows = (): string => `SELECT * FROM ${LEGACY_VIEW_TABLE}`;
+export const countViewRows = (): string => `SELECT count() FROM ${VIEW_TABLE} GROUP ALL`;
+
+export const listRefSelect = (table: string): string =>
+  `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
+export const subqueryListRefSelect = (table: string): string =>
+  `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NOT NONE`;
+export const queryRowCountSelect = (): string => 'SELECT VALUE { rowCount: rowCount, state: state } FROM ONLY $in';
+export const listRefBatchSelect = (table: string): string =>
+  `SELECT in, out, version, parent FROM ${table} WHERE in IN $ins`;
+export const queryRowCountBatchSelect = (): string => 'SELECT VALUE { id: id, rowCount: rowCount, state: state } FROM $ins';
+
+/** The single-query read: edges, meta, subquery children in one request. */
+export const singleSnapshotSelect = (table: string): string =>
+  `${listRefSelect(table)};\n${queryRowCountSelect()};\n${subqueryListRefSelect(table)}`;
+/** The many-query read: all edges + all metas in one request. */
+export const batchSnapshotSelect = (table: string): string =>
+  `${listRefBatchSelect(table)};\n${queryRowCountBatchSelect()}`;
+
+export interface RegisterPayload {
+  id: RecordId<string>;
+  surql: string;
+  params: Record<string, unknown>;
+  ttl: string;
+}
+
+/** Register + read back edges/meta/children in ONE request. */
+export const registerSelect = (table: string): string =>
+  `fn::query::register($config);\n${singleSnapshotSelect(table)}`;
+export const registerVars = (payload: RegisterPayload): Vars => ({ config: payload, in: payload.id });
+
+/** One heartbeat statement per view id; result index i answers id i. */
+export function heartbeatBatch(ids: ReadonlyArray<RecordId<string>>): { sql: string; vars: Vars } {
+  const vars: Vars = {};
+  const stmts = ids.map((id, i) => {
+    vars[`id${i}`] = id;
+    return `fn::query::heartbeat($id${i})`;
+  });
+  return { sql: stmts.join(';\n'), vars };
+}
+
+/** A heartbeat answer whose statement matched no row: the view was reclaimed. */
+export const heartbeatRowGone = (result: unknown): boolean => Array.isArray(result) && result.length === 0;
+
+export const bodySelect = (): string => 'SELECT * FROM $ids';
+
+/** Local transaction that MERGEs many fetched bodies. */
+export function upsertBodiesTx(records: ReadonlyArray<{ id: unknown; content: Record<string, unknown> }>): {
+  query: SealedQuery<unknown>;
+  vars: Vars;
+} {
+  const vars: Vars = {};
+  const stmts = records.map((r, i) => {
+    vars[`id${i}`] = r.id;
+    vars[`content${i}`] = r.content;
+    return surql.upsertMerge(`id${i}`, `content${i}`);
+  });
+  return { query: surql.seal<void>(surql.tx(stmts)) as SealedQuery<unknown>, vars };
+}

package/src/{modules/data → query}/window-query.test.ts

@@ -1,6 +1,24 @@
 import { describe, it, expect } from 'vitest';
 import { buildWindowMaterialization } from './window-query';
 
+describe('scanner edge cases', () => {
+  it('skips escaped quotes inside strings and ignores clauses in parens', () => {
+    const q = "SELECT * FROM t WHERE a = 'it\\'s (FROM)' AND b IN (SELECT id FROM u LIMIT 1 START 1) LIMIT 5 START 5";
+    expect(buildWindowMaterialization(q)).toEqual({ query: 'SELECT * FROM $__win' });
+  });
+  it('requires whitespace inside ORDER BY and a number after START', () => {
+    expect(buildWindowMaterialization('SELECT * FROM t ORDERBY a LIMIT 5 START 5')).toEqual({ query: 'SELECT * FROM $__win' });
+    expect(buildWindowMaterialization('SELECT * FROM t LIMIT 5 START $n')).toBeNull();
+    expect(buildWindowMaterialization('SELECT * FROM t LIMIT 5 START')).toBeNull();
+  });
+  it('matches a keyword at the very start and at the end of the string', () => {
+    expect(buildWindowMaterialization('FROM t LIMIT 5 START 5')).toEqual({ query: ' FROM $__win' });
+    expect(buildWindowMaterialization('SELECT * FROM t ORDER BY a LIMIT 5 START 5 ORDER')).toEqual({
+      query: 'SELECT * FROM $__win ORDER BY a',
+    });
+  });
+});
+
 describe('buildWindowMaterialization', () => {
   it('rewrites the game-list window query (START 30) to select the id-set, keeping ORDER BY', () => {
     const surql =

package/src/{modules/data → query}/window-query.ts

@@ -104,8 +104,9 @@ export function buildWindowMaterialization(
   idsParam = '__win'
 ): { query: string } | null {
   const kw = scanTopLevelClauses(surql);
+  // `START` is only recorded once `FROM` was seen, so a positive start value
+  // implies a FROM index.
   if (kw.startValue === null || kw.startValue <= 0) return null;
-  if (kw.fromIndex === null) return null;
   return { query: renderIdSetSelect(surql, kw, idsParam) };
 }
 

package/src/services/database/connection-supervisor.test.ts

@@ -229,6 +229,35 @@ describe('ConnectionSupervisor', () => {
     sup.dispose();
   });
 
+  it('un-sticks the forced offline state when the SDK slept through the outage', async () => {
+    const listeners = new Map<string, Array<() => void>>();
+    vi.stubGlobal('window', {
+      addEventListener: (e: string, cb: () => void) => {
+        listeners.set(e, [...(listeners.get(e) ?? []), cb]);
+      },
+      removeEventListener: () => {},
+    });
+    const { remote, emit } = makeRemote();
+    const sup = makeSupervisor(remote);
+    emit('connected');
+    sup.start();
+
+    // The browser reports offline, but the SDK's own socket never notices: its
+    // status stays `connected` throughout, so it emits no transition at all.
+    listeners.get('offline')?.forEach((cb) => cb());
+    expect(sup.connection).toBe('disconnected');
+
+    // Back online. Nothing but the probe can correct the forced state here -
+    // there is no SDK event coming - and traffic flowing under a permanent
+    // "offline" indicator is exactly the bug.
+    listeners.get('online')?.forEach((cb) => cb());
+    await vi.advanceTimersByTimeAsync(10);
+    expect(remote.query).toHaveBeenCalled();
+    expect(sup.connection).toBe('connected');
+
+    sup.dispose();
+  });
+
   it('probes immediately when a hidden tab becomes visible', async () => {
     const listeners = new Map<string, Array<() => void>>();
     vi.stubGlobal('document', {

package/src/services/database/connection-supervisor.ts

@@ -286,6 +286,13 @@ export class ConnectionSupervisor {
         `Heartbeat timed out after ${this.config.heartbeatTimeoutMs}ms`
       );
       this.heartbeatFailures = 0;
+      // A round trip is the strongest evidence there is that the socket works,
+      // so it also RECONCILES the reported state. The `offline` handler forces
+      // `disconnected` without the SDK's agreement; if the SDK never noticed
+      // the outage it emits no transition when the network returns, and that
+      // forced state would otherwise stick for the life of the page - traffic
+      // flowing while every indicator says offline.
+      this.setState('connected');
       this.startHeartbeat();
     } catch (err) {
       this.heartbeatFailures++;

package/src/services/database/database.responses.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it } from 'vitest';
+import { AbstractDatabaseService } from './database';
+import { createDatabaseEventSystem, DatabaseEventTypes } from './events/index';
+
+class Fake extends AbstractDatabaseService {
+  protected eventType = DatabaseEventTypes.RemoteQuery;
+  constructor(client: any) {
+    const noop = () => {};
+    const logger: any = { debug: noop, trace: noop, error: noop, info: noop, warn: noop, child: () => logger };
+    super(client, logger, createDatabaseEventSystem());
+  }
+  async connect() {}
+}
+
+describe('queryResponses', () => {
+  it('maps per-statement successes and failures without rejecting', async () => {
+    const client = {
+      query: () => ({
+        responses: async () => [
+          { success: true, result: [1] },
+          { success: false, error: new Error('denied') },
+          { success: false, error: 'plain' },
+        ],
+      }),
+    };
+    const out = await new Fake(client).queryResponses('A; B; C');
+    expect(out).toEqual([
+      { status: 'OK', result: [1] },
+      { status: 'ERR', error: 'denied' },
+      { status: 'ERR', error: 'plain' },
+    ]);
+  });
+  it('still rejects on a transport failure', async () => {
+    const client = {
+      query: () => ({
+        responses: async () => {
+          throw new Error('socket closed');
+        },
+      }),
+    };
+    await expect(new Fake(client).queryResponses('A')).rejects.toThrow('socket closed');
+  });
+});

package/src/services/database/database.ts

@@ -104,6 +104,30 @@ export abstract class AbstractDatabaseService {
    * SDK's pending map.
    */
   async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
+    return this.runStatement<T>(query, vars, () => this.client.query(query, vars) as unknown as Promise<T>);
+  }
+
+  /**
+   * Like {@link query} but never rejects on a statement error: every
+   * statement answers on its own as `{ status: 'OK', result }` or
+   * `{ status: 'ERR', error }`. This is what lets a batched outbox push judge
+   * each mutation separately.
+   */
+  async queryResponses(
+    query: string,
+    vars?: Record<string, unknown>
+  ): Promise<Array<{ status: 'OK'; result: unknown } | { status: 'ERR'; error: string }>> {
+    return this.runStatement(query, vars, async () => {
+      const responses = await this.client.query(query, vars).responses();
+      return responses.map((r) =>
+        r.success
+          ? { status: 'OK' as const, result: r.result }
+          : { status: 'ERR' as const, error: r.error instanceof Error ? r.error.message : String(r.error) }
+      );
+    });
+  }
+
+  private async runStatement<T>(query: string, vars: Record<string, unknown> | undefined, dispatch: () => Promise<T>): Promise<T> {
     await this.acquireQuerySlot();
     try {
       await this.beforeQuery();
@@ -113,20 +137,12 @@ export abstract class AbstractDatabaseService {
           { query, vars, Category: 'sp00ky-client::Database::query' },
           'Executing query'
         );
-        const pending = this.client.query(query, vars);
-        // In SurrealDB 2.0, .query() collects results by default.
-        // We cast to T directly as proper typing depends on the caller knowing the return structure.
         // "timed out" in the message is load-bearing: `classifySyncError`
         // keys off it to classify this as `network` so the sync queues
         // retry rather than rolling the mutation back.
-        const result = (await withTimeout(
-          pending as unknown as Promise<T>,
-          this.queryTimeoutMs,
-          () => this.timeoutError(query)
-        )) as T;
+        const result = await withTimeout(dispatch(), this.queryTimeoutMs, () => this.timeoutError(query));
         const duration = performance.now() - startTime;
 
-        // Emit query event
         this.events.emit(this.eventType, {
           query,
           vars,
@@ -143,7 +159,6 @@ export abstract class AbstractDatabaseService {
       } catch (err) {
         const duration = performance.now() - startTime;
 
-        // Emit query event with error
         this.events.emit(this.eventType, {
           query,
           vars,

package/src/services/database/local-migrator.ts

@@ -44,9 +44,9 @@ export class LocalMigrator {
 
     const systemSchema = `
       DEFINE TABLE IF NOT EXISTS _00_stream_processor_state SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _00_query SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
-      DEFINE TABLE IF NOT EXISTS _00_preload SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_view SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_window SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_failed_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_blob SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;

package/src/services/database/plan-render.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import type { QueryPlan } from '@spooky-sync/query-builder';
 import { renderBaseSelectSurql, renderRelationFetchSurql } from './plan-render';
-import { buildWindowMaterializationPlan } from '../../modules/data/window-query';
+import { buildWindowMaterializationPlan } from '../../query/window-query';
 
 describe('renderBaseSelectSurql', () => {
   it('renders projection, where, order, limit, offset', () => {

package/src/services/database/sqlite-cache-engine.test.ts

@@ -91,7 +91,8 @@ describe('SqliteCacheEngine system-table seeding', () => {
 
     await engine.connect('user:fresh');
     const open = msgs.find((m) => m.type === 'open');
-    expect(open?.payload?.systemTables).toContain('_00_query');
+    expect(open?.payload?.systemTables).toContain('_00_view');
+    expect(open?.payload?.systemTables).toContain('_00_failed_mutations');
     expect(open?.payload?.systemTables).toContain('_00_pending_mutations');
   });
 });
@@ -271,7 +272,7 @@ describe('SqliteCacheEngine role modes', () => {
     return port;
   }
 
-  it('adoptOwner opens under the worker lock and wipes stale _00_query rows', async () => {
+  it('adoptOwner opens under the worker lock without wiping anything', async () => {
     const { engine, log } = makeSharedEngine();
     const health = await engine.adoptOwner('anon', {
       workerLockName: 'sp00ky-tabs:fp:anon:worker:1',
@@ -281,7 +282,7 @@ describe('SqliteCacheEngine role modes', () => {
     expect(health.status).toBe('persistent');
     expect(engine.storageHealth.role).toBe('leader');
     expect(log[0]).toBe('open');
-    expect(log).toContain('run'); // the DELETE _00_query wipe
+    expect(log).not.toContain('run'); // no local _00_query table any more, nothing to wipe
     expect(engine.currentBucketId).toBe('anon');
   });
 

package/src/services/database/sqlite-cache-engine.ts

@@ -60,9 +60,9 @@ const SYSTEM_TABLES = [
   // In-browser circuit snapshot: one BLOB row (`circuit`) + a JSON meta row.
   // Excluded from DevTools' table listing, since its data column is not JSON.
   '_00_circuit_snapshot',
-  '_00_query',
-  '_00_preload',
+  '_00_view',
   '_00_window',
+  '_00_failed_mutations',
   '_00_schema',
   '_00_pending_mutations',
   // Blob cache manifest. Read before its first write on every boot (reconcile
@@ -605,15 +605,6 @@ export class SqliteCacheEngine implements LocalStore {
           disallowMemoryFallback: !opts.allowMemoryFallback,
         },
       });
-      // Leader wipe-on-pool-open: local `_00_query` rows from earlier sessions
-      // are dead (query hashes are session-salted) and this is the one moment
-      // no other tab is attached, so clearing here replaces the per-switch
-      // DELETE that solo mode does in doSwitchBucket. Followers never wipe.
-      try {
-        await this.rawCall('run', { sql: 'DELETE FROM "_00_query"' });
-      } catch {
-        /* fresh bucket: table just seeded, nothing to wipe */
-      }
       getStats().roleChanges = (getStats().roleChanges ?? 0) + 1;
       this.hadStore = true;
       this.closeRoleGate();

package/src/services/database/sqlite-devtools-queries.integration.test.ts

@@ -122,7 +122,7 @@ describe('DevTools explorer against real SQLite', () => {
     expect(Object.keys(info.tables)).toContain('game_insight');
     // Seeded system tables show up too — the panel's "internal" toggle filters
     // them, the engine must not pre-filter.
-    expect(Object.keys(info.tables)).toContain('_00_query');
+    expect(Object.keys(info.tables)).toContain('_00_view');
     expect(Object.keys(info.tables).some((t) => t.startsWith('sqlite_'))).toBe(false);
   });
 

package/src/services/tabs/protocol.ts

@@ -247,12 +247,16 @@ export type LeaderToFollowerMessage =
       error: string;
     }
   /** The leader's drain pushed a mutation and deleted its outbox row from the
-   *  SHARED store. Every follower starts its settled-write grace so a row it
-   *  was rendering as a pending write does not blink out before its
+   *  SHARED store. Every follower keeps the row in its overlay until its
    *  `_00_list_ref` membership arrives. */
   | {
       type: 'mutation-settled';
       mutationId: string;
       recordId: string;
       eventType: 'create' | 'update' | 'delete';
-    };
+    }
+  /** The leader's LIVE saw edges of these queries change; followers re-read
+   *  membership for the ones they hold. */
+  | { type: 'membership-dirty'; hashes: string[] }
+  /** The failed-writes tray changed size on the shared store. */
+  | { type: 'failed-mutations-changed'; count: number };