@spooky-sync/core 0.0.1-canary.25 → 0.0.1-canary.255

package/src/query/render-set.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from 'vitest';
+import { buildRenderIds, resolveMembership } from './render-set';
+
+const ov = (writes: string[] = [], deletes: string[] = []) => ({ writes: new Set(writes), deletes: new Set(deletes) });
+
+describe('resolveMembership', () => {
+  it('cold scans, cold window uses the local view, anything else uses membership', () => {
+    const remoteArray: [string, number][] = [['t:1', 1]];
+    const localArray: [string, number][] = [['t:2', 1]];
+    expect(resolveMembership({ phase: 'cold', remoteArray, localArray, isWindow: false })).toBeNull();
+    expect(resolveMembership({ phase: 'cold', remoteArray, localArray, isWindow: true })).toBe(localArray);
+    expect(resolveMembership({ phase: 'cached', remoteArray, localArray, isWindow: false })).toBe(remoteArray);
+    expect(resolveMembership({ phase: 'view-lost', remoteArray, localArray, isWindow: true })).toBe(remoteArray);
+  });
+});
+
+describe('buildRenderIds', () => {
+  const opts = { hasExplicitOrder: false, isWindow: false };
+  it('membership minus deletes, sorted', () => {
+    expect(buildRenderIds([['t:b', 1], ['t:a', 1], ['t:c', 1]], [], ov([], ['t:c']), opts)).toEqual(['t:a', 't:b']);
+  });
+  it('adds pending writes the local view admits, never twice, never deleted', () => {
+    const ids = buildRenderIds(
+      [['t:1', 1]],
+      [['t:1', 1], ['t:2', 1], ['t:3', 1], ['t:4', 1]],
+      ov(['t:1', 't:2', 't:3', 't:9'], ['t:3']),
+      opts
+    );
+    expect(ids).toEqual(['t:1', 't:2']);
+  });
+  it('keeps server order for windows and explicit orderBy; dedupes membership', () => {
+    const m: [string, number][] = [['t:b', 1], ['t:a', 1], ['t:b', 1]];
+    expect(buildRenderIds(m, [], ov(), { hasExplicitOrder: true, isWindow: false })).toEqual(['t:b', 't:a']);
+    expect(buildRenderIds(m, [], ov(), { hasExplicitOrder: false, isWindow: true })).toEqual(['t:b', 't:a']);
+  });
+});

package/src/query/render-set.ts

@@ -0,0 +1,60 @@
+import type { RecordVersionArray } from '../types';
+import type { QueryPhase } from '../state/lifecycle';
+import type { Overlay } from '../state/selectors';
+
+export interface RenderInput {
+  phase: QueryPhase;
+  remoteArray: RecordVersionArray;
+  localArray: RecordVersionArray;
+  isWindow: boolean;
+}
+
+/**
+ * The id-set a query renders from, or `null` when it must fall back to a
+ * local predicate scan (a cold query that was never resolved on this device).
+ * A cold window has no usable scan (re-applying `START m` against the shared
+ * store returns the wrong rows), so it renders the SSP's local window instead.
+ */
+export function resolveMembership(input: RenderInput): RecordVersionArray | null {
+  if (input.phase !== 'cold') return input.remoteArray;
+  if (input.isWindow) return input.localArray;
+  return null;
+}
+
+export interface RenderOptions {
+  hasExplicitOrder: boolean;
+  isWindow: boolean;
+}
+
+/**
+ *   render = (membership ∪ (writes ∩ localView)) − deletes
+ *
+ * `writes`/`deletes` are the outbox overlay (pending and acked items). The
+ * middle term keeps optimistic writes visible without re-admitting stale rows:
+ * `localView` (the SSP's local id-set) says whether the written row matches
+ * the query per local truth. Sorted unless the query orders itself or is a
+ * window (whose id order is the slice order).
+ */
+export function buildRenderIds(
+  membership: RecordVersionArray,
+  localView: RecordVersionArray,
+  overlay: Overlay,
+  opts: RenderOptions
+): string[] {
+  const ordered: string[] = [];
+  const seen = new Set<string>();
+  for (const [id] of membership) {
+    if (overlay.deletes.has(id) || seen.has(id)) continue;
+    seen.add(id);
+    ordered.push(id);
+  }
+  if (overlay.writes.size > 0) {
+    for (const [id] of localView) {
+      if (!overlay.writes.has(id) || overlay.deletes.has(id) || seen.has(id)) continue;
+      seen.add(id);
+      ordered.push(id);
+    }
+  }
+  if (!opts.hasExplicitOrder && !opts.isWindow) ordered.sort();
+  return ordered;
+}

package/src/query/sql.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import * as sql from './sql';
+import {
+  buildListRefBatchSelect,
+  buildListRefSelect,
+  buildQueryRowCountBatchSelect,
+  buildQueryRowCountSelect,
+  buildSubqueryListRefSelect,
+} from '../sync/policy';
+
+describe('query sql builders (golden against the previous builders)', () => {
+  it('edge and meta selects are byte-identical to the old helpers', () => {
+    expect(sql.listRefSelect('_00_list_ref_user_x')).toBe(buildListRefSelect('_00_list_ref_user_x'));
+    expect(sql.subqueryListRefSelect('t')).toBe(buildSubqueryListRefSelect('t'));
+    expect(sql.queryRowCountSelect()).toBe(buildQueryRowCountSelect());
+    expect(sql.listRefBatchSelect('t')).toBe(buildListRefBatchSelect('t'));
+    expect(sql.queryRowCountBatchSelect()).toBe(buildQueryRowCountBatchSelect());
+  });
+  it('composes the single, batch and register requests', () => {
+    expect(sql.singleSnapshotSelect('t').split(';\n')).toEqual([
+      sql.listRefSelect('t'),
+      sql.queryRowCountSelect(),
+      sql.subqueryListRefSelect('t'),
+    ]);
+    expect(sql.batchSnapshotSelect('t')).toBe(`${sql.listRefBatchSelect('t')};\n${sql.queryRowCountBatchSelect()}`);
+    expect(sql.registerSelect('t').startsWith('fn::query::register($config);\n')).toBe(true);
+    const id = new RecordId('_00_query', 'h');
+    expect(sql.registerVars({ id, surql: 's', params: { a: 1 }, ttl: '10m' })).toEqual({
+      config: { id, surql: 's', params: { a: 1 }, ttl: '10m' },
+      in: id,
+    });
+  });
+  it('view rows and ids', () => {
+    expect(sql.viewRecordId('k')).toEqual(new RecordId('_00_view', 'k'));
+    expect(sql.viewRow([['t:1', 1]], true, 5)).toEqual({ ids: [['t:1', 1]], confirmed: true, updatedAt: 5 });
+    expect(sql.readLegacyViewRows()).toBe('SELECT * FROM _00_window');
+    expect(sql.countViewRows()).toContain('_00_view');
+    expect(sql.bodySelect()).toBe('SELECT * FROM $ids');
+  });
+  it('heartbeat batch answers per index and detects reclaimed rows', () => {
+    const a = new RecordId('_00_query', 'a');
+    const b = new RecordId('_00_query', 'b');
+    expect(sql.heartbeatBatch([a, b])).toEqual({
+      sql: 'fn::query::heartbeat($id0);\nfn::query::heartbeat($id1)',
+      vars: { id0: a, id1: b },
+    });
+    expect(sql.heartbeatRowGone([])).toBe(true);
+    expect(sql.heartbeatRowGone([{ id: a }])).toBe(false);
+    expect(sql.heartbeatRowGone(undefined)).toBe(false);
+  });
+  it('upsertBodiesTx MERGEs each body inside one transaction', () => {
+    const { query, vars } = sql.upsertBodiesTx([
+      { id: 'thing:1', content: { a: 1 } },
+      { id: 'thing:2', content: { b: 2 } },
+    ]);
+    expect(query.sql).toBe(
+      'BEGIN TRANSACTION;\nUPSERT ONLY $id0 MERGE $content0;UPSERT ONLY $id1 MERGE $content1;\nCOMMIT TRANSACTION;'
+    );
+    expect(vars).toEqual({ id0: 'thing:1', content0: { a: 1 }, id1: 'thing:2', content1: { b: 2 } });
+  });
+});

package/src/query/sql.ts

@@ -0,0 +1,79 @@
+import { RecordId } from 'surrealdb';
+import type { RecordVersionArray } from '../types';
+import type { Vars } from '../kernel/effects';
+import { surql } from '../utils/surql';
+import type { SealedQuery } from '../utils/surql';
+
+/** Every SurrealQL statement the query side sends, as pure builders. */
+
+export const VIEW_TABLE = '_00_view';
+export const LEGACY_VIEW_TABLE = '_00_window';
+
+export const viewRecordId = (viewKey: string): RecordId<string> => new RecordId(VIEW_TABLE, viewKey);
+
+export const viewRow = (ids: RecordVersionArray, confirmed: boolean, now: number): Record<string, unknown> => ({
+  ids,
+  confirmed,
+  updatedAt: now,
+});
+
+/** Copy every legacy `_00_window` row into `_00_view` once. */
+export const readLegacyViewRows = (): string => `SELECT * FROM ${LEGACY_VIEW_TABLE}`;
+export const countViewRows = (): string => `SELECT count() FROM ${VIEW_TABLE} GROUP ALL`;
+
+export const listRefSelect = (table: string): string =>
+  `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
+export const subqueryListRefSelect = (table: string): string =>
+  `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NOT NONE`;
+export const queryRowCountSelect = (): string => 'SELECT VALUE { rowCount: rowCount, state: state } FROM ONLY $in';
+export const listRefBatchSelect = (table: string): string =>
+  `SELECT in, out, version, parent FROM ${table} WHERE in IN $ins`;
+export const queryRowCountBatchSelect = (): string => 'SELECT VALUE { id: id, rowCount: rowCount, state: state } FROM $ins';
+
+/** The single-query read: edges, meta, subquery children in one request. */
+export const singleSnapshotSelect = (table: string): string =>
+  `${listRefSelect(table)};\n${queryRowCountSelect()};\n${subqueryListRefSelect(table)}`;
+/** The many-query read: all edges + all metas in one request. */
+export const batchSnapshotSelect = (table: string): string =>
+  `${listRefBatchSelect(table)};\n${queryRowCountBatchSelect()}`;
+
+export interface RegisterPayload {
+  id: RecordId<string>;
+  surql: string;
+  params: Record<string, unknown>;
+  ttl: string;
+}
+
+/** Register + read back edges/meta/children in ONE request. */
+export const registerSelect = (table: string): string =>
+  `fn::query::register($config);\n${singleSnapshotSelect(table)}`;
+export const registerVars = (payload: RegisterPayload): Vars => ({ config: payload, in: payload.id });
+
+/** One heartbeat statement per view id; result index i answers id i. */
+export function heartbeatBatch(ids: ReadonlyArray<RecordId<string>>): { sql: string; vars: Vars } {
+  const vars: Vars = {};
+  const stmts = ids.map((id, i) => {
+    vars[`id${i}`] = id;
+    return `fn::query::heartbeat($id${i})`;
+  });
+  return { sql: stmts.join(';\n'), vars };
+}
+
+/** A heartbeat answer whose statement matched no row: the view was reclaimed. */
+export const heartbeatRowGone = (result: unknown): boolean => Array.isArray(result) && result.length === 0;
+
+export const bodySelect = (): string => 'SELECT * FROM $ids';
+
+/** Local transaction that MERGEs many fetched bodies. */
+export function upsertBodiesTx(records: ReadonlyArray<{ id: unknown; content: Record<string, unknown> }>): {
+  query: SealedQuery<unknown>;
+  vars: Vars;
+} {
+  const vars: Vars = {};
+  const stmts = records.map((r, i) => {
+    vars[`id${i}`] = r.id;
+    vars[`content${i}`] = r.content;
+    return surql.upsertMerge(`id${i}`, `content${i}`);
+  });
+  return { query: surql.seal<void>(surql.tx(stmts)) as SealedQuery<unknown>, vars };
+}

package/src/query/window-query.test.ts

@@ -0,0 +1,70 @@
+import { describe, it, expect } from 'vitest';
+import { buildWindowMaterialization } from './window-query';
+
+describe('scanner edge cases', () => {
+  it('skips escaped quotes inside strings and ignores clauses in parens', () => {
+    const q = "SELECT * FROM t WHERE a = 'it\\'s (FROM)' AND b IN (SELECT id FROM u LIMIT 1 START 1) LIMIT 5 START 5";
+    expect(buildWindowMaterialization(q)).toEqual({ query: 'SELECT * FROM $__win' });
+  });
+  it('requires whitespace inside ORDER BY and a number after START', () => {
+    expect(buildWindowMaterialization('SELECT * FROM t ORDERBY a LIMIT 5 START 5')).toEqual({ query: 'SELECT * FROM $__win' });
+    expect(buildWindowMaterialization('SELECT * FROM t LIMIT 5 START $n')).toBeNull();
+    expect(buildWindowMaterialization('SELECT * FROM t LIMIT 5 START')).toBeNull();
+  });
+  it('matches a keyword at the very start and at the end of the string', () => {
+    expect(buildWindowMaterialization('FROM t LIMIT 5 START 5')).toEqual({ query: ' FROM $__win' });
+    expect(buildWindowMaterialization('SELECT * FROM t ORDER BY a LIMIT 5 START 5 ORDER')).toEqual({
+      query: 'SELECT * FROM $__win ORDER BY a',
+    });
+  });
+});
+
+describe('buildWindowMaterialization', () => {
+  it('rewrites the game-list window query (START 30) to select the id-set, keeping ORDER BY', () => {
+    const surql =
+      'SELECT * FROM game WHERE database = $database ORDER BY sort_index asc, date desc LIMIT 30 START 30;';
+    const r = buildWindowMaterialization(surql);
+    expect(r).not.toBeNull();
+    expect(r!.query).toBe('SELECT * FROM $__win ORDER BY sort_index asc, date desc');
+  });
+
+  it('returns null for START 0 (offset-free windows still work via the normal re-query)', () => {
+    const surql =
+      'SELECT * FROM game WHERE database = $database ORDER BY sort_index asc, date desc LIMIT 30 START 0;';
+    expect(buildWindowMaterialization(surql)).toBeNull();
+  });
+
+  it('returns null when there is no START clause', () => {
+    expect(
+      buildWindowMaterialization('SELECT * FROM game WHERE database = $database LIMIT 30;')
+    ).toBeNull();
+    expect(buildWindowMaterialization('SELECT * FROM game;')).toBeNull();
+  });
+
+  it('preserves a custom projection', () => {
+    const surql = 'SELECT id, white, black FROM game ORDER BY date desc LIMIT 30 START 60;';
+    expect(buildWindowMaterialization(surql)!.query).toBe(
+      'SELECT id, white, black FROM $__win ORDER BY date desc'
+    );
+  });
+
+  it('omits ORDER BY when the query had none', () => {
+    const surql = 'SELECT * FROM game LIMIT 30 START 30;';
+    expect(buildWindowMaterialization(surql)!.query).toBe('SELECT * FROM $__win');
+  });
+
+  it('ignores FROM/ORDER BY/LIMIT/START inside subqueries (paren-aware)', () => {
+    const surql =
+      'SELECT *, (SELECT * FROM comment WHERE game = $parent.id ORDER BY created_at desc LIMIT 5) AS comments ' +
+      'FROM game ORDER BY sort_index asc LIMIT 30 START 90;';
+    expect(buildWindowMaterialization(surql)!.query).toBe(
+      'SELECT *, (SELECT * FROM comment WHERE game = $parent.id ORDER BY created_at desc LIMIT 5) AS comments FROM $__win ORDER BY sort_index asc'
+    );
+  });
+
+  it('does not treat a START inside a string literal as the offset', () => {
+    const surql = "SELECT * FROM game WHERE note = 'LIMIT 30 START 30' ORDER BY date desc LIMIT 30 START 30;";
+    const r = buildWindowMaterialization(surql);
+    expect(r!.query).toBe('SELECT * FROM $__win ORDER BY date desc');
+  });
+});

package/src/query/window-query.ts

@@ -0,0 +1,195 @@
+/**
+ * Rewrite a windowed (`LIMIT n START m`, m>0) SELECT so its rows are
+ * materialized from an explicit record-id set — the window the SSP already
+ * computed — instead of re-running the original query (with its `START m`)
+ * against the shared local DB.
+ *
+ * Why: `DataModule.processStreamUpdate` materializes a query's rows by
+ * re-querying the local in-browser SurrealDB with the original surql. For an
+ * offset query that re-applies `START m` against the *shared* local store —
+ * which, with sparse windowing, may hold only this window's rows — so it skips
+ * them all and returns nothing (the "page 2 returns 0 rows" bug). The SSP's
+ * materialized view (`StreamUpdate.localArray`) is exactly this window's row
+ * ids, so we select those ids directly and re-apply the original `ORDER BY` for
+ * stable display order.
+ *
+ * Returns `null` for non-offset queries (`START` absent or 0) — the caller
+ * keeps the normal re-query path, so only the broken case changes behavior.
+ *
+ * Preserves the `SELECT <projection>` clause and the top-level `ORDER BY`;
+ * drops the original `FROM`/`WHERE`/`LIMIT`/`START`. Subqueries inside the
+ * projection are preserved verbatim (the scanner is paren/quote aware), with
+ * one caveat: SurrealDB v3 drops `*` in the `SELECT *, <subquery> FROM $param`
+ * shape — such windowed+subquery queries are rare and were already returning 0,
+ * so this is no regression.
+ */
+import type { QueryPlan } from '@spooky-sync/query-builder';
+
+/**
+ * Plan-level window materialization: restrict a windowed query's base rows to
+ * exactly the SSP-computed id-set, dropping `where`/`limit`/`offset` but keeping
+ * `orderBy`, `select` and `relations`. The engine-neutral counterpart of
+ * {@link buildWindowMaterialization} (which does the same by string surgery for
+ * the raw-SurrealQL path). Returns `null` for non-offset queries so the caller
+ * keeps the normal re-query path.
+ */
+export function buildWindowMaterializationPlan(
+  plan: QueryPlan,
+  ids: unknown[]
+): QueryPlan | null {
+  if (plan.offset === undefined || plan.offset <= 0) return null;
+  return buildIdSetPlan(plan, ids);
+}
+
+/**
+ * Raw-SurrealQL counterpart of {@link buildIdSetPlan}, for queries that arrived
+ * without a plan. Same rewrite as {@link buildWindowMaterialization} without the
+ * offset guard: keep the projection and top-level `ORDER BY`, replace the source
+ * with the bound id-set.
+ */
+export function buildIdSetSurql(
+  surql: string,
+  idsParam = '__win'
+): { query: string } | null {
+  const kw = scanTopLevelClauses(surql);
+  if (kw.fromIndex === null) return null;
+  return { query: renderIdSetSelect(surql, kw, idsParam) };
+}
+
+/** Shared body of the two id-set surql rewrites: keep the projection and the
+ *  top-level `ORDER BY`, replace the source with the bound id-set. */
+function renderIdSetSelect(surql: string, kw: TopLevelClauses, idsParam: string): string {
+  const selectClause = surql.slice(0, kw.fromIndex!).trimEnd(); // "SELECT <projection>"
+
+  let orderBy = '';
+  const orderByIndex = kw.orderByIndex;
+  if (orderByIndex !== null) {
+    const ends = [kw.limitIndex, kw.startIndex, kw.semicolonIndex, surql.length].filter(
+      (n): n is number => n !== null && n > orderByIndex
+    );
+    orderBy = ' ' + surql.slice(orderByIndex, Math.min(...ends)).trim();
+  }
+
+  return `${selectClause} FROM $${idsParam}${orderBy}`;
+}
+
+/**
+ * Restrict any plan's base rows to an explicit id-set, regardless of offset.
+ *
+ * Same rewrite as {@link buildWindowMaterializationPlan} without the offset
+ * guard, for rendering a query from its authoritative membership list rather
+ * than by re-running its predicate.
+ *
+ * `where` is dropped deliberately: the id-set already IS the answer to the
+ * predicate as the server evaluated it, and re-applying the `WHERE` locally is
+ * exactly the bug — a row that left the window keeps its (stale, never
+ * re-fetched) local body, so it keeps matching and keeps rendering.
+ * `limit`/`offset` go for the same reason: both are already baked into which ids
+ * are in the set, so the set itself is the bound. (Both engines ignore them on
+ * the `ids` path regardless — only `select` and `orderBy` are honored there.)
+ * `orderBy` is kept so display order stays deterministic.
+ */
+export function buildIdSetPlan(plan: QueryPlan, ids: unknown[]): QueryPlan {
+  return {
+    ...plan,
+    ids,
+    where: undefined,
+    limit: undefined,
+    offset: undefined,
+  };
+}
+
+export function buildWindowMaterialization(
+  surql: string,
+  idsParam = '__win'
+): { query: string } | null {
+  const kw = scanTopLevelClauses(surql);
+  // `START` is only recorded once `FROM` was seen, so a positive start value
+  // implies a FROM index.
+  if (kw.startValue === null || kw.startValue <= 0) return null;
+  return { query: renderIdSetSelect(surql, kw, idsParam) };
+}
+
+interface TopLevelClauses {
+  fromIndex: number | null;
+  orderByIndex: number | null;
+  limitIndex: number | null;
+  startIndex: number | null;
+  startValue: number | null;
+  semicolonIndex: number | null;
+}
+
+// Single pass over the query, tracking paren depth and single-quoted strings so
+// that clauses inside subqueries (which live in parens) are ignored — only the
+// outermost (depth-0) clause keywords are recorded.
+function scanTopLevelClauses(sql: string): TopLevelClauses {
+  const out: TopLevelClauses = {
+    fromIndex: null,
+    orderByIndex: null,
+    limitIndex: null,
+    startIndex: null,
+    startValue: null,
+    semicolonIndex: null,
+  };
+  let depth = 0;
+  let inStr = false;
+
+  for (let i = 0; i < sql.length; i++) {
+    const ch = sql[i];
+    if (inStr) {
+      if (ch === '\\') i++; // skip escaped char
+      else if (ch === "'") inStr = false;
+      continue;
+    }
+    if (ch === "'") { inStr = true; continue; }
+    if (ch === '(') { depth++; continue; }
+    if (ch === ')') { depth--; continue; }
+    if (depth !== 0) continue;
+    if (ch === ';' && out.semicolonIndex === null) { out.semicolonIndex = i; continue; }
+
+    // Only test keywords at a word boundary.
+    if (!isWordBoundary(sql, i)) continue;
+    if (out.fromIndex === null && matchKeyword(sql, i, 'FROM')) { out.fromIndex = i; continue; }
+    // FROM must come before the trailing clauses; only record them once seen.
+    if (out.fromIndex === null) continue;
+    if (out.orderByIndex === null && matchKeyword(sql, i, 'ORDER BY')) { out.orderByIndex = i; continue; }
+    if (out.limitIndex === null && matchKeyword(sql, i, 'LIMIT')) { out.limitIndex = i; continue; }
+    if (out.startIndex === null && matchKeyword(sql, i, 'START')) {
+      out.startIndex = i;
+      out.startValue = readNumberAfter(sql, i + 'START'.length);
+      continue;
+    }
+  }
+  return out;
+}
+
+function isWordBoundary(sql: string, i: number): boolean {
+  if (i === 0) return true;
+  return !/[A-Za-z0-9_]/.test(sql[i - 1]);
+}
+
+// Case-insensitive keyword match where internal whitespace (e.g. "ORDER BY")
+// matches any run of whitespace, and the keyword ends on a non-word char.
+function matchKeyword(sql: string, i: number, keyword: string): boolean {
+  const parts = keyword.split(' ');
+  let pos = i;
+  for (let p = 0; p < parts.length; p++) {
+    const word = parts[p];
+    if (sql.slice(pos, pos + word.length).toUpperCase() !== word) return false;
+    pos += word.length;
+    if (p < parts.length - 1) {
+      const wsStart = pos;
+      while (pos < sql.length && /\s/.test(sql[pos])) pos++;
+      if (pos === wsStart) return false; // required whitespace
+    }
+  }
+  // Must end on a non-word char (or end of string).
+  return pos >= sql.length || !/[A-Za-z0-9_]/.test(sql[pos]);
+}
+
+function readNumberAfter(sql: string, from: number): number | null {
+  let pos = from;
+  while (pos < sql.length && /\s/.test(sql[pos])) pos++;
+  const m = /^\d+/.exec(sql.slice(pos));
+  return m ? parseInt(m[0], 10) : null;
+}