@spooky-sync/core 0.0.1-canary.30 → 0.0.1-canary.301

package/src/query/materialize.saga.test.ts

@@ -0,0 +1,115 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { runPure } from '../testing/run-pure';
+import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
+import * as R from '../state/reducers';
+import { materialize, streamUpdate } from './materialize.saga';
+
+describe('materialize', () => {
+  const plan = { table: 'thing' } as any;
+  it('no entry: nothing', async () => {
+    const out = await runPure(materialize('zz'), { state: buildState() });
+    expect(out.log.filter((e) => e.kind !== 'state.read')).toHaveLength(0);
+  });
+  const vouch = (...ids: string[]) => R.reloadViews(new Map([['other-view', ids]]));
+  it('cold: predicate scan through the plan; rows land, dirt clears, subscribers hear once', async () => {
+    const s = R.markDirty(['a'])(buildState([buildEntry({ def: { hash: 'a', plan } })], vouch('thing:1')));
+    const out = await runPure(materialize('a'), { state: s, handlers: { 'local.select': (e: any) => (e.plan.ids ? [] : [{ id: 'thing:1' }]) } });
+    const en = out.state.queries.get('a')!;
+    expect(en.records).toEqual([{ id: 'thing:1' }]);
+    expect(en.lifecycle.notified).toBe(true);
+    expect(en.telemetry.updateCount).toBe(1);
+    expect(en.telemetry.lastUpdatedAt).toBe(1_700_000_000_000);
+    expect(out.state.dirty.has('a')).toBe(false);
+    expect(out.emitted).toEqual([{ type: 'query:records', hash: 'a', records: [{ id: 'thing:1' }] }]);
+    const again = await runPure(materialize('a'), { state: out.state, handlers: { 'local.select': () => [{ id: 'thing:1' }] } });
+    expect(again.emitted).toEqual([]);
+    expect(again.state.queries.get('a')!.telemetry.updateCount).toBe(1);
+  });
+  it('cold: a body no view vouches for (an orphan) does not paint; vouched, optimistic, _00_ and id-less rows do', async () => {
+    const s = buildState(
+      [buildEntry({ def: { hash: 'a', plan } })],
+      vouch('thing:1', 'thing:4'),
+      R.outboxReplace([buildOutboxItem({ id: 'c', type: 'create', recordId: 'thing:3' })])
+    );
+    const scanned = [
+      { id: 'thing:1' },
+      { id: 'thing:2' },
+      { id: 'thing:3' },
+      { id: '_00_user_feature:f' },
+      { name: 'no id' },
+      { id: new RecordId('thing', '4') },
+      { id: new RecordId('thing', '5') },
+    ];
+    const out = await runPure(materialize('a'), { state: s, handlers: { 'local.select': () => scanned } });
+    expect(out.state.queries.get('a')!.records).toEqual([
+      { id: 'thing:1' },
+      { id: 'thing:3' },
+      { id: '_00_user_feature:f' },
+      { name: 'no id' },
+      { id: new RecordId('thing', '4') },
+    ]);
+    // Once the server has answered, membership decides and the index is not consulted.
+    const live = buildState([buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1]] })]);
+    const l = await runPure(materialize('a'), { state: live, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
+    expect(l.state.queries.get('a')!.records).toEqual([{ id: 'thing:2' }]);
+  });
+  it('cold window renders the SSP local window (orphans dropped); live renders membership with the overlay', async () => {
+    const win = buildState(
+      [buildEntry({ def: { hash: 'w', plan: { ...plan, offset: 5 }, surql: 'SELECT * FROM thing LIMIT 5 START 5' }, localArray: [['thing:7', 1], ['thing:8', 1]] })],
+      vouch('thing:7')
+    );
+    const w = await runPure(materialize('w'), { state: win, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
+    expect(w.state.queries.get('w')!.records).toEqual([{ id: 'thing:7' }]);
+    const live = buildState(
+      [buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1], ['thing:1', 1]], localArray: [['thing:9', 1]] })],
+      R.outboxReplace([buildOutboxItem({ id: 'c', type: 'create', recordId: 'thing:9' }), buildOutboxItem({ id: 'd', type: 'delete', recordId: 'thing:2' })])
+    );
+    const l = await runPure(materialize('a'), { state: live, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
+    expect(l.state.queries.get('a')!.records).toEqual([{ id: 'thing:1' }, { id: 'thing:9' }]);
+    const ordered = buildState([buildEntry({ def: { hash: 'o', plan: { ...plan, orderBy: [['a', 'asc']] } }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1], ['thing:1', 1]] })]);
+    const o = await runPure(materialize('o'), { state: ordered, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
+    expect(o.state.queries.get('o')!.records).toEqual([{ id: 'thing:2' }, { id: 'thing:1' }]);
+  });
+  it('a failing read counts an error and clears dirt; an entry removed mid-read is left alone', async () => {
+    const s = R.markDirty(['a'])(buildState([buildEntry({ def: { hash: 'a', plan } })]));
+    const fail = await runPure(materialize('a'), {
+      state: s,
+      handlers: {
+        'local.select': () => {
+          throw new Error('locked');
+        },
+      },
+    });
+    expect(fail.state.queries.get('a')!.telemetry.errorCount).toBe(1);
+    expect(fail.state.dirty.has('a')).toBe(false);
+    expect(fail.emitted).toEqual([expect.objectContaining({ level: 'warn' })]);
+    const gone = await runPure(materialize('a'), {
+      state: s,
+      handlers: {
+        'local.select': (_e, ctx) => {
+          ctx.state = R.removeQuery('a')(ctx.state);
+          return [];
+        },
+      },
+    });
+    expect(gone.emitted).toEqual([]);
+  });
+});
+
+describe('streamUpdate', () => {
+  it('takes the local id-set (dirtying the query) and records ingest timings; unknown hashes are ignored', async () => {
+    const s = buildState([buildEntry({ def: { hash: 'a' } })]);
+    const out = await runPure(streamUpdate({ queryHash: 'a', localArray: [['thing:1', 1]], op: 'CREATE', materializationTimeMs: 4, storeApplyMs: 1, circuitStepMs: 2 }), { state: s });
+    const en = out.state.queries.get('a')!;
+    expect(en.localArray).toEqual([['thing:1', 1]]);
+    expect(out.state.dirty.has('a')).toBe(true);
+    expect(en.telemetry.lastIngestLatencyMs).toBe(4);
+    expect(en.telemetry.materializationSamples).toEqual([4]);
+    expect(en.telemetry.phaseLast).toEqual({ sspStoreApply: 1, sspCircuitStep: 2 });
+    const bare = await runPure(streamUpdate({ queryHash: 'a', localArray: [] }), { state: s });
+    expect(bare.state.queries.get('a')!.telemetry.lastIngestLatencyMs).toBeNull();
+    const unknown = await runPure(streamUpdate({ queryHash: 'zz', localArray: [] }), { state: s });
+    expect(unknown.log.filter((e) => e.kind === 'state.update')).toHaveLength(0);
+  });
+});

package/src/query/materialize.saga.ts

@@ -0,0 +1,84 @@
+import type { QueryHash } from '../types';
+import type { StreamUpdate } from '../services/stream-processor/index';
+import type { Saga } from '../kernel/saga';
+import { fx } from '../kernel/effects';
+import type { QueryEntry } from '../state/client-state';
+import * as R from '../state/reducers';
+import { overlay, vouched } from '../state/selectors';
+import type { Overlay } from '../state/selectors';
+import { isWindowed, materializeEffect, rowKey, rowsEqual, rowsFromResult } from './materialize';
+import { buildRenderIds, paintsCold, resolveMembership } from './render-set';
+
+const NONE: ReadonlySet<string> = new Set();
+
+/**
+ * Render one query from state: membership (or a cold scan), the outbox
+ * overlay, one local read, then notify subscribers if the rows changed. The
+ * only writer of `records`. Runs on the `mat:<hash>` dedupe lane whenever the
+ * query is dirty. Before the server has answered (`cold`) only bodies some
+ * durable view vouches for paint (see `paintsCold`).
+ */
+export function* materialize(hash: QueryHash): Saga<void> {
+  const [entry, ov, known] = (yield fx.state.read((s) => {
+    const e = s.queries.get(hash);
+    return [e, overlay(s), e?.lifecycle.phase === 'cold' ? vouched(s) : NONE];
+  })) as [QueryEntry | undefined, Overlay, ReadonlySet<string>];
+  if (!entry) return;
+  const t0 = (yield fx.now()) as number;
+  const isWindow = isWindowed(entry.def.surql);
+  const paints = (id: string | null) => id === null || paintsCold(id, known, ov);
+  const coldWindow = entry.lifecycle.phase === 'cold' && isWindow;
+  const membership = resolveMembership({
+    phase: entry.lifecycle.phase,
+    remoteArray: entry.remoteArray,
+    localArray: coldWindow ? entry.localArray.filter(([id]) => paints(id)) : entry.localArray,
+    isWindow,
+  });
+  const ids = membership
+    ? buildRenderIds(membership, entry.localArray, ov, {
+        hasExplicitOrder: (entry.def.plan?.orderBy?.length ?? 0) > 0,
+        isWindow,
+      })
+    : null;
+  const effect = materializeEffect(entry.def, ids);
+  let rows: Record<string, unknown>[];
+  try {
+    rows = rowsFromResult(effect, yield effect);
+    if (ids === null) rows = rows.filter((row) => paints(rowKey(row)));
+  } catch (error) {
+    yield fx.state.update(R.compose(R.recordError(hash), R.clearDirty(hash)));
+    yield fx.emit({ type: 'log', level: 'warn', message: 'materialize failed', data: { hash, error } });
+    return;
+  }
+  const current = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+  if (!current) return;
+  const t1 = (yield fx.now()) as number;
+  const changed = !rowsEqual(rows, current.records);
+  yield fx.state.update(
+    R.compose(R.setRecords(hash, rows, changed, t1 - t0), changed ? R.stampUpdated(hash, t1) : (s) => s)
+  );
+  if (changed) yield fx.emit({ type: 'query:records', hash, records: rows });
+}
+
+/**
+ * A view update from the in-browser SSP: the query's local id-set moved.
+ * State takes the array (which dirties the query); the ingest timings feed
+ * telemetry. Synthetic updates no longer exist: re-materialization is dirt.
+ */
+export function* streamUpdate(update: StreamUpdate): Saga<void> {
+  const hash = update.queryHash;
+  const exists = (yield fx.state.read((s) => s.queries.has(hash))) as boolean;
+  if (!exists) return;
+  const phases: Array<[string, number | undefined]> = [
+    ['sspStoreApply', update.storeApplyMs],
+    ['sspCircuitStep', update.circuitStepMs],
+    ['sspTransform', update.transformMs],
+  ];
+  yield fx.state.update(
+    R.compose(
+      R.setLocalArray(hash, update.localArray),
+      typeof update.materializationTimeMs === 'number' ? R.recordIngest(hash, update.materializationTimeMs) : (s) => s,
+      ...phases.filter((p): p is [string, number] => typeof p[1] === 'number').map(([phase, ms]) => R.recordPhase(hash, phase, ms))
+    )
+  );
+}

package/src/query/materialize.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { isWindowed, materializeEffect, rowKey, rowsEqual, rowsFromResult } from './materialize';
+
+describe('materializeEffect', () => {
+  const plan = { table: 'thing', where: [{ field: 'a', op: '=', value: 1 }] } as any;
+  it('plan + ids: id-set plan select without where/limit/offset', () => {
+    const e = materializeEffect({ surql: 'SELECT * FROM thing', params: { a: 1 }, plan: { ...plan, limit: 5, offset: 2 } }, ['thing:1']);
+    expect(e.kind).toBe('local.select');
+    if (e.kind !== 'local.select') throw new Error();
+    expect(e.plan.ids).toEqual([new RecordId('thing', '1')]);
+    expect(e.plan.where).toBeUndefined();
+    expect(e.plan.limit).toBeUndefined();
+    expect(e.params).toEqual({ a: 1 });
+  });
+  it('surql + ids: id-set rewrite bound to __win; unparseable surql falls back to the raw query', () => {
+    const e = materializeEffect({ surql: 'SELECT * FROM thing WHERE a = $a ORDER BY b LIMIT 3', params: { a: 1 } }, ['thing:1']);
+    expect(e).toMatchObject({ kind: 'local.query', sql: 'SELECT * FROM $__win ORDER BY b' });
+    if (e.kind !== 'local.query') throw new Error();
+    expect(e.vars).toEqual({ a: 1, __win: [new RecordId('thing', '1')] });
+    const raw = materializeEffect({ surql: 'RETURN 1', params: {} }, ['thing:1']);
+    expect(raw).toMatchObject({ kind: 'local.query', sql: 'RETURN 1' });
+  });
+  it('null ids: predicate scan via plan or raw surql', () => {
+    expect(materializeEffect({ surql: 's', params: {}, plan }, null)).toMatchObject({ kind: 'local.select', plan });
+    expect(materializeEffect({ surql: 'SELECT * FROM thing', params: { x: 1 } }, null)).toMatchObject({
+      kind: 'local.query',
+      sql: 'SELECT * FROM thing',
+      vars: { x: 1 },
+    });
+  });
+  it('rowsFromResult reads both effect shapes; rowsEqual compares structurally', () => {
+    const sel = materializeEffect({ surql: 's', params: {}, plan }, null);
+    const q = materializeEffect({ surql: 'SELECT * FROM thing', params: {} }, null);
+    expect(rowsFromResult(sel, [{ id: 1 }])).toEqual([{ id: 1 }]);
+    expect(rowsFromResult(sel, undefined)).toEqual([]);
+    expect(rowsFromResult(q, [[{ id: 1 }]])).toEqual([{ id: 1 }]);
+    expect(rowsFromResult(q, [null])).toEqual([]);
+    expect(rowsFromResult(q, 'x')).toEqual([]);
+    const rows = [{ id: 1 }];
+    expect(rowsEqual(rows, rows)).toBe(true);
+    expect(rowsEqual(rows, [{ id: 1 }])).toBe(true);
+    expect(rowsEqual(rows, [{ id: 2 }])).toBe(false);
+    expect(rowsEqual(rows, [])).toBe(false);
+  });
+  it('isWindowed', () => {
+    expect(isWindowed('SELECT * FROM t LIMIT 10 START 10')).toBe(true);
+    expect(isWindowed('SELECT * FROM t LIMIT 10')).toBe(false);
+  });
+});
+
+describe('rowKey', () => {
+  it('reads a string id (SQLite) or a RecordId (SurrealDB); null when the row carries none', () => {
+    expect(rowKey({ id: 'thing:1' })).toBe('thing:1');
+    expect(rowKey({ id: new RecordId('thing', '2') })).toBe('thing:2');
+    for (const row of [{ name: 'x' }, { id: null }, { id: { other: 1 } }, null, 5]) expect(rowKey(row)).toBeNull();
+  });
+});

package/src/query/materialize.ts

@@ -0,0 +1,57 @@
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import type { RecordId } from 'surrealdb';
+import type { Effect } from '../kernel/effects';
+import { fx } from '../kernel/effects';
+import { encodeRecordId, parseRecordIdString } from '../utils/index';
+import { buildIdSetPlan, buildIdSetSurql, buildWindowMaterialization } from './window-query';
+
+export interface MaterializeSource {
+  surql: string;
+  params: Record<string, unknown>;
+  plan?: QueryPlan;
+}
+
+export const isWindowed = (surql: string): boolean => buildWindowMaterialization(surql) !== null;
+
+/**
+ * The one read that materializes a query. `ids` is the render set (membership
+ * with the overlay applied); `null` means "no membership yet, scan the local
+ * store with the query's own predicate".
+ */
+export function materializeEffect(src: MaterializeSource, ids: string[] | null): Effect {
+  if (ids !== null) {
+    const parsed = ids.map((id) => parseRecordIdString(id));
+    if (src.plan) return fx.local.select(buildIdSetPlan(src.plan, parsed), src.params);
+    const idSet = buildIdSetSurql(src.surql);
+    if (idSet) return fx.local.query(idSet.query, { ...src.params, __win: parsed });
+    return fx.local.query(src.surql, src.params);
+  }
+  if (src.plan) return fx.local.select(src.plan, src.params);
+  return fx.local.query(src.surql, src.params);
+}
+
+/** Rows out of either read effect's result. */
+export function rowsFromResult(effect: Effect, result: unknown): Record<string, unknown>[] {
+  if (effect.kind === 'local.select') return (result as Record<string, unknown>[] | undefined) ?? [];
+  const first = Array.isArray(result) ? result[0] : undefined;
+  return Array.isArray(first) ? (first as Record<string, unknown>[]) : [];
+}
+
+/**
+ * A row's encoded record id: a string on the SQLite engine, a `RecordId` on
+ * the SurrealDB one. `null` for a row that carries none (a projection
+ * without `id`, a `SELECT VALUE`).
+ */
+export function rowKey(row: unknown): string | null {
+  const id = row !== null && typeof row === 'object' ? (row as { id?: unknown }).id : undefined;
+  if (typeof id === 'string') return id;
+  if (id !== null && typeof id === 'object' && 'table' in id && 'id' in id) return encodeRecordId(id as RecordId<string>);
+  return null;
+}
+
+/** Cheap structural equality for the "did the rows change" check. */
+export function rowsEqual(a: ReadonlyArray<unknown>, b: ReadonlyArray<unknown>): boolean {
+  if (a === b) return true;
+  if (a.length !== b.length) return false;
+  return JSON.stringify(a) === JSON.stringify(b);
+}