@spooky-sync/core 0.0.1-canary.286 → 0.0.1-canary.288

package/src/boot/boot.saga.ts

@@ -6,6 +6,8 @@ import { ANON_USER_ID } from '../modules/ref-tables';
 import type { SagaEnv } from '../query/env';
 import * as sql from '../query/sql';
 import { loadOutbox } from '../mutation/push.saga';
+import { loadViews } from '../query/lifecycle.saga';
+import { GC_BOOT_DELAY_MS } from '../kernel/constants';
 
 export interface BootOptions {
   /** A shared-tabs coordinator is available: ask it for a role instead of opening the store alone. */
@@ -34,6 +36,7 @@ export function* boot(env: SagaEnv, opts: BootOptions): Saga<void> {
   yield fx.state.update(R.setIdentity({ bucketId: bucket }));
   if ((yield fx.service('local.usesSurqlSchema')) as boolean) yield fx.service('migrator.provision');
   yield* migrateWindowToView();
+  yield* loadViews();
   yield fx.dispatch({ type: 'WarmBlobs' });
   yield fx.service('ssp.init');
   yield fx.service('ssp.setPermissions');
@@ -51,7 +54,7 @@ export function* boot(env: SagaEnv, opts: BootOptions): Saga<void> {
   yield fx.service('features.init');
   yield fx.service('releases.init');
   yield fx.dispatch({ type: 'LifecycleTick' });
-  yield fx.dispatch({ type: 'GcTick' });
+  yield fx.timer.set('gc', GC_BOOT_DELAY_MS, { type: 'GcTick' });
   yield fx.state.update(R.setIdentity({ localReady: true }));
   yield fx.dispatch({ type: 'StartRemote' });
 }

package/src/boot/bucket-switch.saga.ts

@@ -10,6 +10,7 @@ import type { SagaEnv } from '../query/env';
 import { queryHashInput } from '../query/hash';
 import { isResolvedBefore, parseViewRow } from '../query/membership';
 import * as sql from '../query/sql';
+import { loadViews } from '../query/lifecycle.saga';
 import { loadOutbox } from '../mutation/push.saga';
 
 export interface BucketSwitchOptions {
@@ -71,6 +72,7 @@ export function* bucketSwitch(env: SagaEnv, target: string, release: (() => void
       yield fx.emit({ type: 'log', level: 'warn', message: 'failed to re-persist the auth token', data: { error } });
     }
   }
+  yield* loadViews();
   yield* rebindQueries();
   yield* loadOutbox(env);
   yield fx.dispatch({ type: 'EnsureRegistered' });

package/src/client/runtime.test.ts

@@ -73,7 +73,10 @@ describe('Runtime lanes', () => {
 
 describe('Runtime state hooks', () => {
   it('dirty queries are materialized after the debounce, once per burst', async () => {
-    const { a, rt } = make({ local: { select: async () => [{ id: 'thing:1' }] } }, buildState([buildEntry({ def: { hash: 'q', plan: { table: 'thing' } as any } })]));
+    const { a, rt } = make(
+      { local: { select: async () => [{ id: 'thing:1' }] } },
+      buildState([buildEntry({ def: { hash: 'q', plan: { table: 'thing' } as any } })], R.reloadViews(new Map([['v', ['thing:1']]])))
+    );
     const seen: unknown[] = [];
     rt.subscribe('q', (rows) => seen.push(rows));
     rt.update(R.markDirty(['q']));

package/src/client/sp00ky-client.test.ts

@@ -49,7 +49,14 @@ describe('Sp00kyClient facade', () => {
   });
 
   it('queryRaw registers locally and returns the hash before any remote work; subscribers paint from the local store', async () => {
-    const { client, a } = makeClient({ local: { query: (async (sql: string) => (sql.startsWith('SELECT * FROM thing') ? [[{ id: 'thing:1' }]] : [])) as any } });
+    // Boot loads the view index: `thing:1` is vouched for, `thing:2` is an orphan and stays hidden.
+    const stored = (sql: string) =>
+      sql.startsWith('SELECT * FROM thing')
+        ? [[{ id: 'thing:1' }, { id: 'thing:2' }]]
+        : sql === 'SELECT * FROM _00_view'
+          ? [[{ id: '_00_view:k', ids: [['thing:1', 1]] }]]
+          : [];
+    const { client, a } = makeClient({ local: { query: (async (sql: string) => stored(sql)) as any } });
     await client.init();
     const hash = await client.queryRaw('SELECT * FROM thing', {}, '10m');
     expect(client.state.queries.get(hash)!.lifecycle.phase).toBe('cold');

package/src/kernel/constants.ts

@@ -48,8 +48,21 @@ export const AUTH_READY_RETRY_MS = 500;
 export const AUTH_READY_MAX_ATTEMPTS = 10;
 /** Heartbeat at this fraction of the shortest ttl in state. */
 export const TTL_HEARTBEAT_FRACTION = 0.5;
-/** Orphan body garbage collection cadence. */
-export const GC_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000;
+/**
+ * Orphan body garbage collection: the first sweep waits this long after boot
+ * (off the first paint and the first sync round), then repeats on the interval.
+ */
+export const GC_BOOT_DELAY_MS = 30_000;
+export const GC_INTERVAL_MS = 60 * 60 * 1000;
+/** Ids deleted per GC step. */
+export const GC_CHUNK = 200;
+/**
+ * A `_00_view` row no query has resolved for this long stops vouching for its
+ * ids (the sweep deletes it, so its bodies can be collected). Each session
+ * that resolves a query rewrites its row, so this only retires query shapes
+ * the app no longer runs.
+ */
+export const VIEW_RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
 /** Remote adapter slot limit. */
 export const REMOTE_CONCURRENCY = 16;
 /** Rolling telemetry sample window per query. */

package/src/query/lifecycle.saga.test.ts

@@ -4,7 +4,7 @@ import { runPure } from '../testing/run-pure';
 import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
 import * as R from '../state/reducers';
 import { defaultEnv } from './env';
-import { ackPrune, evictQuery, gcTick, lifecycleTick } from './lifecycle.saga';
+import { ackPrune, evictQuery, gcTick, lifecycleTick, loadViews } from './lifecycle.saga';
 import type { StatementResult } from '../kernel/effects';
 
 const env = defaultEnv({ tables: [] } as any);
@@ -75,31 +75,158 @@ describe('ackPrune', () => {
   });
 });
 
+describe('loadViews', () => {
+  it('fills the index from every _00_view row (members and children); a failed read leaves it empty', async () => {
+    const rows = [
+      { id: '_00_view:a', ids: [['t:1', 1]], children: [['c:1', 1], 'junk'] },
+      { id: new RecordId('_00_view', 'b'), ids: [] },
+      { id: '', ids: [] },
+      { ids: [['t:no-key', 1]] },
+      null,
+    ];
+    const out = await runPure(loadViews(), { state: buildState(), handlers: { 'local.query': () => [rows] } });
+    expect(out.state.views).toEqual(new Map([['a', ['t:1', 'c:1']], ['b', []]]));
+    const odd = await runPure(loadViews(), { state: buildState(), handlers: { 'local.query': () => undefined } });
+    expect(odd.state.views.size).toBe(0);
+    const failed = await runPure(loadViews(), {
+      state: buildState(),
+      handlers: {
+        'local.query': () => {
+          throw new Error('no table');
+        },
+      },
+    });
+    expect(failed.state.views.size).toBe(0);
+    expect(failed.emitted).toEqual([expect.objectContaining({ level: 'warn', message: 'view index read failed' })]);
+  });
+});
+
 describe('gcTick', () => {
-  it('deletes bodies no view names and no outbox item touches; keeps _00_ rows; reschedules', async () => {
+  const day = 24 * 60 * 60 * 1000;
+  const now = 100 * day;
+  const ready = R.setIdentity({ primed: true, bucketId: 'u1' });
+  const deletedIds = (log: Array<{ kind: string }>) =>
+    log.filter((e: any) => e.kind === 'local.delete').map((e: any) => (e.table === '_00_view' ? `view:${e.id.id}` : e.id));
+
+  it('retires stale view rows, deletes the bodies nothing retains from the store and the circuit, re-fetches, reschedules', async () => {
     const s = buildState(
-      [],
-      R.setVersions([['thing:keep', 1], ['thing:gone', 1], ['thing:pending', 1], ['_00_view:x', 1], ['thing:failed', 1]]),
+      [buildEntry({ def: { hash: 'h', viewKey: 'held' }, lifecycle: { phase: 'cached' }, remoteArray: [['thing:mem', 1]] })],
+      ready,
+      R.setVersions(
+        ['thing:keep', 'child:1', 'thing:stale', 'thing:held', 'thing:stuck', 'thing:mem', 'thing:gone', 'thing:pending', '_00_view:x', 'thing:failed'].map(
+          (id) => [id, 1] as const
+        )
+      ),
       R.outboxReplace([buildOutboxItem({ recordId: 'thing:pending' })])
     );
-    const deleted: string[] = [];
+    const rows = [
+      { id: '_00_view:fresh', ids: [['thing:keep', 1]], children: [['child:1', 1]], updatedAt: now - day },
+      // Nobody resolved this query for 15 days: the row is retired and stops vouching.
+      { id: new RecordId('_00_view', 'old'), ids: [['thing:stale', 1]], updatedAt: now - 15 * day },
+      // As old, but a query in state holds it.
+      { id: '_00_view:held', ids: [['thing:held', 1]] },
+      // As old, but its delete fails: it keeps vouching.
+      { id: '_00_view:stuck', ids: [['thing:stuck', 1]], updatedAt: 0 },
+      { ids: 'bad' },
+    ];
     const out = await runPure(gcTick(), {
       state: s,
+      now,
       handlers: {
-        'local.query': () => [[{ ids: [['thing:keep', 1]] }, { ids: 'bad' }]],
+        'local.query': (e: any) => {
+          expect(e.sql).toBe('SELECT * FROM _00_view');
+          return [rows];
+        },
         'local.delete': (e: any) => {
-          deleted.push(e.id);
-          if (e.id === 'thing:failed') throw new Error('busy');
+          if (e.id === 'thing:failed' || e.id?.id === 'stuck') throw new Error('busy');
         },
         'ssp.ingest': () => undefined,
       },
     });
-    expect(deleted).toEqual(['thing:gone', 'thing:failed']);
-    expect([...out.state.versions.keys()]).toEqual(['thing:keep', 'thing:pending', '_00_view:x', 'thing:failed']);
+    expect(deletedIds(out.log)).toEqual(['view:old', 'view:stuck', 'thing:stale', 'thing:gone', 'thing:failed']);
+    expect(out.state.views).toEqual(
+      new Map([
+        ['fresh', ['thing:keep', 'child:1']],
+        ['held', ['thing:held']],
+        ['stuck', ['thing:stuck']],
+      ])
+    );
+    expect([...out.state.versions.keys()]).toEqual(['thing:keep', 'child:1', 'thing:held', 'thing:stuck', 'thing:mem', 'thing:pending', '_00_view:x', 'thing:failed']);
     const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
-    expect(ingest.records).toEqual([{ table: 'thing', op: 'DELETE', id: 'thing:gone', record: {} }]);
-    expect(out.timers.get('gc')!.event).toEqual({ type: 'GcTick' });
-    const failing = await runPure(gcTick(), {
+    expect(ingest.records).toEqual([
+      { table: 'thing', op: 'DELETE', id: 'thing:stale', record: {} },
+      { table: 'thing', op: 'DELETE', id: 'thing:gone', record: {} },
+    ]);
+    // Solo: nobody to relay to.
+    expect(out.emitted.filter((e) => e.type === 'tabs:broadcast')).toEqual([]);
+    expect(out.emitted).toContainEqual(expect.objectContaining({ message: 'orphan gc done', data: { removed: 2, retiredViews: 1 } }));
+    expect(out.dispatched).toEqual([{ type: 'FetchRows' }]);
+    expect(out.timers.get('gc')).toEqual({ ms: 60 * 60 * 1000, event: { type: 'GcTick' } });
+  });
+
+  it('waits for the circuit prime (the versions); a leader relays its deletes to the other tabs', async () => {
+    const s = buildState([], R.setIdentity({ bucketId: 'u1' }), R.setTabRole('leader'), R.setVersions([['thing:gone', 1]]));
+    const out = await runPure(gcTick(), {
+      state: s,
+      handlers: {
+        'state.wait': (e: any, ctx) => {
+          expect(e.until(ctx.state)).toBe(false);
+          ctx.state = R.setIdentity({ primed: true })(ctx.state);
+        },
+        'local.query': () => [[]],
+        'local.delete': () => undefined,
+        'ssp.ingest': () => undefined,
+      },
+    });
+    expect(out.emitted).toContainEqual({
+      type: 'tabs:broadcast',
+      message: { type: 'ingest', records: [{ table: 'thing', op: 'DELETE', id: 'thing:gone', record: {} }] },
+    });
+  });
+
+  it('a follower only refreshes its index: no view row or body is deleted', async () => {
+    const s = buildState([], ready, R.setTabRole('follower'), R.setVersions([['thing:gone', 1]]));
+    const out = await runPure(gcTick(), {
+      state: s,
+      now,
+      handlers: { 'local.query': () => [[{ id: '_00_view:old', ids: [['thing:x', 1]], updatedAt: 0 }]] },
+    });
+    expect(out.state.views).toEqual(new Map([['old', ['thing:x']]]));
+    expect(deletedIds(out.log)).toEqual([]);
+    expect(out.state.versions.has('thing:gone')).toBe(true);
+    expect(out.timers.has('gc')).toBe(true);
+  });
+
+  it('each chunk is re-checked against fresh state, and the sweep stops when the bucket moves', async () => {
+    const ids = Array.from({ length: 250 }, (_, i) => `thing:o${i}`);
+    const s = buildState([], ready, R.setVersions(ids.map((id) => [id, 1] as const)));
+    const handlers = (onFirst: (ctx: any) => void) => ({
+      'local.query': () => [[]],
+      'local.delete': (e: any, ctx: any) => {
+        if (e.id === 'thing:o0') onFirst(ctx);
+      },
+      'ssp.ingest': () => undefined,
+    });
+    // Meanwhile a query commits o210 and o220's body goes another way: neither is deleted.
+    const recheck = await runPure(gcTick(), {
+      state: s,
+      handlers: handlers((ctx) => {
+        ctx.state = R.compose(R.putQuery(buildEntry({ def: { hash: 'q' }, remoteArray: [['thing:o210', 1]] })), R.deleteVersions(['thing:o220']))(ctx.state);
+      }),
+    });
+    const removed = deletedIds(recheck.log);
+    expect(removed).toHaveLength(248);
+    expect(removed).not.toContain('thing:o210');
+    expect(removed).not.toContain('thing:o220');
+    expect(recheck.log.filter((e) => e.kind === 'ssp.ingest')).toHaveLength(2);
+    const moved = await runPure(gcTick(), { state: s, handlers: handlers((ctx) => (ctx.state = R.setIdentity({ bucketId: 'u2' })(ctx.state))) });
+    expect(deletedIds(moved.log)).toHaveLength(200);
+    expect(moved.timers.has('gc')).toBe(true);
+  });
+
+  it('failures are logged and the tick still reschedules; a sweep that removes nothing asks for nothing', async () => {
+    const s = buildState([], ready, R.setVersions([['thing:gone', 1]]));
+    const readFails = await runPure(gcTick(), {
       state: s,
       handlers: {
         'local.query': () => {
@@ -107,10 +234,35 @@ describe('gcTick', () => {
         },
       },
     });
-    expect(failing.emitted).toEqual([expect.objectContaining({ message: 'orphan gc failed' })]);
-    expect(failing.timers.has('gc')).toBe(true);
-    const nothing = await runPure(gcTick(), { state: buildState(), handlers: { 'local.query': () => [] } });
-    expect(nothing.log.filter((e) => e.kind === 'ssp.ingest')).toHaveLength(0);
+    expect(readFails.emitted).toEqual([expect.objectContaining({ message: 'orphan gc failed' })]);
+    expect(readFails.timers.has('gc')).toBe(true);
+    const ingestFails = await runPure(gcTick(), {
+      state: s,
+      handlers: {
+        'local.query': () => [[]],
+        'local.delete': () => undefined,
+        'ssp.ingest': () => {
+          throw new Error('wasm');
+        },
+      },
+    });
+    expect(ingestFails.emitted).toEqual([expect.objectContaining({ message: 'orphan gc failed' })]);
+    expect(ingestFails.timers.has('gc')).toBe(true);
+    const deleteFails = await runPure(gcTick(), {
+      state: s,
+      handlers: {
+        'local.query': () => [[]],
+        'local.delete': () => {
+          throw new Error('busy');
+        },
+      },
+    });
+    expect(deleteFails.log.filter((e) => e.kind === 'ssp.ingest')).toHaveLength(0);
+    expect(deleteFails.dispatched).toEqual([]);
+    expect(deleteFails.emitted).toContainEqual(expect.objectContaining({ message: 'orphan gc done', data: { removed: 0, retiredViews: 0 } }));
+    const nothing = await runPure(gcTick(), { state: buildState([], ready), handlers: { 'local.query': () => [] } });
+    expect(nothing.log.filter((e) => e.kind === 'local.delete')).toHaveLength(0);
+    expect(nothing.dispatched).toEqual([]);
   });
 });
 

package/src/query/lifecycle.saga.ts

@@ -1,11 +1,13 @@
 import type { Saga } from '../kernel/saga';
 import type { Settled, StatementResult } from '../kernel/effects';
 import { fx } from '../kernel/effects';
-import { ACK_GRACE_MS, GC_INTERVAL_MS, TTL_HEARTBEAT_FRACTION } from '../kernel/constants';
-import type { ClientState, QueryEntry } from '../state/client-state';
+import { ACK_GRACE_MS, GC_CHUNK, GC_INTERVAL_MS, TTL_HEARTBEAT_FRACTION, VIEW_RETENTION_MS } from '../kernel/constants';
+import type { IngestRecord } from '../services/stream-processor/index';
+import type { ClientState, QueryEntry, TabRole } from '../state/client-state';
 import * as R from '../state/reducers';
-import { evictable, shortestTtlMs } from '../state/selectors';
+import { evictable, retained, shortestTtlMs } from '../state/selectors';
 import type { SagaEnv } from './env';
+import { parseViewIndex, type ViewIndexRow } from './membership';
 import * as sql from './sql';
 
 /**
@@ -65,37 +67,93 @@ export function* ackPrune(): Saga<void> {
   if (stillAcked) yield fx.timer.set('ack-prune', ACK_GRACE_MS, { type: 'AckPrune' });
 }
 
+/** Every `_00_view` row, parsed. Throws when the read fails. */
+function* readViews(): Saga<ViewIndexRow[]> {
+  const res = (yield fx.local.query(sql.readViewRows())) as unknown[];
+  return parseViewIndex(Array.isArray(res) ? res[0] : undefined);
+}
+
+const indexOf = (rows: ViewIndexRow[]): Map<string, string[]> => new Map(rows.map((r) => [r.key, r.ids]));
+
+/**
+ * Load the durable view index (boot, bucket switch). A failed read leaves it
+ * empty, which fails closed: cold scans paint nothing until the server answers.
+ */
+export function* loadViews(): Saga<void> {
+  try {
+    yield fx.state.update(R.reloadViews(indexOf(yield* readViews())));
+  } catch (error) {
+    yield fx.emit({ type: 'log', level: 'warn', message: 'view index read failed', data: { error } });
+  }
+}
+
+const tableOf = (id: string): string => id.slice(0, id.indexOf(':'));
+
 /**
- * Weekly orphan collection: bodies the store holds that no `_00_view` row
- * names and no outbox item touches are invisible; delete them in chunks.
+ * Orphan collection, `GC_BOOT_DELAY_MS` after boot and then every
+ * `GC_INTERVAL_MS`. Re-reads every `_00_view` row into the index, retires the
+ * rows no query has resolved for `VIEW_RETENTION_MS`, then deletes the bodies
+ * nothing retains (see `retained`) from the store and the circuit, a chunk at
+ * a time. Only a leader or solo tab deletes, and a leader relays the deletes
+ * so followers drop the rows too; a follower only refreshes its index.
  */
 export function* gcTick(): Saga<void> {
+  yield fx.state.wait((s) => s.primed);
   try {
-    const views = (yield fx.local.query(`SELECT ids FROM ${sql.VIEW_TABLE}`)) as unknown[];
-    const rows = Array.isArray(views?.[0]) ? (views[0] as Array<{ ids?: unknown }>) : [];
-    const keep = new Set<string>();
-    for (const row of rows) {
-      if (!Array.isArray(row.ids)) continue;
-      for (const pair of row.ids as Array<[string, number]>) keep.add(pair[0]);
+    const rows = yield* readViews();
+    const [role, bucket, held] = (yield fx.state.read((s) => [
+      s.tabRole,
+      s.bucketId,
+      new Set([...s.queries.values()].map((e) => e.def.viewKey)),
+    ])) as [TabRole, string | null, Set<string>];
+    const now = (yield fx.now()) as number;
+    const expired = role === 'follower' ? [] : rows.filter((r) => !held.has(r.key) && now - r.updatedAt >= VIEW_RETENTION_MS);
+    const retired = new Set<string>();
+    for (let i = 0; i < expired.length; i += GC_CHUNK) {
+      const chunk = expired.slice(i, i + GC_CHUNK);
+      const settled = (yield fx.all(chunk.map((r) => fx.local.delete(sql.VIEW_TABLE, sql.viewRecordId(r.key))))) as Settled[];
+      chunk.forEach((r, j) => settled[j].ok && retired.add(r.key));
     }
-    const state = (yield fx.state.read((s) => s)) as ClientState;
-    for (const item of state.outbox) keep.add(item.recordId);
-    const orphans = [...state.versions.keys()].filter((id) => !keep.has(id) && !id.startsWith('_00_'));
-    const deletes = orphans.map((id) => {
-      const table = id.slice(0, id.indexOf(':'));
-      return fx.local.delete(table, id);
-    });
-    for (let i = 0; i < deletes.length; i += 200) {
-      const settled = (yield fx.all(deletes.slice(i, i + 200))) as Settled[];
-      const done = orphans.slice(i, i + 200).filter((_, j) => settled[j].ok);
-      yield fx.state.update(R.deleteVersions(done));
-      if (done.length > 0) {
-        yield fx.ssp.ingest(done.map((id) => ({ table: id.slice(0, id.indexOf(':')), op: 'DELETE', id, record: {} })));
-      }
-    }
-    yield fx.emit({ type: 'log', level: 'info', message: 'orphan gc done', data: { removed: orphans.length } });
+    yield fx.state.update(R.reloadViews(indexOf(rows.filter((r) => !retired.has(r.key)))));
+    if (role === 'follower') return;
+    const removed = yield* collectOrphans(bucket, role === 'leader');
+    yield fx.emit({ type: 'log', level: 'info', message: 'orphan gc done', data: { removed, retiredViews: retired.size } });
   } catch (error) {
     yield fx.emit({ type: 'log', level: 'warn', message: 'orphan gc failed', data: { error } });
+  } finally {
+    yield fx.timer.set('gc', GC_INTERVAL_MS, { type: 'GcTick' });
+  }
+}
+
+/**
+ * Delete every body nothing retains, `GC_CHUNK` at a time. Each chunk is
+ * re-checked against fresh state (a membership committed meanwhile may name
+ * an id again) and the sweep stops if the bucket moved under it.
+ */
+function* collectOrphans(bucket: string | null, relay: boolean): Saga<number> {
+  const candidates = (yield fx.state.read((s) => {
+    const keep = retained(s);
+    return [...s.versions.keys()].filter((id) => !keep(id));
+  })) as string[];
+  let removed = 0;
+  for (let i = 0; i < candidates.length; i += GC_CHUNK) {
+    const chunk = (yield fx.state.read((s) => {
+      if (!s.primed || s.bucketId !== bucket) return null;
+      const keep = retained(s);
+      return candidates.slice(i, i + GC_CHUNK).filter((id) => s.versions.has(id) && !keep(id));
+    })) as string[] | null;
+    if (chunk === null) break;
+    const settled = (yield fx.all(chunk.map((id) => fx.local.delete(tableOf(id), id)))) as Settled[];
+    const done = chunk.filter((_, j) => settled[j].ok);
+    if (done.length === 0) continue;
+    const records: IngestRecord[] = done.map((id) => ({ table: tableOf(id), op: 'DELETE', id, record: {} }));
+    yield fx.state.update(R.deleteVersions(done));
+    yield fx.ssp.ingest(records);
+    if (relay) yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records } });
+    removed += done.length;
   }
-  yield fx.timer.set('gc', GC_INTERVAL_MS, { type: 'GcTick' });
+  // A query may have committed a deleted id between its chunk's check and the
+  // delete; with its version gone, the fetch plan pulls it back.
+  if (removed > 0) yield fx.dispatch({ type: 'FetchRows' });
+  return removed;
 }

package/src/query/materialize.saga.test.ts

@@ -1,4 +1,5 @@
 import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
 import { runPure } from '../testing/run-pure';
 import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
 import * as R from '../state/reducers';
@@ -10,8 +11,9 @@ describe('materialize', () => {
     const out = await runPure(materialize('zz'), { state: buildState() });
     expect(out.log.filter((e) => e.kind !== 'state.read')).toHaveLength(0);
   });
+  const vouch = (...ids: string[]) => R.reloadViews(new Map([['other-view', ids]]));
   it('cold: predicate scan through the plan; rows land, dirt clears, subscribers hear once', async () => {
-    const s = R.markDirty(['a'])(buildState([buildEntry({ def: { hash: 'a', plan } })]));
+    const s = R.markDirty(['a'])(buildState([buildEntry({ def: { hash: 'a', plan } })], vouch('thing:1')));
     const out = await runPure(materialize('a'), { state: s, handlers: { 'local.select': (e: any) => (e.plan.ids ? [] : [{ id: 'thing:1' }]) } });
     const en = out.state.queries.get('a')!;
     expect(en.records).toEqual([{ id: 'thing:1' }]);
@@ -24,8 +26,39 @@ describe('materialize', () => {
     expect(again.emitted).toEqual([]);
     expect(again.state.queries.get('a')!.telemetry.updateCount).toBe(1);
   });
-  it('cold window renders the SSP local window; live renders membership with the overlay', async () => {
-    const win = buildState([buildEntry({ def: { hash: 'w', plan: { ...plan, offset: 5 }, surql: 'SELECT * FROM thing LIMIT 5 START 5' }, localArray: [['thing:7', 1]] })]);
+  it('cold: a body no view vouches for (an orphan) does not paint; vouched, optimistic, _00_ and id-less rows do', async () => {
+    const s = buildState(
+      [buildEntry({ def: { hash: 'a', plan } })],
+      vouch('thing:1', 'thing:4'),
+      R.outboxReplace([buildOutboxItem({ id: 'c', type: 'create', recordId: 'thing:3' })])
+    );
+    const scanned = [
+      { id: 'thing:1' },
+      { id: 'thing:2' },
+      { id: 'thing:3' },
+      { id: '_00_user_feature:f' },
+      { name: 'no id' },
+      { id: new RecordId('thing', '4') },
+      { id: new RecordId('thing', '5') },
+    ];
+    const out = await runPure(materialize('a'), { state: s, handlers: { 'local.select': () => scanned } });
+    expect(out.state.queries.get('a')!.records).toEqual([
+      { id: 'thing:1' },
+      { id: 'thing:3' },
+      { id: '_00_user_feature:f' },
+      { name: 'no id' },
+      { id: new RecordId('thing', '4') },
+    ]);
+    // Once the server has answered, membership decides and the index is not consulted.
+    const live = buildState([buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1]] })]);
+    const l = await runPure(materialize('a'), { state: live, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
+    expect(l.state.queries.get('a')!.records).toEqual([{ id: 'thing:2' }]);
+  });
+  it('cold window renders the SSP local window (orphans dropped); live renders membership with the overlay', async () => {
+    const win = buildState(
+      [buildEntry({ def: { hash: 'w', plan: { ...plan, offset: 5 }, surql: 'SELECT * FROM thing LIMIT 5 START 5' }, localArray: [['thing:7', 1], ['thing:8', 1]] })],
+      vouch('thing:7')
+    );
     const w = await runPure(materialize('w'), { state: win, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
     expect(w.state.queries.get('w')!.records).toEqual([{ id: 'thing:7' }]);
     const live = buildState(

package/src/query/materialize.saga.ts

@@ -4,26 +4,34 @@ import type { Saga } from '../kernel/saga';
 import { fx } from '../kernel/effects';
 import type { QueryEntry } from '../state/client-state';
 import * as R from '../state/reducers';
-import { overlay } from '../state/selectors';
+import { overlay, vouched } from '../state/selectors';
 import type { Overlay } from '../state/selectors';
-import { isWindowed, materializeEffect, rowsEqual, rowsFromResult } from './materialize';
-import { buildRenderIds, resolveMembership } from './render-set';
+import { isWindowed, materializeEffect, rowKey, rowsEqual, rowsFromResult } from './materialize';
+import { buildRenderIds, paintsCold, resolveMembership } from './render-set';
+
+const NONE: ReadonlySet<string> = new Set();
 
 /**
  * Render one query from state: membership (or a cold scan), the outbox
  * overlay, one local read, then notify subscribers if the rows changed. The
  * only writer of `records`. Runs on the `mat:<hash>` dedupe lane whenever the
- * query is dirty.
+ * query is dirty. Before the server has answered (`cold`) only bodies some
+ * durable view vouches for paint (see `paintsCold`).
  */
 export function* materialize(hash: QueryHash): Saga<void> {
-  const [entry, ov] = (yield fx.state.read((s) => [s.queries.get(hash), overlay(s)])) as [QueryEntry | undefined, Overlay];
+  const [entry, ov, known] = (yield fx.state.read((s) => {
+    const e = s.queries.get(hash);
+    return [e, overlay(s), e?.lifecycle.phase === 'cold' ? vouched(s) : NONE];
+  })) as [QueryEntry | undefined, Overlay, ReadonlySet<string>];
   if (!entry) return;
   const t0 = (yield fx.now()) as number;
   const isWindow = isWindowed(entry.def.surql);
+  const paints = (id: string | null) => id === null || paintsCold(id, known, ov);
+  const coldWindow = entry.lifecycle.phase === 'cold' && isWindow;
   const membership = resolveMembership({
     phase: entry.lifecycle.phase,
     remoteArray: entry.remoteArray,
-    localArray: entry.localArray,
+    localArray: coldWindow ? entry.localArray.filter(([id]) => paints(id)) : entry.localArray,
     isWindow,
   });
   const ids = membership
@@ -36,6 +44,7 @@ export function* materialize(hash: QueryHash): Saga<void> {
   let rows: Record<string, unknown>[];
   try {
     rows = rowsFromResult(effect, yield effect);
+    if (ids === null) rows = rows.filter((row) => paints(rowKey(row)));
   } catch (error) {
     yield fx.state.update(R.compose(R.recordError(hash), R.clearDirty(hash)));
     yield fx.emit({ type: 'log', level: 'warn', message: 'materialize failed', data: { hash, error } });

package/src/query/materialize.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest';
 import { RecordId } from 'surrealdb';
-import { isWindowed, materializeEffect, rowsEqual, rowsFromResult } from './materialize';
+import { isWindowed, materializeEffect, rowKey, rowsEqual, rowsFromResult } from './materialize';
 
 describe('materializeEffect', () => {
   const plan = { table: 'thing', where: [{ field: 'a', op: '=', value: 1 }] } as any;
@@ -48,3 +48,11 @@ describe('materializeEffect', () => {
     expect(isWindowed('SELECT * FROM t LIMIT 10')).toBe(false);
   });
 });
+
+describe('rowKey', () => {
+  it('reads a string id (SQLite) or a RecordId (SurrealDB); null when the row carries none', () => {
+    expect(rowKey({ id: 'thing:1' })).toBe('thing:1');
+    expect(rowKey({ id: new RecordId('thing', '2') })).toBe('thing:2');
+    for (const row of [{ name: 'x' }, { id: null }, { id: { other: 1 } }, null, 5]) expect(rowKey(row)).toBeNull();
+  });
+});

package/src/query/materialize.ts

@@ -1,7 +1,8 @@
 import type { QueryPlan } from '@spooky-sync/query-builder';
+import type { RecordId } from 'surrealdb';
 import type { Effect } from '../kernel/effects';
 import { fx } from '../kernel/effects';
-import { parseRecordIdString } from '../utils/index';
+import { encodeRecordId, parseRecordIdString } from '../utils/index';
 import { buildIdSetPlan, buildIdSetSurql, buildWindowMaterialization } from './window-query';
 
 export interface MaterializeSource {
@@ -36,6 +37,18 @@ export function rowsFromResult(effect: Effect, result: unknown): Record<string,
   return Array.isArray(first) ? (first as Record<string, unknown>[]) : [];
 }
 
+/**
+ * A row's encoded record id: a string on the SQLite engine, a `RecordId` on
+ * the SurrealDB one. `null` for a row that carries none (a projection
+ * without `id`, a `SELECT VALUE`).
+ */
+export function rowKey(row: unknown): string | null {
+  const id = row !== null && typeof row === 'object' ? (row as { id?: unknown }).id : undefined;
+  if (typeof id === 'string') return id;
+  if (id !== null && typeof id === 'object' && 'table' in id && 'id' in id) return encodeRecordId(id as RecordId<string>);
+  return null;
+}
+
 /** Cheap structural equality for the "did the rows change" check. */
 export function rowsEqual(a: ReadonlyArray<unknown>, b: ReadonlyArray<unknown>): boolean {
   if (a === b) return true;