@spooky-sync/core 0.0.1-canary.33 → 0.0.1-canary.331

package/src/query/membership.saga.ts

@@ -0,0 +1,282 @@
+import type { QueryHash, RecordVersionArray, ServerViewMeta } from '../types';
+import type { Saga } from '../kernel/saga';
+import type { Settled, StatementResult } from '../kernel/effects';
+import { fx } from '../kernel/effects';
+import {
+  MATERIALIZING_REREAD_LADDER_MS,
+  MEMBERSHIP_COALESCE_MS,
+  VIEW_LOST_RETRY_BASE_MS,
+  VIEW_LOST_RETRY_MAX_MS,
+  backoffMs,
+} from '../kernel/constants';
+import type { ClientState, QueryEntry } from '../state/client-state';
+import { isAuthoritative } from '../state/lifecycle';
+import * as R from '../state/reducers';
+import { encodeRecordId } from '../utils/index';
+import { departedIds } from '../state/selectors';
+import { forgetDeparted } from './lifecycle.saga';
+import { planListRefPollChunks, recordVersionArraysEqual } from '../sync/policy';
+import type { SagaEnv } from './env';
+import { listRefTable } from './env';
+import * as sql from './sql';
+import {
+  FAILED,
+  decideMembershipOutcome,
+  snapshotFromSingle,
+  snapshotsFromBatch,
+  suspectHashes,
+  type ListRefSnapshot,
+} from './membership';
+
+type MembershipOutcome = 'applied' | 'ignored' | 'view-lost';
+
+/**
+ * Accept (or refuse) a server id-set for one query. The only writer of
+ * `remoteArray`, the `_00_view` row and the cold -> live transition.
+ */
+export function* applyMembership(
+  hash: QueryHash,
+  remoteArray: RecordVersionArray,
+  meta?: ServerViewMeta,
+  verifiedRemoval?: boolean
+): Saga<MembershipOutcome> {
+  const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+  if (!entry) return 'ignored';
+  if (meta) yield fx.state.update(R.setServerState(hash, meta.present ? meta.state : null));
+  // The row is there: whatever made it read as gone has passed.
+  if (meta?.present && (entry.viewLostCount > 0 || entry.viewLostRetryAt !== null)) {
+    yield fx.state.update(R.setViewLost(hash, 0, null));
+  }
+  const outcome = decideMembershipOutcome({
+    phase: entry.lifecycle.phase,
+    held: entry.remoteArray.length,
+    remoteArray,
+    meta,
+    verifiedRemoval,
+  });
+  if (outcome === 'ignored') return outcome;
+  if (outcome === 'view-lost') {
+    if (entry.lifecycle.phase !== 'view-lost') {
+      yield fx.state.update(R.applyLifecycle(hash, { type: 'row-missing' }));
+      yield fx.emit({ type: 'query:view-lost', hash });
+    }
+    // A recovery is already scheduled: every read in the meantime would
+    // otherwise re-register again.
+    if (entry.viewLostRetryAt !== null) return outcome;
+    if (entry.viewLostCount === 0) {
+      yield fx.state.update(R.setViewLost(hash, 1, null));
+      if (entry.lifecycle.remote !== 'unregistered') {
+        yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-dropped' }));
+      }
+      yield fx.dispatch({ type: 'EnsureRegistered' });
+      return outcome;
+    }
+    // Re-registered already and the row still reads as gone. Under load that
+    // repeated every read, each one costing the server a registration while it
+    // was still publishing the last, so from here on it is paced.
+    const delay = backoffMs(entry.viewLostCount - 1, VIEW_LOST_RETRY_BASE_MS, VIEW_LOST_RETRY_MAX_MS);
+    const now = (yield fx.now()) as number;
+    yield fx.state.update(R.setViewLost(hash, entry.viewLostCount + 1, now + delay));
+    yield fx.emit({
+      type: 'log',
+      level: 'debug',
+      message: 'view still lost; re-registering after a delay',
+      data: { hash, attempt: entry.viewLostCount + 1, delayMs: delay },
+    });
+    yield fx.timer.set(`view-lost:${hash}`, delay, { type: 'RecoverLostView', hash });
+    return outcome;
+  }
+  const wasAuthoritative = isAuthoritative(entry.lifecycle);
+  yield fx.state.update(R.commitMembership(hash, remoteArray, meta?.present ?? true));
+  yield* forgetDeparted(hash, departedIds(entry.remoteArray, remoteArray));
+  if (!wasAuthoritative) yield fx.emit({ type: 'query:authority', hash, known: true });
+  const now = (yield fx.now()) as number;
+  try {
+    yield fx.local.upsert(
+      sql.VIEW_TABLE,
+      sql.viewRecordId(entry.def.viewKey),
+      sql.viewRow(remoteArray, entry.subqueryRemoteArray, true, now),
+      'replace'
+    );
+  } catch (err) {
+    yield fx.emit({ type: 'log', level: 'debug', message: 'view row write failed', data: { hash, err } });
+  }
+  yield fx.dispatch({ type: 'FetchRows' });
+  return outcome;
+}
+
+/**
+ * The paced view-lost re-registration. A no-op when the row has been seen
+ * since (the pacing was reset) or the query recovered or went away.
+ */
+export function* recoverLostView(hash: QueryHash): Saga<void> {
+  const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+  if (!entry || entry.viewLostRetryAt === null) return;
+  yield fx.state.update(R.setViewLost(hash, entry.viewLostCount, null));
+  if (entry.lifecycle.phase !== 'view-lost') return;
+  if (entry.lifecycle.remote !== 'unregistered') {
+    yield fx.state.update(R.applyLifecycle(hash, { type: 'remote-dropped' }));
+  }
+  yield fx.dispatch({ type: 'RegisterRemote', hash });
+}
+
+/**
+ * Replace the subquery child set; bodies follow through the fetch plan. A
+ * query with a `_00_view` row (past `cold`) records the children there too,
+ * so the GC keeps their bodies and a cold scan may paint them.
+ */
+export function* applySubqueryChildren(hash: QueryHash, children: RecordVersionArray): Saga<void> {
+  const entry = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+  if (!entry || recordVersionArraysEqual(entry.subqueryRemoteArray, children)) return;
+  yield fx.state.update(R.setSubqueryRemoteArray(hash, children));
+  yield* forgetDeparted(hash, departedIds(entry.subqueryRemoteArray, children));
+  if (entry.lifecycle.phase !== 'cold') {
+    try {
+      yield fx.local.upsert(sql.VIEW_TABLE, sql.viewRecordId(entry.def.viewKey), { children }, 'merge');
+    } catch (err) {
+      yield fx.emit({ type: 'log', level: 'debug', message: 'view row write failed', data: { hash, err } });
+    }
+  }
+  yield fx.dispatch({ type: 'FetchRows' });
+}
+
+const ok = (r: Settled): r is { ok: true; value: unknown } => r.ok;
+/** One statement's result, or `FAILED` when it did not answer (see `FAILED`). */
+const stmt = (results: unknown, i: number): unknown => {
+  if (!Array.isArray(results)) return FAILED;
+  const r = (results as StatementResult[])[i];
+  return r && r.status === 'OK' ? r.result : FAILED;
+};
+
+/**
+ * Read the server membership of many queries in as few round trips as the
+ * row budget allows, then apply each changed set. This is the ONLY place a
+ * server id-set enters state: registration, the poll, LIVE dirt and view-lost
+ * recovery all come through here.
+ */
+export function* readMembership(
+  env: SagaEnv,
+  hashes: QueryHash[],
+  opts: { force?: boolean } = {}
+): Saga<{ changed: boolean; failed: boolean }> {
+  const state = (yield fx.state.read((s) => s)) as ClientState;
+  const entries = hashes.map((h) => state.queries.get(h)).filter((e): e is QueryEntry => !!e);
+  if (entries.length === 0) return { changed: false, failed: false };
+  const table = listRefTable(env, state);
+  const now = (yield fx.now()) as number;
+  const chunks = planListRefPollChunks(
+    entries.map((e) => ({ hash: e.def.hash, rows: e.remoteArray.length, lastPolledAt: opts.force ? 0 : (e.lastPolledAt ?? 0) })),
+    { now }
+  );
+  const byHash = new Map(entries.map((e) => [e.def.hash, e]));
+  const single = (e: QueryEntry) =>
+    fx.remote.query(sql.singleSnapshotSelect(table), { in: e.def.id }, env.remoteTimeoutMs);
+  const requests = chunks.map((chunk) =>
+    chunk.length === 1
+      ? single(byHash.get(chunk[0])!)
+      : fx.remote.query(sql.batchSnapshotSelect(table), { ins: chunk.map((h) => byHash.get(h)!.def.id) }, env.remoteTimeoutMs)
+  );
+  const results = (yield fx.all(requests)) as Settled[];
+
+  const snapshots = new Map<QueryHash, ListRefSnapshot>();
+  let failed = false;
+  const rereads: QueryEntry[] = [];
+  chunks.forEach((chunk, i) => {
+    const r = results[i];
+    if (!ok(r)) {
+      failed = true;
+      return;
+    }
+    if (chunk.length === 1) {
+      const snap = snapshotFromSingle(stmt(r.value, 0) as never, stmt(r.value, 1) as never, stmt(r.value, 2) as never);
+      if (snap) snapshots.set(chunk[0], snap);
+      else failed = true;
+      return;
+    }
+    const hashById = new Map(chunk.map((h) => [encodeRecordId(byHash.get(h)!.def.id), h]));
+    const edges = stmt(r.value, 0) as never[] | null;
+    const counts = stmt(r.value, 1);
+    if (!Array.isArray(edges) || counts === FAILED) {
+      failed = true;
+      return;
+    }
+    const batch = snapshotsFromBatch(edges, counts as never, hashById);
+    const held = new Map(chunk.map((h) => [h, byHash.get(h)!.remoteArray.length]));
+    const suspect = new Set(suspectHashes(batch, held, edges.length));
+    for (const [h, snap] of batch) {
+      if (suspect.has(h)) rereads.push(byHash.get(h)!);
+      else snapshots.set(h, snap);
+    }
+  });
+  if (rereads.length > 0) {
+    const again = (yield fx.all(rereads.map(single))) as Settled[];
+    again.forEach((r, i) => {
+      if (!ok(r)) {
+        failed = true;
+        return;
+      }
+      const snap = snapshotFromSingle(stmt(r.value, 0) as never, stmt(r.value, 1) as never, stmt(r.value, 2) as never);
+      if (snap) snapshots.set(rereads[i].def.hash, snap);
+      else failed = true;
+    });
+  }
+
+  let changed = false;
+  const stillMaterializing: QueryHash[] = [];
+  for (const [hash, snap] of snapshots) {
+    const current = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
+    if (!current) continue;
+    const metaChanged = current.serverState !== (snap.meta.present ? snap.meta.state : null);
+    if (!recordVersionArraysEqual(snap.primary, current.remoteArray) || metaChanged || current.lifecycle.phase === 'cold' || current.lifecycle.phase === 'cached') {
+      const outcome = yield* applyMembership(hash, snap.primary, snap.meta);
+      if (outcome === 'applied') changed = true;
+      if (outcome === 'ignored' && snap.meta.present && snap.meta.state === 'materializing') stillMaterializing.push(hash);
+    }
+    yield* applySubqueryChildren(hash, snap.subquery);
+  }
+  yield fx.state.update(R.compose(R.clearMembershipDirty([...snapshots.keys()]), R.stampPolled(snapshots.keys(), now)));
+
+  // A view whose edges are still in flight: re-read on a short ladder rather
+  // than waiting for a backed-off poll tick.
+  for (const hash of stillMaterializing) {
+    const attempt = (yield fx.state.read((s) => s.membershipReread.get(hash) ?? 0)) as number;
+    if (attempt >= MATERIALIZING_REREAD_LADDER_MS.length) {
+      yield fx.state.update(R.setMembershipReread(hash, null));
+      continue;
+    }
+    yield fx.state.update(R.compose(R.setMembershipReread(hash, attempt + 1), R.markMembershipDirty([hash])));
+    yield fx.timer.set('membership', MATERIALIZING_REREAD_LADDER_MS[attempt], { type: 'ReadDirtyMembership' });
+  }
+  for (const hash of snapshots.keys()) {
+    if (!stillMaterializing.includes(hash)) yield fx.state.update(R.setMembershipReread(hash, null));
+  }
+  yield fx.dispatch({ type: 'SyncOutcome', ok: !failed, error: failed ? 'membership read failed' : undefined });
+  return { changed, failed };
+}
+
+/**
+ * LIVE handler: mark dirty and coalesce into one batched re-read.
+ *
+ * The window is armed ONCE per burst, when the dirty set goes from empty to
+ * non-empty. `timer.set` replaces a pending timer, so re-arming on every event
+ * would push the read out for as long as events keep arriving: on a busy
+ * `_00_list_ref` table (registrations, an import, another tab syncing) the
+ * events never stop for a whole 50 ms and the read simply never ran, leaving
+ * membership frozen until a poll tick happened to cover it.
+ */
+export function* markMembershipDirty(hashes: QueryHash[]): Saga<void> {
+  const armed = (yield fx.state.read((s) => s.membershipDirty.size > 0)) as boolean;
+  yield fx.state.update(R.markMembershipDirty(hashes));
+  if (!armed) yield fx.timer.set('membership', MEMBERSHIP_COALESCE_MS, { type: 'ReadDirtyMembership' });
+}
+
+export function* readDirtyMembership(env: SagaEnv): Saga<void> {
+  const dirty = (yield fx.state.read((s) => [...s.membershipDirty])) as QueryHash[];
+  if (dirty.length === 0) return;
+  yield* readMembership(env, dirty, { force: true });
+  // Dirt that arrived while the read was in flight, or a hash the read could
+  // not answer, holds the set non-empty - and with the window armed only on the
+  // empty -> non-empty edge, nothing else would arm it again.
+  const left = (yield fx.state.read((s) => s.membershipDirty.size)) as number;
+  if (left > 0) yield fx.timer.set('membership', MEMBERSHIP_COALESCE_MS, { type: 'ReadDirtyMembership' });
+}

package/src/query/membership.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import {
+  decideMembershipOutcome,
+  dedupeRecordVersions,
+  isResolvedBefore,
+  metaFromRow,
+  parseViewRow,
+  snapshotFromSingle,
+  snapshotsFromBatch,
+  suspectHashes,
+} from './membership';
+
+describe('durable view row', () => {
+  it('parses and gates resolved-before', () => {
+    expect(parseViewRow(null)).toBeNull();
+    expect(parseViewRow({ ids: 'nope' })).toBeNull();
+    expect(parseViewRow({ ids: [['t:1', 1]], confirmed: true })).toEqual({ ids: [['t:1', 1]], confirmed: true });
+    expect(parseViewRow({ ids: [] })).toEqual({ ids: [], confirmed: false });
+    expect(isResolvedBefore(null)).toBe(false);
+    expect(isResolvedBefore({ ids: [], confirmed: false })).toBe(false);
+    expect(isResolvedBefore({ ids: [], confirmed: true })).toBe(true);
+    expect(isResolvedBefore({ ids: [['t:1', 1]], confirmed: false })).toBe(true);
+  });
+});
+
+describe('metaFromRow / dedupe', () => {
+  it('folds the row-count select', () => {
+    expect(metaFromRow(null)).toEqual({ present: false, rowCount: null, state: null });
+    expect(metaFromRow({ rowCount: 3, state: 'ready' })).toEqual({ present: true, rowCount: 3, state: 'ready' });
+    expect(metaFromRow({ rowCount: null, state: 'weird' })).toEqual({ present: true, rowCount: null, state: null });
+    expect(metaFromRow({ state: 'materializing' })).toEqual({ present: true, rowCount: null, state: 'materializing' });
+  });
+  it('keeps the highest version per id', () => {
+    const single: [string, number][] = [['t:1', 1]];
+    expect(dedupeRecordVersions(single)).toBe(single);
+    const clean: [string, number][] = [['t:1', 1], ['t:2', 1]];
+    expect(dedupeRecordVersions(clean)).toBe(clean);
+    expect(dedupeRecordVersions([['t:1', 1], ['t:1', 3], ['t:1', 2]])).toEqual([['t:1', 3]]);
+  });
+});
+
+describe('decideMembershipOutcome', () => {
+  const ready0 = { present: true, rowCount: 0, state: 'ready' as const };
+  it('non-empty or verified removal is always applied', () => {
+    expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [['t:1', 1]] })).toBe('applied');
+    expect(decideMembershipOutcome({ phase: 'live', held: 2, remoteArray: [], verifiedRemoval: true })).toBe('applied');
+  });
+  it('empty with no readable row: view-lost when holding, ignored when cold and empty', () => {
+    expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [] })).toBe('ignored');
+    expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: { present: false, rowCount: null, state: null } })).toBe('ignored');
+    expect(decideMembershipOutcome({ phase: 'cold', held: 2, remoteArray: [] })).toBe('view-lost');
+    expect(decideMembershipOutcome({ phase: 'cached', held: 0, remoteArray: [] })).toBe('view-lost');
+    expect(decideMembershipOutcome({ phase: 'live', held: 1, remoteArray: [], meta: { present: false, rowCount: null, state: null } })).toBe('view-lost');
+  });
+  it('empty with a present row: applied only when ready and zero rows', () => {
+    expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: ready0 })).toBe('applied');
+    expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: { present: true, rowCount: 0, state: null } })).toBe('applied');
+    expect(decideMembershipOutcome({ phase: 'cold', held: 0, remoteArray: [], meta: { present: true, rowCount: 0, state: 'materializing' } })).toBe('ignored');
+    expect(decideMembershipOutcome({ phase: 'live', held: 3, remoteArray: [], meta: { present: true, rowCount: 3, state: 'ready' } })).toBe('ignored');
+    expect(decideMembershipOutcome({ phase: 'live', held: 3, remoteArray: [], meta: { present: true, rowCount: null, state: 'ready' } })).toBe('ignored');
+  });
+});
+
+describe('snapshots', () => {
+  const q1 = new RecordId('_00_query', 'a');
+  const q2 = new RecordId('_00_query', 'b');
+  const out = (id: string) => new RecordId('thing', id);
+  it('single: edges, meta, children; null when the primary select is not an array', () => {
+    expect(snapshotFromSingle(null, null, null)).toBeNull();
+    const snap = snapshotFromSingle(
+      [{ out: out('1'), version: 1 }, { out: out('1'), version: 2 }],
+      { rowCount: 1, state: 'ready' },
+      [{ out: out('c'), version: 1 }]
+    );
+    expect(snap).toEqual({
+      primary: [['thing:1', 2]],
+      subquery: [['thing:c', 1]],
+      meta: { present: true, rowCount: 1, state: 'ready' },
+    });
+    expect(snapshotFromSingle([], null, null)!.subquery).toEqual([]);
+  });
+  it('batch: splits by hash and parent, fills meta, marks missing rows absent', () => {
+    const hashById = new Map([['_00_query:a', 'ha'], ['_00_query:b', 'hb']]);
+    expect(snapshotsFromBatch(null, null, hashById).size).toBe(0);
+    const snaps = snapshotsFromBatch(
+      [
+        { in: q1, out: out('1'), version: 1 },
+        { in: q1, out: out('1'), version: 1 },
+        { in: q1, out: out('c'), version: 1, parent: q1 },
+        { in: new RecordId('_00_query', 'zz'), out: out('9'), version: 1 },
+      ],
+      [{ id: q1, rowCount: 1, state: 'ready' }, null, { rowCount: 5 }, { id: new RecordId('_00_query', 'zz') }],
+      hashById
+    );
+    expect(snaps.get('ha')).toEqual({
+      primary: [['thing:1', 1]],
+      subquery: [['thing:c', 1]],
+      meta: { present: true, rowCount: 1, state: 'ready' },
+    });
+    expect(snaps.get('hb')).toEqual({ primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } });
+    expect(snapshotsFromBatch([], null, hashById).get('ha')!.meta.present).toBe(false);
+  });
+  it('suspectHashes flags held queries with a missing row or a silently empty batch', () => {
+    const snaps = new Map([
+      ['missing', { primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } }],
+      ['empty', { primary: [], subquery: [], meta: { present: true, rowCount: 4, state: 'ready' as const } }],
+      ['fine', { primary: [], subquery: [], meta: { present: true, rowCount: 0, state: 'ready' as const } }],
+      ['unheld', { primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } }],
+    ]);
+    const held = new Map([['missing', 2], ['empty', 4], ['fine', 1], ['unheld', 0]]);
+    expect(suspectHashes(snaps, held, 0)).toEqual(['missing', 'empty']);
+    expect(suspectHashes(snaps, held, 1)).toEqual(['missing']);
+    const nullCount = new Map([['q', { primary: [], subquery: [], meta: { present: true, rowCount: null, state: 'ready' as const } }]]);
+    expect(suspectHashes(nullCount, new Map([['q', 3]]), 0)).toEqual([]);
+    expect(suspectHashes(nullCount, new Map(), 0)).toEqual([]);
+  });
+});

package/src/query/membership.ts

@@ -0,0 +1,201 @@
+import type { RecordId } from 'surrealdb';
+import type { MembershipOutcome, RecordVersionArray, ServerViewMeta } from '../types';
+import type { QueryPhase } from '../state/lifecycle';
+import { encodeRecordId } from '../utils/index';
+
+/** The durable `_00_view` row: the last server-confirmed membership. */
+export interface DurableView {
+  ids: RecordVersionArray;
+  confirmed: boolean;
+}
+
+export function parseViewRow(row: unknown): DurableView | null {
+  if (!row || typeof row !== 'object') return null;
+  const ids = (row as { ids?: unknown }).ids;
+  if (!Array.isArray(ids)) return null;
+  return { ids: ids as RecordVersionArray, confirmed: (row as { confirmed?: unknown }).confirmed === true };
+}
+
+/** One `_00_view` row as the view index and the GC see it. */
+export interface ViewIndexRow {
+  key: string;
+  /** Members and subquery children: what the row vouches for. */
+  ids: string[];
+  /** Last time a session resolved the query (0 when the row predates the field). */
+  updatedAt: number;
+}
+
+const pairIds = (pairs: unknown): string[] =>
+  Array.isArray(pairs) ? pairs.filter((p): p is [string, number] => Array.isArray(p) && typeof p[0] === 'string').map((p) => p[0]) : [];
+
+/** `SELECT * FROM _00_view` rows, keyed by view key; junk rows are skipped. */
+export function parseViewIndex(rows: unknown): ViewIndexRow[] {
+  if (!Array.isArray(rows)) return [];
+  const out: ViewIndexRow[] = [];
+  for (const row of rows as Array<Record<string, unknown> | null>) {
+    if (!row || typeof row !== 'object' || !Array.isArray(row.ids)) continue;
+    const id = row.id;
+    const key = typeof id === 'string' ? id.slice(id.indexOf(':') + 1) : String((id as { id?: unknown } | undefined)?.id ?? '');
+    if (!key) continue;
+    out.push({
+      key,
+      ids: [...new Set([...pairIds(row.ids), ...pairIds(row.children)])],
+      updatedAt: typeof row.updatedAt === 'number' ? row.updatedAt : 0,
+    });
+  }
+  return out;
+}
+
+/**
+ * "Resolved before": the server answered this query on this device at some
+ * point. An unconfirmed empty row cannot be told apart from one written before
+ * the marker existed, so it does not count.
+ */
+export const isResolvedBefore = (view: DurableView | null): boolean =>
+  view !== null && (view.ids.length > 0 || view.confirmed);
+
+/** The `{ rowCount, state }` object the row-count selects return per row. */
+export interface QueryMetaRow {
+  id?: RecordId<string>;
+  rowCount?: number | null;
+  state?: string | null;
+}
+
+export function metaFromRow(row: QueryMetaRow | null | undefined): ServerViewMeta {
+  if (!row || typeof row !== 'object') return { present: false, rowCount: null, state: null };
+  const state = row.state === 'materializing' || row.state === 'ready' ? row.state : null;
+  return { present: true, rowCount: typeof row.rowCount === 'number' ? row.rowCount : null, state };
+}
+
+/** Collapse duplicate `(id, version)` pairs to one per id, keeping the highest version. */
+export function dedupeRecordVersions(pairs: RecordVersionArray): RecordVersionArray {
+  if (pairs.length < 2) return pairs;
+  const best = new Map<string, number>();
+  for (const [id, version] of pairs) {
+    const prev = best.get(id);
+    if (prev === undefined || version > prev) best.set(id, version);
+  }
+  if (best.size === pairs.length) return pairs;
+  return Array.from(best.entries());
+}
+
+export interface MembershipDecisionInput {
+  phase: QueryPhase;
+  /** Number of ids currently held as membership. */
+  held: number;
+  remoteArray: RecordVersionArray;
+  meta?: ServerViewMeta;
+  verifiedRemoval?: boolean;
+}
+
+/**
+ * Whether a server id-set may be applied. A NON-EMPTY set is always the
+ * server's answer. An EMPTY set is believed only when the server stands
+ * behind it: the `_00_query` row is present, `ready` (or pre-`state`), and
+ * reports zero rows. A missing row while we hold membership is a lost view.
+ */
+export function decideMembershipOutcome(input: MembershipDecisionInput): MembershipOutcome {
+  const { phase, held, remoteArray, meta, verifiedRemoval } = input;
+  if (remoteArray.length > 0 || verifiedRemoval) return 'applied';
+  if (!meta || !meta.present) {
+    return held > 0 || phase !== 'cold' ? 'view-lost' : 'ignored';
+  }
+  const knownEmpty = meta.rowCount === 0 && meta.state !== 'materializing';
+  return knownEmpty ? 'applied' : 'ignored';
+}
+
+// ---- list_ref snapshots ------------------------------------------------------
+
+/**
+ * A statement that did not answer (an ERR, or no result at all). Kept apart
+ * from a `null` result, which is the server saying "no such row": reading a
+ * failed statement as a missing row turned every timeout under load into a
+ * lost view and re-registered it, over and over (whitepawn, 2026-09-16).
+ */
+export const FAILED: unique symbol = Symbol('statement failed');
+export type Answer<T> = T | null | typeof FAILED;
+
+export interface ListRefEdgeRow {
+  in: RecordId<string>;
+  out: RecordId<string>;
+  version: number;
+  parent?: unknown;
+}
+
+export interface ListRefSnapshot {
+  primary: RecordVersionArray;
+  subquery: RecordVersionArray;
+  meta: ServerViewMeta;
+}
+
+const toPairs = (rows: Array<{ out: RecordId<string>; version: number }> | null | undefined): RecordVersionArray =>
+  dedupeRecordVersions(Array.isArray(rows) ? rows.map((r) => [encodeRecordId(r.out), r.version]) : []);
+
+/**
+ * Fold the single-query statement batch (edges, meta, children) into a
+ * snapshot. `null` when any of the three did not answer: half an answer is
+ * not one, and a missing meta would read as a lost view.
+ */
+export function snapshotFromSingle(
+  items: Answer<Array<{ out: RecordId<string>; version: number }>>,
+  metaRow: Answer<QueryMetaRow>,
+  children: Answer<Array<{ out: RecordId<string>; version: number }>>
+): ListRefSnapshot | null {
+  if (!Array.isArray(items) || metaRow === FAILED || children === FAILED) return null;
+  return { primary: toPairs(items), subquery: toPairs(children), meta: metaFromRow(metaRow) };
+}
+
+/**
+ * Fold the many-query statement batch into one snapshot per hash. Every hash
+ * in `hashById` gets a snapshot; a query whose row did not come back reads
+ * `present: false`. Empty when either statement did not answer.
+ */
+export function snapshotsFromBatch(
+  edges: Answer<ListRefEdgeRow[]>,
+  counts: Answer<Array<QueryMetaRow | null>>,
+  hashById: ReadonlyMap<string, string>
+): Map<string, ListRefSnapshot> {
+  const out = new Map<string, ListRefSnapshot>();
+  if (!Array.isArray(edges) || counts === FAILED) return out;
+  for (const hash of hashById.values()) {
+    out.set(hash, { primary: [], subquery: [], meta: { present: false, rowCount: null, state: null } });
+  }
+  for (const row of edges) {
+    const hash = hashById.get(encodeRecordId(row.in));
+    if (!hash) continue;
+    const snapshot = out.get(hash)!;
+    const pair: [string, number] = [encodeRecordId(row.out), row.version];
+    if (row.parent == null) snapshot.primary.push(pair);
+    else snapshot.subquery.push(pair);
+  }
+  for (const snapshot of out.values()) {
+    snapshot.primary = dedupeRecordVersions(snapshot.primary);
+    snapshot.subquery = dedupeRecordVersions(snapshot.subquery);
+  }
+  for (const count of Array.isArray(counts) ? counts : []) {
+    if (!count || !count.id) continue;
+    const hash = hashById.get(encodeRecordId(count.id));
+    if (!hash) continue;
+    out.get(hash)!.meta = metaFromRow(count);
+  }
+  return out;
+}
+
+/**
+ * A batch that must not be believed at face value: a query we hold rows for
+ * whose `_00_query` row did not come back, or no edge at all while the server
+ * still reports rows for a held query. Such hashes are re-read one at a time.
+ */
+export function suspectHashes(
+  snapshots: ReadonlyMap<string, ListRefSnapshot>,
+  heldCounts: ReadonlyMap<string, number>,
+  edgeCount: number
+): string[] {
+  const out: string[] = [];
+  for (const [hash, snapshot] of snapshots) {
+    const held = heldCounts.get(hash) ?? 0;
+    if (held === 0) continue;
+    if (!snapshot.meta.present || (edgeCount === 0 && (snapshot.meta.rowCount ?? 0) > 0)) out.push(hash);
+  }
+  return out;
+}