@spooky-sync/core 0.0.1-canary.27 → 0.0.1-canary.270

package/src/mutation/write.saga.ts

@@ -0,0 +1,166 @@
+import { RecordId } from 'surrealdb';
+import type { MutationEventType, UpdateOptions } from '../types';
+import type { Saga } from '../kernel/saga';
+import { fx } from '../kernel/effects';
+import type { ClientState, PendingWrite } from '../state/client-state';
+import * as R from '../state/reducers';
+import { parseParams } from '../utils/parser';
+import { extractTablePart, parseRecordIdString } from '../utils/index';
+import type { SagaEnv } from '../query/env';
+import { columnsFor } from '../query/env';
+import { planCreateTx, planDeferredOutboxRowTx, planDeleteTx, planLocalOnlyUpdateTx, planUpdateTx, type LocalTx } from './rows';
+
+export interface WriteInput {
+  kind: MutationEventType;
+  recordId: string;
+  data?: Record<string, unknown>;
+  options?: UpdateOptions;
+}
+
+export interface WriteResult {
+  mutationId: string;
+  record: Record<string, unknown> | null;
+}
+
+const debounceKey = (input: WriteInput): string | null => {
+  const d = input.options?.debounced;
+  if (!d) return null;
+  const opts = d === true ? {} : d;
+  const keyMode = opts.key ?? 'recordId_x_fields';
+  return keyMode === 'recordId'
+    ? input.recordId
+    : `${input.recordId}::${Object.keys(input.data ?? {}).sort().join(',')}`;
+};
+const debounceDelay = (input: WriteInput): number => {
+  const d = input.options?.debounced;
+  return d && d !== true && typeof d.delay === 'number' ? d.delay : 300;
+};
+
+/**
+ * One optimistic write: read the previous row (update/delete), run the local
+ * transaction that writes the row AND its outbox entry, publish the outbox
+ * item (which puts the id into the overlay and dirties the table's queries),
+ * feed the circuit, and start a drain. A debounced update applies locally at
+ * once but accumulates into one outbox row written on flush.
+ */
+export function* write(env: SagaEnv, input: WriteInput): Saga<WriteResult> {
+  const table = extractTablePart(input.recordId);
+  const columns = columnsFor(env, table);
+  if (!columns) throw new Error(`Table ${table} not found`);
+  const rid = parseRecordIdString(input.recordId);
+  const params = input.data ? parseParams(columns as never, input.data) : undefined;
+  const key = input.kind === 'update' ? debounceKey(input) : null;
+  if (key !== null) return yield* debouncedUpdate(input, key, rid, table, params ?? {});
+  const mutationId = (yield fx.id('mutation')) as string;
+  const now = (yield fx.now()) as number;
+  let before: Record<string, unknown> | null = null;
+  if (input.kind !== 'create') before = yield* readBefore(rid);
+  const tx = planTx(input.kind, { recordId: rid, mutationId: parseRecordIdString(mutationId), table, data: params, before, now });
+  return yield* commitWrite(input.kind, tx, { mutationId, recordId: input.recordId, table, before, now });
+}
+
+function planTx(kind: MutationEventType, planInput: Parameters<typeof planCreateTx>[0]): LocalTx {
+  if (kind === 'create') return planCreateTx(planInput);
+  if (kind === 'update') return planUpdateTx(planInput);
+  return planDeleteTx(planInput);
+}
+
+function* readBefore(rid: RecordId<string>): Saga<Record<string, unknown> | null> {
+  const res = (yield fx.local.query('SELECT * FROM ONLY $id', { id: rid })) as unknown[];
+  const row = Array.isArray(res) ? res[0] : null;
+  return row && typeof row === 'object' && !Array.isArray(row) ? (row as Record<string, unknown>) : null;
+}
+
+interface CommitCtx {
+  mutationId: string;
+  recordId: string;
+  table: string;
+  before: Record<string, unknown> | null;
+  now: number;
+}
+
+function* commitWrite(kind: MutationEventType, tx: LocalTx, ctx: CommitCtx): Saga<WriteResult> {
+  const result = (yield fx.local.execute(tx.query, tx.vars)) as unknown;
+  const target = kind === 'update' ? (result as { target?: unknown } | undefined)?.target : result;
+  const record = kind === 'delete' || !target || typeof target !== 'object' ? null : (target as Record<string, unknown>);
+  yield fx.state.update(
+    R.outboxPush({ id: ctx.mutationId, type: kind, recordId: ctx.recordId, table: ctx.table, status: 'pending', ackedAt: null, attempts: 0 })
+  );
+  const version = kind === 'create' ? 1 : typeof record?._00_rv === 'number' ? (record._00_rv as number) : ((ctx.before?._00_rv as number) ?? 0) + 1;
+  const circuitRecord = kind === 'delete' ? (ctx.before ?? {}) : { ...record, _00_rv: version };
+  const circuitTuple = { table: ctx.table, op: kind.toUpperCase() as 'CREATE' | 'UPDATE' | 'DELETE', id: ctx.recordId, record: circuitRecord };
+  try {
+    yield fx.ssp.ingest([circuitTuple]);
+    if (kind === 'delete') yield fx.state.update(R.deleteVersions([ctx.recordId]));
+    else yield fx.state.update(R.setVersions([[ctx.recordId, version]]));
+  } catch (error) {
+    yield fx.emit({ type: 'log', level: 'error', message: 'circuit ingest failed after a local write', data: { recordId: ctx.recordId, error } });
+  }
+  yield fx.emit({
+    type: 'mutation:event',
+    event: { type: kind, mutation_id: parseRecordIdString(ctx.mutationId), record_id: parseRecordIdString(ctx.recordId), data: tx.vars.data, record, beforeRecord: ctx.before ?? undefined, tableName: ctx.table },
+  });
+  const role = (yield fx.state.read((s) => s.tabRole)) as ClientState['tabRole'];
+  if (role !== 'solo') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records: [circuitTuple] } });
+  if (role === 'follower') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'outbox-changed', mutationId: ctx.mutationId } });
+  else yield fx.dispatch({ type: 'Drain' });
+  return { mutationId: ctx.mutationId, record };
+}
+
+/** Apply locally now, remember the merged patch, write the outbox row on flush. */
+function* debouncedUpdate(
+  input: WriteInput,
+  key: string,
+  rid: RecordId<string>,
+  table: string,
+  params: Record<string, unknown>
+): Saga<WriteResult> {
+  const existing = (yield fx.state.read((s) => s.pendingWrites.get(key))) as PendingWrite | undefined;
+  const now = (yield fx.now()) as number;
+  const before = existing ? existing.before : yield* readBefore(rid);
+  const local = planLocalOnlyUpdateTx({ recordId: rid, data: params });
+  const res = (yield fx.local.execute(local.query, local.vars)) as { target?: Record<string, unknown> } | undefined;
+  const record = res?.target ?? null;
+  yield fx.state.update(R.mergePendingWrite({ key, table, recordId: input.recordId, data: params, before, firstAt: existing?.firstAt ?? now }));
+  yield fx.state.update(R.markTableDirty(table));
+  const version = typeof record?._00_rv === 'number' ? (record._00_rv as number) : 0;
+  const tuple = { table, op: 'UPDATE' as const, id: input.recordId, record: { ...record, _00_rv: version } };
+  try {
+    yield fx.ssp.ingest([tuple]);
+    yield fx.state.update(R.setVersions([[input.recordId, version]]));
+  } catch (error) {
+    yield fx.emit({ type: 'log', level: 'error', message: 'circuit ingest failed after a local write', data: { recordId: input.recordId, error } });
+  }
+  const role = (yield fx.state.read((s) => s.tabRole)) as ClientState['tabRole'];
+  if (role !== 'solo') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records: [tuple] } });
+  yield fx.timer.set(`debounce:${key}`, debounceDelay(input), { type: 'FlushWrite', key });
+  return { mutationId: '', record };
+}
+
+/** Timer target: turn the accumulated patch into one outbox row and drain. */
+export function* flushWrite(env: SagaEnv, key: string): Saga<void> {
+  const pending = (yield fx.state.read((s) => s.pendingWrites.get(key))) as PendingWrite | undefined;
+  if (!pending) return;
+  yield fx.state.update(R.clearPendingWrite(key));
+  const mutationId = (yield fx.id('mutation')) as string;
+  const now = (yield fx.now()) as number;
+  const rid = parseRecordIdString(pending.recordId);
+  const insert = planDeferredOutboxRowTx({
+    recordId: rid,
+    mutationId: parseRecordIdString(mutationId),
+    table: pending.table,
+    data: { ...pending.data },
+    before: pending.before,
+    now,
+  });
+  yield fx.local.execute(insert.query, insert.vars);
+  yield fx.state.update(R.outboxPush({ id: mutationId, type: 'update', recordId: pending.recordId, table: pending.table, status: 'pending', ackedAt: null, attempts: 0 }));
+  yield fx.emit({
+    type: 'mutation:event',
+    event: { type: 'update', mutation_id: parseRecordIdString(mutationId), record_id: rid, data: { ...pending.data }, beforeRecord: pending.before ?? undefined, tableName: pending.table },
+  });
+  const role = (yield fx.state.read((s) => s.tabRole)) as ClientState['tabRole'];
+  if (role === 'follower') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'outbox-changed', mutationId } });
+  else yield fx.dispatch({ type: 'Drain' });
+}
+

package/src/otel/index.ts

@@ -1,5 +1,5 @@
-import { Level } from 'pino';
-import { PinoTransmit } from '../types';
+import type { Level } from 'pino';
+import type { PinoTransmit } from '../types';
 
 // Map pino levels to OTEL severity numbers
 // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#severity-fields
@@ -32,7 +32,7 @@ async function loadOtelModules(otelEndpoint: string) {
     ]);
 
   const resource = resourceFromAttributes({
-    [ATTR_SERVICE_NAME]: 'spooky-client',
+    [ATTR_SERVICE_NAME]: 'sp00ky-client',
   });
 
   const exporter = new OTLPLogExporter({
@@ -61,7 +61,7 @@ async function loadOtelModules(otelEndpoint: string) {
  * ```ts
  * import { createOtelTransmit } from '@spooky-sync/core/otel';
  *
- * new SpookyClient({
+ * new Sp00kyClient({
  *   // ...
  *   otelTransmit: createOtelTransmit('http://localhost:4318/v1/logs'),
  * });
@@ -74,6 +74,7 @@ export function createOtelTransmit(endpoint: string, level: Level = 'info'): Pin
   return {
     level: level,
     send: (levelLabel: string, logEvent: any) => {
+      // oxlint-disable-next-line promise/always-return
       otelReady.then((getOtelLogger) => {
         try {
           const messages = [...logEvent.messages];
@@ -81,15 +82,15 @@ export function createOtelTransmit(endpoint: string, level: Level = 'info'): Pin
 
           // Construct the message body
           let body = '';
-          const msg = messages.pop();
+          const lastMsg = messages.pop();
 
-          if (typeof msg === 'string') {
-            body = msg;
-          } else if (msg) {
-            body = JSON.stringify(msg);
+          if (typeof lastMsg === 'string') {
+            body = lastMsg;
+          } else if (lastMsg) {
+            body = JSON.stringify(lastMsg);
           }
 
-          let category = 'spooky-client::unknown';
+          let category = 'sp00ky-client::unknown';
 
           const attributes = {};
           for (const msg of messages) {
@@ -114,9 +115,11 @@ export function createOtelTransmit(endpoint: string, level: Level = 'info'): Pin
             timestamp: new Date(logEvent.ts),
           });
         } catch (e) {
+          // oxlint-disable-next-line no-console
           console.warn('Failed to transmit log to OTEL endpoint', e);
         }
       }).catch((e) => {
+        // oxlint-disable-next-line no-console
         console.warn('Failed to load OpenTelemetry modules', e);
       });
     },

package/src/query/env.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { columnsFor, defaultEnv, listRefTable } from './env';
+import { emptyState } from '../state/client-state';
+
+const schema = { tables: [{ name: 'thing', columns: { a: { type: 'string' } } }] } as any;
+
+describe('SagaEnv', () => {
+  it('defaultEnv applies overrides', () => {
+    expect(defaultEnv(schema).outboxBatchSize).toBe(50);
+    expect(defaultEnv(schema, { outboxBatchSize: 5 }).outboxBatchSize).toBe(5);
+  });
+  it('listRefTable: per-user, anonymous when enabled, global otherwise', () => {
+    const s = emptyState({ tabId: 't' });
+    expect(listRefTable(defaultEnv(schema), { ...s, userId: 'user:abc' })).toBe('_00_list_ref_user_abc');
+    expect(listRefTable(defaultEnv(schema, { anonLive: true }), s)).toBe('_00_list_ref_anon');
+    expect(listRefTable(defaultEnv(schema), s)).toBe('_00_list_ref');
+  });
+  it('columnsFor: schema table, framework table, unknown', () => {
+    expect(columnsFor(defaultEnv(schema), 'thing')).toEqual({ a: { type: 'string' } });
+    expect(columnsFor(defaultEnv(schema), '_00_feature_flag')).toEqual({});
+    expect(columnsFor(defaultEnv(schema), 'nope')).toBeNull();
+  });
+});

package/src/query/env.ts

@@ -0,0 +1,56 @@
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import type { RefMode } from '../modules/ref-tables';
+import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor } from '../modules/ref-tables';
+import type { ClientState } from '../state/client-state';
+import {
+  DEGRADE_AFTER_FAILURES,
+  LIST_REF_POLL_BASE_MS,
+  MATERIALIZE_DEBOUNCE_MS,
+  OUTBOX_BATCH_SIZE,
+  PUSH_TIMEOUT_MS,
+} from '../kernel/constants';
+
+/**
+ * Static configuration the sagas read. Plain data, built once by the runtime
+ * from `Sp00kyConfig`; sagas never see the config object itself.
+ */
+export interface SagaEnv {
+  readonly schema: SchemaStructure;
+  readonly refMode: RefMode;
+  readonly anonLive: boolean;
+  readonly remoteTimeoutMs: number;
+  readonly pushTimeoutMs: number;
+  readonly outboxBatchSize: number;
+  readonly degradeAfter: number;
+  readonly materializeDebounceMs: number;
+  readonly pollBaseMs: number;
+  readonly defaultTtlMs: number;
+}
+
+export function defaultEnv(schema: SchemaStructure, over: Partial<SagaEnv> = {}): SagaEnv {
+  return {
+    schema,
+    refMode: DEFAULT_REF_MODE,
+    anonLive: false,
+    remoteTimeoutMs: 60_000,
+    pushTimeoutMs: PUSH_TIMEOUT_MS,
+    outboxBatchSize: OUTBOX_BATCH_SIZE,
+    degradeAfter: DEGRADE_AFTER_FAILURES,
+    materializeDebounceMs: MATERIALIZE_DEBOUNCE_MS,
+    pollBaseMs: LIST_REF_POLL_BASE_MS,
+    defaultTtlMs: 600_000,
+    ...over,
+  };
+}
+
+/** The `_00_list_ref` table this session reads: per-user, anonymous, or global. */
+export function listRefTable(env: SagaEnv, s: ClientState): string {
+  const userId = s.userId ?? (env.anonLive ? ANON_USER_ID : null);
+  return listRefTableFor(env.refMode, userId);
+}
+
+export function columnsFor(env: SagaEnv, table: string): Record<string, unknown> | null {
+  const found = env.schema.tables.find((t) => t.name === table);
+  if (found) return found.columns as Record<string, unknown>;
+  return table.startsWith('_00_') ? {} : null;
+}

package/src/query/fetch.saga.test.ts

@@ -0,0 +1,156 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { runPure } from '../testing/run-pure';
+import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
+import * as R from '../state/reducers';
+import { defaultEnv } from './env';
+import { fetchRows } from './fetch.saga';
+import type { StatementResult } from '../kernel/effects';
+
+const env = defaultEnv({ tables: [{ name: 'thing', columns: { a: {} } }] } as any);
+const ok = (result: unknown): StatementResult => ({ status: 'OK', result });
+const live = { phase: 'live' } as const;
+const primed = (s: ReturnType<typeof buildState>) => ({ ...s, primed: true });
+
+describe('fetchRows', () => {
+  it('waits for the prime and returns on an empty plan (resetting the attempt counter)', async () => {
+    const s = R.patchSync({ fetchAttempts: 3 })(buildState());
+    const out = await runPure(fetchRows(env), { state: s, handlers: { 'state.wait': (_e, ctx) => void (ctx.state = { ...ctx.state, primed: true }) } });
+    expect(out.state.sync.fetchAttempts).toBe(0);
+    expect(out.log.filter((e) => e.kind === 'remote.query')).toHaveLength(0);
+  });
+
+  it('fetches the deduped set once, writes + ingests bodies, records versions, balances fetch depth', async () => {
+    const s = primed(
+      buildState(
+        [
+          buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 2], ['thing:2', 1]] }),
+          buildEntry({ def: { hash: 'b' }, lifecycle: live, remoteArray: [['thing:2', 1], ['thing:3', 1]] }),
+        ],
+        R.setVersions([['thing:3', 1], ['thing:1', 1]]),
+        R.outboxReplace([buildOutboxItem({ type: 'delete', recordId: 'thing:2' })])
+      )
+    );
+    const depths: number[] = [];
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      epoch: 7,
+      handlers: {
+        'remote.query': (e: any, ctx) => {
+          depths.push(ctx.state.queries.get('a')!.lifecycle.fetchDepth);
+          expect(e.sql).toBe('SELECT * FROM $ids');
+          expect(e.vars.ids).toEqual([new RecordId('thing', '1')]);
+          return [ok([{ id: new RecordId('thing', '1'), a: 'x', junk: true }])];
+        },
+        'local.execute': () => undefined,
+        'ssp.ingest': () => undefined,
+      },
+    });
+    expect(depths).toEqual([1]);
+    expect(out.state.queries.get('a')!.lifecycle.fetchDepth).toBe(0);
+    expect(out.state.versions.get('thing:1')).toBe(2);
+    const exec = out.log.find((e) => e.kind === 'local.execute') as any;
+    expect(exec.vars.content0).toEqual({ a: 'x', _00_rv: 2 });
+    expect(exec.epoch).toBe(7);
+    expect(out.log.findIndex((e) => e.kind === 'local.epoch')).toBeLessThan(out.log.findIndex((e) => e.kind === 'all'));
+    const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
+    expect(ingest.records).toEqual([{ table: 'thing', op: 'UPDATE', id: 'thing:1', record: { id: new RecordId('thing', '1'), a: 'x', _00_rv: 2 } }]);
+    expect(out.dispatched).toEqual([{ type: 'SyncOutcome', ok: true, error: undefined }]);
+    expect(out.state.dirty.has('a')).toBe(true);
+  });
+
+  it('a body the server does not return is remembered at the requested version so the plan converges', async () => {
+    const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1], ['other:2', 1]] })]));
+    let calls = 0;
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': () => {
+          calls++;
+          return [ok([{ id: new RecordId('other', '2'), z: 1 }, { nope: true }])];
+        },
+        'local.execute': () => undefined,
+        'ssp.ingest': () => undefined,
+      },
+    });
+    expect(calls).toBe(1);
+    expect(out.state.versions.get('thing:1')).toBe(1);
+    expect(out.state.versions.get('other:2')).toBe(1);
+    const extra = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': () => [ok([{ id: new RecordId('thing', '1') }, { id: new RecordId('thing', 'unrequested') }])],
+        'local.execute': () => undefined,
+        'ssp.ingest': (e: any) => {
+          expect(e.records[1]).toMatchObject({ id: 'thing:unrequested', record: { _00_rv: 0 } });
+        },
+      },
+    });
+    expect(extra.state.versions.has('thing:unrequested')).toBe(false);
+    const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
+    expect(ingest.records[0]).toMatchObject({ table: 'other', op: 'CREATE', record: { z: 1, _00_rv: 1 } });
+  });
+
+  it('failures back off: transport error, ERR statement, non-array result, local write error; ingest errors are logged only', async () => {
+    const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })]));
+    const expectBackoff = (out: Awaited<ReturnType<typeof runPure>>, attempts: number) => {
+      expect(out.state.sync.fetchAttempts).toBe(attempts);
+      expect(out.timers.get('fetch')).toEqual({ ms: 500 * 2 ** (attempts - 1), event: { type: 'FetchRows' } });
+      expect(out.state.queries.get('a')!.lifecycle.fetchDepth).toBe(0);
+      expect(out.dispatched.at(-1)).toMatchObject({ type: 'SyncOutcome', ok: false });
+    };
+    expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => { throw new Error('offline'); } } }), 1);
+    expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => [{ status: 'ERR', error: 'x' }] } }), 1);
+    expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => [ok('nope')] } }), 1);
+    expectBackoff(
+      await runPure(fetchRows(env), {
+        state: R.patchSync({ fetchAttempts: 1 })(s),
+        handlers: {
+          'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])],
+          'local.execute': () => {
+            throw new Error('stale epoch');
+          },
+        },
+      }),
+      2
+    );
+    const ingestFail = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])],
+        'local.execute': () => undefined,
+        'ssp.ingest': () => {
+          throw new Error('wasm');
+        },
+      },
+    });
+    expect(ingestFail.state.sync.fetchAttempts).toBe(0);
+    expect(ingestFail.emitted).toContainEqual(expect.objectContaining({ message: 'circuit ingest failed' }));
+  });
+
+  it('a leader relays landed bodies to followers', async () => {
+    const s = { ...primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })])), tabRole: 'leader' as const };
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      handlers: { 'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])], 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
+    });
+    expect(out.emitted).toContainEqual({ type: 'tabs:broadcast', message: { type: 'ingest', records: [expect.objectContaining({ id: 'thing:1' })] } });
+  });
+
+  it('loops until the plan is empty', async () => {
+    const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })]));
+    let calls = 0;
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': (_e, ctx) => {
+          calls++;
+          if (calls === 1) ctx.state = R.commitMembership('a', [['thing:1', 1], ['thing:2', 1]], true)(ctx.state);
+          return [ok([])];
+        },
+      },
+    });
+    expect(calls).toBe(2);
+    expect(out.state.versions.get('thing:2')).toBe(1);
+  });
+});

package/src/query/fetch.saga.ts

@@ -0,0 +1,115 @@
+import type { RecordId } from 'surrealdb';
+import type { Saga } from '../kernel/saga';
+import type { Settled, StatementResult } from '../kernel/effects';
+import { fx } from '../kernel/effects';
+import { backoffMs } from '../kernel/constants';
+import type { IngestRecord } from '../services/stream-processor/index';
+import type { ClientState } from '../state/client-state';
+import * as R from '../state/reducers';
+import { planFetch } from '../state/selectors';
+import { encodeRecordId, parseRecordIdString } from '../utils/index';
+import { cleanRecord } from '../utils/parser';
+import type { SagaEnv } from './env';
+import * as sql from './sql';
+
+/**
+ * The unified row fetcher (the poke analogue). Runs on the serial `fetch`
+ * lane: computes the cross-query set of bodies missing or stale locally,
+ * pulls them in parallel chunks, writes + ingests them, and loops until the
+ * plan is empty. A row shared by N queries is fetched once. Bodies are never
+ * deleted here; ids that left membership simply stop rendering.
+ */
+export function* fetchRows(env: SagaEnv): Saga<void> {
+  yield fx.state.wait((s) => s.primed);
+  for (;;) {
+    const state = (yield fx.state.read((s) => s)) as ClientState;
+    const plan = planFetch(state);
+    if (plan.chunks.length === 0) {
+      if (state.sync.fetchAttempts !== 0) yield fx.state.update(R.patchSync({ fetchAttempts: 0 }));
+      return;
+    }
+    // The engine's epoch, not a mirrored counter: it also moves on transport
+    // failover, and a stale value makes the store drop every body write.
+    const epoch = (yield fx.local.epoch()) as number;
+    yield fx.state.update(R.compose(...plan.hashes.map((h) => R.applyLifecycle(h, { type: 'fetch-begin' }))));
+    let failed = 0;
+    try {
+      const results = (yield fx.all(
+        plan.chunks.map((ids) => fx.remote.query(sql.bodySelect(), { ids: ids.map(parseRecordIdString) }, env.remoteTimeoutMs))
+      )) as Settled[];
+      for (let i = 0; i < results.length; i++) {
+        const r = results[i];
+        const first = r.ok ? (r.value as StatementResult[])[0] : undefined;
+        if (!r.ok || !first || first.status === 'ERR' || !Array.isArray(first.result)) {
+          failed++;
+          continue;
+        }
+        const landed = yield* landChunk(env, plan.chunks[i], first.result as Array<Record<string, unknown>>, plan.versions, state, epoch);
+        if (!landed) failed++;
+      }
+    } finally {
+      yield fx.state.update(R.compose(...plan.hashes.map((h) => R.applyLifecycle(h, { type: 'fetch-end' }))));
+    }
+    yield fx.dispatch({ type: 'SyncOutcome', ok: failed === 0, error: failed > 0 ? 'body fetch failed' : undefined });
+    if (failed > 0) {
+      const attempt = (yield fx.state.read((s) => s.sync.fetchAttempts)) as number;
+      yield fx.state.update(R.patchSync({ fetchAttempts: attempt + 1 }));
+      yield fx.timer.set('fetch', backoffMs(attempt), { type: 'FetchRows' });
+      return;
+    }
+  }
+}
+
+/**
+ * Write one chunk's bodies to the store and circuit; record their versions.
+ *
+ * Shared with the LIVE path: a body that arrives on a notification is landed
+ * through here so it is indistinguishable from a fetched one, which is what
+ * makes `planFetch` stop asking for it.
+ */
+export function* landChunk(
+  env: SagaEnv,
+  requested: string[],
+  rows: Array<Record<string, unknown>>,
+  versions: ReadonlyMap<string, number>,
+  state: ClientState,
+  epoch: number
+): Saga<boolean> {
+  const bodies: Array<{ id: unknown; content: Record<string, unknown> }> = [];
+  const ingest: IngestRecord[] = [];
+  const seen = new Set<string>();
+  for (const row of rows) {
+    const rid = row?.id as RecordId<string> | undefined;
+    if (!rid || typeof rid !== 'object') continue;
+    const id = encodeRecordId(rid);
+    const table = String(rid.table);
+    const version = versions.get(id) ?? 0;
+    const columns = env.schema.tables.find((t) => t.name === table)?.columns;
+    const cleaned = columns ? cleanRecord(columns, row) : row;
+    const { id: _id, ...content } = cleaned;
+    bodies.push({ id: rid, content: { ...content, _00_rv: version } });
+    ingest.push({ table, op: state.versions.has(id) ? 'UPDATE' : 'CREATE', id, record: { ...cleaned, _00_rv: version } });
+    seen.add(id);
+  }
+  if (bodies.length > 0) {
+    try {
+      const tx = sql.upsertBodiesTx(bodies);
+      yield fx.local.execute(tx.query, tx.vars, epoch);
+    } catch (error) {
+      yield fx.emit({ type: 'log', level: 'warn', message: 'body write failed', data: { error } });
+      return false;
+    }
+    try {
+      yield fx.ssp.ingest(ingest);
+    } catch (error) {
+      yield fx.emit({ type: 'log', level: 'warn', message: 'circuit ingest failed', data: { error } });
+    }
+    if (state.tabRole === 'leader') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records: ingest } });
+  }
+  // Ids the server did not return (deleted or unreadable upstream) count as
+  // known at the requested version, so the plan stops asking for them; their
+  // absence from the store is what the render shows.
+  const entries: Array<readonly [string, number]> = requested.map((id) => [id, versions.get(id) as number]);
+  yield fx.state.update(R.setVersions(entries));
+  return true;
+}

package/src/query/hash.test.ts

@@ -0,0 +1,14 @@
+import { describe, expect, it } from 'vitest';
+import { queryHashInput, viewKeyInput } from './hash';
+
+describe('hash inputs (golden against the previous DataModule formulas)', () => {
+  const input = { surql: 'SELECT * FROM t WHERE a = $a', params: { a: 1, b: 'x' } };
+  it('salted key is JSON.stringify({surql, params, sessionId}) in that key order', () => {
+    expect(queryHashInput(input, 'sess')).toBe(JSON.stringify({ ...input, sessionId: 'sess' }));
+    expect(queryHashInput(input, null)).toBe(JSON.stringify({ ...input, sessionId: null }));
+  });
+  it('view key is the same input minus the salt', () => {
+    expect(viewKeyInput(input)).toBe(JSON.stringify(input));
+    expect(viewKeyInput(input)).not.toBe(queryHashInput(input, 'sess'));
+  });
+});

package/src/query/hash.ts

@@ -0,0 +1,18 @@
+/**
+ * Hash inputs for the two query keys. Kept byte-identical to the previous
+ * implementation (`DataModule.calculateHash` / `calculateMembershipKey`):
+ * the salted key names the remote `_00_query` row, the unsalted one names the
+ * durable `_00_view` row, and both are `sha256(JSON.stringify(input))`.
+ */
+export interface QueryKeyInput {
+  surql: string;
+  params: Record<string, unknown>;
+}
+
+/** Session-salted: two tabs of one user must not share a remote view. */
+export const queryHashInput = (input: QueryKeyInput, sessionId: string | null): string =>
+  JSON.stringify({ surql: input.surql, params: input.params, sessionId });
+
+/** Session-independent: survives reload and bucket switch. */
+export const viewKeyInput = (input: QueryKeyInput): string =>
+  JSON.stringify({ surql: input.surql, params: input.params });