@spooky-sync/core 0.0.1-canary.30 → 0.0.1-canary.301

package/src/pure.ts

@@ -0,0 +1,65 @@
+/**
+ * `@spooky-sync/core/pure`: the side-effect-free builders of the sync protocol.
+ *
+ * Everything here is plain functions and constants over `surrealdb` value
+ * types: no wasm, no logger, no `window`/`document`/`localStorage`. Safe to
+ * import from a service worker, a dedicated worker or Node. The full client
+ * imports the same modules, so what this entry builds is byte for byte what
+ * `Sp00kyClient` sends.
+ */
+import { queryHashInput, type QueryKeyInput } from './query/hash';
+import { sha256Hex } from './utils/sha256';
+
+// Statement builders: register + snapshot, heartbeat, bodies, views.
+export * from './query/sql';
+// Query keys.
+export { queryHashInput, viewKeyInput, type QueryKeyInput } from './query/hash';
+// `_00_list_ref` table naming and user-id sanitizing (mirrors ssp-protocol).
+export {
+  ANON_USER_ID,
+  DEFAULT_REF_MODE,
+  bucketIdForUser,
+  listRefTableFor,
+  sanitizeUserId,
+  type RefMode,
+} from './modules/ref-tables';
+// Membership snapshot parsing (the answer to the register read-back).
+export {
+  FAILED,
+  dedupeRecordVersions,
+  metaFromRow,
+  snapshotFromSingle,
+  type Answer,
+  type ListRefSnapshot,
+  type QueryMetaRow,
+} from './query/membership';
+// LIVE edge decoding.
+export { hashOfEdge, rowOfEdge } from './utils/edges';
+// SurrealQL string helpers.
+export {
+  surql,
+  type SealOptions,
+  type SealedQuery,
+  type SurqlHelper,
+  type TxQuery,
+} from './utils/surql';
+// Param typing the full client applies before registering.
+export { parseQueryParams } from './utils/parser';
+export { encodeRecordId, parseRecordIdString, parseDuration } from './utils/index';
+// Unverified JWT claim reading (user id, access method, impersonation).
+export {
+  decodeTokenClaims,
+  IMPERSONATION_ACCESS,
+  type TokenClaims,
+} from './modules/auth/impersonation';
+export { sha256Hex } from './utils/sha256';
+// Web Push wire types.
+export * from './push/types';
+
+/**
+ * The remote `_00_query` key of a query for one session salt: what the full
+ * client names its view (`_00_query:<key>`).
+ */
+export function queryKey(input: QueryKeyInput, sessionId: string | null): Promise<string> {
+  return sha256Hex(queryHashInput(input, sessionId));
+}

package/src/push/types.ts

@@ -0,0 +1,174 @@
+/**
+ * Web Push wire types shared by the page (`db.webPush`), the service worker
+ * bridge (`@spooky-sync/core/sw`) and the live feed. Pure: types plus a few
+ * dependency-free helpers, safe in any context.
+ *
+ * `PushPayload` mirrors `packages/push-core/src/config.rs` (`PushPayload`,
+ * `PayloadKind`, `Op`, `NotificationTemplate`). Keep the two in step.
+ */
+
+/** Payload format version this build understands. */
+export const PUSH_PAYLOAD_VERSION = 1;
+
+export type PushPayloadKind = 'rule' | 'message';
+export type PushOp = 'create' | 'update' | 'delete';
+export type PushUrgency = 'very-low' | 'low' | 'normal' | 'high';
+
+/** One button on a notification (`NotificationAction` plus a per-action `url`). */
+export interface PushNotificationAction {
+  action: string;
+  title: string;
+  icon?: string;
+  /** Opened instead of the notification's `url` when this action is clicked. */
+  url?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * What a content push shows. Every key other than `title` and `url` is passed
+ * to `showNotification` as is, unknown ones included (`vibrate`, `timestamp`,
+ * whatever a browser adds later).
+ */
+export interface PushNotification {
+  title?: string;
+  body?: string;
+  icon?: string;
+  badge?: string;
+  image?: string;
+  lang?: string;
+  dir?: string;
+  /** Defaults to the push's `topic`. */
+  tag?: string;
+  /** Opened (or focused) on click. */
+  url?: string;
+  requireInteraction?: boolean;
+  renotify?: boolean;
+  silent?: boolean;
+  actions?: PushNotificationAction[];
+  /** Merged into the notification's `data` next to the bridge's own keys. */
+  data?: Record<string, unknown>;
+  [key: string]: unknown;
+}
+
+/**
+ * The JSON every push carries (then RFC 8291 encrypted). `notification`
+ * present: a content push the service worker shows as is. Absent: a
+ * content-free nudge, the app renders from its own data.
+ */
+export interface PushPayload {
+  v: number;
+  kind: PushPayloadKind;
+  /** Rule name (`kind: 'rule'`). */
+  rule?: string;
+  /** `_00_push_message` id (`kind: 'message'`). */
+  message?: string;
+  /** The row that fired the rule. */
+  table?: string;
+  id?: string;
+  op?: PushOp;
+  topic?: string;
+  notification?: PushNotification;
+  data?: Record<string, unknown>;
+  /** Epoch ms when the engine built it. */
+  ts: number;
+}
+
+/** Page -> service worker: the session to use for live rendering. */
+export interface BridgeTokenMessage {
+  type: 'sp00ky:token';
+  token: string;
+  userId: string;
+  /** The SurrealDB endpoint (`wss://host/rpc`), not the push endpoint. */
+  endpoint: string;
+  namespace: string;
+  database: string;
+  /** VAPID public key (base64url) the page subscribed with, for `pushsubscriptionchange`. */
+  publicKey?: string;
+  /** This browser's push endpoint, when subscribed. */
+  pushEndpoint?: string;
+}
+
+/** Page -> service worker: forget the session and close every notification. */
+export interface BridgeSignOutMessage {
+  type: 'sp00ky:signout';
+}
+
+/** Service worker -> page messages. */
+export type WebPushClientMessage =
+  /** A push arrived while this page was visible, so nothing was shown. */
+  | { type: 'sp00ky:push'; payload: PushPayload }
+  /** A notification was clicked and the bridge was told to route in-app. */
+  | { type: 'sp00ky:navigate'; url: string; payload?: PushPayload; action?: string }
+  /** The browser rotated the push subscription; the page should `sync()`. */
+  | { type: 'sp00ky:subscriptionchange'; endpoint?: string };
+
+/** `urlBase64ToUint8Array`: the VAPID key as `applicationServerKey` wants it. */
+export function base64UrlToBytes(value: string): Uint8Array {
+  const padded =
+    value.replace(/-/g, '+').replace(/_/g, '/') + '='.repeat((4 - (value.length % 4)) % 4);
+  const raw = typeof atob === 'function' ? atob(padded) : decodeBase64Fallback(padded);
+  const out = new Uint8Array(raw.length);
+  for (let i = 0; i < raw.length; i++) out[i] = raw.charCodeAt(i);
+  return out;
+}
+
+export function bytesToBase64Url(bytes: ArrayBuffer | Uint8Array): string {
+  const view = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+  let raw = '';
+  for (let i = 0; i < view.length; i++) raw += String.fromCharCode(view[i]);
+  const b64 = typeof btoa === 'function' ? btoa(raw) : encodeBase64Fallback(raw);
+  return b64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+}
+
+export function sameBytes(
+  a: ArrayBuffer | Uint8Array | null | undefined,
+  b: ArrayBuffer | Uint8Array | null | undefined
+): boolean {
+  if (!a || !b) return false;
+  const x = a instanceof Uint8Array ? a : new Uint8Array(a);
+  const y = b instanceof Uint8Array ? b : new Uint8Array(b);
+  if (x.length !== y.length) return false;
+  for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) return false;
+  return true;
+}
+
+/** A payload this build can act on (`v` known, `kind` set). */
+export function isPushPayload(value: unknown): value is PushPayload {
+  if (!value || typeof value !== 'object') return false;
+  const v = value as { v?: unknown; kind?: unknown };
+  return v.v === PUSH_PAYLOAD_VERSION && (v.kind === 'rule' || v.kind === 'message');
+}
+
+const B64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+
+function decodeBase64Fallback(input: string): string {
+  const clean = input.replace(/=+$/, '');
+  let out = '';
+  let buffer = 0;
+  let bits = 0;
+  for (const ch of clean) {
+    const idx = B64.indexOf(ch);
+    if (idx < 0) continue;
+    buffer = (buffer << 6) | idx;
+    bits += 6;
+    if (bits >= 8) {
+      bits -= 8;
+      out += String.fromCharCode((buffer >> bits) & 0xff);
+    }
+  }
+  return out;
+}
+
+function encodeBase64Fallback(raw: string): string {
+  let out = '';
+  for (let i = 0; i < raw.length; i += 3) {
+    const a = raw.charCodeAt(i);
+    const b = i + 1 < raw.length ? raw.charCodeAt(i + 1) : 0;
+    const c = i + 2 < raw.length ? raw.charCodeAt(i + 2) : 0;
+    const n = (a << 16) | (b << 8) | c;
+    out += B64[(n >> 18) & 63] + B64[(n >> 12) & 63];
+    out += i + 1 < raw.length ? B64[(n >> 6) & 63] : '=';
+    out += i + 2 < raw.length ? B64[n & 63] : '=';
+  }
+  return out;
+}

package/src/query/env.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { columnsFor, defaultEnv, listRefTable } from './env';
+import { emptyState } from '../state/client-state';
+
+const schema = { tables: [{ name: 'thing', columns: { a: { type: 'string' } } }] } as any;
+
+describe('SagaEnv', () => {
+  it('defaultEnv applies overrides', () => {
+    expect(defaultEnv(schema).outboxBatchSize).toBe(50);
+    expect(defaultEnv(schema, { outboxBatchSize: 5 }).outboxBatchSize).toBe(5);
+  });
+  it('listRefTable: per-user, anonymous when enabled, global otherwise', () => {
+    const s = emptyState({ tabId: 't' });
+    expect(listRefTable(defaultEnv(schema), { ...s, userId: 'user:abc' })).toBe('_00_list_ref_user_abc');
+    expect(listRefTable(defaultEnv(schema, { anonLive: true }), s)).toBe('_00_list_ref_anon');
+    expect(listRefTable(defaultEnv(schema), s)).toBe('_00_list_ref');
+  });
+  it('columnsFor: schema table, framework table, unknown', () => {
+    expect(columnsFor(defaultEnv(schema), 'thing')).toEqual({ a: { type: 'string' } });
+    expect(columnsFor(defaultEnv(schema), '_00_feature_flag')).toEqual({});
+    expect(columnsFor(defaultEnv(schema), 'nope')).toBeNull();
+  });
+});

package/src/query/env.ts

@@ -0,0 +1,56 @@
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import type { RefMode } from '../modules/ref-tables';
+import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor } from '../modules/ref-tables';
+import type { ClientState } from '../state/client-state';
+import {
+  DEGRADE_AFTER_FAILURES,
+  LIST_REF_POLL_BASE_MS,
+  MATERIALIZE_DEBOUNCE_MS,
+  OUTBOX_BATCH_SIZE,
+  PUSH_TIMEOUT_MS,
+} from '../kernel/constants';
+
+/**
+ * Static configuration the sagas read. Plain data, built once by the runtime
+ * from `Sp00kyConfig`; sagas never see the config object itself.
+ */
+export interface SagaEnv {
+  readonly schema: SchemaStructure;
+  readonly refMode: RefMode;
+  readonly anonLive: boolean;
+  readonly remoteTimeoutMs: number;
+  readonly pushTimeoutMs: number;
+  readonly outboxBatchSize: number;
+  readonly degradeAfter: number;
+  readonly materializeDebounceMs: number;
+  readonly pollBaseMs: number;
+  readonly defaultTtlMs: number;
+}
+
+export function defaultEnv(schema: SchemaStructure, over: Partial<SagaEnv> = {}): SagaEnv {
+  return {
+    schema,
+    refMode: DEFAULT_REF_MODE,
+    anonLive: false,
+    remoteTimeoutMs: 60_000,
+    pushTimeoutMs: PUSH_TIMEOUT_MS,
+    outboxBatchSize: OUTBOX_BATCH_SIZE,
+    degradeAfter: DEGRADE_AFTER_FAILURES,
+    materializeDebounceMs: MATERIALIZE_DEBOUNCE_MS,
+    pollBaseMs: LIST_REF_POLL_BASE_MS,
+    defaultTtlMs: 600_000,
+    ...over,
+  };
+}
+
+/** The `_00_list_ref` table this session reads: per-user, anonymous, or global. */
+export function listRefTable(env: SagaEnv, s: ClientState): string {
+  const userId = s.userId ?? (env.anonLive ? ANON_USER_ID : null);
+  return listRefTableFor(env.refMode, userId);
+}
+
+export function columnsFor(env: SagaEnv, table: string): Record<string, unknown> | null {
+  const found = env.schema.tables.find((t) => t.name === table);
+  if (found) return found.columns as Record<string, unknown>;
+  return table.startsWith('_00_') ? {} : null;
+}

package/src/query/fetch.saga.test.ts

@@ -0,0 +1,199 @@
+import { describe, expect, it } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { runPure } from '../testing/run-pure';
+import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
+import * as R from '../state/reducers';
+import { defaultEnv } from './env';
+import { fetchRows, landChunk } from './fetch.saga';
+import type { StatementResult } from '../kernel/effects';
+
+const env = defaultEnv({ tables: [{ name: 'thing', columns: { a: {} } }] } as any);
+const ok = (result: unknown): StatementResult => ({ status: 'OK', result });
+const live = { phase: 'live' } as const;
+const primed = (s: ReturnType<typeof buildState>) => ({ ...s, primed: true });
+
+describe('fetchRows', () => {
+  it('waits for the prime and returns on an empty plan (resetting the attempt counter)', async () => {
+    const s = R.patchSync({ fetchAttempts: 3 })(buildState());
+    const out = await runPure(fetchRows(env), { state: s, handlers: { 'state.wait': (_e, ctx) => void (ctx.state = { ...ctx.state, primed: true }) } });
+    expect(out.state.sync.fetchAttempts).toBe(0);
+    expect(out.log.filter((e) => e.kind === 'remote.query')).toHaveLength(0);
+  });
+
+  it('fetches the deduped set once, writes + ingests bodies, records versions, balances fetch depth', async () => {
+    const s = primed(
+      buildState(
+        [
+          buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 2], ['thing:2', 1]] }),
+          buildEntry({ def: { hash: 'b' }, lifecycle: live, remoteArray: [['thing:2', 1], ['thing:3', 1]] }),
+        ],
+        R.setVersions([['thing:3', 1], ['thing:1', 1]]),
+        R.outboxReplace([buildOutboxItem({ type: 'delete', recordId: 'thing:2' })])
+      )
+    );
+    const depths: number[] = [];
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      epoch: 7,
+      handlers: {
+        'remote.query': (e: any, ctx) => {
+          depths.push(ctx.state.queries.get('a')!.lifecycle.fetchDepth);
+          expect(e.sql).toBe('SELECT * FROM $ids');
+          expect(e.vars.ids).toEqual([new RecordId('thing', '1')]);
+          return [ok([{ id: new RecordId('thing', '1'), a: 'x', junk: true }])];
+        },
+        'local.execute': () => undefined,
+        'ssp.ingest': () => undefined,
+      },
+    });
+    expect(depths).toEqual([1]);
+    expect(out.state.queries.get('a')!.lifecycle.fetchDepth).toBe(0);
+    expect(out.state.versions.get('thing:1')).toBe(2);
+    const exec = out.log.find((e) => e.kind === 'local.execute') as any;
+    expect(exec.vars.content0).toEqual({ a: 'x', _00_rv: 2 });
+    expect(exec.epoch).toBe(7);
+    expect(out.log.findIndex((e) => e.kind === 'local.epoch')).toBeLessThan(out.log.findIndex((e) => e.kind === 'all'));
+    const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
+    expect(ingest.records).toEqual([{ table: 'thing', op: 'UPDATE', id: 'thing:1', record: { id: new RecordId('thing', '1'), a: 'x', _00_rv: 2 } }]);
+    expect(out.dispatched).toEqual([{ type: 'SyncOutcome', ok: true, error: undefined }]);
+    expect(out.state.dirty.has('a')).toBe(true);
+  });
+
+  it('a body the server does not return is remembered at the requested version so the plan converges', async () => {
+    const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1], ['other:2', 1]] })]));
+    let calls = 0;
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': () => {
+          calls++;
+          return [ok([{ id: new RecordId('other', '2'), z: 1 }, { nope: true }])];
+        },
+        'local.execute': () => undefined,
+        'ssp.ingest': () => undefined,
+      },
+    });
+    expect(calls).toBe(1);
+    expect(out.state.versions.get('thing:1')).toBe(1);
+    expect(out.state.versions.get('other:2')).toBe(1);
+    const extra = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': () => [ok([{ id: new RecordId('thing', '1') }, { id: new RecordId('thing', 'unrequested') }])],
+        'local.execute': () => undefined,
+        'ssp.ingest': (e: any) => {
+          expect(e.records[1]).toMatchObject({ id: 'thing:unrequested', record: { _00_rv: 0 } });
+        },
+      },
+    });
+    expect(extra.state.versions.has('thing:unrequested')).toBe(false);
+    const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
+    expect(ingest.records[0]).toMatchObject({ table: 'other', op: 'CREATE', record: { z: 1, _00_rv: 1 } });
+  });
+
+  it('failures back off: transport error, ERR statement, non-array result, local write error; ingest errors are logged only', async () => {
+    const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })]));
+    const expectBackoff = (out: Awaited<ReturnType<typeof runPure>>, attempts: number) => {
+      expect(out.state.sync.fetchAttempts).toBe(attempts);
+      expect(out.timers.get('fetch')).toEqual({ ms: 500 * 2 ** (attempts - 1), event: { type: 'FetchRows' } });
+      expect(out.state.queries.get('a')!.lifecycle.fetchDepth).toBe(0);
+      expect(out.dispatched.at(-1)).toMatchObject({ type: 'SyncOutcome', ok: false });
+    };
+    expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => { throw new Error('offline'); } } }), 1);
+    expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => [{ status: 'ERR', error: 'x' }] } }), 1);
+    expectBackoff(await runPure(fetchRows(env), { state: s, handlers: { 'remote.query': () => [ok('nope')] } }), 1);
+    expectBackoff(
+      await runPure(fetchRows(env), {
+        state: R.patchSync({ fetchAttempts: 1 })(s),
+        handlers: {
+          'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])],
+          'local.execute': () => {
+            throw new Error('stale epoch');
+          },
+        },
+      }),
+      2
+    );
+    const ingestFail = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])],
+        'local.execute': () => undefined,
+        'ssp.ingest': () => {
+          throw new Error('wasm');
+        },
+      },
+    });
+    expect(ingestFail.state.sync.fetchAttempts).toBe(0);
+    expect(ingestFail.emitted).toContainEqual(expect.objectContaining({ message: 'circuit ingest failed' }));
+  });
+
+  it('a leader relays landed bodies to followers', async () => {
+    const s = { ...primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })])), tabRole: 'leader' as const };
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      handlers: { 'remote.query': () => [ok([{ id: new RecordId('thing', '1') }])], 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
+    });
+    expect(out.emitted).toContainEqual({ type: 'tabs:broadcast', message: { type: 'ingest', records: [expect.objectContaining({ id: 'thing:1' })] } });
+  });
+
+  it('loops until the plan is empty', async () => {
+    const s = primed(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: live, remoteArray: [['thing:1', 1]] })]));
+    let calls = 0;
+    const out = await runPure(fetchRows(env), {
+      state: s,
+      handlers: {
+        'remote.query': (_e, ctx) => {
+          calls++;
+          if (calls === 1) ctx.state = R.commitMembership('a', [['thing:1', 1], ['thing:2', 1]], true)(ctx.state);
+          return [ok([])];
+        },
+      },
+    });
+    expect(calls).toBe(2);
+    expect(out.state.versions.get('thing:2')).toBe(1);
+  });
+});
+
+describe('landChunk', () => {
+  const typed = defaultEnv({
+    tables: [
+      {
+        name: 'broadcast',
+        columns: {
+          title: { type: 'string', optional: false },
+          owner: { type: 'string', recordId: true, optional: false },
+          club: { type: 'string', recordId: true, optional: true },
+          note: { type: 'string', optional: true },
+        },
+      },
+    ],
+  } as any);
+
+  // `UPDATE broadcast:x UNSET club` leaves no `club` key in the body the server
+  // returns, and a MERGE that does not name it kept the stale local club.
+  it('names each optional column the body lacks as undefined so the MERGE clears it', async () => {
+    const rid = new RecordId('broadcast', 'x');
+    const out = await runPure(landChunk(typed, ['broadcast:x'], [{ id: rid, title: 't', note: 'n', _00_extra: 1 }], new Map([['broadcast:x', 4]]), buildState(), 3), {
+      handlers: { 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
+    });
+    expect(out.result).toBe(true);
+    const exec = out.log.find((e) => e.kind === 'local.execute') as any;
+    expect(exec.query.sql).toContain('UPSERT ONLY $id0 MERGE $content0');
+    expect(exec.vars.content0).toStrictEqual({ club: undefined, title: 't', note: 'n', _00_extra: 1, _00_rv: 4 });
+    expect(Object.prototype.hasOwnProperty.call(exec.vars.content0, 'club')).toBe(true);
+    // A required column (`owner`) is never cleared: its absence means hidden, not unset.
+    expect(Object.prototype.hasOwnProperty.call(exec.vars.content0, 'owner')).toBe(false);
+    // The circuit record carries no clear: CREATE/UPDATE replace the circuit row.
+    const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
+    expect(ingest.records[0].record).toStrictEqual({ id: rid, title: 't', note: 'n', _00_extra: 1, _00_rv: 4 });
+  });
+
+  it('names nothing for a table without a schema', async () => {
+    const out = await runPure(landChunk(typed, ['other:1'], [{ id: new RecordId('other', '1'), z: 1 }], new Map([['other:1', 1]]), buildState(), 0), {
+      handlers: { 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
+    });
+    const exec = out.log.find((e) => e.kind === 'local.execute') as any;
+    expect(exec.vars.content0).toStrictEqual({ z: 1, _00_rv: 1 });
+  });
+});

package/src/query/fetch.saga.ts

@@ -0,0 +1,120 @@
+import type { RecordId } from 'surrealdb';
+import type { Saga } from '../kernel/saga';
+import type { Settled, StatementResult } from '../kernel/effects';
+import { fx } from '../kernel/effects';
+import { backoffMs } from '../kernel/constants';
+import type { IngestRecord } from '../services/stream-processor/index';
+import type { ClientState } from '../state/client-state';
+import * as R from '../state/reducers';
+import { planFetch } from '../state/selectors';
+import { encodeRecordId, parseRecordIdString } from '../utils/index';
+import { absentOptionalColumns, cleanRecord } from '../utils/parser';
+import type { SagaEnv } from './env';
+import * as sql from './sql';
+
+/**
+ * The unified row fetcher (the poke analogue). Runs on the serial `fetch`
+ * lane: computes the cross-query set of bodies missing or stale locally,
+ * pulls them in parallel chunks, writes + ingests them, and loops until the
+ * plan is empty. A row shared by N queries is fetched once. Bodies are never
+ * deleted here; ids that left membership simply stop rendering.
+ */
+export function* fetchRows(env: SagaEnv): Saga<void> {
+  yield fx.state.wait((s) => s.primed);
+  for (;;) {
+    const state = (yield fx.state.read((s) => s)) as ClientState;
+    const plan = planFetch(state);
+    if (plan.chunks.length === 0) {
+      if (state.sync.fetchAttempts !== 0) yield fx.state.update(R.patchSync({ fetchAttempts: 0 }));
+      return;
+    }
+    // The engine's epoch, not a mirrored counter: it also moves on transport
+    // failover, and a stale value makes the store drop every body write.
+    const epoch = (yield fx.local.epoch()) as number;
+    yield fx.state.update(R.compose(...plan.hashes.map((h) => R.applyLifecycle(h, { type: 'fetch-begin' }))));
+    let failed = 0;
+    try {
+      const results = (yield fx.all(
+        plan.chunks.map((ids) => fx.remote.query(sql.bodySelect(), { ids: ids.map(parseRecordIdString) }, env.remoteTimeoutMs))
+      )) as Settled[];
+      for (let i = 0; i < results.length; i++) {
+        const r = results[i];
+        const first = r.ok ? (r.value as StatementResult[])[0] : undefined;
+        if (!r.ok || !first || first.status === 'ERR' || !Array.isArray(first.result)) {
+          failed++;
+          continue;
+        }
+        const landed = yield* landChunk(env, plan.chunks[i], first.result as Array<Record<string, unknown>>, plan.versions, state, epoch);
+        if (!landed) failed++;
+      }
+    } finally {
+      yield fx.state.update(R.compose(...plan.hashes.map((h) => R.applyLifecycle(h, { type: 'fetch-end' }))));
+    }
+    yield fx.dispatch({ type: 'SyncOutcome', ok: failed === 0, error: failed > 0 ? 'body fetch failed' : undefined });
+    if (failed > 0) {
+      const attempt = (yield fx.state.read((s) => s.sync.fetchAttempts)) as number;
+      yield fx.state.update(R.patchSync({ fetchAttempts: attempt + 1 }));
+      yield fx.timer.set('fetch', backoffMs(attempt), { type: 'FetchRows' });
+      return;
+    }
+  }
+}
+
+/**
+ * Write one chunk's bodies to the store and circuit; record their versions.
+ *
+ * Shared with the LIVE path: a body that arrives on a notification is landed
+ * through here so it is indistinguishable from a fetched one, which is what
+ * makes `planFetch` stop asking for it.
+ */
+export function* landChunk(
+  env: SagaEnv,
+  requested: string[],
+  rows: Array<Record<string, unknown>>,
+  versions: ReadonlyMap<string, number>,
+  state: ClientState,
+  epoch: number
+): Saga<boolean> {
+  const bodies: Array<{ id: unknown; content: Record<string, unknown> }> = [];
+  const ingest: IngestRecord[] = [];
+  const seen = new Set<string>();
+  for (const row of rows) {
+    const rid = row?.id as RecordId<string> | undefined;
+    if (!rid || typeof rid !== 'object') continue;
+    const id = encodeRecordId(rid);
+    const table = String(rid.table);
+    const version = versions.get(id) ?? 0;
+    const columns = env.schema.tables.find((t) => t.name === table)?.columns;
+    const cleaned = columns ? cleanRecord(columns, row) : row;
+    const { id: _id, ...content } = cleaned;
+    // The row is the server's whole body, so an optional column it lacks was
+    // cleared upstream: name it (as `undefined`) so the MERGE removes the local
+    // copy instead of keeping it. `_00_*` local bookkeeping is never named, so
+    // MERGE (not REPLACE) still preserves it. The circuit record below needs no
+    // such key: CREATE/UPDATE replace the circuit's row outright.
+    bodies.push({ id: rid, content: { ...absentOptionalColumns(columns, row), ...content, _00_rv: version } });
+    ingest.push({ table, op: state.versions.has(id) ? 'UPDATE' : 'CREATE', id, record: { ...cleaned, _00_rv: version } });
+    seen.add(id);
+  }
+  if (bodies.length > 0) {
+    try {
+      const tx = sql.upsertBodiesTx(bodies);
+      yield fx.local.execute(tx.query, tx.vars, epoch);
+    } catch (error) {
+      yield fx.emit({ type: 'log', level: 'warn', message: 'body write failed', data: { error } });
+      return false;
+    }
+    try {
+      yield fx.ssp.ingest(ingest);
+    } catch (error) {
+      yield fx.emit({ type: 'log', level: 'warn', message: 'circuit ingest failed', data: { error } });
+    }
+    if (state.tabRole === 'leader') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'ingest', records: ingest } });
+  }
+  // Ids the server did not return (deleted or unreadable upstream) count as
+  // known at the requested version, so the plan stops asking for them; their
+  // absence from the store is what the render shows.
+  const entries: Array<readonly [string, number]> = requested.map((id) => [id, versions.get(id) as number]);
+  yield fx.state.update(R.setVersions(entries));
+  return true;
+}

package/src/query/hash.test.ts

@@ -0,0 +1,14 @@
+import { describe, expect, it } from 'vitest';
+import { queryHashInput, viewKeyInput } from './hash';
+
+describe('hash inputs (golden against the previous DataModule formulas)', () => {
+  const input = { surql: 'SELECT * FROM t WHERE a = $a', params: { a: 1, b: 'x' } };
+  it('salted key is JSON.stringify({surql, params, sessionId}) in that key order', () => {
+    expect(queryHashInput(input, 'sess')).toBe(JSON.stringify({ ...input, sessionId: 'sess' }));
+    expect(queryHashInput(input, null)).toBe(JSON.stringify({ ...input, sessionId: null }));
+  });
+  it('view key is the same input minus the salt', () => {
+    expect(viewKeyInput(input)).toBe(JSON.stringify(input));
+    expect(viewKeyInput(input)).not.toBe(queryHashInput(input, 'sess'));
+  });
+});

package/src/query/hash.ts

@@ -0,0 +1,18 @@
+/**
+ * Hash inputs for the two query keys. Kept byte-identical to the previous
+ * implementation (`DataModule.calculateHash` / `calculateMembershipKey`):
+ * the salted key names the remote `_00_query` row, the unsalted one names the
+ * durable `_00_view` row, and both are `sha256(JSON.stringify(input))`.
+ */
+export interface QueryKeyInput {
+  surql: string;
+  params: Record<string, unknown>;
+}
+
+/** Session-salted: two tabs of one user must not share a remote view. */
+export const queryHashInput = (input: QueryKeyInput, sessionId: string | null): string =>
+  JSON.stringify({ surql: input.surql, params: input.params, sessionId });
+
+/** Session-independent: survives reload and bucket switch. */
+export const viewKeyInput = (input: QueryKeyInput): string =>
+  JSON.stringify({ surql: input.surql, params: input.params });