create-walle 0.10.2 → 0.10.3

package/template/wall-e/brain/brain-join.js

@@ -0,0 +1,859 @@
+'use strict';
+
+// WP-4 (§5 D-4/D-6/D-7, §6.7) — the JOIN half of shared memory: the home produces ONE
+// consistent snapshot of its brain; the member imports only the four replicated tables (plus
+// their clock rows), never host-local state (D-7).
+//
+//   createJoinSnapshot({ db, brainDir, now, fresh })  -> { ok, joinId, path, bytes, sha256, vector, createdAt }
+//   startJoinImport(db, { joinId, path, sha256, homeInstanceId }) -> { ok, state, resumed? }
+//     (a same-joinId call whose record is a resumable failure re-enters at the stored cursor and
+//      answers `resumed: true` — R-1..R-5)
+//   joinImportStep(db, { limit, now, _beforePage }) -> { state, table, done, retry? }  (one bounded page)
+//   joinStatus(db)                             -> the status route's shape
+//
+// WP-3 (§5 R-11/R-12): a step is bounded by TIME, not only by rows. `_importPage` walks its step in
+// JOIN_CHUNK_ROWS-row chunks and stops after the chunk during which the elapsed time inside the
+// transaction reached JOIN_STEP_TIME_BUDGET_MS, so one hold stays under the brain write lock's
+// WRITE_LOCK_HOLD_BUDGET_MS. A page the lock aborts (hold-budget/busy) is transient: the cursor is
+// unchanged, the record stays 'importing', and the same page runs next tick (bounded retries).
+//
+// The snapshot's own brain_row_clock IS the consistency point (P-1): the vector is read FROM
+// THE SNAPSHOT FILE (readonly), never from the live DB, so a change the home writes during
+// transfer arrives later through the normal v2 feed (P-2). The import runs on the member's
+// OWNER connection with the connection-scoped TEMP sentinel on, so nothing it writes is
+// captured as a local edit and no host-local table (kv_store, loop_checkpoints, owner, …) is
+// ever read from the snapshot.
+//
+// The cursor and the imported/rekeyed counters live in kv_store under 'brain_sync.join' — one
+// host-local record, self-describing, resumable after a crash. This module owns NO DDL: the
+// sentinel's CREATE/DROP TEMP TABLE text is brain/schema.js createBrainSyncSentinel /
+// dropBrainSyncSentinel.
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const Database = require('better-sqlite3');
+
+const { createBrainSyncSentinel, dropBrainSyncSentinel } = require('./schema');
+const { clockSeedState, localBrainInstanceId } = require('./brain-sync-store');
+const { markMembership } = require('./brain-sync-v2');
+const { WRITE_LOCK_HOLD_BUDGET_MS } = require('../shared/sqlite-write-lock');
+const { strFlag } = require('../lib/env-flags');
+const errorSink = require('../lib/error-sink');
+
+const JOIN_KEY = 'brain_sync.join';
+const JOIN_DIR_NAME = 'sync-join';
+const INBOX_DIR_NAME = 'sync-inbox';
+// The snapshot's four tables, in import order (matches the clock-seed / backfill walk).
+const JOIN_TABLE_ORDER = Object.freeze(['entities', 'people', 'knowledge', 'memories']);
+// The import CURSOR order (WP-0 §5 R-2): the four tables plus a `knowledge_links` pass between
+// knowledge and memories. `knowledge_links` is a cursor POSITION, not a table — it restores the
+// superseded_by pointers the knowledge pages deliberately insert as NULL (R-1).
+const JOIN_STEP_ORDER = Object.freeze(['entities', 'people', 'knowledge', 'knowledge_links', 'memories']);
+const JOIN_STEP_DEFAULT = 5000;
+// R-11 (§5): rows fetched, re-keyed and inserted per chunk. Small enough that one chunk's work is a
+// fraction of the lock budget under any load, large enough to keep the per-chunk statement count low.
+const JOIN_CHUNK_ROWS = 500;
+// R-11: the time a single import transaction may hold the brain write lock. Derived from the lock's
+// own budget, never re-typed: a hold stays ≤ budget/3 plus one chunk, so the lock's over-budget
+// abort (which rolls the page back and fails the join) is unreachable in normal operation.
+const JOIN_STEP_TIME_BUDGET_MS = Math.floor(WRITE_LOCK_HOLD_BUDGET_MS / 3);
+// R-12: consecutive transient (hold-budget / busy) aborts of the SAME cursor before the join fails.
+const JOIN_MAX_TRANSIENT_RETRIES = 5;
+// R-3 (§6.4): how many times a FAILED join may be resumed under the SAME joinId before the caller
+// has to give up on the snapshot (WP-2 then asks the home for a fresh one). A transient failure
+// costs one backoff instead of a multi-GB re-transfer, and the cap keeps a permanently broken
+// cursor from looping forever.
+const JOIN_MAX_RESUMES = 3;
+// R-2 (§6.4): the two failures a resume can never fix — the snapshot's bytes are wrong, or the
+// file is gone. Both are the exact strings joinImportStep's verifying step stores via _markFailed.
+const NON_RESUMABLE_ERRORS = Object.freeze(new Set(['sha256_mismatch', 'snapshot_missing']));
+// R-12 (§5): the two codes a rolled-back write hold raises. Both leave the committed cursor intact,
+// so the page is safe to retry; anything else is a real failure.
+const TRANSIENT_LOCK_CODES = Object.freeze(new Set(['SQLITE_WRITE_LOCK_HOLD_BUDGET', 'SQLITE_BUSY']));
+const REUSE_WINDOW_MS = 6 * 60 * 60 * 1000; // a second call within 6 h reuses the newest snapshot
+const RETENTION_MS = 48 * 60 * 60 * 1000; // sync-join/*.db older than this is swept on every call
+const MAX_ERROR_LEN = 500;
+const TERMINAL_STATES = Object.freeze(new Set(['done', 'failed']));
+const EMPTY_IMPORTED = Object.freeze({ memories: 0, knowledge: 0, entities: 0, people: 0 });
+// D-7 / §6.7: an imported memory is visible but not re-extracted here — the home's extraction
+// produced its knowledge rows, and this host's read stamps/retry counters describe ITS work.
+// The inserted rows get these values (the SELECT below), never the snapshot's.
+const MEMORY_INSERT_OVERRIDES = Object.freeze({
+  extraction_status: "'replicated'",
+  extraction_attempts: '0',
+  last_accessed: 'NULL',
+  access_count: '0',
+  embed_failed_count: '0',
+});
+// R-1 (§5): a snapshot knowledge row's superseded_by points FORWARD — at a row with a LARGER
+// rowid — because a supersede chain is written oldest-first (34,244 of the live hub's 312,638
+// knowledge rows, every one of them forward, none dangling). The knowledge page therefore inserts
+// the pointer as NULL; the later `knowledge_links` step restores it once every row is present.
+// FKs stay ON — the two passes are each individually valid (§3, P-1).
+const KNOWLEDGE_INSERT_OVERRIDES = Object.freeze({ superseded_by: 'NULL' });
+// Per-table column overrides applied by _insertSnapshotPage.
+const INSERT_OVERRIDES = Object.freeze({
+  memories: MEMORY_INSERT_OVERRIDES,
+  knowledge: KNOWLEDGE_INSERT_OVERRIDES,
+});
+
+function _string(value, max = 4000) {
+  if (typeof value === 'string') return value.slice(0, max);
+  return value == null ? '' : String(value).slice(0, max);
+}
+
+function _coded(message, code) {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+// Wall-E's own data dir — lib/env-flags.js is the one sanctioned env reader (flag-read lint).
+// Mirrors brain.js's DATA_DIR (os.homedir(), not process.env.HOME, which is unset under launchd).
+function _dataDir() {
+  const configured = _string(strFlag('WALL_E_DATA_DIR', ''), 4096).trim();
+  if (configured) return path.resolve(configured);
+  return path.join(os.homedir(), '.walle', 'data');
+}
+
+function _resolveBrainDir(brainDir) {
+  const given = _string(brainDir, 4096).trim();
+  if (given) return path.resolve(given);
+  return path.dirname(require('../brain').getDbPath());
+}
+
+function _resolveDataDir(dataDir) {
+  const given = _string(dataDir, 4096).trim();
+  return given ? path.resolve(given) : _dataDir();
+}
+
+// §6.7/the wire contract: the member's join file must live in the CTM data dir's sync-inbox/ — the
+// daemon passes that root down as `dataDir` (R-6/I-5: the owner worker's env whitelist cannot see
+// CTM_DATA_DIR) — or beside the brain file's own sync-inbox/. Resolve both sides before comparing
+// (a symlinked temp path must not pass by spelling alone).
+function isInsideJoinInbox(filePath, { brainDir, dataDir } = {}) {
+  const resolved = path.resolve(_string(filePath, 4096));
+  const dirs = [
+    path.join(_resolveBrainDir(brainDir), INBOX_DIR_NAME),
+    path.join(_resolveDataDir(dataDir), INBOX_DIR_NAME),
+  ];
+  return dirs.some((dir) => resolved !== dir && resolved.startsWith(`${dir}${path.sep}`));
+}
+
+function _sha256File(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash('sha256');
+    const stream = fs.createReadStream(filePath);
+    stream.on('error', reject);
+    stream.on('data', (chunk) => hash.update(chunk));
+    stream.on('end', () => resolve(hash.digest('hex')));
+  });
+}
+
+// The snapshot's vector, read FROM THE SNAPSHOT FILE (P-1) — never from the live DB. origin ''
+// ("this host" before v2 identity, or a v1 row) is not addressable and is excluded (§6.4).
+function vectorFromSnapshotFile(filePath) {
+  const d = new Database(filePath, { readonly: true, fileMustExist: true });
+  try {
+    const vector = {};
+    const rows = d.prepare(`
+      SELECT origin_instance_id AS origin, MAX(origin_seq) AS max_seq
+        FROM brain_row_clock
+       WHERE origin_instance_id <> ''
+       GROUP BY origin_instance_id
+       ORDER BY origin_instance_id ASC
+    `).all();
+    for (const row of rows) vector[String(row.origin)] = Number(row.max_seq) || 0;
+    return vector;
+  } finally {
+    try { d.close(); } catch { /* expected: the snapshot handle is only read from; a close failure must not mask the vector */ }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Read/write of the one host-local join record (kv_store; no capture trigger exists on kv_store)
+// ---------------------------------------------------------------------------
+
+function readJoinRecord(db) {
+  const row = db.prepare('SELECT value FROM kv_store WHERE key = ?').get(JOIN_KEY);
+  if (!row || !row.value) return null;
+  try {
+    const parsed = JSON.parse(row.value);
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
+  } catch {
+    return null; // HANDLE: a corrupt join record reads as "no join" — the next start overwrites it
+  }
+}
+
+function _writeJoinRecord(db, record) {
+  db.prepare(`
+    INSERT INTO kv_store (key, value, updated_at) VALUES (?, ?, datetime('now'))
+    ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
+  `).run(JOIN_KEY, JSON.stringify(record));
+}
+
+function _normalizeRecord(record) {
+  const imported = {};
+  for (const table of Object.keys(EMPTY_IMPORTED)) imported[table] = Math.max(0, Number(record.imported?.[table]) || 0);
+  // `knowledge_links` is a valid cursor position alongside the four tables (R-2); anything else
+  // normalizes to null, which _importPage reads as "start at the first step".
+  const table = _string(record.table, 40);
+  return {
+    joinId: _string(record.joinId, 200) || null,
+    path: _string(record.path, 4096) || null,
+    sha256: _string(record.sha256, 128) || null,
+    homeInstanceId: _string(record.homeInstanceId, 160) || null,
+    state: _string(record.state, 40) || 'none',
+    table: JOIN_STEP_ORDER.includes(table) ? table : null,
+    after: Math.max(0, Math.floor(Number(record.after) || 0)),
+    imported,
+    rekeyed: Math.max(0, Number(record.rekeyed) || 0),
+    linksDropped: Math.max(0, Number(record.linksDropped) || 0),
+    error: record.error ? _string(record.error, MAX_ERROR_LEN) : null,
+    startedAt: _string(record.startedAt, 64) || null,
+    finishedAt: _string(record.finishedAt, 64) || null,
+    // R-12: consecutive transient lock aborts of the SAME (table, after) cursor, the cursor they
+    // happened at, and the last message — so a load spike is visible in the record, not just a tick.
+    transientRetries: Math.max(0, Math.floor(Number(record.transientRetries) || 0)),
+    lastTransientAt: record.lastTransientAt ? _string(record.lastTransientAt, 120) : null,
+    lastTransientError: record.lastTransientError ? _string(record.lastTransientError, MAX_ERROR_LEN) : null,
+    // R-3 (§6.4): how many times this joinId has been resumed after a failure, and when the last
+    // resume happened — both survive in the record so the cap and the audit trail are durable.
+    resumes: Math.max(0, Math.floor(Number(record.resumes) || 0)),
+    resumedAt: record.resumedAt ? _string(record.resumedAt, 64) : null,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Home: create (or reuse) the join snapshot
+// ---------------------------------------------------------------------------
+
+function _joinFileCreatedAtMs(joinId) {
+  const match = /-(\d+)$/.exec(String(joinId || ''));
+  return match ? Number(match[1]) : 0;
+}
+
+function _pruneJoinDir(dir, now) {
+  let names;
+  try { names = fs.readdirSync(dir); } catch { return 0; } // HANDLE: the dir is created by the caller; an unreadable dir prunes nothing
+  let removed = 0;
+  for (const name of names) {
+    if (!name.endsWith('.db')) continue;
+    const full = path.join(dir, name);
+    try {
+      if (now - fs.statSync(full).mtimeMs > RETENTION_MS) { fs.unlinkSync(full); removed += 1; }
+    } catch { /* expected: a file another process is sweeping at the same moment is already gone */ }
+  }
+  return removed;
+}
+
+// Newest snapshot THIS instance produced (its name carries the local instance id) whose mtime is
+// inside the reuse window. Reuse is what makes a repeated call within 6 h cheap and idempotent.
+function _reusableJoinFile(dir, localId, now) {
+  if (!localId) return null;
+  let names;
+  try { names = fs.readdirSync(dir); } catch { return null; } // HANDLE: no dir yet — nothing to reuse
+  const prefix = `brain-${localId}-`;
+  let best = null;
+  for (const name of names) {
+    if (!name.startsWith(prefix) || !name.endsWith('.db')) continue;
+    const joinId = name.slice('brain-'.length, name.length - '.db'.length);
+    const full = path.join(dir, name);
+    let mtimeMs;
+    try { mtimeMs = fs.statSync(full).mtimeMs; } catch { continue; } // HANDLE: a file that vanished between readdir and stat is not reusable
+    if (now - mtimeMs > REUSE_WINDOW_MS) continue;
+    if (!best || mtimeMs > best.mtimeMs) best = { path: full, joinId, mtimeMs };
+  }
+  if (!best) return null;
+  return { path: best.path, joinId: best.joinId, createdAt: new Date(_joinFileCreatedAtMs(best.joinId) || best.mtimeMs).toISOString() };
+}
+
+// Concurrent calls share ONE in-flight build (§6.7): the second caller awaits the same promise.
+let _inFlightSnapshot = null;
+
+async function createJoinSnapshot({ db, brainDir, now, fresh = false } = {}) {
+  const handle = db || require('../brain').getDb();
+  const seed = clockSeedState(handle);
+  if (!seed.done) throw _coded('clock_seed_pending', 'JOIN_CLOCK_SEED_PENDING');
+  if (_inFlightSnapshot) return _inFlightSnapshot;
+  const build = _buildJoinSnapshot({
+    db: handle,
+    brainDir: _resolveBrainDir(brainDir),
+    now: Number(now) || Date.now(),
+    fresh: fresh === true,
+  });
+  _inFlightSnapshot = build.finally(() => { _inFlightSnapshot = null; });
+  return _inFlightSnapshot;
+}
+
+async function _buildJoinSnapshot({ db, brainDir, now, fresh = false }) {
+  const dir = path.join(brainDir, JOIN_DIR_NAME);
+  fs.mkdirSync(dir, { recursive: true });
+  _pruneJoinDir(dir, now);
+  const localId = localBrainInstanceId(db);
+
+  // R-4/R-5 (§6.4): the caller whose member could not resume asks for a NEW snapshot, because the
+  // one inside the reuse window is the one the member just rejected. An in-flight build is still
+  // shared (`_inFlightSnapshot`) — the build it awaits is new by construction.
+  const reused = fresh ? null : _reusableJoinFile(dir, localId, now);
+  let joinId;
+  let filePath;
+  let createdAt;
+  if (reused) {
+    ({ joinId, path: filePath, createdAt } = reused);
+  } else {
+    createdAt = new Date(now).toISOString();
+    joinId = `${localId}-${now}`;
+    filePath = path.join(dir, `brain-${joinId}.db`);
+    await require('../brain').createConsistentSnapshot(filePath);
+  }
+
+  const bytes = fs.statSync(filePath).size;
+  const sha256 = await _sha256File(filePath);
+  const vector = vectorFromSnapshotFile(filePath);
+  return { ok: true, joinId, path: filePath, bytes, sha256, vector, createdAt };
+}
+
+// ---------------------------------------------------------------------------
+// Member: record, then one bounded import page per call
+// ---------------------------------------------------------------------------
+
+// The inbox file names the member may reclaim: CTM's snapshot name and its two in-progress
+// spellings (`join-<joinId>.db`, `join-<joinId>.part`, `join-<joinId>.part.json`).
+const JOIN_INBOX_NAME_RE = /^join-.*\.(?:db|part|part\.json)$/;
+
+// R-8 (§5/D-5): reclaim the superseded inbox files. When startJoinImport ACCEPTS a new join file
+// (`keepPath`), every other regular `join-*.{db,part,part.json}` in `keepPath`'s OWN directory
+// whose mtime is strictly older is deleted. `dir` is the accepted file's directory, already
+// validated by isInsideJoinInbox, so nothing outside an inbox is ever considered; the accepted
+// file itself is never touched. A readdir/stat fault is HANDLE — reclamation must never be a
+// reason to refuse a join. Returns the number of files deleted.
+function _sweepJoinInbox(dir, keepPath) {
+  let names;
+  try {
+    names = fs.readdirSync(dir);
+  } catch {
+    errorSink.expected('brain-join', 'join-inbox-sweep');
+    return 0;
+  }
+  let keepMtimeMs;
+  try {
+    keepMtimeMs = fs.statSync(keepPath).mtimeMs;
+  } catch {
+    // Without the accepted file's mtime there is no honest "older than" — the sweep reclaims
+    // nothing; the verifying step reports the file's absence in its own terms.
+    errorSink.expected('brain-join', 'join-inbox-sweep');
+    return 0;
+  }
+  const keep = path.resolve(keepPath);
+  let deleted = 0;
+  for (const name of names) {
+    if (!JOIN_INBOX_NAME_RE.test(name)) continue;
+    const full = path.join(dir, name);
+    if (path.resolve(full) === keep) continue;
+    let stat;
+    try {
+      stat = fs.statSync(full);
+      if (!stat.isFile() || stat.mtimeMs >= keepMtimeMs) continue;
+    } catch {
+      errorSink.expected('brain-join', 'join-inbox-sweep');
+      continue;
+    }
+    _deleteJoinFile(full);
+    deleted += 1;
+  }
+  return deleted;
+}
+
+// R-1..R-5 (§6.4): a same-joinId call may only RESUME a failed record. `normalized` is the stored
+// record after normalization and `{ filePath, sha256 }` the incoming call (sha256 already trimmed
+// and lowercased; '' means the caller did not assert one). The snapshot file must still be there —
+// a record whose file vanished has nothing to resume from, and the caller gets the stored `failed`.
+function _isResumableRecord(normalized, { filePath, sha256 } = {}) {
+  if (normalized.state !== 'failed') return false;
+  if (!normalized.path || path.resolve(normalized.path) !== path.resolve(filePath)) return false;
+  if (sha256 && normalized.sha256 !== sha256) return false;
+  if (NON_RESUMABLE_ERRORS.has(String(normalized.error || ''))) return false;
+  if (normalized.resumes >= JOIN_MAX_RESUMES) return false;
+  return fs.existsSync(normalized.path);
+}
+
+function startJoinImport(db, options = {}) {
+  const joinId = _string(options.joinId, 200).trim();
+  if (!joinId) throw _coded('join_id_required', 'JOIN_ID_REQUIRED');
+  const filePath = _string(options.path, 4096).trim();
+  if (!filePath) throw _coded('join_path_required', 'JOIN_PATH_INVALID');
+  const sha256 = _string(options.sha256, 128).trim().toLowerCase();
+  const homeInstanceId = _string(options.homeInstanceId, 160).trim();
+
+  const current = readJoinRecord(db);
+  // Idempotent: the same joinId already recorded is answered with its current state — EXCEPT when
+  // the record is a resumable failure (R-1..R-5, §6.4). Every import page is idempotent and commits
+  // its cursor inside the same IMMEDIATE transaction (`_importPage`), so re-entering at the stored
+  // (table, after) is safe and costs one hub backoff instead of a fresh multi-GB snapshot.
+  if (current && current.joinId === joinId) {
+    const normalized = _normalizeRecord(current);
+    if (_isResumableRecord(normalized, { filePath, sha256 })) {
+      const resumed = _normalizeRecord({
+        ...normalized,
+        state: normalized.table ? 'importing' : 'verifying',
+        error: null,
+        finishedAt: null,
+        transientRetries: 0,
+        lastTransientAt: null,
+        lastTransientError: null,
+        resumes: normalized.resumes + 1,
+        resumedAt: new Date().toISOString(),
+      });
+      _writeJoinRecord(db, resumed);
+      // A resume is NOT an acceptance: the inbox sweep (R-8) runs only when a NEW file is taken.
+      return { ok: true, state: resumed.state, joinId, resumed: true };
+    }
+    return { ok: true, state: normalized.state, joinId };
+  }
+  // A DIFFERENT joinId while one is still in flight is refused; a terminal record is replaced.
+  if (current && current.joinId && !TERMINAL_STATES.has(current.state)) {
+    const error = _coded('join_in_progress', 'JOIN_IN_PROGRESS');
+    error.joinId = current.joinId;
+    throw error;
+  }
+  if (!isInsideJoinInbox(filePath, { brainDir: options.brainDir, dataDir: options.dataDir })) {
+    throw _coded('join_path_invalid', 'JOIN_PATH_INVALID');
+  }
+
+  // R-7 (§5): replacing a TERMINAL record (done/failed) under a DIFFERENT joinId is the hub
+  // retrying after a failure, so the replaced record's stranded snapshot is removed — but only
+  // when it sits inside a join inbox and is not the file being started now. A path outside the
+  // inbox is never deleted (the member's only sweep is this one). A non-terminal record never
+  // reaches here: it already threw join_in_progress above.
+  if (current && current.joinId && TERMINAL_STATES.has(current.state)) {
+    const replaced = _string(current.path, 4096).trim();
+    if (replaced) {
+      const resolvedReplaced = path.resolve(replaced);
+      if (resolvedReplaced !== path.resolve(filePath)
+        && isInsideJoinInbox(resolvedReplaced, { brainDir: options.brainDir, dataDir: options.dataDir })) {
+        _deleteJoinFile(resolvedReplaced);
+      }
+    }
+  }
+
+  // R-8 (§5/D-5, I-4): this is the ONE moment the member knows which inbox file is current —
+  // it is about to write a new record — so every OLDER join file beside it is reclaimed here. A
+  // refused or superseded transfer used to strand its multi-GB snapshot in the inbox forever;
+  // the same-joinId early return above and every refusal path deliberately do not sweep.
+  _sweepJoinInbox(path.dirname(path.resolve(filePath)), filePath);
+
+  const record = _normalizeRecord({
+    joinId,
+    path: path.resolve(filePath),
+    sha256,
+    homeInstanceId,
+    state: 'verifying',
+    table: null,
+    after: 0,
+    imported: EMPTY_IMPORTED,
+    rekeyed: 0,
+    error: null,
+    startedAt: new Date().toISOString(),
+    finishedAt: null,
+  });
+  _writeJoinRecord(db, record);
+  return { ok: true, state: record.state, joinId };
+}
+
+// The columns of `qualifiedTable` ("main"."x" / "snap"."x") as SQLite sees them on this connection.
+function _columns(db, qualifiedTable) {
+  try {
+    return db.prepare(`SELECT * FROM ${qualifiedTable} LIMIT 0`).columns().map((column) => column.name);
+  } catch {
+    return []; // HANDLE: a missing table (or a schema a migration has not yet reached) has no columns to copy
+  }
+}
+
+// The snapshot's brain_row_clock rows for a page's ids — INSERT OR REPLACE: the snapshot clock is
+// the truth for those rows (§6.7). Runs with the sentinel on, so it stamps no local origin.
+function _copySnapshotClock(db, table, ids) {
+  if (!ids.length) return 0;
+  const placeholders = ids.map(() => '?').join(', ');
+  const rows = db.prepare(`
+    SELECT table_name, row_id, updated_at, origin_instance_id, origin_seq, deleted
+      FROM snap.brain_row_clock
+     WHERE table_name = ? AND row_id IN (${placeholders})
+  `).all(table, ...ids);
+  const upsert = db.prepare(`
+    INSERT OR REPLACE INTO brain_row_clock (table_name, row_id, updated_at, origin_instance_id, origin_seq, deleted)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+  let copied = 0;
+  for (const row of rows) {
+    upsert.run(
+      String(row.table_name), String(row.row_id), _string(row.updated_at, 64),
+      _string(row.origin_instance_id, 160), Number(row.origin_seq) || 0, Number(row.deleted) ? 1 : 0,
+    );
+    copied += 1;
+  }
+  return copied;
+}
+
+// D-4/§6.7 re-keying: a snapshot memory whose (source, source_id) is already here under a
+// DIFFERENT id replaces it — every local knowledge.source_memory_ids array that names the local
+// id is rewritten to the snapshot id, then the local row and its clock row are deleted (with the
+// sentinel on, so the delete leaves no tombstone). Returns how many rows were re-keyed.
+// WP-0 (§5 R-1): the rewrite is ONE batched knowledge scan for the whole page, not one per
+// collision — the live hub page held 2,792 collisions and 105.5 s of the write lock, while one
+// batched scan of the same rows measures 77 ms. Deletes keep their exact per-collision order.
+function _rekeyCollidingMemories(db, ids) {
+  if (!ids.length) return 0;
+  const placeholders = ids.map(() => '?').join(', ');
+  const snaps = db.prepare('SELECT id, source, source_id FROM snap.memories WHERE id IN (' + placeholders + ')').all(...ids);
+  const findLocal = db.prepare('SELECT id FROM memories WHERE source = ? AND source_id = ? AND id <> ? LIMIT 1');
+  const deleteMemory = db.prepare('DELETE FROM memories WHERE id = ?');
+  const deleteClock = db.prepare('DELETE FROM brain_row_clock WHERE table_name = ? AND row_id = ?');
+  const mapping = new Map(); // local id -> snapshot id, both String; insertion order == `snaps` order
+  let rekeyed = 0;
+  for (const snap of snaps) {
+    if (snap.source_id == null || snap.source_id === '') continue; // the identity pair needs a non-null source_id
+    const local = findLocal.get(_string(snap.source, 200), _string(snap.source_id, 400), String(snap.id));
+    if (!local) continue;
+    mapping.set(String(local.id), String(snap.id));
+    deleteMemory.run(local.id);
+    deleteClock.run('memories', local.id);
+    rekeyed += 1;
+  }
+  if (mapping.size > 0) {
+    let chained = false;
+    for (const target of mapping.values()) if (mapping.has(target)) { chained = true; break; }
+    // K-1 (§11): the batch maps each element ONCE, but the old code composed a chain — a local id
+    // that is also a later collision's snapshot id (L1->S1, then L2=S1->S2 left S2). A chain needs
+    // one id to name different content on two machines, which per-insert id generation prevents
+    // (never observed on the live snapshot); when one appears, replay the pairs one at a time in
+    // `snaps` order to reproduce the composed result instead of the single-hop one.
+    if (chained) {
+      for (const [fromId, toId] of mapping) _replaceKnowledgeMemoryIds(db, new Map([[fromId, toId]]));
+    } else {
+      _replaceKnowledgeMemoryIds(db, mapping);
+    }
+  }
+  return rekeyed;
+}
+
+// R-3/R-4 (§5): one statement finds every main.knowledge row whose source_memory_ids carries one
+// of `mapping`'s local ids, then the JS rewrite applies today's rules. The CASE keeps json_each
+// away from malformed text (P-2) whatever order SQLite evaluates the AND terms in; CAST … AS TEXT
+// keeps today's String(value) equality for numeric elements. Returns how many rows changed.
+function _replaceKnowledgeMemoryIds(db, mapping) {
+  if (!mapping.size) return 0;
+  const candidates = db.prepare(`
+    SELECT k.id, k.source_memory_ids FROM main.knowledge k
+     WHERE k.source_memory_ids IS NOT NULL
+       AND EXISTS (SELECT 1 FROM json_each(CASE WHEN json_valid(k.source_memory_ids) THEN k.source_memory_ids ELSE '[]' END) j
+                   WHERE CAST(j.value AS TEXT) IN (SELECT value FROM json_each(?)))
+  `).all(JSON.stringify([...mapping.keys()]));
+  const update = db.prepare('UPDATE main.knowledge SET source_memory_ids = ? WHERE id = ?');
+  let updated = 0;
+  for (const row of candidates) {
+    let ids;
+    try {
+      ids = JSON.parse(row.source_memory_ids);
+    } catch {
+      continue; // HANDLE: an unparseable source_memory_ids is reported by utils/repair.js — this sweep leaves it alone
+    }
+    if (!Array.isArray(ids)) continue;
+    let changed = false;
+    const next = ids.map((value) => {
+      const key = String(value);
+      if (!mapping.has(key)) return value;
+      changed = true;
+      return mapping.get(key);
+    });
+    if (changed) {
+      update.run(JSON.stringify(next), row.id);
+      updated += 1;
+    }
+  }
+  return updated;
+}
+
+// INSERT OR IGNORE ... SELECT over the INTERSECTION of the local and snapshot column lists (the two
+// schemas may differ by a migration), restricted to the page's snapshot rowids.
+function _insertSnapshotPage(db, table, page) {
+  if (!page.length) return 0;
+  const localCols = _columns(db, `main."${table}"`);
+  const snapCols = _columns(db, `snap."${table}"`);
+  const cols = localCols.filter((column) => snapCols.includes(column));
+  if (!cols.length) return 0;
+  const overrides = INSERT_OVERRIDES[table];
+  const selectExpr = cols.map((column) => {
+    if (overrides && overrides[column]) return `${overrides[column]} AS "${column}"`;
+    return `"${column}"`;
+  });
+  const placeholders = page.map(() => '?').join(', ');
+  const sql = `INSERT OR IGNORE INTO main."${table}" (${cols.map((c) => `"${c}"`).join(', ')})
+    SELECT ${selectExpr.join(', ')} FROM snap."${table}" WHERE rowid IN (${placeholders})`;
+  return db.prepare(sql).run(...page.map((row) => row.rid)).changes || 0;
+}
+
+// brain_vv := MAX(local, snapshot vector) per origin (P-2). The snapshot vector is read from the
+// file, so a home write made during transfer keeps a higher seq and still travels the normal feed.
+function _mergeVector(db, snapshotPath) {
+  const vector = vectorFromSnapshotFile(snapshotPath);
+  const upsert = db.prepare(`
+    INSERT INTO brain_vv (origin_instance_id, max_seq) VALUES (?, ?)
+    ON CONFLICT(origin_instance_id) DO UPDATE SET max_seq = MAX(brain_vv.max_seq, excluded.max_seq)
+  `);
+  for (const [origin, seq] of Object.entries(vector)) {
+    if (!origin) continue;
+    upsert.run(_string(origin, 160), Math.max(0, Number(seq) || 0));
+  }
+  return vector;
+}
+
+// R-3/R-4 (§5): one bounded CHUNK of the `knowledge_links` step — the next `budget` snapshot
+// knowledge rows (rowid > after) that CARRY a pointer. For each, main.knowledge.superseded_by is
+// set to the snapshot value only when the main row exists, its pointer IS NULL and the target id
+// exists in main.knowledge. A pointer whose target is missing stays NULL and is counted in
+// record.linksDropped (R-4, the same repair the feed makes). A row whose main id is absent or
+// whose pointer is already non-NULL is neither linked nor dropped (R-5: local truth wins).
+// Runs inside the step's IMMEDIATE transaction with the sentinel on, so no clock/seq/vv is stamped.
+function _linkKnowledgePage(db, record, after, budget) {
+  const page = db.prepare(`
+    SELECT rowid AS rid, id, superseded_by
+      FROM snap."knowledge"
+     WHERE rowid > ? AND superseded_by IS NOT NULL
+     ORDER BY rowid ASC LIMIT ?
+  `).all(after, budget);
+  const current = db.prepare('SELECT superseded_by FROM main."knowledge" WHERE id = ?');
+  const targetExists = db.prepare('SELECT 1 AS hit FROM main."knowledge" WHERE id = ? LIMIT 1');
+  const setPointer = db.prepare('UPDATE main."knowledge" SET superseded_by = ? WHERE id = ?');
+  let linked = 0;
+  let dropped = 0;
+  for (const row of page) {
+    const id = row.id == null ? '' : String(row.id);
+    const target = row.superseded_by == null ? '' : String(row.superseded_by);
+    if (!id || !target) continue;
+    const local = current.get(id);
+    if (!local) continue; // the main row is absent: nothing to point at (neither linked nor dropped)
+    if (local.superseded_by !== null && local.superseded_by !== undefined) continue; // R-5
+    if (!targetExists.get(target)) { dropped += 1; continue; } // R-4: dangling pointer stays NULL
+    setPointer.run(target, id);
+    linked += 1;
+  }
+  record.linksDropped = (Number(record.linksDropped) || 0) + dropped;
+  return { page, linked, dropped };
+}
+
+// One bounded page of ONE step, inside the caller's IMMEDIATE transaction. R-11 (§5): the page is
+// walked in JOIN_CHUNK_ROWS-row chunks and stops after the chunk during which the elapsed time
+// reached JOIN_STEP_TIME_BUDGET_MS, so the transaction never outlives the write lock's own budget.
+// The record is mutated in place and written before returning so a crash resumes exactly here.
+function _importPage(db, record, budget, now) {
+  const startedAt = now();
+  const step = JOIN_STEP_ORDER.includes(record.table) ? record.table : JOIN_STEP_ORDER[0];
+  let after = Math.max(0, Math.floor(Number(record.after) || 0));
+
+  let processed = 0;
+  let lastRid = after;
+  let exhausted = false; // a chunk came back shorter than asked — the step has no more rows
+  while (processed < budget) {
+    const chunk = Math.min(JOIN_CHUNK_ROWS, budget - processed);
+    let pageLength;
+    let chunkLastRid = after;
+    if (step === 'knowledge_links') {
+      const linked = _linkKnowledgePage(db, record, after, chunk);
+      pageLength = linked.page.length;
+      if (pageLength) chunkLastRid = Number(linked.page[pageLength - 1].rid) || after;
+    } else {
+      const page = db.prepare(`SELECT rowid AS rid, id FROM snap."${step}" WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`).all(after, chunk);
+      const ids = page.map((row) => row.id).filter((id) => id != null && id !== '').map(String);
+      const rekeyed = step === 'memories' ? _rekeyCollidingMemories(db, ids) : 0;
+      const inserted = _insertSnapshotPage(db, step, page);
+      _copySnapshotClock(db, step, ids);
+      record.imported[step] = (Number(record.imported[step]) || 0) + inserted;
+      record.rekeyed = (Number(record.rekeyed) || 0) + rekeyed;
+      pageLength = page.length;
+      if (pageLength) chunkLastRid = Number(page[pageLength - 1].rid) || after;
+    }
+    processed += pageLength;
+    lastRid = chunkLastRid;
+    after = chunkLastRid;
+    // A short chunk means the step is exhausted; the cursor moves on. A chunk that filled the ask
+    // either used the row allowance or ran out of time — both keep the cursor INSIDE this step.
+    if (pageLength < chunk) { exhausted = true; break; }
+    if (processed >= budget) break;
+    if (now() - startedAt >= JOIN_STEP_TIME_BUDGET_MS) break;
+  }
+
+  // R-12: the page committed, so any earlier transient abort is behind us.
+  record.transientRetries = 0;
+  delete record.lastTransientAt;
+  delete record.lastTransientError;
+
+  const index = JOIN_STEP_ORDER.indexOf(step) + (exhausted ? 1 : 0);
+
+  if (index >= JOIN_STEP_ORDER.length) {
+    _mergeVector(db, record.path);
+    markMembership(db, { homeInstanceId: record.homeInstanceId });
+    record.state = 'done';
+    record.table = null;
+    record.after = 0;
+    record.finishedAt = new Date().toISOString();
+    _writeJoinRecord(db, record);
+    return { state: 'done', table: null, done: true, deleteSnapshot: true };
+  }
+  record.state = 'importing';
+  record.table = JOIN_STEP_ORDER[index];
+  record.after = exhausted ? 0 : lastRid;
+  _writeJoinRecord(db, record);
+  return { state: 'importing', table: record.table, done: false, deleteSnapshot: false };
+}
+
+function _markFailed(db, record, err) {
+  const message = _string(typeof err === 'string' ? err : (err?.message || String(err)), MAX_ERROR_LEN);
+  record.state = 'failed';
+  record.error = message;
+  record.finishedAt = new Date().toISOString();
+  try {
+    _writeJoinRecord(db, record);
+  } catch (writeError) {
+    // A locked/broken kv_store must not turn a join failure into a route-level throw: the
+    // in-memory record still answers this tick, and the next tick re-reads the same 'failed' row.
+    errorSink.report('brain-join', 'join-failure-record-write', writeError);
+  }
+  return { state: 'failed', table: record.table || null, done: true };
+}
+
+// R-12 (§5): a page whose transaction threw a write-lock hold-budget / busy error has ROLLED BACK,
+// so the committed cursor is exactly where it was and the same page runs again next tick. The
+// in-memory record was mutated inside the rolled-back transaction, so re-read the committed record
+// from kv, count the retry against the (table, after) it happened at, and stay 'importing'. A
+// deterministic failure at one cursor still ends the join after JOIN_MAX_TRANSIENT_RETRIES tries.
+function _recordTransientRetry(db, failure) {
+  const fresh = _normalizeRecord(readJoinRecord(db) || {});
+  const cursor = `${fresh.table == null ? '' : fresh.table}:${fresh.after}`;
+  fresh.transientRetries = fresh.lastTransientAt === cursor ? fresh.transientRetries + 1 : 1;
+  fresh.lastTransientAt = cursor;
+  fresh.lastTransientError = _string(failure?.message || String(failure), MAX_ERROR_LEN);
+  fresh.state = 'importing';
+  if (fresh.transientRetries >= JOIN_MAX_TRANSIENT_RETRIES) return _markFailed(db, fresh, failure);
+  _writeJoinRecord(db, fresh);
+  return { state: 'importing', table: fresh.table, done: false, retry: true };
+}
+
+// §6.7: attach the snapshot READONLY, so the import can never write to it. The documented shape
+// is the URI form `file:<path>?mode=ro`; a SQLite build with URI filenames disabled for ATTACH
+// (this one — verified: `file:` is only honoured when SQLITE_OPEN_URI is set) rejects it, so fall
+// back to a plain attach. The import only ever SELECTs from `snap`, and the file is deleted once
+// the import reaches 'done'.
+function _attachSnapshot(db, filePath) {
+  try {
+    db.prepare('ATTACH DATABASE ? AS snap').run(`file:${filePath}?mode=ro`);
+  } catch {
+    db.prepare('ATTACH DATABASE ? AS snap').run(filePath);
+  }
+  return true;
+}
+
+function _deleteJoinFile(filePath) {
+  try {
+    fs.rmSync(filePath, { force: true });
+  } catch {
+    // `force` already covers ENOENT; anything left (EPERM/EBUSY) is benign — a MEMBER inbox file
+    // is reclaimed by the R-7 replacement delete or the R-8 inbox sweep on the next accepted join,
+    // never by the hub's 48 h sweep of <brainDir>/sync-join (that directory is the home's).
+    errorSink.expected('brain-join', 'join-file-unlink');
+  }
+}
+
+// `now` is the injectable clock used for the R-11 time budget (defaults to the real one); `_beforePage`
+// is a TEST-ONLY seam: called inside the import transaction before the page work, so a test can make
+// one page throw a coded error and exercise the R-12 transient path. Production passes neither.
+async function joinImportStep(db, { limit = JOIN_STEP_DEFAULT, now = Date.now, _beforePage } = {}) {
+  const record = readJoinRecord(db);
+  if (!record) return { state: 'none', table: null, done: true };
+  if (TERMINAL_STATES.has(record.state)) return { state: record.state, table: record.table || null, done: true };
+
+  const budget = Math.max(1, Math.floor(Number(limit) || 0) || JOIN_STEP_DEFAULT);
+
+  // verifying: the hash is a READ of the file — never inside the import transaction.
+  if (record.state === 'verifying') {
+    if (!fs.existsSync(record.path)) return _markFailed(db, record, 'snapshot_missing');
+    const actual = await _sha256File(record.path);
+    if (record.sha256 && actual !== record.sha256) return _markFailed(db, record, 'sha256_mismatch');
+    record.state = 'importing';
+    record.table = JOIN_STEP_ORDER[0];
+    record.after = 0;
+    _writeJoinRecord(db, record);
+    return { state: 'importing', table: record.table, done: false };
+  }
+
+  // importing: ATTACH the snapshot readonly, sentinel on, ONE IMMEDIATE transaction for the page.
+  let attached = false;
+  let outcome = null;
+  let failure = null;
+  createBrainSyncSentinel(db);
+  try {
+    attached = _attachSnapshot(db, record.path);
+    outcome = db.transaction(() => {
+      if (typeof _beforePage === 'function') _beforePage(record);
+      return _importPage(db, record, budget, now);
+    }).immediate();
+  } catch (err) {
+    failure = err;
+  } finally {
+    if (attached) {
+      try { db.exec('DETACH DATABASE snap'); } catch { /* expected: a rolled-back page may already have released the attachment; the failure is reported below */ }
+    }
+    dropBrainSyncSentinel(db);
+  }
+  if (failure) {
+    if (TRANSIENT_LOCK_CODES.has(failure.code)) return _recordTransientRetry(db, failure);
+    return _markFailed(db, record, failure);
+  }
+  if (outcome.deleteSnapshot) _deleteJoinFile(record.path);
+  return { state: outcome.state, table: outcome.table, done: outcome.done };
+}
+
+function joinStatus(db) {
+  const record = readJoinRecord(db);
+  if (!record) {
+    return { ok: true, joinId: null, state: 'none', table: null, imported: { ...EMPTY_IMPORTED }, rekeyed: 0, linksDropped: 0, embeddings: 'skipped', error: null, transientRetries: 0, resumes: 0, resumedAt: null };
+  }
+  const normalized = _normalizeRecord(record);
+  return {
+    ok: true,
+    joinId: normalized.joinId,
+    state: normalized.state,
+    table: normalized.table,
+    imported: normalized.imported,
+    rekeyed: normalized.rekeyed,
+    linksDropped: normalized.linksDropped,
+    embeddings: 'skipped',
+    error: normalized.error,
+    // R-12: visible so a retry loop under load is diagnosable from the status route.
+    transientRetries: normalized.transientRetries,
+    // R-3 (§6.4): the resume budget's state, so a member stuck at the cap is visible from outside.
+    resumes: normalized.resumes,
+    resumedAt: normalized.resumedAt,
+  };
+}
+
+module.exports = {
+  JOIN_KEY,
+  JOIN_TABLE_ORDER,
+  JOIN_STEP_ORDER,
+  JOIN_STEP_DEFAULT,
+  JOIN_CHUNK_ROWS,
+  JOIN_STEP_TIME_BUDGET_MS,
+  JOIN_MAX_TRANSIENT_RETRIES,
+  JOIN_MAX_RESUMES,
+  EMPTY_IMPORTED,
+  readJoinRecord,
+  isInsideJoinInbox,
+  vectorFromSnapshotFile,
+  createJoinSnapshot,
+  startJoinImport,
+  joinImportStep,
+  joinStatus,
+};