@humwick/server 0.0.0-stage → 0.130.0

package/src/db/connection.ts

@@ -0,0 +1,532 @@
+import { resetMainConnectionWrites } from "./main-connection-writes";
+import { databases, instrumentPostBindDatabase } from "./connection-instrumentation";
+export { instrumentPostBindDatabase } from "./connection-instrumentation";
+import { Database } from "bun:sqlite";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { DB_SLOW_QUERY_MS, dbCheckpointOwner } from "../config";
+
+// The shared SQLite handle. Set by initDb() (schema.ts) or an existing-database
+// attachment, and read everywhere else through getDb(). Kept module-private so
+// the only mutation path is setDb().
+let db: Database;
+let dbPath: string | undefined;
+let dbGeneration = 0;
+
+
+export function setDb(conn: Database, path?: string): void {
+  if (db && db !== conn) databases.get(db)?.();
+  instrumentPostBindDatabase(conn);
+  db = conn;
+  dbPath = path;
+  dbGeneration += 1;
+  resetMainConnectionWrites();
+}
+
+/**
+ * The raw synchronous `bun:sqlite` handle.
+ *
+ * @deprecated @internal — DEPRECATED INTERNAL escape hatch (ADR 0001 §5/§7, epic #1869). The async DB
+ * gateway (`src/db/gateway.ts`, S2.3 #1878) is now the SOLE SANCTIONED public DB access surface: feature
+ * code must go through `getGateway()` (`read`/`write`/`tx`), never call `getDb()` directly. A synchronous
+ * `.run()` on the event loop is what freezes the hub (#1868/#1765/#1847); the gateway makes it
+ * unrepresentable through the sanctioned surface. This export is NOT removed yet only because the S3
+ * migration (#1870) has not yet drained the ~1298 remaining callsites across `db/` + feature modules —
+ * unexporting it now would fail to compile. It is DELETED at S4 (#1880) once those callsites are gone,
+ * at which point the CI ratchet flips to ban the raw handle outside the allowlisted bootstrap layer.
+ * Do NOT add new callers. New DB access goes through the gateway.
+ */
+export function getDb(): Database {
+  if (!db) throw new Error("database not initialized");
+  return db;
+}
+
+export function getDbPath(): string | undefined {
+  return dbPath;
+}
+
+/** A non-handle identity for caches that must rebuild after the shared connection is replaced. */
+export function getDbGeneration(): number {
+  return dbGeneration;
+}
+
+/**
+ * Close the shared connection and clear the handle; getDb() throws until the next open.
+ *
+ * For short-lived ATTACHERS (e.g. the cd-watcher child process) that must RELEASE their connection
+ * to a database the live Relay owns before a long non-DB operation. An open connection — even an idle
+ * reader — holds a WAL read-mark that prevents the live server's synchronous checkpoint from
+ * reclaiming frames past it, so a connection held across a multi-second `git fetch` lets the WAL grow
+ * and stalls the server's event-loop thread on its next checkpoint (#1765; general #1724 rule). The
+ * live Relay never calls this on its own primary handle — it owns the DB for its whole lifetime.
+ */
+export function closeDb(): void {
+  if (db) {
+    try { db.close(); } catch { /* a half-open handle must not wedge the caller's cleanup */ }
+  }
+  db = undefined as unknown as Database;
+  dbPath = undefined;
+  closeFastFailConnection();
+}
+
+/**
+ * #2276 — the inline-checkpoint TRIGGER threshold, in PAGES (page_size is 4096, so ≈32MB). Large
+ * enough that a commit rarely checkpoints, finite so the WAL is still bounded wherever reader-free
+ * instants exist — without depending on any maintenance job running. NOT a hard cap on the file: see
+ * the measured overshoot under a saturating reader in `applyConnectionPragmas`.
+ */
+export const WAL_AUTOCHECKPOINT_PAGES = 8192;
+
+/** The ceiling on one write's synchronous main-thread wait for a held lock (#1428). Every handle that
+ *  shares a hub connection must keep it: a larger value set later silently lengthens hub stalls (#3000). */
+export const CONNECTION_BUSY_TIMEOUT_MS = 1000;
+
+/** Rows ANALYZE samples per index — SQLite's recommended bound for approximate planner stats. */
+export const ANALYSIS_LIMIT_ROWS = 400;
+
+// Connection-scoped pragmas. journal_mode and auto_vacuum persist in the file;
+// the rest reset per connection, so they must be re-applied on every open.
+export function applyConnectionPragmas(conn: Database, options: { ensureWal?: boolean } = {}): void {
+  if (options.ensureWal !== false) conn.run("PRAGMA journal_mode = WAL");
+  // Wait for a held lock instead of failing instantly with SQLITE_BUSY. bun:sqlite is SYNCHRONOUS,
+  // so this wait BLOCKS the event-loop thread — it is the ceiling on a single write's main-thread
+  // stall. Kept deliberately SHORT (#1428): a long timeout trades BUSY errors for event-loop-lag
+  // spikes (the owner saw ~200–2000ms). With BEGIN IMMEDIATE (writeTransaction) removing the
+  // deferred-upgrade instant-BUSY and PASSIVE checkpoints removing long lock holds, real contention
+  // resolves in ms; the rare spike that exceeds this is caught by the ASYNC retry below, which
+  // yields the loop between attempts instead of extending the synchronous block.
+  conn.run(`PRAGMA busy_timeout = ${CONNECTION_BUSY_TIMEOUT_MS}`);
+  // NORMAL is the recommended durability level under WAL: as safe as FULL except
+  // for a power loss mid-checkpoint, and meaningfully faster on writes.
+  conn.run("PRAGMA synchronous = NORMAL");
+  // #2276 — THROTTLE the inline checkpoint; do NOT remove it. SQLite's autocheckpoint runs a PASSIVE
+  // checkpoint INSIDE the COMMIT of whichever connection crosses its threshold. The relay's writer
+  // half is in-process (ADR 0001 §7 Phase-R: only the READER half is env-gated onto worker threads),
+  // so that connection is the EVENT LOOP: at the 1000-page (≈4MB) default, a busy relay past the
+  // threshold pays a checkpoint on essentially EVERY commit — copying frames into a multi-GB main DB
+  // file, plus its fsync, on the hub's only thread. That is the uniform ~50–65ms #2276 measured
+  // across wholly unrelated single-row ops (providerQuotas.upsert, kernelPersistence.*,
+  // commands.patch, orchestrators.heartbeat): a cost that belongs to COMMIT, not to any op body.
+  //
+  // WHAT THE CHECKPOINT IS ACTUALLY DOING (the first cut of this fix got this wrong, and PR #2279's
+  // review falsified it with a harness): it is NOT futile. It performs real, expensive backfill —
+  // measured at ~150MB moved into the main DB over a run where autocheckpoint=0 backfilled ZERO. What
+  // it fails to do under a continuous read-mark is SHRINK the file. Those are different things, and
+  // the distinction is load-bearing: `nBackfill` only ever advances via a checkpoint, and a WAL is
+  // rewound only when a writer finds `nBackfill == mxFrame` with no reader holding a mark. Frequent
+  // checkpoints keep `nBackfill` pinned at max, so any reader-free instant rewinds the log — which is
+  // exactly what holds a live relay's WAL at a STABLE size instead of letting it grow.
+  //
+  // Hence a large-but-FINITE ceiling rather than 0. Measured here (bun:sqlite, WAL,
+  // synchronous=NORMAL, 20k×4KB commits, a second connection reading between chunks):
+  //
+  //   wal_autocheckpoint = 0     → WAL 246MB and strictly monotonic; main DB backfilled 0 bytes.
+  //   wal_autocheckpoint = 4096  → WAL held at 16.1MB.
+  //   wal_autocheckpoint = 8192  → WAL held at 32.2MB.
+  //
+  // Over 3 repetitions of a burst workload, 8192 took ~4.4× fewer ≥5ms commits than the 1000-page
+  // default (≈146 → ≈33) at a lower average per-commit cost.
+  //
+  // WHAT THE CEILING IS AND IS NOT. `pages × page_size` is the size at which a checkpoint is
+  // TRIGGERED, not a hard cap on the file: a rewind still needs a reader-free instant, so under a
+  // pool reading almost continuously the WAL overshoots it. Measured against a reader SUBPROCESS at
+  // three cadences (60k × 4KB commits; peak WAL / multiple of the nominal ceiling):
+  //
+  //                     read every 2ms     every 20ms     every 100ms
+  //   pages = 8192      287.8MB (9.0×)     41.1MB (1.3×)   34.9MB (1.1×)
+  //   pages = 1000      416.5MB (106×)     13.0MB (3.3×)    5.8MB (1.5×)
+  //
+  // So the honest claim is: this bounds the WAL wherever reader-free instants exist, and heavy
+  // overshoot is a property of reader SATURATION rather than of the raised ceiling — under a
+  // saturating reader the 1000-page default overshoots further in absolute bytes than 8192 does.
+  // `db-wal-reclaim` is the second line of defence (off the loop, in a subprocess) and the
+  // `database-wal` health check is what makes a failure of both visible. The ceiling itself does not
+  // depend on either, nor on which backend the reader half runs. (For the record, since the two
+  // behave differently for reclaim: the repo default is `in-process` (`gatewayReaderBackendFromEnv`),
+  // while the production relay is verified to run `AGENT_RELAY_GATEWAY_READER_BACKEND=worker`.)
+  //
+  // #3022 — the hub also runs src/db/wal-checkpointer.ts, which backfills off the loop so the writer
+  // rewinds the WAL long before this threshold. Reaching it means that worker is down or outrun.
+  conn.run(`PRAGMA wal_autocheckpoint = ${dbCheckpointOwner() === "litestream" ? 0 : WAL_AUTOCHECKPOINT_PAGES}`);
+  conn.run("PRAGMA foreign_keys = ON");
+  // ~8MB page cache (negative = KiB) and 256MB mmap window for the read-heavy
+  // poll workload. Both are best-effort; SQLite ignores them if unsupported.
+  conn.run("PRAGMA cache_size = -8000");
+  conn.run("PRAGMA mmap_size = 268435456");
+  // #3000 — every ANALYZE (startup optimize, migrations, db-maintenance) holds the write lock for its
+  // whole statement; bound how many rows it samples per index.
+  conn.run(`PRAGMA analysis_limit = ${ANALYSIS_LIMIT_ROWS}`);
+}
+
+/**
+ * Attach a short-lived client to a database the running Relay already owns.
+ * Unlike initDb(), this never runs catalog DDL or migrations: an external
+ * deploy must not contend with the live server's persistence bootstrap.
+ */
+export const openExistingDb = (path: string): Database => {
+  const conn = new Database(path, { readwrite: true, create: false });
+  // The live Relay has already established WAL; changing a persistent pragma
+  // from a deploy-side connection would itself require a schema-level lock.
+  applyConnectionPragmas(conn, { ensureWal: false });
+  setDb(conn, path);
+  return conn;
+};
+
+/** A read-only connection to `path` for a reader that must not share the main RW handle (#3012). */
+export const openReadOnlyDb = (path: string): Database => new Database(path, { readonly: true });
+
+// #1835/#1840 — a SEPARATE connection to the same database file, opened with a much shorter
+// busy_timeout than the shared app connection, lazily created and cached by (path, timeout).
+let fastFailConn: Database | undefined;
+let fastFailConnPath: string | undefined;
+let fastFailConnBusyTimeoutMs: number | undefined;
+
+function closeFastFailConnection(): void {
+  if (fastFailConn) {
+    try { fastFailConn.close(); } catch { /* a half-open handle must not wedge the caller */ }
+  }
+  fastFailConn = undefined;
+  fastFailConnPath = undefined;
+  fastFailConnBusyTimeoutMs = undefined;
+}
+
+/** Forbids `fn` from returning a `PromiseLike` — see `withFastFailConnection`'s own doc for why an
+ *  async callback would silently defeat the whole point of this helper. Resolves to `never` for any
+ *  `T` that is (or could be) a promise, which TypeScript then rejects at the CALL SITE (the argument
+ *  no longer matches `() => never`) rather than merely in a comment. */
+type NotPromiseLike<T> = T extends PromiseLike<unknown> ? never : T;
+
+function isThenable(value: unknown): value is PromiseLike<unknown> {
+  return typeof value === "object" && value !== null && typeof (value as { then?: unknown }).then === "function";
+}
+
+/**
+ * Run `fn` with `getDb()` temporarily pointing at a SEPARATE connection to the same database file,
+ * using `busyTimeoutMs` instead of the shared app connection's own (much longer) busy_timeout — so a
+ * write-lock conflict during `fn` fails FAST (a `SQLITE_BUSY`/`SQLITE_LOCKED` throw after
+ * `busyTimeoutMs`) instead of blocking the event loop for up to the shared connection's own budget.
+ * Built for `memory-store-health-probe.ts`'s per-scope DB cycle (#1840): a health probe under real
+ * write-lock contention should report `cannot-tell` fast, not become a multi-second freeze.
+ *
+ * LOAD-BEARING INVARIANT — `fn` MUST be fully synchronous: no `await`, no callback deferred to a
+ * later microtask/macrotask. The swap works by reassigning the SAME module-level `db` binding
+ * `getDb()` reads, for the exact duration of `fn`'s synchronous call; nothing else can observe the
+ * swapped connection only because nothing else CAN run while `fn` occupies the (single-threaded)
+ * call stack, and the shared connection is restored in `finally` before control ever returns to the
+ * event loop. This is exactly why the shared app connection's OWN busy_timeout is never touched by
+ * this helper. A caller that needs fast-fail behavior spanning a real await needs a genuinely
+ * different design (e.g. #1809/#1822's offload primitive), not this one.
+ *
+ * ENFORCED AT COMPILE TIME, backstopped (not fixed) at runtime (#1840 re-review MED-1/MED-2) — an
+ * `async fn` would still return its Promise SYNCHRONOUSLY (the function body only runs to its first
+ * `await` before returning), so a naive version of this helper would restore the shared connection in
+ * `finally` and only THEN let the callback's post-`await` continuation run — silently against the
+ * ordinary shared connection, with no error, defeating the fast-fail guarantee for exactly the call it
+ * was supposed to protect. `NotPromiseLike<T>` is the REAL protection: it rejects an `async fn` at the
+ * CALL SITE, so no caller reachable by the type checker can ever produce this shape.
+ *
+ * The runtime `isThenable` check below is NOT a second way to prevent the same outcome — it CANNOT be:
+ * by the time `fn()` has returned a thenable, that thenable's continuation is already scheduled
+ * independent of anything this function does next, so throwing here does not (and cannot) stop it
+ * from eventually running against whatever connection is active when it resumes. What the runtime
+ * check buys is FAIL-LOUD, not fail-safe: a caller that routes an async callback around the type
+ * system (a cast through `any`, for instance) gets an immediate, unmissable `TypeError` at the call
+ * site instead of a silent, delayed connection mix-up nobody notices until a later bug report. Treat a
+ * caller ever seeing this error as a real bug in that call site, not a recoverable runtime condition.
+ *
+ * Falls back to running `fn` against the ordinary shared connection for `:memory:` databases (this
+ * repo's whole test suite) or before any database is open: `bun:sqlite` gives every `:memory:` handle
+ * a DISTINCT, empty database (no shared-cache URI in use here), so a second `:memory:` connection
+ * would silently see no data at all rather than fail fast on contention — and there is no meaningful
+ * lock contention to guard against in a single-process test run in the first place. The thenable
+ * guard still applies on this path, so misuse is caught the same way regardless of environment.
+ */
+export function withFastFailConnection<T>(busyTimeoutMs: number, fn: () => NotPromiseLike<T>): T {
+  if (!Number.isSafeInteger(busyTimeoutMs) || busyTimeoutMs < 0) {
+    throw new RangeError("busyTimeoutMs must be a non-negative integer");
+  }
+  const path = getDbPath();
+  const useFastFail = Boolean(path) && path !== ":memory:";
+  let original: Database | undefined;
+  if (useFastFail) {
+    if (!fastFailConn || fastFailConnPath !== path || fastFailConnBusyTimeoutMs !== busyTimeoutMs) {
+      closeFastFailConnection();
+      const conn = new Database(path!, { readwrite: true, create: false });
+      // The live connection already established WAL/foreign_keys/etc.; only busy_timeout needs to
+      // differ, applied AFTER the shared pragmas so it overrides the ordinary default.
+      applyConnectionPragmas(conn, { ensureWal: false });
+      conn.run(`PRAGMA busy_timeout = ${busyTimeoutMs}`);
+      fastFailConn = conn;
+      fastFailConnPath = path;
+      fastFailConnBusyTimeoutMs = busyTimeoutMs;
+    }
+    original = db;
+    db = fastFailConn;
+  }
+  try {
+    const result = fn();
+    if (isThenable(result)) {
+      // Fail LOUD, not fail-safe — see this function's doc for why this can only make the misuse
+      // visible, not undo it: the thenable's continuation is already scheduled and will still run
+      // against whatever connection is active when it resumes, regardless of this throw.
+      throw new TypeError(
+        "withFastFailConnection: fn returned a thenable — this is a caller bug, not a recoverable " +
+          "condition. fn must be fully synchronous; an async callback's post-await continuation runs " +
+          "AFTER this call restores the shared connection, so it silently escapes the fast-fail " +
+          "guarantee even though this check caught it here. Fix the caller — do not catch this and " +
+          "continue.",
+      );
+    }
+    return result as T;
+  } finally {
+    if (useFastFail) db = original!;
+  }
+}
+
+/** #2712 — is the write lock free right now? One empty `BEGIN IMMEDIATE` on the fast-fail connection,
+ *  so a paced sweep can step aside for another writer instead of waiting out `busy_timeout` per write.
+ *  Always true for `:memory:` (no second writer can exist), like {@link withFastFailConnection}. */
+export function writerLockAvailable(busyTimeoutMs: number): boolean {
+  try {
+    return withFastFailConnection(busyTimeoutMs, () => {
+      const conn = getDb();
+      if (conn.inTransaction) return true;
+      conn.run("BEGIN IMMEDIATE");
+      conn.run("ROLLBACK");
+      return true;
+    });
+  } catch (err) {
+    if (isSqliteBusyError(err)) return false;
+    throw err;
+  }
+}
+
+// SQLITE_BUSY resilience for WRITE transactions (#1428).
+//
+// bun:sqlite's `db.transaction(fn)()` begins DEFERRED: the write lock is only taken when the first
+// write statement runs. A read-then-write transaction (the hot message-send path reads getAgent /
+// idempotency / duplicate checks, THEN inserts) therefore starts as a reader and UPGRADES to a
+// writer. If another connection holds the write lock at that instant — e.g. the maintenance-worker
+// subprocess mid-checkpoint/prune — SQLite returns SQLITE_BUSY *immediately* and IGNORES
+// busy_timeout, because waiting for the upgrade could deadlock two readers. That is exactly the
+// production `database is locked` on `POST /api/messages`.
+//
+// `BEGIN IMMEDIATE` takes the write lock up front, so busy_timeout governs contention: a concurrent
+// writer WAITS for the lock instead of erroring on the upgrade. Route every read-then-write
+// transaction through `writeTransaction`; keep pure read-only transactions on the deferred default
+// (they never upgrade, and IMMEDIATE would needlessly serialize reads).
+//
+// SHORT sync-block bias (#1428): `writeTransaction` makes a SINGLE immediate attempt — it never
+// synchronously sleeps-and-retries, because that would only lengthen the main-thread stall. With
+// busy_timeout kept low, that one attempt blocks the loop only briefly. Resilience for the rare BUSY
+// that survives busy_timeout lives in `runWriteWithBusyRetry`, which awaits between attempts so the
+// event loop breathes (other requests/timers run) instead of the thread spinning.
+const SQLITE_BUSY_RETRY_ATTEMPTS = 4;
+const SQLITE_BUSY_RETRY_BASE_MS = 15;
+const SQLITE_BUSY_RETRY_MAX_MS = 120;
+
+/** True for the transient lock-contention codes SQLite raises under concurrency. */
+export function isSqliteBusyError(err: unknown): boolean {
+  if (!err || typeof err !== "object") return false;
+  const code = (err as { code?: unknown }).code;
+  if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED") return true;
+  // errno 5 = SQLITE_BUSY, 6 = SQLITE_LOCKED (bun:sqlite surfaces these on SQLiteError).
+  const errno = (err as { errno?: unknown }).errno;
+  if (errno === 5 || errno === 6) return true;
+  const message = (err as { message?: unknown }).message;
+  return typeof message === "string" && /database (is|table is) locked|SQLITE_BUSY|SQLITE_LOCKED/i.test(message);
+}
+
+/**
+ * Run `fn` inside a single `BEGIN IMMEDIATE` transaction. The write lock is taken at BEGIN, so
+ * busy_timeout (short, by design) governs contention and the loop stalls only briefly. On a BUSY
+ * that survives busy_timeout this THROWS — the caller on the hot path wraps it in
+ * `runWriteWithBusyRetry` for an async, loop-yielding retry; other callers surface the error as
+ * before. Keep the transaction body small: it runs synchronously with the write lock held.
+ */
+declare const WriteTransactionContextBrand: unique symbol;
+
+/** Unforgeable caller capability proving code is executing synchronously inside `writeTransaction`.
+ * Transaction-bound read→write helpers require this instead of accepting a service-level precheck. */
+export interface WriteTransactionContext {
+  readonly [WriteTransactionContextBrand]: true;
+}
+
+const WRITE_TRANSACTION_CONTEXT = Object.freeze({}) as WriteTransactionContext;
+let activeWriteTransactionDepth = 0;
+
+export function assertWriteTransactionContext(tx: WriteTransactionContext): void {
+  if (tx !== WRITE_TRANSACTION_CONTEXT || activeWriteTransactionDepth < 1 || !getDb().inTransaction) {
+    throw new Error("write transaction context is not active");
+  }
+}
+
+/** Internal composition seam for a transaction-bound callback (for example ingress `bind`) to call
+ * an existing DB facade without opening a nested SQLite transaction. */
+export function currentWriteTransactionContext(): WriteTransactionContext | undefined {
+  return activeWriteTransactionDepth > 0 && getDb().inTransaction ? WRITE_TRANSACTION_CONTEXT : undefined;
+}
+
+// #2111 — every `BEGIN IMMEDIATE` this process opens passes through `writeTransaction` (the gateway's
+// single-op and batch executors both call it, as does every remaining sync path), and
+// `inWriteTransaction` JOINS an open one rather than nesting. So this monotonic counter is an exact
+// count of write transactions — the unit the reconnection-storm regression is measured in, since each
+// one costs a lock acquisition and its own set of WAL frames on the main event loop.
+let writeTransactionsOpened = 0;
+
+/** Monotonic count of write transactions opened since process start. */
+export function writeTransactionCount(): number {
+  return writeTransactionsOpened;
+}
+
+export function writeTransaction<T>(fn: (tx: WriteTransactionContext) => T): T {
+  writeTransactionsOpened++;
+  return getDb().transaction(() => {
+    activeWriteTransactionDepth++;
+    try {
+      return fn(WRITE_TRANSACTION_CONTEXT);
+    } finally {
+      activeWriteTransactionDepth--;
+    }
+  }).immediate() as T;
+}
+
+/**
+ * Run `fn` under a write transaction, JOINING the caller's open one when there already is one and
+ * opening a fresh `BEGIN IMMEDIATE` otherwise.
+ *
+ * This is the seam-safe form of `writeTransaction` (ADR 0001 §3.4): a registered writer op runs
+ * inside a transaction the EXECUTOR already opened — either the single `BEGIN IMMEDIATE` of one job
+ * or a per-caller `SAVEPOINT` of a coalesced batch — so an op body that called `writeTransaction`
+ * itself would nest a second SQLite transaction under it. Joining instead keeps the whole op body
+ * one atomic unit under the executor's transaction, while an ordinary synchronous caller (no
+ * ambient transaction) still gets its own exactly as before.
+ *
+ * `scheduled-timers`/`timer-message-*` used to spell this composition out inline; it is factored
+ * here so every db-domain body relocated onto the gateway shares one implementation.
+ */
+export function inWriteTransaction<T>(fn: (tx: WriteTransactionContext, db: Database) => T): T {
+  const active = currentWriteTransactionContext();
+  // The handle is supplied only while the transaction is live. A writer op reaches this branch on
+  // the dedicated writer thread, so its entire callback stays inside that one worker-thread
+  // transaction rather than borrowing a main-thread handle or opening a nested transaction.
+  return active ? fn(active, getDb()) : writeTransaction((tx) => fn(tx, getDb()));
+}
+
+/**
+ * Hot-path write with a bounded, ASYNC retry around SQLITE_BUSY (#1428). Each attempt is one short
+ * synchronous `writeTransaction`; between attempts we `await` a jittered backoff via setTimeout,
+ * which YIELDS the event loop so other work runs and the contending writer (another process, a
+ * checkpoint) can release the lock — the opposite of a long synchronous busy_timeout spin. A rare
+ * contention spike thus degrades to a slightly slower request instead of a 500 or a multi-second
+ * main-thread stall.
+ */
+export async function runWriteWithBusyRetry<T>(fn: () => T, attempts: number = SQLITE_BUSY_RETRY_ATTEMPTS): Promise<T> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return writeTransaction(fn);
+    } catch (err) {
+      if (attempt < attempts && isSqliteBusyError(err)) {
+        const cap = Math.min(SQLITE_BUSY_RETRY_MAX_MS, SQLITE_BUSY_RETRY_BASE_MS * 2 ** (attempt - 1));
+        const delay = cap / 2 + Math.random() * (cap / 2);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+        continue;
+      }
+      throw err;
+    }
+  }
+}
+
+// Slow-query log. Off unless AGENT_RELAY_DB_SLOW_MS is set (>0). Wraps the
+// execution of the heavy dynamic queries so the next slow query surfaces before
+// it wedges something (see issue #197 — the 8.6s reply-obligation query).
+const SLOW_QUERY_MS = DB_SLOW_QUERY_MS;
+export function timedQuery<T>(label: string, run: () => T): T {
+  if (SLOW_QUERY_MS <= 0) return run();
+  const start = performance.now();
+  try {
+    return run();
+  } finally {
+    const ms = performance.now() - start;
+    if (ms >= SLOW_QUERY_MS) {
+      console.warn(`[db.slow] ${label} took ${ms.toFixed(1)}ms (threshold ${SLOW_QUERY_MS}ms)`);
+    }
+  }
+}
+
+export type DbMaintenanceResult = { vacuumed: boolean };
+
+// Periodic DB upkeep, invoked by the daily maintenance scheduler. WAL checkpoint/reclaim is a
+// separate frequent, lightweight job so VACUUM cadence never governs WAL size. Planner statistics are
+// refreshed at startup instead (`refreshPlannerStats`, #3000).
+export function runDbMaintenance(options: { vacuum?: boolean } = {}): DbMaintenanceResult {
+  if (options.vacuum) db.run("VACUUM");
+  return { vacuumed: options.vacuum === true };
+}
+
+/** #3000 — the one ANALYZE, run at startup before the server binds. ANALYZE holds the write lock for
+ *  its whole statement, and a connection loads sqlite_stat1 only when it reads the schema (at open or
+ *  after a schema change), so an ANALYZE while serving stalls live writes for stats the running hub
+ *  does not pick up. `0x10002` checks every table, not only those this connection used;
+ *  `analysis_limit` bounds each table's scan. Skewed tables get exact rows from db-planner-stats (#3086). */
+export function refreshPlannerStats(conn: Database): void {
+  try {
+    const hasStats = conn.query("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'sqlite_stat1'").get();
+    if (!hasStats) conn.run("ANALYZE");
+    conn.run("PRAGMA optimize = 0x10002");
+  } catch (error) {
+    // Another process holding the write lock must not fail boot; the next start refreshes instead.
+    if (!isSqliteBusyError(error)) throw error;
+    console.warn(`[db] planner statistics refresh skipped: ${(error as Error).message}`);
+  }
+}
+
+export async function runDbMaintenanceWorker<T>(operation: string, args: string[], signal?: AbortSignal): Promise<T> {
+  if (!dbPath || dbPath === ":memory:") {
+    throw new Error("database worker requires a file-backed database");
+  }
+  if (signal?.aborted) throw new Error("database worker aborted");
+  const bunPath = existsSync(process.execPath) ? process.execPath : (Bun.which("bun") ?? process.execPath);
+  const proc = Bun.spawn([bunPath, join(import.meta.dir, "maintenance-worker.ts"), operation, dbPath, ...args], {
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  let killTimer: Timer | undefined;
+  const abort = () => {
+    proc.kill("SIGTERM");
+    killTimer = setTimeout(() => proc.kill("SIGKILL"), 2_000);
+  };
+  signal?.addEventListener("abort", abort, { once: true });
+  try {
+    const [stdout, stderr, exitCode] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
+    if (signal?.aborted) throw new Error("database worker aborted");
+    if (exitCode !== 0) {
+      const detail = stderr.trim() || stdout.trim() || `exit ${exitCode}`;
+      throw new Error(`database worker ${operation} failed: ${detail}`);
+    }
+    return JSON.parse(stdout) as T;
+  } finally {
+    signal?.removeEventListener("abort", abort);
+    if (killTimer) clearTimeout(killTimer);
+  }
+}
+
+export async function runDbMaintenanceAsync(options: { vacuum?: boolean; signal?: AbortSignal } = {}): Promise<DbMaintenanceResult & { offThread: boolean }> {
+  if (!options.vacuum) return { vacuumed: false, offThread: false };
+  if (!dbPath || dbPath === ":memory:") return { ...runDbMaintenance(options), offThread: false };
+  const result = await runDbMaintenanceWorker<DbMaintenanceResult>(
+    "db-maintenance",
+    [options.vacuum ? "vacuum" : "no-vacuum"],
+    options.signal,
+  );
+  return { ...result, offThread: true };
+}
+
+// Shared error types thrown across query domains and unwrapped at call sites.
+// `ValidationError` now lives in the SDK (@humwick/sdk) as the single canonical
+// home shared with the orchestrator package; re-exported here so the ~120 server
+// call sites (and #892's `toThrow(ValidationError)` tests) stay byte-unchanged and
+// keep the same class identity.
+export { ValidationError } from "@humwick/sdk";
+export class ClaimError extends Error {}