@humwick/server 0.0.0-stage → 0.130.0

package/src/db/writer-gateway.ts

@@ -0,0 +1,840 @@
+import { snapshotBufferedPresence, type PresenceSnapshot } from "./presence-buffer";
+import { WriterDrain } from "./writer-drain";
+// The async WRITER gateway (ADR 0001 §3.3/§3.6/§3.7/§3.9, S2.1 #1871).
+//
+// This is the writer HALF of the async DB seam. It routes every write through a serialized async
+// queue owned by a size-1 writer worker (the sole RW connection), so:
+//   - write ordering is guaranteed by a single gateway FIFO feeding the size-1 pool (INV-2). EVERY
+//     path — dbWrite, dbTx, dbWriteBatch, and same-tick dbCoalesce — enqueues into that ONE ordered
+//     queue at submission time, so submission order is preserved ACROSS coalesced + normal writes
+//     (finding 2: a microtask-deferred coalesce must not let a same-tick dbWrite overtake it);
+//   - SQLite write-contention is impossible on the event loop by construction — any `busy_timeout`
+//     wait now happens on the worker thread, never the loop (no loop spin path for writes);
+//   - atomic multi-statement transactions run as ONE writer job wrapped in BEGIN IMMEDIATE (§3.4);
+//   - db-thrown errors are reconstructed into their real classes on the main thread (H2);
+//   - a worker death after commit surfaces as an explicit INDETERMINATE outcome (phantom-failure, H5).
+//
+// WEDGED-WRITER POLICY (H4 §3.6.1, finding 1 / Option B). The offload substrate replaces a worker on
+// a job deadline — but a Bun Worker is a THREAD, so `terminate()` cannot interrupt a synchronously
+// wedged op (e.g. one spinning in `busy_timeout`/`Atomics.wait`): the dead thread keeps holding the
+// SQLite write lock (its fd survives thread death), so a replacement worker's fresh connection just
+// hits `database is locked` and the NEXT write times out too. The honest thread-level guarantee is a
+// BOUNDED STALL, not zero stall: we therefore do NOT arm the pool's per-job deadline (which would
+// replace the wedged worker and deadlock the replacement). Instead the gateway runs its own deadline;
+// on expiry it marks the writer WEDGED (kept, not replaced — its op still holds the lock and will
+// complete), surfaces the timed-out caller's INDETERMINATE outcome (H5), and HOLDS the FIFO: new
+// non-critical writes shed (503) and critical writes are held until the wedged op completes and
+// releases the lock, at which point the SAME warm connection drains the held FIFO. The wedged state is
+// observable on `writerStats()` (NOT `busy:0`). A true force-kill that preempts a held lock needs a
+// separate WRITER PROCESS (SIGKILL → OS releases the lock); that subprocess substrate is a deliberate
+// follow-up behind this same gateway interface and out of scope here.
+//
+// SCOPE (S2.1): this is the writer surface only. It deliberately does NOT unexport `getDb()` and does
+// NOT finalize the unified read+write gateway — that is the S2.3 barrier (#1878). The reader pool is
+// the parallel S2.2 (#1872). Everything here is additive.
+//
+// Two backends behind one async interface (§3.8):
+//   - worker-backed: a file DB in production — the size-1 writer pool.
+//   - in-process: `:memory:` / bootstrap — runs the op synchronously in-thread, but ROUND-TRIPS args,
+//     results, and errors through the SAME marshaling path as the worker (H1 §3.8.1), so the default
+//     `bun test` suite (which runs entirely on `:memory:`) exercises the seam and fails exactly where
+//     production would.
+
+import { errMessage } from "@humwick/sdk";
+import { createWorkerPool, type WorkerPool } from "@humwick/offload";
+import { getDbGeneration, getDbPath } from "./connection.ts";
+import { onLoopWriteTransaction } from "./on-loop-write-transaction.ts";
+// On-loop-block attribution (#1882 / ADR §5). `resolveLoopCallsite` pins the effective callsite onto a
+// submission in the CALLER's context; `captureSyncFrame` times the op body where it actually runs — in
+// the in-process drain microtask, NOT the synchronous enqueue. One-way import (loop-block-attribution
+// imports only event-loop-lag + config), so no cycle.
+import { captureSyncFrame, resolveLoopCallsite } from "../loop-block-attribution.ts";
+import { reconstructDbError, serializeDbError } from "./db-error-registry.ts";
+import {
+  isIdempotentWriterOp,
+  isOffLoopWriterOp,
+  runWriterBatch,
+  runWriterOp,
+  WriteRetrySafetyError,
+  type BatchEntry,
+  type SerializedBatchEntry,
+  type WriterJobInput,
+  type WriterJobResult,
+} from "./write-ops.ts";
+
+// The structural idempotency error is defined at the executor layer (write-ops) so the invariant is
+// enforced regardless of entry point (H5 §3.7.1); re-exported here to keep the gateway's public error
+// surface unchanged for existing importers.
+export { WriteRetrySafetyError };
+
+// ── Errors surfaced by the gateway (distinct from the db-thrown taxonomy) ─────────────────────────
+
+/**
+ * Backpressure shed (§3.6). The writer queue is past its soft high-water mark (or the writer is
+ * wedged) and this write was non-critical, so it is rejected fast rather than deepening the queue.
+ * Maps to a 503 at the edge. Critical writes (message send) are never shed — they keep enqueuing.
+ */
+export class WriterOverloadedError extends Error {
+  readonly retriable = true;
+  constructor(readonly op: string, readonly queued: number) {
+    super(`writer overloaded: ${queued} jobs queued, shedding non-critical op ${op}`);
+    this.name = "WriterOverloadedError";
+  }
+}
+
+/**
+ * Phantom-failure (H5 §3.7.1). The writer worker died (crash/OOM) OR the gateway deadline expired with
+ * this job in flight (the wedged-writer path, §3.6.1), so its promise rejects — but the write MAY have
+ * already committed before the ack was lost. The outcome is INDETERMINATE, not "definitely did not
+ * happen": a caller must reconcile via the idempotency key or a follow-up read, NEVER via blind
+ * compensation or a naive retry (which would double-apply a non-idempotent write). At-most-once
+ * redefined: "a write may be committed even when reported as failed."
+ */
+export class WriteIndeterminateError extends Error {
+  readonly indeterminate = true as const;
+  readonly op: string;
+  readonly idempotencyKey?: string;
+  constructor(op: string, cause: unknown, idempotencyKey?: string) {
+    super(
+      `writer op ${op} is INDETERMINATE (worker died or wedged before ack): the write may or may not ` +
+        `have committed. Reconcile via the idempotency key or a follow-up read; do not blindly retry.`,
+      { cause }, // available as this.cause (Error.cause)
+    );
+    this.name = "WriteIndeterminateError";
+    this.op = op;
+    this.idempotencyKey = idempotencyKey;
+  }
+}
+
+// ── Options & per-call opts ───────────────────────────────────────────────────────────────────────
+
+export interface InitWriterOptions {
+  /** Overrides the path from `getDbPath()`; `:memory:`/undefined selects the in-process backend. */
+  path?: string;
+  /** Soft high-water mark on the writer queue; past it, non-critical writes shed (§3.6). */
+  queueHighWater?: number;
+  /**
+   * Hard cap on total pending writes (gateway-held + pool-queued). Past it EVERY write rejects —
+   * including critical ones — so a write storm (or a long wedge holding critical writes) can never
+   * grow the queue without bound (§3.6). Defaults to `queueHighWater * 4`.
+   */
+  queueHardCap?: number;
+  /**
+   * Writer-job deadline (H4 §3.6.1). Enforced by the GATEWAY (not the offload pool): on expiry the
+   * caller sees a phantom-failure and the writer is marked WEDGED — the worker is NOT torn down (a
+   * thread `terminate()` cannot release the held SQLite lock; a replacement would only deadlock on
+   * `database is locked`), it is kept until its op completes and releases the lock, then the FIFO
+   * resumes. Default is a few seconds — above a healthy commit + a full `busy_timeout` wait, below an
+   * unbounded stall. The exact value is a #1871 tuning item, to be measured against #1868 traces.
+   */
+  writerTimeoutMs?: number;
+  /**
+   * #2276 — the IN-PROCESS drain's on-loop time slice, in ms. See {@link DEFAULT_DRAIN_SLICE_MS}.
+   * Worker-backed drains ignore it (their op bodies never run on the loop). 0 disables slicing.
+   */
+  drainSliceMs?: number;
+  /** Reader-pool sizing is out of scope here (S2.2); this gateway only stands up the writer. */
+}
+
+export interface WriteOpts {
+  /**
+   * A critical write (e.g. message send) is NEVER shed under backpressure OR while the writer is
+   * wedged — it keeps enqueuing and degrades only to higher latency (§3.6). Best-effort writes shed
+   * past the high-water mark or while wedged. A critical write is STILL bounded by the hard queue cap
+   * (§3.6) — nothing may grow the writer queue without bound.
+   */
+  critical?: boolean;
+  /**
+   * REQUIRED on any op the registry declares non-idempotent (`idempotent:false`) — the gateway rejects
+   * a keyless non-idempotent write ALWAYS (H5 §3.7.1). It is what makes a post-phantom-failure retry
+   * deduplicate: the writer records (op, key) atomically with the effect and a repeat submission with a
+   * seen key returns the prior result WITHOUT re-applying. Also carried on `WriteIndeterminateError` so
+   * the caller can reconcile. Idempotent ops (reads, ON-CONFLICT inserts, pure throws) may omit it.
+   */
+  idempotencyKey?: string;
+  /**
+   * Build-time `file:line` of the originating callsite (#1882 / ADR §5). Attribution-only — the writer
+   * executor ignores it; the gateway facade stamps it into the loop-callsite context so a synchronous
+   * in-process write body that blocks the loop names its own `file:line` in the >50ms regression alarm.
+   * Symmetric with `ReadOptions.callsite`.
+   */
+  callsite?: string;
+}
+
+export interface WriterStats {
+  backend: "worker" | "in-process" | "uninitialized";
+  /** Pending writes not yet acked: gateway-held submissions + the pool's own queue depth. */
+  queued: number;
+  busy: number;
+  /** Soft high-water mark; `queued >= highWater` is the shed threshold. */
+  highWater: number;
+  /**
+   * True when a writer job exceeded its gateway deadline and the sole writer is stalled holding the
+   * SQLite lock (H4 §3.6.1). A wedged writer is NOT idle — this is deliberately distinct from
+   * `busy:0`, which is how the naive pool-replace path mis-reported the stall. New non-critical
+   * writes shed and critical writes are held until the wedged op completes and the FIFO resumes.
+   */
+  wedged: boolean;
+  /**
+   * #2276 — how many times the IN-PROCESS drain has yielded the event loop mid-convoy because its
+   * time slice was spent. 0 on the worker backend (nothing to slice). A steadily climbing count is
+   * the observable signal that write convoys are long enough to need the throttle.
+   */
+  drainYields: number;
+}
+
+// ── Backend state ─────────────────────────────────────────────────────────────────────────────────
+
+// Exported so a test that lowers the high-water mark can RESTORE it afterward: initWriter does NOT reset
+// queueHighWater when the option is omitted (a lazy re-init keeps the last value), so a test that sets a
+// tiny mark must set it back to this default or the low mark LEAKS into later files in the same shard
+// bucket and sheds their concurrent writes (the ordering-dependent singleton bleed).
+export const DEFAULT_QUEUE_HIGH_WATER = 256;
+const DEFAULT_WRITER_TIMEOUT_MS = 5_000;
+// How far above the soft high-water mark the HARD cap sits when not set explicitly. The hard cap is
+// the absolute memory bound that even critical writes cannot exceed (§3.6).
+const HARD_CAP_MULTIPLIER = 4;
+const WRITER_TASK_NAME = "db.write";
+// Hard cap on the arg count of a SINGLE coalesced/batch writer job (finding 5): one job must never
+// pin the sole writer under an unbounded `argsList`. A dbWriteBatch beyond this is rejected (dropping
+// entries would silently lose writes); same-tick dbCoalesce rolls over into a fresh FIFO entry.
+const MAX_WRITE_BATCH = 512;
+/**
+ * #2276 — the IN-PROCESS drain's on-loop time slice.
+ *
+ * `kickDrain` defers the drain to a `queueMicrotask`, and `dispatchEntry` on the in-process backend
+ * is fully synchronous, so `await dispatchEntry(...)` only reaches a MICROTASK checkpoint between
+ * entries — I/O, timers and the bus never get a turn. A convoy of queued writes therefore executes
+ * as ONE uninterrupted on-loop block whose length is the SUM of its op bodies, which is what the
+ * #1882 attributor reports as "<op> blocked the loop N× within the window".
+ *
+ * Slicing bounds the CONTINUOUS block instead: once a drain has spent this much wall time running op
+ * bodies it yields to the MACROTASK queue (so the poll phase runs and pending I/O is served) and
+ * resumes. FIFO is untouched — the queue is only paused, never reordered — and a lone write (the
+ * overwhelmingly common case) never yields at all, so the steady-state path is unchanged.
+ *
+ * This bounds convoys, NOT a single op body: one atomic `messages.sendBatch` is indivisible by
+ * construction (all-or-nothing for one caller), so its own duration is unaffected. Shortening THAT
+ * is what raising the inline WAL-checkpoint threshold does (src/db/connection.ts).
+ *
+ * ONE DELIBERATE TRADE (PR #2279 review, LOW-1): a yield widens the `dbCoalesce` TAIL-APPEND window.
+ * Before this, an in-process drain never yielded, so a still-open coalesce entry could only grow
+ * within a single microtask; now it can grow across a full macrotask turn. That is bounded — the
+ * tail-append refuses past `MAX_WRITE_BATCH` and rolls over into a fresh FIFO entry — and it is the
+ * amortization win working as designed (more independent same-tick callers share one commit). But it
+ * does mean a yield can LENGTHEN an individual op body, which is the one thing this throttle cannot
+ * shorten, so the two mechanisms are complementary rather than redundant.
+ */
+export const DEFAULT_DRAIN_SLICE_MS = 25;
+
+// The in-process marshaling round-trip is what makes the default (`:memory:`) suite exercise the seam
+// (H1). Production's in-process path is only `:memory:`/bootstrap (a file DB uses the worker backend),
+// so gating on non-production keeps prod bootstrap free while every test still fails where prod would.
+const MARSHAL_IN_PROCESS = process.env.NODE_ENV !== "production";
+
+interface WorkerBackend {
+  kind: "worker";
+  pool: WorkerPool;
+  dbPath: string;
+}
+interface InProcessBackend {
+  kind: "in-process";
+  stop: AbortController;
+  busy: boolean;
+  offLoop?: WorkerBackend;
+  offLoopGeneration?: number;
+}
+type Backend = WorkerBackend | InProcessBackend;
+
+let backend: Backend | undefined;
+let shutdownDrain = new WriterDrain();
+let queueHighWater = DEFAULT_QUEUE_HIGH_WATER;
+let queueHardCap = DEFAULT_QUEUE_HIGH_WATER * HARD_CAP_MULTIPLIER;
+let writerTimeoutMs = DEFAULT_WRITER_TIMEOUT_MS;
+let drainSliceMs = DEFAULT_DRAIN_SLICE_MS;
+// Monotonic across re-inits (like any counter): a test that wants a delta reads it before and after.
+let drainYields = 0;
+// Set while a writer job has blown its gateway deadline and the sole writer is stalled holding the
+// lock (§3.6.1). Cleared when that in-flight job finally settles (the op completed and released the
+// lock), which re-kicks the FIFO drain. Worker backend only — an in-process op runs synchronously on
+// the main thread, so it cannot be observed mid-stall.
+let wedged = false;
+
+// Test-only fault injection for the precise H5 window: the op has COMMITTED, but its
+// acknowledgement is lost before the caller observes the result. One-shot by design so a retry
+// exercises the persisted idempotency ledger rather than repeatedly failing at the transport edge.
+let postCommitAckLossForTests: { op: string; remaining: number } | undefined;
+
+export function rejectNextWriterAckAfterCommitForTests(op: string): void {
+  postCommitAckLossForTests = { op, remaining: 1 };
+}
+
+function losePostCommitAckForTests(op: string): boolean {
+  if (!postCommitAckLossForTests || postCommitAckLossForTests.op !== op) return false;
+  postCommitAckLossForTests.remaining -= 1;
+  if (postCommitAckLossForTests.remaining <= 0) postCommitAckLossForTests = undefined;
+  return true;
+}
+
+/**
+ * Stand up the writer gateway. Idempotent per path: calling it again with the same effective path is
+ * a no-op. Chooses the backend from the db path — `:memory:`/undefined → in-process, a file → the
+ * size-1 writer pool.
+ */
+export function initWriter(options: InitWriterOptions = {}): void {
+  const path = options.path ?? getDbPath();
+  // EVERY caller-supplied number is validated BEFORE any of them is assigned: a RangeError thrown
+  // after an earlier field was already written would leave the module in a half-applied config the
+  // caller never asked for.
+  //
+  // queueHighWater: must be a SAFE integer >= 1. A non-finite / non-positive high-water mark would
+  // silently disable shedding (every `depth >= NaN` is false), and an unsafely-large one (e.g.
+  // Number.MAX_VALUE) would make the DERIVED hard cap `*4` overflow to Infinity — an unbounded queue
+  // through an accepted config (MED-1/MED-3). Requiring a safe integer forecloses both at the source.
+  if (options.queueHighWater !== undefined && (!Number.isSafeInteger(options.queueHighWater) || options.queueHighWater < 1)) {
+    throw new RangeError(`queueHighWater must be a safe integer >= 1, got ${options.queueHighWater}`);
+  }
+  if (options.queueHardCap !== undefined && (!Number.isSafeInteger(options.queueHardCap) || options.queueHardCap < 1)) {
+    throw new RangeError(`queueHardCap must be a safe integer >= 1, got ${options.queueHardCap}`);
+  }
+  // drainSliceMs: must be a SAFE integer >= 0. NaN would make every `elapsed >= slice` comparison
+  // false and silently disable the throttle; a negative value would yield after every single op. 0 is
+  // the explicit, legible way to turn slicing off.
+  if (options.drainSliceMs !== undefined && (!Number.isSafeInteger(options.drainSliceMs) || options.drainSliceMs < 0)) {
+    throw new RangeError(`drainSliceMs must be a safe integer >= 0, got ${options.drainSliceMs}`);
+  }
+
+  // Unlike queueHighWater, an omitted slice RESETS to the default rather than keeping the last value,
+  // so a test that lowered it cannot leak a disabled throttle into a later file in the same shard.
+  drainSliceMs = options.drainSliceMs ?? DEFAULT_DRAIN_SLICE_MS;
+  if (options.queueHighWater !== undefined) queueHighWater = options.queueHighWater;
+  if (options.queueHardCap !== undefined) {
+    queueHardCap = options.queueHardCap;
+  } else {
+    // The hard cap follows the high-water mark unless set explicitly.
+    queueHardCap = queueHighWater * HARD_CAP_MULTIPLIER;
+  }
+  // The hard cap must never sit below the soft high-water mark (a critical write may sit above the soft
+  // mark but must still fit under the hard cap).
+  if (queueHardCap < queueHighWater) queueHardCap = queueHighWater;
+  // Final guard on the RESOLVED hard cap: it must be a SAFE integer, not merely finite. A derived cap
+  // like `Number.MAX_SAFE_INTEGER * 4` is finite but beyond 2^53, where `cap + 1 === cap`, so the
+  // admission arithmetic (`occupancy + count > cap`) can no longer faithfully enforce the bound
+  // (cycle-5 MED-1). Rejecting here means the largest accepted high-water is floor(MAX_SAFE_INTEGER/4),
+  // whose *4 stays a safe integer.
+  if (!Number.isSafeInteger(queueHardCap) || queueHardCap < 1) {
+    throw new RangeError(
+      `resolved queueHardCap ${queueHardCap} is not a safe integer >= 1 — reduce queueHighWater ` +
+        `(its derived cap is queueHighWater*${HARD_CAP_MULTIPLIER}) or set an explicit safe queueHardCap`,
+    );
+  }
+  if (options.writerTimeoutMs !== undefined) writerTimeoutMs = options.writerTimeoutMs;
+
+  const wantWorker = Boolean(path) && path !== ":memory:";
+  if (backend) {
+    // Re-init to the same backend shape is a no-op; a genuine path change tears the old pool down.
+    if (backend.kind === "worker" && wantWorker && backend.dbPath === path) return;
+    if (backend.kind === "in-process" && !wantWorker) return;
+    void shutdownWriter().catch((error) => console.error(`[writer-gateway] teardown on re-init failed: ${errMessage(error)}`));
+  }
+
+  shutdownDrain = new WriterDrain();
+  if (wantWorker) {
+    backend = createFileWriter(path as string);
+  } else {
+    backend = { kind: "in-process", stop: new AbortController(), busy: false };
+  }
+}
+
+function createFileWriter(dbPath: string): WorkerBackend {
+  // The gateway owns deadlines; replacing a timed-out worker could strand its SQLite lock.
+  const pool = createWorkerPool({ worker: new URL("./db-writer-worker.ts", import.meta.url), size: 1 });
+  return { kind: "worker", pool, dbPath };
+}
+
+/** Server shutdown keeps already-admitted writes; test/reset teardown remains immediate. */
+export async function drainWriterOnShutdown(): Promise<void> {
+  try { await shutdownDrain.start(() => !draining && !wedged && pendingWriteDepth() === 0 && inFlightWrites() === 0); }
+  finally { await shutdownWriter(); }
+}
+
+export async function shutdownWriter(): Promise<void> {
+  const current = backend;
+  backend = undefined;
+  wedged = false;
+  // Reject anything still queued at the gateway so no caller hangs across a teardown.
+  const stranded = submissionQueue.splice(0, submissionQueue.length);
+  for (const entry of stranded) {
+    rejectEntry(entry, new Error("writer gateway shut down before this write was dispatched"));
+  }
+  if (current?.kind === "in-process") current.stop.abort(new Error("writer gateway shut down"));
+  if (current?.kind === "worker") await current.pool.destroy();
+  if (current?.kind === "in-process") await current.offLoop?.pool.destroy();
+}
+
+/** Whether the gateway is initialized (some callers guard bootstrap ordering on this). */
+export function isWriterInitialized(): boolean {
+  return backend !== undefined;
+}
+
+export function writerStats(): WriterStats {
+  if (!backend) return { backend: "uninitialized", queued: 0, busy: 0, highWater: queueHighWater, wedged: false, drainYields };
+  if (backend.kind === "in-process") {
+    return { backend: "in-process", queued: pendingWriteDepth(), busy: inFlightWrites(), highWater: queueHighWater, wedged, drainYields };
+  }
+  const stats = backend.pool.stats;
+  return { backend: "worker", queued: pendingWriteDepth(), busy: stats.busy, highWater: queueHighWater, wedged, drainYields };
+}
+
+function requireBackend(): Backend {
+  if (!backend) throw new Error("writer gateway not initialized — call initWriter() after initDb()");
+  return backend;
+}
+
+// ── Marshaling helpers (identical path in both backends — H1) ─────────────────────────────────────
+
+function cloneIn<T>(value: T): T {
+  return MARSHAL_IN_PROCESS ? (structuredClone(value) as T) : value;
+}
+
+/** Round-trip a thrown db error through serialize→reconstruct so `:memory:` tests see the reconstructed
+ *  typed instance exactly as the worker path produces it. */
+function reconstructInProcess(error: unknown): unknown {
+  return MARSHAL_IN_PROCESS ? reconstructDbError(serializeDbError(error)) : error;
+}
+
+// ── Submission FIFO (one ordered queue in front of the size-1 pool) ───────────────────────────────
+
+/** A per-caller outcome from a coalesced batch: the reconstructed typed result or error. */
+export type CoalescedEntry<R = unknown> = { ok: true; result: R } | { ok: false; error: unknown };
+
+// A coalesced caller's own promise settlers plus its own idempotency key, so a phantom-failure that
+// fans out to the whole batch still hands EACH caller ITS key to reconcile (not the first caller's).
+type Resolver = { resolve: (value: unknown) => void; reject: (reason: unknown) => void; idempotencyKey?: string };
+
+// A single logical submission, ordered by arrival. `now` is captured at submission (main thread, H6).
+// `callsite` (#1882) is likewise pinned at submission — the effective `file:line` resolved in the
+// CALLER's context — because the in-process op body runs later, in the drain microtask, where the
+// ambient callsite context is gone (or belongs to a different caller). For a coalesced entry it is the
+// FIRST accumulating caller's callsite (one physical block = one savepoint tx = one frame). CAVEAT
+// (r2-review LOW): that first caller is not necessarily the one whose savepoint blocks — a fast-first /
+// slow-second coalesce names the fast caller's `file:line`. Acceptable: coalescing is same-`op` only,
+// so the `op` field always scopes the alarm correctly; only the `file:line` among same-op same-tick
+// callers can be off. Not worth splitting one savepoint tx into per-caller frames to fix.
+type Submission =
+  // dbWrite / dbTx — one caller, one op.
+  | { kind: "op"; op: string; args: unknown; opts?: WriteOpts; now: number; presence?: PresenceSnapshot; callsite?: string; resolve: (v: unknown) => void; reject: (e: unknown) => void }
+  // dbWriteBatch — one caller submits an argsList and gets the whole CoalescedEntry[] array.
+  | { kind: "batch"; op: string; argsList: unknown[]; opts?: WriteOpts; now: number; presence?: PresenceSnapshot; callsite?: string; resolve: (v: unknown) => void; reject: (e: unknown) => void }
+  // dbCoalesce — N independent callers accumulated in the same tick; each resolver fans out its own entry.
+  | { kind: "coalesce"; op: string; opts?: WriteOpts; now: number; presence?: PresenceSnapshot; callsite?: string; argsList: unknown[]; resolvers: Resolver[] };
+
+const submissionQueue: Submission[] = [];
+let draining = false;
+let drainScheduled = false;
+
+/** QUEUED writes not yet dispatched: gateway-held args plus the pool's own queue depth. This is the
+ *  `queued` health metric — deliberately does NOT include the in-flight (busy) job, which is reported
+ *  separately as `busy`. */
+function pendingWriteDepth(): number {
+  let n = 0;
+  for (const entry of submissionQueue) {
+    n += entry.kind === "op" ? 1 : entry.argsList.length;
+  }
+  if (backend?.kind === "worker") n += backend.pool.stats.queued;
+  return n;
+}
+
+/** Includes an in-process writer waiting asynchronously for BEGIN. */
+function inFlightWrites(): number {
+  return backend?.kind === "worker" ? backend.pool.stats.busy : Math.max(Number(backend?.busy ?? false), backend?.offLoop?.pool.stats.busy ?? 0);
+}
+
+/** Reject every caller attached to a submission (used for teardown / dispatch-time backend loss). */
+function rejectEntry(entry: Submission, error: unknown): void {
+  if (entry.kind === "coalesce") for (const r of entry.resolvers) r.reject(error);
+  else entry.reject(error);
+}
+
+/** Structural idempotency gate (H5 §3.7.1). A non-idempotent op MUST carry a key, ALWAYS — throws
+ *  before the write is ever submitted, so a keyless non-idempotent write can never be retried into a
+ *  duplicate. `key` is the per-write key (the batch/coalesce paths pass each entry's own key). */
+function guardIdempotency(op: string, key: string | undefined): void {
+  if (key === undefined && !isIdempotentWriterOp(op)) throw new WriteRetrySafetyError(op);
+}
+
+/** Backpressure admission (§3.6/§3.6.1). `count` is the number of writes this submission would ADD (1
+ *  for a single op / one coalesce call, `argsList.length` for a batch): the caps reserve the FULL count
+ *  so a multi-entry batch cannot slip past a cap that its size would breach. The HARD cap bounds total
+ *  pending writes and rejects EVERY submission past it — including critical ones — so the queue (and a
+ *  wedge's held critical writes) can never grow without bound. Below the hard cap, non-critical writes
+ *  still shed past the soft high-water mark or while the writer is wedged; critical writes are admitted
+ *  (and held/queued) up to the hard cap. */
+function admitOrThrow(op: string, opts: WriteOpts | undefined, count: number): void {
+  if (shutdownDrain.closing) throw new Error("writer gateway is draining for shutdown");
+  // Occupancy = TOTAL pending writes = queued + the in-flight (busy) job (ADR §3.6 bounds total
+  // pending, not just queued — cycle-5 MED-2). A wedged writer's in-flight op must count against the
+  // cap, else queued:hardCap + busy:1 exceeds the stated bound. `busy` is at most 1 (single writer) and
+  // transient per write, so under the comfortably-large default cap this only bites near saturation.
+  const occupancy = pendingWriteDepth() + inFlightWrites();
+  if (occupancy + count > queueHardCap) throw new WriterOverloadedError(op, occupancy);
+  if (opts?.critical) return;
+  const workerWedged = wedged;
+  if (workerWedged || occupancy + count > queueHighWater) throw new WriterOverloadedError(op, occupancy);
+}
+
+/** Off-loop bodies consume submission data instead of looking at the main thread's live buffer. */
+function captureWriterPresence(op: string): PresenceSnapshot | undefined {
+  return backend?.kind === "worker" || isOffLoopWriterOp(op) ? snapshotBufferedPresence() : undefined;
+}
+
+function kickDrain(): void {
+  if (draining || drainScheduled) return;
+  drainScheduled = true;
+  // Deferred to a microtask so all same-tick dbCoalesce calls accumulate into one FIFO entry before
+  // the drain closes it (the amortization win, §3.9) — while still preserving global submission order.
+  queueMicrotask(() => {
+    drainScheduled = false;
+    void drainLoop().catch((error) => console.error(`[writer-gateway] drain loop failed: ${errMessage(error)}`));
+  });
+}
+
+/** Drains submissions to the pool in strict FIFO order, one at a time. Pauses (holds the remaining
+ *  FIFO) while the writer is wedged; resumes when the wedged op settles and re-kicks the drain. */
+async function drainLoop(): Promise<void> {
+  if (draining) return;
+  draining = true;
+  try {
+    // #2276 — start of the current on-loop time slice (in-process backend only; see
+    // DEFAULT_DRAIN_SLICE_MS). `draining` stays true across a yield, so `kickDrain` will not start a
+    // second concurrent drain and anything submitted meanwhile is picked up by this same loop when
+    // it resumes — FIFO is preserved, the queue is only PAUSED.
+    let sliceStartedAt = performance.now();
+    while (submissionQueue.length > 0 && !wedged) {
+      const entry = submissionQueue.shift()!;
+      const onLoop = backend?.kind === "in-process";
+      await dispatchEntry(entry);
+      if (!onLoop || drainSliceMs <= 0) {
+        // A worker dispatch already awaited a real thread hop, so the loop has had its turn and the
+        // slice restarts from here rather than counting the (off-loop) wait against the next entry.
+        sliceStartedAt = performance.now();
+        continue;
+      }
+      if (submissionQueue.length === 0 || wedged) continue;
+      if (performance.now() - sliceStartedAt < drainSliceMs) continue;
+      drainYields += 1;
+      await yieldEventLoop();
+      sliceStartedAt = performance.now();
+    }
+  } finally {
+    draining = false;
+    shutdownDrain.notify(!wedged && pendingWriteDepth() === 0 && inFlightWrites() === 0);
+  }
+}
+
+/** A MACROTASK yield — `setImmediate` runs after the poll phase, so pending I/O is served before the
+ *  drain resumes. A microtask (`await` on a resolved promise) would NOT do that: it stays inside the
+ *  same tick's checkpoint, which is exactly how a convoy became one uninterrupted block. */
+function yieldEventLoop(): Promise<void> {
+  return new Promise<void>((resolve) => {
+    setImmediate(resolve);
+  });
+}
+
+async function dispatchEntry(entry: Submission): Promise<void> {
+  const back = backend;
+  if (!back) {
+    rejectEntry(entry, new Error("writer gateway not initialized — call initWriter() after initDb()"));
+    return;
+  }
+  if (back.kind === "in-process") {
+    back.busy = true;
+    try {
+      const path = getDbPath();
+      if (path && path !== ":memory:" && isOffLoopWriterOp(entry.op)) {
+        if (back.offLoop && back.offLoopGeneration !== getDbGeneration()) {
+          await back.offLoop.pool.destroy();
+          back.offLoop = undefined;
+        }
+        back.stop.signal.throwIfAborted();
+        back.offLoop ??= createFileWriter(path);
+        back.offLoopGeneration = getDbGeneration();
+        await dispatchWorker(entry, back.offLoop);
+      } else await dispatchInProcess(entry, back.stop.signal);
+    } catch (error) { rejectEntry(entry, error); }
+    finally { back.busy = false; }
+    return;
+  }
+  await dispatchWorker(entry, back);
+}
+
+/**
+ * The per-entry dedup keys for a batch/coalesce job (H5 §3.7.1). A dbWriteBatch carries ONE key for
+ * the whole call and derives a stable per-entry key `${key}#${i}`, so a blind retry of the same batch
+ * dedups every entry. A coalesced job carries each independent caller's OWN key. An idempotent op has
+ * no key (undefined), so the executor runs it directly.
+ */
+function entryKeys(entry: Extract<Submission, { kind: "batch" | "coalesce" }>): (string | undefined)[] {
+  if (entry.kind === "batch") {
+    const base = entry.opts?.idempotencyKey;
+    return entry.argsList.map((_, i) => (base === undefined ? undefined : `${base}#${i}`));
+  }
+  return entry.resolvers.map((r) => r.idempotencyKey);
+}
+
+// ── In-process dispatch (synchronous op, marshaling round-trip — H1) ──────────────────────────────
+
+/**
+ * #1882 / ADR §5 — capture the op body HERE, not at the gateway facade. `dbWrite`/`dbTx`/`dbWriteBatch`/
+ * `dbCoalesce` only enqueue synchronously and defer the drain to a `queueMicrotask` (`kickDrain`), so
+ * the actual op body executes in THIS microtask — outside any facade-level timing. For the in-process
+ * backend that body runs synchronously ON the loop (it IS the on-loop block), so timing it here is what
+ * makes a >50ms write name its own op + `file:line` in the regression alarm. The callsite was pinned
+ * onto the submission at submission time (`resolveLoopCallsite`, in the caller's context), so it is
+ * correct even though the drain runs under some other caller's ambient context. The worker path
+ * (`dispatchWorker`) is deliberately NOT wrapped: its op body runs off-thread, so there is no on-loop
+ * block to attribute (only the sub-floor enqueue, which the record floor drops).
+ */
+async function dispatchInProcess(entry: Submission, signal: AbortSignal): Promise<void> {
+  const execute = <T>(fn: () => T) => onLoopWriteTransaction(
+    fn, (attempt) => captureSyncFrame({ op: entry.op, callsite: entry.callsite }, attempt), writerTimeoutMs, signal,
+  );
+  if (entry.kind === "op") {
+    try {
+      const result = cloneIn(await execute(() => runWriterOp(entry.op, cloneIn(entry.args), { now: entry.now, presence: cloneIn(entry.presence) }, entry.opts?.idempotencyKey)));
+      if (losePostCommitAckForTests(entry.op)) {
+        entry.reject(new WriteIndeterminateError(entry.op, new Error("injected post-commit lost acknowledgement"), entry.opts?.idempotencyKey));
+        return;
+      }
+      entry.resolve(result);
+    } catch (error) {
+      entry.reject(reconstructInProcess(error));
+    }
+    return;
+  }
+  // batch + coalesce share the SAVEPOINT-per-caller executor; only the fan-out differs.
+  let coalesced: CoalescedEntry[];
+  try {
+    const entries: BatchEntry[] = await execute(() => runWriterBatch(
+      entry.op,
+      entry.argsList.map((a) => cloneIn(a)),
+      entryKeys(entry),
+      { now: entry.now, presence: cloneIn(entry.presence) },
+    ));
+    coalesced = entries.map((e): CoalescedEntry =>
+      e.ok ? { ok: true, result: cloneIn(e.result) } : { ok: false, error: reconstructInProcess(e.error) },
+    );
+  } catch (error) {
+    // The OUTER transaction itself failed (not a single caller) — reject the whole batch.
+    const outer = reconstructInProcess(error);
+    if (entry.kind === "batch") entry.reject(outer);
+    else for (const r of entry.resolvers) r.reject(outer);
+    return;
+  }
+  if (entry.kind === "batch") entry.resolve(coalesced);
+  else fanOutCoalesce(entry, coalesced);
+}
+
+// ── Worker dispatch (real thread hop + gateway deadline / wedged policy — H4/H5) ───────────────────
+
+async function dispatchWorker(entry: Submission, back: WorkerBackend): Promise<void> {
+  const input: WriterJobInput =
+    entry.kind === "op"
+      ? { kind: "op", op: entry.op, args: entry.args, now: entry.now, presence: entry.presence, dbPath: back.dbPath, idempotencyKey: entry.opts?.idempotencyKey }
+      : { kind: "batch", op: entry.op, argsList: entry.argsList, keys: entryKeys(entry), now: entry.now, presence: entry.presence, dbPath: back.dbPath };
+  const labels: Record<string, string> =
+    entry.kind === "op" ? { op: entry.op } : { op: entry.op, batch: String(entry.argsList.length) };
+
+  // retrySafe stays false (default): a write must NEVER be auto-re-dispatched (it would repeat its
+  // effect). No pool timeout is armed — the deadline is the gateway's own (§3.6.1).
+  const poolPromise = back.pool.run<WriterJobInput, WriterJobResult>(WRITER_TASK_NAME, input, { labels });
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const raced = await Promise.race<{ done: "ok"; value: WriterJobResult } | { done: "err"; error: unknown } | { done: "timeout" }>([
+    poolPromise.then((value) => ({ done: "ok" as const, value }), (error) => ({ done: "err" as const, error })),
+    new Promise((resolve) => {
+      timer = setTimeout(() => resolve({ done: "timeout" as const }), writerTimeoutMs);
+    }),
+  ]);
+  if (timer) clearTimeout(timer);
+
+  if (raced.done === "timeout") {
+    // WEDGED (§3.6.1). The op is still running and still holds the lock; do NOT replace the worker.
+    // Surface the caller's INDETERMINATE outcome, hold the FIFO, and resume when the op finally
+    // settles (it commits/rolls back and releases the lock on the SAME warm connection).
+    wedged = true;
+    rejectIndeterminate(entry);
+    poolPromise.then(
+      () => {},
+      () => {},
+    ).finally(() => {
+      wedged = false;
+      kickDrain();
+    }).catch((error) => console.error(`[writer-gateway] resume after wedge failed: ${errMessage(error)}`));
+    return;
+  }
+  if (raced.done === "err") {
+    // The worker died (crash/OOM) with the job in flight: the write MAY have committed (H5). The pool
+    // has already respawned the slot, so the FIFO simply continues on the fresh worker.
+    rejectIndeterminate(entry, raced.error);
+    return;
+  }
+  fanOutWorker(entry, raced.value);
+}
+
+function rejectIndeterminate(entry: Submission, cause?: unknown): void {
+  const reason = cause ?? new Error("writer op exceeded its deadline (wedged)");
+  if (entry.kind === "coalesce") {
+    // Each coalesced caller reconciles via ITS OWN key.
+    for (const r of entry.resolvers) r.reject(new WriteIndeterminateError(entry.op, reason, r.idempotencyKey));
+  } else {
+    entry.reject(new WriteIndeterminateError(entry.op, reason, entry.opts?.idempotencyKey));
+  }
+}
+
+function fanOutWorker(entry: Submission, result: WriterJobResult): void {
+  if (entry.kind === "op") {
+    if (result.kind !== "op") {
+      entry.reject(new Error(`writer worker returned a ${result.kind} result for a single op`));
+      return;
+    }
+    if (result.ok) entry.resolve(result.result);
+    else entry.reject(reconstructDbError(result.error));
+    return;
+  }
+  if (result.kind !== "batch" || "error" in result) {
+    const failed = result.kind === "batch" && "error" in result ? reconstructDbError(result.error)
+      : new Error(`writer worker returned a ${result.kind} result for a batch`);
+    if (entry.kind === "batch") entry.reject(failed);
+    else for (const r of entry.resolvers) r.reject(failed);
+    return;
+  }
+  const coalesced: CoalescedEntry[] = result.entries.map((e: SerializedBatchEntry) =>
+    e.ok ? { ok: true, result: e.result } : { ok: false, error: reconstructDbError(e.error) },
+  );
+  if (entry.kind === "batch") entry.resolve(coalesced);
+  else fanOutCoalesce(entry, coalesced);
+}
+
+function fanOutCoalesce(entry: Extract<Submission, { kind: "coalesce" }>, coalesced: CoalescedEntry[]): void {
+  for (let i = 0; i < entry.resolvers.length; i++) {
+    const outcome = coalesced[i];
+    const r = entry.resolvers[i]!;
+    if (outcome && outcome.ok) r.resolve(outcome.result);
+    else r.reject(outcome ? outcome.error : new Error("coalesced entry missing"));
+  }
+}
+
+// ── Single-op write / tx ──────────────────────────────────────────────────────────────────────────
+
+/**
+ * Route a single write through the serialized writer queue. `op` is a registered op name (never a
+ * closure); `args` must be structured-clone-safe. Resolves only AFTER the commit has durably landed
+ * (INV-1), so a read issued after this resolves sees the write.
+ */
+export function dbWrite<R = unknown>(op: string, args: unknown, opts?: WriteOpts): Promise<R> {
+  return new Promise<R>((resolve, reject) => {
+    // The guards run inside the returned promise so an uninitialized gateway, a keyless non-idempotent
+    // write, or a backpressure shed REJECTS rather than throwing synchronously — an async facade must
+    // never throw before returning.
+    try {
+      requireBackend();
+      guardIdempotency(op, opts?.idempotencyKey);
+      admitOrThrow(op, opts, 1);
+    } catch (guard) {
+      reject(guard);
+      return;
+    }
+    submissionQueue.push({ kind: "op", op, args, opts, now: Date.now(), presence: captureWriterPresence(op), callsite: resolveLoopCallsite(opts?.callsite), resolve: resolve as (v: unknown) => void, reject });
+    kickDrain();
+  });
+}
+
+/**
+ * Atomic multi-statement transaction across the boundary. Semantically identical to `dbWrite` — the
+ * op body already runs inside one BEGIN IMMEDIATE on the writer thread (§3.4) — but named separately
+ * so callers reading the code can see a multi-statement unit is atomic. Interactive/loop-driven
+ * transactions are not supported by design (the pin-the-writer anti-pattern); a transaction is always
+ * a single registered op.
+ */
+export function dbTx<R = unknown>(op: string, args: unknown, opts?: WriteOpts): Promise<R> {
+  return dbWrite<R>(op, args, opts);
+}
+
+// ── Coalesced batch (SAVEPOINT-per-caller, H3) ────────────────────────────────────────────────────
+
+/**
+ * Coalesce INDEPENDENT callers of the same op into ONE writer job with per-caller SAVEPOINT isolation
+ * (§3.9.1 / H3). One outer BEGIN IMMEDIATE amortizes the lock + fsync across the batch; a throw from
+ * one caller rolls back only that caller and the rest still commit. Returns a per-caller `result |
+ * error` array aligned to `argsList`. Sheds under backpressure/wedge and rejects an over-cap batch
+ * (finding 5) — one job must never pin the sole writer on an unbounded `argsList`.
+ */
+export function dbWriteBatch<R = unknown>(op: string, argsList: readonly unknown[], opts?: WriteOpts): Promise<CoalescedEntry<R>[]> {
+  return new Promise<CoalescedEntry<R>[]>((resolve, reject) => {
+    if (argsList.length === 0) {
+      resolve([]);
+      return;
+    }
+    try {
+      requireBackend();
+      guardIdempotency(op, opts?.idempotencyKey);
+      if (argsList.length > MAX_WRITE_BATCH) {
+        throw new WriterOverloadedError(op, argsList.length);
+      }
+      // Reserve the FULL batch size against the caps — a batch is one FIFO entry but N pending writes,
+      // so a critical N-entry batch must be rejected whole if it would breach the hard cap (HIGH-1).
+      admitOrThrow(op, opts, argsList.length);
+    } catch (guard) {
+      reject(guard);
+      return;
+    }
+    submissionQueue.push({
+      kind: "batch",
+      op,
+      argsList: [...argsList],
+      opts,
+      now: Date.now(),
+      presence: captureWriterPresence(op),
+      callsite: resolveLoopCallsite(opts?.callsite),
+      resolve: resolve as (v: unknown) => void,
+      reject,
+    });
+    kickDrain();
+  });
+}
+
+// ── Same-tick coalescing (§3.9 / open-question 3: queueMicrotask, not a timer) ────────────────────
+
+/**
+ * Buffer this call with other same-op calls made in the SAME tick and flush them as one coalesced
+ * batch (§3.9). Unlike a per-op side buffer, the coalesced job is placed into the SAME global FIFO at
+ * submission time (finding 2), so a same-tick `dbWrite` can never overtake an earlier `dbCoalesce`.
+ * Each caller's promise resolves/rejects INDEPENDENTLY from its own savepoint entry. Sheds under
+ * backpressure/wedge and caps a single coalesced job at `MAX_WRITE_BATCH` (finding 5), rolling over
+ * into a fresh FIFO entry rather than growing one job without bound.
+ */
+export function dbCoalesce<R = unknown>(op: string, args: unknown, opts?: WriteOpts): Promise<R> {
+  return new Promise<R>((resolve, reject) => {
+    try {
+      requireBackend();
+      guardIdempotency(op, opts?.idempotencyKey);
+      admitOrThrow(op, opts, 1);
+    } catch (guard) {
+      reject(guard);
+      return;
+    }
+    const resolver: Resolver = { resolve: resolve as (v: unknown) => void, reject, idempotencyKey: opts?.idempotencyKey };
+    // Append to the tail entry ONLY if it is still an open coalesce of the same op and under the cap —
+    // any intervening single/batch/other-op submission starts a fresh entry, which is exactly what
+    // preserves global FIFO across coalesced + normal writes.
+    const tail = submissionQueue[submissionQueue.length - 1];
+    if (tail && tail.kind === "coalesce" && tail.op === op && tail.argsList.length < MAX_WRITE_BATCH) {
+      tail.argsList.push(args);
+      tail.resolvers.push(resolver);
+    } else {
+      submissionQueue.push({ kind: "coalesce", op, opts, now: Date.now(), presence: captureWriterPresence(op), callsite: resolveLoopCallsite(opts?.callsite), argsList: [args], resolvers: [resolver] });
+    }
+    kickDrain();
+  });
+}