@haverstack/server 0.8.0

package/src/lib/queryWorker/pool.ts

@@ -0,0 +1,324 @@
+/**
+ * Main-thread side of the query worker boundary. Owns a small, fixed-size
+ * pool of workers (src/lib/queryWorker/worker.ts), each with its own
+ * LocalAdapter connection to the same stack.db, and dispatches
+ * ScopedStack.query() calls to them. The operator-facing account of why
+ * this exists is docs/deployment.md § Bounding query cost.
+ *
+ * Two bounds, deliberately not one:
+ *
+ *   - `deadlineMs` bounds **execution**, and its clock starts when a
+ *     request reaches a worker, not when it was accepted here. Execution
+ *     is the only part that needs bounding: node:sqlite exposes no
+ *     `sqlite3_interrupt` equivalent, so a query already inside the engine
+ *     cannot be cancelled in place. Queueing makes a request slow, never
+ *     failed.
+ *
+ *   - `queueLimit` bounds **the wait**, so "slow" cannot quietly become
+ *     unbounded in latency or retained memory. Past it the pool sheds load
+ *     with StackTimeoutError — a retryable 503, which is honest about
+ *     overload in a way a silently growing queue is not.
+ *
+ * Past its deadline the only lever left is termination: the caller is
+ * answered at once and the worker is terminated and replaced, so a stuck
+ * query costs one slot rather than wedging the pool. That recycle needs no
+ * backoff, being rate-limited by construction at `poolSize` terminations
+ * per `deadlineMs`.
+ *
+ * An *unexpected* death is the opposite case and does back off: respawning
+ * a worker that can't boot — an unreadable DB_PATH, say — as fast as the
+ * loop allows would spin threads and flood logs. After
+ * MAX_CONSECUTIVE_SPAWN_FAILURES the pool reports itself unhealthy and
+ * fails queries with the underlying spawn error, so a fatal
+ * misconfiguration surfaces as itself rather than as a pile of timeouts.
+ * Respawns continue at the capped interval, so the pool heals once the
+ * cause clears.
+ *
+ * Terminating mid-write is safe: WAL journaling leaves an uncommitted
+ * transaction simply absent once the file is reopened. Only query() routes
+ * through this pool — src/routes/records.ts holds the reason writes and
+ * by-id reads stay on the main thread.
+ */
+import { Worker } from 'node:worker_threads';
+import type { StackQuery, QueryResult, TokenSession } from '@haverstack/core';
+import { StackTimeoutError } from '@haverstack/core';
+import { deserializeError } from '@haverstack/wire-types';
+import type { Logger } from 'pino';
+import type { QueryRequest, QueryResponse, QueryWorkerInit } from './protocol.js';
+
+// import.meta.url ends in .ts under tsx (dev) and vitest (test), and in
+// .js once tsc has compiled this file into dist/ — so which worker file
+// and loader to use falls out of how *this* module was loaded, with no
+// separate NODE_ENV check to keep in sync.
+const RUNNING_FROM_SOURCE = import.meta.url.endsWith('.ts');
+const WORKER_URL = new URL(RUNNING_FROM_SOURCE ? './worker.ts' : './worker.js', import.meta.url);
+const WORKER_EXEC_ARGV = RUNNING_FROM_SOURCE ? ['--import', 'tsx'] : [];
+
+/** Backoff schedule for respawning a worker that died unexpectedly. */
+const RESPAWN_BASE_DELAY_MS = 100;
+const RESPAWN_MAX_DELAY_MS = 30_000;
+/** Consecutive unexpected deaths after which the pool reports itself unhealthy. */
+const MAX_CONSECUTIVE_SPAWN_FAILURES = 5;
+
+type Pending = {
+  resolve: (result: QueryResult) => void;
+  reject: (err: Error) => void;
+  deadlineMs: number;
+  /** The execution clock. Null while the request is still queued. */
+  timer: NodeJS.Timeout | null;
+};
+
+type Slot = {
+  /** Null while the worker is dead and its respawn is pending. */
+  worker: Worker | null;
+  /** id of the request currently running on this worker, or null if idle. */
+  busyWith: number | null;
+  respawnTimer: NodeJS.Timeout | null;
+};
+
+export type QueryWorkerPoolOptions = {
+  init: QueryWorkerInit;
+  poolSize: number;
+  /** Maximum requests waiting for a slot before the pool sheds load. */
+  queueLimit: number;
+  logger: Logger;
+  /**
+   * Worker entry point. Defaults to this directory's worker; overridable
+   * so tests can drive the pool with a fixture worker whose timing they
+   * control, rather than inferring pool behavior from real query latency.
+   */
+  workerUrl?: URL;
+};
+
+export class QueryWorkerPool {
+  private readonly slots: Slot[] = [];
+  private readonly pending = new Map<number, Pending>();
+  /** FIFO of requests waiting for a slot to free up. */
+  private readonly waiting: Array<{ id: number; req: QueryRequest }> = [];
+  private nextId = 1;
+  private closed = false;
+  private consecutiveFailures = 0;
+  private lastFailure: Error | null = null;
+
+  private readonly init: QueryWorkerInit;
+  private readonly queueLimit: number;
+  private readonly logger: Logger;
+  private readonly workerUrl: URL;
+
+  constructor(opts: QueryWorkerPoolOptions) {
+    this.init = opts.init;
+    this.queueLimit = opts.queueLimit;
+    this.logger = opts.logger;
+    this.workerUrl = opts.workerUrl ?? WORKER_URL;
+    for (let i = 0; i < opts.poolSize; i++) {
+      const slot: Slot = { worker: null, busyWith: null, respawnTimer: null };
+      this.slots.push(slot);
+      this.startWorker(slot);
+    }
+  }
+
+  /**
+   * True once every recent spawn attempt has failed — a standing fault
+   * (bad DB_PATH, unreadable file) rather than one unlucky crash.
+   */
+  private get unhealthy(): boolean {
+    return this.consecutiveFailures >= MAX_CONSECUTIVE_SPAWN_FAILURES;
+  }
+
+  private startWorker(slot: Slot): void {
+    if (this.closed) return;
+    slot.respawnTimer = null;
+    const worker = new Worker(this.workerUrl, {
+      workerData: this.init,
+      execArgv: WORKER_EXEC_ARGV,
+    });
+    slot.worker = worker;
+    slot.busyWith = null;
+    worker.on('message', (msg: QueryResponse) => this.handleMessage(slot, msg));
+    worker.on('error', (err: Error) => this.handleWorkerDeath(slot, err));
+    worker.on('exit', (code) => {
+      if (!this.closed && code !== 0) {
+        this.handleWorkerDeath(slot, new Error(`query worker exited with code ${code}`));
+      }
+    });
+    // postMessage before a worker finishes booting is fine — worker_threads
+    // queues it until the thread installs its listener.
+    this.drainWaiting(slot);
+  }
+
+  private handleMessage(slot: Slot, msg: QueryResponse): void {
+    // A worker that answers is a working worker: clear any standing
+    // failure state so an earlier rough patch can't leave the pool
+    // permanently marked unhealthy.
+    this.consecutiveFailures = 0;
+    this.lastFailure = null;
+    slot.busyWith = null;
+    const pending = this.pending.get(msg.id);
+    // Already timed out (and rejected) before the worker answered — the
+    // late reply just frees the slot, above.
+    if (pending) {
+      this.pending.delete(msg.id);
+      if (pending.timer) clearTimeout(pending.timer);
+      if (msg.ok) pending.resolve(msg.result);
+      else pending.reject(msg.wire ? deserializeError(msg.wire.body) : new Error(msg.message));
+    }
+    this.drainWaiting(slot);
+  }
+
+  /**
+   * A worker crashed or exited on its own — not a deadline recycle, which
+   * is deliberate and handled by recycleSlot() without backoff.
+   */
+  private handleWorkerDeath(slot: Slot, err: Error): void {
+    const failedId = slot.busyWith;
+    if (failedId !== null) {
+      const pending = this.pending.get(failedId);
+      if (pending) {
+        this.pending.delete(failedId);
+        if (pending.timer) clearTimeout(pending.timer);
+        pending.reject(err);
+      }
+    }
+    this.teardown(slot);
+    this.consecutiveFailures++;
+    this.lastFailure = err;
+    const delay = Math.min(
+      RESPAWN_BASE_DELAY_MS * 2 ** (this.consecutiveFailures - 1),
+      RESPAWN_MAX_DELAY_MS,
+    );
+    this.logger.error(
+      { err, consecutiveFailures: this.consecutiveFailures, retryInMs: delay },
+      'Query worker died unexpectedly; respawning after backoff',
+    );
+    // Nothing is going to serve the queue any time soon — answer it with
+    // the real cause instead of letting it wait out a fault that isn't
+    // going to clear on its own.
+    if (this.unhealthy) this.failQueued(err);
+    this.scheduleRespawn(slot, delay);
+  }
+
+  /** Deliberate recycle after a deadline: capacity is restored at once, no backoff. */
+  private recycleSlot(slot: Slot): void {
+    if (this.closed) return;
+    this.teardown(slot);
+    this.startWorker(slot);
+  }
+
+  private teardown(slot: Slot): void {
+    const worker = slot.worker;
+    slot.worker = null;
+    slot.busyWith = null;
+    if (!worker) return;
+    worker.removeAllListeners();
+    void worker.terminate().catch(() => {});
+  }
+
+  private scheduleRespawn(slot: Slot, delay: number): void {
+    if (this.closed) return;
+    if (slot.respawnTimer) clearTimeout(slot.respawnTimer);
+    slot.respawnTimer = setTimeout(() => this.startWorker(slot), delay);
+    // A pending respawn must not by itself keep the process alive.
+    slot.respawnTimer.unref();
+  }
+
+  private failQueued(err: Error): void {
+    for (const { id } of this.waiting.splice(0)) {
+      const pending = this.pending.get(id);
+      if (!pending) continue;
+      this.pending.delete(id);
+      if (pending.timer) clearTimeout(pending.timer);
+      pending.reject(err);
+    }
+  }
+
+  private drainWaiting(slot: Slot): void {
+    if (this.closed || !slot.worker || slot.busyWith !== null) return;
+    // Looped, not recursive: a long run of already-abandoned queue entries
+    // shouldn't cost stack depth.
+    for (;;) {
+      const next = this.waiting.shift();
+      if (!next) return;
+      const pending = this.pending.get(next.id);
+      // Abandoned while queued (pool went unhealthy, or close()).
+      if (!pending) continue;
+      this.startExecution(slot, next.id, next.req, pending);
+      return;
+    }
+  }
+
+  /** Hand a request to a worker and start its execution clock — see the module doc. */
+  private startExecution(slot: Slot, id: number, req: QueryRequest, pending: Pending): void {
+    pending.timer = setTimeout(() => this.onDeadline(id), pending.deadlineMs);
+    slot.busyWith = id;
+    slot.worker!.postMessage(req);
+  }
+
+  private onDeadline(id: number): void {
+    const pending = this.pending.get(id);
+    if (!pending) return;
+    this.pending.delete(id);
+    pending.reject(
+      new StackTimeoutError(`Query exceeded the ${pending.deadlineMs}ms execution deadline`),
+    );
+    const slot = this.slots.find((s) => s.busyWith === id);
+    if (slot) this.recycleSlot(slot);
+  }
+
+  /**
+   * Run a query on the pool. `deadlineMs` bounds execution only; a request
+   * that waits for a slot waits without penalty, until the queue itself is
+   * full.
+   */
+  query(
+    session: TokenSession | null,
+    stackQuery: StackQuery,
+    deadlineMs: number,
+  ): Promise<QueryResult> {
+    if (this.closed) return Promise.reject(new Error('Query worker pool is closed'));
+    // A standing spawn fault is a server fault, not a slow query: report
+    // the cause rather than making every caller wait out a deadline.
+    if (this.unhealthy) {
+      return Promise.reject(this.lastFailure ?? new Error('Query worker pool is unavailable'));
+    }
+    if (this.waiting.length >= this.queueLimit) {
+      return Promise.reject(
+        new StackTimeoutError(
+          `Query queue is full (${this.queueLimit} waiting); the server is shedding load rather than queueing further`,
+        ),
+      );
+    }
+    const id = this.nextId++;
+    const req: QueryRequest = { id, session, query: stackQuery };
+    return new Promise<QueryResult>((resolve, reject) => {
+      const pending: Pending = { resolve, reject, deadlineMs, timer: null };
+      this.pending.set(id, pending);
+      const idle = this.slots.find((s) => s.worker !== null && s.busyWith === null);
+      if (idle) this.startExecution(idle, id, req, pending);
+      else this.waiting.push({ id, req });
+    });
+  }
+
+  async close(): Promise<void> {
+    this.closed = true;
+    for (const pending of this.pending.values()) {
+      if (pending.timer) clearTimeout(pending.timer);
+      pending.reject(new Error('Query worker pool is closing'));
+    }
+    this.pending.clear();
+    this.waiting.length = 0;
+    await Promise.all(
+      this.slots.map(async (slot) => {
+        if (slot.respawnTimer) {
+          clearTimeout(slot.respawnTimer);
+          slot.respawnTimer = null;
+        }
+        const worker = slot.worker;
+        slot.worker = null;
+        slot.busyWith = null;
+        if (!worker) return;
+        worker.removeAllListeners();
+        await worker.terminate();
+      }),
+    );
+  }
+}

package/src/lib/queryWorker/protocol.ts

@@ -0,0 +1,39 @@
+/**
+ * Message shapes exchanged between the main thread and a query worker.
+ * Both sides structured-clone these over `postMessage` — no custom
+ * serialization needed for the plain-data fields (StackQuery's Date
+ * filters and QueryResult's records clone natively), but a thrown
+ * StackError needs the same wire encoding used at the HTTP boundary
+ * (serializeError/deserializeError from @haverstack/wire-types) since a
+ * custom Error subclass doesn't survive structured clone as itself.
+ */
+import type { StackQuery, QueryResult, TokenSession } from '@haverstack/core';
+import type { WireError } from '@haverstack/wire-types';
+
+/**
+ * Passed as `workerData` when a query worker is spawned. dbPath is the
+ * whole of it: a worker calls LocalAdapter.open(), which reads timezone
+ * (and the rest of the stack config) out of the database itself — only
+ * the initialize path takes one as input, and workers never initialize.
+ */
+export type QueryWorkerInit = {
+  dbPath: string;
+};
+
+export type QueryRequest = {
+  id: number;
+  /** null = anonymous (Stack.asEntity(null)), matching records.ts's scopeFor(). */
+  session: TokenSession | null;
+  query: StackQuery;
+};
+
+export type QueryResponse =
+  | { id: number; ok: true; result: QueryResult }
+  | {
+      id: number;
+      ok: false;
+      /** Set when the failure is a StackError with a wire mapping; reconstructed via deserializeError(). */
+      wire: { status: number; body: WireError } | null;
+      /** Always set, for logging — the only signal left when `wire` is null (an unanticipated bug). */
+      message: string;
+    };

package/src/lib/queryWorker/worker.ts

@@ -0,0 +1,51 @@
+/**
+ * Query worker entry point. One of these owns a second (or third, ...)
+ * LocalAdapter connection to the same stack.db as the main thread's.
+ * That's legal against docs/spec/adapters.md's storage-ownership lock —
+ * acquireLock() in @haverstack/sqlite-shared keys purely on process.pid,
+ * and worker_threads share process.pid with the thread that spawned
+ * them, so every worker's open() sees `ownedBySelf: true` — and legal
+ * against SQLite itself, since WAL mode is built for exactly this:
+ * multiple connections against one file, coordinated by the engine.
+ *
+ * Runs Stack.query()/ScopedStack.query() off the request thread per
+ * docs/spec/wire-format.md § Bounding query cost, so a slow full-text
+ * search blocks only this worker — the main thread stays free to accept
+ * and answer other requests, and can terminate this worker outright if
+ * the query outlives its deadline (see pool.ts).
+ */
+import { parentPort, workerData } from 'node:worker_threads';
+import { LocalAdapter } from '@haverstack/adapter-local';
+import { Stack } from '@haverstack/core';
+import { serializeError } from '@haverstack/wire-types';
+import type { QueryRequest, QueryResponse, QueryWorkerInit } from './protocol.js';
+
+if (!parentPort) {
+  throw new Error('query worker must be run as a worker_thread, not the main thread');
+}
+
+const init = workerData as QueryWorkerInit;
+
+// The database already exists — the main thread's initStack() created or
+// opened it, and seeded system types, before any worker is spawned (see
+// pool.ts). open() (not openOrInitialize()) reflects that: there is no
+// first-run path to handle here.
+const adapter = await LocalAdapter.open({ path: init.dbPath });
+const stack = await Stack.create(adapter);
+
+parentPort.on('message', async (req: QueryRequest) => {
+  const port = parentPort!;
+  try {
+    const scoped = req.session ? stack.forSession(req.session) : stack.asEntity(null);
+    const result = await scoped.query(req.query);
+    port.postMessage({ id: req.id, ok: true, result } satisfies QueryResponse);
+  } catch (err) {
+    const wire = serializeError(err);
+    port.postMessage({
+      id: req.id,
+      ok: false,
+      wire,
+      message: err instanceof Error ? err.message : String(err),
+    } satisfies QueryResponse);
+  }
+});

package/src/lib/resumeBuffer.ts

@@ -0,0 +1,268 @@
+/**
+ * Per-(session, filter) resume buffers for GET /changes. Sits downstream
+ * of each connection's `ScopedStack.subscribe()`, so every entry here is a
+ * frame that subscription already delivered — something this session was
+ * cleared to see at the moment it happened. That is what keeps replay a
+ * pure re-send needing nothing from core, where a single server-wide
+ * buffer would owe a permission check per frame per reader.
+ *
+ * A buffer mints its own cursors (its position is a private counter, not a
+ * slice of some global sequence) and is fed independently of any one HTTP
+ * connection: it keeps collecting for a bounded window *after* the last
+ * connection using it disconnects, which is what lets a reconnect learn
+ * about changes made while nobody was listening. Depth-bounded on top of
+ * that: a session that stays away long enough to overflow the ring gets
+ * `overflow` rather than a silently incomplete replay.
+ */
+import type { ChangeFilter, RecordChange, Unsubscribe } from '@haverstack/core';
+import { serializeChange } from '@haverstack/wire-types';
+import type { WireRecordChange } from '@haverstack/wire-types';
+import { encodeCursor } from './resumeCursor.js';
+
+export type ResumeEntry = {
+  n: number;
+  recordId: string;
+  isPurge: boolean;
+  /** Wire-ready, seq already attached. */
+  frame: WireRecordChange;
+};
+
+export type EntriesAfterResult =
+  | { status: 'ok'; entries: ResumeEntry[] }
+  /** The buffer is the right one, but depth eviction already dropped what would be needed. */
+  | { status: 'overflow' }
+  /** A position this buffer never held, or no longer holds any record of. */
+  | { status: 'cursor_expired' };
+
+/** One session+filter's resume buffer. Data and notification only — lifecycle lives in the registry. */
+export class ResumeBuffer {
+  readonly id: string = crypto.randomUUID();
+  private readonly entries: ResumeEntry[] = [];
+  private currentN = 0;
+  private evictedUpTo = 0;
+  private readonly listeners = new Set<(entry: ResumeEntry) => void>();
+
+  /** Connections currently holding this buffer open. Registry-managed. */
+  liveCount = 0;
+  /** Set by the registry when liveCount drops to 0; cleared on reacquire. */
+  disconnectedAt: number | null = null;
+
+  constructor(private readonly depth: number) {}
+
+  /** The cursor naming this buffer's current position — what `ready` reports. */
+  headCursor(): string {
+    return encodeCursor(this.id, this.currentN);
+  }
+
+  /**
+   * Record one already-permission-filtered change, minting its cursor.
+   * Notifies every currently attached live connection with the same frame
+   * a replay would later produce.
+   */
+  append(change: RecordChange): ResumeEntry {
+    this.currentN += 1;
+    const seq = encodeCursor(this.id, this.currentN);
+    const frame = serializeChange({ ...change, seq });
+    const entry: ResumeEntry = {
+      n: this.currentN,
+      recordId: change.recordId,
+      isPurge: change.kind === 'purged',
+      frame,
+    };
+    this.entries.push(entry);
+    if (this.entries.length > this.depth) {
+      const dropped = this.entries.shift();
+      if (dropped) this.evictedUpTo = dropped.n;
+    }
+    for (const listener of this.listeners) listener(entry);
+    return entry;
+  }
+
+  /** Attach a live connection. Returns the detach function. */
+  subscribeLive(listener: (entry: ResumeEntry) => void): () => void {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  /**
+   * Everything after position `n`, or why that can't be answered. `n` is
+   * this server's own minted position, already decoded and matched to
+   * this buffer's id by the caller.
+   */
+  entriesAfter(n: number): EntriesAfterResult {
+    if (n > this.currentN || n < 0) return { status: 'cursor_expired' };
+    if (n < this.evictedUpTo) return { status: 'overflow' };
+    return { status: 'ok', entries: this.entries.filter((e) => e.n > n) };
+  }
+}
+
+export type ResumeBufferKeyParts = {
+  /** Both null for an anonymous connection. */
+  principalId: string | null;
+  subjectId: string | null;
+  filter: ChangeFilter;
+  includeRecords: boolean;
+  includeUnlisted: boolean;
+};
+
+/**
+ * Reproducible across a reconnect that re-sends the same query params —
+ * order-independent on the parts of `filter` that don't carry order of
+ * their own (`typeId`/`kinds` are matched as sets, not sequences, so
+ * re-sending them in a different order must key the same buffer).
+ *
+ * Two filters that mean different things must never key the same buffer.
+ * A buffer opens exactly one `ScopedStack.subscribe()`, carrying the
+ * filter of whichever connection minted it, so a collision hands one of
+ * the two colliding connections the *other's* filter: either a silent gap
+ * (events it asked for and never hears about, with no `reset` to tell it
+ * so) or frames outside its filter, which "filtering is exact, not
+ * advisory" forbids. See docs/spec/wire-format.md § Change feed.
+ *
+ * Hence an object rather than a positional array: `JSON.stringify` drops
+ * an undefined *property* but renders an undefined *array element* as
+ * `null`, which would collapse "no parentId filter" onto `parentId: null`
+ * — "root records only", a filter the wire format defines. An absent part
+ * is an absent key, which no present value can imitate. Key order is fixed
+ * by construction below.
+ */
+export function resumeBufferKey(parts: ResumeBufferKeyParts): string {
+  const { filter } = parts;
+  const typeId = filter.typeId
+    ? [...(Array.isArray(filter.typeId) ? filter.typeId : [filter.typeId])].sort()
+    : undefined;
+  const kinds = filter.kinds ? [...filter.kinds].sort() : undefined;
+  return JSON.stringify({
+    principalId: parts.principalId,
+    subjectId: parts.subjectId,
+    includeRecords: parts.includeRecords,
+    includeUnlisted: parts.includeUnlisted,
+    ...(typeId !== undefined && { typeId }),
+    ...(filter.parentId !== undefined && { parentId: filter.parentId }),
+    ...(filter.entityId !== undefined && { entityId: filter.entityId }),
+    ...(kinds !== undefined && { kinds }),
+  });
+}
+
+export type ResumeBufferRegistryOptions = {
+  /** Max entries retained per buffer before the oldest is dropped. */
+  depth: number;
+  /** How long a buffer keeps being fed after its last connection disconnects, before it's dropped entirely. */
+  retentionMs: number;
+};
+
+/**
+ * Owns every live resume buffer, keyed by (session, filter, includeRecords).
+ * A buffer's underlying `ScopedStack.subscribe()` is opened once, when the
+ * buffer is first minted, and kept open — including across a gap with zero
+ * live connections — until the buffer itself is evicted.
+ */
+export class ResumeBufferRegistry {
+  private readonly buffers = new Map<string, ResumeBuffer>();
+  private readonly unsubscribes = new Map<string, Unsubscribe>();
+  private readonly evictionTimers = new Map<string, NodeJS.Timeout>();
+  /**
+   * Buffers whose subscription is still opening. A key lands in `buffers`
+   * only once its `ScopedStack.subscribe()` has resolved, so this is what
+   * keeps two connections arriving in the same tick from opening two
+   * subscriptions for one key — they await the same attempt and share
+   * whichever way it settles.
+   */
+  private readonly acquiring = new Map<string, Promise<ResumeBuffer>>();
+
+  constructor(private readonly opts: ResumeBufferRegistryOptions) {}
+
+  /**
+   * Get this key's buffer, creating one and opening its scoped subscription
+   * if none is currently retained. `subscribeToStack` is called at most
+   * once per buffer instance.
+   *
+   * A failed subscription registers nothing: the next connection on this
+   * key retries it. Registering a buffer that nothing is feeding would
+   * hand every later connection on that key a `ready` frame followed by
+   * permanent silence — a gap with no `reset` to announce it, which is
+   * the one failure the feed contract calls untrustworthy.
+   */
+  async acquire(
+    key: string,
+    subscribeToStack: (onChange: (change: RecordChange) => void) => Promise<Unsubscribe>,
+  ): Promise<ResumeBuffer> {
+    const existing = this.buffers.get(key);
+    if (existing) {
+      this.cancelEviction(key);
+      existing.liveCount += 1;
+      existing.disconnectedAt = null;
+      return existing;
+    }
+
+    const inFlight = this.acquiring.get(key);
+    if (inFlight) {
+      const buffer = await inFlight;
+      buffer.liveCount += 1;
+      return buffer;
+    }
+
+    const attempt = (async () => {
+      const buffer = new ResumeBuffer(this.opts.depth);
+      // Awaited before either map is touched — the handler closes over
+      // `buffer`, so anything the stack emits while this is in flight is
+      // already being recorded by the time the buffer becomes reachable.
+      const unsubscribe = await subscribeToStack((change) => {
+        buffer.append(change);
+      });
+      this.buffers.set(key, buffer);
+      this.unsubscribes.set(key, unsubscribe);
+      return buffer;
+    })();
+    this.acquiring.set(key, attempt);
+
+    try {
+      const buffer = await attempt;
+      buffer.liveCount += 1;
+      return buffer;
+    } finally {
+      this.acquiring.delete(key);
+    }
+  }
+
+  /** Detach one connection from this key's buffer, starting its retention countdown once none remain. */
+  release(key: string): void {
+    const buffer = this.buffers.get(key);
+    if (!buffer) return;
+    buffer.liveCount = Math.max(0, buffer.liveCount - 1);
+    if (buffer.liveCount > 0) return;
+    buffer.disconnectedAt = Date.now();
+    const timer = setTimeout(() => this.evict(key, buffer), this.opts.retentionMs);
+    timer.unref?.();
+    this.evictionTimers.set(key, timer);
+  }
+
+  private cancelEviction(key: string): void {
+    const timer = this.evictionTimers.get(key);
+    if (timer) {
+      clearTimeout(timer);
+      this.evictionTimers.delete(key);
+    }
+  }
+
+  private evict(key: string, buffer: ResumeBuffer): void {
+    if (this.buffers.get(key) !== buffer) return; // already replaced
+    if (buffer.liveCount > 0) return; // reacquired since the timer was set
+    this.buffers.delete(key);
+    this.evictionTimers.delete(key);
+    this.unsubscribes.get(key)?.();
+    this.unsubscribes.delete(key);
+  }
+
+  /** Tear down every retained buffer immediately — for shutdown. */
+  closeAll(): void {
+    for (const timer of this.evictionTimers.values()) clearTimeout(timer);
+    this.evictionTimers.clear();
+    this.acquiring.clear();
+    for (const unsubscribe of this.unsubscribes.values()) unsubscribe();
+    this.unsubscribes.clear();
+    this.buffers.clear();
+  }
+}