@mlx-node/server 0.0.12 → 0.0.15

package/src/session-registry.ts

@@ -0,0 +1,1371 @@
+/**
+ * SessionRegistry -- per-model cache holding AT MOST one live
+ * `ChatSession` whose native KV state is currently valid.
+ *
+ * **Tier-2 `prompt_cache_key` reuse is ON by default** so the server
+ * is compatible with any stateless LLM agent that sends the full
+ * conversation history each turn. The key is caller-controlled and
+ * HMAC-scoped with a boot-time nonce (raw value never stored on the
+ * entry), but two clients that pick the same raw key will still
+ * lease the same warm session — a session-hijack surface in
+ * multi-tenant settings. For multi-tenant deployments, opt out via
+ * `MLX_DISABLE_PROMPT_CACHE_KEY=1` or front the server with an auth
+ * proxy that rewrites or namespaces `prompt_cache_key` per tenant
+ * before it reaches this process. See also the comments on
+ * {@link scopePromptCacheKey}.
+ *
+ * Design notes:
+ *
+ *   - **One registry per model.** Composed alongside each registered
+ *     `ServableModel` in `ModelRegistry`. Sessions are keyed purely
+ *     by response id — no secondary keying on model name because the
+ *     registry is already scoped per model.
+ *
+ *   - **Single-warm-session invariant.** The JS warm registry retains at
+ *     most ONE `ChatSession` entry — both `getOrCreate` and `adopt` clear
+ *     the map before returning or inserting. This remains the sole safe
+ *     reuse mechanism for flat-cache models whose native cache is one
+ *     mutable vector. Block-paged schedulers instead isolate live turns by
+ *     cache owner and reuse verified physical blocks through the native
+ *     prefix table; those models may run fresh JS sessions concurrently.
+ *
+ *   - **Lease semantics on hit.** Clear-on-hit also gives single-
+ *     flight lease semantics: two overlapping requests referencing
+ *     the same `previous_response_id` cannot share the same live
+ *     `ChatSession`. The first wins the cleared entry; the second
+ *     finds the map empty and cold-replays from `ResponseStore` on
+ *     a fresh session. Without this, the second would hit
+ *     `ChatSession`'s single-flight "concurrent send() not allowed"
+ *     guard.
+ *
+ *   - **Prefix compatibility changes miss.** Each entry records its
+ *     `instructions` plus an opaque fingerprint of the cache salt used to
+ *     adopt it. `getOrCreate` compares both against the new request;
+ *     mismatch forces owner release and cold replay instead of reusing a
+ *     stale prompt or changing the security domain of one live native
+ *     request. OpenAI `instructions` and Anthropic `system` share the same
+ *     parameter; both endpoints also thread their mapped `cache_salt`.
+ *
+ *   - **Cache miss fallback.** On a miss (eviction, interleaved turn
+ *     on a different chain, restart, lease-on-hit) the endpoint
+ *     layer reconstructs the conversation from the `ResponseStore`
+ *     history, primes a fresh `ChatSession` via `primeHistory()`,
+ *     and resumes through `startFromHistory()` /
+ *     `startFromHistoryStream()`. That pair dispatches one
+ *     `chatSessionStart*` call that rebuilds the full KV cache and
+ *     atomically appends the new user turn, so cold replay is
+ *     indistinguishable from a hot hit.
+ *
+ *   - **TTL.** Default 1800 seconds mirrors `DEFAULT_RESPONSE_RETENTION_SECONDS`
+ *     in `packages/server/src/server.ts` so the cached
+ *     entry ages out alongside its stored response metadata. With
+ *     at most one entry there is no LRU bookkeeping — just a single
+ *     expiry check on lookup.
+ *
+ *   - **Thread safety.** Node.js is single-threaded within one
+ *     event-loop tick, so the internal `Map` is safe against
+ *     concurrent mutation by design. `sweep()` can be scheduled
+ *     via `setInterval` without colliding with in-flight calls.
+ *
+ *   - **Per-model admission lane.** Flat and not-yet-batched families use
+ *     `withExclusive(fn)`, the original FIFO mutex across the full dispatch.
+ *     A model that explicitly reports `maxConcurrentSequences() > 1` uses
+ *     `withAdmission(fn)`, a counting semaphore sized to that native
+ *     scheduler. Per-session serialization still lives in `ChatSession`;
+ *     only independent sessions share the model lane.
+ */
+
+import { createHash, createHmac, randomBytes } from 'node:crypto';
+
+import type { ChatConfig } from '@mlx-node/core';
+import { ChatSession, type SessionCapableModel } from '@mlx-node/lm';
+
+/**
+ * Tier-2 `prompt_cache_key` reuse is **ON by default** so the server
+ * is immediately compatible with any stateless LLM agent (pi-mono,
+ * Aider, Codex CLI, Claude Code, Cline, Continue) that sends the full
+ * transcript every turn. Agents that set OpenAI's standard
+ * `prompt_cache_key` field — which most do — get automatic KV cache
+ * reuse across turns of the same logical session.
+ *
+ * Opt out via `MLX_DISABLE_PROMPT_CACHE_KEY=1` for **multi-tenant
+ * deployments**, where the tier-2 lookup becomes unsafe: two clients
+ * that pick the same raw `prompt_cache_key` (by accident or on
+ * purpose) would share a warm `ChatSession`, leaking conversation
+ * history and sampling state across principals.
+ *
+ * Multi-tenant isolation is out of scope for this registry. The
+ * HMAC-scoping applied below only hides the raw key from memory /
+ * dumps — it does NOT protect against two clients sharing the same
+ * raw input. Operators who need multi-tenant isolation must either
+ * disable the feature or front the server with an auth proxy that
+ * rewrites `prompt_cache_key` per-tenant before it reaches the
+ * process.
+ *
+ * Read at call time (not cached at module load) so tests can flip the
+ * env via `vi.stubEnv()` between cases without re-importing the module.
+ * The check is a single env-var read plus a string compare — negligible
+ * against the rest of the lookup work.
+ */
+function isPromptCacheKeyEnabled(): boolean {
+  return process.env.MLX_DISABLE_PROMPT_CACHE_KEY !== '1';
+}
+
+/**
+ * Minimum length accepted for a caller-supplied `prompt_cache_key`
+ * before tier-2 scoping. Short keys make trivial guessing collisions
+ * plausible; reject anything shorter than this as if the caller had
+ * not supplied a key at all. Chosen to reject one- / two- / few-byte
+ * values a client might accidentally pass through while still allowing
+ * any reasonable opaque id (UUID prefix, short client token, etc.).
+ */
+const PROMPT_CACHE_KEY_MIN_LENGTH = 8;
+
+/**
+ * Lazily-initialized boot-time nonce used to HMAC every caller-supplied
+ * `prompt_cache_key` before it is stored or looked up. Held in memory
+ * only — never persisted to disk. A process restart invalidates every
+ * tier-2 entry because the next module instance produces a fresh
+ * nonce.
+ *
+ * The nonce makes pre-existing entries unmatchable from outside the
+ * process: an attacker who knows a victim's raw `prompt_cache_key` but
+ * cannot read the nonce from the server's memory also cannot craft a
+ * lookup that collides with the stored HMAC'd key. Combined with the
+ * opt-in gate above, the tier-2 surface is off-by-default and bound to
+ * a server-instance secret when enabled.
+ *
+ * Populated lazily on first use so the cost is not paid when tier-2 is
+ * disabled. Module-scope so it is shared across every `SessionRegistry`
+ * in the process (each per-model registry's HMAC'd keys are still
+ * distinct via their per-registry `entries` map — there is no
+ * cross-model leakage).
+ */
+let cachedNonce: Buffer | null = null;
+
+/** Lazily obtain the module-scoped HMAC nonce. */
+function getNonce(): Buffer {
+  if (cachedNonce === null) {
+    cachedNonce = randomBytes(32);
+  }
+  return cachedNonce;
+}
+
+/** Opaque equality token for a cache salt; the caller's raw value is never retained. */
+function fingerprintCacheSalt(cacheSalt: string | null | undefined): string | null {
+  if (cacheSalt == null) return null;
+  return createHmac('sha256', getNonce()).update(cacheSalt).digest('hex').slice(0, 32);
+}
+
+/**
+ * Test-only hook used by the scoping unit tests to simulate a server
+ * restart: resets the module-scoped HMAC nonce (so every previously stored
+ * tier-2 key and cache-salt fingerprint misses) and clears the silent-miss
+ * dedupe cache so tests can re-exercise the once-per-key diagnostic path.
+ *
+ * **Not exported from the package's public `index.ts` surface** —
+ * exporting it there would let downstream consumers nuke tier-2
+ * state in production (every stored entry would go unreachable).
+ * Tests reach the function via the deep path
+ * `packages/server/src/session-registry.ts` instead; that import is
+ * deliberately noisy to signal "test-only, do not use from app
+ * code". The `__` prefix is a loud-enough convention for the
+ * ergonomic test path but NOT sufficient for a public package
+ * export.
+ */
+export function __resetPromptCacheKeyNonceForTests(): void {
+  cachedNonce = null;
+  loggedSilentMissKeys.clear();
+}
+
+/**
+ * Test-only probe for the silent-miss dedupe Map's live size. Same
+ * "do not use from app code" rationale as
+ * {@link __resetPromptCacheKeyNonceForTests}; the `__` prefix and the
+ * test-only deep-import path are the load-bearing signals. Exists so
+ * the flooding regression test can assert the FIFO cap actually bounds
+ * the stored set — an invariant that is otherwise unobservable from
+ * outside the module and is NOT implied by the warning count (each
+ * call emits at most one warning regardless of whether the cap works).
+ */
+export function __loggedSilentMissKeysSizeForTests(): number {
+  return loggedSilentMissKeys.size;
+}
+
+/**
+ * Normalize and HMAC-scope a caller-supplied `prompt_cache_key` before
+ * it is stored or used for lookup.
+ *
+ * **Single-tenant trust boundary.** HMAC-scoping hides the raw key
+ * from memory dumps and keeps one process instance's stored keys
+ * unreachable from another instance (a restart rerolls the nonce),
+ * but it does NOT protect against two clients supplying the same raw
+ * key — by construction both lookups HMAC to the same scoped key and
+ * share the entry. Multi-tenant isolation is out of scope; see the
+ * module docstring.
+ *
+ * Returns `null` when:
+ *   - The tier-2 feature is disabled
+ *     (`MLX_DISABLE_PROMPT_CACHE_KEY` is set to `"1"`).
+ *   - `rawKey` is `null`, `undefined`, or the empty string (callers
+ *     that forget to thread the key must not accidentally opt into
+ *     tier-2 reuse).
+ *   - `rawKey` is shorter than {@link PROMPT_CACHE_KEY_MIN_LENGTH}
+ *     characters, which keeps trivial guessing collisions off the
+ *     table.
+ *
+ * Otherwise returns the first 32 hex chars of
+ * `HMAC-SHA256(cachedNonce, rawKey)` — opaque, server-instance-scoped,
+ * and long enough to preserve the 64-bit entropy floor that the
+ * pre-scoped path relied on for key uniqueness.
+ */
+function scopePromptCacheKey(rawKey: string | null | undefined): string | null {
+  if (!isPromptCacheKeyEnabled()) return null;
+  if (rawKey == null || rawKey.length < PROMPT_CACHE_KEY_MIN_LENGTH) return null;
+  return createHmac('sha256', getNonce()).update(rawKey).digest('hex').slice(0, 32);
+}
+
+/**
+ * Bounded set of SHA-256-digest prefixes tracking which
+ * `prompt_cache_key` values have already had a silent-miss warning
+ * emitted in this process. Stored as 64-bit hex digests (not raw
+ * strings) so an attacker flooding the endpoint with distinct
+ * attacker-controlled keys cannot drive unbounded memory growth.
+ * FIFO-evicted at {@link LOGGED_SILENT_MISS_KEYS_MAX} entries via the
+ * Map insertion-order guarantee. Reset alongside the nonce / warning
+ * flag in {@link __resetPromptCacheKeyNonceForTests} so unit tests
+ * can re-exercise the once-per-key path.
+ */
+const LOGGED_SILENT_MISS_KEYS_MAX = 256;
+const loggedSilentMissKeys = new Map<string, true>();
+
+/** Hash a raw key to a bounded digest for dedupe storage. */
+function digestSilentMissKey(rawKey: string): string {
+  // 64-bit prefix is enough for dedupe across a 256-entry window.
+  return createHash('sha256').update(rawKey).digest('hex').slice(0, 16);
+}
+
+/**
+ * Emit a once-per-raw-key stderr debug warning when the caller
+ * supplies a non-empty `prompt_cache_key` but at least one tier-2
+ * prerequisite is missing — the env gate is off, or the key is
+ * shorter than {@link PROMPT_CACHE_KEY_MIN_LENGTH}. The silent-miss
+ * fallback (cold-start as if no key were supplied) is the documented
+ * behaviour but is easy to miss during integration; this nudge
+ * surfaces the cause once per distinct key so operators don't have
+ * to grep source to diagnose a flat `X-Session-Cache: fresh`.
+ *
+ * No-op when `rawKey` is null / undefined / empty (the caller did
+ * not ask for tier-2 at all) or when scoping would succeed (the
+ * gate already accepted the key). Called by the endpoint layer
+ * right after it has decided `effectivePromptCacheKey`.
+ */
+export function maybeWarnPromptCacheKeyIneligible(rawKey: string | null | undefined): void {
+  if (rawKey == null || rawKey.length === 0) return;
+  // Happy-path branch: scoping would succeed, no nudge needed.
+  if (isPromptCacheKeyEnabled() && rawKey.length >= PROMPT_CACHE_KEY_MIN_LENGTH) return;
+  const digest = digestSilentMissKey(rawKey);
+  if (loggedSilentMissKeys.has(digest)) return;
+  // FIFO eviction via Map insertion order — bounds memory under
+  // adversarial key flooding while preserving once-per-key semantics
+  // within the recent-key window.
+  if (loggedSilentMissKeys.size >= LOGGED_SILENT_MISS_KEYS_MAX) {
+    const oldest = loggedSilentMissKeys.keys().next().value;
+    if (oldest !== undefined) loggedSilentMissKeys.delete(oldest);
+  }
+  loggedSilentMissKeys.set(digest, true);
+  if (!isPromptCacheKeyEnabled()) {
+    console.warn(
+      `[mlx-node] prompt_cache_key supplied but tier-2 reuse is disabled ` +
+        `(MLX_DISABLE_PROMPT_CACHE_KEY=1). The key will be ignored and this ` +
+        `turn will cold-start. This message is logged once per distinct key.`,
+    );
+    return;
+  }
+  console.warn(
+    `[mlx-node] prompt_cache_key is shorter than ${PROMPT_CACHE_KEY_MIN_LENGTH} chars; tier-2 reuse requires ` +
+      `at least ${PROMPT_CACHE_KEY_MIN_LENGTH} characters. The key will be ignored and this turn will ` +
+      `cold-start. This message is logged once per distinct key.`,
+  );
+}
+
+/** Constructor options for {@link SessionRegistry}. */
+export interface SessionRegistryOptions {
+  /** The model that every session in this registry wraps. Single-model per registry. */
+  model: SessionCapableModel;
+  /** TTL in seconds before an unused session is evicted. Default: 1800 (30 min). */
+  ttlSec?: number;
+  /**
+   * Maximum number of requests that may be WAITING for the
+   * per-model execution mutex at the same time (the in-flight holder
+   * does NOT count toward this). When set and the cap is exceeded at
+   * `withExclusive` entry, the call throws {@link QueueFullError}
+   * synchronously so the endpoint layer can emit HTTP 429 and the
+   * client can retry later.
+   *
+   * Default at THIS layer: `undefined` (unbounded) — a directly
+   * constructed `SessionRegistry` has no cap. `createServer` supplies
+   * its own default of 16 per {@link ServerConfig.maxQueueDepthPerModel}
+   * (or the `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var; config
+   * `'unbounded'` opts out), so server-allocated registries are capped
+   * unless the operator explicitly opted out.
+   */
+  maxQueueDepth?: number;
+  /**
+   * Number of independent dispatches the native model scheduler can advance
+   * concurrently. Values below two retain the legacy exclusive FIFO. The
+   * model registry derives this from `maxConcurrentSequences()`; direct
+   * construction defaults to one.
+   */
+  maxConcurrentDispatches?: number;
+  /**
+   * Optional sampling defaults applied to every `ChatSession` this
+   * registry allocates. Forwarded verbatim into `new ChatSession(model,
+   * { defaultConfig })` so the session's `mergeConfig(overlay)` shallow-
+   * merges per-call config on top. Intended for server operators who
+   * want to pin per-model sampling knobs (temperature, topK, penalties,
+   * etc.) without client cooperation — per-request values from the
+   * OpenAI `/v1/responses` or Anthropic `/v1/messages` body still win
+   * where present because `ChatSession` treats them as an overlay.
+   *
+   * When `undefined`, behaviour is unchanged from the pre-defaults era
+   * (each `new ChatSession(model)` uses an empty `defaultConfig`).
+   */
+  samplingDefaults?: ChatConfig;
+  /**
+   * Optional per-model cap for generated output tokens. Endpoint handlers
+   * apply it after request mapping, before dispatching into native decode.
+   */
+  maxOutputTokens?: number;
+}
+
+/**
+ * Thrown synchronously by {@link SessionRegistry.withExclusive} when
+ * the per-model queue cap (`maxQueueDepth`) is exceeded. The error is
+ * raised BEFORE awaiting the previous lock holder so endpoint handlers
+ * can reliably catch it without racing the chain.
+ */
+export class QueueFullError extends Error {
+  readonly queueDepth: number;
+  readonly preDispatchAdmissions: number;
+  readonly admissionFootprint: number;
+  readonly limit: number;
+
+  constructor(queueDepth: number, preDispatchAdmissions: number, limit: number) {
+    const admissionFootprint = queueDepth + preDispatchAdmissions;
+    super(
+      `Model queue full: ${queueDepth} queued, ${preDispatchAdmissions} pre-dispatch ` +
+        `(${admissionFootprint} admitted outside the active runner; waiter limit ${limit})`,
+    );
+    this.name = 'QueueFullError';
+    this.queueDepth = queueDepth;
+    this.preDispatchAdmissions = preDispatchAdmissions;
+    this.admissionFootprint = admissionFootprint;
+    this.limit = limit;
+  }
+}
+
+/**
+ * Admission permit returned by
+ * {@link SessionRegistry.beginPreDispatchAdmission}. Represents exactly
+ * ONE unit of the per-model admission budget, and ends in exactly one of
+ * two terminal states:
+ *
+ *  - **handed off**: passed as the second argument of
+ *    `withExclusive(fn, permit)` or `withAdmission(fn, permit)`, which
+ *    consumes it atomically as that
+ *    call's admission — the unit converts into the waiter charge (or is
+ *    retired when the caller wins the runner slot). `release()` becomes
+ *    a no-op afterwards.
+ *  - **released**: `release()` frees the unit without a dispatch. Every
+ *    exit between admission and lock placement must do this — the
+ *    recommended pattern is one unconditional `release()` in a
+ *    `finally`, which is safe on every path because `release()` is
+ *    idempotent and a no-op after handoff.
+ */
+export interface PreDispatchAdmission {
+  /** Idempotent: free the budget unit unless already handed off. */
+  release(): void;
+}
+
+/**
+ * Result of {@link SessionRegistry.getOrCreate}. `hit` reflects whether
+ * the call consumed a live warm entry (single-use lease) or returned a
+ * fresh `ChatSession` on a miss. The endpoint layer uses `hit` to
+ * classify the per-request session-cache status emitted to clients via
+ * the `X-Session-Cache` observability header.
+ */
+export interface SessionLookupResult {
+  session: ChatSession<SessionCapableModel>;
+  hit: boolean;
+}
+
+interface SessionEntry {
+  session: ChatSession<SessionCapableModel>;
+  /**
+   * The `instructions` / `system` string the caller adopted this
+   * session with. `null` if the caller did not supply any. Compared
+   * byte-for-byte against the caller's `requestedInstructions` in
+   * `getOrCreate` to detect prefix/system-state changes that would
+   * otherwise let a hit silently reuse a stale warmed prompt.
+   */
+  instructions: string | null;
+  /**
+   * Prefix-cache security domain used when this warm session was adopted.
+   * Omission is represented as `null` (native salt 0) and must remain
+   * distinct from every explicit salt because a live native request cannot
+   * change domains.
+   */
+  cacheSaltFingerprint: string | null;
+  /**
+   * Stable caller-supplied key identifying the logical conversation
+   * chain for warm-session reuse across stateless turns that do NOT
+   * carry a `previous_response_id`. `null` when the adopting caller
+   * supplied no key (or when the request came through an endpoint
+   * that does not honor the key, e.g. the chain terminated on an
+   * in-`previous_response_id` hop). See
+   * {@link SessionRegistry.getOrCreate} for the tier-2 lookup
+   * semantics.
+   *
+   * The distinction between `null` (no key supplied) and the empty
+   * string `""` (key explicitly set to empty) is load-bearing — tier-2
+   * lookup treats them as different keys so a client that forgets to
+   * thread the key does not accidentally collide with another client
+   * that did set it to empty.
+   */
+  promptCacheKey: string | null;
+  /** Unix seconds at which this entry becomes eligible for eviction. */
+  expiresAt: number;
+}
+
+/** Current time in unix seconds. Kept as a helper so tests can patch `Date.now` via fake timers. */
+function nowSec(): number {
+  return Math.floor(Date.now() / 1000);
+}
+
+export class SessionRegistry {
+  private readonly model: SessionCapableModel;
+  private readonly ttlSec: number;
+  private readonly maxQueueDepth: number | undefined;
+  private readonly maxConcurrentDispatches: number;
+  /**
+   * Per-model sampling defaults forwarded into every new `ChatSession`
+   * via its `defaultConfig` constructor option. `undefined` preserves
+   * the pre-defaults behaviour (empty `defaultConfig`). See
+   * {@link SessionRegistryOptions.samplingDefaults}.
+   */
+  private samplingDefaults: ChatConfig | undefined;
+  private maxOutputTokens: number | undefined;
+  /**
+   * Number of callers that are currently WAITING for the per-model
+   * execution mutex — i.e. have entered `withExclusive` but have not
+   * yet started running their closure. The caller that is actively
+   * running inside `fn()` is NOT counted here, so a cap of
+   * `maxQueueDepth = N` means "1 running + up to N waiting".
+   *
+   * Mutated strictly inside `withExclusive`: the admitting caller is
+   * counted as a waiter ONLY when the execution chain is already
+   * non-idle (i.e. some earlier caller still holds the mutex). The
+   * first caller into an idle chain is admitted directly as the
+   * runner slot and never contributes to `queuedCount`. Waiters
+   * decrement exactly once as they transition from waiting to
+   * running (after `await prev`). The counter is intentionally
+   * NEVER touched on cap-reject paths (the caller never queued) so
+   * the cap check is stable across concurrent entries, and runner-
+   * slot admissions leave it alone so a synchronous burst
+   * (e.g. `Promise.all([fn, fn])`) does not spuriously bill the
+   * runner-slot caller against the waiter cap.
+   */
+  private queuedCount = 0;
+  /**
+   * Requests admitted by {@link beginPreDispatchAdmission} whose permit
+   * is still outstanding — parked in the `ModelWorkCoordinator` writer
+   * queue (host mode), blocked in pre-lock store lookups
+   * (`previous_response_id` continuations), or anywhere else between the
+   * endpoint gate and resident-lane placement. None of that parking is
+   * visible to `queuedCount`; this counter is what lets the gate bound
+   * it. Decremented ONLY by the permit itself: `release()` on a bail-out
+   * or the atomic consume inside the selected lane on handoff.
+   */
+  private preDispatchAdmits = 0;
+  /**
+   * Consume hooks for outstanding permits, keyed by permit identity.
+   * Registry-scoped on purpose: both execution lanes consult THIS map, so a
+   * permit minted by a different registry is simply not found and the
+   * call falls back to normal waiter charging — a cross-registry handoff
+   * cannot corrupt either registry's counters.
+   */
+  private readonly permitConsumers = new WeakMap<PreDispatchAdmission, () => boolean>();
+  /**
+   * Holds AT MOST ONE entry under the single-warm invariant (see the
+   * module-level rustdoc). `getOrCreate` and `adopt` both clear the
+   * map as part of their contract so a later lookup cannot hand out
+   * a wrapper whose assumed native state has been overwritten by a
+   * turn on another cached entry.
+   */
+  private readonly entries: Map<string, SessionEntry> = new Map();
+  /**
+   * Eviction is synchronous at the map boundary, but releasing a native
+   * scheduler owner is asynchronous. Start every disposal immediately and
+   * retain its promise so endpoint admission lanes can wait for command-order
+   * visibility before dispatching a replacement turn.
+   */
+  private readonly pendingDisposals = new Set<Promise<void>>();
+  private readonly disposalBySession = new WeakMap<ChatSession<SessionCapableModel>, Promise<void>>();
+  private readonly failedDisposals = new Set<ChatSession<SessionCapableModel>>();
+  /**
+   * Shared sentinel representing "the execution chain is idle" — a
+   * pre-resolved promise. `execLock` starts at this value and is
+   * reset to it whenever the last holder releases without a
+   * successor chained behind it. `withExclusive` uses reference
+   * equality against this sentinel (`execLock === initialLock`) to
+   * tell "I am the runner slot on an idle chain" apart from "I am a
+   * waiter behind someone else", which is how the burst
+   * (`Promise.all([fn, fn])`) admission bug is avoided.
+   */
+  private readonly initialLock: Promise<void> = Promise.resolve();
+  /**
+   * Tail of the per-model execution FIFO. Every `withExclusive` call
+   * captures this value as its predecessor, then overwrites it with
+   * its own pending promise so the next waiter chains after it. The
+   * chain is resolved only when the current holder's `fn` has
+   * settled (success or failure), guaranteeing that at most one
+   * dispatch runs through this registry's native model at a time.
+   * Initialized to `initialLock` so the first caller proceeds
+   * without waiting AND is recognised as the runner slot (no waiter
+   * increment). When a holder releases as the current chain tail it
+   * restores `execLock` to `initialLock` so the next burst starts
+   * cleanly from the idle state.
+   */
+  private execLock: Promise<void> = this.initialLock;
+  /** Active holders in the continuous-batching admission lane. */
+  private activeAdmissions = 0;
+  /** FIFO waiters parked behind the continuous-batching admission limit. */
+  private readonly admissionWaiters: Array<() => void> = [];
+
+  constructor(opts: SessionRegistryOptions) {
+    this.model = opts.model;
+    this.ttlSec = opts.ttlSec ?? 1800;
+    this.maxQueueDepth = opts.maxQueueDepth;
+    const requestedConcurrency = opts.maxConcurrentDispatches ?? 1;
+    this.maxConcurrentDispatches =
+      Number.isSafeInteger(requestedConcurrency) && requestedConcurrency > 1 ? requestedConcurrency : 1;
+    this.samplingDefaults = opts.samplingDefaults;
+    this.maxOutputTokens = opts.maxOutputTokens;
+  }
+
+  /**
+   * Construct a fresh `ChatSession` bound to this registry's model and
+   * pre-seeded with the operator-configured `samplingDefaults` (if any).
+   * Centralized so every cache-miss branch of `getOrCreate` produces a
+   * session whose per-call overlay will merge on top of the same
+   * defaults — clients cannot accidentally stray from the server's
+   * pinned sampling knobs by picking a cold-replay path.
+   */
+  private newSession(): ChatSession<SessionCapableModel> {
+    if (this.samplingDefaults === undefined) {
+      return new ChatSession(this.model);
+    }
+    return new ChatSession(this.model, {
+      defaultConfig: this.samplingDefaults,
+    });
+  }
+
+  private scheduleDispose(session: ChatSession<SessionCapableModel>): void {
+    if (this.disposalBySession.has(session)) return;
+
+    const disposal = this.disposeSession(session)
+      .catch((error: unknown) => {
+        console.error('[server] failed to release an evicted chat-session cache owner:', error);
+      })
+      .finally(() => {
+        this.pendingDisposals.delete(disposal);
+        this.disposalBySession.delete(session);
+      });
+    this.disposalBySession.set(session, disposal);
+    this.pendingDisposals.add(disposal);
+  }
+
+  /**
+   * Dispose one leased session while retaining failed cleanup for a later
+   * registry flush. `ChatSession.dispose()` removes successful owners as it
+   * goes, so a retry only revisits owners whose native release failed.
+   */
+  async disposeSession(session: ChatSession<SessionCapableModel>): Promise<void> {
+    try {
+      await session.dispose();
+      this.failedDisposals.delete(session);
+    } catch (error) {
+      this.failedDisposals.add(session);
+      throw error;
+    }
+  }
+
+  /** Remove every cached entry except an optional session being leased. */
+  private evictEntriesExcept(keep?: ChatSession<SessionCapableModel>): void {
+    for (const entry of this.entries.values()) {
+      if (entry.session !== keep) this.scheduleDispose(entry.session);
+    }
+    this.entries.clear();
+  }
+
+  /**
+   * Wait until every disposal scheduled so far has settled, retrying each
+   * failure discovered during this flush once. A persistent failure remains
+   * recorded for a later flush instead of spinning forever. Disposals log and
+   * absorb their own failures so cleanup cannot rewrite a response that has
+   * already reached the client.
+   */
+  async flushPendingDisposals(): Promise<void> {
+    const retried = new Set<ChatSession<SessionCapableModel>>();
+    while (true) {
+      const retries = Array.from(this.failedDisposals).filter((session) => !retried.has(session));
+      for (const session of retries) {
+        this.failedDisposals.delete(session);
+        retried.add(session);
+        this.scheduleDispose(session);
+      }
+      if (this.pendingDisposals.size === 0) return;
+      await Promise.all(Array.from(this.pendingDisposals));
+    }
+  }
+
+  /**
+   * Disposals started but not yet settled — each one is an in-flight
+   * native `releaseCacheOwner` round-trip — plus disposals whose initial
+   * attempt and bounded retry both failed and remain owed to the native
+   * scheduler ({@link failedDisposals}), retried by the next
+   * {@link flushPendingDisposals}. Endpoints await that flush before
+   * leaving the admission lane, but it runs after the response has
+   * finished, so an observer keyed on request completion can still beat
+   * the release. `adopt`/`drop`/`sweep` schedule synchronously, so once
+   * the request counters read zero any disposal those requests will ever
+   * cause is already counted here. Primarily for diagnostics/tests.
+   */
+  get pendingDisposalCount(): number {
+    return this.pendingDisposals.size + this.failedDisposals.size;
+  }
+
+  /**
+   * Number of requests waiting for this model's selected admission lane.
+   * Active dispatches are not included. Primarily for diagnostics/tests.
+   */
+  get queueDepth(): number {
+    return this.queuedCount;
+  }
+
+  /**
+   * Configured waiter cap for this model's admission lane, or `undefined`
+   * when unbounded. Paired with {@link queueDepth} so a readiness probe can
+   * tell "3 waiters, unbounded" (fine) from "3 waiters, cap of 3" (the next
+   * request gets a 429) without reaching into private state.
+   */
+  get queueDepthLimit(): number | undefined {
+    return this.maxQueueDepth;
+  }
+
+  /** Native continuous-batching capacity used by the endpoint route switch. */
+  get concurrentAdmissionLimit(): number {
+    return this.maxConcurrentDispatches;
+  }
+
+  /**
+   * Outstanding pre-dispatch permits — requests admitted by
+   * {@link beginPreDispatchAdmission} that have neither handed their
+   * permit to the selected execution lane nor released it yet. These permits,
+   * queued callers, and active dispatches share one bounded budget; see
+   * {@link assertAdmissionCapacity}. For probes and diagnostics.
+   */
+  get preDispatchAdmitCount(): number {
+    return this.preDispatchAdmits;
+  }
+
+  /**
+   * Single source of truth for the per-model admission budget — the
+   * ONLY place the cap arithmetic lives. Every admission path calls
+   * this: {@link beginPreDispatchAdmission} before minting a permit,
+   * and both execution lanes for every non-handed-off caller. A handed-off
+   * permit skips the call for its OWN
+   * token only: that token was charged here at acquisition and its
+   * conversion keeps the total constant, so re-checking would
+   * double-charge an already-admitted request. It never exempts anyone
+   * else — all other outstanding state stays counted for every caller
+   * that did not pay.
+   *
+   * Budget invariant: active dispatches + queued callers + outstanding
+   * permits never exceed `maxConcurrentDispatches + maxQueueDepth`.
+   * On the exclusive lane `maxConcurrentDispatches` is one, exactly the
+   * original runner entitlement. On the batched lane it is the native
+   * scheduler's sequence capacity.
+   *
+   * Charging stays at the call sites (`preDispatchAdmits += 1` at the
+   * gate, `queuedCount += 1` for waiters, and the selected lane's active
+   * count for runners); every admitted unit is counted by exactly one
+   * at any time, which is what makes the footprint sum complete across
+   * any interleaving of permitted and permitless callers.
+   *
+   * Throws {@link QueueFullError} — reporting the footprint and the
+   * cap — when the caller does not fit; returns normally otherwise.
+   * No-op when the registry is unbounded.
+   */
+  private assertAdmissionCapacity(): void {
+    if (this.maxQueueDepth === undefined) return;
+    const activeDispatches =
+      this.maxConcurrentDispatches > 1 ? this.activeAdmissions : this.execLock === this.initialLock ? 0 : 1;
+    const footprint = activeDispatches + this.queuedCount + this.preDispatchAdmits;
+    if (footprint >= this.maxQueueDepth + this.maxConcurrentDispatches) {
+      throw new QueueFullError(this.queuedCount, this.preDispatchAdmits, this.maxQueueDepth);
+    }
+  }
+
+  /**
+   * Endpoint-side early admission against this registry's cap, taken BEFORE
+   * the request enters any pre-dispatch parking spot the resident lane cannot
+   * see. Resident host traffic deliberately bypasses the model-load writer so
+   * continuous batching remains reachable, but a continuation can still block
+   * in `await store.getChain(...)` (or other pre-lock work) with `queuedCount`
+   * unchanged. This permit keeps all such work inside the same bounded budget.
+   *
+   * Accounting: pre-dispatch permits, active dispatches, and queued callers
+   * draw from ONE budget: `maxQueueDepth` waiter slots plus the selected
+   * lane's active capacity. Both lanes and this early gate route through
+   * {@link assertAdmissionCapacity}, so permitted and permitless arrivals
+   * cannot double-spend a slot.
+   *
+   * Throws {@link QueueFullError} synchronously when over cap (the
+   * caller maps it to the same 429 envelope as resident-lane rejection
+   * reject). On admission returns a {@link PreDispatchAdmission} permit
+   * the caller must RETAIN through ALL pre-lock asynchronous work and
+   * then hand to `withExclusive(fn, permit)` or `withAdmission(fn, permit)`,
+   * which consumes it
+   * atomically as that call's admission — one budget, one token per
+   * request, never double-counted. Releasing the permit early instead
+   * of handing it off re-opens the hole this gate closes: the request
+   * would be counted by NEITHER counter while parked, arrivals would
+   * refill the budget, and the resident lane would then admit a second
+   * full waiter budget on top. `release()` belongs on bail-out paths
+   * only (idempotent, no-op after handoff — an unconditional `finally`
+   * release is the recommended shape).
+   */
+  beginPreDispatchAdmission(): PreDispatchAdmission {
+    this.assertAdmissionCapacity();
+    this.preDispatchAdmits += 1;
+    let settled = false;
+    const settle = (): boolean => {
+      if (settled) return false;
+      settled = true;
+      this.preDispatchAdmits -= 1;
+      if (this.preDispatchAdmits < 0) this.preDispatchAdmits = 0;
+      this.permitConsumers.delete(permit);
+      return true;
+    };
+    const permit: PreDispatchAdmission = {
+      release: (): void => {
+        void settle();
+      },
+    };
+    this.permitConsumers.set(permit, settle);
+    return permit;
+  }
+
+  /**
+   * Current sampling defaults applied to every new `ChatSession` this
+   * registry allocates. Exposed primarily for tests and diagnostics.
+   */
+  get defaultSamplingConfig(): ChatConfig | undefined {
+    return this.samplingDefaults;
+  }
+
+  get outputTokenLimit(): number | undefined {
+    return this.maxOutputTokens;
+  }
+
+  /**
+   * Replace the sampling defaults forwarded into every future
+   * `ChatSession` this registry allocates. Called by `ModelRegistry`
+   * on a `register(name, model, { samplingDefaults })` refresh so a
+   * fresh registration's defaults immediately apply to the next
+   * cache-miss cold-start. Sessions already cached at call time keep
+   * the defaults they were constructed with — they settle naturally
+   * through the single-warm cache rotation.
+   */
+  setSamplingDefaults(defaults: ChatConfig | undefined): void {
+    this.samplingDefaults = defaults;
+  }
+
+  setMaxOutputTokens(limit: number | undefined): void {
+    this.maxOutputTokens = limit;
+  }
+
+  /** Number of sessions currently cached. Primarily for tests and diagnostics. Always 0 or 1. */
+  get size(): number {
+    return this.entries.size;
+  }
+
+  /**
+   * Look up or allocate a session for the given previous response id.
+   * Always returns a `SessionLookupResult` and always leaves the cache
+   * empty after return (single-warm invariant).
+   *
+   * Lookup proceeds in two tiers:
+   *
+   *   1. **Tier 1 — `previousResponseId`.** The existing hot path:
+   *      exact id match on a live, non-expired entry whose stored
+   *      `instructions` and cache-salt fingerprint match the request. On
+   *      a match the entry is leased out (single-use: removed from the
+   *      map so a concurrent second request cannot share the live
+   *      `ChatSession`). On a miss — unknown id, expired, instructions
+   *      drift, or cache-salt drift — the method falls through to a FRESH
+   *      session regardless of whether tier 2 would have hit.
+   *
+   *      `previousResponseId` wins unconditionally when supplied. The
+   *      two keys could legitimately identify different conversation
+   *      branches (e.g. a client fork where one arm chose the prev-id
+   *      path and the other arm chose to set `prompt_cache_key`
+   *      without one), so routing the prev-id branch through tier 2
+   *      on miss risks splicing the wrong warm state into the wrong
+   *      chain. Cold-replay is the safe default.
+   *
+   *   2. **Tier 2 — `promptCacheKey`.** Only runs when
+   *      `previousResponseId` is `null`. Stateless agent clients
+   *      (pi-mono, Aider, Codex CLI, Continue, etc.) never use
+   *      `previous_response_id` — they own the conversation history
+   *      client-side and resend the full transcript on every turn —
+   *      so the only way to reuse a warm session across those turns
+   *      is to key on the client-supplied `prompt_cache_key`. Scans
+   *      for any live, non-expired entry whose stored
+   *      `promptCacheKey` is non-null AND byte-equal to the caller's
+   *      `promptCacheKey`, whose stored `instructions` are byte-equal,
+   *      AND whose cache-salt fingerprint matches. Empty string is treated
+   *      as a distinct key from `null` — an opt-out sentinel from a client that forgot to
+   *      thread the key must NOT collide with another client that
+   *      did set it to empty. On a match the entry is leased out
+   *      (same single-use semantics as tier 1). On a miss, fall
+   *      through to a fresh session.
+   *
+   * The `hit` flag drives the `X-Session-Cache` observability header
+   * emitted by both `/v1/responses` and `/v1/messages`: when the caller
+   * supplied a `previous_response_id`, `hit === true` yields `hit` and
+   * `hit === false` yields `cold_replay` (the endpoint then rebuilds
+   * from the `ResponseStore` on a fresh session). Requests with no
+   * `previous_response_id` yield either `fresh` (tier-2 miss) or
+   * `prefix_hit` (tier-2 hit — only classified as such once the
+   * native `cachedTokens > 0` confirms the prefix-cache machinery
+   * actually reused the cached tokens).
+   */
+  getOrCreate(
+    previousResponseId: string | null,
+    requestedInstructions: string | null,
+    promptCacheKey: string | null = null,
+    requestedCacheSalt: string | null = null,
+  ): SessionLookupResult {
+    const requestedCacheSaltFingerprint = fingerprintCacheSalt(requestedCacheSalt);
+    // Tier 1: previousResponseId exact match.
+    //
+    // Every call is about to overwrite native KV state, so drop any
+    // other cached entry now — a later `getOrCreate` must not hand
+    // out a wrapper whose assumed state has been stomped. Under the
+    // single-warm invariant the map holds at most one entry, so the
+    // common case is either "the entry we want" or "nothing".
+    if (previousResponseId !== null) {
+      const entry = this.entries.get(previousResponseId);
+      if (entry === undefined) {
+        this.evictEntriesExcept();
+        return { session: this.newSession(), hit: false };
+      }
+      if (entry.expiresAt < nowSec()) {
+        this.evictEntriesExcept();
+        return { session: this.newSession(), hit: false };
+      }
+      // Prefix-state mismatch forces cold replay so new instructions are
+      // re-primed; cache-salt mismatch below likewise prevents a live native
+      // request from crossing prefix-cache security domains.
+      if (entry.instructions !== requestedInstructions) {
+        this.evictEntriesExcept();
+        return { session: this.newSession(), hit: false };
+      }
+      if (entry.cacheSaltFingerprint !== requestedCacheSaltFingerprint) {
+        this.evictEntriesExcept();
+        return { session: this.newSession(), hit: false };
+      }
+      // Tier-1 hit: clear and hand the session out as a single-use
+      // lease so a concurrent second request against the same id
+      // cold-replays instead of sharing this live ChatSession. Note
+      // that even on a prev-id tier-1 MISS we do NOT fall through to
+      // tier 2 — see the docstring above for the precedence
+      // rationale.
+      this.evictEntriesExcept(entry.session);
+      return { session: entry.session, hit: true };
+    }
+
+    // Tier 2: promptCacheKey scan (only reached when previousResponseId is null).
+    //
+    // The registry holds at most one entry under the single-warm
+    // invariant, so the "scan" is actually a single lookup — walk the
+    // map, check the one entry if present, hit or miss. A non-null
+    // scoped key on both the request and the entry plus byte-equal
+    // instructions and cache-salt fingerprints is the match condition.
+    //
+    // SECURITY: raw caller-supplied keys never touch the map. They
+    // are run through {@link scopePromptCacheKey}, which (a) returns
+    // `null` when the tier-2 opt-in env var is unset, (b) enforces a
+    // minimum length, and (c) HMACs the key with a boot-time nonce
+    // held only in this process's memory. Without the opt-in every
+    // tier-2 lookup below immediately misses; with the opt-in,
+    // attackers who cannot read the process-local nonce cannot craft
+    // a lookup that matches a stored entry by guessing the raw key.
+    const scopedKey = scopePromptCacheKey(promptCacheKey);
+    if (scopedKey !== null) {
+      for (const entry of this.entries.values()) {
+        if (entry.expiresAt < nowSec()) continue;
+        if (entry.promptCacheKey === null) continue;
+        if (entry.promptCacheKey !== scopedKey) continue;
+        if (entry.instructions !== requestedInstructions) continue;
+        if (entry.cacheSaltFingerprint !== requestedCacheSaltFingerprint) continue;
+        // Tier-2 hit: clear and lease (same single-warm / single-use
+        // semantics as tier 1).
+        this.evictEntriesExcept(entry.session);
+        return { session: entry.session, hit: true };
+      }
+    }
+
+    // Fall through: fresh session. Clear any leftover entry so a
+    // later lookup cannot hand out a wrapper whose assumed state has
+    // been overwritten by this dispatch.
+    this.evictEntriesExcept();
+    return { session: this.newSession(), hit: false };
+  }
+
+  /**
+   * Allocate a fresh `ChatSession` bound to this registry's model
+   * without touching the warm slot. Intended for the `/v1/messages`
+   * endpoint when the underlying model has a block-paged KV cache
+   * active: the native cache already reuses SYS blocks across requests
+   * via content-addressing in `BlockAllocator`'s prefix-hash table, so
+   * the JS-side warm slot in
+   * {@link SessionRegistry.getOrCreateWarmAny} is redundant.
+   *
+   * Crucially, this call is purely additive — it does **NOT** clear,
+   * read, or evict the warm slot. Two parallel `/v1/messages` requests
+   * sharing a system prompt both call `createFreshSession` and both
+   * get distinct sessions; the native cache transparently refcounts
+   * the shared SYS blocks across them. This is the routing decision
+   * the long block comment in `packages/server/src/endpoints/messages.ts`
+   * documents: paged → fresh session, non-paged → warm-any lookup.
+   *
+   * The returned session is pre-seeded with the operator-configured
+   * `samplingDefaults` (matching every other cache-miss branch) so a
+   * client that picks the paged path does not silently stray from the
+   * server's pinned sampling knobs.
+   *
+   * Returned with `hit: false` to keep the result shape uniform with
+   * {@link SessionRegistry.getOrCreate} and
+   * {@link SessionRegistry.getOrCreateWarmAny}; callers that care
+   * about the cache header semantics should observe
+   * `result.cachedTokens` from the dispatch instead — that's the
+   * authoritative signal for whether the native engine recovered any
+   * prefix on this turn (paged or otherwise).
+   */
+  createFreshSession(): SessionLookupResult {
+    return { session: this.newSession(), hit: false };
+  }
+
+  /**
+   * @deprecated **Redundant on `/v1/messages` for paged-active models.**
+   * There is no call site for paged-active full-attention models (Qwen3 +
+   * LFM2 + Gemma4 today): the native block-paged KV adapter (`PagedKVCacheAdapter` +
+   * `BlockAllocator` + `LayerKVPool`) recovers a turn's prefix from
+   * refcounted KV blocks keyed by token-prefix hash, so the JS-side
+   * single-warm slot this method walks is redundant — the native
+   * cache picks up the same cross-turn reuse without the
+   * byte-equal-`instructions` gate, and additionally supports
+   * cross-conversation prefix sharing the warm slot cannot.
+   *
+   * The `/v1/messages` endpoint now branches at request time on
+   * {@link SessionCapableModel.hasBlockPagedCache}: paged-active models
+   * call {@link SessionRegistry.createFreshSession} per request and
+   * never touch the warm slot; non-paged models (Qwen3.5 dense + MoE —
+   * default-OFF pending a perf decision; the `QianfanOCRModel` VLM —
+   * no adapter wired) still call this method because the JS-side warm
+   * slot is the ONLY cross-conversation reuse mechanism available to
+   * them. Removing this method would silently disable cross-turn reuse
+   * on every non-paged model, so it stays load-bearing until ALL
+   * session-capable models have paged enabled by default. Treat
+   * `@deprecated` as an intent signal that paged-active callers should
+   * use `createFreshSession` instead.
+   *
+   * Third lookup mode — for STATELESS full-history endpoints that have
+   * no `previous_response_id` to thread and do not propagate
+   * `prompt_cache_key` back to the server. The Anthropic
+   * `/v1/messages` endpoint is the canonical caller: clients (e.g.
+   * Claude Code) POST the entire conversation each turn, so the only
+   * remaining signal that a turn N continues turn N-1's prefix is the
+   * registry's own warm slot.
+   *
+   * Behaviour: walk the registry's at-most-one warm entry. If it is
+   * non-expired, its stored `instructions` are byte-equal to
+   * `requestedInstructions`, AND its stored cache salt equals
+   * `requestedCacheSalt`, lease it out (single-use — `entries.clear()`
+   * before return, mirroring the tier-1 / tier-2 lease-on-hit semantics).
+   * Otherwise clear the map and return a fresh session.
+   *
+   * Crucially, this lookup IGNORES `entry.promptCacheKey` and ignores
+   * the entry's prior `previousResponseId` keying — any warm slot is
+   * fair game for `/v1/messages` reuse. Byte-equal instructions and cache
+   * salt are the correctness gates: a system prompt or prefix-cache
+   * security-domain change forces cold replay instead of reusing stale state
+   * or asking the native adapter to mutate a live request's salt.
+   *
+   * **Adoption sentinel.** `/v1/messages` adopts back under the literal
+   * sentinel id `'__msg_warm__'`. That sentinel will never appear as a
+   * `previous_response_id` on a `/v1/responses` request — the
+   * Anthropic Messages API does not produce a `previous_response_id`
+   * value clients could echo back, and the OpenAI side mints fresh
+   * `resp_*` ids — so cross-endpoint capture via tier-1 is impossible
+   * by construction. The two endpoints still SHARE the single warm
+   * slot under the registry's single-warm invariant: a
+   * `/v1/messages` turn that follows a `/v1/responses` turn can evict
+   * (and vice versa). That is the explicit trade-off of holding at
+   * most one warm entry per model.
+   *
+   * **Trust model.** Multi-tenant isolation on this endpoint requires
+   * fronting the server with an auth proxy that scopes warm-slot
+   * visibility per tenant — same trust boundary documented at the top
+   * of this file for the tier-2 `prompt_cache_key` path. The single-
+   * warm invariant plus `withExclusive`'s per-model serialization make
+   * the lookup safe under SINGLE-tenant assumptions: no two requests
+   * race the slot, and there is at most one slot to lease.
+   *
+   * **Caller contract on miss.** If `instructions` drifts between
+   * turns (system prompt changed) this returns `hit: false` and a
+   * fresh session — and the caller MUST then run a full
+   * `session.reset()` before priming history, NOT the JS-only
+   * `resetPreservingNativeCacheForWarmReuse` path. A fresh JS session
+   * does NOT imply a fresh native cache (the underlying
+   * `SessionCapableModel` is shared and its native
+   * `cached_token_history` persists across requests), so skipping the
+   * native wipe on a miss would let the next `chatSessionStart` reuse
+   * an unrelated previous request's prefix — the cross-request
+   * cache-affinity side channel that the long block comment in
+   * `responses.ts` (around the `runSessionNonStreaming` /
+   * `runSessionStreaming` branches) describes.
+   */
+  getOrCreateWarmAny(
+    requestedInstructions: string | null,
+    requestedCacheSalt: string | null = null,
+  ): SessionLookupResult {
+    const requestedCacheSaltFingerprint = fingerprintCacheSalt(requestedCacheSalt);
+    // Single-warm invariant: at most one entry. Walk it once, lease
+    // on a fresh + instructions-and-salt-matched hit, otherwise clear and
+    // cold-start. The ignored fields (promptCacheKey,
+    // previousResponseId-keying) are deliberate — see the docstring.
+    for (const entry of this.entries.values()) {
+      if (entry.expiresAt < nowSec()) continue;
+      if (entry.instructions !== requestedInstructions) continue;
+      if (entry.cacheSaltFingerprint !== requestedCacheSaltFingerprint) continue;
+      // Hit: clear and lease (single-use semantics, same as tiers 1/2).
+      this.evictEntriesExcept(entry.session);
+      return { session: entry.session, hit: true };
+    }
+    // Miss (no entry, expired, instructions drift, or salt drift). Clear the map
+    // so a stale wrapper cannot leak into a later lookup, and return
+    // a fresh session.
+    this.evictEntriesExcept();
+    return { session: this.newSession(), hit: false };
+  }
+
+  /**
+   * Insert a session under a newly allocated response id. Clears the
+   * map before inserting to keep the single-warm invariant explicit
+   * regardless of caller ordering.
+   *
+   * `instructions` is the prefix/system state used for this turn;
+   * stored on the entry and compared on the next `getOrCreate` to
+   * detect prefix changes that must force a cold replay.
+   * `cacheSalt` is HMAC-fingerprinted before storage and compared alongside
+   * the prefix so the raw security-domain key is not retained.
+   *
+   * `promptCacheKey` is the client-supplied conversation-chain key
+   * that enables the registry's tier-2 lookup for stateless agent
+   * turns that do not carry a `previous_response_id`. `null` /
+   * `undefined` means "no key supplied" — stored verbatim so a
+   * subsequent stateless lookup that also omits the key does NOT
+   * accidentally pick up this entry (only explicit non-null
+   * key-equality on both sides can hit tier 2). See
+   * {@link SessionRegistry.getOrCreate} for the precedence rules.
+   */
+  adopt(
+    responseId: string,
+    session: ChatSession<SessionCapableModel>,
+    instructions: string | null,
+    promptCacheKey: string | null | undefined = null,
+    cacheSalt: string | null | undefined = null,
+  ): void {
+    // Scope the caller-supplied key BEFORE storing so a later
+    // `getOrCreate` can only resolve entries via the same opt-in +
+    // HMAC path. When tier-2 reuse is disabled (or the key is too
+    // short / absent) `scopePromptCacheKey` returns `null`, which
+    // disables this entry from ever matching a tier-2 lookup — the
+    // raw caller-supplied key is NEVER stored.
+    this.evictEntriesExcept(session);
+    this.entries.set(responseId, {
+      session,
+      instructions,
+      cacheSaltFingerprint: fingerprintCacheSalt(cacheSalt),
+      promptCacheKey: scopePromptCacheKey(promptCacheKey ?? null),
+      expiresAt: nowSec() + this.ttlSec,
+    });
+  }
+
+  /**
+   * Remove a session by response id. No-op if the key is not present.
+   */
+  drop(responseId: string): void {
+    const entry = this.entries.get(responseId);
+    if (entry === undefined) return;
+    this.entries.delete(responseId);
+    this.scheduleDispose(entry.session);
+  }
+
+  /**
+   * Walk the map and drop the entry if its TTL has expired.
+   * Intended for periodic cleanup via `setInterval`. Under the
+   * single-warm invariant the map holds at most one entry.
+   */
+  sweep(): void {
+    const cutoff = nowSec();
+    for (const [key, entry] of this.entries) {
+      if (entry.expiresAt < cutoff) {
+        this.entries.delete(key);
+        this.scheduleDispose(entry.session);
+      }
+    }
+  }
+
+  /** Empty the registry. Useful at shutdown and in tests. */
+  clear(): void {
+    this.evictEntriesExcept();
+  }
+
+  /**
+   * Serialize `fn` against every other dispatch through this
+   * registry's model. The caller must hold the lock across the
+   * entire per-model dispatch span — `getOrCreate` ->
+   * `primeHistory`/`send*` -> `adopt`/`drop`. Without it, two
+   * concurrent `primeHistory()` / `send*()` calls would race on
+   * the single mutable native KV cache and whichever finished last
+   * would corrupt the other's chain.
+   *
+   * FIFO chaining via a rolling `execLock` promise: each caller
+   * captures the current tail, publishes a fresh pending promise as
+   * the new tail, awaits the old tail, then runs `fn`. The
+   * `finally` releases regardless of whether `fn` threw.
+   *
+   * **Admission control.** Every non-handed-off call — waiter AND
+   * runner — routes through {@link assertAdmissionCapacity}, the single
+   * source of truth for the budget: the combined admission footprint
+   * (`queuedCount + preDispatchAdmits`) plus this caller must fit
+   * within `maxQueueDepth` plus the idle-chain runner entitlement.
+   * Over-budget calls throw {@link QueueFullError} — SYNCHRONOUSLY
+   * from the caller's perspective, not merely before `await prev`. The
+   * wrapper is deliberately NOT declared `async` so the admission gate
+   * throws on the caller's stack frame, letting endpoint handlers wrap
+   * the call site in a plain try/catch without racing promise
+   * microtasks. On acceptance the async body takes over via the
+   * returned `Promise<T>`.
+   *
+   * A cap of N permits one running dispatch plus N admitted-but-not-
+   * running requests (queued waiters and outstanding permits
+   * combined), rejecting the next. The default (undefined) preserves
+   * the original unbounded behaviour.
+   *
+   * **Runner-slot admission.** Whether a given caller counts as the
+   * runner slot or as a waiter is decided up front by comparing
+   * `execLock` against the idle sentinel `initialLock`. If they are
+   * identical, nobody is currently in-flight and this caller is
+   * admitted against the extra runner entitlement rather than a waiter
+   * slot — it never touches `queuedCount`. Otherwise it is a waiter
+   * and the normal charge / increment / decrement cycle applies. This
+   * is what keeps a synchronous burst such as `Promise.all([fn, fn])`
+   * admissible under `maxQueueDepth = 1` — Call 1 is the runner,
+   * Call 2 is the one allowed waiter, Call 3 would throw. The runner
+   * path is NOT exempt from the budget: while the chain is idle,
+   * {@link beginPreDispatchAdmission} lends out `maxQueueDepth + 1`
+   * permits precisely because one of them is entitled to become the
+   * runner, so a permitless call that would take that seat while the
+   * whole runner-plus-waiter capacity is already spoken for must
+   * reject — otherwise the outstanding permits would later convert on
+   * top of it and breach the cap.
+   *
+   * **Permit handoff.** A caller that was already admitted by
+   * {@link beginPreDispatchAdmission} passes its permit as the second
+   * argument; the permit is consumed atomically as this call's
+   * admission instead of charging `queuedCount` a second time — one
+   * budget, one token per request. See the handoff comment in the
+   * body and {@link PreDispatchAdmission}.
+   */
+  withExclusive<T>(fn: () => Promise<T>, permit?: PreDispatchAdmission): Promise<T> {
+    // Distinguish runner-slot from waiter admission. If the chain is
+    // idle (`execLock === initialLock`) the current caller is about
+    // to become the active holder on its very first `await prev`
+    // microtask — it must NOT be billed against the waiter cap and
+    // must NOT touch `queuedCount`. Only chained callers (someone
+    // else still holds or is ahead in the FIFO) count as waiters.
+    const asWaiter = this.execLock !== this.initialLock;
+
+    // Atomic permit handoff (see `beginPreDispatchAdmission`). A caller
+    // handing in a still-outstanding permit from THIS registry already
+    // owns one unit of the shared budget: consuming it here — in the
+    // same synchronous block as the waiter accounting below — converts
+    // that unit into this call's admission (waiter path increments
+    // `queuedCount`, keeping the total constant) or retires it (runner
+    // path). The handoff skips only the caller's OWN token: its budget
+    // share was charged at acquisition and conversion keeps the total
+    // constant, so the handoff itself can never create a breach —
+    // whereas re-checking would double-charge and could reject a
+    // request that was already admitted at the endpoint gate. A permit
+    // minted by a DIFFERENT registry is not in `permitConsumers` and
+    // falls through to normal charging (its own budget stays balanced
+    // by the caller's `finally` release).
+    const consume = permit === undefined ? undefined : this.permitConsumers.get(permit);
+    const handedOff = consume !== undefined && consume();
+
+    // Admission check — raised synchronously so endpoint handlers
+    // can reliably catch `QueueFullError` without racing any `await`.
+    // Every non-handed-off caller — runner AND waiter — runs the same
+    // shared predicate as `beginPreDispatchAdmission`: the combined
+    // footprint (queued waiters PLUS outstanding pre-dispatch permits)
+    // against the cap plus the idle-chain runner entitlement. Checking
+    // `queuedCount` alone would let a permitless caller spend a slot
+    // an outstanding permit already owns (e.g. a continuation parked
+    // in `store.getChain`); exempting the runner path would let a
+    // permitless call take the runner seat the gate already lent to
+    // one of `maxQueueDepth + 1` idle-chain permits — either way the
+    // cap breaches once those permits convert. Nothing is mutated on
+    // the reject path; the request never queued.
+    if (!handedOff) {
+      this.assertAdmissionCapacity();
+    }
+
+    const prev = this.execLock;
+    let release!: () => void;
+    const myLock = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    this.execLock = myLock;
+    if (asWaiter) {
+      this.queuedCount += 1;
+    }
+    return this._runExclusive(prev, myLock, release, fn, asWaiter);
+  }
+
+  /**
+   * Admit one dispatch to the model's continuous-batching lane.
+   *
+   * Admission rejection is synchronous, matching {@link withExclusive} and
+   * preserving the endpoint's existing QueueFullError-to-429 mapping. Once
+   * accepted, at most {@link concurrentAdmissionLimit} closures run at once;
+   * excess accepted callers wait FIFO and contribute to {@link queueDepth}.
+   * A pre-dispatch permit is consumed atomically into either an active slot or
+   * a queued slot, so the early endpoint gate and this semaphore share one
+   * bounded budget.
+   */
+  withAdmission<T>(fn: () => Promise<T>, permit?: PreDispatchAdmission): Promise<T> {
+    const consume = permit === undefined ? undefined : this.permitConsumers.get(permit);
+    const handedOff = consume !== undefined && consume();
+    if (!handedOff) {
+      this.assertAdmissionCapacity();
+    }
+
+    let acquire!: () => void;
+    const acquired = new Promise<void>((resolve) => {
+      acquire = resolve;
+    });
+    if (this.activeAdmissions < this.maxConcurrentDispatches) {
+      this.activeAdmissions += 1;
+      acquire();
+    } else {
+      this.queuedCount += 1;
+      this.admissionWaiters.push(() => {
+        this.queuedCount -= 1;
+        if (this.queuedCount < 0) this.queuedCount = 0;
+        this.activeAdmissions += 1;
+        acquire();
+      });
+    }
+    return this._runAdmission(acquired, fn);
+  }
+
+  private async _runAdmission<T>(acquired: Promise<void>, fn: () => Promise<T>): Promise<T> {
+    await acquired;
+    try {
+      return await fn();
+    } finally {
+      this.activeAdmissions -= 1;
+      if (this.activeAdmissions < 0) this.activeAdmissions = 0;
+      const next = this.admissionWaiters.shift();
+      next?.();
+    }
+  }
+
+  /**
+   * Async tail of {@link withExclusive}. Kept separate so the public
+   * wrapper stays a plain (non-async) function whose admission-gate
+   * throw lands on the caller's stack synchronously. This helper
+   * owns the post-acceptance bookkeeping: awaiting the predecessor
+   * lock, transitioning from waiter to holder (`queuedCount`
+   * decrement for waiters only), running `fn`, releasing the FIFO
+   * tail, and resetting the chain to `initialLock` when this caller
+   * is still the tail (so a future burst admits its first entry as
+   * a runner-slot rather than as a waiter).
+   */
+  private async _runExclusive<T>(
+    prev: Promise<void>,
+    myLock: Promise<void>,
+    release: () => void,
+    fn: () => Promise<T>,
+    asWaiter: boolean,
+  ): Promise<T> {
+    // Track whether the waiting-counter has already been balanced so
+    // an error raised by `await prev` (should never happen today but
+    // is cheap to defend against) cannot double-decrement via the
+    // outer `finally` below. Runner-slot admissions never touch the
+    // counter, so the flag starts already-balanced for them.
+    let waitingDecremented = !asWaiter;
+    try {
+      try {
+        await prev;
+      } finally {
+        // Transition from "waiting" to "running" — the counter must
+        // drop exactly here regardless of whether `prev` fulfilled
+        // or rejected, because from this point forward the caller is
+        // the active holder and no longer part of the queue depth.
+        if (!waitingDecremented) {
+          this.queuedCount -= 1;
+          if (this.queuedCount < 0) this.queuedCount = 0;
+          waitingDecremented = true;
+        }
+      }
+      return await fn();
+    } finally {
+      // Belt-and-suspenders: if `await prev` managed to throw before
+      // reaching the inner `finally` (extremely unlikely given the
+      // chain is always resolved with `undefined`), still balance the
+      // queued counter so a future cap check doesn't drift upward.
+      if (!waitingDecremented) {
+        this.queuedCount -= 1;
+        if (this.queuedCount < 0) this.queuedCount = 0;
+        waitingDecremented = true;
+      }
+      release();
+      // Reset the chain to the idle sentinel ONLY when this caller
+      // is still the tail — if someone else has already extended the
+      // FIFO behind us, leave their tail in place. Reference-equality
+      // gate here is what lets the next burst see `execLock ===
+      // initialLock` and admit its first caller as a runner slot.
+      if (this.execLock === myLock) {
+        this.execLock = this.initialLock;
+      }
+    }
+  }
+}