@mlx-node/server 0.0.0 → 0.0.8

package/dist/session-registry.js

@@ -2,6 +2,18 @@
  * SessionRegistry -- per-model cache holding AT MOST one live
  * `ChatSession` whose native KV state is currently valid.
  *
+ * **Tier-2 `prompt_cache_key` reuse is ON by default** so the server
+ * is compatible with any stateless LLM agent that sends the full
+ * conversation history each turn. The key is caller-controlled and
+ * HMAC-scoped with a boot-time nonce (raw value never stored on the
+ * entry), but two clients that pick the same raw key will still
+ * lease the same warm session — a session-hijack surface in
+ * multi-tenant settings. For multi-tenant deployments, opt out via
+ * `MLX_DISABLE_PROMPT_CACHE_KEY=1` or front the server with an auth
+ * proxy that rewrites or namespaces `prompt_cache_key` per tenant
+ * before it reaches this process. See also the comments on
+ * {@link scopePromptCacheKey}.
+ *
  * Design notes:
  *
  *   - **One registry per model.** Composed alongside each registered
@@ -50,8 +62,8 @@
  *     atomically appends the new user turn, so cold replay is
  *     indistinguishable from a hot hit.
  *
- *   - **TTL.** Default 1800 seconds mirrors `RESPONSE_TTL_SECONDS`
- *     in `packages/server/src/endpoints/responses.ts` so the cached
+ *   - **TTL.** Default 1800 seconds mirrors `DEFAULT_RESPONSE_RETENTION_SECONDS`
+ *     in `packages/server/src/server.ts` so the cached
  *     entry ages out alongside its stored response metadata. With
  *     at most one entry there is no LRU bookkeeping — just a single
  *     expiry check on lookup.
@@ -79,7 +91,204 @@
  *     time. A weaker epoch-token scheme would let the losing
  *     `adopt()` no-op but the native KV would already be wrong.
  */
+import { createHash, createHmac, randomBytes } from 'node:crypto';
 import { ChatSession } from '@mlx-node/lm';
+/**
+ * Tier-2 `prompt_cache_key` reuse is **ON by default** so the server
+ * is immediately compatible with any stateless LLM agent (pi-mono,
+ * Aider, Codex CLI, Claude Code, Cline, Continue) that sends the full
+ * transcript every turn. Agents that set OpenAI's standard
+ * `prompt_cache_key` field — which most do — get automatic KV cache
+ * reuse across turns of the same logical session.
+ *
+ * Opt out via `MLX_DISABLE_PROMPT_CACHE_KEY=1` for **multi-tenant
+ * deployments**, where the tier-2 lookup becomes unsafe: two clients
+ * that pick the same raw `prompt_cache_key` (by accident or on
+ * purpose) would share a warm `ChatSession`, leaking conversation
+ * history and sampling state across principals.
+ *
+ * Multi-tenant isolation is out of scope for this registry. The
+ * HMAC-scoping applied below only hides the raw key from memory /
+ * dumps — it does NOT protect against two clients sharing the same
+ * raw input. Operators who need multi-tenant isolation must either
+ * disable the feature or front the server with an auth proxy that
+ * rewrites `prompt_cache_key` per-tenant before it reaches the
+ * process.
+ *
+ * Read at call time (not cached at module load) so tests can flip the
+ * env via `vi.stubEnv()` between cases without re-importing the module.
+ * The check is a single env-var read plus a string compare — negligible
+ * against the rest of the lookup work.
+ */
+function isPromptCacheKeyEnabled() {
+    return process.env.MLX_DISABLE_PROMPT_CACHE_KEY !== '1';
+}
+/**
+ * Minimum length accepted for a caller-supplied `prompt_cache_key`
+ * before tier-2 scoping. Short keys make trivial guessing collisions
+ * plausible; reject anything shorter than this as if the caller had
+ * not supplied a key at all. Chosen to reject one- / two- / few-byte
+ * values a client might accidentally pass through while still allowing
+ * any reasonable opaque id (UUID prefix, short client token, etc.).
+ */
+const PROMPT_CACHE_KEY_MIN_LENGTH = 8;
+/**
+ * Lazily-initialized boot-time nonce used to HMAC every caller-supplied
+ * `prompt_cache_key` before it is stored or looked up. Held in memory
+ * only — never persisted to disk. A process restart invalidates every
+ * tier-2 entry because the next module instance produces a fresh
+ * nonce.
+ *
+ * The nonce makes pre-existing entries unmatchable from outside the
+ * process: an attacker who knows a victim's raw `prompt_cache_key` but
+ * cannot read the nonce from the server's memory also cannot craft a
+ * lookup that collides with the stored HMAC'd key. Combined with the
+ * opt-in gate above, the tier-2 surface is off-by-default and bound to
+ * a server-instance secret when enabled.
+ *
+ * Populated lazily on first use so the cost is not paid when tier-2 is
+ * disabled. Module-scope so it is shared across every `SessionRegistry`
+ * in the process (each per-model registry's HMAC'd keys are still
+ * distinct via their per-registry `entries` map — there is no
+ * cross-model leakage).
+ */
+let cachedNonce = null;
+/** Lazily obtain the module-scoped HMAC nonce. */
+function getNonce() {
+    if (cachedNonce === null) {
+        cachedNonce = randomBytes(32);
+    }
+    return cachedNonce;
+}
+/**
+ * Test-only hook used by the scoping unit tests to simulate a
+ * server restart: resets the module-scoped HMAC nonce (so every
+ * previously stored tier-2 key misses) and clears the silent-miss
+ * dedupe cache so tests can re-exercise the once-per-key diagnostic
+ * path.
+ *
+ * **Not exported from the package's public `index.ts` surface** —
+ * exporting it there would let downstream consumers nuke tier-2
+ * state in production (every stored entry would go unreachable).
+ * Tests reach the function via the deep path
+ * `packages/server/src/session-registry.ts` instead; that import is
+ * deliberately noisy to signal "test-only, do not use from app
+ * code". The `__` prefix is a loud-enough convention for the
+ * ergonomic test path but NOT sufficient for a public package
+ * export.
+ */
+export function __resetPromptCacheKeyNonceForTests() {
+    cachedNonce = null;
+    loggedSilentMissKeys.clear();
+}
+/**
+ * Test-only probe for the silent-miss dedupe Map's live size. Same
+ * "do not use from app code" rationale as
+ * {@link __resetPromptCacheKeyNonceForTests}; the `__` prefix and the
+ * test-only deep-import path are the load-bearing signals. Exists so
+ * the flooding regression test can assert the FIFO cap actually bounds
+ * the stored set — an invariant that is otherwise unobservable from
+ * outside the module and is NOT implied by the warning count (each
+ * call emits at most one warning regardless of whether the cap works).
+ */
+export function __loggedSilentMissKeysSizeForTests() {
+    return loggedSilentMissKeys.size;
+}
+/**
+ * Normalize and HMAC-scope a caller-supplied `prompt_cache_key` before
+ * it is stored or used for lookup.
+ *
+ * **Single-tenant trust boundary.** HMAC-scoping hides the raw key
+ * from memory dumps and keeps one process instance's stored keys
+ * unreachable from another instance (a restart rerolls the nonce),
+ * but it does NOT protect against two clients supplying the same raw
+ * key — by construction both lookups HMAC to the same scoped key and
+ * share the entry. Multi-tenant isolation is out of scope; see the
+ * module docstring.
+ *
+ * Returns `null` when:
+ *   - The tier-2 feature is disabled
+ *     (`MLX_DISABLE_PROMPT_CACHE_KEY` is set to `"1"`).
+ *   - `rawKey` is `null`, `undefined`, or the empty string (callers
+ *     that forget to thread the key must not accidentally opt into
+ *     tier-2 reuse).
+ *   - `rawKey` is shorter than {@link PROMPT_CACHE_KEY_MIN_LENGTH}
+ *     characters, which keeps trivial guessing collisions off the
+ *     table.
+ *
+ * Otherwise returns the first 32 hex chars of
+ * `HMAC-SHA256(cachedNonce, rawKey)` — opaque, server-instance-scoped,
+ * and long enough to preserve the 64-bit entropy floor that the
+ * pre-scoped path relied on for key uniqueness.
+ */
+function scopePromptCacheKey(rawKey) {
+    if (!isPromptCacheKeyEnabled())
+        return null;
+    if (rawKey == null || rawKey.length < PROMPT_CACHE_KEY_MIN_LENGTH)
+        return null;
+    return createHmac('sha256', getNonce()).update(rawKey).digest('hex').slice(0, 32);
+}
+/**
+ * Bounded set of SHA-256-digest prefixes tracking which
+ * `prompt_cache_key` values have already had a silent-miss warning
+ * emitted in this process. Stored as 64-bit hex digests (not raw
+ * strings) so an attacker flooding the endpoint with distinct
+ * attacker-controlled keys cannot drive unbounded memory growth.
+ * FIFO-evicted at {@link LOGGED_SILENT_MISS_KEYS_MAX} entries via the
+ * Map insertion-order guarantee. Reset alongside the nonce / warning
+ * flag in {@link __resetPromptCacheKeyNonceForTests} so unit tests
+ * can re-exercise the once-per-key path.
+ */
+const LOGGED_SILENT_MISS_KEYS_MAX = 256;
+const loggedSilentMissKeys = new Map();
+/** Hash a raw key to a bounded digest for dedupe storage. */
+function digestSilentMissKey(rawKey) {
+    // 64-bit prefix is enough for dedupe across a 256-entry window.
+    return createHash('sha256').update(rawKey).digest('hex').slice(0, 16);
+}
+/**
+ * Emit a once-per-raw-key stderr debug warning when the caller
+ * supplies a non-empty `prompt_cache_key` but at least one tier-2
+ * prerequisite is missing — the env gate is off, or the key is
+ * shorter than {@link PROMPT_CACHE_KEY_MIN_LENGTH}. The silent-miss
+ * fallback (cold-start as if no key were supplied) is the documented
+ * behaviour but is easy to miss during integration; this nudge
+ * surfaces the cause once per distinct key so operators don't have
+ * to grep source to diagnose a flat `X-Session-Cache: fresh`.
+ *
+ * No-op when `rawKey` is null / undefined / empty (the caller did
+ * not ask for tier-2 at all) or when scoping would succeed (the
+ * gate already accepted the key). Called by the endpoint layer
+ * right after it has decided `effectivePromptCacheKey`.
+ */
+export function maybeWarnPromptCacheKeyIneligible(rawKey) {
+    if (rawKey == null || rawKey.length === 0)
+        return;
+    // Happy-path branch: scoping would succeed, no nudge needed.
+    if (isPromptCacheKeyEnabled() && rawKey.length >= PROMPT_CACHE_KEY_MIN_LENGTH)
+        return;
+    const digest = digestSilentMissKey(rawKey);
+    if (loggedSilentMissKeys.has(digest))
+        return;
+    // FIFO eviction via Map insertion order — bounds memory under
+    // adversarial key flooding while preserving once-per-key semantics
+    // within the recent-key window.
+    if (loggedSilentMissKeys.size >= LOGGED_SILENT_MISS_KEYS_MAX) {
+        const oldest = loggedSilentMissKeys.keys().next().value;
+        if (oldest !== undefined)
+            loggedSilentMissKeys.delete(oldest);
+    }
+    loggedSilentMissKeys.set(digest, true);
+    if (!isPromptCacheKeyEnabled()) {
+        console.warn(`[mlx-node] prompt_cache_key supplied but tier-2 reuse is disabled ` +
+            `(MLX_DISABLE_PROMPT_CACHE_KEY=1). The key will be ignored and this ` +
+            `turn will cold-start. This message is logged once per distinct key.`);
+        return;
+    }
+    console.warn(`[mlx-node] prompt_cache_key is shorter than ${PROMPT_CACHE_KEY_MIN_LENGTH} chars; tier-2 reuse requires ` +
+        `at least ${PROMPT_CACHE_KEY_MIN_LENGTH} characters. The key will be ignored and this turn will ` +
+        `cold-start. This message is logged once per distinct key.`);
+}
 /**
  * Thrown synchronously by {@link SessionRegistry.withExclusive} when
  * the per-model queue cap (`maxQueueDepth`) is exceeded. The error is
@@ -104,6 +313,14 @@ export class SessionRegistry {
     model;
     ttlSec;
     maxQueueDepth;
+    /**
+     * Per-model sampling defaults forwarded into every new `ChatSession`
+     * via its `defaultConfig` constructor option. `undefined` preserves
+     * the pre-defaults behaviour (empty `defaultConfig`). See
+     * {@link SessionRegistryOptions.samplingDefaults}.
+     */
+    samplingDefaults;
+    maxOutputTokens;
     /**
      * Number of callers that are currently WAITING for the per-model
      * execution mutex — i.e. have entered `withExclusive` but have not
@@ -162,6 +379,24 @@ export class SessionRegistry {
         this.model = opts.model;
         this.ttlSec = opts.ttlSec ?? 1800;
         this.maxQueueDepth = opts.maxQueueDepth;
+        this.samplingDefaults = opts.samplingDefaults;
+        this.maxOutputTokens = opts.maxOutputTokens;
+    }
+    /**
+     * Construct a fresh `ChatSession` bound to this registry's model and
+     * pre-seeded with the operator-configured `samplingDefaults` (if any).
+     * Centralized so every cache-miss branch of `getOrCreate` produces a
+     * session whose per-call overlay will merge on top of the same
+     * defaults — clients cannot accidentally stray from the server's
+     * pinned sampling knobs by picking a cold-replay path.
+     */
+    newSession() {
+        if (this.samplingDefaults === undefined) {
+            return new ChatSession(this.model);
+        }
+        return new ChatSession(this.model, {
+            defaultConfig: this.samplingDefaults,
+        });
     }
     /**
      * Number of requests currently WAITING to acquire the per-model
@@ -171,6 +406,31 @@ export class SessionRegistry {
     get queueDepth() {
         return this.queuedCount;
     }
+    /**
+     * Current sampling defaults applied to every new `ChatSession` this
+     * registry allocates. Exposed primarily for tests and diagnostics.
+     */
+    get defaultSamplingConfig() {
+        return this.samplingDefaults;
+    }
+    get outputTokenLimit() {
+        return this.maxOutputTokens;
+    }
+    /**
+     * Replace the sampling defaults forwarded into every future
+     * `ChatSession` this registry allocates. Called by `ModelRegistry`
+     * on a `register(name, model, { samplingDefaults })` refresh so a
+     * fresh registration's defaults immediately apply to the next
+     * cache-miss cold-start. Sessions already cached at call time keep
+     * the defaults they were constructed with — they settle naturally
+     * through the single-warm cache rotation.
+     */
+    setSamplingDefaults(defaults) {
+        this.samplingDefaults = defaults;
+    }
+    setMaxOutputTokens(limit) {
+        this.maxOutputTokens = limit;
+    }
     /** Number of sessions currently cached. Primarily for tests and diagnostics. Always 0 or 1. */
     get size() {
         return this.entries.size;
@@ -180,60 +440,256 @@ export class SessionRegistry {
      * Always returns a `SessionLookupResult` and always leaves the cache
      * empty after return (single-warm invariant).
      *
-     * On a null id, missing key, expired entry, or prefix-state
-     * mismatch: clear and return `{ session: new ChatSession(model), hit: false }`.
-     * The caller primes / cold-replays from the `ResponseStore` and
-     * re-adopts after the turn commits.
+     * Lookup proceeds in two tiers:
      *
-     * On a hit: the entry is removed and its live session is returned
-     * alongside `hit: true`. Overlapping requests against the same
-     * `previous_response_id` cannot share the same live `ChatSession` —
-     * the first wins, the second misses and cold-replays.
+     *   1. **Tier 1 — `previousResponseId`.** The existing hot path:
+     *      exact id match on a live, non-expired entry whose stored
+     *      `instructions` are byte-equal to `requestedInstructions`. On
+     *      a match the entry is leased out (single-use: removed from the
+     *      map so a concurrent second request cannot share the live
+     *      `ChatSession`). On a miss — unknown id, expired, or
+     *      instructions drift — the method falls through to a FRESH
+     *      session regardless of whether tier 2 would have hit.
      *
-     * `requestedInstructions` is the caller's prefix/system state
-     * (OpenAI `instructions`, Anthropic `system`, or `null`); byte-for-
-     * byte mismatch against the cached entry forces cold replay so
-     * the new prefix is re-primed.
+     *      `previousResponseId` wins unconditionally when supplied. The
+     *      two keys could legitimately identify different conversation
+     *      branches (e.g. a client fork where one arm chose the prev-id
+     *      path and the other arm chose to set `prompt_cache_key`
+     *      without one), so routing the prev-id branch through tier 2
+     *      on miss risks splicing the wrong warm state into the wrong
+     *      chain. Cold-replay is the safe default.
+     *
+     *   2. **Tier 2 — `promptCacheKey`.** Only runs when
+     *      `previousResponseId` is `null`. Stateless agent clients
+     *      (pi-mono, Aider, Codex CLI, Continue, etc.) never use
+     *      `previous_response_id` — they own the conversation history
+     *      client-side and resend the full transcript on every turn —
+     *      so the only way to reuse a warm session across those turns
+     *      is to key on the client-supplied `prompt_cache_key`. Scans
+     *      for any live, non-expired entry whose stored
+     *      `promptCacheKey` is non-null AND byte-equal to the caller's
+     *      `promptCacheKey` AND whose stored `instructions` are byte-
+     *      equal. Empty string is treated as a distinct key from
+     *      `null` — an opt-out sentinel from a client that forgot to
+     *      thread the key must NOT collide with another client that
+     *      did set it to empty. On a match the entry is leased out
+     *      (same single-use semantics as tier 1). On a miss, fall
+     *      through to a fresh session.
      *
      * The `hit` flag drives the `X-Session-Cache` observability header
      * emitted by both `/v1/responses` and `/v1/messages`: when the caller
      * supplied a `previous_response_id`, `hit === true` yields `hit` and
      * `hit === false` yields `cold_replay` (the endpoint then rebuilds
      * from the `ResponseStore` on a fresh session). Requests with no
-     * `previous_response_id` (or the stateless `/v1/messages` endpoint,
-     * which always passes `null`) yield `fresh` regardless of this flag.
+     * `previous_response_id` yield either `fresh` (tier-2 miss) or
+     * `prefix_hit` (tier-2 hit — only classified as such once the
+     * native `cachedTokens > 0` confirms the prefix-cache machinery
+     * actually reused the cached tokens).
      */
-    getOrCreate(previousResponseId, requestedInstructions) {
+    getOrCreate(previousResponseId, requestedInstructions, promptCacheKey = null) {
+        // Tier 1: previousResponseId exact match.
+        //
         // Every call is about to overwrite native KV state, so drop any
         // other cached entry now — a later `getOrCreate` must not hand
         // out a wrapper whose assumed state has been stomped. Under the
         // single-warm invariant the map holds at most one entry, so the
         // common case is either "the entry we want" or "nothing".
-        if (previousResponseId === null) {
-            this.entries.clear();
-            return { session: new ChatSession(this.model), hit: false };
-        }
-        const entry = this.entries.get(previousResponseId);
-        if (entry === undefined) {
+        if (previousResponseId !== null) {
+            const entry = this.entries.get(previousResponseId);
+            if (entry === undefined) {
+                this.entries.clear();
+                return { session: this.newSession(), hit: false };
+            }
+            if (entry.expiresAt < nowSec()) {
+                this.entries.clear();
+                return { session: this.newSession(), hit: false };
+            }
+            // Prefix-state mismatch forces cold replay so the new
+            // instructions are re-primed; without this guard, output would
+            // silently depend on cache state instead of request contents.
+            if (entry.instructions !== requestedInstructions) {
+                this.entries.clear();
+                return { session: this.newSession(), hit: false };
+            }
+            // Tier-1 hit: clear and hand the session out as a single-use
+            // lease so a concurrent second request against the same id
+            // cold-replays instead of sharing this live ChatSession. Note
+            // that even on a prev-id tier-1 MISS we do NOT fall through to
+            // tier 2 — see the docstring above for the precedence
+            // rationale.
             this.entries.clear();
-            return { session: new ChatSession(this.model), hit: false };
+            return { session: entry.session, hit: true };
         }
-        if (entry.expiresAt < nowSec()) {
-            this.entries.clear();
-            return { session: new ChatSession(this.model), hit: false };
+        // Tier 2: promptCacheKey scan (only reached when previousResponseId is null).
+        //
+        // The registry holds at most one entry under the single-warm
+        // invariant, so the "scan" is actually a single lookup — walk the
+        // map, check the one entry if present, hit or miss. A non-null
+        // scoped key on both the request and the entry plus byte-equal
+        // instructions is the match condition.
+        //
+        // SECURITY: raw caller-supplied keys never touch the map. They
+        // are run through {@link scopePromptCacheKey}, which (a) returns
+        // `null` when the tier-2 opt-in env var is unset, (b) enforces a
+        // minimum length, and (c) HMACs the key with a boot-time nonce
+        // held only in this process's memory. Without the opt-in every
+        // tier-2 lookup below immediately misses; with the opt-in,
+        // attackers who cannot read the process-local nonce cannot craft
+        // a lookup that matches a stored entry by guessing the raw key.
+        const scopedKey = scopePromptCacheKey(promptCacheKey);
+        if (scopedKey !== null) {
+            for (const entry of this.entries.values()) {
+                if (entry.expiresAt < nowSec())
+                    continue;
+                if (entry.promptCacheKey === null)
+                    continue;
+                if (entry.promptCacheKey !== scopedKey)
+                    continue;
+                if (entry.instructions !== requestedInstructions)
+                    continue;
+                // Tier-2 hit: clear and lease (same single-warm / single-use
+                // semantics as tier 1).
+                this.entries.clear();
+                return { session: entry.session, hit: true };
+            }
         }
-        // Prefix-state mismatch forces cold replay so the new
-        // instructions are re-primed; without this guard, output would
-        // silently depend on cache state instead of request contents.
-        if (entry.instructions !== requestedInstructions) {
+        // Fall through: fresh session. Clear any leftover entry so a
+        // later lookup cannot hand out a wrapper whose assumed state has
+        // been overwritten by this dispatch.
+        this.entries.clear();
+        return { session: this.newSession(), hit: false };
+    }
+    /**
+     * Allocate a fresh `ChatSession` bound to this registry's model
+     * without touching the warm slot. Intended for the `/v1/messages`
+     * endpoint when the underlying model has a block-paged KV cache
+     * active: the native cache already reuses SYS blocks across requests
+     * via content-addressing in `BlockAllocator`'s prefix-hash table, so
+     * the JS-side warm slot in
+     * {@link SessionRegistry.getOrCreateWarmAny} is redundant.
+     *
+     * Crucially, this call is purely additive — it does **NOT** clear,
+     * read, or evict the warm slot. Two parallel `/v1/messages` requests
+     * sharing a system prompt both call `createFreshSession` and both
+     * get distinct sessions; the native cache transparently refcounts
+     * the shared SYS blocks across them. This is the routing decision
+     * the long block comment in `packages/server/src/endpoints/messages.ts`
+     * documents: paged → fresh session, non-paged → warm-any lookup.
+     *
+     * The returned session is pre-seeded with the operator-configured
+     * `samplingDefaults` (matching every other cache-miss branch) so a
+     * client that picks the paged path does not silently stray from the
+     * server's pinned sampling knobs.
+     *
+     * Returned with `hit: false` to keep the result shape uniform with
+     * {@link SessionRegistry.getOrCreate} and
+     * {@link SessionRegistry.getOrCreateWarmAny}; callers that care
+     * about the cache header semantics should observe
+     * `result.cachedTokens` from the dispatch instead — that's the
+     * authoritative signal for whether the native engine recovered any
+     * prefix on this turn (paged or otherwise).
+     */
+    createFreshSession() {
+        return { session: this.newSession(), hit: false };
+    }
+    /**
+     * @deprecated **Redundant on `/v1/messages` for paged-active models.**
+     * There is no call site for paged-active full-attention models (Qwen3 +
+     * LFM2 + Gemma4 today): the native block-paged KV adapter (`PagedKVCacheAdapter` +
+     * `BlockAllocator` + `LayerKVPool`) recovers a turn's prefix from
+     * refcounted KV blocks keyed by token-prefix hash, so the JS-side
+     * single-warm slot this method walks is redundant — the native
+     * cache picks up the same cross-turn reuse without the
+     * byte-equal-`instructions` gate, and additionally supports
+     * cross-conversation prefix sharing the warm slot cannot.
+     *
+     * The `/v1/messages` endpoint now branches at request time on
+     * {@link SessionCapableModel.hasBlockPagedCache}: paged-active models
+     * call {@link SessionRegistry.createFreshSession} per request and
+     * never touch the warm slot; non-paged models (Qwen3.5 dense + MoE —
+     * default-OFF pending a perf decision; the `QianfanOCRModel` VLM —
+     * no adapter wired) still call this method because the JS-side warm
+     * slot is the ONLY cross-conversation reuse mechanism available to
+     * them. Removing this method would silently disable cross-turn reuse
+     * on every non-paged model, so it stays load-bearing until ALL
+     * session-capable models have paged enabled by default. Treat
+     * `@deprecated` as an intent signal that paged-active callers should
+     * use `createFreshSession` instead.
+     *
+     * Third lookup mode — for STATELESS full-history endpoints that have
+     * no `previous_response_id` to thread and do not propagate
+     * `prompt_cache_key` back to the server. The Anthropic
+     * `/v1/messages` endpoint is the canonical caller: clients (e.g.
+     * Claude Code) POST the entire conversation each turn, so the only
+     * remaining signal that a turn N continues turn N-1's prefix is the
+     * registry's own warm slot.
+     *
+     * Behaviour: walk the registry's at-most-one warm entry. If it is
+     * non-expired AND its stored `instructions` are byte-equal to
+     * `requestedInstructions`, lease it out (single-use — `entries.clear()`
+     * before return, mirroring the tier-1 / tier-2 lease-on-hit
+     * semantics). Otherwise clear the map and return a fresh session.
+     *
+     * Crucially, this lookup IGNORES `entry.promptCacheKey` and ignores
+     * the entry's prior `previousResponseId` keying — any warm slot is
+     * fair game for `/v1/messages` reuse. The byte-equal `instructions`
+     * compare is the SOLE correctness gate: a system prompt change
+     * forces cold replay so the new prefix state is re-primed instead
+     * of silently reusing a stale warmed prompt.
+     *
+     * **Adoption sentinel.** `/v1/messages` adopts back under the literal
+     * sentinel id `'__msg_warm__'`. That sentinel will never appear as a
+     * `previous_response_id` on a `/v1/responses` request — the
+     * Anthropic Messages API does not produce a `previous_response_id`
+     * value clients could echo back, and the OpenAI side mints fresh
+     * `resp_*` ids — so cross-endpoint capture via tier-1 is impossible
+     * by construction. The two endpoints still SHARE the single warm
+     * slot under the registry's single-warm invariant: a
+     * `/v1/messages` turn that follows a `/v1/responses` turn can evict
+     * (and vice versa). That is the explicit trade-off of holding at
+     * most one warm entry per model.
+     *
+     * **Trust model.** Multi-tenant isolation on this endpoint requires
+     * fronting the server with an auth proxy that scopes warm-slot
+     * visibility per tenant — same trust boundary documented at the top
+     * of this file for the tier-2 `prompt_cache_key` path. The single-
+     * warm invariant plus `withExclusive`'s per-model serialization make
+     * the lookup safe under SINGLE-tenant assumptions: no two requests
+     * race the slot, and there is at most one slot to lease.
+     *
+     * **Caller contract on miss.** If `instructions` drifts between
+     * turns (system prompt changed) this returns `hit: false` and a
+     * fresh session — and the caller MUST then run a full
+     * `session.reset()` before priming history, NOT the JS-only
+     * `resetPreservingNativeCacheForWarmReuse` path. A fresh JS session
+     * does NOT imply a fresh native cache (the underlying
+     * `SessionCapableModel` is shared and its native
+     * `cached_token_history` persists across requests), so skipping the
+     * native wipe on a miss would let the next `chatSessionStart` reuse
+     * an unrelated previous request's prefix — the cross-request
+     * cache-affinity side channel that the long block comment in
+     * `responses.ts` (around the `runSessionNonStreaming` /
+     * `runSessionStreaming` branches) describes.
+     */
+    getOrCreateWarmAny(requestedInstructions) {
+        // Single-warm invariant: at most one entry. Walk it once, lease
+        // on a fresh + instructions-matched hit, otherwise clear and
+        // cold-start. The ignored fields (promptCacheKey,
+        // previousResponseId-keying) are deliberate — see the docstring.
+        for (const entry of this.entries.values()) {
+            if (entry.expiresAt < nowSec())
+                continue;
+            if (entry.instructions !== requestedInstructions)
+                continue;
+            // Hit: clear and lease (single-use semantics, same as tiers 1/2).
             this.entries.clear();
-            return { session: new ChatSession(this.model), hit: false };
+            return { session: entry.session, hit: true };
         }
-        // Hit: clear and hand the session out as a single-use lease so
-        // a concurrent second request against the same id cold-replays
-        // instead of sharing this live ChatSession.
+        // Miss (no entry, expired, or instructions drift). Clear the map
+        // so a stale wrapper cannot leak into a later lookup, and return
+        // a fresh session.
         this.entries.clear();
-        return { session: entry.session, hit: true };
+        return { session: this.newSession(), hit: false };
     }
     /**
      * Insert a session under a newly allocated response id. Clears the
@@ -243,12 +699,28 @@ export class SessionRegistry {
      * `instructions` is the prefix/system state used for this turn;
      * stored on the entry and compared on the next `getOrCreate` to
      * detect prefix changes that must force a cold replay.
+     *
+     * `promptCacheKey` is the client-supplied conversation-chain key
+     * that enables the registry's tier-2 lookup for stateless agent
+     * turns that do not carry a `previous_response_id`. `null` /
+     * `undefined` means "no key supplied" — stored verbatim so a
+     * subsequent stateless lookup that also omits the key does NOT
+     * accidentally pick up this entry (only explicit non-null
+     * key-equality on both sides can hit tier 2). See
+     * {@link SessionRegistry.getOrCreate} for the precedence rules.
      */
-    adopt(responseId, session, instructions) {
+    adopt(responseId, session, instructions, promptCacheKey = null) {
+        // Scope the caller-supplied key BEFORE storing so a later
+        // `getOrCreate` can only resolve entries via the same opt-in +
+        // HMAC path. When tier-2 reuse is disabled (or the key is too
+        // short / absent) `scopePromptCacheKey` returns `null`, which
+        // disables this entry from ever matching a tier-2 lookup — the
+        // raw caller-supplied key is NEVER stored.
         this.entries.clear();
         this.entries.set(responseId, {
             session,
             instructions,
+            promptCacheKey: scopePromptCacheKey(promptCacheKey ?? null),
             expiresAt: nowSec() + this.ttlSec,
         });
     }