@mlx-node/server 0.0.9 → 0.0.12

package/dist/session-registry.d.ts

@@ -21,18 +21,13 @@
  *     by response id — no secondary keying on model name because the
  *     registry is already scoped per model.
  *
- *   - **Single-warm-session invariant.** `ChatSession<M>` is a thin
- *     JS wrapper — it does NOT own any native KV cache. The cache
- *     lives on the underlying `SessionCapableModel` (one shared
- *     `cached_token_history` / `caches` vector per model instance).
- *     Any call that runs a turn overwrites that shared native state,
- *     silently invalidating every other `ChatSession` wrapper
- *     pointing at the same model. Caching multiple wrappers per
- *     model is therefore an illusion: at most ONE matches real
- *     native state (whichever ran most recently). To prevent
- *     cross-session corruption this registry holds at most ONE
- *     entry — both `getOrCreate` and `adopt` clear the map before
- *     returning or inserting.
+ *   - **Single-warm-session invariant.** The JS warm registry retains at
+ *     most ONE `ChatSession` entry — both `getOrCreate` and `adopt` clear
+ *     the map before returning or inserting. This remains the sole safe
+ *     reuse mechanism for flat-cache models whose native cache is one
+ *     mutable vector. Block-paged schedulers instead isolate live turns by
+ *     cache owner and reuse verified physical blocks through the native
+ *     prefix table; those models may run fresh JS sessions concurrently.
  *
  *   - **Lease semantics on hit.** Clear-on-hit also gives single-
  *     flight lease semantics: two overlapping requests referencing
@@ -43,14 +38,13 @@
  *     `ChatSession`'s single-flight "concurrent send() not allowed"
  *     guard.
  *
- *   - **Instructions / prefix-state change also misses.** Each entry
- *     records the `instructions` string used to adopt it.
- *     `getOrCreate` compares the caller's `requestedInstructions`
- *     against the cached value; mismatch forces cold replay so the
- *     new prefix state is re-primed instead of silently reusing a
- *     stale warmed prompt. The OpenAI `instructions` field and the
- *     Anthropic `system` field both flow through the same parameter
- *     — the registry does not care which is which.
+ *   - **Prefix compatibility changes miss.** Each entry records its
+ *     `instructions` plus an opaque fingerprint of the cache salt used to
+ *     adopt it. `getOrCreate` compares both against the new request;
+ *     mismatch forces owner release and cold replay instead of reusing a
+ *     stale prompt or changing the security domain of one live native
+ *     request. OpenAI `instructions` and Anthropic `system` share the same
+ *     parameter; both endpoints also thread their mapped `cache_salt`.
  *
  *   - **Cache miss fallback.** On a miss (eviction, interleaved turn
  *     on a different chain, restart, lease-on-hit) the endpoint
@@ -73,32 +67,20 @@
  *     concurrent mutation by design. `sweep()` can be scheduled
  *     via `setInterval` without colliding with in-flight calls.
  *
- *   - **Per-model execution mutex.** A dispatch that spans multiple
- *     awaits (map -> prefill -> decode -> persist -> adopt) is NOT
- *     atomic from the registry's POV. Two requests against the
- *     same model would both receive a `ChatSession` pointing at
- *     the same native model; even though the lease-on-hit clear
- *     prevents sharing one `ChatSession` object, the native KV
- *     cache is a single mutable resource and two parallel
- *     `primeHistory()` / `send*()` calls would race. Whichever
- *     finished last would win `adopt()`, poisoning the hot path
- *     for every subsequent chained turn.
- *
- *     `withExclusive(fn)` serializes every per-model dispatch via
- *     a FIFO `execLock` chain. `/v1/responses` and `/v1/messages`
- *     wrap the full `getOrCreate -> run -> adopt/drop` span in one
- *     `withExclusive` so at most one request holds the model at a
- *     time. A weaker epoch-token scheme would let the losing
- *     `adopt()` no-op but the native KV would already be wrong.
+ *   - **Per-model admission lane.** Flat and not-yet-batched families use
+ *     `withExclusive(fn)`, the original FIFO mutex across the full dispatch.
+ *     A model that explicitly reports `maxConcurrentSequences() > 1` uses
+ *     `withAdmission(fn)`, a counting semaphore sized to that native
+ *     scheduler. Per-session serialization still lives in `ChatSession`;
+ *     only independent sessions share the model lane.
  */
 import type { ChatConfig } from '@mlx-node/core';
 import { ChatSession, type SessionCapableModel } from '@mlx-node/lm';
 /**
- * Test-only hook used by the scoping unit tests to simulate a
- * server restart: resets the module-scoped HMAC nonce (so every
- * previously stored tier-2 key misses) and clears the silent-miss
- * dedupe cache so tests can re-exercise the once-per-key diagnostic
- * path.
+ * Test-only hook used by the scoping unit tests to simulate a server
+ * restart: resets the module-scoped HMAC nonce (so every previously stored
+ * tier-2 key and cache-salt fingerprint misses) and clears the silent-miss
+ * dedupe cache so tests can re-exercise the once-per-key diagnostic path.
  *
  * **Not exported from the package's public `index.ts` surface** —
  * exporting it there would let downstream consumers nuke tier-2
@@ -152,11 +134,21 @@ export interface SessionRegistryOptions {
      * synchronously so the endpoint layer can emit HTTP 429 and the
      * client can retry later.
      *
-     * Default: `undefined` (unbounded — current behaviour). Opt-in per
-     * {@link ServerConfig.maxQueueDepthPerModel} or the
-     * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var.
+     * Default at THIS layer: `undefined` (unbounded) — a directly
+     * constructed `SessionRegistry` has no cap. `createServer` supplies
+     * its own default of 16 per {@link ServerConfig.maxQueueDepthPerModel}
+     * (or the `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var; config
+     * `'unbounded'` opts out), so server-allocated registries are capped
+     * unless the operator explicitly opted out.
      */
     maxQueueDepth?: number;
+    /**
+     * Number of independent dispatches the native model scheduler can advance
+     * concurrently. Values below two retain the legacy exclusive FIFO. The
+     * model registry derives this from `maxConcurrentSequences()`; direct
+     * construction defaults to one.
+     */
+    maxConcurrentDispatches?: number;
     /**
      * Optional sampling defaults applied to every `ChatSession` this
      * registry allocates. Forwarded verbatim into `new ChatSession(model,
@@ -184,9 +176,33 @@ export interface SessionRegistryOptions {
  * can reliably catch it without racing the chain.
  */
 export declare class QueueFullError extends Error {
-    readonly queuedCount: number;
+    readonly queueDepth: number;
+    readonly preDispatchAdmissions: number;
+    readonly admissionFootprint: number;
     readonly limit: number;
-    constructor(queuedCount: number, limit: number);
+    constructor(queueDepth: number, preDispatchAdmissions: number, limit: number);
+}
+/**
+ * Admission permit returned by
+ * {@link SessionRegistry.beginPreDispatchAdmission}. Represents exactly
+ * ONE unit of the per-model admission budget, and ends in exactly one of
+ * two terminal states:
+ *
+ *  - **handed off**: passed as the second argument of
+ *    `withExclusive(fn, permit)` or `withAdmission(fn, permit)`, which
+ *    consumes it atomically as that
+ *    call's admission — the unit converts into the waiter charge (or is
+ *    retired when the caller wins the runner slot). `release()` becomes
+ *    a no-op afterwards.
+ *  - **released**: `release()` frees the unit without a dispatch. Every
+ *    exit between admission and lock placement must do this — the
+ *    recommended pattern is one unconditional `release()` in a
+ *    `finally`, which is safe on every path because `release()` is
+ *    idempotent and a no-op after handoff.
+ */
+export interface PreDispatchAdmission {
+    /** Idempotent: free the budget unit unless already handed off. */
+    release(): void;
 }
 /**
  * Result of {@link SessionRegistry.getOrCreate}. `hit` reflects whether
@@ -203,6 +219,7 @@ export declare class SessionRegistry {
     private readonly model;
     private readonly ttlSec;
     private readonly maxQueueDepth;
+    private readonly maxConcurrentDispatches;
     /**
      * Per-model sampling defaults forwarded into every new `ChatSession`
      * via its `defaultConfig` constructor option. `undefined` preserves
@@ -232,6 +249,25 @@ export declare class SessionRegistry {
      * runner-slot caller against the waiter cap.
      */
     private queuedCount;
+    /**
+     * Requests admitted by {@link beginPreDispatchAdmission} whose permit
+     * is still outstanding — parked in the `ModelWorkCoordinator` writer
+     * queue (host mode), blocked in pre-lock store lookups
+     * (`previous_response_id` continuations), or anywhere else between the
+     * endpoint gate and resident-lane placement. None of that parking is
+     * visible to `queuedCount`; this counter is what lets the gate bound
+     * it. Decremented ONLY by the permit itself: `release()` on a bail-out
+     * or the atomic consume inside the selected lane on handoff.
+     */
+    private preDispatchAdmits;
+    /**
+     * Consume hooks for outstanding permits, keyed by permit identity.
+     * Registry-scoped on purpose: both execution lanes consult THIS map, so a
+     * permit minted by a different registry is simply not found and the
+     * call falls back to normal waiter charging — a cross-registry handoff
+     * cannot corrupt either registry's counters.
+     */
+    private readonly permitConsumers;
     /**
      * Holds AT MOST ONE entry under the single-warm invariant (see the
      * module-level rustdoc). `getOrCreate` and `adopt` both clear the
@@ -240,6 +276,15 @@ export declare class SessionRegistry {
      * turn on another cached entry.
      */
     private readonly entries;
+    /**
+     * Eviction is synchronous at the map boundary, but releasing a native
+     * scheduler owner is asynchronous. Start every disposal immediately and
+     * retain its promise so endpoint admission lanes can wait for command-order
+     * visibility before dispatching a replacement turn.
+     */
+    private readonly pendingDisposals;
+    private readonly disposalBySession;
+    private readonly failedDisposals;
     /**
      * Shared sentinel representing "the execution chain is idle" — a
      * pre-resolved promise. `execLock` starts at this value and is
@@ -265,6 +310,10 @@ export declare class SessionRegistry {
      * cleanly from the idle state.
      */
     private execLock;
+    /** Active holders in the continuous-batching admission lane. */
+    private activeAdmissions;
+    /** FIFO waiters parked behind the continuous-batching admission limit. */
+    private readonly admissionWaiters;
     constructor(opts: SessionRegistryOptions);
     /**
      * Construct a fresh `ChatSession` bound to this registry's model and
@@ -275,19 +324,117 @@ export declare class SessionRegistry {
      * pinned sampling knobs by picking a cold-replay path.
      */
     private newSession;
+    private scheduleDispose;
+    /**
+     * Dispose one leased session while retaining failed cleanup for a later
+     * registry flush. `ChatSession.dispose()` removes successful owners as it
+     * goes, so a retry only revisits owners whose native release failed.
+     */
+    disposeSession(session: ChatSession<SessionCapableModel>): Promise<void>;
+    /** Remove every cached entry except an optional session being leased. */
+    private evictEntriesExcept;
+    /**
+     * Wait until every disposal scheduled so far has settled, retrying each
+     * failure discovered during this flush once. A persistent failure remains
+     * recorded for a later flush instead of spinning forever. Disposals log and
+     * absorb their own failures so cleanup cannot rewrite a response that has
+     * already reached the client.
+     */
+    flushPendingDisposals(): Promise<void>;
     /**
-     * Number of requests currently WAITING to acquire the per-model
-     * execution mutex. Does NOT include the one actively running inside
-     * `fn`. Primarily for tests and diagnostics.
+     * Disposals started but not yet settled — each one is an in-flight
+     * native `releaseCacheOwner` round-trip — plus disposals whose initial
+     * attempt and bounded retry both failed and remain owed to the native
+     * scheduler ({@link failedDisposals}), retried by the next
+     * {@link flushPendingDisposals}. Endpoints await that flush before
+     * leaving the admission lane, but it runs after the response has
+     * finished, so an observer keyed on request completion can still beat
+     * the release. `adopt`/`drop`/`sweep` schedule synchronously, so once
+     * the request counters read zero any disposal those requests will ever
+     * cause is already counted here. Primarily for diagnostics/tests.
+     */
+    get pendingDisposalCount(): number;
+    /**
+     * Number of requests waiting for this model's selected admission lane.
+     * Active dispatches are not included. Primarily for diagnostics/tests.
      */
     get queueDepth(): number;
     /**
-     * Configured waiter cap for this model's execution mutex, or `undefined`
+     * Configured waiter cap for this model's admission lane, or `undefined`
      * when unbounded. Paired with {@link queueDepth} so a readiness probe can
      * tell "3 waiters, unbounded" (fine) from "3 waiters, cap of 3" (the next
      * request gets a 429) without reaching into private state.
      */
     get queueDepthLimit(): number | undefined;
+    /** Native continuous-batching capacity used by the endpoint route switch. */
+    get concurrentAdmissionLimit(): number;
+    /**
+     * Outstanding pre-dispatch permits — requests admitted by
+     * {@link beginPreDispatchAdmission} that have neither handed their
+     * permit to the selected execution lane nor released it yet. These permits,
+     * queued callers, and active dispatches share one bounded budget; see
+     * {@link assertAdmissionCapacity}. For probes and diagnostics.
+     */
+    get preDispatchAdmitCount(): number;
+    /**
+     * Single source of truth for the per-model admission budget — the
+     * ONLY place the cap arithmetic lives. Every admission path calls
+     * this: {@link beginPreDispatchAdmission} before minting a permit,
+     * and both execution lanes for every non-handed-off caller. A handed-off
+     * permit skips the call for its OWN
+     * token only: that token was charged here at acquisition and its
+     * conversion keeps the total constant, so re-checking would
+     * double-charge an already-admitted request. It never exempts anyone
+     * else — all other outstanding state stays counted for every caller
+     * that did not pay.
+     *
+     * Budget invariant: active dispatches + queued callers + outstanding
+     * permits never exceed `maxConcurrentDispatches + maxQueueDepth`.
+     * On the exclusive lane `maxConcurrentDispatches` is one, exactly the
+     * original runner entitlement. On the batched lane it is the native
+     * scheduler's sequence capacity.
+     *
+     * Charging stays at the call sites (`preDispatchAdmits += 1` at the
+     * gate, `queuedCount += 1` for waiters, and the selected lane's active
+     * count for runners); every admitted unit is counted by exactly one
+     * at any time, which is what makes the footprint sum complete across
+     * any interleaving of permitted and permitless callers.
+     *
+     * Throws {@link QueueFullError} — reporting the footprint and the
+     * cap — when the caller does not fit; returns normally otherwise.
+     * No-op when the registry is unbounded.
+     */
+    private assertAdmissionCapacity;
+    /**
+     * Endpoint-side early admission against this registry's cap, taken BEFORE
+     * the request enters any pre-dispatch parking spot the resident lane cannot
+     * see. Resident host traffic deliberately bypasses the model-load writer so
+     * continuous batching remains reachable, but a continuation can still block
+     * in `await store.getChain(...)` (or other pre-lock work) with `queuedCount`
+     * unchanged. This permit keeps all such work inside the same bounded budget.
+     *
+     * Accounting: pre-dispatch permits, active dispatches, and queued callers
+     * draw from ONE budget: `maxQueueDepth` waiter slots plus the selected
+     * lane's active capacity. Both lanes and this early gate route through
+     * {@link assertAdmissionCapacity}, so permitted and permitless arrivals
+     * cannot double-spend a slot.
+     *
+     * Throws {@link QueueFullError} synchronously when over cap (the
+     * caller maps it to the same 429 envelope as resident-lane rejection
+     * reject). On admission returns a {@link PreDispatchAdmission} permit
+     * the caller must RETAIN through ALL pre-lock asynchronous work and
+     * then hand to `withExclusive(fn, permit)` or `withAdmission(fn, permit)`,
+     * which consumes it
+     * atomically as that call's admission — one budget, one token per
+     * request, never double-counted. Releasing the permit early instead
+     * of handing it off re-opens the hole this gate closes: the request
+     * would be counted by NEITHER counter while parked, arrivals would
+     * refill the budget, and the resident lane would then admit a second
+     * full waiter budget on top. `release()` belongs on bail-out paths
+     * only (idempotent, no-op after handoff — an unconditional `finally`
+     * release is the recommended shape).
+     */
+    beginPreDispatchAdmission(): PreDispatchAdmission;
     /**
      * Current sampling defaults applied to every new `ChatSession` this
      * registry allocates. Exposed primarily for tests and diagnostics.
@@ -316,11 +463,11 @@ export declare class SessionRegistry {
      *
      *   1. **Tier 1 — `previousResponseId`.** The existing hot path:
      *      exact id match on a live, non-expired entry whose stored
-     *      `instructions` are byte-equal to `requestedInstructions`. On
+     *      `instructions` and cache-salt fingerprint match the request. On
      *      a match the entry is leased out (single-use: removed from the
      *      map so a concurrent second request cannot share the live
-     *      `ChatSession`). On a miss — unknown id, expired, or
-     *      instructions drift — the method falls through to a FRESH
+     *      `ChatSession`). On a miss — unknown id, expired, instructions
+     *      drift, or cache-salt drift — the method falls through to a FRESH
      *      session regardless of whether tier 2 would have hit.
      *
      *      `previousResponseId` wins unconditionally when supplied. The
@@ -340,9 +487,9 @@ export declare class SessionRegistry {
      *      is to key on the client-supplied `prompt_cache_key`. Scans
      *      for any live, non-expired entry whose stored
      *      `promptCacheKey` is non-null AND byte-equal to the caller's
-     *      `promptCacheKey` AND whose stored `instructions` are byte-
-     *      equal. Empty string is treated as a distinct key from
-     *      `null` — an opt-out sentinel from a client that forgot to
+     *      `promptCacheKey`, whose stored `instructions` are byte-equal,
+     *      AND whose cache-salt fingerprint matches. Empty string is treated
+     *      as a distinct key from `null` — an opt-out sentinel from a client that forgot to
      *      thread the key must NOT collide with another client that
      *      did set it to empty. On a match the entry is leased out
      *      (same single-use semantics as tier 1). On a miss, fall
@@ -358,7 +505,7 @@ export declare class SessionRegistry {
      * native `cachedTokens > 0` confirms the prefix-cache machinery
      * actually reused the cached tokens).
      */
-    getOrCreate(previousResponseId: string | null, requestedInstructions: string | null, promptCacheKey?: string | null): SessionLookupResult;
+    getOrCreate(previousResponseId: string | null, requestedInstructions: string | null, promptCacheKey?: string | null, requestedCacheSalt?: string | null): SessionLookupResult;
     /**
      * Allocate a fresh `ChatSession` bound to this registry's model
      * without touching the warm slot. Intended for the `/v1/messages`
@@ -423,17 +570,18 @@ export declare class SessionRegistry {
      * registry's own warm slot.
      *
      * Behaviour: walk the registry's at-most-one warm entry. If it is
-     * non-expired AND its stored `instructions` are byte-equal to
-     * `requestedInstructions`, lease it out (single-use — `entries.clear()`
-     * before return, mirroring the tier-1 / tier-2 lease-on-hit
-     * semantics). Otherwise clear the map and return a fresh session.
+     * non-expired, its stored `instructions` are byte-equal to
+     * `requestedInstructions`, AND its stored cache salt equals
+     * `requestedCacheSalt`, lease it out (single-use — `entries.clear()`
+     * before return, mirroring the tier-1 / tier-2 lease-on-hit semantics).
+     * Otherwise clear the map and return a fresh session.
      *
      * Crucially, this lookup IGNORES `entry.promptCacheKey` and ignores
      * the entry's prior `previousResponseId` keying — any warm slot is
-     * fair game for `/v1/messages` reuse. The byte-equal `instructions`
-     * compare is the SOLE correctness gate: a system prompt change
-     * forces cold replay so the new prefix state is re-primed instead
-     * of silently reusing a stale warmed prompt.
+     * fair game for `/v1/messages` reuse. Byte-equal instructions and cache
+     * salt are the correctness gates: a system prompt or prefix-cache
+     * security-domain change forces cold replay instead of reusing stale state
+     * or asking the native adapter to mutate a live request's salt.
      *
      * **Adoption sentinel.** `/v1/messages` adopts back under the literal
      * sentinel id `'__msg_warm__'`. That sentinel will never appear as a
@@ -469,7 +617,7 @@ export declare class SessionRegistry {
      * `responses.ts` (around the `runSessionNonStreaming` /
      * `runSessionStreaming` branches) describes.
      */
-    getOrCreateWarmAny(requestedInstructions: string | null): SessionLookupResult;
+    getOrCreateWarmAny(requestedInstructions: string | null, requestedCacheSalt?: string | null): SessionLookupResult;
     /**
      * Insert a session under a newly allocated response id. Clears the
      * map before inserting to keep the single-warm invariant explicit
@@ -478,6 +626,8 @@ export declare class SessionRegistry {
      * `instructions` is the prefix/system state used for this turn;
      * stored on the entry and compared on the next `getOrCreate` to
      * detect prefix changes that must force a cold replay.
+     * `cacheSalt` is HMAC-fingerprinted before storage and compared alongside
+     * the prefix so the raw security-domain key is not retained.
      *
      * `promptCacheKey` is the client-supplied conversation-chain key
      * that enables the registry's tier-2 lookup for stateless agent
@@ -488,7 +638,7 @@ export declare class SessionRegistry {
      * key-equality on both sides can hit tier 2). See
      * {@link SessionRegistry.getOrCreate} for the precedence rules.
      */
-    adopt(responseId: string, session: ChatSession<SessionCapableModel>, instructions: string | null, promptCacheKey?: string | null | undefined): void;
+    adopt(responseId: string, session: ChatSession<SessionCapableModel>, instructions: string | null, promptCacheKey?: string | null | undefined, cacheSalt?: string | null | undefined): void;
     /**
      * Remove a session by response id. No-op if the key is not present.
      */
@@ -515,33 +665,63 @@ export declare class SessionRegistry {
      * the new tail, awaits the old tail, then runs `fn`. The
      * `finally` releases regardless of whether `fn` threw.
      *
-     * **Admission control.** When `maxQueueDepth` is configured and the
-     * current number of waiters (`queuedCount`, excluding the active
-     * holder) is already at or above the cap, the call throws
-     * {@link QueueFullError} synchronously — SYNCHRONOUSLY from the
-     * caller's perspective, not merely before `await prev`. The wrapper
-     * is deliberately NOT declared `async` so the admission gate
-     * throws on the caller's stack frame, letting endpoint handlers
-     * wrap the call site in a plain try/catch without racing promise
+     * **Admission control.** Every non-handed-off call — waiter AND
+     * runner — routes through {@link assertAdmissionCapacity}, the single
+     * source of truth for the budget: the combined admission footprint
+     * (`queuedCount + preDispatchAdmits`) plus this caller must fit
+     * within `maxQueueDepth` plus the idle-chain runner entitlement.
+     * Over-budget calls throw {@link QueueFullError} — SYNCHRONOUSLY
+     * from the caller's perspective, not merely before `await prev`. The
+     * wrapper is deliberately NOT declared `async` so the admission gate
+     * throws on the caller's stack frame, letting endpoint handlers wrap
+     * the call site in a plain try/catch without racing promise
      * microtasks. On acceptance the async body takes over via the
      * returned `Promise<T>`.
      *
-     * The cap is "waiters-only" — a cap of N permits one running
-     * dispatch plus N queued ones, rejecting the (N+1)th waiter. The
-     * default (undefined) preserves the original unbounded behaviour.
+     * A cap of N permits one running dispatch plus N admitted-but-not-
+     * running requests (queued waiters and outstanding permits
+     * combined), rejecting the next. The default (undefined) preserves
+     * the original unbounded behaviour.
      *
      * **Runner-slot admission.** Whether a given caller counts as the
      * runner slot or as a waiter is decided up front by comparing
      * `execLock` against the idle sentinel `initialLock`. If they are
-     * identical, nobody is currently in-flight and this caller wins
-     * the runner slot: it is not counted against the waiter cap and
-     * never touches `queuedCount`. Otherwise it is a waiter and the
-     * normal cap check / increment / decrement cycle applies. This is
-     * what keeps a synchronous burst such as `Promise.all([fn, fn])`
+     * identical, nobody is currently in-flight and this caller is
+     * admitted against the extra runner entitlement rather than a waiter
+     * slot — it never touches `queuedCount`. Otherwise it is a waiter
+     * and the normal charge / increment / decrement cycle applies. This
+     * is what keeps a synchronous burst such as `Promise.all([fn, fn])`
      * admissible under `maxQueueDepth = 1` — Call 1 is the runner,
-     * Call 2 is the one allowed waiter, Call 3 would throw.
+     * Call 2 is the one allowed waiter, Call 3 would throw. The runner
+     * path is NOT exempt from the budget: while the chain is idle,
+     * {@link beginPreDispatchAdmission} lends out `maxQueueDepth + 1`
+     * permits precisely because one of them is entitled to become the
+     * runner, so a permitless call that would take that seat while the
+     * whole runner-plus-waiter capacity is already spoken for must
+     * reject — otherwise the outstanding permits would later convert on
+     * top of it and breach the cap.
+     *
+     * **Permit handoff.** A caller that was already admitted by
+     * {@link beginPreDispatchAdmission} passes its permit as the second
+     * argument; the permit is consumed atomically as this call's
+     * admission instead of charging `queuedCount` a second time — one
+     * budget, one token per request. See the handoff comment in the
+     * body and {@link PreDispatchAdmission}.
+     */
+    withExclusive<T>(fn: () => Promise<T>, permit?: PreDispatchAdmission): Promise<T>;
+    /**
+     * Admit one dispatch to the model's continuous-batching lane.
+     *
+     * Admission rejection is synchronous, matching {@link withExclusive} and
+     * preserving the endpoint's existing QueueFullError-to-429 mapping. Once
+     * accepted, at most {@link concurrentAdmissionLimit} closures run at once;
+     * excess accepted callers wait FIFO and contribute to {@link queueDepth}.
+     * A pre-dispatch permit is consumed atomically into either an active slot or
+     * a queued slot, so the early endpoint gate and this semaphore share one
+     * bounded budget.
      */
-    withExclusive<T>(fn: () => Promise<T>): Promise<T>;
+    withAdmission<T>(fn: () => Promise<T>, permit?: PreDispatchAdmission): Promise<T>;
+    private _runAdmission;
     /**
      * Async tail of {@link withExclusive}. Kept separate so the public
      * wrapper stays a plain (non-async) function whose admission-gate

package/dist/session-registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyErE;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kCAAkC,IAAI,IAAI,CAGzD;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAE3D;AAuDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CA2BzF;AAED,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAK7C;CACF;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AAsCD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,eAAe,CAAqB;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;IAEnD,YAAY,IAAI,EAAE,sBAAsB,EAMvC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IASlB;;;;OAIG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;;;OAKG;IACH,IAAI,eAAe,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;OAGG;IACH,IAAI,qBAAqB,IAAI,UAAU,GAAG,SAAS,CAElD;IAED,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED;;;;;;;;OAQG;IACH,mBAAmB,CAAC,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE1D;IAED,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;IACH,WAAW,CACT,kBAAkB,EAAE,MAAM,GAAG,IAAI,EACjC,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,cAAc,GAAE,MAAM,GAAG,IAAW,GACnC,mBAAmB,CAsErB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,kBAAkB,IAAI,mBAAmB,CAExC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8EG;IACH,kBAAkB,CAAC,qBAAqB,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB,CAiB5E;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EACzC,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,cAAc,GAAE,MAAM,GAAG,IAAI,GAAG,SAAgB,GAC/C,IAAI,CAcN;IAED;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAE7B;IAED;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAOZ;IAED,2DAA2D;IAC3D,KAAK,IAAI,IAAI,CAEZ;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CA4BjD;IAED;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}
+{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2EG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AA+ErE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kCAAkC,IAAI,IAAI,CAGzD;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAE3D;AAuDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CA2BzF;AAED,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,UAAU,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAW3E;CACF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,oBAAoB;IACnC,kEAAkE;IAClE,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AA6CD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,eAAe,CAAqB;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB,CAAK;IAC9B;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsD;IACtF;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA4B;IAC7D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAkE;IACpG,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA+C;IAC/E;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;IACnD,gEAAgE;IAChE,OAAO,CAAC,gBAAgB,CAAK;IAC7B,0EAA0E;IAC1E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAyB;IAE1D,YAAY,IAAI,EAAE,sBAAsB,EASvC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,eAAe;IAevB;;;;OAIG;IACG,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAQ7E;IAED,yEAAyE;IACzE,OAAO,CAAC,kBAAkB;IAO1B;;;;;;OAMG;IACG,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAY3C;IAED;;;;;;;;;;;OAWG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;;;OAKG;IACH,IAAI,eAAe,IAAI,MAAM,GAAG,SAAS,CAExC;IAED,6EAA6E;IAC7E,IAAI,wBAAwB,IAAI,MAAM,CAErC;IAED;;;;;;OAMG;IACH,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,yBAAyB,IAAI,oBAAoB,CAmBhD;IAED;;;OAGG;IACH,IAAI,qBAAqB,IAAI,UAAU,GAAG,SAAS,CAElD;IAED,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED;;;;;;;;OAQG;IACH,mBAAmB,CAAC,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE1D;IAED,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;IACH,WAAW,CACT,kBAAkB,EAAE,MAAM,GAAG,IAAI,EACjC,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,cAAc,GAAE,MAAM,GAAG,IAAW,EACpC,kBAAkB,GAAE,MAAM,GAAG,IAAW,GACvC,mBAAmB,CA4ErB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,kBAAkB,IAAI,mBAAmB,CAExC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+EG;IACH,kBAAkB,CAChB,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,kBAAkB,GAAE,MAAM,GAAG,IAAW,GACvC,mBAAmB,CAmBrB;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EACzC,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,cAAc,GAAE,MAAM,GAAG,IAAI,GAAG,SAAgB,EAChD,SAAS,GAAE,MAAM,GAAG,IAAI,GAAG,SAAgB,GAC1C,IAAI,CAeN;IAED;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAK7B;IAED;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAQZ;IAED,2DAA2D;IAC3D,KAAK,IAAI,IAAI,CAEZ;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwDG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,CAAC,CAAC,CAqDhF;IAED;;;;;;;;;;OAUG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,CAAC,CAAC,CAwBhF;YAEa,aAAa;IAY3B;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}