@mlx-node/server 0.0.7 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/auth.d.ts +56 -0
  2. package/dist/auth.d.ts.map +1 -0
  3. package/dist/auth.js +106 -0
  4. package/dist/chat-session-warm-reuse.d.ts +51 -0
  5. package/dist/chat-session-warm-reuse.d.ts.map +1 -0
  6. package/dist/chat-session-warm-reuse.js +72 -0
  7. package/dist/endpoints/messages-count-tokens.d.ts +8 -0
  8. package/dist/endpoints/messages-count-tokens.d.ts.map +1 -0
  9. package/dist/endpoints/messages-count-tokens.js +121 -0
  10. package/dist/endpoints/messages.d.ts +57 -5
  11. package/dist/endpoints/messages.d.ts.map +1 -1
  12. package/dist/endpoints/messages.js +1043 -147
  13. package/dist/endpoints/models.d.ts +2 -1
  14. package/dist/endpoints/models.d.ts.map +1 -1
  15. package/dist/endpoints/models.js +2 -2
  16. package/dist/endpoints/responses.d.ts +22 -7
  17. package/dist/endpoints/responses.d.ts.map +1 -1
  18. package/dist/endpoints/responses.js +608 -85
  19. package/dist/errors.d.ts +1 -0
  20. package/dist/errors.d.ts.map +1 -1
  21. package/dist/errors.js +3 -0
  22. package/dist/handler.d.ts +69 -1
  23. package/dist/handler.d.ts.map +1 -1
  24. package/dist/handler.js +70 -16
  25. package/dist/health.d.ts +146 -0
  26. package/dist/health.d.ts.map +1 -0
  27. package/dist/health.js +107 -0
  28. package/dist/host/discover.d.ts +19 -0
  29. package/dist/host/discover.d.ts.map +1 -0
  30. package/dist/host/discover.js +50 -0
  31. package/dist/host/env-policy.d.ts +62 -0
  32. package/dist/host/env-policy.d.ts.map +1 -0
  33. package/dist/host/env-policy.js +69 -0
  34. package/dist/host/index.d.ts +202 -0
  35. package/dist/host/index.d.ts.map +1 -0
  36. package/dist/host/index.js +325 -0
  37. package/dist/host/logger.d.ts +36 -0
  38. package/dist/host/logger.d.ts.map +1 -0
  39. package/dist/host/logger.js +376 -0
  40. package/dist/host/net.d.ts +65 -0
  41. package/dist/host/net.d.ts.map +1 -0
  42. package/dist/host/net.js +97 -0
  43. package/dist/host/paths.d.ts +28 -0
  44. package/dist/host/paths.d.ts.map +1 -0
  45. package/dist/host/paths.js +71 -0
  46. package/dist/host/swap.d.ts +27 -0
  47. package/dist/host/swap.d.ts.map +1 -0
  48. package/dist/host/swap.js +178 -0
  49. package/dist/host/temp-root.d.ts +57 -0
  50. package/dist/host/temp-root.d.ts.map +1 -0
  51. package/dist/host/temp-root.js +99 -0
  52. package/dist/idle-sweeper.d.ts +245 -0
  53. package/dist/idle-sweeper.d.ts.map +1 -0
  54. package/dist/idle-sweeper.js +408 -0
  55. package/dist/index.d.ts +19 -4
  56. package/dist/index.d.ts.map +1 -1
  57. package/dist/index.js +17 -1
  58. package/dist/load-model.d.ts +69 -0
  59. package/dist/load-model.d.ts.map +1 -0
  60. package/dist/load-model.js +63 -0
  61. package/dist/mappers/anthropic-request.d.ts +24 -2
  62. package/dist/mappers/anthropic-request.d.ts.map +1 -1
  63. package/dist/mappers/anthropic-request.js +222 -24
  64. package/dist/mappers/anthropic-response.d.ts +29 -4
  65. package/dist/mappers/anthropic-response.d.ts.map +1 -1
  66. package/dist/mappers/anthropic-response.js +143 -21
  67. package/dist/mappers/request.d.ts +48 -0
  68. package/dist/mappers/request.d.ts.map +1 -1
  69. package/dist/mappers/request.js +211 -35
  70. package/dist/mappers/response.d.ts.map +1 -1
  71. package/dist/mappers/response.js +13 -1
  72. package/dist/model-work-coordinator.d.ts +95 -0
  73. package/dist/model-work-coordinator.d.ts.map +1 -0
  74. package/dist/model-work-coordinator.js +201 -0
  75. package/dist/pending-writes.d.ts.map +1 -1
  76. package/dist/presets.d.ts +82 -0
  77. package/dist/presets.d.ts.map +1 -0
  78. package/dist/presets.js +98 -0
  79. package/dist/registry.d.ts +31 -1
  80. package/dist/registry.d.ts.map +1 -1
  81. package/dist/registry.js +33 -5
  82. package/dist/router.d.ts +37 -1
  83. package/dist/router.d.ts.map +1 -1
  84. package/dist/router.js +79 -7
  85. package/dist/server.d.ts +193 -3
  86. package/dist/server.d.ts.map +1 -1
  87. package/dist/server.js +173 -10
  88. package/dist/session-registry.d.ts +279 -18
  89. package/dist/session-registry.d.ts.map +1 -1
  90. package/dist/session-registry.js +518 -37
  91. package/dist/stop-sequence-buffer.d.ts +58 -0
  92. package/dist/stop-sequence-buffer.d.ts.map +1 -0
  93. package/dist/stop-sequence-buffer.js +148 -0
  94. package/dist/streaming.d.ts +14 -0
  95. package/dist/streaming.d.ts.map +1 -1
  96. package/dist/streaming.js +45 -0
  97. package/dist/text-recovery.d.ts +35 -0
  98. package/dist/text-recovery.d.ts.map +1 -0
  99. package/dist/text-recovery.js +41 -0
  100. package/dist/timing.d.ts +80 -0
  101. package/dist/timing.d.ts.map +1 -0
  102. package/dist/timing.js +121 -0
  103. package/dist/tool-call-buffer.d.ts +5 -5
  104. package/dist/tool-call-buffer.d.ts.map +1 -1
  105. package/dist/tool-call-buffer.js +28 -8
  106. package/dist/types-anthropic.d.ts +161 -1
  107. package/dist/types-anthropic.d.ts.map +1 -1
  108. package/dist/types.d.ts +172 -2
  109. package/dist/types.d.ts.map +1 -1
  110. package/package.json +15 -3
@@ -2,6 +2,18 @@
2
2
  * SessionRegistry -- per-model cache holding AT MOST one live
3
3
  * `ChatSession` whose native KV state is currently valid.
4
4
  *
5
+ * **Tier-2 `prompt_cache_key` reuse is ON by default** so the server
6
+ * is compatible with any stateless LLM agent that sends the full
7
+ * conversation history each turn. The key is caller-controlled and
8
+ * HMAC-scoped with a boot-time nonce (raw value never stored on the
9
+ * entry), but two clients that pick the same raw key will still
10
+ * lease the same warm session — a session-hijack surface in
11
+ * multi-tenant settings. For multi-tenant deployments, opt out via
12
+ * `MLX_DISABLE_PROMPT_CACHE_KEY=1` or front the server with an auth
13
+ * proxy that rewrites or namespaces `prompt_cache_key` per tenant
14
+ * before it reaches this process. See also the comments on
15
+ * {@link scopePromptCacheKey}.
16
+ *
5
17
  * Design notes:
6
18
  *
7
19
  * - **One registry per model.** Composed alongside each registered
@@ -50,8 +62,8 @@
50
62
  * atomically appends the new user turn, so cold replay is
51
63
  * indistinguishable from a hot hit.
52
64
  *
53
- * - **TTL.** Default 1800 seconds mirrors `RESPONSE_TTL_SECONDS`
54
- * in `packages/server/src/endpoints/responses.ts` so the cached
65
+ * - **TTL.** Default 1800 seconds mirrors `DEFAULT_RESPONSE_RETENTION_SECONDS`
66
+ * in `packages/server/src/server.ts` so the cached
55
67
  * entry ages out alongside its stored response metadata. With
56
68
  * at most one entry there is no LRU bookkeeping — just a single
57
69
  * expiry check on lookup.
@@ -79,7 +91,53 @@
79
91
  * time. A weaker epoch-token scheme would let the losing
80
92
  * `adopt()` no-op but the native KV would already be wrong.
81
93
  */
94
+ import type { ChatConfig } from '@mlx-node/core';
82
95
  import { ChatSession, type SessionCapableModel } from '@mlx-node/lm';
96
+ /**
97
+ * Test-only hook used by the scoping unit tests to simulate a
98
+ * server restart: resets the module-scoped HMAC nonce (so every
99
+ * previously stored tier-2 key misses) and clears the silent-miss
100
+ * dedupe cache so tests can re-exercise the once-per-key diagnostic
101
+ * path.
102
+ *
103
+ * **Not exported from the package's public `index.ts` surface** —
104
+ * exporting it there would let downstream consumers nuke tier-2
105
+ * state in production (every stored entry would go unreachable).
106
+ * Tests reach the function via the deep path
107
+ * `packages/server/src/session-registry.ts` instead; that import is
108
+ * deliberately noisy to signal "test-only, do not use from app
109
+ * code". The `__` prefix is a loud-enough convention for the
110
+ * ergonomic test path but NOT sufficient for a public package
111
+ * export.
112
+ */
113
+ export declare function __resetPromptCacheKeyNonceForTests(): void;
114
+ /**
115
+ * Test-only probe for the silent-miss dedupe Map's live size. Same
116
+ * "do not use from app code" rationale as
117
+ * {@link __resetPromptCacheKeyNonceForTests}; the `__` prefix and the
118
+ * test-only deep-import path are the load-bearing signals. Exists so
119
+ * the flooding regression test can assert the FIFO cap actually bounds
120
+ * the stored set — an invariant that is otherwise unobservable from
121
+ * outside the module and is NOT implied by the warning count (each
122
+ * call emits at most one warning regardless of whether the cap works).
123
+ */
124
+ export declare function __loggedSilentMissKeysSizeForTests(): number;
125
+ /**
126
+ * Emit a once-per-raw-key stderr debug warning when the caller
127
+ * supplies a non-empty `prompt_cache_key` but at least one tier-2
128
+ * prerequisite is missing — the env gate is off, or the key is
129
+ * shorter than {@link PROMPT_CACHE_KEY_MIN_LENGTH}. The silent-miss
130
+ * fallback (cold-start as if no key were supplied) is the documented
131
+ * behaviour but is easy to miss during integration; this nudge
132
+ * surfaces the cause once per distinct key so operators don't have
133
+ * to grep source to diagnose a flat `X-Session-Cache: fresh`.
134
+ *
135
+ * No-op when `rawKey` is null / undefined / empty (the caller did
136
+ * not ask for tier-2 at all) or when scoping would succeed (the
137
+ * gate already accepted the key). Called by the endpoint layer
138
+ * right after it has decided `effectivePromptCacheKey`.
139
+ */
140
+ export declare function maybeWarnPromptCacheKeyIneligible(rawKey: string | null | undefined): void;
83
141
  /** Constructor options for {@link SessionRegistry}. */
84
142
  export interface SessionRegistryOptions {
85
143
  /** The model that every session in this registry wraps. Single-model per registry. */
@@ -99,6 +157,25 @@ export interface SessionRegistryOptions {
99
157
  * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var.
100
158
  */
101
159
  maxQueueDepth?: number;
160
+ /**
161
+ * Optional sampling defaults applied to every `ChatSession` this
162
+ * registry allocates. Forwarded verbatim into `new ChatSession(model,
163
+ * { defaultConfig })` so the session's `mergeConfig(overlay)` shallow-
164
+ * merges per-call config on top. Intended for server operators who
165
+ * want to pin per-model sampling knobs (temperature, topK, penalties,
166
+ * etc.) without client cooperation — per-request values from the
167
+ * OpenAI `/v1/responses` or Anthropic `/v1/messages` body still win
168
+ * where present because `ChatSession` treats them as an overlay.
169
+ *
170
+ * When `undefined`, behaviour is unchanged from the pre-defaults era
171
+ * (each `new ChatSession(model)` uses an empty `defaultConfig`).
172
+ */
173
+ samplingDefaults?: ChatConfig;
174
+ /**
175
+ * Optional per-model cap for generated output tokens. Endpoint handlers
176
+ * apply it after request mapping, before dispatching into native decode.
177
+ */
178
+ maxOutputTokens?: number;
102
179
  }
103
180
  /**
104
181
  * Thrown synchronously by {@link SessionRegistry.withExclusive} when
@@ -126,6 +203,14 @@ export declare class SessionRegistry {
126
203
  private readonly model;
127
204
  private readonly ttlSec;
128
205
  private readonly maxQueueDepth;
206
+ /**
207
+ * Per-model sampling defaults forwarded into every new `ChatSession`
208
+ * via its `defaultConfig` constructor option. `undefined` preserves
209
+ * the pre-defaults behaviour (empty `defaultConfig`). See
210
+ * {@link SessionRegistryOptions.samplingDefaults}.
211
+ */
212
+ private samplingDefaults;
213
+ private maxOutputTokens;
129
214
  /**
130
215
  * Number of callers that are currently WAITING for the per-model
131
216
  * execution mutex — i.e. have entered `withExclusive` but have not
@@ -181,12 +266,45 @@ export declare class SessionRegistry {
181
266
  */
182
267
  private execLock;
183
268
  constructor(opts: SessionRegistryOptions);
269
+ /**
270
+ * Construct a fresh `ChatSession` bound to this registry's model and
271
+ * pre-seeded with the operator-configured `samplingDefaults` (if any).
272
+ * Centralized so every cache-miss branch of `getOrCreate` produces a
273
+ * session whose per-call overlay will merge on top of the same
274
+ * defaults — clients cannot accidentally stray from the server's
275
+ * pinned sampling knobs by picking a cold-replay path.
276
+ */
277
+ private newSession;
184
278
  /**
185
279
  * Number of requests currently WAITING to acquire the per-model
186
280
  * execution mutex. Does NOT include the one actively running inside
187
281
  * `fn`. Primarily for tests and diagnostics.
188
282
  */
189
283
  get queueDepth(): number;
284
+ /**
285
+ * Configured waiter cap for this model's execution mutex, or `undefined`
286
+ * when unbounded. Paired with {@link queueDepth} so a readiness probe can
287
+ * tell "3 waiters, unbounded" (fine) from "3 waiters, cap of 3" (the next
288
+ * request gets a 429) without reaching into private state.
289
+ */
290
+ get queueDepthLimit(): number | undefined;
291
+ /**
292
+ * Current sampling defaults applied to every new `ChatSession` this
293
+ * registry allocates. Exposed primarily for tests and diagnostics.
294
+ */
295
+ get defaultSamplingConfig(): ChatConfig | undefined;
296
+ get outputTokenLimit(): number | undefined;
297
+ /**
298
+ * Replace the sampling defaults forwarded into every future
299
+ * `ChatSession` this registry allocates. Called by `ModelRegistry`
300
+ * on a `register(name, model, { samplingDefaults })` refresh so a
301
+ * fresh registration's defaults immediately apply to the next
302
+ * cache-miss cold-start. Sessions already cached at call time keep
303
+ * the defaults they were constructed with — they settle naturally
304
+ * through the single-warm cache rotation.
305
+ */
306
+ setSamplingDefaults(defaults: ChatConfig | undefined): void;
307
+ setMaxOutputTokens(limit: number | undefined): void;
190
308
  /** Number of sessions currently cached. Primarily for tests and diagnostics. Always 0 or 1. */
191
309
  get size(): number;
192
310
  /**
@@ -194,30 +312,164 @@ export declare class SessionRegistry {
194
312
  * Always returns a `SessionLookupResult` and always leaves the cache
195
313
  * empty after return (single-warm invariant).
196
314
  *
197
- * On a null id, missing key, expired entry, or prefix-state
198
- * mismatch: clear and return `{ session: new ChatSession(model), hit: false }`.
199
- * The caller primes / cold-replays from the `ResponseStore` and
200
- * re-adopts after the turn commits.
315
+ * Lookup proceeds in two tiers:
316
+ *
317
+ * 1. **Tier 1 `previousResponseId`.** The existing hot path:
318
+ * exact id match on a live, non-expired entry whose stored
319
+ * `instructions` are byte-equal to `requestedInstructions`. On
320
+ * a match the entry is leased out (single-use: removed from the
321
+ * map so a concurrent second request cannot share the live
322
+ * `ChatSession`). On a miss — unknown id, expired, or
323
+ * instructions drift — the method falls through to a FRESH
324
+ * session regardless of whether tier 2 would have hit.
201
325
  *
202
- * On a hit: the entry is removed and its live session is returned
203
- * alongside `hit: true`. Overlapping requests against the same
204
- * `previous_response_id` cannot share the same live `ChatSession`
205
- * the first wins, the second misses and cold-replays.
326
+ * `previousResponseId` wins unconditionally when supplied. The
327
+ * two keys could legitimately identify different conversation
328
+ * branches (e.g. a client fork where one arm chose the prev-id
329
+ * path and the other arm chose to set `prompt_cache_key`
330
+ * without one), so routing the prev-id branch through tier 2
331
+ * on miss risks splicing the wrong warm state into the wrong
332
+ * chain. Cold-replay is the safe default.
206
333
  *
207
- * `requestedInstructions` is the caller's prefix/system state
208
- * (OpenAI `instructions`, Anthropic `system`, or `null`); byte-for-
209
- * byte mismatch against the cached entry forces cold replay so
210
- * the new prefix is re-primed.
334
+ * 2. **Tier 2 `promptCacheKey`.** Only runs when
335
+ * `previousResponseId` is `null`. Stateless agent clients
336
+ * (pi-mono, Aider, Codex CLI, Continue, etc.) never use
337
+ * `previous_response_id` they own the conversation history
338
+ * client-side and resend the full transcript on every turn —
339
+ * so the only way to reuse a warm session across those turns
340
+ * is to key on the client-supplied `prompt_cache_key`. Scans
341
+ * for any live, non-expired entry whose stored
342
+ * `promptCacheKey` is non-null AND byte-equal to the caller's
343
+ * `promptCacheKey` AND whose stored `instructions` are byte-
344
+ * equal. Empty string is treated as a distinct key from
345
+ * `null` — an opt-out sentinel from a client that forgot to
346
+ * thread the key must NOT collide with another client that
347
+ * did set it to empty. On a match the entry is leased out
348
+ * (same single-use semantics as tier 1). On a miss, fall
349
+ * through to a fresh session.
211
350
  *
212
351
  * The `hit` flag drives the `X-Session-Cache` observability header
213
352
  * emitted by both `/v1/responses` and `/v1/messages`: when the caller
214
353
  * supplied a `previous_response_id`, `hit === true` yields `hit` and
215
354
  * `hit === false` yields `cold_replay` (the endpoint then rebuilds
216
355
  * from the `ResponseStore` on a fresh session). Requests with no
217
- * `previous_response_id` (or the stateless `/v1/messages` endpoint,
218
- * which always passes `null`) yield `fresh` regardless of this flag.
356
+ * `previous_response_id` yield either `fresh` (tier-2 miss) or
357
+ * `prefix_hit` (tier-2 hit only classified as such once the
358
+ * native `cachedTokens > 0` confirms the prefix-cache machinery
359
+ * actually reused the cached tokens).
360
+ */
361
+ getOrCreate(previousResponseId: string | null, requestedInstructions: string | null, promptCacheKey?: string | null): SessionLookupResult;
362
+ /**
363
+ * Allocate a fresh `ChatSession` bound to this registry's model
364
+ * without touching the warm slot. Intended for the `/v1/messages`
365
+ * endpoint when the underlying model has a block-paged KV cache
366
+ * active: the native cache already reuses SYS blocks across requests
367
+ * via content-addressing in `BlockAllocator`'s prefix-hash table, so
368
+ * the JS-side warm slot in
369
+ * {@link SessionRegistry.getOrCreateWarmAny} is redundant.
370
+ *
371
+ * Crucially, this call is purely additive — it does **NOT** clear,
372
+ * read, or evict the warm slot. Two parallel `/v1/messages` requests
373
+ * sharing a system prompt both call `createFreshSession` and both
374
+ * get distinct sessions; the native cache transparently refcounts
375
+ * the shared SYS blocks across them. This is the routing decision
376
+ * the long block comment in `packages/server/src/endpoints/messages.ts`
377
+ * documents: paged → fresh session, non-paged → warm-any lookup.
378
+ *
379
+ * The returned session is pre-seeded with the operator-configured
380
+ * `samplingDefaults` (matching every other cache-miss branch) so a
381
+ * client that picks the paged path does not silently stray from the
382
+ * server's pinned sampling knobs.
383
+ *
384
+ * Returned with `hit: false` to keep the result shape uniform with
385
+ * {@link SessionRegistry.getOrCreate} and
386
+ * {@link SessionRegistry.getOrCreateWarmAny}; callers that care
387
+ * about the cache header semantics should observe
388
+ * `result.cachedTokens` from the dispatch instead — that's the
389
+ * authoritative signal for whether the native engine recovered any
390
+ * prefix on this turn (paged or otherwise).
219
391
  */
220
- getOrCreate(previousResponseId: string | null, requestedInstructions: string | null): SessionLookupResult;
392
+ createFreshSession(): SessionLookupResult;
393
+ /**
394
+ * @deprecated **Redundant on `/v1/messages` for paged-active models.**
395
+ * There is no call site for paged-active full-attention models (Qwen3 +
396
+ * LFM2 + Gemma4 today): the native block-paged KV adapter (`PagedKVCacheAdapter` +
397
+ * `BlockAllocator` + `LayerKVPool`) recovers a turn's prefix from
398
+ * refcounted KV blocks keyed by token-prefix hash, so the JS-side
399
+ * single-warm slot this method walks is redundant — the native
400
+ * cache picks up the same cross-turn reuse without the
401
+ * byte-equal-`instructions` gate, and additionally supports
402
+ * cross-conversation prefix sharing the warm slot cannot.
403
+ *
404
+ * The `/v1/messages` endpoint now branches at request time on
405
+ * {@link SessionCapableModel.hasBlockPagedCache}: paged-active models
406
+ * call {@link SessionRegistry.createFreshSession} per request and
407
+ * never touch the warm slot; non-paged models (Qwen3.5 dense + MoE —
408
+ * default-OFF pending a perf decision; the `QianfanOCRModel` VLM —
409
+ * no adapter wired) still call this method because the JS-side warm
410
+ * slot is the ONLY cross-conversation reuse mechanism available to
411
+ * them. Removing this method would silently disable cross-turn reuse
412
+ * on every non-paged model, so it stays load-bearing until ALL
413
+ * session-capable models have paged enabled by default. Treat
414
+ * `@deprecated` as an intent signal that paged-active callers should
415
+ * use `createFreshSession` instead.
416
+ *
417
+ * Third lookup mode — for STATELESS full-history endpoints that have
418
+ * no `previous_response_id` to thread and do not propagate
419
+ * `prompt_cache_key` back to the server. The Anthropic
420
+ * `/v1/messages` endpoint is the canonical caller: clients (e.g.
421
+ * Claude Code) POST the entire conversation each turn, so the only
422
+ * remaining signal that a turn N continues turn N-1's prefix is the
423
+ * registry's own warm slot.
424
+ *
425
+ * Behaviour: walk the registry's at-most-one warm entry. If it is
426
+ * non-expired AND its stored `instructions` are byte-equal to
427
+ * `requestedInstructions`, lease it out (single-use — `entries.clear()`
428
+ * before return, mirroring the tier-1 / tier-2 lease-on-hit
429
+ * semantics). Otherwise clear the map and return a fresh session.
430
+ *
431
+ * Crucially, this lookup IGNORES `entry.promptCacheKey` and ignores
432
+ * the entry's prior `previousResponseId` keying — any warm slot is
433
+ * fair game for `/v1/messages` reuse. The byte-equal `instructions`
434
+ * compare is the SOLE correctness gate: a system prompt change
435
+ * forces cold replay so the new prefix state is re-primed instead
436
+ * of silently reusing a stale warmed prompt.
437
+ *
438
+ * **Adoption sentinel.** `/v1/messages` adopts back under the literal
439
+ * sentinel id `'__msg_warm__'`. That sentinel will never appear as a
440
+ * `previous_response_id` on a `/v1/responses` request — the
441
+ * Anthropic Messages API does not produce a `previous_response_id`
442
+ * value clients could echo back, and the OpenAI side mints fresh
443
+ * `resp_*` ids — so cross-endpoint capture via tier-1 is impossible
444
+ * by construction. The two endpoints still SHARE the single warm
445
+ * slot under the registry's single-warm invariant: a
446
+ * `/v1/messages` turn that follows a `/v1/responses` turn can evict
447
+ * (and vice versa). That is the explicit trade-off of holding at
448
+ * most one warm entry per model.
449
+ *
450
+ * **Trust model.** Multi-tenant isolation on this endpoint requires
451
+ * fronting the server with an auth proxy that scopes warm-slot
452
+ * visibility per tenant — same trust boundary documented at the top
453
+ * of this file for the tier-2 `prompt_cache_key` path. The single-
454
+ * warm invariant plus `withExclusive`'s per-model serialization make
455
+ * the lookup safe under SINGLE-tenant assumptions: no two requests
456
+ * race the slot, and there is at most one slot to lease.
457
+ *
458
+ * **Caller contract on miss.** If `instructions` drifts between
459
+ * turns (system prompt changed) this returns `hit: false` and a
460
+ * fresh session — and the caller MUST then run a full
461
+ * `session.reset()` before priming history, NOT the JS-only
462
+ * `resetPreservingNativeCacheForWarmReuse` path. A fresh JS session
463
+ * does NOT imply a fresh native cache (the underlying
464
+ * `SessionCapableModel` is shared and its native
465
+ * `cached_token_history` persists across requests), so skipping the
466
+ * native wipe on a miss would let the next `chatSessionStart` reuse
467
+ * an unrelated previous request's prefix — the cross-request
468
+ * cache-affinity side channel that the long block comment in
469
+ * `responses.ts` (around the `runSessionNonStreaming` /
470
+ * `runSessionStreaming` branches) describes.
471
+ */
472
+ getOrCreateWarmAny(requestedInstructions: string | null): SessionLookupResult;
221
473
  /**
222
474
  * Insert a session under a newly allocated response id. Clears the
223
475
  * map before inserting to keep the single-warm invariant explicit
@@ -226,8 +478,17 @@ export declare class SessionRegistry {
226
478
  * `instructions` is the prefix/system state used for this turn;
227
479
  * stored on the entry and compared on the next `getOrCreate` to
228
480
  * detect prefix changes that must force a cold replay.
481
+ *
482
+ * `promptCacheKey` is the client-supplied conversation-chain key
483
+ * that enables the registry's tier-2 lookup for stateless agent
484
+ * turns that do not carry a `previous_response_id`. `null` /
485
+ * `undefined` means "no key supplied" — stored verbatim so a
486
+ * subsequent stateless lookup that also omits the key does NOT
487
+ * accidentally pick up this entry (only explicit non-null
488
+ * key-equality on both sides can hit tier 2). See
489
+ * {@link SessionRegistry.getOrCreate} for the precedence rules.
229
490
  */
230
- adopt(responseId: string, session: ChatSession<SessionCapableModel>, instructions: string | null): void;
491
+ adopt(responseId: string, session: ChatSession<SessionCapableModel>, instructions: string | null, promptCacheKey?: string | null | undefined): void;
231
492
  /**
232
493
  * Remove a session by response id. No-op if the key is not present.
233
494
  */
@@ -1 +1 @@
1
- {"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgFG;AAEH,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAErE,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;CAM/C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AAqBD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;gBAEvC,IAAI,EAAE,sBAAsB;IAMxC;;;;OAIG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,WAAW,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,EAAE,qBAAqB,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB;IAiCzG;;;;;;;;OAQG;IACH,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IASvG;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAI9B;;;;OAIG;IACH,KAAK,IAAI,IAAI;IASb,2DAA2D;IAC3D,KAAK,IAAI,IAAI;IAIb;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA8BlD;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}
1
+ {"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyErE;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kCAAkC,IAAI,IAAI,CAGzD;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAE3D;AAuDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CA2BzF;AAED,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAK7C;CACF;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AAsCD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,eAAe,CAAqB;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;IAEnD,YAAY,IAAI,EAAE,sBAAsB,EAMvC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IASlB;;;;OAIG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;;;OAKG;IACH,IAAI,eAAe,IAAI,MAAM,GAAG,SAAS,CAExC;IAED;;;OAGG;IACH,IAAI,qBAAqB,IAAI,UAAU,GAAG,SAAS,CAElD;IAED,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED;;;;;;;;OAQG;IACH,mBAAmB,CAAC,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE1D;IAED,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;IACH,WAAW,CACT,kBAAkB,EAAE,MAAM,GAAG,IAAI,EACjC,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,cAAc,GAAE,MAAM,GAAG,IAAW,GACnC,mBAAmB,CAsErB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,kBAAkB,IAAI,mBAAmB,CAExC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8EG;IACH,kBAAkB,CAAC,qBAAqB,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB,CAiB5E;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EACzC,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,cAAc,GAAE,MAAM,GAAG,IAAI,GAAG,SAAgB,GAC/C,IAAI,CAcN;IAED;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAE7B;IAED;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAOZ;IAED,2DAA2D;IAC3D,KAAK,IAAI,IAAI,CAEZ;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CA4BjD;IAED;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}