@mlx-node/server 0.0.0 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat-session-warm-reuse.d.ts +51 -0
- package/dist/chat-session-warm-reuse.d.ts.map +1 -0
- package/dist/chat-session-warm-reuse.js +68 -0
- package/dist/endpoints/messages-count-tokens.d.ts +8 -0
- package/dist/endpoints/messages-count-tokens.d.ts.map +1 -0
- package/dist/endpoints/messages-count-tokens.js +121 -0
- package/dist/endpoints/messages.d.ts +57 -5
- package/dist/endpoints/messages.d.ts.map +1 -1
- package/dist/endpoints/messages.js +1043 -147
- package/dist/endpoints/models.d.ts +2 -1
- package/dist/endpoints/models.d.ts.map +1 -1
- package/dist/endpoints/models.js +2 -2
- package/dist/endpoints/responses.d.ts +20 -7
- package/dist/endpoints/responses.d.ts.map +1 -1
- package/dist/endpoints/responses.js +572 -82
- package/dist/errors.d.ts +1 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +3 -0
- package/dist/handler.d.ts +42 -0
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +6 -1
- package/dist/idle-sweeper.d.ts +245 -0
- package/dist/idle-sweeper.d.ts.map +1 -0
- package/dist/idle-sweeper.js +408 -0
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -0
- package/dist/mappers/anthropic-request.d.ts +24 -2
- package/dist/mappers/anthropic-request.d.ts.map +1 -1
- package/dist/mappers/anthropic-request.js +222 -24
- package/dist/mappers/anthropic-response.d.ts +29 -4
- package/dist/mappers/anthropic-response.d.ts.map +1 -1
- package/dist/mappers/anthropic-response.js +143 -21
- package/dist/mappers/request.d.ts +48 -0
- package/dist/mappers/request.d.ts.map +1 -1
- package/dist/mappers/request.js +211 -35
- package/dist/mappers/response.d.ts.map +1 -1
- package/dist/mappers/response.js +13 -1
- package/dist/model-work-coordinator.d.ts +70 -0
- package/dist/model-work-coordinator.d.ts.map +1 -0
- package/dist/model-work-coordinator.js +120 -0
- package/dist/pending-writes.d.ts.map +1 -1
- package/dist/presets.d.ts +82 -0
- package/dist/presets.d.ts.map +1 -0
- package/dist/presets.js +98 -0
- package/dist/registry.d.ts +31 -1
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +33 -5
- package/dist/router.d.ts +4 -1
- package/dist/router.d.ts.map +1 -1
- package/dist/router.js +34 -4
- package/dist/server.d.ts +76 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +48 -1
- package/dist/session-registry.d.ts +272 -18
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +509 -37
- package/dist/stop-sequence-buffer.d.ts +58 -0
- package/dist/stop-sequence-buffer.d.ts.map +1 -0
- package/dist/stop-sequence-buffer.js +148 -0
- package/dist/text-recovery.d.ts +35 -0
- package/dist/text-recovery.d.ts.map +1 -0
- package/dist/text-recovery.js +41 -0
- package/dist/timing.d.ts +80 -0
- package/dist/timing.d.ts.map +1 -0
- package/dist/timing.js +121 -0
- package/dist/tool-call-buffer.d.ts +5 -5
- package/dist/tool-call-buffer.d.ts.map +1 -1
- package/dist/tool-call-buffer.js +28 -8
- package/dist/types-anthropic.d.ts +161 -1
- package/dist/types-anthropic.d.ts.map +1 -1
- package/dist/types.d.ts +172 -2
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -5
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
* SessionRegistry -- per-model cache holding AT MOST one live
|
|
3
3
|
* `ChatSession` whose native KV state is currently valid.
|
|
4
4
|
*
|
|
5
|
+
* **Tier-2 `prompt_cache_key` reuse is ON by default** so the server
|
|
6
|
+
* is compatible with any stateless LLM agent that sends the full
|
|
7
|
+
* conversation history each turn. The key is caller-controlled and
|
|
8
|
+
* HMAC-scoped with a boot-time nonce (raw value never stored on the
|
|
9
|
+
* entry), but two clients that pick the same raw key will still
|
|
10
|
+
* lease the same warm session — a session-hijack surface in
|
|
11
|
+
* multi-tenant settings. For multi-tenant deployments, opt out via
|
|
12
|
+
* `MLX_DISABLE_PROMPT_CACHE_KEY=1` or front the server with an auth
|
|
13
|
+
* proxy that rewrites or namespaces `prompt_cache_key` per tenant
|
|
14
|
+
* before it reaches this process. See also the comments on
|
|
15
|
+
* {@link scopePromptCacheKey}.
|
|
16
|
+
*
|
|
5
17
|
* Design notes:
|
|
6
18
|
*
|
|
7
19
|
* - **One registry per model.** Composed alongside each registered
|
|
@@ -50,8 +62,8 @@
|
|
|
50
62
|
* atomically appends the new user turn, so cold replay is
|
|
51
63
|
* indistinguishable from a hot hit.
|
|
52
64
|
*
|
|
53
|
-
* - **TTL.** Default 1800 seconds mirrors `
|
|
54
|
-
* in `packages/server/src/
|
|
65
|
+
* - **TTL.** Default 1800 seconds mirrors `DEFAULT_RESPONSE_RETENTION_SECONDS`
|
|
66
|
+
* in `packages/server/src/server.ts` so the cached
|
|
55
67
|
* entry ages out alongside its stored response metadata. With
|
|
56
68
|
* at most one entry there is no LRU bookkeeping — just a single
|
|
57
69
|
* expiry check on lookup.
|
|
@@ -79,7 +91,53 @@
|
|
|
79
91
|
* time. A weaker epoch-token scheme would let the losing
|
|
80
92
|
* `adopt()` no-op but the native KV would already be wrong.
|
|
81
93
|
*/
|
|
94
|
+
import type { ChatConfig } from '@mlx-node/core';
|
|
82
95
|
import { ChatSession, type SessionCapableModel } from '@mlx-node/lm';
|
|
96
|
+
/**
|
|
97
|
+
* Test-only hook used by the scoping unit tests to simulate a
|
|
98
|
+
* server restart: resets the module-scoped HMAC nonce (so every
|
|
99
|
+
* previously stored tier-2 key misses) and clears the silent-miss
|
|
100
|
+
* dedupe cache so tests can re-exercise the once-per-key diagnostic
|
|
101
|
+
* path.
|
|
102
|
+
*
|
|
103
|
+
* **Not exported from the package's public `index.ts` surface** —
|
|
104
|
+
* exporting it there would let downstream consumers nuke tier-2
|
|
105
|
+
* state in production (every stored entry would go unreachable).
|
|
106
|
+
* Tests reach the function via the deep path
|
|
107
|
+
* `packages/server/src/session-registry.ts` instead; that import is
|
|
108
|
+
* deliberately noisy to signal "test-only, do not use from app
|
|
109
|
+
* code". The `__` prefix is a loud-enough convention for the
|
|
110
|
+
* ergonomic test path but NOT sufficient for a public package
|
|
111
|
+
* export.
|
|
112
|
+
*/
|
|
113
|
+
export declare function __resetPromptCacheKeyNonceForTests(): void;
|
|
114
|
+
/**
|
|
115
|
+
* Test-only probe for the silent-miss dedupe Map's live size. Same
|
|
116
|
+
* "do not use from app code" rationale as
|
|
117
|
+
* {@link __resetPromptCacheKeyNonceForTests}; the `__` prefix and the
|
|
118
|
+
* test-only deep-import path are the load-bearing signals. Exists so
|
|
119
|
+
* the flooding regression test can assert the FIFO cap actually bounds
|
|
120
|
+
* the stored set — an invariant that is otherwise unobservable from
|
|
121
|
+
* outside the module and is NOT implied by the warning count (each
|
|
122
|
+
* call emits at most one warning regardless of whether the cap works).
|
|
123
|
+
*/
|
|
124
|
+
export declare function __loggedSilentMissKeysSizeForTests(): number;
|
|
125
|
+
/**
|
|
126
|
+
* Emit a once-per-raw-key stderr debug warning when the caller
|
|
127
|
+
* supplies a non-empty `prompt_cache_key` but at least one tier-2
|
|
128
|
+
* prerequisite is missing — the env gate is off, or the key is
|
|
129
|
+
* shorter than {@link PROMPT_CACHE_KEY_MIN_LENGTH}. The silent-miss
|
|
130
|
+
* fallback (cold-start as if no key were supplied) is the documented
|
|
131
|
+
* behaviour but is easy to miss during integration; this nudge
|
|
132
|
+
* surfaces the cause once per distinct key so operators don't have
|
|
133
|
+
* to grep source to diagnose a flat `X-Session-Cache: fresh`.
|
|
134
|
+
*
|
|
135
|
+
* No-op when `rawKey` is null / undefined / empty (the caller did
|
|
136
|
+
* not ask for tier-2 at all) or when scoping would succeed (the
|
|
137
|
+
* gate already accepted the key). Called by the endpoint layer
|
|
138
|
+
* right after it has decided `effectivePromptCacheKey`.
|
|
139
|
+
*/
|
|
140
|
+
export declare function maybeWarnPromptCacheKeyIneligible(rawKey: string | null | undefined): void;
|
|
83
141
|
/** Constructor options for {@link SessionRegistry}. */
|
|
84
142
|
export interface SessionRegistryOptions {
|
|
85
143
|
/** The model that every session in this registry wraps. Single-model per registry. */
|
|
@@ -99,6 +157,25 @@ export interface SessionRegistryOptions {
|
|
|
99
157
|
* `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var.
|
|
100
158
|
*/
|
|
101
159
|
maxQueueDepth?: number;
|
|
160
|
+
/**
|
|
161
|
+
* Optional sampling defaults applied to every `ChatSession` this
|
|
162
|
+
* registry allocates. Forwarded verbatim into `new ChatSession(model,
|
|
163
|
+
* { defaultConfig })` so the session's `mergeConfig(overlay)` shallow-
|
|
164
|
+
* merges per-call config on top. Intended for server operators who
|
|
165
|
+
* want to pin per-model sampling knobs (temperature, topK, penalties,
|
|
166
|
+
* etc.) without client cooperation — per-request values from the
|
|
167
|
+
* OpenAI `/v1/responses` or Anthropic `/v1/messages` body still win
|
|
168
|
+
* where present because `ChatSession` treats them as an overlay.
|
|
169
|
+
*
|
|
170
|
+
* When `undefined`, behaviour is unchanged from the pre-defaults era
|
|
171
|
+
* (each `new ChatSession(model)` uses an empty `defaultConfig`).
|
|
172
|
+
*/
|
|
173
|
+
samplingDefaults?: ChatConfig;
|
|
174
|
+
/**
|
|
175
|
+
* Optional per-model cap for generated output tokens. Endpoint handlers
|
|
176
|
+
* apply it after request mapping, before dispatching into native decode.
|
|
177
|
+
*/
|
|
178
|
+
maxOutputTokens?: number;
|
|
102
179
|
}
|
|
103
180
|
/**
|
|
104
181
|
* Thrown synchronously by {@link SessionRegistry.withExclusive} when
|
|
@@ -126,6 +203,14 @@ export declare class SessionRegistry {
|
|
|
126
203
|
private readonly model;
|
|
127
204
|
private readonly ttlSec;
|
|
128
205
|
private readonly maxQueueDepth;
|
|
206
|
+
/**
|
|
207
|
+
* Per-model sampling defaults forwarded into every new `ChatSession`
|
|
208
|
+
* via its `defaultConfig` constructor option. `undefined` preserves
|
|
209
|
+
* the pre-defaults behaviour (empty `defaultConfig`). See
|
|
210
|
+
* {@link SessionRegistryOptions.samplingDefaults}.
|
|
211
|
+
*/
|
|
212
|
+
private samplingDefaults;
|
|
213
|
+
private maxOutputTokens;
|
|
129
214
|
/**
|
|
130
215
|
* Number of callers that are currently WAITING for the per-model
|
|
131
216
|
* execution mutex — i.e. have entered `withExclusive` but have not
|
|
@@ -181,12 +266,38 @@ export declare class SessionRegistry {
|
|
|
181
266
|
*/
|
|
182
267
|
private execLock;
|
|
183
268
|
constructor(opts: SessionRegistryOptions);
|
|
269
|
+
/**
|
|
270
|
+
* Construct a fresh `ChatSession` bound to this registry's model and
|
|
271
|
+
* pre-seeded with the operator-configured `samplingDefaults` (if any).
|
|
272
|
+
* Centralized so every cache-miss branch of `getOrCreate` produces a
|
|
273
|
+
* session whose per-call overlay will merge on top of the same
|
|
274
|
+
* defaults — clients cannot accidentally stray from the server's
|
|
275
|
+
* pinned sampling knobs by picking a cold-replay path.
|
|
276
|
+
*/
|
|
277
|
+
private newSession;
|
|
184
278
|
/**
|
|
185
279
|
* Number of requests currently WAITING to acquire the per-model
|
|
186
280
|
* execution mutex. Does NOT include the one actively running inside
|
|
187
281
|
* `fn`. Primarily for tests and diagnostics.
|
|
188
282
|
*/
|
|
189
283
|
get queueDepth(): number;
|
|
284
|
+
/**
|
|
285
|
+
* Current sampling defaults applied to every new `ChatSession` this
|
|
286
|
+
* registry allocates. Exposed primarily for tests and diagnostics.
|
|
287
|
+
*/
|
|
288
|
+
get defaultSamplingConfig(): ChatConfig | undefined;
|
|
289
|
+
get outputTokenLimit(): number | undefined;
|
|
290
|
+
/**
|
|
291
|
+
* Replace the sampling defaults forwarded into every future
|
|
292
|
+
* `ChatSession` this registry allocates. Called by `ModelRegistry`
|
|
293
|
+
* on a `register(name, model, { samplingDefaults })` refresh so a
|
|
294
|
+
* fresh registration's defaults immediately apply to the next
|
|
295
|
+
* cache-miss cold-start. Sessions already cached at call time keep
|
|
296
|
+
* the defaults they were constructed with — they settle naturally
|
|
297
|
+
* through the single-warm cache rotation.
|
|
298
|
+
*/
|
|
299
|
+
setSamplingDefaults(defaults: ChatConfig | undefined): void;
|
|
300
|
+
setMaxOutputTokens(limit: number | undefined): void;
|
|
190
301
|
/** Number of sessions currently cached. Primarily for tests and diagnostics. Always 0 or 1. */
|
|
191
302
|
get size(): number;
|
|
192
303
|
/**
|
|
@@ -194,30 +305,164 @@ export declare class SessionRegistry {
|
|
|
194
305
|
* Always returns a `SessionLookupResult` and always leaves the cache
|
|
195
306
|
* empty after return (single-warm invariant).
|
|
196
307
|
*
|
|
197
|
-
*
|
|
198
|
-
* mismatch: clear and return `{ session: new ChatSession(model), hit: false }`.
|
|
199
|
-
* The caller primes / cold-replays from the `ResponseStore` and
|
|
200
|
-
* re-adopts after the turn commits.
|
|
308
|
+
* Lookup proceeds in two tiers:
|
|
201
309
|
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
* the
|
|
310
|
+
* 1. **Tier 1 — `previousResponseId`.** The existing hot path:
|
|
311
|
+
* exact id match on a live, non-expired entry whose stored
|
|
312
|
+
* `instructions` are byte-equal to `requestedInstructions`. On
|
|
313
|
+
* a match the entry is leased out (single-use: removed from the
|
|
314
|
+
* map so a concurrent second request cannot share the live
|
|
315
|
+
* `ChatSession`). On a miss — unknown id, expired, or
|
|
316
|
+
* instructions drift — the method falls through to a FRESH
|
|
317
|
+
* session regardless of whether tier 2 would have hit.
|
|
206
318
|
*
|
|
207
|
-
*
|
|
208
|
-
*
|
|
209
|
-
*
|
|
210
|
-
* the
|
|
319
|
+
* `previousResponseId` wins unconditionally when supplied. The
|
|
320
|
+
* two keys could legitimately identify different conversation
|
|
321
|
+
* branches (e.g. a client fork where one arm chose the prev-id
|
|
322
|
+
* path and the other arm chose to set `prompt_cache_key`
|
|
323
|
+
* without one), so routing the prev-id branch through tier 2
|
|
324
|
+
* on miss risks splicing the wrong warm state into the wrong
|
|
325
|
+
* chain. Cold-replay is the safe default.
|
|
326
|
+
*
|
|
327
|
+
* 2. **Tier 2 — `promptCacheKey`.** Only runs when
|
|
328
|
+
* `previousResponseId` is `null`. Stateless agent clients
|
|
329
|
+
* (pi-mono, Aider, Codex CLI, Continue, etc.) never use
|
|
330
|
+
* `previous_response_id` — they own the conversation history
|
|
331
|
+
* client-side and resend the full transcript on every turn —
|
|
332
|
+
* so the only way to reuse a warm session across those turns
|
|
333
|
+
* is to key on the client-supplied `prompt_cache_key`. Scans
|
|
334
|
+
* for any live, non-expired entry whose stored
|
|
335
|
+
* `promptCacheKey` is non-null AND byte-equal to the caller's
|
|
336
|
+
* `promptCacheKey` AND whose stored `instructions` are byte-
|
|
337
|
+
* equal. Empty string is treated as a distinct key from
|
|
338
|
+
* `null` — an opt-out sentinel from a client that forgot to
|
|
339
|
+
* thread the key must NOT collide with another client that
|
|
340
|
+
* did set it to empty. On a match the entry is leased out
|
|
341
|
+
* (same single-use semantics as tier 1). On a miss, fall
|
|
342
|
+
* through to a fresh session.
|
|
211
343
|
*
|
|
212
344
|
* The `hit` flag drives the `X-Session-Cache` observability header
|
|
213
345
|
* emitted by both `/v1/responses` and `/v1/messages`: when the caller
|
|
214
346
|
* supplied a `previous_response_id`, `hit === true` yields `hit` and
|
|
215
347
|
* `hit === false` yields `cold_replay` (the endpoint then rebuilds
|
|
216
348
|
* from the `ResponseStore` on a fresh session). Requests with no
|
|
217
|
-
* `previous_response_id`
|
|
218
|
-
*
|
|
349
|
+
* `previous_response_id` yield either `fresh` (tier-2 miss) or
|
|
350
|
+
* `prefix_hit` (tier-2 hit — only classified as such once the
|
|
351
|
+
* native `cachedTokens > 0` confirms the prefix-cache machinery
|
|
352
|
+
* actually reused the cached tokens).
|
|
353
|
+
*/
|
|
354
|
+
getOrCreate(previousResponseId: string | null, requestedInstructions: string | null, promptCacheKey?: string | null): SessionLookupResult;
|
|
355
|
+
/**
|
|
356
|
+
* Allocate a fresh `ChatSession` bound to this registry's model
|
|
357
|
+
* without touching the warm slot. Intended for the `/v1/messages`
|
|
358
|
+
* endpoint when the underlying model has a block-paged KV cache
|
|
359
|
+
* active: the native cache already reuses SYS blocks across requests
|
|
360
|
+
* via content-addressing in `BlockAllocator`'s prefix-hash table, so
|
|
361
|
+
* the JS-side warm slot in
|
|
362
|
+
* {@link SessionRegistry.getOrCreateWarmAny} is redundant.
|
|
363
|
+
*
|
|
364
|
+
* Crucially, this call is purely additive — it does **NOT** clear,
|
|
365
|
+
* read, or evict the warm slot. Two parallel `/v1/messages` requests
|
|
366
|
+
* sharing a system prompt both call `createFreshSession` and both
|
|
367
|
+
* get distinct sessions; the native cache transparently refcounts
|
|
368
|
+
* the shared SYS blocks across them. This is the routing decision
|
|
369
|
+
* the long block comment in `packages/server/src/endpoints/messages.ts`
|
|
370
|
+
* documents: paged → fresh session, non-paged → warm-any lookup.
|
|
371
|
+
*
|
|
372
|
+
* The returned session is pre-seeded with the operator-configured
|
|
373
|
+
* `samplingDefaults` (matching every other cache-miss branch) so a
|
|
374
|
+
* client that picks the paged path does not silently stray from the
|
|
375
|
+
* server's pinned sampling knobs.
|
|
376
|
+
*
|
|
377
|
+
* Returned with `hit: false` to keep the result shape uniform with
|
|
378
|
+
* {@link SessionRegistry.getOrCreate} and
|
|
379
|
+
* {@link SessionRegistry.getOrCreateWarmAny}; callers that care
|
|
380
|
+
* about the cache header semantics should observe
|
|
381
|
+
* `result.cachedTokens` from the dispatch instead — that's the
|
|
382
|
+
* authoritative signal for whether the native engine recovered any
|
|
383
|
+
* prefix on this turn (paged or otherwise).
|
|
219
384
|
*/
|
|
220
|
-
|
|
385
|
+
createFreshSession(): SessionLookupResult;
|
|
386
|
+
/**
|
|
387
|
+
* @deprecated **Redundant on `/v1/messages` for paged-active models.**
|
|
388
|
+
* There is no call site for paged-active full-attention models (Qwen3 +
|
|
389
|
+
* LFM2 + Gemma4 today): the native block-paged KV adapter (`PagedKVCacheAdapter` +
|
|
390
|
+
* `BlockAllocator` + `LayerKVPool`) recovers a turn's prefix from
|
|
391
|
+
* refcounted KV blocks keyed by token-prefix hash, so the JS-side
|
|
392
|
+
* single-warm slot this method walks is redundant — the native
|
|
393
|
+
* cache picks up the same cross-turn reuse without the
|
|
394
|
+
* byte-equal-`instructions` gate, and additionally supports
|
|
395
|
+
* cross-conversation prefix sharing the warm slot cannot.
|
|
396
|
+
*
|
|
397
|
+
* The `/v1/messages` endpoint now branches at request time on
|
|
398
|
+
* {@link SessionCapableModel.hasBlockPagedCache}: paged-active models
|
|
399
|
+
* call {@link SessionRegistry.createFreshSession} per request and
|
|
400
|
+
* never touch the warm slot; non-paged models (Qwen3.5 dense + MoE —
|
|
401
|
+
* default-OFF pending a perf decision; the `QianfanOCRModel` VLM —
|
|
402
|
+
* no adapter wired) still call this method because the JS-side warm
|
|
403
|
+
* slot is the ONLY cross-conversation reuse mechanism available to
|
|
404
|
+
* them. Removing this method would silently disable cross-turn reuse
|
|
405
|
+
* on every non-paged model, so it stays load-bearing until ALL
|
|
406
|
+
* session-capable models have paged enabled by default. Treat
|
|
407
|
+
* `@deprecated` as an intent signal that paged-active callers should
|
|
408
|
+
* use `createFreshSession` instead.
|
|
409
|
+
*
|
|
410
|
+
* Third lookup mode — for STATELESS full-history endpoints that have
|
|
411
|
+
* no `previous_response_id` to thread and do not propagate
|
|
412
|
+
* `prompt_cache_key` back to the server. The Anthropic
|
|
413
|
+
* `/v1/messages` endpoint is the canonical caller: clients (e.g.
|
|
414
|
+
* Claude Code) POST the entire conversation each turn, so the only
|
|
415
|
+
* remaining signal that a turn N continues turn N-1's prefix is the
|
|
416
|
+
* registry's own warm slot.
|
|
417
|
+
*
|
|
418
|
+
* Behaviour: walk the registry's at-most-one warm entry. If it is
|
|
419
|
+
* non-expired AND its stored `instructions` are byte-equal to
|
|
420
|
+
* `requestedInstructions`, lease it out (single-use — `entries.clear()`
|
|
421
|
+
* before return, mirroring the tier-1 / tier-2 lease-on-hit
|
|
422
|
+
* semantics). Otherwise clear the map and return a fresh session.
|
|
423
|
+
*
|
|
424
|
+
* Crucially, this lookup IGNORES `entry.promptCacheKey` and ignores
|
|
425
|
+
* the entry's prior `previousResponseId` keying — any warm slot is
|
|
426
|
+
* fair game for `/v1/messages` reuse. The byte-equal `instructions`
|
|
427
|
+
* compare is the SOLE correctness gate: a system prompt change
|
|
428
|
+
* forces cold replay so the new prefix state is re-primed instead
|
|
429
|
+
* of silently reusing a stale warmed prompt.
|
|
430
|
+
*
|
|
431
|
+
* **Adoption sentinel.** `/v1/messages` adopts back under the literal
|
|
432
|
+
* sentinel id `'__msg_warm__'`. That sentinel will never appear as a
|
|
433
|
+
* `previous_response_id` on a `/v1/responses` request — the
|
|
434
|
+
* Anthropic Messages API does not produce a `previous_response_id`
|
|
435
|
+
* value clients could echo back, and the OpenAI side mints fresh
|
|
436
|
+
* `resp_*` ids — so cross-endpoint capture via tier-1 is impossible
|
|
437
|
+
* by construction. The two endpoints still SHARE the single warm
|
|
438
|
+
* slot under the registry's single-warm invariant: a
|
|
439
|
+
* `/v1/messages` turn that follows a `/v1/responses` turn can evict
|
|
440
|
+
* (and vice versa). That is the explicit trade-off of holding at
|
|
441
|
+
* most one warm entry per model.
|
|
442
|
+
*
|
|
443
|
+
* **Trust model.** Multi-tenant isolation on this endpoint requires
|
|
444
|
+
* fronting the server with an auth proxy that scopes warm-slot
|
|
445
|
+
* visibility per tenant — same trust boundary documented at the top
|
|
446
|
+
* of this file for the tier-2 `prompt_cache_key` path. The single-
|
|
447
|
+
* warm invariant plus `withExclusive`'s per-model serialization make
|
|
448
|
+
* the lookup safe under SINGLE-tenant assumptions: no two requests
|
|
449
|
+
* race the slot, and there is at most one slot to lease.
|
|
450
|
+
*
|
|
451
|
+
* **Caller contract on miss.** If `instructions` drifts between
|
|
452
|
+
* turns (system prompt changed) this returns `hit: false` and a
|
|
453
|
+
* fresh session — and the caller MUST then run a full
|
|
454
|
+
* `session.reset()` before priming history, NOT the JS-only
|
|
455
|
+
* `resetPreservingNativeCacheForWarmReuse` path. A fresh JS session
|
|
456
|
+
* does NOT imply a fresh native cache (the underlying
|
|
457
|
+
* `SessionCapableModel` is shared and its native
|
|
458
|
+
* `cached_token_history` persists across requests), so skipping the
|
|
459
|
+
* native wipe on a miss would let the next `chatSessionStart` reuse
|
|
460
|
+
* an unrelated previous request's prefix — the cross-request
|
|
461
|
+
* cache-affinity side channel that the long block comment in
|
|
462
|
+
* `responses.ts` (around the `runSessionNonStreaming` /
|
|
463
|
+
* `runSessionStreaming` branches) describes.
|
|
464
|
+
*/
|
|
465
|
+
getOrCreateWarmAny(requestedInstructions: string | null): SessionLookupResult;
|
|
221
466
|
/**
|
|
222
467
|
* Insert a session under a newly allocated response id. Clears the
|
|
223
468
|
* map before inserting to keep the single-warm invariant explicit
|
|
@@ -226,8 +471,17 @@ export declare class SessionRegistry {
|
|
|
226
471
|
* `instructions` is the prefix/system state used for this turn;
|
|
227
472
|
* stored on the entry and compared on the next `getOrCreate` to
|
|
228
473
|
* detect prefix changes that must force a cold replay.
|
|
474
|
+
*
|
|
475
|
+
* `promptCacheKey` is the client-supplied conversation-chain key
|
|
476
|
+
* that enables the registry's tier-2 lookup for stateless agent
|
|
477
|
+
* turns that do not carry a `previous_response_id`. `null` /
|
|
478
|
+
* `undefined` means "no key supplied" — stored verbatim so a
|
|
479
|
+
* subsequent stateless lookup that also omits the key does NOT
|
|
480
|
+
* accidentally pick up this entry (only explicit non-null
|
|
481
|
+
* key-equality on both sides can hit tier 2). See
|
|
482
|
+
* {@link SessionRegistry.getOrCreate} for the precedence rules.
|
|
229
483
|
*/
|
|
230
|
-
adopt(responseId: string, session: ChatSession<SessionCapableModel>, instructions: string | null): void;
|
|
484
|
+
adopt(responseId: string, session: ChatSession<SessionCapableModel>, instructions: string | null, promptCacheKey?: string | null | undefined): void;
|
|
231
485
|
/**
|
|
232
486
|
* Remove a session by response id. No-op if the key is not present.
|
|
233
487
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAyErE;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kCAAkC,IAAI,IAAI,CAGzD;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,IAAI,MAAM,CAE3D;AAuDD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CA2BzF;AAED,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAK7C;CACF;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AAsCD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,eAAe,CAAqB;IAC5C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;IAEnD,YAAY,IAAI,EAAE,sBAAsB,EAMvC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IASlB;;;;OAIG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,qBAAqB,IAAI,UAAU,GAAG,SAAS,CAElD;IAED,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED;;;;;;;;OAQG;IACH,mBAAmB,CAAC,QAAQ,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI,CAE1D;IAED,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkDG;IACH,WAAW,CACT,kBAAkB,EAAE,MAAM,GAAG,IAAI,EACjC,qBAAqB,EAAE,MAAM,GAAG,IAAI,EACpC,cAAc,GAAE,MAAM,GAAG,IAAW,GACnC,mBAAmB,CAsErB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,kBAAkB,IAAI,mBAAmB,CAExC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8EG;IACH,kBAAkB,CAAC,qBAAqB,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB,CAiB5E;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EACzC,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,cAAc,GAAE,MAAM,GAAG,IAAI,GAAG,SAAgB,GAC/C,IAAI,CAcN;IAED;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAE7B;IAED;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAOZ;IAED,2DAA2D;IAC3D,KAAK,IAAI,IAAI,CAEZ;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CA4BjD;IAED;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}
|