@mlx-node/server 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/endpoints/messages.d.ts +13 -0
- package/dist/endpoints/messages.d.ts.map +1 -0
- package/dist/endpoints/messages.js +511 -0
- package/dist/endpoints/models.d.ts +5 -0
- package/dist/endpoints/models.d.ts.map +1 -0
- package/dist/endpoints/models.js +10 -0
- package/dist/endpoints/responses.d.ts +79 -0
- package/dist/endpoints/responses.d.ts.map +1 -0
- package/dist/endpoints/responses.js +2816 -0
- package/dist/errors.d.ts +43 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +84 -0
- package/dist/handler.d.ts +18 -0
- package/dist/handler.d.ts.map +1 -0
- package/dist/handler.js +35 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/mappers/anthropic-request.d.ts +9 -0
- package/dist/mappers/anthropic-request.d.ts.map +1 -0
- package/dist/mappers/anthropic-request.js +241 -0
- package/dist/mappers/anthropic-response.d.ts +14 -0
- package/dist/mappers/anthropic-response.d.ts.map +1 -0
- package/dist/mappers/anthropic-response.js +112 -0
- package/dist/mappers/request.d.ts +18 -0
- package/dist/mappers/request.d.ts.map +1 -0
- package/dist/mappers/request.js +206 -0
- package/dist/mappers/response.d.ts +13 -0
- package/dist/mappers/response.d.ts.map +1 -0
- package/dist/mappers/response.js +116 -0
- package/dist/pending-writes.d.ts +337 -0
- package/dist/pending-writes.d.ts.map +1 -0
- package/dist/pending-writes.js +468 -0
- package/dist/registry.d.ts +363 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/registry.js +497 -0
- package/dist/router.d.ts +6 -0
- package/dist/router.d.ts.map +1 -0
- package/dist/router.js +78 -0
- package/dist/server.d.ts +80 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +158 -0
- package/dist/session-registry.d.ts +297 -0
- package/dist/session-registry.d.ts.map +1 -0
- package/dist/session-registry.js +403 -0
- package/dist/streaming.d.ts +7 -0
- package/dist/streaming.d.ts.map +1 -0
- package/dist/streaming.js +16 -0
- package/dist/tool-call-buffer.d.ts +26 -0
- package/dist/tool-call-buffer.d.ts.map +1 -0
- package/dist/tool-call-buffer.js +51 -0
- package/dist/transport-visibility.d.ts +56 -0
- package/dist/transport-visibility.d.ts.map +1 -0
- package/dist/transport-visibility.js +161 -0
- package/dist/types-anthropic.d.ts +144 -0
- package/dist/types-anthropic.d.ts.map +1 -0
- package/dist/types-anthropic.js +2 -0
- package/dist/types.d.ts +220 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/package.json +36 -0
package/dist/server.js
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/** Full HTTP server lifecycle: wires up the handler and periodically sweeps expired `ResponseStore` rows and sessions. */
|
|
2
|
+
import { mkdir } from 'node:fs/promises';
|
|
3
|
+
import { createServer as httpCreateServer } from 'node:http';
|
|
4
|
+
import { homedir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { ResponseStore } from '@mlx-node/core';
|
|
7
|
+
import { createHandler } from './handler.js';
|
|
8
|
+
import { ModelRegistry } from './registry.js';
|
|
9
|
+
/** Cleanup interval for expired responses (ms). */
|
|
10
|
+
const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
|
11
|
+
/**
|
|
12
|
+
* Default retention for persisted response rows in SQLite, in seconds.
|
|
13
|
+
*
|
|
14
|
+
* Decoupled from the in-memory `SessionRegistry` TTL (30 min) so a client
|
|
15
|
+
* sending `previous_response_id` after the warm KV cache has been evicted
|
|
16
|
+
* can still cold-replay from disk via `reconstructMessagesFromChain` +
|
|
17
|
+
* `ChatSession.startFromHistory`. 7 days trades disk for continuity at the
|
|
18
|
+
* cost of a one-time prefill on recovery.
|
|
19
|
+
*/
|
|
20
|
+
const DEFAULT_RESPONSE_RETENTION_SECONDS = 7 * 24 * 60 * 60; // 7 days
|
|
21
|
+
/**
|
|
22
|
+
* Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
|
|
23
|
+
*
|
|
24
|
+
* Non-integer positive values (e.g. `"1.5"`) are rejected rather than
|
|
25
|
+
* silently truncated — a typo like `"1.5"` meant as `"15"` would otherwise
|
|
26
|
+
* be accepted as 1 second, expiring persisted response rows almost
|
|
27
|
+
* immediately and breaking `previous_response_id` continuity. We prefer
|
|
28
|
+
* falling through to the caller's default over crashing on startup so a
|
|
29
|
+
* config-template typo in a Dockerfile / CI manifest does not take the
|
|
30
|
+
* service down.
|
|
31
|
+
*
|
|
32
|
+
* Exported for unit tests.
|
|
33
|
+
*/
|
|
34
|
+
export function parseEnvSeconds(name) {
|
|
35
|
+
const raw = process.env[name];
|
|
36
|
+
if (raw == null || raw === '')
|
|
37
|
+
return undefined;
|
|
38
|
+
const parsed = Number(raw);
|
|
39
|
+
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
40
|
+
return undefined;
|
|
41
|
+
if (!Number.isInteger(parsed))
|
|
42
|
+
return undefined;
|
|
43
|
+
return parsed;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Parse a positive integer count from env; shares the reject-unset-or-invalid
|
|
47
|
+
* semantics used by {@link parseEnvSeconds} (including the non-integer
|
|
48
|
+
* reject) so callers can fall back to their own default when the var is
|
|
49
|
+
* missing or malformed.
|
|
50
|
+
*
|
|
51
|
+
* Exported for unit tests.
|
|
52
|
+
*/
|
|
53
|
+
export function parseEnvPositiveInt(name) {
|
|
54
|
+
const raw = process.env[name];
|
|
55
|
+
if (raw == null || raw === '')
|
|
56
|
+
return undefined;
|
|
57
|
+
const parsed = Number(raw);
|
|
58
|
+
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
59
|
+
return undefined;
|
|
60
|
+
if (!Number.isInteger(parsed))
|
|
61
|
+
return undefined;
|
|
62
|
+
return parsed;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Validate a caller-supplied positive-integer config knob.
|
|
66
|
+
*
|
|
67
|
+
* Mirrors the reject-invalid semantics of {@link parseEnvPositiveInt} /
|
|
68
|
+
* {@link parseEnvSeconds} but fails fast with a descriptive error when
|
|
69
|
+
* the caller explicitly passes a bogus value. Silent coercion would hide
|
|
70
|
+
* a config bug that can take the model offline (e.g. a
|
|
71
|
+
* `maxQueueDepthPerModel: 0` makes `queuedCount >= limit` true for every
|
|
72
|
+
* request, immediately returning HTTP 429; a `responseRetentionSec: 0`
|
|
73
|
+
* stamps `expires_at = now` on every row and the next cleanup sweep
|
|
74
|
+
* deletes it). `undefined` falls through so the env/default path still
|
|
75
|
+
* applies.
|
|
76
|
+
*/
|
|
77
|
+
function normalizePositiveIntConfig(value, name) {
|
|
78
|
+
if (value === undefined)
|
|
79
|
+
return undefined;
|
|
80
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || !Number.isInteger(value) || value <= 0) {
|
|
81
|
+
throw new Error(`${name} must be a positive integer; received ${String(value)}`);
|
|
82
|
+
}
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Start an MLX-Node HTTP server exposing `POST /v1/responses`,
|
|
87
|
+
* `POST /v1/messages`, and `GET /v1/models`.
|
|
88
|
+
*
|
|
89
|
+
* @example
|
|
90
|
+
* ```typescript
|
|
91
|
+
* const { registry, close } = await createServer({ port: 8080 });
|
|
92
|
+
* registry.register('qwen3.5-3b', await Qwen35Model.load('./models/qwen3.5-3b'));
|
|
93
|
+
* ```
|
|
94
|
+
*/
|
|
95
|
+
export async function createServer(config) {
|
|
96
|
+
const port = config?.port ?? 8080;
|
|
97
|
+
const host = config?.host ?? '127.0.0.1';
|
|
98
|
+
const cors = config?.cors ?? true;
|
|
99
|
+
const disableStore = config?.disableStore ?? false;
|
|
100
|
+
// Validate caller-supplied numeric knobs BEFORE consulting env fallbacks
|
|
101
|
+
// so a bogus explicit value surfaces as a descriptive error instead of
|
|
102
|
+
// silently falling through to env / default. See
|
|
103
|
+
// `normalizePositiveIntConfig` for the failure modes we're guarding.
|
|
104
|
+
const configRetentionSec = normalizePositiveIntConfig(config?.responseRetentionSec, 'responseRetentionSec');
|
|
105
|
+
const responseRetentionSec = configRetentionSec ?? parseEnvSeconds('MLX_RESPONSE_RETENTION_SECONDS') ?? DEFAULT_RESPONSE_RETENTION_SECONDS;
|
|
106
|
+
// Opt-in queue-depth cap; resolved exactly once at server construction
|
|
107
|
+
// so the registry (and its per-model `SessionRegistry` instances
|
|
108
|
+
// allocated on `register()`) all share a single effective value.
|
|
109
|
+
const configMaxQueueDepth = normalizePositiveIntConfig(config?.maxQueueDepthPerModel, 'maxQueueDepthPerModel');
|
|
110
|
+
const maxQueueDepthPerModel = configMaxQueueDepth ?? parseEnvPositiveInt('MLX_MAX_QUEUE_DEPTH_PER_MODEL');
|
|
111
|
+
const registry = new ModelRegistry({ maxQueueDepth: maxQueueDepthPerModel });
|
|
112
|
+
let store = null;
|
|
113
|
+
if (!disableStore) {
|
|
114
|
+
const storePath = config?.storePath ?? join(homedir(), '.mlx-node', 'responses.db');
|
|
115
|
+
const storeDir = join(storePath, '..');
|
|
116
|
+
await mkdir(storeDir, { recursive: true });
|
|
117
|
+
store = await ResponseStore.open(storePath);
|
|
118
|
+
}
|
|
119
|
+
// Always schedule the sweep — sessions need TTL sweeps even without a store.
|
|
120
|
+
const cleanupTimer = setInterval(() => {
|
|
121
|
+
if (store) {
|
|
122
|
+
store.cleanupExpired().catch(() => { });
|
|
123
|
+
}
|
|
124
|
+
for (const sessReg of registry.listSessionRegistries()) {
|
|
125
|
+
sessReg.sweep();
|
|
126
|
+
}
|
|
127
|
+
}, CLEANUP_INTERVAL_MS);
|
|
128
|
+
cleanupTimer.unref();
|
|
129
|
+
const handler = createHandler(registry, { cors, store, responseRetentionSec });
|
|
130
|
+
const server = httpCreateServer(handler);
|
|
131
|
+
await new Promise((resolve, reject) => {
|
|
132
|
+
const onError = (err) => {
|
|
133
|
+
server.removeListener('error', onError);
|
|
134
|
+
reject(err);
|
|
135
|
+
};
|
|
136
|
+
server.on('error', onError);
|
|
137
|
+
server.listen(port, host, () => {
|
|
138
|
+
server.removeListener('error', onError);
|
|
139
|
+
resolve();
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
return {
|
|
143
|
+
server,
|
|
144
|
+
registry,
|
|
145
|
+
store,
|
|
146
|
+
async close() {
|
|
147
|
+
clearInterval(cleanupTimer);
|
|
148
|
+
await new Promise((resolve, reject) => {
|
|
149
|
+
server.close((err) => {
|
|
150
|
+
if (err)
|
|
151
|
+
reject(err);
|
|
152
|
+
else
|
|
153
|
+
resolve();
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
},
|
|
157
|
+
};
|
|
158
|
+
}
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionRegistry -- per-model cache holding AT MOST one live
|
|
3
|
+
* `ChatSession` whose native KV state is currently valid.
|
|
4
|
+
*
|
|
5
|
+
* Design notes:
|
|
6
|
+
*
|
|
7
|
+
* - **One registry per model.** Composed alongside each registered
|
|
8
|
+
* `ServableModel` in `ModelRegistry`. Sessions are keyed purely
|
|
9
|
+
* by response id — no secondary keying on model name because the
|
|
10
|
+
* registry is already scoped per model.
|
|
11
|
+
*
|
|
12
|
+
* - **Single-warm-session invariant.** `ChatSession<M>` is a thin
|
|
13
|
+
* JS wrapper — it does NOT own any native KV cache. The cache
|
|
14
|
+
* lives on the underlying `SessionCapableModel` (one shared
|
|
15
|
+
* `cached_token_history` / `caches` vector per model instance).
|
|
16
|
+
* Any call that runs a turn overwrites that shared native state,
|
|
17
|
+
* silently invalidating every other `ChatSession` wrapper
|
|
18
|
+
* pointing at the same model. Caching multiple wrappers per
|
|
19
|
+
* model is therefore an illusion: at most ONE matches real
|
|
20
|
+
* native state (whichever ran most recently). To prevent
|
|
21
|
+
* cross-session corruption this registry holds at most ONE
|
|
22
|
+
* entry — both `getOrCreate` and `adopt` clear the map before
|
|
23
|
+
* returning or inserting.
|
|
24
|
+
*
|
|
25
|
+
* - **Lease semantics on hit.** Clear-on-hit also gives single-
|
|
26
|
+
* flight lease semantics: two overlapping requests referencing
|
|
27
|
+
* the same `previous_response_id` cannot share the same live
|
|
28
|
+
* `ChatSession`. The first wins the cleared entry; the second
|
|
29
|
+
* finds the map empty and cold-replays from `ResponseStore` on
|
|
30
|
+
* a fresh session. Without this, the second would hit
|
|
31
|
+
* `ChatSession`'s single-flight "concurrent send() not allowed"
|
|
32
|
+
* guard.
|
|
33
|
+
*
|
|
34
|
+
* - **Instructions / prefix-state change also misses.** Each entry
|
|
35
|
+
* records the `instructions` string used to adopt it.
|
|
36
|
+
* `getOrCreate` compares the caller's `requestedInstructions`
|
|
37
|
+
* against the cached value; mismatch forces cold replay so the
|
|
38
|
+
* new prefix state is re-primed instead of silently reusing a
|
|
39
|
+
* stale warmed prompt. The OpenAI `instructions` field and the
|
|
40
|
+
* Anthropic `system` field both flow through the same parameter
|
|
41
|
+
* — the registry does not care which is which.
|
|
42
|
+
*
|
|
43
|
+
* - **Cache miss fallback.** On a miss (eviction, interleaved turn
|
|
44
|
+
* on a different chain, restart, lease-on-hit) the endpoint
|
|
45
|
+
* layer reconstructs the conversation from the `ResponseStore`
|
|
46
|
+
* history, primes a fresh `ChatSession` via `primeHistory()`,
|
|
47
|
+
* and resumes through `startFromHistory()` /
|
|
48
|
+
* `startFromHistoryStream()`. That pair dispatches one
|
|
49
|
+
* `chatSessionStart*` call that rebuilds the full KV cache and
|
|
50
|
+
* atomically appends the new user turn, so cold replay is
|
|
51
|
+
* indistinguishable from a hot hit.
|
|
52
|
+
*
|
|
53
|
+
* - **TTL.** Default 1800 seconds mirrors `RESPONSE_TTL_SECONDS`
|
|
54
|
+
* in `packages/server/src/endpoints/responses.ts` so the cached
|
|
55
|
+
* entry ages out alongside its stored response metadata. With
|
|
56
|
+
* at most one entry there is no LRU bookkeeping — just a single
|
|
57
|
+
* expiry check on lookup.
|
|
58
|
+
*
|
|
59
|
+
* - **Thread safety.** Node.js is single-threaded within one
|
|
60
|
+
* event-loop tick, so the internal `Map` is safe against
|
|
61
|
+
* concurrent mutation by design. `sweep()` can be scheduled
|
|
62
|
+
* via `setInterval` without colliding with in-flight calls.
|
|
63
|
+
*
|
|
64
|
+
* - **Per-model execution mutex.** A dispatch that spans multiple
|
|
65
|
+
* awaits (map -> prefill -> decode -> persist -> adopt) is NOT
|
|
66
|
+
* atomic from the registry's POV. Two requests against the
|
|
67
|
+
* same model would both receive a `ChatSession` pointing at
|
|
68
|
+
* the same native model; even though the lease-on-hit clear
|
|
69
|
+
* prevents sharing one `ChatSession` object, the native KV
|
|
70
|
+
* cache is a single mutable resource and two parallel
|
|
71
|
+
* `primeHistory()` / `send*()` calls would race. Whichever
|
|
72
|
+
* finished last would win `adopt()`, poisoning the hot path
|
|
73
|
+
* for every subsequent chained turn.
|
|
74
|
+
*
|
|
75
|
+
* `withExclusive(fn)` serializes every per-model dispatch via
|
|
76
|
+
* a FIFO `execLock` chain. `/v1/responses` and `/v1/messages`
|
|
77
|
+
* wrap the full `getOrCreate -> run -> adopt/drop` span in one
|
|
78
|
+
* `withExclusive` so at most one request holds the model at a
|
|
79
|
+
* time. A weaker epoch-token scheme would let the losing
|
|
80
|
+
* `adopt()` no-op but the native KV would already be wrong.
|
|
81
|
+
*/
|
|
82
|
+
import { ChatSession, type SessionCapableModel } from '@mlx-node/lm';
|
|
83
|
+
/** Constructor options for {@link SessionRegistry}. */
|
|
84
|
+
export interface SessionRegistryOptions {
|
|
85
|
+
/** The model that every session in this registry wraps. Single-model per registry. */
|
|
86
|
+
model: SessionCapableModel;
|
|
87
|
+
/** TTL in seconds before an unused session is evicted. Default: 1800 (30 min). */
|
|
88
|
+
ttlSec?: number;
|
|
89
|
+
/**
|
|
90
|
+
* Maximum number of requests that may be WAITING for the
|
|
91
|
+
* per-model execution mutex at the same time (the in-flight holder
|
|
92
|
+
* does NOT count toward this). When set and the cap is exceeded at
|
|
93
|
+
* `withExclusive` entry, the call throws {@link QueueFullError}
|
|
94
|
+
* synchronously so the endpoint layer can emit HTTP 429 and the
|
|
95
|
+
* client can retry later.
|
|
96
|
+
*
|
|
97
|
+
* Default: `undefined` (unbounded — current behaviour). Opt-in per
|
|
98
|
+
* {@link ServerConfig.maxQueueDepthPerModel} or the
|
|
99
|
+
* `MLX_MAX_QUEUE_DEPTH_PER_MODEL` env var.
|
|
100
|
+
*/
|
|
101
|
+
maxQueueDepth?: number;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Thrown synchronously by {@link SessionRegistry.withExclusive} when
|
|
105
|
+
* the per-model queue cap (`maxQueueDepth`) is exceeded. The error is
|
|
106
|
+
* raised BEFORE awaiting the previous lock holder so endpoint handlers
|
|
107
|
+
* can reliably catch it without racing the chain.
|
|
108
|
+
*/
|
|
109
|
+
export declare class QueueFullError extends Error {
|
|
110
|
+
readonly queuedCount: number;
|
|
111
|
+
readonly limit: number;
|
|
112
|
+
constructor(queuedCount: number, limit: number);
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Result of {@link SessionRegistry.getOrCreate}. `hit` reflects whether
|
|
116
|
+
* the call consumed a live warm entry (single-use lease) or returned a
|
|
117
|
+
* fresh `ChatSession` on a miss. The endpoint layer uses `hit` to
|
|
118
|
+
* classify the per-request session-cache status emitted to clients via
|
|
119
|
+
* the `X-Session-Cache` observability header.
|
|
120
|
+
*/
|
|
121
|
+
export interface SessionLookupResult {
|
|
122
|
+
session: ChatSession<SessionCapableModel>;
|
|
123
|
+
hit: boolean;
|
|
124
|
+
}
|
|
125
|
+
export declare class SessionRegistry {
|
|
126
|
+
private readonly model;
|
|
127
|
+
private readonly ttlSec;
|
|
128
|
+
private readonly maxQueueDepth;
|
|
129
|
+
/**
|
|
130
|
+
* Number of callers that are currently WAITING for the per-model
|
|
131
|
+
* execution mutex — i.e. have entered `withExclusive` but have not
|
|
132
|
+
* yet started running their closure. The caller that is actively
|
|
133
|
+
* running inside `fn()` is NOT counted here, so a cap of
|
|
134
|
+
* `maxQueueDepth = N` means "1 running + up to N waiting".
|
|
135
|
+
*
|
|
136
|
+
* Mutated strictly inside `withExclusive`: the admitting caller is
|
|
137
|
+
* counted as a waiter ONLY when the execution chain is already
|
|
138
|
+
* non-idle (i.e. some earlier caller still holds the mutex). The
|
|
139
|
+
* first caller into an idle chain is admitted directly as the
|
|
140
|
+
* runner slot and never contributes to `queuedCount`. Waiters
|
|
141
|
+
* decrement exactly once as they transition from waiting to
|
|
142
|
+
* running (after `await prev`). The counter is intentionally
|
|
143
|
+
* NEVER touched on cap-reject paths (the caller never queued) so
|
|
144
|
+
* the cap check is stable across concurrent entries, and runner-
|
|
145
|
+
* slot admissions leave it alone so a synchronous burst
|
|
146
|
+
* (e.g. `Promise.all([fn, fn])`) does not spuriously bill the
|
|
147
|
+
* runner-slot caller against the waiter cap.
|
|
148
|
+
*/
|
|
149
|
+
private queuedCount;
|
|
150
|
+
/**
|
|
151
|
+
* Holds AT MOST ONE entry under the single-warm invariant (see the
|
|
152
|
+
* module-level rustdoc). `getOrCreate` and `adopt` both clear the
|
|
153
|
+
* map as part of their contract so a later lookup cannot hand out
|
|
154
|
+
* a wrapper whose assumed native state has been overwritten by a
|
|
155
|
+
* turn on another cached entry.
|
|
156
|
+
*/
|
|
157
|
+
private readonly entries;
|
|
158
|
+
/**
|
|
159
|
+
* Shared sentinel representing "the execution chain is idle" — a
|
|
160
|
+
* pre-resolved promise. `execLock` starts at this value and is
|
|
161
|
+
* reset to it whenever the last holder releases without a
|
|
162
|
+
* successor chained behind it. `withExclusive` uses reference
|
|
163
|
+
* equality against this sentinel (`execLock === initialLock`) to
|
|
164
|
+
* tell "I am the runner slot on an idle chain" apart from "I am a
|
|
165
|
+
* waiter behind someone else", which is how the burst
|
|
166
|
+
* (`Promise.all([fn, fn])`) admission bug is avoided.
|
|
167
|
+
*/
|
|
168
|
+
private readonly initialLock;
|
|
169
|
+
/**
|
|
170
|
+
* Tail of the per-model execution FIFO. Every `withExclusive` call
|
|
171
|
+
* captures this value as its predecessor, then overwrites it with
|
|
172
|
+
* its own pending promise so the next waiter chains after it. The
|
|
173
|
+
* chain is resolved only when the current holder's `fn` has
|
|
174
|
+
* settled (success or failure), guaranteeing that at most one
|
|
175
|
+
* dispatch runs through this registry's native model at a time.
|
|
176
|
+
* Initialized to `initialLock` so the first caller proceeds
|
|
177
|
+
* without waiting AND is recognised as the runner slot (no waiter
|
|
178
|
+
* increment). When a holder releases as the current chain tail it
|
|
179
|
+
* restores `execLock` to `initialLock` so the next burst starts
|
|
180
|
+
* cleanly from the idle state.
|
|
181
|
+
*/
|
|
182
|
+
private execLock;
|
|
183
|
+
constructor(opts: SessionRegistryOptions);
|
|
184
|
+
/**
|
|
185
|
+
* Number of requests currently WAITING to acquire the per-model
|
|
186
|
+
* execution mutex. Does NOT include the one actively running inside
|
|
187
|
+
* `fn`. Primarily for tests and diagnostics.
|
|
188
|
+
*/
|
|
189
|
+
get queueDepth(): number;
|
|
190
|
+
/** Number of sessions currently cached. Primarily for tests and diagnostics. Always 0 or 1. */
|
|
191
|
+
get size(): number;
|
|
192
|
+
/**
|
|
193
|
+
* Look up or allocate a session for the given previous response id.
|
|
194
|
+
* Always returns a `SessionLookupResult` and always leaves the cache
|
|
195
|
+
* empty after return (single-warm invariant).
|
|
196
|
+
*
|
|
197
|
+
* On a null id, missing key, expired entry, or prefix-state
|
|
198
|
+
* mismatch: clear and return `{ session: new ChatSession(model), hit: false }`.
|
|
199
|
+
* The caller primes / cold-replays from the `ResponseStore` and
|
|
200
|
+
* re-adopts after the turn commits.
|
|
201
|
+
*
|
|
202
|
+
* On a hit: the entry is removed and its live session is returned
|
|
203
|
+
* alongside `hit: true`. Overlapping requests against the same
|
|
204
|
+
* `previous_response_id` cannot share the same live `ChatSession` —
|
|
205
|
+
* the first wins, the second misses and cold-replays.
|
|
206
|
+
*
|
|
207
|
+
* `requestedInstructions` is the caller's prefix/system state
|
|
208
|
+
* (OpenAI `instructions`, Anthropic `system`, or `null`); byte-for-
|
|
209
|
+
* byte mismatch against the cached entry forces cold replay so
|
|
210
|
+
* the new prefix is re-primed.
|
|
211
|
+
*
|
|
212
|
+
* The `hit` flag drives the `X-Session-Cache` observability header
|
|
213
|
+
* emitted by both `/v1/responses` and `/v1/messages`: when the caller
|
|
214
|
+
* supplied a `previous_response_id`, `hit === true` yields `hit` and
|
|
215
|
+
* `hit === false` yields `cold_replay` (the endpoint then rebuilds
|
|
216
|
+
* from the `ResponseStore` on a fresh session). Requests with no
|
|
217
|
+
* `previous_response_id` (or the stateless `/v1/messages` endpoint,
|
|
218
|
+
* which always passes `null`) yield `fresh` regardless of this flag.
|
|
219
|
+
*/
|
|
220
|
+
getOrCreate(previousResponseId: string | null, requestedInstructions: string | null): SessionLookupResult;
|
|
221
|
+
/**
|
|
222
|
+
* Insert a session under a newly allocated response id. Clears the
|
|
223
|
+
* map before inserting to keep the single-warm invariant explicit
|
|
224
|
+
* regardless of caller ordering.
|
|
225
|
+
*
|
|
226
|
+
* `instructions` is the prefix/system state used for this turn;
|
|
227
|
+
* stored on the entry and compared on the next `getOrCreate` to
|
|
228
|
+
* detect prefix changes that must force a cold replay.
|
|
229
|
+
*/
|
|
230
|
+
adopt(responseId: string, session: ChatSession<SessionCapableModel>, instructions: string | null): void;
|
|
231
|
+
/**
|
|
232
|
+
* Remove a session by response id. No-op if the key is not present.
|
|
233
|
+
*/
|
|
234
|
+
drop(responseId: string): void;
|
|
235
|
+
/**
|
|
236
|
+
* Walk the map and drop the entry if its TTL has expired.
|
|
237
|
+
* Intended for periodic cleanup via `setInterval`. Under the
|
|
238
|
+
* single-warm invariant the map holds at most one entry.
|
|
239
|
+
*/
|
|
240
|
+
sweep(): void;
|
|
241
|
+
/** Empty the registry. Useful at shutdown and in tests. */
|
|
242
|
+
clear(): void;
|
|
243
|
+
/**
|
|
244
|
+
* Serialize `fn` against every other dispatch through this
|
|
245
|
+
* registry's model. The caller must hold the lock across the
|
|
246
|
+
* entire per-model dispatch span — `getOrCreate` ->
|
|
247
|
+
* `primeHistory`/`send*` -> `adopt`/`drop`. Without it, two
|
|
248
|
+
* concurrent `primeHistory()` / `send*()` calls would race on
|
|
249
|
+
* the single mutable native KV cache and whichever finished last
|
|
250
|
+
* would corrupt the other's chain.
|
|
251
|
+
*
|
|
252
|
+
* FIFO chaining via a rolling `execLock` promise: each caller
|
|
253
|
+
* captures the current tail, publishes a fresh pending promise as
|
|
254
|
+
* the new tail, awaits the old tail, then runs `fn`. The
|
|
255
|
+
* `finally` releases regardless of whether `fn` threw.
|
|
256
|
+
*
|
|
257
|
+
* **Admission control.** When `maxQueueDepth` is configured and the
|
|
258
|
+
* current number of waiters (`queuedCount`, excluding the active
|
|
259
|
+
* holder) is already at or above the cap, the call throws
|
|
260
|
+
* {@link QueueFullError} synchronously — SYNCHRONOUSLY from the
|
|
261
|
+
* caller's perspective, not merely before `await prev`. The wrapper
|
|
262
|
+
* is deliberately NOT declared `async` so the admission gate
|
|
263
|
+
* throws on the caller's stack frame, letting endpoint handlers
|
|
264
|
+
* wrap the call site in a plain try/catch without racing promise
|
|
265
|
+
* microtasks. On acceptance the async body takes over via the
|
|
266
|
+
* returned `Promise<T>`.
|
|
267
|
+
*
|
|
268
|
+
* The cap is "waiters-only" — a cap of N permits one running
|
|
269
|
+
* dispatch plus N queued ones, rejecting the (N+1)th waiter. The
|
|
270
|
+
* default (undefined) preserves the original unbounded behaviour.
|
|
271
|
+
*
|
|
272
|
+
* **Runner-slot admission.** Whether a given caller counts as the
|
|
273
|
+
* runner slot or as a waiter is decided up front by comparing
|
|
274
|
+
* `execLock` against the idle sentinel `initialLock`. If they are
|
|
275
|
+
* identical, nobody is currently in-flight and this caller wins
|
|
276
|
+
* the runner slot: it is not counted against the waiter cap and
|
|
277
|
+
* never touches `queuedCount`. Otherwise it is a waiter and the
|
|
278
|
+
* normal cap check / increment / decrement cycle applies. This is
|
|
279
|
+
* what keeps a synchronous burst such as `Promise.all([fn, fn])`
|
|
280
|
+
* admissible under `maxQueueDepth = 1` — Call 1 is the runner,
|
|
281
|
+
* Call 2 is the one allowed waiter, Call 3 would throw.
|
|
282
|
+
*/
|
|
283
|
+
withExclusive<T>(fn: () => Promise<T>): Promise<T>;
|
|
284
|
+
/**
|
|
285
|
+
* Async tail of {@link withExclusive}. Kept separate so the public
|
|
286
|
+
* wrapper stays a plain (non-async) function whose admission-gate
|
|
287
|
+
* throw lands on the caller's stack synchronously. This helper
|
|
288
|
+
* owns the post-acceptance bookkeeping: awaiting the predecessor
|
|
289
|
+
* lock, transitioning from waiter to holder (`queuedCount`
|
|
290
|
+
* decrement for waiters only), running `fn`, releasing the FIFO
|
|
291
|
+
* tail, and resetting the chain to `initialLock` when this caller
|
|
292
|
+
* is still the tail (so a future burst admits its first entry as
|
|
293
|
+
* a runner-slot rather than as a waiter).
|
|
294
|
+
*/
|
|
295
|
+
private _runExclusive;
|
|
296
|
+
}
|
|
297
|
+
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgFG;AAEH,OAAO,EAAE,WAAW,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAErE,uDAAuD;AACvD,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,KAAK,EAAE,mBAAmB,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;CAM/C;AAED;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC;CACd;AAqBD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,WAAW,CAAK;IACxB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,QAAQ,CAAmC;gBAEvC,IAAI,EAAE,sBAAsB;IAMxC;;;;OAIG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,+FAA+F;IAC/F,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,WAAW,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,EAAE,qBAAqB,EAAE,MAAM,GAAG,IAAI,GAAG,mBAAmB;IAiCzG;;;;;;;;OAQG;IACH,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,mBAAmB,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IASvG;;OAEG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAI9B;;;;OAIG;IACH,KAAK,IAAI,IAAI;IASb,2DAA2D;IAC3D,KAAK,IAAI,IAAI;IAIb;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA8BlD;;;;;;;;;;OAUG;YACW,aAAa;CAiD5B"}
|