@mlx-node/server 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.d.ts +56 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +106 -0
- package/dist/chat-session-warm-reuse.d.ts +8 -8
- package/dist/chat-session-warm-reuse.d.ts.map +1 -1
- package/dist/chat-session-warm-reuse.js +12 -8
- package/dist/endpoints/responses.d.ts +3 -1
- package/dist/endpoints/responses.d.ts.map +1 -1
- package/dist/endpoints/responses.js +38 -5
- package/dist/handler.d.ts +27 -1
- package/dist/handler.d.ts.map +1 -1
- package/dist/handler.js +65 -16
- package/dist/health.d.ts +146 -0
- package/dist/health.d.ts.map +1 -0
- package/dist/health.js +107 -0
- package/dist/host/discover.d.ts +19 -0
- package/dist/host/discover.d.ts.map +1 -0
- package/dist/host/discover.js +50 -0
- package/dist/host/env-policy.d.ts +62 -0
- package/dist/host/env-policy.d.ts.map +1 -0
- package/dist/host/env-policy.js +69 -0
- package/dist/host/index.d.ts +202 -0
- package/dist/host/index.d.ts.map +1 -0
- package/dist/host/index.js +325 -0
- package/dist/host/logger.d.ts +36 -0
- package/dist/host/logger.d.ts.map +1 -0
- package/dist/host/logger.js +376 -0
- package/dist/host/net.d.ts +65 -0
- package/dist/host/net.d.ts.map +1 -0
- package/dist/host/net.js +97 -0
- package/dist/host/paths.d.ts +28 -0
- package/dist/host/paths.d.ts.map +1 -0
- package/dist/host/paths.js +71 -0
- package/dist/host/swap.d.ts +27 -0
- package/dist/host/swap.d.ts.map +1 -0
- package/dist/host/swap.js +178 -0
- package/dist/host/temp-root.d.ts +57 -0
- package/dist/host/temp-root.d.ts.map +1 -0
- package/dist/host/temp-root.js +99 -0
- package/dist/index.d.ts +11 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/load-model.d.ts +69 -0
- package/dist/load-model.d.ts.map +1 -0
- package/dist/load-model.js +63 -0
- package/dist/model-work-coordinator.d.ts +29 -4
- package/dist/model-work-coordinator.d.ts.map +1 -1
- package/dist/model-work-coordinator.js +97 -16
- package/dist/router.d.ts +34 -1
- package/dist/router.d.ts.map +1 -1
- package/dist/router.js +47 -5
- package/dist/server.d.ts +117 -3
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +125 -9
- package/dist/session-registry.d.ts +7 -0
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +9 -0
- package/dist/streaming.d.ts +14 -0
- package/dist/streaming.d.ts.map +1 -1
- package/dist/streaming.js +45 -0
- package/package.json +15 -3
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Render a thrown value for {@link ModelLoadRecord.error}.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately avoids `String(unknown)`: a rejection carrying a plain object
|
|
5
|
+
* would render as the useless `[object Object]` in the one field a supervisor
|
|
6
|
+
* reads to find out why the model would not load.
|
|
7
|
+
*/
|
|
8
|
+
function describeLoadFailure(error) {
|
|
9
|
+
if (error instanceof Error)
|
|
10
|
+
return error.message;
|
|
11
|
+
if (typeof error === 'string')
|
|
12
|
+
return error;
|
|
13
|
+
if (error == null)
|
|
14
|
+
return 'unknown error';
|
|
15
|
+
try {
|
|
16
|
+
return JSON.stringify(error) ?? 'unknown error';
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
// Circular structure, or a `toJSON` that throws.
|
|
20
|
+
return 'unknown error';
|
|
21
|
+
}
|
|
22
|
+
}
|
|
1
23
|
/**
|
|
2
24
|
* Process-local gate for native MLX work.
|
|
3
25
|
*
|
|
@@ -9,16 +31,67 @@
|
|
|
9
31
|
*/
|
|
10
32
|
export class ModelWorkCoordinator {
|
|
11
33
|
activeReaders = 0;
|
|
12
|
-
|
|
13
|
-
|
|
34
|
+
writerHeld = false;
|
|
35
|
+
queuedWriters = 0;
|
|
14
36
|
readerWaiters = [];
|
|
15
37
|
writerWaiters = [];
|
|
16
|
-
|
|
38
|
+
/**
|
|
39
|
+
* Most recent settled load bracket. Retained here because the coordinator
|
|
40
|
+
* is the ONE place that brackets every load: a `resolveModel` failure in
|
|
41
|
+
* `/v1/messages` becomes a 500 and is otherwise dropped on the floor, so a
|
|
42
|
+
* supervisor polling `/health` afterwards had no way to learn what went
|
|
43
|
+
* wrong. See {@link ModelLoadRecord} for the "successful no-op overwrites
|
|
44
|
+
* an earlier failure" caveat.
|
|
45
|
+
*/
|
|
46
|
+
lastLoadRecord = null;
|
|
47
|
+
/** Read-only: `true` while a load holds the exclusive writer slot. */
|
|
48
|
+
get writerActive() {
|
|
49
|
+
return this.writerHeld;
|
|
50
|
+
}
|
|
51
|
+
/** Read-only: loads parked in `acquireWrite()` waiting for the slot. */
|
|
52
|
+
get waitingWriters() {
|
|
53
|
+
return this.queuedWriters;
|
|
54
|
+
}
|
|
55
|
+
/** Read-only: outcome of the most recent settled load bracket, or `null`. */
|
|
56
|
+
get lastLoad() {
|
|
57
|
+
return this.lastLoadRecord;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Record a settled bracket. Called from the `finally` of both load
|
|
61
|
+
* wrappers so a throw is captured just as reliably as a success.
|
|
62
|
+
*/
|
|
63
|
+
recordLoad(label, startedAt, error, ok) {
|
|
64
|
+
this.lastLoadRecord = {
|
|
65
|
+
label: label ?? null,
|
|
66
|
+
startedAt,
|
|
67
|
+
finishedAt: Date.now(),
|
|
68
|
+
ok,
|
|
69
|
+
error: ok ? null : describeLoadFailure(error),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* @param label Optional identifier (normally the model name) stamped into
|
|
74
|
+
* {@link lastLoad} so `/health` can name what was being loaded.
|
|
75
|
+
*/
|
|
76
|
+
async withModelLoad(fn, label) {
|
|
17
77
|
await this.acquireWrite();
|
|
78
|
+
// Measured from lock acquisition, not from arrival: `startedAt` is meant
|
|
79
|
+
// to answer "how long has the actual materialization been running",
|
|
80
|
+
// which is what a supervisor deciding whether to wait needs.
|
|
81
|
+
const startedAt = Date.now();
|
|
82
|
+
let ok = false;
|
|
83
|
+
let failure;
|
|
18
84
|
try {
|
|
19
|
-
|
|
85
|
+
const result = await fn();
|
|
86
|
+
ok = true;
|
|
87
|
+
return result;
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
failure = err;
|
|
91
|
+
throw err;
|
|
20
92
|
}
|
|
21
93
|
finally {
|
|
94
|
+
this.recordLoad(label, startedAt, failure, ok);
|
|
22
95
|
this.releaseWrite();
|
|
23
96
|
}
|
|
24
97
|
}
|
|
@@ -35,25 +108,33 @@ export class ModelWorkCoordinator {
|
|
|
35
108
|
* 60-second cold-load does not look like 60 seconds of own work for
|
|
36
109
|
* every concurrent request.
|
|
37
110
|
*/
|
|
38
|
-
async withModelLoadInstrumented(fn) {
|
|
111
|
+
async withModelLoadInstrumented(fn, label) {
|
|
39
112
|
// `owner` MUST be decided synchronously, before any await, so the
|
|
40
113
|
// signal reflects coordinator state at arrival rather than after
|
|
41
114
|
// any peer transition. The wait/own split is measured around the
|
|
42
115
|
// actual phase boundaries (lock acquisition, fn completion) so the
|
|
43
116
|
// two intervals partition cleanly instead of both reporting total
|
|
44
117
|
// elapsed time — see `ModelLoadOutcome` for the contract.
|
|
45
|
-
const owner = !this.
|
|
118
|
+
const owner = !this.writerHeld && this.queuedWriters === 0;
|
|
46
119
|
const arrivedAt = Date.now();
|
|
47
120
|
await this.acquireWrite();
|
|
48
121
|
const lockAcquiredAt = Date.now();
|
|
122
|
+
let ok = false;
|
|
123
|
+
let failure;
|
|
49
124
|
try {
|
|
50
125
|
const result = await fn();
|
|
126
|
+
ok = true;
|
|
51
127
|
const fnDoneAt = Date.now();
|
|
52
128
|
const waitMs = Math.max(0, lockAcquiredAt - arrivedAt);
|
|
53
129
|
const ownMs = Math.max(0, fnDoneAt - lockAcquiredAt);
|
|
54
130
|
return { result, owner, waitMs, ownMs };
|
|
55
131
|
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
failure = err;
|
|
134
|
+
throw err;
|
|
135
|
+
}
|
|
56
136
|
finally {
|
|
137
|
+
this.recordLoad(label, lockAcquiredAt, failure, ok);
|
|
57
138
|
this.releaseWrite();
|
|
58
139
|
}
|
|
59
140
|
}
|
|
@@ -67,7 +148,7 @@ export class ModelWorkCoordinator {
|
|
|
67
148
|
}
|
|
68
149
|
}
|
|
69
150
|
acquireRead() {
|
|
70
|
-
if (!this.
|
|
151
|
+
if (!this.writerHeld && this.queuedWriters === 0) {
|
|
71
152
|
this.activeReaders += 1;
|
|
72
153
|
return Promise.resolve();
|
|
73
154
|
}
|
|
@@ -79,16 +160,16 @@ export class ModelWorkCoordinator {
|
|
|
79
160
|
});
|
|
80
161
|
}
|
|
81
162
|
acquireWrite() {
|
|
82
|
-
this.
|
|
83
|
-
if (!this.
|
|
84
|
-
this.
|
|
85
|
-
this.
|
|
163
|
+
this.queuedWriters += 1;
|
|
164
|
+
if (!this.writerHeld && this.activeReaders === 0) {
|
|
165
|
+
this.queuedWriters -= 1;
|
|
166
|
+
this.writerHeld = true;
|
|
86
167
|
return Promise.resolve();
|
|
87
168
|
}
|
|
88
169
|
return new Promise((resolve) => {
|
|
89
170
|
this.writerWaiters.push(() => {
|
|
90
|
-
this.
|
|
91
|
-
this.
|
|
171
|
+
this.queuedWriters -= 1;
|
|
172
|
+
this.writerHeld = true;
|
|
92
173
|
resolve();
|
|
93
174
|
});
|
|
94
175
|
});
|
|
@@ -101,17 +182,17 @@ export class ModelWorkCoordinator {
|
|
|
101
182
|
this.drain();
|
|
102
183
|
}
|
|
103
184
|
releaseWrite() {
|
|
104
|
-
this.
|
|
185
|
+
this.writerHeld = false;
|
|
105
186
|
this.drain();
|
|
106
187
|
}
|
|
107
188
|
drain() {
|
|
108
|
-
if (this.
|
|
189
|
+
if (this.writerHeld)
|
|
109
190
|
return;
|
|
110
191
|
if (this.activeReaders === 0 && this.writerWaiters.length > 0) {
|
|
111
192
|
this.writerWaiters.shift()?.();
|
|
112
193
|
return;
|
|
113
194
|
}
|
|
114
|
-
if (this.
|
|
195
|
+
if (this.queuedWriters === 0 && this.readerWaiters.length > 0) {
|
|
115
196
|
const readers = this.readerWaiters.splice(0);
|
|
116
197
|
for (const resolve of readers)
|
|
117
198
|
resolve();
|
package/dist/router.d.ts
CHANGED
|
@@ -2,8 +2,41 @@
|
|
|
2
2
|
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
3
3
|
import type { ResponseStore } from '@mlx-node/core';
|
|
4
4
|
import type { PublicModelEntry } from './handler.js';
|
|
5
|
+
import { type ServerHealth } from './health.js';
|
|
5
6
|
import type { IdleSweeper } from './idle-sweeper.js';
|
|
6
7
|
import type { ModelWorkCoordinator } from './model-work-coordinator.js';
|
|
7
8
|
import type { ModelRegistry } from './registry.js';
|
|
8
|
-
|
|
9
|
+
/**
|
|
10
|
+
* The request's pathname, parsed against a CONSTANT base.
|
|
11
|
+
*
|
|
12
|
+
* Never against `Host`. That header is attacker-controlled text on every
|
|
13
|
+
* request — `Host: [` makes `new URL()` throw `ERR_INVALID_URL`, and a throw
|
|
14
|
+
* from an async request listener is an unhandled rejection, which under Node's
|
|
15
|
+
* default `--unhandled-rejections=throw` takes the whole process down. One
|
|
16
|
+
* malformed byte from any client that can reach the socket was enough to end
|
|
17
|
+
* inference. A pathname does not depend on the authority anyway, so a fixed
|
|
18
|
+
* base is both safer and equivalent: an absolute-form request URI
|
|
19
|
+
* (`GET http://host/v1/models HTTP/1.1`, legal in HTTP/1.1) still wins over
|
|
20
|
+
* the base and yields the same path it always did.
|
|
21
|
+
*
|
|
22
|
+
* `req.url` itself is guarded too, for the same reason rather than a known
|
|
23
|
+
* input: this function's contract is that no request can make it throw.
|
|
24
|
+
*/
|
|
25
|
+
export declare function requestPathname(req: IncomingMessage): string;
|
|
26
|
+
/**
|
|
27
|
+
* Trailing options bag. Added as an object rather than two more positional
|
|
28
|
+
* parameters — `routeRequest` already carries nine, and the two knobs here
|
|
29
|
+
* are unrelated to each other.
|
|
30
|
+
*/
|
|
31
|
+
export interface RouteExtras {
|
|
32
|
+
/** Builds the `/health` body. Omitted ⇒ the legacy constant `{ status: 'ok' }`. */
|
|
33
|
+
health?: () => ServerHealth;
|
|
34
|
+
/**
|
|
35
|
+
* Whether the caller presented a valid token. Only consulted by `/health`,
|
|
36
|
+
* which is the one route reachable without one. `true` when no token is
|
|
37
|
+
* configured at all, so an unprotected server keeps serving the full body.
|
|
38
|
+
*/
|
|
39
|
+
authenticated?: boolean;
|
|
40
|
+
}
|
|
41
|
+
export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number, idleSweeper?: IdleSweeper | null, resolveModel?: (name: string) => Promise<void>, listModels?: () => PublicModelEntry[], modelWorkCoordinator?: ModelWorkCoordinator, extras?: RouteExtras): Promise<void>;
|
|
9
42
|
//# sourceMappingURL=router.d.ts.map
|
package/dist/router.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAapD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAapD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAmB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAOnD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,CAS5D;AAoBD;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,mFAAmF;IACnF,MAAM,CAAC,EAAE,MAAM,YAAY,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,EAC7B,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,EAChC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EAC9C,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,EACrC,oBAAoB,CAAC,EAAE,oBAAoB,EAC3C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,IAAI,CAAC,CAuHf"}
|
package/dist/router.js
CHANGED
|
@@ -4,8 +4,36 @@ import { handleCreateMessage } from './endpoints/messages.js';
|
|
|
4
4
|
import { handleListModels } from './endpoints/models.js';
|
|
5
5
|
import { handleCreateResponse } from './endpoints/responses.js';
|
|
6
6
|
import { sendAnthropicBadRequest, sendAnthropicMethodNotAllowed, sendBadRequest, sendMethodNotAllowed, sendNotFound, } from './errors.js';
|
|
7
|
+
import { toMinimalHealth } from './health.js';
|
|
7
8
|
/** Max request body size (10 MB). */
|
|
8
9
|
const MAX_BODY_BYTES = 10 * 1024 * 1024;
|
|
10
|
+
/**
|
|
11
|
+
* The request's pathname, parsed against a CONSTANT base.
|
|
12
|
+
*
|
|
13
|
+
* Never against `Host`. That header is attacker-controlled text on every
|
|
14
|
+
* request — `Host: [` makes `new URL()` throw `ERR_INVALID_URL`, and a throw
|
|
15
|
+
* from an async request listener is an unhandled rejection, which under Node's
|
|
16
|
+
* default `--unhandled-rejections=throw` takes the whole process down. One
|
|
17
|
+
* malformed byte from any client that can reach the socket was enough to end
|
|
18
|
+
* inference. A pathname does not depend on the authority anyway, so a fixed
|
|
19
|
+
* base is both safer and equivalent: an absolute-form request URI
|
|
20
|
+
* (`GET http://host/v1/models HTTP/1.1`, legal in HTTP/1.1) still wins over
|
|
21
|
+
* the base and yields the same path it always did.
|
|
22
|
+
*
|
|
23
|
+
* `req.url` itself is guarded too, for the same reason rather than a known
|
|
24
|
+
* input: this function's contract is that no request can make it throw.
|
|
25
|
+
*/
|
|
26
|
+
export function requestPathname(req) {
|
|
27
|
+
const raw = req.url ?? '/';
|
|
28
|
+
try {
|
|
29
|
+
return new URL(raw, 'http://localhost').pathname;
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
const cut = raw.search(/[?#]/);
|
|
33
|
+
const path = cut === -1 ? raw : raw.slice(0, cut);
|
|
34
|
+
return path.startsWith('/') ? path : '/';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
9
37
|
function readBody(req) {
|
|
10
38
|
return new Promise((resolve, reject) => {
|
|
11
39
|
const chunks = [];
|
|
@@ -23,9 +51,8 @@ function readBody(req) {
|
|
|
23
51
|
req.on('error', reject);
|
|
24
52
|
});
|
|
25
53
|
}
|
|
26
|
-
export async function routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator) {
|
|
27
|
-
const
|
|
28
|
-
const path = url.pathname;
|
|
54
|
+
export async function routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator, extras) {
|
|
55
|
+
const path = requestPathname(req);
|
|
29
56
|
if (path === '/v1/models') {
|
|
30
57
|
if (req.method !== 'GET') {
|
|
31
58
|
sendMethodNotAllowed(res, 'GET');
|
|
@@ -49,7 +76,7 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
|
|
|
49
76
|
sendBadRequest(res, msg);
|
|
50
77
|
return;
|
|
51
78
|
}
|
|
52
|
-
await handleCreateResponse(res, body, registry, store, req, responseRetentionSec, idleSweeper, modelWorkCoordinator);
|
|
79
|
+
await handleCreateResponse(res, body, registry, store, req, responseRetentionSec, idleSweeper, modelWorkCoordinator, resolveModel);
|
|
53
80
|
return;
|
|
54
81
|
}
|
|
55
82
|
if (path === '/v1/messages/count_tokens') {
|
|
@@ -89,8 +116,23 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
|
|
|
89
116
|
return;
|
|
90
117
|
}
|
|
91
118
|
if (path === '/health' || path === '/v1/health') {
|
|
119
|
+
// Deliberately NOT bracketed by `idleSweeper.beginRequest/endRequest`
|
|
120
|
+
// and free of any native call: a supervisor polling on an interval must
|
|
121
|
+
// not keep pushing the drain timer out, nor touch the MLX allocator.
|
|
122
|
+
// Every field is read from plain JavaScript state.
|
|
123
|
+
const health = extras?.health?.();
|
|
124
|
+
if (health === undefined) {
|
|
125
|
+
// No reporter wired (a bare `createHandler` mounted by hand): keep the
|
|
126
|
+
// historical constant so existing consumers are unaffected.
|
|
127
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
128
|
+
res.end(JSON.stringify({ status: 'ok' }));
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
// Unauthenticated pollers get liveness only. `models.resident` leaks
|
|
132
|
+
// project names and local paths, so it stays behind the token.
|
|
133
|
+
const body = extras?.authenticated === false ? toMinimalHealth(health) : health;
|
|
92
134
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
93
|
-
res.end(JSON.stringify(
|
|
135
|
+
res.end(JSON.stringify(body));
|
|
94
136
|
return;
|
|
95
137
|
}
|
|
96
138
|
// Liveness probe at `/`. Claude Code issues `HEAD /` before its first
|
package/dist/server.d.ts
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
import type { Server } from 'node:http';
|
|
3
3
|
import { ResponseStore } from '@mlx-node/core';
|
|
4
4
|
import type { PublicModelEntry } from './handler.js';
|
|
5
|
+
import { type ServerHealth } from './health.js';
|
|
6
|
+
import { type LoadModelOptions } from './load-model.js';
|
|
7
|
+
import { ModelWorkCoordinator } from './model-work-coordinator.js';
|
|
5
8
|
import { ModelRegistry } from './registry.js';
|
|
6
9
|
/**
|
|
7
10
|
* Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
|
|
@@ -26,6 +29,33 @@ export declare function parseEnvSeconds(name: string): number | undefined;
|
|
|
26
29
|
* Exported for unit tests.
|
|
27
30
|
*/
|
|
28
31
|
export declare function parseEnvPositiveInt(name: string): number | undefined;
|
|
32
|
+
/**
|
|
33
|
+
* Resolve the effective auth token from an explicit value plus the env
|
|
34
|
+
* fallback.
|
|
35
|
+
*
|
|
36
|
+
* Exported because `createInferenceHost` has to answer "is this server going
|
|
37
|
+
* to be protected?" BEFORE it binds, in order to refuse a non-loopback bind
|
|
38
|
+
* that would serve anonymously. Two independent copies of this rule would
|
|
39
|
+
* drift, and the direction it would drift is a host that refuses to start
|
|
40
|
+
* while `MLX_SERVER_AUTH_TOKEN` is sitting right there in the environment.
|
|
41
|
+
*
|
|
42
|
+
* An empty env var means "not set". An accidental `MLX_SERVER_AUTH_TOKEN=` in
|
|
43
|
+
* a launcher script must not enable auth with an empty secret that every
|
|
44
|
+
* credential-less request would then fail against.
|
|
45
|
+
*
|
|
46
|
+
* An empty EXPLICIT token is a different case and is rejected outright, for the
|
|
47
|
+
* same reason {@link normalizePositiveIntConfig} rejects a bogus explicit knob:
|
|
48
|
+
* somebody asked for a token and supplied nothing, which is
|
|
49
|
+
* `--auth-token "$TOKEN"` with `TOKEN` unset. Returning `''` made the bind
|
|
50
|
+
* guard read "auth is configured" and allow `0.0.0.0`, while the comparator
|
|
51
|
+
* accepted an empty `x-api-key` because both strings were empty — a wildcard
|
|
52
|
+
* bind published under a credential anyone can guess. Quietly downgrading to
|
|
53
|
+
* "no auth" instead would be fail-open in the other direction: on loopback it
|
|
54
|
+
* hands back the unauthenticated, wildcard-CORS server the operator was
|
|
55
|
+
* explicitly trying not to start. Throwing is the only answer that is wrong in
|
|
56
|
+
* neither bind mode, and it happens before anything binds or loads.
|
|
57
|
+
*/
|
|
58
|
+
export declare function resolveAuthToken(explicit: string | undefined): string | undefined;
|
|
29
59
|
export interface ServerConfig {
|
|
30
60
|
/** Port to listen on (default: 8080). */
|
|
31
61
|
port?: number;
|
|
@@ -35,8 +65,26 @@ export interface ServerConfig {
|
|
|
35
65
|
storePath?: string;
|
|
36
66
|
/** Disable response storage entirely (default: false). */
|
|
37
67
|
disableStore?: boolean;
|
|
38
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Enable CORS headers.
|
|
70
|
+
*
|
|
71
|
+
* Default: `true` when no `authToken` is in effect (historical behaviour),
|
|
72
|
+
* `false` once one is. An explicit value always wins. See
|
|
73
|
+
* {@link ServerConfig.authToken}.
|
|
74
|
+
*/
|
|
39
75
|
cors?: boolean;
|
|
76
|
+
/**
|
|
77
|
+
* Shared secret required on every route except `/health` and `/v1/health`.
|
|
78
|
+
*
|
|
79
|
+
* Default: `process.env.MLX_SERVER_AUTH_TOKEN`, or `undefined` (no auth)
|
|
80
|
+
* when that is unset or empty. `undefined` is byte-for-byte identical to
|
|
81
|
+
* the pre-auth behaviour. An explicit `''` is rejected rather than treated as
|
|
82
|
+
* either — see {@link resolveAuthToken}.
|
|
83
|
+
*
|
|
84
|
+
* Accepted as `x-api-key: <token>` or `authorization: Bearer <token>`.
|
|
85
|
+
* Setting it also flips the `cors` default to `false`.
|
|
86
|
+
*/
|
|
87
|
+
authToken?: string;
|
|
40
88
|
/**
|
|
41
89
|
* Retention for persisted response rows, in seconds. Stamped as `expires_at`
|
|
42
90
|
* on each committed response; controls how long `previous_response_id`
|
|
@@ -88,14 +136,80 @@ export interface ServerConfig {
|
|
|
88
136
|
*/
|
|
89
137
|
listModels?: () => PublicModelEntry[];
|
|
90
138
|
}
|
|
139
|
+
/** Options for {@link ServerInstance.close}. */
|
|
140
|
+
export interface CloseOptions {
|
|
141
|
+
/**
|
|
142
|
+
* Grace period, in milliseconds, before still-open connections are
|
|
143
|
+
* destroyed. Default: {@link DEFAULT_CLOSE_TIMEOUT_MS} (5000).
|
|
144
|
+
*
|
|
145
|
+
* Only the FIRST `close()` call's value is honoured — later calls receive
|
|
146
|
+
* the memoized promise of the first, so their timeout is ignored.
|
|
147
|
+
*/
|
|
148
|
+
timeoutMs?: number;
|
|
149
|
+
}
|
|
150
|
+
/** Outcome of {@link ServerInstance.close}. */
|
|
151
|
+
export interface CloseResult {
|
|
152
|
+
/** `true` when the grace period expired and connections were destroyed. */
|
|
153
|
+
forced: boolean;
|
|
154
|
+
/** SSE streams open at the moment of the forced destroy. `0` when not forced. */
|
|
155
|
+
streamsAborted: number;
|
|
156
|
+
/** Wall-clock duration of the shutdown. */
|
|
157
|
+
durationMs: number;
|
|
158
|
+
}
|
|
91
159
|
export interface ServerInstance {
|
|
92
160
|
server: Server;
|
|
93
161
|
/** Register models before or after starting. */
|
|
94
162
|
registry: ModelRegistry;
|
|
95
163
|
/** Null when disabled. */
|
|
96
164
|
store: ResponseStore | null;
|
|
97
|
-
/**
|
|
98
|
-
|
|
165
|
+
/**
|
|
166
|
+
* Coordinates process-wide MLX work: model loads take the exclusive writer
|
|
167
|
+
* slot, inference takes shared reader slots. Exposed so callers can compose
|
|
168
|
+
* their own brackets (or read `writerActive` / `lastLoad` for diagnostics).
|
|
169
|
+
* Prefer {@link loadModel} for the common load case — it also handles the
|
|
170
|
+
* drain suspension, which is easy to get wrong.
|
|
171
|
+
*/
|
|
172
|
+
readonly modelWork: ModelWorkCoordinator;
|
|
173
|
+
/**
|
|
174
|
+
* Current readiness snapshot — the same body an authenticated
|
|
175
|
+
* `GET /health` returns. Pure JavaScript state; no native calls.
|
|
176
|
+
*/
|
|
177
|
+
health(): ServerHealth;
|
|
178
|
+
/**
|
|
179
|
+
* Bounded, idempotent shutdown.
|
|
180
|
+
*
|
|
181
|
+
* Stops accepting new connections, drops idle (keep-alive) ones
|
|
182
|
+
* immediately, then waits up to `timeoutMs` for the rest to finish. On
|
|
183
|
+
* expiry every remaining connection is destroyed, which fires the same
|
|
184
|
+
* `res.on('close')` path a client disconnect fires — so in-flight SSE
|
|
185
|
+
* generations are cancelled through `@mlx-node/lm` down to the native
|
|
186
|
+
* `ChatStreamHandle`.
|
|
187
|
+
*
|
|
188
|
+
* Idempotent: the promise is memoized, so repeated calls return the same
|
|
189
|
+
* promise and the same result. In particular a second call does NOT
|
|
190
|
+
* reject with `ERR_SERVER_NOT_RUNNING`.
|
|
191
|
+
*
|
|
192
|
+
* RESIDUAL: `ResponseStore` has no `close()` (it is a Rust-side handle),
|
|
193
|
+
* so the SQLite connection is released by process exit, not here. A
|
|
194
|
+
* long-lived process that creates and closes many servers will hold one
|
|
195
|
+
* store handle per server.
|
|
196
|
+
*/
|
|
197
|
+
close(opts?: CloseOptions): Promise<CloseResult>;
|
|
198
|
+
/**
|
|
199
|
+
* Load a model out-of-band and register it, with idle drains suspended and
|
|
200
|
+
* inference excluded for the entire operation — including the wait for the
|
|
201
|
+
* coordinator's writer lock.
|
|
202
|
+
*
|
|
203
|
+
* This is the safe way to swap the resident model on a server that is
|
|
204
|
+
* already serving. Hand-rolling the two brackets in the wrong order races
|
|
205
|
+
* the process-wide Metal allocator; see `load-model.ts` for the full
|
|
206
|
+
* rationale.
|
|
207
|
+
*
|
|
208
|
+
* Rejects with the underlying error if `load()` throws; both brackets
|
|
209
|
+
* unwind cleanly, and `health().lastLoad` records the failure under
|
|
210
|
+
* `opts.name`.
|
|
211
|
+
*/
|
|
212
|
+
loadModel(opts: LoadModelOptions): Promise<void>;
|
|
99
213
|
/**
|
|
100
214
|
* Run `fn` with the idle-drain timer suspended for the duration of
|
|
101
215
|
* an unbracketed, allocator-heavy operation — most commonly a hot
|
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAkB,MAAM,WAAW,CAAC;AAIxD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAuB,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA2B9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAUjF;AAED,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;;;;OAUG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC;IACzC;;;OAGG;IACH,MAAM,IAAI,YAAY,CAAC;IACvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACjD;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;OAQG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA4KjF"}
|