@mlx-node/server 0.0.8 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/auth.d.ts +56 -0
  2. package/dist/auth.d.ts.map +1 -0
  3. package/dist/auth.js +106 -0
  4. package/dist/chat-session-warm-reuse.d.ts +8 -8
  5. package/dist/chat-session-warm-reuse.d.ts.map +1 -1
  6. package/dist/chat-session-warm-reuse.js +12 -8
  7. package/dist/endpoints/responses.d.ts +3 -1
  8. package/dist/endpoints/responses.d.ts.map +1 -1
  9. package/dist/endpoints/responses.js +38 -5
  10. package/dist/handler.d.ts +27 -1
  11. package/dist/handler.d.ts.map +1 -1
  12. package/dist/handler.js +65 -16
  13. package/dist/health.d.ts +146 -0
  14. package/dist/health.d.ts.map +1 -0
  15. package/dist/health.js +107 -0
  16. package/dist/host/discover.d.ts +19 -0
  17. package/dist/host/discover.d.ts.map +1 -0
  18. package/dist/host/discover.js +50 -0
  19. package/dist/host/env-policy.d.ts +62 -0
  20. package/dist/host/env-policy.d.ts.map +1 -0
  21. package/dist/host/env-policy.js +69 -0
  22. package/dist/host/index.d.ts +202 -0
  23. package/dist/host/index.d.ts.map +1 -0
  24. package/dist/host/index.js +325 -0
  25. package/dist/host/logger.d.ts +36 -0
  26. package/dist/host/logger.d.ts.map +1 -0
  27. package/dist/host/logger.js +376 -0
  28. package/dist/host/net.d.ts +65 -0
  29. package/dist/host/net.d.ts.map +1 -0
  30. package/dist/host/net.js +97 -0
  31. package/dist/host/paths.d.ts +28 -0
  32. package/dist/host/paths.d.ts.map +1 -0
  33. package/dist/host/paths.js +71 -0
  34. package/dist/host/swap.d.ts +27 -0
  35. package/dist/host/swap.d.ts.map +1 -0
  36. package/dist/host/swap.js +178 -0
  37. package/dist/host/temp-root.d.ts +57 -0
  38. package/dist/host/temp-root.d.ts.map +1 -0
  39. package/dist/host/temp-root.js +99 -0
  40. package/dist/index.d.ts +11 -2
  41. package/dist/index.d.ts.map +1 -1
  42. package/dist/index.js +7 -1
  43. package/dist/load-model.d.ts +69 -0
  44. package/dist/load-model.d.ts.map +1 -0
  45. package/dist/load-model.js +63 -0
  46. package/dist/model-work-coordinator.d.ts +29 -4
  47. package/dist/model-work-coordinator.d.ts.map +1 -1
  48. package/dist/model-work-coordinator.js +97 -16
  49. package/dist/router.d.ts +34 -1
  50. package/dist/router.d.ts.map +1 -1
  51. package/dist/router.js +47 -5
  52. package/dist/server.d.ts +117 -3
  53. package/dist/server.d.ts.map +1 -1
  54. package/dist/server.js +125 -9
  55. package/dist/session-registry.d.ts +7 -0
  56. package/dist/session-registry.d.ts.map +1 -1
  57. package/dist/session-registry.js +9 -0
  58. package/dist/streaming.d.ts +14 -0
  59. package/dist/streaming.d.ts.map +1 -1
  60. package/dist/streaming.js +45 -0
  61. package/package.json +15 -3
@@ -1,3 +1,25 @@
1
+ /**
2
+ * Render a thrown value for {@link ModelLoadRecord.error}.
3
+ *
4
+ * Deliberately avoids `String(unknown)`: a rejection carrying a plain object
5
+ * would render as the useless `[object Object]` in the one field a supervisor
6
+ * reads to find out why the model would not load.
7
+ */
8
+ function describeLoadFailure(error) {
9
+ if (error instanceof Error)
10
+ return error.message;
11
+ if (typeof error === 'string')
12
+ return error;
13
+ if (error == null)
14
+ return 'unknown error';
15
+ try {
16
+ return JSON.stringify(error) ?? 'unknown error';
17
+ }
18
+ catch {
19
+ // Circular structure, or a `toJSON` that throws.
20
+ return 'unknown error';
21
+ }
22
+ }
1
23
  /**
2
24
  * Process-local gate for native MLX work.
3
25
  *
@@ -9,16 +31,67 @@
9
31
  */
10
32
  export class ModelWorkCoordinator {
11
33
  activeReaders = 0;
12
- writerActive = false;
13
- waitingWriters = 0;
34
+ writerHeld = false;
35
+ queuedWriters = 0;
14
36
  readerWaiters = [];
15
37
  writerWaiters = [];
16
- async withModelLoad(fn) {
38
+ /**
39
+ * Most recent settled load bracket. Retained here because the coordinator
40
+ * is the ONE place that brackets every load: a `resolveModel` failure in
41
+ * `/v1/messages` becomes a 500 and is otherwise dropped on the floor, so a
42
+ * supervisor polling `/health` afterwards had no way to learn what went
43
+ * wrong. See {@link ModelLoadRecord} for the "successful no-op overwrites
44
+ * an earlier failure" caveat.
45
+ */
46
+ lastLoadRecord = null;
47
+ /** Read-only: `true` while a load holds the exclusive writer slot. */
48
+ get writerActive() {
49
+ return this.writerHeld;
50
+ }
51
+ /** Read-only: loads parked in `acquireWrite()` waiting for the slot. */
52
+ get waitingWriters() {
53
+ return this.queuedWriters;
54
+ }
55
+ /** Read-only: outcome of the most recent settled load bracket, or `null`. */
56
+ get lastLoad() {
57
+ return this.lastLoadRecord;
58
+ }
59
+ /**
60
+ * Record a settled bracket. Called from the `finally` of both load
61
+ * wrappers so a throw is captured just as reliably as a success.
62
+ */
63
+ recordLoad(label, startedAt, error, ok) {
64
+ this.lastLoadRecord = {
65
+ label: label ?? null,
66
+ startedAt,
67
+ finishedAt: Date.now(),
68
+ ok,
69
+ error: ok ? null : describeLoadFailure(error),
70
+ };
71
+ }
72
+ /**
73
+ * @param label Optional identifier (normally the model name) stamped into
74
+ * {@link lastLoad} so `/health` can name what was being loaded.
75
+ */
76
+ async withModelLoad(fn, label) {
17
77
  await this.acquireWrite();
78
+ // Measured from lock acquisition, not from arrival: `startedAt` is meant
79
+ // to answer "how long has the actual materialization been running",
80
+ // which is what a supervisor deciding whether to wait needs.
81
+ const startedAt = Date.now();
82
+ let ok = false;
83
+ let failure;
18
84
  try {
19
- return await fn();
85
+ const result = await fn();
86
+ ok = true;
87
+ return result;
88
+ }
89
+ catch (err) {
90
+ failure = err;
91
+ throw err;
20
92
  }
21
93
  finally {
94
+ this.recordLoad(label, startedAt, failure, ok);
22
95
  this.releaseWrite();
23
96
  }
24
97
  }
@@ -35,25 +108,33 @@ export class ModelWorkCoordinator {
35
108
  * 60-second cold-load does not look like 60 seconds of own work for
36
109
  * every concurrent request.
37
110
  */
38
- async withModelLoadInstrumented(fn) {
111
+ async withModelLoadInstrumented(fn, label) {
39
112
  // `owner` MUST be decided synchronously, before any await, so the
40
113
  // signal reflects coordinator state at arrival rather than after
41
114
  // any peer transition. The wait/own split is measured around the
42
115
  // actual phase boundaries (lock acquisition, fn completion) so the
43
116
  // two intervals partition cleanly instead of both reporting total
44
117
  // elapsed time — see `ModelLoadOutcome` for the contract.
45
- const owner = !this.writerActive && this.waitingWriters === 0;
118
+ const owner = !this.writerHeld && this.queuedWriters === 0;
46
119
  const arrivedAt = Date.now();
47
120
  await this.acquireWrite();
48
121
  const lockAcquiredAt = Date.now();
122
+ let ok = false;
123
+ let failure;
49
124
  try {
50
125
  const result = await fn();
126
+ ok = true;
51
127
  const fnDoneAt = Date.now();
52
128
  const waitMs = Math.max(0, lockAcquiredAt - arrivedAt);
53
129
  const ownMs = Math.max(0, fnDoneAt - lockAcquiredAt);
54
130
  return { result, owner, waitMs, ownMs };
55
131
  }
132
+ catch (err) {
133
+ failure = err;
134
+ throw err;
135
+ }
56
136
  finally {
137
+ this.recordLoad(label, lockAcquiredAt, failure, ok);
57
138
  this.releaseWrite();
58
139
  }
59
140
  }
@@ -67,7 +148,7 @@ export class ModelWorkCoordinator {
67
148
  }
68
149
  }
69
150
  acquireRead() {
70
- if (!this.writerActive && this.waitingWriters === 0) {
151
+ if (!this.writerHeld && this.queuedWriters === 0) {
71
152
  this.activeReaders += 1;
72
153
  return Promise.resolve();
73
154
  }
@@ -79,16 +160,16 @@ export class ModelWorkCoordinator {
79
160
  });
80
161
  }
81
162
  acquireWrite() {
82
- this.waitingWriters += 1;
83
- if (!this.writerActive && this.activeReaders === 0) {
84
- this.waitingWriters -= 1;
85
- this.writerActive = true;
163
+ this.queuedWriters += 1;
164
+ if (!this.writerHeld && this.activeReaders === 0) {
165
+ this.queuedWriters -= 1;
166
+ this.writerHeld = true;
86
167
  return Promise.resolve();
87
168
  }
88
169
  return new Promise((resolve) => {
89
170
  this.writerWaiters.push(() => {
90
- this.waitingWriters -= 1;
91
- this.writerActive = true;
171
+ this.queuedWriters -= 1;
172
+ this.writerHeld = true;
92
173
  resolve();
93
174
  });
94
175
  });
@@ -101,17 +182,17 @@ export class ModelWorkCoordinator {
101
182
  this.drain();
102
183
  }
103
184
  releaseWrite() {
104
- this.writerActive = false;
185
+ this.writerHeld = false;
105
186
  this.drain();
106
187
  }
107
188
  drain() {
108
- if (this.writerActive)
189
+ if (this.writerHeld)
109
190
  return;
110
191
  if (this.activeReaders === 0 && this.writerWaiters.length > 0) {
111
192
  this.writerWaiters.shift()?.();
112
193
  return;
113
194
  }
114
- if (this.waitingWriters === 0 && this.readerWaiters.length > 0) {
195
+ if (this.queuedWriters === 0 && this.readerWaiters.length > 0) {
115
196
  const readers = this.readerWaiters.splice(0);
116
197
  for (const resolve of readers)
117
198
  resolve();
package/dist/router.d.ts CHANGED
@@ -2,8 +2,41 @@
2
2
  import type { IncomingMessage, ServerResponse } from 'node:http';
3
3
  import type { ResponseStore } from '@mlx-node/core';
4
4
  import type { PublicModelEntry } from './handler.js';
5
+ import { type ServerHealth } from './health.js';
5
6
  import type { IdleSweeper } from './idle-sweeper.js';
6
7
  import type { ModelWorkCoordinator } from './model-work-coordinator.js';
7
8
  import type { ModelRegistry } from './registry.js';
8
- export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number, idleSweeper?: IdleSweeper | null, resolveModel?: (name: string) => Promise<void>, listModels?: () => PublicModelEntry[], modelWorkCoordinator?: ModelWorkCoordinator): Promise<void>;
9
+ /**
10
+ * The request's pathname, parsed against a CONSTANT base.
11
+ *
12
+ * Never against `Host`. That header is attacker-controlled text on every
13
+ * request — `Host: [` makes `new URL()` throw `ERR_INVALID_URL`, and a throw
14
+ * from an async request listener is an unhandled rejection, which under Node's
15
+ * default `--unhandled-rejections=throw` takes the whole process down. One
16
+ * malformed byte from any client that can reach the socket was enough to end
17
+ * inference. A pathname does not depend on the authority anyway, so a fixed
18
+ * base is both safer and equivalent: an absolute-form request URI
19
+ * (`GET http://host/v1/models HTTP/1.1`, legal in HTTP/1.1) still wins over
20
+ * the base and yields the same path it always did.
21
+ *
22
+ * `req.url` itself is guarded too, for the same reason rather than a known
23
+ * input: this function's contract is that no request can make it throw.
24
+ */
25
+ export declare function requestPathname(req: IncomingMessage): string;
26
+ /**
27
+ * Trailing options bag. Added as an object rather than two more positional
28
+ * parameters — `routeRequest` already carries nine, and the two knobs here
29
+ * are unrelated to each other.
30
+ */
31
+ export interface RouteExtras {
32
+ /** Builds the `/health` body. Omitted ⇒ the legacy constant `{ status: 'ok' }`. */
33
+ health?: () => ServerHealth;
34
+ /**
35
+ * Whether the caller presented a valid token. Only consulted by `/health`,
36
+ * which is the one route reachable without one. `true` when no token is
37
+ * configured at all, so an unprotected server keeps serving the full body.
38
+ */
39
+ authenticated?: boolean;
40
+ }
41
+ export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number, idleSweeper?: IdleSweeper | null, resolveModel?: (name: string) => Promise<void>, listModels?: () => PublicModelEntry[], modelWorkCoordinator?: ModelWorkCoordinator, extras?: RouteExtras): Promise<void>;
9
42
  //# sourceMappingURL=router.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAapD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAyBnD,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,EAC7B,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,EAChC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EAC9C,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,EACrC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAwGf"}
1
+ {"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAapD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAmB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAOnD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,CAS5D;AAoBD;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,mFAAmF;IACnF,MAAM,CAAC,EAAE,MAAM,YAAY,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,EAC7B,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,EAChC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EAC9C,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,EACrC,oBAAoB,CAAC,EAAE,oBAAoB,EAC3C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,IAAI,CAAC,CAuHf"}
package/dist/router.js CHANGED
@@ -4,8 +4,36 @@ import { handleCreateMessage } from './endpoints/messages.js';
4
4
  import { handleListModels } from './endpoints/models.js';
5
5
  import { handleCreateResponse } from './endpoints/responses.js';
6
6
  import { sendAnthropicBadRequest, sendAnthropicMethodNotAllowed, sendBadRequest, sendMethodNotAllowed, sendNotFound, } from './errors.js';
7
+ import { toMinimalHealth } from './health.js';
7
8
  /** Max request body size (10 MB). */
8
9
  const MAX_BODY_BYTES = 10 * 1024 * 1024;
10
+ /**
11
+ * The request's pathname, parsed against a CONSTANT base.
12
+ *
13
+ * Never against `Host`. That header is attacker-controlled text on every
14
+ * request — `Host: [` makes `new URL()` throw `ERR_INVALID_URL`, and a throw
15
+ * from an async request listener is an unhandled rejection, which under Node's
16
+ * default `--unhandled-rejections=throw` takes the whole process down. One
17
+ * malformed byte from any client that can reach the socket was enough to end
18
+ * inference. A pathname does not depend on the authority anyway, so a fixed
19
+ * base is both safer and equivalent: an absolute-form request URI
20
+ * (`GET http://host/v1/models HTTP/1.1`, legal in HTTP/1.1) still wins over
21
+ * the base and yields the same path it always did.
22
+ *
23
+ * `req.url` itself is guarded too, for the same reason rather than a known
24
+ * input: this function's contract is that no request can make it throw.
25
+ */
26
+ export function requestPathname(req) {
27
+ const raw = req.url ?? '/';
28
+ try {
29
+ return new URL(raw, 'http://localhost').pathname;
30
+ }
31
+ catch {
32
+ const cut = raw.search(/[?#]/);
33
+ const path = cut === -1 ? raw : raw.slice(0, cut);
34
+ return path.startsWith('/') ? path : '/';
35
+ }
36
+ }
9
37
  function readBody(req) {
10
38
  return new Promise((resolve, reject) => {
11
39
  const chunks = [];
@@ -23,9 +51,8 @@ function readBody(req) {
23
51
  req.on('error', reject);
24
52
  });
25
53
  }
26
- export async function routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator) {
27
- const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
28
- const path = url.pathname;
54
+ export async function routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator, extras) {
55
+ const path = requestPathname(req);
29
56
  if (path === '/v1/models') {
30
57
  if (req.method !== 'GET') {
31
58
  sendMethodNotAllowed(res, 'GET');
@@ -49,7 +76,7 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
49
76
  sendBadRequest(res, msg);
50
77
  return;
51
78
  }
52
- await handleCreateResponse(res, body, registry, store, req, responseRetentionSec, idleSweeper, modelWorkCoordinator);
79
+ await handleCreateResponse(res, body, registry, store, req, responseRetentionSec, idleSweeper, modelWorkCoordinator, resolveModel);
53
80
  return;
54
81
  }
55
82
  if (path === '/v1/messages/count_tokens') {
@@ -89,8 +116,23 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
89
116
  return;
90
117
  }
91
118
  if (path === '/health' || path === '/v1/health') {
119
+ // Deliberately NOT bracketed by `idleSweeper.beginRequest/endRequest`
120
+ // and free of any native call: a supervisor polling on an interval must
121
+ // not keep pushing the drain timer out, nor touch the MLX allocator.
122
+ // Every field is read from plain JavaScript state.
123
+ const health = extras?.health?.();
124
+ if (health === undefined) {
125
+ // No reporter wired (a bare `createHandler` mounted by hand): keep the
126
+ // historical constant so existing consumers are unaffected.
127
+ res.writeHead(200, { 'Content-Type': 'application/json' });
128
+ res.end(JSON.stringify({ status: 'ok' }));
129
+ return;
130
+ }
131
+ // Unauthenticated pollers get liveness only. `models.resident` leaks
132
+ // project names and local paths, so it stays behind the token.
133
+ const body = extras?.authenticated === false ? toMinimalHealth(health) : health;
92
134
  res.writeHead(200, { 'Content-Type': 'application/json' });
93
- res.end(JSON.stringify({ status: 'ok' }));
135
+ res.end(JSON.stringify(body));
94
136
  return;
95
137
  }
96
138
  // Liveness probe at `/`. Claude Code issues `HEAD /` before its first
package/dist/server.d.ts CHANGED
@@ -2,6 +2,9 @@
2
2
  import type { Server } from 'node:http';
3
3
  import { ResponseStore } from '@mlx-node/core';
4
4
  import type { PublicModelEntry } from './handler.js';
5
+ import { type ServerHealth } from './health.js';
6
+ import { type LoadModelOptions } from './load-model.js';
7
+ import { ModelWorkCoordinator } from './model-work-coordinator.js';
5
8
  import { ModelRegistry } from './registry.js';
6
9
  /**
7
10
  * Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
@@ -26,6 +29,33 @@ export declare function parseEnvSeconds(name: string): number | undefined;
26
29
  * Exported for unit tests.
27
30
  */
28
31
  export declare function parseEnvPositiveInt(name: string): number | undefined;
32
+ /**
33
+ * Resolve the effective auth token from an explicit value plus the env
34
+ * fallback.
35
+ *
36
+ * Exported because `createInferenceHost` has to answer "is this server going
37
+ * to be protected?" BEFORE it binds, in order to refuse a non-loopback bind
38
+ * that would serve anonymously. Two independent copies of this rule would
39
+ * drift, and the direction it would drift is a host that refuses to start
40
+ * while `MLX_SERVER_AUTH_TOKEN` is sitting right there in the environment.
41
+ *
42
+ * An empty env var means "not set". An accidental `MLX_SERVER_AUTH_TOKEN=` in
43
+ * a launcher script must not enable auth with an empty secret that every
44
+ * credential-less request would then fail against.
45
+ *
46
+ * An empty EXPLICIT token is a different case and is rejected outright, for the
47
+ * same reason {@link normalizePositiveIntConfig} rejects a bogus explicit knob:
48
+ * somebody asked for a token and supplied nothing, which is
49
+ * `--auth-token "$TOKEN"` with `TOKEN` unset. Returning `''` made the bind
50
+ * guard read "auth is configured" and allow `0.0.0.0`, while the comparator
51
+ * accepted an empty `x-api-key` because both strings were empty — a wildcard
52
+ * bind published under a credential anyone can guess. Quietly downgrading to
53
+ * "no auth" instead would be fail-open in the other direction: on loopback it
54
+ * hands back the unauthenticated, wildcard-CORS server the operator was
55
+ * explicitly trying not to start. Throwing is the only answer that is wrong in
56
+ * neither bind mode, and it happens before anything binds or loads.
57
+ */
58
+ export declare function resolveAuthToken(explicit: string | undefined): string | undefined;
29
59
  export interface ServerConfig {
30
60
  /** Port to listen on (default: 8080). */
31
61
  port?: number;
@@ -35,8 +65,26 @@ export interface ServerConfig {
35
65
  storePath?: string;
36
66
  /** Disable response storage entirely (default: false). */
37
67
  disableStore?: boolean;
38
- /** Enable CORS headers (default: true). */
68
+ /**
69
+ * Enable CORS headers.
70
+ *
71
+ * Default: `true` when no `authToken` is in effect (historical behaviour),
72
+ * `false` once one is. An explicit value always wins. See
73
+ * {@link ServerConfig.authToken}.
74
+ */
39
75
  cors?: boolean;
76
+ /**
77
+ * Shared secret required on every route except `/health` and `/v1/health`.
78
+ *
79
+ * Default: `process.env.MLX_SERVER_AUTH_TOKEN`, or `undefined` (no auth)
80
+ * when that is unset or empty. `undefined` is byte-for-byte identical to
81
+ * the pre-auth behaviour. An explicit `''` is rejected rather than treated as
82
+ * either — see {@link resolveAuthToken}.
83
+ *
84
+ * Accepted as `x-api-key: <token>` or `authorization: Bearer <token>`.
85
+ * Setting it also flips the `cors` default to `false`.
86
+ */
87
+ authToken?: string;
40
88
  /**
41
89
  * Retention for persisted response rows, in seconds. Stamped as `expires_at`
42
90
  * on each committed response; controls how long `previous_response_id`
@@ -88,14 +136,80 @@ export interface ServerConfig {
88
136
  */
89
137
  listModels?: () => PublicModelEntry[];
90
138
  }
139
+ /** Options for {@link ServerInstance.close}. */
140
+ export interface CloseOptions {
141
+ /**
142
+ * Grace period, in milliseconds, before still-open connections are
143
+ * destroyed. Default: {@link DEFAULT_CLOSE_TIMEOUT_MS} (5000).
144
+ *
145
+ * Only the FIRST `close()` call's value is honoured — later calls receive
146
+ * the memoized promise of the first, so their timeout is ignored.
147
+ */
148
+ timeoutMs?: number;
149
+ }
150
+ /** Outcome of {@link ServerInstance.close}. */
151
+ export interface CloseResult {
152
+ /** `true` when the grace period expired and connections were destroyed. */
153
+ forced: boolean;
154
+ /** SSE streams open at the moment of the forced destroy. `0` when not forced. */
155
+ streamsAborted: number;
156
+ /** Wall-clock duration of the shutdown. */
157
+ durationMs: number;
158
+ }
91
159
  export interface ServerInstance {
92
160
  server: Server;
93
161
  /** Register models before or after starting. */
94
162
  registry: ModelRegistry;
95
163
  /** Null when disabled. */
96
164
  store: ResponseStore | null;
97
- /** Graceful shutdown. */
98
- close(): Promise<void>;
165
+ /**
166
+ * Coordinates process-wide MLX work: model loads take the exclusive writer
167
+ * slot, inference takes shared reader slots. Exposed so callers can compose
168
+ * their own brackets (or read `writerActive` / `lastLoad` for diagnostics).
169
+ * Prefer {@link loadModel} for the common load case — it also handles the
170
+ * drain suspension, which is easy to get wrong.
171
+ */
172
+ readonly modelWork: ModelWorkCoordinator;
173
+ /**
174
+ * Current readiness snapshot — the same body an authenticated
175
+ * `GET /health` returns. Pure JavaScript state; no native calls.
176
+ */
177
+ health(): ServerHealth;
178
+ /**
179
+ * Bounded, idempotent shutdown.
180
+ *
181
+ * Stops accepting new connections, drops idle (keep-alive) ones
182
+ * immediately, then waits up to `timeoutMs` for the rest to finish. On
183
+ * expiry every remaining connection is destroyed, which fires the same
184
+ * `res.on('close')` path a client disconnect fires — so in-flight SSE
185
+ * generations are cancelled through `@mlx-node/lm` down to the native
186
+ * `ChatStreamHandle`.
187
+ *
188
+ * Idempotent: the promise is memoized, so repeated calls return the same
189
+ * promise and the same result. In particular a second call does NOT
190
+ * reject with `ERR_SERVER_NOT_RUNNING`.
191
+ *
192
+ * RESIDUAL: `ResponseStore` has no `close()` (it is a Rust-side handle),
193
+ * so the SQLite connection is released by process exit, not here. A
194
+ * long-lived process that creates and closes many servers will hold one
195
+ * store handle per server.
196
+ */
197
+ close(opts?: CloseOptions): Promise<CloseResult>;
198
+ /**
199
+ * Load a model out-of-band and register it, with idle drains suspended and
200
+ * inference excluded for the entire operation — including the wait for the
201
+ * coordinator's writer lock.
202
+ *
203
+ * This is the safe way to swap the resident model on a server that is
204
+ * already serving. Hand-rolling the two brackets in the wrong order races
205
+ * the process-wide Metal allocator; see `load-model.ts` for the full
206
+ * rationale.
207
+ *
208
+ * Rejects with the underlying error if `load()` throws; both brackets
209
+ * unwind cleanly, and `health().lastLoad` records the failure under
210
+ * `opts.name`.
211
+ */
212
+ loadModel(opts: LoadModelOptions): Promise<void>;
99
213
  /**
100
214
  * Run `fn` with the idle-drain timer suspended for the duration of
101
215
  * an unbracketed, allocator-heavy operation — most commonly a hot
@@ -1 +1 @@
1
- {"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAIxC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAIrD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAgB9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B,yBAAyB;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;OAQG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA8FjF"}
1
+ {"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAkB,MAAM,WAAW,CAAC;AAIxD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAuB,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA2B9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAUjF;AAED,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;;;;OAUG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC;IACzC;;;OAGG;IACH,MAAM,IAAI,YAAY,CAAC;IACvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACjD;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;OAQG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA4KjF"}