@mlx-node/server 0.0.8 → 0.0.9

package/dist/auth.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Optional bearer-token gate.
+ *
+ * Enforced at the single choke point in `createHandler`'s returned closure —
+ * after the CORS/OPTIONS early-return, before `routeRequest` — so there is
+ * exactly one place a route can be added without inheriting the gate.
+ *
+ * Scope, deliberately narrow: this is a shared-secret check for a server
+ * bound to loopback and supervised by a local app. It is NOT a
+ * multi-tenant auth system — there is one token, no rotation, no scopes,
+ * no per-caller identity.
+ */
+import type { IncomingMessage, ServerResponse } from 'node:http';
+/**
+ * Extract the presented credential, or `null` when none is usable.
+ *
+ * `x-api-key` is checked FIRST because Anthropic clients (including Claude
+ * Code, the primary consumer of `/v1/messages`) send it. Checking it first
+ * also means a stale `authorization` header injected by an intermediary
+ * cannot shadow the caller's real key.
+ *
+ * Array-valued headers are rejected outright. Node collapses repeated
+ * non-allowlisted headers such as `x-api-key` into a `string[]`; joining or
+ * picking one arbitrarily would let a caller smuggle a second value past a
+ * front proxy that only inspected the first.
+ */
+export declare function extractPresentedToken(req: IncomingMessage): string | null;
+/**
+ * Constant-time-ish credential comparison.
+ *
+ * The length check in front of `timingSafeEqual` is unavoidable — the
+ * primitive throws on mismatched lengths. It therefore LEAKS the length of
+ * the configured token to an attacker who can time responses. That is an
+ * accepted trade: the token is a locally-generated high-entropy secret, and
+ * knowing its length does not meaningfully reduce the search space. The
+ * byte-by-byte content comparison, which is the part that would otherwise
+ * allow incremental guessing, stays constant-time.
+ */
+export declare function tokensMatch(presented: string, expected: string): boolean;
+/**
+ * `true` when the caller offered SOME credential, valid or not.
+ *
+ * Used only by the `/health` carve-out: an anonymous poll degrades to a
+ * liveness-only body, but a caller who presented a wrong (or malformed)
+ * credential gets a 401 so a mistyped token surfaces instead of masquerading
+ * as a healthy 200.
+ */
+export declare function hasCredential(req: IncomingMessage): boolean;
+/** `true` when the request carries a credential matching `expected`. */
+export declare function isAuthorized(req: IncomingMessage, expected: string): boolean;
+/**
+ * 401 with `WWW-Authenticate: Bearer`. The body deliberately says nothing
+ * about whether a credential was absent, malformed, or simply wrong.
+ */
+export declare function sendUnauthorized(res: ServerResponse): void;
+//# sourceMappingURL=auth.d.ts.map

package/dist/auth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAKjE;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,GAAG,IAAI,CAYzE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAaxE;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAE3D;AAED,wEAAwE;AACxE,wBAAgB,YAAY,CAAC,GAAG,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAI5E;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,cAAc,GAAG,IAAI,CAe1D"}

package/dist/auth.js

@@ -0,0 +1,106 @@
+/**
+ * Optional bearer-token gate.
+ *
+ * Enforced at the single choke point in `createHandler`'s returned closure —
+ * after the CORS/OPTIONS early-return, before `routeRequest` — so there is
+ * exactly one place a route can be added without inheriting the gate.
+ *
+ * Scope, deliberately narrow: this is a shared-secret check for a server
+ * bound to loopback and supervised by a local app. It is NOT a
+ * multi-tenant auth system — there is one token, no rotation, no scopes,
+ * no per-caller identity.
+ */
+import { timingSafeEqual } from 'node:crypto';
+/** `Bearer ` prefix length, used to slice the credential out of `authorization`. */
+const BEARER_PREFIX = 'bearer ';
+/**
+ * Extract the presented credential, or `null` when none is usable.
+ *
+ * `x-api-key` is checked FIRST because Anthropic clients (including Claude
+ * Code, the primary consumer of `/v1/messages`) send it. Checking it first
+ * also means a stale `authorization` header injected by an intermediary
+ * cannot shadow the caller's real key.
+ *
+ * Array-valued headers are rejected outright. Node collapses repeated
+ * non-allowlisted headers such as `x-api-key` into a `string[]`; joining or
+ * picking one arbitrarily would let a caller smuggle a second value past a
+ * front proxy that only inspected the first.
+ */
+export function extractPresentedToken(req) {
+    const apiKey = req.headers['x-api-key'];
+    if (apiKey !== undefined) {
+        return typeof apiKey === 'string' ? apiKey : null;
+    }
+    const authorization = req.headers.authorization;
+    if (typeof authorization !== 'string')
+        return null;
+    // Scheme is case-insensitive per RFC 7235; the credential is not.
+    if (authorization.length <= BEARER_PREFIX.length)
+        return null;
+    if (authorization.slice(0, BEARER_PREFIX.length).toLowerCase() !== BEARER_PREFIX)
+        return null;
+    return authorization.slice(BEARER_PREFIX.length);
+}
+/**
+ * Constant-time-ish credential comparison.
+ *
+ * The length check in front of `timingSafeEqual` is unavoidable — the
+ * primitive throws on mismatched lengths. It therefore LEAKS the length of
+ * the configured token to an attacker who can time responses. That is an
+ * accepted trade: the token is a locally-generated high-entropy secret, and
+ * knowing its length does not meaningfully reduce the search space. The
+ * byte-by-byte content comparison, which is the part that would otherwise
+ * allow incremental guessing, stays constant-time.
+ */
+export function tokensMatch(presented, expected) {
+    // The empty string is not a credential, in either position. `timingSafeEqual`
+    // on two zero-length buffers returns TRUE, so a server misconfigured with an
+    // empty token would authenticate every caller who sent an empty `x-api-key`.
+    // `resolveAuthToken` rejects an empty token before it can reach a server built
+    // through `createServer`; this is the second latch, for a caller that
+    // constructs `createHandler` directly. Refusing is the fail-closed direction —
+    // an empty token 401s everything rather than admitting everything.
+    if (presented === '' || expected === '')
+        return false;
+    const presentedBytes = Buffer.from(presented, 'utf8');
+    const expectedBytes = Buffer.from(expected, 'utf8');
+    if (presentedBytes.length !== expectedBytes.length)
+        return false;
+    return timingSafeEqual(presentedBytes, expectedBytes);
+}
+/**
+ * `true` when the caller offered SOME credential, valid or not.
+ *
+ * Used only by the `/health` carve-out: an anonymous poll degrades to a
+ * liveness-only body, but a caller who presented a wrong (or malformed)
+ * credential gets a 401 so a mistyped token surfaces instead of masquerading
+ * as a healthy 200.
+ */
+export function hasCredential(req) {
+    return req.headers['x-api-key'] !== undefined || req.headers.authorization !== undefined;
+}
+/** `true` when the request carries a credential matching `expected`. */
+export function isAuthorized(req, expected) {
+    const presented = extractPresentedToken(req);
+    if (presented === null)
+        return false;
+    return tokensMatch(presented, expected);
+}
+/**
+ * 401 with `WWW-Authenticate: Bearer`. The body deliberately says nothing
+ * about whether a credential was absent, malformed, or simply wrong.
+ */
+export function sendUnauthorized(res) {
+    res.writeHead(401, {
+        'WWW-Authenticate': 'Bearer realm="mlx-node"',
+        'Content-Type': 'application/json',
+    });
+    res.end(JSON.stringify({
+        error: {
+            type: 'authentication_error',
+            message: 'Missing or invalid API key',
+            code: null,
+            param: null,
+        },
+    }));
+}

package/dist/chat-session-warm-reuse.d.ts

@@ -22,14 +22,14 @@
  * native cache is correct there and only there.
  *
  * Fields accessed: `inFlight`, `history`, `lastImagesKey`, `lastAudioKey`, `turnCount`,
- * `unresolvedOkToolCallCount`, `needsFullReplay`. These are TypeScript `private` fields on
- * `ChatSession` (compile-time only) — at runtime they are ordinary
- * properties. The cast through {@link ChatSessionWarmReuseInternals}
- * gives this helper a typed view of the instance without relaxing the
- * class's `private` declarations. The field names MUST stay in sync
- * with `packages/lm/src/chat-session.ts`; a mismatch would silently
- * skip the intended state wipe and is covered by the chat-session
- * unit tests that exercise this path through the endpoint handler.
+ * `unresolvedOkToolCallCount`, `needsFullReplay`, `defaultConfig`, `activeTools`.
+ * These are TypeScript `private` fields on `ChatSession` (compile-time
+ * only) — at runtime they are ordinary properties. The cast through
+ * {@link ChatSessionWarmReuseInternals} gives this helper a typed view
+ * of the instance without relaxing the class's `private` declarations.
+ * The field names MUST stay in sync with
+ * `packages/lm/src/chat-session.ts`; a mismatch would silently skip
+ * the intended state wipe and is covered by the warm-reuse unit tests.
  */
 import type { ChatSession, SessionCapableModel } from '@mlx-node/lm';
 /**

package/dist/chat-session-warm-reuse.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chat-session-warm-reuse.d.ts","sourceRoot":"","sources":["../src/chat-session-warm-reuse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAmBrE;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,sCAAsC,CAAC,CAAC,SAAS,mBAAmB,EACxF,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,GACtB,OAAO,CAAC,IAAI,CAAC,CAqBf"}
+{"version":3,"file":"chat-session-warm-reuse.d.ts","sourceRoot":"","sources":["../src/chat-session-warm-reuse.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAc,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAqBjF;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,sCAAsC,CAAC,CAAC,SAAS,mBAAmB,EACxF,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,GACtB,OAAO,CAAC,IAAI,CAAC,CAyBf"}

package/dist/chat-session-warm-reuse.js

@@ -22,14 +22,14 @@
  * native cache is correct there and only there.
  *
  * Fields accessed: `inFlight`, `history`, `lastImagesKey`, `lastAudioKey`, `turnCount`,
- * `unresolvedOkToolCallCount`, `needsFullReplay`. These are TypeScript `private` fields on
- * `ChatSession` (compile-time only) — at runtime they are ordinary
- * properties. The cast through {@link ChatSessionWarmReuseInternals}
- * gives this helper a typed view of the instance without relaxing the
- * class's `private` declarations. The field names MUST stay in sync
- * with `packages/lm/src/chat-session.ts`; a mismatch would silently
- * skip the intended state wipe and is covered by the chat-session
- * unit tests that exercise this path through the endpoint handler.
+ * `unresolvedOkToolCallCount`, `needsFullReplay`, `defaultConfig`, `activeTools`.
+ * These are TypeScript `private` fields on `ChatSession` (compile-time
+ * only) — at runtime they are ordinary properties. The cast through
+ * {@link ChatSessionWarmReuseInternals} gives this helper a typed view
+ * of the instance without relaxing the class's `private` declarations.
+ * The field names MUST stay in sync with
+ * `packages/lm/src/chat-session.ts`; a mismatch would silently skip
+ * the intended state wipe and is covered by the warm-reuse unit tests.
  */
 /**
  * JS-state-only reset that DELIBERATELY preserves the underlying
@@ -65,4 +65,8 @@ export async function resetPreservingNativeCacheForWarmReuse(session) {
     internals.turnCount = 0;
     internals.unresolvedOkToolCallCount = null;
     internals.needsFullReplay = false;
+    // Tools are conversation state. A warm-any lease may belong to an
+    // unrelated request, so restore constructor defaults exactly like
+    // ChatSession.reset() instead of leaking the prior committed overlay.
+    internals.activeTools = internals.defaultConfig?.tools;
 }

package/dist/endpoints/responses.d.ts

@@ -88,5 +88,7 @@ export declare function __setServerBootIdForTesting(id: string): void;
  *   `tool_result` / `tool_use_id`). Validation logic is identical.
  */
 export declare function validateAndCanonicalizeHistoryToolOrder(messages: ChatMessage[], apiSurface?: 'openai' | 'anthropic'): string | null;
-export declare function handleCreateResponse(res: ServerResponse, body: ResponsesAPIRequest, registry: ModelRegistry, store: ResponseStore | null, httpReq?: IncomingMessage, responseRetentionSec?: number, idleSweeper?: IdleSweeper | null, modelWorkCoordinator?: ModelWorkCoordinator): Promise<void>;
+export declare function handleCreateResponse(res: ServerResponse, body: ResponsesAPIRequest, registry: ModelRegistry, store: ResponseStore | null, httpReq?: IncomingMessage, responseRetentionSec?: number, idleSweeper?: IdleSweeper | null, modelWorkCoordinator?: ModelWorkCoordinator, 
+/** Lazy-load hook. See the call site below and `ServerConfig.resolveModel`. */
+resolveModel?: (name: string) => Promise<void>): Promise<void>;
 //# sourceMappingURL=responses.d.ts.map

package/dist/endpoints/responses.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"responses.d.ts","sourceRoot":"","sources":["../../src/endpoints/responses.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAc,WAAW,EAAc,aAAa,EAAwB,MAAM,gBAAgB,CAAC;AAM/G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAStD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAEzE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAcpD,OAAO,KAAK,EAMV,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAUrB;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,aAAa,CAAC;AAU5C;;;;;;;;;GASG;AACH,MAAM,MAAM,kBAAkB,GAAG,KAAK,GAAG,aAAa,GAAG,OAAO,GAAG,YAAY,CAAC;AAmChF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iCAAiC,IAAI,MAAM,CAM1D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,0BAA0B,IAAI,MAAM,CAMnD;AA0BD,wBAAgB,eAAe,IAAI,MAAM,CAExC;AAED,wBAAgB,2BAA2B,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAE5D;AAs9BD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uCAAuC,CACrD,QAAQ,EAAE,WAAW,EAAE,EACvB,UAAU,GAAE,QAAQ,GAAG,WAAsB,GAC5C,MAAM,GAAG,IAAI,CAiHf;AA8WD,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,cAAc,EACnB,IAAI,EAAE,mBAAmB,EACzB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,OAAO,CAAC,EAAE,eAAe,EACzB,oBAAoB,CAAC,EAAE,MAAM,EAC7B,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,EAChC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAugEf"}
+{"version":3,"file":"responses.d.ts","sourceRoot":"","sources":["../../src/endpoints/responses.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAc,WAAW,EAAc,aAAa,EAAwB,MAAM,gBAAgB,CAAC;AAM/G,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAStD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAEzE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAcpD,OAAO,KAAK,EAMV,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAUrB;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,aAAa,CAAC;AAU5C;;;;;;;;;GASG;AACH,MAAM,MAAM,kBAAkB,GAAG,KAAK,GAAG,aAAa,GAAG,OAAO,GAAG,YAAY,CAAC;AAmChF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iCAAiC,IAAI,MAAM,CAM1D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,0BAA0B,IAAI,MAAM,CAMnD;AA0BD,wBAAgB,eAAe,IAAI,MAAM,CAExC;AAED,wBAAgB,2BAA2B,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAE5D;AAs9BD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uCAAuC,CACrD,QAAQ,EAAE,WAAW,EAAE,EACvB,UAAU,GAAE,QAAQ,GAAG,WAAsB,GAC5C,MAAM,GAAG,IAAI,CAiHf;AA+WD,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,cAAc,EACnB,IAAI,EAAE,mBAAmB,EACzB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,OAAO,CAAC,EAAE,eAAe,EACzB,oBAAoB,CAAC,EAAE,MAAM,EAC7B,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,EAChC,oBAAoB,CAAC,EAAE,oBAAoB;AAC3C,+EAA+E;AAC/E,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAC7C,OAAO,CAAC,IAAI,CAAC,CAsiEf"}

package/dist/endpoints/responses.js

@@ -1175,10 +1175,11 @@ async function runSessionNonStreaming(session, messages, newInputMessages, confi
     }
     // Hot path — session's KV cache is already warmed for this chain.
     // Single-message continuations whose role is `user` or `tool` take
-    // the cheap delta paths (`send` / `sendToolResult`). Any other single
+    // the session paths (`send` / `sendToolResult`), which render the full
+    // transcript and reuse KV on an exact token-prefix match. Any other single
     // role (`assistant`, `system`) is still accepted by `mapRequest` —
     // `reconstructMessagesFromChain` + `primeHistory` tolerate a tail of
-    // either — but the chat-session delta API has no entry point for
+    // either — but the high-level chat-session API has no entry point for
     // them, so fall through to reset + cold re-prime against the fully
     // rebuilt history. Returning 500 here would regress the pre-session-
     // API full-history path, making valid continuation payloads fail
@@ -1412,7 +1413,9 @@ function readStoredModelIdentity(record) {
 // ---------------------------------------------------------------------------
 // Public handler
 // ---------------------------------------------------------------------------
-export async function handleCreateResponse(res, body, registry, store, httpReq, responseRetentionSec, idleSweeper, modelWorkCoordinator) {
+export async function handleCreateResponse(res, body, registry, store, httpReq, responseRetentionSec, idleSweeper, modelWorkCoordinator, 
+/** Lazy-load hook. See the call site below and `ServerConfig.resolveModel`. */
+resolveModel) {
     const handlerStartedAt = Date.now();
     // Validate required fields
     if (body == null || typeof body !== 'object') {
@@ -1472,6 +1475,36 @@ export async function handleCreateResponse(res, body, registry, store, httpReq,
         }
     }
     const effectiveRetentionSec = requestedRetentionSec ?? responseRetentionSec;
+    // Lazy load, exactly as the Anthropic endpoints do. Nothing is resident at
+    // boot — `createInferenceHost` only discovers — so without this the very
+    // first `/v1/responses` 404s against a `/v1/models` list that advertises the
+    // model, and a client id that exists only as an alias 404s forever.
+    //
+    // Placement is pinned on both sides. AFTER the pure validation above, so a
+    // 400 cannot burn a 30 s load or evict the resident model. BEFORE
+    // `registry.get` below, and therefore before the dispatch lease, which needs
+    // a registered name.
+    if (resolveModel) {
+        // Errors serialize through the OpenAI envelope here. Letting them reach
+        // the outer `createHandler` catch would be right for this endpoint by
+        // accident and wrong for the Anthropic one — `messages.ts` has the mirror
+        // of this note for the opposite reason.
+        try {
+            // Suspension OUTSIDE the writer lock, per `load-model.ts`: a load that
+            // parks in `acquireWrite()` must already be covered, or the drain timer
+            // armed by the previous request fires mid-materialization. `messages.ts`
+            // nests these the other way round, which is safe only because
+            // `withSuspendedDrains` is a counter rather than a mutex.
+            const load = () => modelWorkCoordinator
+                ? modelWorkCoordinator.withModelLoad(() => resolveModel(body.model), 'responses')
+                : resolveModel(body.model);
+            await (idleSweeper ? idleSweeper.withSuspendedDrains(load) : load());
+        }
+        catch (err) {
+            sendInternalError(res, err instanceof Error ? err.message : 'Failed to resolve model');
+            return;
+        }
+    }
     // Look up model
     const model = registry.get(body.model);
     if (!model) {
@@ -2126,7 +2159,7 @@ export async function handleCreateResponse(res, body, registry, store, httpReq,
                 // removed on hit so overlapping requests against the same prior id
                 // cannot race on the same single-flight ChatSession).
                 //
-                // Hot-path eligibility gate: the chat-session delta API only
+                // Hot-path eligibility gate: the high-level chat-session API only
                 // serves a SINGLE `user` or `tool` continuation message — the
                 // `send` / `sendToolResult` entry points cover exactly that
                 // shape. A single `assistant` / `system` continuation cannot
@@ -2135,7 +2168,7 @@ export async function handleCreateResponse(res, body, registry, store, httpReq,
                 // still VALID — it just routes through `runSession*`'s
                 // `session.turns === 0` fall-through (`primeHistory` +
                 // `startFromHistory*`) instead of the `send` / `sendToolResult`
-                // delta path. Crucially, a tier-1 HIT on this branch is still
+                // session continuation path. Crucially, a tier-1 HIT on this branch is still
                 // useful: `resetPreservingNativeCacheForWarmReuse(session)` keeps
                 // the warm native KV cache, and the subsequent
                 // `chat_session_start_sync` -> `verify_cache_prefix_direct`

package/dist/handler.d.ts

@@ -1,6 +1,7 @@
 /** Composable `(req, res)` handler for node:http — usable standalone or mounted into an existing server. */
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import type { ResponseStore } from '@mlx-node/core';
+import { type ServerHealth } from './health.js';
 import type { IdleSweeper } from './idle-sweeper.js';
 import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import type { ModelRegistry } from './registry.js';
@@ -17,7 +18,14 @@ export interface PublicModelEntry {
     owned_by: string;
 }
 export interface HandlerOptions {
-    /** Enable CORS headers (default: true). */
+    /**
+     * Enable CORS headers.
+     *
+     * Default: `true` when no `authToken` is set (historical behaviour), and
+     * `false` once one is — `Access-Control-Allow-Origin: *` on a
+     * token-protected server would invite any web page to spend a leaked token
+     * from the user's browser. An explicit value always wins.
+     */
     cors?: boolean;
     /** Response store for previous_response_id support. */
     store?: ResponseStore | null;
@@ -55,6 +63,24 @@ export interface HandlerOptions {
      * returns this list instead of `registry.list()`.
      */
     listModels?: () => PublicModelEntry[];
+    /**
+     * Shared secret required on every route except `/health` and `/v1/health`.
+     *
+     * `undefined` (the default) disables the gate entirely and is byte-for-byte
+     * identical to the pre-auth behaviour: no header is inspected, no
+     * `WWW-Authenticate` is emitted, and CORS keeps its `true` default.
+     *
+     * Accepted as `x-api-key: <token>` (checked first — Anthropic clients send
+     * it) or `authorization: Bearer <token>` (scheme case-insensitive).
+     */
+    authToken?: string;
+    /**
+     * Builds the `/health` body. Supplied by `createServer` so the HTTP
+     * endpoint and `ServerInstance.health()` share one uptime origin. When
+     * omitted, `createHandler` builds its own reporter from `registry`,
+     * `idleSweeper` and `modelWorkCoordinator`.
+     */
+    health?: () => ServerHealth;
 }
 export declare function createHandler(registry: ModelRegistry, options?: HandlerOptions): (req: IncomingMessage, res: ServerResponse) => Promise<void>;
 //# sourceMappingURL=handler.d.ts.map

package/dist/handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA,4GAA4G;AAE5G,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,KAAK,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CA8C9D"}
+{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA,4GAA4G;AAE5G,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAoBnD;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,KAAK,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;IACtC;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,YAAY,CAAC;CAC7B;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAgF9D"}

package/dist/handler.js

@@ -1,31 +1,80 @@
 /** Composable `(req, res)` handler for node:http — usable standalone or mounted into an existing server. */
+import { hasCredential, isAuthorized, sendUnauthorized } from './auth.js';
 import { sendInternalError } from './errors.js';
+import { createHealthReporter } from './health.js';
 import { ModelWorkCoordinator } from './model-work-coordinator.js';
-import { routeRequest } from './router.js';
+import { requestPathname, routeRequest } from './router.js';
+/**
+ * Routes reachable WITHOUT a token when one is configured.
+ *
+ * A supervisor has to poll liveness before it can be handed a token (it may
+ * be the thing that generates it). Everything served here is scrubbed to
+ * `{ status, uptimeMs, pid }` by the router when the caller is
+ * unauthenticated — see `toMinimalHealth`.
+ *
+ * `/` is included because it is a pure liveness stub: the router answers it
+ * with a constant `{ service: 'mlx-node' }` and reads no state, so there is
+ * nothing to leak. Claude Code issues `HEAD /` before its first request — it
+ * does send `x-api-key`, so it would pass the check anyway, but gating a
+ * contentless probe would 401 every other client's liveness check for no
+ * security gain.
+ */
+const UNAUTHENTICATED_PATHS = new Set(['/', '/health', '/v1/health']);
 export function createHandler(registry, options) {
-    const cors = options?.cors ?? true;
+    const authToken = options?.authToken;
+    // CORS defaults follow the auth posture; an explicit value still wins.
+    const cors = options?.cors ?? authToken === undefined;
     const store = options?.store ?? null;
     const responseRetentionSec = options?.responseRetentionSec;
     const idleSweeper = options?.idleSweeper ?? null;
     const resolveModel = options?.resolveModel;
     const modelWorkCoordinator = options?.modelWorkCoordinator ?? (resolveModel ? new ModelWorkCoordinator() : undefined);
     const listModels = options?.listModels;
+    const health = options?.health ?? createHealthReporter({ registry, idleSweeper, modelWorkCoordinator: modelWorkCoordinator });
     return async (req, res) => {
-        if (cors) {
-            res.setHeader('Access-Control-Allow-Origin', '*');
-            res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
-            res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, anthropic-version');
-            if (req.method === 'OPTIONS') {
-                res.writeHead(204);
-                res.end();
-                return;
-            }
-        }
-        // Returning the promise lets tests await the full lifecycle including
-        // post-`res.end()` bookkeeping (e.g. `SessionRegistry.adopt`). `http.createServer`
-        // ignores the return value, so this is transparent to production callers.
+        // The guard spans the WHOLE listener, not just `routeRequest`. `http.createServer`
+        // discards the returned promise, so anything that escapes here is an unhandled
+        // rejection — and Node's default `--unhandled-rejections=throw` turns that into
+        // process death. A crash-by-request is the one failure a request handler must
+        // not have, so nothing before the routing call gets to be outside the try either.
         try {
-            await routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator);
+            if (cors) {
+                res.setHeader('Access-Control-Allow-Origin', '*');
+                res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+                res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, anthropic-version');
+                if (req.method === 'OPTIONS') {
+                    res.writeHead(204);
+                    res.end();
+                    return;
+                }
+            }
+            // Single auth choke point. `routeRequest` is called from exactly one
+            // place (below), so no route can be added that bypasses this.
+            //
+            // `authToken === undefined` short-circuits before any header is touched:
+            // an unprotected server behaves exactly as it did before auth existed.
+            let authenticated = true;
+            if (authToken !== undefined) {
+                authenticated = isAuthorized(req, authToken);
+                if (!authenticated) {
+                    // Host-independent: see `requestPathname`. Building the base from the
+                    // `Host` header threw on `Host: [`, from a branch only an
+                    // UNAUTHENTICATED request reaches — so enabling auth was what made the
+                    // server killable by anyone who could open the socket.
+                    const path = requestPathname(req);
+                    // `/health` degrades to a liveness-only body instead of 401 — but
+                    // ONLY when no credential was offered. A caller who presented a
+                    // WRONG token gets the 401 it needs to notice the typo.
+                    if (!UNAUTHENTICATED_PATHS.has(path) || hasCredential(req)) {
+                        sendUnauthorized(res);
+                        return;
+                    }
+                }
+            }
+            // Returning the promise lets tests await the full lifecycle including
+            // post-`res.end()` bookkeeping (e.g. `SessionRegistry.adopt`). `http.createServer`
+            // ignores the return value, so this is transparent to production callers.
+            await routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator, { health, authenticated });
         }
         catch (err) {
             const message = err instanceof Error ? err.message : 'Internal server error';

package/dist/health.d.ts

@@ -0,0 +1,146 @@
+/**
+ * Server readiness reporting for supervisors (e.g. an Electron app running
+ * the inference server as a child process).
+ *
+ * The old `/health` returned a constant `{ status: 'ok' }`, which cannot
+ * distinguish four states a supervisor genuinely needs to tell apart:
+ *
+ *   - up, nothing loaded          → keep waiting, this is normal
+ *   - up, model resident          → route traffic
+ *   - wedged mid-load             → keep waiting, do NOT restart
+ *   - wedged behind a full queue  → shed load / warn the user
+ *
+ * Every input already exists in-process; this module only exposes it. The
+ * status ladder itself is a PURE function ({@link deriveHealthStatus}) over a
+ * plain record so it can be unit-tested without a server, a registry, or the
+ * native addon.
+ *
+ * IMPORTANT: nothing here may call into `@mlx-node/core`. `/health` is
+ * deliberately excluded from the idle sweeper's `beginRequest`/`endRequest`
+ * bracket (see `HandlerOptions.idleSweeper`), so a native call from this path
+ * would run outside the in-flight accounting the drain timer relies on.
+ * Every field below is read from plain JavaScript state.
+ */
+import type { IdleSweeper } from './idle-sweeper.js';
+import type { ModelWorkCoordinator } from './model-work-coordinator.js';
+import type { ModelRegistry } from './registry.js';
+/**
+ * Coarse readiness classification. Ordered by precedence in
+ * {@link deriveHealthStatus} — a higher rung wins even when a lower rung's
+ * condition also holds.
+ */
+export type ServerHealthStatus = 'ok' | 'loading' | 'degraded' | 'error';
+/**
+ * Outcome of the most recent load bracket, recorded by
+ * {@link ModelWorkCoordinator} in the `finally` of `withModelLoad` /
+ * `withModelLoadInstrumented`.
+ *
+ * Before this existed, a `resolveModel` failure became an HTTP 500 and was
+ * dropped — a supervisor polling after the fact had no way to learn WHY the
+ * server had no resident model.
+ *
+ * Caveat worth knowing when reading this field: the coordinator brackets
+ * EVERY resolve attempt, including the no-op fast path taken when the
+ * requested model is already resident. A successful no-op therefore
+ * overwrites an earlier failure record. That is why the `'error'` rung is
+ * additionally gated on "no resident models" — a server that is answering
+ * requests is not in an error state regardless of what the last bracket did.
+ */
+export interface ModelLoadRecord {
+    /** Caller-supplied label, normally the model name. `null` when unlabelled. */
+    label: string | null;
+    /** `Date.now()` at writer-lock acquisition (when the load actually began). */
+    startedAt: number;
+    /** `Date.now()` when the bracket settled, success or failure. */
+    finishedAt: number;
+    ok: boolean;
+    /** Message of the thrown error, or `null` on success. */
+    error: string | null;
+}
+/** Inputs to the pure status ladder. Deliberately primitive so tests can fixture them. */
+export interface HealthStatusInputs {
+    /** True while a load holds the coordinator's exclusive writer slot. */
+    writerActive: boolean;
+    /** Loads parked waiting for the writer slot. */
+    waitingWriters: number;
+    /** Inference requests currently bracketed by the idle sweeper. */
+    inFlight: number;
+    /** Distinct model names currently registered. */
+    residentModelCount: number;
+    /** True when at least one per-model queue is holding its configured max waiters. */
+    queueSaturated: boolean;
+    /** Most recent load bracket, or `null` if none has settled. */
+    lastLoad: ModelLoadRecord | null;
+}
+/** Full readiness body. `{ status: 'ok' }` remains a strict subset of this. */
+export interface ServerHealth {
+    status: ServerHealthStatus;
+    /** Milliseconds since the reporter was created (≈ server start). */
+    uptimeMs: number;
+    pid: number;
+    models: {
+        /** Registered names, including aliases. */
+        resident: string[];
+        count: number;
+    };
+    work: {
+        inFlight: number;
+        /** True while the idle sweeper has a `clearCache()` drain armed. */
+        drainPending: boolean;
+        writerActive: boolean;
+        waitingWriters: number;
+    };
+    queue: {
+        /** Deepest per-model waiter count across every session registry. */
+        depth: number;
+        /** Configured per-model waiter cap, or `null` when unbounded. */
+        limit: number | null;
+        saturated: boolean;
+    };
+    lastLoad: ModelLoadRecord | null;
+}
+/**
+ * The subset served to an UNAUTHENTICATED `/health` poll on a
+ * token-protected server. A supervisor must be able to poll before it holds a
+ * token, but `models.resident` is user data — model names routinely leak
+ * project names and local paths.
+ */
+export interface ServerHealthMinimal {
+    status: ServerHealthStatus;
+    uptimeMs: number;
+    pid: number;
+}
+/**
+ * Pure status ladder. No I/O, no clock, no native calls — just the four
+ * documented rungs, in precedence order:
+ *
+ *   1. `writerActive`                        → 'loading'
+ *   2. last load failed AND nothing resident → 'error'
+ *   3. queue saturated, OR a load is parked
+ *      behind live inference                 → 'degraded'
+ *   4. otherwise                             → 'ok'
+ *
+ * `'loading'` outranks `'error'` on purpose: a retry that already holds the
+ * writer slot means the supervisor should wait, not restart the process.
+ */
+export declare function deriveHealthStatus(input: HealthStatusInputs): ServerHealthStatus;
+/** Project the full body down to the three fields safe to serve without a token. */
+export declare function toMinimalHealth(health: ServerHealth): ServerHealthMinimal;
+export interface HealthReporterDeps {
+    registry: ModelRegistry;
+    /** Optional: supplies `inFlight` / `drainPending`. Absent ⇒ both read as idle. */
+    idleSweeper?: IdleSweeper | null;
+    /** Optional: supplies writer state + `lastLoad`. Absent ⇒ no load has ever run. */
+    modelWorkCoordinator?: ModelWorkCoordinator | null;
+    /** Defaults to `Date.now()` at construction. */
+    startedAt?: number;
+    /** Injectable clock for tests. */
+    now?: () => number;
+}
+/**
+ * Build a zero-argument reporter closing over the live server objects. Each
+ * call re-reads current state; nothing is cached, so a supervisor polling on
+ * an interval always sees the present moment.
+ */
+export declare function createHealthReporter(deps: HealthReporterDeps): () => ServerHealth;
+//# sourceMappingURL=health.d.ts.map

package/dist/health.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../src/health.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,IAAI,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC;AAEzE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,eAAe;IAC9B,8EAA8E;IAC9E,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,yDAAyD;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,0FAA0F;AAC1F,MAAM,WAAW,kBAAkB;IACjC,uEAAuE;IACvE,YAAY,EAAE,OAAO,CAAC;IACtB,gDAAgD;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oFAAoF;IACpF,cAAc,EAAE,OAAO,CAAC;IACxB,+DAA+D;IAC/D,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAClC;AAED,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,kBAAkB,CAAC;IAC3B,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE;QACN,2CAA2C;QAC3C,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,IAAI,EAAE;QACJ,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,YAAY,EAAE,OAAO,CAAC;QACtB,YAAY,EAAE,OAAO,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,KAAK,EAAE;QACL,oEAAoE;QACpE,KAAK,EAAE,MAAM,CAAC;QACd,iEAAiE;QACjE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC;IACF,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAClC;AAED;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,kBAAkB,GAAG,kBAAkB,CAShF;AAED,oFAAoF;AACpF,wBAAgB,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,mBAAmB,CAEzE;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,aAAa,CAAC;IACxB,kFAAkF;IAClF,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC,mFAAmF;IACnF,oBAAoB,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACnD,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,YAAY,CAiDjF"}