@mlx-node/server 0.0.7 → 0.0.9

package/dist/errors.d.ts

@@ -32,6 +32,7 @@ export declare function sendAnthropicError(res: ServerResponse, status: number,
 export declare function sendAnthropicBadRequest(res: ServerResponse, message: string): void;
 export declare function sendAnthropicNotFound(res: ServerResponse, message: string): void;
 export declare function sendAnthropicInternalError(res: ServerResponse, message: string): void;
+export declare function sendAnthropicNotImplemented(res: ServerResponse, message: string): void;
 export declare function sendAnthropicMethodNotAllowed(res: ServerResponse, allowed: string): void;
 /**
  * 429 Anthropic-shape rate-limit response. Mirror of {@link sendRateLimit}

package/dist/errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,wBAAgB,SAAS,CACvB,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GACpB,IAAI,CAWN;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAEzF;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEvE;AAED,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAO/E;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE5E;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE7E;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAYxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAG3G;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAElF;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEhF;AAED,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAErF;AAED,wBAAgB,6BAA6B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGjF"}
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,wBAAgB,SAAS,CACvB,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GACpB,IAAI,CAWN;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAEzF;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEvE;AAED,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAO/E;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE5E;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE7E;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAYxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAG3G;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAElF;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEhF;AAED,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAErF;AAED,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEtF;AAED,wBAAgB,6BAA6B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGjF"}

package/dist/errors.js

@@ -68,6 +68,9 @@ export function sendAnthropicNotFound(res, message) {
 export function sendAnthropicInternalError(res, message) {
     sendAnthropicError(res, 500, 'api_error', message);
 }
+export function sendAnthropicNotImplemented(res, message) {
+    sendAnthropicError(res, 501, 'not_supported_error', message);
+}
 export function sendAnthropicMethodNotAllowed(res, allowed) {
     res.writeHead(405, { Allow: allowed, 'Content-Type': 'application/json' });
     res.end(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: 'Method not allowed' } }));

package/dist/handler.d.ts

@@ -1,9 +1,31 @@
 /** Composable `(req, res)` handler for node:http — usable standalone or mounted into an existing server. */
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import type { ResponseStore } from '@mlx-node/core';
+import { type ServerHealth } from './health.js';
+import type { IdleSweeper } from './idle-sweeper.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import type { ModelRegistry } from './registry.js';
+/**
+ * Entry in the list returned by `GET /v1/models`. Matches the shape
+ * `ModelRegistry.list()` produces — exported so callers that supply a
+ * custom `listModels` callback (e.g. `mlx launch claude` discovering
+ * every model on disk) can build entries without importing internals.
+ */
+export interface PublicModelEntry {
+    id: string;
+    object: 'model';
+    created: number;
+    owned_by: string;
+}
 export interface HandlerOptions {
-    /** Enable CORS headers (default: true). */
+    /**
+     * Enable CORS headers.
+     *
+     * Default: `true` when no `authToken` is set (historical behaviour), and
+     * `false` once one is — `Access-Control-Allow-Origin: *` on a
+     * token-protected server would invite any web page to spend a leaked token
+     * from the user's browser. An explicit value always wins.
+     */
     cors?: boolean;
     /** Response store for previous_response_id support. */
     store?: ResponseStore | null;
@@ -13,6 +35,52 @@ export interface HandlerOptions {
      * and `ServerConfig.responseRetentionSec`).
      */
     responseRetentionSec?: number;
+    /**
+     * Optional idle sweeper. Forwarded to `routeRequest` so that the
+     * inference endpoints (`/v1/responses` + `/v1/messages`) can bracket
+     * their native-model dispatch with `beginRequest()` / `endRequest()`.
+     *
+     * Note: the begin/end hooks are DELIBERATELY scoped to the inference
+     * endpoints — wrapping every HTTP call (OPTIONS preflights,
+     * `/v1/models`, `/v1/health`, 404s) would let purely observational
+     * traffic keep the allocator pinned forever.
+     */
+    idleSweeper?: IdleSweeper | null;
+    /**
+     * Optional async callback invoked by `/v1/messages` before it looks
+     * the model up in the registry. The callback should register the
+     * model on demand; on return, the endpoint does `registry.get(name)`
+     * and 404s if still unresolved.
+     */
+    resolveModel?: (name: string) => Promise<void>;
+    /**
+     * Coordinates process-wide MLX work so lazy model loads / warmups do not
+     * overlap live inference on another model.
+     */
+    modelWorkCoordinator?: ModelWorkCoordinator;
+    /**
+     * Optional override for `GET /v1/models`. When provided, that endpoint
+     * returns this list instead of `registry.list()`.
+     */
+    listModels?: () => PublicModelEntry[];
+    /**
+     * Shared secret required on every route except `/health` and `/v1/health`.
+     *
+     * `undefined` (the default) disables the gate entirely and is byte-for-byte
+     * identical to the pre-auth behaviour: no header is inspected, no
+     * `WWW-Authenticate` is emitted, and CORS keeps its `true` default.
+     *
+     * Accepted as `x-api-key: <token>` (checked first — Anthropic clients send
+     * it) or `authorization: Bearer <token>` (scheme case-insensitive).
+     */
+    authToken?: string;
+    /**
+     * Builds the `/health` body. Supplied by `createServer` so the HTTP
+     * endpoint and `ServerInstance.health()` share one uptime origin. When
+     * omitted, `createHandler` builds its own reporter from `registry`,
+     * `idleSweeper` and `modelWorkCoordinator`.
+     */
+    health?: () => ServerHealth;
 }
 export declare function createHandler(registry: ModelRegistry, options?: HandlerOptions): (req: IncomingMessage, res: ServerResponse) => Promise<void>;
 //# sourceMappingURL=handler.d.ts.map

package/dist/handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA,4GAA4G;AAE5G,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,MAAM,WAAW,cAAc;IAC7B,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,KAAK,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAgC9D"}
+{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA,4GAA4G;AAE5G,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAoBnD;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,KAAK,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;IACtC;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,YAAY,CAAC;CAC7B;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAgF9D"}

package/dist/handler.js

@@ -1,26 +1,80 @@
 /** Composable `(req, res)` handler for node:http — usable standalone or mounted into an existing server. */
+import { hasCredential, isAuthorized, sendUnauthorized } from './auth.js';
 import { sendInternalError } from './errors.js';
-import { routeRequest } from './router.js';
+import { createHealthReporter } from './health.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
+import { requestPathname, routeRequest } from './router.js';
+/**
+ * Routes reachable WITHOUT a token when one is configured.
+ *
+ * A supervisor has to poll liveness before it can be handed a token (it may
+ * be the thing that generates it). Everything served here is scrubbed to
+ * `{ status, uptimeMs, pid }` by the router when the caller is
+ * unauthenticated — see `toMinimalHealth`.
+ *
+ * `/` is included because it is a pure liveness stub: the router answers it
+ * with a constant `{ service: 'mlx-node' }` and reads no state, so there is
+ * nothing to leak. Claude Code issues `HEAD /` before its first request — it
+ * does send `x-api-key`, so it would pass the check anyway, but gating a
+ * contentless probe would 401 every other client's liveness check for no
+ * security gain.
+ */
+const UNAUTHENTICATED_PATHS = new Set(['/', '/health', '/v1/health']);
 export function createHandler(registry, options) {
-    const cors = options?.cors ?? true;
+    const authToken = options?.authToken;
+    // CORS defaults follow the auth posture; an explicit value still wins.
+    const cors = options?.cors ?? authToken === undefined;
     const store = options?.store ?? null;
     const responseRetentionSec = options?.responseRetentionSec;
+    const idleSweeper = options?.idleSweeper ?? null;
+    const resolveModel = options?.resolveModel;
+    const modelWorkCoordinator = options?.modelWorkCoordinator ?? (resolveModel ? new ModelWorkCoordinator() : undefined);
+    const listModels = options?.listModels;
+    const health = options?.health ?? createHealthReporter({ registry, idleSweeper, modelWorkCoordinator: modelWorkCoordinator });
     return async (req, res) => {
-        if (cors) {
-            res.setHeader('Access-Control-Allow-Origin', '*');
-            res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
-            res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, anthropic-version');
-            if (req.method === 'OPTIONS') {
-                res.writeHead(204);
-                res.end();
-                return;
-            }
-        }
-        // Returning the promise lets tests await the full lifecycle including
-        // post-`res.end()` bookkeeping (e.g. `SessionRegistry.adopt`). `http.createServer`
-        // ignores the return value, so this is transparent to production callers.
+        // The guard spans the WHOLE listener, not just `routeRequest`. `http.createServer`
+        // discards the returned promise, so anything that escapes here is an unhandled
+        // rejection — and Node's default `--unhandled-rejections=throw` turns that into
+        // process death. A crash-by-request is the one failure a request handler must
+        // not have, so nothing before the routing call gets to be outside the try either.
         try {
-            await routeRequest(req, res, registry, store, responseRetentionSec);
+            if (cors) {
+                res.setHeader('Access-Control-Allow-Origin', '*');
+                res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+                res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, anthropic-version');
+                if (req.method === 'OPTIONS') {
+                    res.writeHead(204);
+                    res.end();
+                    return;
+                }
+            }
+            // Single auth choke point. `routeRequest` is called from exactly one
+            // place (below), so no route can be added that bypasses this.
+            //
+            // `authToken === undefined` short-circuits before any header is touched:
+            // an unprotected server behaves exactly as it did before auth existed.
+            let authenticated = true;
+            if (authToken !== undefined) {
+                authenticated = isAuthorized(req, authToken);
+                if (!authenticated) {
+                    // Host-independent: see `requestPathname`. Building the base from the
+                    // `Host` header threw on `Host: [`, from a branch only an
+                    // UNAUTHENTICATED request reaches — so enabling auth was what made the
+                    // server killable by anyone who could open the socket.
+                    const path = requestPathname(req);
+                    // `/health` degrades to a liveness-only body instead of 401 — but
+                    // ONLY when no credential was offered. A caller who presented a
+                    // WRONG token gets the 401 it needs to notice the typo.
+                    if (!UNAUTHENTICATED_PATHS.has(path) || hasCredential(req)) {
+                        sendUnauthorized(res);
+                        return;
+                    }
+                }
+            }
+            // Returning the promise lets tests await the full lifecycle including
+            // post-`res.end()` bookkeeping (e.g. `SessionRegistry.adopt`). `http.createServer`
+            // ignores the return value, so this is transparent to production callers.
+            await routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator, { health, authenticated });
         }
         catch (err) {
             const message = err instanceof Error ? err.message : 'Internal server error';

package/dist/health.d.ts

@@ -0,0 +1,146 @@
+/**
+ * Server readiness reporting for supervisors (e.g. an Electron app running
+ * the inference server as a child process).
+ *
+ * The old `/health` returned a constant `{ status: 'ok' }`, which cannot
+ * distinguish four states a supervisor genuinely needs to tell apart:
+ *
+ *   - up, nothing loaded          → keep waiting, this is normal
+ *   - up, model resident          → route traffic
+ *   - wedged mid-load             → keep waiting, do NOT restart
+ *   - wedged behind a full queue  → shed load / warn the user
+ *
+ * Every input already exists in-process; this module only exposes it. The
+ * status ladder itself is a PURE function ({@link deriveHealthStatus}) over a
+ * plain record so it can be unit-tested without a server, a registry, or the
+ * native addon.
+ *
+ * IMPORTANT: nothing here may call into `@mlx-node/core`. `/health` is
+ * deliberately excluded from the idle sweeper's `beginRequest`/`endRequest`
+ * bracket (see `HandlerOptions.idleSweeper`), so a native call from this path
+ * would run outside the in-flight accounting the drain timer relies on.
+ * Every field below is read from plain JavaScript state.
+ */
+import type { IdleSweeper } from './idle-sweeper.js';
+import type { ModelWorkCoordinator } from './model-work-coordinator.js';
+import type { ModelRegistry } from './registry.js';
+/**
+ * Coarse readiness classification. Ordered by precedence in
+ * {@link deriveHealthStatus} — a higher rung wins even when a lower rung's
+ * condition also holds.
+ */
+export type ServerHealthStatus = 'ok' | 'loading' | 'degraded' | 'error';
+/**
+ * Outcome of the most recent load bracket, recorded by
+ * {@link ModelWorkCoordinator} in the `finally` of `withModelLoad` /
+ * `withModelLoadInstrumented`.
+ *
+ * Before this existed, a `resolveModel` failure became an HTTP 500 and was
+ * dropped — a supervisor polling after the fact had no way to learn WHY the
+ * server had no resident model.
+ *
+ * Caveat worth knowing when reading this field: the coordinator brackets
+ * EVERY resolve attempt, including the no-op fast path taken when the
+ * requested model is already resident. A successful no-op therefore
+ * overwrites an earlier failure record. That is why the `'error'` rung is
+ * additionally gated on "no resident models" — a server that is answering
+ * requests is not in an error state regardless of what the last bracket did.
+ */
+export interface ModelLoadRecord {
+    /** Caller-supplied label, normally the model name. `null` when unlabelled. */
+    label: string | null;
+    /** `Date.now()` at writer-lock acquisition (when the load actually began). */
+    startedAt: number;
+    /** `Date.now()` when the bracket settled, success or failure. */
+    finishedAt: number;
+    ok: boolean;
+    /** Message of the thrown error, or `null` on success. */
+    error: string | null;
+}
+/** Inputs to the pure status ladder. Deliberately primitive so tests can fixture them. */
+export interface HealthStatusInputs {
+    /** True while a load holds the coordinator's exclusive writer slot. */
+    writerActive: boolean;
+    /** Loads parked waiting for the writer slot. */
+    waitingWriters: number;
+    /** Inference requests currently bracketed by the idle sweeper. */
+    inFlight: number;
+    /** Distinct model names currently registered. */
+    residentModelCount: number;
+    /** True when at least one per-model queue is holding its configured max waiters. */
+    queueSaturated: boolean;
+    /** Most recent load bracket, or `null` if none has settled. */
+    lastLoad: ModelLoadRecord | null;
+}
+/** Full readiness body. `{ status: 'ok' }` remains a strict subset of this. */
+export interface ServerHealth {
+    status: ServerHealthStatus;
+    /** Milliseconds since the reporter was created (≈ server start). */
+    uptimeMs: number;
+    pid: number;
+    models: {
+        /** Registered names, including aliases. */
+        resident: string[];
+        count: number;
+    };
+    work: {
+        inFlight: number;
+        /** True while the idle sweeper has a `clearCache()` drain armed. */
+        drainPending: boolean;
+        writerActive: boolean;
+        waitingWriters: number;
+    };
+    queue: {
+        /** Deepest per-model waiter count across every session registry. */
+        depth: number;
+        /** Configured per-model waiter cap, or `null` when unbounded. */
+        limit: number | null;
+        saturated: boolean;
+    };
+    lastLoad: ModelLoadRecord | null;
+}
+/**
+ * The subset served to an UNAUTHENTICATED `/health` poll on a
+ * token-protected server. A supervisor must be able to poll before it holds a
+ * token, but `models.resident` is user data — model names routinely leak
+ * project names and local paths.
+ */
+export interface ServerHealthMinimal {
+    status: ServerHealthStatus;
+    uptimeMs: number;
+    pid: number;
+}
+/**
+ * Pure status ladder. No I/O, no clock, no native calls — just the four
+ * documented rungs, in precedence order:
+ *
+ *   1. `writerActive`                        → 'loading'
+ *   2. last load failed AND nothing resident → 'error'
+ *   3. queue saturated, OR a load is parked
+ *      behind live inference                 → 'degraded'
+ *   4. otherwise                             → 'ok'
+ *
+ * `'loading'` outranks `'error'` on purpose: a retry that already holds the
+ * writer slot means the supervisor should wait, not restart the process.
+ */
+export declare function deriveHealthStatus(input: HealthStatusInputs): ServerHealthStatus;
+/** Project the full body down to the three fields safe to serve without a token. */
+export declare function toMinimalHealth(health: ServerHealth): ServerHealthMinimal;
+export interface HealthReporterDeps {
+    registry: ModelRegistry;
+    /** Optional: supplies `inFlight` / `drainPending`. Absent ⇒ both read as idle. */
+    idleSweeper?: IdleSweeper | null;
+    /** Optional: supplies writer state + `lastLoad`. Absent ⇒ no load has ever run. */
+    modelWorkCoordinator?: ModelWorkCoordinator | null;
+    /** Defaults to `Date.now()` at construction. */
+    startedAt?: number;
+    /** Injectable clock for tests. */
+    now?: () => number;
+}
+/**
+ * Build a zero-argument reporter closing over the live server objects. Each
+ * call re-reads current state; nothing is cached, so a supervisor polling on
+ * an interval always sees the present moment.
+ */
+export declare function createHealthReporter(deps: HealthReporterDeps): () => ServerHealth;
+//# sourceMappingURL=health.d.ts.map

package/dist/health.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../src/health.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,IAAI,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC;AAEzE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,eAAe;IAC9B,8EAA8E;IAC9E,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,yDAAyD;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,0FAA0F;AAC1F,MAAM,WAAW,kBAAkB;IACjC,uEAAuE;IACvE,YAAY,EAAE,OAAO,CAAC;IACtB,gDAAgD;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oFAAoF;IACpF,cAAc,EAAE,OAAO,CAAC;IACxB,+DAA+D;IAC/D,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAClC;AAED,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,kBAAkB,CAAC;IAC3B,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE;QACN,2CAA2C;QAC3C,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,IAAI,EAAE;QACJ,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,YAAY,EAAE,OAAO,CAAC;QACtB,YAAY,EAAE,OAAO,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,KAAK,EAAE;QACL,oEAAoE;QACpE,KAAK,EAAE,MAAM,CAAC;QACd,iEAAiE;QACjE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC;IACF,QAAQ,EAAE,eAAe,GAAG,IAAI,CAAC;CAClC;AAED;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,kBAAkB,GAAG,kBAAkB,CAShF;AAED,oFAAoF;AACpF,wBAAgB,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,mBAAmB,CAEzE;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,aAAa,CAAC;IACxB,kFAAkF;IAClF,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC,mFAAmF;IACnF,oBAAoB,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACnD,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,YAAY,CAiDjF"}

package/dist/health.js

@@ -0,0 +1,107 @@
+/**
+ * Server readiness reporting for supervisors (e.g. an Electron app running
+ * the inference server as a child process).
+ *
+ * The old `/health` returned a constant `{ status: 'ok' }`, which cannot
+ * distinguish four states a supervisor genuinely needs to tell apart:
+ *
+ *   - up, nothing loaded          → keep waiting, this is normal
+ *   - up, model resident          → route traffic
+ *   - wedged mid-load             → keep waiting, do NOT restart
+ *   - wedged behind a full queue  → shed load / warn the user
+ *
+ * Every input already exists in-process; this module only exposes it. The
+ * status ladder itself is a PURE function ({@link deriveHealthStatus}) over a
+ * plain record so it can be unit-tested without a server, a registry, or the
+ * native addon.
+ *
+ * IMPORTANT: nothing here may call into `@mlx-node/core`. `/health` is
+ * deliberately excluded from the idle sweeper's `beginRequest`/`endRequest`
+ * bracket (see `HandlerOptions.idleSweeper`), so a native call from this path
+ * would run outside the in-flight accounting the drain timer relies on.
+ * Every field below is read from plain JavaScript state.
+ */
+/**
+ * Pure status ladder. No I/O, no clock, no native calls — just the four
+ * documented rungs, in precedence order:
+ *
+ *   1. `writerActive`                        → 'loading'
+ *   2. last load failed AND nothing resident → 'error'
+ *   3. queue saturated, OR a load is parked
+ *      behind live inference                 → 'degraded'
+ *   4. otherwise                             → 'ok'
+ *
+ * `'loading'` outranks `'error'` on purpose: a retry that already holds the
+ * writer slot means the supervisor should wait, not restart the process.
+ */
+export function deriveHealthStatus(input) {
+    if (input.writerActive)
+        return 'loading';
+    if (input.lastLoad?.ok === false && input.residentModelCount === 0)
+        return 'error';
+    if (input.queueSaturated)
+        return 'degraded';
+    // A writer parked while readers are still running means the swap cannot
+    // proceed until they drain, and every request for the incoming model
+    // stalls behind it. Either condition alone is an ordinary transient.
+    if (input.waitingWriters > 0 && input.inFlight > 0)
+        return 'degraded';
+    return 'ok';
+}
+/** Project the full body down to the three fields safe to serve without a token. */
+export function toMinimalHealth(health) {
+    return { status: health.status, uptimeMs: health.uptimeMs, pid: health.pid };
+}
+/**
+ * Build a zero-argument reporter closing over the live server objects. Each
+ * call re-reads current state; nothing is cached, so a supervisor polling on
+ * an interval always sees the present moment.
+ */
+export function createHealthReporter(deps) {
+    const now = deps.now ?? (() => Date.now());
+    const startedAt = deps.startedAt ?? now();
+    return () => {
+        const resident = deps.registry.list().map((entry) => entry.id);
+        // Queue saturation is inherently per-model: one wedged model should
+        // surface even while others are idle. We report the WORST case.
+        let depth = 0;
+        let limit = null;
+        let saturated = false;
+        for (const sessionRegistry of deps.registry.listSessionRegistries()) {
+            const registryDepth = sessionRegistry.queueDepth;
+            if (registryDepth > depth)
+                depth = registryDepth;
+            const registryLimit = sessionRegistry.queueDepthLimit;
+            if (registryLimit !== undefined) {
+                if (limit === null || registryLimit < limit)
+                    limit = registryLimit;
+                if (registryDepth >= registryLimit)
+                    saturated = true;
+            }
+        }
+        const writerActive = deps.modelWorkCoordinator?.writerActive ?? false;
+        const waitingWriters = deps.modelWorkCoordinator?.waitingWriters ?? 0;
+        const lastLoad = deps.modelWorkCoordinator?.lastLoad ?? null;
+        const inFlight = deps.idleSweeper?.inFlight ?? 0;
+        const drainPending = deps.idleSweeper?.isPending ?? false;
+        const status = deriveHealthStatus({
+            writerActive,
+            waitingWriters,
+            inFlight,
+            residentModelCount: resident.length,
+            queueSaturated: saturated,
+            lastLoad,
+        });
+        return {
+            status,
+            // Clamped: a backwards clock step (NTP, fake timers in a sibling test)
+            // must not surface a negative uptime a supervisor might read as a wrap.
+            uptimeMs: Math.max(0, now() - startedAt),
+            pid: process.pid,
+            models: { resident, count: resident.length },
+            work: { inFlight, drainPending, writerActive, waitingWriters },
+            queue: { depth, limit, saturated },
+            lastLoad,
+        };
+    };
+}

package/dist/host/discover.d.ts

@@ -0,0 +1,19 @@
+/** Discover locally-downloaded generative models under a given directory. */
+import { type ModelType } from '@mlx-node/lm';
+import { type LaunchPreset } from '../presets.js';
+/** A locally-downloaded model paired with its sampling preset. */
+export interface DiscoveredModel {
+    name: string;
+    path: string;
+    modelType: ModelType;
+    preset: LaunchPreset;
+}
+/**
+ * Scan `dir` for model subdirectories. Each subdirectory with a recognized
+ * `config.json` is returned with its inferred `ModelType` and `LaunchPreset`.
+ * Non-generative types are silently skipped. Entries with no preset or an
+ * undetectable config are skipped (warnings only emitted when `MLX_DEBUG`
+ * is set). Must stay cheap — do not load weights here.
+ */
+export declare function discoverModels(dir: string): Promise<DiscoveredModel[]>;
+//# sourceMappingURL=discover.d.ts.map

package/dist/host/discover.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../../src/host/discover.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAM7E,OAAO,EAAmB,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAE/D,OAAO,EAAkB,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,kEAAkE;AAClE,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,SAAS,CAAC;IACrB,MAAM,EAAE,YAAY,CAAC;CACtB;AAKD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAoC5E"}

package/dist/host/discover.js

@@ -0,0 +1,50 @@
+/** Discover locally-downloaded generative models under a given directory. */
+import { readdir } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+import { detectModelType } from '@mlx-node/lm';
+import { LAUNCH_PRESETS } from '../presets.js';
+// Non-generative detection results that cannot back a chat endpoint.
+const NON_GENERATIVE = new Set(['harrier', 'qianfan-ocr', 'internvl_chat']);
+/**
+ * Scan `dir` for model subdirectories. Each subdirectory with a recognized
+ * `config.json` is returned with its inferred `ModelType` and `LaunchPreset`.
+ * Non-generative types are silently skipped. Entries with no preset or an
+ * undetectable config are skipped (warnings only emitted when `MLX_DEBUG`
+ * is set). Must stay cheap — do not load weights here.
+ */
+export async function discoverModels(dir) {
+    const debug = Boolean(process.env.MLX_DEBUG);
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory())
+            continue;
+        const full = join(dir, entry.name);
+        let modelType;
+        try {
+            modelType = await detectModelType(full);
+        }
+        catch (err) {
+            if (debug)
+                console.warn(`[mlx] skip ${full}: ${err.message}`);
+            continue;
+        }
+        if (NON_GENERATIVE.has(modelType))
+            continue;
+        const preset = LAUNCH_PRESETS[modelType];
+        if (!preset) {
+            if (debug)
+                console.warn(`[mlx] skip ${full}: no LAUNCH_PRESETS entry for ${modelType}`);
+            continue;
+        }
+        out.push({ name: basename(full), path: full, modelType, preset });
+    }
+    out.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    return out;
+}

package/dist/host/env-policy.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Engine tuning that is LAUNCHER POLICY, not an engine default.
+ *
+ * The native engine reads these knobs from the process environment through a
+ * `OnceLock`: the FIRST read latches the value for the life of the process, so
+ * every variable here must be in place before any model is loaded and must
+ * never be mutated afterwards. That one-shot latch is why the policy is a
+ * plain data object rather than something a caller can toggle at runtime.
+ *
+ * Two application shapes, deliberately different:
+ *
+ *   - In-process host (`mlx serve`, `mlx launch claude`): call
+ *     {@link applyEnginePolicy} to write `process.env` BEFORE the first load.
+ *     A value the user already set in their shell always wins — this is a
+ *     default, not an override.
+ *   - Out-of-process host (Electron `utilityProcess.fork`): pass
+ *     {@link engineEnvFor} into `fork({ env })` and never touch
+ *     `process.env` in the child. The returned map is unconditional: it
+ *     describes the child's starting environment, and the parent is expected
+ *     to spread the user's own env under it if it wants shell values to win.
+ *
+ * Keeping the two apart matters. `MLX_PAGED_PREFILL_CHUNK_SIZE` reads as
+ * `0` ("no chunking") in Rust when unset; 2048 is the value
+ * `mlx launch claude` has historically applied to bound the cold-prefill
+ * memory peak, and baking it into the engine would silently change every
+ * other embedder's behaviour.
+ */
+/** Env var names this module owns. Exported so tests can assert the exact set. */
+export declare const ENGINE_POLICY_ENV_VARS: readonly ['MLX_PAGED_PREFILL_CHUNK_SIZE'];
+export interface EnginePolicy {
+    /**
+     * `MLX_PAGED_PREFILL_CHUNK_SIZE` — tokens per paged-prefill chunk.
+     * `0` disables chunking (the Rust default when the var is unset).
+     */
+    pagedPrefillChunkSize?: number;
+}
+/**
+ * The policy `mlx launch claude` has applied since the flag existed, and the
+ * one `mlx serve` and the desktop sidecar inherit.
+ *
+ * 2048 matches the mlx-lm / mlx-vlm default and reduces per-chunk overhead
+ * versus the older 1024 for long Qwen dense contexts.
+ */
+export declare const LAUNCHER_ENGINE_POLICY: EnginePolicy;
+/**
+ * Render a policy as the environment map a forked child should start with.
+ *
+ * Unconditional by construction: there is no "already set" to respect in a
+ * child that does not exist yet.
+ */
+export declare function engineEnvFor(policy: EnginePolicy): Record<string, string>;
+/**
+ * Apply a policy to an in-process environment as a DEFAULT.
+ *
+ * Only writes vars that are currently unset — matching the historical
+ * `if (process.env.X == null)` guard, so an explicit `X=` (empty string) in
+ * the user's shell is treated as "the user has an opinion" and left alone.
+ *
+ * Returns the names actually written, so a caller can log or assert on them.
+ */
+export declare function applyEnginePolicy(policy: EnginePolicy, env?: NodeJS.ProcessEnv): string[];
+//# sourceMappingURL=env-policy.d.ts.map

package/dist/host/env-policy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"env-policy.d.ts","sourceRoot":"","sources":["../../src/host/env-policy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,kFAAkF;AAClF,eAAO,MAAM,sBAAsB,YAAI,8BAA8B,CAAU,CAAC;AAEhF,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,EAAE,YAA6D,CAAC;AAEnG;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAMzE;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,MAAM,EAAE,CAStG"}