@mlx-node/server 0.0.7 → 0.0.9

package/dist/router.js

@@ -1,10 +1,39 @@
 /** Path-based router for /v1/* endpoints. */
+import { handleCountMessageTokens } from './endpoints/messages-count-tokens.js';
 import { handleCreateMessage } from './endpoints/messages.js';
 import { handleListModels } from './endpoints/models.js';
 import { handleCreateResponse } from './endpoints/responses.js';
 import { sendAnthropicBadRequest, sendAnthropicMethodNotAllowed, sendBadRequest, sendMethodNotAllowed, sendNotFound, } from './errors.js';
+import { toMinimalHealth } from './health.js';
 /** Max request body size (10 MB). */
 const MAX_BODY_BYTES = 10 * 1024 * 1024;
+/**
+ * The request's pathname, parsed against a CONSTANT base.
+ *
+ * Never against `Host`. That header is attacker-controlled text on every
+ * request — `Host: [` makes `new URL()` throw `ERR_INVALID_URL`, and a throw
+ * from an async request listener is an unhandled rejection, which under Node's
+ * default `--unhandled-rejections=throw` takes the whole process down. One
+ * malformed byte from any client that can reach the socket was enough to end
+ * inference. A pathname does not depend on the authority anyway, so a fixed
+ * base is both safer and equivalent: an absolute-form request URI
+ * (`GET http://host/v1/models HTTP/1.1`, legal in HTTP/1.1) still wins over
+ * the base and yields the same path it always did.
+ *
+ * `req.url` itself is guarded too, for the same reason rather than a known
+ * input: this function's contract is that no request can make it throw.
+ */
+export function requestPathname(req) {
+    const raw = req.url ?? '/';
+    try {
+        return new URL(raw, 'http://localhost').pathname;
+    }
+    catch {
+        const cut = raw.search(/[?#]/);
+        const path = cut === -1 ? raw : raw.slice(0, cut);
+        return path.startsWith('/') ? path : '/';
+    }
+}
 function readBody(req) {
     return new Promise((resolve, reject) => {
         const chunks = [];
@@ -22,15 +51,14 @@ function readBody(req) {
         req.on('error', reject);
     });
 }
-export async function routeRequest(req, res, registry, store, responseRetentionSec) {
-    const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
-    const path = url.pathname;
+export async function routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator, extras) {
+    const path = requestPathname(req);
     if (path === '/v1/models') {
         if (req.method !== 'GET') {
             sendMethodNotAllowed(res, 'GET');
             return;
         }
-        handleListModels(res, registry);
+        handleListModels(res, registry, listModels);
         return;
     }
     if (path === '/v1/responses') {
@@ -48,7 +76,25 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
             sendBadRequest(res, msg);
             return;
         }
-        await handleCreateResponse(res, body, registry, store, req, responseRetentionSec);
+        await handleCreateResponse(res, body, registry, store, req, responseRetentionSec, idleSweeper, modelWorkCoordinator, resolveModel);
+        return;
+    }
+    if (path === '/v1/messages/count_tokens') {
+        if (req.method !== 'POST') {
+            sendAnthropicMethodNotAllowed(res, 'POST');
+            return;
+        }
+        let body;
+        try {
+            const raw = await readBody(req);
+            body = JSON.parse(raw);
+        }
+        catch (err) {
+            const msg = err instanceof Error && err.message === 'Request body too large' ? err.message : 'Invalid JSON in request body';
+            sendAnthropicBadRequest(res, msg);
+            return;
+        }
+        await handleCountMessageTokens(res, body, registry, idleSweeper, resolveModel, modelWorkCoordinator);
         return;
     }
     if (path === '/v1/messages') {
@@ -66,12 +112,38 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
             sendAnthropicBadRequest(res, msg);
             return;
         }
-        await handleCreateMessage(res, body, registry, req);
+        await handleCreateMessage(res, body, registry, req, idleSweeper, resolveModel, modelWorkCoordinator);
         return;
     }
     if (path === '/health' || path === '/v1/health') {
+        // Deliberately NOT bracketed by `idleSweeper.beginRequest/endRequest`
+        // and free of any native call: a supervisor polling on an interval must
+        // not keep pushing the drain timer out, nor touch the MLX allocator.
+        // Every field is read from plain JavaScript state.
+        const health = extras?.health?.();
+        if (health === undefined) {
+            // No reporter wired (a bare `createHandler` mounted by hand): keep the
+            // historical constant so existing consumers are unaffected.
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({ status: 'ok' }));
+            return;
+        }
+        // Unauthenticated pollers get liveness only. `models.resident` leaks
+        // project names and local paths, so it stays behind the token.
+        const body = extras?.authenticated === false ? toMinimalHealth(health) : health;
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify(body));
+        return;
+    }
+    // Liveness probe at `/`. Claude Code issues `HEAD /` before its first
+    // request; respond 200 so the probe doesn't leave a 404 in the logs.
+    if (path === '/') {
+        if (req.method !== 'GET' && req.method !== 'HEAD') {
+            sendMethodNotAllowed(res, 'GET, HEAD');
+            return;
+        }
         res.writeHead(200, { 'Content-Type': 'application/json' });
-        res.end(JSON.stringify({ status: 'ok' }));
+        res.end(req.method === 'HEAD' ? undefined : JSON.stringify({ service: 'mlx-node' }));
         return;
     }
     sendNotFound(res, `No route matches ${req.method} ${path}`);

package/dist/server.d.ts

@@ -1,6 +1,10 @@
 /** Full HTTP server lifecycle: wires up the handler and periodically sweeps expired `ResponseStore` rows and sessions. */
 import type { Server } from 'node:http';
 import { ResponseStore } from '@mlx-node/core';
+import type { PublicModelEntry } from './handler.js';
+import { type ServerHealth } from './health.js';
+import { type LoadModelOptions } from './load-model.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import { ModelRegistry } from './registry.js';
 /**
  * Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
@@ -25,6 +29,33 @@ export declare function parseEnvSeconds(name: string): number | undefined;
  * Exported for unit tests.
  */
 export declare function parseEnvPositiveInt(name: string): number | undefined;
+/**
+ * Resolve the effective auth token from an explicit value plus the env
+ * fallback.
+ *
+ * Exported because `createInferenceHost` has to answer "is this server going
+ * to be protected?" BEFORE it binds, in order to refuse a non-loopback bind
+ * that would serve anonymously. Two independent copies of this rule would
+ * drift, and the direction it would drift is a host that refuses to start
+ * while `MLX_SERVER_AUTH_TOKEN` is sitting right there in the environment.
+ *
+ * An empty env var means "not set". An accidental `MLX_SERVER_AUTH_TOKEN=` in
+ * a launcher script must not enable auth with an empty secret that every
+ * credential-less request would then fail against.
+ *
+ * An empty EXPLICIT token is a different case and is rejected outright, for the
+ * same reason {@link normalizePositiveIntConfig} rejects a bogus explicit knob:
+ * somebody asked for a token and supplied nothing, which is
+ * `--auth-token "$TOKEN"` with `TOKEN` unset. Returning `''` made the bind
+ * guard read "auth is configured" and allow `0.0.0.0`, while the comparator
+ * accepted an empty `x-api-key` because both strings were empty — a wildcard
+ * bind published under a credential anyone can guess. Quietly downgrading to
+ * "no auth" instead would be fail-open in the other direction: on loopback it
+ * hands back the unauthenticated, wildcard-CORS server the operator was
+ * explicitly trying not to start. Throwing is the only answer that is wrong in
+ * neither bind mode, and it happens before anything binds or loads.
+ */
+export declare function resolveAuthToken(explicit: string | undefined): string | undefined;
 export interface ServerConfig {
     /** Port to listen on (default: 8080). */
     port?: number;
@@ -34,8 +65,26 @@ export interface ServerConfig {
     storePath?: string;
     /** Disable response storage entirely (default: false). */
     disableStore?: boolean;
-    /** Enable CORS headers (default: true). */
+    /**
+     * Enable CORS headers.
+     *
+     * Default: `true` when no `authToken` is in effect (historical behaviour),
+     * `false` once one is. An explicit value always wins. See
+     * {@link ServerConfig.authToken}.
+     */
     cors?: boolean;
+    /**
+     * Shared secret required on every route except `/health` and `/v1/health`.
+     *
+     * Default: `process.env.MLX_SERVER_AUTH_TOKEN`, or `undefined` (no auth)
+     * when that is unset or empty. `undefined` is byte-for-byte identical to
+     * the pre-auth behaviour. An explicit `''` is rejected rather than treated as
+     * either — see {@link resolveAuthToken}.
+     *
+     * Accepted as `x-api-key: <token>` or `authorization: Bearer <token>`.
+     * Setting it also flips the `cors` default to `false`.
+     */
+    authToken?: string;
     /**
      * Retention for persisted response rows, in seconds. Stamped as `expires_at`
      * on each committed response; controls how long `previous_response_id`
@@ -56,6 +105,56 @@ export interface ServerConfig {
      * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` (positive integer).
      */
     maxQueueDepthPerModel?: number;
+    /**
+     * Milliseconds of HTTP inactivity (no request arrivals or completions)
+     * before the server issues a single `clearCache()` to drain the MLX
+     * Metal allocator's free pool. Replaces the old per-request
+     * `ClearCacheOnDrop` guard in the Rust layer, which was unsafe on a
+     * multi-model server because the pool is process-wide.
+     *
+     * Default: 30_000 ms (30 seconds). `0` disables the sweeper. Env
+     * override: `MLX_IDLE_CLEAR_CACHE_MS` (non-negative integer; 0
+     * disables; unset falls through to default).
+     *
+     * The decode-loop drain every 256 tokens inside each generative
+     * model is untouched and covers in-flight memory churn.
+     */
+    idleClearCacheMs?: number;
+    /**
+     * Optional async callback invoked before the endpoint layer looks
+     * the model up in the registry. Intended for lazy-load schemes:
+     * the callback should register the model into `registry` if it can;
+     * on return, the endpoint does `registry.get(body.model)` and 404s
+     * if still unresolved. Callback errors bubble up as 500s.
+     */
+    resolveModel?: (name: string) => Promise<void>;
+    /**
+     * Optional override for `GET /v1/models` enumeration. When provided,
+     * the endpoint returns this list instead of `registry.list()`.
+     * Intended for dynamic discovery schemes (e.g. enumerate every model
+     * on disk while only the currently-resident one is registered).
+     */
+    listModels?: () => PublicModelEntry[];
+}
+/** Options for {@link ServerInstance.close}. */
+export interface CloseOptions {
+    /**
+     * Grace period, in milliseconds, before still-open connections are
+     * destroyed. Default: {@link DEFAULT_CLOSE_TIMEOUT_MS} (5000).
+     *
+     * Only the FIRST `close()` call's value is honoured — later calls receive
+     * the memoized promise of the first, so their timeout is ignored.
+     */
+    timeoutMs?: number;
+}
+/** Outcome of {@link ServerInstance.close}. */
+export interface CloseResult {
+    /** `true` when the grace period expired and connections were destroyed. */
+    forced: boolean;
+    /** SSE streams open at the moment of the forced destroy. `0` when not forced. */
+    streamsAborted: number;
+    /** Wall-clock duration of the shutdown. */
+    durationMs: number;
 }
 export interface ServerInstance {
     server: Server;
@@ -63,8 +162,99 @@ export interface ServerInstance {
     registry: ModelRegistry;
     /** Null when disabled. */
     store: ResponseStore | null;
-    /** Graceful shutdown. */
-    close(): Promise<void>;
+    /**
+     * Coordinates process-wide MLX work: model loads take the exclusive writer
+     * slot, inference takes shared reader slots. Exposed so callers can compose
+     * their own brackets (or read `writerActive` / `lastLoad` for diagnostics).
+     * Prefer {@link loadModel} for the common load case — it also handles the
+     * drain suspension, which is easy to get wrong.
+     */
+    readonly modelWork: ModelWorkCoordinator;
+    /**
+     * Current readiness snapshot — the same body an authenticated
+     * `GET /health` returns. Pure JavaScript state; no native calls.
+     */
+    health(): ServerHealth;
+    /**
+     * Bounded, idempotent shutdown.
+     *
+     * Stops accepting new connections, drops idle (keep-alive) ones
+     * immediately, then waits up to `timeoutMs` for the rest to finish. On
+     * expiry every remaining connection is destroyed, which fires the same
+     * `res.on('close')` path a client disconnect fires — so in-flight SSE
+     * generations are cancelled through `@mlx-node/lm` down to the native
+     * `ChatStreamHandle`.
+     *
+     * Idempotent: the promise is memoized, so repeated calls return the same
+     * promise and the same result. In particular a second call does NOT
+     * reject with `ERR_SERVER_NOT_RUNNING`.
+     *
+     * RESIDUAL: `ResponseStore` has no `close()` (it is a Rust-side handle),
+     * so the SQLite connection is released by process exit, not here. A
+     * long-lived process that creates and closes many servers will hold one
+     * store handle per server.
+     */
+    close(opts?: CloseOptions): Promise<CloseResult>;
+    /**
+     * Load a model out-of-band and register it, with idle drains suspended and
+     * inference excluded for the entire operation — including the wait for the
+     * coordinator's writer lock.
+     *
+     * This is the safe way to swap the resident model on a server that is
+     * already serving. Hand-rolling the two brackets in the wrong order races
+     * the process-wide Metal allocator; see `load-model.ts` for the full
+     * rationale.
+     *
+     * Rejects with the underlying error if `load()` throws; both brackets
+     * unwind cleanly, and `health().lastLoad` records the failure under
+     * `opts.name`.
+     */
+    loadModel(opts: LoadModelOptions): Promise<void>;
+    /**
+     * Run `fn` with the idle-drain timer suspended for the duration of
+     * an unbracketed, allocator-heavy operation — most commonly a hot
+     * `Model::load()` invoked AFTER the server has already served at
+     * least one request. In that scenario the post-request drain timer
+     * armed by `endRequest()` (at t+idleClearCacheMs) can otherwise
+     * fire MID-LOAD, racing the Metal allocator while weight
+     * materialization is still in progress.
+     *
+     * Handles try/finally bracketing so a thrown load never leaks the
+     * internal suspend counter. Accepts both sync and async functions;
+     * returns `fn`'s own return value (or resolved promise). When the
+     * bracket exits (normal or thrown) AND `inFlight === 0` with no
+     * other suspend active, a fresh drain timer is armed.
+     *
+     * Safe to nest — each call allocates its own token-scoped release
+     * so overlapping brackets unwind independently.
+     *
+     * The common `serve.ts` pattern (load all models BEFORE
+     * `createServer()`) does not need this API — there is no armed
+     * timer before the first request, so there is no race.
+     *
+     * Pass-through when the sweeper is disabled (`idleClearCacheMs: 0`
+     * or missing `__internal__.clearCache`): `fn` is invoked directly.
+     *
+     * @example
+     * ```ts
+     * await instance.withSuspendedDrains(async () => {
+     *   const model = await Qwen35Model.load(modelPath);
+     *   instance.registry.register('new-model', model);
+     * });
+     * ```
+     */
+    withSuspendedDrains<T>(fn: () => Promise<T>): Promise<T>;
+    withSuspendedDrains<T>(fn: () => T): T;
+    /**
+     * Low-level: suspend drains and return an idempotent, token-scoped
+     * disposer. Prefer {@link withSuspendedDrains} unless you need
+     * manual control over when the suspend is released. Safe to nest;
+     * calling the disposer more than once is a no-op.
+     *
+     * No-op when the sweeper is disabled (`idleClearCacheMs: 0` or
+     * missing `__internal__.clearCache`): returns a no-op disposer.
+     */
+    suspendDrains(): () => void;
 }
 /**
  * Start an MLX-Node HTTP server exposing `POST /v1/responses`,

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAIxC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAgB9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B,yBAAyB;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAoEjF"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAkB,MAAM,WAAW,CAAC;AAIxD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAuB,KAAK,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA2B9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAUjF;AAED,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;;;;OAUG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,gDAAgD;AAChD,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC;IACvB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC;IACzC;;;OAGG;IACH,MAAM,IAAI,YAAY,CAAC;IACvB;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACjD;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;OAQG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA4KjF"}

package/dist/server.js

@@ -5,9 +5,23 @@ import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { ResponseStore } from '@mlx-node/core';
 import { createHandler } from './handler.js';
+import { createHealthReporter } from './health.js';
+import { createIdleSweeper, DEFAULT_IDLE_CLEAR_CACHE_MS, parseIdleClearCacheEnv } from './idle-sweeper.js';
+import { runGuardedModelLoad } from './load-model.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import { ModelRegistry } from './registry.js';
+import { activeSSEStreamCountForResponses } from './streaming.js';
 /** Cleanup interval for expired responses (ms). */
 const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+/**
+ * Default grace period before {@link ServerInstance.close} destroys
+ * still-open connections.
+ *
+ * 5 s is deliberately a little longer than the ~5 s GPU watchdog window: a
+ * decode loop that is about to yield its next token should get the chance to
+ * unwind cleanly rather than being cut off a hair early.
+ */
+const DEFAULT_CLOSE_TIMEOUT_MS = 5000;
 /**
  * Default retention for persisted response rows in SQLite, in seconds.
  *
@@ -82,6 +96,42 @@ function normalizePositiveIntConfig(value, name) {
     }
     return value;
 }
+/**
+ * Resolve the effective auth token from an explicit value plus the env
+ * fallback.
+ *
+ * Exported because `createInferenceHost` has to answer "is this server going
+ * to be protected?" BEFORE it binds, in order to refuse a non-loopback bind
+ * that would serve anonymously. Two independent copies of this rule would
+ * drift, and the direction it would drift is a host that refuses to start
+ * while `MLX_SERVER_AUTH_TOKEN` is sitting right there in the environment.
+ *
+ * An empty env var means "not set". An accidental `MLX_SERVER_AUTH_TOKEN=` in
+ * a launcher script must not enable auth with an empty secret that every
+ * credential-less request would then fail against.
+ *
+ * An empty EXPLICIT token is a different case and is rejected outright, for the
+ * same reason {@link normalizePositiveIntConfig} rejects a bogus explicit knob:
+ * somebody asked for a token and supplied nothing, which is
+ * `--auth-token "$TOKEN"` with `TOKEN` unset. Returning `''` made the bind
+ * guard read "auth is configured" and allow `0.0.0.0`, while the comparator
+ * accepted an empty `x-api-key` because both strings were empty — a wildcard
+ * bind published under a credential anyone can guess. Quietly downgrading to
+ * "no auth" instead would be fail-open in the other direction: on loopback it
+ * hands back the unauthenticated, wildcard-CORS server the operator was
+ * explicitly trying not to start. Throwing is the only answer that is wrong in
+ * neither bind mode, and it happens before anything binds or loads.
+ */
+export function resolveAuthToken(explicit) {
+    if (explicit === '') {
+        throw new Error('authToken was set to an empty string; pass a real secret or omit it entirely ' +
+            '(an unset shell variable in `--auth-token "$VAR"` is the usual cause).');
+    }
+    if (explicit !== undefined)
+        return explicit;
+    const fromEnv = process.env.MLX_SERVER_AUTH_TOKEN;
+    return fromEnv != null && fromEnv !== '' ? fromEnv : undefined;
+}
 /**
  * Start an MLX-Node HTTP server exposing `POST /v1/responses`,
  * `POST /v1/messages`, and `GET /v1/models`.
@@ -95,7 +145,10 @@ function normalizePositiveIntConfig(value, name) {
 export async function createServer(config) {
     const port = config?.port ?? 8080;
     const host = config?.host ?? '127.0.0.1';
-    const cors = config?.cors ?? true;
+    const authToken = resolveAuthToken(config?.authToken);
+    // Forwarded unresolved so `createHandler` applies the auth-aware default
+    // (`true` without a token, `false` with one) in exactly one place.
+    const cors = config?.cors;
     const disableStore = config?.disableStore ?? false;
     // Validate caller-supplied numeric knobs BEFORE consulting env fallbacks
     // so a bogus explicit value surfaces as a descriptive error instead of
@@ -108,7 +161,18 @@ export async function createServer(config) {
     // allocated on `register()`) all share a single effective value.
     const configMaxQueueDepth = normalizePositiveIntConfig(config?.maxQueueDepthPerModel, 'maxQueueDepthPerModel');
     const maxQueueDepthPerModel = configMaxQueueDepth ?? parseEnvPositiveInt('MLX_MAX_QUEUE_DEPTH_PER_MODEL');
+    // Idle sweeper wiring. `0` is a legal explicit "off" value so we
+    // cannot reuse `normalizePositiveIntConfig` (which rejects 0).
+    // Precedence: explicit config value wins over env wins over default.
+    const idleClearCacheMs = resolveIdleClearCacheMs(config?.idleClearCacheMs);
+    const idleSweeper = createIdleSweeper(idleClearCacheMs);
+    // The sweeper is driven exclusively by `endRequest()` calls in the
+    // inference endpoints (`/v1/responses`, `/v1/messages`). There is no
+    // cold-start drain: a process that loads models but never receives a
+    // request will not fire `clearCache()`. See the `idle-sweeper.ts`
+    // module doc (section "Drain is post-request only") for rationale.
     const registry = new ModelRegistry({ maxQueueDepth: maxQueueDepthPerModel });
+    const modelWorkCoordinator = new ModelWorkCoordinator();
     let store = null;
     if (!disableStore) {
         const storePath = config?.storePath ?? join(homedir(), '.mlx-node', 'responses.db');
@@ -126,8 +190,33 @@ export async function createServer(config) {
         }
     }, CLEANUP_INTERVAL_MS);
     cleanupTimer.unref();
-    const handler = createHandler(registry, { cors, store, responseRetentionSec });
-    const server = httpCreateServer(handler);
+    // One reporter shared by the HTTP endpoint and `ServerInstance.health()`
+    // so both report the same uptime origin.
+    const health = createHealthReporter({ registry, idleSweeper, modelWorkCoordinator });
+    const handler = createHandler(registry, {
+        cors,
+        store,
+        responseRetentionSec,
+        idleSweeper,
+        resolveModel: config?.resolveModel,
+        modelWorkCoordinator,
+        listModels: config?.listModels,
+        authToken,
+        health,
+    });
+    /**
+     * Responses owned by THIS HTTP server. The SSE registry is deliberately
+     * process-wide because `beginSSE` is also used by standalone handlers;
+     * shutdown accounting intersects it with this weak ownership set so one
+     * server cannot claim streams that another server leaves live. A WeakSet
+     * needs no second response cleanup lifecycle.
+     */
+    const ownedResponses = new WeakSet();
+    const server = httpCreateServer((req, res) => {
+        // Synchronous, before `handler` can reach either endpoint's `beginSSE`.
+        ownedResponses.add(res);
+        void handler(req, res);
+    });
     await new Promise((resolve, reject) => {
         const onError = (err) => {
             server.removeListener('error', onError);
@@ -139,20 +228,94 @@ export async function createServer(config) {
             resolve();
         });
     });
-    return {
-        server,
-        registry,
-        store,
-        async close() {
+    // Memoized so a second `close()` returns the first call's promise instead
+    // of re-entering `server.close()` (which invokes its callback with
+    // ERR_SERVER_NOT_RUNNING once the server is already down).
+    let closePromise = null;
+    const close = (opts) => {
+        if (closePromise !== null)
+            return closePromise;
+        const startedAt = Date.now();
+        const requested = opts?.timeoutMs;
+        const timeoutMs = typeof requested === 'number' && Number.isFinite(requested) && requested >= 0
+            ? requested
+            : DEFAULT_CLOSE_TIMEOUT_MS;
+        closePromise = (async () => {
             clearInterval(cleanupTimer);
-            await new Promise((resolve, reject) => {
+            idleSweeper.close();
+            let forced = false;
+            let streamsAborted = 0;
+            // Arm the wait BEFORE dropping idle sockets so nothing can complete in
+            // the gap and settle `server.close()` before we are listening.
+            const serverClosed = new Promise((resolve, reject) => {
                 server.close((err) => {
-                    if (err)
+                    // Tolerated defensively: memoization should make this unreachable,
+                    // but a caller who also closed `instance.server` directly would
+                    // otherwise turn a successful shutdown into a rejection.
+                    if (err && err.code !== 'ERR_SERVER_NOT_RUNNING')
                         reject(err);
                     else
                         resolve();
                 });
             });
+            // Keep-alive sockets parked in a client pool hold no request; without
+            // this the shutdown would sit on them until the client's own timeout.
+            server.closeIdleConnections();
+            const forceTimer = setTimeout(() => {
+                forced = true;
+                // Snapshot BEFORE destroying: `closeAllConnections()` fires each
+                // response's `'close'` event, which unregisters it from the global SSE
+                // registry used by the intersection.
+                streamsAborted = activeSSEStreamCountForResponses(ownedResponses);
+                // Destroying the socket fires exactly the `res.on('close')` path a
+                // client disconnect fires, so the endpoint's AbortController cancels
+                // the native stream handle rather than leaking a running decode.
+                server.closeAllConnections();
+            }, timeoutMs);
+            try {
+                await serverClosed;
+            }
+            finally {
+                clearTimeout(forceTimer);
+            }
+            return { forced, streamsAborted, durationMs: Date.now() - startedAt };
+        })();
+        return closePromise;
+    };
+    return {
+        server,
+        registry,
+        store,
+        modelWork: modelWorkCoordinator,
+        health,
+        close,
+        loadModel(opts) {
+            return runGuardedModelLoad({ idleSweeper, modelWorkCoordinator, registry }, opts);
+        },
+        withSuspendedDrains(fn) {
+            return idleSweeper.withSuspendedDrains(fn);
+        },
+        suspendDrains() {
+            return idleSweeper.suspendDrains();
         },
     };
 }
+/**
+ * Resolve the effective idle-drain delay from (constructor value, env,
+ * default). Unlike the other knobs, `0` is a legal explicit opt-out —
+ * we must NOT fall through to env/default when the caller explicitly
+ * passes `0`. Returns a non-negative integer milliseconds value.
+ */
+function resolveIdleClearCacheMs(configValue) {
+    if (configValue !== undefined) {
+        if (typeof configValue !== 'number' ||
+            !Number.isFinite(configValue) ||
+            !Number.isInteger(configValue) ||
+            configValue < 0) {
+            throw new Error(`idleClearCacheMs must be a non-negative integer; received ${String(configValue)}`);
+        }
+        return configValue;
+    }
+    const envValue = parseIdleClearCacheEnv();
+    return envValue ?? DEFAULT_IDLE_CLEAR_CACHE_MS;
+}