@mlx-node/server 0.0.7 → 0.0.9

package/dist/host/env-policy.js

@@ -0,0 +1,69 @@
+/**
+ * Engine tuning that is LAUNCHER POLICY, not an engine default.
+ *
+ * The native engine reads these knobs from the process environment through a
+ * `OnceLock`: the FIRST read latches the value for the life of the process, so
+ * every variable here must be in place before any model is loaded and must
+ * never be mutated afterwards. That one-shot latch is why the policy is a
+ * plain data object rather than something a caller can toggle at runtime.
+ *
+ * Two application shapes, deliberately different:
+ *
+ *   - In-process host (`mlx serve`, `mlx launch claude`): call
+ *     {@link applyEnginePolicy} to write `process.env` BEFORE the first load.
+ *     A value the user already set in their shell always wins — this is a
+ *     default, not an override.
+ *   - Out-of-process host (Electron `utilityProcess.fork`): pass
+ *     {@link engineEnvFor} into `fork({ env })` and never touch
+ *     `process.env` in the child. The returned map is unconditional: it
+ *     describes the child's starting environment, and the parent is expected
+ *     to spread the user's own env under it if it wants shell values to win.
+ *
+ * Keeping the two apart matters. `MLX_PAGED_PREFILL_CHUNK_SIZE` reads as
+ * `0` ("no chunking") in Rust when unset; 2048 is the value
+ * `mlx launch claude` has historically applied to bound the cold-prefill
+ * memory peak, and baking it into the engine would silently change every
+ * other embedder's behaviour.
+ */
+/** Env var names this module owns. Exported so tests can assert the exact set. */
+export const ENGINE_POLICY_ENV_VARS = ['MLX_PAGED_PREFILL_CHUNK_SIZE'];
+/**
+ * The policy `mlx launch claude` has applied since the flag existed, and the
+ * one `mlx serve` and the desktop sidecar inherit.
+ *
+ * 2048 matches the mlx-lm / mlx-vlm default and reduces per-chunk overhead
+ * versus the older 1024 for long Qwen dense contexts.
+ */
+export const LAUNCHER_ENGINE_POLICY = Object.freeze({ pagedPrefillChunkSize: 2048 });
+/**
+ * Render a policy as the environment map a forked child should start with.
+ *
+ * Unconditional by construction: there is no "already set" to respect in a
+ * child that does not exist yet.
+ */
+export function engineEnvFor(policy) {
+    const env = {};
+    if (policy.pagedPrefillChunkSize !== undefined) {
+        env.MLX_PAGED_PREFILL_CHUNK_SIZE = String(policy.pagedPrefillChunkSize);
+    }
+    return env;
+}
+/**
+ * Apply a policy to an in-process environment as a DEFAULT.
+ *
+ * Only writes vars that are currently unset — matching the historical
+ * `if (process.env.X == null)` guard, so an explicit `X=` (empty string) in
+ * the user's shell is treated as "the user has an opinion" and left alone.
+ *
+ * Returns the names actually written, so a caller can log or assert on them.
+ */
+export function applyEnginePolicy(policy, env = process.env) {
+    const applied = [];
+    for (const [name, value] of Object.entries(engineEnvFor(policy))) {
+        if (env[name] == null) {
+            env[name] = value;
+            applied.push(name);
+        }
+    }
+    return applied;
+}

package/dist/host/index.d.ts

@@ -0,0 +1,202 @@
+/**
+ * `@mlx-node/server/host` — the reusable inference-host bootstrap.
+ *
+ * Everything a process needs to go from "a directory of downloaded models" to
+ * "a listening Anthropic/OpenAI-compatible endpoint with one resident model
+ * and a working `/model` swap", with none of the opinions about WHO supervises
+ * the process.
+ *
+ * That last part is why this is a module rather than a CLI flag. The two known
+ * front-ends invert the supervision relationship:
+ *
+ *   - `mlx launch claude` starts the host, then spawns and supervises a child
+ *     (`claude`), and exits when the child does.
+ *   - The desktop app's Electron `utilityProcess` IS the child; the host is
+ *     supervised, gets told when to shut down, and may be SIGKILLed.
+ *
+ * `mlx serve` is the third, degenerate case — nothing above, nothing below —
+ * and doubles as the terminal-visible reproduction of the sidecar when the
+ * utilityProcess wedges.
+ *
+ * A subpath export rather than part of `@mlx-node/server`'s main entry, so
+ * importing the plain HTTP handler does not drag in `@mlx-node/lm` and model
+ * loading.
+ *
+ * ## Ownership
+ *
+ * The returned host owns, and disposes on {@link InferenceHost.close}:
+ *   1. the HTTP server,
+ *   2. the verbose request logger (when `logDir` is set),
+ *   3. the `PagedConfigOverrideManager`'s temp root.
+ *
+ * The server goes first because the logger records a request from that
+ * request's own `finish`/`close` handler. Ending the log streams while a
+ * request is still draining loses exactly the completion lines a verbose
+ * shutdown exists to capture. The temp root goes last because a
+ * still-draining request may still be reading a cloned config. Each step is
+ * independently guarded — one failing disposer must not strand the ones
+ * after it.
+ */
+import type { Server } from 'node:http';
+import { type LoadableModel } from '@mlx-node/lm';
+import type { ServerHealth } from '../health.js';
+import { type CloseOptions, type ServerInstance } from '../server.js';
+import { type DiscoveredModel } from './discover.js';
+import { type EnginePolicy } from './env-policy.js';
+import { type Logger } from './logger.js';
+/** Families `mlx launch claude` has historically forced onto the paged path. */
+export declare const DEFAULT_PAGED_MODEL_TYPES: readonly ['qwen3_5', 'qwen3_5_moe'];
+/** Thrown when `modelsDir` holds nothing servable. Carries the dir for the caller's message. */
+export declare class NoModelsDiscoveredError extends Error {
+    readonly modelsDir: string;
+    constructor(modelsDir: string);
+}
+/**
+ * Thrown when a bind reachable from the network was asked for with no shared
+ * secret to gate it.
+ *
+ * There is no safe way to serve that: every route but the `/health` liveness
+ * carve-out runs inference, so an unauthenticated LAN-reachable bind hands
+ * anyone who can route to this machine the GPU, the RAM, and the list of
+ * models on disk. Failing at startup is the only outcome the operator can act
+ * on — serving-with-a-warning is a warning nobody reads scrolling past a
+ * model load.
+ */
+export declare class InsecureBindError extends Error {
+    readonly host: string;
+    constructor(host: string);
+}
+/** Thrown when a requested model name is not among the discovered ones. */
+export declare class ModelNotFoundError extends Error {
+    readonly requested: string;
+    readonly modelsDir: string;
+    readonly available: string[];
+    constructor(requested: string, modelsDir: string, available: string[]);
+}
+/** Thrown when a model load is requested after host shutdown reaches its admission boundary. */
+export declare class InferenceHostClosedError extends Error {
+    constructor();
+}
+export interface InferenceHostOptions {
+    /**
+     * Port to bind. Omitted ⇒ a free port is picked up-front (so the URL is
+     * known before the server starts, which `mlx launch claude` needs in order
+     * to bake `ANTHROPIC_BASE_URL` into the child's env). `0` binds an
+     * ephemeral port and the real one is read back off the socket.
+     */
+    port?: number;
+    /**
+     * Host to bind. Default `127.0.0.1`.
+     *
+     * A non-loopback value (including the wildcards `0.0.0.0` / `::`) requires
+     * an auth token — from {@link InferenceHostOptions.authToken} or
+     * `MLX_SERVER_AUTH_TOKEN` — or the call throws {@link InsecureBindError}
+     * instead of listening.
+     */
+    host?: string;
+    /** Model discovery root. Default: {@link resolveModelsDir}'s resolution order. */
+    modelsDir?: string;
+    /**
+     * Which discovered model is the bound/default one. Precedence, highest
+     * first: this option, `ANTHROPIC_MODEL`, `discovered[0]` (alphabetical).
+     * A name that matches nothing discovered throws {@link ModelNotFoundError}
+     * rather than silently falling back.
+     */
+    model?: string;
+    /** Shared secret for every route except `/health`. See `ServerConfig.authToken`. */
+    authToken?: string;
+    /** When set, every HTTP turn is captured under this directory. */
+    logDir?: string;
+    /**
+     * Engine env policy to apply to `process.env` BEFORE anything can load a
+     * model. Omitted ⇒ nothing is written and the engine's own defaults stand.
+     * Out-of-process hosts must pass `engineEnvFor(policy)` to `fork({ env })`
+     * instead and leave this unset — the vars latch via `OnceLock` on first
+     * read, so mutating them in the child is too late.
+     */
+    enginePolicy?: EnginePolicy;
+    /** Model types forced onto the block-paged KV cache. Default {@link DEFAULT_PAGED_MODEL_TYPES}. */
+    pagedModelTypes?: readonly string[];
+    /** Path to the SQLite response store. See `ServerConfig.storePath`. */
+    storePath?: string;
+    /** Disable response persistence entirely. See `ServerConfig.disableStore`. */
+    disableStore?: boolean;
+    /**
+     * Remove temp roots left behind by hosts that were killed without running
+     * `close()`. Default `true`; a supervisor that runs several hosts under one
+     * pid namespace and wants to control the timing can turn it off and call
+     * {@link sweepOrphanHostTempRoots} itself.
+     */
+    sweepOrphanTempRoots?: boolean;
+    /** Test seam: replaces the native model loader. */
+    loadModel?: (path: string) => Promise<LoadableModel>;
+    /** Test seam: replaces the verbose request logger. */
+    attachLogger?: (server: Server, logDir: string) => Logger;
+}
+export interface InferenceHost {
+    /** Connectable base URL, e.g. `http://127.0.0.1:51234`. */
+    url: string;
+    /** The port actually bound (resolved, never `0`). */
+    port: number;
+    /** The host actually bound — the requested value, not the advertised one. */
+    host: string;
+    /** The resolved model discovery root. */
+    modelsDir: string;
+    /** Everything discovered under `modelsDir`, alphabetical. */
+    models: DiscoveredModel[];
+    /** Name of the default/bound model. Always a member of {@link models}. */
+    boundModel: string;
+    /** Verbose log directory, or `null` when logging is off. */
+    logDir: string | null;
+    /** Escape hatch for callers that need the registry, store, or raw `http.Server`. */
+    server: ServerInstance;
+    /** Current readiness snapshot — the same body `GET /health` returns. */
+    health(): ServerHealth;
+    /**
+     * Make `name` the resident model, out of band (no HTTP request needed).
+     *
+     * Runs under the SAME brackets `ServerInstance.loadModel` uses — drains
+     * suspended OUTSIDE the coordinator's exclusive writer slot — so it cannot
+     * race the process-wide Metal allocator against in-flight inference, and
+     * `/health` reports `loading` plus a `lastLoad` record labelled with `name`.
+     *
+     * It deliberately does NOT call `ServerInstance.loadModel` itself. That
+     * helper only ever registers, never unregisters, so calling it per swap
+     * would accumulate every model the user has ever picked in memory. The
+     * single-resident swap controller is the thing that makes a swap a swap;
+     * this method just wraps it in the right brackets.
+     *
+     * Rejects with {@link ModelNotFoundError} for an unknown name — the swap
+     * controller would otherwise treat it as an alias for the current resident,
+     * which is right for Claude Code's hardcoded `claude-haiku-*` but wrong for
+     * a supervisor that asked for a specific model.
+     *
+     * Calls admitted before {@link close} begins are allowed to finish. Calls
+     * made after shutdown begins reject with {@link InferenceHostClosedError}.
+     */
+    loadModel(name: string): Promise<void>;
+    /**
+     * Stop accepting new HTTP and out-of-band load work, wait for every
+     * out-of-band load already admitted, then dispose the logger and temp root.
+     * Idempotent and memoized; disposal steps are individually guarded so one
+     * failure cannot strand the remaining steps.
+     */
+    close(opts?: CloseOptions): Promise<void>;
+}
+/**
+ * Start a local inference host.
+ *
+ * Throws {@link NoModelsDiscoveredError} / {@link ModelNotFoundError} /
+ * {@link InsecureBindError} rather than exiting — a library cannot know
+ * whether its caller is a CLI, a test, or an Electron main process. Front-ends
+ * render those into their own messages.
+ */
+export declare function createInferenceHost(opts?: InferenceHostOptions): Promise<InferenceHost>;
+export { discoverModels, type DiscoveredModel } from './discover.js';
+export { applyEnginePolicy, engineEnvFor, ENGINE_POLICY_ENV_VARS, LAUNCHER_ENGINE_POLICY, type EnginePolicy, } from './env-policy.js';
+export { attachLogger, resolveLogDir, type Logger } from './logger.js';
+export { bracketHost, hostUrl, isLoopbackBindHost, pickFreePort } from './net.js';
+export { resolveMlxNodeHome, resolveModelsDir } from './paths.js';
+export { makeSwapController, type SwapController } from './swap.js';
+export { hostTempDirPrefix, isProcessAlive, sweepOrphanHostTempRoots, HOST_TEMP_DIR_STEM, type SweepOrphanTempRootsOptions, } from './temp-root.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/host/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/host/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAA4D,KAAK,aAAa,EAAE,MAAM,cAAc,CAAC;AAE5G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAkC,KAAK,YAAY,EAAE,KAAK,cAAc,EAAE,MAAM,cAAc,CAAC;AACtG,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAqB,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAuC,KAAK,MAAM,EAAE,MAAM,aAAa,CAAC;AAM/E,gFAAgF;AAChF,eAAO,MAAM,yBAAyB,YAAI,SAAS,EAAE,aAAa,CAAU,CAAC;AAE7E,gGAAgG;AAChG,qBAAa,uBAAwB,SAAQ,KAAK;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM;IAAtC,YAAqB,SAAS,EAAE,MAAM,EAGrC;CACF;AAED;;;;;;;;;;GAUG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM;IAAjC,YAAqB,IAAI,EAAE,MAAM,EAMhC;CACF;AAED,2EAA2E;AAC3E,qBAAa,kBAAmB,SAAQ,KAAK;IAEzC,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE;IAH9B,YACW,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EAAE,EAI7B;CACF;AAED,gGAAgG;AAChG,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,cAGC;CACF;AAED,MAAM,WAAW,oBAAoB;IACnC;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oFAAoF;IACpF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mGAAmG;IACnG,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B,mDAAmD;IACnD,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IACrD,sDAAsD;IACtD,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CAC3D;AAED,MAAM,WAAW,aAAa;IAC5B,2DAA2D;IAC3D,GAAG,EAAE,MAAM,CAAC;IACZ,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,oFAAoF;IACpF,MAAM,EAAE,cAAc,CAAC;IACvB,wEAAwE;IACxE,MAAM,IAAI,YAAY,CAAC;IACvB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3C;AAED;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,GAAE,oBAAyB,GAAG,OAAO,CAAC,aAAa,CAAC,CA0OjG;AAED,OAAO,EAAE,cAAc,EAAE,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EACL,iBAAiB,EACjB,YAAY,EACZ,sBAAsB,EACtB,sBAAsB,EACtB,KAAK,YAAY,GAClB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,KAAK,MAAM,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACpE,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,wBAAwB,EACxB,kBAAkB,EAClB,KAAK,2BAA2B,GACjC,MAAM,gBAAgB,CAAC"}

package/dist/host/index.js

@@ -0,0 +1,325 @@
+/**
+ * `@mlx-node/server/host` — the reusable inference-host bootstrap.
+ *
+ * Everything a process needs to go from "a directory of downloaded models" to
+ * "a listening Anthropic/OpenAI-compatible endpoint with one resident model
+ * and a working `/model` swap", with none of the opinions about WHO supervises
+ * the process.
+ *
+ * That last part is why this is a module rather than a CLI flag. The two known
+ * front-ends invert the supervision relationship:
+ *
+ *   - `mlx launch claude` starts the host, then spawns and supervises a child
+ *     (`claude`), and exits when the child does.
+ *   - The desktop app's Electron `utilityProcess` IS the child; the host is
+ *     supervised, gets told when to shut down, and may be SIGKILLed.
+ *
+ * `mlx serve` is the third, degenerate case — nothing above, nothing below —
+ * and doubles as the terminal-visible reproduction of the sidecar when the
+ * utilityProcess wedges.
+ *
+ * A subpath export rather than part of `@mlx-node/server`'s main entry, so
+ * importing the plain HTTP handler does not drag in `@mlx-node/lm` and model
+ * loading.
+ *
+ * ## Ownership
+ *
+ * The returned host owns, and disposes on {@link InferenceHost.close}:
+ *   1. the HTTP server,
+ *   2. the verbose request logger (when `logDir` is set),
+ *   3. the `PagedConfigOverrideManager`'s temp root.
+ *
+ * The server goes first because the logger records a request from that
+ * request's own `finish`/`close` handler. Ending the log streams while a
+ * request is still draining loses exactly the completion lines a verbose
+ * shutdown exists to capture. The temp root goes last because a
+ * still-draining request may still be reading a cloned config. Each step is
+ * independently guarded — one failing disposer must not strand the ones
+ * after it.
+ */
+import { loadModel as loadModelNative, PagedConfigOverrideManager } from '@mlx-node/lm';
+import { createServer, resolveAuthToken } from '../server.js';
+import { discoverModels } from './discover.js';
+import { applyEnginePolicy } from './env-policy.js';
+import { attachLogger as defaultAttachLogger } from './logger.js';
+import { hostUrl, isLoopbackBindHost, normalizeLoopbackBindHost, pickFreePort } from './net.js';
+import { resolveModelsDir } from './paths.js';
+import { makeSwapController } from './swap.js';
+import { hostTempDirPrefix, sweepOrphanHostTempRoots } from './temp-root.js';
+/** Families `mlx launch claude` has historically forced onto the paged path. */
+export const DEFAULT_PAGED_MODEL_TYPES = ['qwen3_5', 'qwen3_5_moe'];
+/** Thrown when `modelsDir` holds nothing servable. Carries the dir for the caller's message. */
+export class NoModelsDiscoveredError extends Error {
+    modelsDir;
+    constructor(modelsDir) {
+        super(`No models discovered under ${modelsDir}.`);
+        this.modelsDir = modelsDir;
+        this.name = 'NoModelsDiscoveredError';
+    }
+}
+/**
+ * Thrown when a bind reachable from the network was asked for with no shared
+ * secret to gate it.
+ *
+ * There is no safe way to serve that: every route but the `/health` liveness
+ * carve-out runs inference, so an unauthenticated LAN-reachable bind hands
+ * anyone who can route to this machine the GPU, the RAM, and the list of
+ * models on disk. Failing at startup is the only outcome the operator can act
+ * on — serving-with-a-warning is a warning nobody reads scrolling past a
+ * model load.
+ */
+export class InsecureBindError extends Error {
+    host;
+    constructor(host) {
+        super(`Refusing to bind ${host} without an auth token: every route except /health runs inference. ` +
+            `Pass an auth token (mlx serve --auth-token, or MLX_SERVER_AUTH_TOKEN), or bind 127.0.0.1.`);
+        this.host = host;
+        this.name = 'InsecureBindError';
+    }
+}
+/** Thrown when a requested model name is not among the discovered ones. */
+export class ModelNotFoundError extends Error {
+    requested;
+    modelsDir;
+    available;
+    constructor(requested, modelsDir, available) {
+        super(`Model "${requested}" not found under ${modelsDir}.`);
+        this.requested = requested;
+        this.modelsDir = modelsDir;
+        this.available = available;
+        this.name = 'ModelNotFoundError';
+    }
+}
+/** Thrown when a model load is requested after host shutdown reaches its admission boundary. */
+export class InferenceHostClosedError extends Error {
+    constructor() {
+        super('Inference host is closing or closed; model loads are no longer accepted.');
+        this.name = 'InferenceHostClosedError';
+    }
+}
+/**
+ * Start a local inference host.
+ *
+ * Throws {@link NoModelsDiscoveredError} / {@link ModelNotFoundError} /
+ * {@link InsecureBindError} rather than exiting — a library cannot know
+ * whether its caller is a CLI, a test, or an Electron main process. Front-ends
+ * render those into their own messages.
+ */
+export async function createInferenceHost(opts = {}) {
+    const host = opts.host ?? '127.0.0.1';
+    // BEFORE the engine policy, the temp sweep and discovery — all of which
+    // mutate state outside this function — so a refused start leaves nothing
+    // behind. Resolved through `resolveAuthToken` rather than reading
+    // `opts.authToken`, or `MLX_SERVER_AUTH_TOKEN=… mlx serve --host 0.0.0.0`
+    // would be refused despite being fully protected.
+    if (!isLoopbackBindHost(host) && resolveAuthToken(opts.authToken) === undefined) {
+        throw new InsecureBindError(host);
+    }
+    // `[::1]` is valid URL-authority spelling and the security predicate accepts
+    // it as loopback, but Node's listen host must be the bare literal. Normalize
+    // only after the reachability gate, and only the exact safe form.
+    const bindHost = normalizeLoopbackBindHost(host);
+    // FIRST, before anything can touch the engine: the native side latches
+    // these via `OnceLock` on first read, so a policy applied after a load has
+    // silently done nothing.
+    if (opts.enginePolicy !== undefined)
+        applyEnginePolicy(opts.enginePolicy);
+    // Reclaim roots left by hosts that never got to run `close()`. Best effort
+    // and awaited only so a test can observe it; a failure here is not a reason
+    // to refuse to start.
+    if (opts.sweepOrphanTempRoots !== false) {
+        await sweepOrphanHostTempRoots().catch(() => []);
+    }
+    const modelsDir = resolveModelsDir(opts.modelsDir);
+    const models = await discoverModels(modelsDir);
+    if (models.length === 0)
+        throw new NoModelsDiscoveredError(modelsDir);
+    // Precedence: explicit option > ANTHROPIC_MODEL > discovered[0].
+    const requestedModel = opts.model ?? process.env.ANTHROPIC_MODEL;
+    const requestedEntry = requestedModel != null ? models.find((m) => m.name === requestedModel) : undefined;
+    if (requestedModel != null && requestedEntry === undefined) {
+        throw new ModelNotFoundError(requestedModel, modelsDir, models.map((m) => m.name));
+    }
+    const boundEntry = requestedEntry ?? models[0];
+    // `undefined` means "you pick"; `0` means "the kernel picks and I will read
+    // it back". Only the former needs the up-front probe.
+    const requestedPort = opts.port ?? (await pickFreePort());
+    // The swap controller needs the registry from the server instance, but the
+    // server needs the controller's callbacks at construction. Bridge via a
+    // late-bound holder: the callbacks capture `ctrlRef.current` by closure.
+    const ctrlRef = { current: null };
+    let acceptingHttpModelLoads = true;
+    const activeHttpModelLoads = new Set();
+    const resolveHttpModel = (name) => {
+        if (!acceptingHttpModelLoads)
+            return Promise.reject(new InferenceHostClosedError());
+        const operation = ctrlRef.current.resolveModel(name);
+        activeHttpModelLoads.add(operation);
+        void operation.then(() => activeHttpModelLoads.delete(operation), () => activeHttpModelLoads.delete(operation));
+        return operation;
+    };
+    const serverConfig = {
+        port: requestedPort,
+        host: bindHost,
+        resolveModel: resolveHttpModel,
+        listModels: () => ctrlRef.current.listModels(),
+        ...(opts.authToken !== undefined ? { authToken: opts.authToken } : {}),
+        ...(opts.storePath !== undefined ? { storePath: opts.storePath } : {}),
+        ...(opts.disableStore !== undefined ? { disableStore: opts.disableStore } : {}),
+    };
+    const server = await createServer(serverConfig);
+    // Wrap the loader so managed families get `use_block_paged_cache: true`
+    // injected via a temp-dir clone with a patched config.json. The temp root is
+    // named after our pid so a SIGKILLed host's root can be reclaimed by the
+    // next host's startup sweep — see `temp-root.ts`.
+    const pagedConfigOverrides = new PagedConfigOverrideManager({
+        modelTypes: opts.pagedModelTypes ?? DEFAULT_PAGED_MODEL_TYPES,
+        tempDirPrefix: hostTempDirPrefix(),
+    });
+    const loadModelFn = opts.loadModel ?? loadModelNative;
+    const loadModelPagedAware = async (path) => loadModelFn(await pagedConfigOverrides.resolve(path));
+    const controller = makeSwapController(models, server.registry, loadModelPagedAware, boundEntry.name);
+    ctrlRef.current = controller;
+    // Attach AFTER `createServer` so the wrapper sees every incoming request,
+    // including the `GET /v1/models` a client fires on startup.
+    //
+    // By this point the socket is bound and `ctrlRef.current` is wired, so the
+    // endpoint already answers real requests. `attachLogger` opens with a
+    // synchronous `mkdirSync`, which throws on an unwritable `--log-dir`; without
+    // this rollback the rejection would strand a fully working inference server
+    // with no handle left to close it. Every caller in this repo exits the
+    // process on failure, so the leak is only reachable by an in-process
+    // embedder — which is exactly who this module is for.
+    let logger = null;
+    if (opts.logDir !== undefined) {
+        try {
+            logger = (opts.attachLogger ?? defaultAttachLogger)(server.server, opts.logDir);
+        }
+        catch (err) {
+            // Mirror `close()`'s order and its independent guards, and rethrow the
+            // ORIGINAL failure — a secondary close error must not mask the EACCES
+            // that actually explains what went wrong.
+            try {
+                await server.close();
+            }
+            catch {
+                /* already down, or a socket refused to die; fall through */
+            }
+            // A forced close destroys sockets, not the async handler promises
+            // `http.createServer` discarded. Close resolver admission only after
+            // the server has finished its graceful window, then drain any lazy load
+            // that was already inside the controller before removing its temp files.
+            acceptingHttpModelLoads = false;
+            await Promise.allSettled(activeHttpModelLoads);
+            try {
+                await pagedConfigOverrides.cleanup();
+            }
+            catch {
+                /* the startup sweep of the next host will reclaim it */
+            }
+            throw err;
+        }
+    }
+    const address = server.server.address();
+    const boundPort = address !== null && typeof address === 'object' ? address.port : requestedPort;
+    const byName = new Map(models.map((m) => [m.name, m]));
+    // `closeStarted` is an admission latch, checked and flipped synchronously:
+    // once close() returns its promise no later out-of-band load can join this
+    // set. The operations themselves retain the normal coordinator/controller
+    // ordering; shutdown only observes their promises and never takes a lock
+    // they need, so queued swaps can drain without deadlocking against close.
+    let closeStarted = false;
+    const activeHostLoads = new Set();
+    let closePromise = null;
+    return {
+        url: hostUrl(host, boundPort),
+        port: boundPort,
+        host,
+        modelsDir,
+        models,
+        boundModel: boundEntry.name,
+        logDir: logger?.logDir ?? null,
+        server,
+        health: () => server.health(),
+        loadModel(name) {
+            if (closeStarted)
+                return Promise.reject(new InferenceHostClosedError());
+            if (!byName.has(name)) {
+                return Promise.reject(new ModelNotFoundError(name, modelsDir, models.map((m) => m.name)));
+            }
+            // Same nesting as `runGuardedModelLoad`: the drain suspension must be
+            // OUTSIDE the writer lock so the armed `clearCache()` timer cannot fire
+            // while we are parked waiting for the lock. The HTTP path takes the
+            // locks in this same order, so the two can never deadlock against each
+            // other on the controller's internal serialization.
+            const operation = server.withSuspendedDrains(async () => {
+                await server.modelWork.withModelLoad(() => controller.resolveModel(name), name);
+            });
+            activeHostLoads.add(operation);
+            // Use a two-arm `then`, rather than an ignored `finally()` promise:
+            // cleanup must not manufacture an unhandled rejection when the caller
+            // legitimately observes a failed load through `operation`.
+            void operation.then(() => activeHostLoads.delete(operation), () => activeHostLoads.delete(operation));
+            return operation;
+        },
+        close(closeOpts) {
+            if (closePromise !== null)
+                return closePromise;
+            // Flip the latch and snapshot in the same synchronous turn. Every call
+            // in the snapshot was fully admitted before shutdown; no later call can
+            // race into the set after this point.
+            closeStarted = true;
+            const admittedHostLoads = [...activeHostLoads];
+            closePromise = (async () => {
+                // Every step is independently guarded: a server that fails to close
+                // must still leave the log streams ended and the temp root
+                // reclaimed. The temp root is the only one of the three that leaks
+                // OUTSIDE this process.
+                try {
+                    await server.close(closeOpts);
+                }
+                catch {
+                    /* already down, or a socket refused to die; fall through */
+                }
+                // A forced server close destroys request sockets but does not await
+                // the async handler promises. Stop any handler that has not reached
+                // the resolver from starting a late load, then snapshot resolver calls
+                // already inside the controller. Direct calls were closed and
+                // snapshotted synchronously above; both groups can now drain without
+                // shutdown holding a coordinator/controller lock they need.
+                acceptingHttpModelLoads = false;
+                const admittedHttpModelLoads = [...activeHttpModelLoads];
+                // `allSettled` preserves cleanup when either kind of loader rejects;
+                // the original operation still carries that error to its caller.
+                await Promise.allSettled([...admittedHostLoads, ...admittedHttpModelLoads]);
+                // Only once nothing is still serving. The logger writes a request's
+                // record from that request's `finish` handler, so ending the streams
+                // first both drops the record and — with no `error` listener — makes
+                // the write fatal. `attachLogger` guards the second half; this
+                // ordering is what preserves the first.
+                if (logger !== null) {
+                    try {
+                        await logger.close();
+                    }
+                    catch {
+                        /* logging is best effort; never block shutdown */
+                    }
+                }
+                try {
+                    await pagedConfigOverrides.cleanup();
+                }
+                catch {
+                    /* the startup sweep of the next host will reclaim it */
+                }
+            })();
+            return closePromise;
+        },
+    };
+}
+export { discoverModels } from './discover.js';
+export { applyEnginePolicy, engineEnvFor, ENGINE_POLICY_ENV_VARS, LAUNCHER_ENGINE_POLICY, } from './env-policy.js';
+export { attachLogger, resolveLogDir } from './logger.js';
+export { bracketHost, hostUrl, isLoopbackBindHost, pickFreePort } from './net.js';
+export { resolveMlxNodeHome, resolveModelsDir } from './paths.js';
+export { makeSwapController } from './swap.js';
+export { hostTempDirPrefix, isProcessAlive, sweepOrphanHostTempRoots, HOST_TEMP_DIR_STEM, } from './temp-root.js';

package/dist/host/logger.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Request/response logger for `mlx launch claude --verbose`.
+ *
+ * Each HTTP turn is written as one line of newline-delimited JSON to
+ * `requests.ndjson`. Streaming responses capture every chunk written
+ * to the socket so SSE events land verbatim — enough to audit cache
+ * hits (`x-session-cache` header), tool-call round-trips, and the
+ * model's token-level output post-hoc.
+ *
+ * `session.log` is the human-readable companion: one line per request
+ * arrival and completion, for `tail -f` during a live session.
+ */
+import type { Server } from 'node:http';
+export interface Logger {
+    /** Absolute log directory in use. */
+    readonly logDir: string;
+    /** Flush and close the underlying streams. Safe to call multiple times. */
+    close(): Promise<void>;
+}
+/**
+ * Attach request/response capture to `server`. Call `close()` AFTER
+ * `server.close()` resolves: the completion listeners below fire when a
+ * response finishes, so ending the streams first drops the tail of every
+ * request still in flight.
+ */
+export declare function attachLogger(server: Server, logDir: string): Logger;
+/**
+ * Resolve the log directory for a verbose launch.
+ *
+ * Order: explicit `--log-dir` > `MLX_LOG_DIR` env > a fresh timestamped
+ * directory under `<mlxNodeHome>/logs/`. The timestamped default gives
+ * each launch its own dir so concurrent / sequential runs don't
+ * interleave into one file.
+ */
+export declare function resolveLogDir(explicit: string | undefined, mlxNodeHome: string): string;
+//# sourceMappingURL=logger.d.ts.map

package/dist/host/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/host/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,KAAK,EAAmB,MAAM,EAAkB,MAAM,WAAW,CAAC;AAGzE,MAAM,WAAW,MAAM;IACrB,qCAAqC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,2EAA2E;IAC3E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AA+MD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAsKnE;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAMvF"}