@mlx-node/server 0.0.8 → 0.0.9

package/dist/host/index.js

@@ -0,0 +1,325 @@
+/**
+ * `@mlx-node/server/host` — the reusable inference-host bootstrap.
+ *
+ * Everything a process needs to go from "a directory of downloaded models" to
+ * "a listening Anthropic/OpenAI-compatible endpoint with one resident model
+ * and a working `/model` swap", with none of the opinions about WHO supervises
+ * the process.
+ *
+ * That last part is why this is a module rather than a CLI flag. The two known
+ * front-ends invert the supervision relationship:
+ *
+ *   - `mlx launch claude` starts the host, then spawns and supervises a child
+ *     (`claude`), and exits when the child does.
+ *   - The desktop app's Electron `utilityProcess` IS the child; the host is
+ *     supervised, gets told when to shut down, and may be SIGKILLed.
+ *
+ * `mlx serve` is the third, degenerate case — nothing above, nothing below —
+ * and doubles as the terminal-visible reproduction of the sidecar when the
+ * utilityProcess wedges.
+ *
+ * A subpath export rather than part of `@mlx-node/server`'s main entry, so
+ * importing the plain HTTP handler does not drag in `@mlx-node/lm` and model
+ * loading.
+ *
+ * ## Ownership
+ *
+ * The returned host owns, and disposes on {@link InferenceHost.close}:
+ *   1. the HTTP server,
+ *   2. the verbose request logger (when `logDir` is set),
+ *   3. the `PagedConfigOverrideManager`'s temp root.
+ *
+ * The server goes first because the logger records a request from that
+ * request's own `finish`/`close` handler. Ending the log streams while a
+ * request is still draining loses exactly the completion lines a verbose
+ * shutdown exists to capture. The temp root goes last because a
+ * still-draining request may still be reading a cloned config. Each step is
+ * independently guarded — one failing disposer must not strand the ones
+ * after it.
+ */
+import { loadModel as loadModelNative, PagedConfigOverrideManager } from '@mlx-node/lm';
+import { createServer, resolveAuthToken } from '../server.js';
+import { discoverModels } from './discover.js';
+import { applyEnginePolicy } from './env-policy.js';
+import { attachLogger as defaultAttachLogger } from './logger.js';
+import { hostUrl, isLoopbackBindHost, normalizeLoopbackBindHost, pickFreePort } from './net.js';
+import { resolveModelsDir } from './paths.js';
+import { makeSwapController } from './swap.js';
+import { hostTempDirPrefix, sweepOrphanHostTempRoots } from './temp-root.js';
+/** Families `mlx launch claude` has historically forced onto the paged path. */
+export const DEFAULT_PAGED_MODEL_TYPES = ['qwen3_5', 'qwen3_5_moe'];
+/** Thrown when `modelsDir` holds nothing servable. Carries the dir for the caller's message. */
+export class NoModelsDiscoveredError extends Error {
+    modelsDir;
+    constructor(modelsDir) {
+        super(`No models discovered under ${modelsDir}.`);
+        this.modelsDir = modelsDir;
+        this.name = 'NoModelsDiscoveredError';
+    }
+}
+/**
+ * Thrown when a bind reachable from the network was asked for with no shared
+ * secret to gate it.
+ *
+ * There is no safe way to serve that: every route but the `/health` liveness
+ * carve-out runs inference, so an unauthenticated LAN-reachable bind hands
+ * anyone who can route to this machine the GPU, the RAM, and the list of
+ * models on disk. Failing at startup is the only outcome the operator can act
+ * on — serving-with-a-warning is a warning nobody reads scrolling past a
+ * model load.
+ */
+export class InsecureBindError extends Error {
+    host;
+    constructor(host) {
+        super(`Refusing to bind ${host} without an auth token: every route except /health runs inference. ` +
+            `Pass an auth token (mlx serve --auth-token, or MLX_SERVER_AUTH_TOKEN), or bind 127.0.0.1.`);
+        this.host = host;
+        this.name = 'InsecureBindError';
+    }
+}
+/** Thrown when a requested model name is not among the discovered ones. */
+export class ModelNotFoundError extends Error {
+    requested;
+    modelsDir;
+    available;
+    constructor(requested, modelsDir, available) {
+        super(`Model "${requested}" not found under ${modelsDir}.`);
+        this.requested = requested;
+        this.modelsDir = modelsDir;
+        this.available = available;
+        this.name = 'ModelNotFoundError';
+    }
+}
+/** Thrown when a model load is requested after host shutdown reaches its admission boundary. */
+export class InferenceHostClosedError extends Error {
+    constructor() {
+        super('Inference host is closing or closed; model loads are no longer accepted.');
+        this.name = 'InferenceHostClosedError';
+    }
+}
+/**
+ * Start a local inference host.
+ *
+ * Throws {@link NoModelsDiscoveredError} / {@link ModelNotFoundError} /
+ * {@link InsecureBindError} rather than exiting — a library cannot know
+ * whether its caller is a CLI, a test, or an Electron main process. Front-ends
+ * render those into their own messages.
+ */
+export async function createInferenceHost(opts = {}) {
+    const host = opts.host ?? '127.0.0.1';
+    // BEFORE the engine policy, the temp sweep and discovery — all of which
+    // mutate state outside this function — so a refused start leaves nothing
+    // behind. Resolved through `resolveAuthToken` rather than reading
+    // `opts.authToken`, or `MLX_SERVER_AUTH_TOKEN=… mlx serve --host 0.0.0.0`
+    // would be refused despite being fully protected.
+    if (!isLoopbackBindHost(host) && resolveAuthToken(opts.authToken) === undefined) {
+        throw new InsecureBindError(host);
+    }
+    // `[::1]` is valid URL-authority spelling and the security predicate accepts
+    // it as loopback, but Node's listen host must be the bare literal. Normalize
+    // only after the reachability gate, and only the exact safe form.
+    const bindHost = normalizeLoopbackBindHost(host);
+    // FIRST, before anything can touch the engine: the native side latches
+    // these via `OnceLock` on first read, so a policy applied after a load has
+    // silently done nothing.
+    if (opts.enginePolicy !== undefined)
+        applyEnginePolicy(opts.enginePolicy);
+    // Reclaim roots left by hosts that never got to run `close()`. Best effort
+    // and awaited only so a test can observe it; a failure here is not a reason
+    // to refuse to start.
+    if (opts.sweepOrphanTempRoots !== false) {
+        await sweepOrphanHostTempRoots().catch(() => []);
+    }
+    const modelsDir = resolveModelsDir(opts.modelsDir);
+    const models = await discoverModels(modelsDir);
+    if (models.length === 0)
+        throw new NoModelsDiscoveredError(modelsDir);
+    // Precedence: explicit option > ANTHROPIC_MODEL > discovered[0].
+    const requestedModel = opts.model ?? process.env.ANTHROPIC_MODEL;
+    const requestedEntry = requestedModel != null ? models.find((m) => m.name === requestedModel) : undefined;
+    if (requestedModel != null && requestedEntry === undefined) {
+        throw new ModelNotFoundError(requestedModel, modelsDir, models.map((m) => m.name));
+    }
+    const boundEntry = requestedEntry ?? models[0];
+    // `undefined` means "you pick"; `0` means "the kernel picks and I will read
+    // it back". Only the former needs the up-front probe.
+    const requestedPort = opts.port ?? (await pickFreePort());
+    // The swap controller needs the registry from the server instance, but the
+    // server needs the controller's callbacks at construction. Bridge via a
+    // late-bound holder: the callbacks capture `ctrlRef.current` by closure.
+    const ctrlRef = { current: null };
+    let acceptingHttpModelLoads = true;
+    const activeHttpModelLoads = new Set();
+    const resolveHttpModel = (name) => {
+        if (!acceptingHttpModelLoads)
+            return Promise.reject(new InferenceHostClosedError());
+        const operation = ctrlRef.current.resolveModel(name);
+        activeHttpModelLoads.add(operation);
+        void operation.then(() => activeHttpModelLoads.delete(operation), () => activeHttpModelLoads.delete(operation));
+        return operation;
+    };
+    const serverConfig = {
+        port: requestedPort,
+        host: bindHost,
+        resolveModel: resolveHttpModel,
+        listModels: () => ctrlRef.current.listModels(),
+        ...(opts.authToken !== undefined ? { authToken: opts.authToken } : {}),
+        ...(opts.storePath !== undefined ? { storePath: opts.storePath } : {}),
+        ...(opts.disableStore !== undefined ? { disableStore: opts.disableStore } : {}),
+    };
+    const server = await createServer(serverConfig);
+    // Wrap the loader so managed families get `use_block_paged_cache: true`
+    // injected via a temp-dir clone with a patched config.json. The temp root is
+    // named after our pid so a SIGKILLed host's root can be reclaimed by the
+    // next host's startup sweep — see `temp-root.ts`.
+    const pagedConfigOverrides = new PagedConfigOverrideManager({
+        modelTypes: opts.pagedModelTypes ?? DEFAULT_PAGED_MODEL_TYPES,
+        tempDirPrefix: hostTempDirPrefix(),
+    });
+    const loadModelFn = opts.loadModel ?? loadModelNative;
+    const loadModelPagedAware = async (path) => loadModelFn(await pagedConfigOverrides.resolve(path));
+    const controller = makeSwapController(models, server.registry, loadModelPagedAware, boundEntry.name);
+    ctrlRef.current = controller;
+    // Attach AFTER `createServer` so the wrapper sees every incoming request,
+    // including the `GET /v1/models` a client fires on startup.
+    //
+    // By this point the socket is bound and `ctrlRef.current` is wired, so the
+    // endpoint already answers real requests. `attachLogger` opens with a
+    // synchronous `mkdirSync`, which throws on an unwritable `--log-dir`; without
+    // this rollback the rejection would strand a fully working inference server
+    // with no handle left to close it. Every caller in this repo exits the
+    // process on failure, so the leak is only reachable by an in-process
+    // embedder — which is exactly who this module is for.
+    let logger = null;
+    if (opts.logDir !== undefined) {
+        try {
+            logger = (opts.attachLogger ?? defaultAttachLogger)(server.server, opts.logDir);
+        }
+        catch (err) {
+            // Mirror `close()`'s order and its independent guards, and rethrow the
+            // ORIGINAL failure — a secondary close error must not mask the EACCES
+            // that actually explains what went wrong.
+            try {
+                await server.close();
+            }
+            catch {
+                /* already down, or a socket refused to die; fall through */
+            }
+            // A forced close destroys sockets, not the async handler promises
+            // `http.createServer` discarded. Close resolver admission only after
+            // the server has finished its graceful window, then drain any lazy load
+            // that was already inside the controller before removing its temp files.
+            acceptingHttpModelLoads = false;
+            await Promise.allSettled(activeHttpModelLoads);
+            try {
+                await pagedConfigOverrides.cleanup();
+            }
+            catch {
+                /* the startup sweep of the next host will reclaim it */
+            }
+            throw err;
+        }
+    }
+    const address = server.server.address();
+    const boundPort = address !== null && typeof address === 'object' ? address.port : requestedPort;
+    const byName = new Map(models.map((m) => [m.name, m]));
+    // `closeStarted` is an admission latch, checked and flipped synchronously:
+    // once close() returns its promise no later out-of-band load can join this
+    // set. The operations themselves retain the normal coordinator/controller
+    // ordering; shutdown only observes their promises and never takes a lock
+    // they need, so queued swaps can drain without deadlocking against close.
+    let closeStarted = false;
+    const activeHostLoads = new Set();
+    let closePromise = null;
+    return {
+        url: hostUrl(host, boundPort),
+        port: boundPort,
+        host,
+        modelsDir,
+        models,
+        boundModel: boundEntry.name,
+        logDir: logger?.logDir ?? null,
+        server,
+        health: () => server.health(),
+        loadModel(name) {
+            if (closeStarted)
+                return Promise.reject(new InferenceHostClosedError());
+            if (!byName.has(name)) {
+                return Promise.reject(new ModelNotFoundError(name, modelsDir, models.map((m) => m.name)));
+            }
+            // Same nesting as `runGuardedModelLoad`: the drain suspension must be
+            // OUTSIDE the writer lock so the armed `clearCache()` timer cannot fire
+            // while we are parked waiting for the lock. The HTTP path takes the
+            // locks in this same order, so the two can never deadlock against each
+            // other on the controller's internal serialization.
+            const operation = server.withSuspendedDrains(async () => {
+                await server.modelWork.withModelLoad(() => controller.resolveModel(name), name);
+            });
+            activeHostLoads.add(operation);
+            // Use a two-arm `then`, rather than an ignored `finally()` promise:
+            // cleanup must not manufacture an unhandled rejection when the caller
+            // legitimately observes a failed load through `operation`.
+            void operation.then(() => activeHostLoads.delete(operation), () => activeHostLoads.delete(operation));
+            return operation;
+        },
+        close(closeOpts) {
+            if (closePromise !== null)
+                return closePromise;
+            // Flip the latch and snapshot in the same synchronous turn. Every call
+            // in the snapshot was fully admitted before shutdown; no later call can
+            // race into the set after this point.
+            closeStarted = true;
+            const admittedHostLoads = [...activeHostLoads];
+            closePromise = (async () => {
+                // Every step is independently guarded: a server that fails to close
+                // must still leave the log streams ended and the temp root
+                // reclaimed. The temp root is the only one of the three that leaks
+                // OUTSIDE this process.
+                try {
+                    await server.close(closeOpts);
+                }
+                catch {
+                    /* already down, or a socket refused to die; fall through */
+                }
+                // A forced server close destroys request sockets but does not await
+                // the async handler promises. Stop any handler that has not reached
+                // the resolver from starting a late load, then snapshot resolver calls
+                // already inside the controller. Direct calls were closed and
+                // snapshotted synchronously above; both groups can now drain without
+                // shutdown holding a coordinator/controller lock they need.
+                acceptingHttpModelLoads = false;
+                const admittedHttpModelLoads = [...activeHttpModelLoads];
+                // `allSettled` preserves cleanup when either kind of loader rejects;
+                // the original operation still carries that error to its caller.
+                await Promise.allSettled([...admittedHostLoads, ...admittedHttpModelLoads]);
+                // Only once nothing is still serving. The logger writes a request's
+                // record from that request's `finish` handler, so ending the streams
+                // first both drops the record and — with no `error` listener — makes
+                // the write fatal. `attachLogger` guards the second half; this
+                // ordering is what preserves the first.
+                if (logger !== null) {
+                    try {
+                        await logger.close();
+                    }
+                    catch {
+                        /* logging is best effort; never block shutdown */
+                    }
+                }
+                try {
+                    await pagedConfigOverrides.cleanup();
+                }
+                catch {
+                    /* the startup sweep of the next host will reclaim it */
+                }
+            })();
+            return closePromise;
+        },
+    };
+}
+export { discoverModels } from './discover.js';
+export { applyEnginePolicy, engineEnvFor, ENGINE_POLICY_ENV_VARS, LAUNCHER_ENGINE_POLICY, } from './env-policy.js';
+export { attachLogger, resolveLogDir } from './logger.js';
+export { bracketHost, hostUrl, isLoopbackBindHost, pickFreePort } from './net.js';
+export { resolveMlxNodeHome, resolveModelsDir } from './paths.js';
+export { makeSwapController } from './swap.js';
+export { hostTempDirPrefix, isProcessAlive, sweepOrphanHostTempRoots, HOST_TEMP_DIR_STEM, } from './temp-root.js';

package/dist/host/logger.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Request/response logger for `mlx launch claude --verbose`.
+ *
+ * Each HTTP turn is written as one line of newline-delimited JSON to
+ * `requests.ndjson`. Streaming responses capture every chunk written
+ * to the socket so SSE events land verbatim — enough to audit cache
+ * hits (`x-session-cache` header), tool-call round-trips, and the
+ * model's token-level output post-hoc.
+ *
+ * `session.log` is the human-readable companion: one line per request
+ * arrival and completion, for `tail -f` during a live session.
+ */
+import type { Server } from 'node:http';
+export interface Logger {
+    /** Absolute log directory in use. */
+    readonly logDir: string;
+    /** Flush and close the underlying streams. Safe to call multiple times. */
+    close(): Promise<void>;
+}
+/**
+ * Attach request/response capture to `server`. Call `close()` AFTER
+ * `server.close()` resolves: the completion listeners below fire when a
+ * response finishes, so ending the streams first drops the tail of every
+ * request still in flight.
+ */
+export declare function attachLogger(server: Server, logDir: string): Logger;
+/**
+ * Resolve the log directory for a verbose launch.
+ *
+ * Order: explicit `--log-dir` > `MLX_LOG_DIR` env > a fresh timestamped
+ * directory under `<mlxNodeHome>/logs/`. The timestamped default gives
+ * each launch its own dir so concurrent / sequential runs don't
+ * interleave into one file.
+ */
+export declare function resolveLogDir(explicit: string | undefined, mlxNodeHome: string): string;
+//# sourceMappingURL=logger.d.ts.map

package/dist/host/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/host/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,OAAO,KAAK,EAAmB,MAAM,EAAkB,MAAM,WAAW,CAAC;AAGzE,MAAM,WAAW,MAAM;IACrB,qCAAqC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,2EAA2E;IAC3E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AA+MD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAsKnE;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAMvF"}