@mlx-node/server 0.0.8 → 0.0.9

package/dist/host/swap.js

@@ -0,0 +1,178 @@
+/**
+ * Single-resident lazy-load policy for an inference host.
+ *
+ * The host discovers every local model up-front but loads at most one into
+ * the `ModelRegistry` at a time. Switching models (e.g. via Claude Code's
+ * `/model` picker, or the desktop app's model menu) unregisters the previous
+ * instance, letting GC + native destructors reclaim memory, before loading
+ * the new one.
+ */
+/**
+ * Build the `resolveModel` + `listModels` callbacks for the handler.
+ *
+ * `loadModelFn` is injected so tests can stub it without touching native code.
+ * The controller serializes every `resolveModel` invocation on a single
+ * promise chain so two concurrent requests for different-but-currently-
+ * unloaded models cannot race on the native compiled-path globals.
+ */
+export function makeSwapController(discovered, registry, loadModelFn, defaultName) {
+    const byName = new Map();
+    for (const entry of discovered)
+        byName.set(entry.name, entry);
+    const ordered = [...discovered].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    // Which entry unknown names (haiku subagent dispatches, etc.) fall back
+    // to before anything is resident. Defaults to discovered[0], but the
+    // caller can pin it to the user's `--model` pick so the first haiku
+    // title-gen doesn't trigger a load of the alphabetically-first model
+    // followed by an immediate swap to the user's real choice.
+    const fallbackEntry = (defaultName != null ? byName.get(defaultName) : undefined) ?? discovered[0];
+    let resident = null;
+    // Names we registered as aliases to the current resident (e.g.
+    // Claude Code's hardcoded `claude-haiku-*` for subagent dispatches /
+    // title generation). Tracked so we can unregister them on `/model`
+    // swap — otherwise an alias's refcount would keep the old binding
+    // alive past the user's swap.
+    const aliases = new Set();
+    let currentOp = Promise.resolve();
+    async function resolveModel(name) {
+        // Fast path: already registered under this name (either as a real
+        // resident or as an alias we previously installed). Avoid chaining.
+        if (registry.get(name))
+            return;
+        const next = currentOp.then(async () => {
+            // Re-check under the serialized section — a prior waiter may have loaded it.
+            if (registry.get(name))
+                return;
+            // Pick the discovered entry to resolve against. If the requested
+            // name matches a discovered model, use it. Otherwise (unknown
+            // name — Claude Code's hardcoded small-fast-model, etc.) fall
+            // through to the current resident so subagent dispatches don't
+            // 404, loading discovered[0] on first boot if nothing is resident
+            // yet.
+            //
+            // CRITICAL: this must read `resident` at RUN time, not QUEUE time.
+            // If we capture it before chaining onto `currentOp`, a swap that
+            // ran ahead of us will leave us with a stale target — e.g. a haiku
+            // alias request that arrived during a `/model a → b` switch would
+            // capture `targetEntry = a`, then re-bind itself to `a` and undo
+            // the user's switch when its turn finally comes around.
+            const knownEntry = byName.get(name);
+            const targetEntry = knownEntry ?? (resident ? (byName.get(resident.name) ?? fallbackEntry) : fallbackEntry);
+            const isAlias = targetEntry.name !== name;
+            // Swap out any stale resident that isn't the target.
+            //
+            // We do NOT unregister the aliases here: in-flight messages.ts
+            // requests may be microtask-racing between "resolveModel returned"
+            // and "registry.get(body.model)" and dropping the alias in that
+            // window yields a spurious 404. Instead we carry the alias set
+            // across the swap and re-point them to the new resident below,
+            // so the name always resolves to *some* live instance.
+            const oldResident = resident;
+            const carriedAliases = new Set(aliases);
+            if (oldResident && oldResident.name !== targetEntry.name) {
+                // Drop our local alias bookkeeping AND the old resident's primary
+                // name binding, but leave the alias *names* in the registry pointed
+                // at the old model — they hold the only refcount preventing GC,
+                // and an in-flight `registry.get(alias)` must keep resolving to
+                // *some* live instance until the new model is in hand.
+                aliases.clear();
+                registry.unregister(oldResident.name);
+                resident = null;
+            }
+            // Ensure the target is resident. If the load throws, restore the
+            // pre-swap controller state so future swap attempts know about the
+            // aliases we just cleared — otherwise the alias *names* stay bound
+            // in the registry to the old model object forever (alias bindings
+            // hold their own refcount), but the controller forgets they exist
+            // and never repoints them, leaving alias-routed traffic permanently
+            // pinned to a stale model.
+            let instance = registry.get(targetEntry.name);
+            if (!instance) {
+                let loaded;
+                try {
+                    loaded = await loadModelFn(targetEntry.path);
+                }
+                catch (err) {
+                    // Recovery: re-populate the controller's alias set so the next
+                    // resolveModel call still owns them. The alias→old-model bindings
+                    // are still live in the registry (we never unregistered them), so
+                    // we can recover the old model object via any surviving alias and
+                    // re-bind the old resident's primary name.
+                    //
+                    // If `carriedAliases` is empty (no aliases ever existed) AND we
+                    // unregistered `oldResident.name`, the binding's refcount may have
+                    // hit zero and the model is gone. There's nothing the controller
+                    // can do to recover in that case — the user will need to /model-
+                    // pick again. This is acceptable: alias-less load failures are
+                    // rare, and the user-facing symptom is "prior model gone, please
+                    // re-pick", not silently-wrong responses.
+                    for (const aliasName of carriedAliases)
+                        aliases.add(aliasName);
+                    if (oldResident) {
+                        let oldInstance;
+                        for (const aliasName of carriedAliases) {
+                            const probe = registry.get(aliasName);
+                            if (probe) {
+                                oldInstance = probe;
+                                break;
+                            }
+                        }
+                        if (oldInstance) {
+                            const oldEntry = byName.get(oldResident.name);
+                            registry.register(oldResident.name, oldInstance, {
+                                samplingDefaults: oldEntry?.preset.sampling,
+                                maxOutputTokens: oldEntry?.preset.maxOutputTokens,
+                            });
+                            resident = oldResident;
+                        }
+                    }
+                    throw err;
+                }
+                instance = loaded;
+                registry.register(targetEntry.name, instance, {
+                    samplingDefaults: targetEntry.preset.sampling,
+                    maxOutputTokens: targetEntry.preset.maxOutputTokens,
+                });
+                resident = { name: targetEntry.name };
+            }
+            else if (!resident) {
+                resident = { name: targetEntry.name };
+            }
+            // Re-point any aliases carried across the swap onto the new
+            // resident. `registry.register(sameName, differentModel)` drops
+            // the old binding's refcount and installs the new one atomically,
+            // so any concurrent `registry.get(alias)` either sees the old or
+            // new instance — never null.
+            for (const aliasName of carriedAliases) {
+                if (aliasName === targetEntry.name)
+                    continue;
+                registry.register(aliasName, instance, {
+                    samplingDefaults: targetEntry.preset.sampling,
+                    maxOutputTokens: targetEntry.preset.maxOutputTokens,
+                });
+                aliases.add(aliasName);
+            }
+            // For unknown names, register an alias on the resident instance so
+            // the endpoint's `registry.get(name)` lookup succeeds.
+            if (isAlias) {
+                registry.register(name, instance, {
+                    samplingDefaults: targetEntry.preset.sampling,
+                    maxOutputTokens: targetEntry.preset.maxOutputTokens,
+                });
+                aliases.add(name);
+            }
+        });
+        currentOp = next.catch(() => undefined);
+        await next;
+    }
+    function listModels() {
+        const created = Math.floor(Date.now() / 1000);
+        return ordered.map((entry) => ({
+            id: entry.name,
+            object: 'model',
+            created,
+            owned_by: 'mlx-node',
+        }));
+    }
+    return { resolveModel, listModels };
+}

package/dist/host/temp-root.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Pid-scoped temp roots for the host's paged-config overrides, plus the
+ * startup sweep that reclaims the ones a killed host left behind.
+ *
+ * `PagedConfigOverrideManager` clones a checkpoint directory (config.json
+ * rewritten, everything else symlinked) into a temp root and removes that root
+ * in `cleanup()`. `cleanup()` runs on a normal shutdown — but the host's
+ * headline deployment is an Electron `utilityProcess`, and a `utilityProcess`
+ * can be SIGKILLed (app force-quit, OOM killer, `kill -9`). SIGKILL runs no
+ * handler, so without a sweep EVERY hard kill leaks a root. The clones are
+ * symlink farms rather than copies, so the leak is inodes and directory
+ * entries rather than model-sized bytes — but the roots accumulate forever and
+ * a partially-written clone can hold a real config.json.
+ *
+ * The reclaim strategy is "name the owner in the directory name": the manager
+ * gets a `tempDirPrefix` of `mlx-inference-host-<pid>-`, `mkdtemp` appends its
+ * own random suffix, and {@link sweepOrphanHostTempRoots} parses the pid back
+ * out and removes any root whose owner is gone.
+ *
+ * Known limitation: pid reuse. A long-dead host's root whose pid has since
+ * been recycled by an unrelated process is SPARED (never wrongly deleted), so
+ * the failure mode is a leaked directory, not data loss.
+ */
+/** Shared stem. A directory is host-owned iff its name starts with this. */
+export declare const HOST_TEMP_DIR_STEM = "mlx-inference-host-";
+/**
+ * `tempDirPrefix` to hand `PagedConfigOverrideManager` so the root it creates
+ * carries its owner's pid.
+ */
+export declare function hostTempDirPrefix(pid?: number): string;
+/**
+ * Does a process with this pid exist?
+ *
+ * `process.kill(pid, 0)` sends no signal; it only performs the permission +
+ * existence check. `ESRCH` is the sole "gone" answer — `EPERM` means the
+ * process exists but belongs to another user, which must count as ALIVE so a
+ * multi-user box never has one user's sweep delete another user's live root.
+ */
+export declare function isProcessAlive(pid: number): boolean;
+export interface SweepOrphanTempRootsOptions {
+    /** Directory to scan. Defaults to the OS temp dir. */
+    root?: string;
+    /** Our own pid — never swept, however the liveness probe answers. */
+    selfPid?: number;
+    /** Injectable liveness probe. Defaults to {@link isProcessAlive}. */
+    isAlive?: (pid: number) => boolean;
+}
+/**
+ * Remove every host temp root whose owning pid is no longer alive.
+ *
+ * Best effort by contract: a scan or unlink failure (permissions, a root a
+ * concurrently-exiting host is removing under us) is swallowed, because a
+ * housekeeping step must never be the reason a host refuses to start.
+ * Returns the absolute paths actually removed.
+ */
+export declare function sweepOrphanHostTempRoots(opts?: SweepOrphanTempRootsOptions): Promise<string[]>;
+//# sourceMappingURL=temp-root.d.ts.map

package/dist/host/temp-root.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"temp-root.d.ts","sourceRoot":"","sources":["../../src/host/temp-root.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAMH,4EAA4E;AAC5E,eAAO,MAAM,kBAAkB,wBAAwB,CAAC;AAKxD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,GAAE,MAAoB,GAAG,MAAM,CAEnE;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAWnD;AAED,MAAM,WAAW,2BAA2B;IAC1C,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACpC;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,IAAI,GAAE,2BAAgC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA6BxG"}

package/dist/host/temp-root.js

@@ -0,0 +1,99 @@
+/**
+ * Pid-scoped temp roots for the host's paged-config overrides, plus the
+ * startup sweep that reclaims the ones a killed host left behind.
+ *
+ * `PagedConfigOverrideManager` clones a checkpoint directory (config.json
+ * rewritten, everything else symlinked) into a temp root and removes that root
+ * in `cleanup()`. `cleanup()` runs on a normal shutdown — but the host's
+ * headline deployment is an Electron `utilityProcess`, and a `utilityProcess`
+ * can be SIGKILLed (app force-quit, OOM killer, `kill -9`). SIGKILL runs no
+ * handler, so without a sweep EVERY hard kill leaks a root. The clones are
+ * symlink farms rather than copies, so the leak is inodes and directory
+ * entries rather than model-sized bytes — but the roots accumulate forever and
+ * a partially-written clone can hold a real config.json.
+ *
+ * The reclaim strategy is "name the owner in the directory name": the manager
+ * gets a `tempDirPrefix` of `mlx-inference-host-<pid>-`, `mkdtemp` appends its
+ * own random suffix, and {@link sweepOrphanHostTempRoots} parses the pid back
+ * out and removes any root whose owner is gone.
+ *
+ * Known limitation: pid reuse. A long-dead host's root whose pid has since
+ * been recycled by an unrelated process is SPARED (never wrongly deleted), so
+ * the failure mode is a leaked directory, not data loss.
+ */
+import { readdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+/** Shared stem. A directory is host-owned iff its name starts with this. */
+export const HOST_TEMP_DIR_STEM = 'mlx-inference-host-';
+/** Matches `mlx-inference-host-<pid>-<mkdtemp suffix>`. */
+const HOST_TEMP_DIR_RE = /^mlx-inference-host-(\d+)-/;
+/**
+ * `tempDirPrefix` to hand `PagedConfigOverrideManager` so the root it creates
+ * carries its owner's pid.
+ */
+export function hostTempDirPrefix(pid = process.pid) {
+    return `${HOST_TEMP_DIR_STEM}${pid}-`;
+}
+/**
+ * Does a process with this pid exist?
+ *
+ * `process.kill(pid, 0)` sends no signal; it only performs the permission +
+ * existence check. `ESRCH` is the sole "gone" answer — `EPERM` means the
+ * process exists but belongs to another user, which must count as ALIVE so a
+ * multi-user box never has one user's sweep delete another user's live root.
+ */
+export function isProcessAlive(pid) {
+    // pid 0 addresses the caller's whole process group on POSIX and pid < 0 a
+    // group by id; neither is ever a real owner, and signalling them would be
+    // actively dangerous. Treat as alive so they are never swept.
+    if (!Number.isInteger(pid) || pid <= 0)
+        return true;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code !== 'ESRCH';
+    }
+}
+/**
+ * Remove every host temp root whose owning pid is no longer alive.
+ *
+ * Best effort by contract: a scan or unlink failure (permissions, a root a
+ * concurrently-exiting host is removing under us) is swallowed, because a
+ * housekeeping step must never be the reason a host refuses to start.
+ * Returns the absolute paths actually removed.
+ */
+export async function sweepOrphanHostTempRoots(opts = {}) {
+    const root = opts.root ?? tmpdir();
+    const selfPid = opts.selfPid ?? process.pid;
+    const isAlive = opts.isAlive ?? isProcessAlive;
+    let entries;
+    try {
+        entries = await readdir(root);
+    }
+    catch {
+        return [];
+    }
+    const removed = [];
+    for (const name of entries) {
+        const match = HOST_TEMP_DIR_RE.exec(name);
+        if (match === null)
+            continue;
+        const pid = Number.parseInt(match[1], 10);
+        if (pid === selfPid)
+            continue;
+        if (isAlive(pid))
+            continue;
+        const full = join(root, name);
+        try {
+            await rm(full, { recursive: true, force: true });
+            removed.push(full);
+        }
+        catch {
+            /* another sweeper won the race, or we lack permission; leave it */
+        }
+    }
+    return removed;
+}

package/dist/index.d.ts

@@ -5,7 +5,16 @@
  * `GET /v1/models`, in both streaming (SSE) and non-streaming modes.
  */
 export { createServer } from './server.js';
-export type { ServerConfig, ServerInstance } from './server.js';
+export type { CloseOptions, CloseResult, ServerConfig, ServerInstance } from './server.js';
+/**
+ * Readiness reporting. `deriveHealthStatus` is pure — a supervisor can reuse
+ * it to classify a `ServerHealth` body fetched over HTTP.
+ */
+export { createHealthReporter, deriveHealthStatus, toMinimalHealth } from './health.js';
+export type { HealthReporterDeps, HealthStatusInputs, ModelLoadRecord, ServerHealth, ServerHealthMinimal, ServerHealthStatus, } from './health.js';
+export { ModelWorkCoordinator } from './model-work-coordinator.js';
+export type { ModelLoadOutcome } from './model-work-coordinator.js';
+export type { GuardedLoadDeps, LoadModelOptions } from './load-model.js';
 /**
  * Internal helpers re-exported for unit testing only. Not part of the
  * supported public API — names may change without notice.
@@ -25,5 +34,5 @@ export type { LaunchPreset } from './presets.js';
 export type { PublicModelEntry } from './handler.js';
 export type { ResponsesAPIRequest, ResponseObject, ResponseUsage, ResponseError, InputItem, InputMessage, InputFunctionCall, InputFunctionCallOutput, OutputItem, MessageOutputItem, ReasoningOutputItem, FunctionCallOutputItem, OutputTextPart, SummaryTextPart, ResponsesToolDefinition, ContentPart, InputTextPart, StreamEvent, } from './types.js';
 export type { AnthropicCountTokensRequest, AnthropicCountTokensResponse, AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicMessage, AnthropicContentBlock, AnthropicTextContentBlock, AnthropicImageContentBlock, AnthropicToolResultContentBlock, AnthropicToolUseContentBlock, AnthropicThinkingContentBlock, AnthropicToolDefinition, AnthropicToolChoice, AnthropicResponseContent, AnthropicResponseTextBlock, AnthropicResponseThinkingBlock, AnthropicResponseToolUseBlock, AnthropicUsage, AnthropicStreamEvent, AnthropicMessageStartEvent, AnthropicContentBlockStartEvent, AnthropicContentBlockDeltaEvent, AnthropicContentBlockStopEvent, AnthropicMessageDeltaEvent, AnthropicMessageStopEvent, AnthropicDelta, AnthropicTextDelta, AnthropicThinkingDelta, AnthropicInputJsonDelta, SystemBlock, } from './types-anthropic.js';
-export { writeSSEEvent, beginSSE, endSSE } from './streaming.js';
+export { writeSSEEvent, beginSSE, endSSE, activeSSEStreamCount } from './streaming.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE;;;GAGG;AACH,OAAO,EAAE,eAAe,IAAI,iBAAiB,EAAE,mBAAmB,IAAI,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACjH,OAAO,EACL,iBAAiB,IAAI,mBAAmB,EACxC,sBAAsB,IAAI,wBAAwB,EAClD,2BAA2B,IAAI,6BAA6B,GAC7D,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEtG,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxE,YAAY,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACzF,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAS1D,OAAO,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACxH,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,UAAU,EACV,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,uBAAuB,EACvB,WAAW,EACX,aAAa,EACb,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,2BAA2B,EAC3B,4BAA4B,EAC5B,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,EACrB,yBAAyB,EACzB,0BAA0B,EAC1B,+BAA+B,EAC/B,4BAA4B,EAC5B,6BAA6B,EAC7B,uBAAuB,EACvB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,8BAA8B,EAC9B,6BAA6B,EAC7B,cAAc,EACd,oBAAoB,EACpB,0BAA0B,EAC1B,+BAA+B,EAC/B,+BAA+B,EAC/B,8BAA8B,EAC9B,0BAA0B,EAC1B,yBAAyB,EACzB,cAAc,EACd,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE3F;;;GAGG;AACH,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACxF,YAAY,EACV,kBAAkB,EAClB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,kBAAkB,GACnB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,YAAY,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEzE;;;GAGG;AACH,OAAO,EAAE,eAAe,IAAI,iBAAiB,EAAE,mBAAmB,IAAI,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACjH,OAAO,EACL,iBAAiB,IAAI,mBAAmB,EACxC,sBAAsB,IAAI,wBAAwB,EAClD,2BAA2B,IAAI,6BAA6B,GAC7D,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEtG,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxE,YAAY,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACzF,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAS1D,OAAO,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACxH,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,UAAU,EACV,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,uBAAuB,EACvB,WAAW,EACX,aAAa,EACb,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,2BAA2B,EAC3B,4BAA4B,EAC5B,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,EACrB,yBAAyB,EACzB,0BAA0B,EAC1B,+BAA+B,EAC/B,4BAA4B,EAC5B,6BAA6B,EAC7B,uBAAuB,EACvB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,8BAA8B,EAC9B,6BAA6B,EAC7B,cAAc,EACd,oBAAoB,EACpB,0BAA0B,EAC1B,+BAA+B,EAC/B,+BAA+B,EAC/B,8BAA8B,EAC9B,0BAA0B,EAC1B,yBAAyB,EACzB,cAAc,EACd,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/index.js

@@ -5,6 +5,12 @@
  * `GET /v1/models`, in both streaming (SSE) and non-streaming modes.
  */
 export { createServer } from './server.js';
+/**
+ * Readiness reporting. `deriveHealthStatus` is pure — a supervisor can reuse
+ * it to classify a `ServerHealth` body fetched over HTTP.
+ */
+export { createHealthReporter, deriveHealthStatus, toMinimalHealth } from './health.js';
+export { ModelWorkCoordinator } from './model-work-coordinator.js';
 /**
  * Internal helpers re-exported for unit testing only. Not part of the
  * supported public API — names may change without notice.
@@ -23,4 +29,4 @@ export { resolveServerTuningForUsage } from './timing.js';
 // import it from the deep path
 // `packages/server/src/session-registry.js` instead.
 export { QWEN_SAMPLING_DEFAULTS, GEMMA4_SAMPLING_DEFAULTS, LFM2_SAMPLING_DEFAULTS, LAUNCH_PRESETS } from './presets.js';
-export { writeSSEEvent, beginSSE, endSSE } from './streaming.js';
+export { writeSSEEvent, beginSSE, endSSE, activeSSEStreamCount } from './streaming.js';

package/dist/load-model.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Safe out-of-band model load.
+ *
+ * A supervisor (or a `/model`-picker UI) needs to swap the resident model on a
+ * server that is already serving. Doing that naively races the Metal
+ * allocator — a corruption/abort bug, not a latency bug — so the bracketing
+ * order below is load-bearing:
+ *
+ * ```text
+ * idleSweeper.withSuspendedDrains(          ← OUTSIDE
+ *   modelWorkCoordinator.withModelLoad(     ← INSIDE
+ *     load() ; registry.register(...)
+ *   , label))
+ * ```
+ *
+ * Why the suspension must be OUTSIDE the writer lock: `endRequest()` arms a
+ * drain timer at `t + idleClearCacheMs` whose callback runs
+ * `__internal__.clearCache()`, walking the process-wide MLX free pool. A load
+ * that arrives while another load holds the writer slot parks inside
+ * `acquireWrite()` — and if the suspension were taken only after the lock was
+ * won, that armed timer would still be live during the wait AND during the
+ * hand-off, firing mid-materialization. Suspending first covers the wait, the
+ * lock hand-off, `load()`, and `register()` as one interval.
+ *
+ * Why the writer lock must be INSIDE: `withModelLoad` takes the exclusive
+ * writer slot, which excludes inference readers for exactly as long as
+ * weights are being materialized — no longer. Hoisting it outside the
+ * suspension would gain nothing and would hold readers off during the drain
+ * bookkeeping too.
+ *
+ * See `ServerInstance.withSuspendedDrains` and the `__internal__.clearCache`
+ * rustdoc in `packages/core/index.d.cts` for the underlying contracts.
+ */
+import type { ChatConfig } from '@mlx-node/core';
+import type { IdleSweeper } from './idle-sweeper.js';
+import type { ModelWorkCoordinator } from './model-work-coordinator.js';
+import type { ModelRegistry, ServableModel } from './registry.js';
+export interface GuardedLoadDeps {
+    /** Only `withSuspendedDrains` is used; typed narrowly so tests can double it. */
+    idleSweeper: Pick<IdleSweeper, 'withSuspendedDrains'>;
+    modelWorkCoordinator: ModelWorkCoordinator;
+    registry: ModelRegistry;
+}
+export interface LoadModelOptions {
+    /** Primary registration name. Also used as the `/health` load label. */
+    name: string;
+    /** Materializes the model. Invoked exactly once, inside both brackets. */
+    load: () => Promise<ServableModel>;
+    /**
+     * Extra names bound to the SAME instance. Aliases share one
+     * `SessionRegistry`, preserving the single-warm invariant across names.
+     * An alias equal to `name` is ignored rather than re-registered.
+     */
+    aliases?: string[];
+    /** Per-model sampling defaults; forwarded to every alias too. */
+    samplingDefaults?: ChatConfig;
+    /** Per-model output-token clamp; forwarded to every alias too. */
+    maxOutputTokens?: number;
+}
+/**
+ * Load a model and register it under `name` (plus any aliases) with drains
+ * suspended and inference excluded for the whole materialization window.
+ *
+ * Rejects with the underlying error if `load()` (or `register`) throws;
+ * both brackets unwind via their own `finally`, so neither the suspend
+ * counter nor the writer lock can leak.
+ */
+export declare function runGuardedModelLoad(deps: GuardedLoadDeps, opts: LoadModelOptions): Promise<void>;
+//# sourceMappingURL=load-model.d.ts.map

package/dist/load-model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-model.d.ts","sourceRoot":"","sources":["../src/load-model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAmB,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnF,MAAM,WAAW,eAAe;IAC9B,iFAAiF;IACjF,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,qBAAqB,CAAC,CAAC;IACtD,oBAAoB,EAAE,oBAAoB,CAAC;IAC3C,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,wEAAwE;IACxE,IAAI,EAAE,MAAM,CAAC;IACb,0EAA0E;IAC1E,IAAI,EAAE,MAAM,OAAO,CAAC,aAAa,CAAC,CAAC;IACnC;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,iEAAiE;IACjE,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B,kEAAkE;IAClE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBtG"}

package/dist/load-model.js

@@ -0,0 +1,63 @@
+/**
+ * Safe out-of-band model load.
+ *
+ * A supervisor (or a `/model`-picker UI) needs to swap the resident model on a
+ * server that is already serving. Doing that naively races the Metal
+ * allocator — a corruption/abort bug, not a latency bug — so the bracketing
+ * order below is load-bearing:
+ *
+ * ```text
+ * idleSweeper.withSuspendedDrains(          ← OUTSIDE
+ *   modelWorkCoordinator.withModelLoad(     ← INSIDE
+ *     load() ; registry.register(...)
+ *   , label))
+ * ```
+ *
+ * Why the suspension must be OUTSIDE the writer lock: `endRequest()` arms a
+ * drain timer at `t + idleClearCacheMs` whose callback runs
+ * `__internal__.clearCache()`, walking the process-wide MLX free pool. A load
+ * that arrives while another load holds the writer slot parks inside
+ * `acquireWrite()` — and if the suspension were taken only after the lock was
+ * won, that armed timer would still be live during the wait AND during the
+ * hand-off, firing mid-materialization. Suspending first covers the wait, the
+ * lock hand-off, `load()`, and `register()` as one interval.
+ *
+ * Why the writer lock must be INSIDE: `withModelLoad` takes the exclusive
+ * writer slot, which excludes inference readers for exactly as long as
+ * weights are being materialized — no longer. Hoisting it outside the
+ * suspension would gain nothing and would hold readers off during the drain
+ * bookkeeping too.
+ *
+ * See `ServerInstance.withSuspendedDrains` and the `__internal__.clearCache`
+ * rustdoc in `packages/core/index.d.cts` for the underlying contracts.
+ */
+/**
+ * Load a model and register it under `name` (plus any aliases) with drains
+ * suspended and inference excluded for the whole materialization window.
+ *
+ * Rejects with the underlying error if `load()` (or `register`) throws;
+ * both brackets unwind via their own `finally`, so neither the suspend
+ * counter nor the writer lock can leak.
+ */
+export async function runGuardedModelLoad(deps, opts) {
+    // Build the register options once, omitting keys the caller never set.
+    // `ModelRegistry.register` uses `'samplingDefaults' in opts` to decide
+    // whether to OVERWRITE an existing binding's defaults, so passing an
+    // explicit `undefined` would silently clear them on a re-register.
+    const registerOpts = {};
+    if (opts.samplingDefaults !== undefined)
+        registerOpts.samplingDefaults = opts.samplingDefaults;
+    if (opts.maxOutputTokens !== undefined)
+        registerOpts.maxOutputTokens = opts.maxOutputTokens;
+    await deps.idleSweeper.withSuspendedDrains(async () => {
+        await deps.modelWorkCoordinator.withModelLoad(async () => {
+            const instance = await opts.load();
+            deps.registry.register(opts.name, instance, registerOpts);
+            for (const alias of opts.aliases ?? []) {
+                if (alias === opts.name)
+                    continue;
+                deps.registry.register(alias, instance, registerOpts);
+            }
+        }, opts.name);
+    });
+}

package/dist/model-work-coordinator.d.ts

@@ -1,3 +1,4 @@
+import type { ModelLoadRecord } from './health.js';
 /**
  * Result of a `withModelLoad` call that surfaces who actually drove the load
  * vs. who merely parked behind one that was already in flight. Callers use
@@ -41,11 +42,35 @@ export interface ModelLoadOutcome<T> {
  */
 export declare class ModelWorkCoordinator {
     private activeReaders;
-    private writerActive;
-    private waitingWriters;
+    private writerHeld;
+    private queuedWriters;
     private readonly readerWaiters;
     private readonly writerWaiters;
-    withModelLoad<T>(fn: () => Promise<T> | T): Promise<T>;
+    /**
+     * Most recent settled load bracket. Retained here because the coordinator
+     * is the ONE place that brackets every load: a `resolveModel` failure in
+     * `/v1/messages` becomes a 500 and is otherwise dropped on the floor, so a
+     * supervisor polling `/health` afterwards had no way to learn what went
+     * wrong. See {@link ModelLoadRecord} for the "successful no-op overwrites
+     * an earlier failure" caveat.
+     */
+    private lastLoadRecord;
+    /** Read-only: `true` while a load holds the exclusive writer slot. */
+    get writerActive(): boolean;
+    /** Read-only: loads parked in `acquireWrite()` waiting for the slot. */
+    get waitingWriters(): number;
+    /** Read-only: outcome of the most recent settled load bracket, or `null`. */
+    get lastLoad(): ModelLoadRecord | null;
+    /**
+     * Record a settled bracket. Called from the `finally` of both load
+     * wrappers so a throw is captured just as reliably as a success.
+     */
+    private recordLoad;
+    /**
+     * @param label Optional identifier (normally the model name) stamped into
+     *   {@link lastLoad} so `/health` can name what was being loaded.
+     */
+    withModelLoad<T>(fn: () => Promise<T> | T, label?: string): Promise<T>;
     /**
      * Like {@link withModelLoad} but reports whether THIS caller owned the
      * load (acquired the writer lock with no contention) or merely waited
@@ -59,7 +84,7 @@ export declare class ModelWorkCoordinator {
      * 60-second cold-load does not look like 60 seconds of own work for
      * every concurrent request.
      */
-    withModelLoadInstrumented<T>(fn: () => Promise<T> | T): Promise<ModelLoadOutcome<T>>;
+    withModelLoadInstrumented<T>(fn: () => Promise<T> | T, label?: string): Promise<ModelLoadOutcome<T>>;
     withInference<T>(fn: () => Promise<T> | T): Promise<T>;
     private acquireRead;
     private acquireWrite;

package/dist/model-work-coordinator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"model-work-coordinator.d.ts","sourceRoot":"","sources":["../src/model-work-coordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC;IACjC,MAAM,EAAE,CAAC,CAAC;IACV,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IACvD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IAEjD,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAO3D;IAED;;;;;;;;;;;;OAYG;IACG,yBAAyB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAoBzF;IAEK,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAO3D;IAED,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,YAAY;IAgBpB,OAAO,CAAC,WAAW;IAMnB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,KAAK;CAWd"}
+{"version":3,"file":"model-work-coordinator.d.ts","sourceRoot":"","sources":["../src/model-work-coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAqBnD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC;IACjC,MAAM,EAAE,CAAC,CAAC;IACV,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IACvD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IACvD;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc,CAAgC;IAEtD,sEAAsE;IACtE,IAAI,YAAY,IAAI,OAAO,CAE1B;IAED,wEAAwE;IACxE,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,6EAA6E;IAC7E,IAAI,QAAQ,IAAI,eAAe,GAAG,IAAI,CAErC;IAED;;;OAGG;IACH,OAAO,CAAC,UAAU;IAUlB;;;OAGG;IACG,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAmB3E;IAED;;;;;;;;;;;;OAYG;IACG,yBAAyB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CA2BzG;IAEK,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAO3D;IAED,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,YAAY;IAgBpB,OAAO,CAAC,WAAW;IAMnB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,KAAK;CAWd"}