@mlx-node/server 0.0.7 → 0.0.9

package/dist/host/swap.js

@@ -0,0 +1,178 @@
+/**
+ * Single-resident lazy-load policy for an inference host.
+ *
+ * The host discovers every local model up-front but loads at most one into
+ * the `ModelRegistry` at a time. Switching models (e.g. via Claude Code's
+ * `/model` picker, or the desktop app's model menu) unregisters the previous
+ * instance, letting GC + native destructors reclaim memory, before loading
+ * the new one.
+ */
+/**
+ * Build the `resolveModel` + `listModels` callbacks for the handler.
+ *
+ * `loadModelFn` is injected so tests can stub it without touching native code.
+ * The controller serializes every `resolveModel` invocation on a single
+ * promise chain so two concurrent requests for different-but-currently-
+ * unloaded models cannot race on the native compiled-path globals.
+ */
+export function makeSwapController(discovered, registry, loadModelFn, defaultName) {
+    const byName = new Map();
+    for (const entry of discovered)
+        byName.set(entry.name, entry);
+    const ordered = [...discovered].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    // Which entry unknown names (haiku subagent dispatches, etc.) fall back
+    // to before anything is resident. Defaults to discovered[0], but the
+    // caller can pin it to the user's `--model` pick so the first haiku
+    // title-gen doesn't trigger a load of the alphabetically-first model
+    // followed by an immediate swap to the user's real choice.
+    const fallbackEntry = (defaultName != null ? byName.get(defaultName) : undefined) ?? discovered[0];
+    let resident = null;
+    // Names we registered as aliases to the current resident (e.g.
+    // Claude Code's hardcoded `claude-haiku-*` for subagent dispatches /
+    // title generation). Tracked so we can unregister them on `/model`
+    // swap — otherwise an alias's refcount would keep the old binding
+    // alive past the user's swap.
+    const aliases = new Set();
+    let currentOp = Promise.resolve();
+    async function resolveModel(name) {
+        // Fast path: already registered under this name (either as a real
+        // resident or as an alias we previously installed). Avoid chaining.
+        if (registry.get(name))
+            return;
+        const next = currentOp.then(async () => {
+            // Re-check under the serialized section — a prior waiter may have loaded it.
+            if (registry.get(name))
+                return;
+            // Pick the discovered entry to resolve against. If the requested
+            // name matches a discovered model, use it. Otherwise (unknown
+            // name — Claude Code's hardcoded small-fast-model, etc.) fall
+            // through to the current resident so subagent dispatches don't
+            // 404, loading discovered[0] on first boot if nothing is resident
+            // yet.
+            //
+            // CRITICAL: this must read `resident` at RUN time, not QUEUE time.
+            // If we capture it before chaining onto `currentOp`, a swap that
+            // ran ahead of us will leave us with a stale target — e.g. a haiku
+            // alias request that arrived during a `/model a → b` switch would
+            // capture `targetEntry = a`, then re-bind itself to `a` and undo
+            // the user's switch when its turn finally comes around.
+            const knownEntry = byName.get(name);
+            const targetEntry = knownEntry ?? (resident ? (byName.get(resident.name) ?? fallbackEntry) : fallbackEntry);
+            const isAlias = targetEntry.name !== name;
+            // Swap out any stale resident that isn't the target.
+            //
+            // We do NOT unregister the aliases here: in-flight messages.ts
+            // requests may be microtask-racing between "resolveModel returned"
+            // and "registry.get(body.model)" and dropping the alias in that
+            // window yields a spurious 404. Instead we carry the alias set
+            // across the swap and re-point them to the new resident below,
+            // so the name always resolves to *some* live instance.
+            const oldResident = resident;
+            const carriedAliases = new Set(aliases);
+            if (oldResident && oldResident.name !== targetEntry.name) {
+                // Drop our local alias bookkeeping AND the old resident's primary
+                // name binding, but leave the alias *names* in the registry pointed
+                // at the old model — they hold the only refcount preventing GC,
+                // and an in-flight `registry.get(alias)` must keep resolving to
+                // *some* live instance until the new model is in hand.
+                aliases.clear();
+                registry.unregister(oldResident.name);
+                resident = null;
+            }
+            // Ensure the target is resident. If the load throws, restore the
+            // pre-swap controller state so future swap attempts know about the
+            // aliases we just cleared — otherwise the alias *names* stay bound
+            // in the registry to the old model object forever (alias bindings
+            // hold their own refcount), but the controller forgets they exist
+            // and never repoints them, leaving alias-routed traffic permanently
+            // pinned to a stale model.
+            let instance = registry.get(targetEntry.name);
+            if (!instance) {
+                let loaded;
+                try {
+                    loaded = await loadModelFn(targetEntry.path);
+                }
+                catch (err) {
+                    // Recovery: re-populate the controller's alias set so the next
+                    // resolveModel call still owns them. The alias→old-model bindings
+                    // are still live in the registry (we never unregistered them), so
+                    // we can recover the old model object via any surviving alias and
+                    // re-bind the old resident's primary name.
+                    //
+                    // If `carriedAliases` is empty (no aliases ever existed) AND we
+                    // unregistered `oldResident.name`, the binding's refcount may have
+                    // hit zero and the model is gone. There's nothing the controller
+                    // can do to recover in that case — the user will need to /model-
+                    // pick again. This is acceptable: alias-less load failures are
+                    // rare, and the user-facing symptom is "prior model gone, please
+                    // re-pick", not silently-wrong responses.
+                    for (const aliasName of carriedAliases)
+                        aliases.add(aliasName);
+                    if (oldResident) {
+                        let oldInstance;
+                        for (const aliasName of carriedAliases) {
+                            const probe = registry.get(aliasName);
+                            if (probe) {
+                                oldInstance = probe;
+                                break;
+                            }
+                        }
+                        if (oldInstance) {
+                            const oldEntry = byName.get(oldResident.name);
+                            registry.register(oldResident.name, oldInstance, {
+                                samplingDefaults: oldEntry?.preset.sampling,
+                                maxOutputTokens: oldEntry?.preset.maxOutputTokens,
+                            });
+                            resident = oldResident;
+                        }
+                    }
+                    throw err;
+                }
+                instance = loaded;
+                registry.register(targetEntry.name, instance, {
+                    samplingDefaults: targetEntry.preset.sampling,
+                    maxOutputTokens: targetEntry.preset.maxOutputTokens,
+                });
+                resident = { name: targetEntry.name };
+            }
+            else if (!resident) {
+                resident = { name: targetEntry.name };
+            }
+            // Re-point any aliases carried across the swap onto the new
+            // resident. `registry.register(sameName, differentModel)` drops
+            // the old binding's refcount and installs the new one atomically,
+            // so any concurrent `registry.get(alias)` either sees the old or
+            // new instance — never null.
+            for (const aliasName of carriedAliases) {
+                if (aliasName === targetEntry.name)
+                    continue;
+                registry.register(aliasName, instance, {
+                    samplingDefaults: targetEntry.preset.sampling,
+                    maxOutputTokens: targetEntry.preset.maxOutputTokens,
+                });
+                aliases.add(aliasName);
+            }
+            // For unknown names, register an alias on the resident instance so
+            // the endpoint's `registry.get(name)` lookup succeeds.
+            if (isAlias) {
+                registry.register(name, instance, {
+                    samplingDefaults: targetEntry.preset.sampling,
+                    maxOutputTokens: targetEntry.preset.maxOutputTokens,
+                });
+                aliases.add(name);
+            }
+        });
+        currentOp = next.catch(() => undefined);
+        await next;
+    }
+    function listModels() {
+        const created = Math.floor(Date.now() / 1000);
+        return ordered.map((entry) => ({
+            id: entry.name,
+            object: 'model',
+            created,
+            owned_by: 'mlx-node',
+        }));
+    }
+    return { resolveModel, listModels };
+}

package/dist/host/temp-root.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Pid-scoped temp roots for the host's paged-config overrides, plus the
+ * startup sweep that reclaims the ones a killed host left behind.
+ *
+ * `PagedConfigOverrideManager` clones a checkpoint directory (config.json
+ * rewritten, everything else symlinked) into a temp root and removes that root
+ * in `cleanup()`. `cleanup()` runs on a normal shutdown — but the host's
+ * headline deployment is an Electron `utilityProcess`, and a `utilityProcess`
+ * can be SIGKILLed (app force-quit, OOM killer, `kill -9`). SIGKILL runs no
+ * handler, so without a sweep EVERY hard kill leaks a root. The clones are
+ * symlink farms rather than copies, so the leak is inodes and directory
+ * entries rather than model-sized bytes — but the roots accumulate forever and
+ * a partially-written clone can hold a real config.json.
+ *
+ * The reclaim strategy is "name the owner in the directory name": the manager
+ * gets a `tempDirPrefix` of `mlx-inference-host-<pid>-`, `mkdtemp` appends its
+ * own random suffix, and {@link sweepOrphanHostTempRoots} parses the pid back
+ * out and removes any root whose owner is gone.
+ *
+ * Known limitation: pid reuse. A long-dead host's root whose pid has since
+ * been recycled by an unrelated process is SPARED (never wrongly deleted), so
+ * the failure mode is a leaked directory, not data loss.
+ */
+/** Shared stem. A directory is host-owned iff its name starts with this. */
+export declare const HOST_TEMP_DIR_STEM = "mlx-inference-host-";
+/**
+ * `tempDirPrefix` to hand `PagedConfigOverrideManager` so the root it creates
+ * carries its owner's pid.
+ */
+export declare function hostTempDirPrefix(pid?: number): string;
+/**
+ * Does a process with this pid exist?
+ *
+ * `process.kill(pid, 0)` sends no signal; it only performs the permission +
+ * existence check. `ESRCH` is the sole "gone" answer — `EPERM` means the
+ * process exists but belongs to another user, which must count as ALIVE so a
+ * multi-user box never has one user's sweep delete another user's live root.
+ */
+export declare function isProcessAlive(pid: number): boolean;
+export interface SweepOrphanTempRootsOptions {
+    /** Directory to scan. Defaults to the OS temp dir. */
+    root?: string;
+    /** Our own pid — never swept, however the liveness probe answers. */
+    selfPid?: number;
+    /** Injectable liveness probe. Defaults to {@link isProcessAlive}. */
+    isAlive?: (pid: number) => boolean;
+}
+/**
+ * Remove every host temp root whose owning pid is no longer alive.
+ *
+ * Best effort by contract: a scan or unlink failure (permissions, a root a
+ * concurrently-exiting host is removing under us) is swallowed, because a
+ * housekeeping step must never be the reason a host refuses to start.
+ * Returns the absolute paths actually removed.
+ */
+export declare function sweepOrphanHostTempRoots(opts?: SweepOrphanTempRootsOptions): Promise<string[]>;
+//# sourceMappingURL=temp-root.d.ts.map

package/dist/host/temp-root.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"temp-root.d.ts","sourceRoot":"","sources":["../../src/host/temp-root.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAMH,4EAA4E;AAC5E,eAAO,MAAM,kBAAkB,wBAAwB,CAAC;AAKxD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,GAAE,MAAoB,GAAG,MAAM,CAEnE;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAWnD;AAED,MAAM,WAAW,2BAA2B;IAC1C,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACpC;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,IAAI,GAAE,2BAAgC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA6BxG"}

package/dist/host/temp-root.js

@@ -0,0 +1,99 @@
+/**
+ * Pid-scoped temp roots for the host's paged-config overrides, plus the
+ * startup sweep that reclaims the ones a killed host left behind.
+ *
+ * `PagedConfigOverrideManager` clones a checkpoint directory (config.json
+ * rewritten, everything else symlinked) into a temp root and removes that root
+ * in `cleanup()`. `cleanup()` runs on a normal shutdown — but the host's
+ * headline deployment is an Electron `utilityProcess`, and a `utilityProcess`
+ * can be SIGKILLed (app force-quit, OOM killer, `kill -9`). SIGKILL runs no
+ * handler, so without a sweep EVERY hard kill leaks a root. The clones are
+ * symlink farms rather than copies, so the leak is inodes and directory
+ * entries rather than model-sized bytes — but the roots accumulate forever and
+ * a partially-written clone can hold a real config.json.
+ *
+ * The reclaim strategy is "name the owner in the directory name": the manager
+ * gets a `tempDirPrefix` of `mlx-inference-host-<pid>-`, `mkdtemp` appends its
+ * own random suffix, and {@link sweepOrphanHostTempRoots} parses the pid back
+ * out and removes any root whose owner is gone.
+ *
+ * Known limitation: pid reuse. A long-dead host's root whose pid has since
+ * been recycled by an unrelated process is SPARED (never wrongly deleted), so
+ * the failure mode is a leaked directory, not data loss.
+ */
+import { readdir, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+/** Shared stem. A directory is host-owned iff its name starts with this. */
+export const HOST_TEMP_DIR_STEM = 'mlx-inference-host-';
+/** Matches `mlx-inference-host-<pid>-<mkdtemp suffix>`. */
+const HOST_TEMP_DIR_RE = /^mlx-inference-host-(\d+)-/;
+/**
+ * `tempDirPrefix` to hand `PagedConfigOverrideManager` so the root it creates
+ * carries its owner's pid.
+ */
+export function hostTempDirPrefix(pid = process.pid) {
+    return `${HOST_TEMP_DIR_STEM}${pid}-`;
+}
+/**
+ * Does a process with this pid exist?
+ *
+ * `process.kill(pid, 0)` sends no signal; it only performs the permission +
+ * existence check. `ESRCH` is the sole "gone" answer — `EPERM` means the
+ * process exists but belongs to another user, which must count as ALIVE so a
+ * multi-user box never has one user's sweep delete another user's live root.
+ */
+export function isProcessAlive(pid) {
+    // pid 0 addresses the caller's whole process group on POSIX and pid < 0 a
+    // group by id; neither is ever a real owner, and signalling them would be
+    // actively dangerous. Treat as alive so they are never swept.
+    if (!Number.isInteger(pid) || pid <= 0)
+        return true;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code !== 'ESRCH';
+    }
+}
+/**
+ * Remove every host temp root whose owning pid is no longer alive.
+ *
+ * Best effort by contract: a scan or unlink failure (permissions, a root a
+ * concurrently-exiting host is removing under us) is swallowed, because a
+ * housekeeping step must never be the reason a host refuses to start.
+ * Returns the absolute paths actually removed.
+ */
+export async function sweepOrphanHostTempRoots(opts = {}) {
+    const root = opts.root ?? tmpdir();
+    const selfPid = opts.selfPid ?? process.pid;
+    const isAlive = opts.isAlive ?? isProcessAlive;
+    let entries;
+    try {
+        entries = await readdir(root);
+    }
+    catch {
+        return [];
+    }
+    const removed = [];
+    for (const name of entries) {
+        const match = HOST_TEMP_DIR_RE.exec(name);
+        if (match === null)
+            continue;
+        const pid = Number.parseInt(match[1], 10);
+        if (pid === selfPid)
+            continue;
+        if (isAlive(pid))
+            continue;
+        const full = join(root, name);
+        try {
+            await rm(full, { recursive: true, force: true });
+            removed.push(full);
+        }
+        catch {
+            /* another sweeper won the race, or we lack permission; leave it */
+        }
+    }
+    return removed;
+}

package/dist/idle-sweeper.d.ts

@@ -0,0 +1,245 @@
+/**
+ * Idle cache-pool sweeper.
+ *
+ * The MLX Metal allocator holds a process-wide free pool. Without an
+ * explicit drain the pool can sit near the wired ceiling (tens of GB on
+ * an M3 Max) for the entire server uptime. `@mlx-node/core`'s decode
+ * loop already calls `mlx_clear_cache()` every 256 steps, but that
+ * cadence only fires while a generation is in progress — idle periods
+ * between HTTP requests never get a drain.
+ *
+ * An earlier iteration put a `ClearCacheOnDrop` RAII guard around every
+ * session command so the pool was flushed after every turn. That is
+ * wrong on a multi-model server: the free pool is shared across model
+ * instances, so flushing after a request on model A discards blocks
+ * that model B's next turn is about to reuse.
+ *
+ * The round-1 replacement was a debounced `touch()`-based sweeper:
+ * every request start / end reset a 30s timer that fired `clearCache()`
+ * when the timer expired. That design had a correctness bug flagged in
+ * round-2 review: a long-running request (streaming decode of 60+
+ * seconds) had its 30s timer armed at ARRIVAL, and the timer fired
+ * mid-request at t=30s — exactly when the generation thread still
+ * needed those blocks. Worse, `clearCache()` routes through
+ * `mlx_synchronize()` WITHOUT a stream argument, which only drains the
+ * default stream (see `crates/mlx-sys/mlx/mlx/scheduler.cpp`), but
+ * decode runs on custom streams — so the drain could race with live
+ * command buffers, risking buffer use-after-free.
+ *
+ * This version fixes both by tracking `inFlight` explicitly. The timer
+ * is ONLY armed when the counter returns to zero, and is cancelled the
+ * instant any new request arrives — mid-request drains are impossible
+ * by construction.
+ *
+ * # Scope: inference endpoints only
+ *
+ * The counter is bumped by `/v1/responses` and `/v1/messages` ONLY.
+ * Non-inference traffic — `/v1/models`, `/v1/health`, CORS `OPTIONS`
+ * preflights, 404s for unknown routes — deliberately does NOT touch
+ * the sweeper. Counting those would let a `GET /v1/models` on every
+ * client startup, a CORS preflight from a browser, or a cron health
+ * probe keep the allocator pinned forever.
+ *
+ * # Drain is post-request only
+ *
+ * The only drain path is post-request: `endRequest()` decrements the
+ * in-flight counter and, on the 1 -> 0 transition, arms the timer. A
+ * cold-idle server that loaded models but never served a request
+ * never drains. That is deliberate:
+ *
+ * - Earlier iterations armed a cold-start drain from either
+ *   `createServer()` (regressed on slow model loads > `idleClearCacheMs`,
+ *   firing mid-load) or from `ModelRegistry.register()` (round-7 review
+ *   surfaced that it still raced sequential multi-model loads — the
+ *   timer armed by `register(A)` could fire while `await load(B)` was
+ *   still resolving). Loader-bracketing would require API changes on
+ *   the user-facing load path, so we remove the cold-start path
+ *   entirely instead.
+ * - Load-time allocator growth is bounded (a few GB of scratch), not
+ *   the tens-of-GB problem the post-request drain targets. macOS
+ *   handles truly-idle memory pressure via compression / swap.
+ * - Python `mlx-lm` has no cold-start drain either, so this matches
+ *   that baseline for servers that never receive a request.
+ *
+ * # Hot-load bracketing: `withSuspendedDrains`
+ *
+ * Hot-load flows — a `Model::load()` invoked AFTER the server has
+ * already served at least one request — race the post-request drain
+ * timer: the t+delayMs timer armed by the previous `endRequest()` can
+ * fire MID-LOAD while weight materialization is still allocating
+ * through the Metal free pool. The canonical fix is to bracket the
+ * load with `withSuspendedDrains(fn)`:
+ *
+ * ```ts
+ * await server.withSuspendedDrains(async () => {
+ *   const model = await Qwen35MoeModel.load(modelPath);
+ *   server.registry.register('qwen', model);
+ * });
+ * ```
+ *
+ * `withSuspendedDrains` handles try/finally bracketing itself, so a
+ * thrown load never leaks the internal suspend counter — a footgun the
+ * earlier raw `suspendDrains()` / `resumeDrains()` pair exposed
+ * (round-9 MEDIUM). The low-level `suspendDrains()` entry point
+ * remains available for callers that genuinely need manual control;
+ * its returned disposer is token-scoped and idempotent.
+ *
+ * `withSuspendedDrains` is also thenable-safe: non-Promise thenables
+ * (plain objects or even callables that expose a `.then` method) are
+ * normalized via `Promise.resolve()` so callers always get a real
+ * Promise<T> back and the suspend-release fires on settle. A throwing
+ * `.then` getter is treated as non-thenable (the getter access is
+ * guarded); the value is returned directly and the suspend releases
+ * synchronously — we deliberately do NOT propagate a getter throw, on
+ * the theory that leaking the suspend is a far worse failure mode than
+ * losing a pathological value (round-10 MEDIUM).
+ *
+ * # Tuning
+ *
+ * - The default (30_000 ms) balances "give models a chance to reuse
+ *   hot blocks across back-to-back requests" against "don't hold
+ *   tens of GB hostage while the process is truly idle".
+ * - `idleClearCacheMs: 0` disables the sweeper entirely — useful for
+ *   benchmarks or single-model workloads where the only memory
+ *   pressure is the decode-loop cadence already in place.
+ *   `withSuspendedDrains(fn)` on the disabled sweeper is a pass-through
+ *   so call sites can unconditionally bracket.
+ * - `MLX_IDLE_CLEAR_CACHE_MS` env var overrides the server's
+ *   constructor value; explicit constructor value wins over env.
+ *
+ * Drain cost is a single `mlx_synchronize` + `mlx_clear_cache` —
+ * constant time relative to generation length.
+ */
+/** Default idle window before draining the allocator's free pool (ms). */
+export declare const DEFAULT_IDLE_CLEAR_CACHE_MS = 30000;
+/**
+ * Parse `MLX_IDLE_CLEAR_CACHE_MS`. Same semantics as the other env
+ * knobs in `server.ts`: finite non-negative integer or fall through
+ * to the caller's default. A value of `0` explicitly disables the
+ * sweeper; negative / non-integer / unparseable values are ignored.
+ */
+export declare function parseIdleClearCacheEnv(): number | undefined;
+/**
+ * In-flight counter-based idle-drain scheduler.
+ *
+ * - `beginRequest()` is called ONCE per inference request BEFORE the
+ *   native model is dispatched. It increments the in-flight counter
+ *   and cancels any pending drain timer — eliminating the mid-request
+ *   drain race the debounced-`touch()` design had. Only the inference
+ *   endpoints (`/v1/responses` + `/v1/messages`) should call this;
+ *   `/v1/models`, `/v1/health`, and CORS preflights MUST NOT, or they
+ *   would keep the allocator pinned forever on purely observational
+ *   traffic.
+ * - `endRequest()` is called ONCE per request in a `finally` block
+ *   after the model stream has fully ended (covering success, error,
+ *   and client-abort paths). It decrements the counter; when the
+ *   counter reaches zero it arms a `delayMs` timer that calls
+ *   `onDrain()` on expiry. This is the ONLY path that ever schedules
+ *   a drain — see the module-level "Drain is post-request only"
+ *   note for why cold-start arming was removed.
+ * - `close()` cancels any pending drain — used during graceful
+ *   shutdown so the timer does not keep Node alive after
+ *   `server.close()`. It intentionally does NOT reset the counter:
+ *   draining partway through in-flight requests is the exact failure
+ *   mode we're avoiding.
+ *
+ * Thread-safety is provided by Node's single-threaded event loop —
+ * `beginRequest` / `endRequest` are only ever invoked from the HTTP
+ * handler (or tests). Every `beginRequest()` MUST be paired with
+ * exactly one `endRequest()` on every exit path — a missed
+ * `endRequest()` would leave the counter pinned above zero and the
+ * drain would never fire.
+ */
+export interface IdleSweeper {
+    /** Mark a request as arrived. Cancels any pending drain. */
+    beginRequest(): void;
+    /**
+     * Mark a request as completed. When the in-flight counter reaches
+     * zero, arms a drain timer for `delayMs`.
+     */
+    endRequest(): void;
+    /** Cancel the pending drain. Idempotent. Does NOT reset the counter. */
+    close(): void;
+    /**
+     * Run `fn` with drains suspended. Handles try/finally bracketing so
+     * a thrown load never leaks the internal suspend counter. This is
+     * the canonical entry point for hot-load flows — a `Model::load()`
+     * invoked AFTER the server has already served at least one request.
+     * In that scenario the post-request drain timer armed by
+     * `endRequest()` (t+delayMs) can otherwise fire MID-LOAD while
+     * weight materialization is still allocating through the Metal free
+     * pool, racing the allocator state.
+     *
+     * Accepts both sync and async functions; returns `fn`'s own return
+     * value (or resolved promise). On exit (normal or thrown), the
+     * suspend token is disposed exactly once — the drain timer is
+     * re-armed if `inFlight === 0` and no other suspend is active.
+     *
+     * Thenable-safe: non-Promise thenables (including function-typed
+     * values that expose a `.then` method) are normalized via
+     * `Promise.resolve()` so callers always receive a real `Promise<T>`
+     * on the async branch; a throwing `.then` getter is guarded and the
+     * value is treated as non-thenable rather than leaking the suspend.
+     *
+     * The common `serve.ts` pattern — load all models before
+     * `createServer(...)`, i.e. before any request is served — has no
+     * armed timer in the first place and therefore does NOT need to
+     * bracket.
+     *
+     * Pass-through on the disabled sweeper (`delayMs <= 0`): `fn` is
+     * invoked directly and its return value is returned unchanged, so
+     * call sites can unconditionally bracket without branching.
+     *
+     * @example
+     * ```ts
+     * await server.withSuspendedDrains(async () => {
+     *   const model = await Qwen35MoeModel.load(modelPath);
+     *   server.registry.register('qwen', model);
+     * });
+     * ```
+     */
+    withSuspendedDrains<T>(fn: () => Promise<T>): Promise<T>;
+    withSuspendedDrains<T>(fn: () => T): T;
+    /**
+     * Low-level: suspend drains and return an idempotent disposer.
+     * Prefer {@link withSuspendedDrains} unless you need manual
+     * control over when the suspend is released (for example across
+     * async boundaries the caller wants to manage explicitly).
+     *
+     * Semantics: cancels any pending drain and increments an internal
+     * "load" counter. Returns a disposer — calling the disposer
+     * decrements the counter. When the counter reaches zero AND
+     * `inFlight === 0`, a fresh drain timer is armed so a subsequent
+     * idle window still drains.
+     *
+     * Safe to nest: N suspends require N dispose calls. The returned
+     * disposer is token-scoped and idempotent — calling it more than
+     * once is a no-op, so it cannot over-decrement and mysteriously
+     * shift the idle window.
+     *
+     * No-op on the disabled sweeper (`delayMs <= 0`): returns a no-op
+     * disposer so call sites can unconditionally bracket without
+     * branching on whether the sweeper is enabled.
+     */
+    suspendDrains(): () => void;
+    /** Observability hook — `true` while a drain is scheduled. */
+    readonly isPending: boolean;
+    /** Observability hook — current in-flight request count. */
+    readonly inFlight: number;
+}
+/**
+ * Create an idle sweeper. Pass `0` or a non-positive value to opt out —
+ * the returned object becomes a no-op that still satisfies the
+ * interface. Callers can therefore unconditionally wire
+ * `beginRequest()` / `endRequest()` without branching on whether the
+ * sweeper is enabled.
+ *
+ * When `onDrain` is omitted, the sweeper resolves
+ * `__internal__.clearCache` on `@mlx-node/core` at creation time and
+ * caches the result in the returned closure. A missing namespace /
+ * function triggers a one-time `console.warn` and a no-op fallback —
+ * see `resolveClearCache()`. The `delayMs <= 0` path skips the
+ * resolution entirely so the opt-out remains purely passive.
+ */
+export declare function createIdleSweeper(delayMs: number, onDrain?: () => void): IdleSweeper;
+//# sourceMappingURL=idle-sweeper.d.ts.map

package/dist/idle-sweeper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"idle-sweeper.d.ts","sourceRoot":"","sources":["../src/idle-sweeper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+GG;AAsBH,0EAA0E;AAC1E,eAAO,MAAM,2BAA2B,QAAS,CAAC;AA4BlD;;;;;GAKG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,GAAG,SAAS,CAO3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,WAAW,WAAW;IAC1B,4DAA4D;IAC5D,YAAY,IAAI,IAAI,CAAC;IACrB;;;OAGG;IACH,UAAU,IAAI,IAAI,CAAC;IACnB,wEAAwE;IACxE,KAAK,IAAI,IAAI,CAAC;IACd;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;IAC5B,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AA4CD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,IAAI,GAAG,WAAW,CAoNpF"}