@mlx-node/server 0.0.0 → 0.0.8

package/dist/errors.d.ts

@@ -32,6 +32,7 @@ export declare function sendAnthropicError(res: ServerResponse, status: number,
 export declare function sendAnthropicBadRequest(res: ServerResponse, message: string): void;
 export declare function sendAnthropicNotFound(res: ServerResponse, message: string): void;
 export declare function sendAnthropicInternalError(res: ServerResponse, message: string): void;
+export declare function sendAnthropicNotImplemented(res: ServerResponse, message: string): void;
 export declare function sendAnthropicMethodNotAllowed(res: ServerResponse, allowed: string): void;
 /**
  * 429 Anthropic-shape rate-limit response. Mirror of {@link sendRateLimit}

package/dist/errors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,wBAAgB,SAAS,CACvB,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GACpB,IAAI,CAWN;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAEzF;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEvE;AAED,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAO/E;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE5E;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE7E;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAYxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAG3G;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAElF;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEhF;AAED,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAErF;AAED,wBAAgB,6BAA6B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGjF"}
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,wBAAgB,SAAS,CACvB,GAAG,EAAE,cAAc,EACnB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GACpB,IAAI,CAWN;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAEzF;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEvE;AAED,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAO/E;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE5E;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAE7E;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAYxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAG3G;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAElF;AAED,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEhF;AAED,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAErF;AAED,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEtF;AAED,wBAAgB,6BAA6B,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAGjF"}

package/dist/errors.js

@@ -68,6 +68,9 @@ export function sendAnthropicNotFound(res, message) {
 export function sendAnthropicInternalError(res, message) {
     sendAnthropicError(res, 500, 'api_error', message);
 }
+export function sendAnthropicNotImplemented(res, message) {
+    sendAnthropicError(res, 501, 'not_supported_error', message);
+}
 export function sendAnthropicMethodNotAllowed(res, allowed) {
     res.writeHead(405, { Allow: allowed, 'Content-Type': 'application/json' });
     res.end(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: 'Method not allowed' } }));

package/dist/handler.d.ts

@@ -1,7 +1,21 @@
 /** Composable `(req, res)` handler for node:http — usable standalone or mounted into an existing server. */
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import type { ResponseStore } from '@mlx-node/core';
+import type { IdleSweeper } from './idle-sweeper.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import type { ModelRegistry } from './registry.js';
+/**
+ * Entry in the list returned by `GET /v1/models`. Matches the shape
+ * `ModelRegistry.list()` produces — exported so callers that supply a
+ * custom `listModels` callback (e.g. `mlx launch claude` discovering
+ * every model on disk) can build entries without importing internals.
+ */
+export interface PublicModelEntry {
+    id: string;
+    object: 'model';
+    created: number;
+    owned_by: string;
+}
 export interface HandlerOptions {
     /** Enable CORS headers (default: true). */
     cors?: boolean;
@@ -13,6 +27,34 @@ export interface HandlerOptions {
      * and `ServerConfig.responseRetentionSec`).
      */
     responseRetentionSec?: number;
+    /**
+     * Optional idle sweeper. Forwarded to `routeRequest` so that the
+     * inference endpoints (`/v1/responses` + `/v1/messages`) can bracket
+     * their native-model dispatch with `beginRequest()` / `endRequest()`.
+     *
+     * Note: the begin/end hooks are DELIBERATELY scoped to the inference
+     * endpoints — wrapping every HTTP call (OPTIONS preflights,
+     * `/v1/models`, `/v1/health`, 404s) would let purely observational
+     * traffic keep the allocator pinned forever.
+     */
+    idleSweeper?: IdleSweeper | null;
+    /**
+     * Optional async callback invoked by `/v1/messages` before it looks
+     * the model up in the registry. The callback should register the
+     * model on demand; on return, the endpoint does `registry.get(name)`
+     * and 404s if still unresolved.
+     */
+    resolveModel?: (name: string) => Promise<void>;
+    /**
+     * Coordinates process-wide MLX work so lazy model loads / warmups do not
+     * overlap live inference on another model.
+     */
+    modelWorkCoordinator?: ModelWorkCoordinator;
+    /**
+     * Optional override for `GET /v1/models`. When provided, that endpoint
+     * returns this list instead of `registry.list()`.
+     */
+    listModels?: () => PublicModelEntry[];
 }
 export declare function createHandler(registry: ModelRegistry, options?: HandlerOptions): (req: IncomingMessage, res: ServerResponse) => Promise<void>;
 //# sourceMappingURL=handler.d.ts.map

package/dist/handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA,4GAA4G;AAE5G,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,MAAM,WAAW,cAAc;IAC7B,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,KAAK,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAgC9D"}
+{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../src/handler.ts"],"names":[],"mappings":"AAAA,4GAA4G;AAE5G,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,KAAK,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;OAGG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CA8C9D"}

package/dist/handler.js

@@ -1,10 +1,15 @@
 /** Composable `(req, res)` handler for node:http — usable standalone or mounted into an existing server. */
 import { sendInternalError } from './errors.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import { routeRequest } from './router.js';
 export function createHandler(registry, options) {
     const cors = options?.cors ?? true;
     const store = options?.store ?? null;
     const responseRetentionSec = options?.responseRetentionSec;
+    const idleSweeper = options?.idleSweeper ?? null;
+    const resolveModel = options?.resolveModel;
+    const modelWorkCoordinator = options?.modelWorkCoordinator ?? (resolveModel ? new ModelWorkCoordinator() : undefined);
+    const listModels = options?.listModels;
     return async (req, res) => {
         if (cors) {
             res.setHeader('Access-Control-Allow-Origin', '*');
@@ -20,7 +25,7 @@ export function createHandler(registry, options) {
         // post-`res.end()` bookkeeping (e.g. `SessionRegistry.adopt`). `http.createServer`
         // ignores the return value, so this is transparent to production callers.
         try {
-            await routeRequest(req, res, registry, store, responseRetentionSec);
+            await routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator);
         }
         catch (err) {
             const message = err instanceof Error ? err.message : 'Internal server error';

package/dist/idle-sweeper.d.ts

@@ -0,0 +1,245 @@
+/**
+ * Idle cache-pool sweeper.
+ *
+ * The MLX Metal allocator holds a process-wide free pool. Without an
+ * explicit drain the pool can sit near the wired ceiling (tens of GB on
+ * an M3 Max) for the entire server uptime. `@mlx-node/core`'s decode
+ * loop already calls `mlx_clear_cache()` every 256 steps, but that
+ * cadence only fires while a generation is in progress — idle periods
+ * between HTTP requests never get a drain.
+ *
+ * An earlier iteration put a `ClearCacheOnDrop` RAII guard around every
+ * session command so the pool was flushed after every turn. That is
+ * wrong on a multi-model server: the free pool is shared across model
+ * instances, so flushing after a request on model A discards blocks
+ * that model B's next turn is about to reuse.
+ *
+ * The round-1 replacement was a debounced `touch()`-based sweeper:
+ * every request start / end reset a 30s timer that fired `clearCache()`
+ * when the timer expired. That design had a correctness bug flagged in
+ * round-2 review: a long-running request (streaming decode of 60+
+ * seconds) had its 30s timer armed at ARRIVAL, and the timer fired
+ * mid-request at t=30s — exactly when the generation thread still
+ * needed those blocks. Worse, `clearCache()` routes through
+ * `mlx_synchronize()` WITHOUT a stream argument, which only drains the
+ * default stream (see `crates/mlx-sys/mlx/mlx/scheduler.cpp`), but
+ * decode runs on custom streams — so the drain could race with live
+ * command buffers, risking buffer use-after-free.
+ *
+ * This version fixes both by tracking `inFlight` explicitly. The timer
+ * is ONLY armed when the counter returns to zero, and is cancelled the
+ * instant any new request arrives — mid-request drains are impossible
+ * by construction.
+ *
+ * # Scope: inference endpoints only
+ *
+ * The counter is bumped by `/v1/responses` and `/v1/messages` ONLY.
+ * Non-inference traffic — `/v1/models`, `/v1/health`, CORS `OPTIONS`
+ * preflights, 404s for unknown routes — deliberately does NOT touch
+ * the sweeper. Counting those would let a `GET /v1/models` on every
+ * client startup, a CORS preflight from a browser, or a cron health
+ * probe keep the allocator pinned forever.
+ *
+ * # Drain is post-request only
+ *
+ * The only drain path is post-request: `endRequest()` decrements the
+ * in-flight counter and, on the 1 -> 0 transition, arms the timer. A
+ * cold-idle server that loaded models but never served a request
+ * never drains. That is deliberate:
+ *
+ * - Earlier iterations armed a cold-start drain from either
+ *   `createServer()` (regressed on slow model loads > `idleClearCacheMs`,
+ *   firing mid-load) or from `ModelRegistry.register()` (round-7 review
+ *   surfaced that it still raced sequential multi-model loads — the
+ *   timer armed by `register(A)` could fire while `await load(B)` was
+ *   still resolving). Loader-bracketing would require API changes on
+ *   the user-facing load path, so we remove the cold-start path
+ *   entirely instead.
+ * - Load-time allocator growth is bounded (a few GB of scratch), not
+ *   the tens-of-GB problem the post-request drain targets. macOS
+ *   handles truly-idle memory pressure via compression / swap.
+ * - Python `mlx-lm` has no cold-start drain either, so this matches
+ *   that baseline for servers that never receive a request.
+ *
+ * # Hot-load bracketing: `withSuspendedDrains`
+ *
+ * Hot-load flows — a `Model::load()` invoked AFTER the server has
+ * already served at least one request — race the post-request drain
+ * timer: the t+delayMs timer armed by the previous `endRequest()` can
+ * fire MID-LOAD while weight materialization is still allocating
+ * through the Metal free pool. The canonical fix is to bracket the
+ * load with `withSuspendedDrains(fn)`:
+ *
+ * ```ts
+ * await server.withSuspendedDrains(async () => {
+ *   const model = await Qwen35MoeModel.load(modelPath);
+ *   server.registry.register('qwen', model);
+ * });
+ * ```
+ *
+ * `withSuspendedDrains` handles try/finally bracketing itself, so a
+ * thrown load never leaks the internal suspend counter — a footgun the
+ * earlier raw `suspendDrains()` / `resumeDrains()` pair exposed
+ * (round-9 MEDIUM). The low-level `suspendDrains()` entry point
+ * remains available for callers that genuinely need manual control;
+ * its returned disposer is token-scoped and idempotent.
+ *
+ * `withSuspendedDrains` is also thenable-safe: non-Promise thenables
+ * (plain objects or even callables that expose a `.then` method) are
+ * normalized via `Promise.resolve()` so callers always get a real
+ * Promise<T> back and the suspend-release fires on settle. A throwing
+ * `.then` getter is treated as non-thenable (the getter access is
+ * guarded); the value is returned directly and the suspend releases
+ * synchronously — we deliberately do NOT propagate a getter throw, on
+ * the theory that leaking the suspend is a far worse failure mode than
+ * losing a pathological value (round-10 MEDIUM).
+ *
+ * # Tuning
+ *
+ * - The default (30_000 ms) balances "give models a chance to reuse
+ *   hot blocks across back-to-back requests" against "don't hold
+ *   tens of GB hostage while the process is truly idle".
+ * - `idleClearCacheMs: 0` disables the sweeper entirely — useful for
+ *   benchmarks or single-model workloads where the only memory
+ *   pressure is the decode-loop cadence already in place.
+ *   `withSuspendedDrains(fn)` on the disabled sweeper is a pass-through
+ *   so call sites can unconditionally bracket.
+ * - `MLX_IDLE_CLEAR_CACHE_MS` env var overrides the server's
+ *   constructor value; explicit constructor value wins over env.
+ *
+ * Drain cost is a single `mlx_synchronize` + `mlx_clear_cache` —
+ * constant time relative to generation length.
+ */
+/** Default idle window before draining the allocator's free pool (ms). */
+export declare const DEFAULT_IDLE_CLEAR_CACHE_MS = 30000;
+/**
+ * Parse `MLX_IDLE_CLEAR_CACHE_MS`. Same semantics as the other env
+ * knobs in `server.ts`: finite non-negative integer or fall through
+ * to the caller's default. A value of `0` explicitly disables the
+ * sweeper; negative / non-integer / unparseable values are ignored.
+ */
+export declare function parseIdleClearCacheEnv(): number | undefined;
+/**
+ * In-flight counter-based idle-drain scheduler.
+ *
+ * - `beginRequest()` is called ONCE per inference request BEFORE the
+ *   native model is dispatched. It increments the in-flight counter
+ *   and cancels any pending drain timer — eliminating the mid-request
+ *   drain race the debounced-`touch()` design had. Only the inference
+ *   endpoints (`/v1/responses` + `/v1/messages`) should call this;
+ *   `/v1/models`, `/v1/health`, and CORS preflights MUST NOT, or they
+ *   would keep the allocator pinned forever on purely observational
+ *   traffic.
+ * - `endRequest()` is called ONCE per request in a `finally` block
+ *   after the model stream has fully ended (covering success, error,
+ *   and client-abort paths). It decrements the counter; when the
+ *   counter reaches zero it arms a `delayMs` timer that calls
+ *   `onDrain()` on expiry. This is the ONLY path that ever schedules
+ *   a drain — see the module-level "Drain is post-request only"
+ *   note for why cold-start arming was removed.
+ * - `close()` cancels any pending drain — used during graceful
+ *   shutdown so the timer does not keep Node alive after
+ *   `server.close()`. It intentionally does NOT reset the counter:
+ *   draining partway through in-flight requests is the exact failure
+ *   mode we're avoiding.
+ *
+ * Thread-safety is provided by Node's single-threaded event loop —
+ * `beginRequest` / `endRequest` are only ever invoked from the HTTP
+ * handler (or tests). Every `beginRequest()` MUST be paired with
+ * exactly one `endRequest()` on every exit path — a missed
+ * `endRequest()` would leave the counter pinned above zero and the
+ * drain would never fire.
+ */
+export interface IdleSweeper {
+    /** Mark a request as arrived. Cancels any pending drain. */
+    beginRequest(): void;
+    /**
+     * Mark a request as completed. When the in-flight counter reaches
+     * zero, arms a drain timer for `delayMs`.
+     */
+    endRequest(): void;
+    /** Cancel the pending drain. Idempotent. Does NOT reset the counter. */
+    close(): void;
+    /**
+     * Run `fn` with drains suspended. Handles try/finally bracketing so
+     * a thrown load never leaks the internal suspend counter. This is
+     * the canonical entry point for hot-load flows — a `Model::load()`
+     * invoked AFTER the server has already served at least one request.
+     * In that scenario the post-request drain timer armed by
+     * `endRequest()` (t+delayMs) can otherwise fire MID-LOAD while
+     * weight materialization is still allocating through the Metal free
+     * pool, racing the allocator state.
+     *
+     * Accepts both sync and async functions; returns `fn`'s own return
+     * value (or resolved promise). On exit (normal or thrown), the
+     * suspend token is disposed exactly once — the drain timer is
+     * re-armed if `inFlight === 0` and no other suspend is active.
+     *
+     * Thenable-safe: non-Promise thenables (including function-typed
+     * values that expose a `.then` method) are normalized via
+     * `Promise.resolve()` so callers always receive a real `Promise<T>`
+     * on the async branch; a throwing `.then` getter is guarded and the
+     * value is treated as non-thenable rather than leaking the suspend.
+     *
+     * The common `serve.ts` pattern — load all models before
+     * `createServer(...)`, i.e. before any request is served — has no
+     * armed timer in the first place and therefore does NOT need to
+     * bracket.
+     *
+     * Pass-through on the disabled sweeper (`delayMs <= 0`): `fn` is
+     * invoked directly and its return value is returned unchanged, so
+     * call sites can unconditionally bracket without branching.
+     *
+     * @example
+     * ```ts
+     * await server.withSuspendedDrains(async () => {
+     *   const model = await Qwen35MoeModel.load(modelPath);
+     *   server.registry.register('qwen', model);
+     * });
+     * ```
+     */
+    withSuspendedDrains<T>(fn: () => Promise<T>): Promise<T>;
+    withSuspendedDrains<T>(fn: () => T): T;
+    /**
+     * Low-level: suspend drains and return an idempotent disposer.
+     * Prefer {@link withSuspendedDrains} unless you need manual
+     * control over when the suspend is released (for example across
+     * async boundaries the caller wants to manage explicitly).
+     *
+     * Semantics: cancels any pending drain and increments an internal
+     * "load" counter. Returns a disposer — calling the disposer
+     * decrements the counter. When the counter reaches zero AND
+     * `inFlight === 0`, a fresh drain timer is armed so a subsequent
+     * idle window still drains.
+     *
+     * Safe to nest: N suspends require N dispose calls. The returned
+     * disposer is token-scoped and idempotent — calling it more than
+     * once is a no-op, so it cannot over-decrement and mysteriously
+     * shift the idle window.
+     *
+     * No-op on the disabled sweeper (`delayMs <= 0`): returns a no-op
+     * disposer so call sites can unconditionally bracket without
+     * branching on whether the sweeper is enabled.
+     */
+    suspendDrains(): () => void;
+    /** Observability hook — `true` while a drain is scheduled. */
+    readonly isPending: boolean;
+    /** Observability hook — current in-flight request count. */
+    readonly inFlight: number;
+}
+/**
+ * Create an idle sweeper. Pass `0` or a non-positive value to opt out —
+ * the returned object becomes a no-op that still satisfies the
+ * interface. Callers can therefore unconditionally wire
+ * `beginRequest()` / `endRequest()` without branching on whether the
+ * sweeper is enabled.
+ *
+ * When `onDrain` is omitted, the sweeper resolves
+ * `__internal__.clearCache` on `@mlx-node/core` at creation time and
+ * caches the result in the returned closure. A missing namespace /
+ * function triggers a one-time `console.warn` and a no-op fallback —
+ * see `resolveClearCache()`. The `delayMs <= 0` path skips the
+ * resolution entirely so the opt-out remains purely passive.
+ */
+export declare function createIdleSweeper(delayMs: number, onDrain?: () => void): IdleSweeper;
+//# sourceMappingURL=idle-sweeper.d.ts.map

package/dist/idle-sweeper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"idle-sweeper.d.ts","sourceRoot":"","sources":["../src/idle-sweeper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+GG;AAsBH,0EAA0E;AAC1E,eAAO,MAAM,2BAA2B,QAAS,CAAC;AA4BlD;;;;;GAKG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,GAAG,SAAS,CAO3D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,WAAW,WAAW;IAC1B,4DAA4D;IAC5D,YAAY,IAAI,IAAI,CAAC;IACrB;;;OAGG;IACH,UAAU,IAAI,IAAI,CAAC;IACnB,wEAAwE;IACxE,KAAK,IAAI,IAAI,CAAC;IACd;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;IAC5B,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AA4CD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,IAAI,GAAG,WAAW,CAoNpF"}