@mlx-node/server 0.0.7 → 0.0.8

package/dist/presets.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Sampling presets recommended by third-party model authors, exposed
+ * here as `ChatConfig`-shaped objects so an operator can pin them at
+ * `ModelRegistry.register(name, model, { samplingDefaults: ... })`
+ * time with a single import.
+ *
+ * Per-request client values (OpenAI `temperature`/`top_p`, Anthropic
+ * equivalents) still override these defaults where the client sends
+ * them — `ChatSession.mergeConfig` treats per-call config as an
+ * overlay on top of `defaultConfig`. These presets only fill in the
+ * parameters clients never send (`top_k`, `min_p`, penalties).
+ */
+import type { ChatConfig } from '@mlx-node/core';
+/**
+ * Sampling defaults from Unsloth's Qwen3.6 guide:
+ * https://unsloth.ai/docs/models/qwen3.6#recommended-settings
+ *
+ * All modes pin `top_k = 20` and `min_p = 0.0`; they differ in
+ * `temperature`, `top_p`, and `presence_penalty`.
+ *
+ * The native anti-repetition cutoff is now disabled by default
+ * (vLLM-aligned — vLLM ships no repetition-stop heuristic), so these
+ * presets no longer pin `maxConsecutiveTokens` / `maxNgramRepeats` /
+ * `ngramSize`. Repetition is shaped by the sampling penalties above and
+ * bounded by the per-model `maxOutputTokens`. An operator or client can
+ * still opt in by setting those fields explicitly — a per-request config
+ * value wins via `ChatSession.mergeConfig`.
+ */
+export declare const QWEN_SAMPLING_DEFAULTS: {
+    /** Thinking mode for precise coding tasks: temp=0.6, top_p=0.95, pp=0.0 */
+    readonly thinkingCoding: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+    /** Thinking mode for general tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    readonly thinkingGeneral: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+    /** Instruct (non-thinking) for general tasks: temp=0.7, top_p=0.8, pp=1.5 */
+    readonly instructGeneral: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+    /** Instruct (non-thinking) for reasoning tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    readonly instructReasoning: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+};
+/** Sampling defaults for Gemma4 Instruct. */
+export declare const GEMMA4_SAMPLING_DEFAULTS: ChatConfig;
+/** Sampling defaults for LFM2.5 Thinking. */
+export declare const LFM2_SAMPLING_DEFAULTS: ChatConfig;
+/** Sampling + per-model output token cap exposed by {@link LAUNCH_PRESETS}. */
+export interface LaunchPreset {
+    sampling: ChatConfig;
+    maxOutputTokens: number;
+}
+/**
+ * Per-`ModelType` presets used by `mlx launch claude` to pre-wire a
+ * discovered model with sensible sampling defaults + a max output
+ * token budget. Keyed on the string returned by `detectModelType()`.
+ */
+export declare const LAUNCH_PRESETS: Record<string, LaunchPreset>;
+//# sourceMappingURL=presets.d.ts.map

package/dist/presets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"presets.d.ts","sourceRoot":"","sources":["../src/presets.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,sBAAsB;IACjC,2EAA2E;;;;;;;;;IAU3E,oEAAoE;;;;;;;;;IAUpE,6EAA6E;;;;;;;;;IAU7E,gFAAgF;;;;;;;;;CASxE,CAAC;AAEX,6CAA6C;AAC7C,eAAO,MAAM,wBAAwB,EAAE,UAOtC,CAAC;AAEF,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB,EAAE,UAOpC,CAAC;AAEF,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAqBvD,CAAC"}

package/dist/presets.js

@@ -0,0 +1,98 @@
+/**
+ * Sampling defaults from Unsloth's Qwen3.6 guide:
+ * https://unsloth.ai/docs/models/qwen3.6#recommended-settings
+ *
+ * All modes pin `top_k = 20` and `min_p = 0.0`; they differ in
+ * `temperature`, `top_p`, and `presence_penalty`.
+ *
+ * The native anti-repetition cutoff is now disabled by default
+ * (vLLM-aligned — vLLM ships no repetition-stop heuristic), so these
+ * presets no longer pin `maxConsecutiveTokens` / `maxNgramRepeats` /
+ * `ngramSize`. Repetition is shaped by the sampling penalties above and
+ * bounded by the per-model `maxOutputTokens`. An operator or client can
+ * still opt in by setting those fields explicitly — a per-request config
+ * value wins via `ChatSession.mergeConfig`.
+ */
+export const QWEN_SAMPLING_DEFAULTS = {
+    /** Thinking mode for precise coding tasks: temp=0.6, top_p=0.95, pp=0.0 */
+    thinkingCoding: {
+        temperature: 0.6,
+        topP: 0.95,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 0.0,
+        repetitionPenalty: 1.0,
+    },
+    /** Thinking mode for general tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    thinkingGeneral: {
+        temperature: 1.0,
+        topP: 0.95,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 1.5,
+        repetitionPenalty: 1.0,
+    },
+    /** Instruct (non-thinking) for general tasks: temp=0.7, top_p=0.8, pp=1.5 */
+    instructGeneral: {
+        temperature: 0.7,
+        topP: 0.8,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 1.5,
+        repetitionPenalty: 1.0,
+    },
+    /** Instruct (non-thinking) for reasoning tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    instructReasoning: {
+        temperature: 1.0,
+        topP: 0.95,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 1.5,
+        repetitionPenalty: 1.0,
+    },
+};
+/** Sampling defaults for Gemma4 Instruct. */
+export const GEMMA4_SAMPLING_DEFAULTS = {
+    temperature: 0.7,
+    topP: 0.95,
+    topK: 64,
+    minP: 0.0,
+    presencePenalty: 0.0,
+    repetitionPenalty: 1.0,
+};
+/** Sampling defaults for LFM2.5 Thinking. */
+export const LFM2_SAMPLING_DEFAULTS = {
+    temperature: 0.05,
+    topP: 1.0,
+    topK: 50,
+    minP: 0.0,
+    presencePenalty: 0.0,
+    repetitionPenalty: 1.05,
+};
+/**
+ * Per-`ModelType` presets used by `mlx launch claude` to pre-wire a
+ * discovered model with sensible sampling defaults + a max output
+ * token budget. Keyed on the string returned by `detectModelType()`.
+ */
+export const LAUNCH_PRESETS = {
+    qwen3: {
+        sampling: QWEN_SAMPLING_DEFAULTS.thinkingCoding,
+        maxOutputTokens: 38912,
+    },
+    qwen3_5: {
+        sampling: QWEN_SAMPLING_DEFAULTS.thinkingCoding,
+        maxOutputTokens: 81920,
+    },
+    qwen3_5_moe: {
+        sampling: QWEN_SAMPLING_DEFAULTS.thinkingCoding,
+        maxOutputTokens: 81920,
+    },
+    gemma4: {
+        sampling: GEMMA4_SAMPLING_DEFAULTS,
+        maxOutputTokens: 16384,
+    },
+    lfm2: {
+        sampling: LFM2_SAMPLING_DEFAULTS,
+        maxOutputTokens: 8192,
+    },
+};

package/dist/registry.d.ts

@@ -36,6 +36,7 @@
  *     `body.model`. Instance ids recognise them as the same binding
  *     and the continuation is accepted.
  */
+import type { ChatConfig } from '@mlx-node/core';
 import type { SessionCapableModel } from '@mlx-node/lm';
 import { SessionRegistry } from './session-registry.js';
 /** Minimal contract for a model that can be served via chat sessions. */
@@ -64,6 +65,35 @@ export interface ModelRegistryOptions {
      */
     maxQueueDepth?: number;
 }
+/**
+ * Per-registration options for {@link ModelRegistry.register}.
+ */
+export interface RegisterOptions {
+    /**
+     * Per-model sampling defaults forwarded through the bound
+     * `SessionRegistry` into every `ChatSession` it allocates (as the
+     * session's `defaultConfig`). Clients' per-request sampling values
+     * (OpenAI `temperature`/`top_p`, Anthropic equivalents) still win
+     * where present because `ChatSession.mergeConfig` treats them as an
+     * overlay — these defaults only fill the gaps for parameters the
+     * client never sent (`top_k`, `min_p`, penalties, etc.).
+     *
+     * Re-registering the same name with a fresh `samplingDefaults` value
+     * overwrites the binding's defaults in place so the next
+     * `ChatSession` allocated out of the registry picks up the new
+     * values. Warm sessions already in flight keep the previous defaults
+     * until they settle; this matches how the refresh path treats other
+     * per-binding state.
+     */
+    samplingDefaults?: ChatConfig;
+    /**
+     * Optional per-model upper bound for generated output tokens. This is
+     * intentionally separate from `samplingDefaults.maxNewTokens`: client
+     * requests still provide the desired length, while endpoint handlers can
+     * clamp pathological values before dispatch.
+     */
+    maxOutputTokens?: number;
+}
 export declare class ModelRegistry {
     private readonly maxQueueDepth;
     private readonly models;
@@ -140,7 +170,7 @@ export declare class ModelRegistry {
      * preserves the id because `instanceIds.has(model)` is already
      * true.
      */
-    register(name: string, model: ServableModel): void;
+    register(name: string, model: ServableModel, opts?: RegisterOptions): void;
     /**
      * Unregister a model by name.
      *

package/dist/registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,yEAAyE;AACzE,MAAM,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAEhD,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,EAAE,eAAe,CAAC;CAClC;AAsCD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAoD;IAC7F;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAK;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkF;gBAEzG,IAAI,CAAC,EAAE,oBAAoB;IAIvC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,IAAI;IA8DlD;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAQjC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAClB,IAAI,EAAE,MAAM,GACX;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,eAAe,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS;IAWtF;;;;;;;;OAQG;IACH,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAUhD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAMzC;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAU1C;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,+BAA+B,CAAC,KAAK,EAAE,aAAa,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS;IAYzF;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAS5C;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI5C;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAM/C;;;;;;;;;;;;;;OAcG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;IAI7D;;;;;;;;;OASG;IACH,qBAAqB,IAAI,eAAe,EAAE;IAQ1C;;OAEG;IACH,IAAI,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE;IAa3E;;;;;;;;;OASG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO;CAIhD"}
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,yEAAyE;AACzE,MAAM,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAEhD,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,EAAE,eAAe,CAAC;CAClC;AAsCD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAoD;IAC7F;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAK;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkF;IAErH,YAAY,IAAI,CAAC,EAAE,oBAAoB,EAEtC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,IAAI,CAwFzE;IAED;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAMhC;IAED;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAClB,IAAI,EAAE,MAAM,GACX;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,eAAe,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CASrF;IAED;;;;;;;;OAQG;IACH,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQ/C;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAIxC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQzC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,+BAA+B,CAAC,KAAK,EAAE,aAAa,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAUxF;IAED;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAO3C;IAED;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAE3C;IAED;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAI9C;IAED;;;;;;;;;;;;;;OAcG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAE5D;IAED;;;;;;;;;OASG;IACH,qBAAqB,IAAI,eAAe,EAAE,CAMzC;IAED;;OAEG;IACH,IAAI,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAgB1E;IAED;;;;;;;;;OASG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAG9C;CACF"}

package/dist/registry.js

@@ -115,13 +115,23 @@ export class ModelRegistry {
      * preserves the id because `instanceIds.has(model)` is already
      * true.
      */
-    register(name, model) {
+    register(name, model, opts) {
+        const samplingDefaults = opts?.samplingDefaults;
+        const maxOutputTokens = opts?.maxOutputTokens;
         const existing = this.models.get(name);
         if (existing && existing.model === model) {
             // Same name + same model object: leave the binding and refcount
             // alone. Refresh createdAt so `/v1/models` surfaces the most
-            // recent registration time.
+            // recent registration time. A fresh `samplingDefaults` from the
+            // re-registration is applied in place so the operator can tune
+            // per-model knobs without fully unregistering first.
             existing.createdAt = Math.floor(Date.now() / 1000);
+            if (opts && 'samplingDefaults' in opts) {
+                existing.sessionRegistry.setSamplingDefaults(samplingDefaults);
+            }
+            if (opts && 'maxOutputTokens' in opts) {
+                existing.sessionRegistry.setMaxOutputTokens(maxOutputTokens);
+            }
             return;
         }
         if (existing) {
@@ -138,7 +148,12 @@ export class ModelRegistry {
         let binding = this.sessionRegistriesByModel.get(model);
         if (!binding) {
             binding = {
-                registry: new SessionRegistry({ model, maxQueueDepth: this.maxQueueDepth }),
+                registry: new SessionRegistry({
+                    model,
+                    maxQueueDepth: this.maxQueueDepth,
+                    samplingDefaults,
+                    maxOutputTokens,
+                }),
                 refCount: 0,
                 inFlight: 0,
                 pendingPersists: 0,
@@ -146,8 +161,21 @@ export class ModelRegistry {
             };
             this.sessionRegistriesByModel.set(model, binding);
         }
-        else if (binding.pendingTeardown) {
-            binding.pendingTeardown = false;
+        else {
+            if (binding.pendingTeardown) {
+                binding.pendingTeardown = false;
+            }
+            // Aliasing or reviving an existing binding: if this call passed
+            // `samplingDefaults` explicitly, overwrite the shared binding's
+            // defaults so the latest registration wins for every alias.
+            // Call sites that omit the field leave the existing defaults
+            // intact.
+            if (opts && 'samplingDefaults' in opts) {
+                binding.registry.setSamplingDefaults(samplingDefaults);
+            }
+            if (opts && 'maxOutputTokens' in opts) {
+                binding.registry.setMaxOutputTokens(maxOutputTokens);
+            }
         }
         binding.refCount += 1;
         // Allocate a fresh monotonic instance id on first sight of this

package/dist/router.d.ts

@@ -1,6 +1,9 @@
 /** Path-based router for /v1/* endpoints. */
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import type { ResponseStore } from '@mlx-node/core';
+import type { PublicModelEntry } from './handler.js';
+import type { IdleSweeper } from './idle-sweeper.js';
+import type { ModelWorkCoordinator } from './model-work-coordinator.js';
 import type { ModelRegistry } from './registry.js';
-export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number): Promise<void>;
+export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number, idleSweeper?: IdleSweeper | null, resolveModel?: (name: string) => Promise<void>, listModels?: () => PublicModelEntry[], modelWorkCoordinator?: ModelWorkCoordinator): Promise<void>;
 //# sourceMappingURL=router.d.ts.map

package/dist/router.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAYpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAyBnD,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,GAC5B,OAAO,CAAC,IAAI,CAAC,CA8Df"}
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAapD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAyBnD,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,EAC7B,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,EAChC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EAC9C,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,EACrC,oBAAoB,CAAC,EAAE,oBAAoB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAwGf"}

package/dist/router.js

@@ -1,4 +1,5 @@
 /** Path-based router for /v1/* endpoints. */
+import { handleCountMessageTokens } from './endpoints/messages-count-tokens.js';
 import { handleCreateMessage } from './endpoints/messages.js';
 import { handleListModels } from './endpoints/models.js';
 import { handleCreateResponse } from './endpoints/responses.js';
@@ -22,7 +23,7 @@ function readBody(req) {
         req.on('error', reject);
     });
 }
-export async function routeRequest(req, res, registry, store, responseRetentionSec) {
+export async function routeRequest(req, res, registry, store, responseRetentionSec, idleSweeper, resolveModel, listModels, modelWorkCoordinator) {
     const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
     const path = url.pathname;
     if (path === '/v1/models') {
@@ -30,7 +31,7 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
             sendMethodNotAllowed(res, 'GET');
             return;
         }
-        handleListModels(res, registry);
+        handleListModels(res, registry, listModels);
         return;
     }
     if (path === '/v1/responses') {
@@ -48,7 +49,25 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
             sendBadRequest(res, msg);
             return;
         }
-        await handleCreateResponse(res, body, registry, store, req, responseRetentionSec);
+        await handleCreateResponse(res, body, registry, store, req, responseRetentionSec, idleSweeper, modelWorkCoordinator);
+        return;
+    }
+    if (path === '/v1/messages/count_tokens') {
+        if (req.method !== 'POST') {
+            sendAnthropicMethodNotAllowed(res, 'POST');
+            return;
+        }
+        let body;
+        try {
+            const raw = await readBody(req);
+            body = JSON.parse(raw);
+        }
+        catch (err) {
+            const msg = err instanceof Error && err.message === 'Request body too large' ? err.message : 'Invalid JSON in request body';
+            sendAnthropicBadRequest(res, msg);
+            return;
+        }
+        await handleCountMessageTokens(res, body, registry, idleSweeper, resolveModel, modelWorkCoordinator);
         return;
     }
     if (path === '/v1/messages') {
@@ -66,7 +85,7 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
             sendAnthropicBadRequest(res, msg);
             return;
         }
-        await handleCreateMessage(res, body, registry, req);
+        await handleCreateMessage(res, body, registry, req, idleSweeper, resolveModel, modelWorkCoordinator);
         return;
     }
     if (path === '/health' || path === '/v1/health') {
@@ -74,5 +93,16 @@ export async function routeRequest(req, res, registry, store, responseRetentionS
         res.end(JSON.stringify({ status: 'ok' }));
         return;
     }
+    // Liveness probe at `/`. Claude Code issues `HEAD /` before its first
+    // request; respond 200 so the probe doesn't leave a 404 in the logs.
+    if (path === '/') {
+        if (req.method !== 'GET' && req.method !== 'HEAD') {
+            sendMethodNotAllowed(res, 'GET, HEAD');
+            return;
+        }
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(req.method === 'HEAD' ? undefined : JSON.stringify({ service: 'mlx-node' }));
+        return;
+    }
     sendNotFound(res, `No route matches ${req.method} ${path}`);
 }

package/dist/server.d.ts

@@ -1,6 +1,7 @@
 /** Full HTTP server lifecycle: wires up the handler and periodically sweeps expired `ResponseStore` rows and sessions. */
 import type { Server } from 'node:http';
 import { ResponseStore } from '@mlx-node/core';
+import type { PublicModelEntry } from './handler.js';
 import { ModelRegistry } from './registry.js';
 /**
  * Parse a positive integer seconds value; returns undefined for unset/invalid so caller can apply its own default.
@@ -56,6 +57,36 @@ export interface ServerConfig {
      * `MLX_MAX_QUEUE_DEPTH_PER_MODEL` (positive integer).
      */
     maxQueueDepthPerModel?: number;
+    /**
+     * Milliseconds of HTTP inactivity (no request arrivals or completions)
+     * before the server issues a single `clearCache()` to drain the MLX
+     * Metal allocator's free pool. Replaces the old per-request
+     * `ClearCacheOnDrop` guard in the Rust layer, which was unsafe on a
+     * multi-model server because the pool is process-wide.
+     *
+     * Default: 30_000 ms (30 seconds). `0` disables the sweeper. Env
+     * override: `MLX_IDLE_CLEAR_CACHE_MS` (non-negative integer; 0
+     * disables; unset falls through to default).
+     *
+     * The decode-loop drain every 256 tokens inside each generative
+     * model is untouched and covers in-flight memory churn.
+     */
+    idleClearCacheMs?: number;
+    /**
+     * Optional async callback invoked before the endpoint layer looks
+     * the model up in the registry. Intended for lazy-load schemes:
+     * the callback should register the model into `registry` if it can;
+     * on return, the endpoint does `registry.get(body.model)` and 404s
+     * if still unresolved. Callback errors bubble up as 500s.
+     */
+    resolveModel?: (name: string) => Promise<void>;
+    /**
+     * Optional override for `GET /v1/models` enumeration. When provided,
+     * the endpoint returns this list instead of `registry.list()`.
+     * Intended for dynamic discovery schemes (e.g. enumerate every model
+     * on disk while only the currently-resident one is registered).
+     */
+    listModels?: () => PublicModelEntry[];
 }
 export interface ServerInstance {
     server: Server;
@@ -65,6 +96,51 @@ export interface ServerInstance {
     store: ResponseStore | null;
     /** Graceful shutdown. */
     close(): Promise<void>;
+    /**
+     * Run `fn` with the idle-drain timer suspended for the duration of
+     * an unbracketed, allocator-heavy operation — most commonly a hot
+     * `Model::load()` invoked AFTER the server has already served at
+     * least one request. In that scenario the post-request drain timer
+     * armed by `endRequest()` (at t+idleClearCacheMs) can otherwise
+     * fire MID-LOAD, racing the Metal allocator while weight
+     * materialization is still in progress.
+     *
+     * Handles try/finally bracketing so a thrown load never leaks the
+     * internal suspend counter. Accepts both sync and async functions;
+     * returns `fn`'s own return value (or resolved promise). When the
+     * bracket exits (normal or thrown) AND `inFlight === 0` with no
+     * other suspend active, a fresh drain timer is armed.
+     *
+     * Safe to nest — each call allocates its own token-scoped release
+     * so overlapping brackets unwind independently.
+     *
+     * The common `serve.ts` pattern (load all models BEFORE
+     * `createServer()`) does not need this API — there is no armed
+     * timer before the first request, so there is no race.
+     *
+     * Pass-through when the sweeper is disabled (`idleClearCacheMs: 0`
+     * or missing `__internal__.clearCache`): `fn` is invoked directly.
+     *
+     * @example
+     * ```ts
+     * await instance.withSuspendedDrains(async () => {
+     *   const model = await Qwen35Model.load(modelPath);
+     *   instance.registry.register('new-model', model);
+     * });
+     * ```
+     */
+    withSuspendedDrains<T>(fn: () => Promise<T>): Promise<T>;
+    withSuspendedDrains<T>(fn: () => T): T;
+    /**
+     * Low-level: suspend drains and return an idempotent, token-scoped
+     * disposer. Prefer {@link withSuspendedDrains} unless you need
+     * manual control over when the suspend is released. Safe to nest;
+     * calling the disposer more than once is a no-op.
+     *
+     * No-op when the sweeper is disabled (`idleClearCacheMs: 0` or
+     * missing `__internal__.clearCache`): returns a no-op disposer.
+     */
+    suspendDrains(): () => void;
 }
 /**
  * Start an MLX-Node HTTP server exposing `POST /v1/responses`,

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAIxC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAgB9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B,yBAAyB;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAoEjF"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAI1H,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAIxC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAIrD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAgB9C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOhE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOpE;AAuBD,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,2CAA2C;IAC3C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;;;;;OAaG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,aAAa,CAAC;IACxB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC;IAC5B,yBAAyB;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,mBAAmB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC;;;;;;;;OAQG;IACH,aAAa,IAAI,MAAM,IAAI,CAAC;CAC7B;AAED;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA8FjF"}

package/dist/server.js

@@ -5,6 +5,8 @@ import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { ResponseStore } from '@mlx-node/core';
 import { createHandler } from './handler.js';
+import { createIdleSweeper, DEFAULT_IDLE_CLEAR_CACHE_MS, parseIdleClearCacheEnv } from './idle-sweeper.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
 import { ModelRegistry } from './registry.js';
 /** Cleanup interval for expired responses (ms). */
 const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
@@ -108,7 +110,18 @@ export async function createServer(config) {
     // allocated on `register()`) all share a single effective value.
     const configMaxQueueDepth = normalizePositiveIntConfig(config?.maxQueueDepthPerModel, 'maxQueueDepthPerModel');
     const maxQueueDepthPerModel = configMaxQueueDepth ?? parseEnvPositiveInt('MLX_MAX_QUEUE_DEPTH_PER_MODEL');
+    // Idle sweeper wiring. `0` is a legal explicit "off" value so we
+    // cannot reuse `normalizePositiveIntConfig` (which rejects 0).
+    // Precedence: explicit config value wins over env wins over default.
+    const idleClearCacheMs = resolveIdleClearCacheMs(config?.idleClearCacheMs);
+    const idleSweeper = createIdleSweeper(idleClearCacheMs);
+    // The sweeper is driven exclusively by `endRequest()` calls in the
+    // inference endpoints (`/v1/responses`, `/v1/messages`). There is no
+    // cold-start drain: a process that loads models but never receives a
+    // request will not fire `clearCache()`. See the `idle-sweeper.ts`
+    // module doc (section "Drain is post-request only") for rationale.
     const registry = new ModelRegistry({ maxQueueDepth: maxQueueDepthPerModel });
+    const modelWorkCoordinator = new ModelWorkCoordinator();
     let store = null;
     if (!disableStore) {
         const storePath = config?.storePath ?? join(homedir(), '.mlx-node', 'responses.db');
@@ -126,7 +139,15 @@ export async function createServer(config) {
         }
     }, CLEANUP_INTERVAL_MS);
     cleanupTimer.unref();
-    const handler = createHandler(registry, { cors, store, responseRetentionSec });
+    const handler = createHandler(registry, {
+        cors,
+        store,
+        responseRetentionSec,
+        idleSweeper,
+        resolveModel: config?.resolveModel,
+        modelWorkCoordinator,
+        listModels: config?.listModels,
+    });
     const server = httpCreateServer(handler);
     await new Promise((resolve, reject) => {
         const onError = (err) => {
@@ -145,6 +166,7 @@ export async function createServer(config) {
         store,
         async close() {
             clearInterval(cleanupTimer);
+            idleSweeper.close();
             await new Promise((resolve, reject) => {
                 server.close((err) => {
                     if (err)
@@ -154,5 +176,30 @@ export async function createServer(config) {
                 });
             });
         },
+        withSuspendedDrains(fn) {
+            return idleSweeper.withSuspendedDrains(fn);
+        },
+        suspendDrains() {
+            return idleSweeper.suspendDrains();
+        },
     };
 }
+/**
+ * Resolve the effective idle-drain delay from (constructor value, env,
+ * default). Unlike the other knobs, `0` is a legal explicit opt-out —
+ * we must NOT fall through to env/default when the caller explicitly
+ * passes `0`. Returns a non-negative integer milliseconds value.
+ */
+function resolveIdleClearCacheMs(configValue) {
+    if (configValue !== undefined) {
+        if (typeof configValue !== 'number' ||
+            !Number.isFinite(configValue) ||
+            !Number.isInteger(configValue) ||
+            configValue < 0) {
+            throw new Error(`idleClearCacheMs must be a non-negative integer; received ${String(configValue)}`);
+        }
+        return configValue;
+    }
+    const envValue = parseIdleClearCacheEnv();
+    return envValue ?? DEFAULT_IDLE_CLEAR_CACHE_MS;
+}