@mlx-node/server 0.0.7 → 0.0.9

package/dist/model-work-coordinator.js

@@ -0,0 +1,201 @@
+/**
+ * Render a thrown value for {@link ModelLoadRecord.error}.
+ *
+ * Deliberately avoids `String(unknown)`: a rejection carrying a plain object
+ * would render as the useless `[object Object]` in the one field a supervisor
+ * reads to find out why the model would not load.
+ */
+function describeLoadFailure(error) {
+    if (error instanceof Error)
+        return error.message;
+    if (typeof error === 'string')
+        return error;
+    if (error == null)
+        return 'unknown error';
+    try {
+        return JSON.stringify(error) ?? 'unknown error';
+    }
+    catch {
+        // Circular structure, or a `toJSON` that throws.
+        return 'unknown error';
+    }
+}
+/**
+ * Process-local gate for native MLX work.
+ *
+ * Individual model instances already have a per-model execution mutex, but a
+ * lazy `loadModel()` can still run load-time materialization / warmup Metal
+ * work while another model is decoding. MLX's allocator and command queues are
+ * process-wide, so model load/swap takes an exclusive writer slot; inference
+ * takes shared reader slots.
+ */
+export class ModelWorkCoordinator {
+    activeReaders = 0;
+    writerHeld = false;
+    queuedWriters = 0;
+    readerWaiters = [];
+    writerWaiters = [];
+    /**
+     * Most recent settled load bracket. Retained here because the coordinator
+     * is the ONE place that brackets every load: a `resolveModel` failure in
+     * `/v1/messages` becomes a 500 and is otherwise dropped on the floor, so a
+     * supervisor polling `/health` afterwards had no way to learn what went
+     * wrong. See {@link ModelLoadRecord} for the "successful no-op overwrites
+     * an earlier failure" caveat.
+     */
+    lastLoadRecord = null;
+    /** Read-only: `true` while a load holds the exclusive writer slot. */
+    get writerActive() {
+        return this.writerHeld;
+    }
+    /** Read-only: loads parked in `acquireWrite()` waiting for the slot. */
+    get waitingWriters() {
+        return this.queuedWriters;
+    }
+    /** Read-only: outcome of the most recent settled load bracket, or `null`. */
+    get lastLoad() {
+        return this.lastLoadRecord;
+    }
+    /**
+     * Record a settled bracket. Called from the `finally` of both load
+     * wrappers so a throw is captured just as reliably as a success.
+     */
+    recordLoad(label, startedAt, error, ok) {
+        this.lastLoadRecord = {
+            label: label ?? null,
+            startedAt,
+            finishedAt: Date.now(),
+            ok,
+            error: ok ? null : describeLoadFailure(error),
+        };
+    }
+    /**
+     * @param label Optional identifier (normally the model name) stamped into
+     *   {@link lastLoad} so `/health` can name what was being loaded.
+     */
+    async withModelLoad(fn, label) {
+        await this.acquireWrite();
+        // Measured from lock acquisition, not from arrival: `startedAt` is meant
+        // to answer "how long has the actual materialization been running",
+        // which is what a supervisor deciding whether to wait needs.
+        const startedAt = Date.now();
+        let ok = false;
+        let failure;
+        try {
+            const result = await fn();
+            ok = true;
+            return result;
+        }
+        catch (err) {
+            failure = err;
+            throw err;
+        }
+        finally {
+            this.recordLoad(label, startedAt, failure, ok);
+            this.releaseWrite();
+        }
+    }
+    /**
+     * Like {@link withModelLoad} but reports whether THIS caller owned the
+     * load (acquired the writer lock with no contention) or merely waited
+     * behind a load that was already in flight when it arrived.
+     *
+     * Decided at sync-time before any await: if neither a writer is active
+     * nor any writer is queued ahead, this caller is the owner; otherwise
+     * it is parked behind someone else's load and `owner` is `false`. The
+     * distinction is used by `/v1/messages` to split `resolve_ms` (own
+     * load + lookup) from `load_wait_ms` (waiting on a peer's load) so a
+     * 60-second cold-load does not look like 60 seconds of own work for
+     * every concurrent request.
+     */
+    async withModelLoadInstrumented(fn, label) {
+        // `owner` MUST be decided synchronously, before any await, so the
+        // signal reflects coordinator state at arrival rather than after
+        // any peer transition. The wait/own split is measured around the
+        // actual phase boundaries (lock acquisition, fn completion) so the
+        // two intervals partition cleanly instead of both reporting total
+        // elapsed time — see `ModelLoadOutcome` for the contract.
+        const owner = !this.writerHeld && this.queuedWriters === 0;
+        const arrivedAt = Date.now();
+        await this.acquireWrite();
+        const lockAcquiredAt = Date.now();
+        let ok = false;
+        let failure;
+        try {
+            const result = await fn();
+            ok = true;
+            const fnDoneAt = Date.now();
+            const waitMs = Math.max(0, lockAcquiredAt - arrivedAt);
+            const ownMs = Math.max(0, fnDoneAt - lockAcquiredAt);
+            return { result, owner, waitMs, ownMs };
+        }
+        catch (err) {
+            failure = err;
+            throw err;
+        }
+        finally {
+            this.recordLoad(label, lockAcquiredAt, failure, ok);
+            this.releaseWrite();
+        }
+    }
+    async withInference(fn) {
+        await this.acquireRead();
+        try {
+            return await fn();
+        }
+        finally {
+            this.releaseRead();
+        }
+    }
+    acquireRead() {
+        if (!this.writerHeld && this.queuedWriters === 0) {
+            this.activeReaders += 1;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.readerWaiters.push(() => {
+                this.activeReaders += 1;
+                resolve();
+            });
+        });
+    }
+    acquireWrite() {
+        this.queuedWriters += 1;
+        if (!this.writerHeld && this.activeReaders === 0) {
+            this.queuedWriters -= 1;
+            this.writerHeld = true;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.writerWaiters.push(() => {
+                this.queuedWriters -= 1;
+                this.writerHeld = true;
+                resolve();
+            });
+        });
+    }
+    releaseRead() {
+        this.activeReaders -= 1;
+        if (this.activeReaders < 0)
+            this.activeReaders = 0;
+        if (this.activeReaders === 0)
+            this.drain();
+    }
+    releaseWrite() {
+        this.writerHeld = false;
+        this.drain();
+    }
+    drain() {
+        if (this.writerHeld)
+            return;
+        if (this.activeReaders === 0 && this.writerWaiters.length > 0) {
+            this.writerWaiters.shift()?.();
+            return;
+        }
+        if (this.queuedWriters === 0 && this.readerWaiters.length > 0) {
+            const readers = this.readerWaiters.splice(0);
+            for (const resolve of readers)
+                resolve();
+        }
+    }
+}

package/dist/pending-writes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pending-writes.d.ts","sourceRoot":"","sources":["../src/pending-writes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwGG;AAEH;;;;;;;GAOG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyC;IAEjE;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAkC;IAE3E;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CACjB;IAEZ;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAM;IAElD;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI;IA0ClF;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IAInD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,sBAAsB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAWtD;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,YAAY;IAI3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,OAAO;IAyB/E;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IAYpB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAqBnC,iEAAiE;IACjE,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,gBAAgB,IAAI,MAAM,CAG7B;IAED;;;;;;OAMG;IACH,IAAI,4BAA4B,IAAI,MAAM,CAEzC;CACF;AAYD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAOxE"}
+{"version":3,"file":"pending-writes.d.ts","sourceRoot":"","sources":["../src/pending-writes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwGG;AAEH;;;;;;;GAOG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyC;IAEjE;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAkC;IAE3E;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CACjB;IAEZ;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAM;IAElD;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAwCjF;IAED;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAElD;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,sBAAsB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CASrD;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,YAAY;IAI3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAuB9E;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IAYpB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAmBlC;IAED,iEAAiE;IACjE,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,gBAAgB,IAAI,MAAM,CAG7B;IAED;;;;;;OAMG;IACH,IAAI,4BAA4B,IAAI,MAAM,CAEzC;CACF;AAYD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAOxE"}

package/dist/presets.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Sampling presets recommended by third-party model authors, exposed
+ * here as `ChatConfig`-shaped objects so an operator can pin them at
+ * `ModelRegistry.register(name, model, { samplingDefaults: ... })`
+ * time with a single import.
+ *
+ * Per-request client values (OpenAI `temperature`/`top_p`, Anthropic
+ * equivalents) still override these defaults where the client sends
+ * them — `ChatSession.mergeConfig` treats per-call config as an
+ * overlay on top of `defaultConfig`. These presets only fill in the
+ * parameters clients never send (`top_k`, `min_p`, penalties).
+ */
+import type { ChatConfig } from '@mlx-node/core';
+/**
+ * Sampling defaults from Unsloth's Qwen3.6 guide:
+ * https://unsloth.ai/docs/models/qwen3.6#recommended-settings
+ *
+ * All modes pin `top_k = 20` and `min_p = 0.0`; they differ in
+ * `temperature`, `top_p`, and `presence_penalty`.
+ *
+ * The native anti-repetition cutoff is now disabled by default
+ * (vLLM-aligned — vLLM ships no repetition-stop heuristic), so these
+ * presets no longer pin `maxConsecutiveTokens` / `maxNgramRepeats` /
+ * `ngramSize`. Repetition is shaped by the sampling penalties above and
+ * bounded by the per-model `maxOutputTokens`. An operator or client can
+ * still opt in by setting those fields explicitly — a per-request config
+ * value wins via `ChatSession.mergeConfig`.
+ */
+export declare const QWEN_SAMPLING_DEFAULTS: {
+    /** Thinking mode for precise coding tasks: temp=0.6, top_p=0.95, pp=0.0 */
+    readonly thinkingCoding: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+    /** Thinking mode for general tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    readonly thinkingGeneral: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+    /** Instruct (non-thinking) for general tasks: temp=0.7, top_p=0.8, pp=1.5 */
+    readonly instructGeneral: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+    /** Instruct (non-thinking) for reasoning tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    readonly instructReasoning: {
+        temperature: number;
+        topP: number;
+        topK: number;
+        minP: number;
+        presencePenalty: number;
+        repetitionPenalty: number;
+    };
+};
+/** Sampling defaults for Gemma4 Instruct. */
+export declare const GEMMA4_SAMPLING_DEFAULTS: ChatConfig;
+/** Sampling defaults for LFM2.5 Thinking. */
+export declare const LFM2_SAMPLING_DEFAULTS: ChatConfig;
+/** Sampling + per-model output token cap exposed by {@link LAUNCH_PRESETS}. */
+export interface LaunchPreset {
+    sampling: ChatConfig;
+    maxOutputTokens: number;
+}
+/**
+ * Per-`ModelType` presets used by `mlx launch claude` to pre-wire a
+ * discovered model with sensible sampling defaults + a max output
+ * token budget. Keyed on the string returned by `detectModelType()`.
+ */
+export declare const LAUNCH_PRESETS: Record<string, LaunchPreset>;
+//# sourceMappingURL=presets.d.ts.map

package/dist/presets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"presets.d.ts","sourceRoot":"","sources":["../src/presets.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,sBAAsB;IACjC,2EAA2E;;;;;;;;;IAU3E,oEAAoE;;;;;;;;;IAUpE,6EAA6E;;;;;;;;;IAU7E,gFAAgF;;;;;;;;;CASxE,CAAC;AAEX,6CAA6C;AAC7C,eAAO,MAAM,wBAAwB,EAAE,UAOtC,CAAC;AAEF,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB,EAAE,UAOpC,CAAC;AAEF,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAqBvD,CAAC"}

package/dist/presets.js

@@ -0,0 +1,98 @@
+/**
+ * Sampling defaults from Unsloth's Qwen3.6 guide:
+ * https://unsloth.ai/docs/models/qwen3.6#recommended-settings
+ *
+ * All modes pin `top_k = 20` and `min_p = 0.0`; they differ in
+ * `temperature`, `top_p`, and `presence_penalty`.
+ *
+ * The native anti-repetition cutoff is now disabled by default
+ * (vLLM-aligned — vLLM ships no repetition-stop heuristic), so these
+ * presets no longer pin `maxConsecutiveTokens` / `maxNgramRepeats` /
+ * `ngramSize`. Repetition is shaped by the sampling penalties above and
+ * bounded by the per-model `maxOutputTokens`. An operator or client can
+ * still opt in by setting those fields explicitly — a per-request config
+ * value wins via `ChatSession.mergeConfig`.
+ */
+export const QWEN_SAMPLING_DEFAULTS = {
+    /** Thinking mode for precise coding tasks: temp=0.6, top_p=0.95, pp=0.0 */
+    thinkingCoding: {
+        temperature: 0.6,
+        topP: 0.95,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 0.0,
+        repetitionPenalty: 1.0,
+    },
+    /** Thinking mode for general tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    thinkingGeneral: {
+        temperature: 1.0,
+        topP: 0.95,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 1.5,
+        repetitionPenalty: 1.0,
+    },
+    /** Instruct (non-thinking) for general tasks: temp=0.7, top_p=0.8, pp=1.5 */
+    instructGeneral: {
+        temperature: 0.7,
+        topP: 0.8,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 1.5,
+        repetitionPenalty: 1.0,
+    },
+    /** Instruct (non-thinking) for reasoning tasks: temp=1.0, top_p=0.95, pp=1.5 */
+    instructReasoning: {
+        temperature: 1.0,
+        topP: 0.95,
+        topK: 20,
+        minP: 0.0,
+        presencePenalty: 1.5,
+        repetitionPenalty: 1.0,
+    },
+};
+/** Sampling defaults for Gemma4 Instruct. */
+export const GEMMA4_SAMPLING_DEFAULTS = {
+    temperature: 0.7,
+    topP: 0.95,
+    topK: 64,
+    minP: 0.0,
+    presencePenalty: 0.0,
+    repetitionPenalty: 1.0,
+};
+/** Sampling defaults for LFM2.5 Thinking. */
+export const LFM2_SAMPLING_DEFAULTS = {
+    temperature: 0.05,
+    topP: 1.0,
+    topK: 50,
+    minP: 0.0,
+    presencePenalty: 0.0,
+    repetitionPenalty: 1.05,
+};
+/**
+ * Per-`ModelType` presets used by `mlx launch claude` to pre-wire a
+ * discovered model with sensible sampling defaults + a max output
+ * token budget. Keyed on the string returned by `detectModelType()`.
+ */
+export const LAUNCH_PRESETS = {
+    qwen3: {
+        sampling: QWEN_SAMPLING_DEFAULTS.thinkingCoding,
+        maxOutputTokens: 38912,
+    },
+    qwen3_5: {
+        sampling: QWEN_SAMPLING_DEFAULTS.thinkingCoding,
+        maxOutputTokens: 81920,
+    },
+    qwen3_5_moe: {
+        sampling: QWEN_SAMPLING_DEFAULTS.thinkingCoding,
+        maxOutputTokens: 81920,
+    },
+    gemma4: {
+        sampling: GEMMA4_SAMPLING_DEFAULTS,
+        maxOutputTokens: 16384,
+    },
+    lfm2: {
+        sampling: LFM2_SAMPLING_DEFAULTS,
+        maxOutputTokens: 8192,
+    },
+};

package/dist/registry.d.ts

@@ -36,6 +36,7 @@
  *     `body.model`. Instance ids recognise them as the same binding
  *     and the continuation is accepted.
  */
+import type { ChatConfig } from '@mlx-node/core';
 import type { SessionCapableModel } from '@mlx-node/lm';
 import { SessionRegistry } from './session-registry.js';
 /** Minimal contract for a model that can be served via chat sessions. */
@@ -64,6 +65,35 @@ export interface ModelRegistryOptions {
      */
     maxQueueDepth?: number;
 }
+/**
+ * Per-registration options for {@link ModelRegistry.register}.
+ */
+export interface RegisterOptions {
+    /**
+     * Per-model sampling defaults forwarded through the bound
+     * `SessionRegistry` into every `ChatSession` it allocates (as the
+     * session's `defaultConfig`). Clients' per-request sampling values
+     * (OpenAI `temperature`/`top_p`, Anthropic equivalents) still win
+     * where present because `ChatSession.mergeConfig` treats them as an
+     * overlay — these defaults only fill the gaps for parameters the
+     * client never sent (`top_k`, `min_p`, penalties, etc.).
+     *
+     * Re-registering the same name with a fresh `samplingDefaults` value
+     * overwrites the binding's defaults in place so the next
+     * `ChatSession` allocated out of the registry picks up the new
+     * values. Warm sessions already in flight keep the previous defaults
+     * until they settle; this matches how the refresh path treats other
+     * per-binding state.
+     */
+    samplingDefaults?: ChatConfig;
+    /**
+     * Optional per-model upper bound for generated output tokens. This is
+     * intentionally separate from `samplingDefaults.maxNewTokens`: client
+     * requests still provide the desired length, while endpoint handlers can
+     * clamp pathological values before dispatch.
+     */
+    maxOutputTokens?: number;
+}
 export declare class ModelRegistry {
     private readonly maxQueueDepth;
     private readonly models;
@@ -140,7 +170,7 @@ export declare class ModelRegistry {
      * preserves the id because `instanceIds.has(model)` is already
      * true.
      */
-    register(name: string, model: ServableModel): void;
+    register(name: string, model: ServableModel, opts?: RegisterOptions): void;
     /**
      * Unregister a model by name.
      *

package/dist/registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,yEAAyE;AACzE,MAAM,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAEhD,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,EAAE,eAAe,CAAC;CAClC;AAsCD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAoD;IAC7F;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAK;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkF;gBAEzG,IAAI,CAAC,EAAE,oBAAoB;IAIvC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,IAAI;IA8DlD;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAQjC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAClB,IAAI,EAAE,MAAM,GACX;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,eAAe,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS;IAWtF;;;;;;;;OAQG;IACH,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAUhD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAMzC;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAU1C;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,+BAA+B,CAAC,KAAK,EAAE,aAAa,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS;IAYzF;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAS5C;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI5C;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAM/C;;;;;;;;;;;;;;OAcG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;IAI7D;;;;;;;;;OASG;IACH,qBAAqB,IAAI,eAAe,EAAE;IAQ1C;;OAEG;IACH,IAAI,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE;IAa3E;;;;;;;;;OASG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO;CAIhD"}
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,yEAAyE;AACzE,MAAM,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAEhD,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,EAAE,eAAe,CAAC;CAClC;AAsCD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,CAAC,EAAE,UAAU,CAAC;IAC9B;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAoD;IAC7F;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,yDAAyD;IACzD,OAAO,CAAC,cAAc,CAAK;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkF;IAErH,YAAY,IAAI,CAAC,EAAE,oBAAoB,EAEtC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,IAAI,CAwFzE;IAED;;;;;;;;;;OAUG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAMhC;IAED;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAClB,IAAI,EAAE,MAAM,GACX;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,eAAe,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CASrF;IAED;;;;;;;;OAQG;IACH,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQ/C;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAIxC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAQzC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,+BAA+B,CAAC,KAAK,EAAE,aAAa,GAAG;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAUxF;IAED;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAO3C;IAED;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAE3C;IAED;;;;;;;;;;;;;;OAcG;IACH,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAI9C;IAED;;;;;;;;;;;;;;OAcG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAE5D;IAED;;;;;;;;;OASG;IACH,qBAAqB,IAAI,eAAe,EAAE,CAMzC;IAED;;OAEG;IACH,IAAI,IAAI;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,CAgB1E;IAED;;;;;;;;;OASG;IACH,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAG9C;CACF"}

package/dist/registry.js

@@ -115,13 +115,23 @@ export class ModelRegistry {
      * preserves the id because `instanceIds.has(model)` is already
      * true.
      */
-    register(name, model) {
+    register(name, model, opts) {
+        const samplingDefaults = opts?.samplingDefaults;
+        const maxOutputTokens = opts?.maxOutputTokens;
         const existing = this.models.get(name);
         if (existing && existing.model === model) {
             // Same name + same model object: leave the binding and refcount
             // alone. Refresh createdAt so `/v1/models` surfaces the most
-            // recent registration time.
+            // recent registration time. A fresh `samplingDefaults` from the
+            // re-registration is applied in place so the operator can tune
+            // per-model knobs without fully unregistering first.
             existing.createdAt = Math.floor(Date.now() / 1000);
+            if (opts && 'samplingDefaults' in opts) {
+                existing.sessionRegistry.setSamplingDefaults(samplingDefaults);
+            }
+            if (opts && 'maxOutputTokens' in opts) {
+                existing.sessionRegistry.setMaxOutputTokens(maxOutputTokens);
+            }
             return;
         }
         if (existing) {
@@ -138,7 +148,12 @@ export class ModelRegistry {
         let binding = this.sessionRegistriesByModel.get(model);
         if (!binding) {
             binding = {
-                registry: new SessionRegistry({ model, maxQueueDepth: this.maxQueueDepth }),
+                registry: new SessionRegistry({
+                    model,
+                    maxQueueDepth: this.maxQueueDepth,
+                    samplingDefaults,
+                    maxOutputTokens,
+                }),
                 refCount: 0,
                 inFlight: 0,
                 pendingPersists: 0,
@@ -146,8 +161,21 @@ export class ModelRegistry {
             };
             this.sessionRegistriesByModel.set(model, binding);
         }
-        else if (binding.pendingTeardown) {
-            binding.pendingTeardown = false;
+        else {
+            if (binding.pendingTeardown) {
+                binding.pendingTeardown = false;
+            }
+            // Aliasing or reviving an existing binding: if this call passed
+            // `samplingDefaults` explicitly, overwrite the shared binding's
+            // defaults so the latest registration wins for every alias.
+            // Call sites that omit the field leave the existing defaults
+            // intact.
+            if (opts && 'samplingDefaults' in opts) {
+                binding.registry.setSamplingDefaults(samplingDefaults);
+            }
+            if (opts && 'maxOutputTokens' in opts) {
+                binding.registry.setMaxOutputTokens(maxOutputTokens);
+            }
         }
         binding.refCount += 1;
         // Allocate a fresh monotonic instance id on first sight of this

package/dist/router.d.ts

@@ -1,6 +1,42 @@
 /** Path-based router for /v1/* endpoints. */
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import type { ResponseStore } from '@mlx-node/core';
+import type { PublicModelEntry } from './handler.js';
+import { type ServerHealth } from './health.js';
+import type { IdleSweeper } from './idle-sweeper.js';
+import type { ModelWorkCoordinator } from './model-work-coordinator.js';
 import type { ModelRegistry } from './registry.js';
-export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number): Promise<void>;
+/**
+ * The request's pathname, parsed against a CONSTANT base.
+ *
+ * Never against `Host`. That header is attacker-controlled text on every
+ * request — `Host: [` makes `new URL()` throw `ERR_INVALID_URL`, and a throw
+ * from an async request listener is an unhandled rejection, which under Node's
+ * default `--unhandled-rejections=throw` takes the whole process down. One
+ * malformed byte from any client that can reach the socket was enough to end
+ * inference. A pathname does not depend on the authority anyway, so a fixed
+ * base is both safer and equivalent: an absolute-form request URI
+ * (`GET http://host/v1/models HTTP/1.1`, legal in HTTP/1.1) still wins over
+ * the base and yields the same path it always did.
+ *
+ * `req.url` itself is guarded too, for the same reason rather than a known
+ * input: this function's contract is that no request can make it throw.
+ */
+export declare function requestPathname(req: IncomingMessage): string;
+/**
+ * Trailing options bag. Added as an object rather than two more positional
+ * parameters — `routeRequest` already carries nine, and the two knobs here
+ * are unrelated to each other.
+ */
+export interface RouteExtras {
+    /** Builds the `/health` body. Omitted ⇒ the legacy constant `{ status: 'ok' }`. */
+    health?: () => ServerHealth;
+    /**
+     * Whether the caller presented a valid token. Only consulted by `/health`,
+     * which is the one route reachable without one. `true` when no token is
+     * configured at all, so an unprotected server keeps serving the full body.
+     */
+    authenticated?: boolean;
+}
+export declare function routeRequest(req: IncomingMessage, res: ServerResponse, registry: ModelRegistry, store: ResponseStore | null, responseRetentionSec?: number, idleSweeper?: IdleSweeper | null, resolveModel?: (name: string) => Promise<void>, listModels?: () => PublicModelEntry[], modelWorkCoordinator?: ModelWorkCoordinator, extras?: RouteExtras): Promise<void>;
 //# sourceMappingURL=router.d.ts.map

package/dist/router.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAYpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAyBnD,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,GAC5B,OAAO,CAAC,IAAI,CAAC,CA8Df"}
+{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAE7C,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAapD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAmB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAOnD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,CAS5D;AAoBD;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,mFAAmF;IACnF,MAAM,CAAC,EAAE,MAAM,YAAY,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,eAAe,EACpB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,aAAa,GAAG,IAAI,EAC3B,oBAAoB,CAAC,EAAE,MAAM,EAC7B,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,EAChC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EAC9C,UAAU,CAAC,EAAE,MAAM,gBAAgB,EAAE,EACrC,oBAAoB,CAAC,EAAE,oBAAoB,EAC3C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,IAAI,CAAC,CAuHf"}