@mlx-node/server 0.0.0 → 0.0.8

package/dist/idle-sweeper.js

@@ -0,0 +1,408 @@
+/**
+ * Idle cache-pool sweeper.
+ *
+ * The MLX Metal allocator holds a process-wide free pool. Without an
+ * explicit drain the pool can sit near the wired ceiling (tens of GB on
+ * an M3 Max) for the entire server uptime. `@mlx-node/core`'s decode
+ * loop already calls `mlx_clear_cache()` every 256 steps, but that
+ * cadence only fires while a generation is in progress — idle periods
+ * between HTTP requests never get a drain.
+ *
+ * An earlier iteration put a `ClearCacheOnDrop` RAII guard around every
+ * session command so the pool was flushed after every turn. That is
+ * wrong on a multi-model server: the free pool is shared across model
+ * instances, so flushing after a request on model A discards blocks
+ * that model B's next turn is about to reuse.
+ *
+ * The round-1 replacement was a debounced `touch()`-based sweeper:
+ * every request start / end reset a 30s timer that fired `clearCache()`
+ * when the timer expired. That design had a correctness bug flagged in
+ * round-2 review: a long-running request (streaming decode of 60+
+ * seconds) had its 30s timer armed at ARRIVAL, and the timer fired
+ * mid-request at t=30s — exactly when the generation thread still
+ * needed those blocks. Worse, `clearCache()` routes through
+ * `mlx_synchronize()` WITHOUT a stream argument, which only drains the
+ * default stream (see `crates/mlx-sys/mlx/mlx/scheduler.cpp`), but
+ * decode runs on custom streams — so the drain could race with live
+ * command buffers, risking buffer use-after-free.
+ *
+ * This version fixes both by tracking `inFlight` explicitly. The timer
+ * is ONLY armed when the counter returns to zero, and is cancelled the
+ * instant any new request arrives — mid-request drains are impossible
+ * by construction.
+ *
+ * # Scope: inference endpoints only
+ *
+ * The counter is bumped by `/v1/responses` and `/v1/messages` ONLY.
+ * Non-inference traffic — `/v1/models`, `/v1/health`, CORS `OPTIONS`
+ * preflights, 404s for unknown routes — deliberately does NOT touch
+ * the sweeper. Counting those would let a `GET /v1/models` on every
+ * client startup, a CORS preflight from a browser, or a cron health
+ * probe keep the allocator pinned forever.
+ *
+ * # Drain is post-request only
+ *
+ * The only drain path is post-request: `endRequest()` decrements the
+ * in-flight counter and, on the 1 -> 0 transition, arms the timer. A
+ * cold-idle server that loaded models but never served a request
+ * never drains. That is deliberate:
+ *
+ * - Earlier iterations armed a cold-start drain from either
+ *   `createServer()` (regressed on slow model loads > `idleClearCacheMs`,
+ *   firing mid-load) or from `ModelRegistry.register()` (round-7 review
+ *   surfaced that it still raced sequential multi-model loads — the
+ *   timer armed by `register(A)` could fire while `await load(B)` was
+ *   still resolving). Loader-bracketing would require API changes on
+ *   the user-facing load path, so we remove the cold-start path
+ *   entirely instead.
+ * - Load-time allocator growth is bounded (a few GB of scratch), not
+ *   the tens-of-GB problem the post-request drain targets. macOS
+ *   handles truly-idle memory pressure via compression / swap.
+ * - Python `mlx-lm` has no cold-start drain either, so this matches
+ *   that baseline for servers that never receive a request.
+ *
+ * # Hot-load bracketing: `withSuspendedDrains`
+ *
+ * Hot-load flows — a `Model::load()` invoked AFTER the server has
+ * already served at least one request — race the post-request drain
+ * timer: the t+delayMs timer armed by the previous `endRequest()` can
+ * fire MID-LOAD while weight materialization is still allocating
+ * through the Metal free pool. The canonical fix is to bracket the
+ * load with `withSuspendedDrains(fn)`:
+ *
+ * ```ts
+ * await server.withSuspendedDrains(async () => {
+ *   const model = await Qwen35MoeModel.load(modelPath);
+ *   server.registry.register('qwen', model);
+ * });
+ * ```
+ *
+ * `withSuspendedDrains` handles try/finally bracketing itself, so a
+ * thrown load never leaks the internal suspend counter — a footgun the
+ * earlier raw `suspendDrains()` / `resumeDrains()` pair exposed
+ * (round-9 MEDIUM). The low-level `suspendDrains()` entry point
+ * remains available for callers that genuinely need manual control;
+ * its returned disposer is token-scoped and idempotent.
+ *
+ * `withSuspendedDrains` is also thenable-safe: non-Promise thenables
+ * (plain objects or even callables that expose a `.then` method) are
+ * normalized via `Promise.resolve()` so callers always get a real
+ * Promise<T> back and the suspend-release fires on settle. A throwing
+ * `.then` getter is treated as non-thenable (the getter access is
+ * guarded); the value is returned directly and the suspend releases
+ * synchronously — we deliberately do NOT propagate a getter throw, on
+ * the theory that leaking the suspend is a far worse failure mode than
+ * losing a pathological value (round-10 MEDIUM).
+ *
+ * # Tuning
+ *
+ * - The default (30_000 ms) balances "give models a chance to reuse
+ *   hot blocks across back-to-back requests" against "don't hold
+ *   tens of GB hostage while the process is truly idle".
+ * - `idleClearCacheMs: 0` disables the sweeper entirely — useful for
+ *   benchmarks or single-model workloads where the only memory
+ *   pressure is the decode-loop cadence already in place.
+ *   `withSuspendedDrains(fn)` on the disabled sweeper is a pass-through
+ *   so call sites can unconditionally bracket.
+ * - `MLX_IDLE_CLEAR_CACHE_MS` env var overrides the server's
+ *   constructor value; explicit constructor value wins over env.
+ *
+ * Drain cost is a single `mlx_synchronize` + `mlx_clear_cache` —
+ * constant time relative to generation length.
+ */
+// `clearCache` lives under the `__internal__` NAPI namespace — it is
+// deliberately NOT re-exported on the root `@mlx-node/core` object to
+// keep the process-wide, custom-stream drain out of the public
+// surface. User code that deep-imports from `@mlx-node/core` must
+// acknowledge the namespace (`core.__internal__.clearCache`) and
+// read the `@internal` caveat there. See
+// `crates/mlx-core/src/cache_limit.rs`.
+//
+// We import the module namespace (NOT a destructured `__internal__`)
+// so that a stale `.node` binary missing the `__internal__` namespace
+// does NOT hard-fail the import of `@mlx-node/server` itself. The
+// `clearCache` symbol is resolved LAZILY inside `createIdleSweeper`
+// (see round-5 Finding A): if the namespace / function is absent on
+// the loaded binding we warn once and fall back to a no-op drain,
+// keeping the server runnable on partial upgrades / downgrades. The
+// expected resolution path is `core.__internal__.clearCache` — the
+// sweeper never touches the root namespace so there is no ambiguity
+// with the deliberately-omitted root-level `clearCache` export.
+import * as core from '@mlx-node/core';
+/** Default idle window before draining the allocator's free pool (ms). */
+export const DEFAULT_IDLE_CLEAR_CACHE_MS = 30_000;
+/**
+ * Promise/A+ thenable probe that is robust against:
+ *
+ * - Function-typed thenables (awkward but legal — a callable that
+ *   also exposes a `.then` method). The earlier probe only checked
+ *   `typeof === 'object'`, so those values fell through to the
+ *   synchronous branch and released the suspend before the async
+ *   work had even started (round-10 MEDIUM #2).
+ * - A throwing `.then` getter. Accessing the property is guarded by
+ *   try/catch so a pathological value can't escape before the
+ *   caller's `release()` runs. We deliberately prefer "return false
+ *   on getter throw" over "propagate" here: the whole point of this
+ *   helper is to avoid leaking the suspend counter, and the caller
+ *   treats a non-thenable result as immediately-released.
+ */
+function isThenable(value) {
+    if (value == null)
+        return false;
+    const kind = typeof value;
+    if (kind !== 'object' && kind !== 'function')
+        return false;
+    try {
+        return typeof value.then === 'function';
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Parse `MLX_IDLE_CLEAR_CACHE_MS`. Same semantics as the other env
+ * knobs in `server.ts`: finite non-negative integer or fall through
+ * to the caller's default. A value of `0` explicitly disables the
+ * sweeper; negative / non-integer / unparseable values are ignored.
+ */
+export function parseIdleClearCacheEnv() {
+    const raw = process.env.MLX_IDLE_CLEAR_CACHE_MS;
+    if (raw == null || raw === '')
+        return undefined;
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed) || parsed < 0)
+        return undefined;
+    if (!Number.isInteger(parsed))
+        return undefined;
+    return parsed;
+}
+/** Set to `true` after the first missing-binding warn so we don't spam stderr. */
+let __warnedMissingClearCache = false;
+/**
+ * Resolve `__internal__.clearCache` from the loaded `@mlx-node/core`
+ * binding, guarded against stale / partial / downgraded `.node` files.
+ *
+ * Round-5 Finding A: the previous design dereferenced
+ * `__internal__.clearCache` at MODULE scope, so the import of
+ * `@mlx-node/server` itself threw `TypeError: Cannot read properties
+ * of undefined (reading 'clearCache')` if the loaded native binding
+ * lacked the namespace. That was unrecoverable from user code —
+ * there was no escape hatch, not even `idleClearCacheMs: 0`, because
+ * the throw fired before `createServer()` was reached.
+ *
+ * This resolver probes the namespace defensively at sweeper-creation
+ * time (and ONLY when the sweeper is actually enabled, so the
+ * `delayMs <= 0` opt-out short-circuits before we even look). Missing
+ * namespace / missing function / wrong-type function all route to a
+ * one-time `console.warn` + no-op fallback so the server stays up.
+ */
+function resolveClearCache() {
+    // Read through the module namespace — `core.__internal__` returns
+    // `undefined` when the loaded binding predates the namespace, WITHOUT
+    // throwing. Using an optional chain on the function keeps the probe
+    // safe even if `__internal__` exists but is some exotic shape.
+    const fn = core.__internal__?.clearCache;
+    if (typeof fn === 'function') {
+        return fn;
+    }
+    if (!__warnedMissingClearCache) {
+        __warnedMissingClearCache = true;
+        // Intentionally a `console.warn`, NOT a throw — the server must
+        // stay up on a stale native binding; the user's error surface is
+        // stderr, not an uncaught exception on a module-import side-effect.
+        console.warn('[@mlx-node/server] __internal__.clearCache not found on @mlx-node/core; idle cache drains disabled. Rebuild @mlx-node/core.');
+    }
+    return () => { };
+}
+/**
+ * Create an idle sweeper. Pass `0` or a non-positive value to opt out —
+ * the returned object becomes a no-op that still satisfies the
+ * interface. Callers can therefore unconditionally wire
+ * `beginRequest()` / `endRequest()` without branching on whether the
+ * sweeper is enabled.
+ *
+ * When `onDrain` is omitted, the sweeper resolves
+ * `__internal__.clearCache` on `@mlx-node/core` at creation time and
+ * caches the result in the returned closure. A missing namespace /
+ * function triggers a one-time `console.warn` and a no-op fallback —
+ * see `resolveClearCache()`. The `delayMs <= 0` path skips the
+ * resolution entirely so the opt-out remains purely passive.
+ */
+export function createIdleSweeper(delayMs, onDrain) {
+    if (!Number.isFinite(delayMs) || delayMs <= 0) {
+        function passthrough(fn) {
+            return fn();
+        }
+        return {
+            beginRequest() { },
+            endRequest() { },
+            close() { },
+            withSuspendedDrains: passthrough,
+            suspendDrains() {
+                return () => { };
+            },
+            get isPending() {
+                return false;
+            },
+            get inFlight() {
+                return 0;
+            },
+        };
+    }
+    // Resolve the drain callback exactly once per sweeper — either the
+    // caller-supplied `onDrain` (used by tests for observability) or the
+    // guarded `__internal__.clearCache` lookup. Caching in the closure
+    // means the namespace probe runs ONCE per sweeper, not once per
+    // timer firing.
+    const drainFn = onDrain ?? resolveClearCache();
+    let timer = null;
+    let inFlight = 0;
+    // `loadCounter` tracks active `suspendDrains()` brackets. Any value
+    // > 0 means some caller has declared "do not fire a drain right now
+    // — a long-running, unbracketed allocator-heavy operation is in
+    // progress". `scheduleDrain()` bails when it's positive, and the
+    // drain-fire callback re-checks it defensively before calling
+    // `drainFn()` in case a suspend arrived between arming and firing
+    // on the same event-loop tick.
+    let loadCounter = 0;
+    const cancelTimer = () => {
+        if (timer !== null) {
+            clearTimeout(timer);
+            timer = null;
+        }
+    };
+    const onTimerFire = () => {
+        timer = null;
+        // Double-check: if a request arrived between the timer being
+        // armed and the callback firing, `inFlight` is non-zero and
+        // `beginRequest()` should already have cancelled us. This guard
+        // is belt-and-suspenders — shouldn't trip in practice but
+        // protects against a timer racing a synchronous
+        // `beginRequest()` in adversarial tests.
+        if (inFlight !== 0)
+            return;
+        // Equivalent defensive re-check for the suspend path: a
+        // `suspendDrains()` that landed between arming and firing
+        // (possible if the arming `setTimeout` callback is queued
+        // alongside a synchronous suspend in the same tick) should
+        // reschedule rather than drain mid-load. The microtask ordering
+        // of `setTimeout` + `clearTimeout` makes this nearly impossible
+        // in practice on Node's single-threaded loop — but the check is
+        // cheap and future-proofs against timer-pool refactors.
+        if (loadCounter > 0) {
+            // Re-arm for another window. We intentionally do NOT call
+            // `drainFn()` here. If the suspend clears before the next
+            // `delayMs` elapses, the disposer returned by `suspendDrains()`
+            // will cancel this timer and start fresh anyway.
+            timer = setTimeout(onTimerFire, delayMs);
+            timer.unref();
+            return;
+        }
+        try {
+            drainFn();
+        }
+        catch {
+            // Swallow — drain is best-effort and must not crash the
+            // server loop. The underlying FFI call can't currently fail
+            // but we defend against future refactors growing fallibility.
+        }
+    };
+    const scheduleDrain = () => {
+        // Caller must already have confirmed `inFlight === 0`. We
+        // defensively null-out the existing timer first so overlapping
+        // begin→end→begin→end patterns can't leak a timer.
+        cancelTimer();
+        // Skip arming while a suspend bracket is active — the matching
+        // disposer that takes the counter back to zero will arm a fresh
+        // timer on the transition (provided `inFlight === 0`).
+        if (loadCounter > 0)
+            return;
+        timer = setTimeout(onTimerFire, delayMs);
+        // Do not keep the event loop alive on this timer alone. If the
+        // server would otherwise exit (e.g. the HTTP listener closed),
+        // we do not want to force a last-ditch drain.
+        timer.unref();
+    };
+    /**
+     * Suspend drains and return an idempotent, token-scoped disposer.
+     * Each call allocates a fresh token; the returned disposer checks
+     * the token on every invocation so re-calling the same disposer is
+     * a guaranteed no-op. Critically, a *stale* disposer — one that
+     * survived its matching release via `withSuspendedDrains` — also
+     * cannot re-decrement the counter and mysteriously shift the idle
+     * window (round-9 MEDIUM). The `Math.max(0, …)` clamp is retained
+     * as defense-in-depth for the impossible case where two different
+     * disposers both land after cross-thread reordering.
+     */
+    const suspendDrains = () => {
+        cancelTimer();
+        loadCounter += 1;
+        const token = { disposed: false };
+        return () => {
+            if (token.disposed)
+                return;
+            token.disposed = true;
+            loadCounter = Math.max(0, loadCounter - 1);
+            if (loadCounter === 0 && inFlight === 0) {
+                scheduleDrain();
+            }
+        };
+    };
+    function withSuspendedDrains(fn) {
+        const release = suspendDrains();
+        let released = false;
+        const releaseOnce = () => {
+            if (released)
+                return;
+            released = true;
+            release();
+        };
+        let result;
+        try {
+            result = fn();
+        }
+        catch (err) {
+            releaseOnce();
+            throw err;
+        }
+        if (isThenable(result)) {
+            // `Promise.resolve` adopts any thenable via the Promise
+            // resolution procedure, so a non-Promise thenable (plain
+            // object or callable) still hands the caller back a real
+            // `Promise<T>`. `.finally` runs on both fulfil and reject.
+            return Promise.resolve(result).finally(releaseOnce);
+        }
+        releaseOnce();
+        return result;
+    }
+    return {
+        beginRequest() {
+            inFlight += 1;
+            cancelTimer();
+        },
+        endRequest() {
+            if (inFlight === 0) {
+                // Defensive: an `endRequest()` without a matching
+                // `beginRequest()` is a caller bug. Clamp at zero rather
+                // than letting the counter go negative and latch the drain
+                // off forever.
+                return;
+            }
+            inFlight -= 1;
+            if (inFlight === 0) {
+                scheduleDrain();
+            }
+        },
+        close() {
+            cancelTimer();
+        },
+        withSuspendedDrains,
+        suspendDrains,
+        get isPending() {
+            return timer !== null;
+        },
+        get inFlight() {
+            return inFlight;
+        },
+    };
+}

package/dist/index.d.ts

@@ -11,13 +11,19 @@ export type { ServerConfig, ServerInstance } from './server.js';
  * supported public API — names may change without notice.
  */
 export { parseEnvSeconds as __parseEnvSeconds, parseEnvPositiveInt as __parseEnvPositiveInt } from './server.js';
+export { createIdleSweeper as __createIdleSweeper, parseIdleClearCacheEnv as __parseIdleClearCacheEnv, DEFAULT_IDLE_CLEAR_CACHE_MS as __DEFAULT_IDLE_CLEAR_CACHE_MS, } from './idle-sweeper.js';
+export type { IdleSweeper } from './idle-sweeper.js';
 export { createHandler } from './handler.js';
 export type { HandlerOptions } from './handler.js';
 export { ModelRegistry } from './registry.js';
-export type { ServableModel, ModelEntry, ModelRegistryOptions } from './registry.js';
+export type { ServableModel, ModelEntry, ModelRegistryOptions, RegisterOptions } from './registry.js';
 export { QueueFullError, SessionRegistry } from './session-registry.js';
 export type { SessionLookupResult, SessionRegistryOptions } from './session-registry.js';
+export { resolveServerTuningForUsage } from './timing.js';
+export { QWEN_SAMPLING_DEFAULTS, GEMMA4_SAMPLING_DEFAULTS, LFM2_SAMPLING_DEFAULTS, LAUNCH_PRESETS } from './presets.js';
+export type { LaunchPreset } from './presets.js';
+export type { PublicModelEntry } from './handler.js';
 export type { ResponsesAPIRequest, ResponseObject, ResponseUsage, ResponseError, InputItem, InputMessage, InputFunctionCall, InputFunctionCallOutput, OutputItem, MessageOutputItem, ReasoningOutputItem, FunctionCallOutputItem, OutputTextPart, SummaryTextPart, ResponsesToolDefinition, ContentPart, InputTextPart, StreamEvent, } from './types.js';
-export type { AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicMessage, AnthropicContentBlock, AnthropicTextContentBlock, AnthropicImageContentBlock, AnthropicToolResultContentBlock, AnthropicToolUseContentBlock, AnthropicThinkingContentBlock, AnthropicToolDefinition, AnthropicToolChoice, AnthropicResponseContent, AnthropicResponseTextBlock, AnthropicResponseThinkingBlock, AnthropicResponseToolUseBlock, AnthropicUsage, AnthropicStreamEvent, AnthropicMessageStartEvent, AnthropicContentBlockStartEvent, AnthropicContentBlockDeltaEvent, AnthropicContentBlockStopEvent, AnthropicMessageDeltaEvent, AnthropicMessageStopEvent, AnthropicDelta, AnthropicTextDelta, AnthropicThinkingDelta, AnthropicInputJsonDelta, SystemBlock, } from './types-anthropic.js';
+export type { AnthropicCountTokensRequest, AnthropicCountTokensResponse, AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicMessage, AnthropicContentBlock, AnthropicTextContentBlock, AnthropicImageContentBlock, AnthropicToolResultContentBlock, AnthropicToolUseContentBlock, AnthropicThinkingContentBlock, AnthropicToolDefinition, AnthropicToolChoice, AnthropicResponseContent, AnthropicResponseTextBlock, AnthropicResponseThinkingBlock, AnthropicResponseToolUseBlock, AnthropicUsage, AnthropicStreamEvent, AnthropicMessageStartEvent, AnthropicContentBlockStartEvent, AnthropicContentBlockDeltaEvent, AnthropicContentBlockStopEvent, AnthropicMessageDeltaEvent, AnthropicMessageStopEvent, AnthropicDelta, AnthropicTextDelta, AnthropicThinkingDelta, AnthropicInputJsonDelta, SystemBlock, } from './types-anthropic.js';
 export { writeSSEEvent, beginSSE, endSSE } from './streaming.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE;;;GAGG;AACH,OAAO,EAAE,eAAe,IAAI,iBAAiB,EAAE,mBAAmB,IAAI,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEjH,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAErF,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxE,YAAY,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAEzF,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,UAAU,EACV,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,uBAAuB,EACvB,WAAW,EACX,aAAa,EACb,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,EACrB,yBAAyB,EACzB,0BAA0B,EAC1B,+BAA+B,EAC/B,4BAA4B,EAC5B,6BAA6B,EAC7B,uBAAuB,EACvB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,8BAA8B,EAC9B,6BAA6B,EAC7B,cAAc,EACd,oBAAoB,EACpB,0BAA0B,EAC1B,+BAA+B,EAC/B,+BAA+B,EAC/B,8BAA8B,EAC9B,0BAA0B,EAC1B,yBAAyB,EACzB,cAAc,EACd,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE;;;GAGG;AACH,OAAO,EAAE,eAAe,IAAI,iBAAiB,EAAE,mBAAmB,IAAI,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACjH,OAAO,EACL,iBAAiB,IAAI,mBAAmB,EACxC,sBAAsB,IAAI,wBAAwB,EAClD,2BAA2B,IAAI,6BAA6B,GAC7D,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEtG,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxE,YAAY,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACzF,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAS1D,OAAO,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACxH,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAErD,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,UAAU,EACV,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,uBAAuB,EACvB,WAAW,EACX,aAAa,EACb,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,2BAA2B,EAC3B,4BAA4B,EAC5B,wBAAwB,EACxB,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,EACrB,yBAAyB,EACzB,0BAA0B,EAC1B,+BAA+B,EAC/B,4BAA4B,EAC5B,6BAA6B,EAC7B,uBAAuB,EACvB,mBAAmB,EACnB,wBAAwB,EACxB,0BAA0B,EAC1B,8BAA8B,EAC9B,6BAA6B,EAC7B,cAAc,EACd,oBAAoB,EACpB,0BAA0B,EAC1B,+BAA+B,EAC/B,+BAA+B,EAC/B,8BAA8B,EAC9B,0BAA0B,EAC1B,yBAAyB,EACzB,cAAc,EACd,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/index.js

@@ -10,7 +10,17 @@ export { createServer } from './server.js';
  * supported public API — names may change without notice.
  */
 export { parseEnvSeconds as __parseEnvSeconds, parseEnvPositiveInt as __parseEnvPositiveInt } from './server.js';
+export { createIdleSweeper as __createIdleSweeper, parseIdleClearCacheEnv as __parseIdleClearCacheEnv, DEFAULT_IDLE_CLEAR_CACHE_MS as __DEFAULT_IDLE_CLEAR_CACHE_MS, } from './idle-sweeper.js';
 export { createHandler } from './handler.js';
 export { ModelRegistry } from './registry.js';
 export { QueueFullError, SessionRegistry } from './session-registry.js';
+export { resolveServerTuningForUsage } from './timing.js';
+// NOTE: `__resetPromptCacheKeyNonceForTests` is intentionally NOT
+// re-exported here. It is a test-only helper that nukes the module-
+// scoped HMAC nonce (and the once-per-process single-tenant warning
+// flag); exposing it on the public surface would let downstream
+// consumers invalidate every live tier-2 entry with one call. Tests
+// import it from the deep path
+// `packages/server/src/session-registry.js` instead.
+export { QWEN_SAMPLING_DEFAULTS, GEMMA4_SAMPLING_DEFAULTS, LFM2_SAMPLING_DEFAULTS, LAUNCH_PRESETS } from './presets.js';
 export { writeSSEEvent, beginSSE, endSSE } from './streaming.js';

package/dist/mappers/anthropic-request.d.ts

@@ -1,9 +1,31 @@
 /** Anthropic Messages API request → internal `ChatMessage[]` + `ChatConfig`. */
 import type { ChatConfig, ChatMessage } from '@mlx-node/core';
-import type { AnthropicMessagesRequest } from '../types-anthropic.js';
+import type { AnthropicCountTokensRequest, AnthropicMessagesRequest } from '../types-anthropic.js';
 export interface MappedAnthropicRequest {
     messages: ChatMessage[];
     config: ChatConfig;
+    /**
+     * Client-supplied stop strings (Anthropic `stop_sequences`), normalized to
+     * drop absent/empty entries. Carried alongside `config` rather than on it
+     * because `ChatConfig` has no native stop field; a downstream consumer is
+     * responsible for honouring these.
+     */
+    stopSequences: string[];
 }
-export declare function mapAnthropicRequest(req: AnthropicMessagesRequest): MappedAnthropicRequest;
+/**
+ * Canonicalize the Anthropic `system` field into the same string the mapper
+ * bakes into the leading `system` ChatMessage. Used by both
+ * `mapAnthropicRequest` (so the model never sees the billing header) and the
+ * `/v1/messages` warm-slot gate cache-key derivation (so the gate matches
+ * across rotating billing tokens). The two views MUST stay in sync — a
+ * single source of truth prevents drift.
+ *
+ * Asymmetry vs. `mapAnthropicRequest`: the mapper THROWS on non-text blocks
+ * (it's a request-validation gate), but this helper silently skips them.
+ * Safe because `mapAnthropicRequest` runs first as a pre-flight gate, so by
+ * the time the cache-key is computed, the request shape has already been
+ * validated.
+ */
+export declare function canonicalizeSystemForCacheKey(system: AnthropicMessagesRequest['system']): string | null;
+export declare function mapAnthropicRequest(req: AnthropicMessagesRequest | AnthropicCountTokensRequest): MappedAnthropicRequest;
 //# sourceMappingURL=anthropic-request.d.ts.map

package/dist/mappers/anthropic-request.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"anthropic-request.d.ts","sourceRoot":"","sources":["../../src/mappers/anthropic-request.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAkB,MAAM,gBAAgB,CAAC;AAE9E,OAAO,KAAK,EAGV,wBAAwB,EAIzB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,UAAU,CAAC;CACpB;AAiDD,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,wBAAwB,GAAG,sBAAsB,CAyMzF"}
+{"version":3,"file":"anthropic-request.d.ts","sourceRoot":"","sources":["../../src/mappers/anthropic-request.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAkB,MAAM,gBAAgB,CAAC;AAE9E,OAAO,KAAK,EAEV,2BAA2B,EAE3B,wBAAwB,EAIzB,MAAM,uBAAuB,CAAC;AAI/B,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,UAAU,CAAC;IACnB;;;;;OAKG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAmBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,wBAAwB,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,IAAI,CAoBvG;AAiDD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,wBAAwB,GAAG,2BAA2B,GAC1D,sBAAsB,CA8VxB"}