@mlx-node/server 0.0.0 → 0.0.8

package/dist/mappers/request.d.ts

@@ -5,11 +5,59 @@ export interface MappedRequest {
     messages: ChatMessage[];
     config: ChatConfig;
 }
+/**
+ * Shared MTP-extension parser for the `extra_body` carrier on both
+ * `/v1/responses` (OpenAI) and `/v1/messages` (Anthropic) request
+ * shapes. Mutates the passed `config` in place:
+ *
+ *   * `generation_mode: "mtp"` → `enableMtp = true`
+ *   * `generation_mode: "ar"`  → `enableMtp = false`
+ *   * any other / absent value → `enableMtp` untouched, so the
+ *     downstream `ChatSession.mergeConfig` auto-default (true when
+ *     the model ships an MTP head) applies.
+ *
+ *   * `mtp_depth: <positive int ≤ 64>` → `mtpDepth = value`
+ *   * non-integer, out-of-range, or absent → `mtpDepth` untouched.
+ *     The real clamps are per-family and owned by native
+ *     `resolve_params`: qwen3.5 native MTP clamps to [1, 5], gemma4
+ *     DSpark caps at the draft block size (7 on v1), and the gemma4
+ *     assistant draft clamps to [1, 8]. The server therefore only
+ *     rejects garbage — non-integers, non-positives, and values
+ *     > 64 (a generous sanity ceiling far above any family's real
+ *     clamp) — which saves a round-trip into the model thread.
+ *
+ * Kept as a pure helper rather than inlined into each mapper so the
+ * two endpoints can't drift in semantics.
+ */
+export declare function applyExtraBodyMtpOverrides(config: ChatConfig, extraBody: {
+    generation_mode?: string | null;
+    mtp_depth?: number | null;
+} | undefined): void;
 export declare function mapRequest(req: ResponsesAPIRequest, priorMessages?: ChatMessage[]): MappedRequest;
+/**
+ * Serialise a `ChatMessage[]` snapshot for `StoredResponseRecord.inputJson`,
+ * preserving any `Uint8Array` image payloads as base64-encoded sentinels
+ * so a later `reconstructMessagesFromChain` can revive them into real
+ * `Uint8Array`s for the NAPI chat-session boundary.
+ *
+ * The replacer runs AFTER `toJSON`, so a `Buffer` (which defines
+ * `Buffer.prototype.toJSON` returning `{type:"Buffer",data:[...]}`)
+ * would otherwise slip past the `instanceof Uint8Array` check. We
+ * match both shapes defensively — the production `resolveMessageContent`
+ * path now wraps with `new Uint8Array(...)` at decode time, but any
+ * future caller that sneaks a `Buffer` through still round-trips
+ * instead of silently corrupting image state.
+ */
+export declare function stringifyStoredInputMessages(messages: ChatMessage[]): string;
 /**
  * Reconstruct `ChatMessage[]` from a stored response chain. Each record
  * stores `inputJson` (messages sent) and `outputJson` (output items); we
  * interleave them.
+ *
+ * Image payloads encoded as `{__u8__: "<base64>"}` by
+ * `stringifyStoredInputMessages` are rehydrated back into `Buffer`
+ * (a `Uint8Array` subclass) so replayed user turns carry the same
+ * runtime shape the native chat-session APIs expect.
  */
 export declare function reconstructMessagesFromChain(chain: {
     inputJson: string;

package/dist/mappers/request.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../src/mappers/request.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAE9E,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAkB,MAAM,gBAAgB,CAAC;AAE9E,OAAO,KAAK,EAAe,mBAAmB,EAA2B,MAAM,aAAa,CAAC;AAqC7F,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,UAAU,CAAC;CACpB;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,mBAAmB,EAAE,aAAa,CAAC,EAAE,WAAW,EAAE,GAAG,aAAa,CA4GjG;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,WAAW,EAAE,CAoE9G"}
+{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../src/mappers/request.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAE9E,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAkB,MAAM,gBAAgB,CAAC;AAE9E,OAAO,KAAK,EAAe,mBAAmB,EAA2B,MAAM,aAAa,CAAC;AA2G7F,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,UAAU,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,SAAS,GACpF,IAAI,CAcN;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,mBAAmB,EAAE,aAAa,CAAC,EAAE,WAAW,EAAE,GAAG,aAAa,CAmIjG;AAwBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAgB5E;AAED;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,WAAW,EAAE,CAyE9G"}

package/dist/mappers/request.js

@@ -1,17 +1,78 @@
 /** OpenAI Responses API request → internal `ChatMessage[]` + `ChatConfig`. */
-function resolveContent(content) {
+/**
+ * Resolve a message's `content` array into text + optional image bytes.
+ *
+ * Accepts both input-side (`input_text`, `input_image`) and replay-side
+ * (`output_text`, `refusal`, `summary_text`) content parts. Clients that echo
+ * prior assistant turns in `input[]` instead of using `previous_response_id`
+ * (pi-ai, Codex) send `output_text` on assistant messages — rejecting those
+ * would break cold-start replay. `input_image` with a base64 `data:` URL is
+ * decoded to bytes; `http(s)://` URLs are not fetched (the mapper stays sync).
+ */
+function resolveMessageContent(content, role) {
     if (typeof content === 'string')
-        return content;
+        return { text: content };
     const parts = [];
+    const images = [];
+    // The internal `ChatMessage` shape is `{ content: string, images: Uint8Array[] }`
+    // and the downstream Jinja serializer always emits `[{type:"text",...},
+    // {type:"image"}*N]` — it cannot represent a text part that appears AFTER
+    // an image part in the caller's content array. Detect and reject that
+    // shape rather than silently reordering it and changing user intent.
+    // Mirrors the existing rejection in `anthropic-request.ts` for the
+    // tool_result + trailing-mixed case.
+    let seenImage = false;
     for (const p of content) {
-        if (p.type === 'input_text') {
+        if (p.type === 'input_text' || p.type === 'output_text' || p.type === 'summary_text') {
+            if (seenImage) {
+                throw new Error('Unsupported: text content part after an image part in the same message is not representable ' +
+                    'in the internal message model. The flat ChatMessage shape and the Jinja serializer both place ' +
+                    'all text before all images in a user turn, so any mapping would silently reorder your content. ' +
+                    'Place all text parts before any image parts, or split across separate user turns.');
+            }
             parts.push(p.text);
         }
+        else if (p.type === 'refusal') {
+            if (seenImage) {
+                throw new Error('Unsupported: refusal content part after an image part in the same message is not representable ' +
+                    'in the internal message model; the flat ChatMessage shape would silently reorder it.');
+            }
+            parts.push(p.refusal);
+        }
+        else if (p.type === 'input_image') {
+            if (role !== 'user') {
+                throw new Error(`input_image content parts are only allowed on user messages (got role="${role}")`);
+            }
+            if (p.file_id) {
+                throw new Error('input_image.file_id is not supported — inline the image as a data URL');
+            }
+            if (!p.image_url) {
+                throw new Error('input_image is missing image_url');
+            }
+            const match = /^data:[^;,]+;base64,(.+)$/s.exec(p.image_url);
+            if (!match) {
+                throw new Error('input_image.image_url must be a base64 data URL (data:<mime>;base64,<payload>); ' +
+                    'remote http(s) URLs are not fetched by this server');
+            }
+            // Wrap as a plain `Uint8Array` rather than storing the raw
+            // `Buffer`. `Buffer` is a `Uint8Array` subclass, but it defines
+            // its own `toJSON()` that `JSON.stringify` calls BEFORE any
+            // replacer runs — so a Buffer-backed value would serialise as
+            // `{type:"Buffer",data:[...]}` and skip the `__u8__` sentinel in
+            // `stringifyStoredInputMessages`, corrupting image round-trip
+            // through `previous_response_id` chains. A plain `Uint8Array`
+            // has no `toJSON`, so the replacer fires as intended.
+            images.push(new Uint8Array(Buffer.from(match[1], 'base64')));
+            seenImage = true;
+        }
         else {
             throw new Error(`Unsupported content part type: "${p.type}"`);
         }
     }
-    return parts.join('');
+    const out = { text: parts.join('') };
+    if (images.length > 0)
+        out.images = images;
+    return out;
 }
 /** NAPI `ToolDefinition` requires `parameters.properties` to be a JSON string. */
 function mapTool(tool) {
@@ -34,6 +95,46 @@ function mapTool(tool) {
         },
     };
 }
+/**
+ * Shared MTP-extension parser for the `extra_body` carrier on both
+ * `/v1/responses` (OpenAI) and `/v1/messages` (Anthropic) request
+ * shapes. Mutates the passed `config` in place:
+ *
+ *   * `generation_mode: "mtp"` → `enableMtp = true`
+ *   * `generation_mode: "ar"`  → `enableMtp = false`
+ *   * any other / absent value → `enableMtp` untouched, so the
+ *     downstream `ChatSession.mergeConfig` auto-default (true when
+ *     the model ships an MTP head) applies.
+ *
+ *   * `mtp_depth: <positive int ≤ 64>` → `mtpDepth = value`
+ *   * non-integer, out-of-range, or absent → `mtpDepth` untouched.
+ *     The real clamps are per-family and owned by native
+ *     `resolve_params`: qwen3.5 native MTP clamps to [1, 5], gemma4
+ *     DSpark caps at the draft block size (7 on v1), and the gemma4
+ *     assistant draft clamps to [1, 8]. The server therefore only
+ *     rejects garbage — non-integers, non-positives, and values
+ *     > 64 (a generous sanity ceiling far above any family's real
+ *     clamp) — which saves a round-trip into the model thread.
+ *
+ * Kept as a pure helper rather than inlined into each mapper so the
+ * two endpoints can't drift in semantics.
+ */
+export function applyExtraBodyMtpOverrides(config, extraBody) {
+    if (!extraBody)
+        return;
+    const mode = extraBody.generation_mode;
+    if (mode === 'mtp') {
+        config.enableMtp = true;
+    }
+    else if (mode === 'ar') {
+        config.enableMtp = false;
+    }
+    // Any other value (null, undefined, unknown string) → leave alone.
+    const depth = extraBody.mtp_depth;
+    if (depth != null && Number.isInteger(depth) && depth > 0 && depth <= 64) {
+        config.mtpDepth = depth;
+    }
+}
 export function mapRequest(req, priorMessages) {
     const messages = [];
     if (req.instructions) {
@@ -42,18 +143,33 @@ export function mapRequest(req, priorMessages) {
     if (priorMessages) {
         messages.push(...priorMessages);
     }
-    // Coalesce a `message + function_call+` run (or a pure `function_call+` run)
-    // into ONE assistant `ChatMessage` carrying both `content` and `toolCalls`.
-    // `ChatSession.sendStream()` appends exactly one assistant message per turn,
-    // and `validateAndCanonicalizeHistoryToolOrder` requires each fan-out's
-    // `toolCalls` to pair 1:1 with the trailing tool block — splitting would
-    // reshape the conversation and make the walker reject the turn as orphaned.
-    // A `message` item immediately after a `function_call` starts a new turn.
+    // An assistant turn may serialise into any interleaving of `reasoning`,
+    // `message` (assistant), and `function_call` items. We coalesce that run
+    // into ONE assistant `ChatMessage` carrying `content` + `reasoningContent`
+    // + `toolCalls`, matching the hot-path `ChatSession` shape exactly. Any
+    // non-assistant item (user / system / function_call_output) flushes the
+    // current turn. An assistant `message` item that appears AFTER a
+    // `function_call` opens a fresh turn — preserving the pre-existing
+    // convention documented in the fan-out tests.
     if (typeof req.input === 'string') {
         messages.push({ role: 'user', content: req.input });
     }
     else {
-        let prevItemType = null;
+        let currentAssistant = null;
+        let assistantHasToolCalls = false;
+        const flushAssistant = () => {
+            if (currentAssistant) {
+                messages.push(currentAssistant);
+                currentAssistant = null;
+                assistantHasToolCalls = false;
+            }
+        };
+        const ensureAssistant = () => {
+            if (!currentAssistant) {
+                currentAssistant = { role: 'assistant', content: '' };
+            }
+            return currentAssistant;
+        };
         for (const item of req.input) {
             if (item == null || typeof item !== 'object') {
                 throw new Error('Each input item must be a non-null object');
@@ -66,34 +182,40 @@ export function mapRequest(req, priorMessages) {
                 if (role !== 'user' && role !== 'assistant' && role !== 'system') {
                     throw new Error(`Unsupported message role: "${msg.role}"`);
                 }
-                messages.push({
-                    role,
-                    content: resolveContent(msg.content),
-                });
-            }
-            else if (itemType === 'function_call') {
-                // Coalesce onto the preceding assistant turn — see the loop header.
-                const fc = item;
-                const last = messages[messages.length - 1];
-                const canCoalesce = (prevItemType === 'function_call' || prevItemType === 'message') &&
-                    last !== undefined &&
-                    last.role === 'assistant';
-                if (canCoalesce) {
-                    if (last.toolCalls === undefined) {
-                        last.toolCalls = [];
+                if (role === 'assistant') {
+                    // `message` after a `function_call` opens a new turn.
+                    if (assistantHasToolCalls) {
+                        flushAssistant();
                     }
-                    last.toolCalls.push({ name: fc.name, arguments: fc.arguments, id: fc.call_id });
+                    const a = ensureAssistant();
+                    const { text } = resolveMessageContent(msg.content, 'assistant');
+                    a.content = (a.content ?? '') + text;
                 }
                 else {
-                    messages.push({
-                        role: 'assistant',
-                        content: '',
-                        toolCalls: [{ name: fc.name, arguments: fc.arguments, id: fc.call_id }],
-                    });
+                    flushAssistant();
+                    const { text, images } = resolveMessageContent(msg.content, role);
+                    const m = { role, content: text };
+                    if (images)
+                        m.images = images;
+                    messages.push(m);
                 }
             }
+            else if (itemType === 'reasoning') {
+                const r = item;
+                const summary = (r.summary ?? []).map((s) => s.text ?? '').join('');
+                const a = ensureAssistant();
+                a.reasoningContent = (a.reasoningContent ?? '') + summary;
+            }
+            else if (itemType === 'function_call') {
+                const fc = item;
+                const a = ensureAssistant();
+                a.toolCalls ??= [];
+                a.toolCalls.push({ name: fc.name, arguments: fc.arguments, id: fc.call_id });
+                assistantHasToolCalls = true;
+            }
             else if (itemType === 'function_call_output') {
                 const fco = item;
+                flushAssistant();
                 messages.push({
                     role: 'tool',
                     content: fco.output,
@@ -103,8 +225,8 @@ export function mapRequest(req, priorMessages) {
             else {
                 throw new Error(`Unsupported input item type: "${itemType}"`);
             }
-            prevItemType = itemType;
         }
+        flushAssistant();
     }
     const config = {
         reportPerformance: true,
@@ -139,17 +261,71 @@ export function mapRequest(req, priorMessages) {
     if (priorMessages && priorMessages.length > 0) {
         config.reuseCache = true;
     }
+    applyExtraBodyMtpOverrides(config, req.extra_body);
     return { messages, config };
 }
+/**
+ * Sentinel key used to tag base64-encoded `Uint8Array` payloads in
+ * persisted `inputJson`. Plain `JSON.stringify` turns a `Uint8Array`
+ * into a numeric-keyed object (e.g. `{"0":1,"1":2,...}`), which
+ * (a) bloats the row ~8× vs base64 and (b) does not round-trip — the
+ * parsed object fails the NAPI `Uint8Array` type check on cold replay,
+ * breaking `previous_response_id` continuations that carry images.
+ */
+const UINT8_SENTINEL = '__u8__';
+function isEncodedUint8Array(value) {
+    return (value !== null &&
+        typeof value === 'object' &&
+        typeof value[UINT8_SENTINEL] === 'string');
+}
+/**
+ * Serialise a `ChatMessage[]` snapshot for `StoredResponseRecord.inputJson`,
+ * preserving any `Uint8Array` image payloads as base64-encoded sentinels
+ * so a later `reconstructMessagesFromChain` can revive them into real
+ * `Uint8Array`s for the NAPI chat-session boundary.
+ *
+ * The replacer runs AFTER `toJSON`, so a `Buffer` (which defines
+ * `Buffer.prototype.toJSON` returning `{type:"Buffer",data:[...]}`)
+ * would otherwise slip past the `instanceof Uint8Array` check. We
+ * match both shapes defensively — the production `resolveMessageContent`
+ * path now wraps with `new Uint8Array(...)` at decode time, but any
+ * future caller that sneaks a `Buffer` through still round-trips
+ * instead of silently corrupting image state.
+ */
+export function stringifyStoredInputMessages(messages) {
+    return JSON.stringify(messages, (_key, value) => {
+        if (value instanceof Uint8Array) {
+            return { [UINT8_SENTINEL]: Buffer.from(value).toString('base64') };
+        }
+        if (value !== null &&
+            typeof value === 'object' &&
+            value.type === 'Buffer' &&
+            Array.isArray(value.data)) {
+            const data = value.data;
+            return { [UINT8_SENTINEL]: Buffer.from(data).toString('base64') };
+        }
+        return value;
+    });
+}
 /**
  * Reconstruct `ChatMessage[]` from a stored response chain. Each record
  * stores `inputJson` (messages sent) and `outputJson` (output items); we
  * interleave them.
+ *
+ * Image payloads encoded as `{__u8__: "<base64>"}` by
+ * `stringifyStoredInputMessages` are rehydrated back into `Buffer`
+ * (a `Uint8Array` subclass) so replayed user turns carry the same
+ * runtime shape the native chat-session APIs expect.
  */
 export function reconstructMessagesFromChain(chain) {
     const messages = [];
     for (const record of chain) {
-        const inputMessages = JSON.parse(record.inputJson);
+        const inputMessages = JSON.parse(record.inputJson, (_key, value) => {
+            if (isEncodedUint8Array(value)) {
+                return Buffer.from(value[UINT8_SENTINEL], 'base64');
+            }
+            return value;
+        });
         messages.push(...inputMessages);
         const outputItems = JSON.parse(record.outputJson);
         let assistantText = '';

package/dist/mappers/response.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"response.d.ts","sourceRoot":"","sources":["../../src/mappers/response.ts"],"names":[],"mappings":"AAAA,kEAAkE;AAIlE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAGV,UAAU,EAEV,cAAc,EACd,mBAAmB,EACnB,aAAa,EACd,MAAM,aAAa,CAAC;AAErB,wBAAgB,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE5C;AAED,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,WAAW,GAAG,YAAY,CAOxF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,EAAE,CAwCjE;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,aAAa,CAO5D;AAED,8DAA8D;AAC9D,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAU7D;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,mBAAmB,EACxB,UAAU,EAAE,MAAM,EAClB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,cAAc,CAwBhB;AAED,qHAAqH;AACrH,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,mBAAmB,EACxB,UAAU,EAAE,MAAM,EAClB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,cAAc,CAqBhB"}
+{"version":3,"file":"response.d.ts","sourceRoot":"","sources":["../../src/mappers/response.ts"],"names":[],"mappings":"AAAA,kEAAkE;AAIlE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAGV,UAAU,EAEV,cAAc,EACd,mBAAmB,EACnB,aAAa,EACd,MAAM,aAAa,CAAC;AAErB,wBAAgB,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE5C;AAED,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,WAAW,GAAG,YAAY,CAOxF;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,EAAE,CAwCjE;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,aAAa,CAkB5D;AAED,8DAA8D;AAC9D,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAU7D;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,mBAAmB,EACxB,UAAU,EAAE,MAAM,EAClB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,cAAc,CAwBhB;AAED,qHAAqH;AACrH,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,mBAAmB,EACxB,UAAU,EAAE,MAAM,EAClB,kBAAkB,CAAC,EAAE,MAAM,GAC1B,cAAc,CAqBhB"}

package/dist/mappers/response.js

@@ -1,5 +1,6 @@
 /** ChatResult / ChatStreamEvent → OpenAI Responses API output. */
 import { randomUUID } from 'node:crypto';
+import { mergeTimingUsageExtensions } from '../timing.js';
 export function genId(prefix) {
     return `${prefix}${randomUUID().replaceAll('-', '')}`;
 }
@@ -48,12 +49,23 @@ export function buildOutputItems(result) {
     return items;
 }
 export function buildUsage(result) {
-    return {
+    const usage = {
         input_tokens: result.promptTokens,
         output_tokens: result.numTokens,
         output_tokens_details: { reasoning_tokens: result.reasoningTokens },
         total_tokens: result.promptTokens + result.numTokens,
     };
+    // Surface reused KV-cache prefix tokens via the upstream
+    // `input_tokens_details.cached_tokens` field when the native
+    // dispatch reports a non-zero reuse count. Omitted when zero so
+    // consumers can cheaply `usage.input_tokens_details?.cached_tokens`
+    // without also distinguishing a meaningful zero from "feature not
+    // active on this turn".
+    if (result.cachedTokens > 0) {
+        usage.input_tokens_details = { cached_tokens: result.cachedTokens };
+    }
+    mergeTimingUsageExtensions(usage, result.performance, result.promptTokens, result.numTokens, result.cachedTokens);
+    return usage;
 }
 /** Concatenate all `output_text` parts from message items. */
 export function computeOutputText(items) {

package/dist/model-work-coordinator.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Result of a `withModelLoad` call that surfaces who actually drove the load
+ * vs. who merely parked behind one that was already in flight. Callers use
+ * this to split observability between the request that triggered a cold
+ * weight-materialize from one that arrived a millisecond later and merely
+ * inherited the wait — without the split a 60-second cold-load shows up
+ * on every concurrent request as if each one paid for a separate load.
+ *
+ * `owner` reflects the SYNCHRONOUS state observed at lock acquisition:
+ * `true` if the writer lock was free when this caller arrived and the
+ * caller itself executed the supplied `fn`; `false` if there was already
+ * a writer active (or queued ahead of this caller) when it arrived.
+ *
+ * `waitMs` and `ownMs` partition the wall-clock interval between when
+ * the caller arrived at the coordinator and when its `fn` resolved:
+ *  - `waitMs` is time spent blocked inside `acquireWrite()` (zero for a
+ *    no-contention owner; ≈ peer's load duration for a follower).
+ *  - `ownMs` is time spent inside `fn` once the writer lock was held
+ *    (≈ load duration for an owner driving a cold load; near-zero for a
+ *    follower whose `fn` is a no-op cache lookup).
+ * Both are measured from `Date.now()` and clamped at zero to absorb
+ * monotonic-skew. Their sum equals the total elapsed time in the call,
+ * so handlers can plumb them into separate observability fields
+ * (`server_load_wait_ms` vs. `server_model_resolve_ms`) without
+ * double-counting.
+ */
+export interface ModelLoadOutcome<T> {
+    result: T;
+    owner: boolean;
+    waitMs: number;
+    ownMs: number;
+}
+/**
+ * Process-local gate for native MLX work.
+ *
+ * Individual model instances already have a per-model execution mutex, but a
+ * lazy `loadModel()` can still run load-time materialization / warmup Metal
+ * work while another model is decoding. MLX's allocator and command queues are
+ * process-wide, so model load/swap takes an exclusive writer slot; inference
+ * takes shared reader slots.
+ */
+export declare class ModelWorkCoordinator {
+    private activeReaders;
+    private writerActive;
+    private waitingWriters;
+    private readonly readerWaiters;
+    private readonly writerWaiters;
+    withModelLoad<T>(fn: () => Promise<T> | T): Promise<T>;
+    /**
+     * Like {@link withModelLoad} but reports whether THIS caller owned the
+     * load (acquired the writer lock with no contention) or merely waited
+     * behind a load that was already in flight when it arrived.
+     *
+     * Decided at sync-time before any await: if neither a writer is active
+     * nor any writer is queued ahead, this caller is the owner; otherwise
+     * it is parked behind someone else's load and `owner` is `false`. The
+     * distinction is used by `/v1/messages` to split `resolve_ms` (own
+     * load + lookup) from `load_wait_ms` (waiting on a peer's load) so a
+     * 60-second cold-load does not look like 60 seconds of own work for
+     * every concurrent request.
+     */
+    withModelLoadInstrumented<T>(fn: () => Promise<T> | T): Promise<ModelLoadOutcome<T>>;
+    withInference<T>(fn: () => Promise<T> | T): Promise<T>;
+    private acquireRead;
+    private acquireWrite;
+    private releaseRead;
+    private releaseWrite;
+    private drain;
+}
+//# sourceMappingURL=model-work-coordinator.d.ts.map

package/dist/model-work-coordinator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-work-coordinator.d.ts","sourceRoot":"","sources":["../src/model-work-coordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC;IACjC,MAAM,EAAE,CAAC,CAAC;IACV,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IACvD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IAEjD,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAO3D;IAED;;;;;;;;;;;;OAYG;IACG,yBAAyB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAoBzF;IAEK,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAO3D;IAED,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,YAAY;IAgBpB,OAAO,CAAC,WAAW;IAMnB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,KAAK;CAWd"}

package/dist/model-work-coordinator.js

@@ -0,0 +1,120 @@
+/**
+ * Process-local gate for native MLX work.
+ *
+ * Individual model instances already have a per-model execution mutex, but a
+ * lazy `loadModel()` can still run load-time materialization / warmup Metal
+ * work while another model is decoding. MLX's allocator and command queues are
+ * process-wide, so model load/swap takes an exclusive writer slot; inference
+ * takes shared reader slots.
+ */
+export class ModelWorkCoordinator {
+    activeReaders = 0;
+    writerActive = false;
+    waitingWriters = 0;
+    readerWaiters = [];
+    writerWaiters = [];
+    async withModelLoad(fn) {
+        await this.acquireWrite();
+        try {
+            return await fn();
+        }
+        finally {
+            this.releaseWrite();
+        }
+    }
+    /**
+     * Like {@link withModelLoad} but reports whether THIS caller owned the
+     * load (acquired the writer lock with no contention) or merely waited
+     * behind a load that was already in flight when it arrived.
+     *
+     * Decided at sync-time before any await: if neither a writer is active
+     * nor any writer is queued ahead, this caller is the owner; otherwise
+     * it is parked behind someone else's load and `owner` is `false`. The
+     * distinction is used by `/v1/messages` to split `resolve_ms` (own
+     * load + lookup) from `load_wait_ms` (waiting on a peer's load) so a
+     * 60-second cold-load does not look like 60 seconds of own work for
+     * every concurrent request.
+     */
+    async withModelLoadInstrumented(fn) {
+        // `owner` MUST be decided synchronously, before any await, so the
+        // signal reflects coordinator state at arrival rather than after
+        // any peer transition. The wait/own split is measured around the
+        // actual phase boundaries (lock acquisition, fn completion) so the
+        // two intervals partition cleanly instead of both reporting total
+        // elapsed time — see `ModelLoadOutcome` for the contract.
+        const owner = !this.writerActive && this.waitingWriters === 0;
+        const arrivedAt = Date.now();
+        await this.acquireWrite();
+        const lockAcquiredAt = Date.now();
+        try {
+            const result = await fn();
+            const fnDoneAt = Date.now();
+            const waitMs = Math.max(0, lockAcquiredAt - arrivedAt);
+            const ownMs = Math.max(0, fnDoneAt - lockAcquiredAt);
+            return { result, owner, waitMs, ownMs };
+        }
+        finally {
+            this.releaseWrite();
+        }
+    }
+    async withInference(fn) {
+        await this.acquireRead();
+        try {
+            return await fn();
+        }
+        finally {
+            this.releaseRead();
+        }
+    }
+    acquireRead() {
+        if (!this.writerActive && this.waitingWriters === 0) {
+            this.activeReaders += 1;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.readerWaiters.push(() => {
+                this.activeReaders += 1;
+                resolve();
+            });
+        });
+    }
+    acquireWrite() {
+        this.waitingWriters += 1;
+        if (!this.writerActive && this.activeReaders === 0) {
+            this.waitingWriters -= 1;
+            this.writerActive = true;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.writerWaiters.push(() => {
+                this.waitingWriters -= 1;
+                this.writerActive = true;
+                resolve();
+            });
+        });
+    }
+    releaseRead() {
+        this.activeReaders -= 1;
+        if (this.activeReaders < 0)
+            this.activeReaders = 0;
+        if (this.activeReaders === 0)
+            this.drain();
+    }
+    releaseWrite() {
+        this.writerActive = false;
+        this.drain();
+    }
+    drain() {
+        if (this.writerActive)
+            return;
+        if (this.activeReaders === 0 && this.writerWaiters.length > 0) {
+            this.writerWaiters.shift()?.();
+            return;
+        }
+        if (this.waitingWriters === 0 && this.readerWaiters.length > 0) {
+            const readers = this.readerWaiters.splice(0);
+            for (const resolve of readers)
+                resolve();
+        }
+    }
+}

package/dist/pending-writes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pending-writes.d.ts","sourceRoot":"","sources":["../src/pending-writes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwGG;AAEH;;;;;;;GAOG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyC;IAEjE;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAkC;IAE3E;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CACjB;IAEZ;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAM;IAElD;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI;IA0ClF;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IAInD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,sBAAsB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAWtD;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,YAAY;IAI3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,OAAO;IAyB/E;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IAYpB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAqBnC,iEAAiE;IACjE,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,gBAAgB,IAAI,MAAM,CAG7B;IAED;;;;;;OAMG;IACH,IAAI,4BAA4B,IAAI,MAAM,CAEzC;CACF;AAYD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAOxE"}
+{"version":3,"file":"pending-writes.d.ts","sourceRoot":"","sources":["../src/pending-writes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwGG;AAEH;;;;;;;GAOG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyC;IAEjE;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAkC;IAE3E;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CACjB;IAEZ;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAM;IAElD;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAwCjF;IAED;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAElD;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,sBAAsB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CASrD;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,YAAY;IAI3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAuB9E;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IAYpB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAmBlC;IAED,iEAAiE;IACjE,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,gBAAgB,IAAI,MAAM,CAG7B;IAED;;;;;;OAMG;IACH,IAAI,4BAA4B,IAAI,MAAM,CAEzC;CACF;AAYD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAOxE"}