@mlx-node/server 0.0.12 → 0.0.15

package/src/tool-call-buffer.ts

@@ -0,0 +1,78 @@
+/**
+ * Buffers streaming text to detect and suppress model structural tags. Text
+ * that cannot be part of a partial tag is released immediately; once a
+ * full structural tag is seen, everything after it is suppressed until
+ * the stream ends.
+ */
+export class ToolCallTagBuffer {
+  private static readonly TAGS = [
+    '<tool_call>',
+    '</tool_call>',
+    '<|tool_call>',
+    '<tool_call|>',
+    '<|tool_response>',
+    '<tool_response|>',
+    '<|tool>',
+    '<tool|>',
+    '<|channel>',
+    '<channel|>',
+    '<|turn>',
+    '<turn|>',
+  ] as const;
+  private pendingText = '';
+  private _suppressed = false;
+
+  get suppressed(): boolean {
+    return this._suppressed;
+  }
+
+  /**
+   * Feed text in. Returns `safeText` (emit as delta), `tagFound` (a full
+   * structural tag was just seen), and `cleanPrefix` (text before the tag
+   * when `tagFound` — may contain whitespace; use `.trim()` only for
+   * emptiness checks, never for emission).
+   */
+  push(text: string): { safeText: string; tagFound: boolean; cleanPrefix: string } {
+    if (this._suppressed) {
+      return { safeText: '', tagFound: false, cleanPrefix: '' };
+    }
+
+    this.pendingText += text;
+
+    let tagIdx = -1;
+    for (const tag of ToolCallTagBuffer.TAGS) {
+      const idx = this.pendingText.indexOf(tag);
+      if (idx >= 0 && (tagIdx < 0 || idx < tagIdx)) {
+        tagIdx = idx;
+      }
+    }
+    if (tagIdx >= 0) {
+      const cleanPrefix = this.pendingText.slice(0, tagIdx);
+      this._suppressed = true;
+      this.pendingText = '';
+      return { safeText: '', tagFound: true, cleanPrefix };
+    }
+
+    // Hold back any suffix that could be the start of the tag.
+    let safeLen = this.pendingText.length;
+    const maxTagLength = Math.max(...ToolCallTagBuffer.TAGS.map((tag) => tag.length));
+    for (let i = 1; i <= Math.min(this.pendingText.length, maxTagLength - 1); i++) {
+      const suffix = this.pendingText.slice(-i);
+      if (ToolCallTagBuffer.TAGS.some((tag) => tag.startsWith(suffix))) {
+        safeLen = this.pendingText.length - i;
+        break;
+      }
+    }
+
+    const safeText = this.pendingText.slice(0, safeLen);
+    this.pendingText = this.pendingText.slice(safeLen);
+    return { safeText, tagFound: false, cleanPrefix: '' };
+  }
+
+  /** Release any held-back text at stream end. */
+  flush(): string {
+    const text = this.pendingText;
+    this.pendingText = '';
+    return text;
+  }
+}

package/src/transport-visibility.ts

@@ -0,0 +1,185 @@
+/**
+ * Transport visibility tracking shared between the Responses and Messages
+ * endpoints. Gates "safe to suppress" on whether the client actually observed
+ * a terminal artefact for this turn.
+ *
+ * The helpers reject — and leave visibility flags unflipped — on ANY of:
+ *   - the write callback reporting err != null;
+ *   - a `'error'` event on `res`;
+ *   - a `'close'` event on `res` or its socket;
+ *   - a pre-write check finding the response/socket already destroyed.
+ *
+ * Flipping the flags from `res.end()` / `writeSSEEvent`'s synchronous return
+ * is not sufficient: on a dead socket `_writeRaw` can return without ever
+ * firing the callback or `'error'`, which would pin the per-model mutex on a
+ * client that cannot see anything we write.
+ *
+ * Non-terminal SSE write calls remain synchronous, but streaming handlers
+ * await a close-safe drain promise whenever one reports backpressure. Only the
+ * terminal event uses the callback-backed visibility flush. Streaming handlers
+ * independently attach a `res.once('close', …)` listener to flip
+ * `clientAborted` so the decode loop breaks at the next iteration boundary.
+ */
+
+import type { ServerResponse } from 'node:http';
+
+import { writeSSEEvent } from './streaming.js';
+
+/**
+ * Wire format committed by the handler. `null` = pre-headers (outer catch
+ * can still emit a clean 500 JSON). `'json'` = `writeHead(200, 'application/json')`
+ * already fired — the outer catch MUST NOT emit SSE frames. `'sse'` = `beginSSE()`
+ * fired — the outer catch may emit a best-effort streaming `error` event.
+ */
+export type ResponseMode = 'json' | 'sse' | null;
+
+export interface TransportVisibility {
+  responseMode: ResponseMode;
+  /** Set only after `res.end(body)`'s callback fires with err == null — proves kernel acceptance, not buffer queue. */
+  responseBodyWritten: boolean;
+  /** Set only after the terminal SSE event's write callback reports no error. */
+  terminalEmitted: boolean;
+}
+
+export function createVisibility(): TransportVisibility {
+  return {
+    responseMode: null,
+    responseBodyWritten: false,
+    terminalEmitted: false,
+  };
+}
+
+/** True if `res` or its socket is already destroyed — writes will never be seen and callbacks may never fire. */
+function isSocketGone(res: ServerResponse): boolean {
+  if (res.destroyed) return true;
+  const sock = res.socket;
+  if (sock != null && sock.destroyed) return true;
+  return false;
+}
+
+/**
+ * Write an HTTP 200 JSON response and await kernel ack via `res.end(body, cb)`.
+ * `responseMode` is committed AFTER `writeHead` returns so a synchronous throw
+ * from `writeHead` leaves `responseMode === null` for the outer catch.
+ * `responseBodyWritten` is flipped only on the callback's success path.
+ */
+export async function endJson(res: ServerResponse, body: string, visibility: TransportVisibility): Promise<void> {
+  res.writeHead(200, { 'Content-Type': 'application/json' });
+  visibility.responseMode = 'json';
+  await new Promise<void>((resolve, reject) => {
+    let settled = false;
+    const onError = (err: Error): void => {
+      settle(err instanceof Error ? err : new Error(String(err)));
+    };
+    const onClose = (): void => {
+      // `'close'` without a successful end callback = client never saw the body.
+      settle(new Error('response closed before write completion'));
+    };
+    const sock = res.socket;
+    const settle = (err: Error | null): void => {
+      if (settled) return;
+      settled = true;
+      res.removeListener('error', onError);
+      res.removeListener('close', onClose);
+      if (sock != null) sock.removeListener('close', onClose);
+      if (err != null) {
+        reject(err);
+      } else {
+        visibility.responseBodyWritten = true;
+        resolve();
+      }
+    };
+
+    // Pre-check: `'close'` may have already fired and the write callback
+    // may never arrive on a destroyed peer — reject synchronously instead.
+    if (isSocketGone(res)) {
+      settle(new Error('response socket already destroyed before write'));
+      return;
+    }
+
+    res.once('error', onError);
+    res.once('close', onClose);
+    if (sock != null) sock.once('close', onClose);
+
+    try {
+      res.end(body, (err?: Error | null) => {
+        settle(err ?? null);
+      });
+    } catch (err) {
+      settle(err instanceof Error ? err : new Error(String(err)));
+    }
+  });
+}
+
+/**
+ * Emit the terminal SSE event for a streaming response and await kernel ack via
+ * `res.write(chunk, cb)`. `terminalEmitted` is flipped only on the success path.
+ * Used for `response.completed` / `response.failed`, `message_stop`, and the
+ * streaming `error` event. Non-terminal writes stay synchronous.
+ */
+export async function flushTerminalSSE(
+  res: ServerResponse,
+  eventType: string,
+  data: object,
+  visibility: TransportVisibility,
+): Promise<void> {
+  const payload = { type: eventType, ...data };
+  const chunk = `event: ${eventType}\ndata: ${JSON.stringify(payload)}\n\n`;
+  await new Promise<void>((resolve, reject) => {
+    let settled = false;
+    const onError = (err: Error): void => {
+      settle(err instanceof Error ? err : new Error(String(err)));
+    };
+    const onClose = (): void => {
+      settle(new Error('response closed before terminal SSE write completion'));
+    };
+    const sock = res.socket;
+    const settle = (err: Error | null): void => {
+      if (settled) return;
+      settled = true;
+      res.removeListener('error', onError);
+      res.removeListener('close', onClose);
+      if (sock != null) sock.removeListener('close', onClose);
+      if (err != null) {
+        reject(err);
+      } else {
+        visibility.terminalEmitted = true;
+        resolve();
+      }
+    };
+
+    if (isSocketGone(res)) {
+      settle(new Error('response socket already destroyed before terminal SSE write'));
+      return;
+    }
+
+    res.once('error', onError);
+    res.once('close', onClose);
+    if (sock != null) sock.once('close', onClose);
+
+    try {
+      // Ignore backpressure return value — the callback + `'close'` listener
+      // + `isSocketGone` pre-check cover every settle path we need.
+      const ok = res.write(chunk, (err?: Error | null) => {
+        settle(err ?? null);
+      });
+      void ok;
+    } catch (err) {
+      settle(err instanceof Error ? err : new Error(String(err)));
+    }
+  });
+}
+
+/** Commit to SSE mode — call immediately after `beginSSE(res)` so the outer catch routes SSE-shaped failures correctly. */
+export function markSSEMode(visibility: TransportVisibility): void {
+  visibility.responseMode = 'sse';
+}
+
+/** Best-effort synchronous SSE `error` event for the outer catch when the handler threw before flushing a terminal. */
+export function writeFallbackErrorSSE(res: ServerResponse, eventType: string, data: object): void {
+  try {
+    writeSSEEvent(res, eventType, data);
+  } catch {
+    // Socket is gone — let the caller complete the lifecycle via `end` / destroy.
+  }
+}

package/src/types-anthropic.ts

@@ -0,0 +1,409 @@
+/** Anthropic Messages API types: request/response shapes and SSE streaming events for POST /v1/messages. */
+
+export interface AnthropicTextContentBlock {
+  type: 'text';
+  text: string;
+}
+
+export interface AnthropicImageContentBlock {
+  type: 'image';
+  source: {
+    type: 'base64';
+    media_type: string;
+    data: string;
+  };
+}
+
+export interface AnthropicToolResultContentBlock {
+  type: 'tool_result';
+  tool_use_id: string;
+  /** May be a string, text-block array, or mix of text and image blocks. Image-mixed shapes are rejected by the mapper; see `resolveToolResultContent`. */
+  content?: string | (AnthropicTextContentBlock | AnthropicImageContentBlock)[];
+  is_error?: boolean;
+}
+
+export interface AnthropicToolUseContentBlock {
+  type: 'tool_use';
+  id: string;
+  name: string;
+  input: Record<string, unknown>;
+}
+
+export interface AnthropicThinkingContentBlock {
+  type: 'thinking';
+  thinking: string;
+}
+
+export type AnthropicContentBlock =
+  | AnthropicTextContentBlock
+  | AnthropicImageContentBlock
+  | AnthropicToolResultContentBlock
+  | AnthropicToolUseContentBlock
+  | AnthropicThinkingContentBlock;
+
+// ---------------------------------------------------------------------------
+// System block
+// ---------------------------------------------------------------------------
+
+export interface SystemBlock {
+  type: 'text';
+  text: string;
+  cache_control?: { type: string };
+}
+
+// ---------------------------------------------------------------------------
+// Messages
+// ---------------------------------------------------------------------------
+
+export interface AnthropicMessage {
+  // `system` is not part of the Anthropic Messages spec (system prompts go in
+  // the top-level `system` field), but Claude Code's SessionStart hooks inject
+  // a `{ role: 'system' }` message carrying "additional context" into the
+  // `messages` array. We tolerate it by folding its text into the system
+  // prompt (see `mapAnthropicRequest`) rather than rejecting the request.
+  role: 'user' | 'assistant' | 'system';
+  content: string | AnthropicContentBlock[];
+}
+
+// ---------------------------------------------------------------------------
+// Tools
+// ---------------------------------------------------------------------------
+
+export interface AnthropicToolDefinition {
+  name: string;
+  description?: string;
+  input_schema: Record<string, unknown>;
+}
+
+export interface AnthropicToolChoice {
+  type: 'auto' | 'any' | 'tool';
+  name?: string;
+  disable_parallel_tool_use?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Request
+// ---------------------------------------------------------------------------
+
+export interface AnthropicMessagesRequest {
+  model: string;
+  messages: AnthropicMessage[];
+  max_tokens: number;
+  system?: string | SystemBlock[];
+  temperature?: number;
+  top_p?: number;
+  top_k?: number;
+  tools?: AnthropicToolDefinition[];
+  tool_choice?: AnthropicToolChoice;
+  stream?: boolean;
+  stop_sequences?: string[];
+  metadata?: { user_id?: string };
+  /**
+   * MLX-Node extension matching vLLM's Anthropic surface. Separates native
+   * content-addressed prefix-cache reuse between security domains. The value
+   * must be non-empty, at most 256 UTF-8 bytes, stable for the tenant, secret,
+   * and unpredictable.
+   */
+  cache_salt?: string;
+  output_config?: {
+    format?: {
+      type?: string;
+      schema?: unknown;
+    };
+  };
+  // NOTE: `prompt_cache_key` is intentionally NOT advertised on this
+  // endpoint. KV-cache reuse on `/v1/messages` is delivered via the
+  // server-side `getOrCreateWarmAny` warm-slot mechanism keyed on the
+  // mapped system/instructions string and a per-model sentinel id —
+  // see the block comment on `endpoints/messages.ts`. Paged-active
+  // models (Qwen3 / LFM2 / Gemma4, default-on) additionally benefit
+  // from native `BlockAllocator` content-addressed prefix reuse, which
+  // recovers shared SYS/user prefixes across requests without any
+  // client-supplied key. The `prompt_cache_key` field is therefore
+  // unnecessary on this surface and re-adding it without a per-key
+  // tier-2 path on the handler would be a no-op that silently misleads
+  // clients. The equivalent field on `/v1/responses` is still honoured
+  // for that endpoint's tier-2 lookup.
+  /**
+   * MLX-Node extension carrier for non-Anthropic fields, namespaced
+   * under `extra_body` to mirror the OpenAI-side `/v1/responses`
+   * surface. Unknown keys are ignored (additive, forward-compat).
+   *
+   * Currently exposes:
+   *   * `generation_mode`: `"mtp"` forces W6 speculative-decode (sets
+   *     `enableMtp = true`), `"ar"` forces plain autoregressive
+   *     (`enableMtp = false`). Absent / null / unrecognized leaves
+   *     `enableMtp` untouched so the downstream `ChatSession` auto-
+   *     default (true when the model ships an MTP head) applies.
+   *   * `mtp_depth`: positive integer override for the per-call draft
+   *     depth. The server forwards any positive integer ≤ 64 (a sanity
+   *     ceiling that only blocks garbage) and the native per-family
+   *     `resolve_params` owns the real clamps: qwen3.5 native MTP
+   *     [1, 5], gemma4 DSpark capped at the draft block size, gemma4
+   *     assistant drafts [1, 8].
+   */
+  extra_body?: {
+    // Typed as `string | null` (not the literal union `'mtp' | 'ar'`)
+    // because the value arrives off-wire and may carry any client-
+    // supplied payload. The mapper validates by exact-string match;
+    // anything that doesn't match is silently ignored so the auto-
+    // default still applies.
+    generation_mode?: string | null;
+    mtp_depth?: number | null;
+  };
+}
+
+export type AnthropicCountTokensRequest = Omit<AnthropicMessagesRequest, 'max_tokens'> & {
+  max_tokens?: number;
+};
+
+export interface AnthropicCountTokensResponse {
+  input_tokens: number;
+}
+
+// ---------------------------------------------------------------------------
+// Response content blocks
+// ---------------------------------------------------------------------------
+
+export interface AnthropicResponseTextBlock {
+  type: 'text';
+  text: string;
+}
+
+export interface AnthropicResponseThinkingBlock {
+  type: 'thinking';
+  thinking: string;
+}
+
+export interface AnthropicResponseToolUseBlock {
+  type: 'tool_use';
+  id: string;
+  name: string;
+  input: Record<string, unknown>;
+}
+
+export type AnthropicResponseContent =
+  | AnthropicResponseTextBlock
+  | AnthropicResponseThinkingBlock
+  | AnthropicResponseToolUseBlock;
+
+// ---------------------------------------------------------------------------
+// Usage
+// ---------------------------------------------------------------------------
+
+/**
+ * Anthropic Messages API `usage` block.
+ *
+ * Per the Anthropic spec the cache accounting fields are OPTIONAL and
+ * carry distinct semantics that drive client-side cost / billing
+ * displays (Claude Code reads `cache_read_input_tokens` directly):
+ *
+ *   * `input_tokens` — prompt tokens that were processed at full cost
+ *     this turn. When a cached prefix is reused, this MUST be reduced
+ *     to the unsuffixed remainder (`promptTokens - cachedTokens`) so
+ *     the client doesn't double-count the cached prefix.
+ *   * `cache_read_input_tokens` — prompt tokens served from a cached
+ *     prefix on this turn. Emitted only when reuse genuinely happened
+ *     (`cachedTokens > 0`); omitted on a cold turn so the wire matches
+ *     other Anthropic-compatible servers that elide the field on cache
+ *     misses.
+ *   * `cache_creation_input_tokens` — prompt tokens written to a NEW
+ *     cache entry. Currently always omitted: this server's KV reuse
+ *     is implicit and has no `cache_control` breakpoints, so a client
+ *     that did not request explicit caching should never see a
+ *     non-zero creation count.
+ *
+ * The timing fields below are NON-Anthropic extension fields surfaced
+ * for the launcher's verbose log (`requests.ndjson`) so per-turn
+ * decode-rate / TTFT / prefill-rate telemetry rides the same response
+ * envelope. They describe native/server inference timing, not the
+ * HTTP logger's outer `elapsedMs` envelope. They are emitted only when
+ * the underlying native dispatch produced a finite, positive value —
+ * missing or non-finite metrics are elided rather than surfaced as
+ * zero / null. Anthropic-compatible clients (Claude Code, official
+ * Anthropic SDKs) ignore unknown fields, so the extension is wire-safe
+ * — it parallels how `cache_read_input_tokens` is treated above.
+ *
+ * `prefill_input_tokens` and `cached_prefix_tokens` provide the
+ * denominator/context for `prefill_tokens_per_second`: on cached-prefix
+ * turns the prefill rate is for the uncached suffix, not the full
+ * logical prompt.
+ */
+export interface AnthropicUsage {
+  input_tokens: number;
+  output_tokens: number;
+  cache_read_input_tokens?: number;
+  cache_creation_input_tokens?: number;
+  /** Server-extension: time-to-first-token in milliseconds. */
+  time_to_first_token_ms?: number;
+  /** Server-extension: prompt-token throughput during prefill. */
+  prefill_tokens_per_second?: number;
+  /** Server-extension: generated-token throughput during decode. */
+  decode_tokens_per_second?: number;
+  /** Server-extension: native/server inference elapsed, excluding HTTP transport and logging overhead. */
+  server_inference_elapsed_ms?: number;
+  /** Server-extension alias for disambiguating native TTFT from request/HTTP elapsed time. */
+  server_time_to_first_token_ms?: number;
+  /** Server-extension: handler-start to first native token, including model resolve/load and queue wait. */
+  server_total_time_to_first_token_ms?: number;
+  /** Server-extension alias for native prefill throughput. */
+  server_prefill_tokens_per_second?: number;
+  /** Server-extension alias for native decode throughput. */
+  server_decode_tokens_per_second?: number;
+  /** Server-extension: prompt tokens actually prefetched this turn after cached-prefix reuse. */
+  prefill_input_tokens?: number;
+  /** Server-extension: prompt tokens skipped because a cached prefix was reused. */
+  cached_prefix_tokens?: number;
+  /** Server-extension: time spent resolving/loading/aliasing the requested model before registry lookup. */
+  server_model_resolve_ms?: number;
+  /** Server-extension: wall-clock time blocked on the process-wide model-load writer lock. */
+  server_load_wait_ms?: number;
+  /** Server-extension: true when this request acquired the model-load lock without contention. */
+  server_load_owner?: boolean;
+  /** Server-extension: time spent waiting behind the per-model execution mutex. */
+  server_queue_ms?: number;
+  /** Server-extension: handler time before native inference begins, including resolve and queue wait. */
+  server_pre_inference_ms?: number;
+  /** Server-extension: effective process-level paged-prefill chunk size. */
+  server_paged_prefill_chunk_size?: number;
+  /** Server-extension: effective process-level paged-prefill eval/clear cadence. */
+  server_paged_prefill_eval_interval?: number;
+  /** Server-extension: effective process-level paged-decode cache-clear cadence. */
+  server_paged_decode_cache_clear_interval?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Response
+// ---------------------------------------------------------------------------
+
+export interface AnthropicMessagesResponse {
+  id: string;
+  type: 'message';
+  role: 'assistant';
+  model: string;
+  content: AnthropicResponseContent[];
+  stop_reason: 'end_turn' | 'max_tokens' | 'stop_sequence' | 'tool_use' | null;
+  stop_sequence: string | null;
+  usage: AnthropicUsage;
+}
+
+// ---------------------------------------------------------------------------
+// Streaming delta types
+// ---------------------------------------------------------------------------
+
+export interface AnthropicTextDelta {
+  type: 'text_delta';
+  text: string;
+}
+
+export interface AnthropicThinkingDelta {
+  type: 'thinking_delta';
+  thinking: string;
+}
+
+export interface AnthropicInputJsonDelta {
+  type: 'input_json_delta';
+  partial_json: string;
+}
+
+export type AnthropicDelta = AnthropicTextDelta | AnthropicThinkingDelta | AnthropicInputJsonDelta;
+
+// ---------------------------------------------------------------------------
+// Streaming events
+// ---------------------------------------------------------------------------
+
+export interface AnthropicMessageStartEvent {
+  type: 'message_start';
+  message: AnthropicMessagesResponse;
+}
+
+export interface AnthropicContentBlockStartEvent {
+  type: 'content_block_start';
+  index: number;
+  content_block: AnthropicResponseContent;
+}
+
+export interface AnthropicContentBlockDeltaEvent {
+  type: 'content_block_delta';
+  index: number;
+  delta: AnthropicDelta;
+}
+
+export interface AnthropicContentBlockStopEvent {
+  type: 'content_block_stop';
+  index: number;
+}
+
+export interface AnthropicMessageDeltaEvent {
+  type: 'message_delta';
+  delta: {
+    stop_reason: string | null;
+    stop_sequence: string | null;
+  };
+  /**
+   * Streaming `message_delta` carries the SAME cache-accounting
+   * semantics as the non-streaming response `usage` block (see
+   * `AnthropicUsage` above) — `input_tokens` MUST be net of any reused
+   * prefix, and `cache_read_input_tokens` is emitted only on a true
+   * reuse turn.
+   *
+   * Timing/accounting fields are server extensions (not in
+   * Anthropic's spec) surfaced for the launcher's verbose log; see
+   * the docstring on `AnthropicUsage`. Clients that do not recognize
+   * them ignore them.
+   */
+  usage: {
+    input_tokens?: number;
+    output_tokens: number;
+    cache_read_input_tokens?: number;
+    cache_creation_input_tokens?: number;
+    /** Server-extension: time-to-first-token in milliseconds. */
+    time_to_first_token_ms?: number;
+    /** Server-extension: prompt-token throughput during prefill. */
+    prefill_tokens_per_second?: number;
+    /** Server-extension: generated-token throughput during decode. */
+    decode_tokens_per_second?: number;
+    /** Server-extension: native/server inference elapsed, excluding HTTP transport and logging overhead. */
+    server_inference_elapsed_ms?: number;
+    /** Server-extension alias for disambiguating native TTFT from request/HTTP elapsed time. */
+    server_time_to_first_token_ms?: number;
+    /** Server-extension: handler-start to first native token, including model resolve/load and queue wait. */
+    server_total_time_to_first_token_ms?: number;
+    /** Server-extension alias for native prefill throughput. */
+    server_prefill_tokens_per_second?: number;
+    /** Server-extension alias for native decode throughput. */
+    server_decode_tokens_per_second?: number;
+    /** Server-extension: prompt tokens actually prefetched this turn after cached-prefix reuse. */
+    prefill_input_tokens?: number;
+    /** Server-extension: prompt tokens skipped because a cached prefix was reused. */
+    cached_prefix_tokens?: number;
+    /** Server-extension: time spent resolving/loading/aliasing the requested model before registry lookup. */
+    server_model_resolve_ms?: number;
+    /** Server-extension: wall-clock time blocked on the process-wide model-load writer lock. */
+    server_load_wait_ms?: number;
+    /** Server-extension: true when this request acquired the model-load lock without contention. */
+    server_load_owner?: boolean;
+    /** Server-extension: time spent waiting behind the per-model execution mutex. */
+    server_queue_ms?: number;
+    /** Server-extension: handler time before native inference begins, including resolve and queue wait. */
+    server_pre_inference_ms?: number;
+    /** Server-extension: effective process-level paged-prefill chunk size. */
+    server_paged_prefill_chunk_size?: number;
+    /** Server-extension: effective process-level paged-prefill eval/clear cadence. */
+    server_paged_prefill_eval_interval?: number;
+    /** Server-extension: effective process-level paged-decode cache-clear cadence. */
+    server_paged_decode_cache_clear_interval?: number;
+  };
+}
+
+export interface AnthropicMessageStopEvent {
+  type: 'message_stop';
+}
+
+export type AnthropicStreamEvent =
+  | AnthropicMessageStartEvent
+  | AnthropicContentBlockStartEvent
+  | AnthropicContentBlockDeltaEvent
+  | AnthropicContentBlockStopEvent
+  | AnthropicMessageDeltaEvent
+  | AnthropicMessageStopEvent;