@mlx-node/server 0.0.0 → 0.0.8

package/dist/stop-sequence-buffer.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Buffers streaming text to detect configured stop sequences. Text that
+ * cannot be part of a partial stop sequence is released immediately; a
+ * trailing suffix that could be the start of a stop sequence is held back
+ * until a later push resolves it or the stream is flushed. Once a full stop
+ * sequence is seen, everything after it is suppressed.
+ */
+export declare class StopSequenceBuffer {
+    private readonly stopSequences;
+    private readonly maxLength;
+    private pending_;
+    private _matched;
+    constructor(stopSequences: string[]);
+    /**
+     * Earliest index wins; on a tie at the same index the longest wins. Returns
+     * `{ idx, seq }` for the winning stop, or `{ idx: -1, seq: null }` when none
+     * is present in `pending`.
+     */
+    private findMatch;
+    /** The stop sequence that has matched so far, or `null` if none has. */
+    get matched(): string | null;
+    /**
+     * The text currently held back (received but neither emitted nor matched).
+     * The streaming done-path reads this so it can scan the terminal/recovered
+     * text on the SAME buffer with the held partial still in place, and so it
+     * can reconstruct the full received-but-unemitted prefix for overlap math.
+     */
+    get pending(): string;
+    /**
+     * The earliest start index `j` in `[0, limit]` such that `pending.slice(j)`
+     * is a non-empty STRICT prefix of some configured stop — i.e. a partial that
+     * a later push could still grow into that stop. Returns -1 when no pending
+     * suffix at or before `limit` is viable. Scanning from the front yields the
+     * earliest start index, which is the one whose completed stop would win the
+     * earliest-index tiebreak. A suffix longer than `maxLength - 1` can never be
+     * a strict prefix of any stop, so the search starts no earlier than that.
+     */
+    private earliestViablePrefixIndex;
+    /**
+     * Feed text in. Returns `safeText` (emit as delta) and `matched` (the stop
+     * sequence that has been matched, or `null`). After a match every push
+     * returns empty `safeText` and keeps reporting the matched sequence.
+     */
+    push(text: string): {
+        safeText: string;
+        matched: string | null;
+    };
+    /**
+     * Release any held-back text at stream end. If a stop sequence already
+     * matched, nothing more is emitted; otherwise the residue could not
+     * complete any sequence and is released.
+     */
+    flush(): {
+        safeText: string;
+        matched: string | null;
+    };
+}
+//# sourceMappingURL=stop-sequence-buffer.d.ts.map

package/dist/stop-sequence-buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stop-sequence-buffer.d.ts","sourceRoot":"","sources":["../src/stop-sequence-buffer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAW;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAM;IACtB,OAAO,CAAC,QAAQ,CAAuB;IAEvC,YAAY,aAAa,EAAE,MAAM,EAAE,EAOlC;IAED;;;;OAIG;IACH,OAAO,CAAC,SAAS;IAgBjB,wEAAwE;IACxE,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAED;;;;;OAKG;IACH,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,yBAAyB;IAejC;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CA8C/D;IAED;;;;OAIG;IACH,KAAK,IAAI;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAmBpD;CACF"}

package/dist/stop-sequence-buffer.js

@@ -0,0 +1,148 @@
+/**
+ * Buffers streaming text to detect configured stop sequences. Text that
+ * cannot be part of a partial stop sequence is released immediately; a
+ * trailing suffix that could be the start of a stop sequence is held back
+ * until a later push resolves it or the stream is flushed. Once a full stop
+ * sequence is seen, everything after it is suppressed.
+ */
+export class StopSequenceBuffer {
+    stopSequences;
+    maxLength;
+    pending_ = '';
+    _matched = null;
+    constructor(stopSequences) {
+        // Drop empty AND whitespace-only entries: a whitespace-only stop would
+        // truncate normal output at the first space/newline, and the real
+        // Anthropic API rejects such stops outright. Mirrors the same trim filter
+        // in the request mapper so a whitespace-only configuration is a no-op.
+        this.stopSequences = stopSequences.filter((s) => s.trim().length > 0);
+        this.maxLength = this.stopSequences.reduce((max, s) => Math.max(max, s.length), 0);
+    }
+    /**
+     * Earliest index wins; on a tie at the same index the longest wins. Returns
+     * `{ idx, seq }` for the winning stop, or `{ idx: -1, seq: null }` when none
+     * is present in `pending`.
+     */
+    findMatch() {
+        let matchIdx = -1;
+        let matchSeq = null;
+        for (const seq of this.stopSequences) {
+            const idx = this.pending_.indexOf(seq);
+            if (idx < 0) {
+                continue;
+            }
+            if (matchIdx < 0 || idx < matchIdx || (idx === matchIdx && seq.length > (matchSeq?.length ?? 0))) {
+                matchIdx = idx;
+                matchSeq = seq;
+            }
+        }
+        return { idx: matchIdx, seq: matchSeq };
+    }
+    /** The stop sequence that has matched so far, or `null` if none has. */
+    get matched() {
+        return this._matched;
+    }
+    /**
+     * The text currently held back (received but neither emitted nor matched).
+     * The streaming done-path reads this so it can scan the terminal/recovered
+     * text on the SAME buffer with the held partial still in place, and so it
+     * can reconstruct the full received-but-unemitted prefix for overlap math.
+     */
+    get pending() {
+        return this.pending_;
+    }
+    /**
+     * The earliest start index `j` in `[0, limit]` such that `pending.slice(j)`
+     * is a non-empty STRICT prefix of some configured stop — i.e. a partial that
+     * a later push could still grow into that stop. Returns -1 when no pending
+     * suffix at or before `limit` is viable. Scanning from the front yields the
+     * earliest start index, which is the one whose completed stop would win the
+     * earliest-index tiebreak. A suffix longer than `maxLength - 1` can never be
+     * a strict prefix of any stop, so the search starts no earlier than that.
+     */
+    earliestViablePrefixIndex(limit) {
+        if (this.pending_.length === 0) {
+            return -1;
+        }
+        const lowerBound = Math.max(0, this.pending_.length - (this.maxLength - 1));
+        const upper = Math.min(limit, this.pending_.length - 1);
+        for (let j = lowerBound; j <= upper; j++) {
+            const suffix = this.pending_.slice(j);
+            if (this.stopSequences.some((seq) => suffix.length < seq.length && seq.startsWith(suffix))) {
+                return j;
+            }
+        }
+        return -1;
+    }
+    /**
+     * Feed text in. Returns `safeText` (emit as delta) and `matched` (the stop
+     * sequence that has been matched, or `null`). After a match every push
+     * returns empty `safeText` and keeps reporting the matched sequence.
+     */
+    push(text) {
+        if (this._matched !== null) {
+            return { safeText: '', matched: this._matched };
+        }
+        // Transparent pass-through when there is nothing to detect.
+        if (this.stopSequences.length === 0) {
+            return { safeText: text, matched: null };
+        }
+        this.pending_ += text;
+        const { idx: matchIdx, seq: matchSeq } = this.findMatch();
+        // Earliest start index of a still-growable stop prefix. When a full match
+        // exists we only look at or before it (`limit = matchIdx`): a viable prefix
+        // beginning AFTER the match would complete at a later index and lose the
+        // earliest-index tiebreak, and the bytes from the match onward are
+        // suppressed anyway. A viable prefix at or before the match could still
+        // complete into a stop that WINS (earlier index, or longer at the same
+        // index), so the match must be held. With no full match we consider the
+        // whole pending text.
+        const limit = matchIdx >= 0 ? matchIdx : this.pending_.length - 1;
+        const holdIdx = this.earliestViablePrefixIndex(limit);
+        if (matchIdx >= 0 && matchSeq !== null) {
+            if (holdIdx >= 0) {
+                // A longer/earlier stop could still complete from `holdIdx`; emit only
+                // the bytes before it and keep the rest pending for a later push or
+                // `flush()` to resolve.
+                const safeText = this.pending_.slice(0, holdIdx);
+                this.pending_ = this.pending_.slice(holdIdx);
+                return { safeText, matched: null };
+            }
+            const safeText = this.pending_.slice(0, matchIdx);
+            this._matched = matchSeq;
+            this.pending_ = '';
+            return { safeText, matched: matchSeq };
+        }
+        // No full match: release everything before the earliest viable prefix and
+        // hold that suffix back, since a later push could complete it.
+        const safeLen = holdIdx >= 0 ? holdIdx : this.pending_.length;
+        const safeText = this.pending_.slice(0, safeLen);
+        this.pending_ = this.pending_.slice(safeLen);
+        return { safeText, matched: null };
+    }
+    /**
+     * Release any held-back text at stream end. If a stop sequence already
+     * matched, nothing more is emitted; otherwise the residue could not
+     * complete any sequence and is released.
+     */
+    flush() {
+        if (this._matched !== null) {
+            return { safeText: '', matched: this._matched };
+        }
+        // The stream has ended, so any match `push()` held back for a possible
+        // longer same-index stop can no longer be extended — resolve it now.
+        // Re-scan `pending` for the earliest match (longest on tie) and commit it
+        // if present; otherwise the residue could not complete any sequence and is
+        // released verbatim.
+        const { idx: matchIdx, seq: matchSeq } = this.findMatch();
+        if (matchIdx >= 0 && matchSeq !== null) {
+            const safeText = this.pending_.slice(0, matchIdx);
+            this._matched = matchSeq;
+            this.pending_ = '';
+            return { safeText, matched: matchSeq };
+        }
+        const safeText = this.pending_;
+        this.pending_ = '';
+        return { safeText, matched: null };
+    }
+}

package/dist/text-recovery.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Streaming text-recovery helpers shared between the `/v1/messages` and
+ * `/v1/responses` endpoints.
+ *
+ * Both endpoints have a tool-call streaming recovery branch that has to
+ * compute the unsent suffix of `finalText` given that some prefix of the
+ * model's output may already have been streamed to the wire, but native-side
+ * string normalization makes `finalText` diverge from the streamed-prefix
+ * verbatim. Concrete divergences seen in practice:
+ *
+ *   * The native side trims leading whitespace after `</think>` via
+ *     `split_at_think_end`, so the streamed text can end in `"\n\n"` while
+ *     `finalText` starts at `"<tool_call>"` (no overlap — emit `finalText`
+ *     whole).
+ *   * The native side `.trim()`s tool-tag-bracketed content boundaries, so
+ *     the streamed text can have a trailing space that `finalText` lacks
+ *     (also no overlap — emit `finalText` whole).
+ *
+ * Internal-only — not exported from `packages/server/src/index.ts`.
+ */
+/**
+ * Find the largest k such that `streamed.endsWith(final.slice(0, k))`.
+ *
+ * Returns 0 when there is no overlap (caller emits `final` whole).
+ * Returns `final.length` when `final` is fully contained as a suffix of
+ * `streamed` (caller emits nothing).
+ *
+ * Used by the `/v1/messages` and `/v1/responses` streaming tool-call
+ * recovery branches to decide how much of `finalText` is already on the
+ * wire when native-side normalization (e.g. `.trim()`, post-`</think>`
+ * whitespace stripping) makes the streamed prefix diverge from the
+ * `finalText` prefix verbatim.
+ */
+export declare function longestSuffixPrefixOverlap(streamed: string, final: string): number;
+//# sourceMappingURL=text-recovery.d.ts.map

package/dist/text-recovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-recovery.d.ts","sourceRoot":"","sources":["../src/text-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;;;;;;;;GAYG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAMlF"}

package/dist/text-recovery.js

@@ -0,0 +1,41 @@
+/**
+ * Streaming text-recovery helpers shared between the `/v1/messages` and
+ * `/v1/responses` endpoints.
+ *
+ * Both endpoints have a tool-call streaming recovery branch that has to
+ * compute the unsent suffix of `finalText` given that some prefix of the
+ * model's output may already have been streamed to the wire, but native-side
+ * string normalization makes `finalText` diverge from the streamed-prefix
+ * verbatim. Concrete divergences seen in practice:
+ *
+ *   * The native side trims leading whitespace after `</think>` via
+ *     `split_at_think_end`, so the streamed text can end in `"\n\n"` while
+ *     `finalText` starts at `"<tool_call>"` (no overlap — emit `finalText`
+ *     whole).
+ *   * The native side `.trim()`s tool-tag-bracketed content boundaries, so
+ *     the streamed text can have a trailing space that `finalText` lacks
+ *     (also no overlap — emit `finalText` whole).
+ *
+ * Internal-only — not exported from `packages/server/src/index.ts`.
+ */
+/**
+ * Find the largest k such that `streamed.endsWith(final.slice(0, k))`.
+ *
+ * Returns 0 when there is no overlap (caller emits `final` whole).
+ * Returns `final.length` when `final` is fully contained as a suffix of
+ * `streamed` (caller emits nothing).
+ *
+ * Used by the `/v1/messages` and `/v1/responses` streaming tool-call
+ * recovery branches to decide how much of `finalText` is already on the
+ * wire when native-side normalization (e.g. `.trim()`, post-`</think>`
+ * whitespace stripping) makes the streamed prefix diverge from the
+ * `finalText` prefix verbatim.
+ */
+export function longestSuffixPrefixOverlap(streamed, final) {
+    const max = Math.min(streamed.length, final.length);
+    for (let k = max; k > 0; k--) {
+        if (streamed.endsWith(final.slice(0, k)))
+            return k;
+    }
+    return 0;
+}

package/dist/timing.d.ts

@@ -0,0 +1,80 @@
+/** Wire-safe server timing extensions derived from native performance metrics. */
+export interface PerformanceMetricsForUsage {
+    ttftMs?: number;
+    prefillTokensPerSecond?: number;
+    decodeTokensPerSecond?: number;
+}
+interface TimingUsageExtensions {
+    /** Server-extension: native time-to-first-token in milliseconds. */
+    time_to_first_token_ms?: number;
+    /** Server-extension: prompt-token throughput for the tokens actually prefetched this turn. */
+    prefill_tokens_per_second?: number;
+    /** Server-extension: generated-token throughput during decode. */
+    decode_tokens_per_second?: number;
+    /** Server-extension: native/server inference elapsed, excluding HTTP transport and logging overhead. */
+    server_inference_elapsed_ms?: number;
+    /** Server-extension alias for disambiguating native TTFT from request/HTTP elapsed time. */
+    server_time_to_first_token_ms?: number;
+    /** Server-extension: handler-start to first native token, including model resolve/load and queue wait. */
+    server_total_time_to_first_token_ms?: number;
+    /** Server-extension alias for native prefill throughput. */
+    server_prefill_tokens_per_second?: number;
+    /** Server-extension alias for native decode throughput. */
+    server_decode_tokens_per_second?: number;
+    /** Server-extension: prompt tokens actually prefetched this turn after cached-prefix reuse. */
+    prefill_input_tokens?: number;
+    /** Server-extension: prompt tokens skipped because a cached prefix was reused. */
+    cached_prefix_tokens?: number;
+    /**
+     * Server-extension: time spent resolving/loading/aliasing the requested model
+     * before registry lookup. Includes both the synchronous lookup AND any time
+     * spent driving the load. Excludes time spent blocked behind a peer
+     * request's in-flight load — that wait is reported separately via
+     * `server_load_wait_ms` so a fast follower request is not mis-attributed
+     * a long resolve when it merely inherited a cold-load wait.
+     */
+    server_model_resolve_ms?: number;
+    /**
+     * Server-extension: wall-clock time this request spent blocked on the
+     * process-wide model-load writer lock. Set when the load was already
+     * in flight when this request arrived (a peer drove the load and we
+     * waited for it to finish) AND when this request itself drove the
+     * load. Inspect `server_load_owner` to disambiguate. When the writer
+     * lock was free and the resolve was a no-op (model already loaded),
+     * this field is elided.
+     */
+    server_load_wait_ms?: number;
+    /**
+     * Server-extension: `true` when this request acquired the model-load
+     * writer lock with no contention (i.e. either the model was already
+     * loaded and the call was a no-op, or this request itself drove the
+     * load). `false` when the request was parked behind a peer's in-flight
+     * load. Elided when no load coordinator is wired (single-process
+     * tests, embedded callers).
+     */
+    server_load_owner?: boolean;
+    /** Server-extension: time spent waiting behind the per-model execution mutex. */
+    server_queue_ms?: number;
+    /** Server-extension: handler time before native inference begins, including resolve and queue wait. */
+    server_pre_inference_ms?: number;
+    /** Server-extension: effective process-level paged-prefill chunk size. */
+    server_paged_prefill_chunk_size?: number;
+    /** Server-extension: effective process-level paged-prefill eval/clear cadence. */
+    server_paged_prefill_eval_interval?: number;
+    /** Server-extension: effective process-level paged-decode cache-clear cadence. */
+    server_paged_decode_cache_clear_interval?: number;
+}
+export interface ServerTimingForUsage {
+    server_model_resolve_ms?: number;
+    server_load_wait_ms?: number;
+    server_load_owner?: boolean;
+    server_queue_ms?: number;
+    server_pre_inference_ms?: number;
+    server_paged_prefill_chunk_size?: number;
+    server_paged_prefill_eval_interval?: number;
+    server_paged_decode_cache_clear_interval?: number;
+}
+export declare function resolveServerTuningForUsage(env?: Record<string, string | undefined>): Pick<ServerTimingForUsage, 'server_paged_prefill_chunk_size' | 'server_paged_prefill_eval_interval' | 'server_paged_decode_cache_clear_interval'>;
+export declare function mergeTimingUsageExtensions<T extends TimingUsageExtensions>(usage: T, performance: PerformanceMetricsForUsage | undefined, promptTokens: number | undefined, outputTokens: number | undefined, cachedTokens: number | undefined, serverTiming?: ServerTimingForUsage): void;
+export {};
+//# sourceMappingURL=timing.d.ts.map

package/dist/timing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timing.d.ts","sourceRoot":"","sources":["../src/timing.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAElF,MAAM,WAAW,0BAA0B;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,UAAU,qBAAqB;IAC7B,oEAAoE;IACpE,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,8FAA8F;IAC9F,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,kEAAkE;IAClE,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,wGAAwG;IACxG,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,4FAA4F;IAC5F,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,0GAA0G;IAC1G,mCAAmC,CAAC,EAAE,MAAM,CAAC;IAC7C,4DAA4D;IAC5D,gCAAgC,CAAC,EAAE,MAAM,CAAC;IAC1C,2DAA2D;IAC3D,+BAA+B,CAAC,EAAE,MAAM,CAAC;IACzC,+FAA+F;IAC/F,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kFAAkF;IAClF,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;OAOG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;;;;OAQG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,iFAAiF;IACjF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,uGAAuG;IACvG,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,0EAA0E;IAC1E,+BAA+B,CAAC,EAAE,MAAM,CAAC;IACzC,kFAAkF;IAClF,kCAAkC,CAAC,EAAE,MAAM,CAAC;IAC5C,kFAAkF;IAClF,wCAAwC,CAAC,EAAE,MAAM,CAAC;CACnD;AAcD,MAAM,WAAW,oBAAoB;IACnC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,+BAA+B,CAAC,EAAE,MAAM,CAAC;IACzC,kCAAkC,CAAC,EAAE,MAAM,CAAC;IAC5C,wCAAwC,CAAC,EAAE,MAAM,CAAC;CACnD;AAsBD,wBAAgB,2BAA2B,CACzC,GAAG,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAe,GACpD,IAAI,CACL,oBAAoB,EACpB,iCAAiC,GAAG,oCAAoC,GAAG,0CAA0C,CACtH,CAMA;AAuGD,wBAAgB,0BAA0B,CAAC,CAAC,SAAS,qBAAqB,EACxE,KAAK,EAAE,CAAC,EACR,WAAW,EAAE,0BAA0B,GAAG,SAAS,EACnD,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,YAAY,CAAC,EAAE,oBAAoB,GAClC,IAAI,CAEN"}

package/dist/timing.js

@@ -0,0 +1,121 @@
+/** Wire-safe server timing extensions derived from native performance metrics. */
+function finitePositive(value) {
+    return value != null && Number.isFinite(value) && value > 0 ? value : undefined;
+}
+function finiteNonNegativeInteger(value) {
+    return value != null && Number.isFinite(value) && value >= 0 ? Math.max(0, Math.floor(value)) : undefined;
+}
+function finiteNonNegative(value) {
+    return value != null && Number.isFinite(value) && value >= 0 ? value : undefined;
+}
+const I32_MAX = 0x7fff_ffff;
+function parseI32(value) {
+    if (value == null)
+        return undefined;
+    const trimmed = value.trim();
+    if (!/^[+-]?\d+$/.test(trimmed))
+        return undefined;
+    const parsed = Number.parseInt(trimmed, 10);
+    return Number.isSafeInteger(parsed) && parsed >= -0x8000_0000 && parsed <= I32_MAX ? parsed : undefined;
+}
+function parseNonNegativeI32(value, fallback) {
+    const parsed = parseI32(value);
+    return parsed != null && parsed >= 0 ? parsed : fallback;
+}
+function parsePositiveI32(value, fallback) {
+    const parsed = parseI32(value);
+    return parsed != null && parsed > 0 ? parsed : fallback;
+}
+export function resolveServerTuningForUsage(env = process.env) {
+    return {
+        server_paged_prefill_chunk_size: parseNonNegativeI32(env.MLX_PAGED_PREFILL_CHUNK_SIZE, 0),
+        server_paged_prefill_eval_interval: parsePositiveI32(env.MLX_PAGED_PREFILL_EVAL_INTERVAL, 8),
+        server_paged_decode_cache_clear_interval: parsePositiveI32(env.MLX_PAGED_DECODE_CACHE_CLEAR_INTERVAL, 1024),
+    };
+}
+function computeServerInferenceElapsedMs(ttftMs, decodeTokensPerSecond, outputTokens) {
+    if (ttftMs == null)
+        return undefined;
+    const generatedTokens = finiteNonNegativeInteger(outputTokens);
+    if (generatedTokens == null) {
+        return ttftMs;
+    }
+    if (generatedTokens <= 1) {
+        return ttftMs;
+    }
+    if (decodeTokensPerSecond == null) {
+        return undefined;
+    }
+    return ttftMs + ((generatedTokens - 1) / decodeTokensPerSecond) * 1000;
+}
+function buildTimingUsageExtensions(performance, promptTokens, outputTokens, cachedTokens, serverTiming) {
+    const ttftMs = finitePositive(performance?.ttftMs);
+    const prefillTokensPerSecond = finitePositive(performance?.prefillTokensPerSecond);
+    const decodeTokensPerSecond = finitePositive(performance?.decodeTokensPerSecond);
+    const serverInferenceElapsedMs = computeServerInferenceElapsedMs(ttftMs, decodeTokensPerSecond, outputTokens);
+    const preInferenceMs = finiteNonNegative(serverTiming?.server_pre_inference_ms);
+    const extensions = {};
+    if (ttftMs != null) {
+        extensions.time_to_first_token_ms = ttftMs;
+        extensions.server_time_to_first_token_ms = ttftMs;
+        if (preInferenceMs != null) {
+            extensions.server_total_time_to_first_token_ms = preInferenceMs + ttftMs;
+        }
+    }
+    if (prefillTokensPerSecond != null) {
+        extensions.prefill_tokens_per_second = prefillTokensPerSecond;
+        extensions.server_prefill_tokens_per_second = prefillTokensPerSecond;
+    }
+    if (decodeTokensPerSecond != null) {
+        extensions.decode_tokens_per_second = decodeTokensPerSecond;
+        extensions.server_decode_tokens_per_second = decodeTokensPerSecond;
+    }
+    if (serverInferenceElapsedMs != null && Number.isFinite(serverInferenceElapsedMs) && serverInferenceElapsedMs > 0) {
+        extensions.server_inference_elapsed_ms = serverInferenceElapsedMs;
+    }
+    if (performance != null) {
+        const prompt = finiteNonNegativeInteger(promptTokens);
+        const cached = finiteNonNegativeInteger(cachedTokens);
+        if (prompt != null) {
+            const cachedPrefix = cached == null ? 0 : Math.min(cached, prompt);
+            extensions.prefill_input_tokens = prompt - cachedPrefix;
+            if (cachedPrefix > 0) {
+                extensions.cached_prefix_tokens = cachedPrefix;
+            }
+        }
+    }
+    const modelResolveMs = finiteNonNegative(serverTiming?.server_model_resolve_ms);
+    if (modelResolveMs != null) {
+        extensions.server_model_resolve_ms = modelResolveMs;
+    }
+    const loadWaitMs = finiteNonNegative(serverTiming?.server_load_wait_ms);
+    if (loadWaitMs != null) {
+        extensions.server_load_wait_ms = loadWaitMs;
+    }
+    if (typeof serverTiming?.server_load_owner === 'boolean') {
+        extensions.server_load_owner = serverTiming.server_load_owner;
+    }
+    const queueMs = finiteNonNegative(serverTiming?.server_queue_ms);
+    if (queueMs != null) {
+        extensions.server_queue_ms = queueMs;
+    }
+    if (preInferenceMs != null) {
+        extensions.server_pre_inference_ms = preInferenceMs;
+    }
+    const pagedPrefillChunkSize = finiteNonNegativeInteger(serverTiming?.server_paged_prefill_chunk_size);
+    if (pagedPrefillChunkSize != null) {
+        extensions.server_paged_prefill_chunk_size = pagedPrefillChunkSize;
+    }
+    const pagedPrefillEvalInterval = finiteNonNegativeInteger(serverTiming?.server_paged_prefill_eval_interval);
+    if (pagedPrefillEvalInterval != null) {
+        extensions.server_paged_prefill_eval_interval = pagedPrefillEvalInterval;
+    }
+    const pagedDecodeCacheClearInterval = finiteNonNegativeInteger(serverTiming?.server_paged_decode_cache_clear_interval);
+    if (pagedDecodeCacheClearInterval != null) {
+        extensions.server_paged_decode_cache_clear_interval = pagedDecodeCacheClearInterval;
+    }
+    return extensions;
+}
+export function mergeTimingUsageExtensions(usage, performance, promptTokens, outputTokens, cachedTokens, serverTiming) {
+    Object.assign(usage, buildTimingUsageExtensions(performance, promptTokens, outputTokens, cachedTokens, serverTiming));
+}

package/dist/tool-call-buffer.d.ts

@@ -1,17 +1,17 @@
 /**
- * Buffers streaming text to detect and suppress `<tool_call>` tags. Text
+ * Buffers streaming text to detect and suppress model structural tags. Text
  * that cannot be part of a partial tag is released immediately; once a
- * full tag is seen, everything after it is suppressed until the stream
- * ends.
+ * full structural tag is seen, everything after it is suppressed until
+ * the stream ends.
  */
 export declare class ToolCallTagBuffer {
-    private static readonly TAG;
+    private static readonly TAGS;
     private pendingText;
     private _suppressed;
     get suppressed(): boolean;
     /**
      * Feed text in. Returns `safeText` (emit as delta), `tagFound` (a full
-     * `<tool_call>` was just seen), and `cleanPrefix` (text before the tag
+     * structural tag was just seen), and `cleanPrefix` (text before the tag
      * when `tagFound` — may contain whitespace; use `.trim()` only for
      * emptiness checks, never for emission).
      */

package/dist/tool-call-buffer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tool-call-buffer.d.ts","sourceRoot":"","sources":["../src/tool-call-buffer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAiB;IAC5C,OAAO,CAAC,WAAW,CAAM;IACzB,OAAO,CAAC,WAAW,CAAS;IAE5B,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;;;;OAKG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE;IA8BhF,gDAAgD;IAChD,KAAK,IAAI,MAAM;CAKhB"}
+{"version":3,"file":"tool-call-buffer.d.ts","sourceRoot":"","sources":["../src/tool-call-buffer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAajB;IACX,OAAO,CAAC,WAAW,CAAM;IACzB,OAAO,CAAC,WAAW,CAAS;IAE5B,IAAI,UAAU,IAAI,OAAO,CAExB;IAED;;;;;OAKG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAmC/E;IAED,gDAAgD;IAChD,KAAK,IAAI,MAAM,CAId;CACF"}

package/dist/tool-call-buffer.js

@@ -1,11 +1,24 @@
 /**
- * Buffers streaming text to detect and suppress `<tool_call>` tags. Text
+ * Buffers streaming text to detect and suppress model structural tags. Text
  * that cannot be part of a partial tag is released immediately; once a
- * full tag is seen, everything after it is suppressed until the stream
- * ends.
+ * full structural tag is seen, everything after it is suppressed until
+ * the stream ends.
  */
 export class ToolCallTagBuffer {
-    static TAG = '<tool_call>';
+    static TAGS = [
+        '<tool_call>',
+        '</tool_call>',
+        '<|tool_call>',
+        '<tool_call|>',
+        '<|tool_response>',
+        '<tool_response|>',
+        '<|tool>',
+        '<tool|>',
+        '<|channel>',
+        '<channel|>',
+        '<|turn>',
+        '<turn|>',
+    ];
     pendingText = '';
     _suppressed = false;
     get suppressed() {
@@ -13,7 +26,7 @@ export class ToolCallTagBuffer {
     }
     /**
      * Feed text in. Returns `safeText` (emit as delta), `tagFound` (a full
-     * `<tool_call>` was just seen), and `cleanPrefix` (text before the tag
+     * structural tag was just seen), and `cleanPrefix` (text before the tag
      * when `tagFound` — may contain whitespace; use `.trim()` only for
      * emptiness checks, never for emission).
      */
@@ -22,7 +35,13 @@ export class ToolCallTagBuffer {
             return { safeText: '', tagFound: false, cleanPrefix: '' };
         }
         this.pendingText += text;
-        const tagIdx = this.pendingText.indexOf(ToolCallTagBuffer.TAG);
+        let tagIdx = -1;
+        for (const tag of ToolCallTagBuffer.TAGS) {
+            const idx = this.pendingText.indexOf(tag);
+            if (idx >= 0 && (tagIdx < 0 || idx < tagIdx)) {
+                tagIdx = idx;
+            }
+        }
         if (tagIdx >= 0) {
             const cleanPrefix = this.pendingText.slice(0, tagIdx);
             this._suppressed = true;
@@ -31,9 +50,10 @@ export class ToolCallTagBuffer {
         }
         // Hold back any suffix that could be the start of the tag.
         let safeLen = this.pendingText.length;
-        for (let i = 1; i <= Math.min(this.pendingText.length, ToolCallTagBuffer.TAG.length - 1); i++) {
+        const maxTagLength = Math.max(...ToolCallTagBuffer.TAGS.map((tag) => tag.length));
+        for (let i = 1; i <= Math.min(this.pendingText.length, maxTagLength - 1); i++) {
             const suffix = this.pendingText.slice(-i);
-            if (ToolCallTagBuffer.TAG.startsWith(suffix)) {
+            if (ToolCallTagBuffer.TAGS.some((tag) => tag.startsWith(suffix))) {
                 safeLen = this.pendingText.length - i;
                 break;
             }