@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/providers/mlx/python/vision_inputs.py

@@ -0,0 +1,64 @@
+"""Bounded in-memory images for MLX vision; never fetch URLs or read paths."""
+
+import base64
+import io
+from contextlib import contextmanager
+
+MAX_IMAGES = 32
+MAX_IMAGE_BYTES = 8 * 1024 * 1024
+MAX_TOTAL_BYTES = 32 * 1024 * 1024
+
+
+@contextmanager
+def isolated_position_state(model):
+    """Qwen's multimodal RoPE state is model-local, outside the KV cache."""
+    tower = getattr(model, 'language_model', model)
+    saved = {key: getattr(tower, key) for key in ('_position_ids', '_rope_deltas')
+             if hasattr(tower, key)}
+    try:
+        for key in saved: setattr(tower, key, None)
+        yield
+    finally:
+        for key, value in saved.items(): setattr(tower, key, value)
+
+
+def decode_images(messages):
+    from PIL import Image
+    encoded = [value for message in messages for value in (message.images or [])]
+    if len(encoded) > MAX_IMAGES:
+        raise ValueError(f"At most {MAX_IMAGES} images may be included in a vision request")
+    images, total = [], 0
+    try:
+        for value in encoded:
+            if not isinstance(value, str) or len(value) > (MAX_IMAGE_BYTES * 4 // 3 + 4):
+                raise ValueError("Image exceeds the 8 MiB encoded-input limit")
+            raw = base64.b64decode(value, validate=True)
+            total += len(raw)
+            if len(raw) > MAX_IMAGE_BYTES or total > MAX_TOTAL_BYTES:
+                raise ValueError("Vision request exceeds its image byte budget")
+            with Image.open(io.BytesIO(raw)) as source:
+                if source.format not in ("PNG", "JPEG", "WEBP"):
+                    raise ValueError("Vision accepts PNG, JPEG and WebP images")
+                if source.width * source.height > 20_000_000:
+                    raise ValueError("Image exceeds 20 megapixels")
+                # Bound patch expansion and transient memory before GPU input.
+                source.thumbnail((1024, 1024))
+                images.append(source.convert("RGB"))
+        return images
+    except Exception:
+        for image in images:
+            image.close()
+        raise
+
+
+def message_content(message):
+    content = message.content
+    if isinstance(content, list):
+        # Reject alternative image encodings instead of silently dropping them.
+        if any(isinstance(part, dict) and part.get("type") != "text" for part in content):
+            raise ValueError("Use the message images array for MLX image input")
+        content = "\n".join(part.get("text", "") for part in content if isinstance(part, dict))
+    if message.images:
+        return ([{"type": "image"} for _ in message.images]
+                + [{"type": "text", "text": content or ""}])
+    return content or ""

package/dist/providers/mlx/python/wave_policy.py

@@ -0,0 +1,147 @@
+"""Who the BatchEngine serves next, and when background work steps aside.
+
+The engine runs static waves: a wave is admitted, prefilled, and decoded to
+completion before the next one starts (mid-wave joins are not supported by
+the mlx_vlm caches). Two consequences used to be invisible from the outside:
+
+  - A request that arrives mid-wave waits for the whole wave — prefill AND
+    generation — while the client only sees an HTTP 200 and silence. The TS
+    watchdog then read the silence as a dead engine. Wild-caught: a person's
+    300-token question waited behind a craftbook turn re-prefilling 55k
+    tokens after a restart, was labelled "Processing prompt" for seven
+    minutes, and died at the pre-first-byte watchdog without ever starting.
+  - Nothing distinguished that person's turn from the background task it
+    was stuck behind.
+
+This module owns the decisions; the engine owns the mechanics (parking a
+wave, swapping generators, printing). Pure stdlib so the decisions are
+unit-tested without MLX — see wave_policy_test.py.
+
+Markers printed by the engine (parsed by the TS stdout-parser, routed to the
+owning session by `cache=`):
+
+  [batch] waiting cache=<id> ahead=<n> behind=<id,id|->
+      queued behind a running wave; liveness for the waiting request
+  [batch] paused cache=<id> for=<id,id|->
+      a parked background wave; liveness for its (silent) stream
+  [batch] admitted cache=<id> waited=<seconds>
+      a previously-waiting request has started
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Callable, Dict, Iterable, List, Optional, Sequence, Tuple
+
+INTERACTIVE = "interactive"
+BACKGROUND = "background"
+
+# Repeat cadence for waiting/paused liveness markers. The TS watchdogs it
+# feeds are minutes long; this only has to be comfortably inside them while
+# keeping a ten-request fanout from flooding the log.
+LIVENESS_INTERVAL_S = 8.0
+
+
+def normalize_priority(value: object) -> str:
+    """Only an explicit "background" is background.
+
+    Absent (an older daemon), misspelled, or None reads as interactive —
+    the direction that never parks a turn somebody is waiting on."""
+    if isinstance(value, str) and value.strip().lower() == BACKGROUND:
+        return BACKGROUND
+    return INTERACTIVE
+
+
+def is_background(sub: object) -> bool:
+    return getattr(sub, "priority", INTERACTIVE) == BACKGROUND
+
+
+def is_live(sub: object) -> bool:
+    return not getattr(sub, "cancelled", False)
+
+
+def select_wave(
+    pending: Sequence[object], admit_count: Callable[[int], int]
+) -> Tuple[List[object], List[object]]:
+    """Pick this wave's members from `pending` (arrival order).
+
+    When anyone interactive is waiting the wave is interactive-only. A
+    batched wave prefills every member before any of them decodes, so a
+    co-admitted background turn would put its whole prefill inside the
+    person's time-to-first-token. Returns (batch, rest); `rest` keeps
+    arrival order."""
+    candidates = [s for s in pending if not is_background(s)] or list(pending)
+    n = int(admit_count(len(candidates))) if candidates else 0
+    batch = candidates[: max(0, min(len(candidates), n))]
+    chosen = {id(s) for s in batch}
+    rest = [s for s in pending if id(s) not in chosen]
+    return batch, rest
+
+
+def preemptors(running: Iterable[object], pending: Sequence[object]) -> List[object]:
+    """Interactive requests that should run ahead of the running wave.
+
+    Only a wave made entirely of background work steps aside: an
+    interactive member means a person is already being served, and parking
+    them for another person just moves the wait around."""
+    members = list(running)
+    if not members or not all(is_background(s) for s in members):
+        return []
+    return [s for s in pending if is_live(s) and not is_background(s)]
+
+
+def cache_ids(subs: Iterable[object]) -> List[str]:
+    """Routable ids of `subs`, de-duplicated in order. A sub with no
+    cache_id cannot be addressed by the TS side, so it is left out."""
+    out: List[str] = []
+    for s in subs:
+        cid = getattr(getattr(s, "request", None), "cache_id", None)
+        if cid and cid not in out:
+            out.append(str(cid))
+    return out
+
+
+def _ids_field(ids: Sequence[str]) -> str:
+    return ",".join(ids) if ids else "-"
+
+
+def waiting_line(cache_id: str, ahead: int, behind: Sequence[str]) -> str:
+    return f"[batch] waiting cache={cache_id} ahead={int(ahead)} behind={_ids_field(behind)}"
+
+
+def paused_line(cache_id: str, for_ids: Sequence[str]) -> str:
+    return f"[batch] paused cache={cache_id} for={_ids_field(for_ids)}"
+
+
+def admitted_line(cache_id: str, waited_s: float) -> str:
+    return f"[batch] admitted cache={cache_id} waited={max(0.0, waited_s):.1f}s"
+
+
+class LivenessThrottle:
+    """Per-key emission gate: the first marker for a key goes out at once,
+    repeats at most every `interval_s`. `forget` re-arms a key so a request
+    that waits a second time announces itself immediately."""
+
+    def __init__(
+        self,
+        interval_s: float = LIVENESS_INTERVAL_S,
+        clock: Callable[[], float] = time.time,
+    ) -> None:
+        self.interval_s = float(interval_s)
+        self._clock = clock
+        self._last: Dict[str, float] = {}
+
+    def due(self, key: str) -> bool:
+        now = self._clock()
+        last = self._last.get(key)
+        if last is not None and now - last < self.interval_s:
+            return False
+        self._last[key] = now
+        return True
+
+    def seen(self, key: str) -> bool:
+        return key in self._last
+
+    def forget(self, key: Optional[str]) -> None:
+        if key:
+            self._last.pop(key, None)

package/dist/providers/mlx/reasoning-stream.d.ts

@@ -0,0 +1,85 @@
+/**
+ * How a chat template's generation prompt leaves the model inside a reasoning
+ * block:
+ *
+ *  - `always` — the opener is unconditional (DeepSeek-R1 distills).
+ *  - `unless-thinking-off` — the generation branch reads `enable_thinking`, so
+ *    the opener depends on the request. Qwen 3.x renders a bare `<think>\n`
+ *    when it is on and a closed, empty `<think>\n\n</think>\n\n` when it is off.
+ */
+export type TemplateReasoningOpen = 'always' | 'unless-thinking-off';
+/**
+ * Whether the model's chat template opens a reasoning block as the last thing
+ * it emits before the model starts generating, and whether that depends on
+ * `enable_thinking`.
+ *
+ * Qwen-family templates end their generation prompt with a bare `<think>`, so
+ * the model's output begins *inside* reasoning and the only tag that ever
+ * appears is the closing one. Reading the template makes that a per-model fact
+ * rather than something guessed from the token stream: without it, a streamed
+ * turn is ambiguous until a close marker arrives, and the choice is between
+ * leaking chain-of-thought as answer text or withholding an answer that may
+ * never have had a reasoning block at all.
+ *
+ * Unreadable or unrecognized templates return undefined — the caller then only
+ * splits on explicit paired tags, which is unambiguous.
+ */
+export declare function templateOpensReasoning(modelDir: string): TemplateReasoningOpen | undefined;
+/**
+ * Exposed for tests: the string-level rule applied to template text.
+ *
+ * Looks only at the generation-prompt tail — the branch guarded by
+ * `add_generation_prompt` — because an opener anywhere else belongs to a
+ * *replayed* assistant turn rather than the one about to be generated.
+ */
+export declare function templateTextOpensReasoning(template: string): TemplateReasoningOpen | undefined;
+/**
+ * Whether THIS request's generation starts inside a reasoning block.
+ *
+ * The template fact alone is not enough: every constrained write turn, and
+ * every turn under a tuning profile with `enableThinking: false`, sends
+ * `enable_thinking: false`. A switch-aware template then closes the block in
+ * the prompt itself, so no close marker ever streams — and a splitter seeded
+ * from the template alone routed the whole reply, `write_file` markup and all,
+ * into the reasoning pane. Absent means on, matching the engine's default.
+ */
+export declare function requestOpensReasoning(templateOpen: TemplateReasoningOpen | undefined, body: Record<string, unknown>): boolean;
+export interface ReasoningSplitChunk {
+    visible: string;
+    reasoning: string;
+}
+/**
+ * Incremental reasoning splitter for a streamed MLX turn.
+ *
+ * MLX inlines chain-of-thought in the same content deltas as the reply, and the
+ * provider only separates them once the turn is complete. Everything reading
+ * live deltas — Gezel's own bubble, and every connected app through the
+ * OpenAI-compatible facade — therefore sees thinking rendered as the answer.
+ * llama.cpp and ds4 avoid this because their engines emit a dedicated
+ * `reasoning_content` channel; this class is that channel for MLX, derived on
+ * our side.
+ *
+ * Mirrors {@link LeakyToolCallStripper}'s contract: `push` returns what is safe
+ * to emit now, holding back only a trailing fragment that could still be the
+ * start of a close marker; `flush` drains whatever is left at end of turn.
+ */
+export declare class StreamingReasoningSplit {
+    private buffer;
+    private inReasoning;
+    private readonly enabled;
+    constructor(opts: {
+        opensInReasoning: boolean;
+        enabled?: boolean;
+    });
+    push(chunk: string): ReasoningSplitChunk;
+    /**
+     * End of turn. A reasoning block the model never closed stays reasoning —
+     * promoting it to visible would publish exactly the chain-of-thought this
+     * exists to withhold.
+     */
+    flush(): ReasoningSplitChunk;
+    private drain;
+    private firstCloseMarker;
+    /** Length of the trailing fragment that could still be a partial marker. */
+    private holdBackLength;
+}

package/dist/providers/mlx/reasoning-stream.js

@@ -0,0 +1,196 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { createLogger } from '@bendyline/gezel';
+const log = createLogger('mlx');
+/**
+ * Closing markers that end a reasoning block the chat template opened for the
+ * model. Kept in sync with the paired forms `extractReasoning` strips at end of
+ * turn — this is the streaming half of the same knowledge.
+ */
+const CLOSE_MARKERS = ['</think>', '</reasoning>', '[/THINK]', '<|end|>'];
+/** The longest close marker, so `push` knows how much tail to hold back. */
+const MAX_MARKER_LENGTH = Math.max(...CLOSE_MARKERS.map((marker) => marker.length));
+/**
+ * Whether the model's chat template opens a reasoning block as the last thing
+ * it emits before the model starts generating, and whether that depends on
+ * `enable_thinking`.
+ *
+ * Qwen-family templates end their generation prompt with a bare `<think>`, so
+ * the model's output begins *inside* reasoning and the only tag that ever
+ * appears is the closing one. Reading the template makes that a per-model fact
+ * rather than something guessed from the token stream: without it, a streamed
+ * turn is ambiguous until a close marker arrives, and the choice is between
+ * leaking chain-of-thought as answer text or withholding an answer that may
+ * never have had a reasoning block at all.
+ *
+ * Unreadable or unrecognized templates return undefined — the caller then only
+ * splits on explicit paired tags, which is unambiguous.
+ */
+export function templateOpensReasoning(modelDir) {
+    let template;
+    try {
+        template = readFileSync(join(modelDir, 'chat_template.jinja'), 'utf8');
+    }
+    catch {
+        try {
+            const config = JSON.parse(readFileSync(join(modelDir, 'tokenizer_config.json'), 'utf8'));
+            if (typeof config.chat_template !== 'string')
+                return undefined;
+            template = config.chat_template;
+        }
+        catch (err) {
+            log.debug(`chat template unreadable in ${modelDir}: ${String(err)}`);
+            return undefined;
+        }
+    }
+    return templateTextOpensReasoning(template);
+}
+/**
+ * Exposed for tests: the string-level rule applied to template text.
+ *
+ * Looks only at the generation-prompt tail — the branch guarded by
+ * `add_generation_prompt` — because an opener anywhere else belongs to a
+ * *replayed* assistant turn rather than the one about to be generated.
+ */
+export function templateTextOpensReasoning(template) {
+    const generationBranch = template.lastIndexOf('add_generation_prompt');
+    if (generationBranch === -1)
+        return undefined;
+    const tail = template.slice(generationBranch);
+    // The opener must be the last literal the template emits. Anything after it
+    // (a closing tag, more prose) means the block is already balanced.
+    const opener = /<think>|<reasoning>|\[THINK\]/gi;
+    let lastOpenerEnd = -1;
+    for (let match = opener.exec(tail); match; match = opener.exec(tail)) {
+        lastOpenerEnd = match.index + match[0].length;
+    }
+    if (lastOpenerEnd === -1)
+        return undefined;
+    if (/<\/think>|<\/reasoning>|\[\/THINK\]/i.test(tail.slice(lastOpenerEnd)))
+        return undefined;
+    return /\benable_thinking\b/.test(tail) ? 'unless-thinking-off' : 'always';
+}
+/**
+ * Whether THIS request's generation starts inside a reasoning block.
+ *
+ * The template fact alone is not enough: every constrained write turn, and
+ * every turn under a tuning profile with `enableThinking: false`, sends
+ * `enable_thinking: false`. A switch-aware template then closes the block in
+ * the prompt itself, so no close marker ever streams — and a splitter seeded
+ * from the template alone routed the whole reply, `write_file` markup and all,
+ * into the reasoning pane. Absent means on, matching the engine's default.
+ */
+export function requestOpensReasoning(templateOpen, body) {
+    if (templateOpen === undefined)
+        return false;
+    if (templateOpen === 'always')
+        return true;
+    const kwargs = body.chat_template_kwargs;
+    if (!kwargs || typeof kwargs !== 'object' || Array.isArray(kwargs))
+        return true;
+    return kwargs.enable_thinking !== false;
+}
+/**
+ * Incremental reasoning splitter for a streamed MLX turn.
+ *
+ * MLX inlines chain-of-thought in the same content deltas as the reply, and the
+ * provider only separates them once the turn is complete. Everything reading
+ * live deltas — Gezel's own bubble, and every connected app through the
+ * OpenAI-compatible facade — therefore sees thinking rendered as the answer.
+ * llama.cpp and ds4 avoid this because their engines emit a dedicated
+ * `reasoning_content` channel; this class is that channel for MLX, derived on
+ * our side.
+ *
+ * Mirrors {@link LeakyToolCallStripper}'s contract: `push` returns what is safe
+ * to emit now, holding back only a trailing fragment that could still be the
+ * start of a close marker; `flush` drains whatever is left at end of turn.
+ */
+export class StreamingReasoningSplit {
+    buffer = '';
+    inReasoning;
+    enabled;
+    constructor(opts) {
+        this.enabled = opts.enabled ?? true;
+        this.inReasoning = this.enabled && opts.opensInReasoning;
+    }
+    push(chunk) {
+        if (!this.enabled)
+            return { visible: chunk, reasoning: '' };
+        this.buffer += chunk;
+        return this.drain(false);
+    }
+    /**
+     * End of turn. A reasoning block the model never closed stays reasoning —
+     * promoting it to visible would publish exactly the chain-of-thought this
+     * exists to withhold.
+     */
+    flush() {
+        if (!this.enabled)
+            return { visible: '', reasoning: '' };
+        return this.drain(true);
+    }
+    drain(final) {
+        let visible = '';
+        let reasoning = '';
+        while (this.buffer.length > 0) {
+            if (this.inReasoning) {
+                const close = this.firstCloseMarker();
+                if (!close)
+                    break;
+                reasoning += this.buffer.slice(0, close.index);
+                this.buffer = this.buffer.slice(close.index + close.marker.length);
+                this.inReasoning = false;
+                continue;
+            }
+            const open = /<think>|<reasoning>|\[THINK\]/i.exec(this.buffer);
+            if (!open)
+                break;
+            visible += this.buffer.slice(0, open.index);
+            this.buffer = this.buffer.slice(open.index + open[0].length);
+            this.inReasoning = true;
+        }
+        if (final) {
+            if (this.inReasoning)
+                reasoning += this.buffer;
+            else
+                visible += this.buffer;
+            this.buffer = '';
+            return { visible, reasoning };
+        }
+        // Hold back a tail that could still grow into a marker; release the rest.
+        const safeLength = this.buffer.length - this.holdBackLength();
+        if (safeLength > 0) {
+            const released = this.buffer.slice(0, safeLength);
+            this.buffer = this.buffer.slice(safeLength);
+            if (this.inReasoning)
+                reasoning += released;
+            else
+                visible += released;
+        }
+        return { visible, reasoning };
+    }
+    firstCloseMarker() {
+        let best = null;
+        for (const marker of CLOSE_MARKERS) {
+            const index = this.buffer.toLowerCase().indexOf(marker.toLowerCase());
+            if (index === -1)
+                continue;
+            if (!best || index < best.index)
+                best = { index, marker };
+        }
+        return best;
+    }
+    /** Length of the trailing fragment that could still be a partial marker. */
+    holdBackLength() {
+        const maxTail = Math.min(MAX_MARKER_LENGTH - 1, this.buffer.length);
+        for (let length = maxTail; length > 0; length -= 1) {
+            const tail = this.buffer.slice(this.buffer.length - length).toLowerCase();
+            const candidates = this.inReasoning
+                ? CLOSE_MARKERS.map((marker) => marker.toLowerCase())
+                : ['<think>', '<reasoning>', '[think]'];
+            if (candidates.some((marker) => marker.startsWith(tail)))
+                return length;
+        }
+        return 0;
+    }
+}

package/dist/providers/mlx/request-shape.d.ts

@@ -0,0 +1,34 @@
+/**
+ * The per-model shape of an MLX chat-completions request, shared by the send
+ * loop and the prefix-cache warm.
+ *
+ * Both must render the same `[system][tools][transcript]` prefix the engine
+ * caches: a warm whose grammar or template override differs from the real
+ * turn's saves a prefix no turn can reuse. The rules lived as two hand-kept
+ * copies, which is exactly how that drifts, so both call sites build here.
+ *
+ * - **Tuning.** Catalog manifests own sampling (`tuning.sampling` on each
+ *   chat-model identity). With none set, mlx-vlm falls back to its own
+ *   defaults, which on older versions is effectively greedy.
+ * - **Tool grammar** (`tools.mlx-grammar`). The gezel MLX server builds an
+ *   llguidance grammar from this hint plus the advertised `tools`, so a
+ *   quantized model cannot hallucinate a tool name. Family-derived because
+ *   Qwen's catalog `toolCallFormat` is the coarse `function-call`.
+ * - **Template fix** (`tools.mlx-template-fix`). Swaps the model's stored
+ *   Jinja template for a curated one at request time, no reinstall.
+ */
+import type { ResolvedTuning } from '../../model-profile/index.js';
+import type { ResolvedModelProfile } from '../../model-profile/types.js';
+/**
+ * Whether tool-bearing requests for this profile carry a decode-time tool
+ * grammar — i.e. whether the engine can confine what a call may NAME. The
+ * chat manager reads it to enforce coordinator routing clamps on what is
+ * callable instead of narrowing the rendered roster.
+ */
+export declare function mlxToolGrammarActive(profile: ResolvedModelProfile | undefined): boolean;
+export declare function applyMlxRequestShape(body: Record<string, unknown>, deps: {
+    tuning?: ResolvedTuning;
+    profile?: ResolvedModelProfile;
+}, opts: {
+    hasTools: boolean;
+}): void;

package/dist/providers/mlx/request-shape.js

@@ -0,0 +1,42 @@
+/**
+ * The per-model shape of an MLX chat-completions request, shared by the send
+ * loop and the prefix-cache warm.
+ *
+ * Both must render the same `[system][tools][transcript]` prefix the engine
+ * caches: a warm whose grammar or template override differs from the real
+ * turn's saves a prefix no turn can reuse. The rules lived as two hand-kept
+ * copies, which is exactly how that drifts, so both call sites build here.
+ *
+ * - **Tuning.** Catalog manifests own sampling (`tuning.sampling` on each
+ *   chat-model identity). With none set, mlx-vlm falls back to its own
+ *   defaults, which on older versions is effectively greedy.
+ * - **Tool grammar** (`tools.mlx-grammar`). The gezel MLX server builds an
+ *   llguidance grammar from this hint plus the advertised `tools`, so a
+ *   quantized model cannot hallucinate a tool name. Family-derived because
+ *   Qwen's catalog `toolCallFormat` is the coarse `function-call`.
+ * - **Template fix** (`tools.mlx-template-fix`). Swaps the model's stored
+ *   Jinja template for a curated one at request time, no reinstall.
+ */
+import { profileBehaviorConfig, profileHasBehavior } from '../../model-profile/runtime.js';
+import { familyToToolGrammarHint } from '../../model-profile/tool-grammar.js';
+import { MLX_TUNING_MAP, applyTuning } from '../../model-profile/tuning.js';
+/**
+ * Whether tool-bearing requests for this profile carry a decode-time tool
+ * grammar — i.e. whether the engine can confine what a call may NAME. The
+ * chat manager reads it to enforce coordinator routing clamps on what is
+ * callable instead of narrowing the rendered roster.
+ */
+export function mlxToolGrammarActive(profile) {
+    return (profileHasBehavior(profile, 'tools.mlx-grammar') &&
+        familyToToolGrammarHint(profile?.style) !== null);
+}
+export function applyMlxRequestShape(body, deps, opts) {
+    if (deps.tuning)
+        applyTuning(body, deps.tuning, MLX_TUNING_MAP);
+    if (opts.hasTools && mlxToolGrammarActive(deps.profile)) {
+        body.tool_grammar = familyToToolGrammarHint(deps.profile?.style);
+    }
+    const templateFix = profileBehaviorConfig(deps.profile, 'tools.mlx-template-fix');
+    if (templateFix?.template)
+        body.chat_template_override = templateFix.template;
+}

package/dist/providers/mlx/runtime-diagnostics.d.ts

@@ -0,0 +1,29 @@
+/** Read a process's resident-set size (bytes) via `ps` for engine telemetry. */
+export declare function readProcessRssBytes(pid: number): Promise<number | null>;
+/** Translate known mlx_vlm response shapes into user-actionable errors. */
+export declare function translateMlxHttpError(status: number, statusText: string, body: string): string;
+/** Detect an engine-side HTTP response stream closing mid-flight. */
+export declare function isMidStreamConnectionDrop(err: unknown): boolean;
+/** Format a tokens-per-second number for the in-chat status line. */
+export declare function formatTps(rate: number): string;
+/**
+ * What a person reads when a turn never started answering. Written for the
+ * person, not the engineer: no "first byte", "prefill", token counts, or
+ * server names. The engine-side facts behind each case go to the service
+ * log instead, where someone diagnosing it will look.
+ */
+export declare function buildPreFirstByteAbortMessage(lastPrefill: {
+    progress: number;
+    detail: string;
+    at: number;
+} | null, busyElsewhere?: {
+    detail: string;
+    secondsAgo: number;
+} | null): string;
+/** Build the user-facing message for an engine stream that died mid-turn. */
+export declare function buildMidStreamDropMessage(charsReceived: number, plannedStop: boolean, engineStillRunning?: boolean): string;
+/** Estimate request tokens from message bodies and serialized tool schemas. */
+export declare function estimatePromptTokens(messages: unknown, tools: unknown): number;
+export declare const PRE_FIRST_BYTE_BASE_MS = 300000;
+/** Scale the pre-first-byte watchdog for slow, large-context MLX prefills. */
+export declare function computePreFirstByteBudgetMs(approxPromptTokens: number): number;