@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/providers/mlx/tool-call-stripper.js

@@ -0,0 +1,153 @@
+/**
+ * Streaming-safe filter for chat-template tool-call markers that leak
+ * into mlx_vlm.server's content stream.
+ *
+ * Gemma 3+ tokenizers emit tool calls with literal special-token text
+ * when the chat template doesn't fully consume them. Two open variants
+ * have been observed in the wild:
+ *
+ *   `<|tool_call|>` — canonical, what the chat template SHOULD emit
+ *   `<|tool_call>`  — short form, missing the trailing `|>`; appears on
+ *                     tighter quants of Gemma 4 E4B
+ *
+ * Close has four observed variants: `<tool_call|>`, `</tool_call|>`,
+ * `<tool_call>`, `</tool_call>`. We accept all four.
+ *
+ * mlx_vlm.server still ships a structured `tool_calls` delta when the
+ * markers parse cleanly, but ALSO leaks the raw text. The structured
+ * call is what we want; the text is noise. Stripped bodies are kept on
+ * the side so a second pass can attempt repair (`parseGemmaToolCall`)
+ * before falling back to a user-visible warning.
+ *
+ * Streaming safety: deltas can split markers across chunk boundaries,
+ * so we hold back any tail that could be a prefix of any open variant
+ * until enough additional bytes arrive to disambiguate. End-of-turn
+ * `flush()` drops anything still inside an unclosed marker (treats it
+ * as truncated noise).
+ */
+const OPEN_VARIANTS = ['<|tool_call|>', '<|tool_call>'];
+const LONGEST_OPEN = OPEN_VARIANTS.reduce((a, b) => (a.length >= b.length ? a : b));
+const CLOSE_RE = /<\/?tool_call\|?>/;
+export class LeakyToolCallStripper {
+    buffer = '';
+    inside = false;
+    currentBody = '';
+    bodies = [];
+    /**
+     * Indices into {@link bodies} that came from an EOS-flush (no
+     * `</tool_call>` close marker arrived before the stream ended).
+     * Layer 3 reads this to decide whether a successfully-parsed Gemma
+     * marker call should also be tagged as truncated — its `content`
+     * arg may be partial, in which case the auto-continuation hint
+     * should fire on the tool result.
+     */
+    eosFlushedIndices = new Set();
+    /**
+     * Push a streamed chunk and return whatever's safe to emit. Holds
+     * back any trailing fragment that could be the start of a marker
+     * until enough additional bytes arrive to disambiguate.
+     */
+    push(chunk) {
+        this.buffer += chunk;
+        return this.drain();
+    }
+    /**
+     * Final drain at end-of-turn. If we're still mid-marker the model
+     * never finished closing it — drop the buffered content as noise
+     * rather than leak a half-marker.
+     */
+    flush() {
+        if (this.inside) {
+            // Unclosed marker — its body still goes into the bodies list so
+            // the parser/warning layer above can act on the model's intent
+            // even though the close tag never arrived. Tag the index as
+            // EOS-flushed so the provider knows to wire continuation-hint
+            // behavior (Layer 3).
+            if (this.currentBody.length > 0 || this.buffer.length > 0) {
+                const idx = this.bodies.length;
+                this.bodies.push((this.currentBody + this.buffer).trim());
+                this.eosFlushedIndices.add(idx);
+            }
+            this.buffer = '';
+            this.currentBody = '';
+            this.inside = false;
+            return '';
+        }
+        const out = this.buffer;
+        this.buffer = '';
+        return out;
+    }
+    /**
+     * Bodies of all complete (or unclosed-at-flush) marker spans seen
+     * this stream. Surface area for `parseGemmaToolCall`.
+     */
+    getStrippedBodies() {
+        return this.bodies;
+    }
+    /**
+     * Indices into {@link getStrippedBodies}() that came from an
+     * EOS-flush — i.e. the close marker never arrived. Used by the
+     * provider's Layer 3 truncation-salvage logic to tag those parsed
+     * calls with the continuation-hint behavior even when they parsed
+     * successfully (since their string args may be partial).
+     */
+    getEosFlushedBodyIndices() {
+        return this.eosFlushedIndices;
+    }
+    drain() {
+        let emitted = '';
+        while (this.buffer.length > 0) {
+            if (this.inside) {
+                const closeMatch = this.buffer.match(CLOSE_RE);
+                if (closeMatch?.index !== undefined) {
+                    this.currentBody += this.buffer.slice(0, closeMatch.index);
+                    this.bodies.push(this.currentBody.trim());
+                    this.currentBody = '';
+                    this.buffer = this.buffer.slice(closeMatch.index + closeMatch[0].length);
+                    this.inside = false;
+                    continue;
+                }
+                // Close not seen yet — capture what we have, hold the rest.
+                // We need to keep enough trailing bytes that a partial close
+                // marker isn't accidentally split across capture and buffer.
+                // The longest close pattern is `</tool_call|>` at 13 chars;
+                // hold back that much from the end.
+                const safeBodyEnd = Math.max(0, this.buffer.length - 13);
+                this.currentBody += this.buffer.slice(0, safeBodyEnd);
+                this.buffer = this.buffer.slice(safeBodyEnd);
+                return emitted;
+            }
+            // Not inside — find earliest open across all variants.
+            let openIdx = -1;
+            let openLen = 0;
+            for (const variant of OPEN_VARIANTS) {
+                const idx = this.buffer.indexOf(variant);
+                if (idx >= 0 && (openIdx < 0 || idx < openIdx)) {
+                    openIdx = idx;
+                    openLen = variant.length;
+                }
+            }
+            if (openIdx >= 0) {
+                emitted += this.buffer.slice(0, openIdx);
+                this.buffer = this.buffer.slice(openIdx + openLen);
+                this.inside = true;
+                continue;
+            }
+            // No complete open marker. Find the earliest position whose
+            // suffix is a prefix of any open variant — that's our holdback.
+            const startScan = Math.max(0, this.buffer.length - LONGEST_OPEN.length);
+            let holdback = this.buffer.length;
+            for (let i = startScan; i < this.buffer.length; i++) {
+                const tail = this.buffer.slice(i);
+                if (OPEN_VARIANTS.some((v) => v.startsWith(tail))) {
+                    holdback = i;
+                    break;
+                }
+            }
+            emitted += this.buffer.slice(0, holdback);
+            this.buffer = this.buffer.slice(holdback);
+            return emitted;
+        }
+        return emitted;
+    }
+}

package/dist/providers/mlx/tool-image-retention.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Tool-returned images ride one request after the tool that returned them.
+ *
+ * `preview_document` hands back slide renders as a vision message, and the
+ * session kept it in history. Every later request resent all of it, and a
+ * request carrying images disables the prefix cache, so each remaining turn
+ * of a powerpoint-deck evaluate step re-encoded six slide images and
+ * re-prefilled ~33K text tokens: about three minutes a turn (qwen3.8-27b,
+ * 2026-09-23). Once the model has answered after seeing them, later requests
+ * carry a note instead of the pixels. A model that needs to look again calls
+ * the tool again.
+ *
+ * Only the runtime's own tool-image message is retired. A picture the user
+ * attached stays, because a follow-up question about it is ordinary chat.
+ */
+export declare const TOOL_IMAGES_MESSAGE = "Images returned by the preceding tools. Inspect the pixels before judging them.";
+interface RetainableMessage {
+    role: string;
+    content?: unknown;
+    images?: string[];
+}
+export declare function retireInspectedToolImages<T extends RetainableMessage>(messages: T[]): T[];
+export {};

package/dist/providers/mlx/tool-image-retention.js

@@ -0,0 +1,43 @@
+/**
+ * Tool-returned images ride one request after the tool that returned them.
+ *
+ * `preview_document` hands back slide renders as a vision message, and the
+ * session kept it in history. Every later request resent all of it, and a
+ * request carrying images disables the prefix cache, so each remaining turn
+ * of a powerpoint-deck evaluate step re-encoded six slide images and
+ * re-prefilled ~33K text tokens: about three minutes a turn (qwen3.8-27b,
+ * 2026-09-23). Once the model has answered after seeing them, later requests
+ * carry a note instead of the pixels. A model that needs to look again calls
+ * the tool again.
+ *
+ * Only the runtime's own tool-image message is retired. A picture the user
+ * attached stays, because a follow-up question about it is ordinary chat.
+ */
+export const TOOL_IMAGES_MESSAGE = 'Images returned by the preceding tools. Inspect the pixels before judging them.';
+export function retireInspectedToolImages(messages) {
+    let lastAssistant = -1;
+    for (let i = messages.length - 1; i >= 0; i--) {
+        if (messages[i]?.role === 'assistant') {
+            lastAssistant = i;
+            break;
+        }
+    }
+    if (lastAssistant < 0)
+        return messages;
+    let changed = false;
+    const out = messages.map((message, index) => {
+        if (index > lastAssistant ||
+            message.role !== 'user' ||
+            !message.images?.length ||
+            message.content !== TOOL_IMAGES_MESSAGE) {
+            return message;
+        }
+        changed = true;
+        const { images, ...rest } = message;
+        return {
+            ...rest,
+            content: `${images.length} image(s) returned by an earlier tool call were inspected in a previous turn and are no longer attached. Call that tool again to look at them again.`,
+        };
+    });
+    return changed ? out : messages;
+}

package/dist/providers/mlx/turn-telemetry.d.ts

@@ -0,0 +1,22 @@
+import type { TurnStatsEvent } from '../streaming-session.js';
+import type { TurnUsage } from '../types.js';
+export interface MlxTurnUsageSnapshot {
+    prompt_tokens: number;
+    completion_tokens: number;
+    prompt_tps?: number;
+    generation_tps?: number;
+    cached_tokens?: number;
+}
+/** Build the two terminal events from the final streamed usage snapshot. */
+export declare function buildMlxTerminalTelemetry(args: {
+    lastUsage: MlxTurnUsageSnapshot | null;
+    model: string;
+    start: number;
+    lastIterationStartedAt: number;
+    lastIterationFirstTokenAt: number | null;
+    lastIterationFinishedAt: number | null;
+    firstTokenAt: number | null;
+}): {
+    usage: TurnUsage;
+    stats: TurnStatsEvent;
+} | null;

package/dist/providers/mlx/turn-telemetry.js

@@ -0,0 +1,45 @@
+/** Build the two terminal events from the final streamed usage snapshot. */
+export function buildMlxTerminalTelemetry(args) {
+    const { lastUsage, model, start, lastIterationStartedAt, lastIterationFirstTokenAt, lastIterationFinishedAt, firstTokenAt, } = args;
+    if (!lastUsage || (lastUsage.prompt_tokens <= 0 && lastUsage.completion_tokens <= 0))
+        return null;
+    const finishedAt = Date.now();
+    const durationMs = finishedAt - start;
+    const generationStartedAt = lastIterationFirstTokenAt ?? lastIterationStartedAt;
+    const generationFinishedAt = lastIterationFinishedAt ?? finishedAt;
+    const generationMs = Math.max(1, generationFinishedAt - generationStartedAt);
+    const wallTps = lastUsage.completion_tokens > 0
+        ? lastUsage.completion_tokens / (generationMs / 1000)
+        : undefined;
+    const tokensPerSec = lastUsage.generation_tps !== undefined && lastUsage.generation_tps > 0
+        ? lastUsage.generation_tps
+        : wallTps;
+    const at = new Date(finishedAt).toISOString();
+    return {
+        usage: {
+            model,
+            inputTokens: lastUsage.prompt_tokens,
+            outputTokens: lastUsage.completion_tokens,
+            ...(lastUsage.cached_tokens !== undefined
+                ? { cachedInputTokens: lastUsage.cached_tokens }
+                : {}),
+            ...(tokensPerSec !== undefined ? { outputTokensPerSec: tokensPerSec } : {}),
+            durationMs,
+            at,
+        },
+        stats: {
+            provider: 'mlx',
+            promptTokens: lastUsage.prompt_tokens,
+            completionTokens: lastUsage.completion_tokens,
+            durationMs,
+            ...(firstTokenAt !== null ? { ttftMs: Math.max(0, firstTokenAt - start) } : {}),
+            ...(lastUsage.prompt_tps !== undefined && lastUsage.prompt_tps > 0
+                ? { promptTokensPerSec: lastUsage.prompt_tps }
+                : {}),
+            ...(lastUsage.cached_tokens !== undefined
+                ? { cachedPromptTokens: lastUsage.cached_tokens }
+                : {}),
+            ...(tokensPerSec !== undefined ? { tokensPerSec } : {}),
+        },
+    };
+}

package/dist/providers/mlx/venv.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Single source of truth for the MLX Python venv spec — the venv name
+ * and the package list `UvRuntime.ensureVenv` provisions.
+ *
+ * Two code paths provision this venv and they MUST request the
+ * identical (name, packages):
+ *
+ *   1. The lazy first-chat path — `buildMlxProvider` awaits
+ *      `ensureVenv` before spawning the engine.
+ *   2. The parallel warm kicked off by `MlxModelManager` at install time,
+ *      which
+ *      overlaps the multi-minute uv/torch install with the model-
+ *      weight download instead of paying it serially on the first turn.
+ *
+ * `ensureVenv` keys its idempotent fast-path on the (sorted) package
+ * set, so any divergence between the two callers makes the second one
+ * reinstall from scratch instead of hitting the warmed venv — which
+ * defeats the entire point of warming. Funnel both through here.
+ *
+ * `torch` + `torchvision` are pinned on by default even though we never
+ * compute in torch — `mlx-vlm` does inference in MLX directly. But many
+ * catalog models (Qwen 3.5/3.6, future Gemma vision variants) ship a
+ * `video_preprocessor_config.json`, which drives
+ * `transformers.AutoProcessor.from_pretrained` to construct the model's
+ * `*VideoProcessor` class — and those classes have hard
+ * `requires_backends('torch', 'torchvision')` guards even when no video
+ * is ever fed in. Without these two packages `mlx_vlm.utils.load()`
+ * dies with `ImportError: <Class> requires the PyTorch library...`
+ * before the engine reports ready. The Apple Silicon wheels are small
+ * (~500 MB combined), so paying the disk cost universally is cheaper
+ * than per-model gating.
+ */
+/** Default pin for the MLX backend. Advanced users override via
+ * `config.mlxPackageSpec` (Settings → This Mac → Advanced).
+ *
+ * PINNED, not a floor. A floating `>=` spec previously pulled mlx-vlm 0.6.4
+ * onto fresh machines and exposed a byte-fallback detokenizer regression in
+ * ordinary Qwen output, killing the engine mid-turn and breaking tool calls.
+ * The 0.6.6 line that replaced it carried the Laguna text architecture and was
+ * compatibility-checked against Gezel's wrapper imports, Laguna tool
+ * templating, Qwen byte-BPE streaming, and a real Gemma 4 generation.
+ *
+ * 0.6.14 was TRIED and REVERTED on 2026-08-18. The theory was that its
+ * rewritten `BatchKVCache` would restore prompt-cache reuse — MLX was
+ * re-prefilling every turn (23.5% hit rate, median 0 cached tokens). That
+ * theory was wrong twice over: 0.6.6's `mlx_vlm.models.cache.BatchKVCache`
+ * already implements `is_trimmable()`/`trim()` (an earlier check misread the
+ * `mlx_lm` copy), and on 0.6.14 the cache did not improve — 20.8% hit rate,
+ * still median 0 cached. Meanwhile a paired A/B on qwen3.8-27b-q4 measured a
+ * large slowdown at unchanged 10/10 task success:
+ *
+ *     tictactoe   8.4 min -> 15.9 min      tankcombat  12.7 min -> 23.3 min
+ *     probe throughput 55.8 -> 40.7 gen tok/s
+ *
+ * The re-prefill cause that bump chased was later found and fixed
+ * gezel-side (roster-shaped cache identity + the §12b snapshot machinery
+ * — docs/kv-prompt-caching-strategy.md).
+ *
+ * 0.6.17 (2026-08-28): bumped for MTP speculative decoding, and the bump
+ * is correctness-forced — 0.6.6's MTP verify is measured-INEXACT (greedy
+ * spec diverged from greedy no-spec; reports/mlx-mtp-rig-20260828.md),
+ * and spec_decode.py refuses to arm speculation below 0.6.17. The line
+ * also carries 0.6.15's batch-row isolation fix and 0.6.16's ArraysCache
+ * per-token Metal buffer leak fix (measured: that leak fix does NOT cure
+ * the vlm-tower decode slope under mlx_lm's BatchGenerator — 1.01 → 0.83
+ * ms/tok per 10k ctx — so the text-tower split stays). Exactness and the
+ * ~1.4x speedup are verified on-device for this list's resolved set
+ * (mlx-lm 0.31.3 with mlx 0.32.0; the rig also verified mlx 0.32.2).
+ *
+ * 0.7.1 (2026-09-14): bumped for Qwen3.8 Flash Next (`qwen4_exp`). Its
+ * Qwen4 implementation carries the sparse-attention cache/batch fixes and
+ * the Qwen3.8 Conv3d layout correction that the checkpoint needs. Gezel's
+ * exact wrapper combination was verified on an M5 Max with mlx-lm 0.31.3
+ * and the resolved mlx 0.32.2: real sidecar generation + prompt-cache reuse,
+ * plus a two-row Qwen4-Exp continuous-batch prefill/decode smoke. The full
+ * 112 GB Flash checkpoint is separately catalog-qualified through 0.7.1's
+ * external-PLE view: its 32,000,153,600-byte n-gram table stays disk-backed
+ * instead of consuming the Metal working set. Do not treat the library pin
+ * alone as proof that every community conversion is correct or fits.
+ *
+ * Keep upgrades explicit and re-sweep the MLX catalog whenever this pin
+ * moves. */
+export declare const MLX_DEFAULT_PACKAGE_SPEC = "mlx-vlm==0.7.1";
+/** Venv name passed to `UvRuntime.ensureVenv` (and its dir basename). */
+export declare const MLX_VENV_NAME = "mlx";
+/**
+ * The package list to install into the MLX venv. `packageSpec` is the
+ * user's optional `config.mlxPackageSpec` override; when unset we pin
+ * {@link MLX_DEFAULT_PACKAGE_SPEC}.
+ */
+export declare function mlxVenvPackages(packageSpec?: string): string[];

package/dist/providers/mlx/venv.js

@@ -0,0 +1,111 @@
+/**
+ * Single source of truth for the MLX Python venv spec — the venv name
+ * and the package list `UvRuntime.ensureVenv` provisions.
+ *
+ * Two code paths provision this venv and they MUST request the
+ * identical (name, packages):
+ *
+ *   1. The lazy first-chat path — `buildMlxProvider` awaits
+ *      `ensureVenv` before spawning the engine.
+ *   2. The parallel warm kicked off by `MlxModelManager` at install time,
+ *      which
+ *      overlaps the multi-minute uv/torch install with the model-
+ *      weight download instead of paying it serially on the first turn.
+ *
+ * `ensureVenv` keys its idempotent fast-path on the (sorted) package
+ * set, so any divergence between the two callers makes the second one
+ * reinstall from scratch instead of hitting the warmed venv — which
+ * defeats the entire point of warming. Funnel both through here.
+ *
+ * `torch` + `torchvision` are pinned on by default even though we never
+ * compute in torch — `mlx-vlm` does inference in MLX directly. But many
+ * catalog models (Qwen 3.5/3.6, future Gemma vision variants) ship a
+ * `video_preprocessor_config.json`, which drives
+ * `transformers.AutoProcessor.from_pretrained` to construct the model's
+ * `*VideoProcessor` class — and those classes have hard
+ * `requires_backends('torch', 'torchvision')` guards even when no video
+ * is ever fed in. Without these two packages `mlx_vlm.utils.load()`
+ * dies with `ImportError: <Class> requires the PyTorch library...`
+ * before the engine reports ready. The Apple Silicon wheels are small
+ * (~500 MB combined), so paying the disk cost universally is cheaper
+ * than per-model gating.
+ */
+/** Default pin for the MLX backend. Advanced users override via
+ * `config.mlxPackageSpec` (Settings → This Mac → Advanced).
+ *
+ * PINNED, not a floor. A floating `>=` spec previously pulled mlx-vlm 0.6.4
+ * onto fresh machines and exposed a byte-fallback detokenizer regression in
+ * ordinary Qwen output, killing the engine mid-turn and breaking tool calls.
+ * The 0.6.6 line that replaced it carried the Laguna text architecture and was
+ * compatibility-checked against Gezel's wrapper imports, Laguna tool
+ * templating, Qwen byte-BPE streaming, and a real Gemma 4 generation.
+ *
+ * 0.6.14 was TRIED and REVERTED on 2026-08-18. The theory was that its
+ * rewritten `BatchKVCache` would restore prompt-cache reuse — MLX was
+ * re-prefilling every turn (23.5% hit rate, median 0 cached tokens). That
+ * theory was wrong twice over: 0.6.6's `mlx_vlm.models.cache.BatchKVCache`
+ * already implements `is_trimmable()`/`trim()` (an earlier check misread the
+ * `mlx_lm` copy), and on 0.6.14 the cache did not improve — 20.8% hit rate,
+ * still median 0 cached. Meanwhile a paired A/B on qwen3.8-27b-q4 measured a
+ * large slowdown at unchanged 10/10 task success:
+ *
+ *     tictactoe   8.4 min -> 15.9 min      tankcombat  12.7 min -> 23.3 min
+ *     probe throughput 55.8 -> 40.7 gen tok/s
+ *
+ * The re-prefill cause that bump chased was later found and fixed
+ * gezel-side (roster-shaped cache identity + the §12b snapshot machinery
+ * — docs/kv-prompt-caching-strategy.md).
+ *
+ * 0.6.17 (2026-08-28): bumped for MTP speculative decoding, and the bump
+ * is correctness-forced — 0.6.6's MTP verify is measured-INEXACT (greedy
+ * spec diverged from greedy no-spec; reports/mlx-mtp-rig-20260828.md),
+ * and spec_decode.py refuses to arm speculation below 0.6.17. The line
+ * also carries 0.6.15's batch-row isolation fix and 0.6.16's ArraysCache
+ * per-token Metal buffer leak fix (measured: that leak fix does NOT cure
+ * the vlm-tower decode slope under mlx_lm's BatchGenerator — 1.01 → 0.83
+ * ms/tok per 10k ctx — so the text-tower split stays). Exactness and the
+ * ~1.4x speedup are verified on-device for this list's resolved set
+ * (mlx-lm 0.31.3 with mlx 0.32.0; the rig also verified mlx 0.32.2).
+ *
+ * 0.7.1 (2026-09-14): bumped for Qwen3.8 Flash Next (`qwen4_exp`). Its
+ * Qwen4 implementation carries the sparse-attention cache/batch fixes and
+ * the Qwen3.8 Conv3d layout correction that the checkpoint needs. Gezel's
+ * exact wrapper combination was verified on an M5 Max with mlx-lm 0.31.3
+ * and the resolved mlx 0.32.2: real sidecar generation + prompt-cache reuse,
+ * plus a two-row Qwen4-Exp continuous-batch prefill/decode smoke. The full
+ * 112 GB Flash checkpoint is separately catalog-qualified through 0.7.1's
+ * external-PLE view: its 32,000,153,600-byte n-gram table stays disk-backed
+ * instead of consuming the Metal working set. Do not treat the library pin
+ * alone as proof that every community conversion is correct or fits.
+ *
+ * Keep upgrades explicit and re-sweep the MLX catalog whenever this pin
+ * moves. */
+export const MLX_DEFAULT_PACKAGE_SPEC = 'mlx-vlm==0.7.1';
+/** Venv name passed to `UvRuntime.ensureVenv` (and its dir basename). */
+export const MLX_VENV_NAME = 'mlx';
+/**
+ * The package list to install into the MLX venv. `packageSpec` is the
+ * user's optional `config.mlxPackageSpec` override; when unset we pin
+ * {@link MLX_DEFAULT_PACKAGE_SPEC}.
+ */
+export function mlxVenvPackages(packageSpec) {
+    // `llguidance` backs decode-time tool-call grammars (see
+    // providers/mlx/python/tool_grammar.py). It ships today as a transitive
+    // mlx-vlm dependency, but pin it explicitly so the grammar safety net
+    // can't silently vanish under a future mlx-vlm that drops it.
+    //
+    // `mlx-lm` is load-bearing twice over — the sidecar's batch engine
+    // (`from mlx_lm.generate import BatchGenerator`) and the qwen3_5 text
+    // tower both live there — and modern mlx-vlm no longer declares it as
+    // a dependency (0.6.6 did, which is the only reason the old list
+    // worked). Dropping this pin kills the batch engine at import time.
+    // 0.31.3 is also the combination used for the 0.7.1 Qwen4-Exp batch
+    // compatibility check above; keep it independent of mlx-vlm's deps.
+    return [
+        packageSpec ?? MLX_DEFAULT_PACKAGE_SPEC,
+        'mlx-lm==0.31.3',
+        'torch',
+        'torchvision',
+        'llguidance',
+    ];
+}

package/dist/providers/mlx/vision-mode.d.ts

@@ -0,0 +1,92 @@
+import type { NativeVisionPreference } from '../vision-capability.js';
+import type { MlxEngineGate } from './engine-gate.js';
+/**
+ * When an MLX engine loads its vision tower.
+ *
+ * `--vision` makes the sidecar load mlx_vlm's full multimodal tower, and for
+ * the architectures that otherwise get mlx_lm's text tower that tax lands on
+ * every text turn: 32 -> 45 ms/token at 11k context and 41 -> 87 at 54k on
+ * qwen3.8-27b (the `[tower]` note in gezel_mlx_server.py). So a checkpoint
+ * that has a vision tower starts text-only (`on-demand`) and reloads once, the
+ * first time a request carries pixels, then stays in vision mode for the rest
+ * of that engine process. An explicit per-model `config.nativeVision` wins:
+ * `true` is `always`, `false` is `never`.
+ */
+export type MlxVisionPolicy = 'always' | 'on-demand' | 'never';
+export declare function resolveMlxVisionPolicy(opts: {
+    preference: NativeVisionPreference;
+    hasVisionTower: boolean;
+}): MlxVisionPolicy;
+/**
+ * Disk-cache segment for one launch. KV persisted by mlx_lm's text tower and
+ * by mlx_vlm's tower is not interchangeable — the saved cache classes come
+ * from different packages, and the vlm tower keeps multimodal position state
+ * the text tower never wrote — so the two must never restore each other's
+ * entries. The text segment keeps the historical fingerprint: every release
+ * before on-demand vision launched text-only, and its caches stay valid.
+ */
+export declare function mlxCacheFingerprint(modelFingerprint: string, vision: boolean): string;
+interface VisionMessage {
+    role: string;
+    content?: unknown;
+    images?: string[];
+}
+/** Why this request needs the vision tower, or `undefined` when it carries no pixels. */
+export declare function describeVisionNeed(messages: readonly VisionMessage[], turnStartIdx: number): string | undefined;
+/**
+ * Suffix for a successful tool result whose images a text-only engine cannot
+ * be shown. The tool's real work — a generated image on disk, a rendered
+ * poster — already happened; reporting it as a failure made the model run
+ * the GPU-heavy generation again until the failure tracker aborted the turn.
+ */
+export declare function unseenToolImagesNote(count: number): string;
+export declare function withoutImages<T extends VisionMessage>(messages: T[]): T[];
+interface VisionSupervisor {
+    ensureRunning(): Promise<unknown>;
+    stop(): Promise<void>;
+    lifecycleSnapshot(): {
+        running: boolean;
+    };
+}
+/**
+ * One MLX engine's tower, decided per launch. The supervisor calls
+ * {@link takeLaunch} from `resolveLaunch`; sessions call {@link prepareRequest}
+ * before every request. A switch holds every engine-gate slot, so it waits for
+ * in-flight requests of other sessions instead of killing them, then reloads
+ * through the supervisor's own stop/start — the pool entry, its reservation
+ * and its idle accounting never change.
+ */
+export declare class MlxVisionMode {
+    readonly policy: MlxVisionPolicy;
+    private readonly label;
+    private pendingReason;
+    private launchedWithVision;
+    private unavailableReason;
+    private supervisor;
+    private gate;
+    private isDisposed;
+    private switching;
+    constructor(policy: MlxVisionPolicy, label?: string);
+    /** A tower fixed at construction: an external server, or a test double. */
+    static fixed(visionEnabled: boolean): MlxVisionMode;
+    bindEngine(supervisor: VisionSupervisor | undefined, gate: Pick<MlxEngineGate, 'acquireAll'>, isDisposed: () => boolean): void;
+    /** Whether images can reach the model — now, or after one reload into the vision tower. */
+    get capable(): boolean;
+    /** Whether the current (or most recent) engine process loaded the vision tower. */
+    get active(): boolean;
+    /**
+     * Decide this launch's tower. Called once per engine start; a pending
+     * request is consumed, so the process after this one starts text-only
+     * again unless something asks for vision.
+     */
+    takeLaunch(): boolean;
+    /**
+     * The request's messages, bound for an engine that can read them. When no
+     * vision tower can be loaded the pixels are replaced with an honest note,
+     * because the sidecar rejects image requests on a text-only launch.
+     */
+    prepareRequest<T extends VisionMessage>(messages: T[], turnStartIdx: number, signal?: AbortSignal): Promise<T[]>;
+    ensureVision(reason: string, signal?: AbortSignal): Promise<boolean>;
+    private switchToVision;
+}
+export {};