@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/mlx/runtime-diagnostics.js

@@ -0,0 +1,162 @@
+import { createLogger } from '@bendyline/gezel';
+const log = createLogger('mlx');
+/** Read a process's resident-set size (bytes) via `ps` for engine telemetry. */
+export async function readProcessRssBytes(pid) {
+    const { spawn } = await import('node:child_process');
+    return await new Promise((resolve) => {
+        try {
+            const proc = spawn('ps', ['-o', 'rss=', '-p', String(pid)], {
+                stdio: ['ignore', 'pipe', 'ignore'],
+            });
+            let stdout = '';
+            proc.stdout.on('data', (chunk) => {
+                stdout += chunk.toString('utf8');
+            });
+            proc.on('error', () => resolve(null));
+            proc.on('close', (code) => {
+                if (code !== 0)
+                    return resolve(null);
+                const kb = Number.parseInt(stdout.trim(), 10);
+                if (!Number.isFinite(kb) || kb <= 0)
+                    return resolve(null);
+                resolve(kb * 1024);
+            });
+        }
+        catch {
+            resolve(null);
+        }
+    });
+}
+/** Translate known mlx_vlm response shapes into user-actionable errors. */
+export function translateMlxHttpError(status, statusText, body) {
+    if (/Repository Not Found for url/i.test(body) || /401 Client Error/i.test(body)) {
+        return ("[Mac AI] The engine couldn't load this model. Try, in order:\n" +
+            '  1. Settings → This Mac → Delete the local model, then download it again (in case the on-disk files are incomplete).\n' +
+            '  2. Settings → This Mac → Advanced → Reset venv (in case mlx-vlm is too old for this architecture).\n' +
+            '  3. Restart gezel and retry.');
+    }
+    if (/Received \d+ parameters not in model/i.test(body)) {
+        return ("[Mac AI] The on-device runtime doesn't recognize this model's architecture.\n" +
+            'Settings → This Mac → Advanced → Reset venv, then retry. If it still fails, the catalog entry may need a newer mlx-vlm.');
+    }
+    if (/out of memory|mps backend|allocation failed/i.test(body)) {
+        return ('[Mac AI] Not enough unified memory to load this model.\n' +
+            'Try the E2B variant, close other memory-heavy apps, or restart this Mac to release cached memory.');
+    }
+    if (/FileNotFoundError|No such file or directory/i.test(body)) {
+        const filename = body.match(/['"]?([^'"\s]+\.(?:json|safetensors|jinja|model))['"]?/)?.[1];
+        return `[Mac AI] The local model is missing a file the engine needs${filename ? ` (\`${filename}\`)` : ''}.\nSettings → This Mac → Delete the local model, then download it again.`;
+    }
+    const detail = tryParseJsonDetail(body) ?? body.slice(0, 200);
+    return `[Mac AI] engine returned ${status} ${statusText}: ${detail}`;
+}
+/** Detect an engine-side HTTP response stream closing mid-flight. */
+export function isMidStreamConnectionDrop(err) {
+    if (!(err instanceof Error))
+        return false;
+    if (err.name === 'AbortError')
+        return false;
+    const haystacks = [err.message];
+    const cause = err.cause;
+    if (cause instanceof Error)
+        haystacks.push(cause.message);
+    const causeCode = cause && typeof cause === 'object' && 'code' in cause
+        ? String(cause.code)
+        : '';
+    if (causeCode)
+        haystacks.push(causeCode);
+    const blob = haystacks.join(' ').toLowerCase();
+    return (/\bterminated\b/.test(blob) ||
+        blob.includes('other side closed') ||
+        blob.includes('premature close') ||
+        blob.includes('und_err_socket') ||
+        blob.includes('econnreset'));
+}
+/** Format a tokens-per-second number for the in-chat status line. */
+export function formatTps(rate) {
+    if (rate >= 10)
+        return rate.toFixed(0);
+    return rate.toFixed(1);
+}
+/**
+ * What a person reads when a turn never started answering. Written for the
+ * person, not the engineer: no "first byte", "prefill", token counts, or
+ * server names. The engine-side facts behind each case go to the service
+ * log instead, where someone diagnosing it will look.
+ */
+export function buildPreFirstByteAbortMessage(lastPrefill, busyElsewhere) {
+    // Checked first, and only when THIS request saw no prefill of its own:
+    // an engine that was demonstrably prefilling someone else's turn is
+    // neither loading nor unhealthy, and saying so sends the reader off to
+    // restart a perfectly good engine. Wild-caught on a restart that resumed
+    // four sessions at once — one turn waited 10m36s behind a neighbour's
+    // 124s prefill and was told the model might be sick.
+    if (!lastPrefill && busyElsewhere) {
+        const what = busyElsewhere.detail ? ` (${busyElsewhere.detail})` : '';
+        log.warn(`pre-first-byte abort: waited behind another session's prefill${what} last seen ${busyElsewhere.secondsAgo}s ago`);
+        return '[Mac AI] This message was waiting for the AI on your Mac to finish another chat, and it never got its turn. Nothing is wrong with the AI — send the message again. When several gezels are working at once, they take turns.';
+    }
+    if (lastPrefill && lastPrefill.progress > 0) {
+        const pct = Math.round(lastPrefill.progress * 100);
+        log.warn(`pre-first-byte abort: own prefill stalled at ${pct}% (${lastPrefill.detail || '-'})`);
+        return `[Mac AI] The AI on your Mac stopped partway through reading this conversation (${pct}% done). It may be too long for this model to get through in time. Try again, start a new chat, or switch to a faster model.`;
+    }
+    if (lastPrefill?.detail) {
+        log.warn(`pre-first-byte abort: still prefilling ${lastPrefill.detail} at the deadline`);
+        return '[Mac AI] The AI on your Mac was still reading this conversation when it ran out of time. Try again — the second try is quicker, because it keeps what it has already read — or switch to a faster model.';
+    }
+    log.warn('pre-first-byte abort: no engine activity observed for this request');
+    return "[Mac AI] The AI on your Mac didn't start answering. It may still be loading the model. Try again in a moment; if it keeps happening, quit and reopen Gezel.";
+}
+/** Build the user-facing message for an engine stream that died mid-turn. */
+export function buildMidStreamDropMessage(charsReceived, plannedStop, engineStillRunning) {
+    const got = charsReceived > 0 ? `after ${charsReceived} chars` : 'before any output';
+    if (plannedStop) {
+        return `[Mac AI] this turn stopped ${got} because Gezel unloaded the on-device engine while it was answering. Changing your settings restarts chat sessions so the new settings apply, and unloading a model in Settings → On-device or quitting the app does the same. The model didn't crash — send the message again to redo this turn.`;
+    }
+    if (engineStillRunning) {
+        return `[Mac AI] the connection to the on-device engine dropped ${got}, but the engine is still running — so it did not crash or run out of memory. Something closed the HTTP request underneath the turn. Send the message again; if it keeps happening at roughly the same elapsed time each turn, that points at a timeout rather than the model, so capture the service log for that window.`;
+    }
+    return `[Mac AI] the on-device engine dropped the connection mid-turn (${got}). This usually means the mlx server crashed, ran out of memory, or was restarted while the turn was streaming — this turn's work was lost. Retry the turn; if it keeps happening, restart the engine in Settings → On-device.`;
+}
+/** Estimate request tokens from message bodies and serialized tool schemas. */
+export function estimatePromptTokens(messages, tools) {
+    let chars = 0;
+    try {
+        chars += JSON.stringify(messages)?.length ?? 0;
+    }
+    catch {
+        /* circular or unstringifiable */
+    }
+    if (tools) {
+        try {
+            chars += JSON.stringify(tools)?.length ?? 0;
+        }
+        catch {
+            /* circular or unstringifiable */
+        }
+    }
+    return Math.ceil(chars / 4);
+}
+export const PRE_FIRST_BYTE_BASE_MS = 300_000;
+const PRE_FIRST_BYTE_BASELINE_TOKENS = 8_000;
+const PRE_FIRST_BYTE_MS_PER_1K_TOKENS = 12_000;
+const PRE_FIRST_BYTE_CAP_MS = 900_000;
+/** Scale the pre-first-byte watchdog for slow, large-context MLX prefills. */
+export function computePreFirstByteBudgetMs(approxPromptTokens) {
+    const over = Math.max(0, approxPromptTokens - PRE_FIRST_BYTE_BASELINE_TOKENS);
+    const scaled = PRE_FIRST_BYTE_BASE_MS + (over / 1000) * PRE_FIRST_BYTE_MS_PER_1K_TOKENS;
+    return Math.min(PRE_FIRST_BYTE_CAP_MS, Math.round(scaled));
+}
+function tryParseJsonDetail(body) {
+    try {
+        const obj = JSON.parse(body);
+        if (obj && typeof obj === 'object' && 'detail' in obj && typeof obj.detail === 'string') {
+            return obj.detail.slice(0, 200);
+        }
+    }
+    catch {
+        /* not JSON */
+    }
+    return null;
+}

package/dist/providers/mlx/spec-drafter.d.ts

@@ -0,0 +1,25 @@
+/** Sidecar-visible drafter selection, or null when speculation stays off. */
+export interface SpecDrafterPlan {
+    /** Absolute path passed to the sidecar as `--spec-draft-model`. */
+    dir: string;
+    /** On-disk size, added to the resident-weights figure for memory planning. */
+    bytes: number;
+    /** Why this drafter was chosen — for the launch log. */
+    source: 'configured' | 'convention';
+}
+/** Directory name a model's drafter is expected to occupy. */
+export declare function drafterDirFor(modelDir: string): string;
+/**
+ * Resolve the drafter for a model build.
+ *
+ * `configured` wins outright (an operator pointing at a shared or
+ * hand-built drafter), otherwise the convention path is used when it
+ * exists. Returns null when speculation is disabled or nothing is present.
+ * Never throws: an unreadable drafter directory is a reason to serve
+ * normally, not a reason to fail a launch.
+ */
+export declare function resolveSpecDrafter(opts: {
+    modelDir?: string;
+    configuredPath?: string | null;
+    enabled?: boolean | null;
+}): SpecDrafterPlan | null;

package/dist/providers/mlx/spec-drafter.js

@@ -0,0 +1,73 @@
+/**
+ * Where an MTP drafter lives, and what it costs to keep resident.
+ *
+ * Speculative decoding is on by default, but "on" can only mean "on when a
+ * drafter for this model is actually present": the drafter is a separate
+ * ~0.8 GB artifact split out of an MTP-preserving source checkpoint, and no
+ * MLX conversion carries the `mtp.*` tensors it is made of (mlx-lm's
+ * `sanitize()` strips them). So the launcher looks beside the model tree
+ * instead of requiring configuration, and a model with no drafter simply
+ * serves normally — the sidecar's probe logs one `[spec] off` line and the
+ * turn takes the ordinary path.
+ *
+ * Convention: `<engines>/mlx/drafters/<modelDirName>-mtp`, a sibling of
+ * `<engines>/mlx/models/<modelDirName>`. Keyed on the model directory, not a
+ * base-model guess, because string-surgering a quantization suffix off an id
+ * is how a drafter ends up silently paired with the wrong checkpoint. A
+ * drafter genuinely shared across quantizations of one base model is a
+ * symlink (or an explicit `mlxSpecDraftModelPath`), which keeps the sharing
+ * decision the operator's rather than a parser's.
+ *
+ * The bytes matter as much as the path. MLX slot planning reserves memory
+ * from the target's weights, and an unpriced second resident model is
+ * exactly the over-commit that makes a Metal command-buffer OOM abort the
+ * whole python process. `drafterBytes` feeds the same weights figure the
+ * ceiling math already uses.
+ */
+import { readdirSync, statSync } from 'node:fs';
+import { basename, join, resolve } from 'node:path';
+/** Directory name a model's drafter is expected to occupy. */
+export function drafterDirFor(modelDir) {
+    return resolve(modelDir, '..', '..', 'drafters', `${basename(modelDir)}-mtp`);
+}
+function directorySize(dir) {
+    let total = 0;
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        if (entry.isDirectory()) {
+            total += directorySize(join(dir, entry.name));
+            continue;
+        }
+        if (!entry.isFile())
+            continue;
+        total += statSync(join(dir, entry.name)).size;
+    }
+    return total;
+}
+/**
+ * Resolve the drafter for a model build.
+ *
+ * `configured` wins outright (an operator pointing at a shared or
+ * hand-built drafter), otherwise the convention path is used when it
+ * exists. Returns null when speculation is disabled or nothing is present.
+ * Never throws: an unreadable drafter directory is a reason to serve
+ * normally, not a reason to fail a launch.
+ */
+export function resolveSpecDrafter(opts) {
+    if (opts.enabled === false)
+        return null;
+    const candidate = opts.configuredPath?.trim()
+        ? { dir: opts.configuredPath.trim(), source: 'configured' }
+        : opts.modelDir
+            ? { dir: drafterDirFor(opts.modelDir), source: 'convention' }
+            : null;
+    if (!candidate)
+        return null;
+    try {
+        if (!statSync(candidate.dir).isDirectory())
+            return null;
+        return { ...candidate, bytes: directorySize(candidate.dir) };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/providers/mlx/stdout-parser.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Classify lines from `mlx_vlm.server`'s stdout/stderr into structured
+ * `EnginePhaseEvent`s so the UI surfaces "loading model N%" instead of
+ * a silent spinner.
+ *
+ * `mlx_vlm.server` logs look like:
+ *
+ *   INFO:     Started server process [12345]
+ *   INFO:     Waiting for application startup.
+ *   INFO:     Application startup complete.
+ *   INFO:     Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
+ *   INFO:     Loading model: /path/to/model
+ *   INFO:     Model loaded
+ *   INFO:     127.0.0.1:58420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+ *
+ * The format isn't guaranteed stable across mlx-vlm versions, so the
+ * matchers are intentionally permissive: we look for substrings rather
+ * than exact lines, and silently return null for anything we don't
+ * recognize. Unknown lines still flow to the log file via the
+ * supervisor's `onLog` — we just don't fan them out as phase events.
+ */
+import type { EnginePhaseEvent } from '../streaming-session.js';
+/**
+ * The sidecar's wave-scheduling markers (see python/wave_policy.py):
+ *
+ *   [batch] waiting cache=<id> ahead=<n> behind=<id,id|->
+ *   [batch] paused cache=<id> for=<id,id|->
+ *   [batch] admitted cache=<id> waited=<seconds>s
+ *
+ * Each names exactly one request, so it is always delivered to that session
+ * alone. The `[batch] queued …` line the sidecar prints beside the first
+ * `waiting` marker is diagnostic only and deliberately does not match.
+ */
+export declare function classifyMlxQueueLine(line: string): EnginePhaseEvent | null;
+export declare function classifyMlxStartupLine(line: string): EnginePhaseEvent | null;
+/**
+ * Detect fatal Python exceptions emitted by `mlx_vlm.server` — model
+ * load failures, OOM, missing dependencies, tokenizer mismatches.
+ * The server's httpd keeps running after the `_generate` worker
+ * thread dies, so `/health` lies and chat requests hang — we need a
+ * log-side signal to fail fast instead of trusting the health probe.
+ *
+ * Specifically catches the `NameError: ...` / `ValueError: ...` / etc.
+ * leaf line of a traceback — they're one-liners that appear on every
+ * Python exception, and the message after the colon is the most
+ * useful bit to show a user. Returns null for lines that aren't the
+ * leaf exception line, including ordinary warnings and info logs.
+ *
+ * Specific recognized hints turn into suggested recovery actions the
+ * UI can surface inline. For anything we don't recognize we still
+ * bubble the raw `<ErrorClass>: <message>` up so the user can copy
+ * it into a search.
+ */
+export interface MlxFatalError {
+    category: 'model-arch-mismatch' | 'model-arch-unsupported' | 'missing-dependency' | 'out-of-memory' | 'tokenizer-mismatch' | 'other';
+    message: string;
+    /** Short human-readable hint on how to recover, when we recognize the pattern. */
+    hint?: string;
+}
+export declare function classifyMlxFatalErrorLine(line: string): MlxFatalError | null;

package/dist/providers/mlx/stdout-parser.js

@@ -0,0 +1,254 @@
+/**
+ * Classify lines from `mlx_vlm.server`'s stdout/stderr into structured
+ * `EnginePhaseEvent`s so the UI surfaces "loading model N%" instead of
+ * a silent spinner.
+ *
+ * `mlx_vlm.server` logs look like:
+ *
+ *   INFO:     Started server process [12345]
+ *   INFO:     Waiting for application startup.
+ *   INFO:     Application startup complete.
+ *   INFO:     Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
+ *   INFO:     Loading model: /path/to/model
+ *   INFO:     Model loaded
+ *   INFO:     127.0.0.1:58420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
+ *
+ * The format isn't guaranteed stable across mlx-vlm versions, so the
+ * matchers are intentionally permissive: we look for substrings rather
+ * than exact lines, and silently return null for anything we don't
+ * recognize. Unknown lines still flow to the log file via the
+ * supervisor's `onLog` — we just don't fan them out as phase events.
+ */
+/**
+ * The sidecar's wave-scheduling markers (see python/wave_policy.py):
+ *
+ *   [batch] waiting cache=<id> ahead=<n> behind=<id,id|->
+ *   [batch] paused cache=<id> for=<id,id|->
+ *   [batch] admitted cache=<id> waited=<seconds>s
+ *
+ * Each names exactly one request, so it is always delivered to that session
+ * alone. The `[batch] queued …` line the sidecar prints beside the first
+ * `waiting` marker is diagnostic only and deliberately does not match.
+ */
+export function classifyMlxQueueLine(line) {
+    const m = line.match(/\[batch\] (waiting|paused|admitted) cache=(\S+)(?: ahead=(\d+))?(?: (?:behind|for)=(\S+))?/);
+    if (!m)
+        return null;
+    const state = m[1];
+    const cacheId = m[2];
+    const ids = m[4] && m[4] !== '-' ? m[4].split(',').filter(Boolean) : [];
+    return {
+        provider: 'mlx',
+        phase: 'prefill',
+        cacheId,
+        engineQueue: {
+            state,
+            ...(state === 'admitted' ? {} : { behind: ids }),
+            ...(m[3] !== undefined ? { ahead: Number.parseInt(m[3], 10) } : {}),
+        },
+    };
+}
+export function classifyMlxStartupLine(line) {
+    if (line.includes('memory-admission:'))
+        return { provider: 'mlx', phase: 'starting', detail: 'Waiting for available memory' };
+    const lower = line.toLowerCase();
+    // ── Process starting ──
+    if (/started server process/i.test(line) || /waiting for application startup/i.test(line)) {
+        // Phase `detail` is what the engine pill renders verbatim
+        // (uppercased by CSS). Keep it human-readable — "Mac AI" beats
+        // shipping "mlx_vlm.server" to the user even though that's the
+        // literal binary name. The internal `provider: 'mlx'` discriminator
+        // and the supervisor's log prefix are unchanged.
+        return { provider: 'mlx', phase: 'starting', detail: 'Mac AI server starting' };
+    }
+    // ── Chunked prefill progress ──
+    // mlx_vlm prints a tqdm bar to stdout once chunked prefill kicks in
+    // for prompts longer than `--prefill-step-size`. The format is:
+    //
+    //   Prefill:   8%|██▍       | 2048/24726 [00:08<01:33, 242.65tok/s]
+    //
+    // Each batch flushes a fresh line because tqdm uses `\r` to overwrite
+    // and the supervisor splits chunks on `\n` (not `\r`), so we see
+    // every step. Surface this as a `prefill` phase event with `progress`
+    // set so the UI can render a real percentage during the long silent
+    // window before the first generated token. The 0% line ships with
+    // `?tok/s` — handle the missing rate gracefully.
+    const prefill = line.match(/^(?:\[[^\]]+\]\s+)?Prefill:\s*(\d+)%[^|]*\|[^|]*\|\s*(\d+)\/(\d+)\s*\[[^,\]]*(?:,\s*([\d.]+)tok\/s)?/i);
+    if (prefill) {
+        const pct = Number.parseInt(prefill[1] ?? '0', 10);
+        const cur = Number.parseInt(prefill[2] ?? '0', 10);
+        const total = Number.parseInt(prefill[3] ?? '0', 10);
+        const tps = prefill[4] ? Number.parseFloat(prefill[4]) : null;
+        const detailParts = [
+            `${cur.toLocaleString('en-US')} / ${total.toLocaleString('en-US')} tokens`,
+        ];
+        if (tps !== null && Number.isFinite(tps) && tps > 0) {
+            detailParts.push(`${tps >= 10 ? tps.toFixed(0) : tps.toFixed(1)} tok/s`);
+        }
+        // `cache=<id>` names the sub this marker belongs to. Matched separately
+        // from the tqdm shape above so the bar itself keeps parsing from engines
+        // that don't emit the tag (older sidecars, the aggregate fallback path).
+        const owner = line.match(/\bcache=(\S+)/);
+        return {
+            provider: 'mlx',
+            phase: 'prefill',
+            detail: detailParts.join(' · '),
+            progress: Math.max(0, Math.min(1, pct / 100)),
+            ...(owner?.[1] ? { cacheId: owner[1] } : {}),
+        };
+    }
+    // ── Loading model ──
+    // Match phrases seen in the wild: "Loading model", "Loading weights",
+    // "Fetching", "Loading tokenizer". Don't try to parse percentages —
+    // mlx-vlm doesn't emit them; UI just shows an indefinite "Loading".
+    if (/loading model|loading weights|loading tokenizer|fetching/i.test(lower)) {
+        return { provider: 'mlx', phase: 'loading_model', detail: 'Loading weights' };
+    }
+    // ── Ready ──
+    // "Model loaded" (mlx-vlm), "Uvicorn running" (the HTTP server's own
+    // readiness line), or "Application startup complete" — any is proof
+    // the engine can accept traffic.
+    if (/model loaded/i.test(line) ||
+        /uvicorn running on/i.test(line) ||
+        /application startup complete/i.test(line)) {
+        return { provider: 'mlx', phase: 'ready' };
+    }
+    // ── Cache events (Phase 2 wrapped server) ──
+    // The wrapper logs `[cache] hit cache_id=... prior_tokens=N`,
+    // `[cache] miss cache_id=...`, `[cache] saved cache_id=... tokens=N`,
+    // `[cache] evicted cache_id=...`, `[cache] warmed cache_id=...`. We
+    // surface hits and warms as `ready` phase events with a friendly
+    // detail so the engine pill briefly flashes "Cache hit · 24K tokens
+    // reused" instead of staying silent on near-instant turns. Saved /
+    // evicted events are bookkeeping only — no UI signal needed.
+    const hit = line.match(/\[cache\] hit cache_id=\S+ prior_tokens=(\d+)/);
+    if (hit) {
+        return {
+            provider: 'mlx',
+            phase: 'ready',
+            detail: `Cache hit · reusing ${Number.parseInt(hit[1] ?? '0', 10).toLocaleString('en-US')} tokens`,
+        };
+    }
+    const warmed = line.match(/\[cache\] warmed cache_id=\S+ tokens=(\d+)/);
+    if (warmed) {
+        return {
+            provider: 'mlx',
+            phase: 'ready',
+            detail: `Cache warmed · ${Number.parseInt(warmed[1] ?? '0', 10).toLocaleString('en-US')} tokens ready`,
+        };
+    }
+    return null;
+}
+export function classifyMlxFatalErrorLine(line) {
+    // ── "X requires the Y library" body line ──
+    // transformers' `requires_backends` raises ImportError with the leaf
+    // class name on one line ("ImportError:") and the actual hint on
+    // following lines: "Qwen3VLVideoProcessor requires the PyTorch
+    // library but it was not found in your environment." We match the
+    // body line directly because the leaf line is empty after the colon
+    // and the classifier needs a single-line signal — there's no
+    // multi-line state in the supervisor's per-line dispatch loop. This
+    // catches the most common "model X needs torch/torchvision" path
+    // even when the bare `ImportError:` line is empty.
+    const requiresBackend = line.match(/^(?:\[(?:mlx|mlx_v?lm\.server)\]\s+)?(\S+) requires the (\S+) library but it was not found/);
+    if (requiresBackend) {
+        const className = requiresBackend[1] ?? '';
+        const backend = requiresBackend[2] ?? '';
+        return {
+            category: 'missing-dependency',
+            message: `ImportError: ${className} requires the ${backend} library but it was not found`,
+            hint: `This model needs the ${backend} backend, which isn't in the on-device venv. Pick a different model from the catalog (the recommended ones don't need ${backend}), or open Settings → This Mac → Advanced and add ${backend.toLowerCase()} to the package spec then Reset venv.`,
+        };
+    }
+    // Match the leaf exception line: optional supervisor `[mlx]` prefix
+    // OR mlx-vlm's own `[mlx_lm.server]` / `[mlx_vlm.server]` prefix
+    // (depending on whether the supervisor or mlx-vlm tagged the line)
+    // then `<ClassName>Error: <message>`. The supervisor's `[mlx]`
+    // prefix is the common case in production; the mlx-vlm prefixes
+    // appear when the inner process logs through its own logger.
+    const match = line.match(/^(?:\[(?:mlx|mlx_v?lm\.server)\]\s+)?([A-Z][A-Za-z]+Error):\s+(.*)$/);
+    if (!match)
+        return null;
+    const errorClass = match[1] ?? '';
+    const message = (match[2] ?? '').trim();
+    if (!errorClass)
+        return null;
+    // Empty-message ImportError is a real signal (transformers'
+    // `requires_backends` produces this) — surface it so we don't keep
+    // accepting requests against a dead engine. The followup "X requires
+    // the Y library" line above gives a more specific message; this
+    // branch is a fallback for cases where that pattern doesn't appear
+    // (or arrives after we've already locked in this generic message).
+    if (!message) {
+        if (errorClass === 'ImportError' || errorClass === 'ModuleNotFoundError') {
+            return {
+                category: 'missing-dependency',
+                message: `${errorClass} (no detail emitted)`,
+                hint: 'The on-device runtime is missing a required Python module. Try a different model from the catalog, or Settings → This Mac → Advanced → Reset venv.',
+            };
+        }
+        return null;
+    }
+    // ── Model architecture mismatch ──
+    // Two distinct flavors hit this branch, both surfaced as the same
+    // strict-load `ValueError` with a list of unrecognized tensors:
+    //   1. Older mlx-vlm builds that don't know a brand-new model
+    //      class at all (the original Gemma 3n / `gemma3n` case).
+    //   2. A specific quant repacker (e.g. lmstudio-community's Gemma 4
+    //      MLX-4bit) chose to quantize a layer that mlx-vlm's model
+    //      defines as a custom non-quantizable class — so the weights
+    //      ship `<layer>.scales` / `<layer>.biases` that mlx_vlm has
+    //      no slot for. The fix in that case is to switch to a quant
+    //      that left the custom layer alone, not to bump the package.
+    // The hint covers both paths — bump mlx-vlm first, then if still
+    // broken delete the install and pick a different catalog entry.
+    if (errorClass === 'ValueError' && /Received \d+ parameters not in model/i.test(message)) {
+        return {
+            category: 'model-arch-mismatch',
+            message: `${errorClass}: ${message}`,
+            hint: "The on-device runtime doesn't recognize this model's architecture (or this quant's layer layout). Try, in order:\n  1. Settings → This Mac → Advanced: leave the package spec blank for the latest release, then Reset venv.\n  2. If the error persists, the model file layout itself isn't compatible — delete the local model and pick a different catalog entry.",
+        };
+    }
+    // ── Model architecture unsupported (checkpoint MISSING params) ──
+    // The mirror of the branch above: `ValueError: Missing N parameters`
+    // means mlx-vlm built a model graph with slots the checkpoint doesn't
+    // fill — i.e. the loader's model class can't fully represent this
+    // architecture yet, so no quant of it will load. Distinct hint from the
+    // extra-params case: the fix is an mlx-vlm that adds the architecture,
+    // not a different quant — and until then the model is llama.cpp-only.
+    if (errorClass === 'ValueError' && /Missing \d+ parameters?/i.test(message)) {
+        return {
+            category: 'model-arch-unsupported',
+            message: `${errorClass}: ${message}`,
+            hint: "The on-device runtime (mlx-vlm) can't fully build this model's architecture — the loader expects parameters this checkpoint doesn't provide, so no quant of it will load on MLX yet. Try, in order:\n  1. Settings → This Mac → Advanced: blank the package spec for the latest mlx-vlm, then Reset venv (a newer release may add support).\n  2. If it persists, run this model via llama instead — same weights, different engine — or pick a different catalog entry.",
+        };
+    }
+    // ── Missing Python dependency ──
+    if (errorClass === 'ImportError' || errorClass === 'ModuleNotFoundError') {
+        return {
+            category: 'missing-dependency',
+            message: `${errorClass}: ${message}`,
+            hint: 'The on-device runtime is missing a required component. Reset venv from Settings → This Mac → Advanced to re-provision it.',
+        };
+    }
+    // ── Out of memory ──
+    if (errorClass === 'MemoryError' || /out of memory|mps backend out of memory/i.test(message)) {
+        return {
+            category: 'out-of-memory',
+            message: `${errorClass}: ${message}`,
+            hint: 'Not enough unified memory for this model. Try a smaller variant (E2B instead of E4B) or close other memory-heavy apps.',
+        };
+    }
+    // ── Tokenizer / chat-template mismatch ──
+    if (/tokenizer|chat.?template/i.test(message)) {
+        return {
+            category: 'tokenizer-mismatch',
+            message: `${errorClass}: ${message}`,
+            hint: 'The tokenizer_config is malformed or missing the expected chat template. Try deleting the local model and downloading it again from Settings → This Mac.',
+        };
+    }
+    return {
+        category: 'other',
+        message: `${errorClass}: ${message}`,
+    };
+}

package/dist/providers/mlx/textual-tool-call-salvage.d.ts

@@ -0,0 +1,46 @@
+import { findTruncatedJsonEnvelope } from '../local-tool-call-salvage.js';
+import type { MlxToolCallAccumulator } from './tool-call-protocol.js';
+type SalvagedToolCalls = ReturnType<MlxToolCallAccumulator['finalize']>;
+/** What the textual passes recovered from one generation, by the shape that matched. */
+export interface TextualToolCallSalvage {
+    /** The visible text with every salvaged call (and a truncated tail) stripped. */
+    turnContent: string;
+    proseRepaired: SalvagedToolCalls;
+    xmlTagRepaired: SalvagedToolCalls;
+    claudeInvokeRepaired: SalvagedToolCalls;
+    glmRepaired: SalvagedToolCalls;
+    hermesRepaired: SalvagedToolCalls;
+    shellRepaired: SalvagedToolCalls;
+    envelopeRepaired: SalvagedToolCalls;
+    envelopeTruncated: ReturnType<typeof findTruncatedJsonEnvelope>;
+    /** A JSON envelope naming a tool this turn does not have. */
+    unknownCall: {
+        wanted: string;
+        suggestion: string | null;
+    } | null;
+    truncatedSalvage: SalvagedToolCalls;
+    bareInvokeRepaired: SalvagedToolCalls;
+}
+/**
+ * The salvage passes that read tool calls out of a generation's visible text,
+ * run after the structured calls and the stripped-marker repair. Each pass is
+ * gated on every earlier one having found nothing, so one intent never fires
+ * twice. Truncated calls are reported through `truncatedCallIds` and
+ * `droppedTruncatedPayloadCalls`, which the tool loop reads afterwards.
+ */
+export declare function salvageTextualToolCalls(input: {
+    turnContent: string;
+    knownToolNames: Set<string>;
+    seq: number;
+    turn: number;
+    structuredCalls: SalvagedToolCalls;
+    repairedCalls: SalvagedToolCalls;
+    codeBlockRepaired: SalvagedToolCalls;
+    truncatedCallIds: Set<string>;
+    droppedTruncatedPayloadCalls: Array<{
+        name: string;
+        args: Record<string, unknown>;
+    }>;
+    emitWarning: (message: string) => void;
+}): TextualToolCallSalvage;
+export {};