@blockrun/franklin 3.15.89 → 3.15.91

package/dist/agent/loop.js

@@ -27,7 +27,7 @@ import { setSessionPersistenceDisabled } from '../session/storage.js';
 import { estimateCost, OPUS_PRICING } from '../pricing.js';
 import { maybeMidSessionExtract } from '../learnings/extractor.js';
 import { extractMentions, buildEntityContext, loadEntities } from '../brain/store.js';
-import { routeRequestAsync, resolveTierToModel, parseRoutingProfile, getFallbackChain, pickFreeFallback } from '../router/index.js';
+import { routeRequestAsync, resolveTierToModel, parseRoutingProfile, getFallbackChain, pickFreeFallback, isVisionModel, messageNeedsVision, pickVisionSibling } from '../router/index.js';
 import { recordOutcome } from '../router/local-elo.js';
 import { shouldPlan, getPlanningPrompt, getExecutorModel, isExecutorStuck, toolCallSignature } from './planner.js';
 import { shouldVerify, runVerification } from './verification.js';
@@ -1118,6 +1118,16 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
                 onProgress: (id, text) => onEvent({ kind: 'capability_progress', id, text }),
                 sessionId,
             });
+            // ── Vision-need detection (per turn) ──
+            // Images enter a turn one of two ways: the user types an image path
+            // and the Read tool will inline bytes mid-turn, or the user references
+            // an image in their last message directly. We can only see (1) at this
+            // point — but that's the case we care about: the router has to decide
+            // BEFORE the model call which model to use. If the model can't see
+            // images, Read's tool_result image blocks get tokenized as base64 text
+            // by the gateway (verified 2026-05-09) and the model hallucinates from
+            // the "Image file: <path>" stub. Detect upfront, route accordingly.
+            const turnNeedsVision = loopCount === 1 && messageNeedsVision(lastUserInput);
             // ── Router: resolve routing profiles to concrete models ──
             // Uses the tier already decided by the turn-analyzer — one LLM call
             // up-front rather than a separate classifier here. Fallback to the
@@ -1129,8 +1139,8 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
             let routingSavings;
             if (routingProfile) {
                 const routing = turnAnalysis
-                    ? resolveTierToModel(turnAnalysis.tier, routingProfile)
-                    : await routeRequestAsync(lastUserInput || '', routingProfile);
+                    ? resolveTierToModel(turnAnalysis.tier, routingProfile, turnNeedsVision)
+                    : await routeRequestAsync(lastUserInput || '', routingProfile, undefined, turnNeedsVision);
                 resolvedModel = routing.model;
                 routingTier = routing.tier;
                 routingConfidence = routing.confidence;
@@ -1138,12 +1148,31 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
                 lastRoutedModel = routing.model;
                 lastRoutedCategory = routing.category || '';
                 if (loopCount === 1) {
+                    const visionTag = turnNeedsVision ? ' 👁️' : '';
                     onEvent({
                         kind: 'text_delta',
-                        text: `*Auto → ${routing.model}*\n\n`,
+                        text: `*Auto → ${routing.model}${visionTag}*\n\n`,
                     });
                 }
             }
+            else if (turnNeedsVision && !isVisionModel(resolvedModel)) {
+                // ── Manual-mode guard ──
+                // User explicitly picked a model that can't see images. Don't silently
+                // send the image — the model would only see the text stub from Read
+                // and hallucinate. Swap to the closest vision sibling JUST for this
+                // turn (next turn's model-recovery block at the top of the user-input
+                // handler resets to baseModel, so the user's intent isn't permanently
+                // overridden). Always emit a visible notice so the user knows their
+                // pick was overridden and why.
+                const original = resolvedModel;
+                const visionSwap = pickVisionSibling(original);
+                resolvedModel = visionSwap;
+                config.model = visionSwap;
+                onEvent({
+                    kind: 'text_delta',
+                    text: `*⚠️ ${original} can't see images — using ${visionSwap} for this turn.*\n\n`,
+                });
+            }
             // Update token estimation model for more accurate byte-per-token ratio
             setEstimationModel(resolvedModel);
             // ── Plan-then-execute: detect and activate ──

package/dist/agent/reduce.js

@@ -44,13 +44,37 @@ export function ageToolResults(history) {
         const aged = parts.map(part => {
             if (part.type !== 'tool_result')
                 return part;
+            // Vision tool_results carry [text, image] arrays. JSON.stringify-ing
+            // them and rebuilding `content` as a truncated string drops the
+            // image block entirely. Measure only the text portion; aging code
+            // below rebuilds as a string only when no image is present, otherwise
+            // we leave the part untouched (image bytes are already context-cheap
+            // once cached, and turn them into placeholders is the wrong fix).
+            let hasImage = false;
+            let textOnly = '';
+            if (Array.isArray(part.content)) {
+                for (const block of part.content) {
+                    if (block?.type === 'text' && typeof block.text === 'string') {
+                        textOnly += (textOnly ? '\n' : '') + block.text;
+                    }
+                    else if (block?.type === 'image') {
+                        hasImage = true;
+                    }
+                }
+            }
             const content = typeof part.content === 'string'
                 ? part.content
-                : JSON.stringify(part.content);
+                : Array.isArray(part.content)
+                    ? textOnly
+                    : JSON.stringify(part.content);
             const charLen = content.length;
             // Recent 3 results: keep full
             if (age <= 3)
                 return part;
+            // Preserve image-bearing tool_results regardless of age — replacing
+            // them with a text stub would silently delete the model's vision context.
+            if (hasImage)
+                return part;
             // Age 4-8: keep first 500 chars
             if (age <= 8 && charLen > 500) {
                 modified = true;
@@ -206,7 +230,32 @@ export function deduplicateToolResultLines(history) {
         const newParts = parts.map(part => {
             if (part.type !== 'tool_result')
                 return part;
-            const raw = typeof part.content === 'string' ? part.content : JSON.stringify(part.content);
+            // Vision tool_results carry [text, image] arrays. JSON.stringify-ing
+            // them and writing back as a string would destroy the image (same
+            // bug class as ageToolResults / budgetToolResults — sibling site,
+            // verified 2026-05-10 during PR #53 review). For arrays, dedupe
+            // only the text segments; image segments pass through untouched.
+            let raw;
+            const imageBlocks = [];
+            if (typeof part.content === 'string') {
+                raw = part.content;
+            }
+            else if (Array.isArray(part.content)) {
+                const blocks = part.content;
+                const texts = [];
+                for (const b of blocks) {
+                    if (b?.type === 'text' && typeof b.text === 'string') {
+                        texts.push(b.text);
+                    }
+                    else if (b?.type === 'image') {
+                        imageBlocks.push(b);
+                    }
+                }
+                raw = texts.join('\n');
+            }
+            else {
+                raw = JSON.stringify(part.content);
+            }
             // Strip ANSI codes
             const stripped = raw.replace(ANSI_RE_REDUCE, '');
             // Collapse repeated consecutive lines
@@ -231,6 +280,19 @@ export function deduplicateToolResultLines(history) {
             if (result === raw)
                 return part;
             partModified = true;
+            // If the original content was an array with image blocks, rebuild
+            // as an array — keep all image segments, replace the joined text
+            // payload with a single deduped text segment. This way dedupe runs
+            // for free on image-bearing results without losing vision context.
+            if (Array.isArray(part.content) && imageBlocks.length > 0) {
+                return {
+                    ...part,
+                    content: [
+                        { type: 'text', text: result },
+                        ...imageBlocks,
+                    ],
+                };
+            }
             return { ...part, content: result };
         });
         if (!partModified)
@@ -301,6 +363,16 @@ export function collapseRepetitiveTools(history) {
         const parts = msg.content.map(part => {
             if (part.type !== 'tool_result' || !oldIds.has(part.tool_use_id))
                 return part;
+            // Image-bearing results (third sibling site of the JSON.stringify
+            // bug class — same pattern as ageToolResults / budgetToolResults /
+            // deduplicateToolResultLines). Don't collapse; replacing them with
+            // a `[first-line...]` string would destroy the vision context.
+            // Image bytes are already cache-cheap upstream once prompt-cached;
+            // the cost-control intent of this collapser is satisfied without
+            // touching them.
+            if (Array.isArray(part.content) && part.content.some((b) => b.type === 'image')) {
+                return part;
+            }
             const content = typeof part.content === 'string' ? part.content : JSON.stringify(part.content);
             if (content.length <= 80)
                 return part;

package/dist/proxy/server.js

@@ -4,7 +4,7 @@ import { recordUsage } from '../stats/tracker.js';
 import { appendSettlementRow } from '../stats/cost-log.js';
 import { appendAudit } from '../stats/audit.js';
 import { buildFallbackChain, DEFAULT_FALLBACK_CONFIG, ROUTING_PROFILES, } from './fallback.js';
-import { routeRequest, parseRoutingProfile, } from '../router/index.js';
+import { routeRequest, parseRoutingProfile, isVisionModel, messagesNeedVision, pickVisionSibling, } from '../router/index.js';
 import { estimateCost } from '../pricing.js';
 import { VERSION } from '../config.js';
 // User-Agent for backend requests
@@ -342,6 +342,13 @@ export function createProxy(options) {
                             parsed.model = currentModel || DEFAULT_MODEL;
                         }
                         requestModel = parsed.model || DEFAULT_MODEL;
+                        // Vision-need detection: does this request carry an image? We
+                        // check messages[] for explicit image / image_url parts AND for
+                        // image paths embedded in text — Anthropic-format proxies stream
+                        // both shapes. Used both by the Auto router (pick a vision-capable
+                        // tier model) and by the manual-mode guard (swap when the user
+                        // explicitly picked a text-only model).
+                        const proxyNeedsVision = messagesNeedVision(parsed.messages || []);
                         // Smart routing: if model is a routing profile, classify and route
                         const routingProfile = parseRoutingProfile(requestModel);
                         if (routingProfile) {
@@ -360,13 +367,27 @@ export function createProxy(options) {
                                         .join('\n');
                                 }
                             }
-                            // Route the request
-                            const routing = routeRequest(promptText, routingProfile);
+                            // Route the request — propagate vision-need so AUTO_TIERS' V4
+                            // Pro default doesn't get picked for an image-bearing turn.
+                            const routing = routeRequest(promptText, routingProfile, proxyNeedsVision);
                             parsed.model = routing.model;
                             requestModel = routing.model;
                             logger.info(`[franklin] 🧠 Smart routing: ${routingProfile} → ${routing.tier} → ${routing.model} ` +
                                 `(${(routing.savings * 100).toFixed(0)}% savings) [${routing.signals.join(', ')}]`);
                         }
+                        else if (proxyNeedsVision && !isVisionModel(requestModel)) {
+                            // Manual-mode guard: user (or an upstream client) passed a
+                            // concrete text-only model alongside an image. Swap to the
+                            // family-closest vision sibling and log loudly — silently
+                            // sending the image would tokenize as base64 text and produce
+                            // a hallucinated answer. Same swap policy as the agent loop's
+                            // interactive path so behavior is consistent across surfaces.
+                            const original = requestModel;
+                            const visionSwap = pickVisionSibling(original);
+                            parsed.model = visionSwap;
+                            requestModel = visionSwap;
+                            logger.warn(`[franklin] 👁️  Vision swap: ${original} can't see images → ${visionSwap}`);
+                        }
                         {
                             const original = parsed.max_tokens;
                             const model = (parsed.model || '').toLowerCase();

package/dist/router/index.d.ts

@@ -10,6 +10,7 @@
  * Local Elo adjustments personalize routing per user over time.
  */
 import { type Category } from './categories.js';
+export { isVisionModel, messageNeedsVision, messagesNeedVision, pickVisionSibling } from './vision.js';
 export type Tier = 'SIMPLE' | 'MEDIUM' | 'COMPLEX' | 'REASONING';
 export type RoutingProfile = 'auto' | 'free';
 export interface RoutingResult {
@@ -33,7 +34,7 @@ export declare function llmClassifyRequest(prompt: string): Promise<Tier | null>
  * Profile-specific tier tables (AUTO / ECO / PREMIUM / FREE) still pick
  * the concrete model; the classifier only picks the TIER.
  */
-export declare function routeRequestAsync(prompt: string, profile?: RoutingProfile, classify?: TierClassifier): Promise<RoutingResult>;
+export declare function routeRequestAsync(prompt: string, profile?: RoutingProfile, classify?: TierClassifier, needsVision?: boolean): Promise<RoutingResult>;
 /**
  * Map a pre-classified tier to a concrete model + savings using the profile's
  * tier table. No classifier call — assumes the caller already decided the
@@ -43,8 +44,8 @@ export declare function routeRequestAsync(prompt: string, profile?: RoutingProfi
  * Use this when you have a tier already. Use `routeRequestAsync` when you
  * need the classifier to produce the tier.
  */
-export declare function resolveTierToModel(tier: Tier, profile?: RoutingProfile): RoutingResult;
-export declare function routeRequest(prompt: string, profile?: RoutingProfile): RoutingResult;
+export declare function resolveTierToModel(tier: Tier, profile?: RoutingProfile, needsVision?: boolean): RoutingResult;
+export declare function routeRequest(prompt: string, profile?: RoutingProfile, needsVision?: boolean): RoutingResult;
 /**
  * Get fallback models for a tier
  */

package/dist/router/index.js

@@ -16,6 +16,8 @@ import { BLOCKRUN_DIR } from '../config.js';
 import { detectCategory, mapCategoryToTier } from './categories.js';
 import { selectModel } from './selector.js';
 import { computeLocalElo, blendElo } from './local-elo.js';
+import { isVisionModel } from './vision.js';
+export { isVisionModel, messageNeedsVision, messagesNeedVision, pickVisionSibling } from './vision.js';
 // ─── Learned Weights Loading ───
 const WEIGHTS_FILE = path.join(BLOCKRUN_DIR, 'router-weights.json');
 let cachedWeights; // undefined = not loaded yet
@@ -69,6 +71,27 @@ const AUTO_TIERS = {
         ],
     },
 };
+/**
+ * If this turn carries an image, the picked tier model must be able to see it.
+ * Walks the tier's primary+fallback chain for the first vision-capable model;
+ * if none of them have vision, escalates to COMPLEX (Opus is always vision).
+ *
+ * Note: only applied when the caller signals needsVision=true. Without that
+ * hint the classic per-tier defaults still rule — V4 Pro's $0.50/$1.00 promo
+ * is the right SIMPLE/MEDIUM pick for text-only turns and we don't want to
+ * blanket-upgrade everyone to a vision model.
+ */
+function pickVisionTierModel(tier) {
+    const chain = [AUTO_TIERS[tier].primary, ...AUTO_TIERS[tier].fallback];
+    const visionInTier = chain.find(isVisionModel);
+    if (visionInTier)
+        return { model: visionInTier, tier, signal: 'vision-required' };
+    // Tier chain is fully text-only (unusual but possible if cheap tiers get
+    // re-tuned). Escalate to COMPLEX whose primary (Opus) is always vision.
+    const escalated = [AUTO_TIERS.COMPLEX.primary, ...AUTO_TIERS.COMPLEX.fallback]
+        .find(isVisionModel) ?? AUTO_TIERS.COMPLEX.primary;
+    return { model: escalated, tier: 'COMPLEX', signal: 'vision-escalated' };
+}
 // ─── Keywords for Classification ───
 //
 // Keyword fast-path uses English only by policy (English-only-source rule).
@@ -250,7 +273,7 @@ function classifyRequest(prompt, tokenCount) {
     return { tier, confidence, signals };
 }
 // ─── Classic Router (keyword-based fallback) ───
-function classicRouteRequest(prompt, profile) {
+function classicRouteRequest(prompt, profile, needsVision = false) {
     // Estimate token count (use byte length / 4 for better accuracy with non-ASCII)
     const byteLen = Buffer.byteLength(prompt, 'utf-8');
     const tokenCount = Math.ceil(byteLen / 4);
@@ -260,11 +283,21 @@ function classicRouteRequest(prompt, profile) {
     // 2026-05-03 — see comment on RoutingProfile above). 'free' is handled
     // earlier by the caller path; if it ever reaches here, fall through to
     // AUTO_TIERS rather than crashing.
-    const tierConfigs = AUTO_TIERS;
-    const model = tierConfigs[tier].primary;
+    let model;
+    let finalTier = tier;
+    const finalSignals = [...signals];
+    if (needsVision) {
+        const v = pickVisionTierModel(tier);
+        model = v.model;
+        finalTier = v.tier;
+        finalSignals.push(v.signal);
+    }
+    else {
+        model = AUTO_TIERS[tier].primary;
+    }
     const savings = computeSavings(model);
     const category = detectCategory(prompt, loadLearnedWeights()?.category_keywords).category;
-    return { model, tier, confidence, signals, savings, category };
+    return { model, tier: finalTier, confidence, signals: finalSignals, savings, category };
 }
 // ─── LLM-based classifier ───
 //
@@ -362,25 +395,35 @@ export async function llmClassifyRequest(prompt) {
  * Profile-specific tier tables (AUTO / ECO / PREMIUM / FREE) still pick
  * the concrete model; the classifier only picks the TIER.
  */
-export async function routeRequestAsync(prompt, profile = 'auto', classify = llmClassifyRequest) {
+export async function routeRequestAsync(prompt, profile = 'auto', classify = llmClassifyRequest, needsVision = false) {
     // Free / short-circuit profiles — no classifier needed.
     if (profile === 'free')
-        return routeRequest(prompt, profile);
+        return routeRequest(prompt, profile, needsVision);
     const tier = await classify(prompt).catch(() => null);
     if (!tier) {
         // Classifier miss or disabled — fall through to the sync keyword router.
-        return routeRequest(prompt, profile);
+        return routeRequest(prompt, profile, needsVision);
     }
     // Build a RoutingResult from the LLM-picked tier using the same tier
     // tables the keyword path uses. Keeps downstream code path-identical.
-    const tierConfigs = AUTO_TIERS;
-    const model = tierConfigs[tier].primary;
+    let model;
+    let finalTier = tier;
+    const signals = ['llm-classified'];
+    if (needsVision) {
+        const v = pickVisionTierModel(tier);
+        model = v.model;
+        finalTier = v.tier;
+        signals.push(v.signal);
+    }
+    else {
+        model = AUTO_TIERS[tier].primary;
+    }
     const category = detectCategory(prompt, loadLearnedWeights()?.category_keywords).category;
     return {
         model,
-        tier,
+        tier: finalTier,
         confidence: 0.85, // LLM classification — medium-high confidence
-        signals: ['llm-classified'],
+        signals,
         savings: computeSavings(model),
         category,
     };
@@ -394,36 +437,51 @@ export async function routeRequestAsync(prompt, profile = 'auto', classify = llm
  * Use this when you have a tier already. Use `routeRequestAsync` when you
  * need the classifier to produce the tier.
  */
-export function resolveTierToModel(tier, profile = 'auto') {
+export function resolveTierToModel(tier, profile = 'auto', needsVision = false) {
     // Free profile short-circuits — everything routes to a single free model.
+    // qwen3-coder-480b is text-only; on a vision turn the free profile can't
+    // help us. Caller should detect this and warn the user that Free won't
+    // handle images — for now we just return the free pick and let the model
+    // fail gracefully. (Open question: should we hard-fall to nvidia/llama-4-
+    // maverick here? Skipped until we see a real user hit this path.)
     if (profile === 'free') {
         return {
             model: 'nvidia/qwen3-coder-480b',
             tier: 'SIMPLE',
             confidence: 1.0,
-            signals: ['free-profile'],
+            signals: needsVision ? ['free-profile', 'vision-unsupported'] : ['free-profile'],
             savings: 1.0,
         };
     }
-    const tierConfigs = AUTO_TIERS;
-    const model = tierConfigs[tier].primary;
+    let model;
+    let finalTier = tier;
+    const signals = ['pre-classified'];
+    if (needsVision) {
+        const v = pickVisionTierModel(tier);
+        model = v.model;
+        finalTier = v.tier;
+        signals.push(v.signal);
+    }
+    else {
+        model = AUTO_TIERS[tier].primary;
+    }
     return {
         model,
-        tier,
+        tier: finalTier,
         confidence: 0.85,
-        signals: ['pre-classified'],
+        signals,
         savings: computeSavings(model),
     };
 }
 // ─── Main Router ───
-export function routeRequest(prompt, profile = 'auto') {
+export function routeRequest(prompt, profile = 'auto', needsVision = false) {
     // Free profile — always use free model
     if (profile === 'free') {
         return {
             model: 'nvidia/qwen3-coder-480b',
             tier: 'SIMPLE',
             confidence: 1.0,
-            signals: ['free-profile'],
+            signals: needsVision ? ['free-profile', 'vision-unsupported'] : ['free-profile'],
             savings: 1.0,
         };
     }
@@ -432,7 +490,7 @@ export function routeRequest(prompt, profile = 'auto') {
     // cheap/weak models on agentic work. Classic AUTO_TIERS defaults are
     // agent-tuned (Sonnet-tier backbone) and more predictable for users.
     if (profile === 'auto') {
-        return classicRouteRequest(prompt, profile);
+        return classicRouteRequest(prompt, profile, needsVision);
     }
     // ── Learned routing (if weights available) ──
     const weights = loadLearnedWeights();
@@ -457,6 +515,21 @@ export function routeRequest(prompt, profile = 'auto') {
         const selected = selectModel(category, profile, adjustedWeights);
         if (selected) {
             const tier = mapCategoryToTier(category);
+            // Vision-aware substitution: if the Elo-picked model is text-only but
+            // the turn needs vision, swap to the tier's first vision-capable model.
+            // We deliberately don't blend Elo with vision capability — vision is a
+            // hard requirement, not a quality dimension.
+            if (needsVision && !isVisionModel(selected.model)) {
+                const v = pickVisionTierModel(tier);
+                return {
+                    model: v.model,
+                    tier: v.tier,
+                    confidence,
+                    signals: [category, v.signal],
+                    savings: computeSavings(v.model),
+                    category,
+                };
+            }
             const savings = computeSavings(selected.model);
             return {
                 model: selected.model,
@@ -470,7 +543,7 @@ export function routeRequest(prompt, profile = 'auto') {
         // Fall through to classic if selectModel returns null (no candidates for category)
     }
     // ── Classic routing (keyword-based fallback) ──
-    return classicRouteRequest(prompt, profile);
+    return classicRouteRequest(prompt, profile, needsVision);
 }
 function computeSavings(model) {
     const opusCostPer1K = (OPUS_PRICING.input + OPUS_PRICING.output) / 2 / 1000;

package/dist/router/vision.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Vision capability + image-attachment detection.
+ *
+ * Two jobs:
+ *   1. isVisionModel(id)        — does this gateway model accept image input?
+ *   2. messageNeedsVision(text) — does this user message reference an image?
+ *
+ * Source of truth: a hand-curated allowlist below. The gateway exposes a
+ * 'vision' category on /v1/models, but resolving it at routing time would
+ * make routeRequest async and gate sync proxy paths on a network call. The
+ * allowlist is small (~18 entries) and changes only when models do, which
+ * already touches the router + pricing tables — updating one more file is
+ * the right tradeoff vs. async fan-out across every routing callsite.
+ *
+ * Background: prior to this module, Auto routing could pick a text-only model
+ * (e.g. deepseek-v4-pro) on an image-bearing turn. The Read tool would still
+ * inline image bytes, the gateway would tokenize the base64 as text, and the
+ * model — having no vision pathway — would hallucinate based on the
+ * `Image file: <path>` description string. Expensive AND wrong.
+ */
+/** Does this concrete gateway model accept image input? */
+export declare function isVisionModel(modelId: string | undefined | null): boolean;
+/**
+ * Pick a vision-capable replacement closest to the user's chosen model.
+ * Prefers same provider family (so the user's intent — "I want Claude" vs
+ * "I want Gemini" — survives the swap), then falls back to a sensible
+ * vision default (Sonnet 4.6 — agent-tuned, mid-tier price).
+ */
+export declare function pickVisionSibling(modelId: string): string;
+/**
+ * Does this user-typed message reference an image file? Used by the router
+ * to bump Auto mode to a vision-capable tier, and by the manual-mode guard
+ * to swap a text-only model for one turn.
+ *
+ * Detection is intentionally a regex over file extensions rather than a
+ * filesystem stat — the user may type a path that doesn't yet exist
+ * (about to wget it) or a glob; what we care about is "does the model need
+ * eyes for this turn?" The false-positive risk is benign (we route to a
+ * slightly stronger model than strictly needed).
+ */
+export declare function messageNeedsVision(text: string | undefined | null): boolean;
+/**
+ * Messages-array variant: scans OpenAI- and Anthropic-format content blocks
+ * for explicit image parts (image / image_url / input_image) and for image
+ * paths embedded in text parts. Used by the proxy router which receives a
+ * fully-formed messages[] payload, not a single string.
+ */
+export declare function messagesNeedVision(messages: Array<{
+    role?: string;
+    content?: unknown;
+}> | undefined | null): boolean;

package/dist/router/vision.js

@@ -0,0 +1,127 @@
+/**
+ * Vision capability + image-attachment detection.
+ *
+ * Two jobs:
+ *   1. isVisionModel(id)        — does this gateway model accept image input?
+ *   2. messageNeedsVision(text) — does this user message reference an image?
+ *
+ * Source of truth: a hand-curated allowlist below. The gateway exposes a
+ * 'vision' category on /v1/models, but resolving it at routing time would
+ * make routeRequest async and gate sync proxy paths on a network call. The
+ * allowlist is small (~18 entries) and changes only when models do, which
+ * already touches the router + pricing tables — updating one more file is
+ * the right tradeoff vs. async fan-out across every routing callsite.
+ *
+ * Background: prior to this module, Auto routing could pick a text-only model
+ * (e.g. deepseek-v4-pro) on an image-bearing turn. The Read tool would still
+ * inline image bytes, the gateway would tokenize the base64 as text, and the
+ * model — having no vision pathway — would hallucinate based on the
+ * `Image file: <path>` description string. Expensive AND wrong.
+ */
+const VISION_MODELS = new Set([
+    // Anthropic — native vision across the line
+    'anthropic/claude-opus-4.7',
+    'anthropic/claude-opus-4.6',
+    'anthropic/claude-sonnet-4.6',
+    'anthropic/claude-haiku-4.5-20251001',
+    // OpenAI — multimodal flagships + o3 (Codex 5.3 is text-only, excluded)
+    'openai/gpt-5.5',
+    'openai/gpt-5.4',
+    'openai/gpt-5.4-pro',
+    'openai/gpt-5.2',
+    'openai/gpt-5.2-pro',
+    'openai/gpt-5-mini',
+    'openai/gpt-4.1',
+    'openai/o3',
+    // Google — vision baked into every Gemini SKU we surface
+    'google/gemini-3.1-pro',
+    'google/gemini-2.5-pro',
+    'google/gemini-2.5-flash',
+    // xAI — only Grok 4 base supports vision; grok-4-1-fast-reasoning is text-only
+    'xai/grok-4-0709',
+    'xai/grok-3',
+    // Moonshot — K2.6 added vision + reasoning when it replaced K2.5
+    'moonshot/kimi-k2.6',
+    // NVIDIA inference — Llama 4 Maverick is multimodal; deepseek/qwen-coder are not
+    'nvidia/llama-4-maverick',
+]);
+/** Does this concrete gateway model accept image input? */
+export function isVisionModel(modelId) {
+    if (!modelId)
+        return false;
+    return VISION_MODELS.has(modelId);
+}
+/** Lower-cased copy used for prefix family matching in pickVisionSibling. */
+const VISION_MODELS_LIST = Array.from(VISION_MODELS);
+/**
+ * Pick a vision-capable replacement closest to the user's chosen model.
+ * Prefers same provider family (so the user's intent — "I want Claude" vs
+ * "I want Gemini" — survives the swap), then falls back to a sensible
+ * vision default (Sonnet 4.6 — agent-tuned, mid-tier price).
+ */
+export function pickVisionSibling(modelId) {
+    const family = modelId.split('/')[0]?.toLowerCase();
+    if (family) {
+        const sibling = VISION_MODELS_LIST.find(m => m.startsWith(`${family}/`));
+        if (sibling)
+            return sibling;
+    }
+    return 'anthropic/claude-sonnet-4.6';
+}
+// Image file extensions Franklin's Read tool inlines as vision content. Keep
+// this in sync with IMAGE_MEDIA_TYPES in src/tools/read.ts — if Read learns a
+// new format (e.g. .avif), this regex needs to learn it too or routing will
+// silently miss it.
+//
+// We match the basename only ("foo.png"), preceded by any path separator or
+// punctuation. Trying to match full path prefixes ("./", "/", "~/", "C:\")
+// in one regex produced false negatives on Windows-style paths because of
+// the `:` and `\` separators. The basename anchor is enough — a bare
+// `foo.png` reference is what the Read tool actually needs to inline bytes.
+const IMAGE_PATH_RE = /(?:^|[\s"'`(<\[\\/])[\w@%+-]+\.(?:png|jpe?g|gif|webp)(?=$|[\s"'`)>\],.?!:;])/i;
+/**
+ * Does this user-typed message reference an image file? Used by the router
+ * to bump Auto mode to a vision-capable tier, and by the manual-mode guard
+ * to swap a text-only model for one turn.
+ *
+ * Detection is intentionally a regex over file extensions rather than a
+ * filesystem stat — the user may type a path that doesn't yet exist
+ * (about to wget it) or a glob; what we care about is "does the model need
+ * eyes for this turn?" The false-positive risk is benign (we route to a
+ * slightly stronger model than strictly needed).
+ */
+export function messageNeedsVision(text) {
+    if (!text)
+        return false;
+    return IMAGE_PATH_RE.test(text);
+}
+/**
+ * Messages-array variant: scans OpenAI- and Anthropic-format content blocks
+ * for explicit image parts (image / image_url / input_image) and for image
+ * paths embedded in text parts. Used by the proxy router which receives a
+ * fully-formed messages[] payload, not a single string.
+ */
+export function messagesNeedVision(messages) {
+    if (!messages || messages.length === 0)
+        return false;
+    for (const msg of messages) {
+        if (msg.role && msg.role !== 'user')
+            continue;
+        const content = msg.content;
+        if (typeof content === 'string') {
+            if (messageNeedsVision(content))
+                return true;
+            continue;
+        }
+        if (!Array.isArray(content))
+            continue;
+        for (const part of content) {
+            const t = part?.type;
+            if (t === 'image' || t === 'image_url' || t === 'input_image')
+                return true;
+            if (t === 'text' && messageNeedsVision(part.text))
+                return true;
+        }
+    }
+    return false;
+}

package/dist/tools/read.js

@@ -103,12 +103,64 @@ async function execute(input, ctx) {
                     output: `Image file: ${resolved} (${ext}, ${sizeStr}). Too large to inline for vision (>${Math.round(IMAGE_MAX_BYTES / 1_000_000)}MB). Resize or crop first.`,
                 };
             }
-            const bytes = fs.readFileSync(resolved);
-            const base64 = bytes.toString('base64');
+            // Client-side normalization to bound vision-token cost. The BlockRun
+            // gateway (verified 2026-05-09) tokenizes image base64 as text on the
+            // /v1/messages forward path, so a 1.9MB PNG → ~2.5M base64 chars →
+            // ~1.36M billed tokens (~$0.50 per call) instead of Anthropic's
+            // native vision tokenization (~1.6k tokens). Resizing the long edge
+            // to 1280px and re-encoding as JPEG q85 cuts payload to ~80KB while
+            // keeping vision usable. Skip work if the file is already small;
+            // preserve PNG when transparency matters (alpha sample).
+            const SKIP_BELOW_BYTES = 150 * 1024;
+            const MAX_LONG_EDGE = 1280;
+            const JPEG_QUALITY = 85;
+            const rawBytes = fs.readFileSync(resolved);
+            let outBytes = rawBytes;
+            let outMedia = IMAGE_MEDIA_TYPES[ext];
+            let normalizeNote = '';
+            if (stat.size > SKIP_BELOW_BYTES) {
+                try {
+                    const sharpMod = await import('sharp');
+                    const sharp = sharpMod.default;
+                    const img = sharp(rawBytes, { failOn: 'none' });
+                    const meta = await img.metadata();
+                    const longEdge = Math.max(meta.width ?? 0, meta.height ?? 0);
+                    // Detect transparency: GIF/WebP/PNG with non-opaque alpha → keep PNG.
+                    let hasAlpha = false;
+                    if (meta.hasAlpha) {
+                        const stats = await sharp(rawBytes).stats();
+                        const alpha = stats.channels[stats.channels.length - 1];
+                        hasAlpha = alpha?.min !== undefined && alpha.min < 255;
+                    }
+                    let pipeline = sharp(rawBytes, { failOn: 'none' });
+                    if (longEdge > MAX_LONG_EDGE) {
+                        pipeline = pipeline.resize({
+                            width: meta.width && meta.width >= (meta.height ?? 0) ? MAX_LONG_EDGE : undefined,
+                            height: meta.height && meta.height > (meta.width ?? 0) ? MAX_LONG_EDGE : undefined,
+                            fit: 'inside',
+                            withoutEnlargement: true,
+                        });
+                    }
+                    if (hasAlpha) {
+                        outBytes = await pipeline.png({ compressionLevel: 9 }).toBuffer();
+                        outMedia = 'image/png';
+                    }
+                    else {
+                        outBytes = await pipeline.jpeg({ quality: JPEG_QUALITY, mozjpeg: true }).toBuffer();
+                        outMedia = 'image/jpeg';
+                    }
+                    const outKb = (outBytes.length / 1024).toFixed(1);
+                    normalizeNote = ` Normalized: ${sizeStr} → ${outKb}KB (${meta.width}×${meta.height}${longEdge > MAX_LONG_EDGE ? ` → long edge ${MAX_LONG_EDGE}` : ''}, ${hasAlpha ? 'PNG/alpha' : `JPEG q${JPEG_QUALITY}`}).`;
+                }
+                catch {
+                    // Best-effort — if sharp fails, fall through with raw bytes.
+                }
+            }
+            const base64 = outBytes.toString('base64');
             fileReadTracker.set(resolved, { mtimeMs: stat.mtimeMs, readAt: Date.now() });
             return {
-                output: `Image file: ${resolved} (${ext}, ${sizeStr}). Rendered below for vision-capable models.`,
-                images: [{ mediaType: IMAGE_MEDIA_TYPES[ext], base64 }],
+                output: `Image file: ${resolved} (${ext}, ${sizeStr}).${normalizeNote} Rendered below for vision-capable models.`,
+                images: [{ mediaType: outMedia, base64 }],
             };
         }
         const binaryExts = new Set(['.ico', '.bmp', '.pdf', '.zip', '.tar', '.gz', '.woff', '.woff2', '.ttf', '.eot', '.mp3', '.mp4', '.wav', '.avi', '.mov', '.exe', '.dll', '.so', '.dylib']);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blockrun/franklin",
-  "version": "3.15.89",
+  "version": "3.15.91",
   "description": "Franklin — The AI agent with a wallet. Spends USDC autonomously to get real work done. Pay per action, no subscriptions.",
   "type": "module",
   "exports": {
@@ -80,6 +80,7 @@
     "playwright-core": "^1.49.1",
     "qrcode": "^1.5.4",
     "react": "^19.2.4",
+    "sharp": "^0.34.5",
     "viem": "^2.48.1"
   },
   "devDependencies": {