@sylphx/image-reader-mcp 0.1.2 → 0.1.3

package/README.md

@@ -43,6 +43,8 @@ Iris is **local-first**: geometry + OCR + **layout blocks** + **agent_map** so a
 
 Spec: [docs/specs/agent-image-read-contract.md](docs/specs/agent-image-read-contract.md)
 
+**Local-first frontier:** Rust decode, Tesseract native layout (no npm ML), optional Ollama VLM; cloud URL optional. Zero API key.
+
 ## Product docs
 
 | Doc | Purpose |

package/dist/doctor-cli.js

@@ -6501,9 +6501,11 @@ var listTesseractLanguages = () => {
   }
   const result = spawnSync("tesseract", ["--list-langs"], {
     encoding: "utf8",
-    timeout: 5000,
+    timeout: OCR_HEALTHCHECK_TIMEOUT_MS,
     windowsHide: true
   });
+  const stdout = typeof result.stdout === "string" ? result.stdout : "";
+  const languages = stdout.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.toLowerCase().includes("list of available"));
   if (result.status !== 0) {
     const stderr = typeof result.stderr === "string" ? result.stderr.trim() : "";
     return {
@@ -6512,9 +6514,6 @@ var listTesseractLanguages = () => {
       warning: stderr || "tesseract --list-langs failed"
     };
   }
-  const out = `${result.stdout ?? ""}
-${result.stderr ?? ""}`;
-  const languages = out.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.toLowerCase().includes("list of available languages") && !l.startsWith("Error"));
   return { available: true, languages };
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sylphx/image-reader-mcp",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "mcpName": "io.github.SylphxAI/image-reader-mcp",
   "description": "Iris \u2014 Evidence-first image reading for AI agents \u2014 metadata, OCR text, regions, and citeable evidence without generative LLM.",
   "type": "module",

package/src/handlers/readImage.ts

@@ -202,6 +202,7 @@ export const readImage = tool()
             ? { languages_warning: ocr.languages_warning }
             : {}),
           ...(ocr.words !== undefined ? { words: ocr.words } : {}),
+          ...(ocr.native_blocks !== undefined ? { native_blocks: ocr.native_blocks } : {}),
         };
       }
 

package/src/schemas/readImage.ts

@@ -79,7 +79,7 @@ export const readImageArgsSchema = z.object({
     .boolean()
     .optional()
     .describe(
-      'Optional caption via IRIS_OPTIONAL_LLM_URL. Off by default; never authority over OCR/layout evidence.'
+      'Optional local frontier caption via Ollama vision models or IRIS_OPTIONAL_LLM_URL. Off by default; never authority over OCR/layout evidence.'
     ),
 });
 
@@ -110,6 +110,17 @@ export const agentMediaTwinSchema = z.object({
           })
         )
         .optional(),
+      native_blocks: z
+        .array(
+          z.object({
+            id: z.string(),
+            kind: z.enum(['block', 'paragraph']),
+            text: z.string(),
+            bbox: boundingBoxSchema,
+            confidence: z.number().optional(),
+          })
+        )
+        .optional(),
     })
     .optional(),
   region_evidence: z
@@ -157,6 +168,7 @@ export const agentMediaTwinSchema = z.object({
           skipped_reason: z.string().optional(),
           route: z.string().optional(),
           caption: z.string().optional(),
+          model: z.string().optional(),
         })
         .optional(),
     })

package/src/utils/agentMap.ts

@@ -20,6 +20,7 @@ export type AgentImageMap = {
     skipped_reason?: string;
     route?: string;
     caption?: string;
+    model?: string;
   };
 };
 

package/src/utils/applyImageIntelligence.ts

@@ -1,6 +1,6 @@
 import type { AgentMediaTwin, ReadImageArgs } from '../schemas/readImage.js';
 import { buildAgentImageMap } from './agentMap.js';
-import { buildLayoutFromOcrLines } from './layout.js';
+import { buildBestEffortLayout } from './layout.js';
 import { maybeOptionalImageCaption } from './optionalLlm.js';
 import { samplePalette } from './palette.js';
 
@@ -18,14 +18,30 @@ export async function applyImageIntelligence(
 
   const includeLayout = input.include_layout ?? includeOcr;
   if (includeLayout && next.ocr?.lines && next.ocr.lines.length > 0) {
-    const layout = buildLayoutFromOcrLines(
-      next.ocr.lines.map((line) => ({
+    const ocrAny = next.ocr as {
+      lines: Array<{
+        text: string;
+        bbox: { x: number; y: number; width: number; height: number };
+        confidence?: number;
+      }>;
+      native_blocks?: Array<{
+        id: string;
+        kind: 'block' | 'paragraph';
+        text: string;
+        bbox: { x: number; y: number; width: number; height: number };
+        confidence?: number;
+      }>;
+    };
+    const layout = buildBestEffortLayout({
+      lines: ocrAny.lines.map((line) => ({
         text: line.text,
         bbox: line.bbox,
         ...(line.confidence !== undefined ? { confidence: line.confidence } : {}),
-      }))
-    );
+      })),
+      ...(ocrAny.native_blocks ? { nativeBlocks: ocrAny.native_blocks } : {}),
+    });
     next.layout = layout;
+    next.trust_warnings.push(`layout_policy: ${layout.policy}`);
     for (const warning of layout.warnings) {
       next.trust_warnings.push(`layout: ${warning}`);
     }

package/src/utils/layout.ts

@@ -1,5 +1,5 @@
 /**
- * Deterministic layout structure from OCR line boxes (no generative model).
+ * Local-first layout: prefer Tesseract native block/paragraph units; fall back to line clustering.
  */
 
 export type BBox = { x: number; y: number; width: number; height: number };
@@ -17,6 +17,7 @@ export type LayoutBlock = {
   bbox: BBox;
   line_count: number;
   reading_order: number;
+  source?: 'tesseract_native' | 'line_cluster';
 };
 
 export type ImageLayout = {
@@ -27,6 +28,14 @@ export type ImageLayout = {
   warnings: string[];
 };
 
+export type NativeBlockInput = {
+  id: string;
+  kind: 'block' | 'paragraph';
+  text: string;
+  bbox: BBox;
+  confidence?: number;
+};
+
 const unionBBox = (boxes: BBox[]): BBox => {
   const minX = Math.min(...boxes.map((b) => b.x));
   const minY = Math.min(...boxes.map((b) => b.y));
@@ -50,7 +59,7 @@ const shouldMerge = (prev: LayoutLine, next: LayoutLine, yGap: number, xSlop: nu
   return verticalClose && horizontalOverlap;
 };
 
-const toBlock = (lines: LayoutLine[], index: number): LayoutBlock => {
+const toClusterBlock = (lines: LayoutLine[], index: number): LayoutBlock => {
   const ordered = [...lines].sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x);
   const text = ordered
     .map((l) => l.text.trim())
@@ -63,9 +72,35 @@ const toBlock = (lines: LayoutLine[], index: number): LayoutBlock => {
     bbox: unionBBox(ordered.map((l) => l.bbox)),
     line_count: ordered.length,
     reading_order: index + 1,
+    source: 'line_cluster',
   };
 };
 
+export function buildLayoutFromNativeBlocks(native: NativeBlockInput[]): ImageLayout {
+  const usable = native
+    .filter((b) => b.text.trim().length > 0 && b.bbox.width > 0 && b.bbox.height > 0)
+    .sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x);
+  const blocks: LayoutBlock[] = usable.map((b, index) => ({
+    id: b.id || `native-${index + 1}`,
+    kind: 'text_block',
+    text: b.text.trim(),
+    bbox: b.bbox,
+    line_count: Math.max(1, b.text.split(/\n/).length),
+    reading_order: index + 1,
+    source: 'tesseract_native',
+  }));
+  return {
+    policy: 'tesseract_native_layout_v1',
+    block_count: blocks.length,
+    blocks,
+    full_text: blocks.map((b) => b.text).join('\n\n'),
+    warnings:
+      blocks.length === 0
+        ? ['Native Tesseract blocks present but empty text; falling back recommended.']
+        : [],
+  };
+}
+
 export function buildLayoutFromOcrLines(lines: LayoutLine[]): ImageLayout {
   if (lines.length === 0) {
     return {
@@ -96,7 +131,7 @@ export function buildLayoutFromOcrLines(lines: LayoutLine[]): ImageLayout {
     }
   }
 
-  const blocks = clusters.map((cluster, index) => toBlock(cluster, index));
+  const blocks = clusters.map((cluster, index) => toClusterBlock(cluster, index));
   const warnings: string[] = [];
   if (blocks.length === 1 && lines.length > 8) {
     warnings.push(
@@ -112,3 +147,15 @@ export function buildLayoutFromOcrLines(lines: LayoutLine[]): ImageLayout {
     warnings,
   };
 }
+
+/** Local-first frontier layout: native Tesseract structure first, heuristic second. */
+export function buildBestEffortLayout(input: {
+  lines: LayoutLine[];
+  nativeBlocks?: NativeBlockInput[];
+}): ImageLayout {
+  if (input.nativeBlocks?.some((b) => b.text.trim().length > 0)) {
+    const native = buildLayoutFromNativeBlocks(input.nativeBlocks);
+    if (native.block_count > 0) return native;
+  }
+  return buildLayoutFromOcrLines(input.lines);
+}

package/src/utils/ocr.ts

@@ -10,6 +10,17 @@ export interface OcrWord {
   confidence?: number;
 }
 
+/** Native Tesseract layout unit (block or paragraph) — frontier-local structure free with classic OCR. */
+export interface OcrNativeBlock {
+  id: string;
+  kind: 'block' | 'paragraph';
+  text: string;
+  bbox: { x: number; y: number; width: number; height: number };
+  block_num?: number;
+  par_num?: number;
+  confidence?: number;
+}
+
 export interface OcrResult {
   available: boolean;
   skipped_reason?: string;
@@ -21,6 +32,8 @@ export interface OcrResult {
   dropped_low_confidence?: number;
   lines: OcrLine[];
   words?: OcrWord[];
+  /** Prefer for layout when present (Tesseract levels 2/3). */
+  native_blocks?: OcrNativeBlock[];
 }
 
 export interface ParseTesseractOptions {
@@ -28,21 +41,31 @@ export interface ParseTesseractOptions {
   includeWords?: boolean;
 }
 
-/** Pure TSV parser — offline-testable evidence path for OCR bbox lines. */
+const num = (v: string | undefined): number => Number.parseFloat(v ?? '');
+
+/**
+ * Pure TSV parser — lines (level 4 aggregated from words), words (level 5),
+ * and native blocks/paragraphs (levels 2/3) for local-first layout frontier.
+ */
 export const parseTesseractTsv = (
   raw: string,
   options: ParseTesseractOptions = {}
-): { lines: OcrLine[]; words: OcrWord[]; dropped_low_confidence: number } => {
+): {
+  lines: OcrLine[];
+  words: OcrWord[];
+  native_blocks: OcrNativeBlock[];
+  dropped_low_confidence: number;
+} => {
   const minConfidence = options.minConfidence ?? 0;
   const includeWords = options.includeWords ?? false;
   const linesRaw = raw.split(/\r?\n/).filter((line) => line.length > 0);
   if (linesRaw.length <= 1) {
-    return { lines: [], words: [], dropped_low_confidence: 0 };
+    return { lines: [], words: [], native_blocks: [], dropped_low_confidence: 0 };
   }
 
   const rows = linesRaw.slice(1).map((line) => line.split('\t'));
   const lineMap = new Map<
-    number,
+    string,
     {
       words: Array<{
         text: string;
@@ -57,22 +80,47 @@ export const parseTesseractTsv = (
 
   let dropped = 0;
   const flatWords: OcrWord[] = [];
+  const nativeBlocks: OcrNativeBlock[] = [];
 
   for (const columns of rows) {
     if (columns.length < 12) continue;
     const level = Number.parseInt(columns[0] ?? '', 10);
-    if (level !== 5) continue;
-
-    const text = columns[11]?.trim() ?? '';
-    if (text.length === 0) continue;
-
+    const blockNum = Number.parseInt(columns[2] ?? '', 10);
+    const parNum = Number.parseInt(columns[3] ?? '', 10);
     const lineNum = Number.parseInt(columns[4] ?? '', 10);
     const left = Number.parseInt(columns[6] ?? '', 10);
     const top = Number.parseInt(columns[7] ?? '', 10);
     const width = Number.parseInt(columns[8] ?? '', 10);
     const height = Number.parseInt(columns[9] ?? '', 10);
-    const conf = Number.parseFloat(columns[10] ?? '');
+    const conf = num(columns[10]);
+    const text = columns[11]?.trim() ?? '';
+
+    // Native layout units from Tesseract itself (better than pure heuristic when present)
+    if ((level === 2 || level === 3) && Number.isFinite(left) && Number.isFinite(top)) {
+      if (text.length > 0 || level === 2) {
+        // level 2 blocks often have empty text; still keep geometry if non-zero
+        if (width > 0 && height > 0) {
+          const nb: OcrNativeBlock = {
+            id: level === 2 ? `tess-block-${blockNum}` : `tess-par-${blockNum}-${parNum}`,
+            kind: level === 2 ? 'block' : 'paragraph',
+            text,
+            bbox: {
+              x: left,
+              y: top,
+              width: Number.isFinite(width) ? width : 0,
+              height: Number.isFinite(height) ? height : 0,
+            },
+          };
+          if (Number.isFinite(blockNum)) nb.block_num = blockNum;
+          if (Number.isFinite(parNum)) nb.par_num = parNum;
+          if (Number.isFinite(conf) && conf >= 0) nb.confidence = conf;
+          nativeBlocks.push(nb);
+        }
+      }
+    }
 
+    if (level !== 5) continue;
+    if (text.length === 0) continue;
     if (!Number.isFinite(lineNum) || !Number.isFinite(left) || !Number.isFinite(top)) continue;
 
     const confVal = Number.isFinite(conf) ? conf : 0;
@@ -81,7 +129,8 @@ export const parseTesseractTsv = (
       continue;
     }
 
-    const bucket = lineMap.get(lineNum) ?? { words: [] };
+    const lineKey = `${blockNum}:${parNum}:${lineNum}`;
+    const bucket = lineMap.get(lineKey) ?? { words: [] };
     bucket.words.push({
       text,
       left,
@@ -90,7 +139,7 @@ export const parseTesseractTsv = (
       height: Number.isFinite(height) ? height : 0,
       conf: confVal,
     });
-    lineMap.set(lineNum, bucket);
+    lineMap.set(lineKey, bucket);
 
     if (includeWords) {
       flatWords.push({
@@ -106,18 +155,34 @@ export const parseTesseractTsv = (
     }
   }
 
-  const ocrLines: OcrLine[] = [];
+  // Fill empty native block text from aggregated words in that block
+  const wordsByBlock = new Map<number, string[]>();
+  for (const [key, bucket] of lineMap) {
+    const blockNum = Number.parseInt(key.split(':')[0] ?? '', 10);
+    if (!Number.isFinite(blockNum)) continue;
+    const list = wordsByBlock.get(blockNum) ?? [];
+    const sorted = [...bucket.words].sort((a, b) => a.top - b.top || a.left - b.left);
+    list.push(sorted.map((w) => w.text).join(' '));
+    wordsByBlock.set(blockNum, list);
+  }
+  for (const block of nativeBlocks) {
+    if (block.text.length > 0) continue;
+    if (block.block_num === undefined) continue;
+    const parts = wordsByBlock.get(block.block_num);
+    if (parts && parts.length > 0) {
+      block.text = parts.join('\n').trim();
+    }
+  }
 
+  const ocrLines: OcrLine[] = [];
   for (const bucket of lineMap.values()) {
     if (bucket.words.length === 0) continue;
-
     const sorted = [...bucket.words].sort((a, b) => a.left - b.left);
     const text = sorted
       .map((word) => word.text)
       .join(' ')
       .trim();
     if (text.length === 0) continue;
-
     const left = Math.min(...sorted.map((word) => word.left));
     const top = Math.min(...sorted.map((word) => word.top));
     const right = Math.max(...sorted.map((word) => word.left + word.width));
@@ -127,7 +192,6 @@ export const parseTesseractTsv = (
       confidenceValues.length > 0
         ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
         : undefined;
-
     ocrLines.push({
       text,
       bbox: {
@@ -140,9 +204,20 @@ export const parseTesseractTsv = (
     });
   }
 
+  // Prefer paragraph native blocks with text; else blocks
+  const paragraphs = nativeBlocks.filter((b) => b.kind === 'paragraph' && b.text.length > 0);
+  const blocksOnly = nativeBlocks.filter((b) => b.kind === 'block');
+  const preferredNative =
+    paragraphs.length > 0
+      ? paragraphs
+      : blocksOnly.filter((b) => b.text.length > 0).length > 0
+        ? blocksOnly.filter((b) => b.text.length > 0)
+        : blocksOnly;
+
   return {
     lines: ocrLines.sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x),
     words: includeWords ? flatWords.sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x) : [],
+    native_blocks: preferredNative.sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x),
     dropped_low_confidence: dropped,
   };
 };
@@ -156,7 +231,6 @@ export const isTesseractAvailable = (): boolean => {
   return result.status === 0;
 };
 
-/** List installed Tesseract traineddata languages (honest empty when unavailable). */
 export const listTesseractLanguages = (): {
   available: boolean;
   languages: string[];
@@ -171,9 +245,14 @@ export const listTesseractLanguages = (): {
   }
   const result = spawnSync('tesseract', ['--list-langs'], {
     encoding: 'utf8',
-    timeout: 5_000,
+    timeout: OCR_HEALTHCHECK_TIMEOUT_MS,
     windowsHide: true,
   });
+  const stdout = typeof result.stdout === 'string' ? result.stdout : '';
+  const languages = stdout
+    .split(/\r?\n/)
+    .map((l) => l.trim())
+    .filter((l) => l && !l.toLowerCase().includes('list of available'));
   if (result.status !== 0) {
     const stderr = typeof result.stderr === 'string' ? result.stderr.trim() : '';
     return {
@@ -182,13 +261,6 @@ export const listTesseractLanguages = (): {
       warning: stderr || 'tesseract --list-langs failed',
     };
   }
-  const out = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
-  const languages = out
-    .split(/\r?\n/)
-    .map((l) => l.trim())
-    .filter(
-      (l) => l && !l.toLowerCase().includes('list of available languages') && !l.startsWith('Error')
-    );
   return { available: true, languages };
 };
 
@@ -205,7 +277,7 @@ export const runTesseractOcr = (
   const options: RunOcrOptions = Array.isArray(languagesOrOptions)
     ? { languages: languagesOrOptions }
     : languagesOrOptions;
-  const languages = options.languages?.length ? options.languages : ['eng'];
+  const languages = options.languages ?? ['eng'];
   const minConfidence = options.minConfidence ?? 0;
   const includeWords = options.includeWords ?? false;
 
@@ -218,6 +290,7 @@ export const runTesseractOcr = (
       lines: [],
       line_count: 0,
       dropped_low_confidence: 0,
+      native_blocks: [],
     };
   }
 
@@ -227,22 +300,27 @@ export const runTesseractOcr = (
   );
 
   const languageArg = languages.join('+');
-  const result = spawnSync('tesseract', [imagePath, 'stdout', '-l', languageArg, 'tsv'], {
-    encoding: 'utf8',
-    timeout: OCR_TIMEOUT_MS,
-    windowsHide: true,
-    maxBuffer: 10 * 1024 * 1024,
-  });
+  const result = spawnSync(
+    'tesseract',
+    [imagePath, 'stdout', '-l', languageArg, 'tsv', '--psm', '3'],
+    {
+      encoding: 'utf8',
+      timeout: OCR_TIMEOUT_MS,
+      windowsHide: true,
+      maxBuffer: 20 * 1024 * 1024,
+    }
+  );
 
   if (result.error) {
     return {
       available: false,
-      skipped_reason: `Tesseract failed to start: ${result.error.message}`,
+      skipped_reason: result.error.message,
       route: 'tesseract_tsv',
       languages,
       lines: [],
       line_count: 0,
       dropped_low_confidence: 0,
+      native_blocks: [],
     };
   }
 
@@ -257,6 +335,7 @@ export const runTesseractOcr = (
       lines: [],
       line_count: 0,
       dropped_low_confidence: 0,
+      native_blocks: [],
     };
   }
 
@@ -264,11 +343,12 @@ export const runTesseractOcr = (
   const parsed = parseTesseractTsv(stdout, { minConfidence, includeWords });
   return {
     available: true,
-    route: 'tesseract_tsv',
+    route: 'tesseract_tsv_psm3_native_layout',
     languages,
     lines: parsed.lines,
     line_count: parsed.lines.length,
     dropped_low_confidence: parsed.dropped_low_confidence,
+    native_blocks: parsed.native_blocks,
     ...(missingLangs.length
       ? {
           languages_warning: `Requested OCR language(s) not listed by tesseract --list-langs: ${missingLangs.join(', ')}. Installed: ${installed.languages.join(', ') || '(none)'}.`,

package/src/utils/optionalLlm.ts

@@ -1,36 +1,51 @@
 /**
- * Optional local/remote vision caption — OFF by default (local-first).
- * Set IRIS_OPTIONAL_LLM_URL to a POST endpoint that accepts JSON
- * { path, mime, purpose: "image_caption" } and returns { caption: string }.
- * Never treated as authority over OCR/layout evidence.
+ * Local-first frontier optional vision caption.
+ *
+ * Priority:
+ * 1) IRIS_OPTIONAL_LLM_URL — custom POST { path, mime, purpose } → { caption }
+ * 2) Local Ollama at IRIS_OLLAMA_URL (default http://127.0.0.1:11434) if a vision model is installed
+ *
+ * Never authority over OCR/layout locators. OFF unless include_optional_llm=true.
  */
 
+import { readFileSync } from 'node:fs';
+
 export type OptionalLlmResult = {
   available: boolean;
   skipped_reason?: string;
   route?: string;
   caption?: string;
+  model?: string;
 };
 
-export async function maybeOptionalImageCaption(input: {
+const DEFAULT_OLLAMA = 'http://127.0.0.1:11434';
+
+const VISION_MODEL_HINTS = [
+  'llava',
+  'minicpm-v',
+  'minicpm_v',
+  'qwen2-vl',
+  'qwen2.5-vl',
+  'qwen3-vl',
+  'bakllava',
+  'moondream',
+  'gemma3',
+  'llama3.2-vision',
+  'pixtral',
+];
+
+function isVisionModel(name: string): boolean {
+  const n = name.toLowerCase();
+  return VISION_MODEL_HINTS.some((h) => n.includes(h));
+}
+
+async function tryCustomUrl(input: {
   path: string;
   mime: string;
-  enabled: boolean;
+  url: string;
 }): Promise<OptionalLlmResult> {
-  if (!input.enabled) {
-    return { available: false, skipped_reason: 'include_optional_llm is false (default).' };
-  }
-  const url = process.env['IRIS_OPTIONAL_LLM_URL'];
-  if (!url) {
-    return {
-      available: false,
-      skipped_reason:
-        'IRIS_OPTIONAL_LLM_URL is not set. Local OCR/layout remain the evidence authority.',
-    };
-  }
-
   try {
-    const res = await fetch(url, {
+    const res = await fetch(input.url, {
       method: 'POST',
       headers: { 'content-type': 'application/json' },
       body: JSON.stringify({
@@ -38,13 +53,13 @@ export async function maybeOptionalImageCaption(input: {
         mime: input.mime,
         purpose: 'image_caption',
       }),
-      signal: AbortSignal.timeout(15_000),
+      signal: AbortSignal.timeout(20_000),
     });
     if (!res.ok) {
       return {
         available: false,
         skipped_reason: `optional LLM HTTP ${res.status}`,
-        route: url,
+        route: input.url,
       };
     }
     const body = (await res.json()) as { caption?: string };
@@ -52,16 +67,162 @@ export async function maybeOptionalImageCaption(input: {
       return {
         available: false,
         skipped_reason: 'optional LLM response missing caption string',
-        route: url,
+        route: input.url,
       };
     }
     return {
       available: true,
-      route: `optional-llm:${url}`,
+      route: `optional-llm:${input.url}`,
       caption: body.caption.slice(0, 4000),
     };
   } catch (error: unknown) {
     const message = error instanceof Error ? error.message : 'optional LLM failed';
-    return { available: false, skipped_reason: message, route: url };
+    return { available: false, skipped_reason: message, route: input.url };
+  }
+}
+
+async function listOllamaModels(base: string): Promise<string[]> {
+  const res = await fetch(`${base.replace(/\/$/, '')}/api/tags`, {
+    signal: AbortSignal.timeout(3_000),
+  });
+  if (!res.ok) return [];
+  const body = (await res.json()) as { models?: Array<{ name?: string }> };
+  return (body.models ?? [])
+    .map((m) => m.name)
+    .filter((n): n is string => typeof n === 'string' && n.length > 0);
+}
+
+async function tryOllamaVision(input: {
+  path: string;
+  baseUrl: string;
+  modelHint?: string;
+}): Promise<OptionalLlmResult> {
+  const base = input.baseUrl.replace(/\/$/, '');
+  let models: string[] = [];
+  try {
+    models = await listOllamaModels(base);
+  } catch {
+    return {
+      available: false,
+      skipped_reason: `Ollama not reachable at ${base}`,
+      route: base,
+    };
   }
+  if (models.length === 0) {
+    return {
+      available: false,
+      skipped_reason: 'Ollama has no models installed',
+      route: base,
+    };
+  }
+
+  const preferred =
+    (input.modelHint &&
+      models.find((m) => m === input.modelHint || m.startsWith(`${input.modelHint}:`))) ||
+    models.find((m) => isVisionModel(m));
+  if (!preferred) {
+    return {
+      available: false,
+      skipped_reason: `No local vision model found in Ollama (have: ${models.slice(0, 6).join(', ')}). Install e.g. llava or minicpm-v.`,
+      route: base,
+    };
+  }
+
+  let b64: string;
+  try {
+    b64 = readFileSync(input.path).toString('base64');
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'read failed';
+    return { available: false, skipped_reason: message, route: base };
+  }
+
+  // Cap image payload ~4MB base64
+  if (b64.length > 5_500_000) {
+    return {
+      available: false,
+      skipped_reason: 'image too large for default Ollama caption path; crop_region first',
+      route: `ollama:${preferred}`,
+    };
+  }
+
+  try {
+    const res = await fetch(`${base}/api/chat`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        model: preferred,
+        stream: false,
+        messages: [
+          {
+            role: 'user',
+            content:
+              'Describe this image for a software agent. Focus on structure, UI/text regions, objects, and layout. Be concise and factual. Do not invent unreadable text.',
+            images: [b64],
+          },
+        ],
+      }),
+      signal: AbortSignal.timeout(60_000),
+    });
+    if (!res.ok) {
+      return {
+        available: false,
+        skipped_reason: `Ollama chat HTTP ${res.status}`,
+        route: `ollama:${preferred}`,
+        model: preferred,
+      };
+    }
+    const body = (await res.json()) as {
+      message?: { content?: string };
+      response?: string;
+    };
+    const caption = body.message?.content ?? body.response;
+    if (!caption || typeof caption !== 'string') {
+      return {
+        available: false,
+        skipped_reason: 'Ollama response missing message content',
+        route: `ollama:${preferred}`,
+        model: preferred,
+      };
+    }
+    return {
+      available: true,
+      route: `ollama-local-vlm:${preferred}`,
+      model: preferred,
+      caption: caption.slice(0, 4000),
+    };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'Ollama caption failed';
+    return {
+      available: false,
+      skipped_reason: message,
+      route: `ollama:${preferred}`,
+      model: preferred,
+    };
+  }
+}
+
+export async function maybeOptionalImageCaption(input: {
+  path: string;
+  mime: string;
+  enabled: boolean;
+}): Promise<OptionalLlmResult> {
+  if (!input.enabled) {
+    return { available: false, skipped_reason: 'include_optional_llm is false (default).' };
+  }
+
+  const custom = process.env['IRIS_OPTIONAL_LLM_URL'];
+  if (custom) {
+    return tryCustomUrl({ path: input.path, mime: input.mime, url: custom });
+  }
+
+  const ollamaBase = process.env['IRIS_OLLAMA_URL'] ?? process.env['OLLAMA_HOST'] ?? DEFAULT_OLLAMA;
+  // OLLAMA_HOST may be host:port without scheme
+  const baseUrl = ollamaBase.startsWith('http') ? ollamaBase : `http://${ollamaBase}`;
+  const modelHint = process.env['IRIS_OLLAMA_VISION_MODEL'] ?? process.env['OLLAMA_VISION_MODEL'];
+
+  return tryOllamaVision({
+    path: input.path,
+    baseUrl,
+    ...(modelHint ? { modelHint } : {}),
+  });
 }