@sylphx/image-reader-mcp 0.1.1 → 0.1.3

package/README.md

@@ -36,6 +36,15 @@ Each instrument is an independent repository (marketplace + stars).
 ---
 
 
+
+## Read images (not vague vision)
+
+Iris is **local-first**: geometry + OCR + **layout blocks** + **agent_map** so a text-only agent can understand picture architecture without a vision model.
+
+Spec: [docs/specs/agent-image-read-contract.md](docs/specs/agent-image-read-contract.md)
+
+**Local-first frontier:** Rust decode, Tesseract native layout (no npm ML), optional Ollama VLM; cloud URL optional. Zero API key.
+
 ## Product docs
 
 | Doc | Purpose |

package/dist/doctor-cli.js

@@ -6501,9 +6501,11 @@ var listTesseractLanguages = () => {
   }
   const result = spawnSync("tesseract", ["--list-langs"], {
     encoding: "utf8",
-    timeout: 5000,
+    timeout: OCR_HEALTHCHECK_TIMEOUT_MS,
     windowsHide: true
   });
+  const stdout = typeof result.stdout === "string" ? result.stdout : "";
+  const languages = stdout.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.toLowerCase().includes("list of available"));
   if (result.status !== 0) {
     const stderr = typeof result.stderr === "string" ? result.stderr.trim() : "";
     return {
@@ -6512,9 +6514,6 @@ var listTesseractLanguages = () => {
       warning: stderr || "tesseract --list-langs failed"
     };
   }
-  const out = `${result.stdout ?? ""}
-${result.stderr ?? ""}`;
-  const languages = out.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.toLowerCase().includes("list of available languages") && !l.startsWith("Error"));
   return { available: true, languages };
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sylphx/image-reader-mcp",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "mcpName": "io.github.SylphxAI/image-reader-mcp",
   "description": "Iris \u2014 Evidence-first image reading for AI agents \u2014 metadata, OCR text, regions, and citeable evidence without generative LLM.",
   "type": "module",

package/src/handlers/readImage.ts

@@ -9,6 +9,7 @@ import {
 } from '../engine/rust-decode.js';
 import { text, tool, toolError } from '../mcp.js';
 import { type AgentMediaTwin, readImageArgsSchema } from '../schemas/readImage.js';
+import { applyImageIntelligence } from '../utils/applyImageIntelligence.js';
 import { ErrorCode, ImageError } from '../utils/errors.js';
 import { collectTrustWarnings, redactGpsFields } from '../utils/metadata.js';
 import { runTesseractOcr } from '../utils/ocr.js';
@@ -74,7 +75,7 @@ const readMetadata = async (
 
 export const readImage = tool()
   .description(
-    'Evidence-first image reader. Returns an Agent Media Twin with filename, mime, dimensions, metadata, optional OCR lines with bounding boxes, and trust warnings. No generative LLM is used.'
+    'Evidence-first image reader for agents (read, not vague vision). Returns Agent Media Twin: geometry, metadata, OCR lines/words, layout blocks, text agent_map, optional palette, optional non-authority LLM caption. Local-first; generative path off by default.'
   )
   .input(readImageArgsSchema)
   .handler(async ({ input }) => {
@@ -201,9 +202,12 @@ export const readImage = tool()
             ? { languages_warning: ocr.languages_warning }
             : {}),
           ...(ocr.words !== undefined ? { words: ocr.words } : {}),
+          ...(ocr.native_blocks !== undefined ? { native_blocks: ocr.native_blocks } : {}),
         };
       }
 
+      twin = await applyImageIntelligence(twin, resolvedPath, input, includeOcr);
+
       if (input.region !== undefined) {
         if (!useRustDecode) {
           throw new ImageError(

package/src/schemas/readImage.ts

@@ -59,6 +59,28 @@ export const readImageArgsSchema = z.object({
     .positive()
     .optional()
     .describe('Maximum width or height when resizing the cropped region for evidence.'),
+  include_layout: z
+    .boolean()
+    .optional()
+    .describe(
+      'When OCR lines exist, cluster them into reading-order text blocks (image architecture). Defaults to true when include_ocr is true.'
+    ),
+  include_agent_map: z
+    .boolean()
+    .optional()
+    .describe(
+      'Return a text-only agent_map so non-vision models can read image structure. Defaults to true.'
+    ),
+  include_palette: z
+    .boolean()
+    .optional()
+    .describe('Sample an approximate local color palette via sharp (not ML). Defaults to false.'),
+  include_optional_llm: z
+    .boolean()
+    .optional()
+    .describe(
+      'Optional local frontier caption via Ollama vision models or IRIS_OPTIONAL_LLM_URL. Off by default; never authority over OCR/layout evidence.'
+    ),
 });
 
 export const agentMediaTwinSchema = z.object({
@@ -88,6 +110,17 @@ export const agentMediaTwinSchema = z.object({
           })
         )
         .optional(),
+      native_blocks: z
+        .array(
+          z.object({
+            id: z.string(),
+            kind: z.enum(['block', 'paragraph']),
+            text: z.string(),
+            bbox: boundingBoxSchema,
+            confidence: z.number().optional(),
+          })
+        )
+        .optional(),
     })
     .optional(),
   region_evidence: z
@@ -101,6 +134,45 @@ export const agentMediaTwinSchema = z.object({
       image_base64: z.string().optional(),
     })
     .optional(),
+  layout: z
+    .object({
+      policy: z.string(),
+      block_count: z.number().int().nonnegative(),
+      blocks: z.array(
+        z.object({
+          id: z.string(),
+          kind: z.literal('text_block'),
+          text: z.string(),
+          bbox: boundingBoxSchema,
+          line_count: z.number().int().nonnegative(),
+          reading_order: z.number().int().positive(),
+        })
+      ),
+      full_text: z.string(),
+      warnings: z.array(z.string()),
+    })
+    .optional(),
+  agent_map: z
+    .object({
+      policy: z.string(),
+      filename: z.string(),
+      mime: z.string(),
+      dimensions: imageDimensionsSchema,
+      outline: z.string(),
+      text_present: z.boolean(),
+      block_count: z.number().int().nonnegative(),
+      palette: z.array(z.object({ hex: z.string(), approx_share: z.number() })).optional(),
+      optional_llm: z
+        .object({
+          available: z.boolean(),
+          skipped_reason: z.string().optional(),
+          route: z.string().optional(),
+          caption: z.string().optional(),
+          model: z.string().optional(),
+        })
+        .optional(),
+    })
+    .optional(),
   trust_warnings: z.array(z.string()),
 });
 

package/src/utils/agentMap.ts

@@ -0,0 +1,94 @@
+/**
+ * Text-only Agent Map: lets a non-vision model "read" the image architecture.
+ * Deterministic, local-first. Optional LLM captions are layered elsewhere.
+ */
+
+import type { ImageLayout } from './layout.js';
+
+export type AgentImageMap = {
+  policy: 'agent_image_map_v1';
+  filename: string;
+  mime: string;
+  dimensions: { width: number; height: number };
+  /** Human/agent readable outline (markdown-ish, not prose hallucination). */
+  outline: string;
+  text_present: boolean;
+  block_count: number;
+  palette?: Array<{ hex: string; approx_share: number }>;
+  optional_llm?: {
+    available: boolean;
+    skipped_reason?: string;
+    route?: string;
+    caption?: string;
+    model?: string;
+  };
+};
+
+export function buildAgentImageMap(input: {
+  filename: string;
+  mime: string;
+  dimensions: { width: number; height: number };
+  layout?: ImageLayout;
+  ocrLineCount?: number;
+  palette?: Array<{ hex: string; approx_share: number }>;
+  optionalLlm?: AgentImageMap['optional_llm'];
+}): AgentImageMap {
+  const { width, height } = input.dimensions;
+  const blocks = input.layout?.blocks ?? [];
+  const textPresent = (input.ocrLineCount ?? 0) > 0 || blocks.length > 0;
+
+  const lines: string[] = [
+    `# Image map: ${input.filename}`,
+    `- mime: ${input.mime}`,
+    `- size: ${width}×${height}px`,
+    `- text_present: ${textPresent}`,
+    `- layout_blocks: ${blocks.length}`,
+  ];
+
+  if (input.palette && input.palette.length > 0) {
+    lines.push(
+      `- palette: ${input.palette.map((p) => `${p.hex}~${Math.round(p.approx_share * 100)}%`).join(', ')}`
+    );
+  }
+
+  if (blocks.length > 0) {
+    lines.push('', '## Reading-order text blocks');
+    for (const b of blocks) {
+      const preview = b.text.replace(/\s+/g, ' ').slice(0, 160);
+      lines.push(
+        `### ${b.id} (order ${b.reading_order}, ${b.line_count} lines)`,
+        `- bbox: x=${b.bbox.x} y=${b.bbox.y} w=${b.bbox.width} h=${b.bbox.height}`,
+        `- text: ${preview}${b.text.length > 160 ? '…' : ''}`,
+        ''
+      );
+    }
+  } else if (!textPresent) {
+    lines.push(
+      '',
+      '## Notes',
+      '- No OCR text recovered. Enable include_ocr for text architecture.',
+      '- Use crop_region / image_probe for geometry; optional LLM caption only if configured.'
+    );
+  }
+
+  if (input.optionalLlm?.available && input.optionalLlm.caption) {
+    lines.push('', '## Optional LLM caption (non-authority)', input.optionalLlm.caption);
+  } else if (input.optionalLlm && !input.optionalLlm.available) {
+    lines.push(
+      '',
+      `## Optional LLM: skipped (${input.optionalLlm.skipped_reason ?? 'not configured'})`
+    );
+  }
+
+  return {
+    policy: 'agent_image_map_v1',
+    filename: input.filename,
+    mime: input.mime,
+    dimensions: input.dimensions,
+    outline: lines.join('\n'),
+    text_present: textPresent,
+    block_count: blocks.length,
+    ...(input.palette ? { palette: input.palette } : {}),
+    ...(input.optionalLlm ? { optional_llm: input.optionalLlm } : {}),
+  };
+}

package/src/utils/applyImageIntelligence.ts

@@ -0,0 +1,84 @@
+import type { AgentMediaTwin, ReadImageArgs } from '../schemas/readImage.js';
+import { buildAgentImageMap } from './agentMap.js';
+import { buildBestEffortLayout } from './layout.js';
+import { maybeOptionalImageCaption } from './optionalLlm.js';
+import { samplePalette } from './palette.js';
+
+/** Attach layout, palette, optional LLM, and agent_map to an Agent Media Twin. */
+export async function applyImageIntelligence(
+  twin: AgentMediaTwin,
+  resolvedPath: string,
+  input: ReadImageArgs,
+  includeOcr: boolean
+): Promise<AgentMediaTwin> {
+  const next: AgentMediaTwin = {
+    ...twin,
+    trust_warnings: [...twin.trust_warnings],
+  };
+
+  const includeLayout = input.include_layout ?? includeOcr;
+  if (includeLayout && next.ocr?.lines && next.ocr.lines.length > 0) {
+    const ocrAny = next.ocr as {
+      lines: Array<{
+        text: string;
+        bbox: { x: number; y: number; width: number; height: number };
+        confidence?: number;
+      }>;
+      native_blocks?: Array<{
+        id: string;
+        kind: 'block' | 'paragraph';
+        text: string;
+        bbox: { x: number; y: number; width: number; height: number };
+        confidence?: number;
+      }>;
+    };
+    const layout = buildBestEffortLayout({
+      lines: ocrAny.lines.map((line) => ({
+        text: line.text,
+        bbox: line.bbox,
+        ...(line.confidence !== undefined ? { confidence: line.confidence } : {}),
+      })),
+      ...(ocrAny.native_blocks ? { nativeBlocks: ocrAny.native_blocks } : {}),
+    });
+    next.layout = layout;
+    next.trust_warnings.push(`layout_policy: ${layout.policy}`);
+    for (const warning of layout.warnings) {
+      next.trust_warnings.push(`layout: ${warning}`);
+    }
+  }
+
+  let palette: Array<{ hex: string; approx_share: number }> | undefined;
+  if (input.include_palette) {
+    try {
+      palette = await samplePalette(resolvedPath);
+    } catch {
+      next.trust_warnings.push('palette: sampling failed; continuing without palette.');
+    }
+  }
+
+  const optionalLlm = await maybeOptionalImageCaption({
+    path: resolvedPath,
+    mime: next.mime,
+    enabled: input.include_optional_llm ?? false,
+  });
+  if (optionalLlm.available) {
+    next.trust_warnings.push(
+      'optional_llm caption is non-authority; prefer OCR/layout locators for claims.'
+    );
+  }
+
+  const includeAgentMap = input.include_agent_map ?? true;
+  if (includeAgentMap) {
+    next.agent_map = buildAgentImageMap({
+      filename: next.filename,
+      mime: next.mime,
+      dimensions: next.dimensions,
+      ...(next.layout ? { layout: next.layout } : {}),
+      ocrLineCount: next.ocr?.line_count ?? next.ocr?.lines?.length ?? 0,
+      ...(palette ? { palette } : {}),
+      optionalLlm,
+    });
+  }
+
+  return next;
+}

package/src/utils/layout.ts

@@ -0,0 +1,161 @@
+/**
+ * Local-first layout: prefer Tesseract native block/paragraph units; fall back to line clustering.
+ */
+
+export type BBox = { x: number; y: number; width: number; height: number };
+
+export type LayoutLine = {
+  text: string;
+  bbox: BBox;
+  confidence?: number;
+};
+
+export type LayoutBlock = {
+  id: string;
+  kind: 'text_block';
+  text: string;
+  bbox: BBox;
+  line_count: number;
+  reading_order: number;
+  source?: 'tesseract_native' | 'line_cluster';
+};
+
+export type ImageLayout = {
+  policy: string;
+  block_count: number;
+  blocks: LayoutBlock[];
+  full_text: string;
+  warnings: string[];
+};
+
+export type NativeBlockInput = {
+  id: string;
+  kind: 'block' | 'paragraph';
+  text: string;
+  bbox: BBox;
+  confidence?: number;
+};
+
+const unionBBox = (boxes: BBox[]): BBox => {
+  const minX = Math.min(...boxes.map((b) => b.x));
+  const minY = Math.min(...boxes.map((b) => b.y));
+  const maxX = Math.max(...boxes.map((b) => b.x + b.width));
+  const maxY = Math.max(...boxes.map((b) => b.y + b.height));
+  return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
+};
+
+const median = (values: number[], fallback: number): number => {
+  if (values.length === 0) return fallback;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.floor(sorted.length / 2)] ?? fallback;
+};
+
+const shouldMerge = (prev: LayoutLine, next: LayoutLine, yGap: number, xSlop: number): boolean => {
+  const verticalClose = next.bbox.y - (prev.bbox.y + prev.bbox.height) <= yGap;
+  const prevRight = prev.bbox.x + prev.bbox.width;
+  const nextRight = next.bbox.x + next.bbox.width;
+  const horizontalOverlap =
+    Math.min(prevRight, nextRight) - Math.max(prev.bbox.x, next.bbox.x) > -xSlop;
+  return verticalClose && horizontalOverlap;
+};
+
+const toClusterBlock = (lines: LayoutLine[], index: number): LayoutBlock => {
+  const ordered = [...lines].sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x);
+  const text = ordered
+    .map((l) => l.text.trim())
+    .filter(Boolean)
+    .join('\n');
+  return {
+    id: `block-${index + 1}`,
+    kind: 'text_block',
+    text,
+    bbox: unionBBox(ordered.map((l) => l.bbox)),
+    line_count: ordered.length,
+    reading_order: index + 1,
+    source: 'line_cluster',
+  };
+};
+
+export function buildLayoutFromNativeBlocks(native: NativeBlockInput[]): ImageLayout {
+  const usable = native
+    .filter((b) => b.text.trim().length > 0 && b.bbox.width > 0 && b.bbox.height > 0)
+    .sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x);
+  const blocks: LayoutBlock[] = usable.map((b, index) => ({
+    id: b.id || `native-${index + 1}`,
+    kind: 'text_block',
+    text: b.text.trim(),
+    bbox: b.bbox,
+    line_count: Math.max(1, b.text.split(/\n/).length),
+    reading_order: index + 1,
+    source: 'tesseract_native',
+  }));
+  return {
+    policy: 'tesseract_native_layout_v1',
+    block_count: blocks.length,
+    blocks,
+    full_text: blocks.map((b) => b.text).join('\n\n'),
+    warnings:
+      blocks.length === 0
+        ? ['Native Tesseract blocks present but empty text; falling back recommended.']
+        : [],
+  };
+}
+
+export function buildLayoutFromOcrLines(lines: LayoutLine[]): ImageLayout {
+  if (lines.length === 0) {
+    return {
+      policy: 'ocr_line_cluster_v1',
+      block_count: 0,
+      blocks: [],
+      full_text: '',
+      warnings: ['No OCR lines available for layout clustering.'],
+    };
+  }
+
+  const sorted = [...lines].sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x);
+  const medianHeight = median(
+    sorted.map((l) => l.bbox.height).filter((h) => h > 0),
+    16
+  );
+  const yGap = Math.max(8, medianHeight * 1.4);
+  const xSlop = medianHeight * 2;
+
+  const clusters: LayoutLine[][] = [];
+  for (const line of sorted) {
+    const last = clusters[clusters.length - 1];
+    const prev = last?.[last.length - 1];
+    if (last && prev && shouldMerge(prev, line, yGap, xSlop)) {
+      last.push(line);
+    } else {
+      clusters.push([line]);
+    }
+  }
+
+  const blocks = clusters.map((cluster, index) => toClusterBlock(cluster, index));
+  const warnings: string[] = [];
+  if (blocks.length === 1 && lines.length > 8) {
+    warnings.push(
+      'Layout collapsed to a single block; image may be dense continuous text or OCR line geometry is coarse.'
+    );
+  }
+
+  return {
+    policy: 'ocr_line_cluster_v1',
+    block_count: blocks.length,
+    blocks,
+    full_text: blocks.map((b) => b.text).join('\n\n'),
+    warnings,
+  };
+}
+
+/** Local-first frontier layout: native Tesseract structure first, heuristic second. */
+export function buildBestEffortLayout(input: {
+  lines: LayoutLine[];
+  nativeBlocks?: NativeBlockInput[];
+}): ImageLayout {
+  if (input.nativeBlocks?.some((b) => b.text.trim().length > 0)) {
+    const native = buildLayoutFromNativeBlocks(input.nativeBlocks);
+    if (native.block_count > 0) return native;
+  }
+  return buildLayoutFromOcrLines(input.lines);
+}

package/src/utils/ocr.ts

@@ -10,6 +10,17 @@ export interface OcrWord {
   confidence?: number;
 }
 
+/** Native Tesseract layout unit (block or paragraph) — frontier-local structure free with classic OCR. */
+export interface OcrNativeBlock {
+  id: string;
+  kind: 'block' | 'paragraph';
+  text: string;
+  bbox: { x: number; y: number; width: number; height: number };
+  block_num?: number;
+  par_num?: number;
+  confidence?: number;
+}
+
 export interface OcrResult {
   available: boolean;
   skipped_reason?: string;
@@ -21,6 +32,8 @@ export interface OcrResult {
   dropped_low_confidence?: number;
   lines: OcrLine[];
   words?: OcrWord[];
+  /** Prefer for layout when present (Tesseract levels 2/3). */
+  native_blocks?: OcrNativeBlock[];
 }
 
 export interface ParseTesseractOptions {
@@ -28,21 +41,31 @@ export interface ParseTesseractOptions {
   includeWords?: boolean;
 }
 
-/** Pure TSV parser — offline-testable evidence path for OCR bbox lines. */
+const num = (v: string | undefined): number => Number.parseFloat(v ?? '');
+
+/**
+ * Pure TSV parser — lines (level 4 aggregated from words), words (level 5),
+ * and native blocks/paragraphs (levels 2/3) for local-first layout frontier.
+ */
 export const parseTesseractTsv = (
   raw: string,
   options: ParseTesseractOptions = {}
-): { lines: OcrLine[]; words: OcrWord[]; dropped_low_confidence: number } => {
+): {
+  lines: OcrLine[];
+  words: OcrWord[];
+  native_blocks: OcrNativeBlock[];
+  dropped_low_confidence: number;
+} => {
   const minConfidence = options.minConfidence ?? 0;
   const includeWords = options.includeWords ?? false;
   const linesRaw = raw.split(/\r?\n/).filter((line) => line.length > 0);
   if (linesRaw.length <= 1) {
-    return { lines: [], words: [], dropped_low_confidence: 0 };
+    return { lines: [], words: [], native_blocks: [], dropped_low_confidence: 0 };
   }
 
   const rows = linesRaw.slice(1).map((line) => line.split('\t'));
   const lineMap = new Map<
-    number,
+    string,
     {
       words: Array<{
         text: string;
@@ -57,22 +80,47 @@ export const parseTesseractTsv = (
 
   let dropped = 0;
   const flatWords: OcrWord[] = [];
+  const nativeBlocks: OcrNativeBlock[] = [];
 
   for (const columns of rows) {
     if (columns.length < 12) continue;
     const level = Number.parseInt(columns[0] ?? '', 10);
-    if (level !== 5) continue;
-
-    const text = columns[11]?.trim() ?? '';
-    if (text.length === 0) continue;
-
+    const blockNum = Number.parseInt(columns[2] ?? '', 10);
+    const parNum = Number.parseInt(columns[3] ?? '', 10);
     const lineNum = Number.parseInt(columns[4] ?? '', 10);
     const left = Number.parseInt(columns[6] ?? '', 10);
     const top = Number.parseInt(columns[7] ?? '', 10);
     const width = Number.parseInt(columns[8] ?? '', 10);
     const height = Number.parseInt(columns[9] ?? '', 10);
-    const conf = Number.parseFloat(columns[10] ?? '');
+    const conf = num(columns[10]);
+    const text = columns[11]?.trim() ?? '';
+
+    // Native layout units from Tesseract itself (better than pure heuristic when present)
+    if ((level === 2 || level === 3) && Number.isFinite(left) && Number.isFinite(top)) {
+      if (text.length > 0 || level === 2) {
+        // level 2 blocks often have empty text; still keep geometry if non-zero
+        if (width > 0 && height > 0) {
+          const nb: OcrNativeBlock = {
+            id: level === 2 ? `tess-block-${blockNum}` : `tess-par-${blockNum}-${parNum}`,
+            kind: level === 2 ? 'block' : 'paragraph',
+            text,
+            bbox: {
+              x: left,
+              y: top,
+              width: Number.isFinite(width) ? width : 0,
+              height: Number.isFinite(height) ? height : 0,
+            },
+          };
+          if (Number.isFinite(blockNum)) nb.block_num = blockNum;
+          if (Number.isFinite(parNum)) nb.par_num = parNum;
+          if (Number.isFinite(conf) && conf >= 0) nb.confidence = conf;
+          nativeBlocks.push(nb);
+        }
+      }
+    }
 
+    if (level !== 5) continue;
+    if (text.length === 0) continue;
     if (!Number.isFinite(lineNum) || !Number.isFinite(left) || !Number.isFinite(top)) continue;
 
     const confVal = Number.isFinite(conf) ? conf : 0;
@@ -81,7 +129,8 @@ export const parseTesseractTsv = (
       continue;
     }
 
-    const bucket = lineMap.get(lineNum) ?? { words: [] };
+    const lineKey = `${blockNum}:${parNum}:${lineNum}`;
+    const bucket = lineMap.get(lineKey) ?? { words: [] };
     bucket.words.push({
       text,
       left,
@@ -90,7 +139,7 @@ export const parseTesseractTsv = (
       height: Number.isFinite(height) ? height : 0,
       conf: confVal,
     });
-    lineMap.set(lineNum, bucket);
+    lineMap.set(lineKey, bucket);
 
     if (includeWords) {
       flatWords.push({
@@ -106,18 +155,34 @@ export const parseTesseractTsv = (
     }
   }
 
-  const ocrLines: OcrLine[] = [];
+  // Fill empty native block text from aggregated words in that block
+  const wordsByBlock = new Map<number, string[]>();
+  for (const [key, bucket] of lineMap) {
+    const blockNum = Number.parseInt(key.split(':')[0] ?? '', 10);
+    if (!Number.isFinite(blockNum)) continue;
+    const list = wordsByBlock.get(blockNum) ?? [];
+    const sorted = [...bucket.words].sort((a, b) => a.top - b.top || a.left - b.left);
+    list.push(sorted.map((w) => w.text).join(' '));
+    wordsByBlock.set(blockNum, list);
+  }
+  for (const block of nativeBlocks) {
+    if (block.text.length > 0) continue;
+    if (block.block_num === undefined) continue;
+    const parts = wordsByBlock.get(block.block_num);
+    if (parts && parts.length > 0) {
+      block.text = parts.join('\n').trim();
+    }
+  }
 
+  const ocrLines: OcrLine[] = [];
   for (const bucket of lineMap.values()) {
     if (bucket.words.length === 0) continue;
-
     const sorted = [...bucket.words].sort((a, b) => a.left - b.left);
     const text = sorted
       .map((word) => word.text)
       .join(' ')
       .trim();
     if (text.length === 0) continue;
-
     const left = Math.min(...sorted.map((word) => word.left));
     const top = Math.min(...sorted.map((word) => word.top));
     const right = Math.max(...sorted.map((word) => word.left + word.width));
@@ -127,7 +192,6 @@ export const parseTesseractTsv = (
       confidenceValues.length > 0
         ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
         : undefined;
-
     ocrLines.push({
       text,
       bbox: {
@@ -140,9 +204,20 @@ export const parseTesseractTsv = (
     });
   }
 
+  // Prefer paragraph native blocks with text; else blocks
+  const paragraphs = nativeBlocks.filter((b) => b.kind === 'paragraph' && b.text.length > 0);
+  const blocksOnly = nativeBlocks.filter((b) => b.kind === 'block');
+  const preferredNative =
+    paragraphs.length > 0
+      ? paragraphs
+      : blocksOnly.filter((b) => b.text.length > 0).length > 0
+        ? blocksOnly.filter((b) => b.text.length > 0)
+        : blocksOnly;
+
   return {
     lines: ocrLines.sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x),
     words: includeWords ? flatWords.sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x) : [],
+    native_blocks: preferredNative.sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x),
     dropped_low_confidence: dropped,
   };
 };
@@ -156,7 +231,6 @@ export const isTesseractAvailable = (): boolean => {
   return result.status === 0;
 };
 
-/** List installed Tesseract traineddata languages (honest empty when unavailable). */
 export const listTesseractLanguages = (): {
   available: boolean;
   languages: string[];
@@ -171,9 +245,14 @@ export const listTesseractLanguages = (): {
   }
   const result = spawnSync('tesseract', ['--list-langs'], {
     encoding: 'utf8',
-    timeout: 5_000,
+    timeout: OCR_HEALTHCHECK_TIMEOUT_MS,
     windowsHide: true,
   });
+  const stdout = typeof result.stdout === 'string' ? result.stdout : '';
+  const languages = stdout
+    .split(/\r?\n/)
+    .map((l) => l.trim())
+    .filter((l) => l && !l.toLowerCase().includes('list of available'));
   if (result.status !== 0) {
     const stderr = typeof result.stderr === 'string' ? result.stderr.trim() : '';
     return {
@@ -182,13 +261,6 @@ export const listTesseractLanguages = (): {
       warning: stderr || 'tesseract --list-langs failed',
     };
   }
-  const out = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
-  const languages = out
-    .split(/\r?\n/)
-    .map((l) => l.trim())
-    .filter(
-      (l) => l && !l.toLowerCase().includes('list of available languages') && !l.startsWith('Error')
-    );
   return { available: true, languages };
 };
 
@@ -205,7 +277,7 @@ export const runTesseractOcr = (
   const options: RunOcrOptions = Array.isArray(languagesOrOptions)
     ? { languages: languagesOrOptions }
     : languagesOrOptions;
-  const languages = options.languages?.length ? options.languages : ['eng'];
+  const languages = options.languages ?? ['eng'];
   const minConfidence = options.minConfidence ?? 0;
   const includeWords = options.includeWords ?? false;
 
@@ -218,6 +290,7 @@ export const runTesseractOcr = (
       lines: [],
       line_count: 0,
       dropped_low_confidence: 0,
+      native_blocks: [],
     };
   }
 
@@ -227,22 +300,27 @@ export const runTesseractOcr = (
   );
 
   const languageArg = languages.join('+');
-  const result = spawnSync('tesseract', [imagePath, 'stdout', '-l', languageArg, 'tsv'], {
-    encoding: 'utf8',
-    timeout: OCR_TIMEOUT_MS,
-    windowsHide: true,
-    maxBuffer: 10 * 1024 * 1024,
-  });
+  const result = spawnSync(
+    'tesseract',
+    [imagePath, 'stdout', '-l', languageArg, 'tsv', '--psm', '3'],
+    {
+      encoding: 'utf8',
+      timeout: OCR_TIMEOUT_MS,
+      windowsHide: true,
+      maxBuffer: 20 * 1024 * 1024,
+    }
+  );
 
   if (result.error) {
     return {
       available: false,
-      skipped_reason: `Tesseract failed to start: ${result.error.message}`,
+      skipped_reason: result.error.message,
       route: 'tesseract_tsv',
       languages,
       lines: [],
       line_count: 0,
       dropped_low_confidence: 0,
+      native_blocks: [],
     };
   }
 
@@ -257,6 +335,7 @@ export const runTesseractOcr = (
       lines: [],
       line_count: 0,
       dropped_low_confidence: 0,
+      native_blocks: [],
     };
   }
 
@@ -264,11 +343,12 @@ export const runTesseractOcr = (
   const parsed = parseTesseractTsv(stdout, { minConfidence, includeWords });
   return {
     available: true,
-    route: 'tesseract_tsv',
+    route: 'tesseract_tsv_psm3_native_layout',
     languages,
     lines: parsed.lines,
     line_count: parsed.lines.length,
     dropped_low_confidence: parsed.dropped_low_confidence,
+    native_blocks: parsed.native_blocks,
     ...(missingLangs.length
       ? {
           languages_warning: `Requested OCR language(s) not listed by tesseract --list-langs: ${missingLangs.join(', ')}. Installed: ${installed.languages.join(', ') || '(none)'}.`,

package/src/utils/optionalLlm.ts

@@ -0,0 +1,228 @@
+/**
+ * Local-first frontier optional vision caption.
+ *
+ * Priority:
+ * 1) IRIS_OPTIONAL_LLM_URL — custom POST { path, mime, purpose } → { caption }
+ * 2) Local Ollama at IRIS_OLLAMA_URL (default http://127.0.0.1:11434) if a vision model is installed
+ *
+ * Never authority over OCR/layout locators. OFF unless include_optional_llm=true.
+ */
+
+import { readFileSync } from 'node:fs';
+
+export type OptionalLlmResult = {
+  available: boolean;
+  skipped_reason?: string;
+  route?: string;
+  caption?: string;
+  model?: string;
+};
+
+const DEFAULT_OLLAMA = 'http://127.0.0.1:11434';
+
+const VISION_MODEL_HINTS = [
+  'llava',
+  'minicpm-v',
+  'minicpm_v',
+  'qwen2-vl',
+  'qwen2.5-vl',
+  'qwen3-vl',
+  'bakllava',
+  'moondream',
+  'gemma3',
+  'llama3.2-vision',
+  'pixtral',
+];
+
+function isVisionModel(name: string): boolean {
+  const n = name.toLowerCase();
+  return VISION_MODEL_HINTS.some((h) => n.includes(h));
+}
+
+async function tryCustomUrl(input: {
+  path: string;
+  mime: string;
+  url: string;
+}): Promise<OptionalLlmResult> {
+  try {
+    const res = await fetch(input.url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        path: input.path,
+        mime: input.mime,
+        purpose: 'image_caption',
+      }),
+      signal: AbortSignal.timeout(20_000),
+    });
+    if (!res.ok) {
+      return {
+        available: false,
+        skipped_reason: `optional LLM HTTP ${res.status}`,
+        route: input.url,
+      };
+    }
+    const body = (await res.json()) as { caption?: string };
+    if (!body.caption || typeof body.caption !== 'string') {
+      return {
+        available: false,
+        skipped_reason: 'optional LLM response missing caption string',
+        route: input.url,
+      };
+    }
+    return {
+      available: true,
+      route: `optional-llm:${input.url}`,
+      caption: body.caption.slice(0, 4000),
+    };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'optional LLM failed';
+    return { available: false, skipped_reason: message, route: input.url };
+  }
+}
+
+async function listOllamaModels(base: string): Promise<string[]> {
+  const res = await fetch(`${base.replace(/\/$/, '')}/api/tags`, {
+    signal: AbortSignal.timeout(3_000),
+  });
+  if (!res.ok) return [];
+  const body = (await res.json()) as { models?: Array<{ name?: string }> };
+  return (body.models ?? [])
+    .map((m) => m.name)
+    .filter((n): n is string => typeof n === 'string' && n.length > 0);
+}
+
+async function tryOllamaVision(input: {
+  path: string;
+  baseUrl: string;
+  modelHint?: string;
+}): Promise<OptionalLlmResult> {
+  const base = input.baseUrl.replace(/\/$/, '');
+  let models: string[] = [];
+  try {
+    models = await listOllamaModels(base);
+  } catch {
+    return {
+      available: false,
+      skipped_reason: `Ollama not reachable at ${base}`,
+      route: base,
+    };
+  }
+  if (models.length === 0) {
+    return {
+      available: false,
+      skipped_reason: 'Ollama has no models installed',
+      route: base,
+    };
+  }
+
+  const preferred =
+    (input.modelHint &&
+      models.find((m) => m === input.modelHint || m.startsWith(`${input.modelHint}:`))) ||
+    models.find((m) => isVisionModel(m));
+  if (!preferred) {
+    return {
+      available: false,
+      skipped_reason: `No local vision model found in Ollama (have: ${models.slice(0, 6).join(', ')}). Install e.g. llava or minicpm-v.`,
+      route: base,
+    };
+  }
+
+  let b64: string;
+  try {
+    b64 = readFileSync(input.path).toString('base64');
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'read failed';
+    return { available: false, skipped_reason: message, route: base };
+  }
+
+  // Cap image payload ~4MB base64
+  if (b64.length > 5_500_000) {
+    return {
+      available: false,
+      skipped_reason: 'image too large for default Ollama caption path; crop_region first',
+      route: `ollama:${preferred}`,
+    };
+  }
+
+  try {
+    const res = await fetch(`${base}/api/chat`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        model: preferred,
+        stream: false,
+        messages: [
+          {
+            role: 'user',
+            content:
+              'Describe this image for a software agent. Focus on structure, UI/text regions, objects, and layout. Be concise and factual. Do not invent unreadable text.',
+            images: [b64],
+          },
+        ],
+      }),
+      signal: AbortSignal.timeout(60_000),
+    });
+    if (!res.ok) {
+      return {
+        available: false,
+        skipped_reason: `Ollama chat HTTP ${res.status}`,
+        route: `ollama:${preferred}`,
+        model: preferred,
+      };
+    }
+    const body = (await res.json()) as {
+      message?: { content?: string };
+      response?: string;
+    };
+    const caption = body.message?.content ?? body.response;
+    if (!caption || typeof caption !== 'string') {
+      return {
+        available: false,
+        skipped_reason: 'Ollama response missing message content',
+        route: `ollama:${preferred}`,
+        model: preferred,
+      };
+    }
+    return {
+      available: true,
+      route: `ollama-local-vlm:${preferred}`,
+      model: preferred,
+      caption: caption.slice(0, 4000),
+    };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'Ollama caption failed';
+    return {
+      available: false,
+      skipped_reason: message,
+      route: `ollama:${preferred}`,
+      model: preferred,
+    };
+  }
+}
+
+export async function maybeOptionalImageCaption(input: {
+  path: string;
+  mime: string;
+  enabled: boolean;
+}): Promise<OptionalLlmResult> {
+  if (!input.enabled) {
+    return { available: false, skipped_reason: 'include_optional_llm is false (default).' };
+  }
+
+  const custom = process.env['IRIS_OPTIONAL_LLM_URL'];
+  if (custom) {
+    return tryCustomUrl({ path: input.path, mime: input.mime, url: custom });
+  }
+
+  const ollamaBase = process.env['IRIS_OLLAMA_URL'] ?? process.env['OLLAMA_HOST'] ?? DEFAULT_OLLAMA;
+  // OLLAMA_HOST may be host:port without scheme
+  const baseUrl = ollamaBase.startsWith('http') ? ollamaBase : `http://${ollamaBase}`;
+  const modelHint = process.env['IRIS_OLLAMA_VISION_MODEL'] ?? process.env['OLLAMA_VISION_MODEL'];
+
+  return tryOllamaVision({
+    path: input.path,
+    baseUrl,
+    ...(modelHint ? { modelHint } : {}),
+  });
+}

package/src/utils/palette.ts

@@ -0,0 +1,36 @@
+import sharp from 'sharp';
+
+/** Cheap local palette sample via sharp stats (not ML segmentation). */
+export async function samplePalette(
+  filePath: string,
+  maxColors = 5
+): Promise<Array<{ hex: string; approx_share: number }>> {
+  const image = sharp(filePath, { failOn: 'none' }).resize(64, 64, { fit: 'inside' });
+  const stats = await image.stats();
+  const dominant = stats.dominant;
+  if (!dominant) return [];
+
+  const hex = (r: number, g: number, b: number) =>
+    `#${[r, g, b]
+      .map((v) =>
+        Math.max(0, Math.min(255, Math.round(v)))
+          .toString(16)
+          .padStart(2, '0')
+      )
+      .join('')}`;
+
+  const colors: Array<{ hex: string; approx_share: number }> = [
+    { hex: hex(dominant.r, dominant.g, dominant.b), approx_share: 0.55 },
+  ];
+
+  // Channel means as a second honest swatch (not true multi-color clustering).
+  const ch = stats.channels;
+  if (ch && ch.length >= 3) {
+    colors.push({
+      hex: hex(ch[0]?.mean ?? dominant.r, ch[1]?.mean ?? dominant.g, ch[2]?.mean ?? dominant.b),
+      approx_share: 0.45,
+    });
+  }
+
+  return colors.slice(0, maxColors);
+}