@sylphx/image-reader-mcp 0.1.1 → 0.1.2

package/README.md

@@ -36,6 +36,13 @@ Each instrument is an independent repository (marketplace + stars).
 ---
 
 
+
+## Read images (not vague vision)
+
+Iris is **local-first**: geometry + OCR + **layout blocks** + **agent_map** so a text-only agent can understand picture architecture without a vision model.
+
+Spec: [docs/specs/agent-image-read-contract.md](docs/specs/agent-image-read-contract.md)
+
 ## Product docs
 
 | Doc | Purpose |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sylphx/image-reader-mcp",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "mcpName": "io.github.SylphxAI/image-reader-mcp",
   "description": "Iris \u2014 Evidence-first image reading for AI agents \u2014 metadata, OCR text, regions, and citeable evidence without generative LLM.",
   "type": "module",

package/src/handlers/readImage.ts

@@ -9,6 +9,7 @@ import {
 } from '../engine/rust-decode.js';
 import { text, tool, toolError } from '../mcp.js';
 import { type AgentMediaTwin, readImageArgsSchema } from '../schemas/readImage.js';
+import { applyImageIntelligence } from '../utils/applyImageIntelligence.js';
 import { ErrorCode, ImageError } from '../utils/errors.js';
 import { collectTrustWarnings, redactGpsFields } from '../utils/metadata.js';
 import { runTesseractOcr } from '../utils/ocr.js';
@@ -74,7 +75,7 @@ const readMetadata = async (
 
 export const readImage = tool()
   .description(
-    'Evidence-first image reader. Returns an Agent Media Twin with filename, mime, dimensions, metadata, optional OCR lines with bounding boxes, and trust warnings. No generative LLM is used.'
+    'Evidence-first image reader for agents (read, not vague vision). Returns Agent Media Twin: geometry, metadata, OCR lines/words, layout blocks, text agent_map, optional palette, optional non-authority LLM caption. Local-first; generative path off by default.'
   )
   .input(readImageArgsSchema)
   .handler(async ({ input }) => {
@@ -204,6 +205,8 @@ export const readImage = tool()
         };
       }
 
+      twin = await applyImageIntelligence(twin, resolvedPath, input, includeOcr);
+
       if (input.region !== undefined) {
         if (!useRustDecode) {
           throw new ImageError(

package/src/schemas/readImage.ts

@@ -59,6 +59,28 @@ export const readImageArgsSchema = z.object({
     .positive()
     .optional()
     .describe('Maximum width or height when resizing the cropped region for evidence.'),
+  include_layout: z
+    .boolean()
+    .optional()
+    .describe(
+      'When OCR lines exist, cluster them into reading-order text blocks (image architecture). Defaults to true when include_ocr is true.'
+    ),
+  include_agent_map: z
+    .boolean()
+    .optional()
+    .describe(
+      'Return a text-only agent_map so non-vision models can read image structure. Defaults to true.'
+    ),
+  include_palette: z
+    .boolean()
+    .optional()
+    .describe('Sample an approximate local color palette via sharp (not ML). Defaults to false.'),
+  include_optional_llm: z
+    .boolean()
+    .optional()
+    .describe(
+      'Optional caption via IRIS_OPTIONAL_LLM_URL. Off by default; never authority over OCR/layout evidence.'
+    ),
 });
 
 export const agentMediaTwinSchema = z.object({
@@ -101,6 +123,44 @@ export const agentMediaTwinSchema = z.object({
       image_base64: z.string().optional(),
     })
     .optional(),
+  layout: z
+    .object({
+      policy: z.string(),
+      block_count: z.number().int().nonnegative(),
+      blocks: z.array(
+        z.object({
+          id: z.string(),
+          kind: z.literal('text_block'),
+          text: z.string(),
+          bbox: boundingBoxSchema,
+          line_count: z.number().int().nonnegative(),
+          reading_order: z.number().int().positive(),
+        })
+      ),
+      full_text: z.string(),
+      warnings: z.array(z.string()),
+    })
+    .optional(),
+  agent_map: z
+    .object({
+      policy: z.string(),
+      filename: z.string(),
+      mime: z.string(),
+      dimensions: imageDimensionsSchema,
+      outline: z.string(),
+      text_present: z.boolean(),
+      block_count: z.number().int().nonnegative(),
+      palette: z.array(z.object({ hex: z.string(), approx_share: z.number() })).optional(),
+      optional_llm: z
+        .object({
+          available: z.boolean(),
+          skipped_reason: z.string().optional(),
+          route: z.string().optional(),
+          caption: z.string().optional(),
+        })
+        .optional(),
+    })
+    .optional(),
   trust_warnings: z.array(z.string()),
 });
 

package/src/utils/agentMap.ts

@@ -0,0 +1,93 @@
+/**
+ * Text-only Agent Map: lets a non-vision model "read" the image architecture.
+ * Deterministic, local-first. Optional LLM captions are layered elsewhere.
+ */
+
+import type { ImageLayout } from './layout.js';
+
+export type AgentImageMap = {
+  policy: 'agent_image_map_v1';
+  filename: string;
+  mime: string;
+  dimensions: { width: number; height: number };
+  /** Human/agent readable outline (markdown-ish, not prose hallucination). */
+  outline: string;
+  text_present: boolean;
+  block_count: number;
+  palette?: Array<{ hex: string; approx_share: number }>;
+  optional_llm?: {
+    available: boolean;
+    skipped_reason?: string;
+    route?: string;
+    caption?: string;
+  };
+};
+
+export function buildAgentImageMap(input: {
+  filename: string;
+  mime: string;
+  dimensions: { width: number; height: number };
+  layout?: ImageLayout;
+  ocrLineCount?: number;
+  palette?: Array<{ hex: string; approx_share: number }>;
+  optionalLlm?: AgentImageMap['optional_llm'];
+}): AgentImageMap {
+  const { width, height } = input.dimensions;
+  const blocks = input.layout?.blocks ?? [];
+  const textPresent = (input.ocrLineCount ?? 0) > 0 || blocks.length > 0;
+
+  const lines: string[] = [
+    `# Image map: ${input.filename}`,
+    `- mime: ${input.mime}`,
+    `- size: ${width}×${height}px`,
+    `- text_present: ${textPresent}`,
+    `- layout_blocks: ${blocks.length}`,
+  ];
+
+  if (input.palette && input.palette.length > 0) {
+    lines.push(
+      `- palette: ${input.palette.map((p) => `${p.hex}~${Math.round(p.approx_share * 100)}%`).join(', ')}`
+    );
+  }
+
+  if (blocks.length > 0) {
+    lines.push('', '## Reading-order text blocks');
+    for (const b of blocks) {
+      const preview = b.text.replace(/\s+/g, ' ').slice(0, 160);
+      lines.push(
+        `### ${b.id} (order ${b.reading_order}, ${b.line_count} lines)`,
+        `- bbox: x=${b.bbox.x} y=${b.bbox.y} w=${b.bbox.width} h=${b.bbox.height}`,
+        `- text: ${preview}${b.text.length > 160 ? '…' : ''}`,
+        ''
+      );
+    }
+  } else if (!textPresent) {
+    lines.push(
+      '',
+      '## Notes',
+      '- No OCR text recovered. Enable include_ocr for text architecture.',
+      '- Use crop_region / image_probe for geometry; optional LLM caption only if configured.'
+    );
+  }
+
+  if (input.optionalLlm?.available && input.optionalLlm.caption) {
+    lines.push('', '## Optional LLM caption (non-authority)', input.optionalLlm.caption);
+  } else if (input.optionalLlm && !input.optionalLlm.available) {
+    lines.push(
+      '',
+      `## Optional LLM: skipped (${input.optionalLlm.skipped_reason ?? 'not configured'})`
+    );
+  }
+
+  return {
+    policy: 'agent_image_map_v1',
+    filename: input.filename,
+    mime: input.mime,
+    dimensions: input.dimensions,
+    outline: lines.join('\n'),
+    text_present: textPresent,
+    block_count: blocks.length,
+    ...(input.palette ? { palette: input.palette } : {}),
+    ...(input.optionalLlm ? { optional_llm: input.optionalLlm } : {}),
+  };
+}

package/src/utils/applyImageIntelligence.ts

@@ -0,0 +1,68 @@
+import type { AgentMediaTwin, ReadImageArgs } from '../schemas/readImage.js';
+import { buildAgentImageMap } from './agentMap.js';
+import { buildLayoutFromOcrLines } from './layout.js';
+import { maybeOptionalImageCaption } from './optionalLlm.js';
+import { samplePalette } from './palette.js';
+
+/** Attach layout, palette, optional LLM, and agent_map to an Agent Media Twin. */
+export async function applyImageIntelligence(
+  twin: AgentMediaTwin,
+  resolvedPath: string,
+  input: ReadImageArgs,
+  includeOcr: boolean
+): Promise<AgentMediaTwin> {
+  const next: AgentMediaTwin = {
+    ...twin,
+    trust_warnings: [...twin.trust_warnings],
+  };
+
+  const includeLayout = input.include_layout ?? includeOcr;
+  if (includeLayout && next.ocr?.lines && next.ocr.lines.length > 0) {
+    const layout = buildLayoutFromOcrLines(
+      next.ocr.lines.map((line) => ({
+        text: line.text,
+        bbox: line.bbox,
+        ...(line.confidence !== undefined ? { confidence: line.confidence } : {}),
+      }))
+    );
+    next.layout = layout;
+    for (const warning of layout.warnings) {
+      next.trust_warnings.push(`layout: ${warning}`);
+    }
+  }
+
+  let palette: Array<{ hex: string; approx_share: number }> | undefined;
+  if (input.include_palette) {
+    try {
+      palette = await samplePalette(resolvedPath);
+    } catch {
+      next.trust_warnings.push('palette: sampling failed; continuing without palette.');
+    }
+  }
+
+  const optionalLlm = await maybeOptionalImageCaption({
+    path: resolvedPath,
+    mime: next.mime,
+    enabled: input.include_optional_llm ?? false,
+  });
+  if (optionalLlm.available) {
+    next.trust_warnings.push(
+      'optional_llm caption is non-authority; prefer OCR/layout locators for claims.'
+    );
+  }
+
+  const includeAgentMap = input.include_agent_map ?? true;
+  if (includeAgentMap) {
+    next.agent_map = buildAgentImageMap({
+      filename: next.filename,
+      mime: next.mime,
+      dimensions: next.dimensions,
+      ...(next.layout ? { layout: next.layout } : {}),
+      ocrLineCount: next.ocr?.line_count ?? next.ocr?.lines?.length ?? 0,
+      ...(palette ? { palette } : {}),
+      optionalLlm,
+    });
+  }
+
+  return next;
+}

package/src/utils/layout.ts

@@ -0,0 +1,114 @@
+/**
+ * Deterministic layout structure from OCR line boxes (no generative model).
+ */
+
+export type BBox = { x: number; y: number; width: number; height: number };
+
+export type LayoutLine = {
+  text: string;
+  bbox: BBox;
+  confidence?: number;
+};
+
+export type LayoutBlock = {
+  id: string;
+  kind: 'text_block';
+  text: string;
+  bbox: BBox;
+  line_count: number;
+  reading_order: number;
+};
+
+export type ImageLayout = {
+  policy: string;
+  block_count: number;
+  blocks: LayoutBlock[];
+  full_text: string;
+  warnings: string[];
+};
+
+const unionBBox = (boxes: BBox[]): BBox => {
+  const minX = Math.min(...boxes.map((b) => b.x));
+  const minY = Math.min(...boxes.map((b) => b.y));
+  const maxX = Math.max(...boxes.map((b) => b.x + b.width));
+  const maxY = Math.max(...boxes.map((b) => b.y + b.height));
+  return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
+};
+
+const median = (values: number[], fallback: number): number => {
+  if (values.length === 0) return fallback;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.floor(sorted.length / 2)] ?? fallback;
+};
+
+const shouldMerge = (prev: LayoutLine, next: LayoutLine, yGap: number, xSlop: number): boolean => {
+  const verticalClose = next.bbox.y - (prev.bbox.y + prev.bbox.height) <= yGap;
+  const prevRight = prev.bbox.x + prev.bbox.width;
+  const nextRight = next.bbox.x + next.bbox.width;
+  const horizontalOverlap =
+    Math.min(prevRight, nextRight) - Math.max(prev.bbox.x, next.bbox.x) > -xSlop;
+  return verticalClose && horizontalOverlap;
+};
+
+const toBlock = (lines: LayoutLine[], index: number): LayoutBlock => {
+  const ordered = [...lines].sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x);
+  const text = ordered
+    .map((l) => l.text.trim())
+    .filter(Boolean)
+    .join('\n');
+  return {
+    id: `block-${index + 1}`,
+    kind: 'text_block',
+    text,
+    bbox: unionBBox(ordered.map((l) => l.bbox)),
+    line_count: ordered.length,
+    reading_order: index + 1,
+  };
+};
+
+export function buildLayoutFromOcrLines(lines: LayoutLine[]): ImageLayout {
+  if (lines.length === 0) {
+    return {
+      policy: 'ocr_line_cluster_v1',
+      block_count: 0,
+      blocks: [],
+      full_text: '',
+      warnings: ['No OCR lines available for layout clustering.'],
+    };
+  }
+
+  const sorted = [...lines].sort((a, b) => a.bbox.y - b.bbox.y || a.bbox.x - b.bbox.x);
+  const medianHeight = median(
+    sorted.map((l) => l.bbox.height).filter((h) => h > 0),
+    16
+  );
+  const yGap = Math.max(8, medianHeight * 1.4);
+  const xSlop = medianHeight * 2;
+
+  const clusters: LayoutLine[][] = [];
+  for (const line of sorted) {
+    const last = clusters[clusters.length - 1];
+    const prev = last?.[last.length - 1];
+    if (last && prev && shouldMerge(prev, line, yGap, xSlop)) {
+      last.push(line);
+    } else {
+      clusters.push([line]);
+    }
+  }
+
+  const blocks = clusters.map((cluster, index) => toBlock(cluster, index));
+  const warnings: string[] = [];
+  if (blocks.length === 1 && lines.length > 8) {
+    warnings.push(
+      'Layout collapsed to a single block; image may be dense continuous text or OCR line geometry is coarse.'
+    );
+  }
+
+  return {
+    policy: 'ocr_line_cluster_v1',
+    block_count: blocks.length,
+    blocks,
+    full_text: blocks.map((b) => b.text).join('\n\n'),
+    warnings,
+  };
+}

package/src/utils/optionalLlm.ts

@@ -0,0 +1,67 @@
+/**
+ * Optional local/remote vision caption — OFF by default (local-first).
+ * Set IRIS_OPTIONAL_LLM_URL to a POST endpoint that accepts JSON
+ * { path, mime, purpose: "image_caption" } and returns { caption: string }.
+ * Never treated as authority over OCR/layout evidence.
+ */
+
+export type OptionalLlmResult = {
+  available: boolean;
+  skipped_reason?: string;
+  route?: string;
+  caption?: string;
+};
+
+export async function maybeOptionalImageCaption(input: {
+  path: string;
+  mime: string;
+  enabled: boolean;
+}): Promise<OptionalLlmResult> {
+  if (!input.enabled) {
+    return { available: false, skipped_reason: 'include_optional_llm is false (default).' };
+  }
+  const url = process.env['IRIS_OPTIONAL_LLM_URL'];
+  if (!url) {
+    return {
+      available: false,
+      skipped_reason:
+        'IRIS_OPTIONAL_LLM_URL is not set. Local OCR/layout remain the evidence authority.',
+    };
+  }
+
+  try {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        path: input.path,
+        mime: input.mime,
+        purpose: 'image_caption',
+      }),
+      signal: AbortSignal.timeout(15_000),
+    });
+    if (!res.ok) {
+      return {
+        available: false,
+        skipped_reason: `optional LLM HTTP ${res.status}`,
+        route: url,
+      };
+    }
+    const body = (await res.json()) as { caption?: string };
+    if (!body.caption || typeof body.caption !== 'string') {
+      return {
+        available: false,
+        skipped_reason: 'optional LLM response missing caption string',
+        route: url,
+      };
+    }
+    return {
+      available: true,
+      route: `optional-llm:${url}`,
+      caption: body.caption.slice(0, 4000),
+    };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'optional LLM failed';
+    return { available: false, skipped_reason: message, route: url };
+  }
+}

package/src/utils/palette.ts

@@ -0,0 +1,36 @@
+import sharp from 'sharp';
+
+/** Cheap local palette sample via sharp stats (not ML segmentation). */
+export async function samplePalette(
+  filePath: string,
+  maxColors = 5
+): Promise<Array<{ hex: string; approx_share: number }>> {
+  const image = sharp(filePath, { failOn: 'none' }).resize(64, 64, { fit: 'inside' });
+  const stats = await image.stats();
+  const dominant = stats.dominant;
+  if (!dominant) return [];
+
+  const hex = (r: number, g: number, b: number) =>
+    `#${[r, g, b]
+      .map((v) =>
+        Math.max(0, Math.min(255, Math.round(v)))
+          .toString(16)
+          .padStart(2, '0')
+      )
+      .join('')}`;
+
+  const colors: Array<{ hex: string; approx_share: number }> = [
+    { hex: hex(dominant.r, dominant.g, dominant.b), approx_share: 0.55 },
+  ];
+
+  // Channel means as a second honest swatch (not true multi-color clustering).
+  const ch = stats.channels;
+  if (ch && ch.length >= 3) {
+    colors.push({
+      hex: hex(ch[0]?.mean ?? dominant.r, ch[1]?.mean ?? dominant.g, ch[2]?.mean ?? dominant.b),
+      approx_share: 0.45,
+    });
+  }
+
+  return colors.slice(0, maxColors);
+}