@sylphx/image-reader-mcp 0.1.2 → 0.1.5

package/README.md

@@ -4,7 +4,7 @@
 
 ### Image evidence for agents 
 
-**Iris** (transitional package `@sylphx/image-reader-mcp`) — local-first image facts, not default VLM guesses.
+**Iris** (transitional package `@sylphx/image-reader-mcp`) — **Rust-first** local image facts, not default VLM guesses. `sharp`/`exifr` are optional, not required for MCP.
 
 <p align="center">
   <img src="https://mark.sylphx.com/api/v1/banner?type=holo&theme=tokyonight&text=image+reader+mcp&desc=Evidence-first+image+reading+for+AI+agents+%E2%80%94+metadata%2C+OCR+text%2C+regions%2C+and+ci&height=200&animation=rise&credit=0" alt="image-reader-mcp — Sylphx Mark banner" width="100%" />
@@ -43,6 +43,8 @@ Iris is **local-first**: geometry + OCR + **layout blocks** + **agent_map** so a
 
 Spec: [docs/specs/agent-image-read-contract.md](docs/specs/agent-image-read-contract.md)
 
+**Local-first frontier:** Rust decode, Tesseract native layout (no npm ML), optional Ollama VLM; cloud URL optional. Zero API key.
+
 ## Product docs
 
 | Doc | Purpose |
@@ -76,7 +78,7 @@ an image, not a creative caption.**
 | Paraphrased OCR | Optional Tesseract lines with bounding boxes and confidence |
 | GPS and EXIF leak into context | GPS redacted; trust warnings for suspicious metadata |
 | No provenance | Agent Media Twin JSON with measurable, citeable fields |
-| Cloud API by default | **Local-first** — sharp + exifr on your machine |
+| Cloud API by default | **Local-first** — Rust decode + optional Tesseract OCR; sharp/exifr only optional fallbacks |
 | Ship and pray | **23** unit tests on schema, metadata, OCR hooks, safety limits, doctor, and release gate |
 
 ## See it work
@@ -139,7 +141,7 @@ Abbreviated shape — optional OCR skips gracefully when Tesseract is not instal
 | --- | --- |
 | `read_image` | Read a local image and return dimensions, mime, metadata, optional OCR, and trust warnings. |
 
-Supported formats: PNG, JPEG, GIF, WebP, TIFF, and other formats sharp can decode.
+Supported formats: PNG, JPEG, GIF, WebP, TIFF, and other formats the **Rust decode engine** supports (optional sharp covers additional formats when installed).
 
 ## Quick Start
 

package/bin/image-reader-mcp

@@ -1,25 +1,96 @@
 #!/usr/bin/env bash
-# Image Reader MCP launcher — Rust rmcp only (fail-closed, no TS stdio fallback).
+# Iris MCP launcher — Rust rmcp only (fail-closed).
+# Resolves native via optionalDependencies, staged bin/native, or cargo target.
 set -euo pipefail
 
-ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+PACKAGE_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+host_platform_key() {
+	local os arch
+	os="$(uname -s 2>/dev/null || echo unknown)"
+	arch="$(uname -m 2>/dev/null || echo unknown)"
+	case "$os" in
+		Darwin) os="darwin" ;;
+		Linux) os="linux" ;;
+		*) os="$(printf '%s' "$os" | tr '[:upper:]' '[:lower:]')" ;;
+	esac
+	case "$arch" in
+		x86_64|amd64) arch="x64" ;;
+		aarch64|arm64) arch="arm64" ;;
+	esac
+	printf '%s-%s\n' "$os" "$arch"
+}
+
+platform_package_name() {
+	case "$(host_platform_key)" in
+		darwin-arm64) printf '%s\n' '@sylphx/image-reader-mcp-darwin-arm64' ;;
+		darwin-x64) printf '%s\n' '@sylphx/image-reader-mcp-darwin-x64' ;;
+		linux-x64) printf '%s\n' '@sylphx/image-reader-mcp-linux-x64-gnu' ;;
+		linux-arm64) printf '%s\n' '@sylphx/image-reader-mcp-linux-arm64-gnu' ;;
+		*) return 1 ;;
+	esac
+}
+
+is_runnable_native() {
+	local candidate="$1"
+	[[ -n "$candidate" && -f "$candidate" && -x "$candidate" ]] || return 1
+	return 0
+}
+
+resolve_from_optional_dep() {
+	local pkg_name candidate
+	pkg_name="$(platform_package_name 2>/dev/null || true)"
+	[[ -n "$pkg_name" ]] || return 1
+	local search_roots=(
+		"$PACKAGE_ROOT/node_modules"
+		"$PACKAGE_ROOT/../node_modules"
+		"$PACKAGE_ROOT/../../node_modules"
+	)
+	for root in "${search_roots[@]}"; do
+		candidate="$root/$pkg_name/image-reader-mcp-server"
+		if is_runnable_native "$candidate"; then
+			printf '%s\n' "$candidate"
+			return 0
+		fi
+	done
+	if command -v node >/dev/null 2>&1; then
+		candidate="$(
+			node -e "
+        try {
+          const p = require('node:path');
+          const { createRequire } = require('node:module');
+          const r = createRequire(p.join(process.argv[1], 'package.json'));
+          const pkg = r.resolve(process.argv[2] + '/package.json');
+          process.stdout.write(p.join(p.dirname(pkg), 'image-reader-mcp-server'));
+        } catch { process.exit(1); }
+      " "$PACKAGE_ROOT" "$pkg_name" 2>/dev/null || true
+		)"
+		if is_runnable_native "${candidate:-}"; then
+			printf '%s\n' "$candidate"
+			return 0
+		fi
+	fi
+	return 1
+}
 
 resolve_rust_bin() {
 	if [[ -n "${IMAGE_READER_MCP_RUST_BIN:-}" && -x "${IMAGE_READER_MCP_RUST_BIN}" ]]; then
 		printf '%s\n' "${IMAGE_READER_MCP_RUST_BIN}"
 		return 0
 	fi
-
+	if bin="$(resolve_from_optional_dep)"; then
+		printf '%s\n' "$bin"
+		return 0
+	fi
 	for candidate in \
-		"$ROOT/bin/native/image-reader-mcp-server" \
-		"$ROOT/target/release/image-reader-mcp-server" \
-		"$ROOT/target/debug/image-reader-mcp-server"; do
-		if [[ -x "$candidate" ]]; then
+		"$PACKAGE_ROOT/bin/native/image-reader-mcp-server" \
+		"$PACKAGE_ROOT/target/release/image-reader-mcp-server" \
+		"$PACKAGE_ROOT/target/debug/image-reader-mcp-server"; do
+		if is_runnable_native "$candidate"; then
 			printf '%s\n' "$candidate"
 			return 0
 		fi
 	done
-
 	return 1
 }
 
@@ -44,5 +115,5 @@ if bin="$(resolve_rust_bin)"; then
 	exec "$bin" "$@"
 fi
 
-echo "[image-reader-mcp] Rust MCP server (rmcp) is not built. Run: bun run build:rust" >&2
+echo "[iris] Rust MCP server not found. Install matching optional native package (@sylphx/image-reader-mcp-<platform>) or run: bun run build:rust" >&2
 exit 1

package/dist/doctor-cli.js

@@ -118,15 +118,15 @@ var require_filesystem = __commonJS((exports, module) => {
   var LDD_PATH = "/usr/bin/ldd";
   var SELF_PATH = "/proc/self/exe";
   var MAX_LENGTH = 2048;
-  var readFileSync = (path) => {
-    const fd = fs.openSync(path, "r");
+  var readFileSync = (path2) => {
+    const fd = fs.openSync(path2, "r");
     const buffer = Buffer.alloc(MAX_LENGTH);
     const bytesRead = fs.readSync(fd, buffer, 0, MAX_LENGTH, 0);
     fs.close(fd, () => {});
     return buffer.subarray(0, bytesRead);
   };
-  var readFile = (path) => new Promise((resolve, reject) => {
-    fs.open(path, "r", (err, fd) => {
+  var readFile = (path2) => new Promise((resolve, reject) => {
+    fs.open(path2, "r", (err, fd) => {
       if (err) {
         reject(err);
       } else {
@@ -238,11 +238,11 @@ var require_detect_libc = __commonJS((exports, module) => {
     }
     return null;
   };
-  var familyFromInterpreterPath = (path) => {
-    if (path) {
-      if (path.includes("/ld-musl-")) {
+  var familyFromInterpreterPath = (path2) => {
+    if (path2) {
+      if (path2.includes("/ld-musl-")) {
         return MUSL;
-      } else if (path.includes("/ld-linux-")) {
+      } else if (path2.includes("/ld-linux-")) {
         return GLIBC;
       }
     }
@@ -287,8 +287,8 @@ var require_detect_libc = __commonJS((exports, module) => {
     cachedFamilyInterpreter = null;
     try {
       const selfContent = await readFile(SELF_PATH);
-      const path = interpreterPath(selfContent);
-      cachedFamilyInterpreter = familyFromInterpreterPath(path);
+      const path2 = interpreterPath(selfContent);
+      cachedFamilyInterpreter = familyFromInterpreterPath(path2);
     } catch (e) {}
     return cachedFamilyInterpreter;
   };
@@ -299,8 +299,8 @@ var require_detect_libc = __commonJS((exports, module) => {
     cachedFamilyInterpreter = null;
     try {
       const selfContent = readFileSync(SELF_PATH);
-      const path = interpreterPath(selfContent);
-      cachedFamilyInterpreter = familyFromInterpreterPath(path);
+      const path2 = interpreterPath(selfContent);
+      cachedFamilyInterpreter = familyFromInterpreterPath(path2);
     } catch (e) {}
     return cachedFamilyInterpreter;
   };
@@ -1777,7 +1777,7 @@ var require_libvips = __commonJS((exports, module) => {
     Copyright 2013 Lovell Fuller and others.
     SPDX-License-Identifier: Apache-2.0
   */
-  var { spawnSync } = __require("node:child_process");
+  var { spawnSync: spawnSync2 } = __require("node:child_process");
   var { createHash } = __require("node:crypto");
   var semverCoerce = require_coerce();
   var semverGreaterThanOrEqualTo = require_gte();
@@ -1861,7 +1861,7 @@ var require_libvips = __commonJS((exports, module) => {
   };
   var isRosetta = () => {
     if (process.platform === "darwin" && process.arch === "x64") {
-      const translated = spawnSync("sysctl sysctl.proc_translated", spawnSyncOptions).stdout;
+      const translated = spawnSync2("sysctl sysctl.proc_translated", spawnSyncOptions).stdout;
       return (translated || "").trim() === "sysctl.proc_translated: 1";
     }
     return false;
@@ -1877,13 +1877,13 @@ var require_libvips = __commonJS((exports, module) => {
     } catch {}
     return "";
   };
-  var spawnRebuild = () => spawnSync(`node-gyp rebuild --directory=src ${isEmscripten() ? "--nodedir=emscripten" : ""}`, {
+  var spawnRebuild = () => spawnSync2(`node-gyp rebuild --directory=src ${isEmscripten() ? "--nodedir=emscripten" : ""}`, {
     ...spawnSyncOptions,
     stdio: "inherit"
   }).status;
   var globalLibvipsVersion = () => {
     if (process.platform !== "win32") {
-      const globalLibvipsVersion2 = spawnSync("pkg-config --modversion vips-cpp", {
+      const globalLibvipsVersion2 = spawnSync2("pkg-config --modversion vips-cpp", {
         ...spawnSyncOptions,
         env: {
           ...process.env,
@@ -1897,7 +1897,7 @@ var require_libvips = __commonJS((exports, module) => {
   };
   var pkgConfigPath = () => {
     if (process.platform !== "win32") {
-      const brewPkgConfigPath = spawnSync('which brew >/dev/null 2>&1 && brew environment --plain | grep PKG_CONFIG_LIBDIR | cut -d" " -f2', spawnSyncOptions).stdout || "";
+      const brewPkgConfigPath = spawnSync2('which brew >/dev/null 2>&1 && brew environment --plain | grep PKG_CONFIG_LIBDIR | cut -d" " -f2', spawnSyncOptions).stdout || "";
       return [
         brewPkgConfigPath.trim(),
         process.env.PKG_CONFIG_PATH,
@@ -1962,18 +1962,18 @@ var require_sharp = __commonJS((exports, module) => {
     `@img/sharp-${runtimePlatform}/sharp.node`,
     "@img/sharp-wasm32/sharp.node"
   ];
-  var path;
+  var path2;
   var sharp;
   var errors = [];
-  for (path of paths) {
+  for (path2 of paths) {
     try {
-      sharp = __require(path);
+      sharp = __require(path2);
       break;
     } catch (err) {
       errors.push(err);
     }
   }
-  if (sharp && path.startsWith("@img/sharp-linux-x64") && !sharp._isUsingX64V2()) {
+  if (sharp && path2.startsWith("@img/sharp-linux-x64") && !sharp._isUsingX64V2()) {
     const err = new Error("Prebuilt binaries for linux-x64 require v2 microarchitecture");
     err.code = "Unsupported CPU";
     errors.push(err);
@@ -4835,15 +4835,15 @@ var require_color = __commonJS((exports, module) => {
     };
   }
   function wrapConversion(toModel, graph) {
-    const path = [graph[toModel].parent, toModel];
+    const path2 = [graph[toModel].parent, toModel];
     let fn = conversions_default[graph[toModel].parent][toModel];
     let cur = graph[toModel].parent;
     while (graph[cur].parent) {
-      path.unshift(graph[cur].parent);
+      path2.unshift(graph[cur].parent);
       fn = link(conversions_default[graph[cur].parent][cur], fn);
       cur = graph[cur].parent;
     }
-    fn.conversion = path;
+    fn.conversion = path2;
     return fn;
   }
   function route(fromModel) {
@@ -5448,7 +5448,7 @@ var require_output = __commonJS((exports, module) => {
     Copyright 2013 Lovell Fuller and others.
     SPDX-License-Identifier: Apache-2.0
   */
-  var path = __require("node:path");
+  var path2 = __require("node:path");
   var is = require_is();
   var sharp = require_sharp();
   var formats = new Map([
@@ -5479,9 +5479,9 @@ var require_output = __commonJS((exports, module) => {
     let err;
     if (!is.string(fileOut)) {
       err = new Error("Missing output file path");
-    } else if (is.string(this.options.input.file) && path.resolve(this.options.input.file) === path.resolve(fileOut)) {
+    } else if (is.string(this.options.input.file) && path2.resolve(this.options.input.file) === path2.resolve(fileOut)) {
       err = new Error("Cannot use same file for input and output");
-    } else if (jp2Regex.test(path.extname(fileOut)) && !this.constructor.format.jp2k.output.file) {
+    } else if (jp2Regex.test(path2.extname(fileOut)) && !this.constructor.format.jp2k.output.file) {
       err = errJp2Save();
     }
     if (err) {
@@ -6453,7 +6453,6 @@ var require_lib = __commonJS((exports, module) => {
 import { createRequire as createRequire2 } from "node:module";
 
 // src/doctor.ts
-var import_sharp = __toESM(require_lib(), 1);
 import { spawnSync as spawnSync2 } from "node:child_process";
 import { existsSync as existsSync2 } from "node:fs";
 import path2 from "node:path";
@@ -6501,9 +6500,11 @@ var listTesseractLanguages = () => {
   }
   const result = spawnSync("tesseract", ["--list-langs"], {
     encoding: "utf8",
-    timeout: 5000,
+    timeout: OCR_HEALTHCHECK_TIMEOUT_MS,
     windowsHide: true
   });
+  const stdout = typeof result.stdout === "string" ? result.stdout : "";
+  const languages = stdout.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.toLowerCase().includes("list of available"));
   if (result.status !== 0) {
     const stderr = typeof result.stderr === "string" ? result.stderr.trim() : "";
     return {
@@ -6512,9 +6513,6 @@ var listTesseractLanguages = () => {
       warning: stderr || "tesseract --list-langs failed"
     };
   }
-  const out = `${result.stdout ?? ""}
-${result.stderr ?? ""}`;
-  const languages = out.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.toLowerCase().includes("list of available languages") && !l.startsWith("Error"));
   return { available: true, languages };
 };
 
@@ -6550,7 +6548,18 @@ var probeTesseract = () => {
 };
 var probeSharp = async () => {
   try {
-    const buffer = await import_sharp.default({
+    let sharpMod;
+    try {
+      sharpMod = (await Promise.resolve().then(() => __toESM(require_lib(), 1))).default ?? await Promise.resolve().then(() => __toESM(require_lib(), 1));
+    } catch {
+      return {
+        id: "sharp",
+        status: "warn",
+        message: "sharp not installed (optional). Rust decode path is preferred for MCP."
+      };
+    }
+    const sharp = sharpMod;
+    const buffer = await sharp({
       create: {
         width: 1,
         height: 1,
@@ -6568,7 +6577,7 @@ var probeSharp = async () => {
     return {
       id: "sharp",
       status: "ok",
-      message: `sharp decode pipeline is available (v${import_sharp.default.versions.sharp}).`
+      message: `sharp decode pipeline is available (v${sharp.versions?.sharp ?? "unknown"}).`
     };
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
@@ -6643,7 +6652,7 @@ var probeRustDecodeFlag = () => {
   return {
     id: "rust_decode_flag",
     status: "warn",
-    message: "Rust decode CLI is not built. sharp remains the default probe path until `cargo build --release`."
+    message: "Rust decode CLI is not built. Install/stage image-reader-cli for native decode, or optional `sharp` for TS fallback."
   };
 };
 var probeNode = () => {

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@sylphx/image-reader-mcp",
-  "version": "0.1.2",
+  "version": "0.1.5",
   "mcpName": "io.github.SylphxAI/image-reader-mcp",
-  "description": "Iris \u2014 Evidence-first image reading for AI agents \u2014 metadata, OCR text, regions, and citeable evidence without generative LLM.",
+  "description": "Iris — Evidence-first image reading for AI agents — metadata, OCR text, regions, and citeable evidence without generative LLM.",
   "type": "module",
   "bin": {
     "image-reader-mcp": "./bin/image-reader-mcp",
     "iris": "./bin/image-reader-mcp"
   },
   "files": [
-    "bin/",
+    "bin/image-reader-mcp",
     "dist/",
     "README.md",
     "LICENSE",
@@ -69,8 +69,6 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "exifr": "^7.1.3",
-    "sharp": "^0.34.3",
     "zod": "^4.4.3"
   },
   "devDependencies": {
@@ -82,5 +80,13 @@
     "typescript": "^7.0.2"
   },
   "packageManager": "bun@1.3.12",
-  "private": false
+  "private": false,
+  "optionalDependencies": {
+    "sharp": "^0.34.3",
+    "exifr": "^7.1.3",
+    "@sylphx/image-reader-mcp-darwin-arm64": "0.1.5",
+    "@sylphx/image-reader-mcp-darwin-x64": "0.1.5",
+    "@sylphx/image-reader-mcp-linux-x64-gnu": "0.1.5",
+    "@sylphx/image-reader-mcp-linux-arm64-gnu": "0.1.5"
+  }
 }

package/src/doctor.ts

@@ -2,7 +2,6 @@ import { spawnSync } from 'node:child_process';
 import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import sharp from 'sharp';
 import { resolveRustCliBinary } from './engine/rust-decode.js';
 import { listTesseractLanguages } from './utils/ocr.js';
 import { IMAGE_SAFETY_LIMITS } from './utils/safety.js';
@@ -53,6 +52,26 @@ const probeTesseract = (): DoctorCheck => {
 
 const probeSharp = async (): Promise<DoctorCheck> => {
   try {
+    let sharpMod: unknown;
+    try {
+      sharpMod =
+        ((await import('sharp')) as { default?: unknown }).default ?? (await import('sharp'));
+    } catch {
+      return {
+        id: 'sharp',
+        status: 'warn',
+        message: 'sharp not installed (optional). Rust decode path is preferred for MCP.',
+      };
+    }
+    const sharp = sharpMod as {
+      (
+        input: unknown,
+        opts?: unknown
+      ): {
+        png: () => { toBuffer: () => Promise<Buffer> };
+      };
+      versions?: { sharp?: string };
+    };
     const buffer = await sharp({
       create: {
         width: 1,
@@ -75,7 +94,7 @@ const probeSharp = async (): Promise<DoctorCheck> => {
     return {
       id: 'sharp',
       status: 'ok',
-      message: `sharp decode pipeline is available (v${sharp.versions.sharp}).`,
+      message: `sharp decode pipeline is available (v${sharp.versions?.sharp ?? 'unknown'}).`,
     };
   } catch (error: unknown) {
     const message = error instanceof Error ? error.message : String(error);
@@ -162,7 +181,7 @@ const probeRustDecodeFlag = (): DoctorCheck => {
     id: 'rust_decode_flag',
     status: 'warn',
     message:
-      'Rust decode CLI is not built. sharp remains the default probe path until `cargo build --release`.',
+      'Rust decode CLI is not built. Install/stage image-reader-cli for native decode, or optional `sharp` for TS fallback.',
   };
 };
 

package/src/handlers/readImage.ts

@@ -1,7 +1,5 @@
 import fs, { stat } from 'node:fs/promises';
 import path from 'node:path';
-import exifr from 'exifr';
-import sharp from 'sharp';
 import {
   cropRegionViaRustEngine,
   probeImageViaRustEngine,
@@ -46,6 +44,23 @@ const readMetadata = async (
   }
 
   try {
+    let exifr: {
+      parse: (
+        path: string,
+        opts: Record<string, unknown>
+      ) => Promise<Record<string, unknown> | null>;
+    };
+    try {
+      exifr =
+        ((await import('exifr')) as { default?: typeof exifr }).default ??
+        ((await import('exifr')) as unknown as typeof exifr);
+    } catch {
+      return {
+        trustWarnings: [
+          'Optional dependency `exifr` is not installed; EXIF/XMP/IPTC metadata skipped (geometry/OCR still available).',
+        ],
+      };
+    }
     const parsed = await exifr.parse(filePath, {
       tiff: true,
       xmp: true,
@@ -140,6 +155,28 @@ export const readImage = tool()
           twin.metadata = extractedMetadata;
         }
       } else {
+        type SharpMeta = {
+          width?: number;
+          height?: number;
+          format?: string;
+          orientation?: number;
+          space?: string;
+          hasAlpha?: boolean;
+        };
+        type SharpFn = (
+          path: string,
+          opts?: { failOn?: string }
+        ) => { metadata: () => Promise<SharpMeta> };
+        let sharp: SharpFn;
+        try {
+          const mod = await import('sharp');
+          sharp = (mod as { default?: SharpFn }).default ?? (mod as unknown as SharpFn);
+        } catch {
+          throw new ImageError(
+            ErrorCode.InvalidRequest,
+            'Rust decode engine is unavailable and optional `sharp` is not installed. Build/stage image-reader-cli or install sharp for the TS fallback.'
+          );
+        }
         const image = sharp(resolvedPath, { failOn: 'none' });
         const metadata = await image.metadata();
         validateImageSafety({
@@ -202,6 +239,7 @@ export const readImage = tool()
             ? { languages_warning: ocr.languages_warning }
             : {}),
           ...(ocr.words !== undefined ? { words: ocr.words } : {}),
+          ...(ocr.native_blocks !== undefined ? { native_blocks: ocr.native_blocks } : {}),
         };
       }
 

package/src/schemas/readImage.ts

@@ -74,12 +74,14 @@ export const readImageArgsSchema = z.object({
   include_palette: z
     .boolean()
     .optional()
-    .describe('Sample an approximate local color palette via sharp (not ML). Defaults to false.'),
+    .describe(
+      'Sample an approximate local color palette via optional sharp when installed (not ML). Defaults to false.'
+    ),
   include_optional_llm: z
     .boolean()
     .optional()
     .describe(
-      'Optional caption via IRIS_OPTIONAL_LLM_URL. Off by default; never authority over OCR/layout evidence.'
+      'Optional local frontier caption via Ollama vision models or IRIS_OPTIONAL_LLM_URL. Off by default; never authority over OCR/layout evidence.'
     ),
 });
 
@@ -110,6 +112,17 @@ export const agentMediaTwinSchema = z.object({
           })
         )
         .optional(),
+      native_blocks: z
+        .array(
+          z.object({
+            id: z.string(),
+            kind: z.enum(['block', 'paragraph']),
+            text: z.string(),
+            bbox: boundingBoxSchema,
+            confidence: z.number().optional(),
+          })
+        )
+        .optional(),
     })
     .optional(),
   region_evidence: z
@@ -157,6 +170,7 @@ export const agentMediaTwinSchema = z.object({
           skipped_reason: z.string().optional(),
           route: z.string().optional(),
           caption: z.string().optional(),
+          model: z.string().optional(),
         })
         .optional(),
     })

package/src/utils/agentMap.ts

@@ -20,6 +20,7 @@ export type AgentImageMap = {
     skipped_reason?: string;
     route?: string;
     caption?: string;
+    model?: string;
   };
 };
 

package/src/utils/applyImageIntelligence.ts

@@ -1,6 +1,6 @@
 import type { AgentMediaTwin, ReadImageArgs } from '../schemas/readImage.js';
 import { buildAgentImageMap } from './agentMap.js';
-import { buildLayoutFromOcrLines } from './layout.js';
+import { buildBestEffortLayout } from './layout.js';
 import { maybeOptionalImageCaption } from './optionalLlm.js';
 import { samplePalette } from './palette.js';
 
@@ -18,14 +18,30 @@ export async function applyImageIntelligence(
 
   const includeLayout = input.include_layout ?? includeOcr;
   if (includeLayout && next.ocr?.lines && next.ocr.lines.length > 0) {
-    const layout = buildLayoutFromOcrLines(
-      next.ocr.lines.map((line) => ({
+    const ocrAny = next.ocr as {
+      lines: Array<{
+        text: string;
+        bbox: { x: number; y: number; width: number; height: number };
+        confidence?: number;
+      }>;
+      native_blocks?: Array<{
+        id: string;
+        kind: 'block' | 'paragraph';
+        text: string;
+        bbox: { x: number; y: number; width: number; height: number };
+        confidence?: number;
+      }>;
+    };
+    const layout = buildBestEffortLayout({
+      lines: ocrAny.lines.map((line) => ({
         text: line.text,
         bbox: line.bbox,
         ...(line.confidence !== undefined ? { confidence: line.confidence } : {}),
-      }))
-    );
+      })),
+      ...(ocrAny.native_blocks ? { nativeBlocks: ocrAny.native_blocks } : {}),
+    });
     next.layout = layout;
+    next.trust_warnings.push(`layout_policy: ${layout.policy}`);
     for (const warning of layout.warnings) {
       next.trust_warnings.push(`layout: ${warning}`);
     }