@sylphx/iris 0.1.8 → 0.2.0

package/README.md

@@ -2,9 +2,11 @@
 
 # Iris
 
-### Image evidence for agents 
+### Image evidence for agents
 
-**Iris** (transitional package `@sylphx/image-reader-mcp`) — **Rust-first** local image facts, not default VLM guesses. `sharp`/`exifr` are optional, not required for MCP.
+Canonical package: **`@sylphx/iris`** · bin **`iris`** 
+
+**Iris** — **Rust-first** local image facts (`@sylphx/iris`) local image facts, not default VLM guesses. `sharp`/`exifr` are optional, not required for MCP.
 
 <p align="center">
   <img src="https://mark.sylphx.com/api/v1/banner?type=holo&theme=tokyonight&text=image+reader+mcp&desc=Evidence-first+image+reading+for+AI+agents+%E2%80%94+metadata%2C+OCR+text%2C+regions%2C+and+ci&height=200&animation=rise&credit=0" alt="image-reader-mcp — Sylphx Mark banner" width="100%" />
@@ -16,7 +18,7 @@ Evidence-first image reading for AI agents. One call turns any local image into
 **Agent Media Twin** — dimensions, metadata, optional OCR with bounding boxes, and
 trust warnings you can cite without asking a vision LLM to guess.
 
-[![npm version](https://img.shields.io/npm/v/@sylphx/image-reader-mcp?style=flat-square)](https://www.npmjs.com/package/@sylphx/image-reader-mcp)
+[![npm version](https://img.shields.io/npm/v/@sylphx/iris?style=flat-square)](https://www.npmjs.com/package/@sylphx/iris)
 [![License](https://img.shields.io/badge/License-MIT-blue?style=flat-square)](https://opensource.org/licenses/MIT)
 [![CI/CD](https://img.shields.io/github/actions/workflow/status/SylphxAI/image-reader-mcp/ci.yml?style=flat-square&label=CI/CD)](https://github.com/SylphxAI/image-reader-mcp/actions/workflows/ci.yml)
 [![TypeScript](https://img.shields.io/badge/TypeScript-7.0-blue.svg?style=flat-square)](https://www.typescriptlang.org/)
@@ -37,13 +39,23 @@ Each instrument is an independent repository (marketplace + stars).
 
 
 
+## See objects (L2, optional)
+
+With a local Florence-class sidecar or Ollama, the same `read_image` can return **open-vocab objects** with pixel bboxes and scores:
+
+```json
+{ "path": "/abs/photo.jpg", "include_semantics": true, "semantics_prompt": "people and animals" }
+```
+
+Objects are `scored_non_locator` evidence — deterministic L0/L1 facts (geometry/OCR/layout) stay authoritative and always on.
+
 ## Read images (not vague vision)
 
 Iris is **local-first**: geometry + OCR + **layout blocks** + **agent_map** so a text-only agent can understand picture architecture without a vision model.
 
 Spec: [docs/specs/agent-image-read-contract.md](docs/specs/agent-image-read-contract.md)
 
-**Local-first frontier:** Rust decode, Tesseract native layout (no npm ML), optional Ollama VLM; cloud URL optional. Zero API key.
+**Local-first frontier:** Rust decode, Tesseract native layout (no npm ML), optional Ollama VLM; cloud URL optional. Zero API key. Optional **L2 local semantics** (include_semantics) detects open-vocab objects (people/animals/things) with pixel bboxes via an official Florence-class sidecar (examples/florence-sidecar/) or Ollama -- never authority over OCR/layout locators.
 
 ## Product docs
 
@@ -86,15 +98,15 @@ an image, not a creative caption.**
 ### Install (30 seconds)
 
 ```bash
-npm install -g @sylphx/image-reader-mcp
+npm install -g @sylphx/iris
 iris doctor
-claude mcp add iris -- npx @sylphx/image-reader-mcp
+claude mcp add iris -- npx @sylphx/iris
 ```
 
 **Install once. Call once.**
 
 ```bash
-claude mcp add image-reader -- npx @sylphx/image-reader-mcp
+claude mcp add image-reader -- npx @sylphx/iris
 ```
 
 ```json
@@ -148,7 +160,7 @@ Supported formats: PNG, JPEG, GIF, WebP, TIFF, and other formats the **Rust deco
 ### Claude Code
 
 ```bash
-claude mcp add image-reader -- npx @sylphx/image-reader-mcp
+claude mcp add image-reader -- npx @sylphx/iris
 ```
 
 ### Claude Desktop
@@ -160,7 +172,7 @@ Add this to `claude_desktop_config.json`:
   "mcpServers": {
     "image-reader": {
       "command": "npx",
-      "args": ["@sylphx/image-reader-mcp"]
+      "args": ["@sylphx/iris"]
     }
   }
 }
@@ -169,7 +181,7 @@ Add this to `claude_desktop_config.json`:
 ### Any MCP Client
 
 ```bash
-npx @sylphx/image-reader-mcp
+npx @sylphx/iris
 ```
 
 Node.js `>=22.13` is required. Optional OCR uses a local Tesseract adapter when
@@ -219,7 +231,7 @@ Example `read_image` requests live in [`examples/`](examples/).
 ## Support
 
 - [Issues](https://github.com/SylphxAI/image-reader-mcp/issues)
-- [npm package](https://www.npmjs.com/package/@sylphx/image-reader-mcp)
+- [npm package](https://www.npmjs.com/package/@sylphx/iris)
 - Portfolio orchestration: [smart-reader-mcp](https://github.com/SylphxAI/smart-reader-mcp)
 
 ## Help this reach more builders

package/bin/{image-reader-mcp → iris}

@@ -23,10 +23,10 @@ host_platform_key() {
 
 platform_package_name() {
 	case "$(host_platform_key)" in
-		darwin-arm64) printf '%s\n' '@sylphx/image-reader-mcp-darwin-arm64' ;;
-		darwin-x64) printf '%s\n' '@sylphx/image-reader-mcp-darwin-x64' ;;
-		linux-x64) printf '%s\n' '@sylphx/image-reader-mcp-linux-x64-gnu' ;;
-		linux-arm64) printf '%s\n' '@sylphx/image-reader-mcp-linux-arm64-gnu' ;;
+		darwin-arm64) printf '%s\n' '@sylphx/iris-darwin-arm64' ;;
+		darwin-x64) printf '%s\n' '@sylphx/iris-darwin-x64' ;;
+		linux-x64) printf '%s\n' '@sylphx/iris-linux-x64-gnu' ;;
+		linux-arm64) printf '%s\n' '@sylphx/iris-linux-arm64-gnu' ;;
 		*) return 1 ;;
 	esac
 }
@@ -47,7 +47,7 @@ resolve_from_optional_dep() {
 		"$PACKAGE_ROOT/../../node_modules"
 	)
 	for root in "${search_roots[@]}"; do
-		candidate="$root/$pkg_name/image-reader-mcp-server"
+		candidate="$root/$pkg_name/iris-mcp-server"
 		if is_runnable_native "$candidate"; then
 			printf '%s\n' "$candidate"
 			return 0
@@ -61,7 +61,7 @@ resolve_from_optional_dep() {
           const { createRequire } = require('node:module');
           const r = createRequire(p.join(process.argv[1], 'package.json'));
           const pkg = r.resolve(process.argv[2] + '/package.json');
-          process.stdout.write(p.join(p.dirname(pkg), 'image-reader-mcp-server'));
+          process.stdout.write(p.join(p.dirname(pkg), 'iris-mcp-server'));
         } catch { process.exit(1); }
       " "$PACKAGE_ROOT" "$pkg_name" 2>/dev/null || true
 		)"
@@ -74,8 +74,8 @@ resolve_from_optional_dep() {
 }
 
 resolve_rust_bin() {
-	if [[ -n "${IMAGE_READER_MCP_RUST_BIN:-}" && -x "${IMAGE_READER_MCP_RUST_BIN}" ]]; then
-		printf '%s\n' "${IMAGE_READER_MCP_RUST_BIN}"
+	if [[ -n "${IRIS_RUST_BIN:-}" && -x "${IRIS_RUST_BIN}" ]]; then
+		printf '%s\n' "${IRIS_RUST_BIN}"
 		return 0
 	fi
 	if bin="$(resolve_from_optional_dep)"; then
@@ -83,9 +83,9 @@ resolve_rust_bin() {
 		return 0
 	fi
 	for candidate in \
-		"$PACKAGE_ROOT/bin/native/image-reader-mcp-server" \
-		"$PACKAGE_ROOT/target/release/image-reader-mcp-server" \
-		"$PACKAGE_ROOT/target/debug/image-reader-mcp-server"; do
+		"$PACKAGE_ROOT/bin/native/iris-mcp-server" \
+		"$PACKAGE_ROOT/target/release/iris-mcp-server" \
+		"$PACKAGE_ROOT/target/debug/iris-mcp-server"; do
 		if is_runnable_native "$candidate"; then
 			printf '%s\n' "$candidate"
 			return 0
@@ -115,5 +115,5 @@ if bin="$(resolve_rust_bin)"; then
 	exec "$bin" "$@"
 fi
 
-echo "[iris] Rust MCP server not found. Install matching optional native package (@sylphx/image-reader-mcp-<platform>) or run: bun run build:rust" >&2
+echo "[iris] Rust MCP server not found. Install matching optional native package (@sylphx/iris-<platform>) or run: bun run build:rust" >&2
 exit 1

package/package.json

@@ -1,15 +1,14 @@
 {
   "name": "@sylphx/iris",
-  "version": "0.1.8",
-  "mcpName": "io.github.SylphxAI/image-reader-mcp",
-  "description": "Iris — Evidence-first image reading for AI agents — metadata, OCR text, regions, and citeable evidence without generative LLM. (brand alias of @sylphx/image-reader-mcp)",
+  "version": "0.2.0",
+  "mcpName": "io.github.SylphxAI/iris",
+  "description": "Iris \u2014 evidence-first image reading for AI agents. Metadata, OCR text, regions, citeable evidence. Local-first. No generative LLM authority.",
   "type": "module",
   "bin": {
-    "image-reader-mcp": "./bin/image-reader-mcp",
-    "iris": "./bin/image-reader-mcp"
+    "iris": "./bin/iris"
   },
   "files": [
-    "bin/image-reader-mcp",
+    "bin/iris",
     "dist/",
     "README.md",
     "LICENSE",
@@ -17,8 +16,7 @@
   ],
   "exports": {
     ".": "./dist/doctor-cli.js",
-    "./sdk": "./src/sdk.ts",
-    "./iris": "./src/sdk.ts"
+    "./sdk": "./src/sdk.ts"
   },
   "publishConfig": {
     "access": "public"
@@ -60,8 +58,9 @@
     "benchmark:release-gate": "bun scripts/release-gate.ts",
     "sync:server-json": "bun scripts/sync-server-json.ts",
     "version-packages": "changeset version && bun run sync:server-json",
-    "start": "./bin/image-reader-mcp",
+    "start": "./bin/iris",
     "clean": "rm -rf dist coverage",
+    "prepublishOnly": "bun run clean && bun run build && bun run build:rust",
     "release": "bun run typecheck && bun run check && bun run build && bun test && changeset publish",
     "benchmark:public-proof": "bun scripts/public-proof.ts",
     "brand:pack-plan": "bun scripts/brand-pack-plan.ts"
@@ -83,9 +82,9 @@
   "optionalDependencies": {
     "sharp": "^0.35.0",
     "exifr": "^7.1.3",
-    "@sylphx/image-reader-mcp-darwin-arm64": "0.1.8",
-    "@sylphx/image-reader-mcp-darwin-x64": "0.1.8",
-    "@sylphx/image-reader-mcp-linux-x64-gnu": "0.1.8",
-    "@sylphx/image-reader-mcp-linux-arm64-gnu": "0.1.8"
+    "@sylphx/iris-darwin-arm64": "0.2.0",
+    "@sylphx/iris-darwin-x64": "0.2.0",
+    "@sylphx/iris-linux-x64-gnu": "0.2.0",
+    "@sylphx/iris-linux-arm64-gnu": "0.2.0"
   }
-}
+}

package/src/handlers/readImage.ts

@@ -90,7 +90,7 @@ const readMetadata = async (
 
 export const readImage = tool()
   .description(
-    'Evidence-first image reader for agents (read, not vague vision). Returns Agent Media Twin: geometry, metadata, OCR lines/words, layout blocks, text agent_map, optional palette, optional non-authority LLM caption. Local-first; generative path off by default.'
+    'Evidence-first image reader for agents (read, not vague vision). Returns Agent Media Twin: geometry, metadata, OCR lines/words, layout blocks, text agent_map, optional palette, optional non-authority LLM caption, optional L2 semantics (objects). Local-first; generative path off by default.'
   )
   .input(readImageArgsSchema)
   .handler(async ({ input }) => {

package/src/schemas/readImage.ts

@@ -83,6 +83,17 @@ export const readImageArgsSchema = z.object({
     .describe(
       'Optional local frontier caption via Ollama vision models or IRIS_OPTIONAL_LLM_URL. Off by default; never authority over OCR/layout evidence.'
     ),
+  include_semantics: z
+    .union([z.boolean(), z.literal('auto')])
+    .optional()
+    .describe(
+      'L2 local semantics: open-vocab objects + optional caption via IRIS_SEMANTICS_URL (Florence/DINO/SAM adapter) or Ollama structured vision. Default false (zero-config). Never authority over OCR/layout locators.'
+    ),
+  semantics_prompt: z
+    .string()
+    .max(500)
+    .optional()
+    .describe('Optional focus prompt for L2 semantics (e.g. "animals", "UI widgets").'),
 });
 
 export const agentMediaTwinSchema = z.object({
@@ -175,6 +186,30 @@ export const agentMediaTwinSchema = z.object({
         .optional(),
     })
     .optional(),
+  semantics: z
+    .object({
+      available: z.boolean(),
+      authority: z.literal('scored_non_locator'),
+      skipped_reason: z.string().optional(),
+      route: z.string().optional(),
+      model: z.string().optional(),
+      caption: z.string().optional(),
+      object_count: z.number().int().nonnegative().optional(),
+      objects: z
+        .array(
+          z.object({
+            id: z.string(),
+            label: z.string(),
+            category: z.string().optional(),
+            bbox: boundingBoxSchema.optional(),
+            score: z.number().min(0).max(1).optional(),
+            mask_ref: z.string().nullable().optional(),
+          })
+        )
+        .optional(),
+      warnings: z.array(z.string()).optional(),
+    })
+    .optional(),
   trust_warnings: z.array(z.string()),
 });
 

package/src/utils/agentMap.ts

@@ -1,41 +1,22 @@
-/**
- * Text-only Agent Map: lets a non-vision model "read" the image architecture.
- * Deterministic, local-first. Optional LLM captions are layered elsewhere.
- */
+import type { AgentMediaTwin } from '../schemas/readImage.js';
+import type { SemanticsResult } from './optionalSemantics.js';
 
-import type { ImageLayout } from './layout.js';
-
-export type AgentImageMap = {
-  policy: 'agent_image_map_v1';
-  filename: string;
-  mime: string;
-  dimensions: { width: number; height: number };
-  /** Human/agent readable outline (markdown-ish, not prose hallucination). */
-  outline: string;
-  text_present: boolean;
-  block_count: number;
-  palette?: Array<{ hex: string; approx_share: number }>;
-  optional_llm?: {
-    available: boolean;
-    skipped_reason?: string;
-    route?: string;
-    caption?: string;
-    model?: string;
-  };
-};
+export type AgentImageMap = NonNullable<AgentMediaTwin['agent_map']>;
 
 export function buildAgentImageMap(input: {
   filename: string;
   mime: string;
   dimensions: { width: number; height: number };
-  layout?: ImageLayout;
+  layout?: AgentMediaTwin['layout'];
   ocrLineCount?: number;
   palette?: Array<{ hex: string; approx_share: number }>;
   optionalLlm?: AgentImageMap['optional_llm'];
+  semantics?: SemanticsResult | undefined;
 }): AgentImageMap {
   const { width, height } = input.dimensions;
   const blocks = input.layout?.blocks ?? [];
   const textPresent = (input.ocrLineCount ?? 0) > 0 || blocks.length > 0;
+  const objects = input.semantics?.available ? (input.semantics.objects ?? []) : [];
 
   const lines: string[] = [
     `# Image map: ${input.filename}`,
@@ -43,6 +24,7 @@ export function buildAgentImageMap(input: {
     `- size: ${width}×${height}px`,
     `- text_present: ${textPresent}`,
     `- layout_blocks: ${blocks.length}`,
+    `- semantics_objects: ${objects.length}`,
   ];
 
   if (input.palette && input.palette.length > 0) {
@@ -62,15 +44,32 @@ export function buildAgentImageMap(input: {
         ''
       );
     }
-  } else if (!textPresent) {
+  } else if (!textPresent && objects.length === 0) {
     lines.push(
       '',
       '## Notes',
       '- No OCR text recovered. Enable include_ocr for text architecture.',
-      '- Use crop_region / image_probe for geometry; optional LLM caption only if configured.'
+      '- Enable include_semantics for local open-vocab objects (IRIS_SEMANTICS_URL or Ollama).',
+      '- Use crop_region / image_probe for geometry.'
     );
   }
 
+  if (objects.length > 0) {
+    lines.push('', '## L2 semantics objects (scored, non-locator authority)');
+    for (const obj of objects.slice(0, 24)) {
+      const box = obj.bbox
+        ? `bbox x=${obj.bbox.x} y=${obj.bbox.y} w=${obj.bbox.width} h=${obj.bbox.height}`
+        : 'bbox: n/a';
+      const score = obj.score !== undefined ? ` score=${obj.score.toFixed(2)}` : '';
+      lines.push(`- ${obj.id}: ${obj.label}${score}; ${box}`);
+    }
+    if (input.semantics?.caption) {
+      lines.push('', '## L2 caption (scored_non_locator)', input.semantics.caption);
+    }
+  } else if (input.semantics && !input.semantics.available) {
+    lines.push('', `## L2 semantics: skipped (${input.semantics.skipped_reason ?? 'unavailable'})`);
+  }
+
   if (input.optionalLlm?.available && input.optionalLlm.caption) {
     lines.push('', '## Optional LLM caption (non-authority)', input.optionalLlm.caption);
   } else if (input.optionalLlm && !input.optionalLlm.available) {

package/src/utils/applyImageIntelligence.ts

@@ -2,9 +2,10 @@ import type { AgentMediaTwin, ReadImageArgs } from '../schemas/readImage.js';
 import { buildAgentImageMap } from './agentMap.js';
 import { buildBestEffortLayout } from './layout.js';
 import { maybeOptionalImageCaption } from './optionalLlm.js';
+import { maybeImageSemantics } from './optionalSemantics.js';
 import { samplePalette } from './palette.js';
 
-/** Attach layout, palette, optional LLM, and agent_map to an Agent Media Twin. */
+/** Attach layout, palette, optional LLM/semantics, and agent_map to an Agent Media Twin. */
 export async function applyImageIntelligence(
   twin: AgentMediaTwin,
   resolvedPath: string,
@@ -67,6 +68,28 @@ export async function applyImageIntelligence(
     );
   }
 
+  const semantics = await maybeImageSemantics({
+    path: resolvedPath,
+    mime: next.mime,
+    width: next.dimensions.width,
+    height: next.dimensions.height,
+    include: input.include_semantics,
+    prompt: input.semantics_prompt,
+  });
+  if (semantics) {
+    next.semantics = semantics;
+    if (semantics.available) {
+      next.trust_warnings.push(
+        `semantics L2 via ${semantics.route ?? 'unknown'} is scored_non_locator; never overrides OCR/layout.`
+      );
+      if (semantics.object_count) {
+        next.trust_warnings.push(`semantics_objects: ${semantics.object_count}`);
+      }
+    } else if (semantics.skipped_reason) {
+      next.trust_warnings.push(`semantics_skipped: ${semantics.skipped_reason}`);
+    }
+  }
+
   const includeAgentMap = input.include_agent_map ?? true;
   if (includeAgentMap) {
     next.agent_map = buildAgentImageMap({
@@ -77,6 +100,7 @@ export async function applyImageIntelligence(
       ocrLineCount: next.ocr?.line_count ?? next.ocr?.lines?.length ?? 0,
       ...(palette ? { palette } : {}),
       optionalLlm,
+      ...(semantics ? { semantics } : {}),
     });
   }
 

package/src/utils/optionalSemantics.ts

@@ -0,0 +1,402 @@
+/**
+ * Iris L2 local semantics — open-vocab objects + optional caption.
+ *
+ * Priority:
+ * 1) IRIS_SEMANTICS_URL — Florence/Grounding-DINO/SAM-class adapter
+ * 2) Local Ollama vision with structured JSON objects
+ *
+ * Authority: scored_non_locator — never overrides OCR/layout locators.
+ * Default OFF (include_semantics false) for zero-config surprise control.
+ */
+
+import { readFileSync } from 'node:fs';
+
+export type SemanticsBBox = {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+};
+
+export type SemanticsObject = {
+  id: string;
+  label: string;
+  category?: string | undefined;
+  bbox?: SemanticsBBox | undefined;
+  score?: number | undefined;
+  mask_ref?: string | null | undefined;
+};
+
+export type SemanticsResult = {
+  available: boolean;
+  authority: 'scored_non_locator';
+  skipped_reason?: string | undefined;
+  route?: string | undefined;
+  model?: string | undefined;
+  caption?: string | undefined;
+  object_count?: number | undefined;
+  objects?: SemanticsObject[] | undefined;
+  warnings?: string[] | undefined;
+};
+
+export type IncludeSemantics = boolean | 'auto';
+
+const DEFAULT_OLLAMA = 'http://127.0.0.1:11434';
+
+const VISION_MODEL_HINTS = [
+  'llava',
+  'minicpm-v',
+  'minicpm_v',
+  'qwen2-vl',
+  'qwen2.5-vl',
+  'qwen3-vl',
+  'bakllava',
+  'moondream',
+  'gemma3',
+  'llama3.2-vision',
+  'pixtral',
+  'florence',
+];
+
+function isVisionModel(name: string): boolean {
+  const n = name.toLowerCase();
+  return VISION_MODEL_HINTS.some((h) => n.includes(h));
+}
+
+function clampBBox(
+  raw: Partial<SemanticsBBox> | undefined,
+  maxW: number,
+  maxH: number
+): SemanticsBBox | undefined {
+  if (!raw || typeof raw !== 'object') return undefined;
+  const x = Number(raw.x);
+  const y = Number(raw.y);
+  const width = Number(raw.width);
+  const height = Number(raw.height);
+  if (![x, y, width, height].every((n) => Number.isFinite(n))) return undefined;
+  if (width <= 0 || height <= 0) return undefined;
+  const cx = Math.max(0, Math.min(maxW, Math.round(x)));
+  const cy = Math.max(0, Math.min(maxH, Math.round(y)));
+  const cw = Math.max(1, Math.min(maxW - cx, Math.round(width)));
+  const ch = Math.max(1, Math.min(maxH - cy, Math.round(height)));
+  return { x: cx, y: cy, width: cw, height: ch };
+}
+
+function parseScore(scoreRaw: unknown): number | undefined {
+  if (typeof scoreRaw !== 'number' || !Number.isFinite(scoreRaw)) return undefined;
+  return Math.max(0, Math.min(1, scoreRaw > 1 ? scoreRaw / 100 : scoreRaw));
+}
+
+function parseObjectItem(
+  item: unknown,
+  index: number,
+  maxW: number,
+  maxH: number,
+  warnings: string[]
+): SemanticsObject | undefined {
+  if (!item || typeof item !== 'object') return undefined;
+  const rec = item as Record<string, unknown>;
+  const label = typeof rec['label'] === 'string' ? rec['label'].trim() : '';
+  if (!label) return undefined;
+  const score = parseScore(rec['score'] ?? rec['confidence']);
+  const bbox = clampBBox(rec['bbox'] as Partial<SemanticsBBox> | undefined, maxW, maxH);
+  if (rec['bbox'] && !bbox) warnings.push(`dropped invalid bbox for ${label}`);
+  const id = typeof rec['id'] === 'string' && rec['id'] ? rec['id'] : `obj_${index}`;
+  const category = typeof rec['category'] === 'string' ? rec['category'].slice(0, 64) : undefined;
+  const maskRaw = rec['mask_ref'];
+  const mask_ref = typeof maskRaw === 'string' ? maskRaw : maskRaw === null ? null : undefined;
+  return {
+    id,
+    label: label.slice(0, 128),
+    ...(category !== undefined ? { category } : {}),
+    ...(bbox ? { bbox } : {}),
+    ...(score !== undefined ? { score } : {}),
+    ...(mask_ref !== undefined ? { mask_ref } : {}),
+  };
+}
+
+function normalizeObjects(
+  objects: unknown,
+  maxW: number,
+  maxH: number
+): { objects: SemanticsObject[]; warnings: string[] } {
+  const warnings: string[] = [];
+  if (!Array.isArray(objects)) {
+    return { objects: [], warnings: ['semantics objects missing or not an array'] };
+  }
+  const out: SemanticsObject[] = [];
+  let i = 0;
+  for (const item of objects.slice(0, 64)) {
+    i += 1;
+    const obj = parseObjectItem(item, i, maxW, maxH, warnings);
+    if (obj) out.push(obj);
+  }
+  return { objects: out, warnings };
+}
+
+function extractJsonObject(text: string): unknown {
+  const trimmed = text.trim();
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const start = trimmed.indexOf('{');
+    const end = trimmed.lastIndexOf('}');
+    if (start >= 0 && end > start) {
+      return JSON.parse(trimmed.slice(start, end + 1));
+    }
+    throw new Error('no JSON object in model response');
+  }
+}
+
+async function tryHttpSemantics(input: {
+  path: string;
+  mime: string;
+  url: string;
+  prompt?: string | undefined;
+  width: number;
+  height: number;
+}): Promise<SemanticsResult> {
+  try {
+    const res = await fetch(input.url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        path: input.path,
+        mime: input.mime,
+        purpose: 'image_semantics',
+        prompt: input.prompt,
+        dimensions: { width: input.width, height: input.height },
+      }),
+      signal: AbortSignal.timeout(45_000),
+    });
+    if (!res.ok) {
+      return {
+        available: false,
+        authority: 'scored_non_locator',
+        skipped_reason: `semantics HTTP ${res.status}`,
+        route: input.url,
+      };
+    }
+    const body = (await res.json()) as {
+      caption?: string;
+      model?: string;
+      objects?: unknown;
+      warnings?: string[];
+    };
+    const { objects, warnings } = normalizeObjects(body.objects ?? [], input.width, input.height);
+    const caption = typeof body.caption === 'string' ? body.caption.slice(0, 4000) : undefined;
+    if (objects.length === 0 && !caption) {
+      return {
+        available: false,
+        authority: 'scored_non_locator',
+        skipped_reason: 'semantics HTTP response had no objects or caption',
+        route: input.url,
+        model: body.model,
+        warnings,
+      };
+    }
+    return {
+      available: true,
+      authority: 'scored_non_locator',
+      route: `iris-semantics-http:${input.url}`,
+      model: body.model,
+      caption,
+      object_count: objects.length,
+      objects,
+      warnings: [...warnings, ...(Array.isArray(body.warnings) ? body.warnings.map(String) : [])],
+    };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'semantics HTTP failed';
+    return {
+      available: false,
+      authority: 'scored_non_locator',
+      skipped_reason: message,
+      route: input.url,
+    };
+  }
+}
+
+async function listOllamaModels(base: string): Promise<string[]> {
+  const res = await fetch(`${base.replace(/\/$/, '')}/api/tags`, {
+    signal: AbortSignal.timeout(3_000),
+  });
+  if (!res.ok) return [];
+  const body = (await res.json()) as { models?: Array<{ name?: string }> };
+  return (body.models ?? [])
+    .map((m) => m.name)
+    .filter((n): n is string => typeof n === 'string' && n.length > 0);
+}
+
+async function tryOllamaStructured(input: {
+  path: string;
+  baseUrl: string;
+  width: number;
+  height: number;
+  prompt?: string | undefined;
+  modelHint?: string | undefined;
+}): Promise<SemanticsResult> {
+  const base = input.baseUrl.replace(/\/$/, '');
+  try {
+    const models = await listOllamaModels(base);
+    const preferred =
+      input.modelHint && models.includes(input.modelHint)
+        ? input.modelHint
+        : models.find(isVisionModel);
+    if (!preferred) {
+      return {
+        available: false,
+        authority: 'scored_non_locator',
+        skipped_reason: 'no local Ollama vision model for structured semantics',
+        route: `ollama:${base}`,
+      };
+    }
+
+    const bytes = readFileSync(input.path);
+    // Keep payloads modest for local VLM
+    if (bytes.byteLength > 8_000_000) {
+      return {
+        available: false,
+        authority: 'scored_non_locator',
+        skipped_reason: 'image too large for default Ollama semantics path; crop_region first',
+        route: `ollama:${preferred}`,
+        model: preferred,
+      };
+    }
+    const b64 = bytes.toString('base64');
+    const focus = input.prompt?.trim() ? ` Focus: ${input.prompt.trim().slice(0, 200)}.` : '';
+    const system = `You extract structured visual evidence for software agents.
+Image size is ${input.width}x${input.height} pixels (x right, y down).
+Return ONLY a JSON object (no markdown) with shape:
+{"caption":"short factual caption","objects":[{"label":"person|dog|car|…","category":"optional","bbox":{"x":0,"y":0,"width":10,"height":10},"score":0.0}]}
+Rules: bbox in pixels for the full image; invent nothing unreadable; empty objects array if unsure; max 24 objects; scores 0..1.${focus}`;
+
+    const res = await fetch(`${base}/api/chat`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        model: preferred,
+        stream: false,
+        format: 'json',
+        messages: [
+          { role: 'system', content: system },
+          {
+            role: 'user',
+            content: 'Extract objects and caption from this image as JSON.',
+            images: [b64],
+          },
+        ],
+        options: { temperature: 0.1 },
+      }),
+      signal: AbortSignal.timeout(90_000),
+    });
+    if (!res.ok) {
+      return {
+        available: false,
+        authority: 'scored_non_locator',
+        skipped_reason: `ollama chat HTTP ${res.status}`,
+        route: `ollama:${preferred}`,
+        model: preferred,
+      };
+    }
+    const body = (await res.json()) as {
+      message?: { content?: string };
+      response?: string;
+    };
+    const text = body.message?.content ?? body.response;
+    if (!text || typeof text !== 'string') {
+      return {
+        available: false,
+        authority: 'scored_non_locator',
+        skipped_reason: 'ollama response missing content',
+        route: `ollama-structured:${preferred}`,
+        model: preferred,
+      };
+    }
+    const parsed = extractJsonObject(text) as {
+      caption?: string;
+      objects?: unknown;
+    };
+    const { objects, warnings } = normalizeObjects(parsed.objects ?? [], input.width, input.height);
+    const caption = typeof parsed.caption === 'string' ? parsed.caption.slice(0, 4000) : undefined;
+    if (objects.length === 0 && !caption) {
+      return {
+        available: false,
+        authority: 'scored_non_locator',
+        skipped_reason: 'structured VLM returned no objects or caption',
+        route: `ollama-structured:${preferred}`,
+        model: preferred,
+        warnings,
+      };
+    }
+    return {
+      available: true,
+      authority: 'scored_non_locator',
+      route: `ollama-structured:${preferred}`,
+      model: preferred,
+      caption,
+      object_count: objects.length,
+      objects,
+      warnings,
+    };
+  } catch (error: unknown) {
+    const message = error instanceof Error ? error.message : 'ollama semantics failed';
+    return {
+      available: false,
+      authority: 'scored_non_locator',
+      skipped_reason: message,
+      route: `ollama:${base}`,
+    };
+  }
+}
+
+export async function maybeImageSemantics(input: {
+  path: string;
+  mime: string;
+  width: number;
+  height: number;
+  include: IncludeSemantics | undefined;
+  prompt?: string | undefined;
+}): Promise<SemanticsResult | undefined> {
+  const flag = input.include ?? false;
+  if (flag === false) return undefined;
+
+  const httpUrl = process.env['IRIS_SEMANTICS_URL']?.trim();
+  if (httpUrl) {
+    const http = await tryHttpSemantics({
+      path: input.path,
+      mime: input.mime,
+      url: httpUrl,
+      prompt: input.prompt,
+      width: input.width,
+      height: input.height,
+    });
+    if (http.available) return http;
+    // Prefer explicit HTTP error when URL is set and mode is true
+    if (flag === true) return http;
+  }
+
+  const ollamaBase = process.env['IRIS_OLLAMA_URL'] ?? process.env['OLLAMA_HOST'] ?? DEFAULT_OLLAMA;
+  const baseUrl = ollamaBase.startsWith('http') ? ollamaBase : `http://${ollamaBase}`;
+  const modelHint = process.env['IRIS_OLLAMA_VISION_MODEL']?.trim();
+  const ollama = await tryOllamaStructured({
+    path: input.path,
+    baseUrl,
+    width: input.width,
+    height: input.height,
+    prompt: input.prompt,
+    modelHint,
+  });
+  if (ollama.available) return ollama;
+
+  return {
+    available: false,
+    authority: 'scored_non_locator',
+    skipped_reason:
+      ollama.skipped_reason ??
+      (httpUrl
+        ? 'semantics backends unavailable'
+        : 'no IRIS_SEMANTICS_URL and no local Ollama vision model'),
+    route: ollama.route,
+    model: ollama.model,
+  };
+}