@mono-agent/agent-runtime 0.21.0 → 0.21.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mono-agent/agent-runtime",
-  "version": "0.21.0",
+  "version": "0.21.1",
   "description": "Agent runtime backed by the Pi SDK provider catalog",
   "type": "module",
   "license": "GPL-3.0-only",

package/src/agent/tools/pi-bridge.js

@@ -31,6 +31,7 @@ import {
 import { formatSkillBodyWithPathNote } from "../prompt/skill-index.js";
 import { MAX_TOOL_RESULT_BYTES, summarisePayload, wrapToolsWithBloatGuard } from "../tool-bloat.js";
 import { wrapToolsWithApprovalGate } from "../approval.js";
+import { normalizeImageForModel } from "./shared/image.js";
 import { isInsidePath } from "./shared/path-resolver.js";
 import { readToolRuntime } from "./shared/runtime-context.js";
 import { resolveSandboxPolicy } from "./shared/tool-context.js";
@@ -850,7 +851,27 @@ async function connectMcpClient(name, cfg, { cwd, sandboxPolicy, sandboxEngine,
   }
 }
 
-export function coerceMcpContent(out, {
+// MCP servers hand back screenshots at whatever size they captured — the
+// Playwright browser tools in particular follow browser_resize, so a wide desktop
+// capture arrives well past the provider ceiling while staying tiny in bytes and
+// sailing through the byte cap below. Normalize pixels before measuring bytes: a
+// shrunk screenshot may now fit the inline budget instead of being dropped for a
+// text pointer.
+async function normalizeMcpImage(data, mimeType) {
+  const raw = typeof data === "string" ? data : String(data ?? "");
+  const source = Buffer.from(raw, "base64");
+  if (source.length === 0) return { data: raw, mimeType };
+  const normalized = await normalizeImageForModel(source, mimeType);
+  // Shrinking is best-effort. An undecodable payload keeps the block exactly as
+  // the server sent it: losing the tool result would be worse than an oversized one.
+  if (normalized.reason !== undefined) return { data: raw, mimeType };
+  // Images already within the ceiling come back as the same buffer. Return the
+  // original base64 so the common path stays byte-identical and re-encodes nothing.
+  if (normalized.data === source) return { data: raw, mimeType };
+  return { data: normalized.data.toString("base64"), mimeType: normalized.mimeType };
+}
+
+export async function coerceMcpContent(out, {
   textLimit = MCP_TEXT_RESULT_LIMIT,
   imageInlineMaxBytes = MCP_IMAGE_INLINE_MAX_BYTES,
   persistArtifact = null,
@@ -859,15 +880,16 @@ export function coerceMcpContent(out, {
   onTruncate = null,
 } = {}) {
   if (Array.isArray(out?.content) && out.content.length) {
-    return out.content.map((part) => {
+    return await Promise.all(out.content.map(async (part) => {
       if (part.type === "text") return { type: "text", text: truncateMcpText(part.text || "", textLimit).text };
       if (part.type === "image") {
-        const bytes = base64Bytes(part.data);
+        const image = await normalizeMcpImage(part.data, part.mimeType || part.mime_type || "image/png");
+        const bytes = base64Bytes(image.data);
         if (bytes > imageInlineMaxBytes) {
           const summary = summarisePayload(toolName, [{
             type: "image",
-            data: part.data,
-            mimeType: part.mimeType || part.mime_type || "image/png",
+            data: image.data,
+            mimeType: image.mimeType,
           }], persistArtifact, { maxBytes: imageInlineMaxBytes, toolUseId });
           if (summary.truncated && typeof onTruncate === "function") {
             try {
@@ -887,21 +909,24 @@ export function coerceMcpContent(out, {
         }
         return {
           type: "image",
-          data: part.data,
-          mimeType: part.mimeType || part.mime_type || "image/png",
+          data: image.data,
+          mimeType: image.mimeType,
         };
       }
       return { type: "text", text: truncateMcpText(JSON.stringify(part), textLimit).text };
-    });
+    }));
   }
   return [{ type: "text", text: truncateMcpText(JSON.stringify(out || {}), textLimit).text }];
 }
 
-function mcpContentWasTruncated(out, { textLimit = MCP_TEXT_RESULT_LIMIT, imageInlineMaxBytes = MCP_IMAGE_INLINE_MAX_BYTES } = {}) {
+// Text-only. Images are measured after dimension normalization inside
+// coerceMcpContent, which reports a real truncation through onTruncate; judging
+// the raw part here would flag a screenshot that shrinking brought back under budget.
+function mcpContentWasTruncated(out, { textLimit = MCP_TEXT_RESULT_LIMIT } = {}) {
   if (Array.isArray(out?.content) && out.content.length) {
     return out.content.some((part) => {
       if (part.type === "text") return truncateMcpText(part.text || "", textLimit).truncated;
-      if (part.type === "image") return base64Bytes(part.data) > imageInlineMaxBytes;
+      if (part.type === "image") return false;
       return truncateMcpText(JSON.stringify(part), textLimit).truncated;
     });
   }
@@ -1116,7 +1141,7 @@ export async function initPiMcpTools(mcpConfig, reservedNames = new Set(), {
           }
           const imageTruncations = [];
           return {
-            content: coerceMcpContent(out, {
+            content: await coerceMcpContent(out, {
               textLimit,
               imageInlineMaxBytes,
               persistArtifact,
@@ -1136,7 +1161,7 @@ export async function initPiMcpTools(mcpConfig, reservedNames = new Set(), {
               // throwing away the bounded content or structuredContent below.
               ...(out?.isError === true ? { mcp_result_is_error: true } : {}),
               mcp_call_duration_ms: mcpCallDurationMs,
-              result_truncated: mcpContentWasTruncated(out, { textLimit, imageInlineMaxBytes }),
+              result_truncated: mcpContentWasTruncated(out, { textLimit }) || imageTruncations.length > 0,
               raw: compactRawMcpResult(out),
               ...(imageTruncations.length ? {
                 tool_payload_truncated: true,

package/src/agent/tools/read.js

@@ -1,13 +1,12 @@
 import { existsSync, readFileSync } from "node:fs";
 import { extname } from "node:path";
-import { decode as decodeBmp } from "bmp-ts";
-import sharp from "sharp";
 import {
   DEFAULT_MAX_READ_CHARS,
   DEFAULT_READ_LINES,
   MAX_READ_LINES,
 } from "./shared/constants.js";
 import { boundedInt, rememberRead, trimLine } from "./shared/dedup.js";
+import { normalizeImageForModel } from "./shared/image.js";
 import { capChars } from "./shared/output-truncation.js";
 import {
   isPathAllowed,
@@ -31,17 +30,6 @@ const IMAGE_MIME_BY_EXT = {
   ".bmp": "image/bmp",
 };
 
-// Anthropic rejects images with an edge longer than 8,000 px. Normalize Read
-// results to that shared provider-safe ceiling before the tool-result byte cap
-// runs, while leaving the source file untouched.
-const MAX_INLINE_IMAGE_EDGE_PX = 8_000;
-const ANIMATED_IMAGE_MIME_TYPES = new Set(["image/gif", "image/webp"]);
-const OUTPUT_MIME_BY_FORMAT = {
-  png: "image/png",
-  jpeg: "image/jpeg",
-  gif: "image/gif",
-  webp: "image/webp",
-};
 const PROTECTED_READ_MAX_BUFFER_BYTES = 64 * 1024 * 1024;
 const PROTECTED_READ_SOURCE = String.raw`
 "use strict";
@@ -49,69 +37,6 @@ const { readFileSync } = require("node:fs");
 process.stdout.write(readFileSync(process.argv[1]).toString("base64"));
 `;
 
-/**
- * @param {Buffer} source
- * @param {string} filePath
- * @param {string} imageMime
- */
-async function readImageForModel(source, filePath, imageMime) {
-  const inputOptions = { animated: ANIMATED_IMAGE_MIME_TYPES.has(imageMime) };
-
-  try {
-    let width;
-    let height;
-    let createPipeline;
-
-    if (imageMime === "image/bmp") {
-      // The prebuilt Sharp binaries do not include a BMP loader. Decode to raw
-      // RGBA first, then let Sharp handle the provider-safe resize and PNG output.
-      const decoded = decodeBmp(source, { toRGBA: true });
-      width = decoded.width;
-      height = Math.abs(decoded.height);
-      createPipeline = () => sharp(decoded.data, {
-        raw: { width, height, channels: 4 },
-      });
-    } else {
-      const metadata = await sharp(source, inputOptions).metadata();
-      width = metadata.width;
-      // Sharp exposes animated images as a vertical stack internally. Providers
-      // care about the dimensions of each frame, not the height of that stack.
-      height = metadata.pageHeight ?? metadata.height;
-      createPipeline = () => sharp(source, inputOptions);
-    }
-
-    if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) {
-      throw new Error("could not determine positive pixel dimensions");
-    }
-
-    if (width <= MAX_INLINE_IMAGE_EDGE_PX && height <= MAX_INLINE_IMAGE_EDGE_PX) {
-      return { data: source, mimeType: imageMime };
-    }
-
-    let pipeline = createPipeline()
-      .autoOrient()
-      .resize({
-        width: MAX_INLINE_IMAGE_EDGE_PX,
-        height: MAX_INLINE_IMAGE_EDGE_PX,
-        fit: "inside",
-        withoutEnlargement: true,
-      });
-
-    // Sharp cannot emit BMP, so resized BMP input becomes lossless PNG.
-    if (imageMime === "image/bmp") pipeline = pipeline.png();
-
-    const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
-    const mimeType = OUTPUT_MIME_BY_FORMAT[info.format];
-    if (mimeType === undefined) {
-      throw new Error(`unsupported normalized image format: ${info.format}`);
-    }
-    return { data, mimeType };
-  } catch (error) {
-    const reason = error instanceof Error ? error.message : String(error);
-    return { error: `Error: Unable to read image ${filePath}: ${reason}` };
-  }
-}
-
 /**
  * @param {{file_path: string, offset?: number, start_line?: number, limit?: number, max_output_chars?: number, workdir?: string}} params
  * @param {{sandboxPolicy?: any, sandboxEngine?: any, ctx?: any}} [options]
@@ -152,8 +77,8 @@ export async function readToolImpl({ file_path, offset = 0, start_line, limit, m
   // capped by the shared tool-result bloat guard.
   const imageMime = IMAGE_MIME_BY_EXT[extname(target).toLowerCase()];
   if (imageMime !== undefined) {
-    const image = await readImageForModel(source, file_path, imageMime);
-    if (image.error !== undefined) return image.error;
+    const image = await normalizeImageForModel(source, imageMime);
+    if (image.reason !== undefined) return `Error: Unable to read image ${file_path}: ${image.reason}`;
     return { kind: "image", data: image.data.toString("base64"), mimeType: image.mimeType };
   }
   const content = source.toString("utf8");

package/src/agent/tools/shared/image.js

@@ -0,0 +1,89 @@
+import { decode as decodeBmp } from "bmp-ts";
+import sharp from "sharp";
+
+// Anthropic allows an 8,000 px edge per image, but drops to 2,000 px per edge as
+// soon as a single request carries more than 20 image blocks — and images nested
+// in tool results, plus every image replayed from an earlier turn, count toward
+// that threshold. A screenshot-heavy conversation crosses 20 easily, and one
+// oversized image then rejects the whole request with an invalid_request_error
+// that no retry or model failover can clear. Normalize every inline image to the
+// stricter ceiling so the count never matters.
+//
+// 2,000 px sits above the standard tier's 1,568 px native long edge and just under
+// the 2,576 px high-resolution tier, so legibility is effectively unchanged: the
+// provider would downscale past this point for token accounting anyway.
+export const MAX_INLINE_IMAGE_EDGE_PX = 2_000;
+
+const ANIMATED_IMAGE_MIME_TYPES = new Set(["image/gif", "image/webp"]);
+const OUTPUT_MIME_BY_FORMAT = {
+  png: "image/png",
+  jpeg: "image/jpeg",
+  gif: "image/gif",
+  webp: "image/webp",
+};
+
+/**
+ * Cap an image's pixel dimensions to the provider-safe ceiling. Resolves to
+ * `{ data, mimeType }` on success — with `data` being the source buffer itself when
+ * it already fits, so callers can skip a pointless re-encode — or `{ reason }` when
+ * the bytes could not be decoded. Never mutates the source.
+ *
+ * @param {Buffer} source
+ * @param {string} imageMime
+ */
+export async function normalizeImageForModel(source, imageMime) {
+  const inputOptions = { animated: ANIMATED_IMAGE_MIME_TYPES.has(imageMime) };
+
+  try {
+    let width;
+    let height;
+    let createPipeline;
+
+    if (imageMime === "image/bmp") {
+      // The prebuilt Sharp binaries do not include a BMP loader. Decode to raw
+      // RGBA first, then let Sharp handle the provider-safe resize and PNG output.
+      const decoded = decodeBmp(source, { toRGBA: true });
+      width = decoded.width;
+      height = Math.abs(decoded.height);
+      createPipeline = () => sharp(decoded.data, {
+        raw: { width, height, channels: 4 },
+      });
+    } else {
+      const metadata = await sharp(source, inputOptions).metadata();
+      width = metadata.width;
+      // Sharp exposes animated images as a vertical stack internally. Providers
+      // care about the dimensions of each frame, not the height of that stack.
+      height = metadata.pageHeight ?? metadata.height;
+      createPipeline = () => sharp(source, inputOptions);
+    }
+
+    if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) {
+      throw new Error("could not determine positive pixel dimensions");
+    }
+
+    if (width <= MAX_INLINE_IMAGE_EDGE_PX && height <= MAX_INLINE_IMAGE_EDGE_PX) {
+      return { data: source, mimeType: imageMime };
+    }
+
+    let pipeline = createPipeline()
+      .autoOrient()
+      .resize({
+        width: MAX_INLINE_IMAGE_EDGE_PX,
+        height: MAX_INLINE_IMAGE_EDGE_PX,
+        fit: "inside",
+        withoutEnlargement: true,
+      });
+
+    // Sharp cannot emit BMP, so resized BMP input becomes lossless PNG.
+    if (imageMime === "image/bmp") pipeline = pipeline.png();
+
+    const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
+    const mimeType = OUTPUT_MIME_BY_FORMAT[info.format];
+    if (mimeType === undefined) {
+      throw new Error(`unsupported normalized image format: ${info.format}`);
+    }
+    return { data, mimeType };
+  } catch (error) {
+    return { reason: error instanceof Error ? error.message : String(error) };
+  }
+}

package/types/agent/tools/pi-bridge.d.ts

@@ -79,7 +79,7 @@ export function coerceMcpContent(out: any, { textLimit, imageInlineMaxBytes, per
     toolName?: string;
     toolUseId?: any;
     onTruncate?: any;
-}): any;
+}): Promise<any[]>;
 /**
  * @param {any} mcpConfig
  * @param {Set<any>} [reservedNames]

package/types/agent/tools/shared/image.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Cap an image's pixel dimensions to the provider-safe ceiling. Resolves to
+ * `{ data, mimeType }` on success — with `data` being the source buffer itself when
+ * it already fits, so callers can skip a pointless re-encode — or `{ reason }` when
+ * the bytes could not be decoded. Never mutates the source.
+ *
+ * @param {Buffer} source
+ * @param {string} imageMime
+ */
+export function normalizeImageForModel(source: Buffer, imageMime: string): Promise<{
+    data: Buffer<ArrayBufferLike>;
+    mimeType: string;
+    reason?: undefined;
+} | {
+    data: Buffer<ArrayBuffer>;
+    mimeType: any;
+    reason?: undefined;
+} | {
+    reason: string;
+    data?: undefined;
+    mimeType?: undefined;
+}>;
+export const MAX_INLINE_IMAGE_EDGE_PX: 2000;