codex-copilot-dx 0.4.42 → 0.4.44

package/src/image-optimization.mjs

@@ -1,15 +1,22 @@
 import { createHash } from "node:crypto";
+import {
+  canonicalInlineImageIdentity,
+  readResponsesImagePart,
+  readResponsesToolOutputParts,
+} from "./responses-content.mjs";
+import { createBoundedResultCache } from "./bounded-result-cache.mjs";
 import { status } from "./status.mjs";
 
 const DEFAULT_IMG_CONCURRENCY = 2;
 const IMG_MAX_CONCURRENCY = 12;
 const DEFAULT_IMG_MAX_INPUT_PIXELS = 40 * 1000 * 1000;
+const DEFAULT_IMG_CACHE_MAX_BYTES = 64 * 1024 * 1024;
 const DEFAULT_MAX_UPSTREAM_BODY_BYTES = 30 * 1024 * 1024;
-const MAX_OPTIMIZED_IMAGE_DIGESTS = 4096;
 const IMG_MAX_DIM = positiveInt(process.env.CCDX_IMG_MAX_DIM, 2048);
-const IMG_QUALITY = positiveInt(process.env.CCDX_IMG_QUALITY, 85, 100);
+const IMG_QUALITY = positiveInt(process.env.CCDX_IMG_QUALITY, 82, 100);
 const IMG_MIN_BYTES = nonNegativeInt(process.env.CCDX_IMG_MIN_BYTES, 100000);
 const IMG_MAX_INPUT_PIXELS = positiveInt(process.env.CCDX_IMG_MAX_INPUT_PIXELS, DEFAULT_IMG_MAX_INPUT_PIXELS);
+const IMG_CACHE_MAX_BYTES = nonNegativeInt(process.env.CCDX_IMG_CACHE_MAX_BYTES, DEFAULT_IMG_CACHE_MAX_BYTES);
 const MAX_UPSTREAM_BODY_BYTES = positiveInt(process.env.CCDX_MAX_UPSTREAM_BODY_BYTES, DEFAULT_MAX_UPSTREAM_BODY_BYTES);
 const IMG_OPT_DISABLED = process.env.CCDX_DISABLE_IMG_OPT === "1";
 const IMG_CONCURRENCY = parseImageConcurrency(process.env.CCDX_IMG_CONCURRENCY);
@@ -17,7 +24,8 @@ const OVERSIZE_IMAGE_PROFILES = [
   { maxDim: Math.min(IMG_MAX_DIM, 1600), quality: Math.min(IMG_QUALITY, 75) },
   { maxDim: Math.min(IMG_MAX_DIM, 1280), quality: Math.min(IMG_QUALITY, 65) },
 ];
-const optimizedImageDigests = new Set();
+const IMAGE_ENCODER_CONFIG = "webp:v1:effort=4:autorotate=1:fit=inside:without_enlargement=1";
+const imageResultCache = createBoundedResultCache({ maxBytes: IMG_CACHE_MAX_BYTES });
 let sharpImport = null;
 
 function positiveInt(value, fallback, max = Number.MAX_SAFE_INTEGER) {
@@ -30,6 +38,39 @@ function nonNegativeInt(value, fallback) {
   return Number.isFinite(n) && n >= 0 ? n : fallback;
 }
 
+export function imageConstraintsForModel(modelId, { maxDim = IMG_MAX_DIM } = {}) {
+  const model = String(modelId || "").trim().toLowerCase();
+  const maxLongEdge = Math.min(positiveInt(maxDim, IMG_MAX_DIM), IMG_MAX_DIM, 2048);
+  const fullSizeGpt4 = /^(?:gpt-4o|gpt-4\.1)(?:$|-)/.test(model)
+    && !/(?:^|-)(?:mini|nano)(?:$|-)/.test(model);
+  const gpt5Mini = /^gpt-5-mini(?:$|-)/.test(model);
+
+  if (fullSizeGpt4 || gpt5Mini) {
+    return { maxLongEdge, maxShortEdge: Math.min(maxLongEdge, 768) };
+  }
+  if (/^gpt-5\.(?:4|5|6)(?:$|-)/.test(model)) {
+    return { maxLongEdge, maxArea: 2_560_000 };
+  }
+  return { maxLongEdge };
+}
+
+export function fitImageDimensions(width, height, constraints = {}) {
+  const sourceWidth = Number(width);
+  const sourceHeight = Number(height);
+  if (!(sourceWidth > 0) || !(sourceHeight > 0)) return null;
+
+  const longEdge = Math.max(sourceWidth, sourceHeight);
+  const shortEdge = Math.min(sourceWidth, sourceHeight);
+  let scale = 1;
+  if (constraints.maxLongEdge > 0) scale = Math.min(scale, constraints.maxLongEdge / longEdge);
+  if (constraints.maxShortEdge > 0) scale = Math.min(scale, constraints.maxShortEdge / shortEdge);
+  if (constraints.maxArea > 0) scale = Math.min(scale, Math.sqrt(constraints.maxArea / (sourceWidth * sourceHeight)));
+  return {
+    width: Math.max(1, Math.floor(sourceWidth * scale)),
+    height: Math.max(1, Math.floor(sourceHeight * scale)),
+  };
+}
+
 export function parseImageConcurrency(value) {
   const n = Number.parseInt(value, 10);
   return Number.isFinite(n) && n > 0 ? Math.min(n, IMG_MAX_CONCURRENCY) : DEFAULT_IMG_CONCURRENCY;
@@ -102,42 +143,51 @@ export function createTaskLimiter(concurrency) {
 const runGlobalImageTask = createTaskLimiter(IMG_CONCURRENCY);
 
 export function imageOptimizationStats() {
+  const cache = imageResultCache.stats();
   return {
     ...runGlobalImageTask.stats(),
     disabled: IMG_OPT_DISABLED,
-    cache_entries: optimizedImageDigests.size,
+    cache_entries: cache.entries,
+    cache_bytes: cache.bytes,
+    cache_max_bytes: cache.max_bytes,
+    cache_hits: cache.hits,
+    cache_misses: cache.misses,
+    cache_evictions: cache.evictions,
+    cache_inflight: cache.inflight,
     sharp_loaded: sharpImport !== null,
   };
 }
 
+export function resetImageOptimizationCacheForTests() {
+  imageResultCache.clear();
+}
+
 async function sharp() {
   sharpImport ||= import("sharp");
   const mod = await sharpImport;
   return mod.default || mod;
 }
 
-function imageDigest(raw) {
-  return createHash("sha256").update(raw).digest("base64url");
+function imageDigest(value, encoding) {
+  return createHash("sha256").update(value, encoding).digest("base64url");
 }
 
-function touchOptimizedImage(digest) {
-  if (!optimizedImageDigests.delete(digest)) return false;
-  optimizedImageDigests.add(digest);
-  return true;
-}
-
-function rememberOptimizedImage(digest) {
-  optimizedImageDigests.delete(digest);
-  optimizedImageDigests.add(digest);
-  if (optimizedImageDigests.size > MAX_OPTIMIZED_IMAGE_DIGESTS) {
-    optimizedImageDigests.delete(optimizedImageDigests.values().next().value);
-  }
+function imageOptimizationKey(digest, mime, constraints, quality) {
+  return [
+    digest,
+    mime,
+    constraints.maxLongEdge || "",
+    constraints.maxShortEdge || "",
+    constraints.maxArea || "",
+    quality,
+    IMAGE_ENCODER_CONFIG,
+  ].join(":");
 }
 
 export async function optimizeImageDataUrl(dataUrl, {
   maxDim = IMG_MAX_DIM,
   quality = IMG_QUALITY,
-  force = false,
+  model,
   signal,
 } = {}) {
   if (IMG_OPT_DISABLED) return dataUrl;
@@ -147,34 +197,66 @@ export async function optimizeImageDataUrl(dataUrl, {
   const mime = match[1].toLowerCase();
   const encoded = match[2];
   const inputBytes = Buffer.byteLength(encoded, "base64");
-  if (inputBytes < IMG_MIN_BYTES || mime.includes("gif")) return dataUrl;
+  if (mime.includes("gif")) return dataUrl;
+  if (signal?.aborted) throw abortError(signal);
+  const constraints = imageConstraintsForModel(model, { maxDim });
+  const outputQuality = positiveInt(quality, IMG_QUALITY, 100);
+  // Hash decoded source bytes without retaining a decoded Buffer. Cache hits
+  // therefore bypass both Sharp and the global image work queue.
+  const digest = imageDigest(encoded, "base64");
+  const optimizationKey = imageOptimizationKey(digest, mime, constraints, outputQuality);
 
   try {
-    return await runGlobalImageTask(async () => {
-      const raw = Buffer.from(encoded, "base64");
-      if (raw.length < IMG_MIN_BYTES) return dataUrl;
-      const digest = imageDigest(raw);
-      if (!force && touchOptimizedImage(digest)) return dataUrl;
-      const resize = await sharp();
-      const image = resize(raw, { failOn: "none", limitInputPixels: IMG_MAX_INPUT_PIXELS });
-      const out = await image
-        .rotate()
-        .resize(positiveInt(maxDim, IMG_MAX_DIM), positiveInt(maxDim, IMG_MAX_DIM), { fit: "inside", withoutEnlargement: true })
-        .webp({ quality: positiveInt(quality, IMG_QUALITY, 100), effort: 4 })
-        .toBuffer();
-      if (signal?.aborted) throw abortError(signal);
-      if (out.length >= raw.length) {
-        rememberOptimizedImage(digest);
-        return dataUrl;
-      }
-      rememberOptimizedImage(imageDigest(out));
-      const ratio = ((out.length / raw.length) * 100).toFixed(1);
-      console.log(status("info", `image ${(raw.length / 1024).toFixed(0)}KB ${mime} -> ${(out.length / 1024).toFixed(0)}KB webp (${ratio}%)`));
-      return `data:image/webp;base64,${out.toString("base64")}`;
-    }, { signal });
+    return await imageResultCache.getOrCreate(optimizationKey, (sharedSignal) => (
+      runGlobalImageTask(async () => {
+        const raw = Buffer.from(encoded, "base64");
+        const resize = await sharp();
+        const image = resize(raw, { failOn: "none", limitInputPixels: IMG_MAX_INPUT_PIXELS });
+        const metadata = await image.metadata();
+        if (sharedSignal.aborted) throw abortError(sharedSignal);
+        const orientation = Number(metadata.orientation);
+        const rotated = orientation >= 5 && orientation <= 8;
+        const sourceWidth = rotated ? metadata.height : metadata.width;
+        const sourceHeight = rotated ? metadata.width : metadata.height;
+        const target = fitImageDimensions(sourceWidth, sourceHeight, constraints);
+        const resizeRequired = target
+          && (target.width < sourceWidth || target.height < sourceHeight);
+        if (raw.length < IMG_MIN_BYTES && !resizeRequired) return dataUrl;
+        const resizeOptions = target
+          ? { ...target, fit: "inside", withoutEnlargement: true }
+          : {
+              width: constraints.maxLongEdge,
+              height: constraints.maxLongEdge,
+              fit: "inside",
+              withoutEnlargement: true,
+            };
+        const out = await image
+          .rotate()
+          .resize(resizeOptions)
+          .webp({ quality: outputQuality, effort: 4 })
+          .toBuffer();
+        if (sharedSignal.aborted) throw abortError(sharedSignal);
+        if (out.length >= raw.length) return dataUrl;
+        const outputBase64 = out.toString("base64");
+        const outputDataUrl = `data:image/webp;base64,${outputBase64}`;
+        const outputKey = imageOptimizationKey(
+          imageDigest(outputBase64, "base64"),
+          "image/webp",
+          constraints,
+          outputQuality,
+        );
+        // Preserve idempotence when optimized history is submitted again.
+        imageResultCache.set(outputKey, outputDataUrl);
+        const ratio = ((out.length / raw.length) * 100).toFixed(1);
+        console.log(status("info", `image ${(raw.length / 1024).toFixed(0)}KB ${mime} -> ${(out.length / 1024).toFixed(0)}KB webp (${ratio}%)`));
+        return outputDataUrl;
+      }, { signal: sharedSignal })
+    ), { signal });
   } catch (e) {
     if (signal?.aborted || e?.name === "AbortError") throw e;
-    console.warn(status("warn", `image optimize failed (${mime}, ${inputBytes}b): ${e.message}`));
+    if (inputBytes >= IMG_MIN_BYTES) {
+      console.warn(status("warn", `image optimize failed (${mime}, ${inputBytes}b): ${e.message}`));
+    }
     return dataUrl;
   }
 }
@@ -191,102 +273,225 @@ export async function runWithConcurrency(taskFns, concurrency) {
   }));
 }
 
-function visitImageParts(parts, tasks, optimizeImage) {
+function visitImageParts(parts, references, commit = null) {
   if (!Array.isArray(parts)) return;
   for (const part of parts) {
-    if (!part) continue;
-    if (part.type === "input_image" && typeof part.image_url === "string") {
-      tasks.push(async () => { part.image_url = await optimizeImage(part.image_url); });
-    } else if (part.type === "image" && part.source?.type === "base64" && part.source?.data) {
-      tasks.push(async () => {
-        const dataUrl = `data:${part.source.media_type || "image/png"};base64,${part.source.data}`;
-        const optimized = await optimizeImage(dataUrl);
-        const match = /^data:([^;]+);base64,(.+)$/.exec(optimized);
-        if (match) {
-          part.source.media_type = match[1];
-          part.source.data = match[2];
-        }
-      });
+    const image = readResponsesImagePart(part);
+    if (!image?.identity) continue;
+    references.push({
+      commit,
+      get: () => image.dataUrl,
+      set: (value) => image.setDataUrl(value),
+    });
+  }
+}
+
+function collectImageReferences(reqBody) {
+  const references = [];
+  const dirtyCommits = new Set();
+  if (!Array.isArray(reqBody?.input)) {
+    return { references, markDirty() {}, commitDirty() {} };
+  }
+
+  for (const item of reqBody.input) {
+    if (!item) continue;
+    visitImageParts([item], references);
+    if (item.type === "message" && Array.isArray(item.content)) {
+      visitImageParts(item.content, references);
+    }
+    const toolOutput = readResponsesToolOutputParts(item);
+    if (toolOutput) {
+      visitImageParts(toolOutput.parts, references, toolOutput.commit);
     }
   }
+
+  return {
+    references,
+    markDirty(changedReferences) {
+      for (const reference of changedReferences) {
+        if (reference.commit) dirtyCommits.add(reference.commit);
+      }
+    },
+    commitDirty() {
+      for (const apply of dirtyCommits) apply();
+      dirtyCommits.clear();
+    },
+  };
 }
 
-export async function optimizeImagesInBody(reqBody, {
-  concurrency = IMG_CONCURRENCY,
-  optimizeImage = optimizeImageDataUrl,
+async function optimizeImageReferences(collection, {
+  concurrency,
+  model,
+  optimizeImage,
   signal,
-} = {}) {
-  if (IMG_OPT_DISABLED || !Array.isArray(reqBody.input)) return reqBody;
-  const tasks = [];
-  const finalize = [];
+}) {
   const optimizedByDataUrl = new Map();
   const optimizeOnce = (dataUrl) => {
-    if (!optimizedByDataUrl.has(dataUrl)) {
-      optimizedByDataUrl.set(dataUrl, Promise.resolve().then(() => optimizeImage(dataUrl, { signal })));
+    const identity = canonicalInlineImageIdentity(dataUrl) || dataUrl;
+    if (!optimizedByDataUrl.has(identity)) {
+      optimizedByDataUrl.set(identity, Promise.resolve().then(() => optimizeImage(dataUrl, { model, signal })));
     }
-    return optimizedByDataUrl.get(dataUrl);
+    return optimizedByDataUrl.get(identity);
   };
+  const tasks = collection.references.map((reference) => async () => {
+    if (reference.set(await optimizeOnce(reference.get()))) collection.markDirty([reference]);
+  });
+  if (tasks.length) await runWithConcurrency(tasks, concurrency);
+  collection.commitDirty();
+}
 
-  for (const item of reqBody.input) {
-    if (!item) continue;
-    if (item.type === "message" && Array.isArray(item.content)) {
-      visitImageParts(item.content, tasks, optimizeOnce);
-    }
-    if (item.type === "function_call_output") {
-      if (Array.isArray(item.output)) {
-        visitImageParts(item.output, tasks, optimizeOnce);
-      } else if (typeof item.output === "string") {
-        const trimmed = item.output.trim();
-        if (trimmed.startsWith("[")) {
-          try {
-            const parsed = JSON.parse(trimmed);
-            if (Array.isArray(parsed)) {
-              const taskCount = tasks.length;
-              visitImageParts(parsed, tasks, optimizeOnce);
-              if (tasks.length > taskCount) finalize.push(() => { item.output = JSON.stringify(parsed); });
-            }
-          } catch {
-            // Leave non-JSON tool output untouched.
-          }
-        }
-      }
+function imageInputBytes(dataUrl) {
+  if (typeof dataUrl !== "string") return 0;
+  const match = /^data:image\/[a-z+.-]+;base64,(.+)$/i.exec(dataUrl);
+  return match ? Buffer.byteLength(match[1], "base64") : 0;
+}
+
+function uniqueOriginalImages(references) {
+  const unique = new Map();
+  for (const reference of references) {
+    const dataUrl = reference.get();
+    const identity = canonicalInlineImageIdentity(dataUrl);
+    if (!identity) continue;
+    const inputBytes = imageInputBytes(dataUrl);
+    if (inputBytes <= 0) continue;
+    let image = unique.get(identity);
+    if (!image) {
+      image = { dataUrl, inputBytes, references: [] };
+      unique.set(identity, image);
     }
+    image.references.push(reference);
+  }
+  return [...unique.values()];
+}
+
+function currentImageWireBytes(image) {
+  return image.references.reduce(
+    (total, reference) => total + Buffer.byteLength(reference.get()),
+    0,
+  );
+}
+
+function orderImagesByCurrentWireBytes(images) {
+  return [...images].sort((left, right) => (
+    currentImageWireBytes(right) - currentImageWireBytes(left)
+      || right.inputBytes - left.inputBytes
+  ));
+}
+
+function applyImageCandidate(collection, image, candidate, bodyBytes) {
+  if (!canonicalInlineImageIdentity(candidate)) {
+    return { bodyBytes, changed: false };
+  }
+  const previous = image.references.map((reference) => reference.get());
+  if (previous.every((value) => value === candidate)) return { bodyBytes, changed: false };
+  // Base64 data URLs need no JSON escaping, so their byte delta also applies to the serialized body.
+  const candidateBytes = Buffer.byteLength(candidate);
+  const byteDelta = previous.reduce(
+    (delta, value) => delta + candidateBytes - Buffer.byteLength(value),
+    0,
+  );
+  if (byteDelta >= 0) return { bodyBytes, changed: false };
+  const changedReferences = [];
+  for (const reference of image.references) {
+    if (reference.set(candidate)) changedReferences.push(reference);
   }
+  collection.markDirty(changedReferences);
+  return { bodyBytes: bodyBytes + byteDelta, changed: true };
+}
 
-  if (tasks.length) await runWithConcurrency(tasks, concurrency);
-  for (const apply of finalize) apply();
+function normalizeCurrentInputStart(value, reqBody) {
+  const parsed = Number.parseInt(value, 10);
+  const itemCount = Array.isArray(reqBody?.input) ? reqBody.input.length : 0;
+  return Number.isFinite(parsed) && parsed >= 0 ? Math.min(parsed, itemCount) : 0;
+}
+
+export async function optimizeImagesInBody(reqBody, {
+  concurrency = IMG_CONCURRENCY,
+  optimizeImage = optimizeImageDataUrl,
+  signal,
+} = {}) {
+  if (IMG_OPT_DISABLED || !Array.isArray(reqBody.input)) return reqBody;
+  const collection = collectImageReferences(reqBody);
+  await optimizeImageReferences(collection, {
+    concurrency,
+    model: reqBody.model,
+    optimizeImage,
+    signal,
+  });
   return reqBody;
 }
 
 export async function prepareResponsesPayload(reqBody, {
+  currentInputStart = 0,
   maxBytes = MAX_UPSTREAM_BODY_BYTES,
+  optimizeImage = optimizeImageDataUrl,
   profiles = OVERSIZE_IMAGE_PROFILES,
+  skipInitialOptimization = false,
   signal,
 } = {}) {
-  await optimizeImagesInBody(reqBody, { signal });
+  const shouldOptimizeImages = !IMG_OPT_DISABLED && !skipInitialOptimization;
+  const collection = shouldOptimizeImages ? collectImageReferences(reqBody) : null;
+  const originalImages = collection ? uniqueOriginalImages(collection.references) : [];
+  if (collection) {
+    await optimizeImageReferences(collection, {
+      concurrency: IMG_CONCURRENCY,
+      model: reqBody.model,
+      optimizeImage: (dataUrl, options) => optimizeImage(dataUrl, { ...options, quality: IMG_QUALITY }),
+      signal,
+    });
+  }
   let bodyText = JSON.stringify(reqBody);
   let bodyBytes = Buffer.byteLength(bodyText);
   const summary = summarizeReqBody(reqBody);
   const targetBytes = positiveInt(maxBytes, MAX_UPSTREAM_BODY_BYTES);
   let adapted = false;
+  let stage = skipInitialOptimization ? "preoptimized" : `q${IMG_QUALITY}`;
 
-  if (!IMG_OPT_DISABLED && summary.images > 0 && bodyBytes > targetBytes) {
-    for (const profile of profiles) {
+  if (collection && originalImages.length > 0 && bodyBytes > targetBytes) {
+    const batchSize = Math.max(1, IMG_CONCURRENCY);
+    for (const profile of Array.isArray(profiles) ? profiles : []) {
       const beforeBytes = bodyBytes;
-      bodyText = "";
-      await optimizeImagesInBody(reqBody, {
-        optimizeImage: (dataUrl) => optimizeImageDataUrl(dataUrl, { ...profile, force: true, signal }),
-        signal,
-      });
+      let processed = 0;
+      stage = `q${positiveInt(profile.quality, IMG_QUALITY, 100)}`;
+      const orderedImages = orderImagesByCurrentWireBytes(originalImages);
+      for (let index = 0; index < orderedImages.length && bodyBytes > targetBytes; index += batchSize) {
+        const batch = orderedImages.slice(index, index + batchSize);
+        const candidates = await Promise.all(batch.map((image) => optimizeImage(image.dataUrl, {
+          ...profile,
+          force: true,
+          model: reqBody.model,
+          signal,
+        })));
+        for (let offset = 0; offset < batch.length && bodyBytes > targetBytes; offset += 1) {
+          processed += 1;
+          const applied = applyImageCandidate(collection, batch[offset], candidates[offset], bodyBytes);
+          bodyBytes = applied.bodyBytes;
+          adapted ||= applied.changed;
+          if (bodyBytes <= targetBytes) {
+            collection.commitDirty();
+            bodyText = JSON.stringify(reqBody);
+            bodyBytes = Buffer.byteLength(bodyText);
+          }
+        }
+      }
+      collection.commitDirty();
       bodyText = JSON.stringify(reqBody);
       bodyBytes = Buffer.byteLength(bodyText);
-      adapted ||= bodyBytes < beforeBytes;
-      console.warn(status("warn", `responses payload ${beforeBytes}b exceeds ${targetBytes}b; image profile max_dim=${profile.maxDim} quality=${profile.quality} -> ${bodyBytes}b`));
+      console.warn(status("warn", `responses payload ${beforeBytes}b exceeds ${targetBytes}b; image profile max_dim=${profile.maxDim} quality=${profile.quality} processed=${processed}/${orderedImages.length} -> ${bodyBytes}b`));
       if (bodyBytes <= targetBytes) break;
     }
   }
 
-  return { bodyText, bodyBytes, summary, adapted };
+  return {
+    bodyText,
+    bodyBytes,
+    summary,
+    adapted,
+    stage,
+    targetBytes,
+    overBudget: bodyBytes > targetBytes,
+    currentInputStart: normalizeCurrentInputStart(currentInputStart, reqBody),
+  };
 }
 
 export function summarizeReqBody(reqBody) {
@@ -297,24 +502,15 @@ export function summarizeReqBody(reqBody) {
     const countImages = (parts) => {
       if (!Array.isArray(parts)) return;
       for (const part of parts) {
-        if (part?.type === "input_image" || part?.type === "image") images += 1;
+        if (["input_image", "image", "image_url"].includes(part?.type)) images += 1;
       }
     };
 
     for (const item of input) {
+      if (["input_image", "image", "image_url"].includes(item?.type)) countImages([item]);
       if (item?.type === "message") countImages(item.content);
-      if (item?.type === "function_call_output") {
-        if (Array.isArray(item.output)) {
-          countImages(item.output);
-        } else if (typeof item.output === "string" && item.output.trim().startsWith("[")) {
-          try {
-            const parsed = JSON.parse(item.output);
-            if (Array.isArray(parsed)) countImages(parsed);
-          } catch {
-            // Ignore non-JSON tool output.
-          }
-        }
-      }
+      const toolOutput = readResponsesToolOutputParts(item);
+      if (toolOutput) countImages(toolOutput.parts);
     }
 
     return { items: input.length, images };

package/src/observability.mjs

@@ -118,7 +118,7 @@ function modelRegistryStatus(modelRegistry) {
   };
 }
 
-export function runtimeStatusPayload({ metrics, admission, modelRegistry } = {}) {
+export function runtimeStatusPayload({ metrics, streamPerformance, admission, modelRegistry } = {}) {
   const memory = process.memoryUsage();
   return {
     ...adapterHealthPayload(),
@@ -131,6 +131,7 @@ export function runtimeStatusPayload({ metrics, admission, modelRegistry } = {})
       array_buffers_bytes: memory.arrayBuffers,
     },
     requests: metrics?.snapshot?.() || createRequestMetrics().snapshot(),
+    stream_performance: streamPerformance?.snapshot?.() || null,
     admission: admission?.diagnostics?.() || admission?.stats?.() || null,
     response_history: responseHistoryStats(),
     image_optimization: imageOptimizationStats(),