@oh-my-pi/pi-coding-agent 13.7.6 → 13.8.0

package/src/tools/fetch.ts

@@ -1,5 +1,6 @@
 import * as path from "node:path";
 import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
+import type { ImageContent, TextContent } from "@oh-my-pi/pi-ai";
 import { htmlToMarkdown } from "@oh-my-pi/pi-natives";
 import { type Component, Text } from "@oh-my-pi/pi-tui";
 import { ptree, truncate } from "@oh-my-pi/pi-utils";
@@ -12,6 +13,7 @@ import fetchDescription from "../prompts/tools/fetch.md" with { type: "text" };
 import { DEFAULT_MAX_BYTES, truncateHead } from "../session/streaming-output";
 import { renderStatusLine } from "../tui";
 import { CachedOutputBlock } from "../tui/output-block";
+import { formatDimensionNote, resizeImage } from "../utils/image-resize";
 import { ensureTool } from "../utils/tools-manager";
 import { summarizeUrlWithKagi } from "../web/kagi";
 import { specialHandlers } from "../web/scrapers";
@@ -72,6 +74,10 @@ const CONVERTIBLE_EXTENSIONS = new Set([
 	".ogg",
 ]);
 
+const SUPPORTED_INLINE_IMAGE_MIME_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
+const MAX_INLINE_IMAGE_SOURCE_BYTES = 20 * 1024 * 1024;
+const MAX_INLINE_IMAGE_OUTPUT_BYTES = 0.75 * 1024 * 1024;
+
 // =============================================================================
 // Utilities
 // =============================================================================
@@ -145,6 +151,14 @@ function isConvertible(mime: string, extensionHint: string): boolean {
 	return false;
 }
 
+function resolveImageMimeType(mime: string): string | null {
+	return mime.startsWith("image/") ? mime : null;
+}
+
+function isInlineImageMimeTypeSupported(mimeType: string): boolean {
+	return SUPPORTED_INLINE_IMAGE_MIME_TYPES.has(mimeType);
+}
+
 /**
  * Check if content looks like HTML
  */
@@ -542,6 +556,15 @@ function formatJson(content: string): string {
 	}
 }
 
+interface FetchImagePayload {
+	data: string;
+	mimeType: string;
+}
+
+type FetchRenderResult = RenderResult & {
+	image?: FetchImagePayload;
+};
+
 // =============================================================================
 // Unified Special Handler Dispatch
 // =============================================================================
@@ -549,7 +572,11 @@ function formatJson(content: string): string {
 /**
  * Try all special handlers
  */
-async function handleSpecialUrls(url: string, timeout: number, signal?: AbortSignal): Promise<RenderResult | null> {
+async function handleSpecialUrls(
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<FetchRenderResult | null> {
 	for (const handler of specialHandlers) {
 		if (signal?.aborted) {
 			throw new ToolAbortError();
@@ -573,7 +600,7 @@ async function renderUrl(
 	raw: boolean,
 	useKagiSummarizer: boolean,
 	signal?: AbortSignal,
-): Promise<RenderResult> {
+): Promise<FetchRenderResult> {
 	const notes: string[] = [];
 	const fetchedAt = new Date().toISOString();
 	if (signal?.aborted) {
@@ -626,8 +653,124 @@ async function renderUrl(
 	const mime = normalizeMime(response.contentType);
 	const extHint = getExtensionHint(finalUrl);
 
+	const imageMimeType = resolveImageMimeType(mime);
+	const canInlineImage = Boolean(imageMimeType && isInlineImageMimeTypeSupported(imageMimeType));
+	let skipConvertibleBinaryRetry = false;
+	if (imageMimeType && !canInlineImage) {
+		notes.push(
+			`Image MIME type ${imageMimeType} is unsupported for inline model serialization; falling back to textual rendering`,
+		);
+	}
+	if (canInlineImage && imageMimeType) {
+		const binary = await fetchBinary(finalUrl, timeout, signal);
+		if (binary.ok) {
+			notes.push("Fetched image binary");
+			const conversionExtension = getExtensionHint(finalUrl, binary.contentDisposition) || extHint;
+			let convertedText: string | null = null;
+			const converted = await convertWithMarkitdown(binary.buffer, conversionExtension, timeout, signal);
+			if (converted.ok) {
+				if (converted.content.trim().length > 50) {
+					notes.push("Converted with markitdown");
+					convertedText = converted.content;
+				} else {
+					notes.push("markitdown conversion produced no usable output");
+				}
+			} else if (converted.error) {
+				notes.push(`markitdown conversion failed: ${converted.error}`);
+			} else {
+				notes.push("markitdown conversion failed");
+			}
+
+			if (binary.buffer.byteLength > MAX_INLINE_IMAGE_SOURCE_BYTES) {
+				notes.push(
+					`Image exceeds inline source limit (${binary.buffer.byteLength} bytes > ${MAX_INLINE_IMAGE_SOURCE_BYTES} bytes)`,
+				);
+				const output = finalizeOutput(
+					convertedText ?? `Fetched image content (${imageMimeType}), but it is too large to inline render.`,
+				);
+				return {
+					url,
+					finalUrl,
+					contentType: imageMimeType,
+					method: convertedText ? "markitdown" : "image-too-large",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+
+			const resized = await resizeImage(
+				{ type: "image", data: binary.buffer.toBase64(), mimeType: imageMimeType },
+				{ maxBytes: MAX_INLINE_IMAGE_OUTPUT_BYTES },
+			);
+			const isDecodedImage =
+				resized.originalWidth > 0 && resized.originalHeight > 0 && resized.width > 0 && resized.height > 0;
+			if (!isDecodedImage) {
+				notes.push(`Fetched payload could not be decoded as ${imageMimeType}; returning text metadata only`);
+				const output = finalizeOutput(
+					convertedText ??
+						rawContent ??
+						`Fetched payload was labeled ${imageMimeType}, but bytes were not a valid image.`,
+				);
+				return {
+					url,
+					finalUrl,
+					contentType: imageMimeType,
+					method: convertedText ? "markitdown" : "image-invalid",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+			if (resized.buffer.length > MAX_INLINE_IMAGE_OUTPUT_BYTES) {
+				notes.push(
+					`Image exceeds inline output limit after resize (${resized.buffer.length} bytes > ${MAX_INLINE_IMAGE_OUTPUT_BYTES} bytes)`,
+				);
+				const output = finalizeOutput(
+					convertedText ?? `Fetched image content (${imageMimeType}), but it is too large to inline render.`,
+				);
+				return {
+					url,
+					finalUrl,
+					contentType: imageMimeType,
+					method: convertedText ? "markitdown" : "image-too-large",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+
+			const dimensionNote = formatDimensionNote(resized);
+			let imageSummary = convertedText ?? `Fetched image content (${resized.mimeType}).`;
+			if (dimensionNote) {
+				imageSummary += `\n${dimensionNote}`;
+			}
+			const output = finalizeOutput(imageSummary);
+			return {
+				url,
+				finalUrl,
+				contentType: resized.mimeType,
+				method: "image",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes,
+				image: {
+					data: resized.data,
+					mimeType: resized.mimeType,
+				},
+			};
+		}
+		notes.push(binary.error ? `Binary fetch failed: ${binary.error}` : "Binary fetch failed");
+		notes.push("Falling back to textual rendering from initial response");
+		skipConvertibleBinaryRetry = true;
+	}
+
 	// Step 3: Handle convertible binary files (PDF, DOCX, etc.)
-	if (isConvertible(mime, extHint)) {
+	if (!skipConvertibleBinaryRetry && isConvertible(mime, extHint)) {
 		const binary = await fetchBinary(finalUrl, timeout, signal);
 		if (binary.ok) {
 			const ext = getExtensionHint(finalUrl, binary.contentDisposition) || extHint;
@@ -976,7 +1119,12 @@ export class FetchTool implements AgentTool<typeof fetchSchema, FetchToolDetails
 			notes: result.notes,
 		};
 
-		const resultBuilder = toolResult(details).text(output).sourceUrl(result.finalUrl);
+		const contentBlocks: Array<TextContent | ImageContent> = [{ type: "text", text: output }];
+		if (result.image) {
+			contentBlocks.push({ type: "image", data: result.image.data, mimeType: result.image.mimeType });
+		}
+
+		const resultBuilder = toolResult(details).content(contentBlocks).sourceUrl(result.finalUrl);
 		if (needsArtifact) {
 			resultBuilder.truncation(truncation, { direction: "head", artifactId });
 		} else if (result.truncated) {