@oh-my-pi/pi-coding-agent 13.9.12 → 13.9.13

package/src/tools/read.ts

@@ -23,7 +23,12 @@ import {
 import { renderCodeCell, renderStatusLine } from "../tui";
 import { CachedOutputBlock } from "../tui/output-block";
 import { resolveFileDisplayMode } from "../utils/file-display-mode";
-import { formatDimensionNote, resizeImage } from "../utils/image-resize";
+import {
+	ImageInputTooLargeError,
+	loadImageInput,
+	MAX_IMAGE_INPUT_BYTES,
+	readImageMetadata,
+} from "../utils/image-input";
 import { detectSupportedImageMimeTypeFromFile } from "../utils/mime";
 import { ensureTool } from "../utils/tools-manager";
 import { applyListLimit } from "./list-limit";
@@ -253,7 +258,7 @@ async function streamLinesFromFile(
 }
 
 // Maximum image file size (20MB) - larger images will be rejected to prevent OOM during serialization
-const MAX_IMAGE_SIZE = 20 * 1024 * 1024;
+const MAX_IMAGE_SIZE = MAX_IMAGE_INPUT_BYTES;
 const GLOB_TIMEOUT_MS = 5000;
 
 function isNotFoundError(error: unknown): boolean {
@@ -366,6 +371,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
 
 	readonly #autoResizeImages: boolean;
 	readonly #defaultLimit: number;
+	readonly #inspectImageEnabled: boolean;
 
 	constructor(private readonly session: ToolSession) {
 		const displayMode = resolveFileDisplayMode(session);
@@ -374,6 +380,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
 			1,
 			Math.min(session.settings.get("read.defaultLimit") ?? DEFAULT_MAX_LINES, DEFAULT_MAX_LINES),
 		);
+		this.#inspectImageEnabled = session.settings.get("inspect_image.enabled");
 		this.description = renderPromptTemplate(readDescription, {
 			DEFAULT_LIMIT: String(this.#defaultLimit),
 			DEFAULT_MAX_LINES: String(DEFAULT_MAX_LINES),
@@ -455,57 +462,63 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
 			| undefined;
 
 		if (mimeType) {
-			if (fileSize > MAX_IMAGE_SIZE) {
-				const sizeStr = formatBytes(fileSize);
-				const maxStr = formatBytes(MAX_IMAGE_SIZE);
-				throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`);
+			if (this.#inspectImageEnabled) {
+				const metadata = await readImageMetadata({
+					path: readPath,
+					cwd: this.session.cwd,
+					resolvedPath: absolutePath,
+					detectedMimeType: mimeType,
+				});
+				const outputMime = metadata?.mimeType ?? mimeType;
+				const outputBytes = metadata?.bytes ?? fileSize;
+				const metadataLines = [
+					"Image metadata:",
+					`- MIME: ${outputMime}`,
+					`- Bytes: ${outputBytes} (${formatBytes(outputBytes)})`,
+					metadata?.width !== undefined && metadata.height !== undefined
+						? `- Dimensions: ${metadata.width}x${metadata.height}`
+						: "- Dimensions: unknown",
+					metadata?.channels !== undefined ? `- Channels: ${metadata.channels}` : "- Channels: unknown",
+					metadata?.hasAlpha === true
+						? "- Alpha: yes"
+						: metadata?.hasAlpha === false
+							? "- Alpha: no"
+							: "- Alpha: unknown",
+					"",
+					`If you want to analyze the image, call inspect_image with path="${readPath}" and a question describing what to inspect and the desired output format.`,
+				];
+				content = [{ type: "text", text: metadataLines.join("\n") }];
+				details = {};
+				sourcePath = absolutePath;
 			} else {
-				// Read as image (binary)
-				const file = Bun.file(absolutePath);
-				const buffer = await file.arrayBuffer();
-
-				// Check actual buffer size after reading to prevent OOM during serialization
-				if (buffer.byteLength > MAX_IMAGE_SIZE) {
-					const sizeStr = formatBytes(buffer.byteLength);
+				if (fileSize > MAX_IMAGE_SIZE) {
+					const sizeStr = formatBytes(fileSize);
 					const maxStr = formatBytes(MAX_IMAGE_SIZE);
 					throw new ToolError(`Image file too large: ${sizeStr} exceeds ${maxStr} limit.`);
-				} else {
-					const base64 = new Uint8Array(buffer).toBase64();
-
-					if (this.#autoResizeImages) {
-						// Resize image if needed - catch errors from Photon
-						try {
-							const resized = await resizeImage({ type: "image", data: base64, mimeType });
-							const dimensionNote = formatDimensionNote(resized);
-
-							let textNote = `Read image file [${resized.mimeType}]`;
-							if (dimensionNote) {
-								textNote += `\n${dimensionNote}`;
-							}
-
-							content = [
-								{ type: "text", text: textNote },
-								{ type: "image", data: resized.data, mimeType: resized.mimeType },
-							];
-							details = {};
-							sourcePath = absolutePath;
-						} catch {
-							// Fall back to original image on resize failure
-							content = [
-								{ type: "text", text: `Read image file [${mimeType}]` },
-								{ type: "image", data: base64, mimeType },
-							];
-							details = {};
-							sourcePath = absolutePath;
-						}
-					} else {
-						content = [
-							{ type: "text", text: `Read image file [${mimeType}]` },
-							{ type: "image", data: base64, mimeType },
-						];
-						details = {};
-						sourcePath = absolutePath;
+				}
+				try {
+					const imageInput = await loadImageInput({
+						path: readPath,
+						cwd: this.session.cwd,
+						autoResize: this.#autoResizeImages,
+						maxBytes: MAX_IMAGE_SIZE,
+						resolvedPath: absolutePath,
+						detectedMimeType: mimeType,
+					});
+					if (!imageInput) {
+						throw new ToolError(`Read image file [${mimeType}] failed: unsupported image format.`);
+					}
+					content = [
+						{ type: "text", text: imageInput.textNote },
+						{ type: "image", data: imageInput.data, mimeType: imageInput.mimeType },
+					];
+					details = {};
+					sourcePath = imageInput.resolvedPath;
+				} catch (error) {
+					if (error instanceof ImageInputTooLargeError) {
+						throw new ToolError(error.message);
 					}
+					throw error;
 				}
 			}
 		} else if (CONVERTIBLE_EXTENSIONS.has(ext)) {

package/src/tools/renderers.ts

@@ -18,6 +18,7 @@ import { calculatorToolRenderer } from "./calculator";
 import { fetchToolRenderer } from "./fetch";
 import { findToolRenderer } from "./find";
 import { grepToolRenderer } from "./grep";
+import { inspectImageToolRenderer } from "./inspect-image-renderer";
 import { notebookToolRenderer } from "./notebook";
 import { pythonToolRenderer } from "./python";
 import { readToolRenderer } from "./read";
@@ -51,6 +52,7 @@ export const toolRenderers: Record<string, ToolRenderer> = {
 	grep: grepToolRenderer as ToolRenderer,
 	lsp: lspToolRenderer as ToolRenderer,
 	notebook: notebookToolRenderer as ToolRenderer,
+	inspect_image: inspectImageToolRenderer as ToolRenderer,
 	read: readToolRenderer as ToolRenderer,
 	resolve: resolveToolRenderer as ToolRenderer,
 	ssh: sshToolRenderer as ToolRenderer,

package/src/utils/image-input.ts

@@ -0,0 +1,264 @@
+import * as fs from "node:fs/promises";
+import { formatBytes } from "@oh-my-pi/pi-utils";
+import { resolveReadPath } from "../tools/path-utils";
+import { formatDimensionNote, resizeImage } from "./image-resize";
+import { detectSupportedImageMimeTypeFromFile } from "./mime";
+
+export const MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024;
+const MAX_IMAGE_METADATA_HEADER_BYTES = 256 * 1024;
+
+export interface ImageMetadata {
+	mimeType: string;
+	bytes: number;
+	width?: number;
+	height?: number;
+	channels?: number;
+	hasAlpha?: boolean;
+}
+
+export interface LoadedImageInput {
+	resolvedPath: string;
+	mimeType: string;
+	data: string;
+	textNote: string;
+	dimensionNote?: string;
+	bytes: number;
+}
+
+export interface ReadImageMetadataOptions {
+	path: string;
+	cwd: string;
+	resolvedPath?: string;
+	detectedMimeType?: string;
+}
+
+export interface LoadImageInputOptions extends ReadImageMetadataOptions {
+	autoResize: boolean;
+	maxBytes?: number;
+}
+
+export class ImageInputTooLargeError extends Error {
+	readonly bytes: number;
+	readonly maxBytes: number;
+
+	constructor(bytes: number, maxBytes: number) {
+		super(`Image file too large: ${formatBytes(bytes)} exceeds ${formatBytes(maxBytes)} limit.`);
+		this.name = "ImageInputTooLargeError";
+		this.bytes = bytes;
+		this.maxBytes = maxBytes;
+	}
+}
+
+interface ParsedImageHeaderMetadata {
+	width?: number;
+	height?: number;
+	channels?: number;
+	hasAlpha?: boolean;
+}
+
+function parsePngMetadata(header: Buffer): ParsedImageHeaderMetadata {
+	if (header.length < 26) return {};
+	if (
+		header[0] !== 0x89 ||
+		header[1] !== 0x50 ||
+		header[2] !== 0x4e ||
+		header[3] !== 0x47 ||
+		header[4] !== 0x0d ||
+		header[5] !== 0x0a ||
+		header[6] !== 0x1a ||
+		header[7] !== 0x0a
+	) {
+		return {};
+	}
+	if (header.slice(12, 16).toString("ascii") !== "IHDR") return {};
+	const width = header.readUInt32BE(16);
+	const height = header.readUInt32BE(20);
+	const colorType = header[25];
+	if (colorType === 0) return { width, height, channels: 1, hasAlpha: false };
+	if (colorType === 2) return { width, height, channels: 3, hasAlpha: false };
+	if (colorType === 3) return { width, height, channels: 3 };
+	if (colorType === 4) return { width, height, channels: 2, hasAlpha: true };
+	if (colorType === 6) return { width, height, channels: 4, hasAlpha: true };
+	return { width, height };
+}
+
+function parseJpegMetadata(header: Buffer): ParsedImageHeaderMetadata {
+	if (header.length < 4) return {};
+	if (header[0] !== 0xff || header[1] !== 0xd8) return {};
+
+	let offset = 2;
+	while (offset + 9 < header.length) {
+		if (header[offset] !== 0xff) {
+			offset += 1;
+			continue;
+		}
+
+		let markerOffset = offset + 1;
+		while (markerOffset < header.length && header[markerOffset] === 0xff) {
+			markerOffset += 1;
+		}
+		if (markerOffset >= header.length) break;
+
+		const marker = header[markerOffset];
+		const segmentOffset = markerOffset + 1;
+
+		if (marker === 0xd8 || marker === 0xd9 || marker === 0x01) {
+			offset = segmentOffset;
+			continue;
+		}
+		if (marker >= 0xd0 && marker <= 0xd7) {
+			offset = segmentOffset;
+			continue;
+		}
+		if (segmentOffset + 1 >= header.length) break;
+
+		const segmentLength = header.readUInt16BE(segmentOffset);
+		if (segmentLength < 2) break;
+
+		const isStartOfFrame = marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc;
+		if (isStartOfFrame) {
+			if (segmentOffset + 7 >= header.length) break;
+			const height = header.readUInt16BE(segmentOffset + 3);
+			const width = header.readUInt16BE(segmentOffset + 5);
+			const channels = header[segmentOffset + 7];
+			return {
+				width,
+				height,
+				channels: Number.isFinite(channels) ? channels : undefined,
+				hasAlpha: false,
+			};
+		}
+
+		offset = segmentOffset + segmentLength;
+	}
+
+	return {};
+}
+
+function parseGifMetadata(header: Buffer): ParsedImageHeaderMetadata {
+	if (header.length < 10) return {};
+	const signature = header.slice(0, 6).toString("ascii");
+	if (signature !== "GIF87a" && signature !== "GIF89a") return {};
+	return {
+		width: header.readUInt16LE(6),
+		height: header.readUInt16LE(8),
+		channels: 3,
+	};
+}
+
+function parseWebpMetadata(header: Buffer): ParsedImageHeaderMetadata {
+	if (header.length < 30) return {};
+	if (header.slice(0, 4).toString("ascii") !== "RIFF") return {};
+	if (header.slice(8, 12).toString("ascii") !== "WEBP") return {};
+
+	const chunkType = header.slice(12, 16).toString("ascii");
+	if (chunkType === "VP8X") {
+		const hasAlpha = (header[20] & 0x10) !== 0;
+		const width = (header[24] | (header[25] << 8) | (header[26] << 16)) + 1;
+		const height = (header[27] | (header[28] << 8) | (header[29] << 16)) + 1;
+		return { width, height, channels: hasAlpha ? 4 : 3, hasAlpha };
+	}
+	if (chunkType === "VP8L") {
+		if (header.length < 25) return {};
+		const bits = header.readUInt32LE(21);
+		const width = (bits & 0x3fff) + 1;
+		const height = ((bits >> 14) & 0x3fff) + 1;
+		const hasAlpha = ((bits >> 28) & 0x1) === 1;
+		return { width, height, channels: hasAlpha ? 4 : 3, hasAlpha };
+	}
+	if (chunkType === "VP8 ") {
+		const width = header.readUInt16LE(26) & 0x3fff;
+		const height = header.readUInt16LE(28) & 0x3fff;
+		return { width, height, channels: 3, hasAlpha: false };
+	}
+	return {};
+}
+
+function parseImageHeaderMetadata(header: Buffer, mimeType: string): ParsedImageHeaderMetadata {
+	if (mimeType === "image/png") return parsePngMetadata(header);
+	if (mimeType === "image/jpeg") return parseJpegMetadata(header);
+	if (mimeType === "image/gif") return parseGifMetadata(header);
+	if (mimeType === "image/webp") return parseWebpMetadata(header);
+	return {};
+}
+
+async function readHeader(filePath: string, maxBytes: number): Promise<Buffer> {
+	if (maxBytes <= 0) return Buffer.alloc(0);
+	const fileHandle = await fs.open(filePath, "r");
+	try {
+		const buffer = Buffer.allocUnsafe(maxBytes);
+		const { bytesRead } = await fileHandle.read(buffer, 0, maxBytes, 0);
+		return buffer.subarray(0, bytesRead);
+	} finally {
+		await fileHandle.close();
+	}
+}
+
+export async function readImageMetadata(options: ReadImageMetadataOptions): Promise<ImageMetadata | null> {
+	const resolvedPath = options.resolvedPath ?? resolveReadPath(options.path, options.cwd);
+	const mimeType = options.detectedMimeType ?? (await detectSupportedImageMimeTypeFromFile(resolvedPath));
+	if (!mimeType) return null;
+
+	const stats = await Bun.file(resolvedPath).stat();
+	const bytes = stats.size;
+	const headerBytes = Math.max(0, Math.min(bytes, MAX_IMAGE_METADATA_HEADER_BYTES));
+	const header = await readHeader(resolvedPath, headerBytes);
+	const parsed = parseImageHeaderMetadata(header, mimeType);
+
+	return {
+		mimeType,
+		bytes,
+		width: parsed.width,
+		height: parsed.height,
+		channels: parsed.channels,
+		hasAlpha: parsed.hasAlpha,
+	};
+}
+
+export async function loadImageInput(options: LoadImageInputOptions): Promise<LoadedImageInput | null> {
+	const maxBytes = options.maxBytes ?? MAX_IMAGE_INPUT_BYTES;
+	const resolvedPath = options.resolvedPath ?? resolveReadPath(options.path, options.cwd);
+	const mimeType = options.detectedMimeType ?? (await detectSupportedImageMimeTypeFromFile(resolvedPath));
+	if (!mimeType) return null;
+
+	const stat = await Bun.file(resolvedPath).stat();
+	if (stat.size > maxBytes) {
+		throw new ImageInputTooLargeError(stat.size, maxBytes);
+	}
+
+	const inputBuffer = await fs.readFile(resolvedPath);
+	if (inputBuffer.byteLength > maxBytes) {
+		throw new ImageInputTooLargeError(inputBuffer.byteLength, maxBytes);
+	}
+
+	let outputData = new Uint8Array(inputBuffer).toBase64();
+	let outputMimeType = mimeType;
+	let outputBytes = inputBuffer.byteLength;
+	let dimensionNote: string | undefined;
+
+	if (options.autoResize) {
+		try {
+			const resized = await resizeImage({ type: "image", data: outputData, mimeType });
+			outputData = resized.data;
+			outputMimeType = resized.mimeType;
+			outputBytes = resized.buffer.byteLength;
+			dimensionNote = formatDimensionNote(resized);
+		} catch {
+			// keep original image when resize fails
+		}
+	}
+
+	let textNote = `Read image file [${outputMimeType}]`;
+	if (dimensionNote) {
+		textNote += `\n${dimensionNote}`;
+	}
+
+	return {
+		resolvedPath,
+		mimeType: outputMimeType,
+		data: outputData,
+		textNote,
+		dimensionNote,
+		bytes: outputBytes,
+	};
+}