@oh-my-pi/pi-coding-agent 13.9.12 → 13.9.13

package/src/prompts/tools/ast-grep.md

@@ -8,11 +8,12 @@ Performs structural code search using AST matching via native ast-grep.
 - Multiple patterns run in one native pass; results are merged and then `offset`/`limit` are applied to the combined match set
 - Use `sel` only for contextual pattern mode; otherwise provide direct patterns
 - In contextual pattern mode, results are returned for the selected node (`sel`), not the outer wrapper used to make the pattern parse
-- For variadic arguments/fields, use `$$$NAME` (not `$$NAME`)
+- For variadic captures (arguments, fields, statement lists), use `$$$NAME` (not `$$NAME`)
 - Patterns must parse as a single valid AST node for the target language; if a bare pattern fails, wrap it in valid context or use `sel`
+- If ast-grep reports `Multiple AST nodes are detected`, your pattern is not a single parseable node; wrap method snippets in valid context (for example `class $_ { … }`) and use `sel` to target the inner node
 - Patterns match AST structure, not text — whitespace/formatting differences are ignored
 - When the same metavariable appears multiple times, all occurrences must match identical code
-- For TypeScript declarations and methods, prefer shapes that tolerate annotations you do not care about, e.g. `async function $NAME($$$ARGS): $_ { $$$BODY }` or `class $_ { method($$$ARGS): $_ { $$$BODY } }` instead of omitting the return type entirely
+- For TypeScript declarations and methods, prefer shapes that tolerate annotations you do not care about, e.g. `async function $NAME($$$ARGS): $_ { $$$BODY }` or `class $_ { method($ARG: $_): $_ { $$$BODY } }` instead of omitting annotations entirely
 - Metavariables must be the sole content of an AST node; partial-text metavariables like `prefix$VAR`, `"hello $NAME"`, or `a $OP b` do NOT work — match the whole node instead
 - `$$$` captures are lazy (non-greedy): they stop when the next element in the pattern can match; place the most specific node after `$$$` to control where capture ends
 - `$_` is a non-capturing wildcard (matches any single node without binding); use it when you need to tolerate a node but don't need its value
@@ -38,6 +39,8 @@ Performs structural code search using AST matching via native ast-grep.
   `{"pat":["foo()"],"sel":"identifier","lang":"typescript","path":"src/utils.ts"}`
 - Match a TypeScript function declaration without caring about its exact return type:
   `{"pat":["async function processItems($$$ARGS): $_ { $$$BODY }"],"sel":"function_declaration","lang":"typescript","path":"src/worker.ts"}`
+- Match a TypeScript method body fragment by wrapping it in parseable context and selecting the method node:
+  `{"pat":["class $_ { async execute($INPUT: $_) { $$$BEFORE; const $PARSED = $_.parse($INPUT); $$$AFTER } }"],"sel":"method_definition","lang":"typescript","path":"src/tools/todo.ts"}`
 - Loosest existence check for a symbol in one file:
   `{"pat":["processItems"],"sel":"identifier","lang":"typescript","path":"src/worker.ts"}`
 </examples>

package/src/prompts/tools/inspect-image-system.md

@@ -0,0 +1,20 @@
+You are an image-analysis assistant.
+
+Core behavior:
+- Be evidence-first: distinguish direct observations from inferences.
+- If something is unclear, say uncertain rather than guessing.
+- Do not fabricate unreadable or occluded details.
+- Keep output compact and useful.
+
+Default output format (unless the requested question asks for another format):
+1) Answer
+2) Key evidence
+3) Caveats / uncertainty
+
+For OCR-style requests:
+- Preserve exact visible text, including casing and punctuation.
+- If text is partially unreadable, mark the unreadable segments explicitly.
+
+For UI/screenshot debugging requests:
+- Focus on visible states, labels, toggles, error messages, disabled controls, and relevant affordances.
+- Separate observed UI state from probable root cause.

package/src/prompts/tools/inspect-image.md

@@ -0,0 +1,32 @@
+Inspects an image file with a vision-capable model and returns compact text analysis.
+
+<instruction>
+- Use this for image understanding tasks (OCR, UI/screenshot debugging, scene/object questions)
+- Provide `path` to the local image file
+- Write a specific `question`:
+  - what to inspect
+  - constraints (for example: "quote visible text verbatim", "only report confirmed findings")
+  - desired output format (bullets/table/JSON/short answer)
+- Keep `question` grounded in observable evidence and ask for uncertainty when details are unclear
+- Use this tool over `read` when the goal is image analysis
+</instruction>
+
+<examples>
+- OCR with strict formatting:
+  - `{"path":"screenshots/error.png","question":"Extract all visible text verbatim. Return as bullet list in reading order."}`
+- Screenshot debugging:
+  - `{"path":"screenshots/settings.png","question":"Identify the likely cause of the disabled Save button. Return: (1) observations, (2) likely cause, (3) confidence."}`
+- Scene/object question:
+  - `{"path":"photos/shelf.jpg","question":"List all clearly visible product labels and their shelf positions (top/middle/bottom). If unreadable, say unreadable."}`
+</examples>
+
+<output>
+- Returns text-only analysis from the vision model
+- No image content blocks are returned in tool output
+</output>
+
+<critical>
+- Parameters are strict: only `path` and `question` are allowed
+- If image submission is blocked by settings, the tool will fail with an actionable error
+- If configured model does not support image input, configure a vision-capable model role before retrying
+</critical>

package/src/session/compaction/compaction.ts

@@ -13,7 +13,11 @@ import {
 	OPENAI_HEADERS,
 } from "@oh-my-pi/pi-ai/providers/openai-codex/constants";
 import { transformMessages } from "@oh-my-pi/pi-ai/providers/transform-messages";
-import { normalizeResponsesToolCallId } from "@oh-my-pi/pi-ai/utils";
+import {
+	getOpenAIResponsesHistoryItems,
+	getOpenAIResponsesHistoryPayload,
+	normalizeResponsesToolCallId,
+} from "@oh-my-pi/pi-ai/utils";
 import { logger } from "@oh-my-pi/pi-utils";
 import { renderPromptTemplate } from "../../config/prompt-templates";
 import compactionShortSummaryPrompt from "../../prompts/compaction/compaction-short-summary.md" with { type: "text" };
@@ -473,6 +477,7 @@ type OpenAiRemoteCompactionItem = {
 };
 
 interface OpenAiRemoteCompactionPreserveData {
+	provider?: string;
 	replacementHistory: Array<Record<string, unknown>>;
 	compactionItem: OpenAiRemoteCompactionItem;
 }
@@ -523,7 +528,7 @@ function getPreservedOpenAiRemoteCompactionData(
 ): OpenAiRemoteCompactionPreserveData | undefined {
 	const candidate = preserveData?.[OPENAI_REMOTE_COMPACTION_PRESERVE_KEY];
 	if (!candidate || typeof candidate !== "object") return undefined;
-	const maybeData = candidate as { replacementHistory?: unknown; compactionItem?: unknown };
+	const maybeData = candidate as { provider?: unknown; replacementHistory?: unknown; compactionItem?: unknown };
 	if (!Array.isArray(maybeData.replacementHistory)) return undefined;
 	const maybeItem = maybeData.compactionItem;
 	if (!maybeItem || typeof maybeItem !== "object") return undefined;
@@ -535,6 +540,7 @@ function getPreservedOpenAiRemoteCompactionData(
 		return undefined;
 	}
 	return {
+		provider: typeof maybeData.provider === "string" ? maybeData.provider : undefined,
 		replacementHistory: maybeData.replacementHistory as Array<Record<string, unknown>>,
 		compactionItem: compactionItem as unknown as OpenAiRemoteCompactionItem,
 	};
@@ -634,15 +640,6 @@ function trimOpenAiCompactInput(
 	return trimmed;
 }
 
-function getOpenAIResponsesHistoryItems(
-	providerPayload: { type?: string; items?: unknown } | undefined,
-): Array<Record<string, unknown>> | undefined {
-	if (providerPayload?.type !== "openaiResponsesHistory" || !Array.isArray(providerPayload.items)) {
-		return undefined;
-	}
-	return providerPayload.items as Array<Record<string, unknown>>;
-}
-
 function collectKnownOpenAiCallIds(items: Array<Record<string, unknown>>): Set<string> {
 	const knownCallIds = new Set<string>();
 	for (const item of items) {
@@ -667,8 +664,8 @@ function buildOpenAiNativeHistory(
 	let knownCallIds = collectKnownOpenAiCallIds(input);
 	for (const message of transformedMessages) {
 		if (message.role === "user" || message.role === "developer") {
-			const providerPayload = (message as { providerPayload?: { type?: string; items?: unknown } }).providerPayload;
-			const historyItems = getOpenAIResponsesHistoryItems(providerPayload);
+			const providerPayload = (message as { providerPayload?: AssistantMessage["providerPayload"] }).providerPayload;
+			const historyItems = getOpenAIResponsesHistoryItems(providerPayload, model.provider);
 			if (historyItems) {
 				input.push(...historyItems);
 				knownCallIds = collectKnownOpenAiCallIds(input);
@@ -705,22 +702,22 @@ function buildOpenAiNativeHistory(
 		}
 
 		if (message.role === "assistant") {
-			const providerPayload = (
-				message as { providerPayload?: { type?: string; incremental?: boolean; items?: unknown } }
-			).providerPayload;
-			const historyItems = getOpenAIResponsesHistoryItems(providerPayload);
-			if (historyItems) {
-				if (providerPayload?.incremental) {
-					input.push(...historyItems);
+			const assistant = message as AssistantMessage;
+			const providerPayload = getOpenAIResponsesHistoryPayload(
+				assistant.providerPayload,
+				model.provider,
+				assistant.provider,
+			);
+			if (providerPayload) {
+				if (providerPayload.dt) {
+					input.push(...providerPayload.items);
 				} else {
-					input.splice(0, input.length, ...historyItems);
+					input.splice(0, input.length, ...providerPayload.items);
 				}
 				knownCallIds = collectKnownOpenAiCallIds(input);
 				msgIndex++;
 				continue;
 			}
-
-			const assistant = message as AssistantMessage;
 			const isDifferentModel =
 				assistant.model !== model.id && assistant.provider === model.provider && assistant.api === model.api;
 
@@ -889,7 +886,7 @@ async function requestOpenAiRemoteCompaction(
 		});
 		throw new Error("Remote compaction response missing compaction item");
 	}
-	return { replacementHistory, compactionItem };
+	return { provider: model.provider, replacementHistory, compactionItem };
 }
 
 interface RemoteCompactionRequest {
@@ -1224,11 +1221,11 @@ export async function compact(
 	if (settings.remoteEnabled !== false && shouldUseOpenAiRemoteCompaction(model)) {
 		const previousRemoteCompaction = getPreservedOpenAiRemoteCompactionData(previousPreserveData);
 		const remoteMessages = [...messagesToSummarize, ...turnPrefixMessages, ...recentMessages];
-		const remoteHistory = buildOpenAiNativeHistory(
-			remoteMessages,
-			model,
-			previousRemoteCompaction?.replacementHistory,
-		);
+		const previousReplacementHistory =
+			previousRemoteCompaction?.provider === model.provider
+				? previousRemoteCompaction.replacementHistory
+				: undefined;
+		const remoteHistory = buildOpenAiNativeHistory(remoteMessages, model, previousReplacementHistory);
 		if (remoteHistory.length > 0) {
 			try {
 				const remote = await requestOpenAiRemoteCompaction(

package/src/session/session-manager.ts

@@ -532,16 +532,19 @@ export function buildSessionContext(
 	};
 
 	if (compaction) {
-		const remoteReplacementHistory = (() => {
+		const providerPayload: ProviderPayload | undefined = (() => {
 			const candidate = compaction.preserveData?.openaiRemoteCompaction;
 			if (!candidate || typeof candidate !== "object") return undefined;
-			const replacementHistory = (candidate as { replacementHistory?: unknown }).replacementHistory;
-			if (!Array.isArray(replacementHistory)) return undefined;
-			return replacementHistory as Array<Record<string, unknown>>;
+			const remote = candidate as { provider?: unknown; replacementHistory?: unknown };
+			if (typeof remote.provider !== "string" || remote.provider.length === 0) return undefined;
+			if (!Array.isArray(remote.replacementHistory)) return undefined;
+			return {
+				type: "openaiResponsesHistory",
+				provider: remote.provider,
+				items: remote.replacementHistory as Array<Record<string, unknown>>,
+			};
 		})();
-		const providerPayload: ProviderPayload | undefined = remoteReplacementHistory
-			? { type: "openaiResponsesHistory", items: remoteReplacementHistory }
-			: undefined;
+		const remoteReplacementHistory = providerPayload?.items;
 
 		// Emit summary first
 		messages.push(

package/src/tools/index.ts

@@ -27,6 +27,7 @@ import { ExitPlanModeTool } from "./exit-plan-mode";
 import { FetchTool } from "./fetch";
 import { FindTool } from "./find";
 import { GrepTool } from "./grep";
+import { InspectImageTool } from "./inspect-image";
 import { NotebookTool } from "./notebook";
 import { wrapToolWithMetaNotice } from "./output-meta";
 import { PythonTool } from "./python";
@@ -62,6 +63,7 @@ export * from "./fetch";
 export * from "./find";
 export * from "./gemini-image";
 export * from "./grep";
+export * from "./inspect-image";
 export * from "./notebook";
 export * from "./pending-action";
 export * from "./python";
@@ -170,6 +172,7 @@ export const BUILTIN_TOOLS: Record<string, ToolFactory> = {
 	lsp: LspTool.createIf,
 	notebook: s => new NotebookTool(s),
 	read: s => new ReadTool(s),
+	inspect_image: s => new InspectImageTool(s),
 	browser: s => new BrowserTool(s),
 	checkpoint: CheckpointTool.createIf,
 	rewind: RewindTool.createIf,
@@ -313,6 +316,7 @@ export async function createTools(session: ToolSession, toolNames?: string[]): P
 		if (name === "ast_edit") return session.settings.get("astEdit.enabled");
 		if (name === "render_mermaid") return session.settings.get("renderMermaid.enabled");
 		if (name === "notebook") return session.settings.get("notebook.enabled");
+		if (name === "inspect_image") return session.settings.get("inspect_image.enabled");
 		if (name === "fetch") return session.settings.get("fetch.enabled");
 		if (name === "web_search") return session.settings.get("web_search.enabled");
 		if (name === "lsp") return session.settings.get("lsp.enabled");

package/src/tools/inspect-image-renderer.ts

@@ -0,0 +1,103 @@
+import type { Component } from "@oh-my-pi/pi-tui";
+import { Text } from "@oh-my-pi/pi-tui";
+import type { RenderResultOptions } from "../extensibility/custom-tools/types";
+import type { Theme } from "../modes/theme/theme";
+import { renderStatusLine } from "../tui";
+import { formatExpandHint, replaceTabs, shortenPath, truncateToWidth } from "./render-utils";
+
+interface InspectImageRenderArgs {
+	path?: string;
+	question?: string;
+}
+
+interface InspectImageRendererDetails {
+	model: string;
+	imagePath: string;
+	mimeType: string;
+}
+
+interface InspectImageRendererResult {
+	content: Array<{ type: string; text?: string }>;
+	details?: InspectImageRendererDetails;
+	isError?: boolean;
+}
+
+const INSPECT_QUESTION_PREVIEW_WIDTH = 100;
+const INSPECT_OUTPUT_COLLAPSED_LINES = 4;
+const INSPECT_OUTPUT_EXPANDED_LINES = 16;
+const INSPECT_OUTPUT_LINE_WIDTH = 120;
+
+export const inspectImageToolRenderer = {
+	renderCall(args: InspectImageRenderArgs, _options: RenderResultOptions, uiTheme: Theme): Component {
+		const rawPath = args.path ?? "";
+		const pathDisplay = rawPath ? shortenPath(rawPath) : "…";
+		const header = renderStatusLine({ icon: "pending", title: "Inspect Image", description: pathDisplay }, uiTheme);
+		const question = args.question?.trim();
+		if (!question) {
+			return new Text(header, 0, 0);
+		}
+		const questionLine = ` ${uiTheme.fg("dim", uiTheme.tree.last)} ${uiTheme.fg("dim", "Question:")} ${uiTheme.fg("accent", truncateToWidth(replaceTabs(question), INSPECT_QUESTION_PREVIEW_WIDTH))}`;
+		return new Text(`${header}\n${questionLine}`, 0, 0);
+	},
+
+	renderResult(
+		result: InspectImageRendererResult,
+		options: RenderResultOptions,
+		uiTheme: Theme,
+		args?: InspectImageRenderArgs,
+	): Component {
+		const details = result.details;
+		const rawPath = details?.imagePath ?? args?.path ?? "";
+		const pathDisplay = rawPath ? shortenPath(rawPath) : "image";
+		const metaParts: string[] = [];
+		if (details?.model) metaParts.push(details.model);
+		if (details?.mimeType) metaParts.push(details.mimeType);
+		const header = renderStatusLine(
+			{
+				icon: result.isError ? "error" : "success",
+				title: "Inspect Image",
+				description: pathDisplay,
+			},
+			uiTheme,
+		);
+
+		const lines: string[] = [header];
+		const question = args?.question?.trim();
+		if (question) {
+			lines.push(
+				` ${uiTheme.fg("dim", uiTheme.tree.branch)} ${uiTheme.fg("dim", "Question:")} ${uiTheme.fg("accent", truncateToWidth(replaceTabs(question), INSPECT_QUESTION_PREVIEW_WIDTH))}`,
+			);
+		}
+
+		const outputText = result.content.find(content => content.type === "text")?.text?.trimEnd() ?? "";
+		if (!outputText) {
+			lines.push(uiTheme.fg("dim", "(no output)"));
+			if (metaParts.length > 0) {
+				lines.push("");
+				lines.push(uiTheme.fg("dim", metaParts.join(" · ")));
+			}
+			return new Text(lines.join("\n"), 0, 0);
+		}
+
+		lines.push("");
+		const outputLines = replaceTabs(outputText).split("\n");
+		const maxLines = options.expanded ? INSPECT_OUTPUT_EXPANDED_LINES : INSPECT_OUTPUT_COLLAPSED_LINES;
+		for (const line of outputLines.slice(0, maxLines)) {
+			lines.push(uiTheme.fg("toolOutput", truncateToWidth(line, INSPECT_OUTPUT_LINE_WIDTH)));
+		}
+
+		if (outputLines.length > maxLines) {
+			const remaining = outputLines.length - maxLines;
+			const hint = formatExpandHint(uiTheme, options.expanded, true);
+			lines.push(`${uiTheme.fg("dim", `… ${remaining} more lines`)}${hint ? ` ${hint}` : ""}`);
+		}
+
+		if (metaParts.length > 0) {
+			lines.push("");
+			lines.push(uiTheme.fg("dim", metaParts.join(" · ")));
+		}
+
+		return new Text(lines.join("\n"), 0, 0);
+	},
+	mergeCallAndResult: true,
+};

package/src/tools/inspect-image.ts

@@ -0,0 +1,168 @@
+import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
+import { type Api, type AssistantMessage, completeSimple, type Model } from "@oh-my-pi/pi-ai";
+import { type Static, Type } from "@sinclair/typebox";
+import { expandRoleAlias, resolveModelFromString } from "../config/model-resolver";
+import { renderPromptTemplate } from "../config/prompt-templates";
+import inspectImageDescription from "../prompts/tools/inspect-image.md" with { type: "text" };
+import inspectImageSystemPromptTemplate from "../prompts/tools/inspect-image-system.md" with { type: "text" };
+import {
+	ImageInputTooLargeError,
+	type LoadedImageInput,
+	loadImageInput,
+	MAX_IMAGE_INPUT_BYTES,
+} from "../utils/image-input";
+import type { ToolSession } from "./index";
+import { ToolError } from "./tool-errors";
+
+const inspectImageSchema = Type.Object(
+	{
+		path: Type.String({ description: "Filesystem path to an image" }),
+		question: Type.String({ description: "Question to ask about the image" }),
+	},
+	{ additionalProperties: false },
+);
+
+export type InspectImageParams = Static<typeof inspectImageSchema>;
+
+export interface InspectImageToolDetails {
+	model: string;
+	imagePath: string;
+	mimeType: string;
+}
+
+function extractResponseText(message: AssistantMessage): string {
+	return message.content
+		.filter(content => content.type === "text")
+		.map(content => content.text)
+		.join("")
+		.trim();
+}
+
+export class InspectImageTool implements AgentTool<typeof inspectImageSchema, InspectImageToolDetails> {
+	readonly name = "inspect_image";
+	readonly label = "InspectImage";
+	readonly description: string;
+	readonly parameters = inspectImageSchema;
+	readonly strict = true;
+
+	constructor(
+		private readonly session: ToolSession,
+		private readonly completeImageRequest: typeof completeSimple = completeSimple,
+	) {
+		this.description = renderPromptTemplate(inspectImageDescription);
+	}
+
+	async execute(
+		_toolCallId: string,
+		params: InspectImageParams,
+		signal?: AbortSignal,
+		_onUpdate?: AgentToolUpdateCallback<InspectImageToolDetails>,
+		_context?: AgentToolContext,
+	): Promise<AgentToolResult<InspectImageToolDetails>> {
+		if (this.session.settings.get("images.blockImages")) {
+			throw new ToolError(
+				"Image submission is disabled by settings (images.blockImages=true). Disable it to use inspect_image.",
+			);
+		}
+
+		const modelRegistry = this.session.modelRegistry;
+		if (!modelRegistry) {
+			throw new ToolError("Model registry is unavailable for inspect_image.");
+		}
+
+		const availableModels = modelRegistry.getAvailable();
+		if (availableModels.length === 0) {
+			throw new ToolError("No models available for inspect_image.");
+		}
+
+		const matchPreferences = { usageOrder: this.session.settings.getStorage()?.getModelUsageOrder() };
+		const resolvePattern = (pattern: string | undefined): Model<Api> | undefined => {
+			if (!pattern) return undefined;
+			const expanded = expandRoleAlias(pattern, this.session.settings);
+			return resolveModelFromString(expanded, availableModels, matchPreferences);
+		};
+
+		const activeModelPattern = this.session.getActiveModelString?.() ?? this.session.getModelString?.();
+		const model =
+			resolvePattern("pi/vision") ??
+			resolvePattern("pi/default") ??
+			resolvePattern(activeModelPattern) ??
+			availableModels[0];
+		if (!model) {
+			throw new ToolError("Unable to resolve a model for inspect_image.");
+		}
+
+		if (!model.input.includes("image")) {
+			throw new ToolError(
+				`Resolved model ${model.provider}/${model.id} does not support image input. Configure a vision-capable model for modelRoles.vision.`,
+			);
+		}
+
+		const apiKey = await modelRegistry.getApiKey(model);
+		if (!apiKey) {
+			throw new ToolError(
+				`No API key available for ${model.provider}/${model.id}. Configure credentials for this provider or choose another vision-capable model.`,
+			);
+		}
+
+		let imageInput: LoadedImageInput | null;
+		try {
+			imageInput = await loadImageInput({
+				path: params.path,
+				cwd: this.session.cwd,
+				autoResize: this.session.settings.get("images.autoResize"),
+				maxBytes: MAX_IMAGE_INPUT_BYTES,
+			});
+		} catch (error) {
+			if (error instanceof ImageInputTooLargeError) {
+				throw new ToolError(error.message);
+			}
+			throw error;
+		}
+
+		if (!imageInput) {
+			throw new ToolError("inspect_image only supports PNG, JPEG, GIF, and WEBP files detected by file content.");
+		}
+
+		const response = await this.completeImageRequest(
+			model,
+			{
+				systemPrompt: renderPromptTemplate(inspectImageSystemPromptTemplate),
+				messages: [
+					{
+						role: "user",
+						content: [
+							{ type: "image", data: imageInput.data, mimeType: imageInput.mimeType },
+							{ type: "text", text: params.question },
+						],
+						timestamp: Date.now(),
+					},
+				],
+			},
+			{ apiKey, signal },
+		);
+
+		if (response.stopReason === "error") {
+			throw new ToolError(response.errorMessage ?? "inspect_image request failed.");
+		}
+		if (response.stopReason === "aborted") {
+			throw new ToolError("inspect_image request aborted.");
+		}
+
+		const text = extractResponseText(response);
+		if (!text) {
+			throw new ToolError("inspect_image model returned no text output.");
+		}
+
+		return {
+			content: [{ type: "text", text }],
+			details: {
+				model: `${model.provider}/${model.id}`,
+				imagePath: imageInput.resolvedPath,
+				mimeType: imageInput.mimeType,
+			},
+		};
+	}
+}
+
+export { inspectImageToolRenderer } from "./inspect-image-renderer";