@wahack/pi-coding-agent 15.11.0

package/src/tools/eval.ts

@@ -0,0 +1,577 @@
+import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
+import type { ImageContent } from "@oh-my-pi/pi-ai";
+import { prompt } from "@oh-my-pi/pi-utils";
+import * as z from "zod/v4";
+import { jsBackend, pythonBackend } from "../eval";
+import type { ExecutorBackend, ExecutorBackendResult } from "../eval/backend";
+import { EVAL_TIMEOUT_PAUSE_OP, EVAL_TIMEOUT_RESUME_OP } from "../eval/bridge-timeout";
+import { IdleTimeout } from "../eval/idle-timeout";
+import { defaultEvalSessionId } from "../eval/session-id";
+import type { EvalCellResult, EvalDisplayOutput, EvalLanguage, EvalStatusEvent, EvalToolDetails } from "../eval/types";
+import evalDescription from "../prompts/tools/eval.md" with { type: "text" };
+import { DEFAULT_MAX_BYTES, OutputSink, type OutputSummary, TailBuffer } from "../session/streaming-output";
+import { formatDimensionNote, resizeImage } from "../utils/image-resize";
+import type { ToolSession } from ".";
+import { truncateForPrompt } from "./approval";
+import { resolveEvalBackends } from "./eval-backends";
+import { upsertStatusEvent } from "./eval-render";
+import { resolveOutputMaxColumns, resolveOutputSinkHeadBytes } from "./output-meta";
+import { ToolAbortError, ToolError } from "./tool-errors";
+import { toolResult } from "./tool-result";
+import { clampTimeout } from "./tool-timeouts";
+
+export { EVAL_DEFAULT_PREVIEW_LINES, evalToolRenderer } from "./eval-render";
+
+/**
+ * Per-cell input. Each cell runs in order; state persists within a language
+ * across cells and across tool calls.
+ */
+const evalCellSchema = z.object({
+	language: z.enum(["py", "js"]).describe('runtime: "py" for the IPython kernel, "js" for the persistent JS VM'),
+	code: z.string().describe("cell body, verbatim. Use top-level await freely."),
+	title: z.string().optional().describe('short label shown in transcript (e.g. "imports", "load config")'),
+	timeout: z.number().int().min(1).max(3600).optional().describe("per-cell timeout in seconds (1-3600, default 30)"),
+	reset: z
+		.boolean()
+		.optional()
+		.describe("wipe this cell's language kernel before running. Other languages are untouched."),
+});
+export type EvalCellInput = z.infer<typeof evalCellSchema>;
+
+export const evalSchema = z.object({
+	cells: z
+		.array(evalCellSchema)
+		.min(1)
+		.describe("cells executed in order. State persists within each language across cells and tool calls."),
+});
+export type EvalToolParams = z.infer<typeof evalSchema>;
+
+export type EvalToolResult = {
+	content: Array<{ type: "text"; text: string }>;
+	details: EvalToolDetails | undefined;
+};
+
+export type EvalProxyExecutor = (params: EvalToolParams, signal?: AbortSignal) => Promise<EvalToolResult>;
+
+/** Cap per `display()` value sent back to the model. */
+const MAX_DISPLAY_TEXT_BYTES = 8000;
+
+function formatDisplayJsonForText(value: unknown): string {
+	let text: string;
+	try {
+		text = JSON.stringify(value, null, 2) ?? String(value);
+	} catch {
+		text = String(value);
+	}
+	if (text.length > MAX_DISPLAY_TEXT_BYTES) {
+		text = `${text.slice(0, MAX_DISPLAY_TEXT_BYTES)}\n… (${text.length - MAX_DISPLAY_TEXT_BYTES} chars truncated)`;
+	}
+	return text;
+}
+
+/**
+ * Format display() JSON values into text the model can see. Images are surfaced
+ * separately as ImageContent so the model can actually inspect them; this helper
+ * intentionally does not touch images.
+ */
+function formatDisplayOutputsForText(outputs: EvalDisplayOutput[]): string {
+	const chunks: string[] = [];
+	let displayIndex = 0;
+	for (const output of outputs) {
+		if (output.type !== "json") continue;
+		displayIndex++;
+		chunks.push(`display[${displayIndex}]:\n${formatDisplayJsonForText(output.data)}`);
+	}
+	return chunks.join("\n\n");
+}
+
+export interface EvalToolDescriptionOptions {
+	py?: boolean;
+	js?: boolean;
+	/**
+	 * Whether `agent()` is allowed in this session. Driven by the parent's
+	 * spawn policy (`getSessionSpawns`). Defaults to `true` for backward
+	 * compatibility — when the session forbids spawning, the prelude doc
+	 * omits the `agent()` entry so the model does not promise itself a
+	 * helper that will only ever throw "spawns disabled".
+	 */
+	spawns?: boolean;
+}
+
+export function getEvalToolDescription(options: EvalToolDescriptionOptions = {}): string {
+	const py = options.py ?? true;
+	const js = options.js ?? true;
+	const spawns = options.spawns ?? true;
+	return prompt.render(evalDescription, { py, js, spawns });
+}
+
+export interface EvalToolOptions {
+	proxyExecutor?: EvalProxyExecutor;
+}
+
+interface ResolvedBackend {
+	backend: ExecutorBackend;
+	notice?: string;
+}
+
+interface ResolvedEvalCell {
+	index: number;
+	title?: string;
+	code: string;
+	timeoutMs: number;
+	reset: boolean;
+	resolved: ResolvedBackend;
+}
+
+function uniqueEvalLanguages(cells: ResolvedEvalCell[]): EvalLanguage[] {
+	return [...new Set(cells.map(cell => cell.resolved.backend.id))];
+}
+
+function detailsNotice(cells: ResolvedEvalCell[]): string | undefined {
+	const notices = [
+		...new Set(cells.map(cell => cell.resolved.notice).filter((notice): notice is string => Boolean(notice))),
+	];
+	return notices.length > 0 ? notices.join(" ") : undefined;
+}
+
+function timeoutSecondsFromMs(timeoutMs: number): number {
+	return clampTimeout("eval", timeoutMs / 1000);
+}
+
+async function resolveBackend(session: ToolSession, language: EvalLanguage): Promise<ResolvedBackend> {
+	const backends = resolveEvalBackends(session);
+	const allowPy = backends.python;
+	const allowJs = backends.js;
+
+	if (language === "python") {
+		if (!allowPy) throw new ToolError("Python backend is disabled (PI_PY=0 or eval.py = false).");
+		if (!(await pythonBackend.isAvailable(session))) {
+			throw new ToolError(
+				'Python backend is unavailable in this session. Pass language: "js" or install the python kernel.',
+			);
+		}
+		return { backend: pythonBackend };
+	}
+	if (!allowJs) throw new ToolError("JavaScript backend is disabled (PI_JS=0 or eval.js = false).");
+	return { backend: jsBackend };
+}
+
+export class EvalTool implements AgentTool<typeof evalSchema> {
+	readonly name = "eval";
+	readonly approval = "exec" as const;
+	readonly formatApprovalDetails = (args: unknown): string[] => {
+		const params = args as Partial<EvalToolParams>;
+		const cells = Array.isArray(params.cells) ? params.cells : [];
+		const firstCell = cells[0] as Partial<EvalCellInput> | undefined;
+		if (!firstCell) return [];
+		const language = typeof firstCell.language === "string" ? firstCell.language : "(missing)";
+		const code = typeof firstCell.code === "string" ? firstCell.code : "";
+		const lines = [`Language: ${language}`, `Code:\n${truncateForPrompt(code)}`];
+		if (cells.length > 1) {
+			lines.push(`+${cells.length - 1} more cell${cells.length === 2 ? "" : "s"}`);
+		}
+		return lines;
+	};
+	readonly summary = "Execute Python or JavaScript code in an in-process eval backend";
+	readonly loadMode = "discoverable";
+	readonly label = "Eval";
+	get description(): string {
+		if (!this.session) return getEvalToolDescription();
+		const backends = resolveEvalBackends(this.session);
+		const sessionSpawns = this.session.getSessionSpawns?.() ?? "*";
+		const spawnsAllowed = sessionSpawns !== "" && sessionSpawns !== null;
+		return getEvalToolDescription({ py: backends.python, js: backends.js, spawns: spawnsAllowed });
+	}
+	readonly parameters = evalSchema;
+	readonly concurrency = "exclusive";
+	readonly strict = true;
+	readonly intent = (args: Partial<z.infer<typeof evalSchema>>): string | undefined => {
+		const cells = Array.isArray(args.cells) ? args.cells : [];
+		const first = cells.find(c => c && typeof c === "object");
+		if (!first) return "evaluating";
+		const title = typeof first.title === "string" ? first.title : undefined;
+		const language = typeof first.language === "string" ? first.language : "?";
+		const label = title || `running ${language}`;
+		return cells.length > 1 ? `${label} (+${cells.length - 1})` : label;
+	};
+
+	readonly #proxyExecutor?: EvalProxyExecutor;
+
+	constructor(
+		private readonly session: ToolSession | null,
+		options?: EvalToolOptions,
+	) {
+		this.#proxyExecutor = options?.proxyExecutor;
+	}
+
+	async execute(
+		_toolCallId: string,
+		params: z.infer<typeof evalSchema>,
+		signal?: AbortSignal,
+		onUpdate?: AgentToolUpdateCallback,
+		_ctx?: AgentToolContext,
+	): Promise<AgentToolResult<EvalToolDetails | undefined>> {
+		if (this.#proxyExecutor) {
+			return this.#proxyExecutor(params, signal);
+		}
+
+		if (!this.session) {
+			throw new ToolError("Eval tool requires a session when not using proxy executor");
+		}
+		const session = this.session;
+
+		const cells: ResolvedEvalCell[] = [];
+		for (let i = 0; i < params.cells.length; i++) {
+			const cell = params.cells[i];
+			const language: EvalLanguage = cell.language === "py" ? "python" : "js";
+			const resolved = await resolveBackend(session, language);
+			cells.push({
+				index: i,
+				title: cell.title,
+				code: cell.code,
+				timeoutMs: (cell.timeout ?? 30) * 1000,
+				reset: cell.reset ?? false,
+				resolved,
+			});
+		}
+		const languages = uniqueEvalLanguages(cells);
+		const notice = detailsNotice(cells);
+		const sessionAbortController = new AbortController();
+		let outputSink: OutputSink | undefined;
+		let outputSummary: OutputSummary | undefined;
+		let outputDumped = false;
+		const finalizeOutput = async (): Promise<OutputSummary | undefined> => {
+			if (outputDumped || !outputSink) return outputSummary;
+			outputSummary = await outputSink.dump();
+			outputDumped = true;
+			return outputSummary;
+		};
+
+		const execution = (async (): Promise<AgentToolResult<EvalToolDetails | undefined>> => {
+			try {
+				if (signal?.aborted) {
+					throw new ToolAbortError();
+				}
+				session.assertEvalExecutionAllowed?.();
+
+				const tailBuffer = new TailBuffer(DEFAULT_MAX_BYTES * 2);
+				const jsonOutputs: unknown[] = [];
+				const images: ImageContent[] = [];
+				const statusEvents: EvalStatusEvent[] = [];
+
+				const cellResults: EvalCellResult[] = cells.map(cell => ({
+					index: cell.index,
+					title: cell.title,
+					code: cell.code,
+					language: cell.resolved.backend.id,
+					output: "",
+					status: "pending",
+				}));
+				const cellOutputs: string[] = [];
+
+				const appendTail = (text: string) => {
+					tailBuffer.append(text);
+				};
+
+				const buildUpdateDetails = (): EvalToolDetails => {
+					const details: EvalToolDetails = {
+						language: languages[0],
+						languages,
+						cells: cellResults.map(cell => ({
+							...cell,
+							statusEvents: cell.statusEvents ? [...cell.statusEvents] : undefined,
+						})),
+					};
+					if (jsonOutputs.length > 0) {
+						details.jsonOutputs = jsonOutputs;
+					}
+					if (images.length > 0) {
+						details.images = images;
+					}
+					if (statusEvents.length > 0) {
+						details.statusEvents = statusEvents;
+					}
+					if (notice) {
+						details.notice = notice;
+					}
+					return details;
+				};
+
+				const pushUpdate = () => {
+					if (!onUpdate) return;
+					const tailText = tailBuffer.text();
+					onUpdate({
+						content: [{ type: "text", text: tailText }],
+						details: buildUpdateDetails(),
+					});
+				};
+
+				const sessionFile = session.getSessionFile?.() ?? undefined;
+				const kernelOwnerId = session.getEvalKernelOwnerId?.() ?? undefined;
+				const { path: artifactPath, id: artifactId } = (await session.allocateOutputArtifact?.("eval")) ?? {};
+				session.assertEvalExecutionAllowed?.();
+				outputSink = new OutputSink({
+					artifactPath,
+					artifactId,
+					headBytes: resolveOutputSinkHeadBytes(session.settings),
+					maxColumns: resolveOutputMaxColumns(session.settings),
+					onChunk: chunk => {
+						appendTail(chunk);
+						pushUpdate();
+					},
+				});
+				const sessionId = session.getEvalSessionId?.() ?? defaultEvalSessionId(session);
+
+				for (let i = 0; i < cells.length; i++) {
+					const cell = cells[i];
+					const backend = cell.resolved.backend;
+					// The per-cell `timeout` is a budget on the cell runtime's *own*
+					// work. Host-side `agent()`/`parallel()`/`completion()` bridge calls suspend
+					// that budget entirely and restart a fresh timeout window when control
+					// returns to Python/JS. Compute, stdout, `log()`/`phase()`, and
+					// ordinary tool calls all count against the budget. The watchdog drives
+					// `combinedSignal`; we pass no wall-clock deadline downstream so the
+					// backends never arm a competing fixed timer.
+					const idleTimeoutMs = timeoutSecondsFromMs(cell.timeoutMs) * 1000;
+					const idle = new IdleTimeout(idleTimeoutMs);
+					const combinedSignal = signal
+						? AbortSignal.any([signal, idle.signal, sessionAbortController.signal])
+						: AbortSignal.any([idle.signal, sessionAbortController.signal]);
+
+					const cellResult = cellResults[i];
+					cellResult.status = "running";
+					cellResult.output = "";
+					cellResult.statusEvents = undefined;
+					cellResult.exitCode = undefined;
+					cellResult.durationMs = undefined;
+					pushUpdate();
+
+					const startTime = Date.now();
+					let result: ExecutorBackendResult;
+					try {
+						result = await backend.execute(cell.code, {
+							cwd: session.cwd,
+							sessionId,
+							sessionFile: sessionFile ?? undefined,
+							kernelOwnerId,
+							signal: combinedSignal,
+							session,
+							idleTimeoutMs,
+							reset: cell.reset,
+							onChunk: chunk => {
+								outputSink!.push(chunk);
+							},
+							onStatus: event => {
+								if (event.op === EVAL_TIMEOUT_PAUSE_OP) {
+									idle.pause();
+									return;
+								}
+								if (event.op === EVAL_TIMEOUT_RESUME_OP) {
+									idle.resume();
+									return;
+								}
+								cellResult.statusEvents ??= [];
+								upsertStatusEvent(cellResult.statusEvents, event);
+								pushUpdate();
+							},
+						});
+					} finally {
+						idle.dispose();
+					}
+					const durationMs = Date.now() - startTime;
+
+					const cellStatusEvents: EvalStatusEvent[] = [];
+					const cellDisplayOutputs: EvalDisplayOutput[] = [];
+					const cellImageNotes: string[] = [];
+					let cellHasMarkdown = false;
+					for (const output of result.displayOutputs) {
+						if (output.type === "json") {
+							jsonOutputs.push(output.data);
+							cellDisplayOutputs.push(output);
+						}
+						if (output.type === "image") {
+							const resized = await resizeImage({
+								type: "image",
+								data: output.data,
+								mimeType: output.mimeType,
+							});
+							const image: ImageContent = {
+								type: "image",
+								data: resized.data,
+								mimeType: resized.mimeType,
+							};
+							images.push(image);
+							cellDisplayOutputs.push({
+								type: "image",
+								data: image.data,
+								mimeType: image.mimeType,
+							});
+							const dimensionNote = formatDimensionNote(resized);
+							if (dimensionNote) {
+								cellImageNotes.push(`display image ${cellImageNotes.length + 1}: ${dimensionNote}`);
+							}
+						}
+						if (output.type === "status") {
+							upsertStatusEvent(statusEvents, output.event);
+							upsertStatusEvent(cellStatusEvents, output.event);
+						}
+						if (output.type === "markdown") {
+							cellHasMarkdown = true;
+						}
+					}
+
+					const stdoutTrimmed = result.output.trim();
+					const imageText = cellImageNotes.join("\n");
+					const displayText = formatDisplayOutputsForText(cellDisplayOutputs);
+					const visibleDisplayText =
+						displayText && imageText ? `${displayText}\n\n${imageText}` : displayText || imageText;
+					const cellOutput =
+						stdoutTrimmed && visibleDisplayText
+							? `${stdoutTrimmed}\n\n${visibleDisplayText}`
+							: stdoutTrimmed || visibleDisplayText;
+					cellResult.output = cellOutput;
+					cellResult.exitCode = result.exitCode;
+					cellResult.durationMs = durationMs;
+					cellResult.statusEvents = cellStatusEvents.length > 0 ? cellStatusEvents : undefined;
+					cellResult.hasMarkdown = cellHasMarkdown || undefined;
+
+					let combinedCellOutput = "";
+					if (cells.length > 1) {
+						const cellHeader = `[${i + 1}/${cells.length}]`;
+						const cellTitle = cell.title ? ` ${cell.title}` : "";
+						if (cellOutput) {
+							combinedCellOutput = `${cellHeader}${cellTitle}\n${cellOutput}`;
+						} else {
+							combinedCellOutput = `${cellHeader}${cellTitle} (ok)`;
+						}
+						cellOutputs.push(combinedCellOutput);
+					} else if (cellOutput) {
+						combinedCellOutput = cellOutput;
+						cellOutputs.push(combinedCellOutput);
+					}
+
+					if (combinedCellOutput) {
+						const prefix = cellOutputs.length > 1 ? "\n\n" : "";
+						appendTail(`${prefix}${combinedCellOutput}`);
+					}
+
+					if (result.cancelled) {
+						cellResult.status = "error";
+						pushUpdate();
+						const errorMsg = result.output || "Command aborted";
+						const combinedOutput = cellOutputs.join("\n\n");
+						const outputText =
+							cells.length > 1
+								? `${combinedOutput}\n\nCell ${i + 1} aborted: ${errorMsg}`
+								: combinedOutput || errorMsg;
+
+						const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput);
+						const details: EvalToolDetails = {
+							language: languages[0],
+							languages,
+							cells: cellResults,
+							jsonOutputs: jsonOutputs.length > 0 ? jsonOutputs : undefined,
+							statusEvents: statusEvents.length > 0 ? statusEvents : undefined,
+							isError: true,
+						};
+						if (notice) details.notice = notice;
+
+						return toolResult(details)
+							.content([{ type: "text", text: outputText }, ...images])
+							.truncationFromSummary(summaryForMeta, { direction: "tail" })
+							.done();
+					}
+
+					if (result.exitCode !== 0 && result.exitCode !== undefined) {
+						cellResult.status = "error";
+						pushUpdate();
+						const combinedOutput = cellOutputs.join("\n\n");
+						const outputText =
+							cells.length > 1
+								? `${combinedOutput}\n\nCell ${i + 1} failed (exit code ${result.exitCode}). Earlier cells succeeded—their state persists. Fix only cell ${i + 1}.`
+								: combinedOutput
+									? `${combinedOutput}\n\nCommand exited with code ${result.exitCode}`
+									: `Command exited with code ${result.exitCode}`;
+
+						const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput);
+						const details: EvalToolDetails = {
+							language: languages[0],
+							languages,
+							cells: cellResults,
+							jsonOutputs: jsonOutputs.length > 0 ? jsonOutputs : undefined,
+							statusEvents: statusEvents.length > 0 ? statusEvents : undefined,
+							isError: true,
+						};
+						if (notice) details.notice = notice;
+
+						return toolResult(details)
+							.content([{ type: "text", text: outputText }, ...images])
+							.truncationFromSummary(summaryForMeta, { direction: "tail" })
+							.done();
+					}
+
+					cellResult.status = "complete";
+					pushUpdate();
+				}
+
+				const combinedOutput = cellOutputs.join("\n\n");
+				const hasImages = images.length > 0;
+				const outputText =
+					combinedOutput ||
+					(hasImages
+						? `(displayed ${images.length} image${images.length === 1 ? "" : "s"}; no text output)`
+						: "(no output)");
+				const summaryForMeta = await summarizeFinal(combinedOutput, finalizeOutput);
+
+				const details: EvalToolDetails = {
+					language: languages[0],
+					languages,
+					cells: cellResults,
+					jsonOutputs: jsonOutputs.length > 0 ? jsonOutputs : undefined,
+					statusEvents: statusEvents.length > 0 ? statusEvents : undefined,
+				};
+				if (notice) details.notice = notice;
+
+				return toolResult(details)
+					.content([{ type: "text", text: outputText }, ...images])
+					.truncationFromSummary(summaryForMeta, { direction: "tail" })
+					.done();
+			} finally {
+				if (!outputDumped) {
+					try {
+						await finalizeOutput();
+					} catch {}
+				}
+			}
+		})();
+
+		return await (session.trackEvalExecution?.(execution, sessionAbortController) ?? execution);
+	}
+}
+
+async function summarizeFinal(
+	combinedOutput: string,
+	finalizeOutput: () => Promise<OutputSummary | undefined>,
+): Promise<OutputSummary> {
+	const rawSummary = (await finalizeOutput()) ?? {
+		output: "",
+		truncated: false,
+		totalLines: 0,
+		totalBytes: 0,
+		outputLines: 0,
+		outputBytes: 0,
+	};
+	const outputLines = combinedOutput.length > 0 ? combinedOutput.split("\n").length : 0;
+	const outputBytes = Buffer.byteLength(combinedOutput, "utf-8");
+	const missingLines = Math.max(0, rawSummary.totalLines - rawSummary.outputLines);
+	const missingBytes = Math.max(0, rawSummary.totalBytes - rawSummary.outputBytes);
+	return {
+		output: combinedOutput,
+		truncated: rawSummary.truncated,
+		totalLines: outputLines + missingLines,
+		totalBytes: outputBytes + missingBytes,
+		outputLines,
+		outputBytes,
+		artifactId: rawSummary.artifactId,
+	};
+}