@wahack/pi-coding-agent 15.11.0

package/src/eval/backend.ts

@@ -0,0 +1,71 @@
+import { buildEvalUrlRoots, type LocalProtocolOptions } from "../internal-urls";
+import type { ToolSession } from "../tools";
+import type { EvalDisplayOutput, EvalLanguage, EvalStatusEvent } from "./types";
+
+/** Per-cell execute() options. */
+export interface ExecutorBackendExecOptions {
+	cwd: string;
+	sessionId: string;
+	sessionFile: string | undefined;
+	kernelOwnerId: string | undefined;
+	signal?: AbortSignal;
+	session: ToolSession;
+	/**
+	 * Runtime-work budget in milliseconds (the cell's `timeout`). Cancellation is
+	 * driven entirely by `signal`, which the eval tool arms as a watchdog that
+	 * pauses on bridge timeout-control status events and fires a `TimeoutError`
+	 * reason only while the Python/JS runtime owns control. Backends use this
+	 * value only for timeout-annotation text and as cold-start headroom; they MUST
+	 * NOT derive a competing wall-clock timer from it.
+	 */
+	idleTimeoutMs: number;
+	reset: boolean;
+	onChunk: (chunk: string) => void;
+	/**
+	 * Live status events (read/write/agent/…) delivered as they are emitted,
+	 * before the cell finishes. The same events are also returned in
+	 * `displayOutputs`; this channel exists so callers can stream long-running
+	 * progress (e.g. `agent()` subagents) into the UI mid-execution.
+	 */
+	onStatus?: (event: EvalStatusEvent) => void;
+}
+
+/** Result returned by a backend's execute(). */
+export interface ExecutorBackendResult {
+	output: string;
+	exitCode: number | undefined;
+	cancelled: boolean;
+	truncated: boolean;
+	artifactId: string | undefined;
+	totalLines: number;
+	totalBytes: number;
+	outputLines: number;
+	outputBytes: number;
+	displayOutputs: EvalDisplayOutput[];
+}
+
+/** Pluggable language backend for the eval tool. */
+export interface ExecutorBackend {
+	readonly id: EvalLanguage;
+	readonly label: string;
+	/** Source language identifier passed to the syntax highlighter (e.g. "python", "javascript"). */
+	readonly highlightLang: string;
+	/** Cheap availability check. Used by fallback resolution. */
+	isAvailable(session: ToolSession): Promise<boolean>;
+	/** Execute one cell. Caller invokes once per cell and aggregates results. */
+	execute(code: string, opts: ExecutorBackendExecOptions): Promise<ExecutorBackendResult>;
+}
+
+/**
+ * Resolve the on-disk roots that the eval helpers substitute for internal-URL
+ * schemes (currently `local://`). Prefers the session's own
+ * {@link LocalProtocolOptions} — the exact mapping `read local://…` uses — so an
+ * eval `write("local://x")` and a later `read local://x` agree on the location.
+ */
+export function resolveEvalUrlRoots(session: ToolSession): Record<string, string> {
+	const options: LocalProtocolOptions = session.localProtocolOptions ?? {
+		getArtifactsDir: () => session.getArtifactsDir?.() ?? null,
+		getSessionId: () => session.getSessionId?.() ?? null,
+	};
+	return buildEvalUrlRoots(options);
+}

package/src/eval/bridge-timeout.ts

@@ -0,0 +1,44 @@
+/**
+ * Timeout suspension for in-flight host-side eval bridge calls.
+ *
+ * The eval watchdog caps a cell's `timeout` as a budget on the cell runtime's
+ * own work. Host-side `agent()` / `parallel()` / `completion()` bridge calls hand
+ * control to the outer TypeScript process, where the Python kernel or JS VM is
+ * only waiting for a result. While that delegated work is in flight, the cell
+ * timeout must be ignored completely; once the bridge returns and the runtime is
+ * back in control, the watchdog starts a fresh timeout window.
+ *
+ * Bridge helpers express that handoff with synthetic pause/resume status events
+ * on the existing `emitStatus → onStatus` path. Consumers MUST treat these as
+ * timeout-control events only: update the watchdog and drop them from rendered
+ * or persisted cell output.
+ */
+import type { JsStatusEvent } from "./js/shared/types";
+
+/** Synthetic status op emitted when a bridge call leaves the cell runtime. */
+export const EVAL_TIMEOUT_PAUSE_OP = "timeout-pause";
+
+/** Synthetic status op emitted when a bridge call returns control to the runtime. */
+export const EVAL_TIMEOUT_RESUME_OP = "timeout-resume";
+
+/** Whether a status event is pure eval-timeout control and should not render. */
+export function isEvalTimeoutControlEvent(event: JsStatusEvent): boolean {
+	return event.op === EVAL_TIMEOUT_PAUSE_OP || event.op === EVAL_TIMEOUT_RESUME_OP;
+}
+
+/**
+ * Run {@link operation} while suspending the eval watchdog through
+ * {@link emitStatus}. A no-op wrapper when no status sink is wired.
+ */
+export async function withBridgeTimeoutPause<T>(
+	emitStatus: ((event: JsStatusEvent) => void) | undefined,
+	operation: () => Promise<T>,
+): Promise<T> {
+	if (!emitStatus) return operation();
+	emitStatus({ op: EVAL_TIMEOUT_PAUSE_OP });
+	try {
+		return await operation();
+	} finally {
+		emitStatus({ op: EVAL_TIMEOUT_RESUME_OP });
+	}
+}

package/src/eval/budget-bridge.ts

@@ -0,0 +1,48 @@
+/**
+ * Host-side handler for the eval `budget` helper.
+ *
+ * Reports the active token ceiling and amount spent so kernel helpers can
+ * compute remaining budget. Precedence: a `+Nk`/`+Nk!` per-turn directive (the
+ * user's immediate intent) wins; otherwise an active Goal Mode budget; otherwise
+ * no ceiling, with `spent` still reflecting this turn's output where available.
+ */
+import type { ToolSession } from "../tools";
+import type { JsStatusEvent } from "./js/shared/types";
+
+/** Synthetic bridge name reserved for the `budget` helper across both runtimes. */
+export const EVAL_BUDGET_BRIDGE_NAME = "__budget__";
+
+export interface EvalBudgetBridgeOptions {
+	session: ToolSession;
+	signal?: AbortSignal;
+	emitStatus?: (event: JsStatusEvent) => void;
+}
+
+export interface EvalBudgetResult {
+	total: number | null;
+	spent: number;
+	/** Whether the ceiling is enforced (eval `agent()` throws past it) vs advisory. */
+	hard: boolean;
+}
+
+/**
+ * Resolve the current token budget snapshot for an eval cell's `budget` helper.
+ * The returned object is JSON-passed verbatim by the bridge transport; kernel
+ * helpers read `.total`/`.spent`/`.hard` directly.
+ */
+export async function runEvalBudget(_args: unknown, options: EvalBudgetBridgeOptions): Promise<EvalBudgetResult> {
+	const turn = options.session.getTurnBudget?.();
+	if (turn && turn.total !== null) {
+		return { total: turn.total, spent: turn.spent, hard: turn.hard };
+	}
+	const goal = options.session.getGoalModeState?.();
+	if (goal?.enabled && goal.goal) {
+		return {
+			total: goal.goal.tokenBudget ?? null,
+			spent: goal.goal.tokensUsed ?? 0,
+			hard: goal.goal.tokenBudget != null,
+		};
+	}
+	const spent = turn?.spent ?? options.session.getUsageStatistics?.()?.output ?? 0;
+	return { total: null, spent, hard: false };
+}

package/src/eval/completion-bridge.ts

@@ -0,0 +1,207 @@
+/**
+ * Host-side handler for the eval `completion()` helper.
+ *
+ * Both eval runtimes (JS worker + Python kernel) route helper→host calls
+ * through {@link callSessionTool}. Reserving the synthetic tool name
+ * {@link EVAL_COMPLETION_BRIDGE_NAME} lets a single host handler serve both
+ * transports without registering an agent-visible tool: cell code calls
+ * `completion(prompt, opts)`, the prelude forwards `{ prompt, model, system?, schema? }`
+ * through the bridge, and this module performs one stateless completion.
+ *
+ * The call is oneshot and toolless from the model's perspective — pure text
+ * in, text (or, with `schema`, a structured object) out.
+ */
+import { instrumentedCompleteSimple, resolveTelemetry } from "@oh-my-pi/pi-agent-core";
+import { type Api, Effort, type Model, type Tool } from "@oh-my-pi/pi-ai";
+import { getSupportedEfforts } from "@oh-my-pi/pi-catalog/model-thinking";
+import * as z from "zod/v4";
+import { extractTextContent, extractToolCall, parseJsonPayload } from "../commit/utils";
+
+import {
+	expandRoleAlias,
+	formatModelString,
+	getModelMatchPreferences,
+	resolveModelFromString,
+} from "../config/model-resolver";
+import type { ToolSession } from "../tools";
+import { ToolError } from "../tools/tool-errors";
+import { withBridgeTimeoutPause } from "./bridge-timeout";
+import type { JsStatusEvent } from "./js/shared/types";
+
+/** Synthetic bridge name reserved for the `completion()` helper across both runtimes. */
+export const EVAL_COMPLETION_BRIDGE_NAME = "__completion__";
+
+/** Synthetic tool the model is forced to call when a `schema` is supplied. */
+const STRUCTURED_TOOL_NAME = "respond";
+
+type CompletionTier = "smol" | "default" | "slow";
+
+const TIER_TO_PATTERN: Record<CompletionTier, string> = {
+	smol: "pi/smol",
+	default: "pi/default",
+	slow: "pi/slow",
+};
+
+const completionArgsSchema = z.object({
+	prompt: z.string().min(1, "prompt must be a non-empty string"),
+	model: z.enum(["smol", "default", "slow"]).default("default"),
+	system: z.string().optional(),
+	schema: z.record(z.string(), z.unknown()).optional(),
+});
+
+export interface EvalCompletionBridgeOptions {
+	session: ToolSession;
+	signal?: AbortSignal;
+	emitStatus?: (event: JsStatusEvent) => void;
+}
+
+export interface EvalCompletionResult {
+	text: string;
+	details: { model: string; tier: CompletionTier; structured: boolean };
+}
+
+/**
+ * Resolve a tier to a concrete {@link Model}. `default` prefers the session's
+ * active model and falls back to the `pi/default` role; `smol`/`slow` resolve
+ * their respective role patterns. Returns `undefined` when nothing matches.
+ */
+function resolveTierModel(tier: CompletionTier, session: ToolSession): Model<Api> | undefined {
+	const modelRegistry = session.modelRegistry;
+	if (!modelRegistry) return undefined;
+	const available = modelRegistry.getAvailable();
+	if (available.length === 0) return undefined;
+
+	const matchPreferences = getModelMatchPreferences(session.settings);
+	const resolve = (pattern: string | undefined): Model<Api> | undefined => {
+		if (!pattern) return undefined;
+		const expanded = expandRoleAlias(pattern, session.settings);
+		return resolveModelFromString(expanded, available, matchPreferences, modelRegistry);
+	};
+
+	if (tier === "default") {
+		const activePattern = session.getActiveModelString?.() ?? session.getModelString?.();
+		return resolve(activePattern) ?? resolve(TIER_TO_PATTERN.default);
+	}
+	return resolve(TIER_TO_PATTERN[tier]);
+}
+
+/**
+ * Choose the reasoning effort for a tier. Only `slow` opts into thinking, and
+ * only on reasoning-capable models — guarding against `requireSupportedEffort`
+ * throwing downstream on models that cannot reason. Clamps to the highest
+ * supported effort so a reasoning model without `high` does not 400.
+ */
+function reasoningForTier(tier: CompletionTier, model: Model<Api>): Effort | undefined {
+	if (tier !== "slow" || !model.reasoning) return undefined;
+	const efforts = getSupportedEfforts(model);
+	if (efforts.length === 0) return undefined;
+	return efforts.includes(Effort.High) ? Effort.High : efforts[efforts.length - 1];
+}
+
+/**
+ * Run a single stateless completion on behalf of an eval cell's `completion()` call.
+ * Returns a `{ text, details }` value shaped like a {@link callSessionTool}
+ * result so the existing bridge transport carries it to either runtime.
+ */
+export async function runEvalCompletion(
+	args: unknown,
+	options: EvalCompletionBridgeOptions,
+): Promise<EvalCompletionResult> {
+	const parsed = completionArgsSchema.safeParse(args);
+	if (!parsed.success) {
+		const issue = parsed.error.issues[0];
+		const where = issue?.path.length ? `${issue.path.join(".")}: ` : "";
+		throw new ToolError(`completion() received invalid arguments: ${where}${issue?.message ?? "bad input"}`);
+	}
+	const { prompt, model: tier, system, schema } = parsed.data;
+
+	const model = resolveTierModel(tier, options.session);
+	if (!model) {
+		throw new ToolError(
+			`completion() could not resolve a model for the "${tier}" tier. Configure modelRoles.${tier === "default" ? "default" : tier} or ensure a provider is available.`,
+		);
+	}
+
+	const registry = options.session.modelRegistry;
+	const apiKey = await registry?.getApiKey(model);
+	if (!registry || !apiKey) {
+		throw new ToolError(
+			`completion() has no API key for ${formatModelString(model)}. Configure credentials for this provider or choose another tier.`,
+		);
+	}
+
+	const tools: Tool[] | undefined = schema
+		? [
+				{
+					name: STRUCTURED_TOOL_NAME,
+					description: "Return your answer by calling this tool with the requested structured fields.",
+					parameters: schema,
+					strict: false,
+				},
+			]
+		: undefined;
+
+	const telemetry = resolveTelemetry(options.session.getTelemetry?.(), options.session.getSessionId?.() ?? undefined);
+
+	// Some providers (notably openai-codex) require a non-empty `instructions`
+	// field on every Responses request and 400 with "Instructions are required"
+	// when it is missing. Fall back to a minimal default so `completion(prompt)` works
+	// without forcing every caller to pass a `system` prompt.
+	const systemPrompt = system ? [system] : ["You are a helpful assistant."];
+
+	// Suspend eval timeout accounting while the model request owns control. The
+	// timeout clock restarts once the bridge returns to the cell runtime.
+	const response = await withBridgeTimeoutPause(options.emitStatus, () =>
+		instrumentedCompleteSimple(
+			model,
+			{
+				systemPrompt,
+				messages: [{ role: "user", content: [{ type: "text", text: prompt }], timestamp: Date.now() }],
+				tools,
+			},
+			{
+				apiKey: registry.resolver(model.provider, {
+					sessionId: options.session.getSessionId?.() ?? undefined,
+					baseUrl: model.baseUrl,
+					modelId: model.id,
+				}),
+				signal: options.signal,
+				reasoning: reasoningForTier(tier, model),
+				toolChoice: schema ? { type: "tool", name: STRUCTURED_TOOL_NAME } : undefined,
+			},
+			{ telemetry, oneshotKind: "eval_completion" },
+		),
+	);
+
+	if (response.stopReason === "error") {
+		throw new ToolError(response.errorMessage ?? "completion() request failed.");
+	}
+	if (response.stopReason === "aborted") {
+		throw new ToolError("completion() request aborted.");
+	}
+
+	let resultText: string;
+	if (schema) {
+		const call = extractToolCall(response, STRUCTURED_TOOL_NAME);
+		let value: unknown;
+		if (call) {
+			value = call.arguments;
+		} else {
+			const text = extractTextContent(response);
+			if (!text) throw new ToolError("completion() returned no structured response.");
+			try {
+				value = parseJsonPayload(text);
+			} catch {
+				throw new ToolError("completion() did not return a structured response matching the schema.");
+			}
+		}
+		resultText = JSON.stringify(value);
+	} else {
+		resultText = extractTextContent(response);
+		if (!resultText) throw new ToolError("completion() returned no text output.");
+	}
+
+	options.emitStatus?.({ op: "completion", model: formatModelString(model), tier, chars: resultText.length });
+
+	return { text: resultText, details: { model: formatModelString(model), tier, structured: Boolean(schema) } };
+}

package/src/eval/concurrency-bridge.ts

@@ -0,0 +1,34 @@
+/**
+ * Host-side handler for the eval `parallel()` / `pipeline()` worker pool.
+ *
+ * The pool ceiling is not a kernel-side knob: it tracks the `task.maxConcurrency`
+ * setting so an eval fan-out runs as wide as a `task` tool batch would. `0` means
+ * unbounded — run every item at once, exactly like `task.maxConcurrency = 0`.
+ */
+import type { ToolSession } from "../tools";
+import type { JsStatusEvent } from "./js/shared/types";
+
+/** Synthetic bridge name reserved for the parallel-pool ceiling across both runtimes. */
+export const EVAL_CONCURRENCY_BRIDGE_NAME = "__concurrency__";
+
+export interface EvalConcurrencyBridgeOptions {
+	session: ToolSession;
+	signal?: AbortSignal;
+	emitStatus?: (event: JsStatusEvent) => void;
+}
+
+export interface EvalConcurrencyResult {
+	/** Worker-pool ceiling; `0` means unbounded (run every item at once). */
+	limit: number;
+}
+
+/**
+ * Resolve the worker-pool ceiling for an eval cell's `parallel()`/`pipeline()`
+ * helpers from the live `task.maxConcurrency` setting. Negative/non-finite
+ * values collapse to `0` (unbounded), matching the `task` tool's own handling.
+ */
+export function runEvalConcurrency(_args: unknown, options: EvalConcurrencyBridgeOptions): EvalConcurrencyResult {
+	const raw = options.session.settings.get("task.maxConcurrency");
+	const limit = Number.isFinite(raw) ? Math.trunc(raw) : 0;
+	return { limit: limit > 0 ? limit : 0 };
+}

package/src/eval/idle-timeout.ts

@@ -0,0 +1,91 @@
+/**
+ * Watchdog for eval cell work.
+ *
+ * A cell's `timeout` bounds time while the Python kernel or JS VM is in control.
+ * Host-side bridge calls can {@link pause} the watchdog so delegated
+ * `agent()`/`parallel()`/`completion()` work is ignored completely, then {@link resume}
+ * starts a fresh timeout window once the runtime gets control back.
+ *
+ * Pause is reference-counted because `parallel()` can have multiple bridge calls
+ * in flight at once.
+ */
+export class IdleTimeout {
+	readonly #controller = new AbortController();
+	readonly #idleMs: number;
+	/** Absolute time (epoch ms) at which inactivity is considered to have expired. */
+	#deadlineMs: number;
+	#timer: NodeJS.Timeout | undefined;
+	#settled = false;
+	#pauseDepth = 0;
+
+	constructor(idleMs: number) {
+		this.#idleMs = Math.max(1, Math.floor(idleMs));
+		this.#deadlineMs = Date.now() + this.#idleMs;
+		this.#arm(this.#idleMs);
+	}
+
+	/** Aborts with a `TimeoutError` reason once the active timeout window is exhausted. */
+	get signal(): AbortSignal {
+		return this.#controller.signal;
+	}
+
+	/** Configured active timeout window in milliseconds. */
+	get idleMs(): number {
+		return this.#idleMs;
+	}
+
+	/** Suspend timeout accounting while control is delegated to host-side work. */
+	pause(): void {
+		if (this.#settled) return;
+		this.#pauseDepth++;
+		if (this.#pauseDepth !== 1) return;
+		if (this.#timer) {
+			clearTimeout(this.#timer);
+			this.#timer = undefined;
+		}
+	}
+
+	/** Resume timeout accounting with a fresh timeout window. */
+	resume(): void {
+		if (this.#settled || this.#pauseDepth === 0) return;
+		this.#pauseDepth--;
+		if (this.#pauseDepth > 0) return;
+		this.#deadlineMs = Date.now() + this.#idleMs;
+		this.#arm(this.#idleMs);
+	}
+
+	/** Stop the watchdog. Safe to call multiple times. */
+	dispose(): void {
+		if (this.#settled) return;
+		this.#settled = true;
+		if (this.#timer) {
+			clearTimeout(this.#timer);
+			this.#timer = undefined;
+		}
+	}
+
+	[Symbol.dispose](): void {
+		this.dispose();
+	}
+
+	#arm(delayMs: number): void {
+		const timer = setTimeout(() => this.#onExpire(), Math.max(0, delayMs));
+		// Never keep the event loop alive for the watchdog itself.
+		timer.unref?.();
+		this.#timer = timer;
+	}
+
+	#onExpire(): void {
+		if (this.#settled || this.#pauseDepth > 0) return;
+		const remainingMs = this.#deadlineMs - Date.now();
+		if (remainingMs > 0) {
+			// The deadline moved forward (resume re-arming) after this timer was
+			// armed; wait out the remaining window instead of firing early.
+			this.#arm(remainingMs);
+			return;
+		}
+		this.#settled = true;
+		this.#timer = undefined;
+		this.#controller.abort(new DOMException(`Idle for ${Math.round(this.#idleMs / 1000)}s`, "TimeoutError"));
+	}
+}

package/src/eval/index.ts

@@ -0,0 +1,4 @@
+export * from "./backend";
+export { default as jsBackend } from "./js";
+export { default as pythonBackend } from "./py";
+export * from "./types";