@oh-my-pi/pi-coding-agent 17.1.5 → 17.1.6

package/src/eval/__tests__/bridge-timeout.test.ts

@@ -1,170 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import {
-	EVAL_TIMEOUT_PAUSE_OP,
-	EVAL_TIMEOUT_RESUME_OP,
-	isEvalTimeoutControlEvent,
-	withBridgeTimeoutPause,
-} from "../bridge-timeout";
-import { executeWithKernelBase, type GenericKernel } from "../executor-base";
-import type { JsStatusEvent } from "../js/shared/types";
-import type { KernelDisplayOutput } from "../py/display";
-
-describe("withBridgeTimeoutPause", () => {
-	it("emits one pause before the operation and one resume after it settles", async () => {
-		const events: JsStatusEvent[] = [];
-
-		const value = await withBridgeTimeoutPause(
-			event => events.push(event),
-			async () => {
-				await Bun.sleep(80);
-				return "done";
-			},
-			{ deferExternalAbort: true },
-		);
-
-		expect(value).toBe("done");
-		expect(events.map(event => event.op)).toEqual([EVAL_TIMEOUT_PAUSE_OP, EVAL_TIMEOUT_RESUME_OP]);
-		expect(events.every(event => event.deferExternalAbort === true)).toBe(true);
-
-		const settledCount = events.length;
-		await Bun.sleep(40);
-		expect(events.length).toBe(settledCount);
-	});
-
-	it("resumes timeout accounting even when the operation throws", async () => {
-		const events: JsStatusEvent[] = [];
-
-		await expect(
-			withBridgeTimeoutPause(
-				event => events.push(event),
-				async () => {
-					await Bun.sleep(20);
-					throw new Error("boom");
-				},
-			),
-		).rejects.toThrow("boom");
-
-		expect(events.map(event => event.op)).toEqual([EVAL_TIMEOUT_PAUSE_OP, EVAL_TIMEOUT_RESUME_OP]);
-	});
-
-	it("runs the operation without emitting when no status sink is wired", async () => {
-		let ran = 0;
-
-		const value = await withBridgeTimeoutPause(undefined, async () => {
-			ran++;
-			await Bun.sleep(20);
-			return 42;
-		});
-
-		expect(value).toBe(42);
-		expect(ran).toBe(1);
-	});
-
-	it("identifies timeout-control events as non-renderable status", () => {
-		expect(isEvalTimeoutControlEvent({ op: EVAL_TIMEOUT_PAUSE_OP })).toBe(true);
-		expect(isEvalTimeoutControlEvent({ op: EVAL_TIMEOUT_RESUME_OP })).toBe(true);
-		expect(isEvalTimeoutControlEvent({ op: "agent", id: "subagent-1" })).toBe(false);
-	});
-});
-
-class TestCancelledError extends Error {
-	readonly timedOut: boolean;
-
-	constructor(timedOut: boolean) {
-		super(timedOut ? "timed out" : "cancelled");
-		this.name = "TestCancelledError";
-		this.timedOut = timedOut;
-	}
-}
-
-it("defers external aborts until an in-flight agent bridge call resumes", async () => {
-	const abortController = new AbortController();
-	const entered = Promise.withResolvers<void>();
-	const triggerAbort = Promise.withResolvers<void>();
-	const observed = Promise.withResolvers<boolean>();
-	const release = Promise.withResolvers<void>();
-	const kernel: GenericKernel<Record<string, string | null>> = {
-		async execute(_code, options) {
-			entered.resolve();
-			await triggerAbort.promise;
-			options.onDisplay({
-				type: "status",
-				event: { op: EVAL_TIMEOUT_PAUSE_OP, deferExternalAbort: true },
-			} satisfies KernelDisplayOutput);
-			abortController.abort(new Error("external interrupt"));
-			observed.resolve(options.signal?.aborted ?? false);
-			await release.promise;
-			options.onDisplay({
-				type: "status",
-				event: { op: EVAL_TIMEOUT_RESUME_OP, deferExternalAbort: true },
-			} satisfies KernelDisplayOutput);
-			return { status: "ok", cancelled: false, timedOut: false };
-		},
-	};
-
-	const resultPromise = executeWithKernelBase({
-		kernel,
-		code: "agent('slow')",
-		options: { signal: abortController.signal },
-		runIdPrefix: "test",
-		errorLogLabel: "test",
-		cancelledErrorClass: TestCancelledError,
-		buildKernelEnvPatch: () => ({}),
-		formatKernelTimeoutAnnotation: () => "kernel timed out",
-		formatTimeoutAnnotation: () => "timed out",
-	});
-
-	await entered.promise;
-	triggerAbort.resolve();
-	expect(await observed.promise).toBe(false);
-	release.resolve();
-	const result = await resultPromise;
-	expect(result.cancelled).toBe(true);
-	expect(result.exitCode).toBeUndefined();
-});
-
-it("does not defer external aborts for a completion bridge call", async () => {
-	const abortController = new AbortController();
-	const entered = Promise.withResolvers<void>();
-	const triggerAbort = Promise.withResolvers<void>();
-	const observed = Promise.withResolvers<boolean>();
-	const release = Promise.withResolvers<void>();
-	const kernel: GenericKernel<Record<string, string | null>> = {
-		async execute(_code, options) {
-			entered.resolve();
-			await triggerAbort.promise;
-			options.onDisplay({
-				type: "status",
-				event: { op: EVAL_TIMEOUT_PAUSE_OP },
-			} satisfies KernelDisplayOutput);
-			abortController.abort(new Error("external interrupt"));
-			observed.resolve(options.signal?.aborted ?? false);
-			await release.promise;
-			options.onDisplay({
-				type: "status",
-				event: { op: EVAL_TIMEOUT_RESUME_OP },
-			} satisfies KernelDisplayOutput);
-			return { status: "ok", cancelled: false, timedOut: false };
-		},
-	};
-
-	const resultPromise = executeWithKernelBase({
-		kernel,
-		code: "completion('slow')",
-		options: { signal: abortController.signal },
-		runIdPrefix: "test",
-		errorLogLabel: "test",
-		cancelledErrorClass: TestCancelledError,
-		buildKernelEnvPatch: () => ({}),
-		formatKernelTimeoutAnnotation: () => "kernel timed out",
-		formatTimeoutAnnotation: () => "timed out",
-	});
-
-	await entered.promise;
-	triggerAbort.resolve();
-	expect(await observed.promise).toBe(true);
-	release.resolve();
-	const result = await resultPromise;
-	expect(result.cancelled).toBe(true);
-	expect(result.exitCode).toBeUndefined();
-});

package/src/eval/__tests__/budget-bridge.test.ts

@@ -1,80 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import type { GoalModeState } from "../../goals/state";
-import type { UsageStatistics } from "../../session/session-entries";
-import type { ToolSession } from "../../tools";
-import { runEvalBudget } from "../budget-bridge";
-
-type TurnBudget = { total: number | null; spent: number; hard: boolean };
-
-function makeSession(parts: { turn?: TurnBudget; goal?: GoalModeState; usage?: UsageStatistics }): ToolSession {
-	return {
-		getTurnBudget: parts.turn ? () => parts.turn as TurnBudget : undefined,
-		getGoalModeState: parts.goal ? () => parts.goal : undefined,
-		getUsageStatistics: parts.usage ? () => parts.usage as UsageStatistics : undefined,
-	} as unknown as ToolSession;
-}
-
-function goalState(extra: Partial<GoalModeState["goal"]>): GoalModeState {
-	return {
-		enabled: true,
-		mode: "active",
-		goal: { id: "g1", status: "active", tokensUsed: 0, timeUsedSeconds: 0, ...extra },
-	} as GoalModeState;
-}
-
-function usage(output: number): UsageStatistics {
-	return {
-		input: 0,
-		output,
-		cacheRead: 0,
-		cacheWrite: 0,
-		totalTokens: output,
-		orchestrationInput: 0,
-		orchestrationOutput: 0,
-		orchestrationCacheRead: 0,
-		premiumRequests: 0,
-		cost: 0,
-	};
-}
-
-describe("runEvalBudget", () => {
-	it("prefers an active +Nk turn directive over Goal Mode", async () => {
-		const session = makeSession({
-			turn: { total: 200_000, spent: 5_000, hard: true },
-			goal: goalState({ tokenBudget: 100_000, tokensUsed: 4_200 }),
-		});
-		expect(await runEvalBudget({}, { session })).toEqual({ total: 200_000, spent: 5_000, hard: true });
-	});
-
-	it("reports an advisory turn budget as hard:false", async () => {
-		const session = makeSession({ turn: { total: 50_000, spent: 1_000, hard: false } });
-		expect(await runEvalBudget({}, { session })).toEqual({ total: 50_000, spent: 1_000, hard: false });
-	});
-
-	it("falls through to Goal Mode when no turn directive set a ceiling", async () => {
-		const session = makeSession({
-			turn: { total: null, spent: 7_777, hard: false },
-			goal: goalState({ tokenBudget: 100_000, tokensUsed: 4_200 }),
-		});
-		expect(await runEvalBudget({}, { session })).toEqual({ total: 100_000, spent: 4_200, hard: true });
-	});
-
-	it("treats a Goal Mode budget as hard, and a budgetless goal as no ceiling", async () => {
-		const withBudget = makeSession({ goal: goalState({ tokenBudget: 80_000, tokensUsed: 9_000 }) });
-		expect(await runEvalBudget({}, { session: withBudget })).toEqual({ total: 80_000, spent: 9_000, hard: true });
-
-		const noBudget = makeSession({ goal: goalState({ tokenBudget: undefined, tokensUsed: 1_234 }) });
-		expect(await runEvalBudget({}, { session: noBudget })).toEqual({ total: null, spent: 1_234, hard: false });
-	});
-
-	it("reports no ceiling but still surfaces spend", async () => {
-		const fromTurn = makeSession({ turn: { total: null, spent: 333, hard: false } });
-		expect(await runEvalBudget({}, { session: fromTurn })).toEqual({ total: null, spent: 333, hard: false });
-
-		const fromUsage = makeSession({ usage: usage(777) });
-		expect(await runEvalBudget({}, { session: fromUsage })).toEqual({ total: null, spent: 777, hard: false });
-
-		const empty = makeSession({});
-		expect(await runEvalBudget({}, { session: empty })).toEqual({ total: null, spent: 0, hard: false });
-	});
-});

package/src/eval/__tests__/completion-bridge.test.ts

@@ -1,412 +0,0 @@
-import { afterAll, afterEach, describe, expect, it, vi } from "bun:test";
-import * as path from "node:path";
-import type { Api, AssistantMessage, Model } from "@oh-my-pi/pi-ai";
-import * as ai from "@oh-my-pi/pi-ai";
-import { Effort } from "@oh-my-pi/pi-ai";
-import { TempDir } from "@oh-my-pi/pi-utils";
-import { $ } from "bun";
-import type { ModelRegistry } from "../../config/model-registry";
-import { Settings } from "../../config/settings";
-import type { ToolSession } from "../../tools";
-import { ToolError } from "../../tools/tool-errors";
-import { EVAL_TIMEOUT_PAUSE_OP, EVAL_TIMEOUT_RESUME_OP } from "../bridge-timeout";
-import { runEvalCompletion } from "../completion-bridge";
-import { IdleTimeout } from "../idle-timeout";
-import { disposeAllVmContexts } from "../js/context-manager";
-import { executeJs } from "../js/executor";
-import { disposeAllKernelSessions, type PythonResult } from "../py/executor";
-
-function makeModel(provider: string, id: string, extra: Partial<Model<Api>> = {}): Model<Api> {
-	return {
-		id,
-		name: id,
-		api: "openai-responses",
-		provider,
-		baseUrl: "https://example.test/v1",
-		reasoning: false,
-		input: ["text"],
-		cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 1 },
-		contextWindow: 128000,
-		maxTokens: 4096,
-		...extra,
-	} as Model<Api>;
-}
-
-const SMOL = makeModel("p", "smol");
-const DEFAULT = makeModel("p", "default");
-const SLOW = makeModel("p", "slow");
-const REASONING_SLOW = makeModel("p", "slow", {
-	api: "anthropic-messages",
-	reasoning: true,
-	thinking: { efforts: [Effort.Low, Effort.Medium, Effort.High], mode: "anthropic-adaptive" },
-});
-
-interface SessionOptions {
-	available?: Model<Api>[];
-	apiKey?: string | null;
-	activeModel?: string;
-	roles?: Partial<Record<"smol" | "default" | "slow", string>>;
-}
-
-function makeSession(opts: SessionOptions = {}): ToolSession {
-	const settings = Settings.isolated({ "async.enabled": false, "task.isolation.mode": "none" });
-	const roles = opts.roles ?? { smol: "p/smol", slow: "p/slow" };
-	for (const role in roles) {
-		const value = roles[role as keyof typeof roles];
-		if (value) settings.setModelRole(role, value);
-	}
-	const modelRegistry = {
-		getAvailable: () => opts.available ?? [SMOL, DEFAULT, SLOW],
-		getApiKey: async () => (opts.apiKey === undefined ? "test-key" : opts.apiKey),
-		resolver: () => async () => (opts.apiKey === undefined ? "test-key" : opts.apiKey),
-	} as unknown as ModelRegistry;
-	return {
-		settings,
-		modelRegistry,
-		getActiveModelString: () => opts.activeModel ?? "p/default",
-	} as unknown as ToolSession;
-}
-
-function assistant(opts: {
-	text?: string;
-	toolCall?: { name: string; arguments: Record<string, unknown> };
-	stopReason?: AssistantMessage["stopReason"];
-	errorMessage?: string;
-}): AssistantMessage {
-	const content: AssistantMessage["content"] = [];
-	if (opts.text) content.push({ type: "text", text: opts.text });
-	if (opts.toolCall) {
-		content.push({ type: "toolCall", id: "tc-1", name: opts.toolCall.name, arguments: opts.toolCall.arguments });
-	}
-	return {
-		role: "assistant",
-		content,
-		api: "openai-responses",
-		provider: "p",
-		model: "default",
-		usage: {
-			input: 0,
-			output: 0,
-			cacheRead: 0,
-			cacheWrite: 0,
-			totalTokens: 0,
-			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
-		},
-		stopReason: opts.stopReason ?? "stop",
-		errorMessage: opts.errorMessage,
-		timestamp: Date.now(),
-	};
-}
-
-async function runPythonCompletionInSubprocess(options: {
-	structured: boolean;
-	tempDir: TempDir;
-}): Promise<PythonResult> {
-	const repoRoot = path.resolve(import.meta.dir, "../../../..");
-	const scriptPath = path.join(options.tempDir.path(), "run-python-completion.ts");
-	const resultPath = path.join(options.tempDir.path(), "python-completion-result.json");
-	const aiPath = path.resolve(import.meta.dir, "../../../../ai/src/index.ts");
-	const executorPath = path.resolve(import.meta.dir, "../py/executor.ts");
-	const settingsPath = path.resolve(import.meta.dir, "../../config/settings.ts");
-	const code = options.structured
-		? 'import json\nprint(json.dumps(completion("hi", schema={"type": "object"})))'
-		: 'print(completion("hi", model="smol"))';
-	const responseContent = options.structured
-		? '[{ type: "toolCall", id: "tc-1", name: "respond", arguments: { ok: true } }]'
-		: '[{ type: "text", text: "hello from python" }]';
-	await Bun.write(
-		scriptPath,
-		`
-import { vi } from "bun:test";
-import * as ai from ${JSON.stringify(aiPath)};
-import { executePython } from ${JSON.stringify(executorPath)};
-import { Settings } from ${JSON.stringify(settingsPath)};
-
-const SMOL = {
-	id: "smol",
-	name: "smol",
-	api: "openai-responses",
-	provider: "p",
-	baseUrl: "https://example.test/v1",
-	reasoning: false,
-	input: ["text"],
-	cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 1 },
-	contextWindow: 128000,
-	maxTokens: 4096,
-};
-const settings = Settings.isolated({ "async.enabled": false, "task.isolation.mode": "none" });
-settings.setModelRole("smol", "p/smol");
-settings.setModelRole("slow", "p/slow");
-const session = {
-	settings,
-	modelRegistry: {
-		getAvailable: () => [SMOL],
-		getApiKey: async () => "test-key",
-		resolver: () => async () => "test-key",
-	},
-	getActiveModelString: () => "p/smol",
-};
-vi.spyOn(ai, "completeSimple").mockResolvedValue({
-	role: "assistant",
-	api: "openai-responses",
-	provider: "p",
-	model: "smol",
-	stopReason: "stop",
-	content: ${responseContent},
-});
-const result = await executePython(${JSON.stringify(code)}, {
-	cwd: ${JSON.stringify(options.tempDir.path())},
-	sessionId: ${JSON.stringify(`py-completion:${options.structured ? "struct" : "plain"}`)},
-	sessionFile: ${JSON.stringify(path.join(options.tempDir.path(), "session.jsonl"))},
-	toolSession: session,
-	kernelMode: "per-call",
-});
-await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify(result));
-process.exit(0);
-`,
-	);
-	const child = await $`bun ${scriptPath}`.cwd(repoRoot).quiet().nothrow();
-	const stdout = child.stdout.toString();
-	const stderr = child.stderr.toString();
-	if (child.exitCode !== 0)
-		throw new Error(stderr || stdout || `Python completion subprocess exited with ${child.exitCode}`);
-	return (await Bun.file(resultPath).json()) as PythonResult;
-}
-
-describe("runEvalCompletion", () => {
-	afterEach(() => {
-		vi.restoreAllMocks();
-	});
-
-	it("resolves each tier to its expected model", async () => {
-		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
-		const session = makeSession();
-
-		await runEvalCompletion({ prompt: "q", model: "smol" }, { session });
-		await runEvalCompletion({ prompt: "q", model: "default" }, { session });
-		await runEvalCompletion({ prompt: "q", model: "slow" }, { session });
-
-		const resolved = spy.mock.calls.map(call => {
-			const model = call[0] as Model<Api>;
-			return `${model.provider}/${model.id}`;
-		});
-		expect(resolved).toEqual(["p/smol", "p/default", "p/slow"]);
-	});
-
-	it("prefers the session active model for the default tier, falling back to @default", async () => {
-		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
-		const session = makeSession({ available: [SMOL, DEFAULT, SLOW], activeModel: "p/slow" });
-
-		await runEvalCompletion({ prompt: "q", model: "default" }, { session });
-
-		const model = spy.mock.calls[0]?.[0] as Model<Api>;
-		expect(`${model.provider}/${model.id}`).toBe("p/slow");
-	});
-
-	it("returns the completion text in plain mode", async () => {
-		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "the answer" }));
-		const result = await runEvalCompletion({ prompt: "q", model: "smol" }, { session: makeSession() });
-		expect(result.text).toBe("the answer");
-		expect(result.details).toEqual({ model: "p/smol", tier: "smol", structured: false });
-	});
-
-	it("supplies a non-empty systemPrompt when system is omitted (codex 'Instructions are required' guard)", async () => {
-		// The openai-codex Responses transformer drops `instructions` when no
-		// system prompt is provided, and the remote endpoint then 400s with
-		// "Instructions are required". runEvalCompletion must always carry a non-empty
-		// systemPrompt so `completion("…")` without a `system` argument works.
-		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
-		await runEvalCompletion({ prompt: "q", model: "smol" }, { session: makeSession() });
-		const ctx = spy.mock.calls[0]?.[1] as { systemPrompt?: string[] };
-		expect(ctx.systemPrompt).toBeDefined();
-		expect(ctx.systemPrompt?.length).toBeGreaterThan(0);
-		expect(ctx.systemPrompt?.[0]).toMatch(/.+/);
-	});
-
-	it("honors an explicit system prompt instead of overriding it", async () => {
-		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
-		await runEvalCompletion({ prompt: "q", model: "smol", system: "Be terse." }, { session: makeSession() });
-		const ctx = spy.mock.calls[0]?.[1] as { systemPrompt?: string[] };
-		expect(ctx.systemPrompt).toEqual(["Be terse."]);
-	});
-
-	it("forces a respond tool call and returns its arguments in structured mode", async () => {
-		const spy = vi
-			.spyOn(ai, "completeSimple")
-			.mockResolvedValue(assistant({ toolCall: { name: "respond", arguments: { answer: 42 } } }));
-		const result = await runEvalCompletion(
-			{ prompt: "q", model: "smol", schema: { type: "object", properties: { answer: { type: "number" } } } },
-			{ session: makeSession() },
-		);
-
-		expect(JSON.parse(result.text)).toEqual({ answer: 42 });
-		expect(result.details.structured).toBe(true);
-
-		const ctx = spy.mock.calls[0]?.[1] as { tools?: Array<{ name: string }> };
-		const opts = spy.mock.calls[0]?.[2] as { toolChoice?: unknown };
-		expect(ctx.tools?.[0]?.name).toBe("respond");
-		expect(opts.toolChoice).toEqual({ type: "tool", name: "respond" });
-	});
-
-	it("falls back to JSON embedded in text when the model skips the respond tool", async () => {
-		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: 'here: {"answer": 7}' }));
-		const result = await runEvalCompletion(
-			{ prompt: "q", model: "smol", schema: { type: "object" } },
-			{ session: makeSession() },
-		);
-		expect(JSON.parse(result.text)).toEqual({ answer: 7 });
-	});
-
-	it("requests reasoning only for the slow tier on a reasoning-capable model", async () => {
-		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
-		const session = makeSession({ available: [SMOL, DEFAULT, REASONING_SLOW] });
-
-		await runEvalCompletion({ prompt: "q", model: "smol" }, { session });
-		await runEvalCompletion({ prompt: "q", model: "slow" }, { session });
-
-		const smolOpts = spy.mock.calls[0]?.[2] as { reasoning?: unknown };
-		const slowOpts = spy.mock.calls[1]?.[2] as { reasoning?: unknown };
-		expect(smolOpts.reasoning).toBeUndefined();
-		expect(slowOpts.reasoning).toBe(Effort.High);
-	});
-
-	it("does not request reasoning for the slow tier on a non-reasoning model", async () => {
-		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
-		// SLOW is reasoning:false — must not trip requireSupportedEffort downstream.
-		const result = await runEvalCompletion({ prompt: "q", model: "slow" }, { session: makeSession() });
-		expect(result.text).toBe("ok");
-		const opts = spy.mock.calls[0]?.[2] as { reasoning?: unknown };
-		expect(opts.reasoning).toBeUndefined();
-	});
-
-	it("throws ToolError on invalid arguments", async () => {
-		await expect(runEvalCompletion({ prompt: "" }, { session: makeSession() })).rejects.toBeInstanceOf(ToolError);
-		await expect(
-			runEvalCompletion({ prompt: "q", model: "huge" }, { session: makeSession() }),
-		).rejects.toBeInstanceOf(ToolError);
-	});
-
-	it("throws ToolError when no model resolves for the tier", async () => {
-		const session = makeSession({ available: [DEFAULT], roles: { smol: "missing/model" } });
-		await expect(runEvalCompletion({ prompt: "q", model: "smol" }, { session })).rejects.toBeInstanceOf(ToolError);
-	});
-
-	it("throws ToolError when the resolved model has no API key", async () => {
-		const session = makeSession({ apiKey: null });
-		await expect(runEvalCompletion({ prompt: "q", model: "smol" }, { session })).rejects.toBeInstanceOf(ToolError);
-	});
-
-	it("maps error and aborted stop reasons to ToolError", async () => {
-		vi.spyOn(ai, "completeSimple").mockResolvedValueOnce(assistant({ stopReason: "error", errorMessage: "boom" }));
-		await expect(runEvalCompletion({ prompt: "q", model: "smol" }, { session: makeSession() })).rejects.toThrow(
-			"boom",
-		);
-
-		vi.spyOn(ai, "completeSimple").mockResolvedValueOnce(assistant({ stopReason: "aborted" }));
-		await expect(
-			runEvalCompletion({ prompt: "q", model: "smol" }, { session: makeSession() }),
-		).rejects.toBeInstanceOf(ToolError);
-	});
-
-	it("throws ToolError when plain mode produces no text", async () => {
-		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "" }));
-		await expect(
-			runEvalCompletion({ prompt: "q", model: "smol" }, { session: makeSession() }),
-		).rejects.toBeInstanceOf(ToolError);
-	});
-
-	it("pauses the idle watchdog while a slow completion() request is in flight", async () => {
-		// A oneshot completion emits no status until it returns; delegated model
-		// time must be invisible to the eval timeout budget.
-		vi.spyOn(ai, "completeSimple").mockImplementation(async () => {
-			await Bun.sleep(200);
-			return assistant({ text: "the answer" });
-		});
-
-		const ops: string[] = [];
-		using idle = new IdleTimeout(60);
-		const result = await runEvalCompletion(
-			{ prompt: "q", model: "smol" },
-			{
-				session: makeSession(),
-				signal: idle.signal,
-				emitStatus: event => {
-					ops.push(event.op);
-					if (event.op === EVAL_TIMEOUT_PAUSE_OP) idle.pause();
-					if (event.op === EVAL_TIMEOUT_RESUME_OP) idle.resume();
-				},
-			},
-		);
-
-		expect(result.text).toBe("the answer");
-		expect(ops).toEqual([EVAL_TIMEOUT_PAUSE_OP, EVAL_TIMEOUT_RESUME_OP, "completion"]);
-		expect(idle.signal.aborted).toBe(false);
-	});
-});
-
-describe("completion() through eval runtimes", () => {
-	afterEach(() => {
-		vi.restoreAllMocks();
-	});
-
-	afterAll(async () => {
-		await disposeAllVmContexts();
-		await disposeAllKernelSessions();
-	});
-
-	it("exposes completion() in the JavaScript runtime", async () => {
-		using tempDir = TempDir.createSync("@omp-eval-completion-js-");
-		const sessionFile = path.join(tempDir.path(), "session.jsonl");
-		const sessionId = `js-completion:${crypto.randomUUID()}`;
-		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "hello from smol" }));
-
-		const result = await executeJs('return await completion("hi", { model: "smol" });', {
-			cwd: tempDir.path(),
-			sessionId,
-			session: makeSession(),
-			sessionFile,
-		});
-
-		expect(result.exitCode).toBe(0);
-		expect(result.output.trim()).toBe("hello from smol");
-	});
-
-	it("parses structured completion() output in the JavaScript runtime", async () => {
-		using tempDir = TempDir.createSync("@omp-eval-completion-js-struct-");
-		const sessionFile = path.join(tempDir.path(), "session.jsonl");
-		const sessionId = `js-completion-struct:${crypto.randomUUID()}`;
-		vi.spyOn(ai, "completeSimple").mockResolvedValue(
-			assistant({ toolCall: { name: "respond", arguments: { ok: true, n: 3 } } }),
-		);
-
-		const result = await executeJs(
-			'const r = await completion("hi", { schema: { type: "object" } }); return JSON.stringify(r);',
-			{ cwd: tempDir.path(), sessionId, session: makeSession(), sessionFile },
-		);
-
-		expect(result.exitCode).toBe(0);
-		expect(JSON.parse(result.output.trim())).toEqual({ ok: true, n: 3 });
-	});
-
-	it("exposes completion() in the Python runtime", async () => {
-		const tempDir = TempDir.createSync("@omp-eval-completion-py-");
-		try {
-			const result = await runPythonCompletionInSubprocess({ structured: false, tempDir });
-			expect(result.exitCode).toBe(0);
-			expect(result.output.trim()).toBe("hello from python");
-		} finally {
-			tempDir.removeSync();
-		}
-	});
-
-	it("parses structured completion() output in the Python runtime", async () => {
-		const tempDir = TempDir.createSync("@omp-eval-completion-py-struct-");
-		try {
-			const result = await runPythonCompletionInSubprocess({ structured: true, tempDir });
-			expect(result.exitCode).toBe(0);
-			expect(JSON.parse(result.output.trim())).toEqual({ ok: true });
-		} finally {
-			tempDir.removeSync();
-		}
-	});
-});