@oh-my-pi/pi-coding-agent 15.5.13 → 15.5.15

package/dist/types/extensibility/plugins/legacy-pi-compat.d.ts

@@ -1,3 +1,18 @@
+import * as path from "node:path";
+/**
+ * Compute the bunfs package root from the compiled binary's `import.meta.dir`
+ * (or any stand-in supplied by tests). Bun 1.3 reports the bunfs mount root
+ * (`/$bunfs/root` or `<drive>:\~BUN\root`) for imported modules as well as the
+ * entrypoint, so the normal path is `<root>/packages`.
+ *
+ * The suffix branch preserves correctness if a future Bun release switches to
+ * module-specific `import.meta.dir` values inside compiled binaries, matching
+ * the source layout:
+ * `<bunfs>/packages/coding-agent/src/extensibility/plugins`.
+ *
+ * Exported for tests; production callers use `BUNFS_PACKAGE_ROOT` below.
+ */
+export declare function __computeBunfsPackageRoot(metaDir: string, pathImpl?: typeof path): string;
 export declare function loadLegacyPiModule(resolvedPath: string): Promise<unknown>;
 export declare function installLegacyPiSpecifierShim(): void;
 /** Test seam: clears the memoized canonical specifier resolutions. */

package/dist/types/modes/theme/theme.d.ts

@@ -6,7 +6,7 @@ export type SymbolPreset = "unicode" | "nerd" | "ascii";
 /**
  * All available symbol keys organized by category.
  */
-export type SymbolKey = "status.success" | "status.error" | "status.warning" | "status.info" | "status.pending" | "status.disabled" | "status.enabled" | "status.running" | "status.shadowed" | "status.aborted" | "nav.cursor" | "nav.selected" | "nav.expand" | "nav.collapse" | "nav.back" | "tree.branch" | "tree.last" | "tree.vertical" | "tree.horizontal" | "tree.hook" | "boxRound.topLeft" | "boxRound.topRight" | "boxRound.bottomLeft" | "boxRound.bottomRight" | "boxRound.horizontal" | "boxRound.vertical" | "boxSharp.topLeft" | "boxSharp.topRight" | "boxSharp.bottomLeft" | "boxSharp.bottomRight" | "boxSharp.horizontal" | "boxSharp.vertical" | "boxSharp.cross" | "boxSharp.teeDown" | "boxSharp.teeUp" | "boxSharp.teeRight" | "boxSharp.teeLeft" | "sep.powerline" | "sep.powerlineThin" | "sep.powerlineLeft" | "sep.powerlineRight" | "sep.powerlineThinLeft" | "sep.powerlineThinRight" | "sep.block" | "sep.space" | "sep.asciiLeft" | "sep.asciiRight" | "sep.dot" | "sep.slash" | "sep.pipe" | "icon.model" | "icon.plan" | "icon.goal" | "icon.pause" | "icon.loop" | "icon.folder" | "icon.scratchFolder" | "icon.file" | "icon.git" | "icon.branch" | "icon.pr" | "icon.tokens" | "icon.context" | "icon.cost" | "icon.time" | "icon.pi" | "icon.agents" | "icon.cache" | "icon.input" | "icon.output" | "icon.host" | "icon.session" | "icon.package" | "icon.warning" | "icon.rewind" | "icon.auto" | "icon.fast" | "icon.extensionSkill" | "icon.extensionTool" | "icon.extensionSlashCommand" | "icon.extensionMcp" | "icon.extensionRule" | "icon.extensionHook" | "icon.extensionPrompt" | "icon.extensionContextFile" | "icon.extensionInstruction" | "icon.mic" | "thinking.minimal" | "thinking.low" | "thinking.medium" | "thinking.high" | "thinking.xhigh" | "checkbox.checked" | "checkbox.unchecked" | "format.bullet" | "format.dash" | "format.bracketLeft" | "format.bracketRight" | "md.quoteBorder" | "md.hrChar" | "md.bullet" | "lang.default" | "lang.typescript" | "lang.javascript" | "lang.python" | "lang.rust" | "lang.go" | "lang.java" | "lang.c" | "lang.cpp" | "lang.csharp" | "lang.ruby" | "lang.php" | "lang.swift" | "lang.kotlin" | "lang.shell" | "lang.html" | "lang.css" | "lang.json" | "lang.yaml" | "lang.markdown" | "lang.sql" | "lang.docker" | "lang.lua" | "lang.text" | "lang.env" | "lang.toml" | "lang.xml" | "lang.ini" | "lang.conf" | "lang.log" | "lang.csv" | "lang.tsv" | "lang.image" | "lang.pdf" | "lang.archive" | "lang.binary" | "tab.appearance" | "tab.model" | "tab.interaction" | "tab.context" | "tab.editing" | "tab.tools" | "tab.memory" | "tab.tasks" | "tab.providers";
+export type SymbolKey = "status.success" | "status.error" | "status.warning" | "status.info" | "status.pending" | "status.disabled" | "status.enabled" | "status.running" | "status.shadowed" | "status.aborted" | "nav.cursor" | "nav.selected" | "nav.expand" | "nav.collapse" | "nav.back" | "tree.branch" | "tree.last" | "tree.vertical" | "tree.horizontal" | "tree.hook" | "boxRound.topLeft" | "boxRound.topRight" | "boxRound.bottomLeft" | "boxRound.bottomRight" | "boxRound.horizontal" | "boxRound.vertical" | "boxSharp.topLeft" | "boxSharp.topRight" | "boxSharp.bottomLeft" | "boxSharp.bottomRight" | "boxSharp.horizontal" | "boxSharp.vertical" | "boxSharp.cross" | "boxSharp.teeDown" | "boxSharp.teeUp" | "boxSharp.teeRight" | "boxSharp.teeLeft" | "sep.powerline" | "sep.powerlineThin" | "sep.powerlineLeft" | "sep.powerlineRight" | "sep.powerlineThinLeft" | "sep.powerlineThinRight" | "sep.block" | "sep.space" | "sep.asciiLeft" | "sep.asciiRight" | "sep.dot" | "sep.slash" | "sep.pipe" | "icon.model" | "icon.plan" | "icon.goal" | "icon.pause" | "icon.loop" | "icon.folder" | "icon.scratchFolder" | "icon.file" | "icon.git" | "icon.branch" | "icon.pr" | "icon.tokens" | "icon.context" | "icon.cost" | "icon.time" | "icon.pi" | "icon.agents" | "icon.cache" | "icon.input" | "icon.output" | "icon.host" | "icon.session" | "icon.package" | "icon.warning" | "icon.rewind" | "icon.auto" | "icon.fast" | "icon.extensionSkill" | "icon.extensionTool" | "icon.extensionSlashCommand" | "icon.extensionMcp" | "icon.extensionRule" | "icon.extensionHook" | "icon.extensionPrompt" | "icon.extensionContextFile" | "icon.extensionInstruction" | "icon.mic" | "thinking.minimal" | "thinking.low" | "thinking.medium" | "thinking.high" | "thinking.xhigh" | "checkbox.checked" | "checkbox.unchecked" | "format.bullet" | "format.dash" | "format.bracketLeft" | "format.bracketRight" | "md.quoteBorder" | "md.hrChar" | "md.bullet" | "md.colorSwatch" | "lang.default" | "lang.typescript" | "lang.javascript" | "lang.python" | "lang.rust" | "lang.go" | "lang.java" | "lang.c" | "lang.cpp" | "lang.csharp" | "lang.ruby" | "lang.php" | "lang.swift" | "lang.kotlin" | "lang.shell" | "lang.html" | "lang.css" | "lang.json" | "lang.yaml" | "lang.markdown" | "lang.sql" | "lang.docker" | "lang.lua" | "lang.text" | "lang.env" | "lang.toml" | "lang.xml" | "lang.ini" | "lang.conf" | "lang.log" | "lang.csv" | "lang.tsv" | "lang.image" | "lang.pdf" | "lang.archive" | "lang.binary" | "tab.appearance" | "tab.model" | "tab.interaction" | "tab.context" | "tab.editing" | "tab.tools" | "tab.memory" | "tab.tasks" | "tab.providers";
 export type SpinnerType = "status" | "activity";
 export type ThemeColor = "accent" | "border" | "borderAccent" | "borderMuted" | "success" | "error" | "warning" | "muted" | "dim" | "text" | "thinkingText" | "userMessageText" | "customMessageText" | "customMessageLabel" | "toolTitle" | "toolOutput" | "mdHeading" | "mdLink" | "mdLinkUrl" | "mdCode" | "mdCodeBlock" | "mdCodeBlockBorder" | "mdQuote" | "mdQuoteBorder" | "mdHr" | "mdListBullet" | "toolDiffAdded" | "toolDiffRemoved" | "toolDiffContext" | "syntaxComment" | "syntaxKeyword" | "syntaxFunction" | "syntaxVariable" | "syntaxString" | "syntaxNumber" | "syntaxType" | "syntaxOperator" | "syntaxPunctuation" | "thinkingOff" | "thinkingMinimal" | "thinkingLow" | "thinkingMedium" | "thinkingHigh" | "thinkingXhigh" | "bashMode" | "pythonMode" | "statusLineSep" | "statusLineModel" | "statusLinePath" | "statusLineGitClean" | "statusLineGitDirty" | "statusLineContext" | "statusLineSpend" | "statusLineStaged" | "statusLineDirty" | "statusLineUntracked" | "statusLineOutput" | "statusLineCost" | "statusLineSubagents";
 /** Check if a string is a valid ThemeColor value */
@@ -165,6 +165,7 @@ export declare class Theme {
         quoteBorder: string;
         hrChar: string;
         bullet: string;
+        colorSwatch: string;
     };
     /**
      * Default spinner frames (status spinner).

package/dist/types/session/agent-session.d.ts

@@ -262,6 +262,8 @@ export interface SessionStats {
     premiumRequests: number;
     cost: number;
 }
+export declare const ANTHROPIC_TOOL_CALL_BATCH_CAP = 4;
+export declare function resolveToolCallBatchCapForModel(model: Model | undefined): number | undefined;
 export declare class AgentSession {
     #private;
     readonly agent: Agent;

package/dist/types/tools/index.d.ts

@@ -31,7 +31,6 @@ export * from "./ast-edit";
 export * from "./ast-grep";
 export * from "./bash";
 export * from "./browser";
-export * from "./calculator";
 export * from "./checkpoint";
 export * from "./debug";
 export * from "./eval";

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-coding-agent",
-	"version": "15.5.13",
+	"version": "15.5.15",
 	"description": "Coding agent CLI with read, bash, edit, write tools and session management",
 	"homepage": "https://omp.sh",
 	"author": "Can Boluk",
@@ -47,13 +47,13 @@
 		"@agentclientprotocol/sdk": "0.21.0",
 		"@babel/parser": "^7.29.3",
 		"@mozilla/readability": "^0.6.0",
-		"@oh-my-pi/hashline": "15.5.13",
-		"@oh-my-pi/omp-stats": "15.5.13",
-		"@oh-my-pi/pi-agent-core": "15.5.13",
-		"@oh-my-pi/pi-ai": "15.5.13",
-		"@oh-my-pi/pi-natives": "15.5.13",
-		"@oh-my-pi/pi-tui": "15.5.13",
-		"@oh-my-pi/pi-utils": "15.5.13",
+		"@oh-my-pi/hashline": "15.5.15",
+		"@oh-my-pi/omp-stats": "15.5.15",
+		"@oh-my-pi/pi-agent-core": "15.5.15",
+		"@oh-my-pi/pi-ai": "15.5.15",
+		"@oh-my-pi/pi-natives": "15.5.15",
+		"@oh-my-pi/pi-tui": "15.5.15",
+		"@oh-my-pi/pi-utils": "15.5.15",
 		"@puppeteer/browsers": "^2.13.0",
 		"@types/turndown": "5.0.6",
 		"@xterm/headless": "^6.0.0",

package/src/config/model-registry.ts

@@ -192,7 +192,7 @@ function validateProviderConfiguration(
 		}
 	}
 
-	if (mode === "models-config" && config.discovery && !config.api) {
+	if (mode === "models-config" && config.discovery && !config.api && config.discovery.type !== "proxy") {
 		throw new Error(`Provider ${providerName}: "api" is required when discovery is enabled at provider level.`);
 	}
 
@@ -1209,13 +1209,17 @@ export class ModelRegistry {
 				keylessProviders.add(providerName);
 			}
 
-			if (providerConfig.discovery && providerConfig.api) {
+			if (providerConfig.discovery && (providerConfig.api || providerConfig.discovery.type === "proxy")) {
+				const disableStrictCompat = providerConfig.disableStrictTools ? { disableStrictTools: true } : undefined;
 				discoverableProviders.push({
 					provider: providerName,
-					api: providerConfig.api as Api,
+					// Proxy discovery derives per-model api from /v1/models's
+					// supported_endpoint_types; the provider-level api is only a
+					// fallback for entries that don't advertise one.
+					api: (providerConfig.api ?? "openai-completions") as Api,
 					baseUrl: providerConfig.baseUrl,
 					headers: providerConfig.headers,
-					compat: providerConfig.compat,
+					compat: mergeCompat(providerConfig.compat, disableStrictCompat),
 					discovery: providerConfig.discovery,
 					optional: false,
 				});
@@ -1385,6 +1389,8 @@ export class ModelRegistry {
 			case "lm-studio":
 			case "openai-models-list":
 				return this.#discoverOpenAIModelsList(providerConfig);
+			case "proxy":
+				return this.#discoverProxyModels(providerConfig);
 		}
 	}
 
@@ -1711,7 +1717,7 @@ export class ModelRegistry {
 
 		const response = await fetch(modelsUrl, {
 			headers,
-			signal: AbortSignal.timeout(250),
+			signal: AbortSignal.timeout(10_000),
 		});
 		if (!response.ok) {
 			throw new Error(`HTTP ${response.status} from ${modelsUrl}`);
@@ -1746,6 +1752,84 @@ export class ModelRegistry {
 		return this.#applyProviderModelOverrides(providerConfig.provider, discovered);
 	}
 
+	/**
+	 * Discover models from an Anthropic+OpenAI-compatible reseller proxy that
+	 * exposes both `/v1/messages` and `/v1/chat/completions`, advertising each
+	 * model's wire capabilities through `supported_endpoint_types` on
+	 * `GET /v1/models` (new-api / one-api-style proxies).
+	 *
+	 * Routing per model:
+	 *   supported_endpoint_types: ["anthropic", ...] -> api: "anthropic-messages"
+	 *   supported_endpoint_types: ["openai"]         -> api: "openai-completions"
+	 *   missing / neither                            -> provider-level api fallback
+	 *
+	 * Anthropic models share the same baseUrl; the Anthropic SDK strips a
+	 * trailing `/v1` itself before appending `/v1/messages`, so the discovery
+	 * URL (which ends in `/v1`) round-trips correctly.
+	 */
+	async #discoverProxyModels(providerConfig: DiscoveryProviderConfig): Promise<Model<Api>[]> {
+		const baseUrl = this.#normalizeOpenAIModelsListBaseUrl(providerConfig.baseUrl);
+		const modelsUrl = `${baseUrl}/models`;
+
+		const headers: Record<string, string> = { ...(providerConfig.headers ?? {}) };
+		const apiKey = await this.authStorage.getApiKey(providerConfig.provider);
+		if (apiKey && apiKey !== DEFAULT_LOCAL_TOKEN && apiKey !== kNoAuth) {
+			headers.Authorization = `Bearer ${apiKey}`;
+		}
+
+		const response = await fetch(modelsUrl, {
+			headers,
+			signal: AbortSignal.timeout(10_000),
+		});
+		if (!response.ok) {
+			throw new Error(`HTTP ${response.status} from ${modelsUrl}`);
+		}
+		const payload = (await response.json()) as {
+			data?: Array<{ id?: string; supported_endpoint_types?: string[] }>;
+		};
+		const items = payload.data ?? [];
+		const discovered: Model<Api>[] = [];
+		for (const item of items) {
+			const id = item.id;
+			if (!id) continue;
+			const endpoints = item.supported_endpoint_types ?? [];
+			const api: Api | undefined = endpoints.includes("anthropic")
+				? "anthropic-messages"
+				: endpoints.includes("openai")
+					? "openai-completions"
+					: providerConfig.api;
+			if (!api) continue;
+			const isAnthropic = api === "anthropic-messages";
+			discovered.push(
+				enrichModelThinking({
+					id,
+					name: id,
+					api,
+					provider: providerConfig.provider,
+					baseUrl,
+					reasoning: false,
+					input: ["text"],
+					cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+					contextWindow: 128000,
+					maxTokens: 8192,
+					headers,
+					// OpenAI-compat fields are no-ops on anthropic models; the
+					// Anthropic SDK ignores them. Provider-level disableStrictTools
+					// flows in via #applyProviderCompat for the third-party-Anthropic
+					// path.
+					compat: isAnthropic
+						? undefined
+						: {
+								supportsStore: false,
+								supportsDeveloperRole: false,
+								supportsReasoningEffort: false,
+							},
+				}),
+			);
+		}
+		return this.#applyProviderModelOverrides(providerConfig.provider, discovered);
+	}
+
 	#normalizeLlamaCppBaseUrl(baseUrl?: string): string {
 		const defaultBaseUrl = "http://127.0.0.1:8080";
 		const raw = baseUrl || defaultBaseUrl;

package/src/config/models-config-schema.ts

@@ -121,7 +121,7 @@ export const ModelOverrideSchema = z.object({
 export type ModelOverride = z.infer<typeof ModelOverrideSchema>;
 
 export const ProviderDiscoverySchema = z.object({
-	type: z.enum(["ollama", "llama.cpp", "lm-studio", "openai-models-list"]),
+	type: z.enum(["ollama", "llama.cpp", "lm-studio", "openai-models-list", "proxy"]),
 });
 
 export const ProviderAuthSchema = z.enum(["apiKey", "none", "oauth"]);

package/src/config/settings-schema.ts

@@ -81,6 +81,7 @@ export type StatusLineSegmentId =
 	| "hostname"
 	| "cache_read"
 	| "cache_write"
+	| "cache_hit"
 	| "session_name"
 	| "usage";
 
@@ -2016,16 +2017,6 @@ export const SETTINGS_SCHEMA = {
 		},
 	},
 
-	"calc.enabled": {
-		type: "boolean",
-		default: false,
-		ui: {
-			tab: "tools",
-			label: "Calculator",
-			description: "Enable the calculator tool for basic calculations",
-		},
-	},
-
 	"tts.enabled": {
 		type: "boolean",
 		default: false,

package/src/eval/__tests__/llm-bridge.test.ts

@@ -0,0 +1,297 @@
+import { afterAll, afterEach, describe, expect, it, vi } from "bun:test";
+import * as path from "node:path";
+import type { Api, AssistantMessage, Model } from "@oh-my-pi/pi-ai";
+import * as ai from "@oh-my-pi/pi-ai";
+import { Effort } from "@oh-my-pi/pi-ai";
+import { TempDir } from "@oh-my-pi/pi-utils";
+import type { ModelRegistry } from "../../config/model-registry";
+import { Settings } from "../../config/settings";
+import type { ToolSession } from "../../tools";
+import { ToolError } from "../../tools/tool-errors";
+import { disposeAllVmContexts } from "../js/context-manager";
+import { executeJs } from "../js/executor";
+import { runEvalLlm } from "../llm-bridge";
+import { disposeAllKernelSessions, executePython } from "../py/executor";
+
+function makeModel(provider: string, id: string, extra: Partial<Model<Api>> = {}): Model<Api> {
+	return {
+		id,
+		name: id,
+		api: "openai-responses",
+		provider,
+		baseUrl: "https://example.test/v1",
+		reasoning: false,
+		input: ["text"],
+		cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 1 },
+		contextWindow: 128000,
+		maxTokens: 4096,
+		...extra,
+	} as Model<Api>;
+}
+
+const SMOL = makeModel("p", "smol");
+const DEFAULT = makeModel("p", "default");
+const SLOW = makeModel("p", "slow");
+const REASONING_SLOW = makeModel("p", "slow", {
+	api: "anthropic-messages",
+	reasoning: true,
+	thinking: { minLevel: Effort.Low, maxLevel: Effort.High, mode: "anthropic-adaptive" },
+});
+
+interface SessionOptions {
+	available?: Model<Api>[];
+	apiKey?: string | null;
+	activeModel?: string;
+	roles?: Partial<Record<"smol" | "default" | "slow", string>>;
+}
+
+function makeSession(opts: SessionOptions = {}): ToolSession {
+	const settings = Settings.isolated({ "async.enabled": false, "task.isolation.mode": "none" });
+	const roles = opts.roles ?? { smol: "p/smol", slow: "p/slow" };
+	for (const role in roles) {
+		const value = roles[role as keyof typeof roles];
+		if (value) settings.setModelRole(role, value);
+	}
+	const modelRegistry = {
+		getAvailable: () => opts.available ?? [SMOL, DEFAULT, SLOW],
+		getApiKey: async () => (opts.apiKey === undefined ? "test-key" : opts.apiKey),
+	} as unknown as ModelRegistry;
+	return {
+		settings,
+		modelRegistry,
+		getActiveModelString: () => opts.activeModel ?? "p/default",
+	} as unknown as ToolSession;
+}
+
+function assistant(opts: {
+	text?: string;
+	toolCall?: { name: string; arguments: Record<string, unknown> };
+	stopReason?: AssistantMessage["stopReason"];
+	errorMessage?: string;
+}): AssistantMessage {
+	const content: AssistantMessage["content"] = [];
+	if (opts.text) content.push({ type: "text", text: opts.text });
+	if (opts.toolCall) {
+		content.push({ type: "toolCall", id: "tc-1", name: opts.toolCall.name, arguments: opts.toolCall.arguments });
+	}
+	return {
+		role: "assistant",
+		content,
+		api: "openai-responses",
+		provider: "p",
+		model: "default",
+		usage: {
+			input: 0,
+			output: 0,
+			cacheRead: 0,
+			cacheWrite: 0,
+			totalTokens: 0,
+			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+		},
+		stopReason: opts.stopReason ?? "stop",
+		errorMessage: opts.errorMessage,
+		timestamp: Date.now(),
+	};
+}
+
+describe("runEvalLlm", () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it("resolves each tier to its expected model", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		const session = makeSession();
+
+		await runEvalLlm({ prompt: "q", model: "smol" }, { session });
+		await runEvalLlm({ prompt: "q", model: "default" }, { session });
+		await runEvalLlm({ prompt: "q", model: "slow" }, { session });
+
+		const resolved = spy.mock.calls.map(call => {
+			const model = call[0] as Model<Api>;
+			return `${model.provider}/${model.id}`;
+		});
+		expect(resolved).toEqual(["p/smol", "p/default", "p/slow"]);
+	});
+
+	it("prefers the session active model for the default tier, falling back to pi/default", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		const session = makeSession({ available: [SMOL, DEFAULT, SLOW], activeModel: "p/slow" });
+
+		await runEvalLlm({ prompt: "q", model: "default" }, { session });
+
+		const model = spy.mock.calls[0]?.[0] as Model<Api>;
+		expect(`${model.provider}/${model.id}`).toBe("p/slow");
+	});
+
+	it("returns the completion text in plain mode", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "the answer" }));
+		const result = await runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() });
+		expect(result.text).toBe("the answer");
+		expect(result.details).toEqual({ model: "p/smol", tier: "smol", structured: false });
+	});
+
+	it("forces a respond tool call and returns its arguments in structured mode", async () => {
+		const spy = vi
+			.spyOn(ai, "completeSimple")
+			.mockResolvedValue(assistant({ toolCall: { name: "respond", arguments: { answer: 42 } } }));
+		const result = await runEvalLlm(
+			{ prompt: "q", model: "smol", schema: { type: "object", properties: { answer: { type: "number" } } } },
+			{ session: makeSession() },
+		);
+
+		expect(JSON.parse(result.text)).toEqual({ answer: 42 });
+		expect(result.details.structured).toBe(true);
+
+		const ctx = spy.mock.calls[0]?.[1] as { tools?: Array<{ name: string }> };
+		const opts = spy.mock.calls[0]?.[2] as { toolChoice?: unknown };
+		expect(ctx.tools?.[0]?.name).toBe("respond");
+		expect(opts.toolChoice).toEqual({ type: "tool", name: "respond" });
+	});
+
+	it("falls back to JSON embedded in text when the model skips the respond tool", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: 'here: {"answer": 7}' }));
+		const result = await runEvalLlm(
+			{ prompt: "q", model: "smol", schema: { type: "object" } },
+			{ session: makeSession() },
+		);
+		expect(JSON.parse(result.text)).toEqual({ answer: 7 });
+	});
+
+	it("requests reasoning only for the slow tier on a reasoning-capable model", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		const session = makeSession({ available: [SMOL, DEFAULT, REASONING_SLOW] });
+
+		await runEvalLlm({ prompt: "q", model: "smol" }, { session });
+		await runEvalLlm({ prompt: "q", model: "slow" }, { session });
+
+		const smolOpts = spy.mock.calls[0]?.[2] as { reasoning?: unknown };
+		const slowOpts = spy.mock.calls[1]?.[2] as { reasoning?: unknown };
+		expect(smolOpts.reasoning).toBeUndefined();
+		expect(slowOpts.reasoning).toBe(Effort.High);
+	});
+
+	it("does not request reasoning for the slow tier on a non-reasoning model", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		// SLOW is reasoning:false — must not trip requireSupportedEffort downstream.
+		const result = await runEvalLlm({ prompt: "q", model: "slow" }, { session: makeSession() });
+		expect(result.text).toBe("ok");
+		const opts = spy.mock.calls[0]?.[2] as { reasoning?: unknown };
+		expect(opts.reasoning).toBeUndefined();
+	});
+
+	it("throws ToolError on invalid arguments", async () => {
+		await expect(runEvalLlm({ prompt: "" }, { session: makeSession() })).rejects.toBeInstanceOf(ToolError);
+		await expect(runEvalLlm({ prompt: "q", model: "huge" }, { session: makeSession() })).rejects.toBeInstanceOf(
+			ToolError,
+		);
+	});
+
+	it("throws ToolError when no model resolves for the tier", async () => {
+		const session = makeSession({ available: [DEFAULT], roles: { smol: "missing/model" } });
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session })).rejects.toBeInstanceOf(ToolError);
+	});
+
+	it("throws ToolError when the resolved model has no API key", async () => {
+		const session = makeSession({ apiKey: null });
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session })).rejects.toBeInstanceOf(ToolError);
+	});
+
+	it("maps error and aborted stop reasons to ToolError", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValueOnce(assistant({ stopReason: "error", errorMessage: "boom" }));
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() })).rejects.toThrow("boom");
+
+		vi.spyOn(ai, "completeSimple").mockResolvedValueOnce(assistant({ stopReason: "aborted" }));
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() })).rejects.toBeInstanceOf(
+			ToolError,
+		);
+	});
+
+	it("throws ToolError when plain mode produces no text", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "" }));
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() })).rejects.toBeInstanceOf(
+			ToolError,
+		);
+	});
+});
+
+describe("llm() through eval runtimes", () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	afterAll(async () => {
+		await disposeAllVmContexts();
+		await disposeAllKernelSessions();
+	});
+
+	it("exposes llm() in the JavaScript runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-js-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `js-llm:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "hello from smol" }));
+
+		const result = await executeJs('return await llm("hi", { model: "smol" });', {
+			cwd: tempDir.path(),
+			sessionId,
+			session: makeSession(),
+			sessionFile,
+		});
+
+		expect(result.exitCode).toBe(0);
+		expect(result.output.trim()).toBe("hello from smol");
+	});
+
+	it("parses structured llm() output in the JavaScript runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-js-struct-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `js-llm-struct:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(
+			assistant({ toolCall: { name: "respond", arguments: { ok: true, n: 3 } } }),
+		);
+
+		const result = await executeJs(
+			'const r = await llm("hi", { schema: { type: "object" } }); return JSON.stringify(r);',
+			{ cwd: tempDir.path(), sessionId, session: makeSession(), sessionFile },
+		);
+
+		expect(result.exitCode).toBe(0);
+		expect(JSON.parse(result.output.trim())).toEqual({ ok: true, n: 3 });
+	});
+
+	it("exposes llm() in the Python runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-py-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `py-llm:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "hello from python" }));
+
+		const result = await executePython('print(llm("hi", model="smol"))', {
+			cwd: tempDir.path(),
+			sessionId,
+			sessionFile,
+			toolSession: makeSession(),
+		});
+
+		expect(result.exitCode).toBe(0);
+		expect(result.output.trim()).toBe("hello from python");
+	});
+
+	it("parses structured llm() output in the Python runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-py-struct-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `py-llm-struct:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(
+			assistant({ toolCall: { name: "respond", arguments: { ok: true } } }),
+		);
+
+		const result = await executePython('import json\nprint(json.dumps(llm("hi", schema={"type": "object"})))', {
+			cwd: tempDir.path(),
+			sessionId,
+			sessionFile,
+			toolSession: makeSession(),
+		});
+
+		expect(result.exitCode).toBe(0);
+		expect(JSON.parse(result.output.trim())).toEqual({ ok: true });
+	});
+});

package/src/eval/js/shared/prelude.txt

@@ -39,6 +39,13 @@ if (!globalThis.__omp_js_prelude_loaded__) {
 		return values.length === 1 ? values[0] : values;
 	};
 
+	const llm = async (prompt, opts = {}) => {
+		const o = toOptions(opts);
+		const res = await globalThis.__omp_call_tool__("__llm__", { prompt, ...o });
+		const text = res && typeof res === "object" ? res.text : res;
+		return o.schema ? JSON.parse(text) : text;
+	};
+
 	const display = value => {
 		globalThis.__omp_display__(value);
 	};
@@ -61,6 +68,7 @@ if (!globalThis.__omp_js_prelude_loaded__) {
 	globalThis.print = consoleBridge.log;
 	globalThis.display = display;
 	globalThis.tool = tool;
+	globalThis.llm = llm;
 	globalThis.output = output;
 	globalThis.read = read;
 	globalThis.write = write;

package/src/eval/js/tool-bridge.ts

@@ -1,6 +1,7 @@
 import type { AgentTool, AgentToolResult } from "@oh-my-pi/pi-agent-core";
 import type { ToolSession } from "../../tools";
 import { ToolError } from "../../tools/tool-errors";
+import { EVAL_LLM_BRIDGE_NAME, runEvalLlm } from "../llm-bridge";
 import type { JsStatusEvent } from "./shared/types";
 
 export type { JsStatusEvent } from "./shared/types";
@@ -101,6 +102,9 @@ function summarizeToolResult(
 }
 
 export async function callSessionTool(name: string, args: unknown, options: ToolBridgeOptions): Promise<ToolValue> {
+	if (name === EVAL_LLM_BRIDGE_NAME) {
+		return await runEvalLlm(args, options);
+	}
 	const tool = getTool(options.session, name);
 	const normalizedArgs = normalizeArgs(args);
 	const toolCallId = `js-${name}-${crypto.randomUUID()}`;