@oh-my-pi/pi-coding-agent 15.5.13 → 15.6.0

package/src/eval/__tests__/llm-bridge.test.ts

@@ -0,0 +1,297 @@
+import { afterAll, afterEach, describe, expect, it, vi } from "bun:test";
+import * as path from "node:path";
+import type { Api, AssistantMessage, Model } from "@oh-my-pi/pi-ai";
+import * as ai from "@oh-my-pi/pi-ai";
+import { Effort } from "@oh-my-pi/pi-ai";
+import { TempDir } from "@oh-my-pi/pi-utils";
+import type { ModelRegistry } from "../../config/model-registry";
+import { Settings } from "../../config/settings";
+import type { ToolSession } from "../../tools";
+import { ToolError } from "../../tools/tool-errors";
+import { disposeAllVmContexts } from "../js/context-manager";
+import { executeJs } from "../js/executor";
+import { runEvalLlm } from "../llm-bridge";
+import { disposeAllKernelSessions, executePython } from "../py/executor";
+
+function makeModel(provider: string, id: string, extra: Partial<Model<Api>> = {}): Model<Api> {
+	return {
+		id,
+		name: id,
+		api: "openai-responses",
+		provider,
+		baseUrl: "https://example.test/v1",
+		reasoning: false,
+		input: ["text"],
+		cost: { input: 1, output: 1, cacheRead: 0, cacheWrite: 1 },
+		contextWindow: 128000,
+		maxTokens: 4096,
+		...extra,
+	} as Model<Api>;
+}
+
+const SMOL = makeModel("p", "smol");
+const DEFAULT = makeModel("p", "default");
+const SLOW = makeModel("p", "slow");
+const REASONING_SLOW = makeModel("p", "slow", {
+	api: "anthropic-messages",
+	reasoning: true,
+	thinking: { minLevel: Effort.Low, maxLevel: Effort.High, mode: "anthropic-adaptive" },
+});
+
+interface SessionOptions {
+	available?: Model<Api>[];
+	apiKey?: string | null;
+	activeModel?: string;
+	roles?: Partial<Record<"smol" | "default" | "slow", string>>;
+}
+
+function makeSession(opts: SessionOptions = {}): ToolSession {
+	const settings = Settings.isolated({ "async.enabled": false, "task.isolation.mode": "none" });
+	const roles = opts.roles ?? { smol: "p/smol", slow: "p/slow" };
+	for (const role in roles) {
+		const value = roles[role as keyof typeof roles];
+		if (value) settings.setModelRole(role, value);
+	}
+	const modelRegistry = {
+		getAvailable: () => opts.available ?? [SMOL, DEFAULT, SLOW],
+		getApiKey: async () => (opts.apiKey === undefined ? "test-key" : opts.apiKey),
+	} as unknown as ModelRegistry;
+	return {
+		settings,
+		modelRegistry,
+		getActiveModelString: () => opts.activeModel ?? "p/default",
+	} as unknown as ToolSession;
+}
+
+function assistant(opts: {
+	text?: string;
+	toolCall?: { name: string; arguments: Record<string, unknown> };
+	stopReason?: AssistantMessage["stopReason"];
+	errorMessage?: string;
+}): AssistantMessage {
+	const content: AssistantMessage["content"] = [];
+	if (opts.text) content.push({ type: "text", text: opts.text });
+	if (opts.toolCall) {
+		content.push({ type: "toolCall", id: "tc-1", name: opts.toolCall.name, arguments: opts.toolCall.arguments });
+	}
+	return {
+		role: "assistant",
+		content,
+		api: "openai-responses",
+		provider: "p",
+		model: "default",
+		usage: {
+			input: 0,
+			output: 0,
+			cacheRead: 0,
+			cacheWrite: 0,
+			totalTokens: 0,
+			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+		},
+		stopReason: opts.stopReason ?? "stop",
+		errorMessage: opts.errorMessage,
+		timestamp: Date.now(),
+	};
+}
+
+describe("runEvalLlm", () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it("resolves each tier to its expected model", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		const session = makeSession();
+
+		await runEvalLlm({ prompt: "q", model: "smol" }, { session });
+		await runEvalLlm({ prompt: "q", model: "default" }, { session });
+		await runEvalLlm({ prompt: "q", model: "slow" }, { session });
+
+		const resolved = spy.mock.calls.map(call => {
+			const model = call[0] as Model<Api>;
+			return `${model.provider}/${model.id}`;
+		});
+		expect(resolved).toEqual(["p/smol", "p/default", "p/slow"]);
+	});
+
+	it("prefers the session active model for the default tier, falling back to pi/default", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		const session = makeSession({ available: [SMOL, DEFAULT, SLOW], activeModel: "p/slow" });
+
+		await runEvalLlm({ prompt: "q", model: "default" }, { session });
+
+		const model = spy.mock.calls[0]?.[0] as Model<Api>;
+		expect(`${model.provider}/${model.id}`).toBe("p/slow");
+	});
+
+	it("returns the completion text in plain mode", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "the answer" }));
+		const result = await runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() });
+		expect(result.text).toBe("the answer");
+		expect(result.details).toEqual({ model: "p/smol", tier: "smol", structured: false });
+	});
+
+	it("forces a respond tool call and returns its arguments in structured mode", async () => {
+		const spy = vi
+			.spyOn(ai, "completeSimple")
+			.mockResolvedValue(assistant({ toolCall: { name: "respond", arguments: { answer: 42 } } }));
+		const result = await runEvalLlm(
+			{ prompt: "q", model: "smol", schema: { type: "object", properties: { answer: { type: "number" } } } },
+			{ session: makeSession() },
+		);
+
+		expect(JSON.parse(result.text)).toEqual({ answer: 42 });
+		expect(result.details.structured).toBe(true);
+
+		const ctx = spy.mock.calls[0]?.[1] as { tools?: Array<{ name: string }> };
+		const opts = spy.mock.calls[0]?.[2] as { toolChoice?: unknown };
+		expect(ctx.tools?.[0]?.name).toBe("respond");
+		expect(opts.toolChoice).toEqual({ type: "tool", name: "respond" });
+	});
+
+	it("falls back to JSON embedded in text when the model skips the respond tool", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: 'here: {"answer": 7}' }));
+		const result = await runEvalLlm(
+			{ prompt: "q", model: "smol", schema: { type: "object" } },
+			{ session: makeSession() },
+		);
+		expect(JSON.parse(result.text)).toEqual({ answer: 7 });
+	});
+
+	it("requests reasoning only for the slow tier on a reasoning-capable model", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		const session = makeSession({ available: [SMOL, DEFAULT, REASONING_SLOW] });
+
+		await runEvalLlm({ prompt: "q", model: "smol" }, { session });
+		await runEvalLlm({ prompt: "q", model: "slow" }, { session });
+
+		const smolOpts = spy.mock.calls[0]?.[2] as { reasoning?: unknown };
+		const slowOpts = spy.mock.calls[1]?.[2] as { reasoning?: unknown };
+		expect(smolOpts.reasoning).toBeUndefined();
+		expect(slowOpts.reasoning).toBe(Effort.High);
+	});
+
+	it("does not request reasoning for the slow tier on a non-reasoning model", async () => {
+		const spy = vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "ok" }));
+		// SLOW is reasoning:false — must not trip requireSupportedEffort downstream.
+		const result = await runEvalLlm({ prompt: "q", model: "slow" }, { session: makeSession() });
+		expect(result.text).toBe("ok");
+		const opts = spy.mock.calls[0]?.[2] as { reasoning?: unknown };
+		expect(opts.reasoning).toBeUndefined();
+	});
+
+	it("throws ToolError on invalid arguments", async () => {
+		await expect(runEvalLlm({ prompt: "" }, { session: makeSession() })).rejects.toBeInstanceOf(ToolError);
+		await expect(runEvalLlm({ prompt: "q", model: "huge" }, { session: makeSession() })).rejects.toBeInstanceOf(
+			ToolError,
+		);
+	});
+
+	it("throws ToolError when no model resolves for the tier", async () => {
+		const session = makeSession({ available: [DEFAULT], roles: { smol: "missing/model" } });
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session })).rejects.toBeInstanceOf(ToolError);
+	});
+
+	it("throws ToolError when the resolved model has no API key", async () => {
+		const session = makeSession({ apiKey: null });
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session })).rejects.toBeInstanceOf(ToolError);
+	});
+
+	it("maps error and aborted stop reasons to ToolError", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValueOnce(assistant({ stopReason: "error", errorMessage: "boom" }));
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() })).rejects.toThrow("boom");
+
+		vi.spyOn(ai, "completeSimple").mockResolvedValueOnce(assistant({ stopReason: "aborted" }));
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() })).rejects.toBeInstanceOf(
+			ToolError,
+		);
+	});
+
+	it("throws ToolError when plain mode produces no text", async () => {
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "" }));
+		await expect(runEvalLlm({ prompt: "q", model: "smol" }, { session: makeSession() })).rejects.toBeInstanceOf(
+			ToolError,
+		);
+	});
+});
+
+describe("llm() through eval runtimes", () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	afterAll(async () => {
+		await disposeAllVmContexts();
+		await disposeAllKernelSessions();
+	});
+
+	it("exposes llm() in the JavaScript runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-js-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `js-llm:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "hello from smol" }));
+
+		const result = await executeJs('return await llm("hi", { model: "smol" });', {
+			cwd: tempDir.path(),
+			sessionId,
+			session: makeSession(),
+			sessionFile,
+		});
+
+		expect(result.exitCode).toBe(0);
+		expect(result.output.trim()).toBe("hello from smol");
+	});
+
+	it("parses structured llm() output in the JavaScript runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-js-struct-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `js-llm-struct:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(
+			assistant({ toolCall: { name: "respond", arguments: { ok: true, n: 3 } } }),
+		);
+
+		const result = await executeJs(
+			'const r = await llm("hi", { schema: { type: "object" } }); return JSON.stringify(r);',
+			{ cwd: tempDir.path(), sessionId, session: makeSession(), sessionFile },
+		);
+
+		expect(result.exitCode).toBe(0);
+		expect(JSON.parse(result.output.trim())).toEqual({ ok: true, n: 3 });
+	});
+
+	it("exposes llm() in the Python runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-py-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `py-llm:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(assistant({ text: "hello from python" }));
+
+		const result = await executePython('print(llm("hi", model="smol"))', {
+			cwd: tempDir.path(),
+			sessionId,
+			sessionFile,
+			toolSession: makeSession(),
+		});
+
+		expect(result.exitCode).toBe(0);
+		expect(result.output.trim()).toBe("hello from python");
+	});
+
+	it("parses structured llm() output in the Python runtime", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-llm-py-struct-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `py-llm-struct:${crypto.randomUUID()}`;
+		vi.spyOn(ai, "completeSimple").mockResolvedValue(
+			assistant({ toolCall: { name: "respond", arguments: { ok: true } } }),
+		);
+
+		const result = await executePython('import json\nprint(json.dumps(llm("hi", schema={"type": "object"})))', {
+			cwd: tempDir.path(),
+			sessionId,
+			sessionFile,
+			toolSession: makeSession(),
+		});
+
+		expect(result.exitCode).toBe(0);
+		expect(JSON.parse(result.output.trim())).toEqual({ ok: true });
+	});
+});

package/src/eval/__tests__/shared-executors.test.ts

@@ -492,6 +492,42 @@ display({"label": "A"})`,
 		expect(reloaded.output.trim()).toBe("2");
 	});
 
+	it("loads TypeScript type-only imports in cells and local modules", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-js-type-imports-");
+		const sessionFile = path.join(tempDir.path(), "session.jsonl");
+		const sessionId = `js-type-imports:${crypto.randomUUID()}`;
+		const session = createToolSession(tempDir.path(), sessionFile);
+		const typesPath = path.join(tempDir.path(), "types.ts");
+		const valuesPath = path.join(tempDir.path(), "values.ts");
+		const entryPath = path.join(tempDir.path(), "entry.ts");
+		const typesSpec = JSON.stringify(typesPath);
+		const entrySpec = JSON.stringify(entryPath);
+		await Bun.write(typesPath, "export interface TypeOnly { value: number }\n");
+		await Bun.write(valuesPath, "export interface InlineOnly { value: number }\nexport const imported = 41;\n");
+		await Bun.write(
+			entryPath,
+			[
+				'import type { TypeOnly } from "./types.ts";',
+				'import { type InlineOnly, imported } from "./values.ts";',
+				"export const typeOnly = 1;",
+				"export const inlineType = imported;",
+				"",
+			].join("\n"),
+		);
+
+		const result = await executeJs(
+			`import type { TypeOnly } from ${typesSpec};\nconst mod = await import(${entrySpec});\nreturn mod.typeOnly + mod.inlineType;`,
+			{
+				sessionId,
+				session,
+				sessionFile,
+			},
+		);
+
+		expect(result.exitCode).toBe(0);
+		expect(result.output.trim()).toBe("42");
+	});
+
 	it("refreshes the Python tool proxy when bridge env appears after kernel warm-up", async () => {
 		using tempDir = TempDir.createSync("@omp-eval-py-tool-proxy-");
 		const sessionFile = path.join(tempDir.path(), "session.jsonl");

package/src/eval/js/shared/local-module-loader.ts

@@ -88,7 +88,10 @@ export class LocalModuleLoader {
 
 	async #buildLocalModule(modulePath: string): Promise<LocalModuleEntry> {
 		const rawSource = fs.readFileSync(modulePath, "utf8");
-		const stripped = stripTypeScriptSyntax(rawSource);
+		const stripped = stripTypeScriptSyntax(rawSource, {
+			force: isTypeScriptModulePath(modulePath),
+			loader: stripLoaderForPath(modulePath),
+		});
 		const moduleDir = path.dirname(modulePath);
 		const localDeps = new Set<string>();
 		for (const specifier of collectModuleSourceSpecifiers(stripped)) {
@@ -251,6 +254,15 @@ function isLocalPathSpecifier(source: string): boolean {
 	);
 }
 
+function isTypeScriptModulePath(modulePath: string): boolean {
+	const ext = path.extname(modulePath);
+	return ext === ".ts" || ext === ".tsx" || ext === ".mts";
+}
+
+function stripLoaderForPath(modulePath: string): "ts" | "tsx" {
+	return path.extname(modulePath) === ".tsx" ? "tsx" : "ts";
+}
+
 function isManagedLocalModulePath(target: string): boolean {
 	return (
 		path.isAbsolute(target) &&

package/src/eval/js/shared/prelude.txt

@@ -39,6 +39,13 @@ if (!globalThis.__omp_js_prelude_loaded__) {
 		return values.length === 1 ? values[0] : values;
 	};
 
+	const llm = async (prompt, opts = {}) => {
+		const o = toOptions(opts);
+		const res = await globalThis.__omp_call_tool__("__llm__", { prompt, ...o });
+		const text = res && typeof res === "object" ? res.text : res;
+		return o.schema ? JSON.parse(text) : text;
+	};
+
 	const display = value => {
 		globalThis.__omp_display__(value);
 	};
@@ -61,6 +68,7 @@ if (!globalThis.__omp_js_prelude_loaded__) {
 	globalThis.print = consoleBridge.log;
 	globalThis.display = display;
 	globalThis.tool = tool;
+	globalThis.llm = llm;
 	globalThis.output = output;
 	globalThis.read = read;
 	globalThis.write = write;

package/src/eval/js/shared/rewrite-imports.ts

@@ -75,6 +75,7 @@ function parseProgram(code: string): { program: { body: ReadonlyArray<BabelProgr
 			allowSuperOutsideMethod: true,
 			allowUndeclaredExports: true,
 			errorRecovery: true,
+			plugins: ["typescript"],
 		}) as unknown as { program: { body: ReadonlyArray<BabelProgramNode> } };
 	} catch {
 		return null;
@@ -178,8 +179,7 @@ export function rewriteImports(code: string): string {
 		if (node.type !== "CallExpression") return;
 		const call = node as unknown as { callee?: { type?: string; start?: number; end?: number } };
 		const callee = call.callee;
-		if (!callee || callee.type !== "Import" || typeof callee.start !== "number" || typeof callee.end !== "number")
-			return;
+		if (callee?.type !== "Import" || typeof callee.start !== "number" || typeof callee.end !== "number") return;
 		edits.push({ start: callee.start, end: callee.end, text: "__omp_import__" });
 	});
 
@@ -252,12 +252,7 @@ export function rewriteDynamicImports(code: string, callee = "__omp_import__"):
 		if (node.type !== "CallExpression") return;
 		const call = node as unknown as { callee?: { type?: string; start?: number; end?: number } };
 		const callCallee = call.callee;
-		if (
-			!callCallee ||
-			callCallee.type !== "Import" ||
-			typeof callCallee.start !== "number" ||
-			typeof callCallee.end !== "number"
-		) {
+		if (callCallee?.type !== "Import" || typeof callCallee.start !== "number" || typeof callCallee.end !== "number") {
 			return;
 		}
 		edits.push({ start: callCallee.start, end: callCallee.end, text: callee });
@@ -453,38 +448,48 @@ function requiresAsyncWrapper(code: string): boolean {
 }
 
 /**
- * Strip TypeScript syntax (type annotations, `interface`, `as`, `satisfies`, generics in
- * call expressions, etc.) before the import/lexical rewriters parse the code. We use Bun's
- * native transpiler in `ts` loader mode — fast, no JSX transforms, preserves `import`/
- * `export` declarations so the downstream Babel rewrites keep working.
+ * Strip TypeScript syntax (type annotations, type-only imports/exports, `interface`, `as`,
+ * `satisfies`, generics in call expressions, etc.) before the import/lexical rewriters parse
+ * the code. Bun's native transpiler preserves `import`/`export` declarations, so downstream
+ * Babel rewrites still control module resolution.
  *
- * Skipped when the code parses as plain JavaScript already (Babel can accept it), so the
- * common case avoids an extra transpile pass. We detect "looks like TS" with a cheap regex
- * before invoking the transpiler.
+ * Eval cells use a cheap "looks like TS" heuristic to avoid transpiling ordinary JS. Known
+ * TypeScript modules pass `force` because a file can contain TS-only module syntax such as
+ * `import type` without any value-level type annotations.
  */
-function stripTypeScript(code: string): string {
-	if (!LOOKS_LIKE_TS.test(code)) return code;
+type TypeScriptStripLoader = "ts" | "tsx";
+
+const TS_TRANSPILER = new Bun.Transpiler({ loader: "ts" });
+const TSX_TRANSPILER = new Bun.Transpiler({ loader: "tsx" });
+
+function stripTypeScript(code: string, options: { force?: boolean; loader?: TypeScriptStripLoader } = {}): string {
+	if (!options.force && !LOOKS_LIKE_TS.test(code)) return code;
 	try {
-		return new Bun.Transpiler({ loader: "ts" }).transformSync(code);
+		const transpiler = options.loader === "tsx" ? TSX_TRANSPILER : TS_TRANSPILER;
+		return transpiler.transformSync(code);
 	} catch {
 		// Transpiler failed (e.g. unrecoverable syntax). Hand the original source back so the
 		// downstream rewriter / VM surfaces the real error to the user.
 		return code;
 	}
 }
-export function stripTypeScriptSyntax(code: string): string {
-	return stripTypeScript(code);
+export function stripTypeScriptSyntax(
+	code: string,
+	options: { force?: boolean; loader?: TypeScriptStripLoader } = {},
+): string {
+	return stripTypeScript(code, options);
 }
 
-// Heuristic: any of the obvious TS-only tokens. Plain JS using `as` only inside strings
-// won't match because we require a leading word boundary plus a colon/keyword neighbor.
+// Heuristic: obvious TS-only tokens, including type-only module syntax. Plain JS using `as`
+// only inside strings won't match because we require a leading word boundary plus a
+// colon/keyword neighbor.
 const LOOKS_LIKE_TS =
-	/(?:\binterface\s+\w|\btype\s+\w+\s*=|\b(?:as|satisfies)\s+(?:[A-Z]|\bconst\b)|:\s*(?:string|number|boolean|any|unknown|void|never|object|[A-Z]\w*)\b|<\s*[A-Z]\w*\s*[,>])/;
+	/(?:\bimport\s+type\b|\bexport\s+type\b|\b(?:import|export)\s*\{[^}\n]*\btype\s+\w|\binterface\s+\w|\btype\s+\w+\s*=|\b(?:as|satisfies)\s+(?:[A-Z]|\bconst\b)|:\s*(?:string|number|boolean|any|unknown|void|never|object|[A-Z]\w*)\b|<\s*[A-Z]\w*\s*[,>])/;
 
 export function wrapCode(code: string): { source: string; asyncWrapped: boolean; finalExpressionReturned: boolean } {
-	const stripped = stripTypeScript(code);
-	const finalExpression = returnFinalExpression(stripped);
-	const importsRewritten = rewriteImports(finalExpression.source);
+	const finalExpression = returnFinalExpression(code);
+	const stripped = stripTypeScript(finalExpression.source);
+	const importsRewritten = rewriteImports(stripped);
 	const needsAsyncWrapper = requiresAsyncWrapper(importsRewritten);
 	const rewritten = {
 		source: demoteTopLevelLexicals(importsRewritten, { publishGlobals: needsAsyncWrapper }),

package/src/eval/js/tool-bridge.ts

@@ -1,6 +1,7 @@
 import type { AgentTool, AgentToolResult } from "@oh-my-pi/pi-agent-core";
 import type { ToolSession } from "../../tools";
 import { ToolError } from "../../tools/tool-errors";
+import { EVAL_LLM_BRIDGE_NAME, runEvalLlm } from "../llm-bridge";
 import type { JsStatusEvent } from "./shared/types";
 
 export type { JsStatusEvent } from "./shared/types";
@@ -101,6 +102,9 @@ function summarizeToolResult(
 }
 
 export async function callSessionTool(name: string, args: unknown, options: ToolBridgeOptions): Promise<ToolValue> {
+	if (name === EVAL_LLM_BRIDGE_NAME) {
+		return await runEvalLlm(args, options);
+	}
 	const tool = getTool(options.session, name);
 	const normalizedArgs = normalizeArgs(args);
 	const toolCallId = `js-${name}-${crypto.randomUUID()}`;