@selesai/code 0.13.33 → 0.13.35

package/dist/extensions/jev-ask-tool.test.ts

@@ -0,0 +1,501 @@
+/**
+ * The agent-callable `ask_jev` tool.
+ *
+ * The tool is driven the way the model drives it: one call carrying its own
+ * prose, the paths it wants read, one command, and a question block. Tests
+ * assert what the agent observes — the state Jev was sent, the answers that come
+ * back, and what never leaves the process — never private helper order.
+ */
+import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import type { AssistantMessage } from "@earendil-works/pi-ai";
+
+const state = vi.hoisted(() => ({
+	settingsPath: "",
+	commands: new Map<string, { output: string; exitCode: number }>(),
+}));
+
+vi.mock("@selesai/code", () => ({
+	getSettingsPath: () => state.settingsPath,
+	// jev_find runs the real ripgrep on PATH against the temp repo.
+	ensureTool: async () => "rg",
+	// The same local shell backend the bash tool uses, scripted per command.
+	createLocalBashOperations: () => ({
+		exec: async (command: string, _cwd: string, options: { onData: (data: Buffer) => void }) => {
+			const scripted = state.commands.get(command);
+			if (!scripted) return { exitCode: 127 };
+			options.onData(Buffer.from(scripted.output, "utf-8"));
+			return { exitCode: scripted.exitCode };
+		},
+	}),
+}));
+
+import jevAskToolExtension, { askPath, FIND_BATCH, fitFindPayload, MAX_ASK_QUESTIONS } from "./jev-ask-tool.ts";
+import { JEV_ROUTING_EVENT, serializeJevRequest } from "./jev/decisions.ts";
+import { jevResponse, providerTemplate } from "./jev/test-support.ts";
+
+const FILE_SENTINEL = "FILE_BODY_MUST_RETURN_TO_JE ONLY";
+const COMMAND_SENTINEL = "FAIL rounding test at round.ts:12";
+
+let root: string;
+let repo: string;
+
+beforeAll(() => {
+	root = mkdtempSync(join(tmpdir(), "jev-ask-tool-"));
+	repo = join(root, "repo");
+	mkdirSync(join(repo, "src"), { recursive: true });
+	writeFileSync(join(repo, "src", "round.ts"), `export const round = () => ${FILE_SENTINEL};\n`, "utf-8");
+	state.settingsPath = join(root, "agent", "settings.json");
+	mkdirSync(join(root, "agent"), { recursive: true });
+});
+
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+
+beforeEach(() => {
+	state.commands.clear();
+});
+
+/** The answers envelope Jev returns, with either verdict field per question. */
+function answerText(answers: Record<string, unknown>): string {
+	return JSON.stringify({ answers });
+}
+
+interface ToolDefinition {
+	name: string;
+	description: string;
+	promptSnippet?: string;
+	promptGuidelines?: string[];
+	execute: (
+		id: string,
+		params: Record<string, unknown>,
+		signal: AbortSignal | undefined,
+		onUpdate: undefined,
+		ctx: unknown,
+	) => Promise<{ content: Array<{ type: string; text: string }>; details: Record<string, unknown> }>;
+}
+
+interface AskHarness {
+	tool: ToolDefinition;
+	complete: ReturnType<typeof vi.fn>;
+	telemetry: Record<string, unknown>[];
+	/** Warnings shown to the user. */
+	notices: string[];
+	/** The live active-tool loadout the extension may trim or restore. */
+	active: string[];
+	/** Fire the extension's `before_agent_start` hook, as the session does before every run. */
+	beforeRun(): Promise<void>;
+	/** Swap credential availability mid-session, as `/tokenin add` would. */
+	setCredential(available: boolean): void;
+	ask(params: Record<string, unknown>, signal?: AbortSignal): Promise<{ text: string; details: Record<string, unknown> }>;
+	/** The decision request Jev was sent, parsed. */
+	sent(): Record<string, unknown> | undefined;
+	/** Every tool the extension registered, by name. */
+	tools: Record<string, ToolDefinition>;
+	ctx: unknown;
+}
+
+function harness(
+	options: { enabled?: boolean; credential?: boolean; template?: boolean; payloadBytes?: number } = {},
+): AskHarness {
+	writeFileSync(
+		state.settingsPath,
+		JSON.stringify({
+			jevAdvisory: {
+				routes: {
+					ask: {
+						enabled: options.enabled ?? true,
+						...(options.payloadBytes === undefined ? {} : { payloadBytes: options.payloadBytes }),
+					},
+				},
+			},
+		}),
+		"utf-8",
+	);
+	const telemetry: Record<string, unknown>[] = [];
+	const notices: string[] = [];
+	const active = ["read", "bash", "ask_jev"];
+	const hooks: Record<string, (event: unknown, ctx: unknown) => unknown> = {};
+	let credential = options.credential !== false;
+	const complete = vi.fn(async () => jevResponse(answerText({})));
+	const tools: Record<string, ToolDefinition> = {};
+	jevAskToolExtension({
+		registerTool: (definition: ToolDefinition) => {
+			tools[definition.name] = definition;
+		},
+		on: (event: string, handler: (event: unknown, ctx: unknown) => unknown) => {
+			hooks[event] = handler;
+		},
+		getActiveTools: () => [...active],
+		setActiveTools: (names: string[]) => {
+			active.splice(0, active.length, ...names);
+		},
+		events: {
+			emit: (channel: string, data: unknown) => {
+				if (channel === JEV_ROUTING_EVENT) telemetry.push(data as Record<string, unknown>);
+			},
+		},
+	} as never);
+	const ctx = {
+		cwd: repo,
+		hasUI: true,
+		ui: { notify: (message: string) => notices.push(message) },
+		modelRegistry: {
+			getAll: () => (options.template === false ? [] : [providerTemplate()]),
+			getApiKeyAndHeaders: async () =>
+				credential ? { ok: true, apiKey: "key", headers: {} } : { ok: false, error: "no key" },
+			complete,
+		},
+	};
+	const tool = tools.ask_jev as ToolDefinition;
+	return {
+		tool,
+		tools,
+		ctx,
+		complete,
+		telemetry,
+		notices,
+		active,
+		async beforeRun() {
+			await hooks.before_agent_start?.({ prompt: "x" }, ctx);
+		},
+		setCredential(available) {
+			credential = available;
+		},
+		async ask(params, signal) {
+			const result = await tool.execute("call-1", params, signal, undefined, ctx);
+			const content = result.content[0];
+			return { text: content?.type === "text" ? content.text : "", details: result.details };
+		},
+		sent() {
+			const call = complete.mock.calls.at(-1) as unknown[] | undefined;
+			const content = (call?.[1] as { messages?: Array<{ content?: string }> } | undefined)?.messages?.[0]?.content;
+			return typeof content === "string" ? (JSON.parse(content) as Record<string, unknown>) : undefined;
+		},
+	};
+}
+
+function sentBytes(complete: ReturnType<typeof vi.fn>): number {
+	const call = complete.mock.calls.at(-1) as unknown[] | undefined;
+	const content = (call?.[1] as { messages?: Array<{ content?: string }> } | undefined)?.messages?.[0]?.content;
+	return Buffer.byteLength(content ?? "", "utf-8");
+}
+
+const CHOICE = {
+	type: "choice",
+	instructions: "What kind of failure is this?",
+	criteria: { bug_in_code: "The code is wrong.", flaky_test: "The test is nondeterministic.", environment: "The setup is wrong." },
+};
+const NOUL = { type: "noul", instructions: "Do the tests pass?", criteria: { true: "They pass.", false: "They do not." } };
+const SCORE = { type: "score", instructions: "How risky is this diff?", criteria: ["safe", "needs review", "dangerous"] };
+
+const TYPICAL = {
+	state: "The tests are red and I have not read anything yet.",
+	paths: ["src/round.ts"],
+	command: "npm test",
+	questions: { failure_kind: CHOICE, tests_pass: NOUL },
+};
+
+describe("registration", () => {
+	it("registers one ask_jev tool carrying its question schema and the prompt nudge", () => {
+		const { tool } = harness();
+		expect(tool.name).toBe("ask_jev");
+		for (const schemaPart of ["choice", "score", "noul", "criteria"]) {
+			expect(tool.description).toContain(schemaPart);
+		}
+		expect(tool.description).toContain("answers only");
+		expect(tool.promptSnippet).toBeTruthy();
+		expect(tool.promptGuidelines?.[0]).toContain("ask_jev");
+	});
+});
+
+describe("one state, one call", () => {
+	it("assembles prose, code, and command output into one state and returns answers only", async () => {
+		state.commands.set("npm test", { output: COMMAND_SENTINEL, exitCode: 1 });
+		const session = harness();
+		session.complete.mockResolvedValue(
+			jevResponse(answerText({ failure_kind: { choice: "bug_in_code", confidence: 0.99 }, tests_pass: { noul: 0.04 } })),
+		);
+
+		const result = await session.ask(TYPICAL);
+		const sent = session.sent() as { state: Record<string, unknown> };
+
+		expect(session.complete).toHaveBeenCalledTimes(1);
+		expect(sent.state.request).toBe(TYPICAL.state);
+		expect(sent.state.command).toMatchObject({ command: "npm test", exitCode: 1 });
+		expect((sent.state.files as Record<string, string>)["src/round.ts"]).toContain(FILE_SENTINEL);
+
+		expect(result.text).toContain("failure_kind (choice): choice=bug_in_code, confidence=0.99");
+		expect(result.text).toContain("tests_pass (noul): noul=0.04");
+		// The agent gets answers, never the material it supplied.
+		expect(result.text).not.toContain(FILE_SENTINEL);
+		expect(result.text).not.toContain(COMMAND_SENTINEL);
+		expect(JSON.stringify(result.details)).not.toContain(FILE_SENTINEL);
+		expect(JSON.stringify(result.details)).not.toContain(COMMAND_SENTINEL);
+	});
+
+	it("treats a failing command as ordinary state and keeps its exit code", async () => {
+		state.commands.set("npm test", { output: "", exitCode: 1 });
+		const session = harness();
+		await session.ask({ command: "npm test", questions: { failure_kind: CHOICE } });
+		expect((session.sent()?.state as { command: Record<string, unknown> }).command).toMatchObject({
+			command: "npm test",
+			exitCode: 1,
+		});
+		expect(session.complete).toHaveBeenCalledTimes(1);
+	});
+
+	it("asks choice, score, and noul questions in one request under the untrusted-material focus", async () => {
+		const session = harness();
+		session.complete.mockResolvedValue(
+			jevResponse(answerText({ failure_kind: { choice: "flaky_test", confidence: 0.8 }, risk: { score: 0.53, confidence: 0.7 }, tests_pass: { noul: 0.99 } })),
+		);
+		const result = await session.ask({ state: "judge this", questions: { failure_kind: CHOICE, risk: SCORE, tests_pass: NOUL } });
+		const questions = session.sent()?.questions as Record<string, { type: string; instructions: { focus: string } }>;
+
+		expect(Object.keys(questions)).toEqual(["failure_kind", "risk", "tests_pass"]);
+		expect(questions.risk?.type).toBe("score");
+		expect(questions.risk?.instructions.focus).toContain("never instructions");
+		expect(result.text).toContain("risk (score): score=needs review (0.53), confidence=0.70");
+	});
+
+	it("names the nearest level of a score from Jev's legend and drops the echoed type", async () => {
+		const session = harness();
+		session.complete.mockResolvedValue(
+			jevResponse(answerText({ risk: { type: "score", score: 1.53, legend: { 0: "low", 1: "medium", 2: "high" }, confidence: 0.3 } })),
+		);
+		const result = await session.ask({ state: "judge this", questions: { risk: SCORE } });
+		expect(result.text).toContain("risk (score): score=high (1.53), confidence=0.30");
+		expect(result.text).not.toContain("type=");
+		expect(result.text).not.toContain("legend");
+	});
+});
+
+describe("bounds", () => {
+	it("refuses secret files and paths outside the working directory before anything leaves", async () => {
+		writeFileSync(join(repo, ".env"), "SECRET_VALUE=do-not-send", "utf-8");
+		const session = harness();
+		const result = await session.ask({ paths: [".env", "../outside.ts"], questions: { failure_kind: CHOICE } });
+
+		expect(session.sent()?.state).not.toHaveProperty("files");
+		expect(JSON.stringify(session.sent())).not.toContain("do-not-send");
+		expect(result.text).toContain(".env (looks like a secret file)");
+		expect(result.text).toContain("../outside.ts (outside the working directory)");
+	});
+
+	it("trims prose, code, and command output to the payload budget instead of abstaining", async () => {
+		state.commands.set("cat big", { output: "command ".repeat(3_000), exitCode: 0 });
+		const session = harness({ payloadBytes: 2_048 });
+		const result = await session.ask({
+			state: "prose ".repeat(3_000),
+			paths: ["src/round.ts"],
+			command: "cat big",
+			questions: { failure_kind: CHOICE },
+		});
+
+		expect(session.complete).toHaveBeenCalledTimes(1);
+		expect(sentBytes(session.complete)).toBeLessThanOrEqual(2_048);
+		expect(result.text).toContain("Truncated to fit the request budget");
+	});
+
+	it("rejects an unusable question block before any call", async () => {
+		const session = harness();
+		const unknownType = await session.ask({ questions: { q: { type: "magic", instructions: "?" } } });
+		expect(unknownType.text).toContain("magic");
+
+		const missingCriteria = await session.ask({ questions: { q: { type: "choice", instructions: "pick one" } } });
+		expect(missingCriteria.text).toContain("criteria");
+
+		const tooMany = await session.ask({
+			questions: Object.fromEntries(Array.from({ length: MAX_ASK_QUESTIONS + 1 }, (_, i) => [`q${i}`, CHOICE])),
+		});
+		expect(tooMany.text).toContain(`At most ${MAX_ASK_QUESTIONS}`);
+
+		expect(session.complete).not.toHaveBeenCalled();
+	});
+});
+
+describe("unavailability", () => {
+	it("stays silent while the route is disabled", async () => {
+		const session = harness({ enabled: false });
+		const result = await session.ask(TYPICAL);
+		expect(result.text).toContain("disabled");
+		expect(session.complete).not.toHaveBeenCalled();
+	});
+
+	it("reports an abstention rather than failing when Jev is unavailable", async () => {
+		const noCredential = harness({ credential: false });
+		const credentialless = await noCredential.ask({ state: "x", questions: { failure_kind: CHOICE } });
+		expect(noCredential.complete).not.toHaveBeenCalled();
+		expect(credentialless.text).toContain("no-credential");
+		expect(credentialless.details.failure).toBe("no-credential");
+		expect(credentialless.text).toContain("failure_kind: no answer (no-credential)");
+
+		const noTemplate = harness({ template: false });
+		expect((await noTemplate.ask({ state: "x", questions: { failure_kind: CHOICE } })).text).toContain("no-template");
+	});
+});
+
+describe("telemetry", () => {
+	it("names the unanswered questions and reports decisions without any content", async () => {
+		const session = harness();
+		session.complete.mockResolvedValue(jevResponse(answerText({ failure_kind: { choice: "flaky_test", confidence: 0.8 } })));
+		const result = await session.ask({ state: "SENTINEL_STATE", questions: { failure_kind: CHOICE, tests_pass: NOUL } });
+
+		expect(result.text).toContain("tests_pass: no answer (missing)");
+		expect(session.telemetry.at(-1)).toMatchObject({
+			event: "decision",
+			route: "ask",
+			outcome: "jev",
+			candidates: 2,
+			confidence: "medium",
+		});
+		const serialized = JSON.stringify(session.telemetry);
+		expect(serialized).not.toContain("SENTINEL_STATE");
+		expect(serialized).not.toContain("flaky_test");
+	});
+});
+
+describe("without a reachable Jev", () => {
+	it("neither reads the files nor runs the command, and warns once per session", async () => {
+		const session = harness({ credential: false });
+		const marker = join(repo, "command-ran.txt");
+		const result = await session.ask({ state: "x", paths: ["src/round.ts"], command: `touch ${marker}`, questions: { failure_kind: CHOICE } });
+
+		expect(result.text).toContain("Nothing was read, run, or sent");
+		expect(existsSync(marker)).toBe(false);
+		expect(session.complete).not.toHaveBeenCalled();
+
+		await session.ask({ state: "x", questions: { failure_kind: CHOICE } });
+		expect(session.notices).toHaveLength(1);
+		expect(session.notices[0]).toContain("/tokenin add");
+	});
+
+	it("drops ask_jev from the loadout while Jev is out of reach and restores it once it is back", async () => {
+		const session = harness({ credential: false });
+		await session.beforeRun();
+		expect(session.active).not.toContain("ask_jev");
+		// Hiding is silent: a user without a subscription is not nagged every session.
+		expect(session.notices).toEqual([]);
+
+		session.setCredential(true);
+		await session.beforeRun();
+		expect(session.active).toContain("ask_jev");
+	});
+
+	it("hides the tool when the route is turned off, and never re-adds a tool it did not remove", async () => {
+		const disabled = harness({ enabled: false });
+		await disabled.beforeRun();
+		expect(disabled.active).not.toContain("ask_jev");
+
+		const trimmed = harness();
+		trimmed.active.splice(trimmed.active.indexOf("ask_jev"), 1);
+		await trimmed.beforeRun();
+		expect(trimmed.active).not.toContain("ask_jev");
+	});
+});
+
+describe("missing material", () => {
+	it("leads with a warning when none of the passed files reached Jev, and says why each was skipped", async () => {
+		const session = harness();
+		session.complete.mockResolvedValue(jevResponse(answerText({ failure_kind: { choice: "bug_in_code", confidence: 0.7 } })));
+		const result = await session.ask({ state: "x", paths: ["src/missing.ts", "src"], questions: { failure_kind: CHOICE } });
+
+		const lines = result.text.split("\n");
+		expect(lines[1]).toContain("WARNING: Jev answered without any of the 2 files you passed");
+		expect(lines[1]).toContain("src/missing.ts (not found)");
+		expect(lines[1]).toContain("src (not a file)");
+		expect(lines[1]).toContain(repo);
+	});
+
+	it("names partly skipped files before the answers, without the all-missing warning", async () => {
+		const session = harness();
+		session.complete.mockResolvedValue(jevResponse(answerText({ failure_kind: { choice: "bug_in_code", confidence: 0.7 } })));
+		const result = await session.ask({ state: "x", paths: ["src/round.ts", "src/missing.ts"], questions: { failure_kind: CHOICE } });
+
+		expect(result.text.split("\n")[1]).toBe("Not sent to Jev: src/missing.ts (not found).");
+		expect(result.text).not.toContain("WARNING");
+	});
+});
+
+describe("jev_find", () => {
+	const find = async (session: AskHarness, params: Record<string, unknown>) => {
+		const result = await session.tools.jev_find.execute("call-f", params, undefined, undefined, session.ctx);
+		return { text: result.content[0]?.text ?? "", details: result.details };
+	};
+
+	beforeAll(() => {
+		writeFileSync(join(repo, "src", "other.ts"), "// mentions round only in passing\nexport const x = 1;\n", "utf-8");
+		writeFileSync(join(repo, ".env"), "round=SECRET_ENV_VALUE\n", "utf-8");
+	});
+
+	it("ranks ripgrep's candidates by Jev relevance and returns pointers, never contents", async () => {
+		const session = harness();
+		// Jev: the file whose question names round.ts is relevant, the rest are not.
+		session.complete.mockImplementation(async (_model: unknown, request: { messages: Array<{ content: string }> }) => {
+			const sent = JSON.parse(request.messages[0].content) as { questions: Record<string, { instructions: { question: string } }> };
+			const answers = Object.fromEntries(
+				Object.entries(sent.questions).map(([name, q]) => [name, { noul: q.instructions.question.includes("round.ts") ? 0.93 : 0.08 }]),
+			);
+			return jevResponse(answerText(answers));
+		});
+		const result = await find(session, { question: "where is rounding implemented", pattern: "round", ignoreCase: true });
+
+		expect(result.text).toMatch(/judged 2 of 2 candidate/);
+		expect(result.text).toContain("- src/round.ts:1 (0.93)");
+		expect(result.text).not.toContain("other.ts");
+		expect(result.text).not.toContain(FILE_SENTINEL);
+		const sent = session.sent() as { state: { files: Record<string, string> } };
+		expect(Object.keys(sent.state.files).sort()).toEqual(["src/other.ts", "src/round.ts"]);
+		expect(sent.state.files["src/round.ts"]).toContain(FILE_SENTINEL);
+		expect(JSON.stringify(sent)).not.toContain("SECRET_ENV_VALUE");
+	});
+
+	it("lists a directory by glob and still returns candidates when Jev is unreachable", async () => {
+		const session = harness({ credential: false });
+		const result = await find(session, { question: "rounding", path: "src", glob: "*.ts" });
+
+		expect(session.complete).not.toHaveBeenCalled();
+		expect(result.text).toMatch(/Jev did not judge \(no-credential\); 2 candidate/);
+		// The question word in the path orders the unjudged listing.
+		expect(result.text.split("\n")[1]).toBe("- src/round.ts");
+	});
+
+	it("shrinks escape-heavy snippets until a full batch fits the request budget", () => {
+		// Quotes and newlines double under JSON escaping, which is what overflowed live.
+		const snippet = '"\n\t\\'.repeat(1_000);
+		const batch = Array.from({ length: FIND_BATCH }, (_, i) => ({ path: `src/f${i}.ts`, lines: [], snippet }));
+		const payload = fitFindPayload("q", batch, 32 * 1024);
+		expect(serializeJevRequest(payload, 32 * 1024)).toBeDefined();
+		expect(Object.keys(payload.questions as object)).toHaveLength(FIND_BATCH);
+	});
+
+	it("refuses a search root outside the working directory", async () => {
+		const result = await find(harness(), { question: "x", path: ".." });
+		expect(result.text).toContain("outside the working directory");
+	});
+});
+
+describe("askPath symlinks", () => {
+	it("refuses links that escape the working directory or point at a secret", () => {
+		const root = mkdtempSync(join(tmpdir(), "ask-path-"));
+		const cwd = join(root, "repo");
+		mkdirSync(cwd);
+		writeFileSync(join(root, "outside.txt"), "x");
+		writeFileSync(join(cwd, ".env"), "SECRET=1");
+		writeFileSync(join(cwd, "ok.ts"), "x");
+		symlinkSync(join(root, "outside.txt"), join(cwd, "escape.txt"));
+		symlinkSync(root, join(cwd, "up"));
+		symlinkSync(join(cwd, ".env"), join(cwd, "notes.txt"));
+		try {
+			expect(askPath("escape.txt", cwd)).toEqual({ refused: "outside the working directory" });
+			expect(askPath("up/outside.txt", cwd)).toEqual({ refused: "outside the working directory" });
+			expect(askPath("notes.txt", cwd)).toEqual({ refused: "looks like a secret file" });
+			expect(askPath("ok.ts", cwd)).toEqual({ full: join(cwd, "ok.ts") });
+			expect(askPath("missing.ts", cwd)).toEqual({ full: join(cwd, "missing.ts") });
+		} finally {
+			rmSync(root, { recursive: true, force: true });
+		}
+	});
+});