@alis-build/harness-eval 0.1.2 → 0.1.4

package/dist/{suite-Dlzl-HI0.js → suite-C3-8EjUW.js}

@@ -1,5 +1,11 @@
-import { t as claudeCodeAdapter } from "./claude-code-DZ4Vkgp6.js";
+import { n as TrajectoryBuilder, t as AdapterError } from "./types-Bac8_Ixb.js";
+import { t as claudeCodeAdapter } from "./claude-code-C_7hxC8z.js";
 import { n as parseCardinality, t as describeCardinality } from "./cardinality-DlE44e-4.js";
+import { t as codexAdapter } from "./codex-0cHO2te9.js";
+import { spawn } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 //#region src/assertions/patterns.ts
 /**
 * Test whether a fully-qualified tool name matches a pattern.
@@ -454,12 +460,536 @@ function evaluateAll(view, assertions) {
 	return assertions.map((a) => evaluate(view, a));
 }
 //#endregion
+//#region src/adapters/gemini-cli/exit-codes.ts
+/**
+* Known Gemini CLI exit codes for headless / stream-json runs.
+*
+* @see spec P-7 — preserve codes in diagnostics and surface human-readable labels.
+*/
+/** Documented Gemini CLI exit codes for headless harness runs (spec P-7). */
+const GEMINI_CLI_EXIT_CODES = {
+	/** Normal completion. */
+	SUCCESS: 0,
+	/** Unhandled CLI or runtime failure. */
+	ERROR: 1,
+	/** Invalid prompt, flags, or stdin (exit 42). */
+	INPUT_ERROR: 42,
+	/** Agent exceeded configured turn budget (exit 53). */
+	TURN_LIMIT: 53
+};
+/**
+* Return a short description for a non-zero Gemini CLI exit code.
+*
+* Used to populate {@link AdapterDiagnostics.exitCodeDescription} so reports
+* surface human-readable failure reasons without re-parsing stderr.
+*/
+function describeGeminiCliExitCode(exitCode) {
+	if (exitCode === null || exitCode === GEMINI_CLI_EXIT_CODES.SUCCESS) return;
+	switch (exitCode) {
+		case GEMINI_CLI_EXIT_CODES.ERROR: return "Gemini CLI exited with a general error (code 1)";
+		case GEMINI_CLI_EXIT_CODES.INPUT_ERROR: return "Gemini CLI input error (code 42)";
+		case GEMINI_CLI_EXIT_CODES.TURN_LIMIT: return "Gemini CLI turn limit exceeded (code 53)";
+		default: return `Gemini CLI exited with code ${exitCode}`;
+	}
+}
+//#endregion
+//#region src/adapters/gemini-cli/map-events.ts
+/** Stateful mapper — tracks session id, delta text, and pending tool calls. */
+var GeminiCliEventMapper = class {
+	sessionId = "gemini-session";
+	model = "";
+	sawInit = false;
+	startedTools = /* @__PURE__ */ new Set();
+	assistantDeltaBuffer = "";
+	turnCount = 0;
+	/** Map one parsed Gemini JSON object to zero or more stream events. */
+	map(event) {
+		const type = event.type;
+		if (!type) return [];
+		switch (type) {
+			case "init": return [this.buildInit(event.session_id ?? this.sessionId, event.model ?? "")];
+			case "message": return this.mapMessage(event);
+			case "tool_use": return this.mapToolUse(event);
+			case "tool_result": return this.mapToolResult(event);
+			case "error": return [];
+			case "result": return this.mapResult(event);
+			default: return [];
+		}
+	}
+	buildInit(sessionId, model) {
+		this.sessionId = sessionId;
+		this.model = model;
+		this.sawInit = true;
+		return {
+			type: "system",
+			subtype: "init",
+			session_id: sessionId,
+			cwd: "",
+			model,
+			tools: [],
+			mcp_servers: []
+		};
+	}
+	ensureInit() {
+		if (this.sawInit) return [];
+		return [this.buildInit(this.sessionId, this.model)];
+	}
+	mapMessage(event) {
+		if (event.role === "user") return [];
+		const chunk = event.content ?? "";
+		if (event.delta) {
+			this.assistantDeltaBuffer += chunk;
+			return this.ensureInit();
+		}
+		const text = this.assistantDeltaBuffer + chunk;
+		this.assistantDeltaBuffer = "";
+		if (!text) return this.ensureInit();
+		return [...this.ensureInit(), {
+			type: "assistant",
+			session_id: this.sessionId,
+			message: {
+				id: `msg_${this.turnCount}`,
+				type: "message",
+				role: "assistant",
+				content: [{
+					type: "text",
+					text
+				}],
+				stop_reason: "end_turn"
+			}
+		}];
+	}
+	mapToolUse(event) {
+		const toolId = event.tool_id ?? `tool_${Math.random().toString(36).slice(2)}`;
+		if (event.tool_id) this.startedTools.add(event.tool_id);
+		const name = resolveGeminiToolName(event.tool_name ?? "unknown", event.parameters ?? {});
+		return [
+			...this.flushDeltaBuffer(),
+			...this.ensureInit(),
+			{
+				type: "assistant",
+				session_id: this.sessionId,
+				message: {
+					id: `assistant_${toolId}`,
+					type: "message",
+					role: "assistant",
+					content: [{
+						type: "tool_use",
+						id: toolId,
+						name,
+						input: event.parameters ?? {}
+					}],
+					stop_reason: "tool_use"
+				}
+			}
+		];
+	}
+	mapToolResult(event) {
+		const toolId = event.tool_id ?? "";
+		const events = [...this.ensureInit()];
+		if (toolId && !this.startedTools.has(toolId)) events.push({
+			type: "assistant",
+			session_id: this.sessionId,
+			message: {
+				id: `assistant_${toolId}`,
+				type: "message",
+				role: "assistant",
+				content: [{
+					type: "tool_use",
+					id: toolId,
+					name: "unknown",
+					input: {}
+				}],
+				stop_reason: "tool_use"
+			}
+		});
+		else if (toolId) this.startedTools.delete(toolId);
+		const isError = event.status === "error" || event.error != null;
+		const content = event.output ?? event.error?.message ?? "";
+		events.push({
+			type: "user",
+			session_id: this.sessionId,
+			message: {
+				role: "user",
+				content: [{
+					type: "tool_result",
+					tool_use_id: toolId,
+					content,
+					is_error: isError
+				}]
+			}
+		});
+		return events;
+	}
+	mapResult(event) {
+		this.turnCount++;
+		const isError = event.status === "error";
+		return [...this.flushDeltaBuffer(), {
+			type: "result",
+			subtype: isError ? "error" : "success",
+			session_id: this.sessionId,
+			is_error: isError,
+			result: event.error?.message ?? "",
+			usage: mapUsage(event.stats),
+			total_cost_usd: event.stats?.total_cost_usd ?? 0,
+			duration_ms: event.stats?.duration_ms ?? 0,
+			num_turns: this.turnCount
+		}];
+	}
+	/** Emit buffered delta text as one assistant message before tool/result events. */
+	flushDeltaBuffer() {
+		if (!this.assistantDeltaBuffer) return [];
+		const text = this.assistantDeltaBuffer;
+		this.assistantDeltaBuffer = "";
+		return [...this.ensureInit(), {
+			type: "assistant",
+			session_id: this.sessionId,
+			message: {
+				id: `msg_delta_${this.turnCount}`,
+				type: "message",
+				role: "assistant",
+				content: [{
+					type: "text",
+					text
+				}],
+				stop_reason: "end_turn"
+			}
+		}];
+	}
+};
+/**
+* Resolve harness tool name from Gemini tool_name + parameters.
+*
+* MCP tools use `mcp__<server>__<tool>`; built-in Gemini tools keep native names.
+*/
+function resolveGeminiToolName(toolName, parameters) {
+	if (toolName.startsWith("mcp__")) return toolName;
+	const server = typeof parameters.server === "string" ? parameters.server : void 0;
+	const tool = typeof parameters.tool === "string" ? parameters.tool : void 0;
+	if (server && tool) return `mcp__${server}__${tool}`;
+	if (toolName.startsWith("mcp_") && !toolName.startsWith("mcp__")) {
+		const rest = toolName.slice(4);
+		const separator = rest.lastIndexOf("_");
+		if (separator > 0) {
+			const geminiServer = rest.slice(0, separator);
+			const geminiTool = rest.slice(separator + 1);
+			if (geminiServer && geminiTool) return `mcp__${geminiServer}__${geminiTool}`;
+		}
+	}
+	return toolName;
+}
+function mapUsage(stats) {
+	return {
+		input_tokens: stats?.input_tokens ?? 0,
+		output_tokens: stats?.output_tokens ?? 0
+	};
+}
+//#endregion
+//#region src/adapters/gemini-cli/parse-json.ts
+/** Parse Gemini JSONL stdout into parsed event objects. */
+async function* parseGeminiCliJson(stream) {
+	let buffer = "";
+	stream.setEncoding("utf8");
+	for await (const chunk of stream) {
+		buffer += chunk;
+		let newlineIdx;
+		while ((newlineIdx = buffer.indexOf("\n")) !== -1) {
+			const line = buffer.slice(0, newlineIdx).trim();
+			buffer = buffer.slice(newlineIdx + 1);
+			if (line.length === 0) continue;
+			yield tryParseLine(line);
+		}
+	}
+	const trailing = buffer.trim();
+	if (trailing.length > 0) yield tryParseLine(trailing);
+}
+function tryParseLine(line) {
+	try {
+		return {
+			ok: true,
+			event: JSON.parse(line),
+			rawLine: line
+		};
+	} catch (err) {
+		return {
+			ok: false,
+			error: err instanceof Error ? err : new Error(String(err)),
+			rawLine: line
+		};
+	}
+}
+//#endregion
+//#region src/adapters/gemini-cli/flags.ts
+function pushRepeatableFlag(args, flag, values) {
+	if (!values) return;
+	for (const value of values) args.push(flag, value);
+}
+function pushOptionalFlag(args, flag, value) {
+	if (value === void 0) return;
+	if (typeof value === "boolean") {
+		if (value) args.push(flag);
+		return;
+	}
+	args.push(flag, String(value));
+}
+/**
+* Append shared Gemini CLI flags (excluding prompt and output format).
+*
+* Harness and judge subprocesses share this helper so both paths stay aligned
+* on approval mode, sandbox, MCP allowlists, and trust settings.
+*/
+function appendGeminiCliFlags(args, config) {
+	pushOptionalFlag(args, "--approval-mode", config.approvalMode ?? "yolo");
+	pushOptionalFlag(args, "--model", config.model);
+	pushOptionalFlag(args, "--sandbox", config.sandbox);
+	pushOptionalFlag(args, "--skip-trust", config.skipTrust);
+	pushRepeatableFlag(args, "--include-directories", config.includeDirectories);
+	pushRepeatableFlag(args, "--allowed-mcp-server-names", config.allowedMcpServerNames);
+	pushRepeatableFlag(args, "--extensions", config.extensions);
+	pushOptionalFlag(args, "--debug", config.debug);
+}
+/**
+* Build argv for `gemini -p "<prompt>" --output-format stream-json …`.
+*
+* Prompt is passed via `-p` and must remain the final positional argument
+* after all flags. Defaults `skipTrust` to true so CI and ephemeral workspaces
+* do not block on interactive folder-trust prompts.
+*/
+function buildArgs(config) {
+	const args = [
+		"-p",
+		config.prompt,
+		"--output-format",
+		"stream-json"
+	];
+	appendGeminiCliFlags(args, {
+		...config,
+		skipTrust: config.skipTrust ?? true
+	});
+	return args;
+}
+/**
+* Build argv for `gemini -p "<prompt>" --output-format json …` (judge).
+*
+* Emits a single JSON object (not NDJSON). The judge grader may read it from
+* stdout or recover it from stderr when Gemini prints warnings first.
+*/
+function buildJudgeArgs(prompt, config = {}) {
+	const args = [
+		"-p",
+		prompt,
+		"--output-format",
+		"json"
+	];
+	appendGeminiCliFlags(args, {
+		...config,
+		approvalMode: config.approvalMode ?? "yolo",
+		skipTrust: config.skipTrust ?? true
+	});
+	return args;
+}
+//#endregion
+//#region src/adapters/gemini-cli/process.ts
+/**
+* Process management for the Gemini CLI adapter.
+*
+* Spawns `gemini -p … --output-format stream-json`, handles timeout/abort,
+* and optional config-directory isolation.
+*/
+const DEFAULT_TIMEOUT_MS = 300 * 1e3;
+/** Grace period between SIGTERM and SIGKILL on timeout or abort. */
+const KILL_GRACE_MS = 5e3;
+/** Env var Gemini CLI uses for config directory isolation. */
+const GEMINI_CONFIG_DIR_ENV = "GEMINI_CONFIG_DIR";
+/** Resolve config dir for isolated runs. Exported for unit tests. */
+function resolveGeminiConfigDir(config, tempDir) {
+	if (config.isolateConfig !== true || !tempDir) return void 0;
+	return tempDir;
+}
+/**
+* Build subprocess env with optional `GEMINI_CONFIG_DIR` isolation.
+*
+* Shared by harness spawn ({@link spawnGeminiCli}) and the Gemini CLI judge
+* ({@link runGeminiCliGrader}) so both paths use the same config-dir semantics.
+*/
+async function prepareGeminiCliEnv(config, baseEnv = process.env) {
+	const tempConfigDir = config.isolateConfig === true ? await mkdtemp(join(tmpdir(), "harness-eval-gemini-")) : null;
+	const env = {
+		...baseEnv,
+		...config.env
+	};
+	const configDir = resolveGeminiConfigDir(config, tempConfigDir);
+	if (configDir) env[GEMINI_CONFIG_DIR_ENV] = configDir;
+	const cleanup = async () => {
+		if (!tempConfigDir) return;
+		try {
+			await rm(tempConfigDir, {
+				recursive: true,
+				force: true
+			});
+		} catch {}
+	};
+	return {
+		env,
+		cleanup
+	};
+}
+/**
+* Spawn `gemini -p … --output-format stream-json` with optional config-dir isolation.
+*
+* Timeout and abort both send SIGTERM to the process group, then SIGKILL after
+* {@link KILL_GRACE_MS} if the group is still alive.
+*/
+async function spawnGeminiCli(config) {
+	const binary = config.binary ?? "gemini";
+	const args = buildArgs(config);
+	const { env, cleanup: envCleanup } = await prepareGeminiCliEnv(config);
+	const child = spawn(binary, args, {
+		cwd: config.cwd ?? process.cwd(),
+		env,
+		stdio: [
+			"ignore",
+			"pipe",
+			"pipe"
+		],
+		detached: true
+	});
+	let timedOut = false;
+	let killEscalation = null;
+	const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+	const scheduleKillEscalation = () => {
+		if (killEscalation) clearTimeout(killEscalation);
+		killEscalation = setTimeout(() => killTree(child, "SIGKILL"), KILL_GRACE_MS);
+	};
+	const timeoutTimer = setTimeout(() => {
+		timedOut = true;
+		killTree(child, "SIGTERM");
+		scheduleKillEscalation();
+	}, timeoutMs);
+	const onAbort = () => {
+		killTree(child, "SIGTERM");
+		scheduleKillEscalation();
+	};
+	config.signal?.addEventListener("abort", onAbort, { once: true });
+	const stderrChunks = [];
+	child.stderr?.setEncoding("utf8");
+	child.stderr?.on("data", (chunk) => {
+		stderrChunks.push(chunk);
+	});
+	const stderrCollected = new Promise((resolve) => {
+		const finalize = () => resolve(stderrChunks.join(""));
+		child.stderr?.on("end", finalize);
+		child.stderr?.on("error", finalize);
+	});
+	const done = new Promise((resolve) => {
+		let settled = false;
+		const finalize = (exitCode, signal) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timeoutTimer);
+			if (killEscalation) clearTimeout(killEscalation);
+			config.signal?.removeEventListener("abort", onAbort);
+			resolve({
+				exitCode,
+				signal
+			});
+		};
+		child.on("close", (code, signal) => finalize(code, signal));
+		child.on("error", () => finalize(null, null));
+	});
+	return {
+		stdout: child.stdout,
+		done,
+		stderrCollected,
+		timedOut: () => timedOut,
+		cleanup: envCleanup
+	};
+}
+function killTree(child, signal) {
+	if (child.pid === void 0) return;
+	try {
+		process.kill(-child.pid, signal);
+	} catch {
+		try {
+			child.kill(signal);
+		} catch {}
+	}
+}
+//#endregion
+//#region src/adapters/gemini-cli/index.ts
+/**
+* Gemini CLI adapter — public API.
+*/
+/**
+* Run Gemini CLI in headless stream-json mode and return a trajectory.
+*
+* Maps NDJSON events through {@link GeminiCliEventMapper}, records parse
+* errors without aborting, and attaches {@link AdapterDiagnostics.exitCodeDescription}
+* for known non-zero exit codes (spec P-7).
+*/
+async function runGeminiCli(config) {
+	const startTs = Date.now();
+	const spawned = await spawnGeminiCli(config);
+	const builder = new TrajectoryBuilder();
+	const mapper = new GeminiCliEventMapper();
+	const rawEvents = [];
+	const parseErrors = [];
+	try {
+		for await (const result of parseGeminiCliJson(spawned.stdout)) {
+			if (!result.ok) {
+				parseErrors.push({
+					line: result.rawLine,
+					error: result.error.message
+				});
+				continue;
+			}
+			for (const event of mapper.map(result.event)) {
+				builder.consume(event);
+				rawEvents.push(event);
+			}
+		}
+		const [{ exitCode, signal }, stderr] = await Promise.all([spawned.done, spawned.stderrCollected]);
+		const exitCodeDescription = describeGeminiCliExitCode(exitCode);
+		const diagnostics = {
+			exitCode,
+			exitCodeDescription,
+			signal,
+			stderr,
+			parseErrors,
+			timedOut: spawned.timedOut(),
+			durationMs: Date.now() - startTs
+		};
+		let view;
+		try {
+			view = builder.build();
+		} catch (err) {
+			const message = err instanceof Error ? err.message : String(err);
+			const stderrHint = stderr.trim().length > 0 ? ` stderr: ${stderr.trim().slice(0, 400)}` : "";
+			throw new AdapterError(`harness produced no usable trajectory: ${message}${exitCodeDescription ? ` (${exitCodeDescription})` : ""}${stderrHint}`, diagnostics);
+		}
+		return {
+			view,
+			diagnostics,
+			rawEvents
+		};
+	} finally {
+		await spawned.cleanup();
+	}
+}
+/** Registered {@link HarnessAdapter} for Gemini CLI headless runs. */
+const geminiCliAdapter = {
+	id: "gemini-cli",
+	run: runGeminiCli
+};
+//#endregion
 //#region src/adapters/registry.ts
 const ADAPTERS = {};
 function registerBuiltIn(id, adapter) {
 	ADAPTERS[id] = adapter;
 }
 registerBuiltIn("claude-code", claudeCodeAdapter);
+registerBuiltIn("codex", codexAdapter);
+registerBuiltIn("gemini-cli", geminiCliAdapter);
 /**
 * Register a harness adapter by id.
 *
@@ -505,12 +1035,36 @@ function toClaudeCodeConfig(layers, prompt) {
 	merged.prompt = prompt;
 	return merged;
 }
+/** Merge generic suite config layers into a flat {@link CodexAdapterConfig}. */
+function toCodexConfig(layers, prompt) {
+	const merged = {};
+	for (const layer of layers) {
+		const { codex, ...generic } = layer;
+		Object.assign(merged, generic);
+		if (codex && typeof codex === "object") Object.assign(merged, codex);
+	}
+	merged.prompt = prompt;
+	return merged;
+}
+/** Merge generic suite config layers into a flat {@link GeminiCliAdapterConfig}. */
+function toGeminiCliConfig(layers, prompt) {
+	const merged = {};
+	for (const layer of layers) {
+		const { geminiCli, ...generic } = layer;
+		Object.assign(merged, generic);
+		if (geminiCli && typeof geminiCli === "object") Object.assign(merged, geminiCli);
+	}
+	merged.prompt = prompt;
+	return merged;
+}
 /**
 * Resolve merged suite layers into the flat config shape expected by the
-* selected harness adapter.
+* selected harness adapter (`claude-code`, `codex`, or `gemini-cli`).
 */
 function resolveRunConfig(adapterId, layers, prompt) {
 	if (adapterId === "claude-code" || adapterId === "claude-code") return toClaudeCodeConfig(layers, prompt);
+	if (adapterId === "codex") return toCodexConfig(layers, prompt);
+	if (adapterId === "gemini-cli") return toGeminiCliConfig(layers, prompt);
 	const merged = {};
 	for (const layer of layers) Object.assign(merged, layer);
 	merged.prompt = prompt;
@@ -733,6 +1287,6 @@ async function runSuite(suite, options = {}) {
 	return report;
 }
 //#endregion
-export { aggregateCell as a, runRepetition as c, getDefaultAdapter as d, listAdapters as f, evaluateAll as h, DEFAULT_THRESHOLD as i, DEFAULT_ADAPTER_ID as l, evaluate as m, createLimit as n, getRepetitions as o, registerAdapter as p, DEFAULT_REPETITIONS as r, mergeConfig as s, runSuite as t, getAdapter as u };
+export { evaluateAll as _, aggregateCell as a, runRepetition as c, getDefaultAdapter as d, listAdapters as f, evaluate as g, buildJudgeArgs as h, DEFAULT_THRESHOLD as i, DEFAULT_ADAPTER_ID as l, prepareGeminiCliEnv as m, createLimit as n, getRepetitions as o, registerAdapter as p, DEFAULT_REPETITIONS as r, mergeConfig as s, runSuite as t, getAdapter as u };
 
-//# sourceMappingURL=suite-Dlzl-HI0.js.map
+//# sourceMappingURL=suite-C3-8EjUW.js.map