@sverka/arena 0.1.7 → 0.1.9

package/dist/report-3Gl3woII.mjs

@@ -0,0 +1,1232 @@
+import { execSync, spawn } from "node:child_process";
+import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import "node:fs";
+import { isAbsolute, join, resolve, sep } from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { Readable, Writable } from "node:stream";
+import * as acp from "@agentclientprotocol/sdk";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { z } from "zod";
+//#region src/acp.ts
+/**
+* Shared ACP (Agent Client Protocol) utilities — extracted from the Devin
+* adapter so they can be reused by other adapters and the benchmark runner.
+*
+* This module contains:
+* - {@link sanitizeEnv}: strip host-contaminating env vars before spawning an agent.
+* - {@link createPermissionHandler}: auto-approve permission requests.
+* - {@link runAcpSession}: drive a full ACP connect/initialize/prompt/collect loop.
+*/
+/** Env-var prefixes to strip from the spawned process environment. */
+const SANITIZE_PREFIXES = [
+	"GC_",
+	"BEADS_",
+	"BD_",
+	"MCP_",
+	"CLAUDECODE",
+	"CLAUDE_CODE_",
+	"CODEX_"
+];
+/** Exact env-var names to strip from the spawned process environment. */
+const SANITIZE_EXACT = /* @__PURE__ */ new Set([
+	"DEVIN_PERMISSION_MODE",
+	"DEVIN_MODEL",
+	"AGENTS_MD",
+	"CLAUDE_PROJECT_MD"
+]);
+/** Check if an env var name should be stripped from the agent environment. */
+function shouldSanitize(key) {
+	if (SANITIZE_EXACT.has(key)) return true;
+	return SANITIZE_PREFIXES.some((p) => key.startsWith(p));
+}
+/** Copy process.env into a clean record, dropping sanitized entries. */
+function copyCleanEnv() {
+	const env = {};
+	for (const [key, value] of Object.entries(process.env)) {
+		if (value === void 0) continue;
+		if (shouldSanitize(key)) continue;
+		env[key] = value;
+	}
+	return env;
+}
+/** Apply model and permission settings to the env record. */
+function applyModelEnv(env, config) {
+	const model = config.model;
+	env.DEVIN_PERMISSION_MODE = config.permissionMode ?? "dangerous";
+	env.DEVIN_MODEL = model.envVar ? env[model.envVar] ?? model.id : model.id;
+	if (model.envVar) env[model.envVar] = env[model.envVar] ?? model.id;
+}
+/** Merge explicit env overrides from config (these win). */
+function mergeEnvOverrides(env, overrides) {
+	if (!overrides) return;
+	for (const [key, value] of Object.entries(overrides)) env[key] = value;
+}
+/**
+* Build a sanitized environment for an agent subprocess.
+*
+* Strips out host contamination: Gas City, Beads, MCP servers, project
+* skills, and other variables that would leak the host's configuration
+* into the benchmark. The agent must only see what we explicitly install.
+*
+* Sets `DEVIN_PERMISSION_MODE` and `DEVIN_MODEL` from the spawn config,
+* then merges any explicit `config.env` overrides (which win).
+*/
+function sanitizeEnv(config) {
+	const env = copyCleanEnv();
+	applyModelEnv(env, config);
+	mergeEnvOverrides(env, config.env);
+	return env;
+}
+/**
+* Create a permission-request handler that auto-approves tool calls.
+*
+* Selects the first `allow_once` or `allow_always` option, falling back to
+* the first available option. This is the same handler used by both the
+* Devin adapter and the benchmark runner.
+*/
+function createPermissionHandler() {
+	return (ctx) => {
+		const options = ctx.params.options;
+		return { outcome: {
+			outcome: "selected",
+			optionId: options.find((o) => o.kind === "allow_once" || o.kind === "allow_always")?.optionId ?? options[0]?.optionId ?? ""
+		} };
+	};
+}
+/**
+* Process a single {@link acp.ActiveSessionMessage} from the session update
+* stream. Returns the final {@link PromptResponse} when the session reports
+* a stop, otherwise invokes the appropriate callback and returns `undefined`
+* so the caller can continue collecting.
+*/
+function handleSessionUpdate(message, onToolCall, onToolUpdate) {
+	if (message.kind === "stop") return message.response;
+	const update = message.update;
+	if (update.sessionUpdate === "tool_call") onToolCall?.(update);
+	else if (update.sessionUpdate === "tool_call_update") onToolUpdate?.(update);
+}
+/**
+* Drive a single ACP session: send the prompt, race the collect loop
+* against a timeout, and return the final {@link PromptResponse}.
+*/
+async function runSessionLoop(session, prompt, timeoutMs, onToolCall, onToolUpdate, onSession) {
+	onSession?.(session.sessionId);
+	const promptPromise = session.prompt(prompt);
+	const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error(`Agent timed out after ${timeoutMs}ms`)), timeoutMs));
+	const collectPromise = (async () => {
+		for (;;) {
+			const result = handleSessionUpdate(await session.nextUpdate(), onToolCall, onToolUpdate);
+			if (result) return result;
+		}
+	})();
+	return Promise.race([
+		promptPromise,
+		collectPromise,
+		timeoutPromise
+	]);
+}
+/**
+* Run a full ACP session against a spawned agent subprocess.
+*
+* Connects to the agent, initializes the protocol, creates a session in
+* `workspace`, sends `prompt`, and collects update events until the session
+* stops or `timeoutMs` elapses.
+*
+* The `onToolCall` and `onToolUpdate` callbacks let callers collect tool
+* calls and observations without duplicating the session loop logic. The
+* optional `onSession` callback reports the session ID once created. All
+* callbacks are optional.
+*
+* @returns The final {@link PromptResponse} from the agent.
+*/
+async function runAcpSession(proc, workspace, prompt, timeoutMs, onToolCall, onToolUpdate, onSession) {
+	if (!proc.stdin || !proc.stdout) throw new Error("Agent process missing stdin/stdout");
+	const input = Writable.toWeb(proc.stdin);
+	const output = Readable.toWeb(proc.stdout);
+	const stream = acp.ndJsonStream(input, output);
+	return acp.client({ name: "sverka-arena" }).onRequest(acp.methods.client.session.requestPermission, createPermissionHandler()).connectWith(stream, async (ctx) => {
+		await ctx.request(acp.methods.agent.initialize, {
+			protocolVersion: acp.PROTOCOL_VERSION,
+			clientCapabilities: {}
+		});
+		return ctx.buildSession(workspace).withSession((session) => runSessionLoop(session, prompt, timeoutMs, onToolCall, onToolUpdate, onSession));
+	});
+}
+//#endregion
+//#region src/adapters/devin.ts
+/**
+* Devin ACP adapter — spawns `devin acp` as a subprocess and drives an
+* Agent Client Protocol session to collect trace data + metrics.
+*
+* The adapter is abstract over models and plugins: it accepts any
+* {@link ModelConfig} and any set of {@link PluginConfig}s, installs the
+* enabled plugins into the workspace, and runs the prompt against the
+* spawned agent. After the session completes it reads the Devin CLI
+* transcript file to build full {@link TraceData}.
+*/
+/** Default transcript directory: ~/.local/share/devin/cli/transcripts */
+function transcriptDir() {
+	return join(homedir(), ".local", "share", "devin", "cli", "transcripts");
+}
+/** Read a transcript JSON file by session ID. Throws if not found. */
+async function readTranscript(sessionId, dir) {
+	const path = join(dir ?? transcriptDir(), `${sessionId}.json`);
+	const content = await readFile(path, "utf-8");
+	return JSON.parse(content);
+}
+/** Count LLM inference calls in a transcript (agent steps with telemetry.operation === "inference"). */
+function countLlmCalls(transcript) {
+	return transcript.steps.filter((s) => s.source === "agent" && s.extra?.telemetry?.operation === "inference").length;
+}
+/** Build a {@link TraceStep} from a raw transcript step. */
+function buildTraceStep(step) {
+	const isLlmCall = step.source === "agent" && step.extra?.telemetry?.operation === "inference";
+	return {
+		stepId: step.step_id,
+		timestamp: step.timestamp,
+		source: step.source,
+		message: step.message,
+		isLlmCall
+	};
+}
+/** Extract text content from a ToolCallContent array (best-effort). */
+function extractTextContent(contents) {
+	if (!contents) return "";
+	const parts = [];
+	for (const c of contents) if (c.type === "content" && c.content.type === "text") parts.push(c.content.text);
+	return parts.join("\n");
+}
+/** Extract token counts from a Usage object, defaulting to 0. */
+function extractUsageTokens(usage) {
+	return {
+		inputTokens: usage?.inputTokens ?? 0,
+		outputTokens: usage?.outputTokens ?? 0,
+		thoughtTokens: usage?.thoughtTokens ?? 0,
+		totalTokens: usage?.totalTokens ?? 0
+	};
+}
+/** Build {@link RunMetrics} from a PromptResponse + collected counts + timing. */
+function buildMetrics(promptResult, toolCallCount, llmCallCount, startTime) {
+	return {
+		...extractUsageTokens(promptResult.usage),
+		toolCallCount,
+		llmCallCount,
+		executionTimeMs: Date.now() - startTime,
+		stopReason: promptResult.stopReason
+	};
+}
+/**
+* Extract the agent's final text output from the prompt response or trace.
+*
+* Prefers the text content from the ACP `PromptResponse`, falling back to
+* the last agent message in the trace.
+*/
+function extractAgentOutput(promptResult, trace) {
+	for (let i = trace.steps.length - 1; i >= 0; i--) {
+		const step = trace.steps[i];
+		if (step?.source === "agent" && step.message.length > 0) return step.message;
+	}
+	return "";
+}
+/**
+* Install enabled plugins and remove disabled ones from the workspace.
+*
+* CRITICAL: This function first removes the entire `.agents/skills/`
+* directory to ensure no inherited skills from the host contaminate the
+* benchmark. Only explicitly enabled plugins are installed.
+*
+* Each enabled plugin's `path` is copied to `workspace/.agents/skills/<id>/`.
+*/
+async function installPlugins(workspace, plugins) {
+	const skillsDir = join(workspace, ".agents", "skills");
+	await rm(skillsDir, {
+		recursive: true,
+		force: true
+	});
+	const enabledPlugins = plugins.filter((p) => p.enabled);
+	if (enabledPlugins.length === 0) return;
+	await mkdir(skillsDir, { recursive: true });
+	for (const plugin of enabledPlugins) {
+		if (plugin.id.includes("/") || plugin.id.includes("\\") || plugin.id.includes("..")) throw new Error(`Invalid plugin id: ${plugin.id}`);
+		const dest = join(skillsDir, plugin.id);
+		await mkdir(dest, { recursive: true });
+		await cp(plugin.path, dest, { recursive: true });
+	}
+}
+/**
+* Devin ACP adapter — spawns `devin acp` as a subprocess.
+*
+* Usage:
+* ```ts
+* const adapter = new DevinAdapter();
+* const proc = adapter.spawn({ model, workspace, plugins });
+* const result = await proc.run("Fix the bug", 120_000);
+* proc.kill();
+* ```
+*/
+var DevinAdapter = class {
+	id = "devin";
+	spawn(config) {
+		const proc = spawnDevin(config);
+		let killed = false;
+		return {
+			run: (prompt, timeoutMs) => runDevinSession(proc, config, prompt, timeoutMs),
+			kill: () => {
+				if (killed) return;
+				killed = true;
+				proc.kill("SIGTERM");
+			}
+		};
+	}
+};
+/** Resolve the full path to the devin binary to avoid PATH-based lookup. */
+function resolveDevinBinary() {
+	try {
+		return execSync("which devin", {
+			encoding: "utf-8",
+			stdio: [
+				"pipe",
+				"pipe",
+				"pipe"
+			]
+		}).trim();
+	} catch {
+		return "devin";
+	}
+}
+/** Spawn `devin acp --model <model.id>` as a subprocess in the workspace. */
+function spawnDevin(config) {
+	const env = sanitizeEnv(config);
+	const devinBin = resolveDevinBinary();
+	return spawn(devinBin, [
+		"acp",
+		"--model",
+		config.model.id
+	], {
+		cwd: config.workspace,
+		stdio: [
+			"pipe",
+			"pipe",
+			"inherit"
+		],
+		env
+	});
+}
+/** Create a {@link ToolCollector} with callbacks that populate shared state. */
+function createToolCollector() {
+	const collectedToolCalls = [];
+	const observationsByCallId = /* @__PURE__ */ new Map();
+	let toolCallCount = 0;
+	const onToolCall = (update) => {
+		toolCallCount++;
+		const name = update.name ?? update.title ?? "unknown";
+		const args = update.rawInput ?? {};
+		collectedToolCalls.push({
+			functionName: name,
+			arguments: args,
+			toolCallId: update.toolCallId
+		});
+	};
+	const onToolUpdate = (update) => {
+		const text = extractTextContent(update.content ?? void 0);
+		if (text) observationsByCallId.set(update.toolCallId, {
+			sourceCallId: update.toolCallId,
+			content: text
+		});
+	};
+	return {
+		collectedToolCalls,
+		observationsByCallId,
+		get toolCallCount() {
+			return toolCallCount;
+		},
+		onToolCall,
+		onToolUpdate
+	};
+}
+/** Build a {@link RunResult} for a failed session (best-effort trace from collected data). */
+function buildErrorResult(model, pluginIds, toolCallCount, sessionId, startTime, error) {
+	const metrics = buildMetrics({
+		stopReason: "cancelled",
+		usage: null
+	}, toolCallCount, 0, startTime);
+	const trace = {
+		sessionId,
+		model: model.id,
+		steps: [],
+		finalMetrics: {
+			totalPromptTokens: 0,
+			totalCompletionTokens: 0,
+			totalCachedTokens: 0,
+			totalSteps: 0
+		}
+	};
+	return {
+		taskId: "",
+		modelId: model.id,
+		pluginIds,
+		metrics,
+		trace,
+		output: "",
+		checkResults: [],
+		verdicts: [],
+		success: false,
+		error: error instanceof Error ? error.message : String(error)
+	};
+}
+/** Run a full ACP session: initialize, create session, prompt, collect events, read transcript. */
+async function runDevinSession(proc, config, prompt, timeoutMs) {
+	const startTime = Date.now();
+	const model = config.model;
+	const pluginIds = config.plugins.filter((p) => p.enabled).map((p) => p.id);
+	const collector = createToolCollector();
+	let sessionId = "";
+	try {
+		const promptResult = await runAcpSession(proc, config.workspace, prompt, timeoutMs, collector.onToolCall, collector.onToolUpdate, (id) => {
+			sessionId = id;
+		});
+		const trace = await buildTrace(sessionId, model, collector.collectedToolCalls, collector.observationsByCallId);
+		const llmCallCount = trace.steps.filter((s) => s.isLlmCall).length;
+		const metrics = buildMetrics(promptResult, collector.toolCallCount, llmCallCount, startTime);
+		const agentOutput = extractAgentOutput(promptResult, trace);
+		return {
+			taskId: "",
+			modelId: model.id,
+			pluginIds,
+			metrics,
+			trace,
+			output: agentOutput,
+			checkResults: [],
+			verdicts: [],
+			success: metrics.stopReason === "end_turn"
+		};
+	} catch (error) {
+		return buildErrorResult(model, pluginIds, collector.toolCallCount, sessionId, startTime, error);
+	}
+}
+/**
+* Build {@link TraceData} from the Devin transcript, enriched with
+* ACP-collected tool calls. Returns `undefined` when no transcript is
+* available so the caller can fall back to {@link buildTraceFallback}.
+*/
+async function buildTraceFromTranscript(sessionId, collectedToolCalls, observationsByCallId) {
+	if (!sessionId) return void 0;
+	try {
+		const transcript = await readTranscript(sessionId);
+		const steps = transcript.steps.map(buildTraceStep);
+		attachToolCallsToSteps(steps, collectedToolCalls, observationsByCallId);
+		return {
+			sessionId: transcript.session_id,
+			model: transcript.agent.model_name,
+			steps,
+			finalMetrics: {
+				totalPromptTokens: transcript.final_metrics.total_prompt_tokens,
+				totalCompletionTokens: transcript.final_metrics.total_completion_tokens,
+				totalCachedTokens: transcript.final_metrics.total_cached_tokens,
+				totalSteps: transcript.final_metrics.total_steps
+			}
+		};
+	} catch {
+		return;
+	}
+}
+/** Attach the next tool call + observation to an agent step. Returns true if attached. */
+function attachNextToolCall(step, collectedToolCalls, observationsByCallId, toolIdx) {
+	if (toolIdx >= collectedToolCalls.length) return {
+		attached: false,
+		nextIdx: toolIdx
+	};
+	const tc = collectedToolCalls[toolIdx];
+	if (!tc) return {
+		attached: false,
+		nextIdx: toolIdx
+	};
+	const calls = [tc];
+	const obs = observationsByCallId.get(tc.toolCallId);
+	const observations = obs ? [obs] : [];
+	step.toolCalls = calls;
+	if (observations.length > 0) step.observations = observations;
+	return {
+		attached: true,
+		nextIdx: toolIdx + 1
+	};
+}
+/** Attach ACP-collected tool calls/observations to agent steps in order. */
+function attachToolCallsToSteps(steps, collectedToolCalls, observationsByCallId) {
+	let toolIdx = 0;
+	for (const step of steps) {
+		if (step.source !== "agent") continue;
+		const result = attachNextToolCall(step, collectedToolCalls, observationsByCallId, toolIdx);
+		if (result.attached) toolIdx = result.nextIdx;
+	}
+}
+/** Build a minimal {@link TraceData} from ACP-collected data only. */
+function buildTraceFallback(sessionId, model, collectedToolCalls, observationsByCallId) {
+	const steps = collectedToolCalls.map((tc, i) => {
+		const obs = observationsByCallId.get(tc.toolCallId);
+		const step = {
+			stepId: i,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			source: "agent",
+			message: tc.functionName,
+			toolCalls: [tc],
+			isLlmCall: false
+		};
+		if (obs) step.observations = [obs];
+		return step;
+	});
+	return {
+		sessionId,
+		model: model.id,
+		steps,
+		finalMetrics: {
+			totalPromptTokens: 0,
+			totalCompletionTokens: 0,
+			totalCachedTokens: 0,
+			totalSteps: steps.length
+		}
+	};
+}
+/** Build {@link TraceData} from the Devin transcript, falling back to ACP-only data. */
+async function buildTrace(sessionId, model, collectedToolCalls, observationsByCallId) {
+	return await buildTraceFromTranscript(sessionId, collectedToolCalls, observationsByCallId) ?? buildTraceFallback(sessionId, model, collectedToolCalls, observationsByCallId);
+}
+//#endregion
+//#region src/judge.ts
+/** Default system prompt for the judge. */
+const DEFAULT_JUDGE_PROMPT = "You are a blind evaluator. You will receive a task prompt and an agent's response. Score the response 0-100 based on how well it completes the task. Be strict but fair. Output JSON: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70+ is passing.";
+/** Default timeout for a single judge run (ms). */
+const DEFAULT_JUDGE_TIMEOUT_MS = 12e4;
+/**
+* Build the judge prompt for a single run.
+*
+* Includes the task prompt, success criteria, and the agent's output. Does
+* NOT include which plugins were active unless `config.revealPlugins` is
+* true. Asks the judge for a JSON response with score, passed, reasoning,
+* and issues.
+*/
+function buildJudgePrompt(task, runOutput, config, pluginIds = []) {
+	const lines = [
+		config.systemPrompt ?? "You are a blind evaluator. You will receive a task prompt and an agent's response. Score the response 0-100 based on how well it completes the task. Be strict but fair. Output JSON: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70+ is passing.",
+		"",
+		"## Task Prompt",
+		task.prompt,
+		""
+	];
+	if (task.successCriteria) lines.push("## Success Criteria", task.successCriteria, "");
+	if (config.revealPlugins && task.id) {
+		lines.push("## Plugins Active During Run");
+		if (pluginIds.length > 0) lines.push(pluginIds.join(", "));
+		else lines.push("(no plugins)");
+		lines.push("");
+	}
+	lines.push("## Agent Response");
+	lines.push(runOutput || "(no output)");
+	lines.push("");
+	lines.push("## Instructions\nEvaluate the agent's response against the task prompt and success criteria. Respond with ONLY a JSON object: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70 or above is passing. Do not include any text outside the JSON.");
+	return lines.join("\n");
+}
+/**
+* Parse a judge response into a {@link JudgeVerdict}.
+*
+* Extracts JSON from the response, handling markdown code fences
+* (` ```json ... ``` `). Falls back to `score=0, passed=false` when the
+* response cannot be parsed.
+*/
+function parseJudgeResponse(response, runIndex, run) {
+	const jsonText = extractJson(response);
+	const fallback = {
+		runIndex,
+		score: 0,
+		passed: false,
+		reasoning: response.trim() || "Failed to parse judge response",
+		issues: [],
+		taskId: run.taskId,
+		modelId: run.modelId,
+		pluginIds: run.pluginIds
+	};
+	if (jsonText === null) return fallback;
+	let parsed;
+	try {
+		parsed = JSON.parse(jsonText);
+	} catch {
+		return fallback;
+	}
+	if (typeof parsed !== "object" || parsed === null) return fallback;
+	const obj = parsed;
+	const score = typeof obj.score === "number" ? obj.score : 0;
+	return {
+		runIndex,
+		score,
+		passed: typeof obj.passed === "boolean" ? obj.passed : score >= 70,
+		reasoning: typeof obj.reasoning === "string" ? obj.reasoning : "",
+		issues: Array.isArray(obj.issues) ? obj.issues.filter((i) => typeof i === "string") : [],
+		taskId: run.taskId,
+		modelId: run.modelId,
+		pluginIds: run.pluginIds
+	};
+}
+/**
+* Judge a single run — spawn the judge agent, send the prompt, collect the
+* response, parse it into a {@link JudgeVerdict}, and kill the agent.
+*/
+async function judgeRun(run, task, config, runIndex) {
+	const prompt = buildJudgePrompt(task, run.output, config, run.pluginIds);
+	const proc = config.agent.spawn({
+		model: config.model,
+		workspace: process.cwd(),
+		plugins: [],
+		permissionMode: "dangerous"
+	});
+	try {
+		return parseJudgeResponse((await proc.run(prompt, DEFAULT_JUDGE_TIMEOUT_MS)).output, runIndex, run);
+	} catch (error) {
+		return {
+			runIndex,
+			score: 0,
+			passed: false,
+			reasoning: error instanceof Error ? error.message : String(error),
+			issues: [],
+			taskId: run.taskId,
+			modelId: run.modelId,
+			pluginIds: run.pluginIds
+		};
+	} finally {
+		proc.kill();
+	}
+}
+/**
+* Judge all runs in a list, pairing each run with its matching task.
+*
+* Supports `config.repetitions` — the judge is run N times per run for
+* confidence. All verdicts are returned (one per repetition).
+*/
+async function judgeAllRuns(results, tasks, config) {
+	const taskById = /* @__PURE__ */ new Map();
+	for (const task of tasks) taskById.set(task.id, task);
+	const repetitions = config.repetitions ?? 1;
+	const verdicts = [];
+	for (let i = 0; i < results.length; i++) {
+		const run = results[i];
+		if (!run) continue;
+		const task = taskById.get(run.taskId);
+		if (!task) continue;
+		for (let rep = 0; rep < repetitions; rep++) {
+			const verdict = await judgeRun(run, task, config, i);
+			verdicts.push(verdict);
+		}
+	}
+	return verdicts;
+}
+/**
+* Compare two plugin combos for a task (for {@link ComboComparison}).
+*
+* Computes deltas as `candidate - baseline`. Determines whether the
+* candidate is better:
+*   - Lower tokens / toolCalls / llmCalls / time = better
+*   - Higher judgeScore = better
+*
+* The judge score for each run is the average of its verdicts' scores
+* (0 when there are no verdicts).
+*/
+function compareCombos(baseline, candidate) {
+	const baselineScore = avgJudgeScore(baseline);
+	const candidateScore = avgJudgeScore(candidate);
+	const deltaTokens = candidate.metrics.totalTokens - baseline.metrics.totalTokens;
+	const deltaToolCalls = candidate.metrics.toolCallCount - baseline.metrics.toolCallCount;
+	const deltaLlmCalls = candidate.metrics.llmCallCount - baseline.metrics.llmCallCount;
+	const deltaTimeMs = candidate.metrics.executionTimeMs - baseline.metrics.executionTimeMs;
+	const deltaJudgeScore = candidateScore - baselineScore;
+	const candidateBetter = deltaJudgeScore > 0 || deltaJudgeScore === 0 && deltaTokens <= 0 && deltaToolCalls <= 0 && deltaLlmCalls <= 0 && deltaTimeMs <= 0 && (deltaTokens < 0 || deltaToolCalls < 0 || deltaLlmCalls < 0 || deltaTimeMs < 0);
+	return {
+		baseline: comboLabel$1(baseline),
+		candidate: comboLabel$1(candidate),
+		deltaTokens,
+		deltaToolCalls,
+		deltaLlmCalls,
+		deltaTimeMs,
+		deltaJudgeScore,
+		candidateBetter
+	};
+}
+/**
+* Extract the first JSON object from a response string.
+*
+* Handles raw JSON and JSON wrapped in markdown code fences
+* (` ```json ... ``` ` or ` ``` ... ``` `). Returns `null` when no JSON
+* object can be located.
+*/
+function extractJson(response) {
+	const trimmed = response.trim();
+	if (trimmed === "") return null;
+	const fenced = extractFromFence(trimmed);
+	if (fenced !== null) return fenced;
+	const braced = extractFromBraces(trimmed);
+	if (braced !== null) return braced;
+	return null;
+}
+/** Extract JSON from inside a markdown code fence (```...```). */
+function extractFromFence(text) {
+	const fenceStart = text.indexOf("```");
+	if (fenceStart === -1) return null;
+	const afterFence = text.slice(fenceStart + 3);
+	let contentStart = 0;
+	if (afterFence.startsWith("json")) contentStart = 4;
+	while (contentStart < afterFence.length && /\s/.test(afterFence[contentStart])) contentStart++;
+	const fenceEnd = afterFence.indexOf("```", contentStart);
+	if (fenceEnd === -1) return null;
+	const inner = afterFence.slice(contentStart, fenceEnd).trim();
+	if (!looksLikeJson(inner)) return null;
+	return inner;
+}
+/** Extract JSON by finding the first `{` and last `}` in the text. */
+function extractFromBraces(text) {
+	const start = text.indexOf("{");
+	const end = text.lastIndexOf("}");
+	if (start === -1 || end === -1 || end <= start) return null;
+	return text.slice(start, end + 1);
+}
+/** Quick check that a string starts with `{` (a JSON object). */
+function looksLikeJson(text) {
+	return text.trim().startsWith("{");
+}
+/** Average judge score across a run's verdicts (0 when no verdicts). */
+function avgJudgeScore(run) {
+	if (run.verdicts.length === 0) return 0;
+	return run.verdicts.reduce((a, v) => a + v.score, 0) / run.verdicts.length;
+}
+/** Build a human-readable combo label for a run's plugin set. */
+function comboLabel$1(run) {
+	if (run.pluginIds.length === 0) return "no-plugins";
+	return run.pluginIds.join(",");
+}
+//#endregion
+//#region src/runner.ts
+/**
+* Matrix runner — executes the full arena matrix:
+*
+*   task × model × plugin combination × repetition
+*
+* For each cell it spawns the agent, runs the prompt, collects the result,
+* and kills the agent. Results are aggregated into {@link ArenaResult}.
+*/
+/** Resolve the arena package root (for fixture paths). */
+const PKG_ROOT = resolve(typeof __dirname !== "undefined" ? __dirname : fileURLToPath(new URL(".", import.meta.url)), "..");
+/**
+* Generate all plugin on/off combinations.
+*
+* For N plugins this produces 2^N combinations, each a full copy of the
+* plugins array with `enabled` set appropriately.
+*
+* - 0 plugins → `[[]]` (one empty combination)
+* - 1 plugin  → `[[off], [on]]`
+* - 2 plugins → `[[off,off], [on,off], [off,on], [on,on]]`
+*/
+function pluginCombinations(plugins) {
+	if (plugins.length === 0) return [[]];
+	const combos = [[]];
+	for (const plugin of plugins) {
+		const next = [];
+		for (const combo of combos) {
+			next.push([...combo, {
+				...plugin,
+				enabled: false
+			}]);
+			next.push([...combo, {
+				...plugin,
+				enabled: true
+			}]);
+		}
+		combos.length = 0;
+		combos.push(...next);
+	}
+	return combos;
+}
+/**
+* Run the full arena matrix: task × model × plugin combo × repetitions.
+*
+* Each cell spawns the configured agent adapter, installs the plugin
+* combination, runs the prompt, collects trace + metrics, and kills the
+* agent. Results are written to `config.outputDir/results.json` and the
+* full {@link ArenaResult} is returned.
+*/
+async function runArena(config) {
+	const repetitions = config.repetitions ?? 1;
+	const combos = pluginCombinations(config.plugins);
+	const results = await runMatrix(config, combos);
+	if (config.judge) {
+		const verdicts = await judgeAllRuns(results, config.tasks, config.judge);
+		for (const verdict of verdicts) {
+			const run = results[verdict.runIndex];
+			if (run) run.verdicts.push(verdict);
+		}
+	}
+	const aggregates = buildAggregates(results, config.models, combos);
+	const analysis = computeAnalysis(results, config.tasks);
+	const arenaResult = {
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		config: {
+			models: config.models.map((m) => m.id),
+			plugins: config.plugins.map((p) => p.id),
+			tasks: config.tasks.map((t) => t.id),
+			repetitions,
+			...config.judge ? { judgeModel: config.judge.model.id } : {}
+		},
+		results,
+		aggregates,
+		analysis
+	};
+	await mkdir(config.outputDir, { recursive: true });
+	await writeFile(join(config.outputDir, "results.json"), JSON.stringify(arenaResult, null, 2), "utf-8");
+	return arenaResult;
+}
+/**
+* Run every cell of the task × model × combo × repetition matrix, collecting
+* results and logging per-cell progress to stderr.
+*/
+async function runMatrix(config, combos) {
+	const repetitions = config.repetitions ?? 1;
+	const results = [];
+	for (const task of config.tasks) for (const model of config.models) for (const combo of combos) for (let rep = 0; rep < repetitions; rep++) {
+		const cellLabel = formatCell(task.id, model.id, combo, rep);
+		process.stderr.write(`[arena] ${cellLabel} running...\n`);
+		const result = await runCell(config.agent, task, model, combo, rep);
+		results.push(result);
+		process.stderr.write(`[arena] ${cellLabel} ${result.success ? "PASS" : "FAIL"} — ${result.metrics.totalTokens} tokens, ${result.metrics.toolCallCount} tools, ${result.metrics.llmCallCount} llm calls, ${result.metrics.executionTimeMs}ms\n`);
+	}
+	return results;
+}
+/**
+* Aggregate metrics per (model × plugin combo) cell.
+*/
+function buildAggregates(results, models, combos) {
+	const aggregates = [];
+	for (const model of models) for (const combo of combos) {
+		const pluginIds = combo.filter((p) => p.enabled).map((p) => p.id);
+		const label = formatAggregateLabel(model.id, pluginIds);
+		aggregates.push(aggregateResults(results, (r) => r.modelId === model.id && samePluginIds(r.pluginIds, pluginIds), label));
+	}
+	return aggregates;
+}
+/** Run a single matrix cell in an ISOLATED temp workspace. */
+async function runCell(agent, task, model, combo, rep) {
+	const tempWorkspace = await createTempWorkspace(task, model, combo, rep);
+	await installPlugins(tempWorkspace, combo);
+	const result = await executeRun(agent, task, model, combo, tempWorkspace);
+	await rm(tempWorkspace, {
+		recursive: true,
+		force: true
+	});
+	return result;
+}
+/**
+* Create an isolated temp workspace with a fresh copy of the fixture (if any).
+* Each run gets its own copy to prevent skill contamination from the host repo.
+*/
+async function createTempWorkspace(task, model, combo, rep) {
+	const tempWorkspace = join(tmpdir(), `arena-${task.id}-${model.id}-${combo.map((p) => p.id + (p.enabled ? "on" : "off")).join(",")}-r${rep}-${Date.now()}`);
+	await mkdir(tempWorkspace, { recursive: true });
+	if (task.fixture) {
+		const fixturePath = resolve(PKG_ROOT, task.fixture);
+		if (!fixturePath.startsWith(PKG_ROOT + sep) && fixturePath !== PKG_ROOT) throw new Error(`Fixture path escapes package root: ${task.fixture}`);
+		await cp(fixturePath, tempWorkspace, { recursive: true });
+	}
+	return tempWorkspace;
+}
+/**
+* Spawn the agent, run the prompt, run deterministic checks, and return the
+* result. On error returns a zeroed {@link RunResult} via {@link errorResult}.
+*/
+async function executeRun(agent, task, model, combo, tempWorkspace) {
+	const proc = agent.spawn({
+		model,
+		workspace: tempWorkspace,
+		plugins: combo,
+		permissionMode: "dangerous"
+	});
+	try {
+		const result = await proc.run(task.prompt, task.timeoutMs ?? 12e4);
+		result.taskId = task.id;
+		if (!result.verdicts) result.verdicts = [];
+		if (!result.checkResults) result.checkResults = [];
+		if (task.checks && task.checks.length > 0) {
+			result.checkResults = await runChecks(tempWorkspace, task.checks);
+			result.success = result.success && result.checkResults.every((c) => c.passed);
+		}
+		return result;
+	} catch (error) {
+		return errorResult(task, model, combo, error);
+	} finally {
+		proc.kill();
+	}
+}
+/** Build a zeroed {@link RunResult} for a failed run. */
+function errorResult(task, model, combo, error) {
+	return {
+		taskId: task.id,
+		modelId: model.id,
+		pluginIds: combo.filter((p) => p.enabled).map((p) => p.id),
+		metrics: {
+			inputTokens: 0,
+			outputTokens: 0,
+			thoughtTokens: 0,
+			totalTokens: 0,
+			toolCallCount: 0,
+			llmCallCount: 0,
+			executionTimeMs: 0,
+			stopReason: "cancelled"
+		},
+		trace: {
+			sessionId: "",
+			model: model.id,
+			steps: [],
+			finalMetrics: {
+				totalPromptTokens: 0,
+				totalCompletionTokens: 0,
+				totalCachedTokens: 0,
+				totalSteps: 0
+			}
+		},
+		output: "",
+		checkResults: [],
+		verdicts: [],
+		success: false,
+		error: error instanceof Error ? error.message : String(error)
+	};
+}
+/** Run deterministic checks in the workspace. */
+async function runChecks(workspace, checks) {
+	if (!checks) return [];
+	const results = [];
+	for (const check of checks) try {
+		const { output, exitCode } = await new Promise((resolve, reject) => {
+			const proc = spawn("bash", ["-c", check.command], {
+				cwd: workspace,
+				stdio: [
+					"pipe",
+					"pipe",
+					"pipe"
+				],
+				env: {
+					...process.env,
+					CI: "true"
+				}
+			});
+			let stdout = "";
+			proc.stdout?.on("data", (d) => {
+				stdout += d.toString();
+			});
+			proc.stderr?.on("data", (d) => {
+				stdout += d.toString();
+			});
+			proc.on("close", (code) => {
+				resolve({
+					output: stdout,
+					exitCode: code
+				});
+			});
+			proc.on("error", reject);
+		});
+		results.push({
+			checkId: check.id,
+			passed: exitCode === 0,
+			output: output.trim(),
+			exitCode
+		});
+	} catch (error) {
+		results.push({
+			checkId: check.id,
+			passed: false,
+			output: error instanceof Error ? error.message : String(error),
+			exitCode: -1
+		});
+	}
+	return results;
+}
+/**
+* Aggregate metrics for a group of runs selected by `filter`.
+*
+* Computes averages across all matching runs. Returns zeroed metrics when
+* no runs match. Includes judge-related fields (`avgJudgeScore`,
+* `judgePassCount`) and a `label` identifying the aggregate group.
+*/
+function aggregateResults(results, filter, label = "") {
+	const filtered = results.filter(filter);
+	const n = filtered.length;
+	if (n === 0) return {
+		totalRuns: 0,
+		successCount: 0,
+		avgInputTokens: 0,
+		avgOutputTokens: 0,
+		avgTotalTokens: 0,
+		avgToolCalls: 0,
+		avgLlmCalls: 0,
+		avgExecutionTimeMs: 0,
+		avgJudgeScore: 0,
+		judgePassCount: 0,
+		label
+	};
+	const sum = (f) => filtered.reduce((a, r) => a + f(r), 0);
+	const round = (v) => Math.round(v);
+	const round2 = (v) => Math.round(v * 100) / 100;
+	const judgedRuns = filtered.filter((r) => r.verdicts.length > 0);
+	const avgJudgeScore = judgedRuns.length === 0 ? 0 : round2(judgedRuns.reduce((a, r) => a + (r.verdicts[0]?.score ?? 0), 0) / judgedRuns.length);
+	const judgePassCount = filtered.filter((r) => r.verdicts[0]?.passed === true).length;
+	return {
+		totalRuns: n,
+		successCount: filtered.filter((r) => r.success).length,
+		avgInputTokens: round(sum((r) => r.metrics.inputTokens) / n),
+		avgOutputTokens: round(sum((r) => r.metrics.outputTokens) / n),
+		avgTotalTokens: round(sum((r) => r.metrics.totalTokens) / n),
+		avgToolCalls: round2(sum((r) => r.metrics.toolCallCount) / n),
+		avgLlmCalls: round2(sum((r) => r.metrics.llmCallCount) / n),
+		avgExecutionTimeMs: round(sum((r) => r.metrics.executionTimeMs) / n),
+		avgJudgeScore,
+		judgePassCount,
+		label
+	};
+}
+/** Compare two plugin-id arrays for equality (order-insensitive). */
+function samePluginIds(a, b) {
+	if (a.length !== b.length) return false;
+	const set = new Set(a);
+	for (const id of b) if (!set.has(id)) return false;
+	return true;
+}
+/** Format a human-readable cell label for logging. */
+function formatCell(taskId, modelId, combo, rep) {
+	return `${taskId}/${modelId}/{${combo.map((p) => `${p.id}=${p.enabled ? "on" : "off"}`).join(",")}}#${rep}`;
+}
+/** Format an aggregate group label: "model-id/plugins=on,off" or "model-id/no-plugins". */
+function formatAggregateLabel(modelId, pluginIds) {
+	if (pluginIds.length === 0) return `${modelId}/no-plugins`;
+	return `${modelId}/plugins=${pluginIds.join(",")}`;
+}
+/** Format a plugin combo label for comparisons: "no-plugins" or comma-joined ids. */
+function comboLabel(pluginIds) {
+	if (pluginIds.length === 0) return "no-plugins";
+	return pluginIds.join(",");
+}
+/**
+* Create a synthetic "average" {@link RunResult} representing a group of
+* runs with the same plugin combo. Used to feed {@link compareCombos} which
+* operates on single runs.
+*/
+function averageRun(runs) {
+	if (runs.length === 0) return null;
+	const n = runs.length;
+	const avg = (f) => Math.round(runs.reduce((a, r) => a + f(r), 0) / n);
+	const avg2 = (f) => Math.round(runs.reduce((a, r) => a + f(r), 0) / n * 100) / 100;
+	const first = runs[0];
+	return {
+		taskId: first.taskId,
+		modelId: first.modelId,
+		pluginIds: first.pluginIds,
+		metrics: {
+			inputTokens: avg((r) => r.metrics.inputTokens),
+			outputTokens: avg((r) => r.metrics.outputTokens),
+			thoughtTokens: avg((r) => r.metrics.thoughtTokens),
+			totalTokens: avg((r) => r.metrics.totalTokens),
+			toolCallCount: avg2((r) => r.metrics.toolCallCount),
+			llmCallCount: avg2((r) => r.metrics.llmCallCount),
+			executionTimeMs: avg((r) => r.metrics.executionTimeMs),
+			stopReason: first.metrics.stopReason
+		},
+		trace: first.trace,
+		output: first.output,
+		checkResults: first.checkResults ?? [],
+		verdicts: first.verdicts,
+		success: runs.every((r) => r.success)
+	};
+}
+/**
+* Compute per-task analysis comparing baseline (no plugins) vs each
+* candidate plugin combo.
+*
+* For each task, groups runs by plugin combo, creates an averaged
+* representative run per group, then compares the baseline (empty
+* `pluginIds`) against every non-empty candidate combo using
+* {@link compareCombos} from `judge.ts`.
+*/
+function computeAnalysis(results, tasks) {
+	const analyses = [];
+	for (const task of tasks) {
+		const taskRuns = results.filter((r) => r.taskId === task.id);
+		const comboGroups = /* @__PURE__ */ new Map();
+		for (const run of taskRuns) {
+			const key = comboLabel(run.pluginIds);
+			const group = comboGroups.get(key);
+			if (group) group.push(run);
+			else comboGroups.set(key, [run]);
+		}
+		const baseline = averageRun(comboGroups.get("no-plugins") ?? []);
+		const comparisons = [];
+		if (baseline) for (const [key, candidateRuns] of comboGroups) {
+			if (key === "no-plugins") continue;
+			const candidate = averageRun(candidateRuns);
+			if (candidate) comparisons.push(compareCombos(baseline, candidate));
+		}
+		analyses.push({
+			taskId: task.id,
+			taskName: task.name,
+			prompt: task.prompt,
+			comparisons
+		});
+	}
+	return analyses;
+}
+//#endregion
+//#region src/config.ts
+/**
+* Arena config file support — `arena.config.ts` exports declarative data;
+* the CLI resolves the agent adapter by name and validates the shape with
+* zod. See specs/49-arena-cli.
+*/
+var ArenaError = class extends Error {
+	code;
+	constructor(message, code) {
+		super(message);
+		this.code = code;
+		this.name = "ArenaError";
+	}
+};
+const ADAPTERS = { devin: () => new DevinAdapter() };
+function resolveAdapter(name) {
+	if (!Object.hasOwn(ADAPTERS, name)) throw new ArenaError(`unknown agent '${name}' — registered adapters: ${Object.keys(ADAPTERS).join(", ")}`, "UNKNOWN_AGENT");
+	return ADAPTERS[name]();
+}
+const modelSchema = z.object({
+	id: z.string(),
+	name: z.string(),
+	envVar: z.string().optional()
+});
+const pluginSchema = z.object({
+	id: z.string(),
+	name: z.string(),
+	path: z.string(),
+	enabled: z.boolean().optional(),
+	installPath: z.string().optional()
+});
+const checkSchema = z.object({
+	id: z.string(),
+	command: z.string(),
+	description: z.string()
+});
+const taskSchema = z.object({
+	id: z.string(),
+	name: z.string(),
+	prompt: z.string(),
+	timeoutMs: z.number().optional(),
+	successCriteria: z.string().optional(),
+	expectedOutput: z.string().optional(),
+	fixture: z.string().optional(),
+	checks: z.array(checkSchema).optional()
+});
+const configFileSchema = z.object({
+	agent: z.string(),
+	models: z.array(modelSchema).min(1),
+	plugins: z.array(pluginSchema).optional(),
+	tasks: z.array(taskSchema).min(1),
+	workspace: z.string().optional(),
+	repetitions: z.number().int().positive().optional(),
+	outputDir: z.string(),
+	judge: z.object({
+		model: modelSchema,
+		agent: z.string().optional(),
+		repetitions: z.number().int().positive().optional(),
+		revealPlugins: z.boolean().optional(),
+		systemPrompt: z.string().optional()
+	}).optional()
+});
+/** Type-only identity helper — gives configs full typechecking. */
+function defineConfig(config) {
+	return config;
+}
+/**
+* Load and validate an arena config file, resolving the named agent
+* (and judge agent, defaulting to the same adapter) to live adapters.
+*/
+async function loadArenaConfig(configPath) {
+	const absPath = isAbsolute(configPath) ? configPath : resolve(process.cwd(), configPath);
+	let mod;
+	try {
+		mod = await import(pathToFileURL(absPath).href);
+	} catch (err) {
+		throw new ArenaError(`cannot load arena config '${absPath}': ${err instanceof Error ? err.message : String(err)}`, "CONFIG_NOT_FOUND");
+	}
+	if (typeof mod.default !== "object" || mod.default === null) throw new ArenaError(`arena config '${absPath}' must export a default config object`, "CONFIG_INVALID");
+	const parsed = configFileSchema.safeParse(mod.default);
+	if (!parsed.success) throw new ArenaError(`invalid arena config '${absPath}':\n${parsed.error.issues.map((i) => `  ${i.path.join(".")}: ${i.message}`).join("\n")}`, "CONFIG_INVALID");
+	const file = parsed.data;
+	return {
+		agent: resolveAdapter(file.agent),
+		models: file.models,
+		plugins: file.plugins ?? [],
+		tasks: file.tasks,
+		outputDir: file.outputDir,
+		...file.workspace !== void 0 ? { workspace: file.workspace } : {},
+		...file.repetitions !== void 0 ? { repetitions: file.repetitions } : {},
+		...file.judge ? { judge: {
+			model: file.judge.model,
+			agent: resolveAdapter(file.judge.agent ?? file.agent),
+			...file.judge.repetitions !== void 0 ? { repetitions: file.judge.repetitions } : {},
+			...file.judge.revealPlugins !== void 0 ? { revealPlugins: file.judge.revealPlugins } : {},
+			...file.judge.systemPrompt !== void 0 ? { systemPrompt: file.judge.systemPrompt } : {}
+		} } : {}
+	};
+}
+//#endregion
+//#region src/report.ts
+function fmtNum(n) {
+	return Number.isInteger(n) ? String(n) : n.toFixed(1);
+}
+function row(cells, widths) {
+	return cells.map((c, i) => String(c).padEnd(widths[i] ?? 10)).join("").trimEnd();
+}
+function aggregateTable(aggregates) {
+	const header = [
+		"label",
+		"runs",
+		"pass",
+		"tokens",
+		"tools",
+		"llm",
+		"ms",
+		"judge"
+	];
+	const data = aggregates.map((a) => [
+		a.label,
+		a.totalRuns,
+		a.successCount,
+		fmtNum(a.avgTotalTokens),
+		fmtNum(a.avgToolCalls),
+		fmtNum(a.avgLlmCalls),
+		fmtNum(a.avgExecutionTimeMs),
+		a.judgePassCount > 0 || a.avgJudgeScore > 0 ? fmtNum(a.avgJudgeScore) : "-"
+	]);
+	const widths = header.map((h, i) => Math.max(h.length + 2, ...data.map((r) => String(r[i] ?? "").length + 2)));
+	const lines = [row(header, widths)];
+	for (const r of data) lines.push(row(r, widths));
+	return lines.join("\n");
+}
+/** Render an ArenaResult as human-readable text. */
+function renderReport(result) {
+	const parts = [];
+	parts.push("Aggregates", aggregateTable(result.aggregates));
+	if (result.analysis.length > 0) {
+		parts.push("", "Analysis");
+		for (const task of result.analysis) {
+			parts.push(`  ${task.taskName} (${task.taskId})`);
+			for (const cmp of task.comparisons) {
+				const verdict = cmp.candidateBetter ? "better" : "worse";
+				parts.push(`    ${cmp.baseline} → ${cmp.candidate}: tokens ${cmp.deltaTokens}, tools ${cmp.deltaToolCalls}, llm ${cmp.deltaLlmCalls}, ms ${cmp.deltaTimeMs}, judge ${cmp.deltaJudgeScore} — ${verdict}`);
+			}
+			if (task.summary !== void 0) parts.push(`    ${task.summary}`);
+		}
+	}
+	return parts.join("\n");
+}
+//#endregion
+export { sanitizeEnv as C, runAcpSession as S, countLlmCalls as _, resolveAdapter as a, transcriptDir as b, pluginCombinations as c, buildJudgePrompt as d, compareCombos as f, DevinAdapter as g, parseJudgeResponse as h, loadArenaConfig as i, runArena as l, judgeRun as m, ArenaError as n, aggregateResults as o, judgeAllRuns as p, defineConfig as r, computeAnalysis as s, renderReport as t, DEFAULT_JUDGE_PROMPT as u, extractUsageTokens as v, createPermissionHandler as x, installPlugins as y };
+
+//# sourceMappingURL=report-3Gl3woII.mjs.map