@sverka/arena 0.0.0 → 0.1.1

package/dist/index.mjs

@@ -0,0 +1,1079 @@
+import { execSync, spawn } from "node:child_process";
+import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { join, resolve, sep } from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { Readable, Writable } from "node:stream";
+import * as acp from "@agentclientprotocol/sdk";
+import { fileURLToPath } from "node:url";
+//#region src/acp.ts
+/**
+* Shared ACP (Agent Client Protocol) utilities — extracted from the Devin
+* adapter so they can be reused by other adapters and the benchmark runner.
+*
+* This module contains:
+* - {@link sanitizeEnv}: strip host-contaminating env vars before spawning an agent.
+* - {@link createPermissionHandler}: auto-approve permission requests.
+* - {@link runAcpSession}: drive a full ACP connect/initialize/prompt/collect loop.
+*/
+/** Env-var prefixes to strip from the spawned process environment. */
+const SANITIZE_PREFIXES = [
+	"GC_",
+	"BEADS_",
+	"BD_",
+	"MCP_",
+	"CLAUDECODE",
+	"CLAUDE_CODE_",
+	"CODEX_"
+];
+/** Exact env-var names to strip from the spawned process environment. */
+const SANITIZE_EXACT = /* @__PURE__ */ new Set([
+	"DEVIN_PERMISSION_MODE",
+	"DEVIN_MODEL",
+	"AGENTS_MD",
+	"CLAUDE_PROJECT_MD"
+]);
+/** Check if an env var name should be stripped from the agent environment. */
+function shouldSanitize(key) {
+	if (SANITIZE_EXACT.has(key)) return true;
+	return SANITIZE_PREFIXES.some((p) => key.startsWith(p));
+}
+/** Copy process.env into a clean record, dropping sanitized entries. */
+function copyCleanEnv() {
+	const env = {};
+	for (const [key, value] of Object.entries(process.env)) {
+		if (value === void 0) continue;
+		if (shouldSanitize(key)) continue;
+		env[key] = value;
+	}
+	return env;
+}
+/** Apply model and permission settings to the env record. */
+function applyModelEnv(env, config) {
+	const model = config.model;
+	env.DEVIN_PERMISSION_MODE = config.permissionMode ?? "dangerous";
+	env.DEVIN_MODEL = model.envVar ? env[model.envVar] ?? model.id : model.id;
+	if (model.envVar) env[model.envVar] = env[model.envVar] ?? model.id;
+}
+/** Merge explicit env overrides from config (these win). */
+function mergeEnvOverrides(env, overrides) {
+	if (!overrides) return;
+	for (const [key, value] of Object.entries(overrides)) env[key] = value;
+}
+/**
+* Build a sanitized environment for an agent subprocess.
+*
+* Strips out host contamination: Gas City, Beads, MCP servers, project
+* skills, and other variables that would leak the host's configuration
+* into the benchmark. The agent must only see what we explicitly install.
+*
+* Sets `DEVIN_PERMISSION_MODE` and `DEVIN_MODEL` from the spawn config,
+* then merges any explicit `config.env` overrides (which win).
+*/
+function sanitizeEnv(config) {
+	const env = copyCleanEnv();
+	applyModelEnv(env, config);
+	mergeEnvOverrides(env, config.env);
+	return env;
+}
+/**
+* Create a permission-request handler that auto-approves tool calls.
+*
+* Selects the first `allow_once` or `allow_always` option, falling back to
+* the first available option. This is the same handler used by both the
+* Devin adapter and the benchmark runner.
+*/
+function createPermissionHandler() {
+	return (ctx) => {
+		const options = ctx.params.options;
+		return { outcome: {
+			outcome: "selected",
+			optionId: options.find((o) => o.kind === "allow_once" || o.kind === "allow_always")?.optionId ?? options[0]?.optionId ?? ""
+		} };
+	};
+}
+/**
+* Process a single {@link acp.ActiveSessionMessage} from the session update
+* stream. Returns the final {@link PromptResponse} when the session reports
+* a stop, otherwise invokes the appropriate callback and returns `undefined`
+* so the caller can continue collecting.
+*/
+function handleSessionUpdate(message, onToolCall, onToolUpdate) {
+	if (message.kind === "stop") return message.response;
+	const update = message.update;
+	if (update.sessionUpdate === "tool_call") onToolCall?.(update);
+	else if (update.sessionUpdate === "tool_call_update") onToolUpdate?.(update);
+}
+/**
+* Drive a single ACP session: send the prompt, race the collect loop
+* against a timeout, and return the final {@link PromptResponse}.
+*/
+async function runSessionLoop(session, prompt, timeoutMs, onToolCall, onToolUpdate, onSession) {
+	onSession?.(session.sessionId);
+	const promptPromise = session.prompt(prompt);
+	const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error(`Agent timed out after ${timeoutMs}ms`)), timeoutMs));
+	const collectPromise = (async () => {
+		for (;;) {
+			const result = handleSessionUpdate(await session.nextUpdate(), onToolCall, onToolUpdate);
+			if (result) return result;
+		}
+	})();
+	return Promise.race([
+		promptPromise,
+		collectPromise,
+		timeoutPromise
+	]);
+}
+/**
+* Run a full ACP session against a spawned agent subprocess.
+*
+* Connects to the agent, initializes the protocol, creates a session in
+* `workspace`, sends `prompt`, and collects update events until the session
+* stops or `timeoutMs` elapses.
+*
+* The `onToolCall` and `onToolUpdate` callbacks let callers collect tool
+* calls and observations without duplicating the session loop logic. The
+* optional `onSession` callback reports the session ID once created. All
+* callbacks are optional.
+*
+* @returns The final {@link PromptResponse} from the agent.
+*/
+async function runAcpSession(proc, workspace, prompt, timeoutMs, onToolCall, onToolUpdate, onSession) {
+	if (!proc.stdin || !proc.stdout) throw new Error("Agent process missing stdin/stdout");
+	const input = Writable.toWeb(proc.stdin);
+	const output = Readable.toWeb(proc.stdout);
+	const stream = acp.ndJsonStream(input, output);
+	return acp.client({ name: "sverka-arena" }).onRequest(acp.methods.client.session.requestPermission, createPermissionHandler()).connectWith(stream, async (ctx) => {
+		await ctx.request(acp.methods.agent.initialize, {
+			protocolVersion: acp.PROTOCOL_VERSION,
+			clientCapabilities: {}
+		});
+		return ctx.buildSession(workspace).withSession((session) => runSessionLoop(session, prompt, timeoutMs, onToolCall, onToolUpdate, onSession));
+	});
+}
+//#endregion
+//#region src/adapters/devin.ts
+/**
+* Devin ACP adapter — spawns `devin acp` as a subprocess and drives an
+* Agent Client Protocol session to collect trace data + metrics.
+*
+* The adapter is abstract over models and plugins: it accepts any
+* {@link ModelConfig} and any set of {@link PluginConfig}s, installs the
+* enabled plugins into the workspace, and runs the prompt against the
+* spawned agent. After the session completes it reads the Devin CLI
+* transcript file to build full {@link TraceData}.
+*/
+/** Default transcript directory: ~/.local/share/devin/cli/transcripts */
+function transcriptDir() {
+	return join(homedir(), ".local", "share", "devin", "cli", "transcripts");
+}
+/** Read a transcript JSON file by session ID. Throws if not found. */
+async function readTranscript(sessionId, dir) {
+	const path = join(dir ?? transcriptDir(), `${sessionId}.json`);
+	const content = await readFile(path, "utf-8");
+	return JSON.parse(content);
+}
+/** Count LLM inference calls in a transcript (agent steps with telemetry.operation === "inference"). */
+function countLlmCalls(transcript) {
+	return transcript.steps.filter((s) => s.source === "agent" && s.extra?.telemetry?.operation === "inference").length;
+}
+/** Build a {@link TraceStep} from a raw transcript step. */
+function buildTraceStep(step) {
+	const isLlmCall = step.source === "agent" && step.extra?.telemetry?.operation === "inference";
+	return {
+		stepId: step.step_id,
+		timestamp: step.timestamp,
+		source: step.source,
+		message: step.message,
+		isLlmCall
+	};
+}
+/** Extract text content from a ToolCallContent array (best-effort). */
+function extractTextContent(contents) {
+	if (!contents) return "";
+	const parts = [];
+	for (const c of contents) if (c.type === "content" && c.content.type === "text") parts.push(c.content.text);
+	return parts.join("\n");
+}
+/** Extract token counts from a Usage object, defaulting to 0. */
+function extractUsageTokens(usage) {
+	return {
+		inputTokens: usage?.inputTokens ?? 0,
+		outputTokens: usage?.outputTokens ?? 0,
+		thoughtTokens: usage?.thoughtTokens ?? 0,
+		totalTokens: usage?.totalTokens ?? 0
+	};
+}
+/** Build {@link RunMetrics} from a PromptResponse + collected counts + timing. */
+function buildMetrics(promptResult, toolCallCount, llmCallCount, startTime) {
+	return {
+		...extractUsageTokens(promptResult.usage),
+		toolCallCount,
+		llmCallCount,
+		executionTimeMs: Date.now() - startTime,
+		stopReason: promptResult.stopReason
+	};
+}
+/**
+* Extract the agent's final text output from the prompt response or trace.
+*
+* Prefers the text content from the ACP `PromptResponse`, falling back to
+* the last agent message in the trace.
+*/
+function extractAgentOutput(promptResult, trace) {
+	for (let i = trace.steps.length - 1; i >= 0; i--) {
+		const step = trace.steps[i];
+		if (step?.source === "agent" && step.message.length > 0) return step.message;
+	}
+	return "";
+}
+/**
+* Install enabled plugins and remove disabled ones from the workspace.
+*
+* CRITICAL: This function first removes the entire `.agents/skills/`
+* directory to ensure no inherited skills from the host contaminate the
+* benchmark. Only explicitly enabled plugins are installed.
+*
+* Each enabled plugin's `path` is copied to `workspace/.agents/skills/<id>/`.
+*/
+async function installPlugins(workspace, plugins) {
+	const skillsDir = join(workspace, ".agents", "skills");
+	await rm(skillsDir, {
+		recursive: true,
+		force: true
+	});
+	const enabledPlugins = plugins.filter((p) => p.enabled);
+	if (enabledPlugins.length === 0) return;
+	await mkdir(skillsDir, { recursive: true });
+	for (const plugin of enabledPlugins) {
+		if (plugin.id.includes("/") || plugin.id.includes("\\") || plugin.id.includes("..")) throw new Error(`Invalid plugin id: ${plugin.id}`);
+		const dest = join(skillsDir, plugin.id);
+		await mkdir(dest, { recursive: true });
+		await cp(plugin.path, dest, { recursive: true });
+	}
+}
+/**
+* Devin ACP adapter — spawns `devin acp` as a subprocess.
+*
+* Usage:
+* ```ts
+* const adapter = new DevinAdapter();
+* const proc = adapter.spawn({ model, workspace, plugins });
+* const result = await proc.run("Fix the bug", 120_000);
+* proc.kill();
+* ```
+*/
+var DevinAdapter = class {
+	id = "devin";
+	spawn(config) {
+		const proc = spawnDevin(config);
+		let killed = false;
+		return {
+			run: (prompt, timeoutMs) => runDevinSession(proc, config, prompt, timeoutMs),
+			kill: () => {
+				if (killed) return;
+				killed = true;
+				proc.kill("SIGTERM");
+			}
+		};
+	}
+};
+/** Resolve the full path to the devin binary to avoid PATH-based lookup. */
+function resolveDevinBinary() {
+	try {
+		return execSync("which devin", {
+			encoding: "utf-8",
+			stdio: [
+				"pipe",
+				"pipe",
+				"pipe"
+			]
+		}).trim();
+	} catch {
+		return "devin";
+	}
+}
+/** Spawn `devin acp --model <model.id>` as a subprocess in the workspace. */
+function spawnDevin(config) {
+	const env = sanitizeEnv(config);
+	const devinBin = resolveDevinBinary();
+	return spawn(devinBin, [
+		"acp",
+		"--model",
+		config.model.id
+	], {
+		cwd: config.workspace,
+		stdio: [
+			"pipe",
+			"pipe",
+			"inherit"
+		],
+		env
+	});
+}
+/** Create a {@link ToolCollector} with callbacks that populate shared state. */
+function createToolCollector() {
+	const collectedToolCalls = [];
+	const observationsByCallId = /* @__PURE__ */ new Map();
+	let toolCallCount = 0;
+	const onToolCall = (update) => {
+		toolCallCount++;
+		const name = update.name ?? update.title ?? "unknown";
+		const args = update.rawInput ?? {};
+		collectedToolCalls.push({
+			functionName: name,
+			arguments: args,
+			toolCallId: update.toolCallId
+		});
+	};
+	const onToolUpdate = (update) => {
+		const text = extractTextContent(update.content ?? void 0);
+		if (text) observationsByCallId.set(update.toolCallId, {
+			sourceCallId: update.toolCallId,
+			content: text
+		});
+	};
+	return {
+		collectedToolCalls,
+		observationsByCallId,
+		get toolCallCount() {
+			return toolCallCount;
+		},
+		onToolCall,
+		onToolUpdate
+	};
+}
+/** Build a {@link RunResult} for a failed session (best-effort trace from collected data). */
+function buildErrorResult(model, pluginIds, toolCallCount, sessionId, startTime, error) {
+	const metrics = buildMetrics({
+		stopReason: "cancelled",
+		usage: null
+	}, toolCallCount, 0, startTime);
+	const trace = {
+		sessionId,
+		model: model.id,
+		steps: [],
+		finalMetrics: {
+			totalPromptTokens: 0,
+			totalCompletionTokens: 0,
+			totalCachedTokens: 0,
+			totalSteps: 0
+		}
+	};
+	return {
+		taskId: "",
+		modelId: model.id,
+		pluginIds,
+		metrics,
+		trace,
+		output: "",
+		checkResults: [],
+		verdicts: [],
+		success: false,
+		error: error instanceof Error ? error.message : String(error)
+	};
+}
+/** Run a full ACP session: initialize, create session, prompt, collect events, read transcript. */
+async function runDevinSession(proc, config, prompt, timeoutMs) {
+	const startTime = Date.now();
+	const model = config.model;
+	const pluginIds = config.plugins.filter((p) => p.enabled).map((p) => p.id);
+	const collector = createToolCollector();
+	let sessionId = "";
+	try {
+		const promptResult = await runAcpSession(proc, config.workspace, prompt, timeoutMs, collector.onToolCall, collector.onToolUpdate, (id) => {
+			sessionId = id;
+		});
+		const trace = await buildTrace(sessionId, model, collector.collectedToolCalls, collector.observationsByCallId);
+		const llmCallCount = trace.steps.filter((s) => s.isLlmCall).length;
+		const metrics = buildMetrics(promptResult, collector.toolCallCount, llmCallCount, startTime);
+		const agentOutput = extractAgentOutput(promptResult, trace);
+		return {
+			taskId: "",
+			modelId: model.id,
+			pluginIds,
+			metrics,
+			trace,
+			output: agentOutput,
+			checkResults: [],
+			verdicts: [],
+			success: metrics.stopReason === "end_turn"
+		};
+	} catch (error) {
+		return buildErrorResult(model, pluginIds, collector.toolCallCount, sessionId, startTime, error);
+	}
+}
+/**
+* Build {@link TraceData} from the Devin transcript, enriched with
+* ACP-collected tool calls. Returns `undefined` when no transcript is
+* available so the caller can fall back to {@link buildTraceFallback}.
+*/
+async function buildTraceFromTranscript(sessionId, collectedToolCalls, observationsByCallId) {
+	if (!sessionId) return void 0;
+	try {
+		const transcript = await readTranscript(sessionId);
+		const steps = transcript.steps.map(buildTraceStep);
+		attachToolCallsToSteps(steps, collectedToolCalls, observationsByCallId);
+		return {
+			sessionId: transcript.session_id,
+			model: transcript.agent.model_name,
+			steps,
+			finalMetrics: {
+				totalPromptTokens: transcript.final_metrics.total_prompt_tokens,
+				totalCompletionTokens: transcript.final_metrics.total_completion_tokens,
+				totalCachedTokens: transcript.final_metrics.total_cached_tokens,
+				totalSteps: transcript.final_metrics.total_steps
+			}
+		};
+	} catch {
+		return;
+	}
+}
+/** Attach the next tool call + observation to an agent step. Returns true if attached. */
+function attachNextToolCall(step, collectedToolCalls, observationsByCallId, toolIdx) {
+	if (toolIdx >= collectedToolCalls.length) return {
+		attached: false,
+		nextIdx: toolIdx
+	};
+	const tc = collectedToolCalls[toolIdx];
+	if (!tc) return {
+		attached: false,
+		nextIdx: toolIdx
+	};
+	const calls = [tc];
+	const obs = observationsByCallId.get(tc.toolCallId);
+	const observations = obs ? [obs] : [];
+	step.toolCalls = calls;
+	if (observations.length > 0) step.observations = observations;
+	return {
+		attached: true,
+		nextIdx: toolIdx + 1
+	};
+}
+/** Attach ACP-collected tool calls/observations to agent steps in order. */
+function attachToolCallsToSteps(steps, collectedToolCalls, observationsByCallId) {
+	let toolIdx = 0;
+	for (const step of steps) {
+		if (step.source !== "agent") continue;
+		const result = attachNextToolCall(step, collectedToolCalls, observationsByCallId, toolIdx);
+		if (result.attached) toolIdx = result.nextIdx;
+	}
+}
+/** Build a minimal {@link TraceData} from ACP-collected data only. */
+function buildTraceFallback(sessionId, model, collectedToolCalls, observationsByCallId) {
+	const steps = collectedToolCalls.map((tc, i) => {
+		const obs = observationsByCallId.get(tc.toolCallId);
+		const step = {
+			stepId: i,
+			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+			source: "agent",
+			message: tc.functionName,
+			toolCalls: [tc],
+			isLlmCall: false
+		};
+		if (obs) step.observations = [obs];
+		return step;
+	});
+	return {
+		sessionId,
+		model: model.id,
+		steps,
+		finalMetrics: {
+			totalPromptTokens: 0,
+			totalCompletionTokens: 0,
+			totalCachedTokens: 0,
+			totalSteps: steps.length
+		}
+	};
+}
+/** Build {@link TraceData} from the Devin transcript, falling back to ACP-only data. */
+async function buildTrace(sessionId, model, collectedToolCalls, observationsByCallId) {
+	return await buildTraceFromTranscript(sessionId, collectedToolCalls, observationsByCallId) ?? buildTraceFallback(sessionId, model, collectedToolCalls, observationsByCallId);
+}
+//#endregion
+//#region src/judge.ts
+/** Default system prompt for the judge. */
+const DEFAULT_JUDGE_PROMPT = "You are a blind evaluator. You will receive a task prompt and an agent's response. Score the response 0-100 based on how well it completes the task. Be strict but fair. Output JSON: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70+ is passing.";
+/** Default timeout for a single judge run (ms). */
+const DEFAULT_JUDGE_TIMEOUT_MS = 12e4;
+/**
+* Build the judge prompt for a single run.
+*
+* Includes the task prompt, success criteria, and the agent's output. Does
+* NOT include which plugins were active unless `config.revealPlugins` is
+* true. Asks the judge for a JSON response with score, passed, reasoning,
+* and issues.
+*/
+function buildJudgePrompt(task, runOutput, config, pluginIds = []) {
+	const lines = [
+		config.systemPrompt ?? "You are a blind evaluator. You will receive a task prompt and an agent's response. Score the response 0-100 based on how well it completes the task. Be strict but fair. Output JSON: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70+ is passing.",
+		"",
+		"## Task Prompt",
+		task.prompt,
+		""
+	];
+	if (task.successCriteria) lines.push("## Success Criteria", task.successCriteria, "");
+	if (config.revealPlugins && task.id) {
+		lines.push("## Plugins Active During Run");
+		if (pluginIds.length > 0) lines.push(pluginIds.join(", "));
+		else lines.push("(no plugins)");
+		lines.push("");
+	}
+	lines.push("## Agent Response");
+	lines.push(runOutput || "(no output)");
+	lines.push("");
+	lines.push("## Instructions\nEvaluate the agent's response against the task prompt and success criteria. Respond with ONLY a JSON object: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70 or above is passing. Do not include any text outside the JSON.");
+	return lines.join("\n");
+}
+/**
+* Parse a judge response into a {@link JudgeVerdict}.
+*
+* Extracts JSON from the response, handling markdown code fences
+* (` ```json ... ``` `). Falls back to `score=0, passed=false` when the
+* response cannot be parsed.
+*/
+function parseJudgeResponse(response, runIndex, run) {
+	const jsonText = extractJson(response);
+	const fallback = {
+		runIndex,
+		score: 0,
+		passed: false,
+		reasoning: response.trim() || "Failed to parse judge response",
+		issues: [],
+		taskId: run.taskId,
+		modelId: run.modelId,
+		pluginIds: run.pluginIds
+	};
+	if (jsonText === null) return fallback;
+	let parsed;
+	try {
+		parsed = JSON.parse(jsonText);
+	} catch {
+		return fallback;
+	}
+	if (typeof parsed !== "object" || parsed === null) return fallback;
+	const obj = parsed;
+	const score = typeof obj.score === "number" ? obj.score : 0;
+	return {
+		runIndex,
+		score,
+		passed: typeof obj.passed === "boolean" ? obj.passed : score >= 70,
+		reasoning: typeof obj.reasoning === "string" ? obj.reasoning : "",
+		issues: Array.isArray(obj.issues) ? obj.issues.filter((i) => typeof i === "string") : [],
+		taskId: run.taskId,
+		modelId: run.modelId,
+		pluginIds: run.pluginIds
+	};
+}
+/**
+* Judge a single run — spawn the judge agent, send the prompt, collect the
+* response, parse it into a {@link JudgeVerdict}, and kill the agent.
+*/
+async function judgeRun(run, task, config, runIndex) {
+	const prompt = buildJudgePrompt(task, run.output, config, run.pluginIds);
+	const proc = config.agent.spawn({
+		model: config.model,
+		workspace: process.cwd(),
+		plugins: [],
+		permissionMode: "dangerous"
+	});
+	try {
+		return parseJudgeResponse((await proc.run(prompt, DEFAULT_JUDGE_TIMEOUT_MS)).output, runIndex, run);
+	} catch (error) {
+		return {
+			runIndex,
+			score: 0,
+			passed: false,
+			reasoning: error instanceof Error ? error.message : String(error),
+			issues: [],
+			taskId: run.taskId,
+			modelId: run.modelId,
+			pluginIds: run.pluginIds
+		};
+	} finally {
+		proc.kill();
+	}
+}
+/**
+* Judge all runs in a list, pairing each run with its matching task.
+*
+* Supports `config.repetitions` — the judge is run N times per run for
+* confidence. All verdicts are returned (one per repetition).
+*/
+async function judgeAllRuns(results, tasks, config) {
+	const taskById = /* @__PURE__ */ new Map();
+	for (const task of tasks) taskById.set(task.id, task);
+	const repetitions = config.repetitions ?? 1;
+	const verdicts = [];
+	for (let i = 0; i < results.length; i++) {
+		const run = results[i];
+		if (!run) continue;
+		const task = taskById.get(run.taskId);
+		if (!task) continue;
+		for (let rep = 0; rep < repetitions; rep++) {
+			const verdict = await judgeRun(run, task, config, i);
+			verdicts.push(verdict);
+		}
+	}
+	return verdicts;
+}
+/**
+* Compare two plugin combos for a task (for {@link ComboComparison}).
+*
+* Computes deltas as `candidate - baseline`. Determines whether the
+* candidate is better:
+*   - Lower tokens / toolCalls / llmCalls / time = better
+*   - Higher judgeScore = better
+*
+* The judge score for each run is the average of its verdicts' scores
+* (0 when there are no verdicts).
+*/
+function compareCombos(baseline, candidate) {
+	const baselineScore = avgJudgeScore(baseline);
+	const candidateScore = avgJudgeScore(candidate);
+	const deltaTokens = candidate.metrics.totalTokens - baseline.metrics.totalTokens;
+	const deltaToolCalls = candidate.metrics.toolCallCount - baseline.metrics.toolCallCount;
+	const deltaLlmCalls = candidate.metrics.llmCallCount - baseline.metrics.llmCallCount;
+	const deltaTimeMs = candidate.metrics.executionTimeMs - baseline.metrics.executionTimeMs;
+	const deltaJudgeScore = candidateScore - baselineScore;
+	const candidateBetter = deltaJudgeScore > 0 || deltaJudgeScore === 0 && deltaTokens <= 0 && deltaToolCalls <= 0 && deltaLlmCalls <= 0 && deltaTimeMs <= 0 && (deltaTokens < 0 || deltaToolCalls < 0 || deltaLlmCalls < 0 || deltaTimeMs < 0);
+	return {
+		baseline: comboLabel$1(baseline),
+		candidate: comboLabel$1(candidate),
+		deltaTokens,
+		deltaToolCalls,
+		deltaLlmCalls,
+		deltaTimeMs,
+		deltaJudgeScore,
+		candidateBetter
+	};
+}
+/**
+* Extract the first JSON object from a response string.
+*
+* Handles raw JSON and JSON wrapped in markdown code fences
+* (` ```json ... ``` ` or ` ``` ... ``` `). Returns `null` when no JSON
+* object can be located.
+*/
+function extractJson(response) {
+	const trimmed = response.trim();
+	if (trimmed === "") return null;
+	const fenced = extractFromFence(trimmed);
+	if (fenced !== null) return fenced;
+	const braced = extractFromBraces(trimmed);
+	if (braced !== null) return braced;
+	return null;
+}
+/** Extract JSON from inside a markdown code fence (```...```). */
+function extractFromFence(text) {
+	const fenceStart = text.indexOf("```");
+	if (fenceStart === -1) return null;
+	const afterFence = text.slice(fenceStart + 3);
+	let contentStart = 0;
+	if (afterFence.startsWith("json")) contentStart = 4;
+	while (contentStart < afterFence.length && /\s/.test(afterFence[contentStart])) contentStart++;
+	const fenceEnd = afterFence.indexOf("```", contentStart);
+	if (fenceEnd === -1) return null;
+	const inner = afterFence.slice(contentStart, fenceEnd).trim();
+	if (!looksLikeJson(inner)) return null;
+	return inner;
+}
+/** Extract JSON by finding the first `{` and last `}` in the text. */
+function extractFromBraces(text) {
+	const start = text.indexOf("{");
+	const end = text.lastIndexOf("}");
+	if (start === -1 || end === -1 || end <= start) return null;
+	return text.slice(start, end + 1);
+}
+/** Quick check that a string starts with `{` (a JSON object). */
+function looksLikeJson(text) {
+	return text.trim().startsWith("{");
+}
+/** Average judge score across a run's verdicts (0 when no verdicts). */
+function avgJudgeScore(run) {
+	if (run.verdicts.length === 0) return 0;
+	return run.verdicts.reduce((a, v) => a + v.score, 0) / run.verdicts.length;
+}
+/** Build a human-readable combo label for a run's plugin set. */
+function comboLabel$1(run) {
+	if (run.pluginIds.length === 0) return "no-plugins";
+	return run.pluginIds.join(",");
+}
+//#endregion
+//#region src/runner.ts
+/**
+* Matrix runner — executes the full arena matrix:
+*
+*   task × model × plugin combination × repetition
+*
+* For each cell it spawns the agent, runs the prompt, collects the result,
+* and kills the agent. Results are aggregated into {@link ArenaResult}.
+*/
+/** Resolve the arena package root (for fixture paths). */
+const PKG_ROOT = resolve(typeof __dirname !== "undefined" ? __dirname : fileURLToPath(new URL(".", import.meta.url)), "..");
+/**
+* Generate all plugin on/off combinations.
+*
+* For N plugins this produces 2^N combinations, each a full copy of the
+* plugins array with `enabled` set appropriately.
+*
+* - 0 plugins → `[[]]` (one empty combination)
+* - 1 plugin  → `[[off], [on]]`
+* - 2 plugins → `[[off,off], [on,off], [off,on], [on,on]]`
+*/
+function pluginCombinations(plugins) {
+	if (plugins.length === 0) return [[]];
+	const combos = [[]];
+	for (const plugin of plugins) {
+		const next = [];
+		for (const combo of combos) {
+			next.push([...combo, {
+				...plugin,
+				enabled: false
+			}]);
+			next.push([...combo, {
+				...plugin,
+				enabled: true
+			}]);
+		}
+		combos.length = 0;
+		combos.push(...next);
+	}
+	return combos;
+}
+/**
+* Run the full arena matrix: task × model × plugin combo × repetitions.
+*
+* Each cell spawns the configured agent adapter, installs the plugin
+* combination, runs the prompt, collects trace + metrics, and kills the
+* agent. Results are written to `config.outputDir/results.json` and the
+* full {@link ArenaResult} is returned.
+*/
+async function runArena(config) {
+	const repetitions = config.repetitions ?? 1;
+	const combos = pluginCombinations(config.plugins);
+	const results = await runMatrix(config, combos);
+	if (config.judge) {
+		const verdicts = await judgeAllRuns(results, config.tasks, config.judge);
+		for (const verdict of verdicts) {
+			const run = results[verdict.runIndex];
+			if (run) run.verdicts.push(verdict);
+		}
+	}
+	const aggregates = buildAggregates(results, config.models, combos);
+	const analysis = computeAnalysis(results, config.tasks);
+	const arenaResult = {
+		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+		config: {
+			models: config.models.map((m) => m.id),
+			plugins: config.plugins.map((p) => p.id),
+			tasks: config.tasks.map((t) => t.id),
+			repetitions,
+			...config.judge ? { judgeModel: config.judge.model.id } : {}
+		},
+		results,
+		aggregates,
+		analysis
+	};
+	await mkdir(config.outputDir, { recursive: true });
+	await writeFile(join(config.outputDir, "results.json"), JSON.stringify(arenaResult, null, 2), "utf-8");
+	return arenaResult;
+}
+/**
+* Run every cell of the task × model × combo × repetition matrix, collecting
+* results and logging per-cell progress to stderr.
+*/
+async function runMatrix(config, combos) {
+	const repetitions = config.repetitions ?? 1;
+	const results = [];
+	for (const task of config.tasks) for (const model of config.models) for (const combo of combos) for (let rep = 0; rep < repetitions; rep++) {
+		const cellLabel = formatCell(task.id, model.id, combo, rep);
+		process.stderr.write(`[arena] ${cellLabel} running...\n`);
+		const result = await runCell(config.agent, task, model, combo, rep);
+		results.push(result);
+		process.stderr.write(`[arena] ${cellLabel} ${result.success ? "PASS" : "FAIL"} — ${result.metrics.totalTokens} tokens, ${result.metrics.toolCallCount} tools, ${result.metrics.llmCallCount} llm calls, ${result.metrics.executionTimeMs}ms\n`);
+	}
+	return results;
+}
+/**
+* Aggregate metrics per (model × plugin combo) cell.
+*/
+function buildAggregates(results, models, combos) {
+	const aggregates = [];
+	for (const model of models) for (const combo of combos) {
+		const pluginIds = combo.filter((p) => p.enabled).map((p) => p.id);
+		const label = formatAggregateLabel(model.id, pluginIds);
+		aggregates.push(aggregateResults(results, (r) => r.modelId === model.id && samePluginIds(r.pluginIds, pluginIds), label));
+	}
+	return aggregates;
+}
+/** Run a single matrix cell in an ISOLATED temp workspace. */
+async function runCell(agent, task, model, combo, rep) {
+	const tempWorkspace = await createTempWorkspace(task, model, combo, rep);
+	await installPlugins(tempWorkspace, combo);
+	const result = await executeRun(agent, task, model, combo, tempWorkspace);
+	await rm(tempWorkspace, {
+		recursive: true,
+		force: true
+	});
+	return result;
+}
+/**
+* Create an isolated temp workspace with a fresh copy of the fixture (if any).
+* Each run gets its own copy to prevent skill contamination from the host repo.
+*/
+async function createTempWorkspace(task, model, combo, rep) {
+	const tempWorkspace = join(tmpdir(), `arena-${task.id}-${model.id}-${combo.map((p) => p.id + (p.enabled ? "on" : "off")).join(",")}-r${rep}-${Date.now()}`);
+	await mkdir(tempWorkspace, { recursive: true });
+	if (task.fixture) {
+		const fixturePath = resolve(PKG_ROOT, task.fixture);
+		if (!fixturePath.startsWith(PKG_ROOT + sep) && fixturePath !== PKG_ROOT) throw new Error(`Fixture path escapes package root: ${task.fixture}`);
+		await cp(fixturePath, tempWorkspace, { recursive: true });
+	}
+	return tempWorkspace;
+}
+/**
+* Spawn the agent, run the prompt, run deterministic checks, and return the
+* result. On error returns a zeroed {@link RunResult} via {@link errorResult}.
+*/
+async function executeRun(agent, task, model, combo, tempWorkspace) {
+	const proc = agent.spawn({
+		model,
+		workspace: tempWorkspace,
+		plugins: combo,
+		permissionMode: "dangerous"
+	});
+	try {
+		const result = await proc.run(task.prompt, task.timeoutMs ?? 12e4);
+		result.taskId = task.id;
+		if (!result.verdicts) result.verdicts = [];
+		if (!result.checkResults) result.checkResults = [];
+		if (task.checks && task.checks.length > 0) {
+			result.checkResults = await runChecks(tempWorkspace, task.checks);
+			result.success = result.success && result.checkResults.every((c) => c.passed);
+		}
+		return result;
+	} catch (error) {
+		return errorResult(task, model, combo, error);
+	} finally {
+		proc.kill();
+	}
+}
+/** Build a zeroed {@link RunResult} for a failed run. */
+function errorResult(task, model, combo, error) {
+	return {
+		taskId: task.id,
+		modelId: model.id,
+		pluginIds: combo.filter((p) => p.enabled).map((p) => p.id),
+		metrics: {
+			inputTokens: 0,
+			outputTokens: 0,
+			thoughtTokens: 0,
+			totalTokens: 0,
+			toolCallCount: 0,
+			llmCallCount: 0,
+			executionTimeMs: 0,
+			stopReason: "cancelled"
+		},
+		trace: {
+			sessionId: "",
+			model: model.id,
+			steps: [],
+			finalMetrics: {
+				totalPromptTokens: 0,
+				totalCompletionTokens: 0,
+				totalCachedTokens: 0,
+				totalSteps: 0
+			}
+		},
+		output: "",
+		checkResults: [],
+		verdicts: [],
+		success: false,
+		error: error instanceof Error ? error.message : String(error)
+	};
+}
+/** Run deterministic checks in the workspace. */
+async function runChecks(workspace, checks) {
+	if (!checks) return [];
+	const results = [];
+	for (const check of checks) try {
+		const { output, exitCode } = await new Promise((resolve, reject) => {
+			const proc = spawn("bash", ["-c", check.command], {
+				cwd: workspace,
+				stdio: [
+					"pipe",
+					"pipe",
+					"pipe"
+				],
+				env: {
+					...process.env,
+					CI: "true"
+				}
+			});
+			let stdout = "";
+			proc.stdout?.on("data", (d) => {
+				stdout += d.toString();
+			});
+			proc.stderr?.on("data", (d) => {
+				stdout += d.toString();
+			});
+			proc.on("close", (code) => {
+				resolve({
+					output: stdout,
+					exitCode: code
+				});
+			});
+			proc.on("error", reject);
+		});
+		results.push({
+			checkId: check.id,
+			passed: exitCode === 0,
+			output: output.trim(),
+			exitCode
+		});
+	} catch (error) {
+		results.push({
+			checkId: check.id,
+			passed: false,
+			output: error instanceof Error ? error.message : String(error),
+			exitCode: -1
+		});
+	}
+	return results;
+}
+/**
+* Aggregate metrics for a group of runs selected by `filter`.
+*
+* Computes averages across all matching runs. Returns zeroed metrics when
+* no runs match. Includes judge-related fields (`avgJudgeScore`,
+* `judgePassCount`) and a `label` identifying the aggregate group.
+*/
+function aggregateResults(results, filter, label = "") {
+	const filtered = results.filter(filter);
+	const n = filtered.length;
+	if (n === 0) return {
+		totalRuns: 0,
+		successCount: 0,
+		avgInputTokens: 0,
+		avgOutputTokens: 0,
+		avgTotalTokens: 0,
+		avgToolCalls: 0,
+		avgLlmCalls: 0,
+		avgExecutionTimeMs: 0,
+		avgJudgeScore: 0,
+		judgePassCount: 0,
+		label
+	};
+	const sum = (f) => filtered.reduce((a, r) => a + f(r), 0);
+	const round = (v) => Math.round(v);
+	const round2 = (v) => Math.round(v * 100) / 100;
+	const judgedRuns = filtered.filter((r) => r.verdicts.length > 0);
+	const avgJudgeScore = judgedRuns.length === 0 ? 0 : round2(judgedRuns.reduce((a, r) => a + (r.verdicts[0]?.score ?? 0), 0) / judgedRuns.length);
+	const judgePassCount = filtered.filter((r) => r.verdicts[0]?.passed === true).length;
+	return {
+		totalRuns: n,
+		successCount: filtered.filter((r) => r.success).length,
+		avgInputTokens: round(sum((r) => r.metrics.inputTokens) / n),
+		avgOutputTokens: round(sum((r) => r.metrics.outputTokens) / n),
+		avgTotalTokens: round(sum((r) => r.metrics.totalTokens) / n),
+		avgToolCalls: round2(sum((r) => r.metrics.toolCallCount) / n),
+		avgLlmCalls: round2(sum((r) => r.metrics.llmCallCount) / n),
+		avgExecutionTimeMs: round(sum((r) => r.metrics.executionTimeMs) / n),
+		avgJudgeScore,
+		judgePassCount,
+		label
+	};
+}
+/** Compare two plugin-id arrays for equality (order-insensitive). */
+function samePluginIds(a, b) {
+	if (a.length !== b.length) return false;
+	const set = new Set(a);
+	for (const id of b) if (!set.has(id)) return false;
+	return true;
+}
+/** Format a human-readable cell label for logging. */
+function formatCell(taskId, modelId, combo, rep) {
+	return `${taskId}/${modelId}/{${combo.map((p) => `${p.id}=${p.enabled ? "on" : "off"}`).join(",")}}#${rep}`;
+}
+/** Format an aggregate group label: "model-id/plugins=on,off" or "model-id/no-plugins". */
+function formatAggregateLabel(modelId, pluginIds) {
+	if (pluginIds.length === 0) return `${modelId}/no-plugins`;
+	return `${modelId}/plugins=${pluginIds.join(",")}`;
+}
+/** Format a plugin combo label for comparisons: "no-plugins" or comma-joined ids. */
+function comboLabel(pluginIds) {
+	if (pluginIds.length === 0) return "no-plugins";
+	return pluginIds.join(",");
+}
+/**
+* Create a synthetic "average" {@link RunResult} representing a group of
+* runs with the same plugin combo. Used to feed {@link compareCombos} which
+* operates on single runs.
+*/
+function averageRun(runs) {
+	if (runs.length === 0) return null;
+	const n = runs.length;
+	const avg = (f) => Math.round(runs.reduce((a, r) => a + f(r), 0) / n);
+	const avg2 = (f) => Math.round(runs.reduce((a, r) => a + f(r), 0) / n * 100) / 100;
+	const first = runs[0];
+	return {
+		taskId: first.taskId,
+		modelId: first.modelId,
+		pluginIds: first.pluginIds,
+		metrics: {
+			inputTokens: avg((r) => r.metrics.inputTokens),
+			outputTokens: avg((r) => r.metrics.outputTokens),
+			thoughtTokens: avg((r) => r.metrics.thoughtTokens),
+			totalTokens: avg((r) => r.metrics.totalTokens),
+			toolCallCount: avg2((r) => r.metrics.toolCallCount),
+			llmCallCount: avg2((r) => r.metrics.llmCallCount),
+			executionTimeMs: avg((r) => r.metrics.executionTimeMs),
+			stopReason: first.metrics.stopReason
+		},
+		trace: first.trace,
+		output: first.output,
+		checkResults: first.checkResults ?? [],
+		verdicts: first.verdicts,
+		success: runs.every((r) => r.success)
+	};
+}
+/**
+* Compute per-task analysis comparing baseline (no plugins) vs each
+* candidate plugin combo.
+*
+* For each task, groups runs by plugin combo, creates an averaged
+* representative run per group, then compares the baseline (empty
+* `pluginIds`) against every non-empty candidate combo using
+* {@link compareCombos} from `judge.ts`.
+*/
+function computeAnalysis(results, tasks) {
+	const analyses = [];
+	for (const task of tasks) {
+		const taskRuns = results.filter((r) => r.taskId === task.id);
+		const comboGroups = /* @__PURE__ */ new Map();
+		for (const run of taskRuns) {
+			const key = comboLabel(run.pluginIds);
+			const group = comboGroups.get(key);
+			if (group) group.push(run);
+			else comboGroups.set(key, [run]);
+		}
+		const baseline = averageRun(comboGroups.get("no-plugins") ?? []);
+		const comparisons = [];
+		if (baseline) for (const [key, candidateRuns] of comboGroups) {
+			if (key === "no-plugins") continue;
+			const candidate = averageRun(candidateRuns);
+			if (candidate) comparisons.push(compareCombos(baseline, candidate));
+		}
+		analyses.push({
+			taskId: task.id,
+			taskName: task.name,
+			prompt: task.prompt,
+			comparisons
+		});
+	}
+	return analyses;
+}
+//#endregion
+export { DEFAULT_JUDGE_PROMPT, DevinAdapter, aggregateResults, buildJudgePrompt, compareCombos, computeAnalysis, countLlmCalls, createPermissionHandler, extractUsageTokens, installPlugins, judgeAllRuns, judgeRun, parseJudgeResponse, pluginCombinations, runAcpSession, runArena, sanitizeEnv, transcriptDir };
+
+//# sourceMappingURL=index.mjs.map