@sverka/arena 0.1.7 → 0.1.9

package/dist/index.mjs

@@ -1,1079 +1,2 @@
-import { execSync, spawn } from "node:child_process";
-import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
-import { join, resolve, sep } from "node:path";
-import { homedir, tmpdir } from "node:os";
-import { Readable, Writable } from "node:stream";
-import * as acp from "@agentclientprotocol/sdk";
-import { fileURLToPath } from "node:url";
-//#region src/acp.ts
-/**
-* Shared ACP (Agent Client Protocol) utilities — extracted from the Devin
-* adapter so they can be reused by other adapters and the benchmark runner.
-*
-* This module contains:
-* - {@link sanitizeEnv}: strip host-contaminating env vars before spawning an agent.
-* - {@link createPermissionHandler}: auto-approve permission requests.
-* - {@link runAcpSession}: drive a full ACP connect/initialize/prompt/collect loop.
-*/
-/** Env-var prefixes to strip from the spawned process environment. */
-const SANITIZE_PREFIXES = [
-	"GC_",
-	"BEADS_",
-	"BD_",
-	"MCP_",
-	"CLAUDECODE",
-	"CLAUDE_CODE_",
-	"CODEX_"
-];
-/** Exact env-var names to strip from the spawned process environment. */
-const SANITIZE_EXACT = /* @__PURE__ */ new Set([
-	"DEVIN_PERMISSION_MODE",
-	"DEVIN_MODEL",
-	"AGENTS_MD",
-	"CLAUDE_PROJECT_MD"
-]);
-/** Check if an env var name should be stripped from the agent environment. */
-function shouldSanitize(key) {
-	if (SANITIZE_EXACT.has(key)) return true;
-	return SANITIZE_PREFIXES.some((p) => key.startsWith(p));
-}
-/** Copy process.env into a clean record, dropping sanitized entries. */
-function copyCleanEnv() {
-	const env = {};
-	for (const [key, value] of Object.entries(process.env)) {
-		if (value === void 0) continue;
-		if (shouldSanitize(key)) continue;
-		env[key] = value;
-	}
-	return env;
-}
-/** Apply model and permission settings to the env record. */
-function applyModelEnv(env, config) {
-	const model = config.model;
-	env.DEVIN_PERMISSION_MODE = config.permissionMode ?? "dangerous";
-	env.DEVIN_MODEL = model.envVar ? env[model.envVar] ?? model.id : model.id;
-	if (model.envVar) env[model.envVar] = env[model.envVar] ?? model.id;
-}
-/** Merge explicit env overrides from config (these win). */
-function mergeEnvOverrides(env, overrides) {
-	if (!overrides) return;
-	for (const [key, value] of Object.entries(overrides)) env[key] = value;
-}
-/**
-* Build a sanitized environment for an agent subprocess.
-*
-* Strips out host contamination: Gas City, Beads, MCP servers, project
-* skills, and other variables that would leak the host's configuration
-* into the benchmark. The agent must only see what we explicitly install.
-*
-* Sets `DEVIN_PERMISSION_MODE` and `DEVIN_MODEL` from the spawn config,
-* then merges any explicit `config.env` overrides (which win).
-*/
-function sanitizeEnv(config) {
-	const env = copyCleanEnv();
-	applyModelEnv(env, config);
-	mergeEnvOverrides(env, config.env);
-	return env;
-}
-/**
-* Create a permission-request handler that auto-approves tool calls.
-*
-* Selects the first `allow_once` or `allow_always` option, falling back to
-* the first available option. This is the same handler used by both the
-* Devin adapter and the benchmark runner.
-*/
-function createPermissionHandler() {
-	return (ctx) => {
-		const options = ctx.params.options;
-		return { outcome: {
-			outcome: "selected",
-			optionId: options.find((o) => o.kind === "allow_once" || o.kind === "allow_always")?.optionId ?? options[0]?.optionId ?? ""
-		} };
-	};
-}
-/**
-* Process a single {@link acp.ActiveSessionMessage} from the session update
-* stream. Returns the final {@link PromptResponse} when the session reports
-* a stop, otherwise invokes the appropriate callback and returns `undefined`
-* so the caller can continue collecting.
-*/
-function handleSessionUpdate(message, onToolCall, onToolUpdate) {
-	if (message.kind === "stop") return message.response;
-	const update = message.update;
-	if (update.sessionUpdate === "tool_call") onToolCall?.(update);
-	else if (update.sessionUpdate === "tool_call_update") onToolUpdate?.(update);
-}
-/**
-* Drive a single ACP session: send the prompt, race the collect loop
-* against a timeout, and return the final {@link PromptResponse}.
-*/
-async function runSessionLoop(session, prompt, timeoutMs, onToolCall, onToolUpdate, onSession) {
-	onSession?.(session.sessionId);
-	const promptPromise = session.prompt(prompt);
-	const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error(`Agent timed out after ${timeoutMs}ms`)), timeoutMs));
-	const collectPromise = (async () => {
-		for (;;) {
-			const result = handleSessionUpdate(await session.nextUpdate(), onToolCall, onToolUpdate);
-			if (result) return result;
-		}
-	})();
-	return Promise.race([
-		promptPromise,
-		collectPromise,
-		timeoutPromise
-	]);
-}
-/**
-* Run a full ACP session against a spawned agent subprocess.
-*
-* Connects to the agent, initializes the protocol, creates a session in
-* `workspace`, sends `prompt`, and collects update events until the session
-* stops or `timeoutMs` elapses.
-*
-* The `onToolCall` and `onToolUpdate` callbacks let callers collect tool
-* calls and observations without duplicating the session loop logic. The
-* optional `onSession` callback reports the session ID once created. All
-* callbacks are optional.
-*
-* @returns The final {@link PromptResponse} from the agent.
-*/
-async function runAcpSession(proc, workspace, prompt, timeoutMs, onToolCall, onToolUpdate, onSession) {
-	if (!proc.stdin || !proc.stdout) throw new Error("Agent process missing stdin/stdout");
-	const input = Writable.toWeb(proc.stdin);
-	const output = Readable.toWeb(proc.stdout);
-	const stream = acp.ndJsonStream(input, output);
-	return acp.client({ name: "sverka-arena" }).onRequest(acp.methods.client.session.requestPermission, createPermissionHandler()).connectWith(stream, async (ctx) => {
-		await ctx.request(acp.methods.agent.initialize, {
-			protocolVersion: acp.PROTOCOL_VERSION,
-			clientCapabilities: {}
-		});
-		return ctx.buildSession(workspace).withSession((session) => runSessionLoop(session, prompt, timeoutMs, onToolCall, onToolUpdate, onSession));
-	});
-}
-//#endregion
-//#region src/adapters/devin.ts
-/**
-* Devin ACP adapter — spawns `devin acp` as a subprocess and drives an
-* Agent Client Protocol session to collect trace data + metrics.
-*
-* The adapter is abstract over models and plugins: it accepts any
-* {@link ModelConfig} and any set of {@link PluginConfig}s, installs the
-* enabled plugins into the workspace, and runs the prompt against the
-* spawned agent. After the session completes it reads the Devin CLI
-* transcript file to build full {@link TraceData}.
-*/
-/** Default transcript directory: ~/.local/share/devin/cli/transcripts */
-function transcriptDir() {
-	return join(homedir(), ".local", "share", "devin", "cli", "transcripts");
-}
-/** Read a transcript JSON file by session ID. Throws if not found. */
-async function readTranscript(sessionId, dir) {
-	const path = join(dir ?? transcriptDir(), `${sessionId}.json`);
-	const content = await readFile(path, "utf-8");
-	return JSON.parse(content);
-}
-/** Count LLM inference calls in a transcript (agent steps with telemetry.operation === "inference"). */
-function countLlmCalls(transcript) {
-	return transcript.steps.filter((s) => s.source === "agent" && s.extra?.telemetry?.operation === "inference").length;
-}
-/** Build a {@link TraceStep} from a raw transcript step. */
-function buildTraceStep(step) {
-	const isLlmCall = step.source === "agent" && step.extra?.telemetry?.operation === "inference";
-	return {
-		stepId: step.step_id,
-		timestamp: step.timestamp,
-		source: step.source,
-		message: step.message,
-		isLlmCall
-	};
-}
-/** Extract text content from a ToolCallContent array (best-effort). */
-function extractTextContent(contents) {
-	if (!contents) return "";
-	const parts = [];
-	for (const c of contents) if (c.type === "content" && c.content.type === "text") parts.push(c.content.text);
-	return parts.join("\n");
-}
-/** Extract token counts from a Usage object, defaulting to 0. */
-function extractUsageTokens(usage) {
-	return {
-		inputTokens: usage?.inputTokens ?? 0,
-		outputTokens: usage?.outputTokens ?? 0,
-		thoughtTokens: usage?.thoughtTokens ?? 0,
-		totalTokens: usage?.totalTokens ?? 0
-	};
-}
-/** Build {@link RunMetrics} from a PromptResponse + collected counts + timing. */
-function buildMetrics(promptResult, toolCallCount, llmCallCount, startTime) {
-	return {
-		...extractUsageTokens(promptResult.usage),
-		toolCallCount,
-		llmCallCount,
-		executionTimeMs: Date.now() - startTime,
-		stopReason: promptResult.stopReason
-	};
-}
-/**
-* Extract the agent's final text output from the prompt response or trace.
-*
-* Prefers the text content from the ACP `PromptResponse`, falling back to
-* the last agent message in the trace.
-*/
-function extractAgentOutput(promptResult, trace) {
-	for (let i = trace.steps.length - 1; i >= 0; i--) {
-		const step = trace.steps[i];
-		if (step?.source === "agent" && step.message.length > 0) return step.message;
-	}
-	return "";
-}
-/**
-* Install enabled plugins and remove disabled ones from the workspace.
-*
-* CRITICAL: This function first removes the entire `.agents/skills/`
-* directory to ensure no inherited skills from the host contaminate the
-* benchmark. Only explicitly enabled plugins are installed.
-*
-* Each enabled plugin's `path` is copied to `workspace/.agents/skills/<id>/`.
-*/
-async function installPlugins(workspace, plugins) {
-	const skillsDir = join(workspace, ".agents", "skills");
-	await rm(skillsDir, {
-		recursive: true,
-		force: true
-	});
-	const enabledPlugins = plugins.filter((p) => p.enabled);
-	if (enabledPlugins.length === 0) return;
-	await mkdir(skillsDir, { recursive: true });
-	for (const plugin of enabledPlugins) {
-		if (plugin.id.includes("/") || plugin.id.includes("\\") || plugin.id.includes("..")) throw new Error(`Invalid plugin id: ${plugin.id}`);
-		const dest = join(skillsDir, plugin.id);
-		await mkdir(dest, { recursive: true });
-		await cp(plugin.path, dest, { recursive: true });
-	}
-}
-/**
-* Devin ACP adapter — spawns `devin acp` as a subprocess.
-*
-* Usage:
-* ```ts
-* const adapter = new DevinAdapter();
-* const proc = adapter.spawn({ model, workspace, plugins });
-* const result = await proc.run("Fix the bug", 120_000);
-* proc.kill();
-* ```
-*/
-var DevinAdapter = class {
-	id = "devin";
-	spawn(config) {
-		const proc = spawnDevin(config);
-		let killed = false;
-		return {
-			run: (prompt, timeoutMs) => runDevinSession(proc, config, prompt, timeoutMs),
-			kill: () => {
-				if (killed) return;
-				killed = true;
-				proc.kill("SIGTERM");
-			}
-		};
-	}
-};
-/** Resolve the full path to the devin binary to avoid PATH-based lookup. */
-function resolveDevinBinary() {
-	try {
-		return execSync("which devin", {
-			encoding: "utf-8",
-			stdio: [
-				"pipe",
-				"pipe",
-				"pipe"
-			]
-		}).trim();
-	} catch {
-		return "devin";
-	}
-}
-/** Spawn `devin acp --model <model.id>` as a subprocess in the workspace. */
-function spawnDevin(config) {
-	const env = sanitizeEnv(config);
-	const devinBin = resolveDevinBinary();
-	return spawn(devinBin, [
-		"acp",
-		"--model",
-		config.model.id
-	], {
-		cwd: config.workspace,
-		stdio: [
-			"pipe",
-			"pipe",
-			"inherit"
-		],
-		env
-	});
-}
-/** Create a {@link ToolCollector} with callbacks that populate shared state. */
-function createToolCollector() {
-	const collectedToolCalls = [];
-	const observationsByCallId = /* @__PURE__ */ new Map();
-	let toolCallCount = 0;
-	const onToolCall = (update) => {
-		toolCallCount++;
-		const name = update.name ?? update.title ?? "unknown";
-		const args = update.rawInput ?? {};
-		collectedToolCalls.push({
-			functionName: name,
-			arguments: args,
-			toolCallId: update.toolCallId
-		});
-	};
-	const onToolUpdate = (update) => {
-		const text = extractTextContent(update.content ?? void 0);
-		if (text) observationsByCallId.set(update.toolCallId, {
-			sourceCallId: update.toolCallId,
-			content: text
-		});
-	};
-	return {
-		collectedToolCalls,
-		observationsByCallId,
-		get toolCallCount() {
-			return toolCallCount;
-		},
-		onToolCall,
-		onToolUpdate
-	};
-}
-/** Build a {@link RunResult} for a failed session (best-effort trace from collected data). */
-function buildErrorResult(model, pluginIds, toolCallCount, sessionId, startTime, error) {
-	const metrics = buildMetrics({
-		stopReason: "cancelled",
-		usage: null
-	}, toolCallCount, 0, startTime);
-	const trace = {
-		sessionId,
-		model: model.id,
-		steps: [],
-		finalMetrics: {
-			totalPromptTokens: 0,
-			totalCompletionTokens: 0,
-			totalCachedTokens: 0,
-			totalSteps: 0
-		}
-	};
-	return {
-		taskId: "",
-		modelId: model.id,
-		pluginIds,
-		metrics,
-		trace,
-		output: "",
-		checkResults: [],
-		verdicts: [],
-		success: false,
-		error: error instanceof Error ? error.message : String(error)
-	};
-}
-/** Run a full ACP session: initialize, create session, prompt, collect events, read transcript. */
-async function runDevinSession(proc, config, prompt, timeoutMs) {
-	const startTime = Date.now();
-	const model = config.model;
-	const pluginIds = config.plugins.filter((p) => p.enabled).map((p) => p.id);
-	const collector = createToolCollector();
-	let sessionId = "";
-	try {
-		const promptResult = await runAcpSession(proc, config.workspace, prompt, timeoutMs, collector.onToolCall, collector.onToolUpdate, (id) => {
-			sessionId = id;
-		});
-		const trace = await buildTrace(sessionId, model, collector.collectedToolCalls, collector.observationsByCallId);
-		const llmCallCount = trace.steps.filter((s) => s.isLlmCall).length;
-		const metrics = buildMetrics(promptResult, collector.toolCallCount, llmCallCount, startTime);
-		const agentOutput = extractAgentOutput(promptResult, trace);
-		return {
-			taskId: "",
-			modelId: model.id,
-			pluginIds,
-			metrics,
-			trace,
-			output: agentOutput,
-			checkResults: [],
-			verdicts: [],
-			success: metrics.stopReason === "end_turn"
-		};
-	} catch (error) {
-		return buildErrorResult(model, pluginIds, collector.toolCallCount, sessionId, startTime, error);
-	}
-}
-/**
-* Build {@link TraceData} from the Devin transcript, enriched with
-* ACP-collected tool calls. Returns `undefined` when no transcript is
-* available so the caller can fall back to {@link buildTraceFallback}.
-*/
-async function buildTraceFromTranscript(sessionId, collectedToolCalls, observationsByCallId) {
-	if (!sessionId) return void 0;
-	try {
-		const transcript = await readTranscript(sessionId);
-		const steps = transcript.steps.map(buildTraceStep);
-		attachToolCallsToSteps(steps, collectedToolCalls, observationsByCallId);
-		return {
-			sessionId: transcript.session_id,
-			model: transcript.agent.model_name,
-			steps,
-			finalMetrics: {
-				totalPromptTokens: transcript.final_metrics.total_prompt_tokens,
-				totalCompletionTokens: transcript.final_metrics.total_completion_tokens,
-				totalCachedTokens: transcript.final_metrics.total_cached_tokens,
-				totalSteps: transcript.final_metrics.total_steps
-			}
-		};
-	} catch {
-		return;
-	}
-}
-/** Attach the next tool call + observation to an agent step. Returns true if attached. */
-function attachNextToolCall(step, collectedToolCalls, observationsByCallId, toolIdx) {
-	if (toolIdx >= collectedToolCalls.length) return {
-		attached: false,
-		nextIdx: toolIdx
-	};
-	const tc = collectedToolCalls[toolIdx];
-	if (!tc) return {
-		attached: false,
-		nextIdx: toolIdx
-	};
-	const calls = [tc];
-	const obs = observationsByCallId.get(tc.toolCallId);
-	const observations = obs ? [obs] : [];
-	step.toolCalls = calls;
-	if (observations.length > 0) step.observations = observations;
-	return {
-		attached: true,
-		nextIdx: toolIdx + 1
-	};
-}
-/** Attach ACP-collected tool calls/observations to agent steps in order. */
-function attachToolCallsToSteps(steps, collectedToolCalls, observationsByCallId) {
-	let toolIdx = 0;
-	for (const step of steps) {
-		if (step.source !== "agent") continue;
-		const result = attachNextToolCall(step, collectedToolCalls, observationsByCallId, toolIdx);
-		if (result.attached) toolIdx = result.nextIdx;
-	}
-}
-/** Build a minimal {@link TraceData} from ACP-collected data only. */
-function buildTraceFallback(sessionId, model, collectedToolCalls, observationsByCallId) {
-	const steps = collectedToolCalls.map((tc, i) => {
-		const obs = observationsByCallId.get(tc.toolCallId);
-		const step = {
-			stepId: i,
-			timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-			source: "agent",
-			message: tc.functionName,
-			toolCalls: [tc],
-			isLlmCall: false
-		};
-		if (obs) step.observations = [obs];
-		return step;
-	});
-	return {
-		sessionId,
-		model: model.id,
-		steps,
-		finalMetrics: {
-			totalPromptTokens: 0,
-			totalCompletionTokens: 0,
-			totalCachedTokens: 0,
-			totalSteps: steps.length
-		}
-	};
-}
-/** Build {@link TraceData} from the Devin transcript, falling back to ACP-only data. */
-async function buildTrace(sessionId, model, collectedToolCalls, observationsByCallId) {
-	return await buildTraceFromTranscript(sessionId, collectedToolCalls, observationsByCallId) ?? buildTraceFallback(sessionId, model, collectedToolCalls, observationsByCallId);
-}
-//#endregion
-//#region src/judge.ts
-/** Default system prompt for the judge. */
-const DEFAULT_JUDGE_PROMPT = "You are a blind evaluator. You will receive a task prompt and an agent's response. Score the response 0-100 based on how well it completes the task. Be strict but fair. Output JSON: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70+ is passing.";
-/** Default timeout for a single judge run (ms). */
-const DEFAULT_JUDGE_TIMEOUT_MS = 12e4;
-/**
-* Build the judge prompt for a single run.
-*
-* Includes the task prompt, success criteria, and the agent's output. Does
-* NOT include which plugins were active unless `config.revealPlugins` is
-* true. Asks the judge for a JSON response with score, passed, reasoning,
-* and issues.
-*/
-function buildJudgePrompt(task, runOutput, config, pluginIds = []) {
-	const lines = [
-		config.systemPrompt ?? "You are a blind evaluator. You will receive a task prompt and an agent's response. Score the response 0-100 based on how well it completes the task. Be strict but fair. Output JSON: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70+ is passing.",
-		"",
-		"## Task Prompt",
-		task.prompt,
-		""
-	];
-	if (task.successCriteria) lines.push("## Success Criteria", task.successCriteria, "");
-	if (config.revealPlugins && task.id) {
-		lines.push("## Plugins Active During Run");
-		if (pluginIds.length > 0) lines.push(pluginIds.join(", "));
-		else lines.push("(no plugins)");
-		lines.push("");
-	}
-	lines.push("## Agent Response");
-	lines.push(runOutput || "(no output)");
-	lines.push("");
-	lines.push("## Instructions\nEvaluate the agent's response against the task prompt and success criteria. Respond with ONLY a JSON object: {\"score\": number, \"passed\": boolean, \"reasoning\": string, \"issues\": string[]}. A score of 70 or above is passing. Do not include any text outside the JSON.");
-	return lines.join("\n");
-}
-/**
-* Parse a judge response into a {@link JudgeVerdict}.
-*
-* Extracts JSON from the response, handling markdown code fences
-* (` ```json ... ``` `). Falls back to `score=0, passed=false` when the
-* response cannot be parsed.
-*/
-function parseJudgeResponse(response, runIndex, run) {
-	const jsonText = extractJson(response);
-	const fallback = {
-		runIndex,
-		score: 0,
-		passed: false,
-		reasoning: response.trim() || "Failed to parse judge response",
-		issues: [],
-		taskId: run.taskId,
-		modelId: run.modelId,
-		pluginIds: run.pluginIds
-	};
-	if (jsonText === null) return fallback;
-	let parsed;
-	try {
-		parsed = JSON.parse(jsonText);
-	} catch {
-		return fallback;
-	}
-	if (typeof parsed !== "object" || parsed === null) return fallback;
-	const obj = parsed;
-	const score = typeof obj.score === "number" ? obj.score : 0;
-	return {
-		runIndex,
-		score,
-		passed: typeof obj.passed === "boolean" ? obj.passed : score >= 70,
-		reasoning: typeof obj.reasoning === "string" ? obj.reasoning : "",
-		issues: Array.isArray(obj.issues) ? obj.issues.filter((i) => typeof i === "string") : [],
-		taskId: run.taskId,
-		modelId: run.modelId,
-		pluginIds: run.pluginIds
-	};
-}
-/**
-* Judge a single run — spawn the judge agent, send the prompt, collect the
-* response, parse it into a {@link JudgeVerdict}, and kill the agent.
-*/
-async function judgeRun(run, task, config, runIndex) {
-	const prompt = buildJudgePrompt(task, run.output, config, run.pluginIds);
-	const proc = config.agent.spawn({
-		model: config.model,
-		workspace: process.cwd(),
-		plugins: [],
-		permissionMode: "dangerous"
-	});
-	try {
-		return parseJudgeResponse((await proc.run(prompt, DEFAULT_JUDGE_TIMEOUT_MS)).output, runIndex, run);
-	} catch (error) {
-		return {
-			runIndex,
-			score: 0,
-			passed: false,
-			reasoning: error instanceof Error ? error.message : String(error),
-			issues: [],
-			taskId: run.taskId,
-			modelId: run.modelId,
-			pluginIds: run.pluginIds
-		};
-	} finally {
-		proc.kill();
-	}
-}
-/**
-* Judge all runs in a list, pairing each run with its matching task.
-*
-* Supports `config.repetitions` — the judge is run N times per run for
-* confidence. All verdicts are returned (one per repetition).
-*/
-async function judgeAllRuns(results, tasks, config) {
-	const taskById = /* @__PURE__ */ new Map();
-	for (const task of tasks) taskById.set(task.id, task);
-	const repetitions = config.repetitions ?? 1;
-	const verdicts = [];
-	for (let i = 0; i < results.length; i++) {
-		const run = results[i];
-		if (!run) continue;
-		const task = taskById.get(run.taskId);
-		if (!task) continue;
-		for (let rep = 0; rep < repetitions; rep++) {
-			const verdict = await judgeRun(run, task, config, i);
-			verdicts.push(verdict);
-		}
-	}
-	return verdicts;
-}
-/**
-* Compare two plugin combos for a task (for {@link ComboComparison}).
-*
-* Computes deltas as `candidate - baseline`. Determines whether the
-* candidate is better:
-*   - Lower tokens / toolCalls / llmCalls / time = better
-*   - Higher judgeScore = better
-*
-* The judge score for each run is the average of its verdicts' scores
-* (0 when there are no verdicts).
-*/
-function compareCombos(baseline, candidate) {
-	const baselineScore = avgJudgeScore(baseline);
-	const candidateScore = avgJudgeScore(candidate);
-	const deltaTokens = candidate.metrics.totalTokens - baseline.metrics.totalTokens;
-	const deltaToolCalls = candidate.metrics.toolCallCount - baseline.metrics.toolCallCount;
-	const deltaLlmCalls = candidate.metrics.llmCallCount - baseline.metrics.llmCallCount;
-	const deltaTimeMs = candidate.metrics.executionTimeMs - baseline.metrics.executionTimeMs;
-	const deltaJudgeScore = candidateScore - baselineScore;
-	const candidateBetter = deltaJudgeScore > 0 || deltaJudgeScore === 0 && deltaTokens <= 0 && deltaToolCalls <= 0 && deltaLlmCalls <= 0 && deltaTimeMs <= 0 && (deltaTokens < 0 || deltaToolCalls < 0 || deltaLlmCalls < 0 || deltaTimeMs < 0);
-	return {
-		baseline: comboLabel$1(baseline),
-		candidate: comboLabel$1(candidate),
-		deltaTokens,
-		deltaToolCalls,
-		deltaLlmCalls,
-		deltaTimeMs,
-		deltaJudgeScore,
-		candidateBetter
-	};
-}
-/**
-* Extract the first JSON object from a response string.
-*
-* Handles raw JSON and JSON wrapped in markdown code fences
-* (` ```json ... ``` ` or ` ``` ... ``` `). Returns `null` when no JSON
-* object can be located.
-*/
-function extractJson(response) {
-	const trimmed = response.trim();
-	if (trimmed === "") return null;
-	const fenced = extractFromFence(trimmed);
-	if (fenced !== null) return fenced;
-	const braced = extractFromBraces(trimmed);
-	if (braced !== null) return braced;
-	return null;
-}
-/** Extract JSON from inside a markdown code fence (```...```). */
-function extractFromFence(text) {
-	const fenceStart = text.indexOf("```");
-	if (fenceStart === -1) return null;
-	const afterFence = text.slice(fenceStart + 3);
-	let contentStart = 0;
-	if (afterFence.startsWith("json")) contentStart = 4;
-	while (contentStart < afterFence.length && /\s/.test(afterFence[contentStart])) contentStart++;
-	const fenceEnd = afterFence.indexOf("```", contentStart);
-	if (fenceEnd === -1) return null;
-	const inner = afterFence.slice(contentStart, fenceEnd).trim();
-	if (!looksLikeJson(inner)) return null;
-	return inner;
-}
-/** Extract JSON by finding the first `{` and last `}` in the text. */
-function extractFromBraces(text) {
-	const start = text.indexOf("{");
-	const end = text.lastIndexOf("}");
-	if (start === -1 || end === -1 || end <= start) return null;
-	return text.slice(start, end + 1);
-}
-/** Quick check that a string starts with `{` (a JSON object). */
-function looksLikeJson(text) {
-	return text.trim().startsWith("{");
-}
-/** Average judge score across a run's verdicts (0 when no verdicts). */
-function avgJudgeScore(run) {
-	if (run.verdicts.length === 0) return 0;
-	return run.verdicts.reduce((a, v) => a + v.score, 0) / run.verdicts.length;
-}
-/** Build a human-readable combo label for a run's plugin set. */
-function comboLabel$1(run) {
-	if (run.pluginIds.length === 0) return "no-plugins";
-	return run.pluginIds.join(",");
-}
-//#endregion
-//#region src/runner.ts
-/**
-* Matrix runner — executes the full arena matrix:
-*
-*   task × model × plugin combination × repetition
-*
-* For each cell it spawns the agent, runs the prompt, collects the result,
-* and kills the agent. Results are aggregated into {@link ArenaResult}.
-*/
-/** Resolve the arena package root (for fixture paths). */
-const PKG_ROOT = resolve(typeof __dirname !== "undefined" ? __dirname : fileURLToPath(new URL(".", import.meta.url)), "..");
-/**
-* Generate all plugin on/off combinations.
-*
-* For N plugins this produces 2^N combinations, each a full copy of the
-* plugins array with `enabled` set appropriately.
-*
-* - 0 plugins → `[[]]` (one empty combination)
-* - 1 plugin  → `[[off], [on]]`
-* - 2 plugins → `[[off,off], [on,off], [off,on], [on,on]]`
-*/
-function pluginCombinations(plugins) {
-	if (plugins.length === 0) return [[]];
-	const combos = [[]];
-	for (const plugin of plugins) {
-		const next = [];
-		for (const combo of combos) {
-			next.push([...combo, {
-				...plugin,
-				enabled: false
-			}]);
-			next.push([...combo, {
-				...plugin,
-				enabled: true
-			}]);
-		}
-		combos.length = 0;
-		combos.push(...next);
-	}
-	return combos;
-}
-/**
-* Run the full arena matrix: task × model × plugin combo × repetitions.
-*
-* Each cell spawns the configured agent adapter, installs the plugin
-* combination, runs the prompt, collects trace + metrics, and kills the
-* agent. Results are written to `config.outputDir/results.json` and the
-* full {@link ArenaResult} is returned.
-*/
-async function runArena(config) {
-	const repetitions = config.repetitions ?? 1;
-	const combos = pluginCombinations(config.plugins);
-	const results = await runMatrix(config, combos);
-	if (config.judge) {
-		const verdicts = await judgeAllRuns(results, config.tasks, config.judge);
-		for (const verdict of verdicts) {
-			const run = results[verdict.runIndex];
-			if (run) run.verdicts.push(verdict);
-		}
-	}
-	const aggregates = buildAggregates(results, config.models, combos);
-	const analysis = computeAnalysis(results, config.tasks);
-	const arenaResult = {
-		timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-		config: {
-			models: config.models.map((m) => m.id),
-			plugins: config.plugins.map((p) => p.id),
-			tasks: config.tasks.map((t) => t.id),
-			repetitions,
-			...config.judge ? { judgeModel: config.judge.model.id } : {}
-		},
-		results,
-		aggregates,
-		analysis
-	};
-	await mkdir(config.outputDir, { recursive: true });
-	await writeFile(join(config.outputDir, "results.json"), JSON.stringify(arenaResult, null, 2), "utf-8");
-	return arenaResult;
-}
-/**
-* Run every cell of the task × model × combo × repetition matrix, collecting
-* results and logging per-cell progress to stderr.
-*/
-async function runMatrix(config, combos) {
-	const repetitions = config.repetitions ?? 1;
-	const results = [];
-	for (const task of config.tasks) for (const model of config.models) for (const combo of combos) for (let rep = 0; rep < repetitions; rep++) {
-		const cellLabel = formatCell(task.id, model.id, combo, rep);
-		process.stderr.write(`[arena] ${cellLabel} running...\n`);
-		const result = await runCell(config.agent, task, model, combo, rep);
-		results.push(result);
-		process.stderr.write(`[arena] ${cellLabel} ${result.success ? "PASS" : "FAIL"} — ${result.metrics.totalTokens} tokens, ${result.metrics.toolCallCount} tools, ${result.metrics.llmCallCount} llm calls, ${result.metrics.executionTimeMs}ms\n`);
-	}
-	return results;
-}
-/**
-* Aggregate metrics per (model × plugin combo) cell.
-*/
-function buildAggregates(results, models, combos) {
-	const aggregates = [];
-	for (const model of models) for (const combo of combos) {
-		const pluginIds = combo.filter((p) => p.enabled).map((p) => p.id);
-		const label = formatAggregateLabel(model.id, pluginIds);
-		aggregates.push(aggregateResults(results, (r) => r.modelId === model.id && samePluginIds(r.pluginIds, pluginIds), label));
-	}
-	return aggregates;
-}
-/** Run a single matrix cell in an ISOLATED temp workspace. */
-async function runCell(agent, task, model, combo, rep) {
-	const tempWorkspace = await createTempWorkspace(task, model, combo, rep);
-	await installPlugins(tempWorkspace, combo);
-	const result = await executeRun(agent, task, model, combo, tempWorkspace);
-	await rm(tempWorkspace, {
-		recursive: true,
-		force: true
-	});
-	return result;
-}
-/**
-* Create an isolated temp workspace with a fresh copy of the fixture (if any).
-* Each run gets its own copy to prevent skill contamination from the host repo.
-*/
-async function createTempWorkspace(task, model, combo, rep) {
-	const tempWorkspace = join(tmpdir(), `arena-${task.id}-${model.id}-${combo.map((p) => p.id + (p.enabled ? "on" : "off")).join(",")}-r${rep}-${Date.now()}`);
-	await mkdir(tempWorkspace, { recursive: true });
-	if (task.fixture) {
-		const fixturePath = resolve(PKG_ROOT, task.fixture);
-		if (!fixturePath.startsWith(PKG_ROOT + sep) && fixturePath !== PKG_ROOT) throw new Error(`Fixture path escapes package root: ${task.fixture}`);
-		await cp(fixturePath, tempWorkspace, { recursive: true });
-	}
-	return tempWorkspace;
-}
-/**
-* Spawn the agent, run the prompt, run deterministic checks, and return the
-* result. On error returns a zeroed {@link RunResult} via {@link errorResult}.
-*/
-async function executeRun(agent, task, model, combo, tempWorkspace) {
-	const proc = agent.spawn({
-		model,
-		workspace: tempWorkspace,
-		plugins: combo,
-		permissionMode: "dangerous"
-	});
-	try {
-		const result = await proc.run(task.prompt, task.timeoutMs ?? 12e4);
-		result.taskId = task.id;
-		if (!result.verdicts) result.verdicts = [];
-		if (!result.checkResults) result.checkResults = [];
-		if (task.checks && task.checks.length > 0) {
-			result.checkResults = await runChecks(tempWorkspace, task.checks);
-			result.success = result.success && result.checkResults.every((c) => c.passed);
-		}
-		return result;
-	} catch (error) {
-		return errorResult(task, model, combo, error);
-	} finally {
-		proc.kill();
-	}
-}
-/** Build a zeroed {@link RunResult} for a failed run. */
-function errorResult(task, model, combo, error) {
-	return {
-		taskId: task.id,
-		modelId: model.id,
-		pluginIds: combo.filter((p) => p.enabled).map((p) => p.id),
-		metrics: {
-			inputTokens: 0,
-			outputTokens: 0,
-			thoughtTokens: 0,
-			totalTokens: 0,
-			toolCallCount: 0,
-			llmCallCount: 0,
-			executionTimeMs: 0,
-			stopReason: "cancelled"
-		},
-		trace: {
-			sessionId: "",
-			model: model.id,
-			steps: [],
-			finalMetrics: {
-				totalPromptTokens: 0,
-				totalCompletionTokens: 0,
-				totalCachedTokens: 0,
-				totalSteps: 0
-			}
-		},
-		output: "",
-		checkResults: [],
-		verdicts: [],
-		success: false,
-		error: error instanceof Error ? error.message : String(error)
-	};
-}
-/** Run deterministic checks in the workspace. */
-async function runChecks(workspace, checks) {
-	if (!checks) return [];
-	const results = [];
-	for (const check of checks) try {
-		const { output, exitCode } = await new Promise((resolve, reject) => {
-			const proc = spawn("bash", ["-c", check.command], {
-				cwd: workspace,
-				stdio: [
-					"pipe",
-					"pipe",
-					"pipe"
-				],
-				env: {
-					...process.env,
-					CI: "true"
-				}
-			});
-			let stdout = "";
-			proc.stdout?.on("data", (d) => {
-				stdout += d.toString();
-			});
-			proc.stderr?.on("data", (d) => {
-				stdout += d.toString();
-			});
-			proc.on("close", (code) => {
-				resolve({
-					output: stdout,
-					exitCode: code
-				});
-			});
-			proc.on("error", reject);
-		});
-		results.push({
-			checkId: check.id,
-			passed: exitCode === 0,
-			output: output.trim(),
-			exitCode
-		});
-	} catch (error) {
-		results.push({
-			checkId: check.id,
-			passed: false,
-			output: error instanceof Error ? error.message : String(error),
-			exitCode: -1
-		});
-	}
-	return results;
-}
-/**
-* Aggregate metrics for a group of runs selected by `filter`.
-*
-* Computes averages across all matching runs. Returns zeroed metrics when
-* no runs match. Includes judge-related fields (`avgJudgeScore`,
-* `judgePassCount`) and a `label` identifying the aggregate group.
-*/
-function aggregateResults(results, filter, label = "") {
-	const filtered = results.filter(filter);
-	const n = filtered.length;
-	if (n === 0) return {
-		totalRuns: 0,
-		successCount: 0,
-		avgInputTokens: 0,
-		avgOutputTokens: 0,
-		avgTotalTokens: 0,
-		avgToolCalls: 0,
-		avgLlmCalls: 0,
-		avgExecutionTimeMs: 0,
-		avgJudgeScore: 0,
-		judgePassCount: 0,
-		label
-	};
-	const sum = (f) => filtered.reduce((a, r) => a + f(r), 0);
-	const round = (v) => Math.round(v);
-	const round2 = (v) => Math.round(v * 100) / 100;
-	const judgedRuns = filtered.filter((r) => r.verdicts.length > 0);
-	const avgJudgeScore = judgedRuns.length === 0 ? 0 : round2(judgedRuns.reduce((a, r) => a + (r.verdicts[0]?.score ?? 0), 0) / judgedRuns.length);
-	const judgePassCount = filtered.filter((r) => r.verdicts[0]?.passed === true).length;
-	return {
-		totalRuns: n,
-		successCount: filtered.filter((r) => r.success).length,
-		avgInputTokens: round(sum((r) => r.metrics.inputTokens) / n),
-		avgOutputTokens: round(sum((r) => r.metrics.outputTokens) / n),
-		avgTotalTokens: round(sum((r) => r.metrics.totalTokens) / n),
-		avgToolCalls: round2(sum((r) => r.metrics.toolCallCount) / n),
-		avgLlmCalls: round2(sum((r) => r.metrics.llmCallCount) / n),
-		avgExecutionTimeMs: round(sum((r) => r.metrics.executionTimeMs) / n),
-		avgJudgeScore,
-		judgePassCount,
-		label
-	};
-}
-/** Compare two plugin-id arrays for equality (order-insensitive). */
-function samePluginIds(a, b) {
-	if (a.length !== b.length) return false;
-	const set = new Set(a);
-	for (const id of b) if (!set.has(id)) return false;
-	return true;
-}
-/** Format a human-readable cell label for logging. */
-function formatCell(taskId, modelId, combo, rep) {
-	return `${taskId}/${modelId}/{${combo.map((p) => `${p.id}=${p.enabled ? "on" : "off"}`).join(",")}}#${rep}`;
-}
-/** Format an aggregate group label: "model-id/plugins=on,off" or "model-id/no-plugins". */
-function formatAggregateLabel(modelId, pluginIds) {
-	if (pluginIds.length === 0) return `${modelId}/no-plugins`;
-	return `${modelId}/plugins=${pluginIds.join(",")}`;
-}
-/** Format a plugin combo label for comparisons: "no-plugins" or comma-joined ids. */
-function comboLabel(pluginIds) {
-	if (pluginIds.length === 0) return "no-plugins";
-	return pluginIds.join(",");
-}
-/**
-* Create a synthetic "average" {@link RunResult} representing a group of
-* runs with the same plugin combo. Used to feed {@link compareCombos} which
-* operates on single runs.
-*/
-function averageRun(runs) {
-	if (runs.length === 0) return null;
-	const n = runs.length;
-	const avg = (f) => Math.round(runs.reduce((a, r) => a + f(r), 0) / n);
-	const avg2 = (f) => Math.round(runs.reduce((a, r) => a + f(r), 0) / n * 100) / 100;
-	const first = runs[0];
-	return {
-		taskId: first.taskId,
-		modelId: first.modelId,
-		pluginIds: first.pluginIds,
-		metrics: {
-			inputTokens: avg((r) => r.metrics.inputTokens),
-			outputTokens: avg((r) => r.metrics.outputTokens),
-			thoughtTokens: avg((r) => r.metrics.thoughtTokens),
-			totalTokens: avg((r) => r.metrics.totalTokens),
-			toolCallCount: avg2((r) => r.metrics.toolCallCount),
-			llmCallCount: avg2((r) => r.metrics.llmCallCount),
-			executionTimeMs: avg((r) => r.metrics.executionTimeMs),
-			stopReason: first.metrics.stopReason
-		},
-		trace: first.trace,
-		output: first.output,
-		checkResults: first.checkResults ?? [],
-		verdicts: first.verdicts,
-		success: runs.every((r) => r.success)
-	};
-}
-/**
-* Compute per-task analysis comparing baseline (no plugins) vs each
-* candidate plugin combo.
-*
-* For each task, groups runs by plugin combo, creates an averaged
-* representative run per group, then compares the baseline (empty
-* `pluginIds`) against every non-empty candidate combo using
-* {@link compareCombos} from `judge.ts`.
-*/
-function computeAnalysis(results, tasks) {
-	const analyses = [];
-	for (const task of tasks) {
-		const taskRuns = results.filter((r) => r.taskId === task.id);
-		const comboGroups = /* @__PURE__ */ new Map();
-		for (const run of taskRuns) {
-			const key = comboLabel(run.pluginIds);
-			const group = comboGroups.get(key);
-			if (group) group.push(run);
-			else comboGroups.set(key, [run]);
-		}
-		const baseline = averageRun(comboGroups.get("no-plugins") ?? []);
-		const comparisons = [];
-		if (baseline) for (const [key, candidateRuns] of comboGroups) {
-			if (key === "no-plugins") continue;
-			const candidate = averageRun(candidateRuns);
-			if (candidate) comparisons.push(compareCombos(baseline, candidate));
-		}
-		analyses.push({
-			taskId: task.id,
-			taskName: task.name,
-			prompt: task.prompt,
-			comparisons
-		});
-	}
-	return analyses;
-}
-//#endregion
-export { DEFAULT_JUDGE_PROMPT, DevinAdapter, aggregateResults, buildJudgePrompt, compareCombos, computeAnalysis, countLlmCalls, createPermissionHandler, extractUsageTokens, installPlugins, judgeAllRuns, judgeRun, parseJudgeResponse, pluginCombinations, runAcpSession, runArena, sanitizeEnv, transcriptDir };
-
-//# sourceMappingURL=index.mjs.map
+import { C as sanitizeEnv, S as runAcpSession, _ as countLlmCalls, a as resolveAdapter, b as transcriptDir, c as pluginCombinations, d as buildJudgePrompt, f as compareCombos, g as DevinAdapter, h as parseJudgeResponse, i as loadArenaConfig, l as runArena, m as judgeRun, n as ArenaError, o as aggregateResults, p as judgeAllRuns, r as defineConfig, s as computeAnalysis, t as renderReport, u as DEFAULT_JUDGE_PROMPT, v as extractUsageTokens, x as createPermissionHandler, y as installPlugins } from "./report-3Gl3woII.mjs";
+export { ArenaError, DEFAULT_JUDGE_PROMPT, DevinAdapter, aggregateResults, buildJudgePrompt, compareCombos, computeAnalysis, countLlmCalls, createPermissionHandler, defineConfig, extractUsageTokens, installPlugins, judgeAllRuns, judgeRun, loadArenaConfig, parseJudgeResponse, pluginCombinations, renderReport, resolveAdapter, runAcpSession, runArena, sanitizeEnv, transcriptDir };