cyber-aced 0.3.0 → 0.4.1

package/dist/aced-bench.js

@@ -0,0 +1,1256 @@
+#!/usr/bin/env node
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { accessSync, constants, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { delimiter, dirname, isAbsolute, join, normalize, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { parseArgs } from "node:util";
+import { gzipSync } from "node:zlib";
+import { headlessCommand, headlessInvocation } from "@cyberuni/agent-harness";
+//#region skills/bench/scripts/bench-claude-code.mts
+/** Reads `--output-format stream-json`: tool calls from assistant messages, the rest from `result`. */
+function parseStreamJson(text) {
+	const out = {
+		inputTokens: 0,
+		outputTokens: 0,
+		cacheReadTokens: 0,
+		cacheCreationTokens: 0,
+		turns: 0,
+		toolCalls: 0,
+		costUsd: 0,
+		capped: true
+	};
+	for (const line of text.split("\n")) {
+		if (!line.startsWith("{")) continue;
+		let event;
+		try {
+			event = JSON.parse(line);
+		} catch {
+			continue;
+		}
+		if (event.type === "system" && typeof event.model === "string" && event.model && out.model === void 0) out.model = event.model;
+		else if (event.type === "assistant") {
+			const content = Array.isArray(event.message?.content) ? event.message.content : [];
+			out.toolCalls += content.filter((c) => c?.type === "tool_use").length;
+			if (out.model === void 0 && typeof event.message?.model === "string" && event.message.model) out.model = event.message.model;
+		} else if (event.type === "result") {
+			const u = event.usage ?? {};
+			out.inputTokens = u.input_tokens ?? 0;
+			out.outputTokens = u.output_tokens ?? 0;
+			out.cacheReadTokens = u.cache_read_input_tokens ?? 0;
+			out.cacheCreationTokens = u.cache_creation_input_tokens ?? 0;
+			out.turns = event.num_turns ?? 0;
+			out.costUsd = event.total_cost_usd ?? 0;
+			out.capped = event.subtype !== "success" || event.is_error === true;
+		}
+	}
+	return out;
+}
+const HARNESS = "claude-code";
+function launchArgs(o) {
+	const facts = headlessInvocation(HARNESS);
+	const cmd = headlessCommand(HARNESS, {
+		prompt: o.prompt,
+		model: o.model,
+		permission: o.permissionMode,
+		outputFormat: "stream-json"
+	});
+	if (!cmd || facts.supported !== true) throw new Error("@cyberuni/agent-harness reports no headless mode for claude-code");
+	const args = [...cmd.args];
+	const disable = facts.transcript?.disable ?? [];
+	const extra = [
+		"--max-budget-usd",
+		String(o.maxBudgetUsd),
+		...disable,
+		...disable.includes("--no-session-persistence") ? [] : ["--no-session-persistence"],
+		"--setting-sources",
+		"project",
+		"--strict-mcp-config"
+	];
+	if (o.pluginDir) extra.push("--plugin-dir", o.pluginDir);
+	if (existsSync(join(o.checkout, ".mcp.json"))) extra.push("--mcp-config", ".mcp.json");
+	return [...args, ...extra];
+}
+const claudeCode = {
+	harness: HARNESS,
+	name: "aced.claude-code",
+	runner: "print",
+	executable: (() => {
+		const facts = headlessInvocation(HARNESS);
+		return facts.supported === true ? facts.executable : "claude";
+	})(),
+	launch(o) {
+		const r = spawnSync(this.executable, launchArgs(o), {
+			cwd: o.checkout,
+			encoding: "utf8",
+			timeout: o.timeoutMs,
+			killSignal: "SIGKILL",
+			maxBuffer: 536870912,
+			stdio: [
+				"ignore",
+				"pipe",
+				"pipe"
+			],
+			env: {
+				...process.env,
+				...o.env
+			}
+		});
+		const code = r.error?.code;
+		if (r.error && code !== "ETIMEDOUT" && code !== "ENOBUFS") throw new Error(`could not launch ${this.executable}: ${r.error.message}`);
+		const transcript = r.stdout ?? "";
+		const parsed = parseStreamJson(transcript);
+		if (r.error || r.signal) parsed.capped = true;
+		return {
+			...parsed,
+			transcript
+		};
+	},
+	isolatedConfig(runTemp) {
+		const dir = join(runTemp, "claude-config");
+		mkdirSync(dir, { recursive: true });
+		const operator = process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude");
+		const credentials = join(operator, ".credentials.json");
+		if (existsSync(credentials)) copyFileSync(credentials, join(dir, ".credentials.json"));
+		return { CLAUDE_CONFIG_DIR: dir };
+	}
+};
+const ADAPTERS = { [HARNESS]: claudeCode };
+function adapterFor(harness) {
+	return Object.hasOwn(ADAPTERS, harness) ? ADAPTERS[harness] : void 0;
+}
+const ADAPTER_HARNESSES = Object.keys(ADAPTERS);
+//#endregion
+//#region skills/bench/scripts/bench-record.mts
+/** `^[a-z0-9]+(?:[-.][a-z0-9]+)*$` — lowercase segments joined by single `-` or `.`. */
+const SUITE_NAME_RULE = /^[a-z0-9]+(?:[-.][a-z0-9]+)*$/;
+const BENCH_ROOT = join(".agents", "aced", "bench");
+const RESULTS_ROOT = join(".agents", "aced", "results", "bench");
+function suiteDir(suite) {
+	return join(BENCH_ROOT, suite);
+}
+function resultsDir(suite) {
+	return join(RESULTS_ROOT, suite);
+}
+/** An ISO timestamp safe in a file name: `:` and `.` become `-`. */
+function stamp(createdAt) {
+	return createdAt.replace(/[:.]/g, "-");
+}
+/** The compared metrics. `pass` is 0/1 and has no pooled row. */
+const METRICS = [
+	"pass",
+	"inputTokens",
+	"outputTokens",
+	"cacheReadTokens",
+	"turns",
+	"toolCalls",
+	"wallMs",
+	"costUsd"
+];
+/** Metrics that can make a verdict `regressed`. Cost and cache reads are compared but never gate. */
+const GATED = /* @__PURE__ */ new Set([
+	"pass",
+	"turns",
+	"toolCalls",
+	"inputTokens",
+	"outputTokens",
+	"wallMs"
+]);
+function median(values) {
+	if (values.length === 0) return 0;
+	const sorted = [...values].sort((a, b) => a - b);
+	const mid = Math.floor(sorted.length / 2);
+	return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+function round(n, places = 4) {
+	const f = 10 ** places;
+	return Math.round(n * f) / f;
+}
+//#endregion
+//#region skills/bench/scripts/bench-compare.mts
+const ALPHA = .05;
+const EXACT_LIMIT = 2e5;
+const SAMPLES = 2e4;
+/** mulberry32: a small seeded generator, so a sampled p-value repeats exactly. */
+function rng(seed) {
+	let s = seed >>> 0;
+	return () => {
+		s = s + 1831565813 >>> 0;
+		let t = s;
+		t = Math.imul(t ^ t >>> 15, t | 1);
+		t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+		return ((t ^ t >>> 14) >>> 0) / 4294967296;
+	};
+}
+/** FNV-1a, so each row draws from its own seed and one row's sampling never shifts another's. */
+function seedOf(key) {
+	let h = 2166136261;
+	for (let i = 0; i < key.length; i++) {
+		h ^= key.charCodeAt(i);
+		h = Math.imul(h, 16777619);
+	}
+	return h >>> 0;
+}
+function choose(n, k) {
+	let c = 1;
+	for (let i = 1; i <= k; i++) c = c * (n - k + i) / i;
+	return Math.round(c);
+}
+/**
+* The smallest p any data could reach with these group sizes, one `[before, after]` pair per task.
+* One pair is a per-task row; several are a pooled row. Exact when every task enumerates and the
+* product of their relabellings stays within the limit: only the real split and, with equal
+* groups, its mirror can be the most extreme. Sampled otherwise: one hit out of N + 1.
+*/
+function minAttainableP(sizes) {
+	let product = 1;
+	let exact = true;
+	for (const [na, nb] of sizes) {
+		const c = choose(na + nb, na);
+		if (c > EXACT_LIMIT) exact = false;
+		product *= c;
+	}
+	if (!exact || product > EXACT_LIMIT) return 1 / 20001;
+	return (sizes.every(([na, nb]) => na === nb) ? 2 : 1) / product;
+}
+/**
+* Every split of the pooled runs into a "before" group of `a.length` and an "after" group of the
+* rest, as `stat(beforeMean, afterMean)`. Enumerated when few enough, sampled otherwise.
+*/
+function relabellings(a, b, stat, random) {
+	const all = [...a, ...b];
+	const na = a.length;
+	const nb = b.length;
+	const total = all.reduce((s, x) => s + x, 0);
+	const stats = [];
+	const split = (before) => stat(before / na, (total - before) / nb);
+	if (choose(all.length, na) <= EXACT_LIMIT) {
+		const walk = (from, left, acc) => {
+			if (left === 0) {
+				stats.push(split(acc));
+				return;
+			}
+			for (let i = from; i <= all.length - left; i++) walk(i + 1, left - 1, acc + all[i]);
+		};
+		walk(0, na, 0);
+		return {
+			stats,
+			exact: true
+		};
+	}
+	const idx = all.map((_, i) => i);
+	for (let n = 0; n < SAMPLES; n++) {
+		let acc = 0;
+		for (let i = 0; i < na; i++) {
+			const j = i + Math.floor(random() * (idx.length - i));
+			const tmp = idx[i];
+			idx[i] = idx[j];
+			idx[j] = tmp;
+			acc += all[idx[i]];
+		}
+		stats.push(split(acc));
+	}
+	return {
+		stats,
+		exact: false
+	};
+}
+/** Two-sided: the share of relabellings whose |statistic| reaches the observed one. */
+function pValue(observed, { stats, exact }, minP) {
+	const tol = 1e-12 * Math.max(1, Math.abs(observed));
+	let hits = 0;
+	for (const x of stats) if (Math.abs(x) >= Math.abs(observed) - tol) hits++;
+	return {
+		p: exact ? hits / stats.length : (hits + 1) / (stats.length + 1),
+		exact,
+		minP,
+		tooFew: minP > ALPHA
+	};
+}
+const difference = (before, after) => after - before;
+const logRatio = (before, after) => Math.log(after / before);
+const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
+const value = (r, m) => m === "pass" ? r.pass ? 1 : 0 : r[m];
+function sideStats(xs) {
+	if (xs.length === 0) return null;
+	return {
+		n: xs.length,
+		mean: mean(xs),
+		median: median(xs),
+		min: Math.min(...xs),
+		max: Math.max(...xs)
+	};
+}
+/** Worse in the metric's direction: lower pass is worse; higher is worse for every other metric. */
+function worse(metric, before, after) {
+	return metric === "pass" ? after < before : after > before;
+}
+function taskRow(task, metric, a, b, tags) {
+	const before = sideStats(a);
+	const after = sideStats(b);
+	const row = {
+		scope: "task",
+		task,
+		metric,
+		before,
+		after,
+		change: before && after && before.mean !== 0 ? after.mean / before.mean - 1 : null,
+		wrongWay: false,
+		significant: false,
+		tags
+	};
+	if (!before || !after) return row;
+	const test = pValue(difference(before.mean, after.mean), relabellings(a, b, difference, rng(seedOf(`${task}\0${metric}`))), minAttainableP([[a.length, b.length]]));
+	Object.assign(row, test);
+	row.wrongWay = worse(metric, before.mean, after.mean);
+	row.significant = test.p < ALPHA;
+	return row;
+}
+function pooledRow(metric, pairs) {
+	const usable = pairs.filter(([a, b]) => a.length > 0 && b.length > 0 && [...a, ...b].every((x) => x > 0));
+	if (usable.length === 0) return void 0;
+	const random = rng(seedOf(`pooled\0${metric}`));
+	const observed = mean(usable.map(([a, b]) => logRatio(mean(a), mean(b))));
+	const perTask = usable.map(([a, b]) => relabellings(a, b, logRatio, random));
+	const size = perTask.reduce((n, t) => n * t.stats.length, 1);
+	const exact = perTask.every((t) => t.exact) && size <= EXACT_LIMIT;
+	const stats = [];
+	if (exact) {
+		const walk = (i, acc) => {
+			if (i === perTask.length) {
+				stats.push(acc / perTask.length);
+				return;
+			}
+			for (const s of perTask[i].stats) walk(i + 1, acc + s);
+		};
+		walk(0, 0);
+	} else for (let n = 0; n < SAMPLES; n++) {
+		let acc = 0;
+		for (const t of perTask) acc += t.stats[Math.floor(random() * t.stats.length)];
+		stats.push(acc / perTask.length);
+	}
+	const test = pValue(observed, {
+		stats,
+		exact
+	}, minAttainableP(usable.map(([a, b]) => [a.length, b.length])));
+	const ratio = Math.exp(observed);
+	return {
+		scope: "pooled",
+		metric,
+		ratio,
+		change: ratio - 1,
+		tasks: usable.length,
+		...test,
+		wrongWay: observed > 1e-12,
+		significant: test.p < ALPHA
+	};
+}
+const MATCHED_FIELDS = [
+	["layer", (r) => String(r.layer)],
+	["model", (r) => String(r.model)],
+	["harness", (r) => String(r.harness)],
+	["adapter", (r) => String(r.adapter)],
+	["runner", (r) => String(r.runner)],
+	["subject kind", (r) => String(r.subject?.kind)],
+	["task-set hash", (r) => String(r.taskSetHash)]
+];
+function incomparableReasons(a, b) {
+	const reasons = [];
+	for (const [field, get] of MATCHED_FIELDS) {
+		const x = get(a);
+		const y = get(b);
+		if (x !== y) reasons.push({
+			field,
+			before: x,
+			after: y,
+			message: `${field} differs: before "${x}", after "${y}"`
+		});
+	}
+	if (a.model === "unknown" && b.model === "unknown" || a.scoring_model === "unknown" && b.scoring_model === "unknown") reasons.push({
+		field: "model",
+		message: "the model is unknown on both sides"
+	});
+	return reasons;
+}
+function verdictOf(rows) {
+	const g = rows.filter((r) => GATED.has(r.metric));
+	if (g.some((r) => r.wrongWay && r.significant)) return "regressed";
+	if (g.some((r) => r.wrongWay)) return "inconclusive";
+	if (g.some((r) => r.significant)) return "improved";
+	return "unchanged";
+}
+/** Compares `after` against `before`. Never refuses: a mismatch is an `incomparable` verdict. */
+function compareRecords(before, after) {
+	const incomparable = incomparableReasons(before, after);
+	const beforeTasks = [...new Set(before.runs.map((r) => r.task))];
+	const afterTasks = [...new Set(after.runs.map((r) => r.task))];
+	const unmatched = [...beforeTasks.filter((t) => !afterTasks.includes(t)), ...afterTasks.filter((t) => !beforeTasks.includes(t))];
+	const empty = {
+		verdict: "incomparable",
+		incomparable,
+		unmatched,
+		errorsExcluded: {
+			before: 0,
+			after: 0
+		},
+		rows: [],
+		footer: {
+			testCount: 0,
+			expectedByChance: 0,
+			alpha: ALPHA
+		}
+	};
+	if (incomparable.length > 0) return empty;
+	const tasks = beforeTasks.filter((t) => afterTasks.includes(t));
+	const errorsExcluded = {
+		before: 0,
+		after: 0
+	};
+	const kept = (record, task, side) => {
+		const rs = record.runs.filter((r) => r.task === task);
+		const ok = rs.filter((r) => !r.error);
+		errorsExcluded[side] += rs.length - ok.length;
+		return ok;
+	};
+	const rows = [];
+	const pairs = /* @__PURE__ */ new Map();
+	for (const task of tasks) {
+		const a = kept(before, task, "before");
+		const b = kept(after, task, "after");
+		const tags = after.tags?.[task] ?? before.tags?.[task] ?? {};
+		for (const metric of METRICS) {
+			const va = a.map((r) => value(r, metric));
+			const vb = b.map((r) => value(r, metric));
+			rows.push(taskRow(task, metric, va, vb, { ...tags }));
+			if (metric !== "pass") pairs.set(metric, [...pairs.get(metric) ?? [], [va, vb]]);
+		}
+	}
+	for (const metric of METRICS) {
+		if (metric === "pass") continue;
+		const row = pooledRow(metric, pairs.get(metric) ?? []);
+		if (row) rows.push(row);
+	}
+	const testCount = rows.filter((r) => r.p !== void 0).length;
+	return {
+		verdict: verdictOf(rows),
+		incomparable,
+		unmatched,
+		errorsExcluded,
+		rows,
+		footer: {
+			testCount,
+			expectedByChance: round(testCount * ALPHA),
+			alpha: ALPHA
+		}
+	};
+}
+//#endregion
+//#region skills/bench/scripts/bench.mts
+var BenchError = class extends Error {};
+const DEFAULTS = {
+	model: "sonnet",
+	runs: 3,
+	maxBudgetUsd: .5,
+	permissionMode: "bypassPermissions",
+	timeoutMinutes: 20
+};
+const TEMPLATE = {
+	model: DEFAULTS.model,
+	runs: DEFAULTS.runs,
+	maxBudgetUsd: DEFAULTS.maxBudgetUsd,
+	permissionMode: DEFAULTS.permissionMode,
+	timeoutMinutes: DEFAULTS.timeoutMinutes,
+	tasks: [{
+		id: "example-task",
+		prompt: "Replace this with one small, self-contained task, then write a one-line summary of what you did to SUMMARY.md.",
+		check: "test -s SUMMARY.md",
+		tags: { area: "example" }
+	}]
+};
+function positive(raw, key, integer = false) {
+	const v = raw[key];
+	if (v === void 0) return DEFAULTS[key];
+	if (typeof v !== "number" || !(v > 0) || integer && !Number.isInteger(v)) throw new BenchError(`tasks.json: "${key}" must be a positive ${integer ? "whole " : ""}number`);
+	return v;
+}
+function parseTaskSet(text) {
+	let raw;
+	try {
+		raw = JSON.parse(text);
+	} catch (e) {
+		throw new BenchError(`tasks.json is not valid JSON: ${e.message}`);
+	}
+	if (raw === null || typeof raw !== "object" || Array.isArray(raw)) throw new BenchError("tasks.json must hold a JSON object");
+	if (!Array.isArray(raw.tasks)) throw new BenchError("tasks.json: \"tasks\" must be a list of tasks");
+	if (raw.tasks.length === 0) throw new BenchError("tasks.json: the tasks list is empty; list at least one task");
+	const ids = /* @__PURE__ */ new Set();
+	const tasks = raw.tasks.map((t, i) => {
+		if (t === null || typeof t !== "object") throw new BenchError(`tasks.json: tasks[${i}] must be an object`);
+		const label = typeof t.id === "string" && t.id ? `tasks[${i}] ("${t.id}")` : `tasks[${i}]`;
+		for (const key of [
+			"id",
+			"prompt",
+			"check"
+		]) if (typeof t[key] !== "string" || t[key] === "") throw new BenchError(`tasks.json: ${label} has no "${key}"`);
+		if (t.setup !== void 0 && typeof t.setup !== "string") throw new BenchError(`tasks.json: ${label} "setup" must be a string`);
+		if (t.tags !== void 0 && (t.tags === null || typeof t.tags !== "object" || Array.isArray(t.tags) || Object.values(t.tags).some((v) => typeof v !== "string"))) throw new BenchError(`tasks.json: ${label} "tags" must map keys to string values`);
+		const id = t.id;
+		if (ids.has(id)) throw new BenchError(`tasks.json: the task id "${id}" appears twice`);
+		ids.add(id);
+		return {
+			id,
+			prompt: t.prompt,
+			check: t.check,
+			...t.setup ? { setup: t.setup } : {},
+			...t.tags ? { tags: t.tags } : {}
+		};
+	});
+	if (raw.setup !== void 0 && typeof raw.setup !== "string") throw new BenchError("tasks.json: \"setup\" must be a string");
+	for (const key of ["model", "permissionMode"]) if (raw[key] !== void 0 && (typeof raw[key] !== "string" || raw[key] === "")) throw new BenchError(`tasks.json: "${key}" must be a non-empty string`);
+	return {
+		model: raw.model ?? DEFAULTS.model,
+		runs: positive(raw, "runs", true),
+		maxBudgetUsd: positive(raw, "maxBudgetUsd"),
+		permissionMode: raw.permissionMode ?? DEFAULTS.permissionMode,
+		timeoutMinutes: positive(raw, "timeoutMinutes"),
+		...raw.setup ? { setup: raw.setup } : {},
+		tasks
+	};
+}
+function git(cwd, args) {
+	return spawnSync("git", args, {
+		cwd,
+		encoding: "utf8",
+		maxBuffer: 268435456
+	});
+}
+function gitOk(cwd, args, what) {
+	const r = git(cwd, args);
+	if (r.status !== 0) throw new BenchError(`${what}: ${(r.stderr || r.error?.message || "").trim()}`);
+	return r.stdout;
+}
+function repoRoot(cwd) {
+	const r = git(cwd, ["rev-parse", "--show-toplevel"]);
+	if (r.status !== 0) throw new BenchError("not inside a git repository");
+	return r.stdout.trim();
+}
+function resolveCommit(root, ref) {
+	const r = git(root, [
+		"rev-parse",
+		"--verify",
+		"--quiet",
+		`${ref}^{commit}`
+	]);
+	return r.status === 0 ? r.stdout.trim() : void 0;
+}
+function showAt(root, commit, path) {
+	const r = spawnSync("git", ["show", `${commit}:${path}`], {
+		cwd: root,
+		maxBuffer: 268435456
+	});
+	return r.status === 0 ? r.stdout : void 0;
+}
+/** The task set's committed files at `commit`: tasks.json and everything under checks/, path-sorted. */
+function taskSetFiles(root, suite, commit) {
+	const dir = suiteDir(suite);
+	const listed = git(root, [
+		"ls-tree",
+		"-r",
+		"--name-only",
+		commit,
+		"--",
+		`${dir}/tasks.json`,
+		`${dir}/checks`
+	]);
+	if (listed.status !== 0) return [];
+	return listed.stdout.split("\n").filter(Boolean).sort().map((path) => ({
+		path,
+		content: showAt(root, commit, path) ?? Buffer.alloc(0)
+	}));
+}
+function taskSetHash(files) {
+	const h = createHash("sha256");
+	for (const f of files) h.update(f.path).update("\0").update(f.content).update("\0");
+	return h.digest("hex");
+}
+const sha256 = (content) => createHash("sha256").update(content).digest("hex");
+const ARM_LABEL = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
+function parseArm(text) {
+	const eq = text.indexOf("=");
+	const label = text.slice(0, eq);
+	const subject = text.slice(eq + 1);
+	if (eq <= 0 || !ARM_LABEL.test(label)) throw new UsageError(`--arm "${text}" needs <label>=<subject>`);
+	if (subject.startsWith("git:") && subject.length > 4) return {
+		label,
+		kind: "git-ref",
+		ref: subject.slice(4)
+	};
+	if (subject.startsWith("package:")) {
+		const spec = subject.slice(8);
+		const at = spec.lastIndexOf("@");
+		if (at > 0 && at < spec.length - 1) return {
+			label,
+			kind: "package",
+			name: spec.slice(0, at),
+			version: spec.slice(at + 1)
+		};
+	}
+	const file = /^file:(.+?)=(ref:.+|path:.+|absent)$/.exec(subject);
+	if (file) return {
+		label,
+		kind: "file",
+		path: file[1],
+		from: file[2]
+	};
+	throw new UsageError(`--arm "${text}": the subject must be git:<ref>, package:<name>@<version>, or file:<path>=ref:<ref>|path:<source>|absent`);
+}
+function insideRepo(path) {
+	const n = normalize(path);
+	return !isAbsolute(n) && n !== ".." && !n.startsWith(`../`) && n !== ".";
+}
+function resolveArm(root, spec) {
+	if (spec.kind === "git-ref") {
+		const commit = resolveCommit(root, spec.ref);
+		if (!commit) throw new BenchError(`arm "${spec.label}": the git ref "${spec.ref}" names no commit in this repository`);
+		return {
+			label: spec.label,
+			subject: {
+				kind: "git-ref",
+				ref: spec.ref,
+				commit
+			}
+		};
+	}
+	if (spec.kind === "package") {
+		const r = spawnSync("npm", [
+			"view",
+			`${spec.name}@${spec.version}`,
+			"version",
+			"--json"
+		], {
+			cwd: root,
+			encoding: "utf8"
+		});
+		if (!(r.status === 0 && r.stdout.trim() !== "" && r.stdout.trim() !== "[]")) throw new BenchError(`arm "${spec.label}": the registry has no package "${spec.name}" at version "${spec.version}"`);
+		return {
+			label: spec.label,
+			subject: {
+				kind: "package",
+				name: spec.name,
+				version: spec.version
+			}
+		};
+	}
+	if (!insideRepo(spec.path)) throw new BenchError(`arm "${spec.label}": the file path "${spec.path}" is outside the repository`);
+	const subject = {
+		kind: "file",
+		path: spec.path,
+		from: spec.from
+	};
+	if (spec.from.startsWith("ref:")) {
+		const ref = spec.from.slice(4);
+		const commit = resolveCommit(root, ref);
+		if (!commit) throw new BenchError(`arm "${spec.label}": the git ref "${ref}" names no commit in this repository`);
+		if (git(root, [
+			"cat-file",
+			"-e",
+			`${commit}:${spec.path}`
+		]).status !== 0) throw new BenchError(`arm "${spec.label}": ${spec.path} does not exist at "${ref}"`);
+		subject.commit = commit;
+	} else if (spec.from.startsWith("path:")) {
+		const source = spec.from.slice(5);
+		if (!insideRepo(source) || !existsSync(join(root, source))) throw new BenchError(`arm "${spec.label}": the source file ${source} does not exist`);
+	}
+	return {
+		label: spec.label,
+		subject
+	};
+}
+function onPath(executable) {
+	for (const dir of (process.env.PATH ?? "").split(delimiter)) {
+		if (!dir) continue;
+		try {
+			accessSync(join(dir, executable), constants.X_OK);
+			return true;
+		} catch {}
+	}
+	return false;
+}
+function checkSuiteName(suite) {
+	if (suite === void 0) throw new UsageError("--suite is required");
+	if (!SUITE_NAME_RULE.test(suite)) throw new BenchError(`suite name "${suite}" is outside the suite naming rule ${SUITE_NAME_RULE.source} (lowercase letters and digits, segments joined by a single "-" or ".")`);
+	return suite;
+}
+function loadCommittedTaskSet(root, suite, commit) {
+	const path = `${suiteDir(suite)}/tasks.json`;
+	const text = showAt(root, commit, path);
+	if (text === void 0) {
+		if (existsSync(join(root, path))) throw new BenchError(`${path} is not committed; the committed task set is what runs, so commit it first`);
+		throw new BenchError(`suite "${suite}" has no task set: ${path} does not exist (create one with \`aced-bench init --suite ${suite}\`)`);
+	}
+	return parseTaskSet(text.toString("utf8"));
+}
+/** Per task: the cost of each stored, error-free run on the same model, harness, and runner. */
+function storedCosts(root, suite, model, harness, runner) {
+	const dir = join(root, resultsDir(suite));
+	const costs = /* @__PURE__ */ new Map();
+	if (!existsSync(dir)) return costs;
+	for (const file of readdirSync(dir)) {
+		if (!file.endsWith(".json") || file.startsWith("compare-")) continue;
+		let record;
+		try {
+			record = JSON.parse(readFileSync(join(dir, file), "utf8"));
+		} catch {
+			continue;
+		}
+		if (record?.schemaVersion !== 3 || record.layer !== "measured" || record.model !== model || record.harness !== harness || record.runner !== runner || !Array.isArray(record.runs)) continue;
+		for (const r of record.runs) {
+			if (r.error || typeof r.costUsd !== "number") continue;
+			costs.set(r.task, [...costs.get(r.task) ?? [], r.costUsd]);
+		}
+	}
+	return costs;
+}
+function porcelain(root, pathspecs) {
+	return gitOk(root, [
+		"status",
+		"--porcelain",
+		"--untracked-files=all",
+		"--",
+		...pathspecs
+	], "git status failed").trim();
+}
+function plan(root, args) {
+	const suite = checkSuiteName(args.suite);
+	const specs = (args.arm ?? []).map(parseArm);
+	if (specs.length === 0) throw new UsageError("plan needs at least one --arm <label>=<subject>");
+	if (new Set(specs.map((s) => s.label)).size !== specs.length) throw new UsageError("each --arm needs its own label");
+	if (!args.out) throw new UsageError("plan needs --out <file>");
+	let runsOverride;
+	if (args.runs !== void 0) {
+		runsOverride = Number(args.runs);
+		if (!Number.isInteger(runsOverride) || runsOverride < 1) throw new UsageError("--runs must be a positive whole number");
+	}
+	const harness = args.harness ?? "claude-code";
+	const taskSetCommit = resolveCommit(root, "HEAD");
+	if (!taskSetCommit) throw new BenchError("the repository has no commit; the task set must be committed");
+	const taskSet = loadCommittedTaskSet(root, suite, taskSetCommit);
+	let tasks = taskSet.tasks;
+	if (args.task !== void 0) {
+		tasks = tasks.filter((t) => t.id === args.task);
+		if (tasks.length === 0) throw new BenchError(`no task "${args.task}" in suite "${suite}" (tasks: ${taskSet.tasks.map((t) => t.id).join(", ")})`);
+	}
+	const adapter = adapterFor(harness);
+	if (!adapter) throw new BenchError(`the harness "${harness}" has no adapter (harnesses with one: ${ADAPTER_HARNESSES.join(", ")})`);
+	if (!onPath(adapter.executable)) throw new BenchError(`the ${harness} command "${adapter.executable}" is not on the PATH, so no run could start; install it or fix PATH before planning`);
+	if (args.baseline && specs.length > 1) throw new BenchError(`a suite records one baseline: --baseline takes exactly one --arm, not ${specs.length}`);
+	const arms = specs.map((s) => resolveArm(root, s));
+	const warnings = [];
+	const dir = suiteDir(suite);
+	if (porcelain(root, [`${dir}/tasks.json`, `${dir}/checks`]) !== "") warnings.push({
+		code: "suite-uncommitted",
+		message: `the suite has uncommitted changes; the committed suite (${taskSetCommit.slice(0, 12)}) is what runs`
+	});
+	if (arms.some((a) => a.subject.kind === "git-ref") && porcelain(root, [
+		".",
+		`:(exclude)${dir}/tasks.json`,
+		`:(exclude)${dir}/checks`,
+		":(exclude).agents/aced/results"
+	]) !== "") warnings.push({
+		code: "uncommitted-not-benched",
+		message: "the working tree has uncommitted changes; they are not benched, since a git-ref arm runs its committed tree"
+	});
+	const runsPerArm = runsOverride ?? taskSet.runs;
+	const ceilingUsd = round(arms.length * tasks.length * runsPerArm * taskSet.maxBudgetUsd);
+	const stored = storedCosts(root, suite, taskSet.model, harness, adapter.runner);
+	const estimateTasks = {};
+	for (const t of tasks) {
+		const costs = stored.get(t.id) ?? [];
+		const perRunUsd = costs.length > 0 ? median(costs) : taskSet.maxBudgetUsd;
+		estimateTasks[t.id] = {
+			usd: round(arms.length * runsPerArm * perRunUsd),
+			perRunUsd: round(perRunUsd),
+			source: costs.length > 0 ? "stored" : "cap",
+			storedRuns: costs.length
+		};
+	}
+	const totalUsd = round(Object.values(estimateTasks).reduce((s, e) => s + e.usd, 0));
+	const perTaskMinP = minAttainableP([[runsPerArm, runsPerArm]]);
+	if (perTaskMinP > .05) {
+		const pooledMinP = minAttainableP(tasks.map(() => [runsPerArm, runsPerArm]));
+		const pooled = tasks.length >= 2 && pooledMinP <= .05 ? `a pooled result across the ${tasks.length} tasks still can (smallest attainable p ${round(pooledMinP, 6)})` : `a pooled result across ${tasks.length === 1 ? "one task" : `the ${tasks.length} tasks`} cannot either`;
+		warnings.push({
+			code: "too-few-to-call",
+			message: `too few to call: at ${runsPerArm} runs per arm no single task's result can be significant (smallest attainable p ${round(perTaskMinP, 6)} > 0.05); ${pooled}`
+		});
+	}
+	return {
+		planVersion: 1,
+		suite,
+		harness,
+		adapter: adapter.name,
+		runner: adapter.runner,
+		model: taskSet.model,
+		permissionMode: taskSet.permissionMode,
+		permissionScope: `the agent runs with --permission-mode ${taskSet.permissionMode}; it applies only inside the throwaway checkout each run gets, never to this working tree`,
+		maxBudgetUsd: taskSet.maxBudgetUsd,
+		timeoutMinutes: taskSet.timeoutMinutes,
+		runsPerArm,
+		baseline: Boolean(args.baseline),
+		arms,
+		tasks: tasks.map((t) => t.id),
+		counts: {
+			arms: arms.length,
+			tasks: tasks.length,
+			runsPerArm,
+			totalRuns: arms.length * tasks.length * runsPerArm
+		},
+		ceilingUsd,
+		estimate: {
+			totalUsd,
+			tasks: estimateTasks
+		},
+		warnings,
+		taskSetCommit,
+		taskSetHash: taskSetHash(taskSetFiles(root, suite, taskSetCommit))
+	};
+}
+const SHELL_TIMEOUT_MS = 9e5;
+function shell(command, cwd) {
+	return spawnSync("sh", ["-c", command], {
+		cwd,
+		stdio: "ignore",
+		timeout: SHELL_TIMEOUT_MS
+	}).status;
+}
+function zeroMetrics() {
+	return {
+		wallMs: 0,
+		inputTokens: 0,
+		outputTokens: 0,
+		cacheReadTokens: 0,
+		cacheCreationTokens: 0,
+		turns: 0,
+		toolCalls: 0,
+		costUsd: 0,
+		capped: false
+	};
+}
+/** Applies the arm's subject to the checkout. Returns what the launch needs for a package arm. */
+function applySubject(ctx, arm, checkout, runTemp) {
+	const s = arm.subject;
+	if (s.kind === "file") {
+		const target = join(checkout, s.path);
+		if (s.from === "absent") rmSync(target, {
+			force: true,
+			recursive: true
+		});
+		else {
+			const content = s.from.startsWith("ref:") ? showAt(ctx.root, s.commit ?? s.from.slice(4), s.path) : readFileSync(join(ctx.root, s.from.slice(5)));
+			if (content === void 0) throw new BenchError(`could not read ${s.path} at ${s.from}`);
+			mkdirSync(dirname(target), { recursive: true });
+			writeFileSync(target, content);
+		}
+		return {};
+	}
+	if (s.kind === "package") {
+		const packDir = join(runTemp, "package-pack");
+		mkdirSync(packDir, { recursive: true });
+		const pack = spawnSync("npm", [
+			"pack",
+			`${s.name}@${s.version}`,
+			"--pack-destination",
+			packDir
+		], {
+			cwd: runTemp,
+			encoding: "utf8"
+		});
+		const tarball = readdirSync(packDir).find((f) => f.endsWith(".tgz"));
+		if (pack.status !== 0 || !tarball) throw new BenchError(`npm pack ${s.name}@${s.version} failed: ${(pack.stderr || "").trim()}`);
+		const unpacked = join(runTemp, "package");
+		mkdirSync(unpacked, { recursive: true });
+		const untar = spawnSync("tar", [
+			"-xzf",
+			join(packDir, tarball),
+			"-C",
+			unpacked
+		], { encoding: "utf8" });
+		if (untar.status !== 0) throw new BenchError(`could not unpack ${tarball}: ${untar.stderr.trim()}`);
+		return {
+			pluginDir: join(unpacked, "package"),
+			env: ctx.adapter.isolatedConfig(runTemp)
+		};
+	}
+	return {};
+}
+/**
+* Overlays the task-set commit's suite, stages the subject's change with it, and commits both with
+* a throwaway identity, so the agent starts on a clean tree.
+*/
+function commitCheckout(ctx, checkout) {
+	const dir = suiteDir(ctx.plan.suite);
+	gitOk(checkout, [
+		"rm",
+		"-r",
+		"-q",
+		"--ignore-unmatch",
+		"--",
+		dir
+	], "could not clear the suite in the checkout");
+	gitOk(checkout, [
+		"checkout",
+		ctx.plan.taskSetCommit,
+		"--",
+		dir
+	], "could not overlay the task-set commit");
+	gitOk(checkout, ["add", "-A"], "could not stage the checkout");
+	if (git(checkout, [
+		"diff",
+		"--cached",
+		"--quiet"
+	]).status === 0) return;
+	gitOk(checkout, [
+		"-c",
+		"user.name=aced-bench",
+		"-c",
+		"user.email=aced-bench@localhost",
+		"-c",
+		"commit.gpgsign=false",
+		"-c",
+		"core.hooksPath=/dev/null",
+		"commit",
+		"--quiet",
+		"--no-verify",
+		"-m",
+		`bench: task set from ${ctx.plan.taskSetCommit}`
+	], "could not commit the overlaid task set");
+}
+function runOne(ctx, arm, task, run) {
+	const runTemp = realpathSync(mkdtempSync(join(tmpdir(), "aced-bench-")));
+	const checkout = join(runTemp, "repo");
+	const base = {
+		task: task.id,
+		run
+	};
+	try {
+		const at = arm.subject.kind === "git-ref" ? arm.subject.commit : "HEAD";
+		gitOk(ctx.root, [
+			"worktree",
+			"add",
+			"--detach",
+			"--quiet",
+			checkout,
+			at
+		], "git worktree add failed");
+		const launch = applySubject(ctx, arm, checkout, runTemp);
+		commitCheckout(ctx, checkout);
+		for (const [name, command] of [["suite setup", ctx.taskSet.setup], [`${task.id} setup`, task.setup]]) if (command && shell(command, checkout) !== 0) return {
+			...base,
+			...zeroMetrics(),
+			pass: false,
+			error: `${name} failed: ${command}`,
+			transcript: null
+		};
+		const start = performance.now();
+		const { transcript, model, ...metrics } = ctx.adapter.launch({
+			prompt: task.prompt,
+			model: ctx.plan.model,
+			maxBudgetUsd: ctx.plan.maxBudgetUsd,
+			permissionMode: ctx.plan.permissionMode,
+			checkout,
+			timeoutMs: ctx.plan.timeoutMinutes * 60 * 1e3,
+			...launch
+		});
+		const wallMs = Math.round(performance.now() - start);
+		const pass = shell(task.check, checkout) === 0;
+		let kept = null;
+		if (transcript) {
+			kept = `${ctx.transcriptDir}/${task.id}-${run}.jsonl.gz`;
+			mkdirSync(dirname(join(ctx.root, kept)), { recursive: true });
+			writeFileSync(join(ctx.root, kept), gzipSync(transcript));
+		}
+		return {
+			...base,
+			...metrics,
+			wallMs,
+			pass,
+			error: null,
+			transcript: kept,
+			...model ? { model } : {}
+		};
+	} finally {
+		git(ctx.root, [
+			"worktree",
+			"remove",
+			"--force",
+			checkout
+		]);
+		rmSync(runTemp, {
+			recursive: true,
+			force: true
+		});
+		git(ctx.root, ["worktree", "prune"]);
+	}
+}
+function summarize(runs) {
+	const passes = runs.filter((r) => r.pass).length;
+	const ok = runs.filter((r) => !r.error);
+	const m = (f) => median(ok.map(f));
+	const totalCostUsd = round(runs.reduce((s, r) => s + r.costUsd, 0));
+	return {
+		runs: runs.length,
+		passes,
+		passRate: runs.length === 0 ? 0 : round(passes / runs.length),
+		capped: runs.filter((r) => r.capped).length,
+		errors: runs.length - ok.length,
+		medianInputTokens: m((r) => r.inputTokens),
+		medianOutputTokens: m((r) => r.outputTokens),
+		medianCacheReadTokens: m((r) => r.cacheReadTokens),
+		medianTurns: m((r) => r.turns),
+		medianToolCalls: m((r) => r.toolCalls),
+		medianWallMs: m((r) => r.wallMs),
+		totalCostUsd,
+		...passes > 0 ? { costPerSuccessUsd: round(totalCostUsd / passes) } : {}
+	};
+}
+/** A createdAt whose record file and transcript folder are both still free. */
+function freshCreatedAt(dir, suffix, last) {
+	let t = Math.max(Date.now(), last + 1);
+	for (;;) {
+		const createdAt = new Date(t).toISOString();
+		if (!existsSync(join(dir, `${stamp(createdAt)}${suffix}`)) && !existsSync(join(dir, stamp(createdAt)))) return createdAt;
+		t++;
+	}
+}
+function readPlan(path) {
+	let plan;
+	try {
+		plan = JSON.parse(readFileSync(path, "utf8"));
+	} catch (e) {
+		throw new BenchError(`cannot read the plan ${path}: ${e.message}`);
+	}
+	if (plan?.planVersion !== 1 || !Array.isArray(plan.arms) || !Array.isArray(plan.tasks)) throw new BenchError(`${path} is not an aced-bench plan`);
+	return plan;
+}
+function runPlan(root, args) {
+	if (!args.plan) throw new UsageError("run needs --plan <file>");
+	if (!args.consent) throw new BenchError("run spends real money and needs explicit consent: re-run with --consent; nothing was launched");
+	const plan = readPlan(resolve(args.plan));
+	checkSuiteName(plan.suite);
+	const adapter = adapterFor(plan.harness);
+	if (!adapter) throw new BenchError(`the harness "${plan.harness}" has no adapter`);
+	const head = resolveCommit(root, "HEAD");
+	if ((head ? taskSetHash(taskSetFiles(root, plan.suite, head)) : "") !== plan.taskSetHash) throw new BenchError(`the suite "${plan.suite}" changed since the plan was written (task-set hash differs); the consent was given to a different plan, so plan again`);
+	const taskSet = loadCommittedTaskSet(root, plan.suite, plan.taskSetCommit);
+	const tasks = plan.tasks.map((id) => {
+		const t = taskSet.tasks.find((x) => x.id === id);
+		if (!t) throw new BenchError(`the plan's task "${id}" is not in the task-set commit`);
+		return t;
+	});
+	const files = taskSetFiles(root, plan.suite, plan.taskSetCommit);
+	const outDir = resultsDir(plan.suite);
+	mkdirSync(join(root, outDir), { recursive: true });
+	const written = [];
+	let baselinePath;
+	let last = 0;
+	for (const arm of plan.arms) {
+		const createdAt = freshCreatedAt(join(root, outDir), `.${arm.label}.json`, last);
+		last = Date.parse(createdAt);
+		const ctx = {
+			root,
+			plan,
+			taskSet,
+			adapter,
+			transcriptDir: `${outDir}/${stamp(createdAt)}`
+		};
+		const runs = [];
+		for (const task of tasks) for (let run = 1; run <= plan.runsPerArm; run++) runs.push(runOne(ctx, arm, task, run));
+		const evaluated = files.map((f) => ({
+			path: f.path,
+			sha256: sha256(f.content),
+			kind: "file"
+		}));
+		if (arm.subject.kind === "file" && arm.subject.from.startsWith("path:")) {
+			const source = arm.subject.from.slice(5);
+			evaluated.push({
+				path: source,
+				sha256: sha256(readFileSync(join(root, source))),
+				kind: "file"
+			});
+		}
+		const reported = runs.find((r) => r.model)?.model;
+		const clean = runs.map(({ model: _, ...r }) => r);
+		const tags = {};
+		for (const t of tasks) if (t.tags) tags[t.id] = t.tags;
+		const record = {
+			schemaVersion: 3,
+			layer: "measured",
+			suite: plan.suite,
+			arm: arm.label,
+			subject: arm.subject,
+			harness: plan.harness,
+			adapter: adapter.name,
+			runner: adapter.runner,
+			taskSetCommit: plan.taskSetCommit,
+			taskSetHash: plan.taskSetHash,
+			tasks: tasks.map((t) => t.id),
+			tags,
+			model: plan.model,
+			scoring_model: reported ?? plan.model,
+			createdAt,
+			evaluated,
+			runs: clean,
+			summary: {
+				overall: summarize(clean),
+				tasks: tasks.map((t) => ({
+					task: t.id,
+					...summarize(clean.filter((r) => r.task === t.id))
+				}))
+			}
+		};
+		const path = `${outDir}/${stamp(createdAt)}.${arm.label}.json`;
+		writeFileSync(join(root, path), `${JSON.stringify(record, null, 2)}\n`);
+		written.push(path);
+		if (plan.baseline) {
+			const baseline = {
+				...record,
+				runs: clean.map(({ transcript: _, ...r }) => r)
+			};
+			baselinePath = `${suiteDir(plan.suite)}/baseline.json`;
+			writeFileSync(join(root, baselinePath), `${JSON.stringify(baseline, null, "	")}\n`);
+		}
+	}
+	return {
+		records: written,
+		...baselinePath ? { baseline: baselinePath } : {}
+	};
+}
+function loadRecord(path) {
+	let record;
+	try {
+		record = JSON.parse(readFileSync(path, "utf8"));
+	} catch (e) {
+		throw new BenchError(`cannot read the record ${path}: ${e.message}`);
+	}
+	if (record?.schemaVersion !== 3) throw new BenchError(`${path} has schemaVersion ${record?.schemaVersion === void 0 ? "absent" : JSON.stringify(record.schemaVersion)}; this engine reads only schema version 3`);
+	if (!Array.isArray(record.runs)) throw new BenchError(`${path} is not a run record: it has no runs`);
+	return record;
+}
+function compare(root, args) {
+	const suite = checkSuiteName(args.suite);
+	if (!args.before || !args.after) throw new UsageError("compare needs --before and --after");
+	const tags = {};
+	for (const t of args.tag ?? []) {
+		const eq = t.indexOf("=");
+		if (eq <= 0) throw new UsageError(`--tag "${t}" needs key=value`);
+		tags[t.slice(0, eq)] = t.slice(eq + 1);
+	}
+	let beforePath = resolve(args.before);
+	if (args.before === "baseline") {
+		beforePath = join(root, suiteDir(suite), "baseline.json");
+		if (!existsSync(beforePath)) throw new BenchError(`suite "${suite}" has no baseline: ${suiteDir(suite)}/baseline.json does not exist`);
+	}
+	const before = loadRecord(beforePath);
+	const after = loadRecord(resolve(args.after));
+	const comparison = compareRecords(before, after);
+	const outDir = join(root, resultsDir(suite));
+	mkdirSync(outDir, { recursive: true });
+	const createdAt = freshCreatedAt(outDir, ".json", 0);
+	const side = (r, source) => ({
+		source,
+		arm: r.arm,
+		createdAt: r.createdAt,
+		subject: r.subject
+	});
+	const record = {
+		schemaVersion: 3,
+		kind: "comparison",
+		suite,
+		createdAt,
+		before: side(before, args.before === "baseline" ? "baseline" : args.before),
+		after: side(after, args.after),
+		tags,
+		...comparison
+	};
+	const path = `${resultsDir(suite)}/compare-${stamp(createdAt)}.json`;
+	writeFileSync(join(root, path), `${JSON.stringify(record, null, 2)}\n`);
+	return {
+		path,
+		record
+	};
+}
+function init(cwd, args) {
+	const suite = checkSuiteName(args.suite);
+	const r = git(cwd, ["rev-parse", "--show-toplevel"]);
+	const root = r.status === 0 ? r.stdout.trim() : cwd;
+	const path = `${suiteDir(suite)}/tasks.json`;
+	if (existsSync(join(root, path))) throw new BenchError(`${path} already exists; init never overwrites a task set`);
+	mkdirSync(join(root, suiteDir(suite)), { recursive: true });
+	writeFileSync(join(root, path), `${JSON.stringify(TEMPLATE, null, "	")}\n`);
+	return path;
+}
+var UsageError = class extends Error {};
+const USAGE = `usage:
+  aced-bench init    --suite <s>
+  aced-bench plan    --suite <s> --arm <label>=<subject> … [--harness claude-code] [--runs N] [--task <id>] [--baseline] --out <file>
+  aced-bench run     --plan <file> --consent
+  aced-bench compare --suite <s> --before <record|baseline> --after <record> [--tag k=v …]`;
+function main(argv) {
+	const cwd = process.cwd();
+	const [verb, ...rest] = argv;
+	if (verb === "help" || verb === "--help" || verb === "-h") {
+		process.stdout.write(`${USAGE}\n`);
+		return 0;
+	}
+	try {
+		const { values } = parseArgs({
+			args: rest,
+			strict: true,
+			options: {
+				suite: { type: "string" },
+				arm: {
+					type: "string",
+					multiple: true
+				},
+				harness: { type: "string" },
+				runs: { type: "string" },
+				task: { type: "string" },
+				baseline: { type: "boolean" },
+				out: { type: "string" },
+				plan: { type: "string" },
+				consent: { type: "boolean" },
+				before: { type: "string" },
+				after: { type: "string" },
+				tag: {
+					type: "string",
+					multiple: true
+				}
+			}
+		});
+		const args = values;
+		let output;
+		if (verb === "init") output = { tasksFile: init(cwd, args) };
+		else if (verb === "plan") {
+			const p = plan(repoRoot(cwd), args);
+			const out = resolve(cwd, args.out);
+			mkdirSync(dirname(out), { recursive: true });
+			writeFileSync(out, `${JSON.stringify(p, null, 2)}\n`);
+			output = p;
+		} else if (verb === "run") output = runPlan(repoRoot(cwd), args);
+		else if (verb === "compare") output = compare(repoRoot(cwd), args).record;
+		else throw new UsageError(verb ? `unknown verb "${verb}"` : "no verb given");
+		process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
+		return 0;
+	} catch (e) {
+		if (e instanceof UsageError || e.code?.startsWith("ERR_PARSE_ARGS")) {
+			process.stderr.write(`aced-bench: ${e.message}\n${USAGE}\n`);
+			return 2;
+		}
+		process.stderr.write(`aced-bench: ${e.message}\n`);
+		return 1;
+	}
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) process.exit(main(process.argv.slice(2)));
+//#endregion
+export {};