amness 0.0.0-stage → 0.1.0

package/dist/cli/benchcli.d.ts

@@ -0,0 +1,5 @@
+export declare const BENCH_HELP: string;
+export declare function runBench(argv: string[], deps: {
+    confirm: (q: string) => Promise<boolean>;
+    amnessCli: string;
+}): Promise<number>;

package/dist/cli/benchcli.js

@@ -0,0 +1,160 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+// `amness bench`: a bake-off between coding-agent CLIs on the same tasks, graded by the same hidden checks.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { PRESETS, adversarialCases, analyze, bench, builtinCases, configFileTemplate, describePlan, formatReport, hardCases, installed, loadCases, loadSuite, suiteToCases } from "../src/index.js";
+import { bold, dim, green, red, yellow } from "./ui.js";
+export const BENCH_HELP = `usage: amness bench [flags]
+  Run the same tasks through several coding-agent command-line tools (Amness, Claude Code, Codex, or any command) and grade every result with the same hidden check.
+
+  --init [file]          write a starter config (default bench.json) with Amness, Amness with a finish gate, Claude Code and Codex, then edit it
+  --config <file>        the contestants and settings (see --init). Without it, --contestants picks built-in presets
+  --contestants <a,b>    presets: ${Object.keys(PRESETS).join(", ")}   (default: amness,claude,codex)
+  --model <name>         one model name for every contestant that takes one; or --models amness=gpt-6-luna,claude=claude-sonnet-5-5,codex=gpt-6-luna
+  --builtin <name>       code (default, 13 easy tasks), hard (6 harder tasks) or adversarial
+  --cases <dir|file>     your own tasks: a folder of *.json, or a .ts/.js module exporting a list of cases
+  --suite <file.jsonl>   a SWE-bench-format file; needs --test-cmd (see amness eval --help)   --limit <n>   --install <cmd>
+  --only <a,b>           only these tasks
+  --trials <n>           attempts per task and contestant (default 3)   --jobs <n>   runs at once (default 1; the tools use your API limits)
+  --timeout <seconds>    per run (default 900)
+  --check                see which of the tools are installed (no model call)
+  --probe                one tiny real task per contestant, to confirm the tool starts, is graded, and its tokens and cost are read correctly (costs a few cents)
+  --dry-run              show exactly what would run, change nothing
+  --out <file>           where to save the raw results (re-read with: amness eval --report <file>)
+  --yes                  skip the cost confirmation
+Safety: each run happens in a throw-away temp folder, but the tools run with their file and shell tools enabled and are NOT sandboxed from the rest of your machine. Run benchmarks in a VM or container if the tasks are not yours.`;
+async function pickCases(val, has) {
+    let cases;
+    const arg = val("--cases"), suite = val("--suite");
+    if (suite) {
+        if (!val("--test-cmd"))
+            throw new Error('a suite needs --test-cmd, for example --test-cmd "python -m pytest -x -q {tests}"');
+        cases = suiteToCases(loadSuite(resolve(suite)).slice(0, Number(val("--limit") ?? 1e9)), { testCmd: val("--test-cmd"), installCmd: val("--install") });
+    }
+    else if (arg && /\.(m?[jt]s)$/.test(arg)) {
+        const mod = await import(__rewriteRelativeImportExtension(pathToFileURL(resolve(arg)).href + `?v=${Date.now()}`));
+        cases = mod.default ?? mod.cases;
+        if (!Array.isArray(cases))
+            throw new Error(`${arg} must export a list of cases`);
+    }
+    else if (arg)
+        cases = loadCases(resolve(arg));
+    else {
+        const b = val("--builtin") ?? "code";
+        if (!["code", "hard", "adversarial"].includes(b))
+            throw new Error(`unknown built-in suite "${b}"`);
+        cases = b === "hard" ? hardCases : b === "adversarial" ? adversarialCases : builtinCases;
+    }
+    if (val("--only")) {
+        const want = val("--only").split(",");
+        cases = cases.filter((c) => want.includes(c.name));
+        if (!cases.length)
+            throw new Error(`no task matches --only ${val("--only")}`);
+    }
+    void has;
+    return cases;
+}
+export async function runBench(argv, deps) {
+    const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; }, has = (f) => argv.includes(f);
+    if (has("--help") || has("-h")) {
+        console.log(BENCH_HELP);
+        return 0;
+    }
+    try {
+        if (has("--init")) {
+            const i = argv.indexOf("--init"), file = resolve(argv[i + 1] && !argv[i + 1].startsWith("--") ? argv[i + 1] : "bench.json");
+            if (existsSync(file)) {
+                console.error(red("✗ ") + `${file} already exists`);
+                return 1;
+            }
+            writeFileSync(file, JSON.stringify(configFileTemplate(deps.amnessCli), null, 2) + "\n");
+            console.log(green("wrote ") + file + "\n" + dim("Edit it: set a model for each contestant, remove the ones you do not have, change trials/jobs/timeout. Then: amness bench --config " + file + " --dry-run"));
+            return 0;
+        }
+        let cfg;
+        if (val("--config"))
+            cfg = JSON.parse(readFileSync(resolve(val("--config")), "utf8"));
+        else {
+            const models = Object.fromEntries((val("--models") ?? "").split(",").filter(Boolean).map((p) => p.split("=")));
+            const names = (val("--contestants") ?? "amness,claude,codex").split(",").map((s) => s.trim()).filter(Boolean);
+            cfg = { contestants: names.map((n) => { if (!PRESETS[n])
+                    throw new Error(`unknown preset "${n}". Presets: ${Object.keys(PRESETS).join(", ")}. For anything else use --config.`); return PRESETS[n]({ model: models[n] ?? val("--model"), amnessCli: deps.amnessCli }); }) };
+        }
+        if (!cfg.contestants?.length)
+            throw new Error("no contestants");
+        for (const k of cfg.contestants) {
+            if (!k.name || !k.command)
+                throw new Error("every contestant needs a name and a command");
+            if (k.args === undefined)
+                k.args = [];
+        }
+        if (new Set(cfg.contestants.map((k) => k.name)).size !== cfg.contestants.length)
+            throw new Error("contestant names must be different");
+        if (val("--trials"))
+            cfg.trials = Number(val("--trials"));
+        if (val("--jobs"))
+            cfg.jobs = Number(val("--jobs"));
+        if (val("--timeout"))
+            cfg.timeoutSec = Number(val("--timeout"));
+        cfg.trials ??= 3;
+        cfg.jobs ??= 1;
+        cfg.logDir ??= join(".amness", "bench", "logs");
+        if (has("--check")) {
+            let bad = 0;
+            for (const k of cfg.contestants) {
+                const ok = installed(k.command);
+                if (!ok)
+                    bad++;
+                console.log(`${ok ? green("found   ") : red("missing ")} ${k.name.padEnd(14)} ${k.command}${ok ? "" : dim("  (not on PATH)")}`);
+            }
+            return bad ? 1 : 0;
+        }
+        const cases = has("--probe") ? [{ name: "probe", files: {}, task: "Create a file named hello.txt that contains exactly: hi", verify: "node -e \"process.exit(require('fs').readFileSync('hello.txt','utf8').trim()==='hi'?0:1)\"", check: async (d) => { try {
+                    return readFileSync(join(d, "hello.txt"), "utf8").trim() === "hi" || "hello.txt is wrong";
+                }
+                catch {
+                    return "no hello.txt";
+                } } }] : await pickCases(val, has);
+        if (has("--probe")) {
+            cfg.trials = 1;
+            cfg.jobs = 1;
+            cfg.timeoutSec = Math.min(cfg.timeoutSec ?? 300, 300);
+        }
+        for (const l of describePlan(cases, cfg))
+            console.log(l);
+        const missing = cfg.contestants.filter((k) => !installed(k.command));
+        for (const k of missing)
+            console.log(yellow("! ") + `${k.name}: "${k.command}" is not on PATH`);
+        if (has("--dry-run"))
+            return 0;
+        if (missing.length === cfg.contestants.length) {
+            console.error(red("✗ ") + "none of the contestants can start");
+            return 1;
+        }
+        const runs = cases.length * cfg.contestants.length * cfg.trials;
+        if (!has("--yes") && !(await deps.confirm(`This starts up to ${runs} agent run(s) that spend real money on your accounts (the tools bill you themselves). Continue?`)))
+            return 1;
+        mkdirSync(join(".amness", "bench"), { recursive: true });
+        const { rows, analysis } = await bench(cases, cfg, { onRow: (r) => console.error(`  ${r.pass ? green("PASS") : red("FAIL")} ${r.case.padEnd(24)} ${r.variant.padEnd(14)} ${dim(`${(r.ms / 1000).toFixed(0)}s${r.tokens ? ` · ${r.tokens} tok` : ""}${r.metrics?.["cost-usd"] !== undefined ? ` · $${r.metrics["cost-usd"].toFixed(3)}` : ""}${r.note ? " · " + r.note : ""}`)}`) });
+        const out = resolve(val("--out") ?? join(".amness", "bench", new Date().toISOString().replace(/[:.]/g, "-") + ".json"));
+        mkdirSync(join(out, ".."), { recursive: true });
+        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), baseline: analysis.baseline, k: Math.min(3, cfg.trials), contestants: cfg.contestants, trials: cfg.trials, rows }, null, 1));
+        console.log("\n" + formatReport(analyze(rows, { k: Math.min(3, cfg.trials) })));
+        console.log("\n" + bold("what each contestant was") + "\n" + cfg.contestants.map((k) => `  ${k.name}: ${k.model ? "model " + k.model : "default model"}${k.note ? "; " + k.note : ""}`).join("\n"));
+        console.log("\n" + dim(`raw results: ${out}  (amness eval --report ${out}). Raw output of every run: ${cfg.logDir}`));
+        console.log(dim("Read this fairly: the tools differ in prompts, default models and permissions, and tokens or cost are shown only when the tool reports them."));
+        return 0;
+    }
+    catch (e) {
+        console.error(red("✗ ") + e.message);
+        return 2;
+    }
+}

package/dist/cli/config.d.ts

@@ -0,0 +1,79 @@
+export declare const PROVIDERS: {
+    readonly anthropic: {
+        readonly label: "Anthropic (Claude)";
+        readonly baseUrl: "https://api.anthropic.com";
+        readonly model: "claude-opus-5-5";
+        readonly needsKey: true;
+    };
+    readonly openai: {
+        readonly label: "OpenAI";
+        readonly baseUrl: "https://api.openai.com/v1";
+        readonly model: "gpt-6.1-sol";
+        readonly needsKey: true;
+    };
+    readonly openrouter: {
+        readonly label: "OpenRouter";
+        readonly baseUrl: "https://openrouter.ai/api/v1";
+        readonly model: "openai/gpt-4.1";
+        readonly needsKey: true;
+    };
+    readonly ollama: {
+        readonly label: "Ollama (local)";
+        readonly baseUrl: "http://localhost:11434/v1";
+        readonly model: "qwen2.5-coder:7b";
+        readonly needsKey: false;
+    };
+    readonly custom: {
+        readonly label: "Custom OpenAI-compatible";
+        readonly baseUrl: "http://localhost:8000/v1";
+        readonly model: "my-model";
+        readonly needsKey: false;
+    };
+};
+export type ProviderId = keyof typeof PROVIDERS;
+export type Config = {
+    provider: ProviderId;
+    baseUrl: string;
+    model: string;
+    apiKey?: string;
+    tavilyKey?: string;
+    maxTokens: number;
+    effort?: "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; /** OpenAI wire API: auto (default) picks Responses for reasoning models and self-corrects */
+    api?: "auto" | "responses" | "chat"; /** full-screen colour theme (see /theme) */
+    theme?: string; /** which UI to start in */
+    ui?: "full" | "inline"; /** while idle, re-send the conversation now and then so the provider keeps its prompt cache (Claude: the cache expires after 5 minutes) */
+    keepWarm?: boolean; /** semantic memory retrieval via an OpenAI-compatible embeddings endpoint (sends note text to that provider) */
+    embeddings?: boolean; /** which shell runs the agent's commands: auto (default), bash, zsh, sh, pwsh, powershell or cmd (see /shell) */
+    shell?: string; /** a system prompt id from .amness/prompts (see /prompt) */
+    prompt?: string; /** a loop strategy: plan-first or explore-then-act (see /strategy) */
+    strategy?: string; /** most model requests in flight at once, shared by the lead agent and its subagents (default 4); halves automatically on a 429 and ramps back up */
+    maxConcurrent?: number; /** OpenAI Responses only: server-side compaction once the prompt passes this many tokens */
+    compactAt?: number; /** Claude only: advisory token budget per turn (task_budget, beta; at least 20000) */
+    taskBudget?: number; /** a second model reviews risky calls before you are asked (extra model call per risky action) */
+    approvalReviewer?: boolean; /** errors from an installed language server are appended to edit results */
+    lsp?: boolean; /** an independent reviewer checks finished work against acceptance criteria (extra model calls) */
+    review?: boolean; /** OS sandbox for shell commands */
+    sandbox?: "off" | "workspace" | "strict"; /** model ids (same provider) for other roles; unset roles use `model` */
+    models?: {
+        explore?: string;
+        compact?: string;
+        review?: string;
+        cheap?: string;
+    }; /** route exploration steps to models.cheap when that pays off (cache-aware) */
+    router?: boolean; /** model id to answer from when the main one is overloaded or unreachable */
+    fallback?: string;
+};
+export declare const home: () => string;
+export declare const defaults: () => Config;
+export declare function loadConfig(): Config;
+export declare function saveConfig(c: Config): void;
+export declare const ready: (c: Config) => boolean;
+export declare const mask: (k?: string) => string;
+export declare const CLAUDE_MODELS: {
+    id: string;
+    hint: string;
+}[];
+export declare const OPENAI_MODELS: {
+    id: string;
+    hint: string;
+}[];

package/dist/cli/config.js

@@ -0,0 +1,54 @@
+import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+export const PROVIDERS = {
+    anthropic: { label: "Anthropic (Claude)", baseUrl: "https://api.anthropic.com", model: "claude-opus-5-5", needsKey: true },
+    openai: { label: "OpenAI", baseUrl: "https://api.openai.com/v1", model: "gpt-6.1-sol", needsKey: true },
+    openrouter: { label: "OpenRouter", baseUrl: "https://openrouter.ai/api/v1", model: "openai/gpt-4.1", needsKey: true },
+    ollama: { label: "Ollama (local)", baseUrl: "http://localhost:11434/v1", model: "qwen2.5-coder:7b", needsKey: false },
+    custom: { label: "Custom OpenAI-compatible", baseUrl: "http://localhost:8000/v1", model: "my-model", needsKey: false },
+};
+export const home = () => process.env.AMNESS_HOME ?? join(homedir(), ".amness");
+const file = () => join(home(), "config.json");
+export const defaults = () => ({ provider: "openai", baseUrl: PROVIDERS.openai.baseUrl, model: PROVIDERS.openai.model, maxTokens: 300_000 });
+export function loadConfig() {
+    let c = defaults();
+    try {
+        if (existsSync(file()))
+            c = { ...c, ...JSON.parse(readFileSync(file(), "utf8")) };
+    }
+    catch { /* corrupt config: start fresh */ }
+    // environment wins, so CI and one-off runs work without touching the saved file
+    const claude = c.provider === "anthropic";
+    const key = claude ? process.env.ANTHROPIC_API_KEY : process.env.OPENAI_API_KEY;
+    if (key)
+        c.apiKey = key;
+    if (process.env.TAVILY_API_KEY)
+        c.tavilyKey = process.env.TAVILY_API_KEY;
+    if (process.env.OPENAI_BASE_URL && !claude)
+        c.baseUrl = process.env.OPENAI_BASE_URL;
+    if (process.env.AMNESS_MODEL)
+        c.model = process.env.AMNESS_MODEL;
+    return c;
+}
+export function saveConfig(c) {
+    mkdirSync(home(), { recursive: true });
+    writeFileSync(file(), JSON.stringify(c, null, 2));
+    chmodSync(file(), 0o600); // may hold API keys
+}
+export const ready = (c) => !PROVIDERS[c.provider].needsKey || !!c.apiKey;
+export const mask = (k) => (k ? k.slice(0, 3) + "…" + k.slice(-4) : "not set");
+export const CLAUDE_MODELS = [
+    { id: "claude-opus-5-5", hint: "most capable everyday model · $4/$20 per M tokens (default)" },
+    { id: "claude-sonnet-5-5", hint: "fast and cheaper · $2/$10" },
+    { id: "claude-haiku-5-5", hint: "cheapest · $0.10/$0.50 · no refusal fallback" },
+    { id: "claude-fable-5-1", hint: "hardest tasks · $10/$50" },
+];
+export const OPENAI_MODELS = [
+    { id: "gpt-6.1-sol", hint: "near-flagship for complex work · $2/$10 per M tokens (default)" },
+    { id: "gpt-6-astra", hint: "flagship, hardest tasks · $10/$50" },
+    { id: "gpt-6.1-terra", hint: "fast + efficient, computer use · $1/$5" },
+    { id: "gpt-6-luna", hint: "cheap, high-volume · $0.10/$0.50" },
+    { id: "gpt-5.6-luna", hint: "previous generation" },
+    { id: "gpt-4.1", hint: "non-reasoning · Chat Completions" },
+];

package/dist/cli/demo.d.ts

@@ -0,0 +1,13 @@
+import { type Approve, type Event, type Model } from "../src/index.ts";
+export declare const DEMO_TASK = "sumTo(n) should return 1+2+...+n but the tests fail. Fix it.";
+export declare function runDemo(o: {
+    approve: Approve;
+    onEvent: (e: Event) => void;
+    delayMs?: number;
+    model?: (m: Model) => Model;
+}): Promise<{
+    dir: string;
+    result: import("../src/harness.ts").RunResult;
+    events: Event[];
+}>;
+export declare const explain: Record<string, string>;

package/dist/cli/demo.js

@@ -0,0 +1,44 @@
+// Guided demo: a scripted "model" that makes realistic mistakes so every harness guard fires, with no API key.
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createCodeAgent } from "../src/index.js";
+let id = 0;
+const call = (name, args, note) => ({
+    role: "assistant", content: note ?? null, usage: { total_tokens: 420 },
+    tool_calls: [{ id: `demo${++id}`, function: { name, arguments: JSON.stringify(args) } }],
+});
+export const DEMO_TASK = "sumTo(n) should return 1+2+...+n but the tests fail. Fix it.";
+export async function runDemo(o) {
+    const dir = mkdtempSync(join(tmpdir(), "amness-demo-"));
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, "sum.js"), "exports.sumTo = (n) => { let s = 0; for (let i = 1; i < n; i++) s += i; return s; };\n");
+    writeFileSync(join(dir, "test.js"), "const a = require('assert'); const { sumTo } = require('./sum'); a.equal(sumTo(3), 6); a.equal(sumTo(1), 1); console.log('tests passed');\n");
+    const script = [
+        { role: "assistant", content: "I'll start by looking at the code and the failing test.", usage: { total_tokens: 300 } }, // intent, no action → nudge
+        call("read_file", {}), // missing arg → repair
+        call("edit_file", { path: "sum.js", old: "i < n", new: "i <= n" }), // edit before read → blocked
+        call("list_files", {}),
+        call("read_file", { path: "sum.js" }),
+        call("sh", { cmd: "rm -rf node_modules && git clean -fdx" }), // dangerous → approval prompt
+        call("finish", { summary: "Looks fine to me." }), // lies → verify gate runs tests → rejected
+        call("edit_file", { path: "sum.js", old: "i < n", new: "i <= n" }),
+        call("sh", { cmd: "node test.js" }),
+        call("finish", { summary: "Fixed off-by-one in sumTo (i < n → i <= n); test.js passes." }),
+    ];
+    const scripted = async () => { await new Promise((r) => setTimeout(r, o.delayMs ?? 500)); return script.shift(); };
+    const events = [];
+    const agent = createCodeAgent({
+        model: o.model ? o.model(scripted) : scripted, cwd: dir, verify: "node test.js", approve: o.approve,
+        onEvent: (e) => (events.push(e), o.onEvent(e)),
+    });
+    const result = await agent.run(DEMO_TASK);
+    return { dir, result, events };
+}
+export const explain = {
+    nudge: "model said what it would do but did nothing → pushed back to act",
+    repair: "malformed tool call → exact error returned, model self-corrected",
+    denied: "dangerous command → human approval required",
+    rejected: "model claimed success → real test run disagreed, finish blocked",
+    tool: "tool calls executed",
+};

package/dist/cli/detect.d.ts

@@ -0,0 +1,2 @@
+/** Best-guess command that proves the project works (used as the finish gate). */
+export declare function detectVerify(dir: string): string;

package/dist/cli/detect.js

@@ -0,0 +1,20 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+/** Best-guess command that proves the project works (used as the finish gate). */
+export function detectVerify(dir) {
+    try {
+        if (existsSync(join(dir, "package.json"))) {
+            const t = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).scripts?.test;
+            if (t && !/no test specified/.test(t))
+                return "npm test";
+        }
+    }
+    catch { /* ignore */ }
+    if (existsSync(join(dir, "Cargo.toml")))
+        return "cargo test";
+    if (existsSync(join(dir, "go.mod")))
+        return "go test ./...";
+    if (["pytest.ini", "pyproject.toml", "setup.py", "tox.ini"].some((f) => existsSync(join(dir, f))))
+        return "pytest -q";
+    return "";
+}

package/dist/cli/evalcli.d.ts

@@ -0,0 +1,15 @@
+import { type Model } from "../src/index.ts";
+export declare const EVAL_HELP: string;
+export declare function runEval(argv: string[], deps: {
+    model: () => Model;
+    modelFor?: (id: string) => Model;
+    maxTokens: number;
+    confirm: (q: string) => Promise<boolean>;
+}): Promise<number>;
+export declare const TUNE_HELP = "usage: amness tune [flags]\n  Try switching guards off (and changing a limit), one at a time, on part of your tasks; keep a change only if it also holds up on tasks the search never saw. Rolled back otherwise.\n  Writes nothing unless something survives both checks. Calls your model: it asks first.\n\n  --cases <dir>        your own tasks (default: the 13 built-in tasks)\n  --guards <a,b>       which guards to try removing (default: nudge, loop, stuck, wrapup, condense, stagnation)\n  --stagnation <n,n>   also try these stagnationTurns values\n  --trials <n>         attempts per task (default 2)\n  --max-steps <n>      steps per session (default 30)\n  --max-tokens <n>     token cap per session\n  --jobs <n>           sessions at once (default 1)\n  --seed <n>           seed for the split and the intervals\n  --profile-model <p>  the model name or pattern to write the profile for (default: the configured model)\n  --out <file>         where to write the profile (default .amness/profile.json)\n  --yes                skip the cost confirmation";
+export declare function runTune(argv: string[], deps: {
+    model: () => Model;
+    modelName: string;
+    maxTokens: number;
+    confirm: (q: string) => Promise<boolean>;
+}): Promise<number>;

package/dist/cli/evalcli.js

@@ -0,0 +1,198 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+// `amness eval`: compare harness variants on a task suite, with statistics you can act on.
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { BUILTIN_METRICS, gateCompare, importPlugins, tune, writeProfile, adversarialCases, hardCases, analyze, builtinCases, evaluate, recommend, leakAudit, loadSuite, suiteToCases, formatReport, harnessVariant, loadCases, selfTest, variantNames, VARIANT_HELP } from "../src/index.js";
+import { bold, dim, green, red } from "./ui.js";
+import { pathToFileURL } from "node:url";
+async function casesFromModule(file) {
+    const mod = await import(__rewriteRelativeImportExtension(pathToFileURL(file).href + `?v=${Date.now()}`)), list = mod.default ?? mod.cases;
+    if (!Array.isArray(list) || list.some((c) => !c || typeof c.name !== "string" || typeof c.task !== "string" || !c.files || !c.check))
+        throw new Error(`${file}: must export a list of cases ({ name, files, task, check })`);
+    return list;
+}
+export const EVAL_HELP = `usage: amness eval [flags]
+  Run the coding agent on a suite of tasks under several harness variants and compare them statistically.
+
+  --variants <spec>    preset or comma list (default: ab).  presets: ab · ablate · all
+                       variants: ${VARIANT_HELP}
+  --trials <n>         attempts per task and variant (default 3)
+  --suite <file.jsonl> a benchmark in SWE-bench format (instance_id, repo, base_commit, problem_statement, test_patch, FAIL_TO_PASS...); needs --test-cmd
+  --test-cmd <cmd>     how to run the hidden tests; \`{tests}\` is replaced by the instance's FAIL_TO_PASS ids, e.g. "python -m pytest -x -q {tests}"
+  --install <cmd>      run once per instance after checkout (dependencies)
+  --limit <n>          only the first n instances of a suite
+  --audit              contamination checks on a suite (fix lines in the problem statement or already in the base checkout); runs no model
+  --seed <n>           seed for the bootstrap intervals (reproducible reports)
+  --builtin <name>     which built-in suite: code (default, 13 easy tasks), hard (6 tasks with hidden edge cases, a misleading symptom and an exploration question) or adversarial (prompt-injection attempts; a run passes only if the injected command never ran)
+  --cases <dir|file>   your own tasks: a folder of *.json ({ name, files, task, verify, check }), or a .ts/.js module exporting a list of cases (checks may be functions)
+  --metrics <a,b>      extra numbers to record per run and compare (diff-lines, tool-calls, tokens-per-step, overclaim, or a metric a plugin defines)
+  --gate <file>        compare with a saved results file (from --out) and exit 1 on a clear regression: pass rate down by more than --gate-tol points (default 5) or cost per solved task up by more than --gate-cost percent (default 25)
+  --plugin <file,...>  load plugins from these files: their tasks join the suite and their variants can be named in --variants (also full+plugins: every loaded plugin active)
+  --only <a,b>         run only these built-in tasks
+  --baseline <name>    variant the others are compared with (default: the first)
+  --cheap-model <id>   the cheaper model for the full+route variant (same provider)
+  --jobs <n>           sessions run at once (default 1)
+  --max-tokens <n>     token cap per session (default: your configured budget)
+  --max-steps <n>      steps per session (default 30)
+  --profile-out <f>    after an ablation, write the recommended profile (guards the data says to switch off) for --profile-model
+  --profile-model <p>  the model name or pattern the profile applies to (default: the configured model)
+  --out <file>         where to save raw results (default .amness/evals/<time>.json)
+  --report <file>      re-analyse saved results; runs nothing
+  --selftest           check that every task is solvable and its held-out check is sound; runs no model
+  --yes                skip the cost confirmation`;
+export async function runEval(argv, deps) {
+    const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; }, has = (f) => argv.includes(f);
+    if (has("--help") || has("-h")) {
+        console.log(EVAL_HELP);
+        return 0;
+    }
+    try {
+        if (has("--report")) {
+            const saved = JSON.parse(readFileSync(resolve(val("--report")), "utf8"));
+            console.log(formatReport(analyze(saved.rows, { baseline: val("--baseline") ?? saved.baseline, k: saved.k })));
+            return 0;
+        }
+        const builtin = val("--builtin") ?? "code";
+        if (!["code", "adversarial", "hard"].includes(builtin))
+            throw new Error(`unknown built-in suite "${builtin}" (code, hard or adversarial)`);
+        const plugins = [];
+        for (const f of (val("--plugin") ?? "").split(",").filter(Boolean)) {
+            const r = await importPlugins(f);
+            for (const w of r.warnings)
+                console.error(red("! ") + w);
+            plugins.push(...r.plugins);
+        }
+        const casesArg = val("--cases"), isModule = !!casesArg && /\.(m?[jt]s)$/.test(casesArg);
+        let cases = isModule ? await casesFromModule(resolve(casesArg)) : casesArg ? loadCases(resolve(casesArg)) : builtin === "adversarial" ? adversarialCases : builtin === "hard" ? hardCases : builtinCases;
+        if (val("--suite")) {
+            const inst = loadSuite(resolve(val("--suite"))).slice(0, Number(val("--limit") ?? 1e9));
+            if (has("--audit")) {
+                const leaks = await leakAudit(inst);
+                for (const l of leaks)
+                    console.log(`${red("leak")} ${l.id}: ${l.problem}`);
+                console.log(leaks.length ? `${leaks.length} instance(s) with problems` : green("no contamination found in " + inst.length + " instance(s)"));
+                return leaks.length ? 1 : 0;
+            }
+            if (!val("--test-cmd"))
+                throw new Error('a suite needs --test-cmd, e.g. --test-cmd "python -m pytest -x -q {tests}"');
+            cases = suiteToCases(inst, { testCmd: val("--test-cmd"), installCmd: val("--install") });
+        }
+        for (const p of plugins)
+            cases.push(...(p.cases ?? []));
+        const metricFns = {};
+        for (const p of plugins)
+            Object.assign(metricFns, p.metrics ?? {});
+        for (const m of (val("--metrics") ?? "").split(",").filter(Boolean)) {
+            const fn = BUILTIN_METRICS[m] ?? metricFns[m];
+            if (!fn)
+                throw new Error(`unknown metric "${m}". Built in: ${Object.keys(BUILTIN_METRICS).join(", ")}${Object.keys(metricFns).length ? "; from plugins: " + Object.keys(metricFns).join(", ") : ""}`);
+            metricFns[m] = fn;
+        }
+        const wanted = (val("--metrics") ?? "").split(",").filter(Boolean), useMetrics = Object.fromEntries(Object.entries(metricFns).filter(([k]) => wanted.includes(k) || plugins.some((p) => p.metrics && k in p.metrics)));
+        if (val("--only")) {
+            const want = val("--only").split(",");
+            cases = cases.filter((c) => want.includes(c.name));
+            if (!cases.length)
+                throw new Error(`no task matches --only ${val("--only")}`);
+        }
+        if (has("--selftest")) {
+            const res = await selfTest(cases);
+            for (const r of res)
+                console.log(`${r.ok ? green("ok ") : red("bad")} ${r.case}${r.why ? dim("  " + r.why) : ""}`);
+            return res.every((r) => r.ok) ? 0 : 1;
+        }
+        const names = variantNames(val("--variants") ?? "ab"), trials = Math.max(1, Number(val("--trials") ?? 3)), jobs = Math.max(1, Number(val("--jobs") ?? 1));
+        const maxTokens = Number(val("--max-tokens") ?? deps.maxTokens), maxSteps = Number(val("--max-steps") ?? 30);
+        const model = deps.model(), cheap = val("--cheap-model") && deps.modelFor ? deps.modelFor(val("--cheap-model")) : undefined, variants = names.map((n) => harnessVariant(model, n, { maxTokens, maxSteps, cheap, plugins, modelFor: deps.modelFor }));
+        const runs = cases.length * variants.length * trials;
+        console.log(`${bold("plan")}  ${cases.length} tasks × ${variants.length} variants (${names.join(", ")}) × ${trials} trials = ${bold(String(runs))} agent sessions, up to ${maxTokens.toLocaleString()} tokens each`);
+        if (!has("--yes") && !(await deps.confirm(`This calls your model up to ${(runs * maxTokens).toLocaleString()} tokens in the worst case. Continue?`)))
+            return 1;
+        const { rows, analysis } = await evaluate(cases, variants, {
+            metrics: Object.keys(useMetrics).length ? useMetrics : undefined, trials, jobs, baseline: val("--baseline"), k: Math.min(3, trials), seed: val("--seed") ? Number(val("--seed")) : undefined,
+            onRow: (r) => console.error(`  ${r.pass ? green("PASS") : red("FAIL")} ${r.case.padEnd(22)} ${r.variant.padEnd(18)} ${dim(`${r.steps} steps · ${r.tokens} tok · ${(r.ms / 1000).toFixed(1)}s${r.note ? " · " + r.note : ""}`)}`),
+        });
+        const out = resolve(val("--out") ?? join(".amness", "evals", new Date().toISOString().replace(/[:.]/g, "-") + ".json"));
+        mkdirSync(join(out, ".."), { recursive: true });
+        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), baseline: analysis.baseline, k: Math.min(3, trials), variants: names, trials, rows }, null, 1));
+        if (val("--profile-out")) {
+            const rec = recommend(analysis, names.includes("full") ? "full" : analysis.baseline), pm = val("--profile-model") ?? "*";
+            console.log("\nguard verdicts:");
+            for (const v of rec.verdicts)
+                console.log(`  ${v.action === "remove" ? "remove" : v.action === "keep" ? "KEEP  " : "keep? "} ${v.guard.padEnd(11)} ${v.why}`);
+            console.log(`  ${rec.strength}`);
+            writeFileSync(resolve(val("--profile-out")), JSON.stringify({ models: { [pm]: rec.profile } }, null, 2) + "\n");
+            console.log(`profile written to ${val("--profile-out")}: copy it to .amness/profile.json`);
+        }
+        console.log("\n" + formatReport(analysis) + "\n\n" + dim(`raw results: ${out}  (re-analyse with: amness eval --report ${out})`));
+        if (val("--gate")) {
+            const base = JSON.parse(readFileSync(resolve(val("--gate")), "utf8")), verdicts = gateCompare(rows, base.rows, { tol: val("--gate-tol") ? Number(val("--gate-tol")) / 100 : undefined, costPct: val("--gate-cost") ? Number(val("--gate-cost")) : undefined });
+            console.log("\n" + bold("regression gate") + dim(` against ${val("--gate")}`));
+            for (const g of verdicts) {
+                console.log(`  ${g.ok ? green("ok  ") : red("FAIL")} ${g.variant}`);
+                for (const l of g.lines)
+                    console.log("       " + l);
+            }
+            if (verdicts.some((g) => !g.ok))
+                return 1;
+        }
+        return 0;
+    }
+    catch (e) {
+        console.error(red("✗ " + e.message));
+        return 2;
+    }
+}
+export const TUNE_HELP = `usage: amness tune [flags]
+  Try switching guards off (and changing a limit), one at a time, on part of your tasks; keep a change only if it also holds up on tasks the search never saw. Rolled back otherwise.
+  Writes nothing unless something survives both checks. Calls your model: it asks first.
+
+  --cases <dir>        your own tasks (default: the 13 built-in tasks)
+  --guards <a,b>       which guards to try removing (default: nudge, loop, stuck, wrapup, condense, stagnation)
+  --stagnation <n,n>   also try these stagnationTurns values
+  --trials <n>         attempts per task (default 2)
+  --max-steps <n>      steps per session (default 30)
+  --max-tokens <n>     token cap per session
+  --jobs <n>           sessions at once (default 1)
+  --seed <n>           seed for the split and the intervals
+  --profile-model <p>  the model name or pattern to write the profile for (default: the configured model)
+  --out <file>         where to write the profile (default .amness/profile.json)
+  --yes                skip the cost confirmation`;
+export async function runTune(argv, deps) {
+    const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; };
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(TUNE_HELP);
+        return 0;
+    }
+    try {
+        const cases = val("--cases") ? loadCases(resolve(val("--cases"))) : builtinCases, model = deps.model(), maxTokens = Number(val("--max-tokens") ?? deps.maxTokens), maxSteps = Number(val("--max-steps") ?? 30);
+        const guards = val("--guards")?.split(",").map((s) => s.trim()), stag = val("--stagnation")?.split(",").map(Number), trials = Math.max(1, Number(val("--trials") ?? 2));
+        const n = (guards?.length ?? 6) + (stag?.length ?? 0), runs = Math.round(cases.length * 0.66) * 2 * trials * n + Math.max(1, Math.round(cases.length * 0.34)) * 2 * trials;
+        console.log(`${bold("plan")}  up to ${runs} agent sessions (${cases.length} tasks, ${n} candidate changes, ${trials} trial(s)), up to ${maxTokens.toLocaleString()} tokens each`);
+        if (!argv.includes("--yes") && !(await deps.confirm(`This calls your model up to ${(runs * maxTokens).toLocaleString()} tokens in the worst case. Continue?`)))
+            return 1;
+        const r = await tune({ cases, trials, guards, stagnationTurns: stag, jobs: Number(val("--jobs") ?? 1), seed: val("--seed") ? Number(val("--seed")) : undefined,
+            makeVariant: (profile, name) => ({ ...harnessVariant(model, "full", { maxTokens, maxSteps, profile }), name }),
+            onStep: (s) => console.log(`  ${s.accepted ? green("keep") : red("drop")} ${s.stage === "heldout" ? "[held-out] " : ""}${s.change}  ${dim(s.why)}`) });
+        console.log(dim(`\n${r.strength}; held-out tasks: ${r.heldOut.join(", ") || "none"}`));
+        if (!r.accepted.length) {
+            console.log("nothing survived both checks; no profile written.");
+            return 0;
+        }
+        const out = resolve(val("--out") ?? join(".amness", "profile.json"));
+        writeProfile(out, val("--profile-model") ?? deps.modelName, r.profile);
+        console.log(`kept: ${r.accepted.join(", ")}\nprofile written to ${out}`);
+        return 0;
+    }
+    catch (e) {
+        console.error(red("✗ " + e.message));
+        return 2;
+    }
+}

package/dist/cli/format.d.ts

@@ -0,0 +1,5 @@
+import type { Event } from "../src/index.ts";
+/** One readable trace line per harness event. */
+export declare function formatEvent(e: Event & {
+    agent?: string;
+}): string;