amness 0.0.0-stage → 0.1.2

package/dist/cli/evalcli.js

@@ -0,0 +1,198 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+// `amness eval`: compare harness variants on a task suite, with statistics you can act on.
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { BUILTIN_METRICS, gateCompare, importPlugins, tune, writeProfile, adversarialCases, hardCases, analyze, builtinCases, evaluate, recommend, leakAudit, loadSuite, suiteToCases, formatReport, harnessVariant, loadCases, selfTest, variantNames, VARIANT_HELP } from "../src/index.js";
+import { bold, dim, green, red } from "./ui.js";
+import { pathToFileURL } from "node:url";
+async function casesFromModule(file) {
+    const mod = await import(__rewriteRelativeImportExtension(pathToFileURL(file).href + `?v=${Date.now()}`)), list = mod.default ?? mod.cases;
+    if (!Array.isArray(list) || list.some((c) => !c || typeof c.name !== "string" || typeof c.task !== "string" || !c.files || !c.check))
+        throw new Error(`${file}: must export a list of cases ({ name, files, task, check })`);
+    return list;
+}
+export const EVAL_HELP = `usage: amness eval [flags]
+  Run the coding agent on a suite of tasks under several harness variants and compare them statistically.
+
+  --variants <spec>    preset or comma list (default: ab).  presets: ab · ablate · all
+                       variants: ${VARIANT_HELP}
+  --trials <n>         attempts per task and variant (default 3)
+  --suite <file.jsonl> a benchmark in SWE-bench format (instance_id, repo, base_commit, problem_statement, test_patch, FAIL_TO_PASS...); needs --test-cmd
+  --test-cmd <cmd>     how to run the hidden tests; \`{tests}\` is replaced by the instance's FAIL_TO_PASS ids, e.g. "python -m pytest -x -q {tests}"
+  --install <cmd>      run once per instance after checkout (dependencies)
+  --limit <n>          only the first n instances of a suite
+  --audit              contamination checks on a suite (fix lines in the problem statement or already in the base checkout); runs no model
+  --seed <n>           seed for the bootstrap intervals (reproducible reports)
+  --builtin <name>     which built-in suite: code (default, 13 easy tasks), hard (6 tasks with hidden edge cases, a misleading symptom and an exploration question) or adversarial (prompt-injection attempts; a run passes only if the injected command never ran)
+  --cases <dir|file>   your own tasks: a folder of *.json ({ name, files, task, verify, check }), or a .ts/.js module exporting a list of cases (checks may be functions)
+  --metrics <a,b>      extra numbers to record per run and compare (diff-lines, tool-calls, tokens-per-step, overclaim, or a metric a plugin defines)
+  --gate <file>        compare with a saved results file (from --out) and exit 1 on a clear regression: pass rate down by more than --gate-tol points (default 5) or cost per solved task up by more than --gate-cost percent (default 25)
+  --plugin <file,...>  load plugins from these files: their tasks join the suite and their variants can be named in --variants (also full+plugins: every loaded plugin active)
+  --only <a,b>         run only these built-in tasks
+  --baseline <name>    variant the others are compared with (default: the first)
+  --cheap-model <id>   the cheaper model for the full+route variant (same provider)
+  --jobs <n>           sessions run at once (default 1)
+  --max-tokens <n>     token cap per session (default: your configured budget)
+  --max-steps <n>      steps per session (default 30)
+  --profile-out <f>    after an ablation, write the recommended profile (guards the data says to switch off) for --profile-model
+  --profile-model <p>  the model name or pattern the profile applies to (default: the configured model)
+  --out <file>         where to save raw results (default .amness/evals/<time>.json)
+  --report <file>      re-analyse saved results; runs nothing
+  --selftest           check that every task is solvable and its held-out check is sound; runs no model
+  --yes                skip the cost confirmation`;
+export async function runEval(argv, deps) {
+    const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; }, has = (f) => argv.includes(f);
+    if (has("--help") || has("-h")) {
+        console.log(EVAL_HELP);
+        return 0;
+    }
+    try {
+        if (has("--report")) {
+            const saved = JSON.parse(readFileSync(resolve(val("--report")), "utf8"));
+            console.log(formatReport(analyze(saved.rows, { baseline: val("--baseline") ?? saved.baseline, k: saved.k })));
+            return 0;
+        }
+        const builtin = val("--builtin") ?? "code";
+        if (!["code", "adversarial", "hard"].includes(builtin))
+            throw new Error(`unknown built-in suite "${builtin}" (code, hard or adversarial)`);
+        const plugins = [];
+        for (const f of (val("--plugin") ?? "").split(",").filter(Boolean)) {
+            const r = await importPlugins(f);
+            for (const w of r.warnings)
+                console.error(red("! ") + w);
+            plugins.push(...r.plugins);
+        }
+        const casesArg = val("--cases"), isModule = !!casesArg && /\.(m?[jt]s)$/.test(casesArg);
+        let cases = isModule ? await casesFromModule(resolve(casesArg)) : casesArg ? loadCases(resolve(casesArg)) : builtin === "adversarial" ? adversarialCases : builtin === "hard" ? hardCases : builtinCases;
+        if (val("--suite")) {
+            const inst = loadSuite(resolve(val("--suite"))).slice(0, Number(val("--limit") ?? 1e9));
+            if (has("--audit")) {
+                const leaks = await leakAudit(inst);
+                for (const l of leaks)
+                    console.log(`${red("leak")} ${l.id}: ${l.problem}`);
+                console.log(leaks.length ? `${leaks.length} instance(s) with problems` : green("no contamination found in " + inst.length + " instance(s)"));
+                return leaks.length ? 1 : 0;
+            }
+            if (!val("--test-cmd"))
+                throw new Error('a suite needs --test-cmd, e.g. --test-cmd "python -m pytest -x -q {tests}"');
+            cases = suiteToCases(inst, { testCmd: val("--test-cmd"), installCmd: val("--install") });
+        }
+        for (const p of plugins)
+            cases.push(...(p.cases ?? []));
+        const metricFns = {};
+        for (const p of plugins)
+            Object.assign(metricFns, p.metrics ?? {});
+        for (const m of (val("--metrics") ?? "").split(",").filter(Boolean)) {
+            const fn = BUILTIN_METRICS[m] ?? metricFns[m];
+            if (!fn)
+                throw new Error(`unknown metric "${m}". Built in: ${Object.keys(BUILTIN_METRICS).join(", ")}${Object.keys(metricFns).length ? "; from plugins: " + Object.keys(metricFns).join(", ") : ""}`);
+            metricFns[m] = fn;
+        }
+        const wanted = (val("--metrics") ?? "").split(",").filter(Boolean), useMetrics = Object.fromEntries(Object.entries(metricFns).filter(([k]) => wanted.includes(k) || plugins.some((p) => p.metrics && k in p.metrics)));
+        if (val("--only")) {
+            const want = val("--only").split(",");
+            cases = cases.filter((c) => want.includes(c.name));
+            if (!cases.length)
+                throw new Error(`no task matches --only ${val("--only")}`);
+        }
+        if (has("--selftest")) {
+            const res = await selfTest(cases);
+            for (const r of res)
+                console.log(`${r.ok ? green("ok ") : red("bad")} ${r.case}${r.why ? dim("  " + r.why) : ""}`);
+            return res.every((r) => r.ok) ? 0 : 1;
+        }
+        const names = variantNames(val("--variants") ?? "ab"), trials = Math.max(1, Number(val("--trials") ?? 3)), jobs = Math.max(1, Number(val("--jobs") ?? 1));
+        const maxTokens = Number(val("--max-tokens") ?? deps.maxTokens), maxSteps = Number(val("--max-steps") ?? 30);
+        const model = deps.model(), cheap = val("--cheap-model") && deps.modelFor ? deps.modelFor(val("--cheap-model")) : undefined, variants = names.map((n) => harnessVariant(model, n, { maxTokens, maxSteps, cheap, plugins, modelFor: deps.modelFor }));
+        const runs = cases.length * variants.length * trials;
+        console.log(`${bold("plan")}  ${cases.length} tasks × ${variants.length} variants (${names.join(", ")}) × ${trials} trials = ${bold(String(runs))} agent sessions, up to ${maxTokens.toLocaleString()} tokens each`);
+        if (!has("--yes") && !(await deps.confirm(`This calls your model up to ${(runs * maxTokens).toLocaleString()} tokens in the worst case. Continue?`)))
+            return 1;
+        const { rows, analysis } = await evaluate(cases, variants, {
+            metrics: Object.keys(useMetrics).length ? useMetrics : undefined, trials, jobs, baseline: val("--baseline"), k: Math.min(3, trials), seed: val("--seed") ? Number(val("--seed")) : undefined,
+            onRow: (r) => console.error(`  ${r.pass ? green("PASS") : red("FAIL")} ${r.case.padEnd(22)} ${r.variant.padEnd(18)} ${dim(`${r.steps} steps · ${r.tokens} tok · ${(r.ms / 1000).toFixed(1)}s${r.note ? " · " + r.note : ""}`)}`),
+        });
+        const out = resolve(val("--out") ?? join(".amness", "evals", new Date().toISOString().replace(/[:.]/g, "-") + ".json"));
+        mkdirSync(join(out, ".."), { recursive: true });
+        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), baseline: analysis.baseline, k: Math.min(3, trials), variants: names, trials, rows }, null, 1));
+        if (val("--profile-out")) {
+            const rec = recommend(analysis, names.includes("full") ? "full" : analysis.baseline), pm = val("--profile-model") ?? "*";
+            console.log("\nguard verdicts:");
+            for (const v of rec.verdicts)
+                console.log(`  ${v.action === "remove" ? "remove" : v.action === "keep" ? "KEEP  " : "keep? "} ${v.guard.padEnd(11)} ${v.why}`);
+            console.log(`  ${rec.strength}`);
+            writeFileSync(resolve(val("--profile-out")), JSON.stringify({ models: { [pm]: rec.profile } }, null, 2) + "\n");
+            console.log(`profile written to ${val("--profile-out")}: copy it to .amness/profile.json`);
+        }
+        console.log("\n" + formatReport(analysis) + "\n\n" + dim(`raw results: ${out}  (re-analyse with: amness eval --report ${out})`));
+        if (val("--gate")) {
+            const base = JSON.parse(readFileSync(resolve(val("--gate")), "utf8")), verdicts = gateCompare(rows, base.rows, { tol: val("--gate-tol") ? Number(val("--gate-tol")) / 100 : undefined, costPct: val("--gate-cost") ? Number(val("--gate-cost")) : undefined });
+            console.log("\n" + bold("regression gate") + dim(` against ${val("--gate")}`));
+            for (const g of verdicts) {
+                console.log(`  ${g.ok ? green("ok  ") : red("FAIL")} ${g.variant}`);
+                for (const l of g.lines)
+                    console.log("       " + l);
+            }
+            if (verdicts.some((g) => !g.ok))
+                return 1;
+        }
+        return 0;
+    }
+    catch (e) {
+        console.error(red("✗ " + e.message));
+        return 2;
+    }
+}
+export const TUNE_HELP = `usage: amness tune [flags]
+  Try switching guards off (and changing a limit), one at a time, on part of your tasks; keep a change only if it also holds up on tasks the search never saw. Rolled back otherwise.
+  Writes nothing unless something survives both checks. Calls your model: it asks first.
+
+  --cases <dir>        your own tasks (default: the 13 built-in tasks)
+  --guards <a,b>       which guards to try removing (default: nudge, loop, stuck, wrapup, condense, stagnation)
+  --stagnation <n,n>   also try these stagnationTurns values
+  --trials <n>         attempts per task (default 2)
+  --max-steps <n>      steps per session (default 30)
+  --max-tokens <n>     token cap per session
+  --jobs <n>           sessions at once (default 1)
+  --seed <n>           seed for the split and the intervals
+  --profile-model <p>  the model name or pattern to write the profile for (default: the configured model)
+  --out <file>         where to write the profile (default .amness/profile.json)
+  --yes                skip the cost confirmation`;
+export async function runTune(argv, deps) {
+    const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; };
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(TUNE_HELP);
+        return 0;
+    }
+    try {
+        const cases = val("--cases") ? loadCases(resolve(val("--cases"))) : builtinCases, model = deps.model(), maxTokens = Number(val("--max-tokens") ?? deps.maxTokens), maxSteps = Number(val("--max-steps") ?? 30);
+        const guards = val("--guards")?.split(",").map((s) => s.trim()), stag = val("--stagnation")?.split(",").map(Number), trials = Math.max(1, Number(val("--trials") ?? 2));
+        const n = (guards?.length ?? 6) + (stag?.length ?? 0), runs = Math.round(cases.length * 0.66) * 2 * trials * n + Math.max(1, Math.round(cases.length * 0.34)) * 2 * trials;
+        console.log(`${bold("plan")}  up to ${runs} agent sessions (${cases.length} tasks, ${n} candidate changes, ${trials} trial(s)), up to ${maxTokens.toLocaleString()} tokens each`);
+        if (!argv.includes("--yes") && !(await deps.confirm(`This calls your model up to ${(runs * maxTokens).toLocaleString()} tokens in the worst case. Continue?`)))
+            return 1;
+        const r = await tune({ cases, trials, guards, stagnationTurns: stag, jobs: Number(val("--jobs") ?? 1), seed: val("--seed") ? Number(val("--seed")) : undefined,
+            makeVariant: (profile, name) => ({ ...harnessVariant(model, "full", { maxTokens, maxSteps, profile }), name }),
+            onStep: (s) => console.log(`  ${s.accepted ? green("keep") : red("drop")} ${s.stage === "heldout" ? "[held-out] " : ""}${s.change}  ${dim(s.why)}`) });
+        console.log(dim(`\n${r.strength}; held-out tasks: ${r.heldOut.join(", ") || "none"}`));
+        if (!r.accepted.length) {
+            console.log("nothing survived both checks; no profile written.");
+            return 0;
+        }
+        const out = resolve(val("--out") ?? join(".amness", "profile.json"));
+        writeProfile(out, val("--profile-model") ?? deps.modelName, r.profile);
+        console.log(`kept: ${r.accepted.join(", ")}\nprofile written to ${out}`);
+        return 0;
+    }
+    catch (e) {
+        console.error(red("✗ " + e.message));
+        return 2;
+    }
+}

package/dist/cli/format.d.ts

@@ -0,0 +1,5 @@
+import type { Event } from "../src/index.ts";
+/** One readable trace line per harness event. */
+export declare function formatEvent(e: Event & {
+    agent?: string;
+}): string;

package/dist/cli/format.js

@@ -0,0 +1,55 @@
+import { bold, cyan, dim, gray, green, magenta, red, yellow, blue } from "./ui.js";
+const one = (s, n = 90) => { const l = s.replace(/\s+/g, " ").trim(); return l.length > n ? l.slice(0, n - 1) + "…" : l; };
+const argSummary = (name, raw) => {
+    try {
+        const a = JSON.parse(raw);
+        return one(a.cmd ?? a.path ?? a.query ?? a.url ?? a.description ?? a.note ?? a.pattern ?? a.summary ?? raw, 80);
+    }
+    catch {
+        return one(raw, 80);
+    }
+};
+const agentColor = (name) => ({ lead: cyan, general: magenta }[name] ?? blue)(name.padEnd(7));
+/** One readable trace line per harness event. */
+export function formatEvent(e) {
+    const who = e.agent ? agentColor(e.agent) + " " : "";
+    switch (e.kind) {
+        case "tool": {
+            const bad = /^(ERROR|DENIED|REJECTED)/.test(e.result) || /^exit=[^0]/.test(e.result);
+            return `${who}${bad ? red("✗") : green("✓")} ${e.name} ${dim(argSummary(e.name, e.args))}${bad ? "  " + red(one(e.result, 70)) : ""}`;
+        }
+        case "nudge": return `${who}${yellow("↻ nudge")} ${dim("model described an action instead of doing it → told to act")}`;
+        case "repair": return `${who}${yellow("⚒ repair")} ${dim("invalid tool call → error returned to model: " + one(e.detail, 70))}`;
+        case "loop": return `${who}${yellow("∞ loop guard")} ${dim("repeated identical call refused")}`;
+        case "rejected": return `${who}${yellow("⛔ finish rejected")} ${dim(one(e.detail, 80))}`;
+        case "denied": return `${who}${red("⊘ denied")} ${dim(e.detail)}`;
+        case "budget": return `${who}${red("$ budget")} ${e.detail}`;
+        case "tools": return `${who}${gray("⚙ toolbox changed")} ${dim(one(e.detail, 70))}`;
+        case "edit": {
+            const body = e.diff.split("\n").filter((l) => !/^(---|\+\+\+)/.test(l));
+            const shown = body.slice(0, 40).map((l) => l.startsWith("+") ? green("    " + l) : l.startsWith("-") ? red("    " + l) : l.startsWith("@@") ? cyan("    " + l) : gray("    " + l));
+            if (body.length > 40)
+                shown.push(dim(`    … ${body.length - 40} more lines`));
+            return `${who}${yellow("✎")} ${bold(e.path)} ${e.created ? green("new file ") : ""}${green("+" + e.added)} ${red("-" + e.removed)}\n${shown.join("\n")}`;
+        }
+        case "aborted": return `${who}${yellow("⏹ stopped")} ${dim("run interrupted; progress is saved and can be resumed")}`;
+        case "turn":
+        case "phase":
+        case "trim":
+        case "cachebreak":
+        case "run":
+        case "model":
+        case "policy":
+        case "toolstart":
+        case "text":
+        case "thinking":
+        case "assistant":
+        case "todos":
+        case "steer": return "";
+        case "rollover": return `${who}${magenta("⟳ context rollover")} ${dim("handoff note written; continuing in a fresh context")}`;
+        case "review": return `${who}${e.pass ? green("✔ review") : yellow("✖ review")} ${dim(e.inconclusive ? "inconclusive (accepted)" : e.pass ? "the reviewer found no problems" : e.findings.slice(0, 2).map((f) => one(f, 70)).join("; "))}`;
+        case "audit": return e.changed.length || e.notes.length ? `${who}${e.notes.length ? yellow("◐ audit") : green("◉ audit")} ${dim(`${e.changed.length} file(s) changed${e.testsAfterLastEdit === true ? " · tests passed after the last edit" : ""}${e.notes.length ? " · " + e.notes.join("; ") : ""}`)}` : "";
+        case "hook": return `${who}${e.blocked ? red("⛔ hook") : yellow("⚙ hook")} ${dim(e.event + ": " + one(e.detail, 90))}`;
+        case "finish": return `${who}${green("✔ finish")} ${dim(one(e.summary, 80))}`;
+    }
+}

package/dist/cli/headless.d.ts

@@ -0,0 +1,2 @@
+export declare const HELP: string;
+export declare function runHeadless(argv: string[]): Promise<number>;

package/dist/cli/headless.js

@@ -0,0 +1,169 @@
+// Non-interactive mode for CI, scripts and containers:  amness run [flags] "task"
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { Budget, Tracer, Hooks, Permissions, projectConfigFingerprint, createCodeAgent, startLive, setShell, loadPlugins, readImage, visionLikely, bestOfN, parallelParts, isRepo } from "../src/index.js";
+import { PROVIDERS, loadConfig, ready } from "./config.js";
+import { formatEvent } from "./format.js";
+import { createModel } from "./model.js";
+export const HELP = `usage: amness run [flags] "task"
+  --cwd <dir>          workspace (default .)
+  --verify <cmd>       command that must exit 0 before the agent may finish
+  --trust-project      run this project's .amness/hooks.json and plugins and honour its allow rules (they execute code; off by default in headless runs)
+  --sandbox <mode>     run shell commands under an OS sandbox: workspace (write only in the project and temp folders) or strict (also no network)
+  --review             an independent reviewer writes acceptance criteria first and checks the work before it is accepted (extra model calls)
+  --trace <file>       write a JSONL trace of every loop event (also AMNESS_OTLP=<collector url> for OpenTelemetry export)
+  --approval-reviewer  a second model looks at each risky call first and approves clearly safe ones (without --yes, calls it does not clear are still denied)
+  --check <cmd>        fast command run after every edit; failures are shown to the agent immediately
+  --provider <id>      ${Object.keys(PROVIDERS).join(" | ")}    --model <id>    --base-url <url>    --effort none|low|medium|high|xhigh|max   --api auto|responses|chat
+  --bundle <file>      continue from a portable session bundle (any provider/model)
+  --export <file>      write a portable session bundle when done
+  --yes                auto-approve risky commands (default: deny them)
+  --json               machine-readable: one JSON event per line on stdout, final {"kind":"result"} line
+  --live [port]        serve the live browser view
+  --best-of <n>        attempt the task up to n times in separate git worktrees (stops at the first attempt that passes --verify, which is required); the winner is applied
+  --parts <file.json>  run independent parts ([{"name","task"}]) concurrently in worktrees, merged one at a time through --verify; conflicts are reported, not applied
+  --image <file>       show the model a picture (png, jpg, gif, webp; up to 5 MB, repeat the flag for more, max 4); needs a vision-capable model
+  --max-tokens <n>     hard token cap (default 300000)      --no-memory   disable memory
+  -                    as the task: read it from standard input (useful on Windows, where quoting a long task is painful)
+exit code: 0 finished, 1 not finished, 2 usage/config error`;
+export async function runHeadless(argv) {
+    if (argv.includes("--help") || argv.includes("-h")) {
+        console.log(HELP);
+        return 0;
+    } // asking for help must never start a model run
+    const args = [...argv];
+    const val = (f) => { const i = args.indexOf(f); return i < 0 ? undefined : args.splice(i, 2)[1]; };
+    const bool = (f) => { const i = args.indexOf(f); return i >= 0 && !!args.splice(i, 1); };
+    const cfg = loadConfig();
+    const provider = val("--provider");
+    if (provider) {
+        if (!(provider in PROVIDERS)) {
+            console.error(`unknown provider ${provider}\n${HELP}`);
+            return 2;
+        }
+        Object.assign(cfg, { provider, baseUrl: PROVIDERS[provider].baseUrl, model: PROVIDERS[provider].model });
+        const k = provider === "anthropic" ? process.env.ANTHROPIC_API_KEY : process.env.OPENAI_API_KEY;
+        cfg.apiKey = k ?? (loadConfig().provider === provider ? loadConfig().apiKey : undefined);
+    }
+    cfg.model = val("--model") ?? cfg.model;
+    cfg.baseUrl = val("--base-url") ?? cfg.baseUrl;
+    cfg.api = val("--api") ?? cfg.api;
+    cfg.effort = val("--effort") ?? cfg.effort;
+    const cwd = resolve(val("--cwd") ?? "."), verify = val("--verify"), check = val("--check"), bundle = val("--bundle"), exportTo = val("--export");
+    const trustHooks = bool("--trust-project") || bool("--trust-hooks"), yes = bool("--yes"), json = bool("--json"), noMemory = bool("--no-memory");
+    const li = args.indexOf("--live");
+    let livePort;
+    if (li >= 0) {
+        const nxt = args[li + 1];
+        livePort = /^\d+$/.test(nxt ?? "") ? Number(args.splice(li, 2)[1]) : (args.splice(li, 1), 0);
+    }
+    const imagePaths = [];
+    for (let v = val("--image"); v !== undefined; v = val("--image"))
+        imagePaths.push(v);
+    const maxTokens = Number(val("--max-tokens") ?? cfg.maxTokens);
+    let task = args.join(" ").trim();
+    if (task === "-")
+        task = (await new Promise((res) => { let s = ""; process.stdin.setEncoding("utf8"); process.stdin.on("data", (d) => (s += d)); process.stdin.on("end", () => res(s)); })).trim(); // `amness run -` reads the task from stdin (no quoting problems on any shell)
+    if (!task && !bundle) {
+        console.error(HELP);
+        return 2;
+    }
+    if (!ready(cfg)) {
+        console.error(`no API key for ${PROVIDERS[cfg.provider].label}. Set ${cfg.provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"} or run \`amness settings\`.`);
+        return 2;
+    }
+    if (imagePaths.length && !visionLikely(cfg.provider, cfg.model)) {
+        console.error(`${cfg.model} does not look like a vision model; pick one that accepts images or drop --image`);
+        return 2;
+    }
+    let images;
+    try {
+        images = imagePaths.map((p) => readImage(resolve(p)));
+    }
+    catch (e) {
+        console.error(e.message);
+        return 2;
+    }
+    const live = livePort === undefined ? undefined : await startLive({ port: livePort });
+    if (live)
+        console.error(`live view: ${live.url}`);
+    const approve = yes ? () => true : () => "non-interactive run: risky commands are denied unless --yes is passed";
+    const budget = new Budget({ maxTokens });
+    const model = createModel(cfg);
+    const bestOf = Number(val("--best-of") ?? 0), partsFile = val("--parts");
+    if (bestOf || partsFile) {
+        if (!verify) {
+            console.error("--best-of and --parts need --verify <cmd>: that command is what decides which result is kept");
+            return 2;
+        }
+        if (!(await isRepo(cwd))) {
+            console.error("--best-of and --parts need a git repository with at least one commit");
+            return 2;
+        }
+        try {
+            if (bestOf) {
+                const r = await bestOfN({ repo: cwd, n: bestOf, verify, apply: true, maxTokens, attempt: async (dir) => { const b = new Budget({ maxTokens: Math.max(1, Math.floor(maxTokens / bestOf)) }); const a = createCodeAgent({ model, cwd: dir, verify, approve, sandbox: cfg.sandbox, memory: false, budget: b }); const x = await a.run(task); await a.procs?.dispose(); return { tokens: b.tokens, summary: x.text }; }, onCandidate: (c) => console.error(`attempt ${c.index + 1}: ${c.passed ? "passed verify" : c.error ? "error: " + c.error : "failed verify"} (${c.tokens} tokens)`) });
+                if (json)
+                    console.log(JSON.stringify({ kind: "result", finished: !!r.winner, applied: r.applied, tokens: r.tokens, attempts: r.candidates.length }));
+                else
+                    console.log(r.winner ? `attempt ${r.winner.index + 1} won and was ${r.applied ? "applied" : "NOT applied: " + r.applyError}\n${r.winner.summary ?? ""}` : `no attempt passed ${verify} (${r.candidates.length} tried, ${r.tokens} tokens)`);
+                return r.winner && r.applied ? 0 : 1;
+            }
+            const parts = JSON.parse(readFileSync(resolve(partsFile), "utf8"));
+            const { results, tokens } = await parallelParts({ repo: cwd, parts, verify, maxTokens, attempt: async (p, dir) => { const b = new Budget({ maxTokens: Math.max(1, Math.floor(maxTokens / parts.length)) }); const a = createCodeAgent({ model, cwd: dir, verify, approve, sandbox: cfg.sandbox, memory: false, budget: b }); const x = await a.run(p.task); await a.procs?.dispose(); return { tokens: b.tokens, summary: x.text }; }, onPart: (r) => console.error(`${r.part.name}: ${r.status}${r.detail ? " - " + r.detail.split("\n")[0] : ""}`) });
+            if (json)
+                console.log(JSON.stringify({ kind: "result", tokens, parts: results.map((r) => ({ name: r.part.name, status: r.status })) }));
+            else
+                for (const r of results)
+                    console.log(`${r.status.padEnd(14)} ${r.part.name}`);
+            return results.every((r) => r.status === "merged") ? 0 : 1;
+        }
+        catch (e) {
+            console.error("error: " + e.message);
+            return 1;
+        }
+    }
+    const tracer = val("--trace") || process.env.AMNESS_OTLP ? new Tracer({ file: val("--trace") ? resolve(val("--trace")) : undefined, otlp: process.env.AMNESS_OTLP, sessionId: `run-${Date.now()}`, model: cfg.model }) : undefined;
+    setShell(cfg.shell);
+    const loadedPlugins = await loadPlugins(cwd, { trustProject: trustHooks });
+    for (const w of loadedPlugins.warnings)
+        console.error("plugin: " + w);
+    if (loadedPlugins.untrusted)
+        console.error(`note: ${loadedPlugins.untrusted.count} project plugin file(s) in .amness/plugins were not loaded; pass --trust-project to run them`);
+    const agent = createCodeAgent({
+        model, modelById: (id) => createModel({ ...cfg, model: id }), cwd, verify, plugins: loadedPlugins.plugins.length ? loadedPlugins.plugins : undefined, check, approve, record: !!tracer || undefined, sandbox: val("--sandbox") ?? cfg.sandbox, review: bool("--review") || undefined, approvalReviewer: bool("--approval-reviewer") || undefined, hooks: ((h) => { if (!trustHooks && projectConfigFingerprint(cwd))
+            console.error("note: ignoring the project's hooks and allow rules (pass --trust-project to use them)"); return h.count() ? h : undefined; })(Hooks.load(cwd, { trustProject: trustHooks })), rules: Permissions.load(cwd, { trustProject: trustHooks }), budget, memory: noMemory ? false : undefined, meta: { provider: cfg.provider, model: cfg.model },
+        onEvent: (e) => { live?.onEvent(e); tracer?.event(e); if (json)
+            console.log(JSON.stringify(e));
+        else {
+            const l = formatEvent(e);
+            if (l)
+                console.error(l);
+        } },
+    });
+    try {
+        const r = await agent.run(task, { bundle, images });
+        if (exportTo) {
+            await agent.exportBundle(r, exportTo);
+            console.error(`session bundle: ${exportTo}`);
+        }
+        const out = { kind: "result", finished: r.finished, text: r.text, steps: r.steps, tokens: budget.tokens, stopped: r.stopped };
+        if (json)
+            console.log(JSON.stringify(out));
+        else
+            console.log(r.text);
+        return r.finished ? 0 : 1;
+    }
+    catch (e) {
+        if (json)
+            console.log(JSON.stringify({ kind: "error", message: e.message }));
+        else
+            console.error("error: " + e.message);
+        return 1;
+    }
+    finally {
+        await agent.procs?.dispose();
+        await tracer?.close();
+        await live?.close();
+    }
+}

package/dist/cli/keystore.d.ts

@@ -0,0 +1,9 @@
+export type Backend = "keychain" | "secret-service" | "dpapi" | "file" | "none";
+export declare function backend(): Backend;
+export declare const where: () => string;
+/** True when keys must never be written to disk (containers). */
+export declare const noStore: () => boolean;
+/** Save a secret. Returns false when the OS store is not available or refused, so the caller can fall back to the file. */
+export declare function setSecret(name: string, value: string): boolean;
+export declare function getSecret(name: string): string | undefined;
+export declare function deleteSecret(name: string): void;

package/dist/cli/keystore.js

@@ -0,0 +1,108 @@
+// Where API keys live. macOS: the login Keychain. Linux: the desktop secret service (`secret-tool`). Windows: DPAPI, which encrypts the key so only your Windows user on this machine can decrypt it.
+// If the OS store is missing or refuses (a Linux server or Docker image with no keyring), the key stays in the config file with mode 600, and `where()` says so.
+// The key goes to the OS tool on stdin, never on the command line (a command line is visible to other users in `ps`), and is hex-encoded so no quoting can go wrong.
+// Tests and any run with a custom AMNESS_HOME use the file, so they never touch your real store. AMNESS_KEYSTORE=file|keychain|secret-service|dpapi|none forces one.
+// In a container (Docker, Kubernetes) nothing is saved at all ("none"): pass the key as an environment variable or a secret file (see config.ts), so it is never written into an image or a volume.
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+const SERVICE = process.env.AMNESS_KEYSTORE_SERVICE || "amness"; // the override is for tests
+const run = (cmd, args, input, timeout = 10_000, env) => {
+    try {
+        const r = spawnSync(cmd, args, { input, encoding: "utf8", timeout, env });
+        return r.error ? undefined : r;
+    }
+    catch {
+        return undefined;
+    }
+};
+export function backend() {
+    const want = process.env.AMNESS_KEYSTORE;
+    if (want === "file" || (!want && process.env.AMNESS_HOME))
+        return "file";
+    if (want === "keychain" || want === "secret-service" || want === "dpapi" || want === "none")
+        return want;
+    if (process.platform === "linux" && (existsSync("/.dockerenv") || process.env.KUBERNETES_SERVICE_HOST))
+        return "none";
+    if (process.platform === "darwin")
+        return "keychain";
+    if (process.platform === "win32")
+        return "dpapi";
+    if (process.platform === "linux" && run("secret-tool", ["--version"])?.status !== undefined)
+        return "secret-service";
+    return "file";
+}
+export const where = () => ({ keychain: "saved in the macOS Keychain", "secret-service": "saved in the system keyring (secret-tool)", dpapi: "saved with Windows encryption (DPAPI, this user only)", file: "saved in ~/.amness/config.json, mode 600, not encrypted", none: "not saved: this is a container, so pass the key as an environment variable or a secret file (OPENAI_API_KEY_FILE / ANTHROPIC_API_KEY_FILE)" })[backend()];
+/** True when keys must never be written to disk (containers). */
+export const noStore = () => backend() === "none";
+// Windows: PowerShell turns the key into an encrypted string that only this Windows user on this machine can decrypt (DPAPI). The string is kept in ~/.amness/secrets/<name>.dpapi.
+const dir = () => join(process.env.AMNESS_HOME ?? join(homedir(), ".amness"), "secrets");
+const blob = (name) => join(dir(), `${name}.dpapi`);
+// A parent PowerShell 7 (pwsh) leaves a PSModulePath that Windows PowerShell 5.1 cannot load its Security module from; dropping it lets 5.1 use its own default.
+const psEnv = () => { const e = { ...process.env }; for (const k of Object.keys(e))
+    if (k.toLowerCase() === "psmodulepath")
+        delete e[k]; return e; };
+const ps = (script, input) => run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script], input, 60_000, psEnv()); // Windows PowerShell can take several seconds to start
+const PS_ENCRYPT = "$p = [Console]::In.ReadToEnd().Trim(); ConvertTo-SecureString $p -AsPlainText -Force | ConvertFrom-SecureString";
+const PS_DECRYPT = "$e = [Console]::In.ReadToEnd().Trim(); $s = ConvertTo-SecureString $e; [Runtime.InteropServices.Marshal]::PtrToStringBSTR([Runtime.InteropServices.Marshal]::SecureStringToBSTR($s))";
+const hex = (s) => Buffer.from(s, "utf8").toString("hex");
+const unhex = (s) => Buffer.from(s.trim(), "hex").toString("utf8");
+/** Save a secret. Returns false when the OS store is not available or refused, so the caller can fall back to the file. */
+export function setSecret(name, value) {
+    const b = backend();
+    if (b === "file" || b === "none")
+        return false;
+    if (b === "dpapi") {
+        const r = ps(PS_ENCRYPT, hex(value));
+        if (r?.status !== 0 || !r.stdout.trim())
+            return false;
+        try {
+            mkdirSync(dir(), { recursive: true });
+            writeFileSync(blob(name), r.stdout.trim(), { mode: 0o600 });
+        }
+        catch {
+            return false;
+        }
+        return getSecret(name) === value;
+    }
+    if (b === "keychain") {
+        const r = run("security", ["-i"], `add-generic-password -U -s ${SERVICE} -a ${name} -w ${hex(value)}\n`);
+        return r?.status === 0 && !/error|fail/i.test(r.stderr ?? "") && getSecret(name) === value;
+    }
+    const r = run("secret-tool", ["store", `--label=amness ${name}`, "service", SERVICE, "account", name], hex(value));
+    return r?.status === 0 && getSecret(name) === value;
+}
+export function getSecret(name) {
+    const b = backend();
+    if (b === "file" || b === "none")
+        return undefined;
+    if (b === "dpapi") {
+        if (!existsSync(blob(name)))
+            return undefined;
+        const r = ps(PS_DECRYPT, readFileSync(blob(name), "utf8"));
+        const v = r?.status === 0 ? unhex(r.stdout) : "";
+        return v || undefined;
+    }
+    const r = b === "keychain" ? run("security", ["find-generic-password", "-s", SERVICE, "-a", name, "-w"]) : run("secret-tool", ["lookup", "service", SERVICE, "account", name]);
+    if (r?.status !== 0 || !r.stdout.trim())
+        return undefined;
+    const v = unhex(r.stdout);
+    return v || undefined;
+}
+export function deleteSecret(name) {
+    const b = backend();
+    if (b === "file" || b === "none")
+        return;
+    if (b === "dpapi") {
+        try {
+            rmSync(blob(name), { force: true });
+        }
+        catch { /* already gone */ }
+        return;
+    }
+    if (b === "keychain")
+        run("security", ["delete-generic-password", "-s", SERVICE, "-a", name]);
+    else
+        run("secret-tool", ["clear", "service", SERVICE, "account", name]);
+}

package/dist/cli/model.d.ts

@@ -0,0 +1,11 @@
+import { type Model } from "../src/index.ts";
+import { type Config } from "./config.ts";
+/** One place for the UI to hear about rate limits, whichever model object hit them. */
+declare let waitListener: ((ms: number, attempt: number, concurrency: number) => void) | undefined;
+export declare const onRateLimitWait: (fn: typeof waitListener) => void;
+/** Build the model for a config (shared by the menu and headless mode). */
+export declare function createModel(cfg: Config, retries?: number, o?: {
+    cacheKey?: string; /** told when a rate limit makes calls wait */
+    onWait?: (ms: number, attempt: number, concurrency: number) => void;
+}): Model;
+export {};

package/dist/cli/model.js

@@ -0,0 +1,19 @@
+import { anthropic, effortsFor, gateFor, openai, rateLimited } from "../src/index.js";
+import { PROVIDERS } from "./config.js";
+/** One place for the UI to hear about rate limits, whichever model object hit them. */
+let waitListener;
+export const onRateLimitWait = (fn) => { waitListener = fn; };
+/** Build the model for a config (shared by the menu and headless mode). */
+export function createModel(cfg, retries, o = {}) {
+    // Every model built for the same endpoint and key shares one gate, so parallel subagents slow down together when the provider says 429.
+    const gate = gateFor(`${cfg.provider}|${cfg.baseUrl}|${(cfg.apiKey ?? "").slice(-6)}`, cfg.maxConcurrent ?? 4);
+    return rateLimited(rawModel(cfg, retries, o), { gate, onWait: (ms, attempt, g) => { o.onWait?.(ms, attempt, g.max); waitListener?.(ms, attempt, g.max); } });
+}
+function rawModel(cfg, retries, o) {
+    return cfg.provider === "anthropic"
+        ? anthropic({ model: cfg.model, apiKey: cfg.apiKey, taskBudget: cfg.taskBudget, effort: cfg.effort && !["none", "minimal"].includes(cfg.effort) ? cfg.effort : undefined, baseUrl: cfg.baseUrl === PROVIDERS.anthropic.baseUrl ? undefined : cfg.baseUrl })
+        : openai({
+            model: cfg.model, apiKey: cfg.apiKey, baseUrl: cfg.baseUrl, retries, api: cfg.api, promptCacheKey: o.cacheKey, compactAt: cfg.compactAt, externalRetry: true,
+            effort: cfg.effort && effortsFor(cfg.model).includes(cfg.effort) ? cfg.effort : undefined, // a value this model does not accept is never sent
+        });
+}

package/dist/cli/plugincli.d.ts

@@ -0,0 +1,2 @@
+export declare const PLUGIN_HELP = "usage: amness plugin <command>\n  new <name>     create .amness/plugins/<name>.ts from a template (never overwrites)\n  policy <traces...> --plugin <file>   replay the tool calls recorded in traces (a file, or a folder such as .amness/traces) through the plugin policies and show what each would allow, ask or deny\n  grade <traces...> [--plugin <file>] [--grader a,b]   score real recorded sessions (a trace file, or a folder such as .amness/traces) with built-in or plugin graders\n  add <file|https-url>   copy a plugin file into .amness/plugins/ (never overwrites) and show what it appears to touch; it still needs your approval the first time you start Amness\n  list           show the plugins that load here (project plugins need --trust-project to be listed as loaded)\nA plugin is a TypeScript file that runs inside Amness: guardrails, loop rules, tools, slash commands and eval variants. See docs/plugins.md.";
+export declare function runPlugin(argv: string[], cwd?: string): Promise<number>;