amness 0.0.0-stage → 0.1.2

package/dist/cli/benchcli.d.ts

@@ -0,0 +1,5 @@
+export declare const BENCH_HELP: string;
+export declare function runBench(argv: string[], deps: {
+    confirm: (q: string) => Promise<boolean>;
+    amnessCli: string;
+}): Promise<number>;

package/dist/cli/benchcli.js

@@ -0,0 +1,160 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+// `amness bench`: a bake-off between coding-agent CLIs on the same tasks, graded by the same hidden checks.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { PRESETS, adversarialCases, analyze, bench, builtinCases, configFileTemplate, describePlan, formatReport, hardCases, installed, loadCases, loadSuite, suiteToCases } from "../src/index.js";
+import { bold, dim, green, red, yellow } from "./ui.js";
+export const BENCH_HELP = `usage: amness bench [flags]
+  Run the same tasks through several coding-agent command-line tools (Amness, Claude Code, Codex, or any command) and grade every result with the same hidden check.
+
+  --init [file]          write a starter config (default bench.json) with Amness, Amness with a finish gate, Claude Code and Codex, then edit it
+  --config <file>        the contestants and settings (see --init). Without it, --contestants picks built-in presets
+  --contestants <a,b>    presets: ${Object.keys(PRESETS).join(", ")}   (default: amness,claude,codex)
+  --model <name>         one model name for every contestant that takes one; or --models amness=gpt-6-luna,claude=claude-sonnet-5-5,codex=gpt-6-luna
+  --builtin <name>       code (default, 13 easy tasks), hard (6 harder tasks) or adversarial
+  --cases <dir|file>     your own tasks: a folder of *.json, or a .ts/.js module exporting a list of cases
+  --suite <file.jsonl>   a SWE-bench-format file; needs --test-cmd (see amness eval --help)   --limit <n>   --install <cmd>
+  --only <a,b>           only these tasks
+  --trials <n>           attempts per task and contestant (default 3)   --jobs <n>   runs at once (default 1; the tools use your API limits)
+  --timeout <seconds>    per run (default 900)
+  --check                see which of the tools are installed (no model call)
+  --probe                one tiny real task per contestant, to confirm the tool starts, is graded, and its tokens and cost are read correctly (costs a few cents)
+  --dry-run              show exactly what would run, change nothing
+  --out <file>           where to save the raw results (re-read with: amness eval --report <file>)
+  --yes                  skip the cost confirmation
+Safety: each run happens in a throw-away temp folder, but the tools run with their file and shell tools enabled and are NOT sandboxed from the rest of your machine. Run benchmarks in a VM or container if the tasks are not yours.`;
+async function pickCases(val, has) {
+    let cases;
+    const arg = val("--cases"), suite = val("--suite");
+    if (suite) {
+        if (!val("--test-cmd"))
+            throw new Error('a suite needs --test-cmd, for example --test-cmd "python -m pytest -x -q {tests}"');
+        cases = suiteToCases(loadSuite(resolve(suite)).slice(0, Number(val("--limit") ?? 1e9)), { testCmd: val("--test-cmd"), installCmd: val("--install") });
+    }
+    else if (arg && /\.(m?[jt]s)$/.test(arg)) {
+        const mod = await import(__rewriteRelativeImportExtension(pathToFileURL(resolve(arg)).href + `?v=${Date.now()}`));
+        cases = mod.default ?? mod.cases;
+        if (!Array.isArray(cases))
+            throw new Error(`${arg} must export a list of cases`);
+    }
+    else if (arg)
+        cases = loadCases(resolve(arg));
+    else {
+        const b = val("--builtin") ?? "code";
+        if (!["code", "hard", "adversarial"].includes(b))
+            throw new Error(`unknown built-in suite "${b}"`);
+        cases = b === "hard" ? hardCases : b === "adversarial" ? adversarialCases : builtinCases;
+    }
+    if (val("--only")) {
+        const want = val("--only").split(",");
+        cases = cases.filter((c) => want.includes(c.name));
+        if (!cases.length)
+            throw new Error(`no task matches --only ${val("--only")}`);
+    }
+    void has;
+    return cases;
+}
+export async function runBench(argv, deps) {
+    const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; }, has = (f) => argv.includes(f);
+    if (has("--help") || has("-h")) {
+        console.log(BENCH_HELP);
+        return 0;
+    }
+    try {
+        if (has("--init")) {
+            const i = argv.indexOf("--init"), file = resolve(argv[i + 1] && !argv[i + 1].startsWith("--") ? argv[i + 1] : "bench.json");
+            if (existsSync(file)) {
+                console.error(red("✗ ") + `${file} already exists`);
+                return 1;
+            }
+            writeFileSync(file, JSON.stringify(configFileTemplate(deps.amnessCli), null, 2) + "\n");
+            console.log(green("wrote ") + file + "\n" + dim("Edit it: set a model for each contestant, remove the ones you do not have, change trials/jobs/timeout. Then: amness bench --config " + file + " --dry-run"));
+            return 0;
+        }
+        let cfg;
+        if (val("--config"))
+            cfg = JSON.parse(readFileSync(resolve(val("--config")), "utf8"));
+        else {
+            const models = Object.fromEntries((val("--models") ?? "").split(",").filter(Boolean).map((p) => p.split("=")));
+            const names = (val("--contestants") ?? "amness,claude,codex").split(",").map((s) => s.trim()).filter(Boolean);
+            cfg = { contestants: names.map((n) => { if (!PRESETS[n])
+                    throw new Error(`unknown preset "${n}". Presets: ${Object.keys(PRESETS).join(", ")}. For anything else use --config.`); return PRESETS[n]({ model: models[n] ?? val("--model"), amnessCli: deps.amnessCli }); }) };
+        }
+        if (!cfg.contestants?.length)
+            throw new Error("no contestants");
+        for (const k of cfg.contestants) {
+            if (!k.name || !k.command)
+                throw new Error("every contestant needs a name and a command");
+            if (k.args === undefined)
+                k.args = [];
+        }
+        if (new Set(cfg.contestants.map((k) => k.name)).size !== cfg.contestants.length)
+            throw new Error("contestant names must be different");
+        if (val("--trials"))
+            cfg.trials = Number(val("--trials"));
+        if (val("--jobs"))
+            cfg.jobs = Number(val("--jobs"));
+        if (val("--timeout"))
+            cfg.timeoutSec = Number(val("--timeout"));
+        cfg.trials ??= 3;
+        cfg.jobs ??= 1;
+        cfg.logDir ??= join(".amness", "bench", "logs");
+        if (has("--check")) {
+            let bad = 0;
+            for (const k of cfg.contestants) {
+                const ok = installed(k.command);
+                if (!ok)
+                    bad++;
+                console.log(`${ok ? green("found   ") : red("missing ")} ${k.name.padEnd(14)} ${k.command}${ok ? "" : dim("  (not on PATH)")}`);
+            }
+            return bad ? 1 : 0;
+        }
+        const cases = has("--probe") ? [{ name: "probe", files: {}, task: "Create a file named hello.txt that contains exactly: hi", verify: "node -e \"process.exit(require('fs').readFileSync('hello.txt','utf8').trim()==='hi'?0:1)\"", check: async (d) => { try {
+                    return readFileSync(join(d, "hello.txt"), "utf8").trim() === "hi" || "hello.txt is wrong";
+                }
+                catch {
+                    return "no hello.txt";
+                } } }] : await pickCases(val, has);
+        if (has("--probe")) {
+            cfg.trials = 1;
+            cfg.jobs = 1;
+            cfg.timeoutSec = Math.min(cfg.timeoutSec ?? 300, 300);
+        }
+        for (const l of describePlan(cases, cfg))
+            console.log(l);
+        const missing = cfg.contestants.filter((k) => !installed(k.command));
+        for (const k of missing)
+            console.log(yellow("! ") + `${k.name}: "${k.command}" is not on PATH`);
+        if (has("--dry-run"))
+            return 0;
+        if (missing.length === cfg.contestants.length) {
+            console.error(red("✗ ") + "none of the contestants can start");
+            return 1;
+        }
+        const runs = cases.length * cfg.contestants.length * cfg.trials;
+        if (!has("--yes") && !(await deps.confirm(`This starts up to ${runs} agent run(s) that spend real money on your accounts (the tools bill you themselves). Continue?`)))
+            return 1;
+        mkdirSync(join(".amness", "bench"), { recursive: true });
+        const { rows, analysis } = await bench(cases, cfg, { onRow: (r) => console.error(`  ${r.pass ? green("PASS") : red("FAIL")} ${r.case.padEnd(24)} ${r.variant.padEnd(14)} ${dim(`${(r.ms / 1000).toFixed(0)}s${r.tokens ? ` · ${r.tokens} tok` : ""}${r.metrics?.["cost-usd"] !== undefined ? ` · $${r.metrics["cost-usd"].toFixed(3)}` : ""}${r.note ? " · " + r.note : ""}`)}`) });
+        const out = resolve(val("--out") ?? join(".amness", "bench", new Date().toISOString().replace(/[:.]/g, "-") + ".json"));
+        mkdirSync(join(out, ".."), { recursive: true });
+        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), baseline: analysis.baseline, k: Math.min(3, cfg.trials), contestants: cfg.contestants, trials: cfg.trials, rows }, null, 1));
+        console.log("\n" + formatReport(analyze(rows, { k: Math.min(3, cfg.trials) })));
+        console.log("\n" + bold("what each contestant was") + "\n" + cfg.contestants.map((k) => `  ${k.name}: ${k.model ? "model " + k.model : "default model"}${k.note ? "; " + k.note : ""}`).join("\n"));
+        console.log("\n" + dim(`raw results: ${out}  (amness eval --report ${out}). Raw output of every run: ${cfg.logDir}`));
+        console.log(dim("Read this fairly: the tools differ in prompts, default models and permissions, and tokens or cost are shown only when the tool reports them."));
+        return 0;
+    }
+    catch (e) {
+        console.error(red("✗ ") + e.message);
+        return 2;
+    }
+}

package/dist/cli/config.d.ts

@@ -0,0 +1,79 @@
+export declare const PROVIDERS: {
+    readonly anthropic: {
+        readonly label: "Anthropic (Claude)";
+        readonly baseUrl: "https://api.anthropic.com";
+        readonly model: "claude-opus-5-5";
+        readonly needsKey: true;
+    };
+    readonly openai: {
+        readonly label: "OpenAI";
+        readonly baseUrl: "https://api.openai.com/v1";
+        readonly model: "gpt-6.1-sol";
+        readonly needsKey: true;
+    };
+    readonly openrouter: {
+        readonly label: "OpenRouter";
+        readonly baseUrl: "https://openrouter.ai/api/v1";
+        readonly model: "openai/gpt-4.1";
+        readonly needsKey: true;
+    };
+    readonly ollama: {
+        readonly label: "Ollama (local)";
+        readonly baseUrl: "http://localhost:11434/v1";
+        readonly model: "qwen2.5-coder:7b";
+        readonly needsKey: false;
+    };
+    readonly custom: {
+        readonly label: "Custom OpenAI-compatible";
+        readonly baseUrl: "http://localhost:8000/v1";
+        readonly model: "my-model";
+        readonly needsKey: false;
+    };
+};
+export type ProviderId = keyof typeof PROVIDERS;
+export type Config = {
+    provider: ProviderId;
+    baseUrl: string;
+    model: string;
+    apiKey?: string;
+    tavilyKey?: string;
+    maxTokens: number;
+    effort?: "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; /** OpenAI wire API: auto (default) picks Responses for reasoning models and self-corrects */
+    api?: "auto" | "responses" | "chat"; /** full-screen colour theme (see /theme) */
+    theme?: string; /** which UI to start in */
+    ui?: "full" | "inline"; /** while idle, re-send the conversation now and then so the provider keeps its prompt cache (Claude: the cache expires after 5 minutes) */
+    keepWarm?: boolean; /** semantic memory retrieval via an OpenAI-compatible embeddings endpoint (sends note text to that provider) */
+    embeddings?: boolean; /** which shell runs the agent's commands: auto (default), bash, zsh, sh, pwsh, powershell or cmd (see /shell) */
+    shell?: string; /** a system prompt id from .amness/prompts (see /prompt) */
+    prompt?: string; /** a loop strategy: plan-first or explore-then-act (see /strategy) */
+    strategy?: string; /** most model requests in flight at once, shared by the lead agent and its subagents (default 4); halves automatically on a 429 and ramps back up */
+    maxConcurrent?: number; /** OpenAI Responses only: server-side compaction once the prompt passes this many tokens */
+    compactAt?: number; /** Claude only: advisory token budget per turn (task_budget, beta; at least 20000) */
+    taskBudget?: number; /** a second model reviews risky calls before you are asked (extra model call per risky action) */
+    approvalReviewer?: boolean; /** errors from an installed language server are appended to edit results */
+    lsp?: boolean; /** an independent reviewer checks finished work against acceptance criteria (extra model calls) */
+    review?: boolean; /** OS sandbox for shell commands */
+    sandbox?: "off" | "workspace" | "strict"; /** model ids (same provider) for other roles; unset roles use `model` */
+    models?: {
+        explore?: string;
+        compact?: string;
+        review?: string;
+        cheap?: string;
+    }; /** route exploration steps to models.cheap when that pays off (cache-aware) */
+    router?: boolean; /** model id to answer from when the main one is overloaded or unreachable */
+    fallback?: string;
+};
+export declare const home: () => string;
+export declare const defaults: () => Config;
+export declare function loadConfig(): Config;
+export declare function saveConfig(c: Config): void;
+export declare const ready: (c: Config) => boolean;
+export declare const mask: (k?: string) => string;
+export declare const CLAUDE_MODELS: {
+    id: string;
+    hint: string;
+}[];
+export declare const OPENAI_MODELS: {
+    id: string;
+    hint: string;
+}[];

package/dist/cli/config.js

@@ -0,0 +1,111 @@
+import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { deleteSecret, getSecret, noStore, setSecret } from "./keystore.js";
+export const PROVIDERS = {
+    anthropic: { label: "Anthropic (Claude)", baseUrl: "https://api.anthropic.com", model: "claude-opus-5-5", needsKey: true },
+    openai: { label: "OpenAI", baseUrl: "https://api.openai.com/v1", model: "gpt-6.1-sol", needsKey: true },
+    openrouter: { label: "OpenRouter", baseUrl: "https://openrouter.ai/api/v1", model: "openai/gpt-4.1", needsKey: true },
+    ollama: { label: "Ollama (local)", baseUrl: "http://localhost:11434/v1", model: "qwen2.5-coder:7b", needsKey: false },
+    custom: { label: "Custom OpenAI-compatible", baseUrl: "http://localhost:8000/v1", model: "my-model", needsKey: false },
+};
+export const home = () => process.env.AMNESS_HOME ?? join(homedir(), ".amness");
+const file = () => join(home(), "config.json");
+export const defaults = () => ({ provider: "openai", baseUrl: PROVIDERS.openai.baseUrl, model: PROVIDERS.openai.model, maxTokens: 300_000 });
+const SECRETS = ["apiKey", "tavilyKey"];
+/** An environment variable, or the file NAME_FILE points to (Docker and Kubernetes secrets are mounted as files, for example /run/secrets/openai). */
+const envSecret = (name) => {
+    const v = process.env[name];
+    if (v)
+        return v;
+    const f = process.env[`${name}_FILE`];
+    if (!f)
+        return undefined;
+    try {
+        return readFileSync(f, "utf8").trim() || undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
+let fromEnv = {};
+export function loadConfig() {
+    let c = defaults(), onDisk = {};
+    try {
+        if (existsSync(file()))
+            onDisk = JSON.parse(readFileSync(file(), "utf8"));
+        c = { ...c, ...onDisk };
+    }
+    catch { /* corrupt config: start fresh */ }
+    // keys live in the OS credential store when there is one; a key still in the file (an older version wrote it there) is moved over, and removed from the file only after it reads back
+    let migrated = false;
+    for (const k of SECRETS) {
+        const stored = getSecret(k);
+        if (stored) {
+            c[k] = stored;
+            if (onDisk[k]) {
+                delete onDisk[k];
+                migrated = true;
+            }
+        }
+        else if (onDisk[k] && setSecret(k, onDisk[k])) {
+            delete onDisk[k];
+            migrated = true;
+        }
+    }
+    if (migrated)
+        writeConfigFile({ ...defaults(), ...onDisk, ...Object.fromEntries(SECRETS.map((k) => [k, undefined])) });
+    // environment wins, so CI and one-off runs work without touching the saved key. It is never written back (see saveConfig).
+    const claude = c.provider === "anthropic";
+    fromEnv = {};
+    const key = envSecret(claude ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY");
+    if (key)
+        c.apiKey = fromEnv.apiKey = key;
+    const tavily = envSecret("TAVILY_API_KEY");
+    if (tavily)
+        c.tavilyKey = fromEnv.tavilyKey = tavily;
+    if (process.env.OPENAI_BASE_URL && !claude)
+        c.baseUrl = process.env.OPENAI_BASE_URL;
+    if (process.env.AMNESS_MODEL)
+        c.model = process.env.AMNESS_MODEL;
+    return c;
+}
+function writeConfigFile(c) {
+    mkdirSync(home(), { recursive: true });
+    writeFileSync(file(), JSON.stringify(c, null, 2), { mode: 0o600 }); // owner-only from the first byte
+    chmodSync(file(), 0o600); // and for a file that already existed
+}
+export function saveConfig(c) {
+    const out = { ...c };
+    for (const k of SECRETS) {
+        const v = c[k];
+        if (v && v === fromEnv[k]) {
+            delete out[k];
+            continue;
+        } // this value came from an environment variable: do not store it
+        if (!v) {
+            deleteSecret(k);
+            delete out[k];
+            continue;
+        }
+        if (setSecret(k, v) || noStore())
+            delete out[k]; // stored in the OS credential store; otherwise it stays in the file (mode 600), except in a container, where it is kept in memory only
+    }
+    writeConfigFile(out);
+}
+export const ready = (c) => !PROVIDERS[c.provider].needsKey || !!c.apiKey;
+export const mask = (k) => (k ? k.slice(0, 3) + "…" + k.slice(-4) : "not set");
+export const CLAUDE_MODELS = [
+    { id: "claude-opus-5-5", hint: "most capable everyday model · $4/$20 per M tokens (default)" },
+    { id: "claude-sonnet-5-5", hint: "fast and cheaper · $2/$10" },
+    { id: "claude-haiku-5-5", hint: "cheapest · $0.10/$0.50 · no refusal fallback" },
+    { id: "claude-fable-5-1", hint: "hardest tasks · $10/$50" },
+];
+export const OPENAI_MODELS = [
+    { id: "gpt-6.1-sol", hint: "near-flagship for complex work · $2/$10 per M tokens (default)" },
+    { id: "gpt-6-astra", hint: "flagship, hardest tasks · $10/$50" },
+    { id: "gpt-6.1-terra", hint: "fast + efficient, computer use · $1/$5" },
+    { id: "gpt-6-luna", hint: "cheap, high-volume · $0.10/$0.50" },
+    { id: "gpt-5.6-luna", hint: "previous generation" },
+    { id: "gpt-4.1", hint: "non-reasoning · Chat Completions" },
+];

package/dist/cli/demo.d.ts

@@ -0,0 +1,13 @@
+import { type Approve, type Event, type Model } from "../src/index.ts";
+export declare const DEMO_TASK = "sumTo(n) should return 1+2+...+n but the tests fail. Fix it.";
+export declare function runDemo(o: {
+    approve: Approve;
+    onEvent: (e: Event) => void;
+    delayMs?: number;
+    model?: (m: Model) => Model;
+}): Promise<{
+    dir: string;
+    result: import("../src/harness.ts").RunResult;
+    events: Event[];
+}>;
+export declare const explain: Record<string, string>;

package/dist/cli/demo.js

@@ -0,0 +1,44 @@
+// Guided demo: a scripted "model" that makes realistic mistakes so every harness guard fires, with no API key.
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createCodeAgent } from "../src/index.js";
+let id = 0;
+const call = (name, args, note) => ({
+    role: "assistant", content: note ?? null, usage: { total_tokens: 420 },
+    tool_calls: [{ id: `demo${++id}`, function: { name, arguments: JSON.stringify(args) } }],
+});
+export const DEMO_TASK = "sumTo(n) should return 1+2+...+n but the tests fail. Fix it.";
+export async function runDemo(o) {
+    const dir = mkdtempSync(join(tmpdir(), "amness-demo-"));
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, "sum.js"), "exports.sumTo = (n) => { let s = 0; for (let i = 1; i < n; i++) s += i; return s; };\n");
+    writeFileSync(join(dir, "test.js"), "const a = require('assert'); const { sumTo } = require('./sum'); a.equal(sumTo(3), 6); a.equal(sumTo(1), 1); console.log('tests passed');\n");
+    const script = [
+        { role: "assistant", content: "I'll start by looking at the code and the failing test.", usage: { total_tokens: 300 } }, // intent, no action → nudge
+        call("read_file", {}), // missing arg → repair
+        call("edit_file", { path: "sum.js", old: "i < n", new: "i <= n" }), // edit before read → blocked
+        call("list_files", {}),
+        call("read_file", { path: "sum.js" }),
+        call("sh", { cmd: "rm -rf node_modules && git clean -fdx" }), // dangerous → approval prompt
+        call("finish", { summary: "Looks fine to me." }), // lies → verify gate runs tests → rejected
+        call("edit_file", { path: "sum.js", old: "i < n", new: "i <= n" }),
+        call("sh", { cmd: "node test.js" }),
+        call("finish", { summary: "Fixed off-by-one in sumTo (i < n → i <= n); test.js passes." }),
+    ];
+    const scripted = async () => { await new Promise((r) => setTimeout(r, o.delayMs ?? 500)); return script.shift(); };
+    const events = [];
+    const agent = createCodeAgent({
+        model: o.model ? o.model(scripted) : scripted, cwd: dir, verify: "node test.js", approve: o.approve,
+        onEvent: (e) => (events.push(e), o.onEvent(e)),
+    });
+    const result = await agent.run(DEMO_TASK);
+    return { dir, result, events };
+}
+export const explain = {
+    nudge: "model said what it would do but did nothing → pushed back to act",
+    repair: "malformed tool call → exact error returned, model self-corrected",
+    denied: "dangerous command → human approval required",
+    rejected: "model claimed success → real test run disagreed, finish blocked",
+    tool: "tool calls executed",
+};

package/dist/cli/detect.d.ts

@@ -0,0 +1,2 @@
+/** Best-guess command that proves the project works (used as the finish gate). */
+export declare function detectVerify(dir: string): string;

package/dist/cli/detect.js

@@ -0,0 +1,20 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+/** Best-guess command that proves the project works (used as the finish gate). */
+export function detectVerify(dir) {
+    try {
+        if (existsSync(join(dir, "package.json"))) {
+            const t = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).scripts?.test;
+            if (t && !/no test specified/.test(t))
+                return "npm test";
+        }
+    }
+    catch { /* ignore */ }
+    if (existsSync(join(dir, "Cargo.toml")))
+        return "cargo test";
+    if (existsSync(join(dir, "go.mod")))
+        return "go test ./...";
+    if (["pytest.ini", "pyproject.toml", "setup.py", "tox.ini"].some((f) => existsSync(join(dir, f))))
+        return "pytest -q";
+    return "";
+}

package/dist/cli/evalcli.d.ts

@@ -0,0 +1,15 @@
+import { type Model } from "../src/index.ts";
+export declare const EVAL_HELP: string;
+export declare function runEval(argv: string[], deps: {
+    model: () => Model;
+    modelFor?: (id: string) => Model;
+    maxTokens: number;
+    confirm: (q: string) => Promise<boolean>;
+}): Promise<number>;
+export declare const TUNE_HELP = "usage: amness tune [flags]\n  Try switching guards off (and changing a limit), one at a time, on part of your tasks; keep a change only if it also holds up on tasks the search never saw. Rolled back otherwise.\n  Writes nothing unless something survives both checks. Calls your model: it asks first.\n\n  --cases <dir>        your own tasks (default: the 13 built-in tasks)\n  --guards <a,b>       which guards to try removing (default: nudge, loop, stuck, wrapup, condense, stagnation)\n  --stagnation <n,n>   also try these stagnationTurns values\n  --trials <n>         attempts per task (default 2)\n  --max-steps <n>      steps per session (default 30)\n  --max-tokens <n>     token cap per session\n  --jobs <n>           sessions at once (default 1)\n  --seed <n>           seed for the split and the intervals\n  --profile-model <p>  the model name or pattern to write the profile for (default: the configured model)\n  --out <file>         where to write the profile (default .amness/profile.json)\n  --yes                skip the cost confirmation";
+export declare function runTune(argv: string[], deps: {
+    model: () => Model;
+    modelName: string;
+    maxTokens: number;
+    confirm: (q: string) => Promise<boolean>;
+}): Promise<number>;