amness 0.1.2 → 0.1.4

package/README.md

@@ -28,13 +28,13 @@ Give Amness the key once, in whichever way suits you:
 
 | How | Command or step | Saved? |
 |---|---|---|
-| **Paste it on first start** | run `amness`, choose a provider, paste the key (hidden). Change it later in the settings or with `/provider`. | Yes, in your system's secure store: the macOS Keychain, the Linux keyring, or Windows DPAPI. The prompt tells you which. |
+| **Paste it on first start** | run `amness`, choose a provider, paste the key (hidden). Add more providers any time with `/connect`, and switch with `/provider`. | Yes, in your system's secure store: the macOS Keychain, the Linux keyring, or Windows DPAPI. The prompt tells you which. |
 | **Environment variable** | `export ANTHROPIC_API_KEY=sk-ant-...` or `export OPENAI_API_KEY=sk-...` (PowerShell: `$env:OPENAI_API_KEY="sk-..."`) | No. Used for that shell only, and never written to disk. It wins over a saved key. |
 | **Secret file** (Docker, Kubernetes) | `ANTHROPIC_API_KEY_FILE=/run/secrets/anthropic` (also `OPENAI_API_KEY_FILE`, `TAVILY_API_KEY_FILE`) | No. Read from the file each run. |
 | **Docker** | `docker run -it --rm -v "$PWD":/work -e ANTHROPIC_API_KEY amness` | No. Inside a container nothing is saved. |
 | **Library or CI** | pass `apiKey` in code, or set the variable as a CI secret | No. |
 
-`OPENAI_API_KEY` is used for every provider except Anthropic, including OpenRouter. To remove a saved key, clear "API key" in the settings. A Linux server with no keyring falls back to `~/.amness/config.json` (mode 600, **not encrypted**), so use an environment variable or your platform's secret manager there. Details: [Safety](docs/safety.md).
+Each provider keeps its own saved key, so you can connect several and switch with `/provider`; `/connect` shows which are connected. `OPENAI_API_KEY` is used for every provider except Anthropic (OpenRouter prefers `OPENROUTER_API_KEY`). To remove a saved key, clear "API key" in the settings. A Linux server with no keyring falls back to `~/.amness/config.json` (mode 600, **not encrypted**), so use an environment variable or your platform's secret manager there. Details: [Safety](docs/safety.md).
 
 More ways to start:
 

package/dist/cli/amness.js

@@ -15,7 +15,7 @@ import { runPlugin } from "./plugincli.js";
 import { runSchedule } from "./schedulecli.js";
 import { ACP_HELP, runAcp } from "./acpcli.js";
 import { runEval, runTune } from "./evalcli.js";
-import { CLAUDE_MODELS, OPENAI_MODELS, PROVIDERS, home, loadConfig, mask, ready, saveConfig } from "./config.js";
+import { CLAUDE_MODELS, OPENAI_MODELS, PROVIDERS, home, keyFor, loadConfig, mask, ready, saveConfig } from "./config.js";
 import { where } from "./keystore.js";
 import { DEMO_TASK, explain, runDemo } from "./demo.js";
 import { formatEvent } from "./format.js";
@@ -91,7 +91,7 @@ async function ensureReady() {
         cfg.model = (await ask("Model", cfg.model)) || cfg.model;
     }
     if (PROVIDERS[id].needsKey)
-        cfg.apiKey = (await ask(`API key (hidden; ${where()})`, "", { secret: true })) || undefined;
+        cfg.apiKey = keyFor(id).key ?? ((await ask(`API key (hidden; ${where()})`, "", { secret: true })) || undefined);
     saveConfig(cfg);
     if (!ready(cfg))
         return false;
@@ -432,7 +432,7 @@ async function settings() {
             if (what === "provider") {
                 const id = await select("Provider", Object.keys(PROVIDERS).map((k) => ({ label: PROVIDERS[k].label, value: k, hint: PROVIDERS[k].baseUrl })));
                 if (id)
-                    Object.assign(cfg, { provider: id, baseUrl: PROVIDERS[id].baseUrl, model: PROVIDERS[id].model });
+                    Object.assign(cfg, { provider: id, baseUrl: PROVIDERS[id].baseUrl, model: PROVIDERS[id].model, apiKey: keyFor(id).key });
             }
             else if (what === "model") {
                 if (cfg.provider === "anthropic" || cfg.provider === "openai") {
@@ -610,7 +610,7 @@ async function main() {
     if (quick === "--help" || quick === "-h" || quick === "help")
         return void console.log(TOP_HELP);
     if (quick === "eval" && argv.length > 1) {
-        process.exitCode = await runEval(argv.slice(1), { model: makeModel, modelFor: (id) => createModel({ ...cfg, model: id }), maxTokens: cfg.maxTokens, confirm });
+        process.exitCode = await runEval(argv.slice(1), { model: makeModel, modelName: cfg.model, modelFor: (id) => createModel({ ...cfg, model: id }), maxTokens: cfg.maxTokens, confirm });
         return;
     }
     if (quick === "bench") {

package/dist/cli/config.d.ts

@@ -35,7 +35,8 @@ export type Config = {
     provider: ProviderId;
     baseUrl: string;
     model: string;
-    apiKey?: string;
+    apiKey?: string; /** saved keys for the other providers, only where there is no OS store */
+    apiKeys?: Partial<Record<ProviderId, string>>;
     tavilyKey?: string;
     maxTokens: number;
     effort?: "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; /** OpenAI wire API: auto (default) picks Responses for reasoning models and self-corrects */
@@ -65,6 +66,26 @@ export type Config = {
 };
 export declare const home: () => string;
 export declare const defaults: () => Config;
+/** The variable that holds a provider's key. Anthropic has its own; OpenRouter may have its own; every other provider uses OPENAI_API_KEY. */
+export declare const envNameFor: (p: ProviderId) => "ANTHROPIC_API_KEY" | "OPENROUTER_API_KEY" | "OPENAI_API_KEY";
+export declare const envKeyFor: (p: ProviderId) => string | undefined;
+export type KeyState = {
+    key?: string;
+    source: "env" | "saved" | "none";
+};
+/** The key to use for a provider: an environment variable wins over a saved key. */
+export declare function keyFor(p: ProviderId): KeyState;
+/** Save (or, with no key, remove) a provider's key. Uses the OS store; else config.json (mode 600); in a container, memory only. */
+export declare function storeKey(p: ProviderId, key?: string): void;
+export type Connection = {
+    id: ProviderId;
+    label: string;
+    needsKey: boolean;
+    source: KeyState["source"];
+    env: string;
+};
+/** Every provider and whether it is ready to use. */
+export declare function connections(): Connection[];
 export declare function loadConfig(): Config;
 export declare function saveConfig(c: Config): void;
 export declare const ready: (c: Config) => boolean;

package/dist/cli/config.js

@@ -12,7 +12,11 @@ export const PROVIDERS = {
 export const home = () => process.env.AMNESS_HOME ?? join(homedir(), ".amness");
 const file = () => join(home(), "config.json");
 export const defaults = () => ({ provider: "openai", baseUrl: PROVIDERS.openai.baseUrl, model: PROVIDERS.openai.model, maxTokens: 300_000 });
-const SECRETS = ["apiKey", "tavilyKey"];
+// ---- keys. One saved key per provider (account "apiKey-<provider>" in the OS store), so you can be connected to several at once. `cfg.apiKey` is always the active provider's key.
+const acct = (p) => `apiKey-${p}`;
+let fileKeys = {}, fileKeysLoaded = false; // only where there is no OS store (a Linux server): kept in config.json, mode 600
+const sessionKeys = {}; // in a container: this run only
+const persisted = {}; // what is stored now, so a save does not rewrite an unchanged key
 /** An environment variable, or the file NAME_FILE points to (Docker and Kubernetes secrets are mounted as files, for example /run/secrets/openai). */
 const envSecret = (name) => {
     const v = process.env[name];
@@ -28,7 +32,58 @@ const envSecret = (name) => {
         return undefined;
     }
 };
-let fromEnv = {};
+/** The variable that holds a provider's key. Anthropic has its own; OpenRouter may have its own; every other provider uses OPENAI_API_KEY. */
+export const envNameFor = (p) => (p === "anthropic" ? "ANTHROPIC_API_KEY" : p === "openrouter" && envSecret("OPENROUTER_API_KEY") ? "OPENROUTER_API_KEY" : "OPENAI_API_KEY");
+export const envKeyFor = (p) => envSecret(envNameFor(p));
+const loadFileKeys = () => { if (!fileKeysLoaded) {
+    fileKeysLoaded = true;
+    fileKeys = { ...readFileCfg().apiKeys };
+} };
+const savedKey = (p) => { loadFileKeys(); return getSecret(acct(p)) ?? fileKeys[p] ?? sessionKeys[p]; };
+/** The key to use for a provider: an environment variable wins over a saved key. */
+export function keyFor(p) {
+    const e = envKeyFor(p);
+    if (e)
+        return { key: e, source: "env" };
+    const s = savedKey(p);
+    if (s) {
+        persisted[p] = s;
+        return { key: s, source: "saved" };
+    }
+    return { source: "none" };
+}
+/** Save (or, with no key, remove) a provider's key. Uses the OS store; else config.json (mode 600); in a container, memory only. */
+export function storeKey(p, key) {
+    loadFileKeys();
+    persisted[p] = key;
+    if (!key) {
+        deleteSecret(acct(p));
+        delete fileKeys[p];
+        delete sessionKeys[p];
+        writeConfigFile(readFileCfg());
+        return;
+    }
+    if (setSecret(acct(p), key)) {
+        delete fileKeys[p];
+        delete sessionKeys[p];
+    }
+    else if (noStore())
+        sessionKeys[p] = key;
+    else
+        fileKeys[p] = key;
+    writeConfigFile(readFileCfg());
+}
+/** Every provider and whether it is ready to use. */
+export function connections() {
+    return Object.keys(PROVIDERS).map((id) => ({ id, label: PROVIDERS[id].label, needsKey: PROVIDERS[id].needsKey, source: PROVIDERS[id].needsKey ? keyFor(id).source : "saved", env: envNameFor(id) }));
+}
+const readFileCfg = () => { try {
+    return { ...defaults(), ...JSON.parse(readFileSync(file(), "utf8")) };
+}
+catch {
+    return defaults();
+} };
+let tavilyEnv;
 export function loadConfig() {
     let c = defaults(), onDisk = {};
     try {
@@ -37,33 +92,40 @@ export function loadConfig() {
         c = { ...c, ...onDisk };
     }
     catch { /* corrupt config: start fresh */ }
-    // keys live in the OS credential store when there is one; a key still in the file (an older version wrote it there) is moved over, and removed from the file only after it reads back
-    let migrated = false;
-    for (const k of SECRETS) {
-        const stored = getSecret(k);
-        if (stored) {
-            c[k] = stored;
-            if (onDisk[k]) {
-                delete onDisk[k];
-                migrated = true;
-            }
-        }
-        else if (onDisk[k] && setSecret(k, onDisk[k])) {
-            delete onDisk[k];
-            migrated = true;
+    fileKeys = { ...onDisk.apiKeys };
+    fileKeysLoaded = true;
+    delete c.apiKeys;
+    let dirty = false;
+    // older versions kept ONE key (account "apiKey", or a field in the file) for whichever provider was active: give it to that provider
+    const legacy = [getSecret("apiKey"), onDisk.apiKey].find(Boolean);
+    if (legacy && !savedKey(c.provider)) {
+        fileKeys[c.provider] = legacy;
+        dirty = true;
+    }
+    if (getSecret("apiKey"))
+        deleteSecret("apiKey");
+    if (onDisk.apiKey) {
+        delete onDisk.apiKey;
+        dirty = true;
+    }
+    // keys still in the file move to the OS store, and leave the file only after they read back
+    for (const p of Object.keys(fileKeys))
+        if (fileKeys[p] && setSecret(acct(p), fileKeys[p])) {
+            delete fileKeys[p];
+            dirty = true;
         }
+    const tv = getSecret("tavilyKey") ?? onDisk.tavilyKey;
+    if (onDisk.tavilyKey && setSecret("tavilyKey", onDisk.tavilyKey)) {
+        delete onDisk.tavilyKey;
+        dirty = true;
     }
-    if (migrated)
-        writeConfigFile({ ...defaults(), ...onDisk, ...Object.fromEntries(SECRETS.map((k) => [k, undefined])) });
-    // environment wins, so CI and one-off runs work without touching the saved key. It is never written back (see saveConfig).
+    if (dirty)
+        writeConfigFile({ ...defaults(), ...onDisk, apiKey: undefined, tavilyKey: onDisk.tavilyKey });
     const claude = c.provider === "anthropic";
-    fromEnv = {};
-    const key = envSecret(claude ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY");
-    if (key)
-        c.apiKey = fromEnv.apiKey = key;
-    const tavily = envSecret("TAVILY_API_KEY");
-    if (tavily)
-        c.tavilyKey = fromEnv.tavilyKey = tavily;
+    c.apiKey = keyFor(c.provider).key;
+    // environment wins, so CI and one-off runs work without touching the saved key. It is never written back (see saveConfig).
+    tavilyEnv = envSecret("TAVILY_API_KEY");
+    c.tavilyKey = tavilyEnv ?? tv;
     if (process.env.OPENAI_BASE_URL && !claude)
         c.baseUrl = process.env.OPENAI_BASE_URL;
     if (process.env.AMNESS_MODEL)
@@ -72,25 +134,25 @@ export function loadConfig() {
 }
 function writeConfigFile(c) {
     mkdirSync(home(), { recursive: true });
-    writeFileSync(file(), JSON.stringify(c, null, 2), { mode: 0o600 }); // owner-only from the first byte
+    const out = { ...c, apiKeys: Object.keys(fileKeys).length ? fileKeys : undefined };
+    writeFileSync(file(), JSON.stringify(out, null, 2), { mode: 0o600 }); // owner-only from the first byte
     chmodSync(file(), 0o600); // and for a file that already existed
 }
 export function saveConfig(c) {
     const out = { ...c };
-    for (const k of SECRETS) {
-        const v = c[k];
-        if (v && v === fromEnv[k]) {
-            delete out[k];
-            continue;
-        } // this value came from an environment variable: do not store it
-        if (!v) {
-            deleteSecret(k);
-            delete out[k];
-            continue;
-        }
-        if (setSecret(k, v) || noStore())
-            delete out[k]; // stored in the OS credential store; otherwise it stays in the file (mode 600), except in a container, where it is kept in memory only
+    delete out.apiKey;
+    delete out.apiKeys;
+    if (c.apiKey !== persisted[c.provider] && !(c.apiKey && c.apiKey === envKeyFor(c.provider)))
+        storeKey(c.provider, c.apiKey); // a key from an environment variable is never stored
+    const tv = c.tavilyKey;
+    if (tv && tv === tavilyEnv)
+        delete out.tavilyKey;
+    else if (!tv) {
+        deleteSecret("tavilyKey");
+        delete out.tavilyKey;
     }
+    else if (setSecret("tavilyKey", tv) || noStore())
+        delete out.tavilyKey;
     writeConfigFile(out);
 }
 export const ready = (c) => !PROVIDERS[c.provider].needsKey || !!c.apiKey;

package/dist/cli/evalcli.d.ts

@@ -1,7 +1,24 @@
-import { type Model } from "../src/index.ts";
+import { type Model, type Row } from "../src/index.ts";
 export declare const EVAL_HELP: string;
+/** Estimated dollars for some rows (prompt, cached and completion tokens priced separately; the rate card is in pricing.ts). Null when the model's price is unknown. */
+export declare function rowsCost(model: string, rows: Row[]): number | null;
+/** What makes a result repeatable: the Amness version and git commit (when run from a clone), and a hash of the tasks (their files, text and checks). */
+export declare function provenance(cases: {
+    name: string;
+    files: Record<string, string>;
+    task: string;
+    verify?: string;
+    check: unknown;
+    solution?: Record<string, string>;
+}[]): {
+    version: string | undefined;
+    commit: string | undefined;
+    suite: string;
+    tasks: number;
+};
 export declare function runEval(argv: string[], deps: {
     model: () => Model;
+    modelName?: string;
     modelFor?: (id: string) => Model;
     maxTokens: number;
     confirm: (q: string) => Promise<boolean>;

package/dist/cli/evalcli.js

@@ -7,10 +7,13 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
     return path;
 };
 // `amness eval`: compare harness variants on a task suite, with statistics you can act on.
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
-import { BUILTIN_METRICS, gateCompare, importPlugins, tune, writeProfile, adversarialCases, hardCases, analyze, builtinCases, evaluate, recommend, leakAudit, loadSuite, suiteToCases, formatReport, harnessVariant, loadCases, selfTest, variantNames, VARIANT_HELP } from "../src/index.js";
+import { BUILTIN_METRICS, gateCompare, importPlugins, tune, writeProfile, adversarialCases, hardCases, analyze, builtinCases, evaluate, recommend, leakAudit, loadSuite, suiteToCases, formatReport, calibrate, harnessVariant, loadCases, selfTest, variantNames, VARIANT_HELP } from "../src/index.js";
 import { bold, dim, green, red } from "./ui.js";
+import { estimateCost, priceOf } from "./tui/pricing.js";
 import { pathToFileURL } from "node:url";
 async function casesFromModule(file) {
     const mod = await import(__rewriteRelativeImportExtension(pathToFileURL(file).href + `?v=${Date.now()}`)), list = mod.default ?? mod.cases;
@@ -46,7 +49,32 @@ export const EVAL_HELP = `usage: amness eval [flags]
   --out <file>         where to save raw results (default .amness/evals/<time>.json)
   --report <file>      re-analyse saved results; runs nothing
   --selftest           check that every task is solvable and its held-out check is sound; runs no model
+  --max-cost <dollars> stop starting new sessions once the estimated spend reaches this amount (sessions already running finish). Off by default. Needs a model with a known price.
+  --calibrate          how hard is each task? runs only \`baseline\` (5 trials unless you set --trials) and lists each task's pass rate: keep 30% to 70%, drop tasks that are too easy or too hard
   --yes                skip the cost confirmation`;
+/** Estimated dollars for some rows (prompt, cached and completion tokens priced separately; the rate card is in pricing.ts). Null when the model's price is unknown. */
+export function rowsCost(model, rows) {
+    if (!priceOf(model))
+        return null;
+    return rows.reduce((n, r) => n + (estimateCost(model, r.prompt ?? r.tokens, Math.max(0, r.tokens - (r.prompt ?? r.tokens)), r.cached ?? 0) ?? 0), 0);
+}
+/** What makes a result repeatable: the Amness version and git commit (when run from a clone), and a hash of the tasks (their files, text and checks). */
+export function provenance(cases) {
+    let commit;
+    try {
+        commit = execFileSync("git", ["rev-parse", "HEAD"], { cwd: new URL("..", import.meta.url), encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
+    }
+    catch { /* not a git checkout (an installed package) */ }
+    let version;
+    try {
+        version = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
+    }
+    catch { /* no package.json next to us */ }
+    const h = createHash("sha256");
+    for (const x of [...cases].sort((a, b) => a.name.localeCompare(b.name)))
+        h.update(JSON.stringify([x.name, x.files, x.task, x.verify ?? "", typeof x.check === "function" ? x.check.toString() : x.check, x.solution ?? {}]));
+    return { version, commit, suite: h.digest("hex").slice(0, 16), tasks: cases.length };
+}
 export async function runEval(argv, deps) {
     const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; }, has = (f) => argv.includes(f);
     if (has("--help") || has("-h")) {
@@ -108,20 +136,41 @@ export async function runEval(argv, deps) {
                 console.log(`${r.ok ? green("ok ") : red("bad")} ${r.case}${r.why ? dim("  " + r.why) : ""}`);
             return res.every((r) => r.ok) ? 0 : 1;
         }
-        const names = variantNames(val("--variants") ?? "ab"), trials = Math.max(1, Number(val("--trials") ?? 3)), jobs = Math.max(1, Number(val("--jobs") ?? 1));
+        const calib = has("--calibrate"), names = variantNames(val("--variants") ?? (calib ? "baseline" : "ab")), trials = Math.max(1, Number(val("--trials") ?? (calib ? 5 : 3))), jobs = Math.max(1, Number(val("--jobs") ?? 1));
         const maxTokens = Number(val("--max-tokens") ?? deps.maxTokens), maxSteps = Number(val("--max-steps") ?? 30);
         const model = deps.model(), cheap = val("--cheap-model") && deps.modelFor ? deps.modelFor(val("--cheap-model")) : undefined, variants = names.map((n) => harnessVariant(model, n, { maxTokens, maxSteps, cheap, plugins, modelFor: deps.modelFor }));
         const runs = cases.length * variants.length * trials;
         console.log(`${bold("plan")}  ${cases.length} tasks × ${variants.length} variants (${names.join(", ")}) × ${trials} trials = ${bold(String(runs))} agent sessions, up to ${maxTokens.toLocaleString()} tokens each`);
+        const mname = deps.modelName ?? "", maxCost = val("--max-cost") === undefined ? undefined : Number(val("--max-cost"));
+        if (maxCost !== undefined && !(maxCost > 0))
+            throw new Error("--max-cost needs a positive number of dollars, for example --max-cost 0.50");
+        if (maxCost !== undefined && !priceOf(mname))
+            throw new Error(`--max-cost needs a known price for the model "${mname}" (see cli/tui/pricing.ts); without one Amness cannot estimate the spend`);
+        const p = priceOf(mname);
+        if (p)
+            console.log(dim(`worst case about $${(runs * maxTokens * p[2] / 1e6).toFixed(2)} (every token priced as output; real runs are far cheaper: most of a session is cached input)${maxCost !== undefined ? ` · stops at about $${maxCost.toFixed(2)}` : " · no spending cap (--max-cost sets one)"}`));
         if (!has("--yes") && !(await deps.confirm(`This calls your model up to ${(runs * maxTokens).toLocaleString()} tokens in the worst case. Continue?`)))
             return 1;
-        const { rows, analysis } = await evaluate(cases, variants, {
+        const { rows, analysis, stopped, skipped } = await evaluate(cases, variants, {
+            stop: maxCost === undefined ? undefined : (rs) => { const spent = rowsCost(mname, rs) ?? 0; return spent >= maxCost ? `reached the cost cap: about $${spent.toFixed(3)} spent of $${maxCost.toFixed(2)}` : undefined; },
             metrics: Object.keys(useMetrics).length ? useMetrics : undefined, trials, jobs, baseline: val("--baseline"), k: Math.min(3, trials), seed: val("--seed") ? Number(val("--seed")) : undefined,
             onRow: (r) => console.error(`  ${r.pass ? green("PASS") : red("FAIL")} ${r.case.padEnd(22)} ${r.variant.padEnd(18)} ${dim(`${r.steps} steps · ${r.tokens} tok · ${(r.ms / 1000).toFixed(1)}s${r.note ? " · " + r.note : ""}`)}`),
         });
         const out = resolve(val("--out") ?? join(".amness", "evals", new Date().toISOString().replace(/[:.]/g, "-") + ".json"));
         mkdirSync(join(out, ".."), { recursive: true });
-        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), baseline: analysis.baseline, k: Math.min(3, trials), variants: names, trials, rows }, null, 1));
+        const spent = rowsCost(mname, rows);
+        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), model: mname || undefined, ...provenance(cases), cliArgs: argv.filter((a, i) => !/key|token/i.test(a) && !/key|token/i.test(argv[i - 1] ?? "")), baseline: analysis.baseline, k: Math.min(3, trials), variants: names, trials, estimatedCost: spent ?? undefined, stopped, rows }, null, 1));
+        if (stopped)
+            console.log(`\n${red("stopped early")}: ${stopped}. ${skipped} planned session${skipped === 1 ? "" : "s"} did not run, so the results below cover only the runs that finished.`);
+        if (spent !== null)
+            console.log(dim(`estimated spend: about $${spent.toFixed(3)} for ${rows.length} session${rows.length === 1 ? "" : "s"}${rows.length ? ` ($${(spent / rows.length).toFixed(4)} each)` : ""}  (an estimate from token counts, not an invoice)`));
+        if (calib) {
+            console.log("\n" + bold("how hard is each task?") + dim(`  (baseline, ${trials} trials; keep 30% to 70%)`));
+            for (const x of calibrate(rows, names[0]))
+                console.log(`  ${x.verdict === "keep" ? green("keep      ") : red(x.verdict.padEnd(10))} ${x.case.padEnd(26)} ${x.passes}/${x.runs} passed  ${dim((x.rate * 100).toFixed(0) + "%")}`);
+            console.log(dim(`raw results: ${out}`));
+            return 0;
+        }
         if (val("--profile-out")) {
             const rec = recommend(analysis, names.includes("full") ? "full" : analysis.baseline), pm = val("--profile-model") ?? "*";
             console.log("\nguard verdicts:");

package/dist/cli/models.d.ts

@@ -0,0 +1,8 @@
+export type ModelProbe = {
+    provider: string;
+    baseUrl: string;
+    apiKey?: string;
+};
+export declare function fetchModels(p: ModelProbe, fetchImpl?: typeof fetch): Promise<string[]>;
+/** For tests. */
+export declare const clearModelCache: () => void;

package/dist/cli/models.js

@@ -0,0 +1,31 @@
+const cache = new Map();
+const TTL = 10 * 60_000;
+const NOT_CHAT = /embed|whisper|tts|dall-e|moderation|davinci|babbage|transcribe|realtime|audio|image|rerank/i;
+export async function fetchModels(p, fetchImpl = fetch) {
+    if (!p.apiKey && p.provider !== "ollama" && p.provider !== "custom")
+        return []; // a hosted provider refuses a request with no key: do not even try
+    const key = `${p.provider}|${p.baseUrl}|${(p.apiKey ?? "").slice(-6)}`, hit = cache.get(key);
+    if (hit && Date.now() - hit.at < TTL)
+        return hit.ids;
+    const base = p.baseUrl.replace(/\/+$/, "");
+    const anthropic = p.provider === "anthropic";
+    try {
+        const res = await fetchImpl(anthropic ? `${base}/v1/models?limit=100` : `${base}/models`, {
+            headers: anthropic ? { "x-api-key": p.apiKey ?? "", "anthropic-version": "2023-06-01" } : p.apiKey ? { authorization: `Bearer ${p.apiKey}` } : {},
+            signal: AbortSignal.timeout(6000),
+        });
+        if (!res.ok)
+            return [];
+        const body = (await res.json());
+        const ids = [...(body.data ?? []).map((m) => m.id), ...(body.models ?? []).map((m) => m.model ?? m.name)] // Ollama's native shape uses `models`
+            .filter((x) => typeof x === "string" && !!x && !NOT_CHAT.test(x));
+        const uniq = [...new Set(ids)].sort();
+        cache.set(key, { at: Date.now(), ids: uniq });
+        return uniq;
+    }
+    catch {
+        return [];
+    }
+}
+/** For tests. */
+export const clearModelCache = () => cache.clear();

package/dist/cli/tui/app.d.ts

@@ -131,9 +131,11 @@ export declare class App {
         items: Suggestion[];
         sel: number;
         token: Token;
-        kind: Token["kind"];
+        kind: Token["kind"]; /** the user moved into the list with the arrow keys */
+        nav?: boolean;
     } | null;
     suggOff: string;
+    private argPending;
     private builtin;
     private customAt;
     private customCmds;

package/dist/cli/tui/app.js

@@ -15,7 +15,8 @@ import { BlockStore } from "./blocks.js";
 import { buildCommands } from "./commands.js";
 import { FullView } from "./full.js";
 import { loadCustomCommands } from "./custom.js";
-import { FileIndex, applySuggestion, rankCommands, rankFiles, tokenAtCursor } from "./complete.js";
+import { argOptions } from "./argcomplete.js";
+import { FileIndex, fuzzyScore, applySuggestion, menuRows, rankCommands, rankFiles, tokenAtCursor } from "./complete.js";
 import { Editor } from "./editor.js";
 import { KeyParser, PASTE_OFF, PASTE_ON } from "./keys.js";
 import { renderMarkdown, takeCompleteBlocks } from "./markdown.js";
@@ -80,6 +81,7 @@ export class App {
     pager = null;
     sugg = null;
     suggOff = "";
+    argPending = new Set();
     builtin;
     customAt = 0;
     customCmds = [];
@@ -370,11 +372,16 @@ export class App {
     setTitle(t) { if (this.io.output.isTTY !== false)
         this.io.output.write(`\x1b]0;${t}\x07`); }
     suggLines(w) {
-        const s = this.sugg, win = 8, top = Math.max(0, Math.min(s.sel - 3, s.items.length - win));
-        return s.items.slice(top, top + win).map((it, i) => {
-            const on = top + i === s.sel, label = s.kind === "file" ? it.label : it.label.padEnd(12);
+        const s = this.sugg, { rows, above, below } = menuRows(s.items, s.sel, Math.min(12, Math.max(4, this.screen.rows - 8)));
+        const lines = rows.map((r) => {
+            if ("head" in r)
+                return dim(`  ── ${r.head} ${"─".repeat(Math.max(2, 20 - r.head.length))}`);
+            const it = s.items[r.item], on = r.item === s.sel, label = s.kind === "file" ? it.label : it.label.padEnd(16);
             return (on ? cyan("❯ ") + bold(cyan(label)) : "  " + label) + (it.detail ? "  " + (on ? gray(it.detail) : dim(it.detail)) : "");
         }).map((l) => (cellWidth(l) > w ? l.slice(0, w + 12) : l));
+        if (above || below)
+            lines.push(dim(`  ${s.sel + 1}/${s.items.length}  ${above ? "↑" : " "}${below ? "↓" : " "}  ↑↓ move · PgUp/PgDn page`));
+        return lines;
     }
     tick = () => { this.frame++; this.render(); };
     ensureTimer() { if (!this.timer)
@@ -451,7 +458,7 @@ export class App {
         if (k.name === "enter" && !k.alt && !k.shift) {
             if (this.sugg && this.sugg.items.length) {
                 const it = this.sugg.items[this.sugg.sel];
-                if (!(this.sugg.kind === "slash" && ed.text.trim() === it.label))
+                if (!(this.sugg.kind === "slash" && ed.text.trim() === it.label) && !(this.sugg.kind === "arg" && (this.sugg.items.some((x) => x.label === this.sugg.token.query) || (it.open && !this.sugg.nav))))
                     return this.acceptSugg(true);
             }
             if (ed.text.slice(0, ed.cursorIndex).endsWith("\\")) {
@@ -461,10 +468,16 @@ export class App {
             }
             return void this.submit();
         }
+        if (this.sugg && (k.name === "pageup" || k.name === "pagedown")) {
+            const n = this.sugg.items.length;
+            this.sugg.sel = Math.max(0, Math.min(n - 1, this.sugg.sel + (k.name === "pageup" ? -8 : 8)));
+            return this.render();
+        }
         if (k.name === "up" || k.name === "down") {
             if (this.sugg) {
                 const n = this.sugg.items.length;
                 this.sugg.sel = (this.sugg.sel + (k.name === "up" ? n - 1 : 1)) % n;
+                this.sugg.nav = true;
                 return this.render();
             }
             if (!ed.vertical(k.name === "up" ? -1 : 1))
@@ -492,13 +505,25 @@ export class App {
         let items = [];
         if (tok.kind === "slash")
             items = rankCommands(tok.query, this.cmds);
+        else if (tok.kind === "arg") {
+            const cmd = this.cmds.find((c) => c.name === tok.cmd || c.alias === tok.cmd);
+            if (cmd) {
+                const r = argOptions(this, cmd.name, tok.words ?? []), pk = `${cmd.name}|${(tok.words ?? []).join(" ")}`;
+                if (r.pending && !this.argPending.has(pk)) {
+                    this.argPending.add(pk);
+                    void r.pending.catch(() => { }).then(() => { this.argPending.delete(pk); this.updateSugg(); this.render(); });
+                }
+                items = r.options.map((o) => ({ o, s: fuzzyScore(tok.query, o.value) })).filter((x) => x.s !== null)
+                    .sort((x, y) => (tok.query ? y.s - x.s : 0)).map(({ o }) => ({ label: o.value, insert: o.value + (o.more ? " " : ""), detail: o.detail, more: o.more, open: r.open }));
+            }
+        }
         else {
             if (!this.files.fresh())
                 void this.files.ensure().then(() => this.updateSugg(), () => { }); // list still loading: fill the popup in once it arrives
             items = rankFiles(tok.query, this.files.files);
         }
-        const keep = this.sugg && this.sugg.kind === tok.kind && this.sugg.token.query === tok.query ? this.sugg.sel : 0;
-        this.sugg = items.length ? { items, sel: Math.min(keep, items.length - 1), token: tok, kind: tok.kind } : null;
+        const same = this.sugg && this.sugg.kind === tok.kind && this.sugg.token.query === tok.query, keep = same ? this.sugg.sel : 0;
+        this.sugg = items.length ? { items, sel: Math.min(keep, items.length - 1), token: tok, kind: tok.kind, nav: same ? this.sugg.nav : false } : null;
         if (!this.sugg)
             return;
         this.schedule();
@@ -512,6 +537,8 @@ export class App {
         this.suggOff = "";
         if (fromEnter && s.kind === "slash" && !this.cmds.find((c) => "/" + c.name === s.items[s.sel].label)?.args)
             return void this.submit();
+        if (fromEnter && s.kind === "arg" && !s.items[s.sel].more)
+            return void this.submit(); // a finished value: run the command
         this.updateSugg();
         this.render();
     }

package/dist/cli/tui/argcomplete.d.ts

@@ -0,0 +1,12 @@
+import type { App } from "./app.ts";
+export type ArgOption = {
+    value: string;
+    detail?: string; /** more words are expected after this one: insert a space and do not submit */
+    more?: boolean;
+};
+export type ArgResult = {
+    options: ArgOption[];
+    pending?: Promise<unknown>; /** any value is allowed (a number, a model id, a command): Enter runs what you typed unless you moved into the list; Tab still completes */
+    open?: boolean;
+};
+export declare function argOptions(a: App, cmd: string, words: string[]): ArgResult;

package/dist/cli/tui/argcomplete.js

@@ -0,0 +1,55 @@
+// Completion for the words after a slash command: `/mode ` lists the modes, `/model ` the models your provider offers, `/theme ` the themes, and so on.
+// A completer returns its options now and, when it needs the network, a promise that settles when the list has been filled in (the menu refreshes then).
+import { effortsFor, loadPrompts, SANDBOX_MODES, SHELL_CHOICES, STRATEGIES } from "../../src/index.js";
+import { CLAUDE_MODELS, OPENAI_MODELS } from "../config.js";
+import { fetchModels } from "../models.js";
+import { themeNames, THEMES } from "./themes.js";
+const onOff = [{ value: "on" }, { value: "off" }];
+const opts = (...xs) => xs.map(([value, detail]) => ({ value, detail }));
+/** Known models with their hints first, then any other model the provider reports. The provider is asked once per ten minutes; until it answers, only the known models show. */
+const fetched = new Map();
+function modelOptions(a, extra = []) {
+    const known = a.cfg.provider === "anthropic" ? CLAUDE_MODELS : a.cfg.provider === "openai" ? OPENAI_MODELS : [];
+    const key = `${a.cfg.provider}|${a.cfg.baseUrl}`, have = fetched.get(key), fresh = have && Date.now() - have.at < 10 * 60_000;
+    const seen = new Set(known.map((m) => m.id));
+    const options = [...extra, ...known.map((m) => ({ value: m.id, detail: m.hint })), ...(have?.ids ?? []).filter((id) => !seen.has(id)).map((id) => ({ value: id, detail: "from your provider" }))];
+    if (fresh)
+        return { options, open: true };
+    return { options, open: true, pending: fetchModels({ provider: a.cfg.provider, baseUrl: a.cfg.baseUrl, apiKey: a.cfg.apiKey }).then((ids) => { fetched.set(key, { at: Date.now(), ids }); }) };
+}
+export function argOptions(a, cmd, words) {
+    const first = words.length === 0;
+    const ok = (options) => ({ options });
+    switch (cmd) {
+        case "mode": return first ? ok(opts(["default", "ask before edits and non-trivial commands"], ["acceptEdits", "edits are automatic, commands still ask"], ["plan", "read-only: investigate and propose a plan"], ["yolo", "never ask (dangerous)"])) : ok([]);
+        case "api": return first ? ok(opts(["auto", "pick Responses for reasoning models, self-correct"], ["responses", "OpenAI Responses API"], ["chat", "Chat Completions API"])) : ok([]);
+        case "effort": return first ? ok([{ value: "default", detail: "let the model decide" }, ...(a.cfg.provider === "anthropic" ? ["low", "medium", "high", "xhigh", "max"] : effortsFor(a.cfg.model)).map((e) => ({ value: e }))]) : ok([]);
+        case "model": return first ? modelOptions(a) : ok([]);
+        case "models": return first ? ok([...["explore", "compact", "review", "cheap"].map((r) => ({ value: r, detail: `model for ${r} steps`, more: true })), { value: "fallback", detail: "answer from this model when the main one is down", more: true }])
+            : words.length === 1 ? modelOptions(a, [{ value: "off", detail: "use the main model" }]) : ok([]);
+        case "theme": return first ? ok(themeNames().map((n) => ({ value: n, detail: THEMES[n].label }))) : ok([]);
+        case "shell": return first ? ok(SHELL_CHOICES.map((n) => ({ value: n, detail: n === "auto" ? "detect from your system" : undefined }))) : ok([]);
+        case "strategy": return first ? ok([...Object.keys(STRATEGIES).map((n) => ({ value: n })), { value: "off", detail: "the default loop" }]) : ok([]);
+        case "prompt": return first ? ok([...loadPrompts(a.cwd).map((p) => ({ value: p.id, detail: "from .amness/prompts" })), { value: "list" }, { value: "off", detail: "the built-in prompt" }]) : ok([]);
+        case "sandbox": return first ? ok(SANDBOX_MODES.map((m) => ({ value: m, detail: m === "off" ? "no isolation" : m === "workspace" ? "writes only in the project" : "no network, caches read-only" }))) : ok([]);
+        case "diffmode": return first ? ok(opts(["auto"], ["stacked"], ["split"])) : ok([]);
+        case "ui": return first ? ok(opts(["full", "full-screen"], ["inline", "scrolls with the terminal"])) : ok([]);
+        case "mouse":
+        case "keepwarm":
+        case "lsp":
+        case "approver":
+        case "reviewer":
+        case "router": return first ? ok(onOff) : ok([]);
+        case "copy": return first ? ok(opts(["code", "the last code block"], ["all", "the whole conversation"])) : ok([]);
+        case "memory": return first ? ok([...["forget", "add"].map((v) => ({ value: v, more: true })), ...["stale", "prune", "consolidate", "graph", "embeddings"].map((v) => ({ value: v }))]) : ok([]);
+        case "permissions": return first ? ok([{ value: "list" }, ...["allow", "ask", "deny"].map((v) => ({ value: v, detail: `e.g. /permissions ${v} sh(npm test*)`, more: true }))]) : ok([]);
+        case "branch": return first ? ok(a.session.turns.map((t, i) => ({ value: String(i + 1), detail: t.text.replace(/\s+/g, " ").slice(0, 60) }))) : ok([]);
+        case "pr": return first ? ok(opts(["draft", "create it as a draft"])) : ok([]);
+        case "report": return first ? ok([{ value: "case", detail: "turn the last request into an eval task", more: true }]) : ok([]);
+        case "mcp": return first ? ok([{ value: "list" }, { value: "off" }]) : ok([]);
+        case "budget": return first ? { options: opts(["100k"], ["300k", "the default"], ["500k"], ["1m"], ["2m"]), open: true } : ok([]);
+        case "check":
+        case "verify": return first ? { options: [{ value: "off", detail: "switch it off" }], open: true } : ok([]);
+        default: return ok([]);
+    }
+}

package/dist/cli/tui/cards.d.ts

@@ -16,6 +16,8 @@ export type RenderCtx = {
     diffMode: "auto" | "stacked" | "split";
     /** The block above is also a tool call: draw no blank line between them, so a run of tool calls reads as one unit. */
     tight?: boolean;
+    /** this is the most recent tool call of the lead agent: its output stays open (a short preview) until the next one starts */
+    last?: boolean;
 };
 export type Hit = {
     row: number;