amness 0.1.2 → 0.1.4

package/dist/cli/tui/looppanel.js

@@ -0,0 +1,218 @@
+// The LOOP section of the sidebar: the cycle the agent goes round (gather context, act, verify), which part of it is running now, and the latest steps, with the harness's
+// interventions shown where they happened. The stages are INFERRED from the tool calls (the harness has no gather/act/verify phases of its own): reads and searches gather,
+// edits and most commands act, test/build/lint/typecheck commands and the verify gate verify. It adapts to the room it has: older steps fold into one summary line, the step cap
+// appears only when it starts to matter, and a clicked step opens to show every tool call and note. The full history is the /loop command.
+import { mix } from "./ansi.js";
+import { BOLD, F } from "./rich.js";
+import { clipWidth } from "./text.js";
+import { toolLabel } from "./view.js";
+const SPIN = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏";
+const fmtMs = (ms) => (ms < 1000 ? `${Math.round(ms)}ms` : ms < 60_000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.floor(ms / 60_000)}m${Math.round((ms % 60_000) / 1000)}s`);
+/** Notes that mean "the harness stepped in" (shown in the warning colour, and kept visible); everything else (cache, routing, trimming) is background. */
+export const isIntervention = (note) => /nudg|reject|denied|loop|stagnat|stuck|budget|step limit|reviewer|wrap|repair/i.test(note);
+/** "Read ×2 → Edit → Run": neighbouring calls of one kind merge, so a burst of reads is one short phrase. */
+export function stepSummary(tools) {
+    const parts = [];
+    for (const t of tools) {
+        const label = toolLabel(t.name), last = parts[parts.length - 1];
+        if (last && last.label === label)
+            last.n++;
+        else
+            parts.push({ label, n: 1 });
+    }
+    return parts.map((p) => (p.n > 1 ? `${p.label} ×${p.n}` : p.label)).join(" → ");
+}
+/** Wall time of a step: the model call, then its tools (they overlap when run in parallel). */
+export const stepMs = (s) => s.ms + (s.parallel && s.parallel > 1 ? Math.max(0, ...s.tools.map((x) => x.ms)) : s.tools.reduce((n, x) => n + x.ms, 0));
+/** The cap is a safety stop, not a goal, so say nothing about it until it could matter. */
+export function capNote(step, cap) {
+    const r = step / Math.max(1, cap);
+    return r >= 0.8 ? { text: `${step}/${cap}: close to the step limit`, level: "warn" } : r >= 0.5 ? { text: `of ${cap} max`, level: "dim" } : { text: "", level: "none" };
+}
+export const STAGES = ["gather", "act", "verify"];
+export const STAGE_GLYPH = { gather: "◉", act: "✎", verify: "✓" };
+const GATHER_TOOLS = new Set(["read_file", "list_files", "grep", "repo_map", "find_refs", "impact", "web_search", "fetch_url", "memory_search", "tool_search", "skill", "task", "sh_list", "sh_read"]);
+const NEUTRAL_TOOLS = new Set(["todo_write", "memory_save", "memory_forget", "exit_plan_mode", "finish"]); // planning and bookkeeping: not part of the cycle
+const VERIFY_CMD = /\b(tests?|pytest|jest|vitest|mocha|tsc|typecheck|type-check|lint|eslint|ruff|mypy|pyright|flake8|clippy|rspec|phpunit|verify|check)\b|\b(cargo|go|npm|yarn|pnpm|make|gradle|mvn|dotnet|swift)\s+(run\s+)?(test|build|check|vet|lint|clippy|verify)\b|node\s+--test|\bbuild\b/i;
+const LOOK_CMD = /^\s*(ls|cat|head|tail|less|grep|rg|find|tree|pwd|wc|stat|file|which|du|df|git\s+(status|diff|log|show|branch|blame)|sed\s+-n)\b/;
+/** Which part of the cycle a tool call belongs to. A guess from the tool and, for shell commands, the command line; undefined for planning and bookkeeping tools. */
+export function stageOfTool(name, args = "") {
+    if (NEUTRAL_TOOLS.has(name))
+        return undefined;
+    if (GATHER_TOOLS.has(name))
+        return "gather";
+    if (name === "sh" || name === "sh_start") {
+        let cmd = "";
+        try {
+            cmd = String(JSON.parse(args).cmd ?? "");
+        }
+        catch {
+            cmd = args;
+        }
+        return LOOK_CMD.test(cmd) ? "gather" : VERIFY_CMD.test(cmd) ? "verify" : "act";
+    }
+    return "act";
+}
+/** The stages a step touched, in cycle order. */
+export const stagesOf = (tools) => STAGES.filter((st) => tools.some((x) => stageOfTool(x.name, x.args) === st));
+/** How many times the agent has come back round: a new round starts when it gathers or acts again after verifying. */
+export function roundOf(steps) {
+    let round = 1, verified = false;
+    for (const s of steps) {
+        const st = stagesOf(s.tools);
+        if (verified && (st.includes("gather") || st.includes("act"))) {
+            round++;
+            verified = false;
+        }
+        if (st.includes("verify"))
+            verified = true;
+    }
+    return round;
+}
+/** Which stages to light: those running now, and (while the model is thinking and nothing runs) the last one used, dimmed. */
+export function litStages(d) {
+    const now = new Set(), recent = new Set(), cur = d.steps[d.steps.length - 1];
+    if (d.live) {
+        if (d.phase === "verify")
+            now.add("verify");
+        else if (d.phase === "tools") {
+            for (const x of d.inflight) {
+                const st = x.name ? stageOfTool(x.name, x.args) : undefined;
+                if (st)
+                    now.add(st);
+            }
+            if (!now.size && cur) {
+                const last = cur.tools[cur.tools.length - 1];
+                const st = last && stageOfTool(last.name, last.args);
+                if (st)
+                    now.add(st);
+            }
+        }
+    }
+    if (!now.size)
+        for (let i = d.steps.length - 1; i >= 0; i--) {
+            const st = stagesOf(d.steps[i].tools);
+            if (st.length) {
+                st.forEach((x) => recent.add(x));
+                break;
+            }
+        }
+    return { now, recent };
+}
+export function renderLoop(d, t, inner, maxRows) {
+    const lines = [], hits = [];
+    const cur = d.live ? d.steps[d.steps.length - 1] : undefined, round = roundOf(d.steps);
+    const tools = d.steps.reduce((n, s) => n + s.tools.length, 0), failed = d.steps.reduce((n, s) => n + s.tools.filter((x) => !x.ok).length, 0);
+    const color = { gather: t.tool.read, act: t.tool.edit, verify: t.ok };
+    const title = d.live ? "LOOP" : "LAST RUN";
+    const cap = cur ? capNote(cur.n, d.cap) : { text: "", level: "none" };
+    const right = d.live && cur ? F(t.dim, `round ${round} · step ${cur.n}${cap.level === "dim" ? " of " + d.cap : ""}`) : F(t.dim, `${Math.round(d.elapsedMs / 1000)}s`);
+    const spin = F(t.accent2, SPIN[d.frame % SPIN.length]);
+    // the cycle: the stage that is running is bright, the one just used (while the model thinks) is half-bright, the rest are dim. A verify that failed stays red until the next one passes.
+    const { now, recent } = litStages(d);
+    let verifyFailed = false;
+    for (const s of d.steps) {
+        const v = s.tools.filter((x) => stageOfTool(x.name, x.args) === "verify");
+        if (v.length)
+            verifyFailed = v.some((x) => !x.ok);
+    }
+    const compact = inner < 27;
+    const cycle = STAGES.map((st) => {
+        const c = st === "verify" && verifyFailed ? t.err : color[st], label = compact ? "" : " " + st, text = STAGE_GLYPH[st] + label;
+        return now.has(st) ? BOLD(F(c, text)) : recent.has(st) || (st === "verify" && verifyFailed) ? F(mix(c, t.surface, 0.35), text) : F(t.faint, text);
+    }).join(F(t.border, " ▸ "));
+    lines.push(cycle);
+    if (d.live && cur) {
+        const act = d.phase === "compact" ? "compacting the history" : d.phase === "verify" ? "verifying the result" : d.phase === "tools" && d.inflight.length ? "" : `thinking ${fmtMs(Math.max(0, d.now - cur.t0))}`;
+        if (act)
+            lines.push(spin + " " + F(t.dim, act));
+        else {
+            const x = d.inflight[0];
+            lines.push(spin + " " + F(t.text, clipWidth((x.agent && x.agent !== "lead" ? "↳ " : "") + x.label, inner - 9)) + F(t.faint, ` ${fmtMs(x.ms)}`));
+            if (d.inflight.length > 1)
+                lines.push("  " + F(t.faint, `+${d.inflight.length - 1} more running`));
+        }
+        if (cap.level === "warn") {
+            const w = Math.max(6, Math.min(20, inner - 4)), f = Math.max(1, Math.min(w, Math.round((cur.n / d.cap) * w)));
+            lines.push(F(t.warn, "▰".repeat(f)) + F(t.border, "▱".repeat(w - f)) + F(t.warn, "  " + cap.text));
+        }
+    }
+    const lastN = d.steps[d.steps.length - 1]?.n ?? 0;
+    const blocks = d.steps.map((s) => {
+        const isCur = d.live && s === cur, open = d.open.includes(s.n), out = [], bad = s.tools.some((x) => !x.ok);
+        const st = isCur && now.size ? [...now] : stagesOf(s.tools);
+        const glyphs = st.length ? st.map((x) => F(bad && !isCur ? t.err : color[x], STAGE_GLYPH[x])).join("") : isCur ? F(t.accent2, "▸") : F(t.faint, "·");
+        const what = isCur && !s.tools.length && !d.inflight.length ? "thinking…" : s.tools.length || isCur ? stepSummary(s.tools) || (d.inflight.length ? "…" : "") : "replied";
+        const time = isCur ? "" : fmtMs(stepMs(s)), num = String(s.n).padStart(2);
+        const body = clipWidth(what, Math.max(4, inner - 8 - st.length - (bad && !isCur ? 2 : 0) - time.length));
+        out.push(F(t.faint, num) + " " + glyphs + " " + (isCur ? F(t.text, body) : F(bad ? t.err : t.dim, body)) + (bad && !isCur ? F(t.err, " ✗") : "") + (time ? F(t.faint, " " + time) : ""));
+        const showAll = open || isCur;
+        if (open)
+            for (const x of s.tools)
+                out.push("     " + (x.ok ? F(t.ok, "✓ ") : F(t.err, "✗ ")) + F(t.dim, clipWidth(`${toolLabel(x.name)} ${shortArgs(x.args)}`, inner - 12)) + F(t.faint, ` ${fmtMs(x.ms)}`));
+        let hotNote = false;
+        for (const n of s.notes) {
+            const hot = isIntervention(n);
+            hotNote ||= hot;
+            if (!(hot || showAll || s.n >= lastN - 2))
+                continue;
+            // what the harness did is worth reading in full: an intervention wraps to two lines, background notes are clipped to one
+            const parts = hot ? wrapWords(n, inner - 7, 2) : [clipWidth(n, inner - 7)];
+            parts.forEach((l, i) => out.push("   " + (i ? "  " : F(hot ? t.warn : t.faint, hot ? "⚑ " : "· ")) + F(hot ? t.dim : t.faint, l)));
+        }
+        return { step: s.n, lines: out, must: s.n > lastN - 12 && (hotNote || open) };
+    });
+    // choose from the newest back: about as many steps as the room allows (3 when it is short), and every step that has an intervention or was opened
+    const room = Math.max(3, maxRows - lines.length - 1), nominal = Math.max(3, Math.min(8, Math.floor(room / 2)));
+    const shown = new Set();
+    let used = 0, count = 0;
+    const fitsAll = blocks.reduce((n, b) => n + b.lines.length, 0) <= room; // everything fits: fold nothing
+    for (let i = blocks.length - 1; i >= 0 && fitsAll; i--)
+        shown.add(i);
+    for (let i = blocks.length - 1; i >= 0 && !fitsAll; i--) {
+        const b = blocks[i], need = b.lines.length;
+        if (!shown.size) {
+            shown.add(i);
+            used += need;
+            count++;
+            continue;
+        } // the newest step is always there
+        if (used + need <= room && (count < nominal || b.must)) {
+            shown.add(i);
+            used += need;
+            count++;
+        }
+    }
+    const hidden = blocks.filter((_, i) => !shown.has(i));
+    const firstShown = Math.min(...shown);
+    if (hidden.length && blocks.findIndex((_, i) => !shown.has(i)) < firstShown) {
+        const old = blocks.slice(0, firstShown), ot = old.reduce((n, b) => n + d.steps.find((s) => s.n === b.step).tools.length, 0), of = old.reduce((n, b) => n + d.steps.find((s) => s.n === b.step).tools.filter((x) => !x.ok).length, 0);
+        lines.push(F(t.faint, `… ${old.length} earlier step${old.length === 1 ? "" : "s"} · ${ot} tool${ot === 1 ? "" : "s"}${of ? ` · ${of} failed` : ""}`));
+    }
+    blocks.forEach((b, i) => { if (!shown.has(i))
+        return; hits.push({ line: lines.length, step: b.step }); lines.push(...b.lines); });
+    if (!d.live || hidden.length)
+        lines.push(F(t.faint, `${d.steps.length} step${d.steps.length === 1 ? "" : "s"} · ${tools} tool${tools === 1 ? "" : "s"}${failed ? " · " : ""}`) + (failed ? F(t.err, `${failed} failed`) : "") + F(t.faint, "  /loop: all"));
+    return { lines, hits, title, right };
+}
+function wrapWords(s, w, maxLines) {
+    const lines = [""];
+    for (const word of s.split(/\s+/)) {
+        const last = lines[lines.length - 1];
+        if ((last + " " + word).trim().length <= w)
+            lines[lines.length - 1] = (last + " " + word).trim();
+        else
+            lines.push(word);
+    }
+    const cut = lines.slice(0, maxLines);
+    if (lines.length > maxLines)
+        cut[maxLines - 1] = clipWidth(cut[maxLines - 1], w - 1) + "…";
+    return cut.map((l) => clipWidth(l, w));
+}
+function shortArgs(raw) { try {
+    const a = JSON.parse(raw);
+    return String(a.path ?? a.cmd ?? a.pattern ?? a.query ?? a.url ?? a.name ?? "").replace(/\s+/g, " ");
+}
+catch {
+    return "";
+} }

package/dist/cli/tui/sidebar.d.ts

@@ -1,5 +1,6 @@
 import { type Cell } from "./ansi.ts";
 import type { Theme } from "./themes.ts";
+import { type LoopData } from "./looppanel.ts";
 export type SidebarData = {
     title: string;
     model: string;
@@ -37,23 +38,7 @@ export type SidebarData = {
         created: boolean;
         blockId: number;
     }[];
-    run?: {
-        phase: string;
-        step: number;
-        max: number;
-        elapsedMs: number;
-        frame: number;
-        live: {
-            label: string;
-            ms: number;
-            agent?: string;
-        }[];
-        done: {
-            label: string;
-            ok: boolean;
-            ms: number;
-        }[];
-    };
+    /** the agent loop of the current (or last) run, step by step */ loop?: LoopData;
     sandbox?: string;
     /** one line per plugin status (see Plugin.status) */ pluginStatus?: string[];
     /** the token cap for the current/last turn: tokens counted (cached input at a tenth), the cap, and whether usage was estimated */ budget?: {
@@ -75,8 +60,9 @@ export type SidebarData = {
 };
 export type SideHit = {
     row: number;
-    kind: "change" | "todos";
+    kind: "change" | "todos" | "step";
     blockId?: number;
+    step?: number;
 };
 export declare function renderSidebar(d: SidebarData, t: Theme, width: number, height: number): {
     rows: Cell[][];

package/dist/cli/tui/sidebar.js

@@ -4,6 +4,7 @@ import { BOLD, F, ITAL, fit, split } from "./rich.js";
 import { cellWidth, clipWidth } from "./text.js";
 import { brandOfModel } from "./brand.js";
 import { fmtCost, fmtTokens } from "./pricing.js";
+import { renderLoop } from "./looppanel.js";
 const BARS = "▁▂▃▄▅▆▇█";
 export function renderSidebar(d, t, width, height) {
     const inner = width - 4, out = [], hits = [], bg = t.surface;
@@ -12,16 +13,13 @@ export function renderSidebar(d, t, width, height) {
         push(); push(right ? split(F(t.faint, BOLD(title)), right, inner, bg) : F(t.faint, BOLD(title))); };
     const brand = brandOfModel(d.model, d.provider, t);
     push();
-    if (d.run) {
-        const r = d.run, SPIN = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏", stages = [["model", "think"], ["tools", "act"], ["verify", "verify"]];
-        head("RUN", F(t.dim, `${(r.elapsedMs / 1000).toFixed(0)}s`));
-        push(stages.map(([k, label], i) => (k === r.phase || (r.phase === "compact" && k === "model") ? BOLD(F(t.accent, "● " + label)) : F(t.faint, "○ " + label)) + (i < 2 ? F(t.border, " ─ ") : "")).join(""));
-        const lbl = `step ${r.step + 1}/${r.max}`, bw = Math.max(6, Math.min(24, inner - cellWidth(lbl) - 2)), f = Math.max(1, Math.round(((r.step + 1) / Math.max(1, r.max)) * bw));
-        push(F(r.step + 1 >= r.max - 2 ? t.warn : t.accent2, "▰".repeat(f)) + F(t.border, "▱".repeat(bw - f)) + F(t.dim, "  " + lbl));
-        for (const x of r.live.slice(0, 5))
-            push(F(t.accent2, SPIN[r.frame % SPIN.length]) + " " + F(t.text, clipWidth(x.label, inner - 9)) + F(t.faint, ` ${fmtMs(x.ms)}`));
-        for (const x of r.done.slice(-4).reverse())
-            push(F(x.ok ? t.ok : t.err, x.ok ? "✓" : "✗") + " " + F(t.dim, clipWidth(x.label, inner - 9)) + F(t.faint, x.ms >= 50 ? ` ${fmtMs(x.ms)}` : ""));
+    if (d.loop && d.loop.steps.length) {
+        const room = Math.max(4, Math.min(14, height - 26)), lp = renderLoop(d.loop, t, inner, room);
+        head(lp.title, lp.right);
+        for (const h of lp.hits)
+            hits.push({ row: out.length + h.line, kind: "step", step: h.step });
+        for (const l of lp.lines)
+            push(l);
     }
     head("SESSION");
     for (const l of wrap(d.title || "New conversation", inner, 2))

package/dist/src/evals.d.ts

@@ -10,7 +10,13 @@ export type Case = {
     /** Shown to the agent: a command it may run to check its work (and the `finish` gate in the full variant). */
     verify?: string;
     /** Held-out judge: shell command (exit 0 = pass) or function. The agent never sees this. */
-    check: string | ((dir: string) => boolean | string | Promise<boolean | string>);
+    check: string | ((dir: string, run?: {
+        text: string;
+        steps: number;
+        tokens: number;
+    }) => boolean | string | Promise<boolean | string>);
+    /** What the agent would say when it finishes with `solution` applied. Only needed when `check` reads `run.text` (for example "the summary names the file", "the summary admits it could not verify"). */
+    solutionSummary?: string;
     /** A known-good fix (files to overwrite). Used by `selfTest` to prove the case is solvable and its check is not trivially true. */
     solution?: Record<string, string>;
     /** Extra tools for this case (a stubbed web page, a fake MCP server): the agent gets them next to its own. */
@@ -60,7 +66,8 @@ export declare function evaluate(cases: Case[], variants: Variant[], o?: {
     onRow?: (r: Row) => void;
     baseline?: string;
     k?: number;
-    seed?: number;
+    seed?: number; /** Called before each run starts with the rows so far; a returned reason stops the rest (runs already started finish). Used for cost caps. */
+    stop?: (rows: Row[]) => string | undefined;
 }): Promise<{
     rows: Row[];
     summary: {
@@ -73,6 +80,8 @@ export declare function evaluate(cases: Case[], variants: Variant[], o?: {
     }[];
     table: string;
     analysis: Analysis;
+    stopped: string | undefined;
+    skipped: number;
 }>;
 export declare function formatTable(s: {
     variant: string;
@@ -195,3 +204,15 @@ export declare function gateCompare(current: Row[], baseline: Row[], o?: {
     costPct?: number;
     seed?: number;
 }): GateVerdict[];
+/** How hard each task is for one variant (use `baseline`): a task every run passes, or none does, cannot tell two variants apart. Keep tasks whose pass rate is between `low` and `high`. */
+export type Calibration = {
+    case: string;
+    runs: number;
+    passes: number;
+    rate: number;
+    verdict: "keep" | "too easy" | "too hard";
+};
+export declare function calibrate(rows: Row[], variant?: string, band?: {
+    low: number;
+    high: number;
+}): Calibration[];

package/dist/src/evals.js

@@ -12,6 +12,7 @@ import { bootstrap, holm, mean, minDetectable, passPowK, signTest } from "./stat
 /** Run every case x variant x trial in a fresh temp workspace and judge each result with the case's held-out check. `jobs` runs that many sessions at once. */
 export async function evaluate(cases, variants, o = {}) {
     const rows = [], work = [];
+    let stopped;
     for (const c of cases)
         for (const v of variants)
             for (let trial = 0; trial < (o.trials ?? 1); trial++)
@@ -39,7 +40,7 @@ export async function evaluate(cases, variants, o = {}) {
                     try {
                         const out = await v.run(c, dir);
                         Object.assign(row, { steps: out.steps, tokens: out.tokens, events: out.events, prompt: out.prompt, cached: out.cached });
-                        const j = typeof c.check === "string" ? ((await runShell(c.check, dir, 120_000)).startsWith("exit=0") || "check command failed") : await c.check(dir);
+                        const j = typeof c.check === "string" ? ((await runShell(c.check, dir, 120_000)).startsWith("exit=0") || "check command failed") : await c.check(dir, { text: out.text, steps: out.steps, tokens: out.tokens });
                         row.pass = j === true;
                         if (j !== true)
                             row.note = String(j || "failed");
@@ -64,14 +65,18 @@ export async function evaluate(cases, variants, o = {}) {
                     rmSync(dir, { recursive: true, force: true });
                 });
     let next = 0;
-    await Promise.all(Array.from({ length: Math.max(1, Math.min(o.jobs ?? 1, work.length)) }, async () => { while (next < work.length)
-        await work[next++](); }));
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(o.jobs ?? 1, work.length)) }, async () => { while (next < work.length) {
+        stopped ??= o.stop?.(rows);
+        if (stopped)
+            return;
+        await work[next++]();
+    } }));
     rows.sort((a, b) => cases.findIndex((c) => c.name === a.case) - cases.findIndex((c) => c.name === b.case) || variants.findIndex((v) => v.name === a.variant) - variants.findIndex((v) => v.name === b.variant) || a.trial - b.trial);
     const summary = variants.map((v) => {
         const r = rows.filter((x) => x.variant === v.name), avg = (f) => Math.round(r.reduce((n, x) => n + f(x), 0) / (r.length || 1));
         return { variant: v.name, runs: r.length, passRate: r.filter((x) => x.pass).length / (r.length || 1), steps: avg((x) => x.steps), tokens: avg((x) => x.tokens), secs: avg((x) => x.ms) / 1000 };
     });
-    return { rows, summary, table: formatTable(summary), analysis: analyze(rows, { baseline: o.baseline, k: o.k, seed: o.seed }) };
+    return { rows, summary, table: formatTable(summary), analysis: analyze(rows, { baseline: o.baseline, k: o.k, seed: o.seed }), /** why the run ended early, if it did */ stopped, /** runs that never started because of that */ skipped: work.length - next };
 }
 export function formatTable(s) {
     const line = (a) => a.map((x, i) => String(x).padEnd([22, 6, 9, 7, 9, 7][i])).join(" ");
@@ -279,7 +284,8 @@ export async function selfTest(cases) {
                 mkdirSync(dirname(join(dir, f)), { recursive: true });
                 writeFileSync(join(dir, f), t);
             }
-            const judge = async () => (typeof c.check === "string" ? (await runShell(c.check, dir, 60_000)).startsWith("exit=0") : (await c.check(dir)) === true);
+            let said = ""; // what the agent "says": nothing before the fix, the reference summary after it
+            const judge = async () => (typeof c.check === "string" ? (await runShell(c.check, dir, 60_000)).startsWith("exit=0") : (await c.check(dir, { text: said, steps: 0, tokens: 0 })) === true);
             if (await judge()) {
                 out.push({ case: c.name, ok: false, why: "check already passes before any fix" });
                 continue;
@@ -292,6 +298,7 @@ export async function selfTest(cases) {
                 mkdirSync(dirname(join(dir, f)), { recursive: true });
                 writeFileSync(join(dir, f), t);
             }
+            said = c.solutionSummary ?? "";
             out.push((await judge()) ? { case: c.name, ok: true } : { case: c.name, ok: false, why: "check still fails with the reference solution" });
         }
         finally {
@@ -338,3 +345,10 @@ export function gateCompare(current, baseline, o = {}) {
     }
     return out;
 }
+export function calibrate(rows, variant = "baseline", band = { low: 0.3, high: 0.7 }) {
+    const names = [...new Set(rows.filter((r) => !r.excluded && r.variant === variant).map((r) => r.case))];
+    return names.map((name) => {
+        const r = rows.filter((x) => !x.excluded && x.variant === variant && x.case === name), passes = r.filter((x) => x.pass).length, rate = passes / r.length;
+        return { case: name, runs: r.length, passes, rate, verdict: rate > band.high ? "too easy" : rate < band.low ? "too hard" : "keep" };
+    });
+}

package/docs/agent-loop.md

@@ -64,7 +64,7 @@ You type `Fix the failing test in src/math.ts` in the chat and press Enter. The
 **Limits and common problems.**
 
 - The exact step numbers differ from run to run. A plain-text answer without `finish` also ends a run, and then `finished` is `false` in the result.
-- The loop stops after `maxSteps` (the code agent uses 60 in chat and 40 otherwise, unless you set it).
+- The loop stops after `maxSteps` (the code agent uses 60 in chat and 40 otherwise, unless you set it; the eval runner and the plain library harness default to 30). The number is a safety stop against a runaway loop, not a target, and no measurement chose it. In our built-in test suites most tasks finish in about 6 steps and about 1 run in 100 reaches 30. The token budget and the loop and stuck guards are what normally end a run.
 - Not every event appears in every run. Guards only speak when they fire.
 
 **Related pages.** [Architecture](architecture.md), [Library](library.md), [Why: what Amness checks](why.md).
@@ -197,11 +197,24 @@ A mismatch is rejected with the reason. This happens at most twice, so it cannot
 
 **Related pages.** [Budgets and subagents](budgets.md), [Providers and models](providers.md).
 
+## See how other tools shape the loop
+
+The basic loop is the same almost everywhere. Anthropic describes it as *gather context, take action, verify work, repeat* ([Building agents with the Claude Agent SDK](https://claude.com/blog/building-agents-with-the-claude-agent-sdk), read in full). OpenAI describes a Codex turn the same way: the model and its tools go round many times until the model gives a final answer ([Unrolling the Codex agent loop](https://openai.com/index/unrolling-the-codex-agent-loop), seen only as a search summary). A survey of eleven coding agents says they run on hand-written loops, not on agent frameworks ([arXiv 2609.00006](https://arxiv.org/abs/2609.00006), abstract only).
+
+So there are no hidden extra stages. What differs between tools is where they put **gates** around the loop:
+
+- **Gather, act and verify are not separate phases in Amness.** The loop has four phases: `model`, `tools`, `compact` and `verify`. *Gathering* is the model using read-only tools, *acting* is edits and commands, and *verifying* is the model's own test runs, the optional `check` command after each edit, and the verify gate at the end. The strategies `explore-then-act` and `plan-first` are the only things that force an order.
+- **A plan at the first edit.** [PAUSE](https://arxiv.org/html/2610.29870v1) (read in full) holds back the first source edit and asks a separate, stronger planner with no tools for one plan. Then the same agent continues. In the paper, 80% of first edits came after the middle of a run, so the agent has already seen most of the evidence by then. The authors report gains of 6 to 17 points in three harnesses at about 4 cents per task. They also report that the plan helped much less when the planner was the same model as the agent, that most comparisons used one run per task, and that edits made through the shell escape the trigger. `plan-first` in Amness plans at step 0, before anything is read, which is a different design.
+- **Checks after every edit.** Anthropic calls rules-based feedback such as linting "the best form of feedback" and calls an LLM judge "not very robust". Amness has a post-edit `check` command and language-server diagnostics, both off by default.
+- **A fixed pipeline instead of a loop.** [Agentless](https://arxiv.org/abs/2407.01489) (search summary only) runs localise, repair, then validate with generated tests, and no model-chosen actions. Amness's `best-of-n` option belongs to the same "sample several, pick one" family.
+
+These are other people's results on their own benchmarks and models, not measurements of Amness.
+
 ## Learn where these choices come from
 
 Recent work on coding-agent loops points the same way. The loop follows it where this can be done without training a model.
 
-- [An Empirical Study of Harness Design for Coding Agents](https://arxiv.org/abs/2609.20804) (2026): cheap rule-based elision before any LLM summary gave the lowest cost. A recall tool for elided content was almost never used, so Amness has none. Stuck detection and light post-edit checks are cheap ways to stop wasted turns.
+- [An Empirical Study of Harness Design for Coding Agents](https://arxiv.org/abs/2609.20804) (2026): managing the context mattered a lot with a small window (a 35.7 point gap in success at 32k tokens) and little with a large one (2.7 points at 128k). The highest level of context management had the lowest mean cost. A recall tool for elided content was rarely used, so Amness has none. The paper held stuck detection and post-edit checks fixed, so it does not test them; Amness keeps them because they are cheap, not because this paper measured them.
 - [AgentDiet](https://arxiv.org/abs/2509.23586) (trajectory reduction): removing useless, redundant and *expired* information cut input tokens by 40-60% with unchanged results. This is the source of the expired-read and repeated-call rules.
 - [CoACT](https://arxiv.org/abs/2607.02911) and [SWE-Pruner](https://arxiv.org/abs/2601.16746): reads use most of the tokens. Compression must keep what the next action needs, and earlier messages stay untouched so the cache stays valid. Amness uses rules for this. The papers use trained compressors, which would add a dependency.
 - [Don't Break the Cache](https://arxiv.org/abs/2601.06007) (prompt caching for long-horizon agents): caching cut API cost by 41-80% across OpenAI, Anthropic and Google, but naive full-context caching can hurt. Put dynamic content at the end, avoid dynamic tool lists, and keep dynamic tool results out of the cached prefix. [Keeping the Cache Warm Pays](https://arxiv.org/abs/2607.19214) motivates the longer OpenAI retention and the opt-in Claude keep-alive.

package/docs/evaluation.md

@@ -262,6 +262,38 @@ Most public benchmarks need a separate environment for each instance, often a Do
 ## Limits and common problems
 
 - A real run costs real tokens. The command prints the worst case and asks first. Use `--max-tokens` and `--max-steps` (default 30) to limit each session.
+- **A spending cap, when you want one.** Nothing is capped by default. Add `--max-cost 0.50` and Amness stops starting new sessions once the estimated spend reaches 50 cents. Sessions that are already running finish, so with `--jobs 4` you can overshoot by up to four sessions. The result file and the report cover only the runs that finished, and say how many were skipped. The estimate prices prompt, cached and completion tokens from the rate card in `cli/tui/pricing.ts`. It is an estimate, not an invoice, and it needs a model with a known price. After every run Amness prints the estimated spend and the cost per session.
+
+## Check how hard your tasks are
+
+A task that every run passes, or that none passes, cannot tell two variants apart. Before you compare variants, run:
+
+```bash
+amness eval --calibrate --yes
+```
+
+This runs only `baseline`, 5 trials per task (change it with `--trials`), and lists each task's pass rate:
+
+```text
+how hard is each task?  (baseline, 5 trials; keep 30% to 70%)
+  keep       explore-answer             3/5 passed  60%
+  too easy   fix-off-by-one             5/5 passed  100%
+  too hard   cross-module-rename        0/5 passed  0%
+```
+
+Keep the tasks marked `keep`. With only 5 trials the pass rate is rough, so treat a task near the edge of the band (30% or 70%) as undecided and run more trials. A few easy tasks are still worth keeping as a "must not get worse" check.
+
+## Checks that read the agent's final message
+
+A task's `check` can be a function that receives the workspace and, as a second argument, what the agent said when it finished:
+
+```ts
+{ name: "say-where", files: {}, task: "create notes.txt and tell me where it is",
+  check: (dir, run) => existsSync(join(dir, "notes.txt")) && /notes\.txt/.test(run?.text ?? ""),
+  solution: { "notes.txt": "x" }, solutionSummary: "Created ./notes.txt" }
+```
+
+`run` also has `steps` and `tokens`. `solutionSummary` is what the reference solution would say. `--selftest` uses it, so a check that depends on the message is still proven sound (it must fail on silence and pass with the reference message). This works for checks written as functions in a `.ts` or `.js` module, not for JSON tasks with a shell command.
 - The built-in suites are small and made by us. They are a regression test for the guards, not proof of quality.
 - `full+route` needs `--cheap-model`. A variant name that Amness does not know stops with an error that lists the known names.
 - `--variants` and `--baseline`: if the baseline name is not among the variants, Amness uses the first variant.

package/docs/providers.md

@@ -69,6 +69,17 @@ The model list in `/model` (with prices per million tokens as shown in the picke
 - If the chat says "check your key with /provider", the provider answered with an authentication error (401 or 403).
 - The test call checks that the model replies. It does not check that the model can call tools.
 
+## Connect several providers
+
+You can keep a key for every provider at the same time. Each key is saved on its own (in the Keychain, the Linux keyring or DPAPI), so switching provider never overwrites another key.
+
+1. In the chat, type `/connect`. You see every provider with `●` (connected) or `○` (not connected), and where the key comes from: a saved key, or an environment variable.
+2. Choose a provider. If it has no key, Amness asks for one (hidden). If it has one, you can use it now, replace the key, or remove the saved key.
+3. The menu stays open, so you can add the next provider. Press `Esc` when you are done.
+4. Switch between them with `/provider`. Amness uses the saved key of the provider you pick, and asks only if there is none.
+
+An environment variable wins over a saved key for its own provider: `ANTHROPIC_API_KEY` for Claude, `OPENROUTER_API_KEY` (or `OPENAI_API_KEY`) for OpenRouter, and `OPENAI_API_KEY` for the others. Environment keys are never saved. **Limit:** a key from an older version moves to the provider that was active when you first start the new version, so check `/connect` once.
+
 ## Switch provider, model or effort
 
 **How to switch in the chat.**
@@ -152,7 +163,8 @@ In `auto` mode Amness fixes these cases for you:
 
 | Variable | What it does |
 |---|---|
-| `OPENAI_API_KEY` | The key for every provider except `anthropic`. |
+| `OPENAI_API_KEY` | The key for every provider except `anthropic` (and for OpenRouter when `OPENROUTER_API_KEY` is not set). |
+| `OPENROUTER_API_KEY` | The key for `openrouter`. |
 | `ANTHROPIC_API_KEY` | The key for `anthropic`. |
 | `OPENAI_BASE_URL` | The base URL for non-Claude providers. |
 | `AMNESS_MODEL` | The model id. It wins over the saved config. |

package/docs/safety.md

@@ -40,15 +40,15 @@ Notes about the provider:
 
   | System | Where the key is saved | Protection |
   |---|---|---|
-  | macOS | the login Keychain (service `amness`) | encrypted by macOS |
+  | macOS | the login Keychain (service `amness`, one entry per provider, for example `apiKey-anthropic`) | encrypted by macOS |
   | Linux with a desktop keyring | the secret service, through `secret-tool` | encrypted by the keyring |
-  | Windows | `~/.amness/secrets/apiKey.dpapi` | encrypted with DPAPI: only your Windows user on this machine can decrypt it |
+  | Windows | `~/.amness/secrets/apiKey-<provider>.dpapi` | encrypted with DPAPI: only your Windows user on this machine can decrypt it |
   | Docker or Kubernetes | nowhere: kept in memory for that run | nothing is written to the image or a volume. Pass the key as an environment variable or a secret file (`ANTHROPIC_API_KEY_FILE=/run/secrets/anthropic`). |
   | Linux server with no keyring | `~/.amness/config.json` with mode 600 | **not encrypted**, only your user can read it. Prefer an environment variable or your platform's secret manager. |
 
 - Amness passes the key to the system tool on standard input, never on the command line, where other users could see it in the process list.
 - A key that an older version saved in `config.json` is moved to the key store the next time you start Amness, and removed from the file once the move is checked.
-- To remove a saved key, open the settings and clear "API key", or on macOS run `security delete-generic-password -s amness -a apiKey`.
+- To remove a saved key, open the settings and clear "API key", or on macOS run `security delete-generic-password -s amness -a apiKey-anthropic` (use `apiKey-openai` and so on for the others).
 - The environment variable wins over the saved key.
 - Amness does not write keys into a project folder.
 - Commands the agent runs do not get your Amness keys. The variables `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `TAVILY_API_KEY`, `OPENROUTER_API_KEY` and `AMNESS_OTLP` are removed from their environment (`childEnv` in `src/secrets.ts`).

package/docs/usage.md

@@ -60,7 +60,30 @@ If you have no key yet, the first start asks you to pick a provider and paste a
 
 - **Collapsed reads.** A run of reads and searches becomes one row, for example "Explored · 3 files read · 2 searches". Click it to open it. Edits, commands and failures always get their own card. Long diffs show the first 16 rows, and you click to see the rest.
 - **Cards.** Your prompts, the assistant replies (with code blocks that have a copy button), tool cards with a status chip, and diffs with line numbers and word-level marks. Wide terminals show a split view.
-- **Live run view.** The status line says what the agent does now, for example "Reading src/app.ts" or "Waiting for you". It also shows elapsed time, tokens and the step counter. The sidebar has a RUN panel with the think, act and verify steps, running tools and the last finished actions.
+- **Live run view.** The status line says what the agent does now, for example "Reading src/app.ts" or "Waiting for you". It also shows elapsed time, tokens and the step number. The sidebar has a **LOOP** panel that draws the cycle the agent goes round and lights the part that is running:
+
+  ```text
+  LOOP                round 2 · step 14
+  ◉ gather ▸ ✎ act ▸ ✓ verify
+  ⠋ Updating src/app.ts  2.1s
+   12 ◉ Search → Read ×2     3.1s
+      ⚑ nudged: no edits after
+        5 reads
+   13 ✎ Update               0.8s
+   14 ✎ …
+  ```
+
+  - **gather** ◉ (blue): reading, searching, listing files, asking a subagent, shell commands such as `ls` or `git diff`.
+  - **act** ✎ (amber): editing and writing files, and shell commands that change things.
+  - **verify** ✓ (green): shell commands that look like tests, builds, linting or type checks (`npm test`, `pytest`, `tsc`, `cargo test`…), the `check` command, and the verify gate when the agent says it is done.
+
+  The bright stage is the one running now. When two kinds run together, both light up. While the model is thinking and nothing runs, the stage it used last stays visible but dimmer. A test that failed keeps the verify icon red until a later one passes. A **round** is one trip round the cycle: it counts up when the agent gathers or acts again after verifying, so "round 3" means it went back to work twice after checking.
+
+  **These stages are a guess.** Amness reads them from the tool calls, and the loop itself has no such phases. An unusual shell command can be put in the wrong stage, and a step where the model only thinks has no stage.
+
+  Under the cycle you see the latest steps, one line each: the stage icons, the tools used, and the time. Calls of one kind in a row merge (`Read ×2`). A red `✗` means a tool failed. A line with `⚑` means the harness stepped in (a nudge, a loop or stagnation warning, a rejected finish, a denied call, a reviewer finding). It stays visible even when the step is older. **Click a step** to see every tool call with its time. A tall sidebar shows more steps than a short one, and older ones fold into one line. Type **`/loop`** for the whole history of the run, with every turn labelled by its stage. When the run ends, the panel stays as **LAST RUN**.
+
+  The step limit is a safety stop, not a goal, so it stays hidden at first. It appears as "of 60" after half the steps and as a yellow bar with a warning at 80%.
 - **Sidebar** (`Ctrl+B`). It shows the session, a context gauge, token use, todos, changed files and the workspace. The *context* is the text the model can read at one time.
 - **Modals.** Windows for approvals (with the diff, command or plan), the command palette (`Ctrl+P`), pickers and help.
 
@@ -92,7 +115,9 @@ amness --inline
 - Set the environment variable `AMNESS_UI=inline`.
 - Type `/ui inline` in the chat. `/ui full` goes back.
 
-**What you see.** Each tool call is one line, for example `● Read(src/a.ts)`, with a short result. `Ctrl+O` shows everything. Each edit shows a diff with line numbers. A live todo list, a status line (elapsed time and tokens) and a footer (permission mode, git branch, model, context gauge, estimated cost) stay visible. Tool bullets have a colour for each kind and turn red when the tool fails. With `NO_COLOR=1`, or when the output is piped, Amness uses plain text.
+**What you see.** Each tool call is one line, for example `● Read(src/a.ts)`, with a short result. `Ctrl+O` shows everything. Each edit shows a diff with line numbers.
+
+In the full-screen view, the **most recent** tool call stays open. A shell command shows `$ the command` and its output, up to 20 lines. When the next tool call starts, the earlier one folds to one line again. Click a line to open or close it yourself. If the output is longer than 20 lines, the card says how many are hidden. Type `/output` to read all of it in a pager (`/output 2` for the call before). A lone file read does not open itself. A live todo list, a status line (elapsed time and tokens) and a footer (permission mode, git branch, model, context gauge, estimated cost) stay visible. Tool bullets have a colour for each kind and turn red when the tool fails. With `NO_COLOR=1`, or when the output is piped, Amness uses plain text.
 
 ## Type in the input box
 
@@ -204,14 +229,36 @@ The file is attached to your message. In the transcript you see a note such as "
 
 ## Slash commands
 
-Type `/` in the input box to see the popup. `/help` lists the commands in the chat.
+Type `/` in the input box to see the popup. It lists **every** command, grouped under titles: Conversation, Model & keys, Safety, Quality, Project, Extend and Interface. Your own and plugin commands go under Custom.
+
+- Move with `↑` and `↓`. `PgUp` and `PgDn` jump eight at a time. The top of the box shows where you are, for example `12/60 ↓`.
+- Keep typing to filter. The titles disappear and the best match comes first.
+- `Tab` or `Enter` picks the command. `Esc` closes the popup.
+
+**Completion after the command.** Type a command and a space, and the popup lists the values it accepts, each with a short explanation:
+
+| You type | The popup shows |
+|---|---|
+| `/mode ` | `default`, `acceptEdits`, `plan`, `yolo` |
+| `/model ` | the known models with their hints, then every other model your provider reports (OpenAI, Claude, OpenRouter, Ollama and other OpenAI-compatible servers). Amness asks the provider once and remembers the answer for ten minutes. If the provider cannot be reached, you still see the known models. |
+| `/models ` | the roles (`explore`, `compact`, `review`, `cheap`, `fallback`), then the models |
+| `/effort `, `/api `, `/theme `, `/shell `, `/strategy `, `/prompt `, `/sandbox `, `/diffmode `, `/ui ` | their choices |
+| `/keepwarm `, `/lsp `, `/approver `, `/reviewer `, `/router `, `/mouse ` | `on`, `off` |
+| `/branch ` | your turns, numbered, with the first words of each |
+| `/memory `, `/permissions `, `/copy `, `/pr `, `/report `, `/mcp `, `/budget `, `/check `, `/verify ` | their sub-commands or common values |
+
+`Enter` picks the highlighted value and runs the command. For commands that take free text or a number (`/model`, `/budget`, `/check`, `/verify`), `Enter` runs exactly what you typed unless you moved into the list with the arrow keys, so a model id that is not in the list still works. `Tab` always completes. Commands that need more words (`/memory add`, `/permissions allow`) add a space and wait.
+
+`/help` lists the commands in the chat.
 
 ### Models and providers
 
 | Command | What it does |
 |---|---|
 | `/model [id]` | Switches the model. Without an id, it opens a picker. |
-| `/provider` | Switches the provider (OpenAI, Claude, OpenRouter, Ollama, custom). The choice is saved. |
+| `/output [n]` (alias `/last`) | Shows the full output of the last tool call in a pager. `/output 2` shows the one before. |
+| `/provider` | Switches the provider (OpenAI, Claude, OpenRouter, Ollama, custom). The list shows `●` for providers that are connected and `○` for those that are not. Each provider uses its own saved key. |
+| `/connect` (alias `/keys`) | Opens the key manager. See which providers are connected, add a key for any of them, replace or remove one, or switch to it. Add as many as you want. |
 | `/effort [level]` | Sets the reasoning effort. Levels depend on the model. `default` lets the model decide. |
 | `/api [auto\|responses\|chat]` | Chooses the OpenAI wire API. It does not apply to Claude. |
 | `/models [role model]` | Uses another model for `explore`, `compact`, `review` or `cheap`. `/models fallback <model>` sets a model for when the main one is overloaded or not reachable. Use `off` to clear a role. |