amness 0.1.3 → 0.1.4

package/dist/cli/amness.js

@@ -610,7 +610,7 @@ async function main() {
     if (quick === "--help" || quick === "-h" || quick === "help")
         return void console.log(TOP_HELP);
     if (quick === "eval" && argv.length > 1) {
-        process.exitCode = await runEval(argv.slice(1), { model: makeModel, modelFor: (id) => createModel({ ...cfg, model: id }), maxTokens: cfg.maxTokens, confirm });
+        process.exitCode = await runEval(argv.slice(1), { model: makeModel, modelName: cfg.model, modelFor: (id) => createModel({ ...cfg, model: id }), maxTokens: cfg.maxTokens, confirm });
         return;
     }
     if (quick === "bench") {

package/dist/cli/evalcli.d.ts

@@ -1,7 +1,24 @@
-import { type Model } from "../src/index.ts";
+import { type Model, type Row } from "../src/index.ts";
 export declare const EVAL_HELP: string;
+/** Estimated dollars for some rows (prompt, cached and completion tokens priced separately; the rate card is in pricing.ts). Null when the model's price is unknown. */
+export declare function rowsCost(model: string, rows: Row[]): number | null;
+/** What makes a result repeatable: the Amness version and git commit (when run from a clone), and a hash of the tasks (their files, text and checks). */
+export declare function provenance(cases: {
+    name: string;
+    files: Record<string, string>;
+    task: string;
+    verify?: string;
+    check: unknown;
+    solution?: Record<string, string>;
+}[]): {
+    version: string | undefined;
+    commit: string | undefined;
+    suite: string;
+    tasks: number;
+};
 export declare function runEval(argv: string[], deps: {
     model: () => Model;
+    modelName?: string;
     modelFor?: (id: string) => Model;
     maxTokens: number;
     confirm: (q: string) => Promise<boolean>;

package/dist/cli/evalcli.js

@@ -7,10 +7,13 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
     return path;
 };
 // `amness eval`: compare harness variants on a task suite, with statistics you can act on.
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
-import { BUILTIN_METRICS, gateCompare, importPlugins, tune, writeProfile, adversarialCases, hardCases, analyze, builtinCases, evaluate, recommend, leakAudit, loadSuite, suiteToCases, formatReport, harnessVariant, loadCases, selfTest, variantNames, VARIANT_HELP } from "../src/index.js";
+import { BUILTIN_METRICS, gateCompare, importPlugins, tune, writeProfile, adversarialCases, hardCases, analyze, builtinCases, evaluate, recommend, leakAudit, loadSuite, suiteToCases, formatReport, calibrate, harnessVariant, loadCases, selfTest, variantNames, VARIANT_HELP } from "../src/index.js";
 import { bold, dim, green, red } from "./ui.js";
+import { estimateCost, priceOf } from "./tui/pricing.js";
 import { pathToFileURL } from "node:url";
 async function casesFromModule(file) {
     const mod = await import(__rewriteRelativeImportExtension(pathToFileURL(file).href + `?v=${Date.now()}`)), list = mod.default ?? mod.cases;
@@ -46,7 +49,32 @@ export const EVAL_HELP = `usage: amness eval [flags]
   --out <file>         where to save raw results (default .amness/evals/<time>.json)
   --report <file>      re-analyse saved results; runs nothing
   --selftest           check that every task is solvable and its held-out check is sound; runs no model
+  --max-cost <dollars> stop starting new sessions once the estimated spend reaches this amount (sessions already running finish). Off by default. Needs a model with a known price.
+  --calibrate          how hard is each task? runs only \`baseline\` (5 trials unless you set --trials) and lists each task's pass rate: keep 30% to 70%, drop tasks that are too easy or too hard
   --yes                skip the cost confirmation`;
+/** Estimated dollars for some rows (prompt, cached and completion tokens priced separately; the rate card is in pricing.ts). Null when the model's price is unknown. */
+export function rowsCost(model, rows) {
+    if (!priceOf(model))
+        return null;
+    return rows.reduce((n, r) => n + (estimateCost(model, r.prompt ?? r.tokens, Math.max(0, r.tokens - (r.prompt ?? r.tokens)), r.cached ?? 0) ?? 0), 0);
+}
+/** What makes a result repeatable: the Amness version and git commit (when run from a clone), and a hash of the tasks (their files, text and checks). */
+export function provenance(cases) {
+    let commit;
+    try {
+        commit = execFileSync("git", ["rev-parse", "HEAD"], { cwd: new URL("..", import.meta.url), encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
+    }
+    catch { /* not a git checkout (an installed package) */ }
+    let version;
+    try {
+        version = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
+    }
+    catch { /* no package.json next to us */ }
+    const h = createHash("sha256");
+    for (const x of [...cases].sort((a, b) => a.name.localeCompare(b.name)))
+        h.update(JSON.stringify([x.name, x.files, x.task, x.verify ?? "", typeof x.check === "function" ? x.check.toString() : x.check, x.solution ?? {}]));
+    return { version, commit, suite: h.digest("hex").slice(0, 16), tasks: cases.length };
+}
 export async function runEval(argv, deps) {
     const val = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; }, has = (f) => argv.includes(f);
     if (has("--help") || has("-h")) {
@@ -108,20 +136,41 @@ export async function runEval(argv, deps) {
                 console.log(`${r.ok ? green("ok ") : red("bad")} ${r.case}${r.why ? dim("  " + r.why) : ""}`);
             return res.every((r) => r.ok) ? 0 : 1;
         }
-        const names = variantNames(val("--variants") ?? "ab"), trials = Math.max(1, Number(val("--trials") ?? 3)), jobs = Math.max(1, Number(val("--jobs") ?? 1));
+        const calib = has("--calibrate"), names = variantNames(val("--variants") ?? (calib ? "baseline" : "ab")), trials = Math.max(1, Number(val("--trials") ?? (calib ? 5 : 3))), jobs = Math.max(1, Number(val("--jobs") ?? 1));
         const maxTokens = Number(val("--max-tokens") ?? deps.maxTokens), maxSteps = Number(val("--max-steps") ?? 30);
         const model = deps.model(), cheap = val("--cheap-model") && deps.modelFor ? deps.modelFor(val("--cheap-model")) : undefined, variants = names.map((n) => harnessVariant(model, n, { maxTokens, maxSteps, cheap, plugins, modelFor: deps.modelFor }));
         const runs = cases.length * variants.length * trials;
         console.log(`${bold("plan")}  ${cases.length} tasks × ${variants.length} variants (${names.join(", ")}) × ${trials} trials = ${bold(String(runs))} agent sessions, up to ${maxTokens.toLocaleString()} tokens each`);
+        const mname = deps.modelName ?? "", maxCost = val("--max-cost") === undefined ? undefined : Number(val("--max-cost"));
+        if (maxCost !== undefined && !(maxCost > 0))
+            throw new Error("--max-cost needs a positive number of dollars, for example --max-cost 0.50");
+        if (maxCost !== undefined && !priceOf(mname))
+            throw new Error(`--max-cost needs a known price for the model "${mname}" (see cli/tui/pricing.ts); without one Amness cannot estimate the spend`);
+        const p = priceOf(mname);
+        if (p)
+            console.log(dim(`worst case about $${(runs * maxTokens * p[2] / 1e6).toFixed(2)} (every token priced as output; real runs are far cheaper: most of a session is cached input)${maxCost !== undefined ? ` · stops at about $${maxCost.toFixed(2)}` : " · no spending cap (--max-cost sets one)"}`));
         if (!has("--yes") && !(await deps.confirm(`This calls your model up to ${(runs * maxTokens).toLocaleString()} tokens in the worst case. Continue?`)))
             return 1;
-        const { rows, analysis } = await evaluate(cases, variants, {
+        const { rows, analysis, stopped, skipped } = await evaluate(cases, variants, {
+            stop: maxCost === undefined ? undefined : (rs) => { const spent = rowsCost(mname, rs) ?? 0; return spent >= maxCost ? `reached the cost cap: about $${spent.toFixed(3)} spent of $${maxCost.toFixed(2)}` : undefined; },
             metrics: Object.keys(useMetrics).length ? useMetrics : undefined, trials, jobs, baseline: val("--baseline"), k: Math.min(3, trials), seed: val("--seed") ? Number(val("--seed")) : undefined,
             onRow: (r) => console.error(`  ${r.pass ? green("PASS") : red("FAIL")} ${r.case.padEnd(22)} ${r.variant.padEnd(18)} ${dim(`${r.steps} steps · ${r.tokens} tok · ${(r.ms / 1000).toFixed(1)}s${r.note ? " · " + r.note : ""}`)}`),
         });
         const out = resolve(val("--out") ?? join(".amness", "evals", new Date().toISOString().replace(/[:.]/g, "-") + ".json"));
         mkdirSync(join(out, ".."), { recursive: true });
-        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), baseline: analysis.baseline, k: Math.min(3, trials), variants: names, trials, rows }, null, 1));
+        const spent = rowsCost(mname, rows);
+        writeFileSync(out, JSON.stringify({ at: new Date().toISOString(), model: mname || undefined, ...provenance(cases), cliArgs: argv.filter((a, i) => !/key|token/i.test(a) && !/key|token/i.test(argv[i - 1] ?? "")), baseline: analysis.baseline, k: Math.min(3, trials), variants: names, trials, estimatedCost: spent ?? undefined, stopped, rows }, null, 1));
+        if (stopped)
+            console.log(`\n${red("stopped early")}: ${stopped}. ${skipped} planned session${skipped === 1 ? "" : "s"} did not run, so the results below cover only the runs that finished.`);
+        if (spent !== null)
+            console.log(dim(`estimated spend: about $${spent.toFixed(3)} for ${rows.length} session${rows.length === 1 ? "" : "s"}${rows.length ? ` ($${(spent / rows.length).toFixed(4)} each)` : ""}  (an estimate from token counts, not an invoice)`));
+        if (calib) {
+            console.log("\n" + bold("how hard is each task?") + dim(`  (baseline, ${trials} trials; keep 30% to 70%)`));
+            for (const x of calibrate(rows, names[0]))
+                console.log(`  ${x.verdict === "keep" ? green("keep      ") : red(x.verdict.padEnd(10))} ${x.case.padEnd(26)} ${x.passes}/${x.runs} passed  ${dim((x.rate * 100).toFixed(0) + "%")}`);
+            console.log(dim(`raw results: ${out}`));
+            return 0;
+        }
         if (val("--profile-out")) {
             const rec = recommend(analysis, names.includes("full") ? "full" : analysis.baseline), pm = val("--profile-model") ?? "*";
             console.log("\nguard verdicts:");

package/dist/cli/tui/commands.js

@@ -6,6 +6,7 @@ import { collectFiles, effortsFor, loadMcp, readBundle, writeBundle } from "../.
 import { CLAUDE_MODELS, OPENAI_MODELS, PROVIDERS, connections, keyFor, storeKey } from "../config.js";
 import { noStore, where } from "../keystore.js";
 import { fetchModels } from "../models.js";
+import { STAGE_GLYPH, roundOf, stagesOf } from "./looppanel.js";
 import { bold, cyan, dim, green, red, yellow } from "../ui.js";
 import { estimateCost, fmtCost, fmtDuration, fmtTokens } from "./pricing.js";
 import { codeBlocks } from "./clipboard.js";
@@ -318,10 +319,12 @@ export function buildCommands() {
                     return a.say(dim("no loop activity yet: send a task first"));
                 const tot = T.reduce((n, s) => n + s.completion + s.prompt, 0), ms = T.reduce((n, s) => n + s.ms, 0);
                 const waste = cacheWaste(a.tracer?.turns ?? []), wasteCost = waste ? estimateCost(a.cfg.model, waste, 0) : null, wasteNote = waste > 500 ? `${fmtTokens(waste)} tokens re-read at full price after cache breaks${wasteCost !== null ? ` (≈ ${fmtCost(wasteCost - estimateCost(a.cfg.model, waste, 0, waste))} wasted)` : ""} · ` : "";
-                const out = ["", bold(`Agent loop · ${T.length} turn${T.length === 1 ? "" : "s"} · ${fmtDuration(ms)} · ${fmtTokens(tot)} tokens`), dim(`${wasteNote}${a.trimmed ? `trimmed ~${fmtTokens(Math.round(a.trimmed / 4))} tokens of stale or noisy output · ` : ""}${a.cacheBreaks ? `${a.cacheBreaks} cache break${a.cacheBreaks === 1 ? "" : "s"} · ` : ""}each turn: the model answers, its tool calls run, results go back; guards are shown where they fired`), ""];
+                const rounds = roundOf(T.map((s) => ({ ...s })));
+                const out = ["", bold(`Agent loop · ${T.length} turn${T.length === 1 ? "" : "s"} · ${rounds} round${rounds === 1 ? "" : "s"} · ${fmtDuration(ms)} · ${fmtTokens(tot)} tokens`), dim("◉ gather: read, search · ✎ act: edit, run · ✓ verify: test, build · guessed from the tool calls"), dim(`${wasteNote}${a.trimmed ? `trimmed ~${fmtTokens(Math.round(a.trimmed / 4))} tokens of stale or noisy output · ` : ""}${a.cacheBreaks ? `${a.cacheBreaks} cache break${a.cacheBreaks === 1 ? "" : "s"} · ` : ""}each turn: the model answers, its tool calls run, results go back; guards are shown where they fired`), ""];
                 for (const s of T) {
                     const pad = String(s.n).padStart(2);
-                    out.push(`${cyan(`${pad} ●`)} ${bold("model")} ${dim(`${fmtDuration(s.ms)} · ${fmtTokens(s.prompt)} in · ${fmtTokens(s.completion)} out${s.cached ? ` · ${fmtTokens(s.cached)} cached` : ""}`)}`);
+                    const st = stagesOf(s.tools);
+                    out.push(`${cyan(`${pad} ●`)} ${bold("model")}${st.length ? " " + cyan(st.map((x) => `${STAGE_GLYPH[x]} ${x}`).join(" + ")) : ""} ${dim(`${fmtDuration(s.ms)} · ${fmtTokens(s.prompt)} in · ${fmtTokens(s.completion)} out${s.cached ? ` · ${fmtTokens(s.cached)} cached` : ""}`)}`);
                     s.tools.forEach((x, i) => out.push(`     ${dim(i === s.tools.length - 1 && !s.notes.length ? "└─" : "├─")} ${x.ok ? green("✓") : red("✗")} ${toolCall(x.name, x.args)} ${dim(fmtDuration(x.ms))}${s.parallel && s.parallel > 1 && i === 0 ? dim(`  (${s.parallel} calls this turn)`) : ""}`));
                     s.notes.forEach((n, i) => out.push(`     ${dim(i === s.notes.length - 1 ? "└─" : "├─")} ${yellow("⚑")} ${dim(n)}`));
                     if (!s.tools.length && !s.notes.length)

package/dist/cli/tui/full.d.ts

@@ -59,6 +59,10 @@ export declare class FullView {
     /** Keep a toggled block's header where it is on screen. */
     private pin;
     private drawEmpty;
+    /** Steps the user opened in the LOOP section by clicking. */
+    private openSteps;
+    private openFor;
+    private loopData;
     private drawSidebar;
     private drawStatus;
     private editorRows;

package/dist/cli/tui/full.js

@@ -1,6 +1,7 @@
 // The full-screen UI: header, scrollable transcript, sidebar, input panel, modals, toasts and mouse handling, drawn on a cell canvas.
 import { copyToClipboard } from "./clipboard.js";
 import { menuRows } from "./complete.js";
+import { capNote } from "./looppanel.js";
 import { basename } from "node:path";
 import { A, mix } from "./ansi.js";
 import { brandOfModel } from "./brand.js";
@@ -332,6 +333,21 @@ export class FullView {
         c.ansi(x0 + Math.max(0, Math.floor((w - cellWidth(TIPS[this.tipIdx]) - 5) / 2)), Math.min(y0 + h - 2, ry + 2), tip);
     }
     // ------------------------------------------------------------------ sidebar
+    /** Steps the user opened in the LOOP section by clicking. */
+    openSteps = new Set();
+    openFor = 0;
+    loopData(now) {
+        const app = this.app, steps = app.trace.filter((s) => s.t0 >= app.runStart - 50); // this run's steps (the trace spans the whole session)
+        if (!steps.length)
+            return undefined;
+        if (this.openFor !== app.runStart) {
+            this.openFor = app.runStart;
+            this.openSteps.clear();
+        } // step numbers start again in a new run
+        return { live: app.running, phase: app.phase?.phase ?? "model", cap: app.phase?.max ?? 30, now, frame: app.frame, steps,
+            elapsedMs: app.running ? now - app.runStart : Math.max(...steps.map((s) => s.t0 + s.ms)) - app.runStart,
+            inflight: [...app.inflight.values()].map((x) => ({ label: toolVerb(x.name, x.args), ms: now - x.t0, agent: x.agent, name: x.name, args: x.args })), open: [...this.openSteps] };
+    }
     drawSidebar(c, t, x0, y0, w, h) {
         const app = this.app, u = app.session.usage, win = contextWindow(app.cfg.model), now = Date.now();
         const mkey = `${app.session.messages.length}|${app.lastPrompt}`;
@@ -355,15 +371,19 @@ export class FullView {
             sandbox: app.cfg.sandbox,
             procs: app.agent.procs?.list().map((p) => ({ id: p.id, name: p.name, status: p.status, secs: Math.round((now - p.startedAt) / 1000) })),
             running: app.running, elapsedMs: now - this.startedAt, tip: TIPS[this.tipIdx],
-            run: app.running ? { phase: app.phase?.phase ?? "model", step: app.phase?.step ?? 0, max: app.phase?.max ?? 30, elapsedMs: now - app.runStart, frame: app.frame,
-                live: [...app.inflight.values()].map((x) => ({ label: toolVerb(x.name, x.args), ms: now - x.t0, agent: x.agent })), done: app.activity.map((x) => ({ label: toolVerb(x.name, x.args, true), ok: x.ok, ms: x.ms })) } : undefined,
+            loop: this.loopData(now),
         }, t, w, h);
         rows.forEach((r, i) => c.put(x0 + 1, y0 + i, r, w - 1));
         for (let i = 0; i < h; i++)
             c.text(x0, y0 + i, "│", { fg: t.border, bg: t.surface });
         for (const hh of hits)
             if (hh.row < h)
-                this.regions.push({ x: x0, y: y0 + hh.row, w, h: 1, kind: "change", click: () => hh.blockId !== undefined && this.scrollToBlock(hh.blockId) });
+                this.regions.push({ x: x0, y: y0 + hh.row, w, h: 1, kind: "change", click: () => { if (hh.kind === "step" && hh.step !== undefined) {
+                        this.openSteps.has(hh.step) ? this.openSteps.delete(hh.step) : this.openSteps.add(hh.step);
+                        this.invalidate();
+                    }
+                    else if (hh.blockId !== undefined)
+                        this.scrollToBlock(hh.blockId); } });
     }
     // ------------------------------------------------------------------ status line, input, footer
     drawStatus(c, t, x, y, w, now) {
@@ -374,7 +394,7 @@ export class FullView {
                 : app.mode === "plan" ? "Planning" : app.thinkBuf.trim() ? "Thinking" : app.streamBuf ? "Writing" : "Working", chars = [...clipWidth(verb, Math.max(8, w - 44)) + "…"];
             const shim = chars.map((ch, i) => F(mix(t.accent, 0xffffff, Math.max(0, 1 - Math.abs(i - ((f * 0.9) % (chars.length + 8) - 4)) / 3)), ch)).join("");
             const spark = app.runSeries.length > 1 ? " " + F(t.accent2, app.runSeries.slice(-10).map((v) => BARS[Math.min(7, Math.round((v / Math.max(1, ...app.runSeries)) * 7))]).join("")) : "";
-            c.ansi(x + 1, y, F(mix(t.accent, t.accent2, (f % 20) / 20), glyph) + " " + BOLD(shim) + F(t.dim, ` ${fmtDuration(now - app.runStart)} · ↓ ${fmtTokens(app.runTokens)}`) + (app.phase ? F(t.faint, ` · step ${app.phase.step + 1}/${app.phase.max}`) : "") + spark + F(t.faint, "   esc to interrupt"), {}, w - 2);
+            c.ansi(x + 1, y, F(mix(t.accent, t.accent2, (f % 20) / 20), glyph) + " " + BOLD(shim) + F(t.dim, ` ${fmtDuration(now - app.runStart)} · ↓ ${fmtTokens(app.runTokens)}`) + (app.phase ? (() => { const n = app.phase.step + 1, cap = capNote(n, app.phase.max); return F(cap.level === "warn" ? t.warn : t.faint, ` · step ${n}${cap.level === "none" ? "" : cap.level === "dim" ? ` of ${app.phase.max}` : ` of ${app.phase.max}, near the limit`}`); })() : "") + spark + F(t.faint, "   esc to interrupt"), {}, w - 2);
             if (app.thinkBuf.trim() && app.queued.length === 0) { /* reasoning summary lives in the transcript's thinking block */ }
         }
         app.queued.slice(0, 2).forEach((q, i) => c.ansi(x + 1, y + (app.running ? 1 : 0) + i, F(t.warn, "▸ queued ") + F(t.dim, clipWidth(q.text.replace(/\s+/g, " "), w - 14)), {}, w - 2));

package/dist/cli/tui/looppanel.d.ts

@@ -0,0 +1,71 @@
+import type { Theme } from "./themes.ts";
+export type LoopStep = {
+    n: number;
+    t0: number;
+    ms: number;
+    tools: {
+        name: string;
+        args: string;
+        ms: number;
+        ok: boolean;
+    }[];
+    notes: string[];
+    parallel?: number;
+};
+export type LoopData = {
+    /** the run is still going */ live: boolean;
+    phase: string;
+    cap: number;
+    now: number;
+    frame: number;
+    elapsedMs: number;
+    /** this run's steps, oldest first; while live, the last one is the current step */ steps: LoopStep[];
+    /** tool calls in flight right now (name and args let the panel tell which stage they belong to) */ inflight: {
+        label: string;
+        ms: number;
+        agent?: string;
+        name?: string;
+        args?: string;
+    }[];
+    /** steps the user opened by clicking */ open: number[];
+};
+export type LoopHit = {
+    line: number;
+    step: number;
+};
+/** Notes that mean "the harness stepped in" (shown in the warning colour, and kept visible); everything else (cache, routing, trimming) is background. */
+export declare const isIntervention: (note: string) => boolean;
+/** "Read ×2 → Edit → Run": neighbouring calls of one kind merge, so a burst of reads is one short phrase. */
+export declare function stepSummary(tools: {
+    name: string;
+}[]): string;
+/** Wall time of a step: the model call, then its tools (they overlap when run in parallel). */
+export declare const stepMs: (s: LoopStep) => number;
+/** The cap is a safety stop, not a goal, so say nothing about it until it could matter. */
+export declare function capNote(step: number, cap: number): {
+    text: string;
+    level: "none" | "dim" | "warn";
+};
+export type Stage = "gather" | "act" | "verify";
+export declare const STAGES: Stage[];
+export declare const STAGE_GLYPH: Record<Stage, string>;
+/** Which part of the cycle a tool call belongs to. A guess from the tool and, for shell commands, the command line; undefined for planning and bookkeeping tools. */
+export declare function stageOfTool(name: string, args?: string): Stage | undefined;
+/** The stages a step touched, in cycle order. */
+export declare const stagesOf: (tools: {
+    name: string;
+    args: string;
+}[]) => Stage[];
+/** How many times the agent has come back round: a new round starts when it gathers or acts again after verifying. */
+export declare function roundOf(steps: LoopStep[]): number;
+/** Which stages to light: those running now, and (while the model is thinking and nothing runs) the last one used, dimmed. */
+export declare function litStages(d: LoopData): {
+    now: Set<Stage>;
+    recent: Set<Stage>;
+};
+export declare function renderLoop(d: LoopData, t: Theme, inner: number, maxRows: number): {
+    lines: string[];
+    hits: LoopHit[];
+    title: string;
+    right: string;
+};

package/dist/cli/tui/looppanel.js

@@ -0,0 +1,218 @@
+// The LOOP section of the sidebar: the cycle the agent goes round (gather context, act, verify), which part of it is running now, and the latest steps, with the harness's
+// interventions shown where they happened. The stages are INFERRED from the tool calls (the harness has no gather/act/verify phases of its own): reads and searches gather,
+// edits and most commands act, test/build/lint/typecheck commands and the verify gate verify. It adapts to the room it has: older steps fold into one summary line, the step cap
+// appears only when it starts to matter, and a clicked step opens to show every tool call and note. The full history is the /loop command.
+import { mix } from "./ansi.js";
+import { BOLD, F } from "./rich.js";
+import { clipWidth } from "./text.js";
+import { toolLabel } from "./view.js";
+const SPIN = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏";
+const fmtMs = (ms) => (ms < 1000 ? `${Math.round(ms)}ms` : ms < 60_000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.floor(ms / 60_000)}m${Math.round((ms % 60_000) / 1000)}s`);
+/** Notes that mean "the harness stepped in" (shown in the warning colour, and kept visible); everything else (cache, routing, trimming) is background. */
+export const isIntervention = (note) => /nudg|reject|denied|loop|stagnat|stuck|budget|step limit|reviewer|wrap|repair/i.test(note);
+/** "Read ×2 → Edit → Run": neighbouring calls of one kind merge, so a burst of reads is one short phrase. */
+export function stepSummary(tools) {
+    const parts = [];
+    for (const t of tools) {
+        const label = toolLabel(t.name), last = parts[parts.length - 1];
+        if (last && last.label === label)
+            last.n++;
+        else
+            parts.push({ label, n: 1 });
+    }
+    return parts.map((p) => (p.n > 1 ? `${p.label} ×${p.n}` : p.label)).join(" → ");
+}
+/** Wall time of a step: the model call, then its tools (they overlap when run in parallel). */
+export const stepMs = (s) => s.ms + (s.parallel && s.parallel > 1 ? Math.max(0, ...s.tools.map((x) => x.ms)) : s.tools.reduce((n, x) => n + x.ms, 0));
+/** The cap is a safety stop, not a goal, so say nothing about it until it could matter. */
+export function capNote(step, cap) {
+    const r = step / Math.max(1, cap);
+    return r >= 0.8 ? { text: `${step}/${cap}: close to the step limit`, level: "warn" } : r >= 0.5 ? { text: `of ${cap} max`, level: "dim" } : { text: "", level: "none" };
+}
+export const STAGES = ["gather", "act", "verify"];
+export const STAGE_GLYPH = { gather: "◉", act: "✎", verify: "✓" };
+const GATHER_TOOLS = new Set(["read_file", "list_files", "grep", "repo_map", "find_refs", "impact", "web_search", "fetch_url", "memory_search", "tool_search", "skill", "task", "sh_list", "sh_read"]);
+const NEUTRAL_TOOLS = new Set(["todo_write", "memory_save", "memory_forget", "exit_plan_mode", "finish"]); // planning and bookkeeping: not part of the cycle
+const VERIFY_CMD = /\b(tests?|pytest|jest|vitest|mocha|tsc|typecheck|type-check|lint|eslint|ruff|mypy|pyright|flake8|clippy|rspec|phpunit|verify|check)\b|\b(cargo|go|npm|yarn|pnpm|make|gradle|mvn|dotnet|swift)\s+(run\s+)?(test|build|check|vet|lint|clippy|verify)\b|node\s+--test|\bbuild\b/i;
+const LOOK_CMD = /^\s*(ls|cat|head|tail|less|grep|rg|find|tree|pwd|wc|stat|file|which|du|df|git\s+(status|diff|log|show|branch|blame)|sed\s+-n)\b/;
+/** Which part of the cycle a tool call belongs to. A guess from the tool and, for shell commands, the command line; undefined for planning and bookkeeping tools. */
+export function stageOfTool(name, args = "") {
+    if (NEUTRAL_TOOLS.has(name))
+        return undefined;
+    if (GATHER_TOOLS.has(name))
+        return "gather";
+    if (name === "sh" || name === "sh_start") {
+        let cmd = "";
+        try {
+            cmd = String(JSON.parse(args).cmd ?? "");
+        }
+        catch {
+            cmd = args;
+        }
+        return LOOK_CMD.test(cmd) ? "gather" : VERIFY_CMD.test(cmd) ? "verify" : "act";
+    }
+    return "act";
+}
+/** The stages a step touched, in cycle order. */
+export const stagesOf = (tools) => STAGES.filter((st) => tools.some((x) => stageOfTool(x.name, x.args) === st));
+/** How many times the agent has come back round: a new round starts when it gathers or acts again after verifying. */
+export function roundOf(steps) {
+    let round = 1, verified = false;
+    for (const s of steps) {
+        const st = stagesOf(s.tools);
+        if (verified && (st.includes("gather") || st.includes("act"))) {
+            round++;
+            verified = false;
+        }
+        if (st.includes("verify"))
+            verified = true;
+    }
+    return round;
+}
+/** Which stages to light: those running now, and (while the model is thinking and nothing runs) the last one used, dimmed. */
+export function litStages(d) {
+    const now = new Set(), recent = new Set(), cur = d.steps[d.steps.length - 1];
+    if (d.live) {
+        if (d.phase === "verify")
+            now.add("verify");
+        else if (d.phase === "tools") {
+            for (const x of d.inflight) {
+                const st = x.name ? stageOfTool(x.name, x.args) : undefined;
+                if (st)
+                    now.add(st);
+            }
+            if (!now.size && cur) {
+                const last = cur.tools[cur.tools.length - 1];
+                const st = last && stageOfTool(last.name, last.args);
+                if (st)
+                    now.add(st);
+            }
+        }
+    }
+    if (!now.size)
+        for (let i = d.steps.length - 1; i >= 0; i--) {
+            const st = stagesOf(d.steps[i].tools);
+            if (st.length) {
+                st.forEach((x) => recent.add(x));
+                break;
+            }
+        }
+    return { now, recent };
+}
+export function renderLoop(d, t, inner, maxRows) {
+    const lines = [], hits = [];
+    const cur = d.live ? d.steps[d.steps.length - 1] : undefined, round = roundOf(d.steps);
+    const tools = d.steps.reduce((n, s) => n + s.tools.length, 0), failed = d.steps.reduce((n, s) => n + s.tools.filter((x) => !x.ok).length, 0);
+    const color = { gather: t.tool.read, act: t.tool.edit, verify: t.ok };
+    const title = d.live ? "LOOP" : "LAST RUN";
+    const cap = cur ? capNote(cur.n, d.cap) : { text: "", level: "none" };
+    const right = d.live && cur ? F(t.dim, `round ${round} · step ${cur.n}${cap.level === "dim" ? " of " + d.cap : ""}`) : F(t.dim, `${Math.round(d.elapsedMs / 1000)}s`);
+    const spin = F(t.accent2, SPIN[d.frame % SPIN.length]);
+    // the cycle: the stage that is running is bright, the one just used (while the model thinks) is half-bright, the rest are dim. A verify that failed stays red until the next one passes.
+    const { now, recent } = litStages(d);
+    let verifyFailed = false;
+    for (const s of d.steps) {
+        const v = s.tools.filter((x) => stageOfTool(x.name, x.args) === "verify");
+        if (v.length)
+            verifyFailed = v.some((x) => !x.ok);
+    }
+    const compact = inner < 27;
+    const cycle = STAGES.map((st) => {
+        const c = st === "verify" && verifyFailed ? t.err : color[st], label = compact ? "" : " " + st, text = STAGE_GLYPH[st] + label;
+        return now.has(st) ? BOLD(F(c, text)) : recent.has(st) || (st === "verify" && verifyFailed) ? F(mix(c, t.surface, 0.35), text) : F(t.faint, text);
+    }).join(F(t.border, " ▸ "));
+    lines.push(cycle);
+    if (d.live && cur) {
+        const act = d.phase === "compact" ? "compacting the history" : d.phase === "verify" ? "verifying the result" : d.phase === "tools" && d.inflight.length ? "" : `thinking ${fmtMs(Math.max(0, d.now - cur.t0))}`;
+        if (act)
+            lines.push(spin + " " + F(t.dim, act));
+        else {
+            const x = d.inflight[0];
+            lines.push(spin + " " + F(t.text, clipWidth((x.agent && x.agent !== "lead" ? "↳ " : "") + x.label, inner - 9)) + F(t.faint, ` ${fmtMs(x.ms)}`));
+            if (d.inflight.length > 1)
+                lines.push("  " + F(t.faint, `+${d.inflight.length - 1} more running`));
+        }
+        if (cap.level === "warn") {
+            const w = Math.max(6, Math.min(20, inner - 4)), f = Math.max(1, Math.min(w, Math.round((cur.n / d.cap) * w)));
+            lines.push(F(t.warn, "▰".repeat(f)) + F(t.border, "▱".repeat(w - f)) + F(t.warn, "  " + cap.text));
+        }
+    }
+    const lastN = d.steps[d.steps.length - 1]?.n ?? 0;
+    const blocks = d.steps.map((s) => {
+        const isCur = d.live && s === cur, open = d.open.includes(s.n), out = [], bad = s.tools.some((x) => !x.ok);
+        const st = isCur && now.size ? [...now] : stagesOf(s.tools);
+        const glyphs = st.length ? st.map((x) => F(bad && !isCur ? t.err : color[x], STAGE_GLYPH[x])).join("") : isCur ? F(t.accent2, "▸") : F(t.faint, "·");
+        const what = isCur && !s.tools.length && !d.inflight.length ? "thinking…" : s.tools.length || isCur ? stepSummary(s.tools) || (d.inflight.length ? "…" : "") : "replied";
+        const time = isCur ? "" : fmtMs(stepMs(s)), num = String(s.n).padStart(2);
+        const body = clipWidth(what, Math.max(4, inner - 8 - st.length - (bad && !isCur ? 2 : 0) - time.length));
+        out.push(F(t.faint, num) + " " + glyphs + " " + (isCur ? F(t.text, body) : F(bad ? t.err : t.dim, body)) + (bad && !isCur ? F(t.err, " ✗") : "") + (time ? F(t.faint, " " + time) : ""));
+        const showAll = open || isCur;
+        if (open)
+            for (const x of s.tools)
+                out.push("     " + (x.ok ? F(t.ok, "✓ ") : F(t.err, "✗ ")) + F(t.dim, clipWidth(`${toolLabel(x.name)} ${shortArgs(x.args)}`, inner - 12)) + F(t.faint, ` ${fmtMs(x.ms)}`));
+        let hotNote = false;
+        for (const n of s.notes) {
+            const hot = isIntervention(n);
+            hotNote ||= hot;
+            if (!(hot || showAll || s.n >= lastN - 2))
+                continue;
+            // what the harness did is worth reading in full: an intervention wraps to two lines, background notes are clipped to one
+            const parts = hot ? wrapWords(n, inner - 7, 2) : [clipWidth(n, inner - 7)];
+            parts.forEach((l, i) => out.push("   " + (i ? "  " : F(hot ? t.warn : t.faint, hot ? "⚑ " : "· ")) + F(hot ? t.dim : t.faint, l)));
+        }
+        return { step: s.n, lines: out, must: s.n > lastN - 12 && (hotNote || open) };
+    });
+    // choose from the newest back: about as many steps as the room allows (3 when it is short), and every step that has an intervention or was opened
+    const room = Math.max(3, maxRows - lines.length - 1), nominal = Math.max(3, Math.min(8, Math.floor(room / 2)));
+    const shown = new Set();
+    let used = 0, count = 0;
+    const fitsAll = blocks.reduce((n, b) => n + b.lines.length, 0) <= room; // everything fits: fold nothing
+    for (let i = blocks.length - 1; i >= 0 && fitsAll; i--)
+        shown.add(i);
+    for (let i = blocks.length - 1; i >= 0 && !fitsAll; i--) {
+        const b = blocks[i], need = b.lines.length;
+        if (!shown.size) {
+            shown.add(i);
+            used += need;
+            count++;
+            continue;
+        } // the newest step is always there
+        if (used + need <= room && (count < nominal || b.must)) {
+            shown.add(i);
+            used += need;
+            count++;
+        }
+    }
+    const hidden = blocks.filter((_, i) => !shown.has(i));
+    const firstShown = Math.min(...shown);
+    if (hidden.length && blocks.findIndex((_, i) => !shown.has(i)) < firstShown) {
+        const old = blocks.slice(0, firstShown), ot = old.reduce((n, b) => n + d.steps.find((s) => s.n === b.step).tools.length, 0), of = old.reduce((n, b) => n + d.steps.find((s) => s.n === b.step).tools.filter((x) => !x.ok).length, 0);
+        lines.push(F(t.faint, `… ${old.length} earlier step${old.length === 1 ? "" : "s"} · ${ot} tool${ot === 1 ? "" : "s"}${of ? ` · ${of} failed` : ""}`));
+    }
+    blocks.forEach((b, i) => { if (!shown.has(i))
+        return; hits.push({ line: lines.length, step: b.step }); lines.push(...b.lines); });
+    if (!d.live || hidden.length)
+        lines.push(F(t.faint, `${d.steps.length} step${d.steps.length === 1 ? "" : "s"} · ${tools} tool${tools === 1 ? "" : "s"}${failed ? " · " : ""}`) + (failed ? F(t.err, `${failed} failed`) : "") + F(t.faint, "  /loop: all"));
+    return { lines, hits, title, right };
+}
+function wrapWords(s, w, maxLines) {
+    const lines = [""];
+    for (const word of s.split(/\s+/)) {
+        const last = lines[lines.length - 1];
+        if ((last + " " + word).trim().length <= w)
+            lines[lines.length - 1] = (last + " " + word).trim();
+        else
+            lines.push(word);
+    }
+    const cut = lines.slice(0, maxLines);
+    if (lines.length > maxLines)
+        cut[maxLines - 1] = clipWidth(cut[maxLines - 1], w - 1) + "…";
+    return cut.map((l) => clipWidth(l, w));
+}
+function shortArgs(raw) { try {
+    const a = JSON.parse(raw);
+    return String(a.path ?? a.cmd ?? a.pattern ?? a.query ?? a.url ?? a.name ?? "").replace(/\s+/g, " ");
+}
+catch {
+    return "";
+} }

package/dist/cli/tui/sidebar.d.ts

@@ -1,5 +1,6 @@
 import { type Cell } from "./ansi.ts";
 import type { Theme } from "./themes.ts";
+import { type LoopData } from "./looppanel.ts";
 export type SidebarData = {
     title: string;
     model: string;
@@ -37,23 +38,7 @@ export type SidebarData = {
         created: boolean;
         blockId: number;
     }[];
-    run?: {
-        phase: string;
-        step: number;
-        max: number;
-        elapsedMs: number;
-        frame: number;
-        live: {
-            label: string;
-            ms: number;
-            agent?: string;
-        }[];
-        done: {
-            label: string;
-            ok: boolean;
-            ms: number;
-        }[];
-    };
+    /** the agent loop of the current (or last) run, step by step */ loop?: LoopData;
     sandbox?: string;
     /** one line per plugin status (see Plugin.status) */ pluginStatus?: string[];
     /** the token cap for the current/last turn: tokens counted (cached input at a tenth), the cap, and whether usage was estimated */ budget?: {
@@ -75,8 +60,9 @@ export type SidebarData = {
 };
 export type SideHit = {
     row: number;
-    kind: "change" | "todos";
+    kind: "change" | "todos" | "step";
     blockId?: number;
+    step?: number;
 };
 export declare function renderSidebar(d: SidebarData, t: Theme, width: number, height: number): {
     rows: Cell[][];

package/dist/cli/tui/sidebar.js

@@ -4,6 +4,7 @@ import { BOLD, F, ITAL, fit, split } from "./rich.js";
 import { cellWidth, clipWidth } from "./text.js";
 import { brandOfModel } from "./brand.js";
 import { fmtCost, fmtTokens } from "./pricing.js";
+import { renderLoop } from "./looppanel.js";
 const BARS = "▁▂▃▄▅▆▇█";
 export function renderSidebar(d, t, width, height) {
     const inner = width - 4, out = [], hits = [], bg = t.surface;
@@ -12,16 +13,13 @@ export function renderSidebar(d, t, width, height) {
         push(); push(right ? split(F(t.faint, BOLD(title)), right, inner, bg) : F(t.faint, BOLD(title))); };
     const brand = brandOfModel(d.model, d.provider, t);
     push();
-    if (d.run) {
-        const r = d.run, SPIN = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏", stages = [["model", "think"], ["tools", "act"], ["verify", "verify"]];
-        head("RUN", F(t.dim, `${(r.elapsedMs / 1000).toFixed(0)}s`));
-        push(stages.map(([k, label], i) => (k === r.phase || (r.phase === "compact" && k === "model") ? BOLD(F(t.accent, "● " + label)) : F(t.faint, "○ " + label)) + (i < 2 ? F(t.border, " ─ ") : "")).join(""));
-        const lbl = `step ${r.step + 1}/${r.max}`, bw = Math.max(6, Math.min(24, inner - cellWidth(lbl) - 2)), f = Math.max(1, Math.round(((r.step + 1) / Math.max(1, r.max)) * bw));
-        push(F(r.step + 1 >= r.max - 2 ? t.warn : t.accent2, "▰".repeat(f)) + F(t.border, "▱".repeat(bw - f)) + F(t.dim, "  " + lbl));
-        for (const x of r.live.slice(0, 5))
-            push(F(t.accent2, SPIN[r.frame % SPIN.length]) + " " + F(t.text, clipWidth(x.label, inner - 9)) + F(t.faint, ` ${fmtMs(x.ms)}`));
-        for (const x of r.done.slice(-4).reverse())
-            push(F(x.ok ? t.ok : t.err, x.ok ? "✓" : "✗") + " " + F(t.dim, clipWidth(x.label, inner - 9)) + F(t.faint, x.ms >= 50 ? ` ${fmtMs(x.ms)}` : ""));
+    if (d.loop && d.loop.steps.length) {
+        const room = Math.max(4, Math.min(14, height - 26)), lp = renderLoop(d.loop, t, inner, room);
+        head(lp.title, lp.right);
+        for (const h of lp.hits)
+            hits.push({ row: out.length + h.line, kind: "step", step: h.step });
+        for (const l of lp.lines)
+            push(l);
     }
     head("SESSION");
     for (const l of wrap(d.title || "New conversation", inner, 2))

package/dist/src/evals.d.ts

@@ -10,7 +10,13 @@ export type Case = {
     /** Shown to the agent: a command it may run to check its work (and the `finish` gate in the full variant). */
     verify?: string;
     /** Held-out judge: shell command (exit 0 = pass) or function. The agent never sees this. */
-    check: string | ((dir: string) => boolean | string | Promise<boolean | string>);
+    check: string | ((dir: string, run?: {
+        text: string;
+        steps: number;
+        tokens: number;
+    }) => boolean | string | Promise<boolean | string>);
+    /** What the agent would say when it finishes with `solution` applied. Only needed when `check` reads `run.text` (for example "the summary names the file", "the summary admits it could not verify"). */
+    solutionSummary?: string;
     /** A known-good fix (files to overwrite). Used by `selfTest` to prove the case is solvable and its check is not trivially true. */
     solution?: Record<string, string>;
     /** Extra tools for this case (a stubbed web page, a fake MCP server): the agent gets them next to its own. */
@@ -60,7 +66,8 @@ export declare function evaluate(cases: Case[], variants: Variant[], o?: {
     onRow?: (r: Row) => void;
     baseline?: string;
     k?: number;
-    seed?: number;
+    seed?: number; /** Called before each run starts with the rows so far; a returned reason stops the rest (runs already started finish). Used for cost caps. */
+    stop?: (rows: Row[]) => string | undefined;
 }): Promise<{
     rows: Row[];
     summary: {
@@ -73,6 +80,8 @@ export declare function evaluate(cases: Case[], variants: Variant[], o?: {
     }[];
     table: string;
     analysis: Analysis;
+    stopped: string | undefined;
+    skipped: number;
 }>;
 export declare function formatTable(s: {
     variant: string;
@@ -195,3 +204,15 @@ export declare function gateCompare(current: Row[], baseline: Row[], o?: {
     costPct?: number;
     seed?: number;
 }): GateVerdict[];
+/** How hard each task is for one variant (use `baseline`): a task every run passes, or none does, cannot tell two variants apart. Keep tasks whose pass rate is between `low` and `high`. */
+export type Calibration = {
+    case: string;
+    runs: number;
+    passes: number;
+    rate: number;
+    verdict: "keep" | "too easy" | "too hard";
+};
+export declare function calibrate(rows: Row[], variant?: string, band?: {
+    low: number;
+    high: number;
+}): Calibration[];

package/dist/src/evals.js

@@ -12,6 +12,7 @@ import { bootstrap, holm, mean, minDetectable, passPowK, signTest } from "./stat
 /** Run every case x variant x trial in a fresh temp workspace and judge each result with the case's held-out check. `jobs` runs that many sessions at once. */
 export async function evaluate(cases, variants, o = {}) {
     const rows = [], work = [];
+    let stopped;
     for (const c of cases)
         for (const v of variants)
             for (let trial = 0; trial < (o.trials ?? 1); trial++)
@@ -39,7 +40,7 @@ export async function evaluate(cases, variants, o = {}) {
                     try {
                         const out = await v.run(c, dir);
                         Object.assign(row, { steps: out.steps, tokens: out.tokens, events: out.events, prompt: out.prompt, cached: out.cached });
-                        const j = typeof c.check === "string" ? ((await runShell(c.check, dir, 120_000)).startsWith("exit=0") || "check command failed") : await c.check(dir);
+                        const j = typeof c.check === "string" ? ((await runShell(c.check, dir, 120_000)).startsWith("exit=0") || "check command failed") : await c.check(dir, { text: out.text, steps: out.steps, tokens: out.tokens });
                         row.pass = j === true;
                         if (j !== true)
                             row.note = String(j || "failed");
@@ -64,14 +65,18 @@ export async function evaluate(cases, variants, o = {}) {
                     rmSync(dir, { recursive: true, force: true });
                 });
     let next = 0;
-    await Promise.all(Array.from({ length: Math.max(1, Math.min(o.jobs ?? 1, work.length)) }, async () => { while (next < work.length)
-        await work[next++](); }));
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(o.jobs ?? 1, work.length)) }, async () => { while (next < work.length) {
+        stopped ??= o.stop?.(rows);
+        if (stopped)
+            return;
+        await work[next++]();
+    } }));
     rows.sort((a, b) => cases.findIndex((c) => c.name === a.case) - cases.findIndex((c) => c.name === b.case) || variants.findIndex((v) => v.name === a.variant) - variants.findIndex((v) => v.name === b.variant) || a.trial - b.trial);
     const summary = variants.map((v) => {
         const r = rows.filter((x) => x.variant === v.name), avg = (f) => Math.round(r.reduce((n, x) => n + f(x), 0) / (r.length || 1));
         return { variant: v.name, runs: r.length, passRate: r.filter((x) => x.pass).length / (r.length || 1), steps: avg((x) => x.steps), tokens: avg((x) => x.tokens), secs: avg((x) => x.ms) / 1000 };
     });
-    return { rows, summary, table: formatTable(summary), analysis: analyze(rows, { baseline: o.baseline, k: o.k, seed: o.seed }) };
+    return { rows, summary, table: formatTable(summary), analysis: analyze(rows, { baseline: o.baseline, k: o.k, seed: o.seed }), /** why the run ended early, if it did */ stopped, /** runs that never started because of that */ skipped: work.length - next };
 }
 export function formatTable(s) {
     const line = (a) => a.map((x, i) => String(x).padEnd([22, 6, 9, 7, 9, 7][i])).join(" ");
@@ -279,7 +284,8 @@ export async function selfTest(cases) {
                 mkdirSync(dirname(join(dir, f)), { recursive: true });
                 writeFileSync(join(dir, f), t);
             }
-            const judge = async () => (typeof c.check === "string" ? (await runShell(c.check, dir, 60_000)).startsWith("exit=0") : (await c.check(dir)) === true);
+            let said = ""; // what the agent "says": nothing before the fix, the reference summary after it
+            const judge = async () => (typeof c.check === "string" ? (await runShell(c.check, dir, 60_000)).startsWith("exit=0") : (await c.check(dir, { text: said, steps: 0, tokens: 0 })) === true);
             if (await judge()) {
                 out.push({ case: c.name, ok: false, why: "check already passes before any fix" });
                 continue;
@@ -292,6 +298,7 @@ export async function selfTest(cases) {
                 mkdirSync(dirname(join(dir, f)), { recursive: true });
                 writeFileSync(join(dir, f), t);
             }
+            said = c.solutionSummary ?? "";
             out.push((await judge()) ? { case: c.name, ok: true } : { case: c.name, ok: false, why: "check still fails with the reference solution" });
         }
         finally {
@@ -338,3 +345,10 @@ export function gateCompare(current, baseline, o = {}) {
     }
     return out;
 }
+export function calibrate(rows, variant = "baseline", band = { low: 0.3, high: 0.7 }) {
+    const names = [...new Set(rows.filter((r) => !r.excluded && r.variant === variant).map((r) => r.case))];
+    return names.map((name) => {
+        const r = rows.filter((x) => !x.excluded && x.variant === variant && x.case === name), passes = r.filter((x) => x.pass).length, rate = passes / r.length;
+        return { case: name, runs: r.length, passes, rate, verdict: rate > band.high ? "too easy" : rate < band.low ? "too hard" : "keep" };
+    });
+}

package/docs/agent-loop.md

@@ -64,7 +64,7 @@ You type `Fix the failing test in src/math.ts` in the chat and press Enter. The
 **Limits and common problems.**
 
 - The exact step numbers differ from run to run. A plain-text answer without `finish` also ends a run, and then `finished` is `false` in the result.
-- The loop stops after `maxSteps` (the code agent uses 60 in chat and 40 otherwise, unless you set it).
+- The loop stops after `maxSteps` (the code agent uses 60 in chat and 40 otherwise, unless you set it; the eval runner and the plain library harness default to 30). The number is a safety stop against a runaway loop, not a target, and no measurement chose it. In our built-in test suites most tasks finish in about 6 steps and about 1 run in 100 reaches 30. The token budget and the loop and stuck guards are what normally end a run.
 - Not every event appears in every run. Guards only speak when they fire.
 
 **Related pages.** [Architecture](architecture.md), [Library](library.md), [Why: what Amness checks](why.md).
@@ -197,11 +197,24 @@ A mismatch is rejected with the reason. This happens at most twice, so it cannot
 
 **Related pages.** [Budgets and subagents](budgets.md), [Providers and models](providers.md).
 
+## See how other tools shape the loop
+
+The basic loop is the same almost everywhere. Anthropic describes it as *gather context, take action, verify work, repeat* ([Building agents with the Claude Agent SDK](https://claude.com/blog/building-agents-with-the-claude-agent-sdk), read in full). OpenAI describes a Codex turn the same way: the model and its tools go round many times until the model gives a final answer ([Unrolling the Codex agent loop](https://openai.com/index/unrolling-the-codex-agent-loop), seen only as a search summary). A survey of eleven coding agents says they run on hand-written loops, not on agent frameworks ([arXiv 2609.00006](https://arxiv.org/abs/2609.00006), abstract only).
+
+So there are no hidden extra stages. What differs between tools is where they put **gates** around the loop:
+
+- **Gather, act and verify are not separate phases in Amness.** The loop has four phases: `model`, `tools`, `compact` and `verify`. *Gathering* is the model using read-only tools, *acting* is edits and commands, and *verifying* is the model's own test runs, the optional `check` command after each edit, and the verify gate at the end. The strategies `explore-then-act` and `plan-first` are the only things that force an order.
+- **A plan at the first edit.** [PAUSE](https://arxiv.org/html/2610.29870v1) (read in full) holds back the first source edit and asks a separate, stronger planner with no tools for one plan. Then the same agent continues. In the paper, 80% of first edits came after the middle of a run, so the agent has already seen most of the evidence by then. The authors report gains of 6 to 17 points in three harnesses at about 4 cents per task. They also report that the plan helped much less when the planner was the same model as the agent, that most comparisons used one run per task, and that edits made through the shell escape the trigger. `plan-first` in Amness plans at step 0, before anything is read, which is a different design.
+- **Checks after every edit.** Anthropic calls rules-based feedback such as linting "the best form of feedback" and calls an LLM judge "not very robust". Amness has a post-edit `check` command and language-server diagnostics, both off by default.
+- **A fixed pipeline instead of a loop.** [Agentless](https://arxiv.org/abs/2407.01489) (search summary only) runs localise, repair, then validate with generated tests, and no model-chosen actions. Amness's `best-of-n` option belongs to the same "sample several, pick one" family.
+
+These are other people's results on their own benchmarks and models, not measurements of Amness.
+
 ## Learn where these choices come from
 
 Recent work on coding-agent loops points the same way. The loop follows it where this can be done without training a model.
 
-- [An Empirical Study of Harness Design for Coding Agents](https://arxiv.org/abs/2609.20804) (2026): cheap rule-based elision before any LLM summary gave the lowest cost. A recall tool for elided content was almost never used, so Amness has none. Stuck detection and light post-edit checks are cheap ways to stop wasted turns.
+- [An Empirical Study of Harness Design for Coding Agents](https://arxiv.org/abs/2609.20804) (2026): managing the context mattered a lot with a small window (a 35.7 point gap in success at 32k tokens) and little with a large one (2.7 points at 128k). The highest level of context management had the lowest mean cost. A recall tool for elided content was rarely used, so Amness has none. The paper held stuck detection and post-edit checks fixed, so it does not test them; Amness keeps them because they are cheap, not because this paper measured them.
 - [AgentDiet](https://arxiv.org/abs/2509.23586) (trajectory reduction): removing useless, redundant and *expired* information cut input tokens by 40-60% with unchanged results. This is the source of the expired-read and repeated-call rules.
 - [CoACT](https://arxiv.org/abs/2607.02911) and [SWE-Pruner](https://arxiv.org/abs/2601.16746): reads use most of the tokens. Compression must keep what the next action needs, and earlier messages stay untouched so the cache stays valid. Amness uses rules for this. The papers use trained compressors, which would add a dependency.
 - [Don't Break the Cache](https://arxiv.org/abs/2601.06007) (prompt caching for long-horizon agents): caching cut API cost by 41-80% across OpenAI, Anthropic and Google, but naive full-context caching can hurt. Put dynamic content at the end, avoid dynamic tool lists, and keep dynamic tool results out of the cached prefix. [Keeping the Cache Warm Pays](https://arxiv.org/abs/2607.19214) motivates the longer OpenAI retention and the opt-in Claude keep-alive.

package/docs/evaluation.md

@@ -262,6 +262,38 @@ Most public benchmarks need a separate environment for each instance, often a Do
 ## Limits and common problems
 
 - A real run costs real tokens. The command prints the worst case and asks first. Use `--max-tokens` and `--max-steps` (default 30) to limit each session.
+- **A spending cap, when you want one.** Nothing is capped by default. Add `--max-cost 0.50` and Amness stops starting new sessions once the estimated spend reaches 50 cents. Sessions that are already running finish, so with `--jobs 4` you can overshoot by up to four sessions. The result file and the report cover only the runs that finished, and say how many were skipped. The estimate prices prompt, cached and completion tokens from the rate card in `cli/tui/pricing.ts`. It is an estimate, not an invoice, and it needs a model with a known price. After every run Amness prints the estimated spend and the cost per session.
+
+## Check how hard your tasks are
+
+A task that every run passes, or that none passes, cannot tell two variants apart. Before you compare variants, run:
+
+```bash
+amness eval --calibrate --yes
+```
+
+This runs only `baseline`, 5 trials per task (change it with `--trials`), and lists each task's pass rate:
+
+```text
+how hard is each task?  (baseline, 5 trials; keep 30% to 70%)
+  keep       explore-answer             3/5 passed  60%
+  too easy   fix-off-by-one             5/5 passed  100%
+  too hard   cross-module-rename        0/5 passed  0%
+```
+
+Keep the tasks marked `keep`. With only 5 trials the pass rate is rough, so treat a task near the edge of the band (30% or 70%) as undecided and run more trials. A few easy tasks are still worth keeping as a "must not get worse" check.
+
+## Checks that read the agent's final message
+
+A task's `check` can be a function that receives the workspace and, as a second argument, what the agent said when it finished:
+
+```ts
+{ name: "say-where", files: {}, task: "create notes.txt and tell me where it is",
+  check: (dir, run) => existsSync(join(dir, "notes.txt")) && /notes\.txt/.test(run?.text ?? ""),
+  solution: { "notes.txt": "x" }, solutionSummary: "Created ./notes.txt" }
+```
+
+`run` also has `steps` and `tokens`. `solutionSummary` is what the reference solution would say. `--selftest` uses it, so a check that depends on the message is still proven sound (it must fail on silence and pass with the reference message). This works for checks written as functions in a `.ts` or `.js` module, not for JSON tasks with a shell command.
 - The built-in suites are small and made by us. They are a regression test for the guards, not proof of quality.
 - `full+route` needs `--cheap-model`. A variant name that Amness does not know stops with an error that lists the known names.
 - `--variants` and `--baseline`: if the baseline name is not among the variants, Amness uses the first variant.

package/docs/usage.md

@@ -60,7 +60,30 @@ If you have no key yet, the first start asks you to pick a provider and paste a
 
 - **Collapsed reads.** A run of reads and searches becomes one row, for example "Explored · 3 files read · 2 searches". Click it to open it. Edits, commands and failures always get their own card. Long diffs show the first 16 rows, and you click to see the rest.
 - **Cards.** Your prompts, the assistant replies (with code blocks that have a copy button), tool cards with a status chip, and diffs with line numbers and word-level marks. Wide terminals show a split view.
-- **Live run view.** The status line says what the agent does now, for example "Reading src/app.ts" or "Waiting for you". It also shows elapsed time, tokens and the step counter. The sidebar has a RUN panel with the think, act and verify steps, running tools and the last finished actions.
+- **Live run view.** The status line says what the agent does now, for example "Reading src/app.ts" or "Waiting for you". It also shows elapsed time, tokens and the step number. The sidebar has a **LOOP** panel that draws the cycle the agent goes round and lights the part that is running:
+
+  ```text
+  LOOP                round 2 · step 14
+  ◉ gather ▸ ✎ act ▸ ✓ verify
+  ⠋ Updating src/app.ts  2.1s
+   12 ◉ Search → Read ×2     3.1s
+      ⚑ nudged: no edits after
+        5 reads
+   13 ✎ Update               0.8s
+   14 ✎ …
+  ```
+
+  - **gather** ◉ (blue): reading, searching, listing files, asking a subagent, shell commands such as `ls` or `git diff`.
+  - **act** ✎ (amber): editing and writing files, and shell commands that change things.
+  - **verify** ✓ (green): shell commands that look like tests, builds, linting or type checks (`npm test`, `pytest`, `tsc`, `cargo test`…), the `check` command, and the verify gate when the agent says it is done.
+
+  The bright stage is the one running now. When two kinds run together, both light up. While the model is thinking and nothing runs, the stage it used last stays visible but dimmer. A test that failed keeps the verify icon red until a later one passes. A **round** is one trip round the cycle: it counts up when the agent gathers or acts again after verifying, so "round 3" means it went back to work twice after checking.
+
+  **These stages are a guess.** Amness reads them from the tool calls, and the loop itself has no such phases. An unusual shell command can be put in the wrong stage, and a step where the model only thinks has no stage.
+
+  Under the cycle you see the latest steps, one line each: the stage icons, the tools used, and the time. Calls of one kind in a row merge (`Read ×2`). A red `✗` means a tool failed. A line with `⚑` means the harness stepped in (a nudge, a loop or stagnation warning, a rejected finish, a denied call, a reviewer finding). It stays visible even when the step is older. **Click a step** to see every tool call with its time. A tall sidebar shows more steps than a short one, and older ones fold into one line. Type **`/loop`** for the whole history of the run, with every turn labelled by its stage. When the run ends, the panel stays as **LAST RUN**.
+
+  The step limit is a safety stop, not a goal, so it stays hidden at first. It appears as "of 60" after half the steps and as a yellow bar with a warning at 80%.
 - **Sidebar** (`Ctrl+B`). It shows the session, a context gauge, token use, todos, changed files and the workspace. The *context* is the text the model can read at one time.
 - **Modals.** Windows for approvals (with the diff, command or plan), the command palette (`Ctrl+P`), pickers and help.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "amness",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "type": "module",
   "description": "A terminal coding agent for any model: streaming chat TUI, permission modes, plan mode, undo, memory, MCP, with a small reliability harness underneath. OpenAI (Responses API), Claude, and local models.",
   "license": "MIT",