amness 0.0.0-stage → 0.1.2

package/dist/src/plugins.js

@@ -0,0 +1,225 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+// Plugins: harness behaviour written as code. A plugin is a TypeScript or JavaScript file in `.amness/plugins/` (or `~/.amness/plugins/`) that can add tools, run code at every
+// point of the loop (the eight hook events, in-process, with the full payload), rewrite the system prompt, and contribute evaluation tasks, graders and harness variants.
+// Everything the built-in guards do can be done from here, and `amness eval` can then compare your harness against the others like any built-in variant.
+// A plugin runs with your full permissions, so project plugins load only after you approve that exact set of files (same rule as project hooks).
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { extname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { execFile } from "node:child_process";
+import { shellInvocation } from "./shell.js";
+/**
+ * Ask every plugin's `policy`. A deny always wins, then an ask, then an allow. The caller decides what each answer may do:
+ * the harness never lets a policy relax the safety floor or the untrusted-text check, only tighten them or skip an ordinary prompt.
+ */
+export async function checkPolicies(plugins, call) {
+    let best;
+    const rank = { allow: 1, ask: 2, deny: 3 };
+    for (const p of plugins) {
+        if (!p.policy)
+            continue;
+        let r;
+        try {
+            r = await p.policy(call);
+        }
+        catch (e) {
+            r = { action: p.failClosed ? "deny" : "ask", reason: `policy of plugin ${p.name} failed: ${e.message}` };
+        } // a broken policy never silently allows
+        if (!r)
+            continue;
+        const d = { ...(typeof r === "string" ? { action: r } : r), plugin: p.name };
+        if (!best || rank[d.action] > rank[best.action])
+            best = d;
+    }
+    return best;
+}
+/** Several reviewers vote: the finish is accepted when at least `quorum` of them pass ("all", "majority", or a number). */
+export function votingGate(name, reviewers, quorum = "majority", maxRejections = 2) {
+    return { name, maxRejections, async check(ctx) {
+            const votes = await Promise.all(reviewers.map(async (r) => { try {
+                return { name: r.name, ...normalizeGate(await r.review(ctx)) };
+            }
+            catch (e) {
+                return { name: r.name, pass: true, findings: [`(${r.name} could not run: ${e.message})`] };
+            } }));
+            const need = quorum === "all" ? votes.length : quorum === "majority" ? Math.floor(votes.length / 2) + 1 : quorum, passes = votes.filter((v) => v.pass).length;
+            return passes >= need ? true : { pass: false, findings: [`only ${passes} of ${votes.length} reviewers accepted this (${need} needed)`, ...votes.filter((v) => !v.pass).flatMap((v) => v.findings.map((f) => `${v.name}: ${f}`))] };
+        } };
+}
+const normalizeGate = (r) => r === undefined || r === true ? { pass: true, findings: [] } : r === false ? { pass: false, findings: ["the check failed"] } : typeof r === "string" ? { pass: false, findings: [r] } : Array.isArray(r) ? { pass: r.length === 0, findings: r } : { pass: r.pass, findings: r.findings ?? [] };
+const shellRun = (cwd) => (cmd, ms = 60_000) => new Promise((res) => execFile(shellInvocation(cmd).file, shellInvocation(cmd).args, { cwd, windowsHide: true, timeout: ms, maxBuffer: 10 << 20 }, (err, out, errOut) => res({ code: err ? (typeof err.code === "number" ? err.code : 1) : 0, output: String(out) + String(errOut) })));
+/** Identity function that gives you types and completion while writing a plugin. */
+export const definePlugin = (p) => p;
+const EXT = new Set([".ts", ".mts", ".js", ".mjs"]);
+const filesIn = (dir) => (existsSync(dir) ? readdirSync(dir).filter((f) => EXT.has(extname(f)) && statSync(join(dir, f)).isFile()).sort().map((f) => join(dir, f)) : []);
+export const projectPluginDir = (cwd) => join(cwd, ".amness", "plugins");
+export const projectPluginFiles = (cwd) => filesIn(projectPluginDir(cwd));
+/** A fingerprint of the project's plugin files: approval is remembered for exactly this content. */
+export function pluginFingerprint(cwd) {
+    const fs = filesIn(projectPluginDir(cwd));
+    if (!fs.length)
+        return undefined;
+    const h = createHash("sha256");
+    for (const f of fs)
+        h.update(f).update(readFileSync(f));
+    return h.digest("hex").slice(0, 16);
+}
+const valid = (p, file) => {
+    if (!p || typeof p !== "object")
+        return `${file}: must export a plugin object (export default definePlugin({ name, ... }))`;
+    if (typeof p.name !== "string" || !/^[\w.-]{1,40}$/.test(p.name))
+        return `${file}: needs a "name" (letters, numbers, . _ -)`;
+    for (const k of ["tools", "cases"])
+        if (p[k] !== undefined && !Array.isArray(p[k]))
+            return `${file}: "${k}" must be a list`;
+    if (p.hooks !== undefined && (typeof p.hooks !== "object" || Object.values(p.hooks).some((h) => typeof h !== "function")))
+        return `${file}: "hooks" must map event names to functions`;
+    return undefined;
+};
+/** Import one plugin file (also used by `amness eval --plugin <file>`, where passing the file is the approval). */
+export async function importPlugins(file) {
+    const warnings = [], plugins = [];
+    try {
+        const mod = await import(__rewriteRelativeImportExtension(pathToFileURL(resolve(file)).href + `?v=${Date.now()}`)), exp = mod.default ?? mod.plugin ?? mod.plugins;
+        for (const p of Array.isArray(exp) ? exp : [exp]) {
+            const bad = valid(p, file);
+            if (bad)
+                warnings.push(bad);
+            else
+                plugins.push(p);
+        }
+    }
+    catch (e) {
+        warnings.push(`${file}: could not load: ${e.message.split("\n")[0]}`);
+    }
+    return { plugins, warnings };
+}
+/** User plugins always load; project plugins only when `trustProject`. Problems are warnings, never exceptions: a broken plugin must not stop the agent starting. */
+export async function loadPlugins(cwd, o = {}) {
+    const out = { plugins: [], warnings: [] }, seen = new Set();
+    const add = async (files) => { for (const f of files) {
+        const r = await importPlugins(f);
+        out.warnings.push(...r.warnings);
+        for (const p of r.plugins) {
+            if (seen.has(p.name))
+                out.warnings.push(`${f}: a plugin named "${p.name}" is already loaded; skipped`);
+            else {
+                seen.add(p.name);
+                out.plugins.push(p);
+            }
+        }
+    } };
+    await add(filesIn(join(o.userDir ?? process.env.AMNESS_HOME ?? join(homedir(), ".amness"), "plugins")));
+    const proj = filesIn(projectPluginDir(cwd));
+    if (proj.length) {
+        if (o.trustProject)
+            await add(proj);
+        else
+            out.untrusted = { count: proj.length, fingerprint: pluginFingerprint(cwd) };
+    }
+    return out;
+}
+/** Hooks from plugins, run in-process after the shell hooks. Same rules as shell hooks: the first block stops the rest, context accumulates, the last rewritten args win. */
+export function composeHooks(shell, plugins, cwd) {
+    const withHooks = plugins.filter((p) => (p.hooks && Object.keys(p.hooks).length) || p.gates?.length);
+    if (!withHooks.length)
+        return shell;
+    const rejections = new Map();
+    let lastTask;
+    return {
+        count: () => (shell?.count() ?? 0) + withHooks.reduce((n, p) => n + Object.keys(p.hooks ?? {}).length + (p.gates?.length ?? 0), 0),
+        async run(event, payload) {
+            const res = shell ? await shell.run(event, payload) : { warnings: [] };
+            if (res.block)
+                return res;
+            for (const p of withHooks) {
+                const h = p.hooks?.[event];
+                if (!h)
+                    continue;
+                try {
+                    const r = await h({ event, cwd, ...payload });
+                    if (r === undefined || r === null)
+                        continue;
+                    if (typeof r === "string") {
+                        res.context = (res.context ? res.context + "\n" : "") + r;
+                        continue;
+                    }
+                    if (r.block !== undefined) {
+                        res.block = r.block;
+                        return res;
+                    }
+                    if (r.context)
+                        res.context = (res.context ? res.context + "\n" : "") + r.context;
+                    if (r.args)
+                        res.args = r.args;
+                    if (r.tools)
+                        res.tools = r.tools;
+                    if (r.then)
+                        res.then = r.then;
+                }
+                catch (e) {
+                    const msg = `plugin ${p.name} (${event}): ${e.message}`;
+                    if (p.failClosed) {
+                        res.block = msg;
+                        return res;
+                    }
+                    res.warnings.push(msg);
+                }
+            }
+            if (event === "Stop") { // finish gates run after the hooks, with what really happened
+                if (payload.task !== lastTask) {
+                    lastTask = payload.task;
+                    rejections.clear();
+                }
+                for (const p of withHooks)
+                    for (const g of p.gates ?? []) {
+                        const key = `${p.name}:${g.name}`, n = rejections.get(key) ?? 0;
+                        let v;
+                        try {
+                            v = normalizeGate(await g.check({ summary: String(payload.summary ?? ""), task: String(payload.task ?? ""), changed: payload.changed ?? [], testsAfterLastEdit: payload.testsAfterLastEdit, notes: payload.notes ?? [], cwd, run: shellRun(cwd), rejections: n }));
+                        }
+                        catch (e) {
+                            res.warnings.push(`plugin ${p.name} gate ${g.name}: ${e.message}`);
+                            continue;
+                        }
+                        if (v.pass)
+                            continue;
+                        const text = `gate "${g.name}" (plugin ${p.name}) found problems:\n${v.findings.map((f) => `- ${f}`).join("\n")}`;
+                        if (n < (g.maxRejections ?? 2)) {
+                            rejections.set(key, n + 1);
+                            res.block = text;
+                            return res;
+                        }
+                        res.warnings.push(`finishing despite ${text}`);
+                    }
+            }
+            return res;
+        },
+    };
+}
+export const applySystem = (plugins, base, cwd) => plugins.reduce((s, p) => (p.system ? p.system(s, { cwd }) : s), base);
+/** A starting point for `amness plugin new <name>`. */
+export const pluginTemplate = (name) => `// .amness/plugins/${name}.ts: a plugin is just an exported object. See docs/plugins.md.
+export default {
+  name: ${JSON.stringify(name)},
+  description: "what this plugin is for",
+
+  // Runs inside the loop. Return nothing to continue, a string to add context, or { block: "why" } to refuse.
+  hooks: {
+    // PreToolUse: ({ tool, args }) => (tool === "sh" && /rm -rf/.test(String(args.cmd)) ? { block: "no recursive deletes" } : undefined),
+    // Stop: ({ summary, changed }) => (changed.length && !/test/i.test(String(summary)) ? { block: "say which tests you ran" } : undefined),
+  },
+
+  // A slash command: /${name}
+  commands: [{ name: ${JSON.stringify(name)}, help: "say hello from the plugin", run: (ctx) => ctx.say("hello from the ${name} plugin") }],
+};
+`;

package/dist/src/pluginscan.d.ts

@@ -0,0 +1,8 @@
+export type Capability = {
+    id: string;
+    label: string;
+};
+export declare function scanSource(source: string): Capability[];
+export declare const scanFile: (file: string) => Capability[];
+/** What each of this project's plugin files can touch, one line per file, for the approval prompt. */
+export declare function describeProjectPlugins(cwd: string): string[];

package/dist/src/pluginscan.js

@@ -0,0 +1,20 @@
+// A plain-text scan of a plugin's source for what it can touch. This is a heads-up for the approval prompt, NOT a sandbox: a plugin still runs with your full permissions,
+// and a determined plugin can hide what it does. Its job is to make the ordinary case visible ("this plugin runs shell commands and uses the network").
+import { readFileSync } from "node:fs";
+import { basename } from "node:path";
+import { projectPluginFiles } from "./plugins.js";
+const CHECKS = [
+    { id: "shell", label: "runs shell commands", re: /\bchild_process\b|\bexecSync\b|\bspawnSync\b|\bexecFile\b|\bBun\.spawn\b|\bDeno\.Command\b/ },
+    { id: "network", label: "uses the network", re: /\bfetch\s*\(|node:(https?|net|dgram|tls)\b|\bWebSocket\b|\bXMLHttpRequest\b/ },
+    { id: "files-write", label: "writes files", re: /\b(writeFile|writeFileSync|appendFile|appendFileSync|rmSync|unlinkSync|rename(Sync)?|mkdirSync|createWriteStream)\b/ },
+    { id: "files-read", label: "reads files", re: /\b(readFile|readFileSync|readdirSync|createReadStream)\b/ },
+    { id: "env", label: "reads environment variables (may include keys)", re: /\bprocess\.env\b/ },
+    { id: "dynamic-code", label: "runs generated code (eval, new Function, dynamic import)", re: /\beval\s*\(|new Function\s*\(|\bimport\s*\(\s*(?!["'`][^"'`]*["'`]\s*\))/ },
+    { id: "home", label: "looks outside the project (home folder, absolute paths)", re: /\bhomedir\s*\(|["'`]\/(etc|Users|home|root)\b|~\// },
+];
+export function scanSource(source) { const code = source.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, ""); return CHECKS.filter((c) => c.re.test(code)).map(({ id, label }) => ({ id, label })); }
+export const scanFile = (file) => scanSource(readFileSync(file, "utf8"));
+/** What each of this project's plugin files can touch, one line per file, for the approval prompt. */
+export function describeProjectPlugins(cwd) {
+    return projectPluginFiles(cwd).map((f) => { const caps = scanFile(f); return `${basename(f)}: ${caps.length ? caps.map((c) => c.label).join(", ") : "no shell, network, file or environment access found"}`; });
+}

package/dist/src/procs.d.ts

@@ -0,0 +1,41 @@
+import { type SandboxMode } from "./sandbox.ts";
+export type ProcInfo = {
+    id: string;
+    name: string;
+    cmd: string;
+    pid?: number;
+    status: "running" | "exited" | "killed";
+    code?: number | null;
+    startedAt: number;
+    bytes: number;
+};
+export declare class ProcessManager {
+    private procs;
+    private n;
+    private cwd;
+    private max;
+    private sandbox;
+    constructor(cwd: string, o?: {
+        max?: number;
+        sandbox?: () => SandboxMode;
+    });
+    start(cmd: string, name?: string): ProcInfo;
+    private get;
+    private info;
+    list(): ProcInfo[];
+    /** Output produced since the last read. With `waitMs`, waits (up to that long) for new output or for the process to end. */
+    read(id: string, o?: {
+        waitMs?: number;
+        maxChars?: number;
+    }): Promise<{
+        output: string;
+        status: ProcInfo["status"];
+        code?: number | null;
+        skipped: number;
+    }>;
+    stop(id: string): Promise<ProcInfo>;
+    /** Kill everything still running (call on exit). */
+    dispose(): Promise<void>;
+    /** Synchronous last-resort kill for process 'exit' handlers. */
+    killAllNow(): void;
+}

package/dist/src/procs.js

@@ -0,0 +1,94 @@
+// Background processes: start a dev server or a watcher, read its output later, stop it. Output is kept in a bounded buffer; everything is killed on dispose().
+import { spawn } from "node:child_process";
+import { wrapCommand } from "./sandbox.js";
+import { childEnv } from "./secrets.js";
+import { shellInvocation } from "./shell.js";
+const CAP = 200_000;
+export class ProcessManager {
+    procs = new Map();
+    n = 0;
+    cwd;
+    max;
+    sandbox;
+    constructor(cwd, o = {}) { this.cwd = cwd; this.max = o.max ?? 5; this.sandbox = o.sandbox ?? (() => "off"); }
+    start(cmd, name) {
+        if ([...this.procs.values()].filter((p) => p.status === "running").length >= this.max)
+            throw new Error(`already running ${this.max} background processes; stop one with sh_stop first`);
+        const id = `p${++this.n}`;
+        const w = wrapCommand(cmd, this.cwd, this.sandbox()); // throws when a sandbox is on but unavailable
+        const child = spawn(w?.file ?? shellInvocation(cmd).file, w?.args ?? shellInvocation(cmd).args, { cwd: this.cwd, env: childEnv(process.env, this.sandbox() !== "off"), detached: true, stdio: ["ignore", "pipe", "pipe"] });
+        const p = { id, name: name?.slice(0, 40) || cmd.split(/\s+/).slice(0, 3).join(" ").slice(0, 40), cmd, pid: child.pid, status: "running", startedAt: Date.now(), bytes: 0, child, buf: "", dropped: 0, readPos: 0, waiters: [] };
+        const take = (d) => {
+            const s = d.toString("utf8");
+            p.bytes += s.length;
+            p.buf += s;
+            if (p.buf.length > CAP) {
+                const cut = p.buf.length - CAP;
+                p.buf = p.buf.slice(cut);
+                p.dropped += cut;
+                p.readPos = Math.max(0, p.readPos - cut);
+            }
+            for (const w of p.waiters.splice(0))
+                w();
+        };
+        child.stdout.on("data", take);
+        child.stderr.on("data", take);
+        child.on("error", (e) => { p.status = "exited"; p.code = -1; take(Buffer.from(`[could not start: ${e.message}]\n`)); });
+        child.on("close", (code) => { if (p.status === "running") {
+            p.status = "exited";
+            p.code = code;
+        } for (const w of p.waiters.splice(0))
+            w(); });
+        this.procs.set(id, p);
+        return this.info(p);
+    }
+    get(id) { const p = this.procs.get(id); if (!p)
+        throw new Error(`no background process "${id}". Running: ${this.list().map((x) => x.id).join(", ") || "none"}`); return p; }
+    info(p) { const { child: _c, buf: _b, dropped: _d, readPos: _r, waiters: _w, ...i } = p; return i; }
+    list() { return [...this.procs.values()].map((p) => this.info(p)); }
+    /** Output produced since the last read. With `waitMs`, waits (up to that long) for new output or for the process to end. */
+    async read(id, o = {}) {
+        const p = this.get(id);
+        if (p.readPos >= p.buf.length && p.status === "running" && o.waitMs)
+            await new Promise((res) => { const t = setTimeout(res, Math.min(o.waitMs, 10_000)); p.waiters.push(() => { clearTimeout(t); res(); }); });
+        const max = o.maxChars ?? 8000;
+        let skipped = p.dropped;
+        p.dropped = 0;
+        let out = p.buf.slice(p.readPos);
+        p.readPos = p.buf.length;
+        if (out.length > max) {
+            skipped += out.length - max;
+            out = out.slice(-max);
+        }
+        return { output: out, status: p.status, code: p.code, skipped };
+    }
+    async stop(id) {
+        const p = this.get(id);
+        if (p.status === "running") {
+            p.status = "killed";
+            const kill = (sig) => { try {
+                if (p.pid)
+                    process.kill(-p.pid, sig);
+            }
+            catch {
+                try {
+                    p.child.kill(sig);
+                }
+                catch { /* already gone */ }
+            } };
+            kill("SIGTERM");
+            await new Promise((res) => { const t = setTimeout(() => { kill("SIGKILL"); res(); }, 2000); p.child.once("close", () => { clearTimeout(t); res(); }); });
+        }
+        return this.info(p);
+    }
+    /** Kill everything still running (call on exit). */
+    async dispose() { await Promise.all([...this.procs.values()].filter((p) => p.status === "running").map((p) => this.stop(p.id))); }
+    /** Synchronous last-resort kill for process 'exit' handlers. */
+    killAllNow() { for (const p of this.procs.values())
+        if (p.status === "running" && p.pid) {
+            try {
+                process.kill(-p.pid, "SIGKILL");
+            }
+            catch { /* gone */ }
+        } }
+}

package/dist/src/profiles.d.ts

@@ -0,0 +1,36 @@
+import type { Analysis } from "./evals.ts";
+import type { Guard } from "./harness.ts";
+export type Profile = {
+    off?: Guard[];
+    stagnationTurns?: number;
+    maxSteps?: number;
+    contextBudget?: number;
+    review?: boolean;
+    note?: string;
+};
+export type ProfileFile = {
+    default?: Profile;
+    models?: Record<string, Profile>;
+};
+/** The profile for `model`: the user's file first, then the project's (the project's values win), then the matching `models` entries (most specific pattern last). */
+export declare function loadProfile(cwd: string, model: string, o?: {
+    userDir?: string;
+}): {
+    profile: Profile;
+    sources: string[];
+};
+export type GuardVerdict = {
+    guard: string;
+    action: "remove" | "keep" | "undecided";
+    why: string;
+};
+/**
+ * Read an ablation (`full` against `full-<guard>` for each guard) and say, per guard, whether removing it is supported by the data.
+ * Conservative on purpose: a guard is only recommended for removal when taking it out is significantly better, or when it is not worse and clearly cheaper;
+ * a guard whose removal significantly hurts is load-bearing; anything else is "undecided" and stays on.
+ */
+export declare function recommend(a: Analysis, fullName?: string): {
+    verdicts: GuardVerdict[];
+    profile: Profile;
+    strength: string;
+};

package/dist/src/profiles.js

@@ -0,0 +1,61 @@
+// Harness profiles: settings that depend on the model. Scaffolding that rescued one model is dead weight (or a hazard) for the next, so each model family can switch guards off or
+// retune limits, and the eval runner can recommend what to switch off from measurements instead of guesswork.
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+const GUARDS = ["nudge", "loop", "compact", "stuck", "wrapup", "condense", "schedule", "ledger", "stagnation", "review", "redact"];
+const glob = (pat) => new RegExp("^" + pat.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".") + "$", "i");
+function check(file, j) {
+    const bad = (m) => new Error(`${file}: ${m}`), one = (name, p) => {
+        if (p.off !== undefined && (!Array.isArray(p.off) || p.off.some((g) => !GUARDS.includes(g))))
+            throw bad(`${name}: "off" must be a list of guards (${GUARDS.join(", ")})`);
+        for (const k of ["stagnationTurns", "maxSteps", "contextBudget"])
+            if (p[k] !== undefined && (typeof p[k] !== "number" || p[k] < 0))
+                throw bad(`${name}: "${k}" must be a non-negative number`);
+    };
+    if (j.default)
+        one("default", j.default);
+    for (const [m, p] of Object.entries(j.models ?? {}))
+        one(m, p);
+    return j;
+}
+/** The profile for `model`: the user's file first, then the project's (the project's values win), then the matching `models` entries (most specific pattern last). */
+export function loadProfile(cwd, model, o = {}) {
+    const files = [join(o.userDir ?? process.env.AMNESS_HOME ?? join(homedir(), ".amness"), "profile.json"), join(cwd, ".amness", "profile.json")], merged = {}, sources = [];
+    for (const f of files) {
+        if (!existsSync(f))
+            continue;
+        const j = check(f, JSON.parse(readFileSync(f, "utf8")));
+        const picks = [j.default, ...Object.entries(j.models ?? {}).filter(([pat]) => glob(pat).test(model)).sort(([a], [b]) => a.length - b.length).map(([, p]) => p)].filter(Boolean);
+        for (const p of picks) {
+            Object.assign(merged, p, { off: [...new Set([...(merged.off ?? []), ...(p.off ?? [])])] });
+            sources.push(f);
+        }
+    }
+    return { profile: merged, sources };
+}
+/**
+ * Read an ablation (`full` against `full-<guard>` for each guard) and say, per guard, whether removing it is supported by the data.
+ * Conservative on purpose: a guard is only recommended for removal when taking it out is significantly better, or when it is not worse and clearly cheaper;
+ * a guard whose removal significantly hurts is load-bearing; anything else is "undecided" and stays on.
+ */
+export function recommend(a, fullName = "full") {
+    const verdicts = [];
+    for (const c of a.comparisons) {
+        const m = /^full-(.+)$/.exec(c.variant);
+        if (!m || c.vs !== fullName)
+            continue;
+        const pts = (x) => `${(x * 100).toFixed(0)} pts`, cheaper = c.dTokensPerSolved < 0 && Number.isFinite(c.dTokensPerSolved);
+        if (c.verdict === "worse")
+            verdicts.push({ guard: m[1], action: "keep", why: `removing it loses ${pts(-c.dPass)} (95% CI ${pts(c.ci.lo)} to ${pts(c.ci.hi)})` });
+        else if (c.verdict === "better")
+            verdicts.push({ guard: m[1], action: "remove", why: `removing it gains ${pts(c.dPass)} (95% CI ${pts(c.ci.lo)} to ${pts(c.ci.hi)})` });
+        else if (Math.abs(c.dPass) <= 0.02 && cheaper && c.ci.lo > -0.1)
+            verdicts.push({ guard: m[1], action: "remove", why: `removing it makes no real difference to the pass rate (${pts(c.dPass)}, CI ${pts(c.ci.lo)} to ${pts(c.ci.hi)}) and lowers tokens per solved task by ${Math.round(-c.dTokensPerSolved).toLocaleString()}` });
+        else
+            verdicts.push({ guard: m[1], action: "undecided", why: `no clear effect (${pts(c.dPass)}, CI ${pts(c.ci.lo)} to ${pts(c.ci.hi)}); it stays on` });
+    }
+    const strength = a.minDetectable > 0.4 ? `weak evidence: ${a.tasks} task(s) can only detect differences above ~${(a.minDetectable * 100).toFixed(0)} points; add tasks before acting on "remove"` : `${a.tasks} tasks × ${a.trials} trial(s): differences above ~${(a.minDetectable * 100).toFixed(0)} points are detectable`;
+    const off = verdicts.filter((v) => v.action === "remove" && GUARDS.includes(v.guard)).map((v) => v.guard);
+    return { verdicts, profile: { ...(off.length ? { off } : {}), note: `from an ablation of ${a.tasks} task(s) × ${a.trials} trial(s)` }, strength };
+}

package/dist/src/progress.d.ts

@@ -0,0 +1,29 @@
+/** Number of failures in a test/build/lint output, or undefined when it does not look like one. A failing exit code with no readable count counts as 1. */
+export declare function countFailures(cmd: string, result: string): number | undefined;
+export type TurnFacts = {
+    calls: {
+        name: string;
+        args: any;
+        result: string;
+    }[];
+    edits: {
+        path: string;
+        diff: string;
+    }[];
+};
+export declare class ProgressTracker {
+    private bestFail;
+    private everFailed;
+    private editedPaths;
+    private seenDiffs;
+    private reads;
+    /** Turns in a row without an objective improvement. */
+    stale: number;
+    lastFailures: number | undefined;
+    reasons: string[];
+    private readCap;
+    constructor(readCap?: number);
+    /** Feed one turn; returns why it counted as progress (empty = it did not). */
+    observe(t: TurnFacts): string[];
+    summary(): string;
+}

package/dist/src/progress.js

@@ -0,0 +1,75 @@
+// Is the run actually getting anywhere? Judged from objective signals (failing-test counts, new files edited, new code written), never from what the model says.
+// A loop that keeps acting without any of them improving is stagnating, however confident its messages sound.
+import { BUILD_CMD, TEST_CMD } from "./ledger.js";
+/** Number of failures in a test/build/lint output, or undefined when it does not look like one. A failing exit code with no readable count counts as 1. */
+export function countFailures(cmd, result) {
+    if (!TEST_CMD.test(cmd) && !BUILD_CMD.test(cmd))
+        return undefined;
+    const exit = /^exit=(\d+)/.exec(result)?.[1], out = result;
+    const num = (re) => { const m = re.exec(out); return m ? Number(m[1]) : undefined; };
+    const counts = [num(/(\d+)\s+(?:tests?\s+)?(?:failed|failing|failures?)\b/i), num(/\bfail(?:ed|ures?|ing)?:\s*(\d+)/i), num(/Tests:\s+(\d+)\s+failed/i), num(/#\s*fail\s+(\d+)/i), num(/ℹ\s*fail\s+(\d+)/i), num(/failures=(\d+)/i)].filter((x) => x !== undefined);
+    const ts = (out.match(/\berror TS\d+/g) ?? []).length, lint = num(/(\d+)\s+problems?\b/i), errs = num(/\b(\d+)\s+errors?\b/i);
+    const total = (counts.length ? Math.max(...counts) : 0) + ts + (lint ?? 0) + (counts.length || ts || lint ? 0 : errs ?? 0);
+    if (total > 0)
+        return total;
+    return exit && exit !== "0" ? 1 : 0;
+}
+/** The inverse of a unified diff hunk set (swap + and -), as a comparable string. */
+const invert = (diff) => diff.split("\n").filter((l) => /^[+-]/.test(l) && !/^(\+\+\+|---)/.test(l)).map((l) => (l[0] === "+" ? "-" : "+") + l.slice(1)).sort().join("\n");
+const body = (diff) => diff.split("\n").filter((l) => /^[+-]/.test(l) && !/^(\+\+\+|---)/.test(l)).sort().join("\n");
+export class ProgressTracker {
+    bestFail = Infinity;
+    everFailed = new Set();
+    editedPaths = new Set();
+    seenDiffs = new Map();
+    reads = new Set();
+    /** Turns in a row without an objective improvement. */
+    stale = 0;
+    lastFailures;
+    reasons = [];
+    readCap;
+    constructor(readCap = 15) { this.readCap = readCap; }
+    /** Feed one turn; returns why it counted as progress (empty = it did not). */
+    observe(t) {
+        const why = [];
+        for (const e of t.edits) {
+            const seen = this.seenDiffs.get(e.path) ?? new Set(), inv = invert(e.diff), b = body(e.diff);
+            if (!this.editedPaths.has(e.path)) {
+                this.editedPaths.add(e.path);
+                why.push(`first edit to ${e.path}`);
+            }
+            else if (b && !seen.has(b) && !seen.has(inv))
+                why.push(`new change to ${e.path}`); // a change that undoes an earlier one is not new
+            seen.add(b);
+            seen.add(inv);
+            this.seenDiffs.set(e.path, seen);
+        }
+        for (const c of t.calls) {
+            if (c.name === "sh" && typeof c.args?.cmd === "string") {
+                const n = countFailures(c.args.cmd, c.result);
+                if (n !== undefined) {
+                    this.lastFailures = n;
+                    if (n < this.bestFail) {
+                        why.push(n === 0 ? "the checks pass" : this.bestFail === Infinity ? `baseline measured: ${n} failing` : `failures down to ${n}`);
+                        this.bestFail = n;
+                    }
+                    else if (n === 0 && this.everFailed.has(c.args.cmd))
+                        why.push("a failing command now passes");
+                    if (n > 0)
+                        this.everFailed.add(c.args.cmd);
+                }
+            }
+            else if ((c.name === "read_file" || c.name === "grep" || c.name === "list_files") && !/^ERROR/.test(c.result)) {
+                const k = c.name + ":" + (c.args?.path ?? c.args?.pattern ?? "");
+                if (!this.reads.has(k) && this.reads.size < this.readCap) {
+                    this.reads.add(k);
+                    why.push("read something new");
+                }
+            }
+        }
+        this.stale = why.length ? 0 : this.stale + 1;
+        this.reasons = why;
+        return why;
+    }
+    summary() { return `${this.editedPaths.size} file(s) edited so far; failing count ${this.lastFailures === undefined ? "unknown" : this.lastFailures} (best ${this.bestFail === Infinity ? "unknown" : this.bestFail})`; }
+}

package/dist/src/prompts.d.ts

@@ -0,0 +1,10 @@
+import type { Plugin } from "./plugins.ts";
+export type PromptDef = {
+    id: string;
+    text: string;
+    file: string;
+    scope: "project" | "user";
+};
+export declare function loadPrompts(cwd: string): PromptDef[];
+/** A plugin that swaps the system prompt for a registry prompt (the id `base` keeps the built-in one). */
+export declare function promptPlugin(cwd: string, id: string): Plugin;

package/dist/src/prompts.js

@@ -0,0 +1,28 @@
+// A small prompt registry: system prompts as versioned files, so a prompt change is something you can name, diff and test.
+// `.amness/prompts/<id>.md` (and `~/.amness/prompts/`) replace the whole system prompt; the id `base` is the built-in one. `amness eval --variants full,prompt:terse` compares them.
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+const dirs = (cwd) => [{ dir: join(cwd, ".amness", "prompts"), scope: "project" }, { dir: join(process.env.AMNESS_HOME ?? join(homedir(), ".amness"), "prompts"), scope: "user" }];
+export function loadPrompts(cwd) {
+    const seen = new Set(), out = [];
+    for (const { dir, scope } of dirs(cwd))
+        if (existsSync(dir))
+            for (const f of readdirSync(dir).filter((x) => x.endsWith(".md")).sort()) {
+                const id = f.replace(/\.md$/, "");
+                if (!/^[\w.-]{1,40}$/.test(id) || seen.has(id))
+                    continue;
+                seen.add(id);
+                out.push({ id, text: readFileSync(join(dir, f), "utf8").replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, "").trim(), file: join(dir, f), scope });
+            }
+    return out;
+}
+/** A plugin that swaps the system prompt for a registry prompt (the id `base` keeps the built-in one). */
+export function promptPlugin(cwd, id) {
+    if (id === "base")
+        return { name: "prompt-base", description: "the built-in system prompt" };
+    const p = loadPrompts(cwd).find((x) => x.id === id);
+    if (!p)
+        throw new Error(`no prompt "${id}" in .amness/prompts or ~/.amness/prompts. Found: ${loadPrompts(cwd).map((x) => x.id).join(", ") || "none"}`);
+    return { name: `prompt-${id}`, description: `system prompt from ${p.file}`, system: () => p.text };
+}