amness 0.0.0-stage → 0.1.0

package/dist/src/stats.d.ts

@@ -0,0 +1,24 @@
+/** Deterministic PRNG (mulberry32) so a report can be reproduced exactly. */
+export declare function rng(seed?: number): () => number;
+export declare const mean: (xs: number[]) => number;
+export declare function quantile(sorted: number[], q: number): number;
+export type Interval = {
+    lo: number;
+    hi: number;
+};
+/** Percentile bootstrap CI of `stat` over tasks (each element is one task's data). */
+export declare function bootstrap<T>(items: T[], stat: (xs: T[]) => number, o?: {
+    B?: number;
+    seed?: number;
+    level?: number;
+}): Interval;
+/** Exact two-sided sign test: of the tasks where two variants differ, `wins` favour A and `losses` favour B (McNemar's exact test on discordant pairs). */
+export declare function signTest(wins: number, losses: number): number;
+/** Unbiased estimate of "all k of k independent attempts pass" from c passes in n trials (reliability, as in pass^k). */
+export declare const passPowK: (c: number, n: number, k: number) => number;
+/** Unbiased estimate of "at least one of k attempts passes" (pass@k). */
+export declare const passAtK: (c: number, n: number, k: number) => number;
+/** Holm-Bonferroni adjusted p-values (controls the family-wise error when comparing several variants at once). */
+export declare function holm(ps: number[]): number[];
+/** Smallest paired difference in pass rate that `n` tasks can reliably detect (rough, 80% power, alpha 0.05, worst-case variance). */
+export declare const minDetectable: (nTasks: number) => number;

package/dist/src/stats.js

@@ -0,0 +1,62 @@
+// Small, dependency-free statistics for comparing agent variants. Tasks are the unit of analysis: repeated trials of one task are
+// correlated, so every interval here resamples tasks, never individual runs.
+/** Deterministic PRNG (mulberry32) so a report can be reproduced exactly. */
+export function rng(seed = 1) {
+    let a = seed >>> 0;
+    return () => { a = (a + 0x6d2b79f5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; };
+}
+export const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : NaN);
+export function quantile(sorted, q) {
+    if (!sorted.length)
+        return NaN;
+    const i = (sorted.length - 1) * q, lo = Math.floor(i), hi = Math.ceil(i);
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * (i - lo);
+}
+/** Percentile bootstrap CI of `stat` over tasks (each element is one task's data). */
+export function bootstrap(items, stat, o = {}) {
+    if (items.length < 2) {
+        const v = items.length ? stat(items) : NaN;
+        return { lo: v, hi: v };
+    }
+    const B = o.B ?? 4000, r = rng(o.seed ?? 7), est = [];
+    for (let b = 0; b < B; b++) {
+        const s = [];
+        for (let i = 0; i < items.length; i++)
+            s.push(items[Math.floor(r() * items.length)]);
+        const v = stat(s);
+        if (!Number.isNaN(v))
+            est.push(v);
+    }
+    est.sort((x, y) => x - y);
+    const a = (1 - (o.level ?? 0.95)) / 2;
+    return { lo: quantile(est, a), hi: quantile(est, 1 - a) };
+}
+/** Exact two-sided sign test: of the tasks where two variants differ, `wins` favour A and `losses` favour B (McNemar's exact test on discordant pairs). */
+export function signTest(wins, losses) {
+    const n = wins + losses;
+    if (n === 0)
+        return 1;
+    const k = Math.min(wins, losses);
+    let p = 0, c = 1; // c = C(n, i) built iteratively
+    for (let i = 0; i <= k; i++) {
+        p += c;
+        c = (c * (n - i)) / (i + 1);
+    }
+    return Math.min(1, (2 * p) / 2 ** n);
+}
+const choose = (n, k) => { if (k < 0 || k > n)
+    return 0; let r = 1; for (let i = 1; i <= k; i++)
+    r = (r * (n - k + i)) / i; return r; };
+/** Unbiased estimate of "all k of k independent attempts pass" from c passes in n trials (reliability, as in pass^k). */
+export const passPowK = (c, n, k) => (k > n ? NaN : choose(c, k) / choose(n, k));
+/** Unbiased estimate of "at least one of k attempts passes" (pass@k). */
+export const passAtK = (c, n, k) => (k > n ? NaN : 1 - choose(n - c, k) / choose(n, k));
+/** Holm-Bonferroni adjusted p-values (controls the family-wise error when comparing several variants at once). */
+export function holm(ps) {
+    const idx = ps.map((p, i) => [p, i]).sort((a, b) => a[0] - b[0]), out = new Array(ps.length);
+    let run = 0;
+    idx.forEach(([p, i], rank) => { run = Math.max(run, Math.min(1, p * (ps.length - rank))); out[i] = run; });
+    return out;
+}
+/** Smallest paired difference in pass rate that `n` tasks can reliably detect (rough, 80% power, alpha 0.05, worst-case variance). */
+export const minDetectable = (nTasks) => (nTasks > 0 ? Math.min(1, (2.8 * Math.sqrt(0.5)) / Math.sqrt(nTasks)) : 1);

package/dist/src/strategies.d.ts

@@ -0,0 +1,16 @@
+import type { Plugin } from "./plugins.ts";
+/** Write a short plan before touching any tool, then work as usual. Costs one extra model call per request. */
+export declare function planFirst(o?: {
+    note?: string;
+}): Plugin;
+/** Two phases: look around with read-only tools for a few steps, then act. Stops the habit of editing before understanding. */
+export declare function exploreThenAct(o?: {
+    readSteps?: number;
+}): Plugin;
+/**
+ * Before finishing a task that points at a document ("as described in docs/SPEC.md", "follow the README"), make the model go through that document requirement by requirement.
+ * Models often pass the visible tests and miss the edge cases a spec lists. It asks once, so it adds one round trip, not a loop.
+ */
+export declare function specCheck(): Plugin;
+export declare const STRATEGIES: Record<string, () => Plugin>;
+export declare const strategyNames: () => string[];

package/dist/src/strategies.js

@@ -0,0 +1,35 @@
+// Loop strategies, written as plugins on the PreModel event: they change *when* the model may use which tools, without replacing the loop (so the budget, permissions, ledger and
+// tracing all keep working). They are also examples of what you can build yourself: copy one and change it. `amness eval --variants full,strategy:plan-first` measures one against the default.
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+const READ_TOOLS = ["list_files", "read_file", "grep", "repo_map", "find_refs", "impact", "task", "memory_search", "web_search", "fetch_url", "todo_write", "tool_search", "skill", "finish"];
+/** Write a short plan before touching any tool, then work as usual. Costs one extra model call per request. */
+export function planFirst(o = {}) {
+    return { name: "strategy-plan-first", description: "plan in text first, then act", hooks: {
+            PreModel: ({ step }) => (step === 0 ? { context: o.note ?? "Before you use any tool, write a short numbered plan (3 to 7 steps): what you will look at, what you will change, and how you will check it. Do not call tools in this reply.", tools: [], then: "Good. Now carry out the plan with the tools. Adjust it if you find something unexpected." } : undefined),
+        } };
+}
+/** Two phases: look around with read-only tools for a few steps, then act. Stops the habit of editing before understanding. */
+export function exploreThenAct(o = {}) {
+    const n = Math.max(1, o.readSteps ?? 6);
+    return { name: "strategy-explore-then-act", description: `read-only for the first ${n} steps, then edit`, hooks: {
+            PreModel: ({ step }) => (step === 0 ? { context: `Work in two phases. Phase 1 (the next ${n} steps): you may only read and search; edits and shell commands are refused. Phase 2: you may change things. Use phase 1 to find where the change belongs.`, tools: READ_TOOLS }
+                : step < n ? { tools: READ_TOOLS }
+                    : step === n ? { context: "Phase 2 starts now: you may edit files and run commands. Make the smallest correct change and verify it." } : undefined),
+        } };
+}
+/**
+ * Before finishing a task that points at a document ("as described in docs/SPEC.md", "follow the README"), make the model go through that document requirement by requirement.
+ * Models often pass the visible tests and miss the edge cases a spec lists. It asks once, so it adds one round trip, not a loop.
+ */
+export function specCheck() {
+    return { name: "strategy-spec-check", description: "check the work against the document the task points to", gates: [{ name: "spec-check", maxRejections: 1, check: ({ task, cwd }) => {
+                    const named = [...new Set(task.match(/[\w./-]+\.(?:md|txt|rst|ya?ml|json|adoc)\b/gi) ?? [])].filter((f) => existsSync(resolve(cwd, f)));
+                    if (!named.length && !/\b(spec|specification|requirements?|as described|according to)\b/i.test(task))
+                        return true;
+                    const where = named.length ? named.join(", ") : "the document or requirements the task refers to";
+                    return `Before you finish, open ${where} again and write its requirements as a numbered list (including edge cases, error cases and anything that says "must" or "always"). For each one say how you checked it: a test, or running the code. Fix whatever is not covered, then finish again.`;
+                } }] };
+}
+export const STRATEGIES = { "plan-first": () => planFirst(), "explore-then-act": () => exploreThenAct(), "spec-check": () => specCheck() };
+export const strategyNames = () => Object.keys(STRATEGIES);

package/dist/src/suites.d.ts

@@ -0,0 +1,30 @@
+import type { Case } from "./evals.ts";
+export type SuiteInstance = {
+    id: string;
+    repo: string;
+    baseCommit: string;
+    problem: string;
+    testPatch?: string;
+    goldPatch?: string;
+    failToPass: string[];
+    passToPass: string[];
+    installCmd?: string;
+};
+/** Read a JSONL file in SWE-bench's field names (instance_id, repo, base_commit, problem_statement, test_patch, patch, FAIL_TO_PASS, PASS_TO_PASS) or in short names (id, ...). */
+export declare function loadSuite(file: string): SuiteInstance[];
+/** Turn instances into cases. `testCmd` is a template: `{tests}` becomes the instance's FAIL_TO_PASS (and PASS_TO_PASS when `regression`) test ids joined by spaces. */
+export declare function suiteToCases(instances: SuiteInstance[], o: {
+    testCmd: string;
+    installCmd?: string;
+    regression?: boolean;
+    limit?: number;
+}): Case[];
+export type Leak = {
+    id: string;
+    problem: string;
+};
+/**
+ * Contamination checks that need no model. For every instance: (1) lines the gold patch ADDS must not appear in the problem statement, (2) they must not already exist in the
+ * base checkout (a fix that is already there makes the task trivial or broken), (3) the base commit must exist.
+ */
+export declare function leakAudit(instances: SuiteInstance[]): Promise<Leak[]>;

package/dist/src/suites.js

@@ -0,0 +1,112 @@
+// Public-benchmark style suites (SWE-bench format) as eval cases: clone a repository, check out the base commit, let the agent work, then apply the hidden tests and run them.
+// Each instance prepares itself in setup(); one that cannot be prepared is EXCLUDED and listed in the report's ledger instead of silently counting as a failure.
+import { execFile } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runShell } from "./code.js";
+const sh = (cwd, file, args, timeout = 300_000) => new Promise((res, rej) => execFile(file, args, { cwd, timeout, maxBuffer: 64 << 20 }, (e, out, err) => (e ? rej(new Error(`${file} ${args.slice(0, 3).join(" ")}: ${(err || e.message).trim().split("\n")[0]}`)) : res(out))));
+const list = (v) => { if (Array.isArray(v))
+    return v.map(String); if (typeof v === "string") {
+    try {
+        const j = JSON.parse(v);
+        if (Array.isArray(j))
+            return j.map(String);
+    }
+    catch { /* a plain string */ }
+    return v ? [v] : [];
+} return []; };
+/** Read a JSONL file in SWE-bench's field names (instance_id, repo, base_commit, problem_statement, test_patch, patch, FAIL_TO_PASS, PASS_TO_PASS) or in short names (id, ...). */
+export function loadSuite(file) {
+    const out = [];
+    for (const [i, l] of readFileSync(file, "utf8").split("\n").entries()) {
+        if (!l.trim())
+            continue;
+        let j;
+        try {
+            j = JSON.parse(l);
+        }
+        catch {
+            throw new Error(`${file}:${i + 1} is not valid JSON`);
+        }
+        const id = String(j.instance_id ?? j.id ?? ""), repo = String(j.repo ?? ""), base = String(j.base_commit ?? j.baseCommit ?? ""), problem = String(j.problem_statement ?? j.problem ?? "");
+        if (!id || !repo || !base || !problem)
+            throw new Error(`${file}:${i + 1}: each instance needs instance_id, repo, base_commit and problem_statement`);
+        out.push({ id, repo: /^[\w.-]+\/[\w.-]+$/.test(repo) && !repo.startsWith(".") ? `https://github.com/${repo}.git` : repo, baseCommit: base, problem, testPatch: (j.test_patch ?? j.testPatch), goldPatch: (j.patch ?? j.gold_patch), failToPass: list(j.FAIL_TO_PASS ?? j.fail_to_pass), passToPass: list(j.PASS_TO_PASS ?? j.pass_to_pass), installCmd: j.install_cmd });
+    }
+    return out;
+}
+/** Turn instances into cases. `testCmd` is a template: `{tests}` becomes the instance's FAIL_TO_PASS (and PASS_TO_PASS when `regression`) test ids joined by spaces. */
+export function suiteToCases(instances, o) {
+    return instances.slice(0, o.limit ?? instances.length).map((inst) => {
+        const tests = [...inst.failToPass, ...(o.regression ? inst.passToPass : [])].join(" "), cmd = o.testCmd.replace("{tests}", tests);
+        return {
+            name: inst.id, files: {}, task: `${inst.problem.trim()}\n\nFix the problem in this repository. Do not modify existing tests.`,
+            setup: async (dir) => {
+                await sh(dir, "git", ["clone", "--quiet", inst.repo, "."]);
+                await sh(dir, "git", ["checkout", "--quiet", inst.baseCommit]);
+                const install = inst.installCmd ?? o.installCmd;
+                if (install) {
+                    const r = await runShell(install, dir, 600_000);
+                    if (!r.startsWith("exit=0"))
+                        throw new Error(`install failed: ${r.split("\n")[1] ?? r.slice(0, 100)}`);
+                }
+            },
+            // The hidden tests are applied only now, after the agent is done, from a file outside the workspace.
+            check: async (dir) => {
+                if (inst.testPatch) {
+                    const tmp = mkdtempSync(join(tmpdir(), "amness-tp-")), f = join(tmp, "test.patch");
+                    writeFileSync(f, inst.testPatch.endsWith("\n") ? inst.testPatch : inst.testPatch + "\n");
+                    try {
+                        await sh(dir, "git", ["apply", "--whitespace=nowarn", f]);
+                    }
+                    catch (e) {
+                        return `could not apply the hidden tests (${e.message})`;
+                    }
+                    finally {
+                        rmSync(tmp, { recursive: true, force: true });
+                    }
+                }
+                return (await runShell(cmd, dir, 900_000)).startsWith("exit=0") || "the hidden tests still fail";
+            },
+        };
+    });
+}
+/**
+ * Contamination checks that need no model. For every instance: (1) lines the gold patch ADDS must not appear in the problem statement, (2) they must not already exist in the
+ * base checkout (a fix that is already there makes the task trivial or broken), (3) the base commit must exist.
+ */
+export async function leakAudit(instances) {
+    const out = [];
+    for (const inst of instances) {
+        const dir = mkdtempSync(join(tmpdir(), "amness-audit-"));
+        try {
+            try {
+                await sh(dir, "git", ["clone", "--quiet", "--no-checkout", inst.repo, "."]);
+                await sh(dir, "git", ["checkout", "--quiet", inst.baseCommit]);
+            }
+            catch (e) {
+                out.push({ id: inst.id, problem: `cannot check out the base commit: ${e.message}` });
+                continue;
+            }
+            const added = [...new Set((inst.goldPatch ?? "").split("\n").filter((l) => l.startsWith("+") && !l.startsWith("+++")).map((l) => l.slice(1).trim()).filter((l) => l.length >= 25))];
+            const inProblem = added.filter((l) => inst.problem.includes(l));
+            if (inProblem.length)
+                out.push({ id: inst.id, problem: `the problem statement contains ${inProblem.length} line(s) of the fix, e.g. "${inProblem[0].slice(0, 60)}"` });
+            let present = 0;
+            for (const l of added.slice(0, 40)) {
+                try {
+                    if ((await sh(dir, "git", ["grep", "-F", "-q", "--", l]).then(() => true, () => false)))
+                        present++;
+                }
+                catch { /* ignore */ }
+            }
+            if (added.length >= 3 && present / Math.min(added.length, 40) > 0.6)
+                out.push({ id: inst.id, problem: `${present} of the ${Math.min(added.length, 40)} lines the fix adds already exist in the base checkout (the fix may already be applied)` });
+        }
+        finally {
+            rmSync(dir, { recursive: true, force: true });
+        }
+    }
+    return out;
+}

package/dist/src/trace.d.ts

@@ -0,0 +1,63 @@
+import type { Event } from "./harness.ts";
+export type TraceLine = {
+    t: string;
+    ms: number;
+    kind: string;
+    [k: string]: unknown;
+};
+export type TracerOptions = {
+    sink?: (line: TraceLine) => void;
+    file?: string;
+    otlp?: string;
+    sessionId: string;
+    model?: string; /** extra resource attributes */
+    attrs?: Record<string, string>;
+};
+export declare class Tracer {
+    readonly file?: string;
+    private sink?;
+    private otlp?;
+    private id;
+    private model?;
+    private attrs;
+    private t0;
+    private traceId;
+    private rootId;
+    private spans;
+    private open;
+    private turnStart;
+    private textChars;
+    private pending;
+    /** Rolling data for the cache-waste figure. */
+    turns: {
+        prompt: number;
+        cached: number;
+        break?: boolean;
+    }[];
+    constructor(o: TracerOptions);
+    private agent?;
+    private write;
+    /** Record the messages the model produced (needed for replay). */
+    message(m: {
+        role: string;
+        content: string | null;
+        tool_calls?: unknown;
+    }): void;
+    event(e: Event & {
+        agent?: string;
+    }): void;
+    private add;
+    /** OTLP/HTTP JSON body for the spans collected so far. */
+    payload(): object;
+    flush(): void;
+    close(): Promise<void>;
+}
+/**
+ * Prompt tokens that were re-read at full price although the prefix should have been cached: for each turn, the uncached part beyond what was newly added since the
+ * previous turn. A fresh start (first turn) and the turns right after a deliberate rewrite are not waste, so only turns where the cache really should have hit count.
+ */
+export declare function cacheWaste(turns: {
+    prompt: number;
+    cached: number;
+    break?: boolean;
+}[]): number;

package/dist/src/trace.js

@@ -0,0 +1,153 @@
+// Tracing: every loop event goes to a local JSONL file (always, cheap, greppable) and optionally to an OpenTelemetry collector (OTLP/HTTP JSON, plain fetch).
+// The JSONL file is also what `replay` and failure mining read, so it keeps what is needed to re-run the harness logic: the model's messages and tool results.
+import { appendFileSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { randomBytes } from "node:crypto";
+import { redactDeep } from "./secrets.js";
+const clip = (s, n) => { const t = typeof s === "string" ? s : JSON.stringify(s) ?? ""; return t.length > n ? t.slice(0, n) + `…[${t.length - n} more]` : t; };
+const hex = (n) => randomBytes(n).toString("hex");
+const ns = (ms) => String(Math.round(ms * 1e6));
+const attrVal = (v) => (typeof v === "string" ? { stringValue: v } : typeof v === "boolean" ? { boolValue: v } : Number.isInteger(v) ? { intValue: String(v) } : { doubleValue: v });
+export class Tracer {
+    file;
+    sink;
+    otlp;
+    id;
+    model;
+    attrs;
+    t0 = Date.now();
+    traceId = hex(16);
+    rootId = hex(8);
+    spans = [];
+    open = new Map();
+    turnStart = 0;
+    textChars = 0;
+    pending = [];
+    /** Rolling data for the cache-waste figure. */
+    turns = [];
+    constructor(o) { this.file = o.file; this.sink = o.sink; this.otlp = o.otlp; this.id = o.sessionId; this.model = o.model; this.attrs = o.attrs ?? {}; if (o.file) {
+        try {
+            mkdirSync(dirname(o.file), { recursive: true });
+        }
+        catch { /* writes below will fail quietly */ }
+    } }
+    agent;
+    write(kind, rest) {
+        if (!this.file && !this.sink)
+            return;
+        const line = { t: new Date().toISOString(), ms: Date.now() - this.t0, kind, ...(this.agent && { agent: this.agent }), ...redactDeep(rest) }; // secrets never reach the trace file
+        this.sink?.(line);
+        if (!this.file)
+            return;
+        try {
+            appendFileSync(this.file, JSON.stringify(line) + "\n");
+        }
+        catch { /* a full disk must not stop the agent */ }
+    }
+    /** Record the messages the model produced (needed for replay). */
+    message(m) { this.write("message", { role: m.role, content: clip(m.content ?? "", 20_000), tool_calls: m.tool_calls }); }
+    event(e) {
+        const now = Date.now();
+        this.agent = e.agent && e.agent !== "lead" ? e.agent : undefined;
+        switch (e.kind) {
+            case "text":
+            case "thinking":
+                this.textChars += e.delta.length;
+                return; // too chatty; the finished message is recorded instead
+            case "run":
+                this.write("run", { messages: e.messages.map((m) => ({ ...m, content: clip(m.content ?? "", 60_000) })) });
+                return;
+            case "model":
+                this.write("model", { message: { ...e.message, content: clip(e.message.content ?? "", 60_000) } });
+                return;
+            case "assistant":
+                this.write("assistant", { chars: e.text.length, streamed: e.streamed });
+                this.textChars = 0;
+                return;
+            case "phase":
+                if (e.phase === "model")
+                    this.turnStart = now;
+                this.write("phase", { phase: e.phase, step: e.step, n: e.n });
+                return;
+            case "turn": {
+                this.turns.push({ prompt: e.prompt, cached: e.cached });
+                this.write("turn", { step: e.step, prompt: e.prompt, completion: e.completion, cached: e.cached, reasoning: e.reasoning, ms: now - (this.turnStart || now) });
+                this.add({ name: `chat ${this.model ?? ""}`.trim(), start: this.turnStart || now, end: now, attrs: { "gen_ai.operation.name": "chat", ...(this.model && { "gen_ai.request.model": this.model }), "gen_ai.usage.input_tokens": e.prompt, "gen_ai.usage.output_tokens": e.completion, "gen_ai.usage.cache_read.input_tokens": e.cached, "amness.step": e.step } });
+                return;
+            }
+            case "toolstart": {
+                const s = { name: `execute_tool ${e.name}`, spanId: hex(8), start: now, end: now, attrs: { "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": e.name } };
+                this.open.set(e.id, s);
+                this.write("toolstart", { id: e.id, name: e.name, args: clip(e.args, 20_000) });
+                return;
+            }
+            case "tool": {
+                const s = this.open.get(e.id ?? "");
+                this.open.delete(e.id ?? "");
+                const failed = /^(ERROR|DENIED|REJECTED|BLOCKED)/.test(e.result);
+                this.write("tool", { id: e.id, name: e.name, ms: s ? now - s.start : undefined, ok: !failed, result: clip(e.result, 100_000) });
+                if (s) {
+                    s.end = now;
+                    s.attrs["amness.tool.ok"] = !failed;
+                    if (failed)
+                        s.error = clip(e.result, 200);
+                    this.add(s);
+                }
+                return;
+            }
+            case "cachebreak": {
+                const last = this.turns.at(-1);
+                if (last)
+                    last.break = true;
+                this.write("cachebreak", { reason: e.reason, at: e.at });
+                return;
+            }
+            case "hook":
+                this.write("hook", { event: e.event, detail: clip(e.detail, 500), blocked: e.blocked });
+                this.add({ name: `hook ${e.event}`, start: now, end: now, attrs: { "amness.hook.event": e.event, "amness.hook.blocked": e.blocked } });
+                return;
+            case "edit":
+                this.write("edit", { path: e.path, created: e.created, added: e.added, removed: e.removed });
+                return;
+            default: {
+                const d = e;
+                this.write(e.kind, { detail: clip(d.detail ?? d.summary ?? d.text ?? (d.saved !== undefined ? `${d.saved} chars: ${d.what}` : ""), 1000) });
+            }
+        }
+    }
+    add(s) {
+        if (!this.otlp)
+            return;
+        this.spans.push({ ...s, spanId: s.spanId ?? hex(8), parent: this.rootId });
+        if (this.spans.length >= 50)
+            this.flush();
+    }
+    /** OTLP/HTTP JSON body for the spans collected so far. */
+    payload() {
+        const root = { name: "amness session", spanId: this.rootId, start: this.t0, end: Date.now(), attrs: { "amness.session.id": this.id } };
+        const all = [root, ...this.spans];
+        return { resourceSpans: [{ resource: { attributes: [{ key: "service.name", value: { stringValue: "amness" } }, ...Object.entries(this.attrs).map(([k, v]) => ({ key: k, value: { stringValue: v } }))] },
+                    scopeSpans: [{ scope: { name: "amness" }, spans: all.map((s) => ({ traceId: this.traceId, spanId: s.spanId, ...(s.parent && { parentSpanId: s.parent }), name: s.name, kind: 1, startTimeUnixNano: ns(s.start), endTimeUnixNano: ns(Math.max(s.end, s.start)),
+                                attributes: Object.entries(s.attrs).map(([key, v]) => ({ key, value: attrVal(v) })), ...(s.error && { status: { code: 2, message: s.error } }) })) }] }] };
+    }
+    flush() {
+        if (!this.otlp || !this.spans.length)
+            return;
+        const body = JSON.stringify(this.payload());
+        this.spans = [];
+        this.pending.push(fetch(this.otlp.replace(/\/$/, "").replace(/\/v1\/traces$/, "") + "/v1/traces", { method: "POST", headers: { "content-type": "application/json" }, body }).catch(() => { }));
+    }
+    async close() { this.write("end", {}); this.flush(); await Promise.allSettled(this.pending); }
+}
+/**
+ * Prompt tokens that were re-read at full price although the prefix should have been cached: for each turn, the uncached part beyond what was newly added since the
+ * previous turn. A fresh start (first turn) and the turns right after a deliberate rewrite are not waste, so only turns where the cache really should have hit count.
+ */
+export function cacheWaste(turns) {
+    let waste = 0;
+    for (let i = 1; i < turns.length; i++) {
+        const grew = Math.max(0, turns[i].prompt - turns[i - 1].prompt), uncached = Math.max(0, turns[i].prompt - turns[i].cached);
+        waste += Math.max(0, uncached - grew);
+    }
+    return waste;
+}

package/dist/src/tune.d.ts

@@ -0,0 +1,38 @@
+import { type Case, type Row, type Variant } from "./evals.ts";
+import type { Profile } from "./profiles.ts";
+import type { Guard } from "./harness.ts";
+export type Step = {
+    change: string;
+    stage: "train" | "heldout";
+    accepted: boolean;
+    why: string;
+};
+export type TuneResult = {
+    profile: Profile;
+    accepted: string[];
+    steps: Step[];
+    train: string[];
+    heldOut: string[];
+    strength: string;
+};
+export type TuneOptions = {
+    cases: Case[];
+    /** Build a runnable variant for a profile. `amness tune` uses the real code agent; tests use a scripted one. */
+    makeVariant: (profile: Profile, name: string) => Variant;
+    base?: Profile;
+    trials?: number;
+    jobs?: number;
+    seed?: number;
+    heldOutFraction?: number;
+    guards?: Guard[];
+    stagnationTurns?: number[];
+    onStep?: (s: Step) => void;
+    onRow?: (r: Row) => void;
+};
+/** A deterministic split of tasks into a training set and a held-out set (at least one task in each when there are two or more). */
+export declare function split(cases: Case[], fraction?: number, seed?: number): {
+    train: Case[];
+    heldOut: Case[];
+};
+export declare function tune(o: TuneOptions): Promise<TuneResult>;
+export declare function writeProfile(file: string, model: string, p: Profile): void;

package/dist/src/tune.js

@@ -0,0 +1,49 @@
+// `amness tune`: bounded self-improvement of the harness's own settings. The search space is small and fixed (switch one guard off, change a limit), never free-form edits.
+// A change is only kept if it holds up twice: on the training tasks, and again on tasks the search never saw. Everything else is rolled back; the profile file is written only on success.
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { analyze, evaluate } from "./evals.js";
+import { rng } from "./stats.js";
+const DEFAULT_GUARDS = ["nudge", "loop", "stuck", "wrapup", "condense", "stagnation"];
+/** A deterministic split of tasks into a training set and a held-out set (at least one task in each when there are two or more). */
+export function split(cases, fraction = 0.34, seed = 1) {
+    const r = rng(seed), order = cases.map((_, i) => [r(), i]).sort((a, b) => a[0] - b[0]).map(([, i]) => i);
+    const n = cases.length < 2 ? 0 : Math.max(1, Math.round(cases.length * fraction)), held = new Set(order.slice(0, n));
+    return { train: cases.filter((_, i) => !held.has(i)), heldOut: cases.filter((_, i) => held.has(i)) };
+}
+const withChange = (p, c) => ({ ...p, ...(c.guard && { off: [...new Set([...(p.off ?? []), c.guard])] }), ...(c.stagnation !== undefined && { stagnationTurns: c.stagnation }) });
+/** Is `cand` at least as good as `cur` on these cases? Never worse on pass rate, and either better or cheaper per solved task. */
+async function compare(cases, cur, cand, o) {
+    const { rows } = await evaluate(cases, [o.makeVariant(cur, "current"), o.makeVariant(cand, "candidate")], { trials: o.trials ?? 2, jobs: o.jobs, baseline: "current", seed: o.seed, onRow: o.onRow });
+    const a = analyze(rows, { baseline: "current", seed: o.seed }), c = a.comparisons.find((x) => x.variant === "candidate");
+    const cheaper = Number.isFinite(c.dTokensPerSolved) && c.dTokensPerSolved < 0;
+    const ok = c.verdict !== "worse" && c.dPass >= -0.02 && (c.verdict === "better" || c.dPass > 0 || cheaper);
+    return { ok, why: `pass ${c.dPass >= 0 ? "+" : ""}${(c.dPass * 100).toFixed(0)} pts (CI ${(c.ci.lo * 100).toFixed(0)} to ${(c.ci.hi * 100).toFixed(0)}), tokens per solved task ${Number.isFinite(c.dTokensPerSolved) ? Math.round(c.dTokensPerSolved).toLocaleString() : "n/a"}`, minDetectable: a.minDetectable, tasks: a.tasks };
+}
+export async function tune(o) {
+    const { train, heldOut } = split(o.cases, o.heldOutFraction, o.seed), steps = [], accepted = [];
+    const note = (s) => { steps.push(s); o.onStep?.(s); };
+    const startingPoint = { ...(o.base ?? {}) };
+    let cur = startingPoint, strength = "";
+    const candidates = [...(o.guards ?? DEFAULT_GUARDS).filter((g) => !(cur.off ?? []).includes(g)).map((g) => ({ label: `switch off ${g}`, c: { guard: g } })), ...(o.stagnationTurns ?? []).map((n) => ({ label: `stagnationTurns = ${n}`, c: { stagnation: n } }))];
+    for (const { label, c } of candidates) {
+        const cand = withChange(cur, c), t = await compare(train, cur, cand, o);
+        strength = `${t.tasks} training task(s) can only detect differences above ~${(t.minDetectable * 100).toFixed(0)} points`;
+        note({ change: label, stage: "train", accepted: t.ok, why: t.why });
+        if (t.ok) {
+            cur = cand;
+            accepted.push(label);
+        }
+    }
+    // The held-out gate: all accepted changes together must not hurt on tasks the search never looked at; otherwise everything is rolled back.
+    if (accepted.length) {
+        const h = heldOut.length ? await compare(heldOut, startingPoint, cur, o) : { ok: false, why: "no held-out tasks (need at least 2 tasks); nothing is kept without that check" };
+        note({ change: accepted.join(" + "), stage: "heldout", accepted: h.ok, why: h.why });
+        if (!h.ok) {
+            cur = startingPoint;
+            accepted.length = 0;
+        }
+    }
+    return { profile: cur, accepted, steps, train: train.map((c) => c.name), heldOut: heldOut.map((c) => c.name), strength };
+}
+export function writeProfile(file, model, p) { mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, JSON.stringify({ models: { [model]: { ...p, note: "tuned with a held-out check (amness tune)" } } }, null, 2) + "\n"); }

package/dist/src/web.d.ts

@@ -0,0 +1,19 @@
+export type SearchHit = {
+    title: string;
+    url: string;
+    snippet: string;
+};
+export type SearchProvider = (query: string, n: number) => Promise<SearchHit[]>;
+/** Tavily (needs TAVILY_API_KEY): returns clean snippets built for agents. */
+export declare const tavily: (key?: string) => SearchProvider;
+/** Keyless fallback: scrapes DuckDuckGo's HTML endpoint. Fragile by nature; prefer a real provider in production. */
+export declare const duckduckgo: SearchProvider;
+/** Fetch a URL and reduce it to readable text. Refuses non-http(s) and local/private hosts. */
+export declare function fetchText(url: string, max?: number, offset?: number): Promise<string>;
+export declare const webTools: (search?: SearchProvider) => (import("./harness.ts").Tool<{
+    query: string;
+    n?: number;
+}> | import("./harness.ts").Tool<{
+    url: string;
+    offset?: number;
+}>)[];