@miller-tech/uap 1.175.13 → 1.178.0

package/dist/telemetry/tool-calls.js

@@ -0,0 +1,280 @@
+/**
+ * Per-tool-call evidence corpus (harness plan areas D1/D3, 2026-07-31).
+ *
+ * Before this module, UAP's telemetry recorded sessions, routing decisions and
+ * time series — but NOT what happened inside the agent loop. The self-harness
+ * propose stage therefore had no evidence to reason from, which is why it could
+ * only search inference-server knobs: those are the only thing it could measure.
+ *
+ * Agentic Harness Engineering (arXiv 2604.25850) calls this layer "experience
+ * observability": distil trajectories into a layered corpus with per-task failure
+ * analysis and a benchmark-level overview, so mutation proposal is grounded
+ * rather than blind. This is that corpus, at the resolution that matters — one
+ * row per tool call, attributed to a harness component and a stable failure
+ * class.
+ *
+ * Every write is best-effort and fails open. Telemetry must never break the
+ * agent loop it is observing.
+ */
+import Database from 'better-sqlite3';
+import { existsSync, mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { classifyToolResult, componentForTool, isFailureClass, } from './tool-failure.js';
+/** Rolling cap: the corpus is a diagnosis aid, not an audit log. */
+const MAX_TOOL_CALL_ROWS = 20_000;
+const dbCache = new Map();
+/**
+ * Handle cap. A paired bench runs each cell in its own temp dir, so an unbounded
+ * per-cwd cache leaks a better-sqlite3 handle (x3 files with WAL/SHM) per cell
+ * and hits the fd limit on a 200-cell run.
+ */
+const MAX_CACHED_DBS = 8;
+let insertCount = 0;
+/**
+ * Where the corpus lives.
+ *
+ * NOT under the target project. Two reasons, both found in review:
+ *
+ *  - A paired bench runs every cell in a throwaway temp dir, so a project-rooted
+ *    corpus is written into directories that are deleted moments later. The
+ *    evidence from the exact workload the loop is meant to learn from vanished,
+ *    and `uap harness evidence` read a different path again (`process.cwd()`).
+ *  - On a customer repo it created `agents/data/memory/telemetry.db` inside
+ *    their tree — untracked noise at best, committed by an `add -A` at worst.
+ *
+ * A single per-user store keeps the corpus continuous across runs, which is what
+ * cross-run failure ranking needs. `UAP_TOOL_CORPUS_DIR` overrides it (tests).
+ */
+export function toolCorpusRoot() {
+    return process.env.UAP_TOOL_CORPUS_DIR || join(homedir(), '.uap', 'telemetry');
+}
+function dbPathFor(root) {
+    return join(root, 'tool-calls.db');
+}
+function openStore(cwd) {
+    const cached = dbCache.get(cwd);
+    if (cached)
+        return cached;
+    try {
+        const dbPath = dbPathFor(cwd);
+        mkdirSync(dirname(dbPath), { recursive: true });
+        const db = new Database(dbPath);
+        // Same pragmas as the other producers of this file: WAL so the dashboard
+        // reader never blocks the agent loop, and a SHORT busy timeout because a
+        // dropped telemetry row beats stalling a tool call.
+        db.pragma('journal_mode = WAL');
+        db.pragma('synchronous = NORMAL');
+        db.pragma('busy_timeout = 2000');
+        db.exec(`
+      CREATE TABLE IF NOT EXISTS tool_calls (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts TEXT NOT NULL DEFAULT (datetime('now')),
+        run_id TEXT,
+        task_id TEXT,
+        component TEXT NOT NULL,
+        tool TEXT NOT NULL,
+        outcome TEXT NOT NULL,
+        failed INTEGER NOT NULL DEFAULT 0,
+        turn INTEGER,
+        path TEXT,
+        latency_ms INTEGER,
+        bytes INTEGER,
+        detail TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_tool_calls_run ON tool_calls(run_id);
+      CREATE INDEX IF NOT EXISTS idx_tool_calls_outcome ON tool_calls(outcome);
+      CREATE INDEX IF NOT EXISTS idx_tool_calls_component ON tool_calls(component);
+    `);
+        // Prune on OPEN, not on an insert counter: a deliver process records ~12
+        // rows and exits, so a `% 100` counter reset per process meant the row cap
+        // was never enforced and the table grew without bound across runs.
+        try {
+            db.prepare('DELETE FROM tool_calls WHERE id <= (SELECT MAX(id) - ? FROM tool_calls)').run(MAX_TOOL_CALL_ROWS);
+        }
+        catch {
+            /* best-effort */
+        }
+        if (dbCache.size >= MAX_CACHED_DBS) {
+            const oldest = dbCache.keys().next().value;
+            if (oldest !== undefined) {
+                try {
+                    dbCache.get(oldest)?.close();
+                }
+                catch {
+                    /* ignore */
+                }
+                dbCache.delete(oldest);
+            }
+        }
+        dbCache.set(cwd, db);
+        return db;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Record one tool call. Returns the class it was filed under so callers can
+ * assert on it in tests without reading the DB back.
+ */
+export function recordToolCall(rec, root = toolCorpusRoot()) {
+    const outcome = classifyToolResult(rec.tool, rec.result);
+    const db = openStore(root);
+    if (!db)
+        return outcome;
+    try {
+        const component = rec.component ?? componentForTool(rec.tool);
+        // Only failures keep a detail excerpt, and only the FIRST LINE of one.
+        // The full text of an edit-miss now embeds `nearestMatchReport` — numbered
+        // lines of the user's current source — which this table would otherwise
+        // persist indefinitely, and into the target repo's tree at that. The first
+        // line carries the reason; the source body carries risk and no signal.
+        const detail = isFailureClass(outcome)
+            ? String(rec.result ?? '').split('\n')[0].slice(0, 300)
+            : null;
+        db.prepare(`INSERT INTO tool_calls
+         (run_id, task_id, component, tool, outcome, failed, turn, path, latency_ms, bytes, detail)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(rec.runId ?? null, rec.taskId ?? null, component, rec.tool, outcome, isFailureClass(outcome) ? 1 : 0, rec.turn ?? null, rec.path ?? null, rec.latencyMs ?? null, rec.bytes ?? null, detail);
+        if (++insertCount % 100 === 0) {
+            db.prepare(`DELETE FROM tool_calls WHERE id <= (SELECT MAX(id) - ? FROM tool_calls)`).run(MAX_TOOL_CALL_ROWS);
+        }
+    }
+    catch {
+        /* best-effort */
+    }
+    return outcome;
+}
+const EMPTY_SUMMARY = {
+    totalCalls: 0,
+    failedCalls: 0,
+    failureRate: 0,
+    byComponent: [],
+    topFailures: [],
+    editHealth: { attempts: 0, exact: 0, tolerant: 0, misses: 0, ambiguous: 0, successRate: 0 },
+};
+/**
+ * Distil the corpus into the layered view the propose stage consumes: an
+ * overall health number, per-component attribution, a ranked failure queue, and
+ * the edit-tool detail.
+ *
+ * `runId` scopes the summary to a single run; omit it for the whole corpus.
+ */
+export function summarizeToolCalls(runId, root = toolCorpusRoot()) {
+    if (!existsSync(dbPathFor(root)))
+        return EMPTY_SUMMARY;
+    const db = openStore(root);
+    if (!db)
+        return EMPTY_SUMMARY;
+    const where = runId ? 'WHERE run_id = ?' : '';
+    const params = runId ? [runId] : [];
+    try {
+        const totals = db
+            .prepare(`SELECT COUNT(*) AS calls, COALESCE(SUM(failed),0) AS failures FROM tool_calls ${where}`)
+            .get(...params);
+        if (!totals || totals.calls === 0)
+            return EMPTY_SUMMARY;
+        const byComponent = db
+            .prepare(`SELECT component, COUNT(*) AS calls, COALESCE(SUM(failed),0) AS failures
+         FROM tool_calls ${where}
+         GROUP BY component ORDER BY failures DESC, calls DESC`)
+            .all(...params);
+        const topFailures = db
+            .prepare(`SELECT component, tool, outcome, COUNT(*) AS count
+         FROM tool_calls ${where ? `${where} AND failed = 1` : 'WHERE failed = 1'}
+         GROUP BY component, tool, outcome ORDER BY count DESC LIMIT 20`)
+            .all(...params);
+        const editRows = db
+            .prepare(`SELECT outcome, COUNT(*) AS count FROM tool_calls
+         ${where ? `${where} AND` : 'WHERE'} tool IN ('edit_file','edit_range')
+         GROUP BY outcome`)
+            .all(...params);
+        const editCount = (o) => editRows.find((r) => r.outcome === o)?.count ?? 0;
+        const exact = editCount('ok');
+        const tolerant = editCount('ok-tolerant');
+        const misses = editCount('edit-miss');
+        const ambiguous = editCount('edit-ambiguous');
+        const attempts = editRows.reduce((n, r) => n + r.count, 0);
+        return {
+            totalCalls: totals.calls,
+            failedCalls: totals.failures,
+            failureRate: totals.calls > 0 ? totals.failures / totals.calls : 0,
+            byComponent: byComponent.map((r) => ({
+                component: r.component,
+                failures: r.failures,
+                calls: r.calls,
+            })),
+            topFailures,
+            editHealth: {
+                attempts,
+                exact,
+                tolerant,
+                misses,
+                ambiguous,
+                successRate: attempts > 0 ? (exact + tolerant) / attempts : 0,
+            },
+        };
+    }
+    catch {
+        return EMPTY_SUMMARY;
+    }
+}
+/**
+ * Render the summary as the prose block the propose stage puts in front of a
+ * model. Kept here (not in the prompt) so evidence and its formatting version
+ * together, and so a human can read the same text with `uap harness evidence`.
+ */
+export function renderEvidence(summary) {
+    if (summary.totalCalls === 0)
+        return 'No tool-call evidence recorded yet.';
+    const lines = [];
+    lines.push(`Tool calls: ${summary.totalCalls} | failed: ${summary.failedCalls} ` +
+        `(${(summary.failureRate * 100).toFixed(1)}%)`);
+    if (summary.byComponent.length > 0) {
+        lines.push('');
+        lines.push('By harness component (failures/calls):');
+        for (const c of summary.byComponent) {
+            lines.push(`  ${c.component.padEnd(10)} ${c.failures}/${c.calls}`);
+        }
+    }
+    const e = summary.editHealth;
+    if (e.attempts > 0) {
+        lines.push('');
+        lines.push(`Edit tool: ${e.attempts} attempts | exact ${e.exact} | tolerant ${e.tolerant} | ` +
+            `miss ${e.misses} | ambiguous ${e.ambiguous} | success ${(e.successRate * 100).toFixed(1)}%`);
+    }
+    if (summary.topFailures.length > 0) {
+        lines.push('');
+        lines.push('Top failure classes:');
+        for (const f of summary.topFailures.slice(0, 10)) {
+            lines.push(`  ${String(f.count).padStart(4)}x ${f.tool}/${f.outcome} [${f.component}]`);
+        }
+    }
+    return lines.join('\n');
+}
+/** Test-only: close cached handles so temp-dir fixtures do not leak. */
+export function closeToolCallStores(root) {
+    if (root) {
+        const db = dbCache.get(root);
+        if (db) {
+            try {
+                db.close();
+            }
+            catch {
+                /* ignore */
+            }
+            dbCache.delete(root);
+        }
+        return;
+    }
+    for (const db of dbCache.values()) {
+        try {
+            db.close();
+        }
+        catch {
+            /* ignore */
+        }
+    }
+    dbCache.clear();
+}
+//# sourceMappingURL=tool-calls.js.map

package/dist/telemetry/tool-calls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.js","sourceRoot":"","sources":["../../src/telemetry/tool-calls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EACL,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,GAGf,MAAM,mBAAmB,CAAC;AAE3B,oEAAoE;AACpE,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;AACrD;;;;GAIG;AACH,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,IAAI,WAAW,GAAG,CAAC,CAAC;AAEpB;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,cAAc;IAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,yEAAyE;QACzE,yEAAyE;QACzE,oDAAoD;QACpD,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAChC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACjC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;KAmBP,CAAC,CAAC;QACH,yEAAyE;QACzE,2EAA2E;QAC3E,mEAAmE;QACnE,IAAI,CAAC;YACH,EAAE,CAAC,OAAO,CAAC,yEAAyE,CAAC,CAAC,GAAG,CACvF,kBAAkB,CACnB,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB;QACnB,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC;gBAC/B,CAAC;gBAAC,MAAM,CAAC;oBACP,YAAY;gBACd,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAgBD;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,GAAmB,EAAE,IAAI,GAAG,cAAc,EAAE;IACzE,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,CAAC,EAAE;QAAE,OAAO,OAAO,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9D,uEAAuE;QACvE,2EAA2E;QAC3E,wEAAwE;QACxE,2EAA2E;QAC3E,uEAAuE;QACvE,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC;YACpC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACvD,CAAC,CAAC,IAAI,CAAC;QACT,EAAE,CAAC,OAAO,CACR;;gDAE0C,CAC3C,CAAC,GAAG,CACH,GAAG,CAAC,KAAK,IAAI,IAAI,EACjB,GAAG,CAAC,MAAM,IAAI,IAAI,EAClB,SAAS,EACT,GAAG,CAAC,IAAI,EACR,OAAO,EACP,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAC/B,GAAG,CAAC,IAAI,IAAI,IAAI,EAChB,GAAG,CAAC,IAAI,IAAI,IAAI,EAChB,GAAG,CAAC,SAAS,IAAI,IAAI,EACrB,GAAG,CAAC,KAAK,IAAI,IAAI,EACjB,MAAM,CACP,CAAC;QACF,IAAI,EAAE,WAAW,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YAC9B,EAAE,CAAC,OAAO,CACR,yEAAyE,CAC1E,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iBAAiB;IACnB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AA8BD,MAAM,aAAa,GAAoB;IACrC,UAAU,EAAE,CAAC;IACb,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,EAAE;IACf,WAAW,EAAE,EAAE;IACf,UAAU,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE;CAC5F,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAc,EAAE,IAAI,GAAG,cAAc,EAAE;IACxE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,aAAa,CAAC;IACvD,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,CAAC,EAAE;QAAE,OAAO,aAAa,CAAC;IAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,EAAE;aACd,OAAO,CACN,iFAAiF,KAAK,EAAE,CACzF;aACA,GAAG,CAAC,GAAG,MAAM,CAAwC,CAAC;QACzD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,CAAC;YAAE,OAAO,aAAa,CAAC;QAExD,MAAM,WAAW,GAAG,EAAE;aACnB,OAAO,CACN;2BACmB,KAAK;+DAC+B,CACxD;aACA,GAAG,CAAC,GAAG,MAAM,CAAkE,CAAC;QAEnF,MAAM,WAAW,GAAG,EAAE;aACnB,OAAO,CACN;2BACmB,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC,CAAC,kBAAkB;wEACT,CACjE;aACA,GAAG,CAAC,GAAG,MAAM,CAAqB,CAAC;QAEtC,MAAM,QAAQ,GAAG,EAAE;aAChB,OAAO,CACN;WACG,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO;0BACjB,CACnB;aACA,GAAG,CAAC,GAAG,MAAM,CAA8C,CAAC;QAE/D,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;QACnF,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAE3D,OAAO;YACL,UAAU,EAAE,MAAM,CAAC,KAAK;YACxB,WAAW,EAAE,MAAM,CAAC,QAAQ;YAC5B,WAAW,EAAE,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAClE,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACnC,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;aACf,CAAC,CAAC;YACH,WAAW;YACX,UAAU,EAAE;gBACV,QAAQ;gBACR,KAAK;gBACL,QAAQ;gBACR,MAAM;gBACN,SAAS;gBACT,WAAW,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC9D;SACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,aAAa,CAAC;IACvB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,OAAwB;IACrD,IAAI,OAAO,CAAC,UAAU,KAAK,CAAC;QAAE,OAAO,qCAAqC,CAAC;IAC3E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CACR,eAAe,OAAO,CAAC,UAAU,cAAc,OAAO,CAAC,WAAW,GAAG;QACnE,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACjD,CAAC;IACF,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAC7B,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,cAAc,CAAC,CAAC,QAAQ,qBAAqB,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC,QAAQ,KAAK;YAChF,QAAQ,CAAC,CAAC,MAAM,gBAAgB,CAAC,CAAC,SAAS,cAAc,CAAC,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC/F,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,mBAAmB,CAAC,IAAa;IAC/C,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC;gBACH,EAAE,CAAC,KAAK,EAAE,CAAC;YACb,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,OAAO;IACT,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;IACD,OAAO,CAAC,KAAK,EAAE,CAAC;AAClB,CAAC"}

package/dist/telemetry/tool-failure.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Tool-call outcome taxonomy (harness plan area D2, 2026-07-31).
+ *
+ * The Agentic-Harness-Engineering result (arXiv 2604.25850) turns on a single
+ * capability we did not have: every tool call is attributed to a harness
+ * COMPONENT and a stable FAILURE CLASS, so the evolve stage can rank "which part
+ * of the harness is bleeding turns" instead of guessing. Their measured gains
+ * concentrate in tools (+3.3pp), middleware (+2.2pp) and memory (+5.6pp) — and
+ * system-prompt-only edits scored NEGATIVE (-2.3pp), which is exactly the search
+ * direction a system without attribution drifts into.
+ *
+ * The executor's tool results are strings by design (they go straight back to the
+ * model), so classification is textual. That is deliberate: the classifier reads
+ * the same bytes the model reads, so a class can never claim a failure mode the
+ * model was not actually shown.
+ */
+/**
+ * Stable outcome classes. Stable is the point — these strings are GROUP BY keys
+ * in the evidence corpus, so renaming one silently splits its history.
+ */
+export type ToolOutcomeClass = 
+/** Call succeeded with no caveat. */
+'ok'
+/** Succeeded, but only via the whitespace-tolerant edit rung (anchor drift). */
+ | 'ok-tolerant'
+/** `old_string` matched nothing. The dominant edit-tool failure. */
+ | 'edit-miss'
+/** Matched several places, or an out-of-range occurrence selector. */
+ | 'edit-ambiguous'
+/** A line range that does not exist in the file. */
+ | 'range-invalid'
+/** Target file/dir does not exist. */
+ | 'path-not-found'
+/** Path escaped the project root. */
+ | 'path-escape'
+/** Refused: protected test/contract/gate-config/agent-internal path. */
+ | 'protected-path'
+/** Refused by the stub-substance guard (would hollow the file out). */
+ | 'stub-refusal'
+/** Write rejected as truncated mid-file. */
+ | 'truncated-write'
+/** `run_bash` returned a non-zero exit code. */
+ | 'command-failed'
+/** Call exceeded its time budget. */
+ | 'timeout'
+/** Model asked for a tool that does not exist. */
+ | 'unknown-tool'
+/** The tool refused to run at all (e.g. run_bash disabled outside a sandbox). */
+ | 'refused'
+/** A command mutated protected files and the harness restored them. */
+ | 'tamper-restored'
+/** An error we have not given a name to yet. */
+ | 'other-error';
+/** The harness component a tool call exercises. Mirrors the AHE ablation axes. */
+export type HarnessComponent = 'tools' | 'middleware' | 'memory' | 'prompt' | 'execution';
+export declare function isFailureClass(c: ToolOutcomeClass): boolean;
+/**
+ * Classify one tool result string.
+ *
+ * Ordering matters: the specific edit classes are tested before the generic
+ * "not found", because an `old_string not found` message also contains the word
+ * "not found" and would otherwise be miscounted as a missing FILE — which would
+ * point the evolve stage at path handling instead of at the edit tool.
+ */
+export declare function classifyToolResult(tool: string, result: string): ToolOutcomeClass;
+/**
+ * Map a tool name to the harness component it exercises, so failures roll up to
+ * the same axes the AHE ablation measured.
+ */
+export declare function componentForTool(tool: string): HarnessComponent;
+//# sourceMappingURL=tool-failure.d.ts.map

package/dist/telemetry/tool-failure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-failure.d.ts","sourceRoot":"","sources":["../../src/telemetry/tool-failure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;;;GAGG;AACH,MAAM,MAAM,gBAAgB;AAC1B,qCAAqC;AACnC,IAAI;AACN,gFAAgF;GAC9E,aAAa;AACf,oEAAoE;GAClE,WAAW;AACb,sEAAsE;GACpE,gBAAgB;AAClB,oDAAoD;GAClD,eAAe;AACjB,sCAAsC;GACpC,gBAAgB;AAClB,qCAAqC;GACnC,aAAa;AACf,wEAAwE;GACtE,gBAAgB;AAClB,uEAAuE;GACrE,cAAc;AAChB,4CAA4C;GAC1C,iBAAiB;AACnB,gDAAgD;GAC9C,gBAAgB;AAClB,qCAAqC;GACnC,SAAS;AACX,kDAAkD;GAChD,cAAc;AAChB,iFAAiF;GAC/E,SAAS;AACX,uEAAuE;GACrE,iBAAiB;AACnB,gDAAgD;GAC9C,aAAa,CAAC;AAElB,kFAAkF;AAClF,MAAM,MAAM,gBAAgB,GAAG,OAAO,GAAG,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAoB1F,wBAAgB,cAAc,CAAC,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAE3D;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAiDjF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CAc/D"}

package/dist/telemetry/tool-failure.js

@@ -0,0 +1,123 @@
+/**
+ * Tool-call outcome taxonomy (harness plan area D2, 2026-07-31).
+ *
+ * The Agentic-Harness-Engineering result (arXiv 2604.25850) turns on a single
+ * capability we did not have: every tool call is attributed to a harness
+ * COMPONENT and a stable FAILURE CLASS, so the evolve stage can rank "which part
+ * of the harness is bleeding turns" instead of guessing. Their measured gains
+ * concentrate in tools (+3.3pp), middleware (+2.2pp) and memory (+5.6pp) — and
+ * system-prompt-only edits scored NEGATIVE (-2.3pp), which is exactly the search
+ * direction a system without attribution drifts into.
+ *
+ * The executor's tool results are strings by design (they go straight back to the
+ * model), so classification is textual. That is deliberate: the classifier reads
+ * the same bytes the model reads, so a class can never claim a failure mode the
+ * model was not actually shown.
+ */
+/** Outcome classes that represent a wasted or degraded turn. */
+const FAILURE_CLASSES = new Set([
+    'refused',
+    'tamper-restored',
+    'edit-miss',
+    'edit-ambiguous',
+    'range-invalid',
+    'path-not-found',
+    'path-escape',
+    'protected-path',
+    'stub-refusal',
+    'truncated-write',
+    'command-failed',
+    'timeout',
+    'unknown-tool',
+    'other-error',
+]);
+export function isFailureClass(c) {
+    return FAILURE_CLASSES.has(c);
+}
+/**
+ * Classify one tool result string.
+ *
+ * Ordering matters: the specific edit classes are tested before the generic
+ * "not found", because an `old_string not found` message also contains the word
+ * "not found" and would otherwise be miscounted as a missing FILE — which would
+ * point the evolve stage at path handling instead of at the edit tool.
+ */
+export function classifyToolResult(tool, result) {
+    const text = String(result ?? '');
+    const lower = text.toLowerCase();
+    if (lower.includes('unknown tool'))
+        return 'unknown-tool';
+    // run_bash NEVER starts with 'error:' — it returns `exit=<code> ...`, or a
+    // refusal when the shell is disabled. Classifying it by prefix filed every
+    // failed command as a success, left 'command-failed' unreachable, and made the
+    // execution component look permanently healthy to the evolve stage.
+    if (tool === 'run_bash') {
+        if (lower.startsWith('run_bash is disabled') || lower.includes('is disabled'))
+            return 'refused';
+        // A command that mutated protected files and was rolled back is a tamper
+        // event, and matters more than its exit code.
+        if (lower.includes('restored') && lower.includes('protected'))
+            return 'tamper-restored';
+        if (lower.includes('timed out'))
+            return 'timeout';
+        const exit = /(?:^|\s)exit=(-?\d+)/.exec(text);
+        if (exit)
+            return exit[1] === '0' ? 'ok' : 'command-failed';
+        return lower.startsWith('error:') ? 'other-error' : 'ok';
+    }
+    if (!lower.startsWith('error:')) {
+        // Success paths. The tolerant-match note is a success WITH a signal: the
+        // model's anchors are drifting, which predicts future misses.
+        if (lower.includes('did not match byte-for-byte'))
+            return 'ok-tolerant';
+        return 'ok';
+    }
+    // The agent-internal refusal is a protection, not a generic error.
+    if (lower.includes('agent') && lower.includes('internal'))
+        return 'protected-path';
+    if (lower.includes('deliverable and tells you nothing'))
+        return 'protected-path';
+    if (lower.includes('old_string not found'))
+        return 'edit-miss';
+    if (lower.includes('old_string matches') || lower.includes('occurrence'))
+        return 'edit-ambiguous';
+    if (lower.includes('start_line') || lower.includes('end_line'))
+        return 'range-invalid';
+    if (lower.includes('escapes project root'))
+        return 'path-escape';
+    if (lower.includes('protected test') ||
+        lower.includes('locked contract') ||
+        lower.includes('refusing to modify') ||
+        lower.includes('change the implementation, not the gate')) {
+        return 'protected-path';
+    }
+    if (lower.includes('stub') || lower.includes('skeleton'))
+        return 'stub-refusal';
+    if (lower.includes('truncated'))
+        return 'truncated-write';
+    if (lower.includes('timed out') || lower.includes('timeout'))
+        return 'timeout';
+    if (lower.includes('not found') || lower.includes('does not exist'))
+        return 'path-not-found';
+    return 'other-error';
+}
+/**
+ * Map a tool name to the harness component it exercises, so failures roll up to
+ * the same axes the AHE ablation measured.
+ */
+export function componentForTool(tool) {
+    switch (tool) {
+        case 'edit_file':
+        case 'edit_range':
+        case 'write_file':
+        case 'read_file':
+        case 'list_dir':
+        case 'finish':
+            return 'tools';
+        case 'run_bash':
+            return 'execution';
+        default:
+            return tool.startsWith('memory') ? 'memory' : 'tools';
+    }
+}
+//# sourceMappingURL=tool-failure.js.map

package/dist/telemetry/tool-failure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-failure.js","sourceRoot":"","sources":["../../src/telemetry/tool-failure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AA2CH,gEAAgE;AAChE,MAAM,eAAe,GAAkC,IAAI,GAAG,CAAmB;IAC/E,SAAS;IACT,iBAAiB;IACjB,WAAW;IACX,gBAAgB;IAChB,eAAe;IACf,gBAAgB;IAChB,aAAa;IACb,gBAAgB;IAChB,cAAc;IACd,iBAAiB;IACjB,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,aAAa;CACd,CAAC,CAAC;AAEH,MAAM,UAAU,cAAc,CAAC,CAAmB;IAChD,OAAO,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,MAAc;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAEjC,IAAI,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,OAAO,cAAc,CAAC;IAE1D,2EAA2E;IAC3E,2EAA2E;IAC3E,+EAA+E;IAC/E,oEAAoE;IACpE,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,IAAI,KAAK,CAAC,UAAU,CAAC,sBAAsB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;YAAE,OAAO,SAAS,CAAC;QAChG,yEAAyE;QACzE,8CAA8C;QAC9C,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,iBAAiB,CAAC;QACxF,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,SAAS,CAAC;QAClD,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAC3D,OAAO,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3D,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,yEAAyE;QACzE,8DAA8D;QAC9D,IAAI,KAAK,CAAC,QAAQ,CAAC,6BAA6B,CAAC;YAAE,OAAO,aAAa,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,mEAAmE;IACnE,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,gBAAgB,CAAC;IACnF,IAAI,KAAK,CAAC,QAAQ,CAAC,mCAAmC,CAAC;QAAE,OAAO,gBAAgB,CAAC;IAEjF,IAAI,KAAK,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAAE,OAAO,WAAW,CAAC;IAC/D,IAAI,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,gBAAgB,CAAC;IAClG,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,eAAe,CAAC;IACvF,IAAI,KAAK,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAAE,OAAO,aAAa,CAAC;IACjE,IACE,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAChC,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACjC,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QACpC,KAAK,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EACzD,CAAC;QACD,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,cAAc,CAAC;IAChF,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,iBAAiB,CAAC;IAC1D,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/E,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAAE,OAAO,gBAAgB,CAAC;IAC7F,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,WAAW,CAAC;QACjB,KAAK,YAAY,CAAC;QAClB,KAAK,YAAY,CAAC;QAClB,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU,CAAC;QAChB,KAAK,QAAQ;YACX,OAAO,OAAO,CAAC;QACjB,KAAK,UAAU;YACb,OAAO,WAAW,CAAC;QACrB;YACE,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;IAC1D,CAAC;AACH,CAAC"}