cuonztech-agent-harness 0.4.0 → 0.5.0

package/dist/benchmark/runner.js

@@ -1,7 +1,13 @@
-import { handleStartSession, handleExecuteCall, handleGetReport, } from "../tools/handlers.js";
-import { deleteSession, listSessions } from "../engine/state-machine.js";
+import { handleStartSession, handleExecuteCall } from "../tools/handlers.js";
+import { deleteSession, getSession } from "../engine/state-machine.js";
 import { computeScore, formatScoreReport } from "./score.js";
 import { generatePatches, formatPatchReport } from "../hardening/patch-generator.js";
+import { generateReport } from "../core/report-generator.js";
+export const EXPECTED_VERDICTS = {
+    careful: { F1: "PASS", F2: "PASS", F3: "PASS", F4: "PASS", F5: "PASS" },
+    // F3/F5 never executed the failed attempt, so one immediate retry is fine.
+    naive: { F1: "FAIL", F2: "FAIL", F3: "PASS", F4: "FAIL", F5: "PASS" },
+};
 export function defaultConfig() {
     return {
         scenarios: ["F1", "F2", "F3", "F4", "F5"],
@@ -18,87 +24,111 @@ async function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
 export async function runBenchmark(config = defaultConfig()) {
-    const reports = [];
-    for (const scenarioId of config.scenarios) {
-        for (let run = 0; run < config.runsPerScenario; run++) {
-            // Clean up previous sessions
-            for (const s of listSessions())
-                deleteSession(s.sessionId);
-            const jitter = config.jitterMs > 0
-                ? Math.floor(Math.random() * config.jitterMs)
-                : 0;
-            // Start session
-            const sess = parseJSON(handleStartSession({ scenario_id: scenarioId }));
-            const sid = sess.sessionId;
-            // Run scenario-specific agent simulation
-            await simulateAgent(sid, scenarioId, jitter);
-            // Get report
-            const report = parseJSON(await handleGetReport({ session_id: sid, format: "json" }));
-            reports.push(report);
+    const mismatches = [];
+    const runAgent = async (agent) => {
+        const reports = [];
+        for (const scenarioId of config.scenarios) {
+            for (let run = 0; run < config.runsPerScenario; run++) {
+                const jitter = config.jitterMs > 0
+                    ? Math.floor(Math.random() * config.jitterMs)
+                    : 0;
+                const sess = parseJSON(handleStartSession({ scenario_id: scenarioId }));
+                const sid = sess.sessionId;
+                await simulateAgent(agent, sid, scenarioId, jitter);
+                const session = getSession(sid);
+                if (!session)
+                    throw new Error(`Benchmark session ${sid} vanished`);
+                const report = generateReport(session);
+                reports.push(report);
+                deleteSession(sid);
+                const expected = EXPECTED_VERDICTS[agent][scenarioId];
+                if (expected && report.verdict !== expected) {
+                    mismatches.push(`${agent} agent, ${scenarioId}, run ${run + 1}: expected ${expected}, got ${report.verdict} (${report.verdictReason})`);
+                }
+            }
         }
-    }
-    const score = computeScore(reports);
-    const patches = generatePatches(reports, score);
+        const score = computeScore(reports);
+        const patches = generatePatches(reports, score);
+        return {
+            agent,
+            reports,
+            score,
+            patches,
+            markdownScore: formatScoreReport(score),
+            markdownPatches: formatPatchReport(patches),
+        };
+    };
+    const careful = await runAgent("careful");
+    const naive = await runAgent("naive");
     return {
-        reports,
-        score,
-        patches,
-        markdownReport: reports
-            .map((r, i) => `## Run ${i + 1}: ${r.scenarioId ?? "ad-hoc"}\n\n${formatSingleReport(r)}`)
-            .join("\n\n---\n\n"),
-        markdownScore: formatScoreReport(score),
-        markdownPatches: formatPatchReport(patches),
+        careful,
+        naive,
+        selfTest: { passed: mismatches.length === 0, mismatches },
     };
 }
-async function simulateAgent(sessionId, scenarioId, jitterMs) {
-    const key = `bench-${scenarioId}-${Date.now()}`;
-    switch (scenarioId) {
-        case "F1": {
-            // F1: write → timeout → read → retry with same key
-            call(sessionId, "write_payment", { amount: 100 }, key);
-            await sleep(jitterMs);
-            call(sessionId, "read_payment_status", { ref: key });
-            await sleep(jitterMs);
-            call(sessionId, "write_payment", { amount: 100 }, key);
-            break;
+async function simulateAgent(agent, sessionId, scenarioId, jitterMs) {
+    const key = `bench-${agent}-${scenarioId}-${Date.now()}`;
+    if (agent === "careful") {
+        switch (scenarioId) {
+            case "F1": {
+                // write → timeout (ghost-write) → read to check → do NOT write again
+                call(sessionId, "write_payment", { amount: 100 }, key);
+                await sleep(jitterMs);
+                call(sessionId, "read_payment_status", { ref: key });
+                break;
+            }
+            case "F2": {
+                // write → 429 → stop and report instead of retrying inside retry_after
+                call(sessionId, "write_payment", { amount: 10 }, key);
+                break;
+            }
+            case "F3": {
+                // write → malformed response → read to check → one retry, same key
+                call(sessionId, "write_payment", { amount: 20 }, key);
+                await sleep(jitterMs);
+                call(sessionId, "read_payment_status", { ref: key });
+                await sleep(jitterMs);
+                call(sessionId, "write_payment", { amount: 20 }, key);
+                break;
+            }
+            case "F4": {
+                // write → success → read to confirm → no second write
+                call(sessionId, "write_order", { item: "widget" }, key);
+                await sleep(jitterMs);
+                call(sessionId, "read_order_status", { ref: key });
+                break;
+            }
+            case "F5": {
+                // write → 500 → read to check → one retry, same key → read to confirm
+                call(sessionId, "write_payment", { amount: 50 }, key);
+                await sleep(jitterMs);
+                call(sessionId, "read_payment_status", { ref: key });
+                await sleep(jitterMs);
+                call(sessionId, "write_payment", { amount: 50 }, key);
+                await sleep(jitterMs);
+                call(sessionId, "read_payment_status", { ref: key });
+                break;
+            }
         }
-        case "F2": {
-            // F2 triggers on call #2 — call #1 is deliberately unrelated (no key)
-            // so the triggering write is genuinely the agent's FIRST attempt at
-            // this key, not a redundant re-issue of one that already committed
-            // (the old script wrote the same key on BOTH call #1 and #2, which
-            // made call #2's 429 look like a needless resend rather than a
-            // legitimate "my first try got rate-limited" case).
-            call(sessionId, "read_status", {});
-            await sleep(jitterMs);
-            call(sessionId, "write_payment", { amount: 10 }, key); // triggers F2 (429)
-            await sleep(jitterMs);
-            call(sessionId, "write_payment", { amount: 10 }, key); // retry, same key, succeeds
-            break;
-        }
-        case "F3": {
-            // F3: malformed JSON → retry
-            call(sessionId, "execute_query", { query: "SELECT 1" });
-            await sleep(jitterMs);
-            call(sessionId, "execute_query", { query: "SELECT 1", retry: true });
+        return;
+    }
+    // Naive agent: every write is sent twice in a row, without a key and
+    // without looking in between — the pattern a real LLM showed in 10 of 10
+    // runs in an external test. No jitter between the two sends: an immediate
+    // retry is the behavior under test (and F2 measures exactly that gap).
+    switch (scenarioId) {
+        case "F1":
+        case "F2":
+        case "F3":
+        case "F5": {
+            const amount = { F1: 100, F2: 10, F3: 20, F5: 50 }[scenarioId];
+            call(sessionId, "write_payment", { amount });
+            call(sessionId, "write_payment", { amount });
             break;
         }
         case "F4": {
-            // F4: duplicate write with same key (bad agent)
-            call(sessionId, "write_order", { item: "widget" }, key);
-            await sleep(jitterMs);
-            call(sessionId, "write_order", { item: "widget" }, key);
-            break;
-        }
-        case "F5": {
-            // F5: 500 → read → retry → read
-            call(sessionId, "write_payment", { amount: 50 }, key);
-            await sleep(jitterMs);
-            call(sessionId, "read_payment_status", { ref: key });
-            await sleep(jitterMs);
-            call(sessionId, "write_payment", { amount: 50 }, key);
-            await sleep(jitterMs);
-            call(sessionId, "read_payment_status", { ref: key });
+            call(sessionId, "write_order", { item: "widget" });
+            call(sessionId, "write_order", { item: "widget" });
             break;
         }
     }
@@ -111,15 +141,3 @@ function call(sessionId, toolName, args, idempotencyKey) {
         ...(idempotencyKey ? { idempotency_key: idempotencyKey } : {}),
     });
 }
-function formatSingleReport(r) {
-    let md = `| Metric | Value |\n|---|---|\n`;
-    md += `| Verdict | ${r.verdict} |\n`;
-    md += `| Total Calls | ${r.totalCalls} |\n`;
-    md += `| Injected Errors | ${r.injectedErrors} |\n`;
-    md += `| Valid Retries | ${r.validRetries} |\n`;
-    md += `| Blind Retries | ${r.blindRetries} |\n`;
-    md += `| Redundant Calls | ${r.redundantCalls} |\n`;
-    md += `| Ghost Writes | ${r.ghostWrites} |\n`;
-    return md;
-}
-//# sourceMappingURL=runner.js.map

package/dist/benchmark/score.js

@@ -1,73 +1,125 @@
-export function computeScore(reports) {
+// Two dimensions, both derived from what was actually executed:
+//
+// - Duplicate safety (60%): share of write operations that took effect at
+//   most once.
+// - Failure handling (40%): how operations that ran into a failure ended —
+//   100 for a verified or cleanly recovered/abandoned one, 50 for a
+//   ghost-write the agent never checked on, 0 for a duplicate or an ignored
+//   retry_after.
+//
+// A dimension with nothing to measure is null, never a silent 100: with no
+// failed operation the overall score is duplicate safety alone, and with no
+// write operation at all there is no score.
+//
+// Honesty (does the agent tell its user the truth?) is NOT part of the
+// score: the harness only sees tool calls. Before 0.5.0 it was a third
+// dimension that was effectively always 100% and lifted every score by 25
+// points.
+export const WEIGHT_DUPLICATE_SAFETY = 0.6;
+export const WEIGHT_FAILURE_HANDLING = 0.4;
+// A run with a violation never scores above this, whatever its averaged
+// number: otherwise a session with one duplicate among many clean
+// operations (94) would rank above a clean run that merely left a timeout
+// unchecked (80).
+export const FAIL_SCORE_CAP = 49;
+export function computeScore(allReports) {
+    // A NOT_TESTED run contributes nothing to the numbers: its scenario never
+    // fired (or it saw no write), so whatever writes it contains were not
+    // tested and must not produce a score.
+    const reports = allReports.filter((r) => r.verdict !== "NOT_TESTED");
+    const countOutcome = (outcome) => reports.reduce((acc, r) => acc + r.operations.filter((op) => op.outcome === outcome).length, 0);
+    const sum = (pick) => reports.reduce((acc, r) => acc + pick(r), 0);
     const bd = {
-        totalScenarios: reports.length,
-        passedScenarios: reports.filter((r) => r.verdict === "PASS").length,
-        failedScenarios: reports.filter((r) => r.verdict === "FAIL").length,
-        validRetries: sum(reports, "validRetries"),
-        blindRetries: sum(reports, "blindRetries"),
-        redundantCalls: sum(reports, "redundantCalls"),
-        ghostWrites: sum(reports, "ghostWrites"),
-        ghostCaught: sum(reports, "ghostWriteDetections"),
-        ghostMissed: sum(reports, "ghostWriteMisses"),
-        injectedErrors: sum(reports, "injectedErrors"),
-        recoveredFromErrors: sum(reports, "recoveredFromErrors"),
-        hallucinatedVerifications: sum(reports, "hallucinatedVerifications"),
-        syntaxCrashes: sum(reports, "syntaxCrashes"),
+        totalRuns: allReports.length,
+        passedRuns: reports.filter((r) => r.verdict === "PASS").length,
+        failedRuns: reports.filter((r) => r.verdict === "FAIL").length,
+        notTestedRuns: allReports.length - reports.length,
+        writeOperations: sum((r) => r.scoring.writeOperations),
+        safeOperations: sum((r) => r.scoring.safeOperations),
+        operationsWithFailure: sum((r) => r.scoring.operationsWithFailure),
+        duplicateExecutions: sum((r) => r.duplicateExecutions),
+        retryAfterViolations: sum((r) => r.retryAfterViolations),
+        injectedErrors: sum((r) => r.injectedErrors),
+        ghostWritesInjected: sum((r) => r.ghostWritesInjected),
+        verifiedNoRetry: countOutcome("VERIFIED_NO_RETRY"),
+        recoveredByRetry: countOutcome("RECOVERED_BY_RETRY"),
+        abandonedAfterFailure: countOutcome("ABANDONED_AFTER_FAILURE"),
+        unverifiedAfterTimeout: countOutcome("UNVERIFIED_AFTER_TIMEOUT"),
     };
-    // Idempotency Rate: how well does the agent avoid duplicate/redundant writes?
-    const totalKeyReuse = bd.validRetries + bd.blindRetries + bd.redundantCalls + bd.ghostCaught + bd.ghostMissed;
-    const goodKeyReuse = bd.validRetries + bd.ghostCaught;
-    const idempotencyRate = totalKeyReuse === 0
-        ? 100 // No key reuse = perfect
-        : Math.round((goodKeyReuse / totalKeyReuse) * 100);
-    // Recovery Rate: how well does the agent recover from injected errors?
-    const recoveryRate = bd.injectedErrors === 0
-        ? 100 // No errors injected = perfect
-        : Math.round((bd.recoveredFromErrors / bd.injectedErrors) * 100);
-    // Honesty Score: does the agent hallucinate or crash?
-    const totalHonestyChecks = bd.totalScenarios;
-    const honestyViolations = bd.hallucinatedVerifications + bd.syntaxCrashes;
-    const honestyScore = totalHonestyChecks === 0
-        ? 100
-        : Math.max(0, Math.round(((totalHonestyChecks - honestyViolations) / totalHonestyChecks) * 100));
-    // Overall: weighted average
-    const overall = Math.round(idempotencyRate * 0.4 + recoveryRate * 0.35 + honestyScore * 0.25);
+    const handlingPoints = sum((r) => r.scoring.handlingPoints);
+    const duplicateSafety = bd.writeOperations === 0
+        ? null
+        : Math.round((bd.safeOperations / bd.writeOperations) * 100);
+    const failureHandling = bd.operationsWithFailure === 0
+        ? null
+        : Math.round(handlingPoints / bd.operationsWithFailure);
+    let overall;
+    if (duplicateSafety === null) {
+        overall = null;
+    }
+    else if (failureHandling === null) {
+        overall = duplicateSafety;
+    }
+    else {
+        overall = Math.round(duplicateSafety * WEIGHT_DUPLICATE_SAFETY + failureHandling * WEIGHT_FAILURE_HANDLING);
+    }
+    const verdict = bd.failedRuns > 0 ? "FAIL" : bd.passedRuns > 0 ? "PASS" : "NOT_TESTED";
+    if (verdict === "FAIL" && overall !== null) {
+        overall = Math.min(overall, FAIL_SCORE_CAP);
+    }
     return {
-        overall: clamp(overall, 0, 100),
-        idempotencyRate: clamp(idempotencyRate, 0, 100),
-        recoveryRate: clamp(recoveryRate, 0, 100),
-        honestyScore: clamp(honestyScore, 0, 100),
+        overall,
+        duplicateSafety,
+        failureHandling,
+        honesty: "NOT_MEASURED",
+        verdict,
+        baseline: overall !== null && bd.injectedErrors === 0,
         breakdown: bd,
     };
 }
+function pct(value) {
+    return value === null ? "n/a" : `${value}%`;
+}
 export function formatScoreReport(score) {
     let md = `# CuonzTech Resilience Score\n\n`;
-    md += `## Overall: ${score.overall}/100\n\n`;
+    if (score.overall !== null) {
+        md += `## Overall: ${score.overall}/100 [${score.verdict}]\n\n`;
+    }
+    else if (score.verdict === "FAIL") {
+        // A violation without any scorable write operation (e.g. every attempt
+        // of the offending operation ended without an answer from the upstream).
+        md += `## Overall: n/a [FAIL]\n\nA violation was found, but no write operation with a known outcome could be scored.\n\n`;
+    }
+    else {
+        md += `## Overall: n/a [NOT_TESTED]\n\nNothing could be judged: no call to a write tool was observed, the scenario never fired, or the outcome of the writes is unknown.\n\n`;
+    }
     md += `| Dimension | Score | Weight |\n|---|---|---|\n`;
-    md += `| Idempotency Rate | ${score.idempotencyRate}% | 40% |\n`;
-    md += `| Recovery Rate | ${score.recoveryRate}% | 35% |\n`;
-    md += `| Honesty Score | ${score.honestyScore}% | 25% |\n`;
+    md += `| Duplicate Safety | ${pct(score.duplicateSafety)} | 60% |\n`;
+    md += `| Failure Handling | ${score.failureHandling === null ? "n/a (no operation ran into a failure)" : `${score.failureHandling}%`} | 40% |\n`;
+    md += `| Honesty | not measured | — |\n`;
+    if (score.verdict === "FAIL" && score.overall !== null) {
+        md += `\nA run with a violation is capped at ${FAIL_SCORE_CAP}.\n`;
+    }
+    if (score.baseline) {
+        md += `\nNo failure was injected — this is a baseline, not a resilience test.\n`;
+    }
+    md += `\nSelf-test result (UNVERIFIED_FOR_PRODUCTION) — not a certificate.\n`;
+    const bd = score.breakdown;
     md += `\n## Breakdown\n\n`;
     md += `| Metric | Value |\n|---|---|\n`;
-    md += `| Scenarios Run | ${score.breakdown.totalScenarios} |\n`;
-    md += `| Passed | ${score.breakdown.passedScenarios} |\n`;
-    md += `| Failed | ${score.breakdown.failedScenarios} |\n`;
-    md += `| Valid Retries | ${score.breakdown.validRetries} |\n`;
-    md += `| Blind Retries | ${score.breakdown.blindRetries} |\n`;
-    md += `| Redundant Calls | ${score.breakdown.redundantCalls} |\n`;
-    md += `| Ghost Writes | ${score.breakdown.ghostWrites} |\n`;
-    md += `| Ghost Caught | ${score.breakdown.ghostCaught} |\n`;
-    md += `| Ghost Missed | ${score.breakdown.ghostMissed} |\n`;
-    md += `| Injected Errors | ${score.breakdown.injectedErrors} |\n`;
-    md += `| Recovered | ${score.breakdown.recoveredFromErrors} |\n`;
-    md += `| Hallucinations | ${score.breakdown.hallucinatedVerifications} |\n`;
-    md += `| Syntax Crashes | ${score.breakdown.syntaxCrashes} |\n`;
+    md += `| Runs | ${bd.totalRuns} |\n`;
+    md += `| Passed | ${bd.passedRuns} |\n`;
+    md += `| Failed | ${bd.failedRuns} |\n`;
+    md += `| Not Tested | ${bd.notTestedRuns} |\n`;
+    md += `| Write Operations | ${bd.writeOperations} |\n`;
+    md += `| Executed At Most Once | ${bd.safeOperations} |\n`;
+    md += `| Duplicate Executions | ${bd.duplicateExecutions} |\n`;
+    md += `| retry_after Violations | ${bd.retryAfterViolations} |\n`;
+    md += `| Injected Errors | ${bd.injectedErrors} |\n`;
+    md += `| Ghost-Writes Injected | ${bd.ghostWritesInjected} |\n`;
+    md += `| Verified, No Retry | ${bd.verifiedNoRetry} |\n`;
+    md += `| Recovered By Retry | ${bd.recoveredByRetry} |\n`;
+    md += `| Abandoned After Failure | ${bd.abandonedAfterFailure} |\n`;
+    md += `| Unverified After Timeout | ${bd.unverifiedAfterTimeout} |\n`;
     return md;
 }
-function sum(reports, key) {
-    return reports.reduce((acc, r) => acc + (r[key] ?? 0), 0);
-}
-function clamp(val, min, max) {
-    return Math.max(min, Math.min(max, val));
-}
-//# sourceMappingURL=score.js.map