@tea-agent/loop-agent 0.1.0-fe-test.0

package/dist/application/evaluation/replay.js

@@ -0,0 +1,289 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { reportDagUseCase } from "../dag/report-dag.js";
+import { readReplaySpec, writeReplayArtifacts, } from "../../infrastructure/evaluation/store.js";
+function sha256(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+function pairKey(input) {
+    return `${input.split}\u0000${input.taskRef}\u0000${input.seed}`;
+}
+async function verifyEvidenceHash(input) {
+    const content = await readFile(input.filePath);
+    const actual = sha256(content);
+    if (actual !== input.expected) {
+        throw new Error(`${input.label} hash mismatch: expected ${input.expected}, got ${actual}`);
+    }
+}
+function sumOptional(values) {
+    if (values.some((value) => value === undefined)) {
+        return { value: null, missing: true };
+    }
+    return {
+        value: values.reduce((sum, value) => sum + (value ?? 0), 0),
+        missing: false,
+    };
+}
+function verifyPassed(input) {
+    if (input.status !== "finished")
+        return false;
+    const verificationNodes = input.nodes.filter((node) => node.executor === "shell" ||
+        node.nodeId.includes("verify") ||
+        node.nodeId.includes("gate"));
+    return (verificationNodes.length > 0 &&
+        verificationNodes.every((node) => node.status === "FINISHED" &&
+            (!node.failureCategory || node.failureCategory === "success")));
+}
+function metricsForRun(run) {
+    const tokens = sumOptional(run.nodes.map((node) => node.tokensUsed));
+    const duration = sumOptional(run.nodes.map((node) => node.durationMs));
+    const missingFields = [];
+    if (tokens.missing)
+        missingFields.push("tokens");
+    if (duration.missing)
+        missingFields.push("durationMs");
+    return {
+        metrics: {
+            verifyPassed: verifyPassed(run),
+            tokens: tokens.value,
+            durationMs: duration.value,
+            executorCalls: run.nodes.length,
+            repairPasses: run.convergence?.currentPass ?? 0,
+        },
+        missingFields,
+    };
+}
+function compareRows(incumbent, challenger) {
+    const reasons = [];
+    let verdict;
+    if (incumbent.metrics.verifyPassed !== challenger.metrics.verifyPassed) {
+        verdict = challenger.metrics.verifyPassed
+            ? "challenger_win"
+            : "incumbent_win";
+        reasons.push("verification_outcome");
+    }
+    else if (!incumbent.metrics.verifyPassed) {
+        verdict = "tie";
+        reasons.push("both_failed_verification");
+    }
+    else if (incumbent.metrics.tokens === null ||
+        challenger.metrics.tokens === null ||
+        incumbent.metrics.durationMs === null ||
+        challenger.metrics.durationMs === null) {
+        verdict = "incomparable";
+        reasons.push("missing_cost_metrics");
+    }
+    else {
+        const challengerNoWorse = challenger.metrics.tokens <= incumbent.metrics.tokens &&
+            challenger.metrics.durationMs <= incumbent.metrics.durationMs;
+        const incumbentNoWorse = incumbent.metrics.tokens <= challenger.metrics.tokens &&
+            incumbent.metrics.durationMs <= challenger.metrics.durationMs;
+        if (challengerNoWorse && !incumbentNoWorse) {
+            verdict = "challenger_win";
+            reasons.push("lower_cost");
+        }
+        else if (incumbentNoWorse && !challengerNoWorse) {
+            verdict = "incumbent_win";
+            reasons.push("lower_cost");
+        }
+        else if (challengerNoWorse && incumbentNoWorse) {
+            verdict = "tie";
+            reasons.push("equal_metrics");
+        }
+        else {
+            verdict = "incomparable";
+            reasons.push("cost_tradeoff");
+        }
+    }
+    return {
+        taskRef: incumbent.taskRef,
+        seed: incumbent.seed,
+        split: incumbent.split,
+        incumbentRunId: incumbent.runId,
+        challengerRunId: challenger.runId,
+        verdict,
+        reasons,
+    };
+}
+function buildScorecard(input) {
+    const rows = [...input.rows].sort((left, right) => [
+        left.split,
+        left.taskRef,
+        String(left.seed).padStart(12, "0"),
+        left.candidateId,
+        left.runId,
+    ]
+        .join("\u0000")
+        .localeCompare([
+        right.split,
+        right.taskRef,
+        String(right.seed).padStart(12, "0"),
+        right.candidateId,
+        right.runId,
+    ].join("\u0000")));
+    const incumbentByKey = new Map(rows
+        .filter((row) => row.candidateId === input.incumbentCandidateId)
+        .map((row) => [pairKey(row), row]));
+    const challengerByKey = new Map(rows
+        .filter((row) => row.candidateId === input.challengerCandidateId)
+        .map((row) => [pairKey(row), row]));
+    const keys = [
+        ...new Set([...incumbentByKey.keys(), ...challengerByKey.keys()]),
+    ].sort();
+    const comparisons = [];
+    let unpairedEvidenceCount = 0;
+    for (const key of keys) {
+        const incumbent = incumbentByKey.get(key);
+        const challenger = challengerByKey.get(key);
+        if (!incumbent || !challenger) {
+            unpairedEvidenceCount +=
+                Number(Boolean(incumbent)) + Number(Boolean(challenger));
+            continue;
+        }
+        comparisons.push(compareRows(incumbent, challenger));
+    }
+    const reasons = ["replay_only"];
+    if (comparisons.length === 0)
+        reasons.push("insufficient_samples");
+    if (unpairedEvidenceCount > 0)
+        reasons.push("unpaired_evidence");
+    return {
+        schemaVersion: 1,
+        replayId: input.replayId,
+        incumbentCandidateId: input.incumbentCandidateId,
+        challengerCandidateId: input.challengerCandidateId,
+        rows,
+        comparisons,
+        aggregate: {
+            pairedSampleCount: comparisons.length,
+            incumbentWins: comparisons.filter((item) => item.verdict === "incumbent_win").length,
+            challengerWins: comparisons.filter((item) => item.verdict === "challenger_win").length,
+            ties: comparisons.filter((item) => item.verdict === "tie").length,
+            incomparable: comparisons.filter((item) => item.verdict === "incomparable").length,
+            unpairedEvidenceCount,
+            promotionEligible: false,
+            reasons,
+        },
+    };
+}
+export function formatReplayMarkdown(scorecard) {
+    const lines = [
+        `# Eval Replay Scorecard: ${scorecard.replayId}`,
+        "",
+        "> Replay-only evidence. This report never authorizes promotion or executes Pi/DAG work.",
+        "",
+        `- incumbent: ${scorecard.incumbentCandidateId}`,
+        `- challenger: ${scorecard.challengerCandidateId}`,
+        `- promotionEligible: ${scorecard.aggregate.promotionEligible}`,
+        `- reasons: ${scorecard.aggregate.reasons.join(", ")}`,
+        "",
+        "## Evidence",
+        "",
+        "| split | task | seed | candidate | run | verified | tokens | durationMs | calls | repairPasses | missing |",
+        "|---|---|---:|---|---|---|---:|---:|---:|---:|---|",
+    ];
+    for (const row of scorecard.rows) {
+        lines.push(`| ${row.split} | ${row.taskRef} | ${row.seed} | ${row.candidateId} | ${row.runId} | ${row.metrics.verifyPassed} | ${row.metrics.tokens ?? "n/a"} | ${row.metrics.durationMs ?? "n/a"} | ${row.metrics.executorCalls} | ${row.metrics.repairPasses} | ${row.missingFields.join(", ") || "none"} |`);
+    }
+    lines.push("", "## Paired Comparisons", "", "| split | task | seed | incumbent run | challenger run | verdict | reasons |", "|---|---|---:|---|---|---|---|");
+    for (const comparison of scorecard.comparisons) {
+        lines.push(`| ${comparison.split} | ${comparison.taskRef} | ${comparison.seed} | ${comparison.incumbentRunId} | ${comparison.challengerRunId} | ${comparison.verdict} | ${comparison.reasons.join(", ")} |`);
+    }
+    if (scorecard.comparisons.length === 0) {
+        lines.push("| - | - | - | - | - | - | no paired evidence |");
+    }
+    return `${lines.join("\n")}\n`;
+}
+export async function replayEvaluation(input) {
+    const spec = await readReplaySpec(input.repoRoot, input.specPath);
+    const rows = [];
+    for (const evidence of spec.evidence) {
+        const runDir = path.join(input.repoRoot, ".harness", "dag-runs", "completed", evidence.runId);
+        const statePath = path.join(runDir, "state.json");
+        const runPath = path.join(runDir, "run.json");
+        const stateRefPath = path
+            .relative(input.repoRoot, statePath)
+            .split(path.sep)
+            .join("/");
+        const runRefPath = path
+            .relative(input.repoRoot, runPath)
+            .split(path.sep)
+            .join("/");
+        await verifyEvidenceHash({
+            filePath: statePath,
+            expected: evidence.stateSha256,
+            label: "state.json",
+        });
+        await verifyEvidenceHash({
+            filePath: runPath,
+            expected: evidence.runSha256,
+            label: "run.json",
+        });
+        const report = await reportDagUseCase({
+            repoRoot: input.repoRoot,
+            runId: evidence.runId,
+            lifecycle: "completed",
+            failedOnly: false,
+            latest: false,
+        });
+        const run = report.runs[0];
+        if (!run) {
+            throw new Error(`completed DAG run not found: ${evidence.runId}`);
+        }
+        if (run.evaluationAssociation.status === "present") {
+            if (run.evaluationAssociation.candidateId !== evidence.candidateId) {
+                throw new Error(`replay evidence candidateId conflict for ${evidence.runId}: evidence=${evidence.candidateId}, run=${run.evaluationAssociation.candidateId}`);
+            }
+            if (run.evaluationAssociation.seed !== evidence.seed) {
+                throw new Error(`replay evidence seed conflict for ${evidence.runId}: evidence=${evidence.seed}, run=${run.evaluationAssociation.seed}`);
+            }
+            if (run.evaluationAssociation.split &&
+                run.evaluationAssociation.split !== evidence.split) {
+                throw new Error(`replay evidence split conflict for ${evidence.runId}: evidence=${evidence.split}, run=${run.evaluationAssociation.split}`);
+            }
+            if (run.evaluationAssociation.taskRef &&
+                run.evaluationAssociation.taskRef !== evidence.taskRef) {
+                throw new Error(`replay evidence taskRef conflict for ${evidence.runId}: evidence=${evidence.taskRef}, run=${run.evaluationAssociation.taskRef}`);
+            }
+        }
+        if (![
+            "finished",
+            "failed",
+            "partial_failed",
+            "superseded",
+            "abandoned",
+        ].includes(run.status)) {
+            throw new Error(`completed DAG run ${evidence.runId} is not terminal (status=${run.status})`);
+        }
+        const { metrics, missingFields } = metricsForRun(run);
+        rows.push({
+            ...evidence,
+            lifecycle: "completed",
+            runStatus: run.status,
+            metrics,
+            missingFields,
+            evidenceRefs: [
+                { path: stateRefPath, sha256: evidence.stateSha256 },
+                { path: runRefPath, sha256: evidence.runSha256 },
+            ],
+        });
+    }
+    const scorecard = buildScorecard({
+        replayId: spec.replayId,
+        incumbentCandidateId: spec.incumbentCandidateId,
+        challengerCandidateId: spec.challengerCandidateId,
+        rows,
+    });
+    const markdown = formatReplayMarkdown(scorecard);
+    if (input.writeArtifacts === false) {
+        return { scorecard, markdown };
+    }
+    const paths = await writeReplayArtifacts({
+        repoRoot: input.repoRoot,
+        replayId: spec.replayId,
+        scorecard,
+        markdown,
+    });
+    return { scorecard, markdown, ...paths };
+}