@tea-agent/loop-agent 0.13.0-alpha.0 → 0.13.0

package/dist/application/evaluation/replay.js

@@ -0,0 +1,289 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { reportDagUseCase } from "../dag/report-dag.js";
+import { readReplaySpec, writeReplayArtifacts, } from "../../infrastructure/evaluation/store.js";
+function sha256(content) {
+    return createHash("sha256").update(content).digest("hex");
+}
+function pairKey(input) {
+    return `${input.split}\u0000${input.taskRef}\u0000${input.seed}`;
+}
+async function verifyEvidenceHash(input) {
+    const content = await readFile(input.filePath);
+    const actual = sha256(content);
+    if (actual !== input.expected) {
+        throw new Error(`${input.label} hash mismatch: expected ${input.expected}, got ${actual}`);
+    }
+}
+function sumOptional(values) {
+    if (values.some((value) => value === undefined)) {
+        return { value: null, missing: true };
+    }
+    return {
+        value: values.reduce((sum, value) => sum + (value ?? 0), 0),
+        missing: false,
+    };
+}
+function verifyPassed(input) {
+    if (input.status !== "finished")
+        return false;
+    const verificationNodes = input.nodes.filter((node) => node.executor === "shell" ||
+        node.nodeId.includes("verify") ||
+        node.nodeId.includes("gate"));
+    return (verificationNodes.length > 0 &&
+        verificationNodes.every((node) => node.status === "FINISHED" &&
+            (!node.failureCategory || node.failureCategory === "success")));
+}
+function metricsForRun(run) {
+    const tokens = sumOptional(run.nodes.map((node) => node.tokensUsed));
+    const duration = sumOptional(run.nodes.map((node) => node.durationMs));
+    const missingFields = [];
+    if (tokens.missing)
+        missingFields.push("tokens");
+    if (duration.missing)
+        missingFields.push("durationMs");
+    return {
+        metrics: {
+            verifyPassed: verifyPassed(run),
+            tokens: tokens.value,
+            durationMs: duration.value,
+            executorCalls: run.nodes.length,
+            repairPasses: run.convergence?.currentPass ?? 0,
+        },
+        missingFields,
+    };
+}
+function compareRows(incumbent, challenger) {
+    const reasons = [];
+    let verdict;
+    if (incumbent.metrics.verifyPassed !== challenger.metrics.verifyPassed) {
+        verdict = challenger.metrics.verifyPassed
+            ? "challenger_win"
+            : "incumbent_win";
+        reasons.push("verification_outcome");
+    }
+    else if (!incumbent.metrics.verifyPassed) {
+        verdict = "tie";
+        reasons.push("both_failed_verification");
+    }
+    else if (incumbent.metrics.tokens === null ||
+        challenger.metrics.tokens === null ||
+        incumbent.metrics.durationMs === null ||
+        challenger.metrics.durationMs === null) {
+        verdict = "incomparable";
+        reasons.push("missing_cost_metrics");
+    }
+    else {
+        const challengerNoWorse = challenger.metrics.tokens <= incumbent.metrics.tokens &&
+            challenger.metrics.durationMs <= incumbent.metrics.durationMs;
+        const incumbentNoWorse = incumbent.metrics.tokens <= challenger.metrics.tokens &&
+            incumbent.metrics.durationMs <= challenger.metrics.durationMs;
+        if (challengerNoWorse && !incumbentNoWorse) {
+            verdict = "challenger_win";
+            reasons.push("lower_cost");
+        }
+        else if (incumbentNoWorse && !challengerNoWorse) {
+            verdict = "incumbent_win";
+            reasons.push("lower_cost");
+        }
+        else if (challengerNoWorse && incumbentNoWorse) {
+            verdict = "tie";
+            reasons.push("equal_metrics");
+        }
+        else {
+            verdict = "incomparable";
+            reasons.push("cost_tradeoff");
+        }
+    }
+    return {
+        taskRef: incumbent.taskRef,
+        seed: incumbent.seed,
+        split: incumbent.split,
+        incumbentRunId: incumbent.runId,
+        challengerRunId: challenger.runId,
+        verdict,
+        reasons,
+    };
+}
+function buildScorecard(input) {
+    const rows = [...input.rows].sort((left, right) => [
+        left.split,
+        left.taskRef,
+        String(left.seed).padStart(12, "0"),
+        left.candidateId,
+        left.runId,
+    ]
+        .join("\u0000")
+        .localeCompare([
+        right.split,
+        right.taskRef,
+        String(right.seed).padStart(12, "0"),
+        right.candidateId,
+        right.runId,
+    ].join("\u0000")));
+    const incumbentByKey = new Map(rows
+        .filter((row) => row.candidateId === input.incumbentCandidateId)
+        .map((row) => [pairKey(row), row]));
+    const challengerByKey = new Map(rows
+        .filter((row) => row.candidateId === input.challengerCandidateId)
+        .map((row) => [pairKey(row), row]));
+    const keys = [
+        ...new Set([...incumbentByKey.keys(), ...challengerByKey.keys()]),
+    ].sort();
+    const comparisons = [];
+    let unpairedEvidenceCount = 0;
+    for (const key of keys) {
+        const incumbent = incumbentByKey.get(key);
+        const challenger = challengerByKey.get(key);
+        if (!incumbent || !challenger) {
+            unpairedEvidenceCount +=
+                Number(Boolean(incumbent)) + Number(Boolean(challenger));
+            continue;
+        }
+        comparisons.push(compareRows(incumbent, challenger));
+    }
+    const reasons = ["replay_only"];
+    if (comparisons.length === 0)
+        reasons.push("insufficient_samples");
+    if (unpairedEvidenceCount > 0)
+        reasons.push("unpaired_evidence");
+    return {
+        schemaVersion: 1,
+        replayId: input.replayId,
+        incumbentCandidateId: input.incumbentCandidateId,
+        challengerCandidateId: input.challengerCandidateId,
+        rows,
+        comparisons,
+        aggregate: {
+            pairedSampleCount: comparisons.length,
+            incumbentWins: comparisons.filter((item) => item.verdict === "incumbent_win").length,
+            challengerWins: comparisons.filter((item) => item.verdict === "challenger_win").length,
+            ties: comparisons.filter((item) => item.verdict === "tie").length,
+            incomparable: comparisons.filter((item) => item.verdict === "incomparable").length,
+            unpairedEvidenceCount,
+            promotionEligible: false,
+            reasons,
+        },
+    };
+}
+export function formatReplayMarkdown(scorecard) {
+    const lines = [
+        `# Eval Replay Scorecard: ${scorecard.replayId}`,
+        "",
+        "> Replay-only evidence. This report never authorizes promotion or executes Pi/DAG work.",
+        "",
+        `- incumbent: ${scorecard.incumbentCandidateId}`,
+        `- challenger: ${scorecard.challengerCandidateId}`,
+        `- promotionEligible: ${scorecard.aggregate.promotionEligible}`,
+        `- reasons: ${scorecard.aggregate.reasons.join(", ")}`,
+        "",
+        "## Evidence",
+        "",
+        "| split | task | seed | candidate | run | verified | tokens | durationMs | calls | repairPasses | missing |",
+        "|---|---|---:|---|---|---|---:|---:|---:|---:|---|",
+    ];
+    for (const row of scorecard.rows) {
+        lines.push(`| ${row.split} | ${row.taskRef} | ${row.seed} | ${row.candidateId} | ${row.runId} | ${row.metrics.verifyPassed} | ${row.metrics.tokens ?? "n/a"} | ${row.metrics.durationMs ?? "n/a"} | ${row.metrics.executorCalls} | ${row.metrics.repairPasses} | ${row.missingFields.join(", ") || "none"} |`);
+    }
+    lines.push("", "## Paired Comparisons", "", "| split | task | seed | incumbent run | challenger run | verdict | reasons |", "|---|---|---:|---|---|---|---|");
+    for (const comparison of scorecard.comparisons) {
+        lines.push(`| ${comparison.split} | ${comparison.taskRef} | ${comparison.seed} | ${comparison.incumbentRunId} | ${comparison.challengerRunId} | ${comparison.verdict} | ${comparison.reasons.join(", ")} |`);
+    }
+    if (scorecard.comparisons.length === 0) {
+        lines.push("| - | - | - | - | - | - | no paired evidence |");
+    }
+    return `${lines.join("\n")}\n`;
+}
+export async function replayEvaluation(input) {
+    const spec = await readReplaySpec(input.repoRoot, input.specPath);
+    const rows = [];
+    for (const evidence of spec.evidence) {
+        const runDir = path.join(input.repoRoot, ".harness", "dag-runs", "completed", evidence.runId);
+        const statePath = path.join(runDir, "state.json");
+        const runPath = path.join(runDir, "run.json");
+        const stateRefPath = path
+            .relative(input.repoRoot, statePath)
+            .split(path.sep)
+            .join("/");
+        const runRefPath = path
+            .relative(input.repoRoot, runPath)
+            .split(path.sep)
+            .join("/");
+        await verifyEvidenceHash({
+            filePath: statePath,
+            expected: evidence.stateSha256,
+            label: "state.json",
+        });
+        await verifyEvidenceHash({
+            filePath: runPath,
+            expected: evidence.runSha256,
+            label: "run.json",
+        });
+        const report = await reportDagUseCase({
+            repoRoot: input.repoRoot,
+            runId: evidence.runId,
+            lifecycle: "completed",
+            failedOnly: false,
+            latest: false,
+        });
+        const run = report.runs[0];
+        if (!run) {
+            throw new Error(`completed DAG run not found: ${evidence.runId}`);
+        }
+        if (run.evaluationAssociation.status === "present") {
+            if (run.evaluationAssociation.candidateId !== evidence.candidateId) {
+                throw new Error(`replay evidence candidateId conflict for ${evidence.runId}: evidence=${evidence.candidateId}, run=${run.evaluationAssociation.candidateId}`);
+            }
+            if (run.evaluationAssociation.seed !== evidence.seed) {
+                throw new Error(`replay evidence seed conflict for ${evidence.runId}: evidence=${evidence.seed}, run=${run.evaluationAssociation.seed}`);
+            }
+            if (run.evaluationAssociation.split &&
+                run.evaluationAssociation.split !== evidence.split) {
+                throw new Error(`replay evidence split conflict for ${evidence.runId}: evidence=${evidence.split}, run=${run.evaluationAssociation.split}`);
+            }
+            if (run.evaluationAssociation.taskRef &&
+                run.evaluationAssociation.taskRef !== evidence.taskRef) {
+                throw new Error(`replay evidence taskRef conflict for ${evidence.runId}: evidence=${evidence.taskRef}, run=${run.evaluationAssociation.taskRef}`);
+            }
+        }
+        if (![
+            "finished",
+            "failed",
+            "partial_failed",
+            "superseded",
+            "abandoned",
+        ].includes(run.status)) {
+            throw new Error(`completed DAG run ${evidence.runId} is not terminal (status=${run.status})`);
+        }
+        const { metrics, missingFields } = metricsForRun(run);
+        rows.push({
+            ...evidence,
+            lifecycle: "completed",
+            runStatus: run.status,
+            metrics,
+            missingFields,
+            evidenceRefs: [
+                { path: stateRefPath, sha256: evidence.stateSha256 },
+                { path: runRefPath, sha256: evidence.runSha256 },
+            ],
+        });
+    }
+    const scorecard = buildScorecard({
+        replayId: spec.replayId,
+        incumbentCandidateId: spec.incumbentCandidateId,
+        challengerCandidateId: spec.challengerCandidateId,
+        rows,
+    });
+    const markdown = formatReplayMarkdown(scorecard);
+    if (input.writeArtifacts === false) {
+        return { scorecard, markdown };
+    }
+    const paths = await writeReplayArtifacts({
+        repoRoot: input.repoRoot,
+        replayId: spec.replayId,
+        scorecard,
+        markdown,
+    });
+    return { scorecard, markdown, ...paths };
+}

package/dist/application/evaluation/types.js

@@ -0,0 +1,130 @@
+import { z } from "zod";
+export const evalSplitSchema = z.enum(["public", "private", "held_out"]);
+export const replayEvidenceRefSchema = z
+    .object({
+    candidateId: z.string().min(1),
+    taskRef: z.string().min(1),
+    seed: z.number().int().nonnegative(),
+    split: evalSplitSchema,
+    runId: z
+        .string()
+        .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, "runId must be one safe path segment"),
+    stateSha256: z.string().regex(/^[a-f0-9]{64}$/),
+    runSha256: z.string().regex(/^[a-f0-9]{64}$/),
+})
+    .strict();
+export const replaySpecSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    replayId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/),
+    incumbentCandidateId: z.string().min(1),
+    challengerCandidateId: z.string().min(1),
+    evidence: z.array(replayEvidenceRefSchema).min(1),
+})
+    .strict()
+    .superRefine((spec, ctx) => {
+    if (spec.incumbentCandidateId === spec.challengerCandidateId) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "incumbentCandidateId and challengerCandidateId must differ",
+            path: ["challengerCandidateId"],
+        });
+    }
+    const allowed = new Set([
+        spec.incumbentCandidateId,
+        spec.challengerCandidateId,
+    ]);
+    const pairKeys = new Set();
+    for (let i = 0; i < spec.evidence.length; i += 1) {
+        const evidence = spec.evidence[i];
+        if (!allowed.has(evidence.candidateId)) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: "evidence candidateId must match incumbent or challenger",
+                path: ["evidence", i, "candidateId"],
+            });
+        }
+        const key = `${evidence.candidateId}\u0000${evidence.split}\u0000${evidence.taskRef}\u0000${evidence.seed}`;
+        if (pairKeys.has(key)) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: "duplicate evidence for candidateId + split + taskRef + seed",
+                path: ["evidence", i],
+            });
+        }
+        pairKeys.add(key);
+    }
+});
+// --- Candidate Registry (M2 W2.1–W2.2) ---
+export const candidateIdSchema = z
+    .string()
+    .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, "candidateId must be one safe path segment");
+export const candidateKindSchema = z.enum([
+    "prompt",
+    "skill",
+    "context_policy",
+    "model_routing",
+    "profile",
+    "composite",
+]);
+export const candidateContentRefSchema = z
+    .object({
+    path: z
+        .string()
+        .min(1)
+        .refine((value) => !pathIsAbsoluteLike(value), "content ref path must be repo-relative"),
+    sha256: z
+        .string()
+        .regex(/^(sha256:)?[a-f0-9]{64}$/i, "sha256 must be 64 hex digits"),
+})
+    .strict();
+function pathIsAbsoluteLike(value) {
+    if (value.startsWith("/") || value.startsWith("\\"))
+        return true;
+    if (/^[A-Za-z]:[\\/]/.test(value))
+        return true;
+    return false;
+}
+export const candidateManifestInputSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    candidateId: candidateIdSchema,
+    parentCandidateId: candidateIdSchema.nullable().optional(),
+    candidateKind: candidateKindSchema,
+    createdAt: z.string().min(1),
+    description: z.string().optional(),
+    contentRefs: z.array(candidateContentRefSchema).min(1),
+    // optional on input; always computed/verified on register/read
+    bundleHash: z
+        .string()
+        .regex(/^(sha256:)?[a-f0-9]{64}$/i)
+        .optional(),
+})
+    .strict();
+export const candidateManifestSchema = candidateManifestInputSchema
+    .extend({
+    parentCandidateId: candidateIdSchema.nullable(),
+    bundleHash: z.string().regex(/^sha256:[a-f0-9]{64}$/),
+})
+    .strict();
+export const lifecycleStateSchema = z.enum([
+    "proposed",
+    "eligible",
+    "experimenting",
+    "accepted",
+    "rejected",
+    "invalid",
+    "retired",
+]);
+export const lifecycleEventSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    seq: z.number().int().positive(),
+    from: lifecycleStateSchema.nullable(),
+    to: lifecycleStateSchema,
+    reason: z.string().min(1),
+    at: z.string().min(1),
+    previousEventHash: z.string().regex(/^[a-f0-9]{64}$/),
+    eventHash: z.string().regex(/^[a-f0-9]{64}$/),
+})
+    .strict();

package/dist/cli/command-definitions.js

@@ -1,6 +1,7 @@
 import { runDoctor } from "../commands/doctor.js";
 import { runDocsArchive } from "../commands/docs-archive.js";
 import { runDocsAudit } from "../commands/docs-audit.js";
+import { runEval } from "../commands/eval.js";
 import { runCoverageAudit } from "../commands/coverage-audit.js";
 import { runExamples } from "../commands/examples.js";
 import { runCloseout } from "../commands/closeout.js";
@@ -10,7 +11,7 @@ import { runInstructions } from "../commands/instructions.js";
 import { runNewTask } from "../commands/new-task.js";
 import { runImportPrd } from "../commands/import-prd.js";
 import { runPlanList } from "../commands/plan-list.js";
-import { runPlanCheck, runPlanComplete, runPlanCreate } from "../commands/plan.js";
+import { runPlanCheck, runPlanComplete, runPlanCreate, } from "../commands/plan.js";
 import { runPromoteRun } from "../commands/promote-run.js";
 import { runSpine } from "../commands/spine.js";
 import { runStats } from "../commands/stats.js";
@@ -76,6 +77,7 @@ const INIT_SUBCOMMANDS = [
     "update",
 ];
 const EXAMPLES_SUBCOMMANDS = ["list", "show", "copy"];
+const EVAL_SUBCOMMANDS = ["replay", "report", "candidate"];
 const CLOSEOUT_SUBCOMMANDS = ["task"];
 const PLAN_SUBCOMMANDS = ["list", "create", "complete", "check"];
 const SPINE_SUBCOMMANDS = ["audit"];
@@ -194,6 +196,17 @@ export const COMMAND_DEFINITIONS = [
             await runExamples(repoRoot, [subcommand, ...rest].filter(Boolean));
         },
     },
+    {
+        name: "eval",
+        adapter: "required",
+        tier: "operator",
+        intent: "Replay completed DAG evidence and manage immutable Candidate Registry lifecycle without live model execution or promotion.",
+        usage: "eval <replay|report|candidate> ...; candidate <register|show|list|transition> [--json|--markdown]",
+        subcommands: [...EVAL_SUBCOMMANDS],
+        handler: async ({ repoRoot, subcommand, rest }) => {
+            await runEval(repoRoot, [subcommand, ...rest].filter((arg) => Boolean(arg)));
+        },
+    },
     {
         name: "new-task",
         adapter: "required",
@@ -286,17 +299,17 @@ export const COMMAND_DEFINITIONS = [
             if (subcommand === "create") {
                 const [planId, ...titleParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan create <plan-id> \"<title>\"");
+                    throw new Error('usage: plan create <plan-id> "<title>"');
                 await runPlanCreate(repoRoot, planId, titleParts.join(" "));
                 return;
             }
             if (subcommand === "complete") {
                 const [planId, ...summaryParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 const summary = parseSummaryFlag(summaryParts);
                 if (!summary)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 await runPlanComplete(repoRoot, planId, { summary });
                 return;
             }

package/dist/cli/program.js

@@ -22,6 +22,7 @@ import { runDagWorkflowValidate } from "../commands/dag-workflow-validate.js";
 import { runDelegate } from "../commands/delegate.js";
 import { runDocsArchive } from "../commands/docs-archive.js";
 import { runDocsAudit } from "../commands/docs-audit.js";
+import { runEval } from "../commands/eval.js";
 import { runDoctor } from "../commands/doctor.js";
 import { runExamples } from "../commands/examples.js";
 import { runGoal } from "../commands/goal.js";
@@ -38,7 +39,7 @@ import { runImportPrd } from "../commands/import-prd.js";
 import { runPiReuseBenchmark } from "../commands/pi-reuse-benchmark.js";
 import { parsePiPromptArgs, printPiPromptUsage, runPiPrompt, } from "../commands/pi-prompt.js";
 import { runPlanList } from "../commands/plan-list.js";
-import { runPlanCheck, runPlanComplete, runPlanCreate } from "../commands/plan.js";
+import { runPlanCheck, runPlanComplete, runPlanCreate, } from "../commands/plan.js";
 import { runPromoteRun } from "../commands/promote-run.js";
 import { runReferenceIndex } from "../commands/reference-index.js";
 import { runRunDag } from "../commands/run-dag.js";
@@ -163,17 +164,17 @@ async function runCommanderAction(ctx, command, subcommand, rest) {
             if (subcommand === "create") {
                 const [planId, ...titleParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan create <plan-id> \"<title>\"");
+                    throw new Error('usage: plan create <plan-id> "<title>"');
                 await runPlanCreate(ctx.repoRoot, planId, titleParts.join(" "));
                 return;
             }
             if (subcommand === "complete") {
                 const [planId, ...summaryParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 const summary = parsePlanSummaryFlag(summaryParts);
                 if (!summary)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 await runPlanComplete(ctx.repoRoot, planId, { summary });
                 return;
             }
@@ -244,6 +245,9 @@ async function runCommanderAction(ctx, command, subcommand, rest) {
         case "knowledge":
             await runKnowledge(ctx.repoRoot, compactArgs([subcommand, ...rest]));
             return;
+        case "eval":
+            await runEval(ctx.repoRoot, compactArgs([subcommand, ...rest]));
+            return;
         case "dag":
             await runDagAction(ctx.repoRoot, subcommand, rest);
             return;