@tea-agent/loop-agent 0.13.0-alpha.0 → 0.13.0-beta.0

package/dist/application/evaluation/types.js

@@ -0,0 +1,130 @@
+import { z } from "zod";
+export const evalSplitSchema = z.enum(["public", "private", "held_out"]);
+export const replayEvidenceRefSchema = z
+    .object({
+    candidateId: z.string().min(1),
+    taskRef: z.string().min(1),
+    seed: z.number().int().nonnegative(),
+    split: evalSplitSchema,
+    runId: z
+        .string()
+        .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, "runId must be one safe path segment"),
+    stateSha256: z.string().regex(/^[a-f0-9]{64}$/),
+    runSha256: z.string().regex(/^[a-f0-9]{64}$/),
+})
+    .strict();
+export const replaySpecSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    replayId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/),
+    incumbentCandidateId: z.string().min(1),
+    challengerCandidateId: z.string().min(1),
+    evidence: z.array(replayEvidenceRefSchema).min(1),
+})
+    .strict()
+    .superRefine((spec, ctx) => {
+    if (spec.incumbentCandidateId === spec.challengerCandidateId) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "incumbentCandidateId and challengerCandidateId must differ",
+            path: ["challengerCandidateId"],
+        });
+    }
+    const allowed = new Set([
+        spec.incumbentCandidateId,
+        spec.challengerCandidateId,
+    ]);
+    const pairKeys = new Set();
+    for (let i = 0; i < spec.evidence.length; i += 1) {
+        const evidence = spec.evidence[i];
+        if (!allowed.has(evidence.candidateId)) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: "evidence candidateId must match incumbent or challenger",
+                path: ["evidence", i, "candidateId"],
+            });
+        }
+        const key = `${evidence.candidateId}\u0000${evidence.split}\u0000${evidence.taskRef}\u0000${evidence.seed}`;
+        if (pairKeys.has(key)) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: "duplicate evidence for candidateId + split + taskRef + seed",
+                path: ["evidence", i],
+            });
+        }
+        pairKeys.add(key);
+    }
+});
+// --- Candidate Registry (M2 W2.1–W2.2) ---
+export const candidateIdSchema = z
+    .string()
+    .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, "candidateId must be one safe path segment");
+export const candidateKindSchema = z.enum([
+    "prompt",
+    "skill",
+    "context_policy",
+    "model_routing",
+    "profile",
+    "composite",
+]);
+export const candidateContentRefSchema = z
+    .object({
+    path: z
+        .string()
+        .min(1)
+        .refine((value) => !pathIsAbsoluteLike(value), "content ref path must be repo-relative"),
+    sha256: z
+        .string()
+        .regex(/^(sha256:)?[a-f0-9]{64}$/i, "sha256 must be 64 hex digits"),
+})
+    .strict();
+function pathIsAbsoluteLike(value) {
+    if (value.startsWith("/") || value.startsWith("\\"))
+        return true;
+    if (/^[A-Za-z]:[\\/]/.test(value))
+        return true;
+    return false;
+}
+export const candidateManifestInputSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    candidateId: candidateIdSchema,
+    parentCandidateId: candidateIdSchema.nullable().optional(),
+    candidateKind: candidateKindSchema,
+    createdAt: z.string().min(1),
+    description: z.string().optional(),
+    contentRefs: z.array(candidateContentRefSchema).min(1),
+    // optional on input; always computed/verified on register/read
+    bundleHash: z
+        .string()
+        .regex(/^(sha256:)?[a-f0-9]{64}$/i)
+        .optional(),
+})
+    .strict();
+export const candidateManifestSchema = candidateManifestInputSchema
+    .extend({
+    parentCandidateId: candidateIdSchema.nullable(),
+    bundleHash: z.string().regex(/^sha256:[a-f0-9]{64}$/),
+})
+    .strict();
+export const lifecycleStateSchema = z.enum([
+    "proposed",
+    "eligible",
+    "experimenting",
+    "accepted",
+    "rejected",
+    "invalid",
+    "retired",
+]);
+export const lifecycleEventSchema = z
+    .object({
+    schemaVersion: z.literal(1),
+    seq: z.number().int().positive(),
+    from: lifecycleStateSchema.nullable(),
+    to: lifecycleStateSchema,
+    reason: z.string().min(1),
+    at: z.string().min(1),
+    previousEventHash: z.string().regex(/^[a-f0-9]{64}$/),
+    eventHash: z.string().regex(/^[a-f0-9]{64}$/),
+})
+    .strict();

package/dist/cli/command-definitions.js

@@ -1,6 +1,7 @@
 import { runDoctor } from "../commands/doctor.js";
 import { runDocsArchive } from "../commands/docs-archive.js";
 import { runDocsAudit } from "../commands/docs-audit.js";
+import { runEval } from "../commands/eval.js";
 import { runCoverageAudit } from "../commands/coverage-audit.js";
 import { runExamples } from "../commands/examples.js";
 import { runCloseout } from "../commands/closeout.js";
@@ -10,7 +11,7 @@ import { runInstructions } from "../commands/instructions.js";
 import { runNewTask } from "../commands/new-task.js";
 import { runImportPrd } from "../commands/import-prd.js";
 import { runPlanList } from "../commands/plan-list.js";
-import { runPlanCheck, runPlanComplete, runPlanCreate } from "../commands/plan.js";
+import { runPlanCheck, runPlanComplete, runPlanCreate, } from "../commands/plan.js";
 import { runPromoteRun } from "../commands/promote-run.js";
 import { runSpine } from "../commands/spine.js";
 import { runStats } from "../commands/stats.js";
@@ -76,6 +77,7 @@ const INIT_SUBCOMMANDS = [
     "update",
 ];
 const EXAMPLES_SUBCOMMANDS = ["list", "show", "copy"];
+const EVAL_SUBCOMMANDS = ["replay", "report", "candidate"];
 const CLOSEOUT_SUBCOMMANDS = ["task"];
 const PLAN_SUBCOMMANDS = ["list", "create", "complete", "check"];
 const SPINE_SUBCOMMANDS = ["audit"];
@@ -194,6 +196,17 @@ export const COMMAND_DEFINITIONS = [
             await runExamples(repoRoot, [subcommand, ...rest].filter(Boolean));
         },
     },
+    {
+        name: "eval",
+        adapter: "required",
+        tier: "operator",
+        intent: "Replay completed DAG evidence and manage immutable Candidate Registry lifecycle without live model execution or promotion.",
+        usage: "eval <replay|report|candidate> ...; candidate <register|show|list|transition> [--json|--markdown]",
+        subcommands: [...EVAL_SUBCOMMANDS],
+        handler: async ({ repoRoot, subcommand, rest }) => {
+            await runEval(repoRoot, [subcommand, ...rest].filter((arg) => Boolean(arg)));
+        },
+    },
     {
         name: "new-task",
         adapter: "required",
@@ -286,17 +299,17 @@ export const COMMAND_DEFINITIONS = [
             if (subcommand === "create") {
                 const [planId, ...titleParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan create <plan-id> \"<title>\"");
+                    throw new Error('usage: plan create <plan-id> "<title>"');
                 await runPlanCreate(repoRoot, planId, titleParts.join(" "));
                 return;
             }
             if (subcommand === "complete") {
                 const [planId, ...summaryParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 const summary = parseSummaryFlag(summaryParts);
                 if (!summary)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 await runPlanComplete(repoRoot, planId, { summary });
                 return;
             }

package/dist/cli/program.js

@@ -22,6 +22,7 @@ import { runDagWorkflowValidate } from "../commands/dag-workflow-validate.js";
 import { runDelegate } from "../commands/delegate.js";
 import { runDocsArchive } from "../commands/docs-archive.js";
 import { runDocsAudit } from "../commands/docs-audit.js";
+import { runEval } from "../commands/eval.js";
 import { runDoctor } from "../commands/doctor.js";
 import { runExamples } from "../commands/examples.js";
 import { runGoal } from "../commands/goal.js";
@@ -38,7 +39,7 @@ import { runImportPrd } from "../commands/import-prd.js";
 import { runPiReuseBenchmark } from "../commands/pi-reuse-benchmark.js";
 import { parsePiPromptArgs, printPiPromptUsage, runPiPrompt, } from "../commands/pi-prompt.js";
 import { runPlanList } from "../commands/plan-list.js";
-import { runPlanCheck, runPlanComplete, runPlanCreate } from "../commands/plan.js";
+import { runPlanCheck, runPlanComplete, runPlanCreate, } from "../commands/plan.js";
 import { runPromoteRun } from "../commands/promote-run.js";
 import { runReferenceIndex } from "../commands/reference-index.js";
 import { runRunDag } from "../commands/run-dag.js";
@@ -163,17 +164,17 @@ async function runCommanderAction(ctx, command, subcommand, rest) {
             if (subcommand === "create") {
                 const [planId, ...titleParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan create <plan-id> \"<title>\"");
+                    throw new Error('usage: plan create <plan-id> "<title>"');
                 await runPlanCreate(ctx.repoRoot, planId, titleParts.join(" "));
                 return;
             }
             if (subcommand === "complete") {
                 const [planId, ...summaryParts] = rest;
                 if (!planId)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 const summary = parsePlanSummaryFlag(summaryParts);
                 if (!summary)
-                    throw new Error("usage: plan complete <plan-id> --summary \"<summary>\"");
+                    throw new Error('usage: plan complete <plan-id> --summary "<summary>"');
                 await runPlanComplete(ctx.repoRoot, planId, { summary });
                 return;
             }
@@ -244,6 +245,9 @@ async function runCommanderAction(ctx, command, subcommand, rest) {
         case "knowledge":
             await runKnowledge(ctx.repoRoot, compactArgs([subcommand, ...rest]));
             return;
+        case "eval":
+            await runEval(ctx.repoRoot, compactArgs([subcommand, ...rest]));
+            return;
         case "dag":
             await runDagAction(ctx.repoRoot, subcommand, rest);
             return;

package/dist/commands/cursor-prompt.js

@@ -141,7 +141,7 @@ async function runCursorPromptBatch(task, cwd, model, timeoutMs) {
 }
 async function runCursorPromptStreaming(task, cwd, model, timeoutMs) {
     const startedAt = Date.now();
-    process.stderr.write(`[cursor-prompt] streaming (model=${model}, cwd=${cwd})
+    process.stderr.write(`[cursor-prompt] streaming (model=${model}, cwd=${cwd})
 `);
     const runDir = (await computeRunDir(cwd, task)) ?? undefined;
     const result = await executeCursorPromptStream({
@@ -164,15 +164,15 @@ async function runCursorPromptStreaming(task, cwd, model, timeoutMs) {
     });
     const elapsed = ((Date.now() - startedAt) / 1000).toFixed(1);
     if (result.ok) {
-        process.stderr.write(`
-[cursor-prompt] done in ${elapsed}s, status=${result.status}
+        process.stderr.write(`
+[cursor-prompt] done in ${elapsed}s, status=${result.status}
 `);
     }
     else {
         const stderr = result.stderr || "(no output)";
-        process.stderr.write(`
-[cursor-prompt] FAILED in ${elapsed}s (${result.failureCategory}):
-${stderr}
+        process.stderr.write(`
+[cursor-prompt] FAILED in ${elapsed}s (${result.failureCategory}):
+${stderr}
 `);
         process.exit(1);
     }

package/dist/commands/eval.js

@@ -0,0 +1,235 @@
+import path from "node:path";
+import { writeTextAtomic } from "../infrastructure/harness/atomic-write.js";
+import { readReplayScorecard } from "../infrastructure/evaluation/store.js";
+import { formatReplayMarkdown, replayEvaluation, } from "../application/evaluation/replay.js";
+import { formatCandidateMarkdown, listCandidates, registerCandidate, showCandidate, transitionCandidate, } from "../application/evaluation/candidate.js";
+import { lifecycleStateSchema, } from "../application/evaluation/types.js";
+const USAGE = "usage: eval <replay|report|candidate> ...; candidate <register|show|list|transition> [--json|--markdown]";
+function parseFormatFlags(args) {
+    const json = args.includes("--json");
+    const markdown = args.includes("--markdown");
+    if (json && markdown) {
+        throw new Error("eval accepts only one of --json or --markdown");
+    }
+    return { json: json || !markdown, markdown };
+}
+function flagValue(args, flag) {
+    const index = args.indexOf(flag);
+    if (index >= 0) {
+        const value = args[index + 1];
+        if (!value || value.startsWith("-")) {
+            throw new Error(`${flag} requires a value`);
+        }
+        return value;
+    }
+    const prefix = `${flag}=`;
+    return args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length);
+}
+function assertKnownFlags(args, allowed) {
+    for (let i = 0; i < args.length; i += 1) {
+        const arg = args[i];
+        if (!arg.startsWith("-"))
+            continue;
+        const key = arg.split("=", 1)[0];
+        if (!allowed.includes(key)) {
+            throw new Error(`unknown eval argument: ${arg}`);
+        }
+        if ([
+            "--spec",
+            "--output",
+            "--replay-id",
+            "--manifest",
+            "--candidate-id",
+            "--to",
+            "--reason",
+        ].includes(key) &&
+            !arg.includes("=")) {
+            i += 1;
+        }
+    }
+}
+function parseCandidateArgs(rest) {
+    const [action, ...tail] = rest;
+    if (action === "register") {
+        assertKnownFlags(tail, ["--manifest", "--json", "--markdown"]);
+        const manifestPath = flagValue(tail, "--manifest");
+        if (!manifestPath) {
+            throw new Error("eval candidate register requires --manifest <path>");
+        }
+        return {
+            command: "candidate",
+            action: "register",
+            manifestPath,
+            ...parseFormatFlags(tail),
+        };
+    }
+    if (action === "show") {
+        assertKnownFlags(tail, ["--candidate-id", "--json", "--markdown"]);
+        const candidateId = flagValue(tail, "--candidate-id");
+        if (!candidateId) {
+            throw new Error("eval candidate show requires --candidate-id <id>");
+        }
+        return {
+            command: "candidate",
+            action: "show",
+            candidateId,
+            ...parseFormatFlags(tail),
+        };
+    }
+    if (action === "list") {
+        assertKnownFlags(tail, ["--json", "--markdown"]);
+        return {
+            command: "candidate",
+            action: "list",
+            ...parseFormatFlags(tail),
+        };
+    }
+    if (action === "transition") {
+        assertKnownFlags(tail, [
+            "--candidate-id",
+            "--to",
+            "--reason",
+            "--json",
+            "--markdown",
+        ]);
+        const candidateId = flagValue(tail, "--candidate-id");
+        const toRaw = flagValue(tail, "--to");
+        const reason = flagValue(tail, "--reason");
+        if (!candidateId) {
+            throw new Error("eval candidate transition requires --candidate-id <id>");
+        }
+        if (!toRaw) {
+            throw new Error("eval candidate transition requires --to <state>");
+        }
+        if (!reason) {
+            throw new Error("eval candidate transition requires --reason <text>");
+        }
+        const to = lifecycleStateSchema.parse(toRaw);
+        return {
+            command: "candidate",
+            action: "transition",
+            candidateId,
+            to,
+            reason,
+            ...parseFormatFlags(tail),
+        };
+    }
+    throw new Error("usage: eval candidate <register|show|list|transition> ...");
+}
+export function parseEvalArgs(args) {
+    const [command, ...rest] = args;
+    if (command === "replay") {
+        assertKnownFlags(rest, ["--spec", "--output", "--json", "--markdown"]);
+        const specPath = flagValue(rest, "--spec");
+        if (!specPath)
+            throw new Error("eval replay requires --spec <path>");
+        return {
+            command,
+            specPath,
+            outputPath: flagValue(rest, "--output"),
+            ...parseFormatFlags(rest),
+        };
+    }
+    if (command === "report") {
+        assertKnownFlags(rest, ["--replay-id", "--json", "--markdown"]);
+        const replayId = flagValue(rest, "--replay-id");
+        if (!replayId)
+            throw new Error("eval report requires --replay-id <id>");
+        return { command, replayId, ...parseFormatFlags(rest) };
+    }
+    if (command === "candidate") {
+        return parseCandidateArgs(rest);
+    }
+    throw new Error(USAGE);
+}
+function printScorecard(input) {
+    if (input.json) {
+        console.log(JSON.stringify(input.scorecard, null, 2));
+        return;
+    }
+    process.stdout.write(input.markdown);
+}
+function printCandidate(input) {
+    if (input.json) {
+        console.log(JSON.stringify(input.record, null, 2));
+        return;
+    }
+    process.stdout.write(formatCandidateMarkdown(input.record));
+}
+export async function runEval(repoRoot, args) {
+    const parsed = parseEvalArgs(args);
+    if (parsed.command === "replay") {
+        const result = await replayEvaluation({
+            repoRoot,
+            specPath: parsed.specPath,
+        });
+        if (parsed.outputPath) {
+            const outputPath = path.resolve(repoRoot, parsed.outputPath);
+            await writeTextAtomic(outputPath, parsed.markdown
+                ? result.markdown
+                : `${JSON.stringify(result.scorecard, null, 2)}\n`, { repoRoot });
+        }
+        printScorecard({
+            scorecard: result.scorecard,
+            markdown: result.markdown,
+            json: parsed.json,
+        });
+        return;
+    }
+    if (parsed.command === "report") {
+        const scorecard = (await readReplayScorecard(repoRoot, parsed.replayId));
+        const markdown = formatReplayMarkdown(scorecard);
+        printScorecard({ scorecard, markdown, json: parsed.json });
+        return;
+    }
+    // candidate subcommands
+    if (parsed.action === "register") {
+        const result = await registerCandidate({
+            repoRoot,
+            manifestPath: parsed.manifestPath,
+        });
+        if (parsed.json) {
+            console.log(JSON.stringify({
+                idempotent: result.idempotent,
+                manifestPath: result.manifestPath,
+                lifecyclePath: result.lifecyclePath,
+                record: result.record,
+            }, null, 2));
+            return;
+        }
+        process.stdout.write(`${result.idempotent ? "idempotent " : ""}registered ${result.record.manifest.candidateId}\n${formatCandidateMarkdown(result.record)}`);
+        return;
+    }
+    if (parsed.action === "show") {
+        const record = await showCandidate({
+            repoRoot,
+            candidateId: parsed.candidateId,
+        });
+        printCandidate({ record, json: parsed.json });
+        return;
+    }
+    if (parsed.action === "list") {
+        const rows = await listCandidates({ repoRoot });
+        if (parsed.json) {
+            console.log(JSON.stringify(rows, null, 2));
+            return;
+        }
+        const lines = [
+            "# Candidates",
+            "",
+            ...rows.map((row) => `- \`${row.candidateId}\` status=\`${row.status}\` hash=\`${row.bundleHash}\` promotionApplied=false`),
+            "",
+        ];
+        process.stdout.write(`${lines.join("\n")}\n`);
+        return;
+    }
+    if (parsed.action === "transition") {
+        const record = await transitionCandidate({
+            repoRoot,
+            candidateId: parsed.candidateId,
+            to: parsed.to,
+            reason: parsed.reason,
+        });
+        printCandidate({ record, json: parsed.json });
+    }
+}