@tea-agent/loop-agent 0.16.1-beta.2 → 0.16.1

package/dist/application/evaluation/campaign-matrix.js

@@ -0,0 +1,372 @@
+import { mkdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { campaignDir, readCampaignManifest, readCampaignPlan, writeCampaignScorecard, } from "../../infrastructure/evaluation/campaign-store.js";
+import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
+import { planCampaign, runCampaignPrivateChecks, } from "./campaign.js";
+import { evaluatePromotionPolicy } from "./promotion-policy.js";
+import { matrixFixtureSchema, matrixRunSchema } from "./types.js";
+function matrixJobsDir(repoRoot, campaignId) {
+    return path.join(campaignDir(repoRoot, campaignId), "matrix-jobs");
+}
+function matrixRunPath(repoRoot, campaignId) {
+    return path.join(campaignDir(repoRoot, campaignId), "matrix-run.json");
+}
+function evidenceDraftPath(repoRoot, campaignId) {
+    return path.join(campaignDir(repoRoot, campaignId), "evidence-draft.json");
+}
+export async function loadMatrixFixture(input) {
+    const absolute = path.resolve(input.repoRoot, input.fixturePath);
+    return matrixFixtureSchema.parse(JSON.parse(await readFile(absolute, "utf-8")));
+}
+function resolveStubResult(fixture, cell) {
+    for (const row of fixture.results) {
+        if (row.cellId && row.cellId === cell.cellId) {
+            return { verifyPassed: row.verifyPassed, runId: row.runId };
+        }
+        const match = (!row.candidateId || row.candidateId === cell.candidateId) &&
+            (!row.taskRef || row.taskRef === cell.taskRef) &&
+            (row.seed === undefined || row.seed === cell.seed) &&
+            (!row.split || row.split === cell.split);
+        if (match &&
+            (row.candidateId || row.taskRef || row.seed !== undefined || row.split)) {
+            return { verifyPassed: row.verifyPassed, runId: row.runId };
+        }
+    }
+    return { verifyPassed: fixture.defaultVerifyPassed };
+}
+export function buildMatrixJobSpec(input) {
+    return {
+        schemaVersion: 1,
+        cellId: input.cell.cellId,
+        evaluation: {
+            campaignId: input.campaignId,
+            candidateId: input.cell.candidateId,
+            bundleHash: input.cell.bundleHash,
+            seed: input.cell.seed,
+            split: input.cell.split,
+            taskRef: input.cell.taskRef,
+        },
+        executor: "pi",
+        autoPromote: false,
+    };
+}
+async function writeMatrixJob(input) {
+    const dir = matrixJobsDir(input.repoRoot, input.campaignId);
+    await mkdir(dir, { recursive: true });
+    const job = buildMatrixJobSpec({
+        cell: input.cell,
+        campaignId: input.campaignId,
+    });
+    const jobPath = path.join(dir, `${input.cell.cellId}.json`);
+    await writeJsonAtomic(jobPath, job, { repoRoot: input.repoRoot });
+    return {
+        jobPath: path.relative(input.repoRoot, jobPath).replace(/\\/g, "/"),
+        evaluation: job.evaluation,
+    };
+}
+/**
+ * Candidate matrix runner (M7 companion).
+ * Modes:
+ * - dry-run: plan public cells only
+ * - stub: resolve verifyPassed from fixture (tests / offline)
+ * - pi-plan: materialize per-cell Pi evaluation job specs (no spawn)
+ * - pi: require injected PiCellExecutor; never auto-promote
+ */
+export async function runCampaignMatrix(input) {
+    const planned = await planCampaign({
+        repoRoot: input.repoRoot,
+        campaignId: input.campaignId,
+    });
+    const publicCells = planned.plan.cells.filter((cell) => cell.executionRole === "candidate_run");
+    const limited = input.maxPublicCells !== undefined
+        ? publicCells.slice(0, input.maxPublicCells)
+        : publicCells;
+    let fixture;
+    if (input.mode === "stub") {
+        if (!input.fixturePath) {
+            throw new Error("matrix mode=stub requires --fixture <path>");
+        }
+        fixture = await loadMatrixFixture({
+            repoRoot: input.repoRoot,
+            fixturePath: input.fixturePath,
+        });
+    }
+    if (input.mode === "pi" && !input.piExecutor) {
+        throw new Error("matrix mode=pi requires an injected PiCellExecutor (CLI refuses live Pi by default; use pi-plan or stub)");
+    }
+    const cellResults = [];
+    for (const cell of limited) {
+        if (input.mode === "dry-run") {
+            cellResults.push({
+                cellId: cell.cellId,
+                candidateId: cell.candidateId,
+                taskRef: cell.taskRef,
+                seed: cell.seed,
+                split: cell.split,
+                executionRole: cell.executionRole,
+                mode: "dry-run",
+                skipped: true,
+                detail: "dry-run: would execute candidate_run cell",
+            });
+            continue;
+        }
+        if (input.mode === "stub") {
+            const resolved = resolveStubResult(fixture, cell);
+            cellResults.push({
+                cellId: cell.cellId,
+                candidateId: cell.candidateId,
+                taskRef: cell.taskRef,
+                seed: cell.seed,
+                split: cell.split,
+                executionRole: cell.executionRole,
+                mode: "stub",
+                verifyPassed: resolved.verifyPassed,
+                runId: resolved.runId,
+                skipped: false,
+            });
+            continue;
+        }
+        const job = await writeMatrixJob({
+            repoRoot: input.repoRoot,
+            campaignId: input.campaignId,
+            cell,
+        });
+        if (input.mode === "pi-plan") {
+            cellResults.push({
+                cellId: cell.cellId,
+                candidateId: cell.candidateId,
+                taskRef: cell.taskRef,
+                seed: cell.seed,
+                split: cell.split,
+                executionRole: cell.executionRole,
+                mode: "pi-plan",
+                jobPath: job.jobPath,
+                skipped: true,
+                detail: "pi-plan: job materialized; Pi not spawned",
+            });
+            continue;
+        }
+        const executed = await input.piExecutor({
+            repoRoot: input.repoRoot,
+            cell,
+            jobPath: job.jobPath,
+            evaluation: job.evaluation,
+        });
+        cellResults.push({
+            cellId: cell.cellId,
+            candidateId: cell.candidateId,
+            taskRef: cell.taskRef,
+            seed: cell.seed,
+            split: cell.split,
+            executionRole: cell.executionRole,
+            mode: "pi",
+            verifyPassed: executed.verifyPassed,
+            runId: executed.runId,
+            jobPath: job.jobPath,
+            detail: executed.detail,
+            skipped: false,
+        });
+    }
+    // Mark held_out / remaining public as skipped for transparency
+    for (const cell of planned.plan.cells) {
+        if (cell.executionRole === "candidate_run") {
+            if (limited.some((row) => row.cellId === cell.cellId))
+                continue;
+            cellResults.push({
+                cellId: cell.cellId,
+                candidateId: cell.candidateId,
+                taskRef: cell.taskRef,
+                seed: cell.seed,
+                split: cell.split,
+                executionRole: cell.executionRole,
+                mode: input.mode,
+                skipped: true,
+                detail: "skipped by maxPublicCells",
+            });
+            continue;
+        }
+        if (cell.executionRole === "promotion_gate_only") {
+            cellResults.push({
+                cellId: cell.cellId,
+                candidateId: cell.candidateId,
+                taskRef: cell.taskRef,
+                seed: cell.seed,
+                split: cell.split,
+                executionRole: cell.executionRole,
+                mode: input.mode,
+                skipped: true,
+                detail: "held_out remains promotion_gate_only (not executed in matrix)",
+            });
+        }
+    }
+    let privateResults = [];
+    if (input.includePrivateChecks !== false && input.mode !== "dry-run") {
+        const privateRun = await runCampaignPrivateChecks({
+            repoRoot: input.repoRoot,
+            campaignId: input.campaignId,
+        });
+        privateResults = privateRun.results;
+        for (const cell of planned.plan.cells) {
+            if (cell.executionRole !== "private_verify")
+                continue;
+            const related = privateResults.filter((row) => row.cellId === cell.cellId);
+            const ok = related.length > 0 && related.every((row) => row.ok);
+            cellResults.push({
+                cellId: cell.cellId,
+                candidateId: cell.candidateId,
+                taskRef: cell.taskRef,
+                seed: cell.seed,
+                split: cell.split,
+                executionRole: "private_verify",
+                mode: input.mode,
+                verifyPassed: ok,
+                skipped: false,
+                detail: `private checks=${related.length}`,
+            });
+        }
+    }
+    const now = input.now ?? new Date().toISOString();
+    let evidence;
+    let evidencePath;
+    if (input.writeEvidenceDraft !== false && input.mode !== "dry-run") {
+        evidence = buildEvidenceDraft({
+            campaignId: input.campaignId,
+            manifest: planned.manifest,
+            cellResults,
+        });
+        evidencePath = path
+            .relative(input.repoRoot, evidenceDraftPath(input.repoRoot, input.campaignId))
+            .replace(/\\/g, "/");
+        await writeJsonAtomic(evidenceDraftPath(input.repoRoot, input.campaignId), evidence, { repoRoot: input.repoRoot });
+    }
+    const matrix = matrixRunSchema.parse({
+        schemaVersion: 1,
+        campaignId: input.campaignId,
+        planHash: planned.plan.planHash,
+        mode: input.mode,
+        createdAt: now,
+        cells: cellResults,
+        privateCheckCount: privateResults.length,
+        privateCheckFailed: privateResults.filter((row) => !row.ok).length,
+        evidencePath,
+        autoPromote: false,
+    });
+    await writeJsonAtomic(matrixRunPath(input.repoRoot, input.campaignId), matrix, {
+        repoRoot: input.repoRoot,
+    });
+    return { matrix, privateResults, evidence };
+}
+function buildEvidenceDraft(input) {
+    const challengerCandidateId = input.manifest.challengerCandidateIds[0];
+    if (!challengerCandidateId) {
+        throw new Error("campaign has no challenger for evidence draft");
+    }
+    const cells = [];
+    for (const row of input.cellResults) {
+        if (row.verifyPassed === undefined)
+            continue;
+        if (row.executionRole !== "candidate_run" &&
+            row.executionRole !== "private_verify" &&
+            row.executionRole !== "promotion_gate_only") {
+            continue;
+        }
+        // Only include executed public/private pairs; held_out needs external evidence.
+        if (row.executionRole === "promotion_gate_only")
+            continue;
+        cells.push({
+            cellId: row.cellId,
+            candidateId: row.candidateId,
+            taskRef: row.taskRef,
+            seed: row.seed,
+            split: row.split,
+            verifyPassed: row.verifyPassed,
+            runId: row.runId,
+        });
+    }
+    if (cells.length === 0) {
+        // Minimal placeholder so draft is schema-valid for inspection; scorecard needs more.
+        cells.push({
+            candidateId: input.manifest.incumbentCandidateId,
+            taskRef: "matrix/placeholder",
+            seed: 0,
+            split: "public",
+            verifyPassed: false,
+        });
+        cells.push({
+            candidateId: challengerCandidateId,
+            taskRef: "matrix/placeholder",
+            seed: 0,
+            split: "public",
+            verifyPassed: false,
+        });
+    }
+    return {
+        schemaVersion: 1,
+        campaignId: input.campaignId,
+        challengerCandidateId,
+        cells,
+        diffs: [],
+        verifyCommands: { incumbent: [], challenger: [] },
+        budgetStatus: { incumbent: "missing", challenger: "missing" },
+    };
+}
+export async function readMatrixRun(repoRoot, campaignId) {
+    try {
+        const raw = await readFile(matrixRunPath(repoRoot, campaignId), "utf-8");
+        return matrixRunSchema.parse(JSON.parse(raw));
+    }
+    catch (error) {
+        const code = error.code;
+        if (code === "ENOENT")
+            return undefined;
+        throw error;
+    }
+}
+export function formatMatrixRunMarkdown(matrix) {
+    const executed = matrix.cells.filter((cell) => !cell.skipped);
+    const lines = [
+        `# Campaign Matrix: ${matrix.campaignId}`,
+        "",
+        `- mode: \`${matrix.mode}\``,
+        `- planHash: \`${matrix.planHash}\``,
+        `- cells: ${matrix.cells.length} (executed/recorded=${executed.length})`,
+        `- privateChecks: ${matrix.privateCheckCount} (failed=${matrix.privateCheckFailed})`,
+        `- evidencePath: \`${matrix.evidencePath ?? "(none)"}\``,
+        `- autoPromote: \`${matrix.autoPromote}\``,
+        "",
+        "## Cell results",
+        "",
+    ];
+    for (const cell of matrix.cells.slice(0, 40)) {
+        lines.push(`- \`${cell.cellId}\` role=${cell.executionRole} skipped=${cell.skipped} passed=${cell.verifyPassed ?? "-"} ${cell.detail ?? ""}`);
+    }
+    if (matrix.cells.length > 40) {
+        lines.push(`- … ${matrix.cells.length - 40} more`);
+    }
+    lines.push("", "> Matrix never auto-promotes. Use `eval campaign scorecard` + human-gated `eval promote` / `eval experiment accept`.", "");
+    return `${lines.join("\n")}\n`;
+}
+/** Optional helper: build scorecard from matrix evidence draft when complete enough. */
+export async function buildScorecardFromMatrixEvidence(input) {
+    const manifest = await readCampaignManifest(input.repoRoot, input.campaignId);
+    const plan = await readCampaignPlan(input.repoRoot, input.campaignId);
+    if (!plan)
+        throw new Error(`campaign ${input.campaignId} has no plan`);
+    const policy = evaluatePromotionPolicy({
+        campaignId: input.campaignId,
+        incumbentCandidateId: manifest.incumbentCandidateId,
+        evidence: input.evidence,
+    });
+    await writeCampaignScorecard({
+        repoRoot: input.repoRoot,
+        scorecard: {
+            schemaVersion: 1,
+            campaignId: input.campaignId,
+            planHash: plan.planHash,
+            createdAt: new Date().toISOString(),
+            evidenceCellCount: input.evidence.cells.length,
+            policy,
+        },
+        evidence: input.evidence,
+    });
+}

package/dist/application/evaluation/campaign-scorecard.js

@@ -0,0 +1,135 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { readCampaignEvidence, readCampaignManifest, readCampaignPlan, readCampaignScorecard, writeCampaignScorecard, } from "../../infrastructure/evaluation/campaign-store.js";
+import { evaluatePromotionPolicy } from "./promotion-policy.js";
+import { campaignEvidenceSchema, } from "./types.js";
+export async function loadCampaignEvidence(input) {
+    const absolute = path.resolve(input.repoRoot, input.evidencePath);
+    const raw = JSON.parse(await readFile(absolute, "utf-8"));
+    return campaignEvidenceSchema.parse(raw);
+}
+export async function buildCampaignScorecard(input) {
+    const manifest = await readCampaignManifest(input.repoRoot, input.campaignId);
+    const plan = await readCampaignPlan(input.repoRoot, input.campaignId);
+    if (!plan) {
+        throw new Error(`campaign ${input.campaignId} has no plan; run eval campaign plan first`);
+    }
+    if (input.evidence.campaignId !== input.campaignId) {
+        throw new Error(`evidence campaignId mismatch: expected ${input.campaignId}, got ${input.evidence.campaignId}`);
+    }
+    const policy = evaluatePromotionPolicy({
+        campaignId: input.campaignId,
+        incumbentCandidateId: manifest.incumbentCandidateId,
+        evidence: input.evidence,
+        acknowledgeReview: input.acknowledgeReview,
+    });
+    const scorecard = {
+        schemaVersion: 1,
+        campaignId: input.campaignId,
+        planHash: plan.planHash,
+        createdAt: new Date().toISOString(),
+        evidenceCellCount: input.evidence.cells.length,
+        policy,
+    };
+    if (input.persist !== false) {
+        await writeCampaignScorecard({
+            repoRoot: input.repoRoot,
+            scorecard,
+            evidence: input.evidence,
+        });
+    }
+    return scorecard;
+}
+export async function auditCampaignEvidence(input) {
+    const scorecard = await buildCampaignScorecard({
+        ...input,
+        persist: false,
+    });
+    return {
+        findings: scorecard.policy.findings,
+        policy: scorecard.policy,
+    };
+}
+export async function requireCampaignPromotionGate(input) {
+    let evidence;
+    if (input.evidencePath) {
+        evidence = await loadCampaignEvidence({
+            repoRoot: input.repoRoot,
+            evidencePath: input.evidencePath,
+        });
+    }
+    else {
+        evidence = await readCampaignEvidence(input.repoRoot, input.campaignId);
+    }
+    let scorecard;
+    if (evidence) {
+        scorecard = await buildCampaignScorecard({
+            repoRoot: input.repoRoot,
+            campaignId: input.campaignId,
+            evidence,
+            acknowledgeReview: input.acknowledgeReview,
+            persist: true,
+        });
+    }
+    else {
+        const existing = await readCampaignScorecard(input.repoRoot, input.campaignId);
+        if (!existing) {
+            throw new Error(`campaign ${input.campaignId} has no scorecard; run eval campaign scorecard --evidence <path> first`);
+        }
+        if (input.acknowledgeReview &&
+            existing.policy.gates.safetyOk === "review" &&
+            !existing.policy.promotionEligible) {
+            throw new Error(`campaign ${input.campaignId} scorecard needs re-evaluation with evidence; provide --evidence or re-run scorecard`);
+        }
+        scorecard = existing;
+    }
+    if (scorecard.policy.challengerCandidateId !== input.toCandidateId) {
+        throw new Error(`promotion target mismatch: scorecard challenger=${scorecard.policy.challengerCandidateId}, --to-candidate=${input.toCandidateId}`);
+    }
+    if (!scorecard.policy.promotionEligible) {
+        throw new Error(`campaign promotion policy rejected: ${scorecard.policy.reasons.join("; ")}`);
+    }
+    return scorecard;
+}
+export function formatCampaignScorecardMarkdown(scorecard) {
+    const { policy } = scorecard;
+    const lines = [
+        `# Campaign Scorecard: ${scorecard.campaignId}`,
+        "",
+        `- planHash: \`${scorecard.planHash}\``,
+        `- createdAt: \`${scorecard.createdAt}\``,
+        `- evidenceCells: ${scorecard.evidenceCellCount}`,
+        `- policy: \`${policy.policyVersion}\``,
+        `- promotionEligible: **${policy.promotionEligible}**`,
+        "",
+        "## Gates",
+        "",
+        `- privateImprovement: \`${policy.gates.privateImprovement}\``,
+        `- heldOutNonRegression: \`${policy.gates.heldOutNonRegression}\``,
+        `- budgetOk: \`${policy.gates.budgetOk}\``,
+        `- safetyOk: \`${policy.gates.safetyOk}\``,
+        `- humanRequired: \`${policy.gates.humanRequired}\``,
+        "",
+        "## Paired tallies",
+        "",
+        `- private: challengerWins=${policy.paired.private.challengerWins} incumbentWins=${policy.paired.private.incumbentWins} ties=${policy.paired.private.ties}`,
+        `- held_out: challengerWins=${policy.paired.held_out.challengerWins} incumbentWins=${policy.paired.held_out.incumbentWins} ties=${policy.paired.held_out.ties}`,
+        `- public: challengerWins=${policy.paired.public.challengerWins} incumbentWins=${policy.paired.public.incumbentWins} ties=${policy.paired.public.ties}`,
+        "",
+    ];
+    if (policy.findings.length > 0) {
+        lines.push("## Integrity findings", "");
+        for (const finding of policy.findings) {
+            lines.push(`- [${finding.severity}] \`${finding.code}\`${finding.path ? ` @ ${finding.path}` : ""}: ${finding.message}`);
+        }
+        lines.push("");
+    }
+    if (policy.reasons.length > 0) {
+        lines.push("## Reasons", "");
+        for (const reason of policy.reasons) {
+            lines.push(`- ${reason}`);
+        }
+        lines.push("");
+    }
+    return `${lines.join("\n")}\n`;
+}