@tea-agent/loop-agent 0.14.0 → 0.16.0

package/dist/application/evaluation/outer-loop.js

@@ -0,0 +1,132 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { createCampaign, validateCampaign, } from "./campaign.js";
+import { buildScorecardFromMatrixEvidence, formatMatrixRunMarkdown, runCampaignMatrix, } from "./campaign-matrix.js";
+import { formatProposerRunMarkdown, proposeFromCurate } from "./proposer.js";
+import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
+import { campaignManifestInputSchema } from "./types.js";
+/**
+ * M7 outer-loop orchestration:
+ * optional bounded propose → ensure/create campaign → matrix → optional scorecard.
+ * Never calls promote/alias APIs.
+ */
+export async function runOuterLoop(input) {
+    let proposerRun;
+    if (input.proposeFromCurate) {
+        proposerRun = await proposeFromCurate({
+            repoRoot: input.repoRoot,
+            maxCandidates: input.maxProposals ?? 1,
+            dryRun: false,
+            now: input.now,
+        });
+        if (proposerRun.proposals.length === 0) {
+            throw new Error(`outer-loop propose produced no candidates: ${proposerRun.skippedReasons.join("; ")}`);
+        }
+    }
+    let campaignId = input.campaignId;
+    if (input.campaignManifestPath) {
+        const absolute = path.resolve(input.repoRoot, input.campaignManifestPath);
+        const raw = JSON.parse(await readFile(absolute, "utf-8"));
+        if (proposerRun && proposerRun.proposals.length > 0) {
+            raw.challengerCandidateIds = proposerRun.proposals.map((row) => row.candidateId);
+        }
+        const parsed = campaignManifestInputSchema.parse(raw);
+        if (campaignId && parsed.campaignId !== campaignId) {
+            throw new Error(`campaignId mismatch: --campaign-id=${campaignId}, manifest=${parsed.campaignId}`);
+        }
+        campaignId = parsed.campaignId;
+        const staging = path.join(input.repoRoot, ".harness/evaluation/outer-loop", `${parsed.campaignId}.manifest.json`);
+        await writeJsonAtomic(staging, parsed, { repoRoot: input.repoRoot });
+        await validateCampaign({
+            repoRoot: input.repoRoot,
+            manifestPath: staging,
+        });
+        await createCampaign({
+            repoRoot: input.repoRoot,
+            manifestPath: staging,
+        });
+    }
+    if (!campaignId) {
+        throw new Error("outer-loop requires --campaign-id or --campaign-manifest");
+    }
+    if (proposerRun && !input.campaignManifestPath) {
+        // Campaign must already list proposed challengers.
+        const { showCampaign } = await import("./campaign.js");
+        const existing = await showCampaign({
+            repoRoot: input.repoRoot,
+            campaignId,
+        });
+        for (const proposal of proposerRun.proposals) {
+            if (!existing.manifest.challengerCandidateIds.includes(proposal.candidateId)) {
+                throw new Error(`proposed candidate ${proposal.candidateId} is not in campaign ${campaignId} challengers; recreate campaign with --campaign-manifest or add challenger ids`);
+            }
+        }
+    }
+    const matrixResult = await runCampaignMatrix({
+        repoRoot: input.repoRoot,
+        campaignId,
+        mode: input.matrixMode,
+        fixturePath: input.fixturePath,
+        maxPublicCells: input.maxPublicCells,
+        includePrivateChecks: true,
+        writeEvidenceDraft: true,
+        piExecutor: input.piExecutor,
+        now: input.now,
+    });
+    let scorecardBuilt = false;
+    let promotionEligible;
+    if (input.buildScorecard && matrixResult.evidence) {
+        // Scorecard from matrix draft alone is usually insufficient for held_out;
+        // still persist for inspection when requested.
+        try {
+            await buildScorecardFromMatrixEvidence({
+                repoRoot: input.repoRoot,
+                campaignId,
+                evidence: matrixResult.evidence,
+            });
+            scorecardBuilt = true;
+            const { readCampaignScorecard } = await import("../../infrastructure/evaluation/campaign-store.js");
+            const scorecard = await readCampaignScorecard(input.repoRoot, campaignId);
+            promotionEligible = scorecard?.policy.promotionEligible;
+        }
+        catch {
+            scorecardBuilt = false;
+        }
+    }
+    const result = {
+        schemaVersion: 1,
+        proposerRun,
+        campaignId,
+        matrix: matrixResult.matrix,
+        scorecardBuilt,
+        promotionEligible,
+        autoPromote: false,
+        next: [
+            `loop-agent eval campaign show --campaign-id ${campaignId} --json`,
+            `loop-agent eval campaign scorecard --campaign-id ${campaignId} --evidence .harness/evaluation/campaigns/${campaignId}/evidence-draft.json --json`,
+            `loop-agent eval promote --alias <name> --to-candidate <id> --campaign-id ${campaignId} --reason <text> --dry-run`,
+            "NEVER auto-promote: human --apply required after policy pass",
+        ],
+    };
+    return result;
+}
+export function formatOuterLoopMarkdown(result) {
+    const lines = [
+        `# Outer Loop: ${result.campaignId}`,
+        "",
+        `- autoPromote: \`${result.autoPromote}\``,
+        `- scorecardBuilt: ${result.scorecardBuilt}`,
+        `- promotionEligible: ${result.promotionEligible ?? "(n/a)"}`,
+        "",
+    ];
+    if (result.proposerRun) {
+        lines.push(formatProposerRunMarkdown(result.proposerRun));
+    }
+    lines.push(formatMatrixRunMarkdown(result.matrix));
+    lines.push("## Next", "");
+    for (const step of result.next) {
+        lines.push(`- ${step}`);
+    }
+    lines.push("");
+    return `${lines.join("\n")}\n`;
+}

package/dist/application/evaluation/pi-cell-executor.js

@@ -0,0 +1,39 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { runDagUseCase } from "../dag/run-dag.js";
+import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
+/**
+ * M7 live-Pi wiring (bounded): materialize a per-cell DagSpec with evaluation
+ * binding and run `runDagUseCase` in dry-run mode only.
+ *
+ * Never auto-promotes. Full live Pi execute remains opt-in via custom PiCellExecutor.
+ */
+export function createDagDryRunPiCellExecutor(input) {
+    return async (ctx) => {
+        const absoluteTemplate = path.resolve(ctx.repoRoot, input.dagTemplatePath);
+        const raw = JSON.parse(await readFile(absoluteTemplate, "utf-8"));
+        raw.version = 3;
+        raw.evaluation = ctx.evaluation;
+        raw.title =
+            typeof raw.title === "string"
+                ? `${raw.title} :: ${ctx.cell.cellId}`
+                : `eval-matrix-${ctx.cell.cellId}`;
+        const cellDagPath = path.join(ctx.repoRoot, path.dirname(ctx.jobPath), `${ctx.cell.cellId}.dag.json`);
+        await writeJsonAtomic(cellDagPath, raw, { repoRoot: ctx.repoRoot });
+        const result = await runDagUseCase({
+            repoRoot: ctx.repoRoot,
+            dagPath: cellDagPath,
+            cwd: ctx.repoRoot,
+            initOnly: false,
+            dryRun: true,
+        });
+        if (!("mode" in result) || result.mode !== "dry-run") {
+            throw new Error("createDagDryRunPiCellExecutor expected dry-run result; refuse non-dry execution");
+        }
+        return {
+            verifyPassed: false,
+            runId: result.runId,
+            detail: `pi dry-run prepared runDir=${result.runDir}; verifyPassed left false until live shell evidence`,
+        };
+    };
+}

package/dist/application/evaluation/private-verifier.js

@@ -0,0 +1,46 @@
+import { listPrivateVerifierIds, loadPrivateVerifierManifestFromPath, materializePrivateVerifierManifest, readPrivateVerifierManifest, registerPrivateVerifierManifest, } from "../../infrastructure/evaluation/private-verifier-store.js";
+export async function validatePrivateVerifier(input) {
+    const raw = await loadPrivateVerifierManifestFromPath(input.repoRoot, input.manifestPath);
+    const manifest = await materializePrivateVerifierManifest(raw);
+    return { manifest };
+}
+export async function registerPrivateVerifier(input) {
+    const validated = await validatePrivateVerifier(input);
+    return registerPrivateVerifierManifest({
+        repoRoot: input.repoRoot,
+        manifest: validated.manifest,
+    });
+}
+export async function showPrivateVerifier(input) {
+    const manifest = await readPrivateVerifierManifest(input.repoRoot, input.privateVerifierId);
+    return { manifest };
+}
+export async function listPrivateVerifiers(input) {
+    const ids = await listPrivateVerifierIds(input.repoRoot);
+    const rows = [];
+    for (const privateVerifierId of ids) {
+        const manifest = await readPrivateVerifierManifest(input.repoRoot, privateVerifierId);
+        rows.push({
+            privateVerifierId,
+            manifestHash: manifest.manifestHash,
+            checkCount: manifest.checks.length,
+        });
+    }
+    return rows;
+}
+export function formatPrivateVerifierMarkdown(manifest) {
+    return [
+        `# Private Verifier: ${manifest.privateVerifierId}`,
+        "",
+        `- manifestHash: \`${manifest.manifestHash}\``,
+        `- checks: ${manifest.checks.length}`,
+        `- createdAt: \`${manifest.createdAt}\``,
+        "",
+        "> Controller-only. Must never enter candidate prompt, skill snapshot, or contentRefs.",
+        "",
+        "## Checks",
+        "",
+        ...manifest.checks.map((check) => `- \`${check.checkId}\` (${check.kind}): command arity=${check.command.length}, timeoutMs=${check.timeoutMs}`),
+        "",
+    ].join("\n");
+}

package/dist/application/evaluation/promotion-policy.js

@@ -0,0 +1,151 @@
+import { runIntegrityAudits } from "./integrity-audit.js";
+export const PROMOTION_POLICY_VERSION = "promotion-policy-v1";
+function emptyTally() {
+    return { incumbentWins: 0, challengerWins: 0, ties: 0 };
+}
+function pairKey(cell) {
+    return `${cell.split}\u0000${cell.taskRef}\u0000${cell.seed}`;
+}
+function tallyPaired(input) {
+    const byKey = new Map();
+    for (const cell of input.cells) {
+        if (cell.split !== input.split)
+            continue;
+        const key = pairKey(cell);
+        const row = byKey.get(key) ?? {};
+        if (cell.candidateId === input.incumbentCandidateId) {
+            row.incumbent = cell.verifyPassed;
+        }
+        else if (cell.candidateId === input.challengerCandidateId) {
+            row.challenger = cell.verifyPassed;
+        }
+        byKey.set(key, row);
+    }
+    const tally = emptyTally();
+    for (const row of byKey.values()) {
+        if (row.incumbent === undefined || row.challenger === undefined)
+            continue;
+        if (row.challenger === row.incumbent) {
+            tally.ties += 1;
+        }
+        else if (row.challenger && !row.incumbent) {
+            tally.challengerWins += 1;
+        }
+        else {
+            tally.incumbentWins += 1;
+        }
+    }
+    return tally;
+}
+function privateImprovementStatus(tally) {
+    const compared = tally.incumbentWins + tally.challengerWins + tally.ties;
+    if (compared === 0)
+        return "insufficient";
+    if (tally.challengerWins > tally.incumbentWins)
+        return "pass";
+    return "fail";
+}
+function heldOutNonRegressionStatus(tally) {
+    const compared = tally.incumbentWins + tally.challengerWins + tally.ties;
+    if (compared === 0)
+        return "insufficient";
+    // Non-regression: challenger must not lose more pairs than incumbent.
+    if (tally.challengerWins + tally.ties >= tally.incumbentWins)
+        return "pass";
+    return "fail";
+}
+function budgetStatusGate(status) {
+    if (status === "ok")
+        return "pass";
+    if (status === "breached")
+        return "fail";
+    return "insufficient";
+}
+function safetyStatus(findings) {
+    if (findings.some((f) => f.severity === "fail"))
+        return "fail";
+    if (findings.some((f) => f.severity === "review"))
+        return "review";
+    return "pass";
+}
+/**
+ * Evaluate Promotion Policy v1 (W5.1).
+ * Never auto-promotes; human gate remains mandatory at apply time.
+ */
+export function evaluatePromotionPolicy(input) {
+    if (input.evidence.campaignId !== input.campaignId) {
+        throw new Error(`evidence campaignId mismatch: expected ${input.campaignId}, got ${input.evidence.campaignId}`);
+    }
+    const findings = runIntegrityAudits({
+        evidence: input.evidence,
+        incumbentCandidateId: input.incumbentCandidateId,
+    });
+    const paired = {
+        private: tallyPaired({
+            incumbentCandidateId: input.incumbentCandidateId,
+            challengerCandidateId: input.evidence.challengerCandidateId,
+            cells: input.evidence.cells,
+            split: "private",
+        }),
+        held_out: tallyPaired({
+            incumbentCandidateId: input.incumbentCandidateId,
+            challengerCandidateId: input.evidence.challengerCandidateId,
+            cells: input.evidence.cells,
+            split: "held_out",
+        }),
+        public: tallyPaired({
+            incumbentCandidateId: input.incumbentCandidateId,
+            challengerCandidateId: input.evidence.challengerCandidateId,
+            cells: input.evidence.cells,
+            split: "public",
+        }),
+    };
+    const safetyOk = safetyStatus(findings);
+    const gates = {
+        privateImprovement: privateImprovementStatus(paired.private),
+        heldOutNonRegression: heldOutNonRegressionStatus(paired.held_out),
+        budgetOk: budgetStatusGate(input.evidence.budgetStatus.challenger),
+        safetyOk,
+        humanRequired: true,
+    };
+    const reasons = [];
+    if (gates.privateImprovement !== "pass") {
+        reasons.push(`privateImprovement=${gates.privateImprovement} (challengerWins=${paired.private.challengerWins}, incumbentWins=${paired.private.incumbentWins}, ties=${paired.private.ties})`);
+    }
+    if (gates.heldOutNonRegression !== "pass") {
+        reasons.push(`heldOutNonRegression=${gates.heldOutNonRegression} (challengerWins=${paired.held_out.challengerWins}, incumbentWins=${paired.held_out.incumbentWins}, ties=${paired.held_out.ties})`);
+    }
+    if (gates.budgetOk !== "pass") {
+        reasons.push(`budgetOk=${gates.budgetOk}`);
+    }
+    if (gates.safetyOk === "fail") {
+        reasons.push(`safetyOk=fail (${findings.filter((f) => f.severity === "fail").map((f) => f.code).join(", ")})`);
+    }
+    else if (gates.safetyOk === "review") {
+        if (input.acknowledgeReview) {
+            reasons.push(`safetyOk=review acknowledged via --acknowledge-review (${findings.filter((f) => f.severity === "review").map((f) => f.code).join(", ")})`);
+        }
+        else {
+            reasons.push(`safetyOk=review (${findings.filter((f) => f.severity === "review").map((f) => f.code).join(", ")}); re-run with --acknowledge-review after human review`);
+        }
+    }
+    reasons.push("humanRequired=true (apply still needs --reason)");
+    const safetyAllows = gates.safetyOk === "pass" ||
+        (gates.safetyOk === "review" && Boolean(input.acknowledgeReview));
+    const promotionEligible = gates.privateImprovement === "pass" &&
+        gates.heldOutNonRegression === "pass" &&
+        gates.budgetOk === "pass" &&
+        safetyAllows;
+    return {
+        schemaVersion: 1,
+        policyVersion: PROMOTION_POLICY_VERSION,
+        campaignId: input.campaignId,
+        incumbentCandidateId: input.incumbentCandidateId,
+        challengerCandidateId: input.evidence.challengerCandidateId,
+        promotionEligible,
+        gates,
+        findings,
+        reasons,
+        paired,
+    };
+}

package/dist/application/evaluation/proposer.js

@@ -0,0 +1,98 @@
+import { createHash } from "node:crypto";
+import { writeProposerRun } from "../../infrastructure/evaluation/proposer-store.js";
+import { createExperimentFromCurate } from "./experiment.js";
+/**
+ * Controlled outer-loop proposer (M7).
+ * Only proposes bounded experimental candidates from knowledge-curate.
+ * Never modifies evaluator/kernel/private verifier and never auto-promotes.
+ */
+export async function proposeFromCurate(input) {
+    const maxCandidates = input.maxCandidates ?? 1;
+    if (maxCandidates < 1 || maxCandidates > 3) {
+        throw new Error("proposer maxCandidates must be between 1 and 3 (bounded)");
+    }
+    const dryRun = input.dryRun !== false;
+    const now = input.now ?? new Date().toISOString();
+    const proposerRunId = input.proposerRunId ??
+        `propose-${createHash("sha256").update(now).digest("hex").slice(0, 12)}`;
+    const proposals = [];
+    const skippedReasons = [];
+    if (dryRun) {
+        skippedReasons.push("dry-run: no candidate registered; re-run with --apply to materialize bounded proposals");
+        const run = {
+            schemaVersion: 1,
+            proposerRunId,
+            source: "knowledge-curate",
+            createdAt: now,
+            maxCandidates,
+            proposals,
+            skippedReasons,
+            autoPromote: false,
+        };
+        return run;
+    }
+    for (let i = 0; i < maxCandidates; i += 1) {
+        try {
+            const created = await createExperimentFromCurate({
+                repoRoot: input.repoRoot,
+                now,
+                candidateId: `proposed-${proposerRunId}-${i + 1}`,
+                experimentId: `exp-${proposerRunId}-${i + 1}`,
+            });
+            proposals.push({
+                candidateId: created.candidate.manifest.candidateId,
+                experimentId: created.record.experimentId,
+                proposalPath: created.record.proposalPath,
+                bundleHash: created.candidate.manifest.bundleHash,
+                status: "proposed_experimenting",
+            });
+        }
+        catch (error) {
+            skippedReasons.push(error instanceof Error ? error.message : String(error));
+            break;
+        }
+    }
+    const run = {
+        schemaVersion: 1,
+        proposerRunId,
+        source: "knowledge-curate",
+        createdAt: now,
+        maxCandidates,
+        proposals,
+        skippedReasons,
+        autoPromote: false,
+    };
+    await writeProposerRun({ repoRoot: input.repoRoot, run });
+    return run;
+}
+export function formatProposerRunMarkdown(run) {
+    const lines = [
+        `# Proposer Run: ${run.proposerRunId}`,
+        "",
+        `- source: \`${run.source}\``,
+        `- maxCandidates: ${run.maxCandidates}`,
+        `- proposals: ${run.proposals.length}`,
+        `- autoPromote: \`${run.autoPromote}\``,
+        "",
+        "## Proposals",
+        "",
+    ];
+    if (run.proposals.length === 0) {
+        lines.push("(none)", "");
+    }
+    else {
+        for (const proposal of run.proposals) {
+            lines.push(`- \`${proposal.candidateId}\` experiment=\`${proposal.experimentId}\` hash=\`${proposal.bundleHash}\``);
+        }
+        lines.push("");
+    }
+    if (run.skippedReasons.length > 0) {
+        lines.push("## Skipped / notes", "");
+        for (const reason of run.skippedReasons) {
+            lines.push(`- ${reason}`);
+        }
+        lines.push("");
+    }
+    lines.push("> M7: proposer only registers experimenting candidates; promotion remains human-gated.", "");
+    return `${lines.join("\n")}\n`;
+}