@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/project-workflow.test.js

@@ -4,11 +4,14 @@ import os from "node:os";
 import path from "node:path";
 import { after, test } from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { CostServiceLive, makeCostService } from "@velum-labs/routekit-registry";
 import { Cause, Effect, Layer } from "effect";
 import { EvalDimensionContrastError, EvalModelCatalogError } from "../errors.js";
+import { compileEvaluationEvidence, renderEvaluationEvidenceContext } from "../evaluation-evidence.js";
+import { renderEvaluationCriteria } from "../evaluation-structure-policy.js";
 import { EvalRepositoryInspectorLive } from "../inspection.js";
 import { EvalProjectArtifactsLive } from "../project-artifacts.js";
-import { scopedExpectedCalls, summarizeEvalRunLedger } from "../project-contracts.js";
+import { estimateEvalPlanCost, scopedExpectedCalls, summarizeEvalRunLedger } from "../project-contracts.js";
 import { EvalProjectStoreLive } from "../project-store.js";
 import { EvalProjectWorkflow, EvalProjectWorkflowLive } from "../project-workflow.js";
 import { EvalModelCatalog } from "../services/model-catalog/service.js";
@@ -28,12 +31,13 @@ const catalogEfforts = (candidateModels) => Effect.sync(() => {
     };
 });
 const EvalModelCatalogTest = Layer.succeed(EvalModelCatalog, EvalModelCatalog.of({ reasoningEfforts: catalogEfforts }));
-const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(EvalModelCatalogTest), Layer.provide(NodeServicesLayer));
+const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(EvalModelCatalogTest), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));
+const testCostService = makeCostService();
 const makeRepository = async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-project-"));
     roots.push(root);
     await mkdir(path.join(root, "src"), { recursive: true });
-    await writeFile(path.join(root, "README.md"), "# Example application\n");
+    await writeFile(path.join(root, "README.md"), "# Example application\n\nA terminal event must be last; later output is invalid and unsupported.\n");
     await writeFile(path.join(root, "src", "model.ts"), 'client.responses.create({ model: "openai/current" });\n');
     return root;
 };
@@ -162,7 +166,7 @@ test("candidate selection fails closed when catalog reasoning metadata is unavai
             detail: "models.list did not advertise candidate reasoning capabilities"
         }))
     }));
-    const workflowLayer = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(unavailableCatalog), Layer.provide(NodeServicesLayer));
+    const workflowLayer = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(unavailableCatalog), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));
     await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
         yield* workflow.setup(root);
@@ -203,15 +207,89 @@ const reviewedProposedDimensions = reviewedDimensions.map((dimension, index) =>
     inScopeRequest: `Solve an in-scope request for dimension ${String(index + 1)}.`,
     nearMissRequest: `Solve a neighboring request outside dimension ${String(index + 1)}.`
 }));
+const reviewedPrompt = (dimensionId, index) => {
+    switch (index % 5) {
+        case 0:
+            return `Transform raw ${dimensionId} capture ${String(index + 1)}`;
+        case 1:
+            return `Locate the fault in ${dimensionId} trace ${String(index + 1)}`;
+        case 2:
+            return `Construct the ${dimensionId} artifact for sample ${String(index + 1)}`;
+        case 3:
+            return `Analyze the ${dimensionId} boundary sample ${String(index + 1)}`;
+        default:
+            return `Explain the ${dimensionId} operational result ${String(index + 1)}`;
+    }
+};
+const reviewedEvidenceSources = compileEvaluationEvidence({
+    sources: [
+        {
+            path: "README.md",
+            content: "# Example application\n\nA terminal event must be last; later output is invalid and unsupported.\n"
+        },
+        {
+            path: "src/model.ts",
+            content: 'client.responses.create({ model: "openai/current" });\n'
+        }
+    ],
+    maximumBlockBytes: 3_000
+});
+const reviewedEvidenceBlock = reviewedEvidenceSources[0].blocks[0];
+const reviewedCompiledFields = (caseId, index) => {
+    const criterionId = `${caseId}-criterion`;
+    const criteria = [
+        {
+            id: criterionId,
+            description: `Derive the observable repository behavior for case ${String(index + 1)}.`,
+            importance: "critical",
+            gradingMode: "semantic"
+        }
+    ];
+    return {
+        context: renderEvaluationEvidenceContext([reviewedEvidenceBlock]),
+        rubric: renderEvaluationCriteria(criteria),
+        evidenceIds: [reviewedEvidenceBlock.id],
+        criteria,
+        positiveControls: [
+            {
+                id: `${caseId}-positive`,
+                response: `The response derives the repository-backed behavior for case ${String(index + 1)}.`
+            }
+        ],
+        negativeControls: [
+            {
+                id: `${caseId}-missing`,
+                response: "The response discusses the topic but omits the required behavior.",
+                targetedCriterionIds: [criterionId],
+                kind: "missing-criterion"
+            },
+            {
+                id: `${caseId}-forbidden`,
+                response: "The response invents behavior contradicted by the repository evidence.",
+                targetedCriterionIds: [criterionId],
+                kind: "forbidden-behavior"
+            },
+            {
+                id: `${caseId}-non-responsive`,
+                response: "This is a polished overview that never answers the concrete request.",
+                targetedCriterionIds: [criterionId],
+                kind: "non-responsive"
+            }
+        ]
+    };
+};
 const reviewedSuites = reviewedDimensions.map((dimension) => ({
     version: 1,
     dimensionId: dimension.id,
     maximumOutputTokens: 256,
-    cases: Array.from({ length: 20 }, (_, index) => ({
-        id: `${dimension.id}-case-${String(index + 1)}`,
-        prompt: `Solve ${dimension.id} case ${String(index + 1)}`,
-        rubric: `The response must satisfy ${dimension.id} case ${String(index + 1)}`
-    }))
+    cases: Array.from({ length: 20 }, (_, index) => {
+        const id = `${dimension.id}-case-${String(index + 1)}`;
+        return {
+            id,
+            prompt: reviewedPrompt(dimension.id, index),
+            ...reviewedCompiledFields(id, index)
+        };
+    })
 }));
 const reviewedDecompositionBenchmark = {
     maximumVectorL1Error: 0.3,
@@ -231,32 +309,53 @@ const reviewedCompositionSuite = {
     maximumOutputTokens: 256,
     minimumWinnerScoreGap: 0.05,
     minimumWinnerAgreement: 0.8,
-    cases: Array.from({ length: 20 }, (_, index) => ({
-        id: `composition-case-${String(index + 1)}`,
-        prompt: `Solve multi-dimension production case ${String(index + 1)}`,
-        rubric: `The response must satisfy multi-dimension case ${String(index + 1)}`,
-        decomposition: {
-            weights: reviewedDimensions.map((dimension, dimensionIndex) => ({
-                dimensionId: dimension.id,
-                weight: dimensionIndex === index % reviewedDimensions.length ||
-                    dimensionIndex === (index + 1) % reviewedDimensions.length
-                    ? 0.5
-                    : 0
-            })),
-            unknownWeight: 0
-        },
-        requirements: {
-            endpoint: "responses",
-            requiresTools: false,
-            requiresVision: false
-        }
-    }))
+    cases: Array.from({ length: 20 }, (_, index) => {
+        const id = `composition-case-${String(index + 1)}`;
+        return {
+            id,
+            prompt: reviewedPrompt("multi-dimension", index),
+            ...reviewedCompiledFields(id, index),
+            decomposition: {
+                weights: reviewedDimensions.map((dimension, dimensionIndex) => ({
+                    dimensionId: dimension.id,
+                    weight: dimensionIndex === index % reviewedDimensions.length ||
+                        dimensionIndex === (index + 1) % reviewedDimensions.length
+                        ? 0.5
+                        : 0
+                })),
+                unknownWeight: 0
+            },
+            requirements: {
+                endpoint: "responses",
+                requiresTools: false,
+                requiresVision: false
+            }
+        };
+    })
 };
 const reviewedEvaluationInput = {
+    evidenceSources: reviewedEvidenceSources,
     suites: reviewedSuites,
     decompositionBenchmark: reviewedDecompositionBenchmark,
     compositionSuite: reviewedCompositionSuite
 };
+const flashcardEvaluationInput = {
+    ...reviewedEvaluationInput,
+    suites: reviewedSuites.map((suite) => ({
+        ...suite,
+        cases: Array.from({ length: 20 }, (_, index) => {
+            const scenario = Math.floor(index / 2);
+            return {
+                id: `${suite.dimensionId}-flashcard-${String(index + 1)}`,
+                prompt: index % 2 === 0
+                    ? "Diagnose the described RouteKit behavior. Use only the supplied context."
+                    : "Give a concise verification criterion. Use only the supplied context.",
+                context: `Captured scenario ${String(scenario + 1)} emitted output after completion. The terminal event must be last.`,
+                rubric: "Reject post-terminal output and require completion to be the final event."
+            };
+        })
+    }))
+};
 async function completeProjectSetup(root, candidateModels = "openai/gpt-5.6-luna openai/gpt-5.6-terra openai/gpt-5.6-sol") {
     await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
@@ -356,7 +455,8 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
         assert.equal(ready.state.stage, "ready");
         assert.equal(ready.nextAction, "run");
         const plan = yield* workflow.createPlan(root, "pilot");
-        return { basisDigest, evaluationDigest, plan };
+        const repeatedPlan = yield* workflow.createPlan(root, "pilot", 2);
+        return { basisDigest, evaluationDigest, plan, repeatedPlan };
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
     const storedBasis = JSON.parse(await readFile(path.join(root, ".routekit", "evals", "routing-basis.proposed.json"), "utf8"));
     assert.equal(storedBasis.dimensionContrasts.length, reviewedDimensions.length);
@@ -374,20 +474,47 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
     assert.equal(result.plan.expectedJudgeCalls, 90);
     assert.equal(result.plan.expectedCallCount, 185);
     assert.equal(result.plan.maximumOutputTokens, 256);
+    assert.equal(result.plan.repetitions, 1);
+    assert.equal(result.repeatedPlan.repetitions, 2);
+    assert.equal(result.repeatedPlan.expectedCallCount, result.plan.expectedCallCount * 2);
+    assert.equal(result.plan.costEstimate !== undefined, true);
+    assert.equal(result.repeatedPlan.costEstimate !== undefined, true);
+    assert.equal(result.repeatedPlan.costEstimate?.knownPricedMaximumUsd, (result.plan.costEstimate?.knownPricedMaximumUsd ?? 0) * 2);
     const planPath = path.join(root, ".routekit", "evals", "plans", `${result.plan.planId}.json`);
     assert.equal((await stat(planPath)).mode & 0o777, 0o600);
     assert.deepEqual(JSON.parse(await readFile(planPath, "utf8")), result.plan);
     const firstDimension = reviewedDimensions[0];
     const generatedSuite = await readFile(path.join(root, ".routekit", "evals", "plans", result.plan.planId, "dimensions", firstDimension.id, `${firstDimension.id}.eval.ts`), "utf8");
+    const candidateCases = JSON.parse(await readFile(path.join(root, ".routekit", "evals", "plans", result.plan.planId, "dimensions", firstDimension.id, "data", "candidate-cases.json"), "utf8"));
+    const hiddenOracles = JSON.parse(await readFile(path.join(root, ".routekit", "evals", "plans", result.plan.planId, "dimensions", firstDimension.id, "data", "hidden-oracles.json"), "utf8"));
     const completionAssertion = generatedSuite.indexOf("run.toComplete();");
+    const oracleLookup = generatedSuite.indexOf("hiddenOracleById.get(candidateCase.id)");
     const judgeCall = generatedSuite.indexOf("await judge.autoEvals");
     const completionRethrow = generatedSuite.indexOf("throw candidateCompletionError;");
     const judgeRethrow = generatedSuite.indexOf("throw judgeError;");
     assert.match(generatedSuite, /emit the complete target-protocol envelope and terminal event/u);
     assert.match(generatedSuite, /include the concrete output rather than only describing it/u);
+    assert.match(generatedSuite, /You are a strict evaluator for AI agent outputs\./u);
+    assert.deepEqual(Object.keys(candidateCases[0]).sort(), [
+        "context",
+        "id",
+        "prompt"
+    ]);
+    assert.equal(Object.hasOwn(candidateCases[0], "rubric"), false);
+    assert.equal(Object.hasOwn(candidateCases[0], "criteria"), false);
+    assert.deepEqual(Object.keys(hiddenOracles[0]).sort(), [
+        "criteria",
+        "id",
+        "negativeControls",
+        "positiveControls",
+        "rubric"
+    ]);
+    assert.equal(candidateCases[0].id, hiddenOracles[0].id);
     assert.ok(judgeCall >= 0);
+    assert.ok(oracleLookup >= 0);
     assert.ok(completionAssertion >= 0);
-    assert.ok(completionAssertion < judgeCall);
+    assert.ok(completionAssertion < oracleLookup);
+    assert.ok(oracleLookup < judgeCall);
     assert.ok(judgeCall < completionRethrow);
     assert.ok(completionRethrow < judgeRethrow);
     for (const dimension of reviewedDimensions) {
@@ -395,6 +522,95 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
             .mode & 0o777, 0o600);
     }
 });
+test("evaluation approval rejects historical cases without compiler provenance", async () => {
+    const root = await makeRepository();
+    await completeProjectSetup(root);
+    const exit = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const proposedBasis = yield* workflow.proposeDimensions(root, reviewedProposedDimensions);
+        yield* workflow.approveDimensions(root, proposedBasis.artifacts.basisProposalDigest);
+        const proposed = yield* workflow.proposeEvaluations(root, flashcardEvaluationInput);
+        return yield* Effect.exit(workflow.approveEvaluations(root, proposed.artifacts.evaluationProposalDigest));
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(exit._tag, "Failure");
+    if (exit._tag === "Failure") {
+        const error = Cause.squash(exit.cause);
+        assert.match(String(error), /is not compiler-backed/u);
+    }
+    const approvalPath = path.join(root, ".routekit", "evals", "evaluations.approved.json");
+    await assert.rejects(stat(approvalPath), /ENOENT/u);
+});
+test("repository evidence changes stale approval before approval and planning", async () => {
+    const beforeApprovalRoot = await makeRepository();
+    await completeProjectSetup(beforeApprovalRoot);
+    const beforeApproval = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const proposedBasis = yield* workflow.proposeDimensions(beforeApprovalRoot, reviewedProposedDimensions);
+        yield* workflow.approveDimensions(beforeApprovalRoot, proposedBasis.artifacts.basisProposalDigest);
+        const proposed = yield* workflow.proposeEvaluations(beforeApprovalRoot, reviewedEvaluationInput);
+        return proposed.artifacts.evaluationProposalDigest;
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    await writeFile(path.join(beforeApprovalRoot, "README.md"), "# Example application\n\nChanged after authoring.\n");
+    const approvalExit = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        return yield* Effect.exit(workflow.approveEvaluations(beforeApprovalRoot, beforeApproval));
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(approvalExit._tag, "Failure");
+    if (approvalExit._tag === "Failure") {
+        assert.match(String(Cause.squash(approvalExit.cause)), /evidence source changed after authoring/u);
+    }
+    const beforePlanRoot = await makeRepository();
+    await completeProjectSetup(beforePlanRoot);
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const proposedBasis = yield* workflow.proposeDimensions(beforePlanRoot, reviewedProposedDimensions);
+        yield* workflow.approveDimensions(beforePlanRoot, proposedBasis.artifacts.basisProposalDigest);
+        const proposed = yield* workflow.proposeEvaluations(beforePlanRoot, reviewedEvaluationInput);
+        yield* workflow.approveEvaluations(beforePlanRoot, proposed.artifacts.evaluationProposalDigest);
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    await writeFile(path.join(beforePlanRoot, "README.md"), "# Example application\n\nChanged before planning.\n");
+    const stale = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const status = yield* workflow.status(beforePlanRoot);
+        const planExit = yield* Effect.exit(workflow.createPlan(beforePlanRoot, "pilot"));
+        return { status, planExit };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(stale.status?.artifacts?.evaluationsApproved, false);
+    assert.equal(stale.status?.nextAction, "approve-evaluations");
+    assert.equal(stale.planExit._tag, "Failure");
+    if (stale.planExit._tag === "Failure") {
+        assert.match(String(stale.planExit.cause), /evidence source changed after authoring/u);
+    }
+});
+test("ready projects can explicitly rebind stale evaluation approvals", async () => {
+    const root = await makeRepository();
+    await completeProjectSetup(root);
+    const rebound = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const proposedBasis = yield* workflow.proposeDimensions(root, reviewedProposedDimensions);
+        yield* workflow.approveDimensions(root, proposedBasis.artifacts.basisProposalDigest);
+        const proposed = yield* workflow.proposeEvaluations(root, reviewedEvaluationInput);
+        const digest = proposed.artifacts.evaluationProposalDigest;
+        const ready = yield* workflow.approveEvaluations(root, digest);
+        yield* Effect.promise(() => writeFile(path.join(root, ".routekit", "evals", "evaluations.approval.json"), `${JSON.stringify({
+            version: 1,
+            kind: "evaluations",
+            digest,
+            approvedAt: "2026-08-30T00:00:00.000Z"
+        })}\n`, { mode: 0o600 }));
+        const stale = yield* workflow.status(root);
+        const reboundStatus = yield* workflow.approveEvaluations(root, digest);
+        return { ready, stale, reboundStatus };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(rebound.ready.state.stage, "ready");
+    assert.ok(rebound.stale !== undefined);
+    assert.equal(rebound.stale.state.stage, "ready");
+    assert.equal(rebound.stale.artifacts?.evaluationsApproved, false);
+    assert.equal(rebound.stale.nextAction, "approve-evaluations");
+    assert.equal(rebound.reboundStatus.state.stage, "ready");
+    assert.equal(rebound.reboundStatus.artifacts?.evaluationsApproved, true);
+    assert.equal(rebound.reboundStatus.nextAction, "run");
+});
 test("full qualification refuses fewer than twenty reviewed cases per dimension", async () => {
     const root = await makeRepository();
     await completeProjectSetup(root);
@@ -403,6 +619,7 @@ test("full qualification refuses fewer than twenty reviewed cases per dimension"
         const proposed = yield* workflow.proposeDimensions(root, reviewedProposedDimensions);
         yield* workflow.approveDimensions(root, proposed.artifacts.basisProposalDigest);
         const evaluations = yield* workflow.proposeEvaluations(root, {
+            evidenceSources: reviewedEvidenceSources,
             suites: reviewedSuites.map((suite) => ({ ...suite, cases: suite.cases.slice(0, 5) })),
             decompositionBenchmark: {
                 ...reviewedDecompositionBenchmark,
@@ -555,18 +772,25 @@ const qualifiedReport = (plan, runId, projectId, publishAllowed = true) => {
                 expectedCases: plan.selectedCompositionCaseIds.length,
                 comparableCases: plan.selectedCompositionCaseIds.length,
                 agreeingCases: plan.selectedCompositionCaseIds.length,
-                winnerAgreement: 1,
+                ...(plan.selectedCompositionCaseIds.length === 0
+                    ? {}
+                    : { winnerAgreement: 1 }),
+                conclusion: plan.selectedCompositionCaseIds.length === 0 ? "inconclusive" : "passed",
+                publishable: plan.scope === "full" && plan.selectedCompositionCaseIds.length > 0,
                 cases: plan.selectedCompositionCaseIds.map((caseId) => ({
                     caseId,
                     predictedWinner: candidateModels[0],
+                    predictedTiedModels: [],
                     observedWinner: candidateModels[0],
+                    observedTiedModels: [],
                     predictedScoreGap: 0.1,
                     observedScoreGap: 0.2,
+                    comparable: true,
                     passed: true
                 }))
             }
         },
-        ledger: summarizeEvalRunLedger(comparisons, plan.expectedCallCount, observations),
+        ledger: Effect.runSync(summarizeEvalRunLedger(comparisons, plan.expectedCallCount, testCostService, observations, plan.classifierModel)),
         activation: {
             version: 2,
             generatedAt: "2026-08-18T00:02:00.000Z",
@@ -590,7 +814,8 @@ const qualifiedReport = (plan, runId, projectId, publishAllowed = true) => {
                 failureRate: 0,
                 averageJudgeScore: 0.9,
                 p95DurationMs: 250,
-                unpricedCalls: plan.scope === "pilot" ? 5 : 20
+                averageCostUsd: 0.01,
+                unpricedCalls: 0
             })))
         }
     };
@@ -624,7 +849,7 @@ test("run lifecycle retains complete sanitized evidence and activates only after
     assert.equal(outcome.loaded?.status, "passed");
     assert.equal(outcome.loaded?.ledger.observedCalls, 740);
     assert.equal(outcome.loaded?.ledger.observedCandidateRows, 360);
-    assert.equal(outcome.loaded?.ledger.unpricedCalls, 740);
+    assert.equal(outcome.loaded?.ledger.unpricedCalls, 0);
     assert.equal(outcome.activated.state.stage, "activated");
     const reportPath = path.join(root, ".routekit", "evals", "runs", outcome.started.runId, "report.json");
     assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
@@ -633,8 +858,9 @@ test("finishRun derives ledger expectations from the selected case slice", async
     const root = await makeRepository();
     const created = await createPlan(root, "full", "openai/gpt-5.6-luna openai/gpt-5.6-terra");
     const selectedDimension = created.selectedCaseIds[0];
-    const stalePlan = {
+    const stalePlanInput = {
         ...created,
+        repetitions: 1,
         selectedCaseIds: [
             {
                 dimensionId: selectedDimension.dimensionId,
@@ -652,6 +878,10 @@ test("finishRun derives ledger expectations from the selected case slice", async
         expectedJudgeCalls: 3_400,
         expectedCallCount: 6_820
     };
+    const stalePlan = {
+        ...stalePlanInput,
+        costEstimate: Effect.runSync(estimateEvalPlanCost(stalePlanInput, testCostService))
+    };
     await writeFile(path.join(root, ".routekit", "evals", "plans", `${created.planId}.json`), `${JSON.stringify(stalePlan)}\n`);
     const outcome = await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
@@ -671,7 +901,7 @@ test("finishRun derives ledger expectations from the selected case slice", async
         });
         const scopedReport = {
             ...report,
-            ledger: summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount)
+            ledger: yield* summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount, testCostService)
         };
         const finished = yield* workflow.finishRun(root, scopedReport);
         return { finished, report: scopedReport };
@@ -681,6 +911,98 @@ test("finishRun derives ledger expectations from the selected case slice", async
     assert.equal(outcome.report.ledger.observedCalls, 4);
     assert.equal(outcome.report.ledger.observedCandidateRows, 2);
 });
+test("zero-comparable pilot persists a completed inconclusive diagnostic", async () => {
+    const root = await makeRepository();
+    const plan = await createPlan(root, "pilot");
+    const outcome = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const started = yield* workflow.startRun(root, plan.planId);
+        const status = yield* workflow.status(root);
+        const passing = qualifiedReport(plan, started.runId, status.state.projectId);
+        assert.equal(passing.status, "passed");
+        if (passing.status !== "passed")
+            return undefined;
+        const report = {
+            ...passing,
+            status: "completed",
+            qualification: {
+                ...passing.qualification,
+                composition: {
+                    expectedCases: plan.selectedCompositionCaseIds.length,
+                    comparableCases: 0,
+                    agreeingCases: 0,
+                    conclusion: "inconclusive",
+                    reason: "insufficient top-two separation",
+                    publishable: false,
+                    cases: plan.selectedCompositionCaseIds.map((caseId) => ({
+                        caseId,
+                        predictedWinner: null,
+                        predictedTiedModels: [...plan.candidateModels],
+                        observedWinner: null,
+                        observedTiedModels: [...plan.candidateModels],
+                        predictedScoreGap: 0,
+                        observedScoreGap: 0,
+                        comparable: false,
+                        passed: false
+                    }))
+                }
+            }
+        };
+        const finished = yield* workflow.finishRun(root, report);
+        const persisted = yield* workflow.result(root, started.runId);
+        return { finished, persisted };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(outcome?.finished.state.stage, "ready");
+    assert.equal(outcome?.persisted?.status, "completed");
+    assert.equal(outcome?.persisted?.qualification?.composition.expectedCases, 5);
+    assert.equal(outcome?.persisted?.qualification?.composition.comparableCases, 0);
+    assert.equal(outcome?.persisted?.qualification?.composition.winnerAgreement, undefined);
+    assert.equal(outcome?.persisted?.qualification?.composition.conclusion, "inconclusive");
+    assert.equal(outcome?.persisted?.qualification?.composition.publishable, false);
+});
+test("zero-comparable full qualification remains fail closed and unpublished", async () => {
+    const root = await makeRepository();
+    const plan = await createPlan(root, "full");
+    const outcome = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const started = yield* workflow.startRun(root, plan.planId);
+        const status = yield* workflow.status(root);
+        const passing = qualifiedReport(plan, started.runId, status.state.projectId);
+        assert.equal(passing.status, "passed");
+        if (passing.status !== "passed")
+            return undefined;
+        const report = {
+            ...passing,
+            qualification: {
+                ...passing.qualification,
+                composition: {
+                    expectedCases: plan.selectedCompositionCaseIds.length,
+                    comparableCases: 0,
+                    agreeingCases: 0,
+                    conclusion: "inconclusive",
+                    reason: "insufficient top-two separation",
+                    publishable: false,
+                    cases: plan.selectedCompositionCaseIds.map((caseId) => ({
+                        caseId,
+                        predictedWinner: null,
+                        predictedTiedModels: [...plan.candidateModels],
+                        observedWinner: null,
+                        observedTiedModels: [...plan.candidateModels],
+                        predictedScoreGap: 0,
+                        observedScoreGap: 0,
+                        comparable: false,
+                        passed: false
+                    }))
+                }
+            }
+        };
+        const finished = yield* Effect.exit(workflow.finishRun(root, report));
+        const current = yield* workflow.status(root);
+        return { finished, current };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(outcome?.finished._tag, "Failure");
+    assert.equal(outcome?.current?.state.stage, "running");
+});
 test("failed runs preserve plan revisions for retry while mismatched plans remain stale", async () => {
     const root = await makeRepository();
     const plan = await createPlan(root, "full");
@@ -738,8 +1060,36 @@ test("failed runs preserve plan revisions for retry while mismatched plans remai
                 knownOutputTokens: 0,
                 unknownTokenMeasurements: 0,
                 knownPricedSubtotalUsd: 0,
-                unpricedCalls: 0
+                unpricedCalls: 0,
+                subscriptionUnitCalls: 0
             },
+            events: [
+                {
+                    name: "plan",
+                    at: "2026-08-21T03:47:27.000Z",
+                    runId: first.runId,
+                    plannedCalls: plan.expectedCallCount
+                },
+                {
+                    name: "failure-detail",
+                    at: "2026-08-21T03:47:48.000Z",
+                    runId: first.runId,
+                    phase: "target",
+                    reason: "qualification timed out before observing any calls",
+                    retryable: true,
+                    accounting: {
+                        expectedCalls: plan.expectedCallCount,
+                        observedCalls: 0,
+                        knownInputTokens: 0,
+                        knownOutputTokens: 0,
+                        unknownTokenMeasurements: 0,
+                        knownPricedSubtotalUsd: 0,
+                        unpricedCalls: 0,
+                        subscriptionUnitCalls: 0
+                    },
+                    cleanup: { sessionOpened: true, sessionClosed: true }
+                }
+            ],
             failure: "qualification timed out before observing any calls",
             errors: []
         });
@@ -764,7 +1114,7 @@ test("startRun rematerializes stale dimension and composition suite templates",
     const dimensionManifestPath = path.join(dimensionRoot, "routekit.eval-manifest.json");
     const compositionManifestPath = path.join(compositionRoot, "routekit.eval-manifest.json");
     const staleTemplate = `// generated by RouteKit 1.0.21
-await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+await judge.autoEvals({ criteria: mixedCase.rubric, prompt, run });
 run.toComplete();
 `;
     await Promise.all([
@@ -793,6 +1143,7 @@ run.toComplete();
         assert.doesNotMatch(suite, /generated by RouteKit 1\.0\.21/u);
         assert.match(suite, /emit the complete target-protocol envelope and terminal event/u);
         assert.match(suite, /include the concrete output rather than only describing it/u);
+        assert.match(suite, /You are a strict evaluator for AI agent outputs\./u);
         assert.ok(judgeCall >= 0);
         assert.ok(completionAssertion >= 0);
         assert.ok(completionAssertion < judgeCall);
@@ -842,8 +1193,36 @@ test("failed run reports persist the nested qualification error chain", async ()
                 knownOutputTokens: 0,
                 unknownTokenMeasurements: 0,
                 knownPricedSubtotalUsd: 0,
-                unpricedCalls: 0
+                unpricedCalls: 0,
+                subscriptionUnitCalls: 0
             },
+            events: [
+                {
+                    name: "plan",
+                    at: "2026-08-21T03:47:27.000Z",
+                    runId: started.runId,
+                    plannedCalls: plan.expectedCallCount
+                },
+                {
+                    name: "failure-detail",
+                    at: "2026-08-21T03:47:48.000Z",
+                    runId: started.runId,
+                    phase: "comparison",
+                    reason: "RouteKit Eval comparison failed",
+                    retryable: false,
+                    accounting: {
+                        expectedCalls: plan.expectedCallCount,
+                        observedCalls: 0,
+                        knownInputTokens: 0,
+                        knownOutputTokens: 0,
+                        unknownTokenMeasurements: 0,
+                        knownPricedSubtotalUsd: 0,
+                        unpricedCalls: 0,
+                        subscriptionUnitCalls: 0
+                    },
+                    cleanup: { sessionOpened: true, sessionClosed: true }
+                }
+            ],
             failure: 'qualification dimension "provider-protocol-translation" failed ' +
                 "(per-test timeout 600000ms); observed call ids: none",
             errors: [

package/dist/test/repository-authoring-artifacts.test.d.ts

@@ -0,0 +1 @@
+export {};