@velum-labs/routekit-eval-setup 1.0.23 → 1.0.25

package/dist/index.d.ts

@@ -7,8 +7,8 @@ export type { OriEvalAuthoringApi, OriEvalResult } from "./ori-result.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export type { EvalAuthoringCompletion, EvalAuthoringSource, EvalAuthoringTransportShape, EvalProjectAuthorShape } from "./project-authoring.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export type { EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionContrast, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectArtifactsStatus, EvalProjectStatus, EvalRoutingBasisProposal, EvalRunCleanup, EvalRunFailureError, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget } from "./project-contracts.js";
-export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, summarizeEvalRunLedger } from "./project-contracts.js";
+export type { EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionContrast, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectArtifactsStatus, EvalProjectStatus, EvalRoutingBasisProposal, EvalRunCleanup, EvalRunFailureError, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget, EvalScopedExpectedCalls } from "./project-contracts.js";
+export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export type { EvalProjectWorkflowError, EvalProjectWorkflowShape } from "./project-workflow.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";

package/dist/index.js

@@ -4,7 +4,7 @@ export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, summarizeEvalRunLedger } from "./project-contracts.js";
+export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";
 export { questionForStage, withOpenQuestion } from "./questions.js";

package/dist/project-artifacts.js

@@ -30,16 +30,20 @@ for (const model of manifest.candidateModels) {
         ? testCase.prompt
         : [testCase.prompt, "", "Reference material:", "-----", testCase.context, "-----"].join("\\n");
       const run = await candidate.run({ prompt, caseId: testCase.id });
-      let candidateCompletionError: unknown;
+      let candidateCompletionError;
       try {
         run.toComplete();
       } catch (error) {
         candidateCompletionError = error;
       }
-      await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
-      if (candidateCompletionError !== undefined) {
-        throw candidateCompletionError;
+      let judgeError;
+      try {
+        await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+      } catch (error) {
+        judgeError = error;
       }
+      if (candidateCompletionError !== undefined) throw candidateCompletionError;
+      if (judgeError !== undefined) throw judgeError;
     });
   }
 }

package/dist/project-contracts.d.ts

@@ -688,6 +688,17 @@ export declare const EvalExecutionPlan: Schema.Struct<{
     readonly expectedCallCount: Schema.Finite;
 }>;
 export type EvalExecutionPlan = typeof EvalExecutionPlan.Type;
+export type EvalScopedExpectedCalls = {
+    readonly expectedDimensionCandidateCalls: number;
+    readonly expectedDimensionJudgeCalls: number;
+    readonly expectedClassifierCalls: number;
+    readonly expectedCompositionCandidateCalls: number;
+    readonly expectedCompositionJudgeCalls: number;
+    readonly expectedCandidateCalls: number;
+    readonly expectedJudgeCalls: number;
+    readonly expectedCallCount: number;
+};
+export declare function scopedExpectedCalls(plan: Pick<EvalExecutionPlan, "candidateModels" | "selectedCaseIds" | "selectedDecompositionCaseIds" | "selectedCompositionCaseIds">): EvalScopedExpectedCalls;
 export declare const EvalRunTarget: Schema.Struct<{
     readonly kind: Schema.Literals<readonly ["configured", "external"]>;
     readonly identity: Schema.String;

package/dist/project-contracts.js

@@ -248,6 +248,27 @@ export const EvalExecutionPlan = Schema.Struct({
     expectedJudgeCalls: NonNegativeInteger,
     expectedCallCount: NonNegativeInteger
 });
+export function scopedExpectedCalls(plan) {
+    const candidateModelCount = plan.candidateModels.length;
+    const dimensionCaseCount = plan.selectedCaseIds.reduce((sum, selection) => sum + selection.caseIds.length, 0);
+    const expectedDimensionCandidateCalls = dimensionCaseCount * candidateModelCount;
+    const expectedDimensionJudgeCalls = expectedDimensionCandidateCalls;
+    const expectedClassifierCalls = plan.selectedDecompositionCaseIds.length;
+    const expectedCompositionCandidateCalls = plan.selectedCompositionCaseIds.length * candidateModelCount;
+    const expectedCompositionJudgeCalls = expectedCompositionCandidateCalls;
+    const expectedCandidateCalls = expectedDimensionCandidateCalls + expectedCompositionCandidateCalls;
+    const expectedJudgeCalls = expectedDimensionJudgeCalls + expectedCompositionJudgeCalls;
+    return {
+        expectedDimensionCandidateCalls,
+        expectedDimensionJudgeCalls,
+        expectedClassifierCalls,
+        expectedCompositionCandidateCalls,
+        expectedCompositionJudgeCalls,
+        expectedCandidateCalls,
+        expectedJudgeCalls,
+        expectedCallCount: expectedCandidateCalls + expectedJudgeCalls + expectedClassifierCalls
+    };
+}
 export const EvalRunTarget = Schema.Struct({
     kind: Schema.Literals(["configured", "external"]),
     identity: Schema.String,

package/dist/project-workflow.js

@@ -4,7 +4,7 @@ import { Clock, Context, Effect, Layer, Path, Schema } from "effect";
 import { EvalDimensionContrastError, EvalProjectTransitionError } from "./errors.js";
 import { EvalRepositoryInspector } from "./inspection.js";
 import { EvalProjectArtifacts, evaluationProposalDigest, routingBasisDigest } from "./project-artifacts.js";
-import { EVAL_PROJECT_VERSION, summarizeEvalRunLedger } from "./project-contracts.js";
+import { EVAL_PROJECT_VERSION, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 import { EvalProjectStore } from "./project-store.js";
 const isoNow = Effect.map(Clock.currentTimeMillis, (millis) => new Date(millis).toISOString());
 const questionForProgress = (progress) => {
@@ -134,7 +134,9 @@ const sameLedger = (left, right) => left.expectedCalls === right.expectedCalls &
     left.knownPricedSubtotalUsd === right.knownPricedSubtotalUsd &&
     left.unpricedCalls === right.unpricedCalls;
 const validateQualifiedReport = (state, plan, proposal, report) => {
-    assertPublishedRoutingActivation(report.activation);
+    assertPublishedRoutingActivation(report.activation, {
+        dimensionIds: plan.selectedCaseIds.map((entry) => entry.dimensionId)
+    });
     if (report.activation.basisDigest !== state.basisDigest ||
         report.activation.classifierModel !== state.configuration.classifierModel ||
         !sameStrings(report.activation.candidateModels, plan.candidateModels)) {
@@ -207,55 +209,77 @@ const validateQualifiedReport = (state, plan, proposal, report) => {
         throw new Error("comparison results are missing a planned dimension");
     }
     const compositionComparisons = report.comparisons.filter((comparison) => comparison.profileId === "composition");
-    if (compositionComparisons.length !== 1 || compositionComparisons[0] === undefined) {
+    if (plan.selectedCompositionCaseIds.length === 0) {
+        if (compositionComparisons.length !== 0) {
+            throw new Error("comparison results do not cover the planned dimensions exactly once");
+        }
+    }
+    else if (compositionComparisons.length !== 1 || compositionComparisons[0] === undefined) {
         throw new Error("qualification requires exactly one composition comparison");
     }
-    validateComparison(compositionComparisons[0], new Set(plan.selectedCompositionCaseIds));
+    else {
+        validateComparison(compositionComparisons[0], new Set(plan.selectedCompositionCaseIds));
+    }
     const qualification = report.qualification;
     if (qualification === undefined) {
-        throw new Error("qualification report is missing decomposition and composition results");
-    }
-    const expectedDecompositionCases = new Set(plan.selectedDecompositionCaseIds);
-    const observedDecompositionCases = new Set();
-    for (const observation of qualification.decomposition.observations) {
-        if (!expectedDecompositionCases.has(observation.caseId) ||
-            observedDecompositionCases.has(observation.caseId) ||
-            !observation.passed ||
-            observation.vectorL1Error > proposal.decompositionBenchmark.maximumVectorL1Error) {
-            throw new Error("decomposition qualification is incomplete, duplicated, or failed");
-        }
-        observedDecompositionCases.add(observation.caseId);
-    }
-    if (qualification.decomposition.expectedCases !== expectedDecompositionCases.size ||
-        qualification.decomposition.passedCases !== expectedDecompositionCases.size ||
-        observedDecompositionCases.size !== expectedDecompositionCases.size ||
-        qualification.decomposition.maximumObservedL1Error >
-            proposal.decompositionBenchmark.maximumVectorL1Error) {
-        throw new Error("decomposition qualification summary does not match the execution plan");
-    }
-    const expectedCompositionCases = new Set(plan.selectedCompositionCaseIds);
-    const observedCompositionCases = new Set();
-    for (const result of qualification.composition.cases) {
-        if (!expectedCompositionCases.has(result.caseId) ||
-            observedCompositionCases.has(result.caseId)) {
-            throw new Error("composition qualification contains an unknown or duplicate case");
-        }
-        observedCompositionCases.add(result.caseId);
+        if (plan.selectedCompositionCaseIds.length !== 0 ||
+            plan.selectedDecompositionCaseIds.length !== 0) {
+            throw new Error("qualification report is missing decomposition and composition results");
+        }
     }
-    if (qualification.composition.expectedCases !== expectedCompositionCases.size ||
-        observedCompositionCases.size !== expectedCompositionCases.size ||
-        qualification.composition.comparableCases < 1 ||
-        qualification.composition.agreeingCases !==
-            qualification.composition.cases.filter((result) => result.passed).length ||
-        Math.abs(qualification.composition.winnerAgreement -
-            qualification.composition.agreeingCases / qualification.composition.comparableCases) > 1e-6 ||
-        qualification.composition.winnerAgreement <
-            proposal.compositionSuite.minimumWinnerAgreement) {
-        throw new Error("composition qualification summary does not meet the reviewed threshold");
+    else {
+        const expectedDecompositionCases = new Set(plan.selectedDecompositionCaseIds);
+        const observedDecompositionCases = new Set();
+        for (const observation of qualification.decomposition.observations) {
+            if (!expectedDecompositionCases.has(observation.caseId) ||
+                observedDecompositionCases.has(observation.caseId) ||
+                !observation.passed ||
+                observation.vectorL1Error > proposal.decompositionBenchmark.maximumVectorL1Error) {
+                throw new Error("decomposition qualification is incomplete, duplicated, or failed");
+            }
+            observedDecompositionCases.add(observation.caseId);
+        }
+        if (qualification.decomposition.expectedCases !== expectedDecompositionCases.size ||
+            qualification.decomposition.passedCases !== expectedDecompositionCases.size ||
+            observedDecompositionCases.size !== expectedDecompositionCases.size ||
+            qualification.decomposition.maximumObservedL1Error >
+                proposal.decompositionBenchmark.maximumVectorL1Error) {
+            throw new Error("decomposition qualification summary does not match the execution plan");
+        }
+        const expectedCompositionCases = new Set(plan.selectedCompositionCaseIds);
+        const observedCompositionCases = new Set();
+        for (const result of qualification.composition.cases) {
+            if (!expectedCompositionCases.has(result.caseId) ||
+                observedCompositionCases.has(result.caseId)) {
+                throw new Error("composition qualification contains an unknown or duplicate case");
+            }
+            observedCompositionCases.add(result.caseId);
+        }
+        if (expectedCompositionCases.size === 0) {
+            if (qualification.composition.expectedCases !== 0 ||
+                observedCompositionCases.size !== 0 ||
+                qualification.composition.comparableCases !== 0 ||
+                qualification.composition.agreeingCases !== 0 ||
+                qualification.composition.cases.length !== 0) {
+                throw new Error("composition qualification summary does not meet the reviewed threshold");
+            }
+        }
+        else if (qualification.composition.expectedCases !== expectedCompositionCases.size ||
+            observedCompositionCases.size !== expectedCompositionCases.size ||
+            qualification.composition.comparableCases < 1 ||
+            qualification.composition.agreeingCases !==
+                qualification.composition.cases.filter((result) => result.passed).length ||
+            Math.abs(qualification.composition.winnerAgreement -
+                qualification.composition.agreeingCases / qualification.composition.comparableCases) > 1e-6 ||
+            qualification.composition.winnerAgreement <
+                proposal.compositionSuite.minimumWinnerAgreement) {
+            throw new Error("composition qualification summary does not meet the reviewed threshold");
+        }
     }
-    const ledger = summarizeEvalRunLedger(report.comparisons, plan.expectedCallCount, qualification.decomposition.observations);
-    if (ledger.observedCalls !== plan.expectedCallCount ||
-        ledger.observedCandidateRows !== plan.expectedCandidateCalls ||
+    const expected = scopedExpectedCalls(plan);
+    const ledger = summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount, qualification?.decomposition.observations ?? []);
+    if (ledger.observedCalls !== expected.expectedCallCount ||
+        ledger.observedCandidateRows !== expected.expectedCandidateCalls ||
         !sameLedger(ledger, report.ledger)) {
         throw new Error("qualification ledger does not match the completed evidence");
     }
@@ -718,20 +742,18 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
                 .slice(0, scope === "pilot" ? 5 : undefined)
                 .map((testCase) => testCase.id)
         }));
-        const caseCount = selectedCaseIds.reduce((sum, entry) => sum + entry.caseIds.length, 0);
         const selectedDecompositionCaseIds = proposal.decompositionBenchmark.cases
             .slice(0, scope === "pilot" ? 5 : undefined)
             .map((testCase) => testCase.id);
         const selectedCompositionCaseIds = proposal.compositionSuite.cases
             .slice(0, scope === "pilot" ? 5 : undefined)
             .map((testCase) => testCase.id);
-        const expectedDimensionCandidateCalls = caseCount * proposal.candidateModels.length;
-        const expectedDimensionJudgeCalls = expectedDimensionCandidateCalls;
-        const expectedClassifierCalls = selectedDecompositionCaseIds.length;
-        const expectedCompositionCandidateCalls = selectedCompositionCaseIds.length * proposal.candidateModels.length;
-        const expectedCompositionJudgeCalls = expectedCompositionCandidateCalls;
-        const expectedCandidateCalls = expectedDimensionCandidateCalls + expectedCompositionCandidateCalls;
-        const expectedJudgeCalls = expectedDimensionJudgeCalls + expectedCompositionJudgeCalls;
+        const expected = scopedExpectedCalls({
+            candidateModels: proposal.candidateModels,
+            selectedCaseIds,
+            selectedDecompositionCaseIds,
+            selectedCompositionCaseIds
+        });
         const plan = {
             version: EVAL_PROJECT_VERSION,
             planId: crypto.randomUUID(),
@@ -749,14 +771,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             selectedDecompositionCaseIds,
             selectedCompositionCaseIds,
             maximumOutputTokens: Math.max(proposal.compositionSuite.maximumOutputTokens, ...proposal.suites.map((suite) => suite.maximumOutputTokens)),
-            expectedDimensionCandidateCalls,
-            expectedDimensionJudgeCalls,
-            expectedClassifierCalls,
-            expectedCompositionCandidateCalls,
-            expectedCompositionJudgeCalls,
-            expectedCandidateCalls,
-            expectedJudgeCalls,
-            expectedCallCount: expectedCandidateCalls + expectedJudgeCalls + expectedClassifierCalls
+            ...expected
         };
         yield* artifacts.materializePlanSuites(root, plan, proposal);
         yield* artifacts.savePlan(root, plan);

package/dist/test/project-workflow.test.js

@@ -8,7 +8,7 @@ import { Cause, Effect, Layer } from "effect";
 import { EvalDimensionContrastError } from "../errors.js";
 import { EvalRepositoryInspectorLive } from "../inspection.js";
 import { EvalProjectArtifactsLive } from "../project-artifacts.js";
-import { summarizeEvalRunLedger } from "../project-contracts.js";
+import { scopedExpectedCalls, summarizeEvalRunLedger } from "../project-contracts.js";
 import { EvalProjectStoreLive } from "../project-store.js";
 import { EvalProjectWorkflow, EvalProjectWorkflowLive } from "../project-workflow.js";
 const roots = [];
@@ -190,12 +190,12 @@ const reviewedEvaluationInput = {
     decompositionBenchmark: reviewedDecompositionBenchmark,
     compositionSuite: reviewedCompositionSuite
 };
-async function completeProjectSetup(root) {
+async function completeProjectSetup(root, candidateModels = "openai/gpt-5.6-luna openai/gpt-5.6-terra openai/gpt-5.6-sol") {
     await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
         yield* workflow.setup(root);
         yield* workflow.answer(root, "Production workload");
-        yield* workflow.answer(root, "openai/gpt-5.6-luna openai/gpt-5.6-terra openai/gpt-5.6-sol");
+        yield* workflow.answer(root, candidateModels);
         yield* workflow.answer(root, "openai/gpt-5.6-luna");
         yield* workflow.answer(root, "openai/gpt-5.6-terra");
         yield* workflow.answer(root, "openai/gpt-5.6-terra");
@@ -311,9 +311,12 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
     const completionAssertion = generatedSuite.indexOf("run.toComplete();");
     const judgeCall = generatedSuite.indexOf("await judge.autoEvals");
     const completionRethrow = generatedSuite.indexOf("throw candidateCompletionError;");
+    const judgeRethrow = generatedSuite.indexOf("throw judgeError;");
+    assert.ok(judgeCall >= 0);
     assert.ok(completionAssertion >= 0);
-    assert.ok(judgeCall > completionAssertion);
-    assert.ok(completionRethrow > judgeCall);
+    assert.ok(completionAssertion < judgeCall);
+    assert.ok(judgeCall < completionRethrow);
+    assert.ok(completionRethrow < judgeRethrow);
     for (const dimension of reviewedDimensions) {
         assert.equal((await stat(path.join(root, ".routekit", "evals", "dimensions", dimension.id, "suite.json")))
             .mode & 0o777, 0o600);
@@ -345,50 +348,48 @@ test("full qualification refuses fewer than twenty reviewed cases per dimension"
         assert.match(String(exit.cause), /at least 20 reviewed cases/u);
     }
 });
-const comparisonResults = (plan, runId) => reviewedSuites.map((suite) => {
+const comparisonResults = (plan, runId) => reviewedSuites.flatMap((suite) => {
     const selected = plan.selectedCaseIds.find((entry) => entry.dimensionId === suite.dimensionId);
+    if (selected === undefined)
+        return [];
     const cases = suite.cases.filter((testCase) => selected.caseIds.includes(testCase.id));
-    const candidateModels = [
-        "openai/gpt-5.6-luna",
-        "openai/gpt-5.6-terra",
-        "openai/gpt-5.6-sol"
-    ];
-    return {
-        version: 1,
-        comparisonId: `${runId}-${suite.dimensionId}`,
-        profileId: suite.dimensionId,
-        suiteDigest: `suite-${suite.dimensionId}`,
-        judgeModel: "openai/gpt-5.6-terra",
-        startedAt: "2026-08-18T00:00:00.000Z",
-        finishedAt: "2026-08-18T00:01:00.000Z",
-        models: candidateModels.map((model) => ({
-            model,
-            cases: cases.map((testCase) => ({
-                caseId: testCase.id,
-                outcome: "passed",
-                measurement: {
-                    judgeScore: 0.9,
-                    inputTokens: 100,
-                    outputTokens: 50,
-                    durationMs: 250
-                }
-            }))
-        })),
-        calls: [
-            ...candidateModels.flatMap((model) => cases.map((testCase) => ({
-                role: "candidate",
+    const candidateModels = [...plan.candidateModels];
+    return [{
+            version: 1,
+            comparisonId: `${runId}-${suite.dimensionId}`,
+            profileId: suite.dimensionId,
+            suiteDigest: `suite-${suite.dimensionId}`,
+            judgeModel: "openai/gpt-5.6-terra",
+            startedAt: "2026-08-18T00:00:00.000Z",
+            finishedAt: "2026-08-18T00:01:00.000Z",
+            models: candidateModels.map((model) => ({
                 model,
-                caseId: testCase.id,
-                measurement: { inputTokens: 100, outputTokens: 50 }
-            }))),
-            ...candidateModels.flatMap(() => cases.map((testCase) => ({
-                role: "judge",
-                model: "openai/gpt-5.6-terra",
-                caseId: testCase.id,
-                measurement: { inputTokens: 80, outputTokens: 20 }
-            })))
-        ]
-    };
+                cases: cases.map((testCase) => ({
+                    caseId: testCase.id,
+                    outcome: "passed",
+                    measurement: {
+                        judgeScore: 0.9,
+                        inputTokens: 100,
+                        outputTokens: 50,
+                        durationMs: 250
+                    }
+                }))
+            })),
+            calls: [
+                ...candidateModels.flatMap((model) => cases.map((testCase) => ({
+                    role: "candidate",
+                    model,
+                    caseId: testCase.id,
+                    measurement: { inputTokens: 100, outputTokens: 50 }
+                }))),
+                ...candidateModels.flatMap(() => cases.map((testCase) => ({
+                    role: "judge",
+                    model: "openai/gpt-5.6-terra",
+                    caseId: testCase.id,
+                    measurement: { inputTokens: 80, outputTokens: 20 }
+                })))
+            ]
+        }];
 });
 const compositionComparison = (plan, runId) => {
     const cases = reviewedCompositionSuite.cases.filter((testCase) => plan.selectedCompositionCaseIds.includes(testCase.id));
@@ -431,7 +432,12 @@ const compositionComparison = (plan, runId) => {
     };
 };
 const qualifiedReport = (plan, runId, projectId, publishAllowed = true) => {
-    const comparisons = [...comparisonResults(plan, runId), compositionComparison(plan, runId)];
+    const comparisons = [
+        ...comparisonResults(plan, runId),
+        ...(plan.selectedCompositionCaseIds.length === 0
+            ? []
+            : [compositionComparison(plan, runId)])
+    ];
     const candidateModels = [...plan.candidateModels];
     const observations = reviewedDecompositionBenchmark.cases
         .filter((testCase) => plan.selectedDecompositionCaseIds.includes(testCase.id))
@@ -516,8 +522,8 @@ const qualifiedReport = (plan, runId, projectId, publishAllowed = true) => {
         }
     };
 };
-async function createPlan(root, scope = "pilot") {
-    await completeProjectSetup(root);
+async function createPlan(root, scope = "pilot", candidateModels) {
+    await completeProjectSetup(root, candidateModels);
     return Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
         const basis = yield* workflow.proposeDimensions(root, reviewedProposedDimensions);
@@ -550,6 +556,58 @@ test("run lifecycle retains complete sanitized evidence and activates only after
     const reportPath = path.join(root, ".routekit", "evals", "runs", outcome.started.runId, "report.json");
     assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
 });
+test("finishRun derives ledger expectations from the selected case slice", async () => {
+    const root = await makeRepository();
+    const created = await createPlan(root, "full", "openai/gpt-5.6-luna openai/gpt-5.6-terra");
+    const selectedDimension = created.selectedCaseIds[0];
+    const stalePlan = {
+        ...created,
+        selectedCaseIds: [
+            {
+                dimensionId: selectedDimension.dimensionId,
+                caseIds: [selectedDimension.caseIds[0]]
+            }
+        ],
+        selectedDecompositionCaseIds: [],
+        selectedCompositionCaseIds: [],
+        expectedDimensionCandidateCalls: 3_400,
+        expectedDimensionJudgeCalls: 3_400,
+        expectedClassifierCalls: 20,
+        expectedCompositionCandidateCalls: 0,
+        expectedCompositionJudgeCalls: 0,
+        expectedCandidateCalls: 3_400,
+        expectedJudgeCalls: 3_400,
+        expectedCallCount: 6_820
+    };
+    await writeFile(path.join(root, ".routekit", "evals", "plans", `${created.planId}.json`), `${JSON.stringify(stalePlan)}\n`);
+    const outcome = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const started = yield* workflow.startRun(root, stalePlan.planId);
+        const current = yield* workflow.status(root);
+        const report = qualifiedReport(started.plan, started.runId, current.state.projectId);
+        const expected = scopedExpectedCalls(started.plan);
+        assert.deepEqual(expected, {
+            expectedDimensionCandidateCalls: 2,
+            expectedDimensionJudgeCalls: 2,
+            expectedClassifierCalls: 0,
+            expectedCompositionCandidateCalls: 0,
+            expectedCompositionJudgeCalls: 0,
+            expectedCandidateCalls: 2,
+            expectedJudgeCalls: 2,
+            expectedCallCount: 4
+        });
+        const scopedReport = {
+            ...report,
+            ledger: summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount)
+        };
+        const finished = yield* workflow.finishRun(root, scopedReport);
+        return { finished, report: scopedReport };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(outcome.finished.state.stage, "qualified");
+    assert.equal(outcome.report.ledger.expectedCalls, 4);
+    assert.equal(outcome.report.ledger.observedCalls, 4);
+    assert.equal(outcome.report.ledger.observedCandidateRows, 2);
+});
 test("failed runs preserve plan revisions for retry while mismatched plans remain stale", async () => {
     const root = await makeRepository();
     const plan = await createPlan(root, "full");
@@ -633,8 +691,8 @@ test("startRun rematerializes stale dimension and composition suite templates",
     const dimensionManifestPath = path.join(dimensionRoot, "routekit.eval-manifest.json");
     const compositionManifestPath = path.join(compositionRoot, "routekit.eval-manifest.json");
     const staleTemplate = `// generated by RouteKit 1.0.21
-run.toComplete();
 await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+run.toComplete();
 `;
     await Promise.all([
         writeFile(dimensionSuitePath, staleTemplate),
@@ -658,10 +716,13 @@ await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
         const completionAssertion = suite.indexOf("run.toComplete();");
         const judgeCall = suite.indexOf("await judge.autoEvals");
         const completionRethrow = suite.indexOf("throw candidateCompletionError;");
+        const judgeRethrow = suite.indexOf("throw judgeError;");
         assert.doesNotMatch(suite, /generated by RouteKit 1\.0\.21/u);
+        assert.ok(judgeCall >= 0);
         assert.ok(completionAssertion >= 0);
-        assert.ok(judgeCall > completionAssertion);
-        assert.ok(completionRethrow > judgeCall);
+        assert.ok(completionAssertion < judgeCall);
+        assert.ok(judgeCall < completionRethrow);
+        assert.ok(completionRethrow < judgeRethrow);
     }
     assert.equal(dimensionManifestAfter, dimensionManifestBefore);
     assert.equal(compositionManifestAfter, compositionManifestBefore);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.0.23",
+  "version": "1.0.25",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,8 +32,8 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.23",
-    "@velum-labs/routekit-runtime": "1.0.23"
+    "@velum-labs/routekit-eval-contracts": "1.0.25",
+    "@velum-labs/routekit-runtime": "1.0.25"
   },
   "keywords": [
     "routekit",