@velum-labs/routekit-eval-setup 1.3.1 → 1.4.0

package/dist/effect-api.d.ts

@@ -10,6 +10,8 @@ export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export type { OriEvalAuthoringApi, OriEvalResult } from "./ori-result.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
+export type { EvalCaseCursor, EvalNamedEvent, EvalNamedEventLog, EvalRunCallEvent } from "./eval-event-log.js";
+export { assertEvalRunCanComplete, evalEventLabel, presentEvalEventLog, resumeEvalCaseCursors } from "./eval-event-log.js";
 export type { EvalAuthoringCompletion, EvalAuthoringSource, EvalAuthoringTransportShape, EvalProjectAuthorShape } from "./project-authoring.js";
 export { EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";

package/dist/effect-api.js

@@ -6,6 +6,7 @@ export { BasisRepair, BasisRepairLive, makeBasisRepair } from "./services/basis-
 export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository } from "./inspection.js";
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
+export { assertEvalRunCanComplete, evalEventLabel, presentEvalEventLog, resumeEvalCaseCursors } from "./eval-event-log.js";
 export { EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";

package/dist/eval-event-log.d.ts

@@ -0,0 +1,89 @@
+import { Schema } from "effect";
+export declare const EvalNamedEvent: Schema.Union<readonly [Schema.Struct<{
+    readonly name: Schema.Literal<"plan">;
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly plannedCalls: Schema.Finite;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly dimensionId: Schema.String;
+    readonly caseId: Schema.String;
+    readonly sequence: Schema.Finite;
+    readonly name: Schema.Literal<"candidate#">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly dimensionId: Schema.String;
+    readonly caseId: Schema.String;
+    readonly sequence: Schema.Finite;
+    readonly name: Schema.Literal<"judge#">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly operationId: Schema.String;
+    readonly name: Schema.Literal<"basis">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly operationId: Schema.String;
+    readonly name: Schema.Literal<"dimensions">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly operationId: Schema.String;
+    readonly name: Schema.Literal<"activation">;
+}>]>;
+export type EvalNamedEvent = typeof EvalNamedEvent.Type;
+export declare const EvalNamedEventLog: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+    readonly name: Schema.Literal<"plan">;
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly plannedCalls: Schema.Finite;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly dimensionId: Schema.String;
+    readonly caseId: Schema.String;
+    readonly sequence: Schema.Finite;
+    readonly name: Schema.Literal<"candidate#">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly dimensionId: Schema.String;
+    readonly caseId: Schema.String;
+    readonly sequence: Schema.Finite;
+    readonly name: Schema.Literal<"judge#">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly operationId: Schema.String;
+    readonly name: Schema.Literal<"basis">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly operationId: Schema.String;
+    readonly name: Schema.Literal<"dimensions">;
+}>, Schema.Struct<{
+    readonly at: Schema.String;
+    readonly operationId: Schema.String;
+    readonly name: Schema.Literal<"activation">;
+}>]>>;
+export type EvalNamedEventLog = typeof EvalNamedEventLog.Type;
+export type EvalRunCallEvent = Extract<EvalNamedEvent, {
+    readonly name: "candidate#" | "judge#";
+}>;
+export type EvalCaseCursor = {
+    readonly dimensionId: string;
+    readonly caseId: string;
+    readonly completePairs: number;
+    readonly lastEvent?: EvalRunCallEvent;
+    readonly hole?: Extract<EvalRunCallEvent, {
+        readonly name: "candidate#";
+    }>;
+};
+export declare function resumeEvalCaseCursors(log: EvalNamedEventLog): readonly EvalCaseCursor[];
+export declare function evalEventLabel(event: EvalNamedEvent): string;
+export type EvalEventLogPresentation = {
+    readonly failed: boolean;
+    readonly lines: readonly string[];
+};
+export declare function presentEvalEventLog(log: EvalNamedEventLog, options?: {
+    readonly failHoles?: boolean;
+}): EvalEventLogPresentation;
+export declare function assertEvalRunCanComplete(log: EvalNamedEventLog): void;

package/dist/eval-event-log.js

@@ -0,0 +1,161 @@
+import { Schema } from "effect";
+const NonNegativeInteger = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 && Number.isInteger(value) ? undefined : "value must be a non-negative integer")));
+const PositiveInteger = NonNegativeInteger.pipe(Schema.check(Schema.makeFilter((value) => value >= 1 ? undefined : "value must be a positive integer")));
+const RunPlanEvent = Schema.Struct({
+    name: Schema.Literal("plan"),
+    at: Schema.String,
+    runId: Schema.String,
+    plannedCalls: NonNegativeInteger
+});
+const RunCallEventFields = {
+    at: Schema.String,
+    runId: Schema.String,
+    dimensionId: Schema.String,
+    caseId: Schema.String,
+    sequence: PositiveInteger
+};
+const RunCandidateEvent = Schema.Struct({
+    name: Schema.Literal("candidate#"),
+    ...RunCallEventFields
+});
+const RunJudgeEvent = Schema.Struct({
+    name: Schema.Literal("judge#"),
+    ...RunCallEventFields
+});
+const AuthoringEventFields = {
+    at: Schema.String,
+    operationId: Schema.String
+};
+const AuthoringBasisEvent = Schema.Struct({
+    name: Schema.Literal("basis"),
+    ...AuthoringEventFields
+});
+const AuthoringDimensionsEvent = Schema.Struct({
+    name: Schema.Literal("dimensions"),
+    ...AuthoringEventFields
+});
+const AuthoringActivationEvent = Schema.Struct({
+    name: Schema.Literal("activation"),
+    ...AuthoringEventFields
+});
+export const EvalNamedEvent = Schema.Union([
+    RunPlanEvent,
+    RunCandidateEvent,
+    RunJudgeEvent,
+    AuthoringBasisEvent,
+    AuthoringDimensionsEvent,
+    AuthoringActivationEvent
+]);
+export const EvalNamedEventLog = Schema.Array(EvalNamedEvent);
+const caseKey = (dimensionId, caseId) => `${dimensionId}\u0000${caseId}`;
+export function resumeEvalCaseCursors(log) {
+    const cursors = new Map();
+    for (const event of log) {
+        if (event.name !== "candidate#" && event.name !== "judge#")
+            continue;
+        const key = caseKey(event.dimensionId, event.caseId);
+        const cursor = cursors.get(key) ??
+            {
+                dimensionId: event.dimensionId,
+                caseId: event.caseId,
+                candidates: new Map(),
+                judges: new Set()
+            };
+        const firstHoleSequence = (() => {
+            let sequence = 1;
+            while (cursor.candidates.has(sequence) && cursor.judges.has(sequence)) {
+                sequence += 1;
+            }
+            return sequence;
+        })();
+        if (cursor.candidates.has(firstHoleSequence) &&
+            !cursor.judges.has(firstHoleSequence) &&
+            (event.name !== "judge#" || event.sequence !== firstHoleSequence)) {
+            cursors.set(key, cursor);
+            continue;
+        }
+        if (event.name === "candidate#")
+            cursor.candidates.set(event.sequence, event);
+        else
+            cursor.judges.add(event.sequence);
+        cursor.lastEvent = event;
+        cursors.set(key, cursor);
+    }
+    return [...cursors.values()].map((cursor) => {
+        let completePairs = 0;
+        while (cursor.candidates.has(completePairs + 1) &&
+            cursor.judges.has(completePairs + 1)) {
+            completePairs += 1;
+        }
+        const hole = cursor.candidates.get(completePairs + 1);
+        return {
+            dimensionId: cursor.dimensionId,
+            caseId: cursor.caseId,
+            completePairs,
+            ...(hole === undefined
+                ? cursor.lastEvent === undefined
+                    ? {}
+                    : { lastEvent: cursor.lastEvent }
+                : { lastEvent: hole }),
+            ...(hole === undefined ? {} : { hole })
+        };
+    });
+}
+export function evalEventLabel(event) {
+    return event.name === "candidate#" || event.name === "judge#"
+        ? `${event.name}${String(event.sequence)}`
+        : event.name;
+}
+const lastPlanEvent = (log) => {
+    for (let index = log.length - 1; index >= 0; index -= 1) {
+        const event = log[index];
+        if (event?.name === "plan")
+            return event;
+    }
+    return undefined;
+};
+export function presentEvalEventLog(log, options = {}) {
+    const plan = lastPlanEvent(log);
+    if (plan !== undefined && plan.plannedCalls < 1) {
+        return {
+            failed: true,
+            lines: ["run fail no planned calls"]
+        };
+    }
+    const cursors = resumeEvalCaseCursors(log);
+    const failHoles = options.failHoles ?? true;
+    const lastEvent = log[log.length - 1];
+    const authoringLine = lastEvent?.name === "basis" ||
+        lastEvent?.name === "dimensions" ||
+        lastEvent?.name === "activation"
+        ? `authoring ${evalEventLabel(lastEvent)}`
+        : undefined;
+    const cursorLines = cursors.map((cursor) => {
+        const label = cursor.lastEvent === undefined ? "plan" : evalEventLabel(cursor.lastEvent);
+        return `${cursor.dimensionId}/${cursor.caseId} ${cursor.hole === undefined || !failHoles
+            ? label
+            : `fail ${label} missing judge#${String(cursor.hole.sequence)}`}`;
+    });
+    return {
+        failed: failHoles && cursors.some((cursor) => cursor.hole !== undefined),
+        lines: cursors.length === 0
+            ? authoringLine === undefined
+                ? plan === undefined
+                    ? []
+                    : [`run ${evalEventLabel(plan)}`]
+                : [authoringLine]
+            : authoringLine === undefined
+                ? cursorLines
+                : [...cursorLines, authoringLine]
+    };
+}
+export function assertEvalRunCanComplete(log) {
+    const plan = lastPlanEvent(log);
+    if (plan === undefined || plan.plannedCalls < 1) {
+        throw new Error("eval run cannot complete without a plan containing at least one call");
+    }
+    const hole = resumeEvalCaseCursors(log).find((cursor) => cursor.hole !== undefined);
+    if (hole?.hole !== undefined) {
+        throw new Error(`eval case ${JSON.stringify(hole.dimensionId)}/${JSON.stringify(hole.caseId)} cannot complete after ${evalEventLabel(hole.hole)} without judge#${String(hole.hole.sequence)}`);
+    }
+}

package/dist/index.d.ts

@@ -15,8 +15,10 @@ export type { OriEvalAuthoringApi, OriEvalResult } from "./ori-result.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export type { EvalAuthoringCompletion, EvalAuthoringSource, EvalAuthoringTransportShape, EvalProjectAuthorShape } from "./project-authoring.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export type { BasisCandidateDatasetV3, BasisCandidatesArtifactsV3, BasisCandidateCheckpointV3, BasisCandidateRecipeV3, BasisCandidateRecordV3, BasisGenerationArtifactsV3, BasisEvaluationProjectionV3, BasisGenerationJobV3, BasisGenerationLedgerV3, BasisGenerationPlanV3, BasisLabelProtocolV3, BasisQualificationArtifactsV3, BasisRepairArtifactsV3, BasisRepairPlanV3, BasisRepairSummaryV3, BasisSelectionReportV3, BasisStructureFindingV3, BasisStructureReportV3, BasisTaskRecordV3, BasisTaskSplitV3, BasisValidationPlanV3, BasisValidationClassifierBindingV3, BasisValidationJobCheckpointV3, BasisValidationResultsArtifactsV3, BasisValidationArtifactsV3, BasisValidationResultV3, EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionContrast, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectArtifactsStatus, EvalProjectDocument, EvalProjectStateV2, EvalProjectStatus, EvalRoutingBasisProposal, EvalRunCleanup, EvalRunFailureError, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget, EvalScopedExpectedCalls, FrozenRepositorySnapshotV3 } from "./project-contracts.js";
-export { EVAL_PROJECT_VERSION, EVAL_PROJECT_V2_VERSION, BasisCandidateDatasetV3 as BasisCandidateDatasetV3Schema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisCandidateRecipeV3 as BasisCandidateRecipeV3Schema, BasisCandidateRecordV3 as BasisCandidateRecordV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisGenerationJobV3 as BasisGenerationJobV3Schema, BasisGenerationLedgerV3 as BasisGenerationLedgerV3Schema, BasisGenerationPlanV3 as BasisGenerationPlanV3Schema, BasisLabelProtocolV3 as BasisLabelProtocolV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisRepairPlanV3 as BasisRepairPlanV3Schema, BasisRepairSummaryV3 as BasisRepairSummaryV3Schema, BasisSelectionReportV3 as BasisSelectionReportV3Schema, BasisStructureReportV3 as BasisStructureReportV3Schema, BasisTaskRecordV3 as BasisTaskRecordV3Schema, BasisTaskSplitV3 as BasisTaskSplitV3Schema, BasisValidationPlanV3 as BasisValidationPlanV3Schema, BasisValidationClassifierBindingV3 as BasisValidationClassifierBindingV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, BasisValidationResultV3 as BasisValidationResultV3Schema, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectDocument as EvalProjectDocumentSchema, EvalProjectSetupProgress, EvalProjectState, EvalProjectStateV2 as EvalProjectStateV2Schema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, FrozenRepositorySnapshotV3 as FrozenRepositorySnapshotV3Schema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
+export type { EvalCaseCursor, EvalNamedEvent, EvalNamedEventLog, EvalRunCallEvent } from "./eval-event-log.js";
+export { EvalNamedEvent as EvalNamedEventSchema, EvalNamedEventLog as EvalNamedEventLogSchema, assertEvalRunCanComplete, evalEventLabel, presentEvalEventLog, resumeEvalCaseCursors } from "./eval-event-log.js";
+export type { BasisCandidateDatasetV3, BasisCandidatesArtifactsV3, BasisCandidateCheckpointV3, BasisCandidateRecipeV3, BasisCandidateRecordV3, BasisGenerationArtifactsV3, BasisEvaluationProjectionV3, BasisGenerationJobV3, BasisGenerationLedgerV3, BasisGenerationPlanV3, BasisLabelProtocolV3, BasisQualificationArtifactsV3, BasisRepairArtifactsV3, BasisRepairPlanV3, BasisRepairSummaryV3, BasisSelectionReportV3, BasisStructureFindingV3, BasisStructureReportV3, BasisTaskRecordV3, BasisTaskSplitV3, BasisValidationPlanV3, BasisValidationClassifierBindingV3, BasisValidationJobCheckpointV3, BasisValidationResultsArtifactsV3, BasisValidationArtifactsV3, BasisValidationResultV3, EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalCostEstimate, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionContrast, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectArtifactsStatus, EvalProjectDocument, EvalProjectStateV2, EvalProjectStatus, EvalRoutingBasisProposal, EvalRunCleanup, EvalRunFailureError, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget, EvalScopedExpectedCalls, FrozenRepositorySnapshotV3 } from "./project-contracts.js";
+export { EVAL_PROJECT_VERSION, EVAL_PROJECT_V2_VERSION, BasisCandidateDatasetV3 as BasisCandidateDatasetV3Schema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisCandidateRecipeV3 as BasisCandidateRecipeV3Schema, BasisCandidateRecordV3 as BasisCandidateRecordV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisGenerationJobV3 as BasisGenerationJobV3Schema, BasisGenerationLedgerV3 as BasisGenerationLedgerV3Schema, BasisGenerationPlanV3 as BasisGenerationPlanV3Schema, BasisLabelProtocolV3 as BasisLabelProtocolV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisRepairPlanV3 as BasisRepairPlanV3Schema, BasisRepairSummaryV3 as BasisRepairSummaryV3Schema, BasisSelectionReportV3 as BasisSelectionReportV3Schema, BasisStructureReportV3 as BasisStructureReportV3Schema, BasisTaskRecordV3 as BasisTaskRecordV3Schema, BasisTaskSplitV3 as BasisTaskSplitV3Schema, BasisValidationPlanV3 as BasisValidationPlanV3Schema, BasisValidationClassifierBindingV3 as BasisValidationClassifierBindingV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, BasisValidationResultV3 as BasisValidationResultV3Schema, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalCostEstimate as EvalCostEstimateSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectDocument as EvalProjectDocumentSchema, EvalProjectSetupProgress, EvalProjectState, EvalProjectStateV2 as EvalProjectStateV2Schema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, FrozenRepositorySnapshotV3 as FrozenRepositorySnapshotV3Schema, estimateEvalPlanCost, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export type { BasisAuthoringShape } from "./services/basis-authoring/service.js";
 export { BasisAuthoring, BasisAuthoringLive, makeBasisAuthoring } from "./services/basis-authoring/service.js";

package/dist/index.js

@@ -10,7 +10,8 @@ export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export { EVAL_PROJECT_VERSION, EVAL_PROJECT_V2_VERSION, BasisCandidateDatasetV3 as BasisCandidateDatasetV3Schema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisCandidateRecipeV3 as BasisCandidateRecipeV3Schema, BasisCandidateRecordV3 as BasisCandidateRecordV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisGenerationJobV3 as BasisGenerationJobV3Schema, BasisGenerationLedgerV3 as BasisGenerationLedgerV3Schema, BasisGenerationPlanV3 as BasisGenerationPlanV3Schema, BasisLabelProtocolV3 as BasisLabelProtocolV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisRepairPlanV3 as BasisRepairPlanV3Schema, BasisRepairSummaryV3 as BasisRepairSummaryV3Schema, BasisSelectionReportV3 as BasisSelectionReportV3Schema, BasisStructureReportV3 as BasisStructureReportV3Schema, BasisTaskRecordV3 as BasisTaskRecordV3Schema, BasisTaskSplitV3 as BasisTaskSplitV3Schema, BasisValidationPlanV3 as BasisValidationPlanV3Schema, BasisValidationClassifierBindingV3 as BasisValidationClassifierBindingV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, BasisValidationResultV3 as BasisValidationResultV3Schema, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectDocument as EvalProjectDocumentSchema, EvalProjectSetupProgress, EvalProjectState, EvalProjectStateV2 as EvalProjectStateV2Schema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, FrozenRepositorySnapshotV3 as FrozenRepositorySnapshotV3Schema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
+export { EvalNamedEvent as EvalNamedEventSchema, EvalNamedEventLog as EvalNamedEventLogSchema, assertEvalRunCanComplete, evalEventLabel, presentEvalEventLog, resumeEvalCaseCursors } from "./eval-event-log.js";
+export { EVAL_PROJECT_VERSION, EVAL_PROJECT_V2_VERSION, BasisCandidateDatasetV3 as BasisCandidateDatasetV3Schema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisCandidateRecipeV3 as BasisCandidateRecipeV3Schema, BasisCandidateRecordV3 as BasisCandidateRecordV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisGenerationJobV3 as BasisGenerationJobV3Schema, BasisGenerationLedgerV3 as BasisGenerationLedgerV3Schema, BasisGenerationPlanV3 as BasisGenerationPlanV3Schema, BasisLabelProtocolV3 as BasisLabelProtocolV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisRepairPlanV3 as BasisRepairPlanV3Schema, BasisRepairSummaryV3 as BasisRepairSummaryV3Schema, BasisSelectionReportV3 as BasisSelectionReportV3Schema, BasisStructureReportV3 as BasisStructureReportV3Schema, BasisTaskRecordV3 as BasisTaskRecordV3Schema, BasisTaskSplitV3 as BasisTaskSplitV3Schema, BasisValidationPlanV3 as BasisValidationPlanV3Schema, BasisValidationClassifierBindingV3 as BasisValidationClassifierBindingV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, BasisValidationResultV3 as BasisValidationResultV3Schema, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalCostEstimate as EvalCostEstimateSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectDocument as EvalProjectDocumentSchema, EvalProjectSetupProgress, EvalProjectState, EvalProjectStateV2 as EvalProjectStateV2Schema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, FrozenRepositorySnapshotV3 as FrozenRepositorySnapshotV3Schema, estimateEvalPlanCost, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export { BasisAuthoring, BasisAuthoringLive, makeBasisAuthoring } from "./services/basis-authoring/service.js";
 export { BasisOnboarding, BasisOnboardingLive, makeBasisOnboarding } from "./services/basis-onboarding/service.js";

package/dist/project-artifacts.d.ts

@@ -1,6 +1,7 @@
 import { type RoutingBasis } from "@velum-labs/routekit-eval-contracts";
 import { Context, Effect, FileSystem, Layer, Path } from "effect";
 import { EvalProjectArtifactError } from "./errors.js";
+import { type EvalNamedEventLog } from "./eval-event-log.js";
 import { type EvalArtifactApproval, type BasisCandidatesArtifactsV3, type BasisEvaluationProjectionV3, type BasisCandidateCheckpointV3, type BasisGenerationArtifactsV3, type BasisQualificationArtifactsV3, type BasisRepairArtifactsV3, type BasisValidationJobCheckpointV3, type BasisValidationResultsArtifactsV3, type BasisValidationArtifactsV3, type EvalEvaluationProposal, type EvalExecutionPlan, type EvalRoutingBasisProposal, type EvalRunReport } from "./project-contracts.js";
 export declare function routingBasisDigest(dimensions: RoutingBasis["dimensions"]): string;
 export declare function evaluationProposalDigest(proposal: Omit<EvalEvaluationProposal, "evaluationDigest">): string;
@@ -47,6 +48,10 @@ export type EvalProjectArtifactsShape = {
     readonly saveRunReport: (repositoryRoot: string, report: EvalRunReport) => Effect.Effect<string, EvalProjectArtifactError, never>;
     readonly loadRunReport: (repositoryRoot: string, runId: string) => Effect.Effect<EvalRunReport | undefined, EvalProjectArtifactError, never>;
     readonly listRunReports: (repositoryRoot: string) => Effect.Effect<readonly string[], EvalProjectArtifactError, never>;
+    readonly saveRunEventLog: (repositoryRoot: string, runId: string, events: EvalNamedEventLog) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly loadRunEventLog: (repositoryRoot: string, runId: string) => Effect.Effect<EvalNamedEventLog, EvalProjectArtifactError, never>;
+    readonly saveAuthoringEventLog: (repositoryRoot: string, events: EvalNamedEventLog) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly loadAuthoringEventLog: (repositoryRoot: string) => Effect.Effect<EvalNamedEventLog, EvalProjectArtifactError, never>;
 };
 declare const EvalProjectArtifacts_base: Context.ServiceClass<EvalProjectArtifacts, "@velum-labs/routekit-eval-setup/EvalProjectArtifacts", EvalProjectArtifactsShape>;
 export declare class EvalProjectArtifacts extends EvalProjectArtifacts_base {

package/dist/project-artifacts.js

@@ -3,6 +3,7 @@ import { assertRoutingBasis } from "@velum-labs/routekit-eval-contracts";
 import { writeFileAtomicEffect } from "@velum-labs/routekit-runtime/effect";
 import { Context, Effect, Exit, FileSystem, Layer, Path, Schema } from "effect";
 import { EvalProjectArtifactError } from "./errors.js";
+import { EvalNamedEventLog as EvalNamedEventLogSchema } from "./eval-event-log.js";
 import { EVAL_PROJECT_VERSION, EvalArtifactApproval as EvalArtifactApprovalSchema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunReport as EvalRunReportSchema } from "./project-contracts.js";
 const BASIS_PROPOSAL = "routing-basis.proposed.json";
 const BASIS_APPROVAL = "routing-basis.approval.json";
@@ -18,6 +19,21 @@ const BASIS_REPAIR_V3 = "basis/repair.v3.json";
 const BASIS_REPAIR_VALIDATION_V3 = "basis/repair-validation.v3.json";
 const BASIS_REPAIR_VALIDATION_RESULTS_V3 = "basis/repair-validation-results.v3.json";
 const ARTIFACT_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u;
+const repeatedCase = (byId, selectedCaseId) => {
+    const exact = byId.get(selectedCaseId);
+    if (exact !== undefined)
+        return exact;
+    const repeated = /^(.*)#[1-9][0-9]*$/u.exec(selectedCaseId);
+    const sourceId = repeated?.[1];
+    if (sourceId === undefined) {
+        throw new Error(`execution plan refers to unknown case ${JSON.stringify(selectedCaseId)}`);
+    }
+    const testCase = byId.get(sourceId);
+    if (testCase === undefined) {
+        throw new Error(`execution plan refers to unknown case ${JSON.stringify(selectedCaseId)}`);
+    }
+    return { ...testCase, id: selectedCaseId };
+};
 const renderCandidatePrompt = () => `const prompt = [
         testCase.prompt,
         "",
@@ -324,13 +340,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     return yield* artifactFailure("writing", selection.dimensionId, new Error("execution plan refers to an unknown dimension suite"));
                 }
                 const byId = new Map(suite.cases.map((testCase) => [testCase.id, testCase]));
-                const cases = selection.caseIds.map((caseId) => {
-                    const testCase = byId.get(caseId);
-                    if (testCase === undefined) {
-                        throw new Error(`execution plan refers to unknown case ${JSON.stringify(caseId)} in ${JSON.stringify(selection.dimensionId)}`);
-                    }
-                    return testCase;
-                });
+                const cases = selection.caseIds.map((caseId) => repeatedCase(byId, caseId));
                 const suiteRoot = paths.join("plans", plan.planId, "dimensions", selection.dimensionId);
                 yield* writeText(repositoryRoot, paths.join(suiteRoot, `${selection.dimensionId}.eval.ts`), renderDimensionSuite(), mode === "rematerialize");
                 yield* write(repositoryRoot, paths.join(suiteRoot, "data", "cases.json"), cases, mode === "rematerialize");
@@ -347,13 +357,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                 }, mode === "rematerialize");
             }
             const compositionById = new Map(proposal.compositionSuite.cases.map((testCase) => [testCase.id, testCase]));
-            const compositionCases = plan.selectedCompositionCaseIds.map((caseId) => {
-                const testCase = compositionById.get(caseId);
-                if (testCase === undefined) {
-                    throw new Error(`execution plan refers to unknown composition case ${JSON.stringify(caseId)}`);
-                }
-                return testCase;
-            });
+            const compositionCases = plan.selectedCompositionCaseIds.map((caseId) => repeatedCase(compositionById, caseId));
             const compositionRoot = paths.join("plans", plan.planId, "composition");
             yield* writeText(repositoryRoot, paths.join(compositionRoot, "composition.eval.ts"), renderDimensionSuite(), mode === "rematerialize");
             yield* write(repositoryRoot, paths.join(compositionRoot, "data", "cases.json"), compositionCases, mode === "rematerialize");
@@ -435,7 +439,11 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     reports.push(entry);
             }
             return reports.sort((left, right) => left.localeCompare(right));
-        })
+        }),
+        saveRunEventLog: (repositoryRoot, runId, events) => requireArtifactId("run", runId).pipe(Effect.andThen(Schema.decodeEffect(EvalNamedEventLogSchema)(events).pipe(Effect.mapError((cause) => artifactFailure("writing", artifactPath(repositoryRoot, paths.join("runs", runId, "events.json")), cause)))), Effect.andThen(write(repositoryRoot, paths.join("runs", runId, "events.json"), events))),
+        loadRunEventLog: (repositoryRoot, runId) => requireArtifactId("run", runId).pipe(Effect.andThen(read(repositoryRoot, paths.join("runs", runId, "events.json"), Schema.decodeUnknownEffect(EvalNamedEventLogSchema))), Effect.map((events) => events ?? [])),
+        saveAuthoringEventLog: (repositoryRoot, events) => Schema.decodeEffect(EvalNamedEventLogSchema)(events).pipe(Effect.mapError((cause) => artifactFailure("writing", artifactPath(repositoryRoot, paths.join("authoring", "events.json")), cause)), Effect.andThen(write(repositoryRoot, paths.join("authoring", "events.json"), events))),
+        loadAuthoringEventLog: (repositoryRoot) => read(repositoryRoot, paths.join("authoring", "events.json"), Schema.decodeUnknownEffect(EvalNamedEventLogSchema)).pipe(Effect.map((events) => events ?? []))
     });
 });
 export const EvalProjectArtifactsLive = Layer.effect(EvalProjectArtifacts, makeFileEvalProjectArtifacts);

package/dist/project-contracts.d.ts

@@ -1,5 +1,6 @@
 import { type EvalComparisonResult as EvalComparisonResultType } from "@velum-labs/routekit-eval-contracts";
-import { Schema } from "effect";
+import { type CostServiceShape } from "@velum-labs/routekit-registry";
+import { Effect, Schema } from "effect";
 export declare const EVAL_PROJECT_VERSION: 1;
 export declare const EVAL_PROJECT_V2_VERSION: 2;
 export declare const EvalProjectQuestion: Schema.Struct<{
@@ -4312,6 +4313,17 @@ export declare const EvalArtifactApproval: Schema.Struct<{
 export type EvalArtifactApproval = typeof EvalArtifactApproval.Type;
 export declare const EvalPlanScope: Schema.Literals<readonly ["pilot", "full"]>;
 export type EvalPlanScope = typeof EvalPlanScope.Type;
+export declare const EvalCostEstimate: Schema.Struct<{
+    readonly callCount: Schema.Finite;
+    readonly maximumInputTokens: Schema.Finite;
+    readonly maximumOutputTokens: Schema.Finite;
+    readonly maximumCostUsd: Schema.optionalKey<Schema.Finite>;
+    readonly knownPricedMaximumUsd: Schema.Finite;
+    readonly unpricedCalls: Schema.Finite;
+    readonly subscriptionUnitCalls: Schema.Finite;
+    readonly unknowns: Schema.$Array<Schema.String>;
+}>;
+export type EvalCostEstimate = typeof EvalCostEstimate.Type;
 export declare const EvalExecutionPlan: Schema.Struct<{
     readonly version: Schema.Literal<1>;
     readonly planId: Schema.String;
@@ -4319,6 +4331,7 @@ export declare const EvalExecutionPlan: Schema.Struct<{
     readonly projectRevision: Schema.Finite;
     readonly createdAt: Schema.String;
     readonly scope: Schema.Literals<readonly ["pilot", "full"]>;
+    readonly repetitions: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     readonly basisDigest: Schema.String;
     readonly evaluationDigest: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
@@ -4341,6 +4354,16 @@ export declare const EvalExecutionPlan: Schema.Struct<{
     readonly expectedCandidateCalls: Schema.Finite;
     readonly expectedJudgeCalls: Schema.Finite;
     readonly expectedCallCount: Schema.Finite;
+    readonly costEstimate: Schema.optionalKey<Schema.Struct<{
+        readonly callCount: Schema.Finite;
+        readonly maximumInputTokens: Schema.Finite;
+        readonly maximumOutputTokens: Schema.Finite;
+        readonly maximumCostUsd: Schema.optionalKey<Schema.Finite>;
+        readonly knownPricedMaximumUsd: Schema.Finite;
+        readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.Finite;
+        readonly unknowns: Schema.$Array<Schema.String>;
+    }>>;
 }>;
 export type EvalExecutionPlan = typeof EvalExecutionPlan.Type;
 export type EvalScopedExpectedCalls = {
@@ -4354,6 +4377,8 @@ export type EvalScopedExpectedCalls = {
     readonly expectedCallCount: number;
 };
 export declare function scopedExpectedCalls(plan: Pick<EvalExecutionPlan, "candidateModels" | "selectedCaseIds" | "selectedDecompositionCaseIds" | "selectedCompositionCaseIds">): EvalScopedExpectedCalls;
+export declare const EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL: number;
+export declare function estimateEvalPlanCost(plan: Pick<EvalExecutionPlan, "candidateModels" | "candidateReasoningEfforts" | "classifierModel" | "judgeModel" | "maximumOutputTokens" | "expectedDimensionCandidateCalls" | "expectedClassifierCalls" | "expectedCompositionCandidateCalls">, costService: CostServiceShape): Effect.Effect<EvalCostEstimate>;
 export declare const EvalRunTarget: Schema.Struct<{
     readonly kind: Schema.Literals<readonly ["configured", "external"]>;
     readonly identity: Schema.String;
@@ -4375,6 +4400,7 @@ export declare const EvalRunLedger: Schema.Struct<{
     readonly unknownTokenMeasurements: Schema.Finite;
     readonly knownPricedSubtotalUsd: Schema.Finite;
     readonly unpricedCalls: Schema.Finite;
+    readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
 }>;
 export type EvalRunLedger = typeof EvalRunLedger.Type;
 export declare const EvalClassifierObservation: Schema.Struct<{
@@ -4443,7 +4469,7 @@ export declare const EvalRunQualification: Schema.Struct<{
     }>;
 }>;
 export type EvalRunQualification = typeof EvalRunQualification.Type;
-export declare function summarizeEvalRunLedger(comparisons: readonly EvalComparisonResultType[], expectedCalls: number, classifierObservations?: readonly EvalClassifierObservation[]): EvalRunLedger;
+export declare function summarizeEvalRunLedger(comparisons: readonly EvalComparisonResultType[], expectedCalls: number, costService: CostServiceShape, classifierObservations?: readonly EvalClassifierObservation[], classifierModel?: string): Effect.Effect<EvalRunLedger>;
 export declare const EvalRunFailureError: Schema.Struct<{
     readonly name: Schema.String;
     readonly message: Schema.String;
@@ -4617,7 +4643,40 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly unknownTokenMeasurements: Schema.Finite;
         readonly knownPricedSubtotalUsd: Schema.Finite;
         readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     }>;
+    readonly events: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly name: Schema.Literal<"plan">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly plannedCalls: Schema.Finite;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"candidate#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"basis">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"dimensions">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"activation">;
+    }>]>>>, never>;
 }>, Schema.Struct<{
     readonly status: Schema.Literal<"completed">;
     readonly activation: Schema.Struct<{
@@ -4777,7 +4836,40 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly unknownTokenMeasurements: Schema.Finite;
         readonly knownPricedSubtotalUsd: Schema.Finite;
         readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     }>;
+    readonly events: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly name: Schema.Literal<"plan">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly plannedCalls: Schema.Finite;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"candidate#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"basis">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"dimensions">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"activation">;
+    }>]>>>, never>;
 }>, Schema.Struct<{
     readonly status: Schema.Literal<"failed">;
     readonly failure: Schema.String;
@@ -4887,7 +4979,40 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly unknownTokenMeasurements: Schema.Finite;
         readonly knownPricedSubtotalUsd: Schema.Finite;
         readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     }>;
+    readonly events: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly name: Schema.Literal<"plan">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly plannedCalls: Schema.Finite;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"candidate#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"basis">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"dimensions">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"activation">;
+    }>]>>>, never>;
 }>]>;
 export type EvalRunReport = typeof EvalRunReport.Type;
 export type EvalProjectArtifactsStatus = {

package/dist/project-contracts.js

@@ -1,5 +1,6 @@
 import { DecompositionResult, DimensionScoreLabelV3, DimensionScorePredictionV3, EvalCandidateReasoningEfforts, EvalComparisonResult, EvalReasoningEffort, PublishedRoutingActivation, RoutingBasisApprovalV3, RoutingBasisV3, RoutingActivationConstraints, RequestRoutingRequirements, RoutingObjectivePolicy, WorkloadDimension } from "@velum-labs/routekit-eval-contracts";
-import { Schema } from "effect";
+import { Effect, Schema } from "effect";
+import { EvalNamedEventLog } from "./eval-event-log.js";
 export const EVAL_PROJECT_VERSION = 1;
 export const EVAL_PROJECT_V2_VERSION = 2;
 export const EvalProjectQuestion = Schema.Struct({
@@ -579,6 +580,16 @@ export const EvalArtifactApproval = Schema.Struct({
     approvedAt: Schema.String
 });
 export const EvalPlanScope = Schema.Literals(["pilot", "full"]);
+export const EvalCostEstimate = Schema.Struct({
+    callCount: NonNegativeInteger,
+    maximumInputTokens: NonNegativeInteger,
+    maximumOutputTokens: NonNegativeInteger,
+    maximumCostUsd: Schema.optionalKey(NonNegativeFinite),
+    knownPricedMaximumUsd: NonNegativeFinite,
+    unpricedCalls: NonNegativeInteger,
+    subscriptionUnitCalls: NonNegativeInteger,
+    unknowns: Schema.Array(Schema.String)
+});
 export const EvalExecutionPlan = Schema.Struct({
     version: Schema.Literal(EVAL_PROJECT_VERSION),
     planId: Schema.String,
@@ -586,6 +597,7 @@ export const EvalExecutionPlan = Schema.Struct({
     projectRevision: NonNegativeInteger,
     createdAt: Schema.String,
     scope: EvalPlanScope,
+    repetitions: NonNegativeInteger.pipe(Schema.withDecodingDefaultKey(Effect.succeed(1))),
     basisDigest: Schema.String,
     evaluationDigest: Schema.String,
     candidateModels: Schema.Array(Schema.String),
@@ -607,7 +619,8 @@ export const EvalExecutionPlan = Schema.Struct({
     expectedCompositionJudgeCalls: NonNegativeInteger,
     expectedCandidateCalls: NonNegativeInteger,
     expectedJudgeCalls: NonNegativeInteger,
-    expectedCallCount: NonNegativeInteger
+    expectedCallCount: NonNegativeInteger,
+    costEstimate: Schema.optionalKey(EvalCostEstimate)
 });
 export function scopedExpectedCalls(plan) {
     const candidateModelCount = plan.candidateModels.length;
@@ -630,6 +643,44 @@ export function scopedExpectedCalls(plan) {
         expectedCallCount: expectedCandidateCalls + expectedJudgeCalls + expectedClassifierCalls
     };
 }
+export const EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL = 256 * 1024;
+export function estimateEvalPlanCost(plan, costService) {
+    const candidateCaseCalls = plan.expectedDimensionCandidateCalls + plan.expectedCompositionCandidateCalls;
+    const callsPerCandidate = plan.candidateModels.length === 0 ? 0 : candidateCaseCalls / plan.candidateModels.length;
+    const calls = [
+        ...plan.candidateModels.map((model) => ({
+            model,
+            callCount: callsPerCandidate,
+            maximumInputTokens: EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: plan.maximumOutputTokens,
+            ...(plan.candidateReasoningEfforts?.[model] === undefined
+                ? {}
+                : { reasoningEffort: plan.candidateReasoningEfforts[model] })
+        })),
+        {
+            model: plan.judgeModel,
+            callCount: candidateCaseCalls,
+            maximumInputTokens: EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: plan.maximumOutputTokens
+        },
+        {
+            model: plan.classifierModel,
+            callCount: plan.expectedClassifierCalls,
+            maximumInputTokens: EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: plan.maximumOutputTokens
+        }
+    ];
+    return costService.estimate(calls).pipe(Effect.map((estimate) => ({
+        callCount: estimate.callCount,
+        maximumInputTokens: estimate.knownInputTokens,
+        maximumOutputTokens: estimate.knownOutputTokens,
+        ...(estimate.maximumCostUsd === undefined ? {} : { maximumCostUsd: estimate.maximumCostUsd }),
+        knownPricedMaximumUsd: estimate.knownPricedMaximumUsd,
+        unpricedCalls: estimate.unpricedCalls + estimate.upstreamManagedCalls + estimate.unknownBillingCalls,
+        subscriptionUnitCalls: estimate.subscriptionUnitCalls,
+        unknowns: estimate.unknowns
+    })));
+}
 export const EvalRunTarget = Schema.Struct({
     kind: Schema.Literals(["configured", "external"]),
     identity: Schema.String,
@@ -648,7 +699,8 @@ export const EvalRunLedger = Schema.Struct({
     knownOutputTokens: NonNegativeInteger,
     unknownTokenMeasurements: NonNegativeInteger,
     knownPricedSubtotalUsd: NonNegativeFinite,
-    unpricedCalls: NonNegativeInteger
+    unpricedCalls: NonNegativeInteger,
+    subscriptionUnitCalls: NonNegativeInteger.pipe(Schema.withDecodingDefaultKey(Effect.succeed(0)))
 });
 export const EvalClassifierObservation = Schema.Struct({
     caseId: Schema.String,
@@ -687,14 +739,10 @@ export const EvalRunQualification = Schema.Struct({
         cases: Schema.Array(EvalCompositionCaseResult)
     })
 });
-export function summarizeEvalRunLedger(comparisons, expectedCalls, classifierObservations = []) {
+export function summarizeEvalRunLedger(comparisons, expectedCalls, costService, classifierObservations = [], classifierModel = "unknown") {
     let observedCalls = 0;
     let observedCandidateRows = 0;
-    let knownInputTokens = 0;
-    let knownOutputTokens = 0;
-    let unknownTokenMeasurements = 0;
-    let knownPricedSubtotalUsd = 0;
-    let unpricedCalls = 0;
+    const actualCalls = [];
     for (const comparison of comparisons) {
         const calls = comparison.calls ??
             comparison.models.flatMap((model) => model.cases.map((testCase) => ({
@@ -707,49 +755,50 @@ export function summarizeEvalRunLedger(comparisons, expectedCalls, classifierObs
             observedCalls += 1;
             if (call.role === "candidate")
                 observedCandidateRows += 1;
-            if (call.measurement.inputTokens === undefined ||
-                call.measurement.outputTokens === undefined) {
-                unknownTokenMeasurements += 1;
-            }
-            else {
-                knownInputTokens += call.measurement.inputTokens;
-                knownOutputTokens += call.measurement.outputTokens;
-            }
-            if (call.measurement.costUsd === undefined) {
-                unpricedCalls += 1;
-            }
-            else {
-                knownPricedSubtotalUsd += call.measurement.costUsd;
-            }
+            actualCalls.push({
+                model: call.model,
+                usage: {
+                    ...(call.measurement.inputTokens === undefined
+                        ? {}
+                        : { inputTokens: call.measurement.inputTokens }),
+                    ...(call.measurement.outputTokens === undefined
+                        ? {}
+                        : { outputTokens: call.measurement.outputTokens })
+                },
+                ...(call.measurement.costUsd === undefined
+                    ? {}
+                    : { providerCostUsd: call.measurement.costUsd })
+            });
         }
     }
     for (const observation of classifierObservations) {
         observedCalls += 1;
-        if (observation.measurement.inputTokens === undefined ||
-            observation.measurement.outputTokens === undefined) {
-            unknownTokenMeasurements += 1;
-        }
-        else {
-            knownInputTokens += observation.measurement.inputTokens;
-            knownOutputTokens += observation.measurement.outputTokens;
-        }
-        if (observation.measurement.costUsd === undefined) {
-            unpricedCalls += 1;
-        }
-        else {
-            knownPricedSubtotalUsd += observation.measurement.costUsd;
-        }
+        actualCalls.push({
+            model: classifierModel,
+            usage: {
+                ...(observation.measurement.inputTokens === undefined
+                    ? {}
+                    : { inputTokens: observation.measurement.inputTokens }),
+                ...(observation.measurement.outputTokens === undefined
+                    ? {}
+                    : { outputTokens: observation.measurement.outputTokens })
+            },
+            ...(observation.measurement.costUsd === undefined
+                ? {}
+                : { providerCostUsd: observation.measurement.costUsd })
+        });
     }
-    return {
+    return costService.actuals(actualCalls).pipe(Effect.map((actuals) => ({
         expectedCalls,
         observedCalls,
         observedCandidateRows,
-        knownInputTokens,
-        knownOutputTokens,
-        unknownTokenMeasurements,
-        knownPricedSubtotalUsd,
-        unpricedCalls
-    };
+        knownInputTokens: actuals.knownInputTokens,
+        knownOutputTokens: actuals.knownOutputTokens,
+        unknownTokenMeasurements: actuals.unknownTokenCalls,
+        knownPricedSubtotalUsd: actuals.knownPricedSubtotalUsd,
+        unpricedCalls: actuals.unpricedCalls + actuals.upstreamManagedCalls + actuals.unknownBillingCalls,
+        subscriptionUnitCalls: actuals.subscriptionUnitCalls
+    })));
 }
 const EvalRunReportCommon = {
     version: Schema.Literal(EVAL_PROJECT_VERSION),
@@ -764,7 +813,8 @@ const EvalRunReportCommon = {
     cleanup: EvalRunCleanup,
     comparisons: Schema.Array(EvalComparisonResult),
     qualification: Schema.optionalKey(EvalRunQualification),
-    ledger: EvalRunLedger
+    ledger: EvalRunLedger,
+    events: Schema.optionalKey(EvalNamedEventLog).pipe(Schema.withDecodingDefaultKey(Effect.succeed([])))
 };
 export const EvalRunFailureError = Schema.Struct({
     name: Schema.String,

package/dist/project-workflow.d.ts

@@ -1,3 +1,4 @@
+import { CostService } from "@velum-labs/routekit-registry";
 import { Context, Effect, Layer, Path } from "effect";
 import type { EvalProjectArtifactError, EvalSetupInspectionError } from "./errors.js";
 import { EvalDimensionContrastError, type EvalModelCatalogError, type EvalProjectStoreError, EvalProjectTransitionError } from "./errors.js";
@@ -15,7 +16,7 @@ export type EvalProjectWorkflowShape = {
     readonly approveDimensions: (repositoryRoot: string, basisDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly proposeEvaluations: (repositoryRoot: string, proposal: Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "candidateReasoningEfforts" | "judgeModel">) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly approveEvaluations: (repositoryRoot: string, evaluationDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
-    readonly createPlan: (repositoryRoot: string, scope: EvalPlanScope) => Effect.Effect<EvalExecutionPlan, EvalProjectWorkflowError>;
+    readonly createPlan: (repositoryRoot: string, scope: EvalPlanScope, repetitions?: number) => Effect.Effect<EvalExecutionPlan, EvalProjectWorkflowError>;
     readonly startRun: (repositoryRoot: string, planId: string) => Effect.Effect<{
         readonly plan: EvalExecutionPlan;
         readonly runId: string;
@@ -28,6 +29,6 @@ export type EvalProjectWorkflowShape = {
 declare const EvalProjectWorkflow_base: Context.ServiceClass<EvalProjectWorkflow, "@velum-labs/routekit-eval-setup/EvalProjectWorkflow", EvalProjectWorkflowShape>;
 export declare class EvalProjectWorkflow extends EvalProjectWorkflow_base {
 }
-export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
-export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
+export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog | CostService>;
+export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog | CostService>;
 export {};

package/dist/project-workflow.js

@@ -1,11 +1,12 @@
 import { assertDecompositionResult, assertPublishedRoutingActivation, assertRoutingObjectivePolicy, isForbiddenEvalModel } from "@velum-labs/routekit-eval-contracts";
 import { RoutingActivationConstraints, RoutingObjectivePolicy } from "@velum-labs/routekit-eval-contracts";
+import { CostService } from "@velum-labs/routekit-registry";
 import { Clock, Context, Effect, Layer, Path, Schema } from "effect";
 import { EvalDimensionContrastError, EvalProjectTransitionError } from "./errors.js";
 import { EvalRepositoryInspector } from "./inspection.js";
 import { candidateReasoningEffortFromAnswer } from "./model-selection.js";
 import { EvalProjectArtifacts, evaluationProposalDigest, routingBasisDigest } from "./project-artifacts.js";
-import { EVAL_PROJECT_VERSION, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
+import { EVAL_PROJECT_VERSION, estimateEvalPlanCost, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 import { EvalProjectStore } from "./project-store.js";
 import { EvalModelCatalog } from "./services/model-catalog/service.js";
 const isoNow = Effect.map(Clock.currentTimeMillis, (millis) => new Date(millis).toISOString());
@@ -157,8 +158,9 @@ const sameLedger = (left, right) => left.expectedCalls === right.expectedCalls &
     left.knownOutputTokens === right.knownOutputTokens &&
     left.unknownTokenMeasurements === right.unknownTokenMeasurements &&
     left.knownPricedSubtotalUsd === right.knownPricedSubtotalUsd &&
-    left.unpricedCalls === right.unpricedCalls;
-const validateQualifiedReport = (state, plan, proposal, report) => {
+    left.unpricedCalls === right.unpricedCalls &&
+    left.subscriptionUnitCalls === right.subscriptionUnitCalls;
+const validateQualifiedReport = (state, plan, proposal, report, costService) => Effect.gen(function* () {
     assertPublishedRoutingActivation(report.activation, {
         dimensionIds: plan.selectedCaseIds.map((entry) => entry.dimensionId)
     });
@@ -302,13 +304,13 @@ const validateQualifiedReport = (state, plan, proposal, report) => {
         }
     }
     const expected = scopedExpectedCalls(plan);
-    const ledger = summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount, qualification?.decomposition.observations ?? []);
+    const ledger = yield* summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount, costService, qualification?.decomposition.observations ?? [], plan.classifierModel);
     if (ledger.observedCalls !== expected.expectedCallCount ||
         ledger.observedCandidateRows !== expected.expectedCandidateCalls ||
         !sameLedger(ledger, report.ledger)) {
         throw new Error("qualification ledger does not match the completed evidence");
     }
-};
+});
 const parseObjective = (state, answer) => Effect.gen(function* () {
     const text = yield* nonEmptyAnswer(state, answer);
     const json = yield* Effect.try({
@@ -512,6 +514,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
     const artifacts = yield* EvalProjectArtifacts;
     const inspector = yield* EvalRepositoryInspector;
     const modelCatalog = yield* EvalModelCatalog;
+    const costService = yield* CostService;
     const paths = yield* Path.Path;
     const resolveRoot = (repositoryRoot) => paths.resolve(repositoryRoot);
     const statusOf = (root, state) => Effect.gen(function* () {
@@ -790,7 +793,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
         yield* store.save(root, next);
         return yield* statusOf(root, next);
     });
-    const createPlan = (repositoryRoot, scope) => Effect.gen(function* () {
+    const createPlan = (repositoryRoot, scope, repetitions = 1) => Effect.gen(function* () {
         const root = resolveRoot(repositoryRoot);
         const state = yield* loadRequired(root);
         if (state.stage !== "ready" && state.stage !== "qualified") {
@@ -805,6 +808,9 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             proposal.basisDigest !== state.basisDigest) {
             return yield* transitionError(state.stage, "approved evaluation artifacts are stale");
         }
+        if (!Number.isSafeInteger(repetitions) || repetitions < 1) {
+            return yield* transitionError(state.stage, "eval repetitions N must be at least 1");
+        }
         if (scope === "full") {
             const shortSuite = proposal.suites.find((suite) => suite.cases.length < 20);
             if (shortSuite !== undefined ||
@@ -817,16 +823,22 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
         }
         const selectedCaseIds = proposal.suites.map((suite) => ({
             dimensionId: suite.dimensionId,
-            caseIds: suite.cases
+            caseIds: Array.from({ length: repetitions }, (_, repetition) => suite.cases
                 .slice(0, scope === "pilot" ? 5 : undefined)
-                .map((testCase) => testCase.id)
+                .map((testCase) => repetitions === 1
+                ? testCase.id
+                : `${testCase.id}#${String(repetition + 1)}`)).flat()
         }));
-        const selectedDecompositionCaseIds = proposal.decompositionBenchmark.cases
+        const selectedDecompositionCaseIds = Array.from({ length: repetitions }, (_, repetition) => proposal.decompositionBenchmark.cases
             .slice(0, scope === "pilot" ? 5 : undefined)
-            .map((testCase) => testCase.id);
-        const selectedCompositionCaseIds = proposal.compositionSuite.cases
+            .map((testCase) => repetitions === 1
+            ? testCase.id
+            : `${testCase.id}#${String(repetition + 1)}`)).flat();
+        const selectedCompositionCaseIds = Array.from({ length: repetitions }, (_, repetition) => proposal.compositionSuite.cases
             .slice(0, scope === "pilot" ? 5 : undefined)
-            .map((testCase) => testCase.id);
+            .map((testCase) => repetitions === 1
+            ? testCase.id
+            : `${testCase.id}#${String(repetition + 1)}`)).flat();
         const expected = scopedExpectedCalls({
             candidateModels: proposal.candidateModels,
             selectedCaseIds,
@@ -840,6 +852,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             projectRevision: state.revision,
             createdAt: yield* isoNow,
             scope,
+            repetitions,
             basisDigest: state.basisDigest,
             evaluationDigest: state.evaluationDigest,
             candidateModels: state.configuration.candidateModels,
@@ -855,7 +868,15 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             selectedDecompositionCaseIds,
             selectedCompositionCaseIds,
             maximumOutputTokens: Math.max(proposal.compositionSuite.maximumOutputTokens, ...proposal.suites.map((suite) => suite.maximumOutputTokens)),
-            ...expected
+            ...expected,
+            costEstimate: yield* estimateEvalPlanCost({
+                candidateModels: state.configuration.candidateModels,
+                candidateReasoningEfforts: state.configuration.candidateReasoningEfforts,
+                classifierModel: state.configuration.classifierModel,
+                judgeModel: state.configuration.judgeModel,
+                maximumOutputTokens: Math.max(proposal.compositionSuite.maximumOutputTokens, ...proposal.suites.map((suite) => suite.maximumOutputTokens)),
+                ...expected
+            }, costService)
         };
         yield* artifacts.materializePlanSuites(root, plan, proposal);
         yield* artifacts.savePlan(root, plan);
@@ -877,6 +898,9 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             plan.evaluationDigest !== state.evaluationDigest ||
             !sameStrings(plan.candidateModels, state.configuration.candidateModels) ||
             !sameReasoningEfforts(plan.candidateReasoningEfforts, state.configuration.candidateReasoningEfforts) ||
+            (plan.costEstimate !== undefined &&
+                JSON.stringify(plan.costEstimate) !==
+                    JSON.stringify(yield* estimateEvalPlanCost(plan, costService))) ||
             plan.classifierModel !== state.configuration.classifierModel ||
             plan.authorModel !== state.configuration.authorModel ||
             plan.judgeModel !== state.configuration.judgeModel) {
@@ -940,10 +964,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             (plan.scope === "pilot" && report.status !== "completed")) {
             return yield* transitionError(state.stage, "only full plans may qualify routing activation; pilot plans are completed diagnostics");
         }
-        yield* Effect.try({
-            try: () => validateQualifiedReport(state, plan, proposal, report),
-            catch: (cause) => transitionError(state.stage, cause instanceof Error ? cause.message : "qualification report is invalid")
-        });
+        yield* validateQualifiedReport(state, plan, proposal, report, costService).pipe(Effect.catchDefect((cause) => Effect.fail(transitionError(state.stage, cause instanceof Error ? cause.message : "qualification report is invalid"))));
         const reportPath = yield* artifacts.saveRunReport(root, report);
         const common = {
             version: state.version,

package/dist/test/eval-event-log.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/eval-event-log.test.js

@@ -0,0 +1,92 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { Schema } from "effect";
+import { EvalNamedEventLog, assertEvalRunCanComplete, presentEvalEventLog, resumeEvalCaseCursors } from "../eval-event-log.js";
+const at = "2026-08-26T00:00:00.000Z";
+const call = (name, caseId, sequence, dimensionId = "dimension") => ({
+    name,
+    at,
+    runId: "run",
+    dimensionId,
+    caseId,
+    sequence
+});
+test("N=4 candidate-only activity stays per-case and fails before completion", () => {
+    const events = [
+        { name: "plan", at, runId: "run", plannedCalls: 80 },
+        ...Array.from({ length: 40 }, (_, index) => call("candidate#", `case-${String((index % 4) + 1)}`, Math.floor(index / 4) + 1))
+    ];
+    const presentation = presentEvalEventLog(events);
+    assert.equal(presentation.failed, true);
+    assert.equal(presentation.lines.length, 4);
+    assert.ok(presentation.lines.every((line) => /fail candidate#1 missing judge#1/u.test(line)));
+    assert.doesNotMatch(presentation.lines.join("\n"), /40\/80|candidate#40/u);
+    assert.throws(() => assertEvalRunCanComplete(events), /cannot complete after candidate#1/u);
+});
+test("resume cursor stops at the first unmatched candidate and does not resume past the hole", () => {
+    const events = [
+        { name: "plan", at, runId: "run", plannedCalls: 8 },
+        call("candidate#", "case-a", 1),
+        call("judge#", "case-a", 1),
+        call("candidate#", "case-a", 2),
+        call("candidate#", "case-a", 3),
+        call("judge#", "case-a", 3)
+    ];
+    assert.deepEqual(resumeEvalCaseCursors(events), [
+        {
+            dimensionId: "dimension",
+            caseId: "case-a",
+            completePairs: 1,
+            lastEvent: call("candidate#", "case-a", 2),
+            hole: call("candidate#", "case-a", 2)
+        }
+    ]);
+});
+test("zero planned calls is a failed pulse, not progress", () => {
+    const events = [{ name: "plan", at, runId: "run", plannedCalls: 0 }];
+    assert.deepEqual(presentEvalEventLog(events), {
+        failed: true,
+        lines: ["run fail no planned calls"]
+    });
+    assert.throws(() => assertEvalRunCanComplete(events), /at least one call/u);
+});
+test("CI JSON round-trips the exact same named event log", () => {
+    const events = [
+        { name: "basis", at, operationId: "author-basis" },
+        { name: "dimensions", at, operationId: "author-dimensions" },
+        { name: "activation", at, operationId: "author-activation" },
+        { name: "plan", at, runId: "run", plannedCalls: 2 },
+        call("candidate#", "case-a", 1),
+        call("judge#", "case-a", 1)
+    ];
+    const decoded = Schema.decodeUnknownSync(EvalNamedEventLog)(JSON.parse(JSON.stringify(events)));
+    assert.deepEqual(decoded, events);
+    assert.doesNotThrow(() => assertEvalRunCanComplete(decoded));
+    assert.deepEqual([...new Set(decoded.map((event) => event.name))], ["basis", "dimensions", "activation", "plan", "candidate#", "judge#"]);
+});
+test("authoring heartbeat is the last named artifact event", () => {
+    const events = [
+        { name: "basis", at, operationId: "author-basis" },
+        { name: "dimensions", at, operationId: "author-dimensions" }
+    ];
+    assert.deepEqual(presentEvalEventLog(events, { failHoles: false }), {
+        failed: false,
+        lines: ["authoring dimensions"]
+    });
+});
+test("live presentation keeps a candidate hole in flight and failure presentation names it", () => {
+    const events = [
+        { name: "plan", at, runId: "run", plannedCalls: 2 },
+        call("candidate#", "case-a", 1)
+    ];
+    const live = presentEvalEventLog(events, { failHoles: false });
+    assert.deepEqual(live, {
+        failed: false,
+        lines: ["dimension/case-a candidate#1"]
+    });
+    assert.doesNotMatch(live.lines.join("\n"), /missing judge#/u);
+    assert.deepEqual(presentEvalEventLog(events, { failHoles: true }), {
+        failed: true,
+        lines: ["dimension/case-a fail candidate#1 missing judge#1"]
+    });
+});

package/dist/test/project-workflow.test.js

@@ -4,11 +4,12 @@ import os from "node:os";
 import path from "node:path";
 import { after, test } from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { CostServiceLive, makeCostService } from "@velum-labs/routekit-registry";
 import { Cause, Effect, Layer } from "effect";
 import { EvalDimensionContrastError, EvalModelCatalogError } from "../errors.js";
 import { EvalRepositoryInspectorLive } from "../inspection.js";
 import { EvalProjectArtifactsLive } from "../project-artifacts.js";
-import { scopedExpectedCalls, summarizeEvalRunLedger } from "../project-contracts.js";
+import { estimateEvalPlanCost, scopedExpectedCalls, summarizeEvalRunLedger } from "../project-contracts.js";
 import { EvalProjectStoreLive } from "../project-store.js";
 import { EvalProjectWorkflow, EvalProjectWorkflowLive } from "../project-workflow.js";
 import { EvalModelCatalog } from "../services/model-catalog/service.js";
@@ -28,7 +29,8 @@ const catalogEfforts = (candidateModels) => Effect.sync(() => {
     };
 });
 const EvalModelCatalogTest = Layer.succeed(EvalModelCatalog, EvalModelCatalog.of({ reasoningEfforts: catalogEfforts }));
-const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(EvalModelCatalogTest), Layer.provide(NodeServicesLayer));
+const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(EvalModelCatalogTest), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));
+const testCostService = makeCostService();
 const makeRepository = async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-project-"));
     roots.push(root);
@@ -162,7 +164,7 @@ test("candidate selection fails closed when catalog reasoning metadata is unavai
             detail: "models.list did not advertise candidate reasoning capabilities"
         }))
     }));
-    const workflowLayer = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(unavailableCatalog), Layer.provide(NodeServicesLayer));
+    const workflowLayer = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(unavailableCatalog), Layer.provide(CostServiceLive), Layer.provide(NodeServicesLayer));
     await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
         yield* workflow.setup(root);
@@ -356,7 +358,8 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
         assert.equal(ready.state.stage, "ready");
         assert.equal(ready.nextAction, "run");
         const plan = yield* workflow.createPlan(root, "pilot");
-        return { basisDigest, evaluationDigest, plan };
+        const repeatedPlan = yield* workflow.createPlan(root, "pilot", 2);
+        return { basisDigest, evaluationDigest, plan, repeatedPlan };
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
     const storedBasis = JSON.parse(await readFile(path.join(root, ".routekit", "evals", "routing-basis.proposed.json"), "utf8"));
     assert.equal(storedBasis.dimensionContrasts.length, reviewedDimensions.length);
@@ -374,6 +377,12 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
     assert.equal(result.plan.expectedJudgeCalls, 90);
     assert.equal(result.plan.expectedCallCount, 185);
     assert.equal(result.plan.maximumOutputTokens, 256);
+    assert.equal(result.plan.repetitions, 1);
+    assert.equal(result.repeatedPlan.repetitions, 2);
+    assert.equal(result.repeatedPlan.expectedCallCount, result.plan.expectedCallCount * 2);
+    assert.equal(result.plan.costEstimate !== undefined, true);
+    assert.equal(result.repeatedPlan.costEstimate !== undefined, true);
+    assert.equal(result.repeatedPlan.costEstimate?.knownPricedMaximumUsd, (result.plan.costEstimate?.knownPricedMaximumUsd ?? 0) * 2);
     const planPath = path.join(root, ".routekit", "evals", "plans", `${result.plan.planId}.json`);
     assert.equal((await stat(planPath)).mode & 0o777, 0o600);
     assert.deepEqual(JSON.parse(await readFile(planPath, "utf8")), result.plan);
@@ -566,7 +575,7 @@ const qualifiedReport = (plan, runId, projectId, publishAllowed = true) => {
                 }))
             }
         },
-        ledger: summarizeEvalRunLedger(comparisons, plan.expectedCallCount, observations),
+        ledger: Effect.runSync(summarizeEvalRunLedger(comparisons, plan.expectedCallCount, testCostService, observations)),
         activation: {
             version: 2,
             generatedAt: "2026-08-18T00:02:00.000Z",
@@ -633,8 +642,9 @@ test("finishRun derives ledger expectations from the selected case slice", async
     const root = await makeRepository();
     const created = await createPlan(root, "full", "openai/gpt-5.6-luna openai/gpt-5.6-terra");
     const selectedDimension = created.selectedCaseIds[0];
-    const stalePlan = {
+    const stalePlanInput = {
         ...created,
+        repetitions: 1,
         selectedCaseIds: [
             {
                 dimensionId: selectedDimension.dimensionId,
@@ -652,6 +662,10 @@ test("finishRun derives ledger expectations from the selected case slice", async
         expectedJudgeCalls: 3_400,
         expectedCallCount: 6_820
     };
+    const stalePlan = {
+        ...stalePlanInput,
+        costEstimate: Effect.runSync(estimateEvalPlanCost(stalePlanInput, testCostService))
+    };
     await writeFile(path.join(root, ".routekit", "evals", "plans", `${created.planId}.json`), `${JSON.stringify(stalePlan)}\n`);
     const outcome = await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
@@ -671,7 +685,7 @@ test("finishRun derives ledger expectations from the selected case slice", async
         });
         const scopedReport = {
             ...report,
-            ledger: summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount)
+            ledger: yield* summarizeEvalRunLedger(report.comparisons, expected.expectedCallCount, testCostService)
         };
         const finished = yield* workflow.finishRun(root, scopedReport);
         return { finished, report: scopedReport };
@@ -738,7 +752,8 @@ test("failed runs preserve plan revisions for retry while mismatched plans remai
                 knownOutputTokens: 0,
                 unknownTokenMeasurements: 0,
                 knownPricedSubtotalUsd: 0,
-                unpricedCalls: 0
+                unpricedCalls: 0,
+                subscriptionUnitCalls: 0
             },
             failure: "qualification timed out before observing any calls",
             errors: []
@@ -842,7 +857,8 @@ test("failed run reports persist the nested qualification error chain", async ()
                 knownOutputTokens: 0,
                 unknownTokenMeasurements: 0,
                 knownPricedSubtotalUsd: 0,
-                unpricedCalls: 0
+                unpricedCalls: 0,
+                subscriptionUnitCalls: 0
             },
             failure: 'qualification dimension "provider-protocol-translation" failed ' +
                 "(per-test timeout 600000ms); observed call ids: none",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.3.1",
+  "version": "1.4.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -31,9 +31,10 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.3.1",
-    "@velum-labs/routekit-eval-core": "1.3.1",
-    "@velum-labs/routekit-runtime": "1.3.1"
+    "@velum-labs/routekit-eval-contracts": "1.4.0",
+    "@velum-labs/routekit-eval-core": "1.4.0",
+    "@velum-labs/routekit-registry": "1.4.0",
+    "@velum-labs/routekit-runtime": "1.4.0"
   },
   "keywords": [
     "routekit",