@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/project-contracts.js

@@ -1,5 +1,6 @@
 import { DecompositionResult, DimensionScoreLabelV3, DimensionScorePredictionV3, EvalCandidateReasoningEfforts, EvalComparisonResult, EvalReasoningEffort, PublishedRoutingActivation, RoutingBasisApprovalV3, RoutingBasisV3, RoutingActivationConstraints, RequestRoutingRequirements, RoutingObjectivePolicy, WorkloadDimension } from "@velum-labs/routekit-eval-contracts";
-import { Schema } from "effect";
+import { Effect, Schema } from "effect";
+import { EvalNamedEventLog } from "./eval-event-log.js";
 export const EVAL_PROJECT_VERSION = 1;
 export const EVAL_PROJECT_V2_VERSION = 2;
 export const EvalProjectQuestion = Schema.Struct({
@@ -526,11 +527,58 @@ export const EvalProjectDocument = Schema.Union([
     EvalProjectState,
     EvalProjectStateV2
 ]);
+export const EvalEvidenceBlock = Schema.Struct({
+    id: Schema.String,
+    path: Schema.String,
+    sourceDigest: Schema.String,
+    contentDigest: Schema.String,
+    ordinal: NonNegativeInteger,
+    startLine: NonNegativeInteger,
+    endLine: NonNegativeInteger,
+    text: Schema.String
+});
+export const EvalEvidenceSource = Schema.Struct({
+    path: Schema.String,
+    sourceDigest: Schema.String,
+    sourceBytes: Schema.optionalKey(NonNegativeInteger),
+    sourceBlockCount: Schema.optionalKey(NonNegativeInteger),
+    blocks: Schema.Array(EvalEvidenceBlock)
+});
+export const EvalCriterion = Schema.Struct({
+    id: Schema.String,
+    description: Schema.String,
+    importance: Schema.Literals(["critical", "supporting"]),
+    gradingMode: Schema.Literal("semantic")
+});
+export const EvalPositiveControl = Schema.Struct({
+    id: Schema.String,
+    response: Schema.String
+});
+export const EvalNegativeControl = Schema.Struct({
+    id: Schema.String,
+    response: Schema.String,
+    targetedCriterionIds: Schema.Array(Schema.String),
+    kind: Schema.Literals([
+        "missing-criterion",
+        "forbidden-behavior",
+        "non-responsive"
+    ])
+});
+const EvalCompiledCase = {
+    // Optional only at the persisted decoding boundary. The current evaluation
+    // structure policy requires compiler-owned evidence and hidden controls
+    // before approval, planning, or execution.
+    evidenceIds: Schema.optionalKey(Schema.Array(Schema.String)),
+    criteria: Schema.optionalKey(Schema.Array(EvalCriterion)),
+    positiveControls: Schema.optionalKey(Schema.Array(EvalPositiveControl)),
+    negativeControls: Schema.optionalKey(Schema.Array(EvalNegativeControl))
+};
 export const EvalDimensionCase = Schema.Struct({
     id: Schema.String,
     prompt: Schema.String,
     context: Schema.optionalKey(Schema.String),
-    rubric: Schema.String
+    rubric: Schema.String,
+    ...EvalCompiledCase
 });
 export const EvalDimensionSuite = Schema.Struct({
     version: Schema.Literal(EVAL_PROJECT_VERSION),
@@ -552,6 +600,7 @@ export const EvalCompositionCase = Schema.Struct({
     prompt: Schema.String,
     context: Schema.optionalKey(Schema.String),
     rubric: Schema.String,
+    ...EvalCompiledCase,
     decomposition: DecompositionResult,
     requirements: RequestRoutingRequirements
 });
@@ -568,6 +617,7 @@ export const EvalEvaluationProposal = Schema.Struct({
     candidateModels: Schema.Array(Schema.String),
     candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     judgeModel: Schema.String,
+    evidenceSources: Schema.optionalKey(Schema.Array(EvalEvidenceSource)),
     suites: Schema.Array(EvalDimensionSuite),
     decompositionBenchmark: EvalDecompositionBenchmark,
     compositionSuite: EvalCompositionSuite
@@ -576,9 +626,21 @@ export const EvalArtifactApproval = Schema.Struct({
     version: Schema.Literal(EVAL_PROJECT_VERSION),
     kind: Schema.Literals(["routing-basis", "evaluations"]),
     digest: Schema.String,
-    approvedAt: Schema.String
+    approvedAt: Schema.String,
+    structurePolicyVersion: Schema.optionalKey(NonNegativeInteger),
+    structureReportDigest: Schema.optionalKey(Schema.String)
 });
 export const EvalPlanScope = Schema.Literals(["pilot", "full"]);
+export const EvalCostEstimate = Schema.Struct({
+    callCount: NonNegativeInteger,
+    maximumInputTokens: NonNegativeInteger,
+    maximumOutputTokens: NonNegativeInteger,
+    maximumCostUsd: Schema.optionalKey(NonNegativeFinite),
+    knownPricedMaximumUsd: NonNegativeFinite,
+    unpricedCalls: NonNegativeInteger,
+    subscriptionUnitCalls: NonNegativeInteger,
+    unknowns: Schema.Array(Schema.String)
+});
 export const EvalExecutionPlan = Schema.Struct({
     version: Schema.Literal(EVAL_PROJECT_VERSION),
     planId: Schema.String,
@@ -586,6 +648,7 @@ export const EvalExecutionPlan = Schema.Struct({
     projectRevision: NonNegativeInteger,
     createdAt: Schema.String,
     scope: EvalPlanScope,
+    repetitions: NonNegativeInteger.pipe(Schema.withDecodingDefaultKey(Effect.succeed(1))),
     basisDigest: Schema.String,
     evaluationDigest: Schema.String,
     candidateModels: Schema.Array(Schema.String),
@@ -607,7 +670,8 @@ export const EvalExecutionPlan = Schema.Struct({
     expectedCompositionJudgeCalls: NonNegativeInteger,
     expectedCandidateCalls: NonNegativeInteger,
     expectedJudgeCalls: NonNegativeInteger,
-    expectedCallCount: NonNegativeInteger
+    expectedCallCount: NonNegativeInteger,
+    costEstimate: Schema.optionalKey(EvalCostEstimate)
 });
 export function scopedExpectedCalls(plan) {
     const candidateModelCount = plan.candidateModels.length;
@@ -630,6 +694,44 @@ export function scopedExpectedCalls(plan) {
         expectedCallCount: expectedCandidateCalls + expectedJudgeCalls + expectedClassifierCalls
     };
 }
+export const EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL = 256 * 1024;
+export function estimateEvalPlanCost(plan, costService) {
+    const candidateCaseCalls = plan.expectedDimensionCandidateCalls + plan.expectedCompositionCandidateCalls;
+    const callsPerCandidate = plan.candidateModels.length === 0 ? 0 : candidateCaseCalls / plan.candidateModels.length;
+    const calls = [
+        ...plan.candidateModels.map((model) => ({
+            model,
+            callCount: callsPerCandidate,
+            maximumInputTokens: EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: plan.maximumOutputTokens,
+            ...(plan.candidateReasoningEfforts?.[model] === undefined
+                ? {}
+                : { reasoningEffort: plan.candidateReasoningEfforts[model] })
+        })),
+        {
+            model: plan.judgeModel,
+            callCount: candidateCaseCalls,
+            maximumInputTokens: EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: plan.maximumOutputTokens
+        },
+        {
+            model: plan.classifierModel,
+            callCount: plan.expectedClassifierCalls,
+            maximumInputTokens: EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL,
+            maximumOutputTokens: plan.maximumOutputTokens
+        }
+    ];
+    return costService.estimate(calls).pipe(Effect.map((estimate) => ({
+        callCount: estimate.callCount,
+        maximumInputTokens: estimate.knownInputTokens,
+        maximumOutputTokens: estimate.knownOutputTokens,
+        ...(estimate.maximumCostUsd === undefined ? {} : { maximumCostUsd: estimate.maximumCostUsd }),
+        knownPricedMaximumUsd: estimate.knownPricedMaximumUsd,
+        unpricedCalls: estimate.unpricedCalls + estimate.upstreamManagedCalls + estimate.unknownBillingCalls,
+        subscriptionUnitCalls: estimate.subscriptionUnitCalls,
+        unknowns: estimate.unknowns
+    })));
+}
 export const EvalRunTarget = Schema.Struct({
     kind: Schema.Literals(["configured", "external"]),
     identity: Schema.String,
@@ -648,7 +750,8 @@ export const EvalRunLedger = Schema.Struct({
     knownOutputTokens: NonNegativeInteger,
     unknownTokenMeasurements: NonNegativeInteger,
     knownPricedSubtotalUsd: NonNegativeFinite,
-    unpricedCalls: NonNegativeInteger
+    unpricedCalls: NonNegativeInteger,
+    subscriptionUnitCalls: NonNegativeInteger.pipe(Schema.withDecodingDefaultKey(Effect.succeed(0)))
 });
 export const EvalClassifierObservation = Schema.Struct({
     caseId: Schema.String,
@@ -666,10 +769,13 @@ export const EvalClassifierObservation = Schema.Struct({
 });
 export const EvalCompositionCaseResult = Schema.Struct({
     caseId: Schema.String,
-    predictedWinner: Schema.String,
-    observedWinner: Schema.String,
+    predictedWinner: Schema.NullOr(Schema.String),
+    predictedTiedModels: Schema.optionalKey(Schema.Array(Schema.String)).pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))),
+    observedWinner: Schema.NullOr(Schema.String),
+    observedTiedModels: Schema.optionalKey(Schema.Array(Schema.String)).pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))),
     predictedScoreGap: NonNegativeFinite,
     observedScoreGap: NonNegativeFinite,
+    comparable: Schema.optionalKey(Schema.Boolean).pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))),
     passed: Schema.Boolean
 });
 export const EvalRunQualification = Schema.Struct({
@@ -683,18 +789,17 @@ export const EvalRunQualification = Schema.Struct({
         expectedCases: NonNegativeInteger,
         comparableCases: NonNegativeInteger,
         agreeingCases: NonNegativeInteger,
-        winnerAgreement: NonNegativeFinite,
+        winnerAgreement: Schema.optionalKey(NonNegativeFinite),
+        conclusion: Schema.optionalKey(Schema.Literals(["passed", "failed", "inconclusive"])).pipe(Schema.withDecodingDefaultKey(Effect.succeed("failed"))),
+        reason: Schema.optionalKey(Schema.String),
+        publishable: Schema.optionalKey(Schema.Boolean).pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))),
         cases: Schema.Array(EvalCompositionCaseResult)
     })
 });
-export function summarizeEvalRunLedger(comparisons, expectedCalls, classifierObservations = []) {
+export function summarizeEvalRunLedger(comparisons, expectedCalls, costService, classifierObservations = [], classifierModel = "unknown") {
     let observedCalls = 0;
     let observedCandidateRows = 0;
-    let knownInputTokens = 0;
-    let knownOutputTokens = 0;
-    let unknownTokenMeasurements = 0;
-    let knownPricedSubtotalUsd = 0;
-    let unpricedCalls = 0;
+    const actualCalls = [];
     for (const comparison of comparisons) {
         const calls = comparison.calls ??
             comparison.models.flatMap((model) => model.cases.map((testCase) => ({
@@ -707,49 +812,52 @@ export function summarizeEvalRunLedger(comparisons, expectedCalls, classifierObs
             observedCalls += 1;
             if (call.role === "candidate")
                 observedCandidateRows += 1;
-            if (call.measurement.inputTokens === undefined ||
-                call.measurement.outputTokens === undefined) {
-                unknownTokenMeasurements += 1;
-            }
-            else {
-                knownInputTokens += call.measurement.inputTokens;
-                knownOutputTokens += call.measurement.outputTokens;
-            }
-            if (call.measurement.costUsd === undefined) {
-                unpricedCalls += 1;
-            }
-            else {
-                knownPricedSubtotalUsd += call.measurement.costUsd;
-            }
+            actualCalls.push({
+                model: call.model,
+                usage: {
+                    ...(call.measurement.inputTokens === undefined
+                        ? {}
+                        : { inputTokens: call.measurement.inputTokens }),
+                    ...(call.measurement.inputTokens === undefined ? {} : { cachedInputTokens: 0 }),
+                    ...(call.measurement.outputTokens === undefined
+                        ? {}
+                        : { outputTokens: call.measurement.outputTokens })
+                },
+                ...(call.measurement.costUsd === undefined
+                    ? {}
+                    : { providerCostUsd: call.measurement.costUsd })
+            });
         }
     }
     for (const observation of classifierObservations) {
         observedCalls += 1;
-        if (observation.measurement.inputTokens === undefined ||
-            observation.measurement.outputTokens === undefined) {
-            unknownTokenMeasurements += 1;
-        }
-        else {
-            knownInputTokens += observation.measurement.inputTokens;
-            knownOutputTokens += observation.measurement.outputTokens;
-        }
-        if (observation.measurement.costUsd === undefined) {
-            unpricedCalls += 1;
-        }
-        else {
-            knownPricedSubtotalUsd += observation.measurement.costUsd;
-        }
+        actualCalls.push({
+            model: classifierModel,
+            usage: {
+                ...(observation.measurement.inputTokens === undefined
+                    ? {}
+                    : { inputTokens: observation.measurement.inputTokens }),
+                ...(observation.measurement.inputTokens === undefined ? {} : { cachedInputTokens: 0 }),
+                ...(observation.measurement.outputTokens === undefined
+                    ? {}
+                    : { outputTokens: observation.measurement.outputTokens })
+            },
+            ...(observation.measurement.costUsd === undefined
+                ? {}
+                : { providerCostUsd: observation.measurement.costUsd })
+        });
     }
-    return {
+    return costService.actuals(actualCalls).pipe(Effect.map((actuals) => ({
         expectedCalls,
         observedCalls,
         observedCandidateRows,
-        knownInputTokens,
-        knownOutputTokens,
-        unknownTokenMeasurements,
-        knownPricedSubtotalUsd,
-        unpricedCalls
-    };
+        knownInputTokens: actuals.knownInputTokens,
+        knownOutputTokens: actuals.knownOutputTokens,
+        unknownTokenMeasurements: actuals.unknownTokenCalls,
+        knownPricedSubtotalUsd: actuals.knownPricedSubtotalUsd,
+        unpricedCalls: actuals.unpricedCalls + actuals.upstreamManagedCalls + actuals.unknownBillingCalls,
+        subscriptionUnitCalls: actuals.subscriptionUnitCalls
+    })));
 }
 const EvalRunReportCommon = {
     version: Schema.Literal(EVAL_PROJECT_VERSION),
@@ -764,7 +872,8 @@ const EvalRunReportCommon = {
     cleanup: EvalRunCleanup,
     comparisons: Schema.Array(EvalComparisonResult),
     qualification: Schema.optionalKey(EvalRunQualification),
-    ledger: EvalRunLedger
+    ledger: EvalRunLedger,
+    events: Schema.optionalKey(EvalNamedEventLog).pipe(Schema.withDecodingDefaultKey(Effect.succeed([])))
 };
 export const EvalRunFailureError = Schema.Struct({
     name: Schema.String,
@@ -778,8 +887,7 @@ const QualifiedEvalRunReport = Schema.Struct({
 });
 const CompletedEvalRunReport = Schema.Struct({
     ...EvalRunReportCommon,
-    status: Schema.Literal("completed"),
-    activation: PublishedRoutingActivation
+    status: Schema.Literal("completed")
 });
 const FailedEvalRunReport = Schema.Struct({
     ...EvalRunReportCommon,

package/dist/project-store.js

@@ -14,6 +14,7 @@ const storeFailure = (operation, path, cause) => new EvalProjectStoreError({
 export const makeFileEvalProjectStore = Effect.gen(function* () {
     const fs = yield* FileSystem.FileSystem;
     const paths = yield* Path.Path;
+    const platform = yield* Effect.context();
     const projectPath = (root) => paths.join(paths.resolve(root), ".routekit", "evals", PROJECT_FILE);
     const loadDocument = (repositoryRoot) => Effect.gen(function* () {
         const root = yield* fs
@@ -45,7 +46,7 @@ export const makeFileEvalProjectStore = Effect.gen(function* () {
             .pipe(Effect.mapError((cause) => storeFailure("creating", directory, cause)));
         yield* writeFileAtomicEffect(target, `${JSON.stringify(state, null, 2)}\n`, {
             mode: 0o600
-        }).pipe(Effect.mapError((cause) => storeFailure("committing", target, cause)), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths));
+        }).pipe(Effect.mapError((cause) => storeFailure("committing", target, cause)), Effect.provide(platform));
     });
     return EvalProjectStore.of({
         loadDocument,

package/dist/project-workflow.d.ts

@@ -1,4 +1,5 @@
-import { Context, Effect, Layer, Path } from "effect";
+import { CostService } from "@velum-labs/routekit-registry";
+import { Context, Effect, FileSystem, Layer, Path } from "effect";
 import type { EvalProjectArtifactError, EvalSetupInspectionError } from "./errors.js";
 import { EvalDimensionContrastError, type EvalModelCatalogError, type EvalProjectStoreError, EvalProjectTransitionError } from "./errors.js";
 import { EvalRepositoryInspector } from "./inspection.js";
@@ -15,7 +16,7 @@ export type EvalProjectWorkflowShape = {
     readonly approveDimensions: (repositoryRoot: string, basisDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly proposeEvaluations: (repositoryRoot: string, proposal: Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "candidateReasoningEfforts" | "judgeModel">) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly approveEvaluations: (repositoryRoot: string, evaluationDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
-    readonly createPlan: (repositoryRoot: string, scope: EvalPlanScope) => Effect.Effect<EvalExecutionPlan, EvalProjectWorkflowError>;
+    readonly createPlan: (repositoryRoot: string, scope: EvalPlanScope, repetitions?: number) => Effect.Effect<EvalExecutionPlan, EvalProjectWorkflowError>;
     readonly startRun: (repositoryRoot: string, planId: string) => Effect.Effect<{
         readonly plan: EvalExecutionPlan;
         readonly runId: string;
@@ -28,6 +29,6 @@ export type EvalProjectWorkflowShape = {
 declare const EvalProjectWorkflow_base: Context.ServiceClass<EvalProjectWorkflow, "@velum-labs/routekit-eval-setup/EvalProjectWorkflow", EvalProjectWorkflowShape>;
 export declare class EvalProjectWorkflow extends EvalProjectWorkflow_base {
 }
-export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
-export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
+export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | FileSystem.FileSystem | EvalRepositoryInspector | EvalProjectStore | EvalModelCatalog | CostService>;
+export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | FileSystem.FileSystem | EvalRepositoryInspector | EvalProjectStore | EvalModelCatalog | CostService>;
 export {};