@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/project-contracts.d.ts

@@ -1,5 +1,6 @@
 import { type EvalComparisonResult as EvalComparisonResultType } from "@velum-labs/routekit-eval-contracts";
-import { Schema } from "effect";
+import { type CostServiceShape } from "@velum-labs/routekit-registry";
+import { Effect, Schema } from "effect";
 export declare const EVAL_PROJECT_VERSION: 1;
 export declare const EVAL_PROJECT_V2_VERSION: 2;
 export declare const EvalProjectQuestion: Schema.Struct<{
@@ -4151,7 +4152,71 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     }>;
 }>]>]>;
 export type EvalProjectDocument = typeof EvalProjectDocument.Type;
+export declare const EvalEvidenceBlock: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly path: Schema.String;
+    readonly sourceDigest: Schema.String;
+    readonly contentDigest: Schema.String;
+    readonly ordinal: Schema.Finite;
+    readonly startLine: Schema.Finite;
+    readonly endLine: Schema.Finite;
+    readonly text: Schema.String;
+}>;
+export type EvalEvidenceBlock = typeof EvalEvidenceBlock.Type;
+export declare const EvalEvidenceSource: Schema.Struct<{
+    readonly path: Schema.String;
+    readonly sourceDigest: Schema.String;
+    readonly sourceBytes: Schema.optionalKey<Schema.Finite>;
+    readonly sourceBlockCount: Schema.optionalKey<Schema.Finite>;
+    readonly blocks: Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly path: Schema.String;
+        readonly sourceDigest: Schema.String;
+        readonly contentDigest: Schema.String;
+        readonly ordinal: Schema.Finite;
+        readonly startLine: Schema.Finite;
+        readonly endLine: Schema.Finite;
+        readonly text: Schema.String;
+    }>>;
+}>;
+export type EvalEvidenceSource = typeof EvalEvidenceSource.Type;
+export declare const EvalCriterion: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly description: Schema.String;
+    readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+    readonly gradingMode: Schema.Literal<"semantic">;
+}>;
+export type EvalCriterion = typeof EvalCriterion.Type;
+export declare const EvalPositiveControl: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly response: Schema.String;
+}>;
+export type EvalPositiveControl = typeof EvalPositiveControl.Type;
+export declare const EvalNegativeControl: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly response: Schema.String;
+    readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+    readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+}>;
+export type EvalNegativeControl = typeof EvalNegativeControl.Type;
 export declare const EvalDimensionCase: Schema.Struct<{
+    readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+    readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly description: Schema.String;
+        readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+        readonly gradingMode: Schema.Literal<"semantic">;
+    }>>>;
+    readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+    }>>>;
+    readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+        readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+        readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+    }>>>;
     readonly id: Schema.String;
     readonly prompt: Schema.String;
     readonly context: Schema.optionalKey<Schema.String>;
@@ -4163,6 +4228,23 @@ export declare const EvalDimensionSuite: Schema.Struct<{
     readonly dimensionId: Schema.String;
     readonly maximumOutputTokens: Schema.Finite;
     readonly cases: Schema.$Array<Schema.Struct<{
+        readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+        readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly description: Schema.String;
+            readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+            readonly gradingMode: Schema.Literal<"semantic">;
+        }>>>;
+        readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+        }>>>;
+        readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+            readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+            readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+        }>>>;
         readonly id: Schema.String;
         readonly prompt: Schema.String;
         readonly context: Schema.optionalKey<Schema.String>;
@@ -4198,10 +4280,6 @@ export declare const EvalDecompositionBenchmark: Schema.Struct<{
 }>;
 export type EvalDecompositionBenchmark = typeof EvalDecompositionBenchmark.Type;
 export declare const EvalCompositionCase: Schema.Struct<{
-    readonly id: Schema.String;
-    readonly prompt: Schema.String;
-    readonly context: Schema.optionalKey<Schema.String>;
-    readonly rubric: Schema.String;
     readonly decomposition: Schema.Struct<{
         readonly weights: Schema.$Array<Schema.Struct<{
             readonly dimensionId: Schema.String;
@@ -4216,6 +4294,27 @@ export declare const EvalCompositionCase: Schema.Struct<{
         readonly inputTokens: Schema.optionalKey<Schema.Finite>;
         readonly maxOutputTokens: Schema.optionalKey<Schema.Finite>;
     }>;
+    readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+    readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly description: Schema.String;
+        readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+        readonly gradingMode: Schema.Literal<"semantic">;
+    }>>>;
+    readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+    }>>>;
+    readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+        readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+        readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+    }>>>;
+    readonly id: Schema.String;
+    readonly prompt: Schema.String;
+    readonly context: Schema.optionalKey<Schema.String>;
+    readonly rubric: Schema.String;
 }>;
 export type EvalCompositionCase = typeof EvalCompositionCase.Type;
 export declare const EvalCompositionSuite: Schema.Struct<{
@@ -4223,10 +4322,6 @@ export declare const EvalCompositionSuite: Schema.Struct<{
     readonly minimumWinnerScoreGap: Schema.Finite;
     readonly minimumWinnerAgreement: Schema.Finite;
     readonly cases: Schema.$Array<Schema.Struct<{
-        readonly id: Schema.String;
-        readonly prompt: Schema.String;
-        readonly context: Schema.optionalKey<Schema.String>;
-        readonly rubric: Schema.String;
         readonly decomposition: Schema.Struct<{
             readonly weights: Schema.$Array<Schema.Struct<{
                 readonly dimensionId: Schema.String;
@@ -4241,6 +4336,27 @@ export declare const EvalCompositionSuite: Schema.Struct<{
             readonly inputTokens: Schema.optionalKey<Schema.Finite>;
             readonly maxOutputTokens: Schema.optionalKey<Schema.Finite>;
         }>;
+        readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+        readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly description: Schema.String;
+            readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+            readonly gradingMode: Schema.Literal<"semantic">;
+        }>>>;
+        readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+        }>>>;
+        readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+            readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+            readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+        }>>>;
+        readonly id: Schema.String;
+        readonly prompt: Schema.String;
+        readonly context: Schema.optionalKey<Schema.String>;
+        readonly rubric: Schema.String;
     }>>;
 }>;
 export type EvalCompositionSuite = typeof EvalCompositionSuite.Type;
@@ -4251,11 +4367,44 @@ export declare const EvalEvaluationProposal: Schema.Struct<{
     readonly candidateModels: Schema.$Array<Schema.String>;
     readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly judgeModel: Schema.String;
+    readonly evidenceSources: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly path: Schema.String;
+        readonly sourceDigest: Schema.String;
+        readonly sourceBytes: Schema.optionalKey<Schema.Finite>;
+        readonly sourceBlockCount: Schema.optionalKey<Schema.Finite>;
+        readonly blocks: Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly path: Schema.String;
+            readonly sourceDigest: Schema.String;
+            readonly contentDigest: Schema.String;
+            readonly ordinal: Schema.Finite;
+            readonly startLine: Schema.Finite;
+            readonly endLine: Schema.Finite;
+            readonly text: Schema.String;
+        }>>;
+    }>>>;
     readonly suites: Schema.$Array<Schema.Struct<{
         readonly version: Schema.Literal<1>;
         readonly dimensionId: Schema.String;
         readonly maximumOutputTokens: Schema.Finite;
         readonly cases: Schema.$Array<Schema.Struct<{
+            readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+            readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly description: Schema.String;
+                readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+                readonly gradingMode: Schema.Literal<"semantic">;
+            }>>>;
+            readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+            }>>>;
+            readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+                readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+                readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+            }>>>;
             readonly id: Schema.String;
             readonly prompt: Schema.String;
             readonly context: Schema.optionalKey<Schema.String>;
@@ -4281,10 +4430,6 @@ export declare const EvalEvaluationProposal: Schema.Struct<{
         readonly minimumWinnerScoreGap: Schema.Finite;
         readonly minimumWinnerAgreement: Schema.Finite;
         readonly cases: Schema.$Array<Schema.Struct<{
-            readonly id: Schema.String;
-            readonly prompt: Schema.String;
-            readonly context: Schema.optionalKey<Schema.String>;
-            readonly rubric: Schema.String;
             readonly decomposition: Schema.Struct<{
                 readonly weights: Schema.$Array<Schema.Struct<{
                     readonly dimensionId: Schema.String;
@@ -4299,6 +4444,27 @@ export declare const EvalEvaluationProposal: Schema.Struct<{
                 readonly inputTokens: Schema.optionalKey<Schema.Finite>;
                 readonly maxOutputTokens: Schema.optionalKey<Schema.Finite>;
             }>;
+            readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+            readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly description: Schema.String;
+                readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+                readonly gradingMode: Schema.Literal<"semantic">;
+            }>>>;
+            readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+            }>>>;
+            readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+                readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+                readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+            }>>>;
+            readonly id: Schema.String;
+            readonly prompt: Schema.String;
+            readonly context: Schema.optionalKey<Schema.String>;
+            readonly rubric: Schema.String;
         }>>;
     }>;
 }>;
@@ -4308,10 +4474,23 @@ export declare const EvalArtifactApproval: Schema.Struct<{
     readonly kind: Schema.Literals<readonly ["routing-basis", "evaluations"]>;
     readonly digest: Schema.String;
     readonly approvedAt: Schema.String;
+    readonly structurePolicyVersion: Schema.optionalKey<Schema.Finite>;
+    readonly structureReportDigest: Schema.optionalKey<Schema.String>;
 }>;
 export type EvalArtifactApproval = typeof EvalArtifactApproval.Type;
 export declare const EvalPlanScope: Schema.Literals<readonly ["pilot", "full"]>;
 export type EvalPlanScope = typeof EvalPlanScope.Type;
+export declare const EvalCostEstimate: Schema.Struct<{
+    readonly callCount: Schema.Finite;
+    readonly maximumInputTokens: Schema.Finite;
+    readonly maximumOutputTokens: Schema.Finite;
+    readonly maximumCostUsd: Schema.optionalKey<Schema.Finite>;
+    readonly knownPricedMaximumUsd: Schema.Finite;
+    readonly unpricedCalls: Schema.Finite;
+    readonly subscriptionUnitCalls: Schema.Finite;
+    readonly unknowns: Schema.$Array<Schema.String>;
+}>;
+export type EvalCostEstimate = typeof EvalCostEstimate.Type;
 export declare const EvalExecutionPlan: Schema.Struct<{
     readonly version: Schema.Literal<1>;
     readonly planId: Schema.String;
@@ -4319,6 +4498,7 @@ export declare const EvalExecutionPlan: Schema.Struct<{
     readonly projectRevision: Schema.Finite;
     readonly createdAt: Schema.String;
     readonly scope: Schema.Literals<readonly ["pilot", "full"]>;
+    readonly repetitions: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     readonly basisDigest: Schema.String;
     readonly evaluationDigest: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
@@ -4341,6 +4521,16 @@ export declare const EvalExecutionPlan: Schema.Struct<{
     readonly expectedCandidateCalls: Schema.Finite;
     readonly expectedJudgeCalls: Schema.Finite;
     readonly expectedCallCount: Schema.Finite;
+    readonly costEstimate: Schema.optionalKey<Schema.Struct<{
+        readonly callCount: Schema.Finite;
+        readonly maximumInputTokens: Schema.Finite;
+        readonly maximumOutputTokens: Schema.Finite;
+        readonly maximumCostUsd: Schema.optionalKey<Schema.Finite>;
+        readonly knownPricedMaximumUsd: Schema.Finite;
+        readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.Finite;
+        readonly unknowns: Schema.$Array<Schema.String>;
+    }>>;
 }>;
 export type EvalExecutionPlan = typeof EvalExecutionPlan.Type;
 export type EvalScopedExpectedCalls = {
@@ -4354,6 +4544,8 @@ export type EvalScopedExpectedCalls = {
     readonly expectedCallCount: number;
 };
 export declare function scopedExpectedCalls(plan: Pick<EvalExecutionPlan, "candidateModels" | "selectedCaseIds" | "selectedDecompositionCaseIds" | "selectedCompositionCaseIds">): EvalScopedExpectedCalls;
+export declare const EVAL_MAXIMUM_INPUT_TOKENS_PER_CALL: number;
+export declare function estimateEvalPlanCost(plan: Pick<EvalExecutionPlan, "candidateModels" | "candidateReasoningEfforts" | "classifierModel" | "judgeModel" | "maximumOutputTokens" | "expectedDimensionCandidateCalls" | "expectedClassifierCalls" | "expectedCompositionCandidateCalls">, costService: CostServiceShape): Effect.Effect<EvalCostEstimate>;
 export declare const EvalRunTarget: Schema.Struct<{
     readonly kind: Schema.Literals<readonly ["configured", "external"]>;
     readonly identity: Schema.String;
@@ -4375,6 +4567,7 @@ export declare const EvalRunLedger: Schema.Struct<{
     readonly unknownTokenMeasurements: Schema.Finite;
     readonly knownPricedSubtotalUsd: Schema.Finite;
     readonly unpricedCalls: Schema.Finite;
+    readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
 }>;
 export type EvalRunLedger = typeof EvalRunLedger.Type;
 export declare const EvalClassifierObservation: Schema.Struct<{
@@ -4397,10 +4590,13 @@ export declare const EvalClassifierObservation: Schema.Struct<{
 export type EvalClassifierObservation = typeof EvalClassifierObservation.Type;
 export declare const EvalCompositionCaseResult: Schema.Struct<{
     readonly caseId: Schema.String;
-    readonly predictedWinner: Schema.String;
-    readonly observedWinner: Schema.String;
+    readonly predictedWinner: Schema.NullOr<Schema.String>;
+    readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+    readonly observedWinner: Schema.NullOr<Schema.String>;
+    readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
     readonly predictedScoreGap: Schema.Finite;
     readonly observedScoreGap: Schema.Finite;
+    readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
     readonly passed: Schema.Boolean;
 }>;
 export type EvalCompositionCaseResult = typeof EvalCompositionCaseResult.Type;
@@ -4431,19 +4627,25 @@ export declare const EvalRunQualification: Schema.Struct<{
         readonly expectedCases: Schema.Finite;
         readonly comparableCases: Schema.Finite;
         readonly agreeingCases: Schema.Finite;
-        readonly winnerAgreement: Schema.Finite;
+        readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+        readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+        readonly reason: Schema.optionalKey<Schema.String>;
+        readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
         readonly cases: Schema.$Array<Schema.Struct<{
             readonly caseId: Schema.String;
-            readonly predictedWinner: Schema.String;
-            readonly observedWinner: Schema.String;
+            readonly predictedWinner: Schema.NullOr<Schema.String>;
+            readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+            readonly observedWinner: Schema.NullOr<Schema.String>;
+            readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
             readonly predictedScoreGap: Schema.Finite;
             readonly observedScoreGap: Schema.Finite;
+            readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly passed: Schema.Boolean;
         }>>;
     }>;
 }>;
 export type EvalRunQualification = typeof EvalRunQualification.Type;
-export declare function summarizeEvalRunLedger(comparisons: readonly EvalComparisonResultType[], expectedCalls: number, classifierObservations?: readonly EvalClassifierObservation[]): EvalRunLedger;
+export declare function summarizeEvalRunLedger(comparisons: readonly EvalComparisonResultType[], expectedCalls: number, costService: CostServiceShape, classifierObservations?: readonly EvalClassifierObservation[], classifierModel?: string): Effect.Effect<EvalRunLedger>;
 export declare const EvalRunFailureError: Schema.Struct<{
     readonly name: Schema.String;
     readonly message: Schema.String;
@@ -4597,13 +4799,19 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
             readonly expectedCases: Schema.Finite;
             readonly comparableCases: Schema.Finite;
             readonly agreeingCases: Schema.Finite;
-            readonly winnerAgreement: Schema.Finite;
+            readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+            readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+            readonly reason: Schema.optionalKey<Schema.String>;
+            readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly cases: Schema.$Array<Schema.Struct<{
                 readonly caseId: Schema.String;
-                readonly predictedWinner: Schema.String;
-                readonly observedWinner: Schema.String;
+                readonly predictedWinner: Schema.NullOr<Schema.String>;
+                readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+                readonly observedWinner: Schema.NullOr<Schema.String>;
+                readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
                 readonly predictedScoreGap: Schema.Finite;
                 readonly observedScoreGap: Schema.Finite;
+                readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
                 readonly passed: Schema.Boolean;
             }>>;
         }>;
@@ -4617,65 +4825,66 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly unknownTokenMeasurements: Schema.Finite;
         readonly knownPricedSubtotalUsd: Schema.Finite;
         readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     }>;
+    readonly events: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly name: Schema.Literal<"plan">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly plannedCalls: Schema.Finite;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"candidate#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly name: Schema.Literal<"failure-detail">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+        readonly reason: Schema.String;
+        readonly retryable: Schema.Boolean;
+        readonly callId: Schema.optionalKey<Schema.String>;
+        readonly reservationId: Schema.optionalKey<Schema.String>;
+        readonly accounting: Schema.Struct<{
+            readonly expectedCalls: Schema.Finite;
+            readonly observedCalls: Schema.Finite;
+            readonly knownInputTokens: Schema.Finite;
+            readonly knownOutputTokens: Schema.Finite;
+            readonly unknownTokenMeasurements: Schema.Finite;
+            readonly knownPricedSubtotalUsd: Schema.Finite;
+            readonly unpricedCalls: Schema.Finite;
+            readonly subscriptionUnitCalls: Schema.Finite;
+        }>;
+        readonly cleanup: Schema.Struct<{
+            readonly sessionOpened: Schema.Boolean;
+            readonly sessionClosed: Schema.Boolean;
+            readonly detail: Schema.optionalKey<Schema.String>;
+        }>;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"basis">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"dimensions">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"activation">;
+    }>]>>>, never>;
 }>, Schema.Struct<{
     readonly status: Schema.Literal<"completed">;
-    readonly activation: Schema.Struct<{
-        readonly version: Schema.Literal<2>;
-        readonly generatedAt: Schema.String;
-        readonly basisDigest: Schema.String;
-        readonly evidenceDigest: Schema.String;
-        readonly classifierModel: Schema.String;
-        readonly objective: Schema.Union<readonly [Schema.Struct<{
-            readonly kind: Schema.Literal<"highest-quality">;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"lowest-cost">;
-            readonly minimumQuality: Schema.Finite;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"lowest-latency">;
-            readonly minimumQuality: Schema.Finite;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"balanced">;
-            readonly minimumQuality: Schema.Finite;
-            readonly weights: Schema.Struct<{
-                readonly quality: Schema.Finite;
-                readonly cost: Schema.Finite;
-                readonly latency: Schema.Finite;
-            }>;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"pareto">;
-            readonly minimumQuality: Schema.Finite;
-            readonly preference: Schema.Literals<readonly ["quality", "cost", "latency"]>;
-        }>]>;
-        readonly maximumUnknownWeight: Schema.Finite;
-        readonly constraints: Schema.optionalKey<Schema.Struct<{
-            readonly minimumDimensionQuality: Schema.optionalKey<Schema.$Record<Schema.String, Schema.Finite>>;
-            readonly maximumFailureRate: Schema.optionalKey<Schema.Finite>;
-        }>>;
-        readonly dimensions: Schema.$Array<Schema.Struct<{
-            readonly id: Schema.String;
-            readonly description: Schema.String;
-            readonly includes: Schema.$Array<Schema.String>;
-            readonly excludes: Schema.$Array<Schema.String>;
-        }>>;
-        readonly candidateModels: Schema.$Array<Schema.String>;
-        readonly evidence: Schema.$Array<Schema.Struct<{
-            readonly model: Schema.String;
-            readonly dimensionId: Schema.String;
-            readonly suiteDigest: Schema.String;
-            readonly evidenceDigest: Schema.String;
-            readonly quality: Schema.Struct<{
-                readonly passRate: Schema.Finite;
-                readonly lowerConfidenceBound: Schema.Finite;
-                readonly sampleCount: Schema.Finite;
-            }>;
-            readonly failureRate: Schema.Finite;
-            readonly averageJudgeScore: Schema.optionalKey<Schema.Finite>;
-            readonly p95DurationMs: Schema.optionalKey<Schema.Finite>;
-            readonly averageCostUsd: Schema.optionalKey<Schema.Finite>;
-            readonly unpricedCalls: Schema.Finite;
-        }>>;
-    }>;
     readonly version: Schema.Literal<1>;
     readonly runId: Schema.String;
     readonly planId: Schema.String;
@@ -4757,13 +4966,19 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
             readonly expectedCases: Schema.Finite;
             readonly comparableCases: Schema.Finite;
             readonly agreeingCases: Schema.Finite;
-            readonly winnerAgreement: Schema.Finite;
+            readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+            readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+            readonly reason: Schema.optionalKey<Schema.String>;
+            readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly cases: Schema.$Array<Schema.Struct<{
                 readonly caseId: Schema.String;
-                readonly predictedWinner: Schema.String;
-                readonly observedWinner: Schema.String;
+                readonly predictedWinner: Schema.NullOr<Schema.String>;
+                readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+                readonly observedWinner: Schema.NullOr<Schema.String>;
+                readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
                 readonly predictedScoreGap: Schema.Finite;
                 readonly observedScoreGap: Schema.Finite;
+                readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
                 readonly passed: Schema.Boolean;
             }>>;
         }>;
@@ -4777,7 +4992,64 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly unknownTokenMeasurements: Schema.Finite;
         readonly knownPricedSubtotalUsd: Schema.Finite;
         readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     }>;
+    readonly events: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly name: Schema.Literal<"plan">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly plannedCalls: Schema.Finite;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"candidate#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly name: Schema.Literal<"failure-detail">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+        readonly reason: Schema.String;
+        readonly retryable: Schema.Boolean;
+        readonly callId: Schema.optionalKey<Schema.String>;
+        readonly reservationId: Schema.optionalKey<Schema.String>;
+        readonly accounting: Schema.Struct<{
+            readonly expectedCalls: Schema.Finite;
+            readonly observedCalls: Schema.Finite;
+            readonly knownInputTokens: Schema.Finite;
+            readonly knownOutputTokens: Schema.Finite;
+            readonly unknownTokenMeasurements: Schema.Finite;
+            readonly knownPricedSubtotalUsd: Schema.Finite;
+            readonly unpricedCalls: Schema.Finite;
+            readonly subscriptionUnitCalls: Schema.Finite;
+        }>;
+        readonly cleanup: Schema.Struct<{
+            readonly sessionOpened: Schema.Boolean;
+            readonly sessionClosed: Schema.Boolean;
+            readonly detail: Schema.optionalKey<Schema.String>;
+        }>;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"basis">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"dimensions">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"activation">;
+    }>]>>>, never>;
 }>, Schema.Struct<{
     readonly status: Schema.Literal<"failed">;
     readonly failure: Schema.String;
@@ -4867,13 +5139,19 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
             readonly expectedCases: Schema.Finite;
             readonly comparableCases: Schema.Finite;
             readonly agreeingCases: Schema.Finite;
-            readonly winnerAgreement: Schema.Finite;
+            readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+            readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+            readonly reason: Schema.optionalKey<Schema.String>;
+            readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly cases: Schema.$Array<Schema.Struct<{
                 readonly caseId: Schema.String;
-                readonly predictedWinner: Schema.String;
-                readonly observedWinner: Schema.String;
+                readonly predictedWinner: Schema.NullOr<Schema.String>;
+                readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+                readonly observedWinner: Schema.NullOr<Schema.String>;
+                readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
                 readonly predictedScoreGap: Schema.Finite;
                 readonly observedScoreGap: Schema.Finite;
+                readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
                 readonly passed: Schema.Boolean;
             }>>;
         }>;
@@ -4887,7 +5165,64 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly unknownTokenMeasurements: Schema.Finite;
         readonly knownPricedSubtotalUsd: Schema.Finite;
         readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.withDecodingDefaultKey<Schema.Finite, never>;
     }>;
+    readonly events: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly name: Schema.Literal<"plan">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly plannedCalls: Schema.Finite;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"candidate#">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly dimensionId: Schema.String;
+        readonly caseId: Schema.String;
+        readonly sequence: Schema.Finite;
+        readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly name: Schema.Literal<"failure-detail">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+        readonly reason: Schema.String;
+        readonly retryable: Schema.Boolean;
+        readonly callId: Schema.optionalKey<Schema.String>;
+        readonly reservationId: Schema.optionalKey<Schema.String>;
+        readonly accounting: Schema.Struct<{
+            readonly expectedCalls: Schema.Finite;
+            readonly observedCalls: Schema.Finite;
+            readonly knownInputTokens: Schema.Finite;
+            readonly knownOutputTokens: Schema.Finite;
+            readonly unknownTokenMeasurements: Schema.Finite;
+            readonly knownPricedSubtotalUsd: Schema.Finite;
+            readonly unpricedCalls: Schema.Finite;
+            readonly subscriptionUnitCalls: Schema.Finite;
+        }>;
+        readonly cleanup: Schema.Struct<{
+            readonly sessionOpened: Schema.Boolean;
+            readonly sessionClosed: Schema.Boolean;
+            readonly detail: Schema.optionalKey<Schema.String>;
+        }>;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"basis">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"dimensions">;
+    }>, Schema.Struct<{
+        readonly at: Schema.String;
+        readonly operationId: Schema.String;
+        readonly name: Schema.Literal<"activation">;
+    }>]>>>, never>;
 }>]>;
 export type EvalRunReport = typeof EvalRunReport.Type;
 export type EvalProjectArtifactsStatus = {