@velum-labs/routekit-eval-setup 1.4.0 → 1.5.0

package/dist/project-contracts.d.ts

@@ -4152,7 +4152,71 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     }>;
 }>]>]>;
 export type EvalProjectDocument = typeof EvalProjectDocument.Type;
+export declare const EvalEvidenceBlock: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly path: Schema.String;
+    readonly sourceDigest: Schema.String;
+    readonly contentDigest: Schema.String;
+    readonly ordinal: Schema.Finite;
+    readonly startLine: Schema.Finite;
+    readonly endLine: Schema.Finite;
+    readonly text: Schema.String;
+}>;
+export type EvalEvidenceBlock = typeof EvalEvidenceBlock.Type;
+export declare const EvalEvidenceSource: Schema.Struct<{
+    readonly path: Schema.String;
+    readonly sourceDigest: Schema.String;
+    readonly sourceBytes: Schema.optionalKey<Schema.Finite>;
+    readonly sourceBlockCount: Schema.optionalKey<Schema.Finite>;
+    readonly blocks: Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly path: Schema.String;
+        readonly sourceDigest: Schema.String;
+        readonly contentDigest: Schema.String;
+        readonly ordinal: Schema.Finite;
+        readonly startLine: Schema.Finite;
+        readonly endLine: Schema.Finite;
+        readonly text: Schema.String;
+    }>>;
+}>;
+export type EvalEvidenceSource = typeof EvalEvidenceSource.Type;
+export declare const EvalCriterion: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly description: Schema.String;
+    readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+    readonly gradingMode: Schema.Literal<"semantic">;
+}>;
+export type EvalCriterion = typeof EvalCriterion.Type;
+export declare const EvalPositiveControl: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly response: Schema.String;
+}>;
+export type EvalPositiveControl = typeof EvalPositiveControl.Type;
+export declare const EvalNegativeControl: Schema.Struct<{
+    readonly id: Schema.String;
+    readonly response: Schema.String;
+    readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+    readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+}>;
+export type EvalNegativeControl = typeof EvalNegativeControl.Type;
 export declare const EvalDimensionCase: Schema.Struct<{
+    readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+    readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly description: Schema.String;
+        readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+        readonly gradingMode: Schema.Literal<"semantic">;
+    }>>>;
+    readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+    }>>>;
+    readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+        readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+        readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+    }>>>;
     readonly id: Schema.String;
     readonly prompt: Schema.String;
     readonly context: Schema.optionalKey<Schema.String>;
@@ -4164,6 +4228,23 @@ export declare const EvalDimensionSuite: Schema.Struct<{
     readonly dimensionId: Schema.String;
     readonly maximumOutputTokens: Schema.Finite;
     readonly cases: Schema.$Array<Schema.Struct<{
+        readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+        readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly description: Schema.String;
+            readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+            readonly gradingMode: Schema.Literal<"semantic">;
+        }>>>;
+        readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+        }>>>;
+        readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+            readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+            readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+        }>>>;
         readonly id: Schema.String;
         readonly prompt: Schema.String;
         readonly context: Schema.optionalKey<Schema.String>;
@@ -4199,10 +4280,6 @@ export declare const EvalDecompositionBenchmark: Schema.Struct<{
 }>;
 export type EvalDecompositionBenchmark = typeof EvalDecompositionBenchmark.Type;
 export declare const EvalCompositionCase: Schema.Struct<{
-    readonly id: Schema.String;
-    readonly prompt: Schema.String;
-    readonly context: Schema.optionalKey<Schema.String>;
-    readonly rubric: Schema.String;
     readonly decomposition: Schema.Struct<{
         readonly weights: Schema.$Array<Schema.Struct<{
             readonly dimensionId: Schema.String;
@@ -4217,6 +4294,27 @@ export declare const EvalCompositionCase: Schema.Struct<{
         readonly inputTokens: Schema.optionalKey<Schema.Finite>;
         readonly maxOutputTokens: Schema.optionalKey<Schema.Finite>;
     }>;
+    readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+    readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly description: Schema.String;
+        readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+        readonly gradingMode: Schema.Literal<"semantic">;
+    }>>>;
+    readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+    }>>>;
+    readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly id: Schema.String;
+        readonly response: Schema.String;
+        readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+        readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+    }>>>;
+    readonly id: Schema.String;
+    readonly prompt: Schema.String;
+    readonly context: Schema.optionalKey<Schema.String>;
+    readonly rubric: Schema.String;
 }>;
 export type EvalCompositionCase = typeof EvalCompositionCase.Type;
 export declare const EvalCompositionSuite: Schema.Struct<{
@@ -4224,10 +4322,6 @@ export declare const EvalCompositionSuite: Schema.Struct<{
     readonly minimumWinnerScoreGap: Schema.Finite;
     readonly minimumWinnerAgreement: Schema.Finite;
     readonly cases: Schema.$Array<Schema.Struct<{
-        readonly id: Schema.String;
-        readonly prompt: Schema.String;
-        readonly context: Schema.optionalKey<Schema.String>;
-        readonly rubric: Schema.String;
         readonly decomposition: Schema.Struct<{
             readonly weights: Schema.$Array<Schema.Struct<{
                 readonly dimensionId: Schema.String;
@@ -4242,6 +4336,27 @@ export declare const EvalCompositionSuite: Schema.Struct<{
             readonly inputTokens: Schema.optionalKey<Schema.Finite>;
             readonly maxOutputTokens: Schema.optionalKey<Schema.Finite>;
         }>;
+        readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+        readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly description: Schema.String;
+            readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+            readonly gradingMode: Schema.Literal<"semantic">;
+        }>>>;
+        readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+        }>>>;
+        readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly response: Schema.String;
+            readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+            readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+        }>>>;
+        readonly id: Schema.String;
+        readonly prompt: Schema.String;
+        readonly context: Schema.optionalKey<Schema.String>;
+        readonly rubric: Schema.String;
     }>>;
 }>;
 export type EvalCompositionSuite = typeof EvalCompositionSuite.Type;
@@ -4252,11 +4367,44 @@ export declare const EvalEvaluationProposal: Schema.Struct<{
     readonly candidateModels: Schema.$Array<Schema.String>;
     readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly judgeModel: Schema.String;
+    readonly evidenceSources: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly path: Schema.String;
+        readonly sourceDigest: Schema.String;
+        readonly sourceBytes: Schema.optionalKey<Schema.Finite>;
+        readonly sourceBlockCount: Schema.optionalKey<Schema.Finite>;
+        readonly blocks: Schema.$Array<Schema.Struct<{
+            readonly id: Schema.String;
+            readonly path: Schema.String;
+            readonly sourceDigest: Schema.String;
+            readonly contentDigest: Schema.String;
+            readonly ordinal: Schema.Finite;
+            readonly startLine: Schema.Finite;
+            readonly endLine: Schema.Finite;
+            readonly text: Schema.String;
+        }>>;
+    }>>>;
     readonly suites: Schema.$Array<Schema.Struct<{
         readonly version: Schema.Literal<1>;
         readonly dimensionId: Schema.String;
         readonly maximumOutputTokens: Schema.Finite;
         readonly cases: Schema.$Array<Schema.Struct<{
+            readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+            readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly description: Schema.String;
+                readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+                readonly gradingMode: Schema.Literal<"semantic">;
+            }>>>;
+            readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+            }>>>;
+            readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+                readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+                readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+            }>>>;
             readonly id: Schema.String;
             readonly prompt: Schema.String;
             readonly context: Schema.optionalKey<Schema.String>;
@@ -4282,10 +4430,6 @@ export declare const EvalEvaluationProposal: Schema.Struct<{
         readonly minimumWinnerScoreGap: Schema.Finite;
         readonly minimumWinnerAgreement: Schema.Finite;
         readonly cases: Schema.$Array<Schema.Struct<{
-            readonly id: Schema.String;
-            readonly prompt: Schema.String;
-            readonly context: Schema.optionalKey<Schema.String>;
-            readonly rubric: Schema.String;
             readonly decomposition: Schema.Struct<{
                 readonly weights: Schema.$Array<Schema.Struct<{
                     readonly dimensionId: Schema.String;
@@ -4300,6 +4444,27 @@ export declare const EvalEvaluationProposal: Schema.Struct<{
                 readonly inputTokens: Schema.optionalKey<Schema.Finite>;
                 readonly maxOutputTokens: Schema.optionalKey<Schema.Finite>;
             }>;
+            readonly evidenceIds: Schema.optionalKey<Schema.$Array<Schema.String>>;
+            readonly criteria: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly description: Schema.String;
+                readonly importance: Schema.Literals<readonly ["critical", "supporting"]>;
+                readonly gradingMode: Schema.Literal<"semantic">;
+            }>>>;
+            readonly positiveControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+            }>>>;
+            readonly negativeControls: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly response: Schema.String;
+                readonly targetedCriterionIds: Schema.$Array<Schema.String>;
+                readonly kind: Schema.Literals<readonly ["missing-criterion", "forbidden-behavior", "non-responsive"]>;
+            }>>>;
+            readonly id: Schema.String;
+            readonly prompt: Schema.String;
+            readonly context: Schema.optionalKey<Schema.String>;
+            readonly rubric: Schema.String;
         }>>;
     }>;
 }>;
@@ -4309,6 +4474,8 @@ export declare const EvalArtifactApproval: Schema.Struct<{
     readonly kind: Schema.Literals<readonly ["routing-basis", "evaluations"]>;
     readonly digest: Schema.String;
     readonly approvedAt: Schema.String;
+    readonly structurePolicyVersion: Schema.optionalKey<Schema.Finite>;
+    readonly structureReportDigest: Schema.optionalKey<Schema.String>;
 }>;
 export type EvalArtifactApproval = typeof EvalArtifactApproval.Type;
 export declare const EvalPlanScope: Schema.Literals<readonly ["pilot", "full"]>;
@@ -4423,10 +4590,13 @@ export declare const EvalClassifierObservation: Schema.Struct<{
 export type EvalClassifierObservation = typeof EvalClassifierObservation.Type;
 export declare const EvalCompositionCaseResult: Schema.Struct<{
     readonly caseId: Schema.String;
-    readonly predictedWinner: Schema.String;
-    readonly observedWinner: Schema.String;
+    readonly predictedWinner: Schema.NullOr<Schema.String>;
+    readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+    readonly observedWinner: Schema.NullOr<Schema.String>;
+    readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
     readonly predictedScoreGap: Schema.Finite;
     readonly observedScoreGap: Schema.Finite;
+    readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
     readonly passed: Schema.Boolean;
 }>;
 export type EvalCompositionCaseResult = typeof EvalCompositionCaseResult.Type;
@@ -4457,13 +4627,19 @@ export declare const EvalRunQualification: Schema.Struct<{
         readonly expectedCases: Schema.Finite;
         readonly comparableCases: Schema.Finite;
         readonly agreeingCases: Schema.Finite;
-        readonly winnerAgreement: Schema.Finite;
+        readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+        readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+        readonly reason: Schema.optionalKey<Schema.String>;
+        readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
         readonly cases: Schema.$Array<Schema.Struct<{
             readonly caseId: Schema.String;
-            readonly predictedWinner: Schema.String;
-            readonly observedWinner: Schema.String;
+            readonly predictedWinner: Schema.NullOr<Schema.String>;
+            readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+            readonly observedWinner: Schema.NullOr<Schema.String>;
+            readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
             readonly predictedScoreGap: Schema.Finite;
             readonly observedScoreGap: Schema.Finite;
+            readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly passed: Schema.Boolean;
         }>>;
     }>;
@@ -4623,13 +4799,19 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
             readonly expectedCases: Schema.Finite;
             readonly comparableCases: Schema.Finite;
             readonly agreeingCases: Schema.Finite;
-            readonly winnerAgreement: Schema.Finite;
+            readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+            readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+            readonly reason: Schema.optionalKey<Schema.String>;
+            readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly cases: Schema.$Array<Schema.Struct<{
                 readonly caseId: Schema.String;
-                readonly predictedWinner: Schema.String;
-                readonly observedWinner: Schema.String;
+                readonly predictedWinner: Schema.NullOr<Schema.String>;
+                readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+                readonly observedWinner: Schema.NullOr<Schema.String>;
+                readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
                 readonly predictedScoreGap: Schema.Finite;
                 readonly observedScoreGap: Schema.Finite;
+                readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
                 readonly passed: Schema.Boolean;
             }>>;
         }>;
@@ -4664,6 +4846,30 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly caseId: Schema.String;
         readonly sequence: Schema.Finite;
         readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly name: Schema.Literal<"failure-detail">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+        readonly reason: Schema.String;
+        readonly retryable: Schema.Boolean;
+        readonly callId: Schema.optionalKey<Schema.String>;
+        readonly reservationId: Schema.optionalKey<Schema.String>;
+        readonly accounting: Schema.Struct<{
+            readonly expectedCalls: Schema.Finite;
+            readonly observedCalls: Schema.Finite;
+            readonly knownInputTokens: Schema.Finite;
+            readonly knownOutputTokens: Schema.Finite;
+            readonly unknownTokenMeasurements: Schema.Finite;
+            readonly knownPricedSubtotalUsd: Schema.Finite;
+            readonly unpricedCalls: Schema.Finite;
+            readonly subscriptionUnitCalls: Schema.Finite;
+        }>;
+        readonly cleanup: Schema.Struct<{
+            readonly sessionOpened: Schema.Boolean;
+            readonly sessionClosed: Schema.Boolean;
+            readonly detail: Schema.optionalKey<Schema.String>;
+        }>;
     }>, Schema.Struct<{
         readonly at: Schema.String;
         readonly operationId: Schema.String;
@@ -4679,62 +4885,6 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
     }>]>>>, never>;
 }>, Schema.Struct<{
     readonly status: Schema.Literal<"completed">;
-    readonly activation: Schema.Struct<{
-        readonly version: Schema.Literal<2>;
-        readonly generatedAt: Schema.String;
-        readonly basisDigest: Schema.String;
-        readonly evidenceDigest: Schema.String;
-        readonly classifierModel: Schema.String;
-        readonly objective: Schema.Union<readonly [Schema.Struct<{
-            readonly kind: Schema.Literal<"highest-quality">;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"lowest-cost">;
-            readonly minimumQuality: Schema.Finite;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"lowest-latency">;
-            readonly minimumQuality: Schema.Finite;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"balanced">;
-            readonly minimumQuality: Schema.Finite;
-            readonly weights: Schema.Struct<{
-                readonly quality: Schema.Finite;
-                readonly cost: Schema.Finite;
-                readonly latency: Schema.Finite;
-            }>;
-        }>, Schema.Struct<{
-            readonly kind: Schema.Literal<"pareto">;
-            readonly minimumQuality: Schema.Finite;
-            readonly preference: Schema.Literals<readonly ["quality", "cost", "latency"]>;
-        }>]>;
-        readonly maximumUnknownWeight: Schema.Finite;
-        readonly constraints: Schema.optionalKey<Schema.Struct<{
-            readonly minimumDimensionQuality: Schema.optionalKey<Schema.$Record<Schema.String, Schema.Finite>>;
-            readonly maximumFailureRate: Schema.optionalKey<Schema.Finite>;
-        }>>;
-        readonly dimensions: Schema.$Array<Schema.Struct<{
-            readonly id: Schema.String;
-            readonly description: Schema.String;
-            readonly includes: Schema.$Array<Schema.String>;
-            readonly excludes: Schema.$Array<Schema.String>;
-        }>>;
-        readonly candidateModels: Schema.$Array<Schema.String>;
-        readonly evidence: Schema.$Array<Schema.Struct<{
-            readonly model: Schema.String;
-            readonly dimensionId: Schema.String;
-            readonly suiteDigest: Schema.String;
-            readonly evidenceDigest: Schema.String;
-            readonly quality: Schema.Struct<{
-                readonly passRate: Schema.Finite;
-                readonly lowerConfidenceBound: Schema.Finite;
-                readonly sampleCount: Schema.Finite;
-            }>;
-            readonly failureRate: Schema.Finite;
-            readonly averageJudgeScore: Schema.optionalKey<Schema.Finite>;
-            readonly p95DurationMs: Schema.optionalKey<Schema.Finite>;
-            readonly averageCostUsd: Schema.optionalKey<Schema.Finite>;
-            readonly unpricedCalls: Schema.Finite;
-        }>>;
-    }>;
     readonly version: Schema.Literal<1>;
     readonly runId: Schema.String;
     readonly planId: Schema.String;
@@ -4816,13 +4966,19 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
             readonly expectedCases: Schema.Finite;
             readonly comparableCases: Schema.Finite;
             readonly agreeingCases: Schema.Finite;
-            readonly winnerAgreement: Schema.Finite;
+            readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+            readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+            readonly reason: Schema.optionalKey<Schema.String>;
+            readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly cases: Schema.$Array<Schema.Struct<{
                 readonly caseId: Schema.String;
-                readonly predictedWinner: Schema.String;
-                readonly observedWinner: Schema.String;
+                readonly predictedWinner: Schema.NullOr<Schema.String>;
+                readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+                readonly observedWinner: Schema.NullOr<Schema.String>;
+                readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
                 readonly predictedScoreGap: Schema.Finite;
                 readonly observedScoreGap: Schema.Finite;
+                readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
                 readonly passed: Schema.Boolean;
             }>>;
         }>;
@@ -4857,6 +5013,30 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly caseId: Schema.String;
         readonly sequence: Schema.Finite;
         readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly name: Schema.Literal<"failure-detail">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+        readonly reason: Schema.String;
+        readonly retryable: Schema.Boolean;
+        readonly callId: Schema.optionalKey<Schema.String>;
+        readonly reservationId: Schema.optionalKey<Schema.String>;
+        readonly accounting: Schema.Struct<{
+            readonly expectedCalls: Schema.Finite;
+            readonly observedCalls: Schema.Finite;
+            readonly knownInputTokens: Schema.Finite;
+            readonly knownOutputTokens: Schema.Finite;
+            readonly unknownTokenMeasurements: Schema.Finite;
+            readonly knownPricedSubtotalUsd: Schema.Finite;
+            readonly unpricedCalls: Schema.Finite;
+            readonly subscriptionUnitCalls: Schema.Finite;
+        }>;
+        readonly cleanup: Schema.Struct<{
+            readonly sessionOpened: Schema.Boolean;
+            readonly sessionClosed: Schema.Boolean;
+            readonly detail: Schema.optionalKey<Schema.String>;
+        }>;
     }>, Schema.Struct<{
         readonly at: Schema.String;
         readonly operationId: Schema.String;
@@ -4959,13 +5139,19 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
             readonly expectedCases: Schema.Finite;
             readonly comparableCases: Schema.Finite;
             readonly agreeingCases: Schema.Finite;
-            readonly winnerAgreement: Schema.Finite;
+            readonly winnerAgreement: Schema.optionalKey<Schema.Finite>;
+            readonly conclusion: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Literals<readonly ["passed", "failed", "inconclusive"]>>, never>;
+            readonly reason: Schema.optionalKey<Schema.String>;
+            readonly publishable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
             readonly cases: Schema.$Array<Schema.Struct<{
                 readonly caseId: Schema.String;
-                readonly predictedWinner: Schema.String;
-                readonly observedWinner: Schema.String;
+                readonly predictedWinner: Schema.NullOr<Schema.String>;
+                readonly predictedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
+                readonly observedWinner: Schema.NullOr<Schema.String>;
+                readonly observedTiedModels: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.$Array<Schema.String>>, never>;
                 readonly predictedScoreGap: Schema.Finite;
                 readonly observedScoreGap: Schema.Finite;
+                readonly comparable: Schema.withDecodingDefaultKey<Schema.optionalKey<Schema.Boolean>, never>;
                 readonly passed: Schema.Boolean;
             }>>;
         }>;
@@ -5000,6 +5186,30 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
         readonly caseId: Schema.String;
         readonly sequence: Schema.Finite;
         readonly name: Schema.Literal<"judge#">;
+    }>, Schema.Struct<{
+        readonly name: Schema.Literal<"failure-detail">;
+        readonly at: Schema.String;
+        readonly runId: Schema.String;
+        readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+        readonly reason: Schema.String;
+        readonly retryable: Schema.Boolean;
+        readonly callId: Schema.optionalKey<Schema.String>;
+        readonly reservationId: Schema.optionalKey<Schema.String>;
+        readonly accounting: Schema.Struct<{
+            readonly expectedCalls: Schema.Finite;
+            readonly observedCalls: Schema.Finite;
+            readonly knownInputTokens: Schema.Finite;
+            readonly knownOutputTokens: Schema.Finite;
+            readonly unknownTokenMeasurements: Schema.Finite;
+            readonly knownPricedSubtotalUsd: Schema.Finite;
+            readonly unpricedCalls: Schema.Finite;
+            readonly subscriptionUnitCalls: Schema.Finite;
+        }>;
+        readonly cleanup: Schema.Struct<{
+            readonly sessionOpened: Schema.Boolean;
+            readonly sessionClosed: Schema.Boolean;
+            readonly detail: Schema.optionalKey<Schema.String>;
+        }>;
     }>, Schema.Struct<{
         readonly at: Schema.String;
         readonly operationId: Schema.String;

package/dist/project-contracts.js

@@ -527,11 +527,58 @@ export const EvalProjectDocument = Schema.Union([
     EvalProjectState,
     EvalProjectStateV2
 ]);
+export const EvalEvidenceBlock = Schema.Struct({
+    id: Schema.String,
+    path: Schema.String,
+    sourceDigest: Schema.String,
+    contentDigest: Schema.String,
+    ordinal: NonNegativeInteger,
+    startLine: NonNegativeInteger,
+    endLine: NonNegativeInteger,
+    text: Schema.String
+});
+export const EvalEvidenceSource = Schema.Struct({
+    path: Schema.String,
+    sourceDigest: Schema.String,
+    sourceBytes: Schema.optionalKey(NonNegativeInteger),
+    sourceBlockCount: Schema.optionalKey(NonNegativeInteger),
+    blocks: Schema.Array(EvalEvidenceBlock)
+});
+export const EvalCriterion = Schema.Struct({
+    id: Schema.String,
+    description: Schema.String,
+    importance: Schema.Literals(["critical", "supporting"]),
+    gradingMode: Schema.Literal("semantic")
+});
+export const EvalPositiveControl = Schema.Struct({
+    id: Schema.String,
+    response: Schema.String
+});
+export const EvalNegativeControl = Schema.Struct({
+    id: Schema.String,
+    response: Schema.String,
+    targetedCriterionIds: Schema.Array(Schema.String),
+    kind: Schema.Literals([
+        "missing-criterion",
+        "forbidden-behavior",
+        "non-responsive"
+    ])
+});
+const EvalCompiledCase = {
+    // Optional only at the persisted decoding boundary. The current evaluation
+    // structure policy requires compiler-owned evidence and hidden controls
+    // before approval, planning, or execution.
+    evidenceIds: Schema.optionalKey(Schema.Array(Schema.String)),
+    criteria: Schema.optionalKey(Schema.Array(EvalCriterion)),
+    positiveControls: Schema.optionalKey(Schema.Array(EvalPositiveControl)),
+    negativeControls: Schema.optionalKey(Schema.Array(EvalNegativeControl))
+};
 export const EvalDimensionCase = Schema.Struct({
     id: Schema.String,
     prompt: Schema.String,
     context: Schema.optionalKey(Schema.String),
-    rubric: Schema.String
+    rubric: Schema.String,
+    ...EvalCompiledCase
 });
 export const EvalDimensionSuite = Schema.Struct({
     version: Schema.Literal(EVAL_PROJECT_VERSION),
@@ -553,6 +600,7 @@ export const EvalCompositionCase = Schema.Struct({
     prompt: Schema.String,
     context: Schema.optionalKey(Schema.String),
     rubric: Schema.String,
+    ...EvalCompiledCase,
     decomposition: DecompositionResult,
     requirements: RequestRoutingRequirements
 });
@@ -569,6 +617,7 @@ export const EvalEvaluationProposal = Schema.Struct({
     candidateModels: Schema.Array(Schema.String),
     candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     judgeModel: Schema.String,
+    evidenceSources: Schema.optionalKey(Schema.Array(EvalEvidenceSource)),
     suites: Schema.Array(EvalDimensionSuite),
     decompositionBenchmark: EvalDecompositionBenchmark,
     compositionSuite: EvalCompositionSuite
@@ -577,7 +626,9 @@ export const EvalArtifactApproval = Schema.Struct({
     version: Schema.Literal(EVAL_PROJECT_VERSION),
     kind: Schema.Literals(["routing-basis", "evaluations"]),
     digest: Schema.String,
-    approvedAt: Schema.String
+    approvedAt: Schema.String,
+    structurePolicyVersion: Schema.optionalKey(NonNegativeInteger),
+    structureReportDigest: Schema.optionalKey(Schema.String)
 });
 export const EvalPlanScope = Schema.Literals(["pilot", "full"]);
 export const EvalCostEstimate = Schema.Struct({
@@ -718,10 +769,13 @@ export const EvalClassifierObservation = Schema.Struct({
 });
 export const EvalCompositionCaseResult = Schema.Struct({
     caseId: Schema.String,
-    predictedWinner: Schema.String,
-    observedWinner: Schema.String,
+    predictedWinner: Schema.NullOr(Schema.String),
+    predictedTiedModels: Schema.optionalKey(Schema.Array(Schema.String)).pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))),
+    observedWinner: Schema.NullOr(Schema.String),
+    observedTiedModels: Schema.optionalKey(Schema.Array(Schema.String)).pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))),
     predictedScoreGap: NonNegativeFinite,
     observedScoreGap: NonNegativeFinite,
+    comparable: Schema.optionalKey(Schema.Boolean).pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))),
     passed: Schema.Boolean
 });
 export const EvalRunQualification = Schema.Struct({
@@ -735,7 +789,10 @@ export const EvalRunQualification = Schema.Struct({
         expectedCases: NonNegativeInteger,
         comparableCases: NonNegativeInteger,
         agreeingCases: NonNegativeInteger,
-        winnerAgreement: NonNegativeFinite,
+        winnerAgreement: Schema.optionalKey(NonNegativeFinite),
+        conclusion: Schema.optionalKey(Schema.Literals(["passed", "failed", "inconclusive"])).pipe(Schema.withDecodingDefaultKey(Effect.succeed("failed"))),
+        reason: Schema.optionalKey(Schema.String),
+        publishable: Schema.optionalKey(Schema.Boolean).pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))),
         cases: Schema.Array(EvalCompositionCaseResult)
     })
 });
@@ -761,6 +818,7 @@ export function summarizeEvalRunLedger(comparisons, expectedCalls, costService,
                     ...(call.measurement.inputTokens === undefined
                         ? {}
                         : { inputTokens: call.measurement.inputTokens }),
+                    ...(call.measurement.inputTokens === undefined ? {} : { cachedInputTokens: 0 }),
                     ...(call.measurement.outputTokens === undefined
                         ? {}
                         : { outputTokens: call.measurement.outputTokens })
@@ -779,6 +837,7 @@ export function summarizeEvalRunLedger(comparisons, expectedCalls, costService,
                 ...(observation.measurement.inputTokens === undefined
                     ? {}
                     : { inputTokens: observation.measurement.inputTokens }),
+                ...(observation.measurement.inputTokens === undefined ? {} : { cachedInputTokens: 0 }),
                 ...(observation.measurement.outputTokens === undefined
                     ? {}
                     : { outputTokens: observation.measurement.outputTokens })
@@ -828,8 +887,7 @@ const QualifiedEvalRunReport = Schema.Struct({
 });
 const CompletedEvalRunReport = Schema.Struct({
     ...EvalRunReportCommon,
-    status: Schema.Literal("completed"),
-    activation: PublishedRoutingActivation
+    status: Schema.Literal("completed")
 });
 const FailedEvalRunReport = Schema.Struct({
     ...EvalRunReportCommon,