@hraness/kb 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/DISCLOSURE +43 -0
  2. package/LICENSE +21 -0
  3. package/README.md +545 -0
  4. package/dist/agent-context.js +35 -0
  5. package/dist/agent-guide-audit.js +18 -0
  6. package/dist/attachments.js +19 -0
  7. package/dist/authoring.js +33 -0
  8. package/dist/benchmark.js +19 -0
  9. package/dist/capture.js +49 -0
  10. package/dist/cli.js +3646 -0
  11. package/dist/clip/acquire.js +57 -0
  12. package/dist/clip/args.js +21 -0
  13. package/dist/clip/bounded-byte-buffer.js +9 -0
  14. package/dist/clip/browser-profiles.js +17 -0
  15. package/dist/clip/bundle-reader.js +11 -0
  16. package/dist/clip/cli.js +27 -0
  17. package/dist/clip/cookies.js +21 -0
  18. package/dist/clip/defuddle-worker.js +84 -0
  19. package/dist/clip/doctor.js +32 -0
  20. package/dist/clip/jobs.js +784 -0
  21. package/dist/clip/network-proxy.js +9 -0
  22. package/dist/clip/network.js +28 -0
  23. package/dist/clip/persist.js +30 -0
  24. package/dist/clip/refresh.js +11 -0
  25. package/dist/clip/terminal.js +11 -0
  26. package/dist/evaluation-builder.js +12238 -0
  27. package/dist/evaluation-kb.js +26 -0
  28. package/dist/evaluation.js +35 -0
  29. package/dist/git.js +39 -0
  30. package/dist/graph.js +39 -0
  31. package/dist/index-01jj6rbv.js +1103 -0
  32. package/dist/index-06c9ctr6.js +738 -0
  33. package/dist/index-0kavxzqj.js +2480 -0
  34. package/dist/index-1gwbassd.js +743 -0
  35. package/dist/index-1n418kb9.js +827 -0
  36. package/dist/index-1vrd1rmn.js +56 -0
  37. package/dist/index-1xxnjn0d.js +171 -0
  38. package/dist/index-3rm7cz6h.js +373 -0
  39. package/dist/index-3v2z4f0q.js +328 -0
  40. package/dist/index-48pz4jpc.js +392 -0
  41. package/dist/index-4j3tt0c3.js +234 -0
  42. package/dist/index-5n05se68.js +307 -0
  43. package/dist/index-5vwpzb5a.js +764 -0
  44. package/dist/index-6g2pv9d2.js +370 -0
  45. package/dist/index-7fzc32gf.js +712 -0
  46. package/dist/index-84x0vjjp.js +432 -0
  47. package/dist/index-adx6khj5.js +488 -0
  48. package/dist/index-b88v3vtm.js +805 -0
  49. package/dist/index-cv6fh7z5.js +462 -0
  50. package/dist/index-cxfrakt7.js +1067 -0
  51. package/dist/index-d13v9ckt.js +150 -0
  52. package/dist/index-dycjbhw2.js +1943 -0
  53. package/dist/index-dyqwejk5.js +531 -0
  54. package/dist/index-e5fbsywq.js +658 -0
  55. package/dist/index-f984hw45.js +927 -0
  56. package/dist/index-gh719d91.js +63 -0
  57. package/dist/index-hgve9rh2.js +2046 -0
  58. package/dist/index-hya40gb2.js +474 -0
  59. package/dist/index-j4zgmzjr.js +152 -0
  60. package/dist/index-jsmvyyvf.js +1166 -0
  61. package/dist/index-kh0pwvdd.js +1974 -0
  62. package/dist/index-mqx4nd6v.js +119 -0
  63. package/dist/index-mxxxytys.js +1254 -0
  64. package/dist/index-n5dd7r0v.js +711 -0
  65. package/dist/index-npg9z1a4.js +431 -0
  66. package/dist/index-pj501bh1.js +140 -0
  67. package/dist/index-qry4vhxk.js +1041 -0
  68. package/dist/index-s2gw5aw9.js +199 -0
  69. package/dist/index-vvdj6kca.js +70 -0
  70. package/dist/index-w2zc0vwa.js +571 -0
  71. package/dist/index-x3fthpsc.js +585 -0
  72. package/dist/index-zr53sf63.js +44 -0
  73. package/dist/index-zxdy5pby.js +2728 -0
  74. package/dist/index-zzhgcwyt.js +966 -0
  75. package/dist/index.js +453 -0
  76. package/dist/navigation.js +13 -0
  77. package/dist/pdf.js +85 -0
  78. package/dist/percolate.js +24 -0
  79. package/dist/portfolio.js +77 -0
  80. package/dist/query.js +29 -0
  81. package/dist/repository-memory.js +60 -0
  82. package/dist/sdk.js +36 -0
  83. package/dist/search-rules.js +34 -0
  84. package/dist/search.js +22 -0
  85. package/dist/semantic.js +39 -0
  86. package/dist/source-inbox.js +17 -0
  87. package/dist/untrusted-content.js +32 -0
  88. package/dist/url-intelligence.js +74 -0
  89. package/dist/workflow.js +23 -0
  90. package/dist/workflows/decision-context.js +20 -0
  91. package/dist/workflows/explain-change.js +8 -0
  92. package/dist/workflows/index.js +28 -0
  93. package/dist/workflows/plan-radar.js +10 -0
  94. package/package.json +409 -0
  95. package/skills/kb/AGENTS.md +19 -0
  96. package/skills/kb/SKILL.md +82 -0
  97. package/skills/kb/agents/openai.yaml +4 -0
  98. package/skills/kb/references/pdf-review.md +51 -0
  99. package/skills/kb/references/percolate.md +144 -0
  100. package/skills/kb/references/plan-structure.md +105 -0
  101. package/skills/kb/references/plan.md +115 -0
  102. package/skills/kb/references/query.md +231 -0
  103. package/skills/kb/references/refresh.md +133 -0
  104. package/skills/kb/references/save-pdf.md +125 -0
  105. package/skills/kb/references/save-url.md +178 -0
  106. package/skills/kb/references/url-authentication.md +75 -0
  107. package/skills/kb/references/url-platforms.md +25 -0
  108. package/src/agent-context.ts +1241 -0
  109. package/src/agent-guide-audit.ts +750 -0
  110. package/src/attachments.ts +723 -0
  111. package/src/authoring.ts +1725 -0
  112. package/src/benchmark.ts +362 -0
  113. package/src/capture.ts +68 -0
  114. package/src/cli.ts +3660 -0
  115. package/src/clip/acquire.ts +1273 -0
  116. package/src/clip/archive-today.ts +402 -0
  117. package/src/clip/args.ts +423 -0
  118. package/src/clip/assets.ts +262 -0
  119. package/src/clip/bounded-byte-buffer.ts +75 -0
  120. package/src/clip/browser-profiles.ts +372 -0
  121. package/src/clip/bundle-reader.ts +608 -0
  122. package/src/clip/capture.ts +1054 -0
  123. package/src/clip/cli.ts +181 -0
  124. package/src/clip/cookies.ts +512 -0
  125. package/src/clip/doctor.ts +1110 -0
  126. package/src/clip/extract.ts +868 -0
  127. package/src/clip/jobs.ts +959 -0
  128. package/src/clip/lib.ts +861 -0
  129. package/src/clip/media.ts +1474 -0
  130. package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
  131. package/src/clip/metadata-search-tool/Cargo.toml +18 -0
  132. package/src/clip/metadata-search-tool/runner.ts +685 -0
  133. package/src/clip/metadata-search-tool/src/main.rs +389 -0
  134. package/src/clip/metadata-search.ts +687 -0
  135. package/src/clip/network-proxy.ts +627 -0
  136. package/src/clip/network.ts +950 -0
  137. package/src/clip/package-root.ts +54 -0
  138. package/src/clip/persist.ts +1472 -0
  139. package/src/clip/platforms.ts +1824 -0
  140. package/src/clip/refresh.ts +197 -0
  141. package/src/clip/structured.ts +432 -0
  142. package/src/clip/terminal.ts +197 -0
  143. package/src/clip/url-intelligence.ts +581 -0
  144. package/src/clip/url-metadata-backfill.ts +616 -0
  145. package/src/clip/url-metadata-cli.ts +251 -0
  146. package/src/clip/url-metadata.ts +1397 -0
  147. package/src/evaluation-analysis-v2.ts +1440 -0
  148. package/src/evaluation-builder.ts +2539 -0
  149. package/src/evaluation-corpus-authoring.ts +2052 -0
  150. package/src/evaluation-evidence.ts +1147 -0
  151. package/src/evaluation-execution-v2.ts +426 -0
  152. package/src/evaluation-implementation.ts +131 -0
  153. package/src/evaluation-kb-closure.ts +1430 -0
  154. package/src/evaluation-kb-runner-v2.ts +735 -0
  155. package/src/evaluation-kb-v2.ts +2567 -0
  156. package/src/evaluation-kb.ts +1018 -0
  157. package/src/evaluation-measurement-v2.ts +1462 -0
  158. package/src/evaluation-packing-v2.ts +452 -0
  159. package/src/evaluation-redaction.ts +28 -0
  160. package/src/evaluation-v2.ts +3857 -0
  161. package/src/evaluation.ts +1220 -0
  162. package/src/git.ts +1276 -0
  163. package/src/graph.ts +1487 -0
  164. package/src/index.ts +20 -0
  165. package/src/init.ts +123 -0
  166. package/src/navigation.ts +277 -0
  167. package/src/note-lock.ts +529 -0
  168. package/src/pdf/args.ts +202 -0
  169. package/src/pdf/capture.ts +298 -0
  170. package/src/pdf/cli.ts +159 -0
  171. package/src/pdf/extract.ts +303 -0
  172. package/src/pdf/layout.ts +618 -0
  173. package/src/pdf/markdown.ts +154 -0
  174. package/src/pdf/model.ts +234 -0
  175. package/src/pdf/ocr.ts +190 -0
  176. package/src/pdf/persist.ts +314 -0
  177. package/src/pdf/source.ts +135 -0
  178. package/src/pdf/tools.ts +146 -0
  179. package/src/pdf.ts +92 -0
  180. package/src/percolate.ts +877 -0
  181. package/src/portfolio-audit.ts +574 -0
  182. package/src/portfolio-identity.ts +162 -0
  183. package/src/portfolio-registry.ts +473 -0
  184. package/src/portfolio.ts +765 -0
  185. package/src/query.ts +646 -0
  186. package/src/repository-memory.ts +1150 -0
  187. package/src/sdk.ts +1452 -0
  188. package/src/search-rules.ts +712 -0
  189. package/src/search.ts +710 -0
  190. package/src/semantic-runtime.ts +1139 -0
  191. package/src/semantic.ts +2480 -0
  192. package/src/source-inbox.ts +232 -0
  193. package/src/untrusted-content.ts +401 -0
  194. package/src/url-intelligence.ts +104 -0
  195. package/src/vault.ts +636 -0
  196. package/src/workflow.ts +653 -0
  197. package/src/workflows/decision-context.ts +81 -0
  198. package/src/workflows/explain-change.ts +73 -0
  199. package/src/workflows/index.ts +3 -0
  200. package/src/workflows/plan-radar.ts +101 -0
@@ -0,0 +1,3857 @@
1
+ import { createHash } from "node:crypto";
2
+
3
+ export const RETRIEVAL_EVALUATION_V2_SCHEMA_VERSION = 2;
4
+ export const RETRIEVAL_EVALUATION_V2_PROTOCOL = "kb-retrieval-evaluation-v2";
5
+ export const PROMOTION_EVALUATION_QUERY_COUNT_V2 = 168;
6
+ export const PROMOTION_DEVELOPMENT_QUERY_COUNT_V2 = 48;
7
+ export const PROMOTION_TEST_QUERY_COUNT_V2 = 120;
8
+ export const PROMOTION_TEST_SUPPORTED_COUNT_V2 = 80;
9
+ export const PROMOTION_TEST_INSUFFICIENT_COUNT_V2 = 40;
10
+ export const PROMOTION_COHORT_COUNT_V2 = 84;
11
+ export const PROMOTION_TEST_COHORT_COUNT_V2 = 60;
12
+ export const PROMOTION_DUAL_ASSESSMENT_MINIMUM_V2 = 42;
13
+ export const PROMOTION_STRATUM_COHORT_DUAL_FRACTION_V2 = 0.25;
14
+ export const PROMOTION_STRATUM_COHORT_DUAL_MINIMUM_V2 = 2;
15
+
16
+ export const MAX_EVALUATION_V2_QUERIES = 2_000;
17
+ export const MAX_EVALUATION_V2_DOCUMENTS = 20_000;
18
+ export const MAX_EVALUATION_V2_EVIDENCE_UNITS = 100_000;
19
+ export const MAX_EVALUATION_V2_JUDGMENTS_PER_QUERY = 2_000;
20
+ export const MAX_EVALUATION_V2_NUGGETS_PER_QUERY = 100;
21
+ export const MAX_EVALUATION_V2_SUPPORT_SETS_PER_NUGGET = 100;
22
+ export const MAX_EVALUATION_V2_RESULTS_PER_LANE = 1_000;
23
+ export const MAX_EVALUATION_V2_TRACE_DECISIONS = 10_000;
24
+ export const MAX_EVALUATION_V2_SAMPLES = 2_000_000;
25
+ export const MAX_EVALUATION_V2_TEXT_BYTES = 16 * 1_024;
26
+ export const MAX_EVALUATION_V2_REPORT_TRACE_ITEMS = 5_000_000;
27
+ export const MAX_EVALUATION_V2_REPORT_PROVENANCE_ITEMS = 5_000_000;
28
+ export const MAX_EVALUATION_V2_REPORT_TRACE_BYTES = 256 * 1_024 * 1_024;
29
+ export const MAX_EVALUATION_V2_REPORT_PROVENANCE_BYTES = 128 * 1_024 * 1_024;
30
+ export const MAX_EVALUATION_V2_REPORT_RAW_EVIDENCE_ITEMS = 5_000_000;
31
+ export const MAX_EVALUATION_V2_REPORT_RAW_EVIDENCE_BYTES = 256 * 1_024 * 1_024;
32
+
33
+ const MAX_EVALUATION_V2_PACKED_CONTEXT_EVIDENCE_UNITS = 10_000;
34
+ const MAX_EVALUATION_V2_RAW_EVIDENCE_PER_SAMPLE = 10_000;
35
+ const MAX_EVALUATION_V2_RAW_EVIDENCE_DEPTH = 12;
36
+ const MAX_EVALUATION_V2_RAW_EVIDENCE_ARRAY_ITEMS = 10_000;
37
+ const MAX_EVALUATION_V2_RAW_EVIDENCE_OBJECT_FIELDS = 1_000;
38
+ const MAX_EVALUATION_V2_RAW_EVIDENCE_STRING_BYTES = 64 * 1_024;
39
+ const MAX_EVALUATION_V2_RAW_EVIDENCE_BYTES_PER_SAMPLE = 8 * 1_024 * 1_024;
40
+ const MAX_EVALUATION_V2_REPORT_PACKED_CONTEXT_ITEMS = 5_000_000;
41
+ const MAX_EVALUATION_V2_REPORT_PACKED_CONTEXT_BYTES = 128 * 1_024 * 1_024;
42
+ const EMPTY_PACKED_CONTEXT_SHA256 = createHash("sha256").update(Buffer.alloc(0)).digest("hex");
43
+
44
+ export type EvaluationSplitV2 = "development" | "test";
45
+ export type EvaluationCohortV2 = "caller-seeded" | "text-only";
46
+ export type EvaluationExpectedSupportV2 = "insufficient" | "supported";
47
+ export type EvaluationNegativeSubtypeV2 =
48
+ | "boundary-near-miss"
49
+ | "conflicting-evidence"
50
+ | "missing-required-support"
51
+ | "stale-only"
52
+ | "topical-near-miss"
53
+ | "unknown-entity";
54
+
55
+ export type EvaluationStratumV2 =
56
+ | "active-current-state"
57
+ | "code-path-context"
58
+ | "conceptual-recall"
59
+ | "exact-identity"
60
+ | "local-context"
61
+ | "metadata-constraint"
62
+ | "multi-note-relational"
63
+ | "no-answer-near-miss"
64
+ | "source-provenance"
65
+ | "temporal-stale-current";
66
+
67
+ export const PROMOTION_CRITICAL_STRATUM_MINIMA_V2 = Object.freeze({
68
+ "local-context": 20,
69
+ "multi-note-relational": 20,
70
+ "source-provenance": 20,
71
+ "temporal-stale-current": 20,
72
+ } satisfies Readonly<Partial<Record<EvaluationStratumV2, number>>>);
73
+
74
+ export const PROMOTION_ACCEPTANCE_STRATUM_MINIMA_V2 = Object.freeze({
75
+ "active-current-state": 8,
76
+ "code-path-context": 8,
77
+ "conceptual-recall": 8,
78
+ "exact-identity": 8,
79
+ "local-context": 20,
80
+ "metadata-constraint": 8,
81
+ "multi-note-relational": 20,
82
+ "source-provenance": 20,
83
+ "temporal-stale-current": 20,
84
+ } satisfies Readonly<Partial<Record<EvaluationStratumV2, number>>>);
85
+
86
+ export const PROMOTION_ACCEPTANCE_STRATUM_COHORT_MINIMA_V2 = Object.freeze({
87
+ "active-current-state": 4,
88
+ "code-path-context": 4,
89
+ "conceptual-recall": 4,
90
+ "exact-identity": 4,
91
+ "local-context": 10,
92
+ "metadata-constraint": 4,
93
+ "multi-note-relational": 10,
94
+ "source-provenance": 10,
95
+ "temporal-stale-current": 10,
96
+ } satisfies Readonly<Partial<Record<EvaluationStratumV2, number>>>);
97
+
98
+ export type EvaluationLaneIdV2 =
99
+ | "exact"
100
+ | "git"
101
+ | "graph"
102
+ | "hybrid"
103
+ | "keyword"
104
+ | "metadata"
105
+ | "note"
106
+ | "path-context"
107
+ | "semantic";
108
+
109
+ export const PROMOTION_CRITICAL_INPUT_MINIMA_V2 = Object.freeze({
110
+ context: 20,
111
+ graph: 20,
112
+ history: 20,
113
+ metadata: 20,
114
+ } as const);
115
+
116
+ export type EvaluationInputLaneV2 =
117
+ | "context"
118
+ | "graph"
119
+ | "history"
120
+ | "metadata"
121
+ | "noteId"
122
+ | "text";
123
+
124
+ export type EvaluationInputOriginV2 = "caller" | "query-text";
125
+
126
+ export type EvaluationTrustClassV2 =
127
+ | "authoritative-current"
128
+ | "authoritative-historical"
129
+ | "captured-primary"
130
+ | "captured-secondary"
131
+ | "maintained-synthesis"
132
+ | "untrusted-capture";
133
+
134
+ export type EvaluationSourceClassV2 =
135
+ | "authored-note"
136
+ | "captured-source"
137
+ | "git-history"
138
+ | "repository-file";
139
+
140
+ export const EVALUATION_SOURCE_TRUST_COMPATIBILITY_V2 = Object.freeze({
141
+ "authored-note": Object.freeze([
142
+ "authoritative-current",
143
+ "authoritative-historical",
144
+ "maintained-synthesis",
145
+ ]),
146
+ "captured-source": Object.freeze([
147
+ "captured-primary",
148
+ "captured-secondary",
149
+ "untrusted-capture",
150
+ ]),
151
+ "git-history": Object.freeze(["authoritative-historical"]),
152
+ "repository-file": Object.freeze(["authoritative-current"]),
153
+ } satisfies Readonly<Record<EvaluationSourceClassV2, readonly EvaluationTrustClassV2[]>>);
154
+
155
+ export type EvaluationMetadataFilterV2 =
156
+ | { readonly kind: "exists"; readonly path: string }
157
+ | {
158
+ readonly kind: "equals";
159
+ readonly path: string;
160
+ readonly value: string | number | boolean | null;
161
+ };
162
+
163
+ export type EvaluationRetrievalInputsV2 = {
164
+ readonly text: string;
165
+ readonly noteId?: string;
166
+ readonly metadata?: {
167
+ readonly filters: readonly EvaluationMetadataFilterV2[];
168
+ readonly tags: readonly string[];
169
+ };
170
+ readonly graph?: {
171
+ readonly seeds: readonly string[];
172
+ readonly depth: 1 | 2;
173
+ };
174
+ readonly context?: {
175
+ readonly repositoryPath: string;
176
+ };
177
+ readonly history?: {
178
+ readonly query: string;
179
+ readonly noteIds: readonly string[];
180
+ };
181
+ };
182
+
183
+ export type EvaluationInputOriginDeclarationV2 = {
184
+ readonly lane: EvaluationInputLaneV2;
185
+ readonly origin: EvaluationInputOriginV2;
186
+ };
187
+
188
+ export type EvaluationSourceFamilyV2 = {
189
+ readonly id: string;
190
+ readonly sourceClass: EvaluationSourceClassV2;
191
+ readonly trustClass: EvaluationTrustClassV2;
192
+ /**
193
+ * Opaque commitment to the private, independently reviewed family assignment.
194
+ * Catalog-only families may omit it, but every family referenced by a promotion
195
+ * query must carry one.
196
+ */
197
+ readonly familyAssignmentSha256?: string;
198
+ };
199
+
200
+ export type EvaluationDocumentV2 = {
201
+ readonly id: string;
202
+ /** Canonical vault-relative path bound independently of graded qrels. */
203
+ readonly sourcePath: string;
204
+ readonly sourceFamilyId: string;
205
+ readonly trustClass: EvaluationTrustClassV2;
206
+ };
207
+
208
+ export type EvaluationLineRangeV2 = {
209
+ readonly start: number;
210
+ readonly end: number;
211
+ };
212
+
213
+ export type EvaluationEvidenceUnitV2 = {
214
+ readonly id: string;
215
+ readonly documentId: string;
216
+ readonly sourceFamilyId: string;
217
+ readonly trustClass: EvaluationTrustClassV2;
218
+ readonly sourcePath: string;
219
+ readonly lineRange: EvaluationLineRangeV2;
220
+ readonly headingPath: readonly string[];
221
+ readonly sourcePage?: number;
222
+ };
223
+
224
+ export type EvaluationDocumentJudgmentV2 = {
225
+ readonly documentId: string;
226
+ readonly relevance: 0 | 1 | 2 | 3;
227
+ };
228
+
229
+ export type EvaluationEvidenceUnitJudgmentV2 = {
230
+ readonly evidenceUnitId: string;
231
+ readonly relevance: 0 | 1 | 2 | 3;
232
+ };
233
+
234
+ export type EvaluationAcceptableSupportSetV2 = {
235
+ readonly id: string;
236
+ readonly evidenceUnitIds: readonly string[];
237
+ };
238
+
239
+ export type EvaluationAtomicNuggetV2 = {
240
+ readonly id: string;
241
+ readonly text: string;
242
+ readonly required: boolean;
243
+ readonly acceptableSupportSets: readonly EvaluationAcceptableSupportSetV2[];
244
+ };
245
+
246
+ export type EvaluationGoldJudgmentV2 = {
247
+ readonly documents: readonly EvaluationDocumentJudgmentV2[];
248
+ readonly evidenceUnits: readonly EvaluationEvidenceUnitJudgmentV2[];
249
+ readonly nuggets: readonly EvaluationAtomicNuggetV2[];
250
+ };
251
+
252
+ export type EvaluationRawAssessorJudgmentV2 = {
253
+ readonly assessorId: string;
254
+ readonly expectedSupport: EvaluationExpectedSupportV2;
255
+ readonly documents: readonly EvaluationDocumentJudgmentV2[];
256
+ readonly evidenceUnits: readonly EvaluationEvidenceUnitJudgmentV2[];
257
+ readonly nuggets: readonly {
258
+ readonly nuggetId: string;
259
+ readonly required: boolean;
260
+ readonly acceptableSupportSetIds: readonly string[];
261
+ }[];
262
+ };
263
+
264
+ export type EvaluationAdjudicationV2 =
265
+ | { readonly status: "single-assessor" }
266
+ | { readonly status: "agreed" }
267
+ | {
268
+ readonly status: "resolved";
269
+ readonly adjudicatorId: string;
270
+ readonly rationale: string;
271
+ };
272
+
273
+ export type EvaluationQueryV2 = {
274
+ readonly id: string;
275
+ readonly text: string;
276
+ readonly split: EvaluationSplitV2;
277
+ readonly cohort: EvaluationCohortV2;
278
+ readonly strata: readonly EvaluationStratumV2[];
279
+ readonly primaryStratum: EvaluationStratumV2;
280
+ readonly expectedSupport: EvaluationExpectedSupportV2;
281
+ readonly primaryLane: EvaluationLaneIdV2;
282
+ readonly negativeSubtype?: EvaluationNegativeSubtypeV2;
283
+ readonly inputs: EvaluationRetrievalInputsV2;
284
+ readonly inputOrigins: readonly EvaluationInputOriginDeclarationV2[];
285
+ readonly gold: EvaluationGoldJudgmentV2;
286
+ readonly rawAssessments: readonly EvaluationRawAssessorJudgmentV2[];
287
+ readonly adjudication: EvaluationAdjudicationV2;
288
+ };
289
+
290
+ export type EvaluationAssessorV2 = {
291
+ readonly id: string;
292
+ readonly displayName?: string;
293
+ readonly affiliation?: string;
294
+ };
295
+
296
+ export type EvaluationMeasurementOperationV2 =
297
+ | "cold-index"
298
+ | "four-reader-query"
299
+ | "incremental-update"
300
+ | "packing"
301
+ | "warm-query";
302
+
303
+ export type EvaluationMeasurementProfileV2 = {
304
+ readonly id: string;
305
+ readonly operation: EvaluationMeasurementOperationV2;
306
+ readonly scope: "query" | "retriever";
307
+ readonly cacheState: "changed-generation" | "cold" | "not-applicable" | "warm";
308
+ readonly concurrency: number;
309
+ readonly repetitions: number;
310
+ };
311
+
312
+ export type EvaluationMinimumUsefulEffectMetricV2 =
313
+ | "document-recall-at-k"
314
+ | "evidence-recall-at-k"
315
+ | "false-abstention-rate"
316
+ | "no-answer-accuracy"
317
+ | "nugget-coverage";
318
+
319
+ export type EvaluationNonInferiorityMetricV2 =
320
+ | "active-current-state-accuracy"
321
+ | "code-path-context-accuracy"
322
+ | "context-precision"
323
+ | "conceptual-recall-accuracy"
324
+ | "document-recall-at-k"
325
+ | "evidence-recall-at-k"
326
+ | "exact-identity-accuracy"
327
+ | "local-context-accuracy"
328
+ | "metadata-constraint-accuracy"
329
+ | "multi-note-relational-accuracy"
330
+ | "source-provenance-accuracy"
331
+ | "temporal-stale-current-accuracy"
332
+ | "four-reader-query-p95-ms"
333
+ | "packing-p95-ms"
334
+ | "warm-query-p95-ms";
335
+
336
+ export type EvaluationPairedPowerV2 = {
337
+ /** One-sided type-I error used for the positive primary effect gate. */
338
+ readonly alpha: number;
339
+ readonly targetPower: number;
340
+ /** Assumed probability that a paired observation differs in either direction. */
341
+ readonly assumedDiscordantRate: number;
342
+ /** Assumed true favorable paired difference used by the prospective calculation. */
343
+ readonly assumedEffect: number;
344
+ /** Lower confidence-bound threshold required by the corresponding MUE gate. */
345
+ readonly minimumUsefulEffect: number;
346
+ /** Exact result of requiredPairedObservationsV2 for the preceding assumptions. */
347
+ readonly requiredPairs: number;
348
+ };
349
+
350
+ export type EvaluationExperimentV2 = {
351
+ readonly protocol: {
352
+ readonly minimumUsefulEffects: readonly {
353
+ readonly metric: EvaluationMinimumUsefulEffectMetricV2;
354
+ readonly cohort: EvaluationCohortV2;
355
+ readonly minimumAbsoluteDifference: number;
356
+ }[];
357
+ readonly nonInferiorityMargins: readonly {
358
+ readonly metric: EvaluationNonInferiorityMetricV2;
359
+ readonly maximumAbsoluteRegression: number;
360
+ readonly maximumRelativeRegression: number;
361
+ }[];
362
+ readonly pairedPower: EvaluationPairedPowerV2;
363
+ readonly contextCeilings: {
364
+ readonly utf8Bytes: number;
365
+ readonly readerTokens: number;
366
+ };
367
+ };
368
+ readonly environment: {
369
+ readonly tokenizer: {
370
+ readonly id: string;
371
+ readonly sha256: string;
372
+ };
373
+ readonly runtime: {
374
+ readonly id: string;
375
+ readonly sha256: string;
376
+ };
377
+ readonly hardware: {
378
+ readonly id: string;
379
+ };
380
+ readonly localModel:
381
+ | { readonly kind: "none" }
382
+ | {
383
+ readonly kind: "model";
384
+ readonly id: string;
385
+ readonly sha256: string;
386
+ };
387
+ readonly cache: {
388
+ readonly preparation: string;
389
+ readonly fingerprintSha256: string;
390
+ };
391
+ readonly fourReaderBatch: {
392
+ readonly id: string;
393
+ readonly sha256: string;
394
+ };
395
+ readonly incrementalMutation: {
396
+ readonly sourcePath: string;
397
+ readonly appendUtf8Sha256: string;
398
+ readonly expectedPostMutationSha256: string;
399
+ };
400
+ };
401
+ };
402
+
403
+ export type EvaluationRetrieverDescriptorV2 = {
404
+ readonly id: string;
405
+ readonly role: "ablation" | "baseline" | "candidate";
406
+ readonly version: string;
407
+ readonly implementationSha256: string;
408
+ readonly lanes: readonly EvaluationLaneIdV2[];
409
+ readonly configuration: Readonly<Record<string, string | number | boolean | null>>;
410
+ };
411
+
412
+ export type EvaluationCandidateLockV2 = {
413
+ readonly baselineRetrieverId: string;
414
+ readonly candidateRetrieverIds: readonly string[];
415
+ readonly descriptorDigests: readonly {
416
+ readonly retrieverId: string;
417
+ readonly sha256: string;
418
+ }[];
419
+ };
420
+
421
+ export type RetrievalEvaluationCorpusV2 = {
422
+ readonly schemaVersion: 2;
423
+ readonly id: string;
424
+ readonly description: string;
425
+ readonly manifest: {
426
+ readonly protocol: "kb-retrieval-evaluation-v2";
427
+ readonly sealedAt: string;
428
+ readonly corpusSha256: string;
429
+ readonly candidateLockSha256: string;
430
+ /** SHA-256 of the canonical immutable build.json bytes used to compile this corpus. */
431
+ readonly buildContractSha256: string;
432
+ };
433
+ readonly frozen: {
434
+ readonly repositoryCommit: string;
435
+ readonly vaultTree: string;
436
+ readonly vaultRoot: string;
437
+ };
438
+ readonly assessment: {
439
+ readonly rubricVersion: string;
440
+ readonly assessors: readonly EvaluationAssessorV2[];
441
+ };
442
+ readonly experiment: EvaluationExperimentV2;
443
+ readonly sourceFamilies: readonly EvaluationSourceFamilyV2[];
444
+ readonly documents: readonly EvaluationDocumentV2[];
445
+ readonly evidenceUnits: readonly EvaluationEvidenceUnitV2[];
446
+ readonly measurementProfiles: readonly EvaluationMeasurementProfileV2[];
447
+ readonly retrievers: readonly EvaluationRetrieverDescriptorV2[];
448
+ readonly candidateLock: EvaluationCandidateLockV2;
449
+ readonly queries: readonly EvaluationQueryV2[];
450
+ };
451
+
452
+ export type EvaluationExecutionQueryV2 = {
453
+ readonly inputs: EvaluationRetrievalInputsV2;
454
+ };
455
+
456
+ export type EvaluationExecutionRequestV2 = {
457
+ readonly corpus: RetrievalEvaluationCorpusV2["frozen"];
458
+ readonly query: EvaluationExecutionQueryV2;
459
+ readonly limit: number;
460
+ readonly signal: AbortSignal;
461
+ };
462
+
463
+ export type EvaluationRetrieverV2 = {
464
+ readonly descriptor: EvaluationRetrieverDescriptorV2;
465
+ readonly retrieve: (request: EvaluationExecutionRequestV2) => Promise<unknown>;
466
+ };
467
+
468
+ export type EvaluationEvidenceLocatorV2 = {
469
+ readonly evidenceUnitId: string;
470
+ readonly sourceFamilyId: string;
471
+ readonly sourceClass: EvaluationSourceClassV2;
472
+ readonly trustClass: EvaluationTrustClassV2;
473
+ readonly sourcePath: string;
474
+ readonly lineRange: EvaluationLineRangeV2;
475
+ readonly headingPath: readonly string[];
476
+ readonly sourcePage?: number;
477
+ };
478
+
479
+ export type EvaluationRankedCandidateV2 = {
480
+ readonly documentId: string;
481
+ readonly evidenceUnitIds: readonly string[];
482
+ readonly rank: number;
483
+ readonly score?: number;
484
+ readonly provenance: readonly EvaluationEvidenceLocatorV2[];
485
+ };
486
+
487
+ export type EvaluationLaneOutcomeV2 = {
488
+ readonly laneId: EvaluationLaneIdV2;
489
+ readonly applicability: "applied" | "skipped";
490
+ readonly status: "degraded" | "ready" | "unavailable";
491
+ readonly reasonCodes: readonly string[];
492
+ readonly rawRanking: readonly EvaluationRankedCandidateV2[];
493
+ };
494
+
495
+ export type EvaluationCandidateReasonV2 =
496
+ | "appended"
497
+ | "boundary"
498
+ | "deduplicated"
499
+ | "missing-provenance"
500
+ | "output-limit"
501
+ | "primary"
502
+ | "primary-retain-limit"
503
+ | "trust"
504
+ | "unsupported";
505
+
506
+ export type EvaluationCandidateDecisionV2 = {
507
+ readonly documentId: string;
508
+ readonly evidenceUnitIds: readonly string[];
509
+ readonly laneId: EvaluationLaneIdV2;
510
+ readonly sourceRank: number;
511
+ readonly disposition: "accepted" | "excluded";
512
+ readonly reasonCodes: readonly EvaluationCandidateReasonV2[];
513
+ readonly outputRank?: number;
514
+ readonly provenance: readonly EvaluationEvidenceLocatorV2[];
515
+ };
516
+
517
+ export type EvaluationRetrieverTraceV2 = {
518
+ readonly laneOutcomes: readonly EvaluationLaneOutcomeV2[];
519
+ readonly candidateDecisions: readonly EvaluationCandidateDecisionV2[];
520
+ };
521
+
522
+ export type EvaluationResourceAccountingV2 = {
523
+ readonly llm: {
524
+ readonly calls: 0;
525
+ readonly inputTokens: 0;
526
+ readonly outputTokens: 0;
527
+ };
528
+ readonly embedding: {
529
+ readonly calls: number;
530
+ /** Exact count when omitted; zero is only a placeholder when explicitly false. */
531
+ readonly inputTokens: number;
532
+ readonly inputTokensMeasured?: false;
533
+ readonly durationMs: number;
534
+ /** Omitted means the duration measures embedding work exactly. */
535
+ readonly durationScope?: "embedding-backed-search-upper-bound";
536
+ };
537
+ readonly packedContext: {
538
+ readonly utf8Bytes: number;
539
+ readonly readerTokens: number;
540
+ };
541
+ readonly peakRssBytes: number;
542
+ readonly cacheBytes: number;
543
+ };
544
+
545
+ export type EvaluationRepeatedSampleV2 = {
546
+ readonly retrieverId: string;
547
+ readonly profileId: string;
548
+ readonly queryId?: string;
549
+ readonly repetition: number;
550
+ readonly concurrencyBatchIdentity?: string;
551
+ readonly status: "degraded" | "failed" | "ready" | "unavailable";
552
+ readonly timings: {
553
+ readonly elapsedMs: number;
554
+ readonly indexMs: number;
555
+ readonly updateMs: number;
556
+ readonly queryMs: number;
557
+ readonly packingMs: number;
558
+ };
559
+ readonly resources: EvaluationResourceAccountingV2;
560
+ readonly trace: EvaluationRetrieverTraceV2;
561
+ /** Lane-native source records, joined one-for-one to the raw ranking. */
562
+ readonly rawEvidence: readonly EvaluationLaneNativeEvidenceV2[];
563
+ readonly packedContextTrace?: {
564
+ readonly evidenceUnitIds: readonly string[];
565
+ readonly truncated: boolean;
566
+ readonly packedBytesSha256: string;
567
+ };
568
+ readonly failure?: {
569
+ readonly kind: "exception" | "invalid-result" | "timeout";
570
+ readonly message: string;
571
+ };
572
+ };
573
+
574
+ export type EvaluationLaneNativeEvidenceV2 = Readonly<{
575
+ readonly laneId: EvaluationLaneIdV2;
576
+ readonly documentId: string;
577
+ readonly rank: number;
578
+ readonly evidence?: unknown;
579
+ }>;
580
+
581
+ export type RetrievalEvaluationReportV2 = {
582
+ readonly schemaVersion: 2;
583
+ readonly suiteSha256: string;
584
+ readonly candidateLockSha256: string;
585
+ readonly split: EvaluationSplitV2 | "all";
586
+ readonly samples: readonly EvaluationRepeatedSampleV2[];
587
+ };
588
+
589
+ export type EvaluationExternalCorpusSealV2 =
590
+ | {
591
+ readonly expectedCorpusSha256: string;
592
+ readonly expectedGitBlob?: never;
593
+ }
594
+ | {
595
+ readonly expectedCorpusSha256?: never;
596
+ readonly expectedGitBlob: string;
597
+ };
598
+
599
+ export type EvaluationCorpusParseOptionsV2 =
600
+ | { readonly claimPromotion: false }
601
+ | {
602
+ readonly claimPromotion: true;
603
+ readonly expectedSeal: EvaluationExternalCorpusSealV2;
604
+ };
605
+
606
+ const sha256Pattern = /^[0-9a-f]{64}$/u;
607
+ const gitObjectPattern = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/u;
608
+ const evidenceUnitIdPattern = /^eeu:[a-z0-9][a-z0-9._-]{0,63}:[0-9a-f]{64}$/u;
609
+ const canonicalIdPattern = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/u;
610
+ const opaquePatterns = Object.freeze({
611
+ evidenceUnit: evidenceUnitIdPattern,
612
+ nugget: /^ng-[0-9a-f]{16}$/u,
613
+ query: /^q-[0-9a-f]{16}$/u,
614
+ sourceFamily: /^sf-[0-9a-f]{16}$/u,
615
+ supportSet: /^ss-[0-9a-f]{16}$/u,
616
+ });
617
+ const windowsAbsolutePattern = /^[a-z]:[\\/]/iu;
618
+
619
+ const strata = new Set<EvaluationStratumV2>([
620
+ "active-current-state",
621
+ "code-path-context",
622
+ "conceptual-recall",
623
+ "exact-identity",
624
+ "local-context",
625
+ "metadata-constraint",
626
+ "multi-note-relational",
627
+ "no-answer-near-miss",
628
+ "source-provenance",
629
+ "temporal-stale-current",
630
+ ]);
631
+ const lanes = new Set<EvaluationLaneIdV2>([
632
+ "exact",
633
+ "git",
634
+ "graph",
635
+ "hybrid",
636
+ "keyword",
637
+ "metadata",
638
+ "note",
639
+ "path-context",
640
+ "semantic",
641
+ ]);
642
+ const trustClasses = new Set<EvaluationTrustClassV2>([
643
+ "authoritative-current",
644
+ "authoritative-historical",
645
+ "captured-primary",
646
+ "captured-secondary",
647
+ "maintained-synthesis",
648
+ "untrusted-capture",
649
+ ]);
650
+ const minimumUsefulEffectMetrics = new Set<EvaluationMinimumUsefulEffectMetricV2>([
651
+ "document-recall-at-k",
652
+ "evidence-recall-at-k",
653
+ "false-abstention-rate",
654
+ "no-answer-accuracy",
655
+ "nugget-coverage",
656
+ ]);
657
+ const nonInferiorityMetrics = new Set<EvaluationNonInferiorityMetricV2>([
658
+ "active-current-state-accuracy",
659
+ "code-path-context-accuracy",
660
+ "context-precision",
661
+ "conceptual-recall-accuracy",
662
+ "document-recall-at-k",
663
+ "evidence-recall-at-k",
664
+ "exact-identity-accuracy",
665
+ "local-context-accuracy",
666
+ "metadata-constraint-accuracy",
667
+ "multi-note-relational-accuracy",
668
+ "source-provenance-accuracy",
669
+ "temporal-stale-current-accuracy",
670
+ "four-reader-query-p95-ms",
671
+ "packing-p95-ms",
672
+ "warm-query-p95-ms",
673
+ ]);
674
+
675
+ const promotionNonInferiorityMetrics = new Set<EvaluationNonInferiorityMetricV2>([
676
+ "active-current-state-accuracy",
677
+ "code-path-context-accuracy",
678
+ "conceptual-recall-accuracy",
679
+ "document-recall-at-k",
680
+ "evidence-recall-at-k",
681
+ "exact-identity-accuracy",
682
+ "local-context-accuracy",
683
+ "metadata-constraint-accuracy",
684
+ "multi-note-relational-accuracy",
685
+ "source-provenance-accuracy",
686
+ "temporal-stale-current-accuracy",
687
+ "four-reader-query-p95-ms",
688
+ "packing-p95-ms",
689
+ "warm-query-p95-ms",
690
+ ]);
691
+
692
+ function record(value: unknown, label: string): Readonly<Record<string, unknown>> {
693
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
694
+ throw new TypeError(`${label} must be an object.`);
695
+ }
696
+ return value as Readonly<Record<string, unknown>>;
697
+ }
698
+
699
+ function strictKeys(
700
+ value: Readonly<Record<string, unknown>>,
701
+ allowed: readonly string[],
702
+ label: string,
703
+ ): void {
704
+ const allowedSet = new Set(allowed);
705
+ const extra = Object.keys(value).filter((key) => !allowedSet.has(key));
706
+ if (extra.length > 0) {
707
+ throw new TypeError(`${label} has unknown fields: ${extra.toSorted().join(", ")}.`);
708
+ }
709
+ }
710
+
711
+ function boundedString(
712
+ value: unknown,
713
+ label: string,
714
+ maximumBytes = MAX_EVALUATION_V2_TEXT_BYTES,
715
+ ): string {
716
+ if (
717
+ typeof value !== "string"
718
+ || value.trim() === ""
719
+ || /[\0\r\n]/u.test(value)
720
+ || Buffer.byteLength(value, "utf8") > maximumBytes
721
+ ) {
722
+ throw new TypeError(
723
+ `${label} must be a non-empty, single-line string of at most ${maximumBytes} UTF-8 bytes.`,
724
+ );
725
+ }
726
+ const normalized = value.normalize("NFC");
727
+ if (normalized !== value) throw new TypeError(`${label} must be NFC-normalized.`);
728
+ return normalized;
729
+ }
730
+
731
+ function bridgeString(value: unknown, label: string, maximumBytes = 512): string {
732
+ const parsed = boundedString(value, label, maximumBytes);
733
+ if (parsed.trim() !== parsed) {
734
+ throw new TypeError(`${label} must not have leading or trailing whitespace.`);
735
+ }
736
+ return parsed;
737
+ }
738
+
739
+ function optionalBoundedString(value: unknown, label: string): string | undefined {
740
+ return value === undefined ? undefined : boundedString(value, label);
741
+ }
742
+
743
+ function canonicalId(value: unknown, label: string): string {
744
+ const id = boundedString(value, label, 256);
745
+ if (!canonicalIdPattern.test(id)) {
746
+ throw new TypeError(`${label} must be a canonical lowercase hyphenated ID.`);
747
+ }
748
+ return id;
749
+ }
750
+
751
+ function confinedPath(value: unknown, label: string, allowRoot = false): string {
752
+ const path = boundedString(value, label, 4_096);
753
+ if (allowRoot && path === ".") return path;
754
+ if (
755
+ path.startsWith("/")
756
+ || path.startsWith("./")
757
+ || path.includes("\\")
758
+ || windowsAbsolutePattern.test(path)
759
+ || path.split("/").some((part) => part === "" || part === "." || part === "..")
760
+ ) {
761
+ throw new TypeError(`${label} must be a canonical confined repository-relative path.`);
762
+ }
763
+ return path;
764
+ }
765
+
766
+ function safeInteger(
767
+ value: unknown,
768
+ label: string,
769
+ minimum: number,
770
+ maximum: number,
771
+ ): number {
772
+ if (!Number.isSafeInteger(value) || (value as number) < minimum || (value as number) > maximum) {
773
+ throw new TypeError(`${label} must be an integer from ${minimum} through ${maximum}.`);
774
+ }
775
+ return value as number;
776
+ }
777
+
778
+ function nonnegativeNumber(value: unknown, label: string): number {
779
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
780
+ throw new TypeError(`${label} must be a non-negative finite number.`);
781
+ }
782
+ return value;
783
+ }
784
+
785
+ function positiveNumber(value: unknown, label: string): number {
786
+ const parsed = nonnegativeNumber(value, label);
787
+ if (parsed === 0) throw new TypeError(`${label} must be greater than zero.`);
788
+ return parsed;
789
+ }
790
+
791
+ /** Peter J. Acklam's bounded rational approximation for the standard-normal quantile. */
792
+ function inverseNormalCdf(probability: number): number {
793
+ if (!Number.isFinite(probability) || probability <= 0 || probability >= 1) {
794
+ throw new RangeError("Normal quantile probability must be between zero and one.");
795
+ }
796
+ const a = [
797
+ -3.969683028665376e1,
798
+ 2.209460984245205e2,
799
+ -2.759285104469687e2,
800
+ 1.38357751867269e2,
801
+ -3.066479806614716e1,
802
+ 2.506628277459239,
803
+ ] as const;
804
+ const b = [
805
+ -5.447609879822406e1,
806
+ 1.615858368580409e2,
807
+ -1.556989798598866e2,
808
+ 6.680131188771972e1,
809
+ -1.328068155288572e1,
810
+ ] as const;
811
+ const c = [
812
+ -7.784894002430293e-3,
813
+ -3.223964580411365e-1,
814
+ -2.400758277161838,
815
+ -2.549732539343734,
816
+ 4.374664141464968,
817
+ 2.938163982698783,
818
+ ] as const;
819
+ const d = [
820
+ 7.784695709041462e-3,
821
+ 3.224671290700398e-1,
822
+ 2.445134137142996,
823
+ 3.754408661907416,
824
+ ] as const;
825
+ const lower = 0.02425;
826
+ const upper = 1 - lower;
827
+ if (probability < lower) {
828
+ const q = Math.sqrt(-2 * Math.log(probability));
829
+ return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
830
+ / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
831
+ }
832
+ if (probability > upper) {
833
+ const q = Math.sqrt(-2 * Math.log(1 - probability));
834
+ return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5])
835
+ / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
836
+ }
837
+ const q = probability - 0.5;
838
+ const r = q * q;
839
+ return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q
840
+ / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
841
+ }
842
+
843
+ /**
844
+ * Prospective paired-observation count for a one-sided lower-bound test.
845
+ * The null sits at the minimum useful effect; the alternative is the assumed
846
+ * favorable effect. Discordance supplies the paired Bernoulli variance.
847
+ */
848
+ export function requiredPairedObservationsV2(
849
+ design: Omit<EvaluationPairedPowerV2, "requiredPairs">,
850
+ ): number {
851
+ const { alpha, targetPower, assumedDiscordantRate, assumedEffect, minimumUsefulEffect } = design;
852
+ if (!Number.isFinite(alpha) || alpha <= 0 || alpha >= 0.5) {
853
+ throw new RangeError("Paired-power alpha must be between zero and 0.5.");
854
+ }
855
+ if (!Number.isFinite(targetPower) || targetPower <= 0.5 || targetPower >= 1) {
856
+ throw new RangeError("Paired-power targetPower must be between 0.5 and one.");
857
+ }
858
+ if (
859
+ !Number.isFinite(assumedDiscordantRate)
860
+ || assumedDiscordantRate <= 0
861
+ || assumedDiscordantRate > 1
862
+ ) throw new RangeError("Paired-power assumedDiscordantRate must be in (0, 1].");
863
+ if (
864
+ !Number.isFinite(minimumUsefulEffect)
865
+ || minimumUsefulEffect < 0
866
+ || minimumUsefulEffect >= assumedDiscordantRate
867
+ ) throw new RangeError("Paired-power minimumUsefulEffect must be in [0, discordance).");
868
+ if (
869
+ !Number.isFinite(assumedEffect)
870
+ || assumedEffect <= minimumUsefulEffect
871
+ || assumedEffect > assumedDiscordantRate
872
+ ) {
873
+ throw new RangeError(
874
+ "Paired-power assumedEffect must exceed the minimum useful effect and not exceed discordance.",
875
+ );
876
+ }
877
+ const nullVariance = assumedDiscordantRate - minimumUsefulEffect ** 2;
878
+ const alternativeVariance = assumedDiscordantRate - assumedEffect ** 2;
879
+ if (nullVariance <= 0 || alternativeVariance < 0) {
880
+ throw new RangeError("Paired-power assumptions imply an invalid paired variance.");
881
+ }
882
+ const numerator = inverseNormalCdf(1 - alpha) * Math.sqrt(nullVariance)
883
+ + inverseNormalCdf(targetPower) * Math.sqrt(alternativeVariance);
884
+ const required = Math.ceil((numerator / (assumedEffect - minimumUsefulEffect)) ** 2);
885
+ if (!Number.isSafeInteger(required) || required < 1 || required > MAX_EVALUATION_V2_QUERIES) {
886
+ throw new RangeError(
887
+ `Paired-power design requires ${String(required)} observations, outside the evaluator bound.`,
888
+ );
889
+ }
890
+ return required;
891
+ }
892
+
893
+ function evidenceUnitId(value: unknown, label: string): string {
894
+ const id = bridgeString(value, label, 160);
895
+ if (!evidenceUnitIdPattern.test(id)) {
896
+ throw new TypeError(`${label} must use the registry-compatible eeu:<parser-version>:<sha256> form.`);
897
+ }
898
+ return id;
899
+ }
900
+
901
+ function assertCanonicalOrder<T>(
902
+ values: readonly T[],
903
+ key: (value: T) => string,
904
+ label: string,
905
+ ): void {
906
+ const keys = values.map(key);
907
+ if (new Set(keys).size !== keys.length) throw new TypeError(`${label} must not repeat an ID.`);
908
+ const sorted = keys.toSorted((left, right) => left.localeCompare(right));
909
+ if (keys.some((candidate, index) => candidate !== sorted[index])) {
910
+ throw new TypeError(`${label} must be in canonical ID order.`);
911
+ }
912
+ }
913
+
914
+ function stringList(
915
+ value: unknown,
916
+ label: string,
917
+ options: { readonly maximum: number; readonly allowEmpty?: boolean; readonly canonical?: boolean },
918
+ ): readonly string[] {
919
+ if (
920
+ !Array.isArray(value)
921
+ || (!options.allowEmpty && value.length === 0)
922
+ || value.length > options.maximum
923
+ ) {
924
+ const lower = options.allowEmpty ? 0 : 1;
925
+ throw new TypeError(`${label} must contain from ${lower} through ${options.maximum} entries.`);
926
+ }
927
+ const parsed = value.map((entry, index) => boundedString(entry, `${label}[${index}]`, 4_096));
928
+ if (new Set(parsed).size !== parsed.length) throw new TypeError(`${label} must not contain duplicates.`);
929
+ if (options.canonical) {
930
+ const sorted = parsed.toSorted((left, right) => left.localeCompare(right));
931
+ if (parsed.some((entry, index) => entry !== sorted[index])) {
932
+ throw new TypeError(`${label} must be in canonical order.`);
933
+ }
934
+ }
935
+ return Object.freeze(parsed);
936
+ }
937
+
938
+ function parseHeadingPath(value: unknown, label: string): readonly string[] {
939
+ if (!Array.isArray(value) || value.length > 32) {
940
+ throw new TypeError(`${label} must contain at most 32 heading components.`);
941
+ }
942
+ return Object.freeze(value.map((entry, index) =>
943
+ bridgeString(entry, `${label}[${index}]`, 4_096)));
944
+ }
945
+
946
+ function canonicalJson(value: unknown): string {
947
+ if (value === null || typeof value === "boolean" || typeof value === "string") {
948
+ return JSON.stringify(value);
949
+ }
950
+ if (typeof value === "number") {
951
+ if (!Number.isFinite(value)) throw new TypeError("Commitment input contains a non-finite number.");
952
+ return JSON.stringify(value);
953
+ }
954
+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
955
+ if (typeof value === "object" && value !== null) {
956
+ const input = value as Readonly<Record<string, unknown>>;
957
+ return `{${Object.keys(input).toSorted().map((key) =>
958
+ `${JSON.stringify(key)}:${canonicalJson(input[key])}`).join(",")}}`;
959
+ }
960
+ throw new TypeError("Commitment input must be JSON-compatible.");
961
+ }
962
+
963
+ function sha256(value: unknown): string {
964
+ return createHash("sha256").update(canonicalJson(value), "utf8").digest("hex");
965
+ }
966
+
967
+ function requireSha256(value: unknown, label: string): string {
968
+ if (typeof value !== "string" || !sha256Pattern.test(value)) {
969
+ throw new TypeError(`${label} must be 64 lowercase hexadecimal characters.`);
970
+ }
971
+ return value;
972
+ }
973
+
974
+ function parseMetadataFilter(
975
+ value: unknown,
976
+ label: string,
977
+ ): EvaluationMetadataFilterV2 {
978
+ const input = record(value, label);
979
+ strictKeys(input, ["kind", "path", "value"], label);
980
+ const path = boundedString(input.path, `${label}.path`, 2_048);
981
+ if (input.kind === "exists") {
982
+ if (input.value !== undefined) throw new TypeError(`${label}.value is forbidden for exists.`);
983
+ return Object.freeze({ kind: "exists", path });
984
+ }
985
+ if (input.kind !== "equals") throw new TypeError(`${label}.kind must be equals or exists.`);
986
+ const filterValue = input.value;
987
+ if (
988
+ filterValue !== null
989
+ && typeof filterValue !== "boolean"
990
+ && typeof filterValue !== "number"
991
+ && typeof filterValue !== "string"
992
+ ) throw new TypeError(`${label}.value must be a JSON scalar.`);
993
+ if (typeof filterValue === "number" && !Number.isFinite(filterValue)) {
994
+ throw new TypeError(`${label}.value must be finite.`);
995
+ }
996
+ return Object.freeze({
997
+ kind: "equals",
998
+ path,
999
+ value: typeof filterValue === "string"
1000
+ ? boundedString(filterValue, `${label}.value`)
1001
+ : filterValue,
1002
+ });
1003
+ }
1004
+
1005
+ function parseRetrievalInputsV2(value: unknown, label: string): EvaluationRetrievalInputsV2 {
1006
+ const input = record(value, label);
1007
+ strictKeys(input, ["context", "graph", "history", "metadata", "noteId", "text"], label);
1008
+ const text = boundedString(input.text, `${label}.text`);
1009
+ const noteId = input.noteId === undefined
1010
+ ? undefined
1011
+ : confinedPath(input.noteId, `${label}.noteId`);
1012
+
1013
+ let metadata: EvaluationRetrievalInputsV2["metadata"];
1014
+ if (input.metadata !== undefined) {
1015
+ const metadataInput = record(input.metadata, `${label}.metadata`);
1016
+ strictKeys(metadataInput, ["filters", "tags"], `${label}.metadata`);
1017
+ if (!Array.isArray(metadataInput.filters) || metadataInput.filters.length > 32) {
1018
+ throw new TypeError(`${label}.metadata.filters must have at most 32 entries.`);
1019
+ }
1020
+ const filters = metadataInput.filters.map((entry, index) =>
1021
+ parseMetadataFilter(entry, `${label}.metadata.filters[${index}]`));
1022
+ const filterKeys = filters.map(canonicalJson);
1023
+ if (new Set(filterKeys).size !== filterKeys.length) {
1024
+ throw new TypeError(`${label}.metadata.filters must not contain duplicates.`);
1025
+ }
1026
+ if (filterKeys.some((entry, index) => entry !== filterKeys.toSorted()[index])) {
1027
+ throw new TypeError(`${label}.metadata.filters must be in canonical order.`);
1028
+ }
1029
+ const tags = stringList(metadataInput.tags, `${label}.metadata.tags`, {
1030
+ allowEmpty: true,
1031
+ canonical: true,
1032
+ maximum: 32,
1033
+ });
1034
+ if (filters.length === 0 && tags.length === 0) {
1035
+ throw new TypeError(`${label}.metadata must contain at least one filter or tag.`);
1036
+ }
1037
+ metadata = Object.freeze({ filters: Object.freeze(filters), tags });
1038
+ }
1039
+
1040
+ let graph: EvaluationRetrievalInputsV2["graph"];
1041
+ if (input.graph !== undefined) {
1042
+ const graphInput = record(input.graph, `${label}.graph`);
1043
+ strictKeys(graphInput, ["depth", "seeds"], `${label}.graph`);
1044
+ const rawSeeds = stringList(graphInput.seeds, `${label}.graph.seeds`, {
1045
+ canonical: true,
1046
+ maximum: 20,
1047
+ });
1048
+ const seeds = rawSeeds.map((seed, index) =>
1049
+ confinedPath(seed, `${label}.graph.seeds[${index}]`));
1050
+ if (graphInput.depth !== 1 && graphInput.depth !== 2) {
1051
+ throw new TypeError(`${label}.graph.depth must be 1 or 2.`);
1052
+ }
1053
+ graph = Object.freeze({ depth: graphInput.depth, seeds: Object.freeze(seeds) });
1054
+ }
1055
+
1056
+ let context: EvaluationRetrievalInputsV2["context"];
1057
+ if (input.context !== undefined) {
1058
+ const contextInput = record(input.context, `${label}.context`);
1059
+ strictKeys(contextInput, ["repositoryPath"], `${label}.context`);
1060
+ context = Object.freeze({
1061
+ repositoryPath: confinedPath(
1062
+ contextInput.repositoryPath,
1063
+ `${label}.context.repositoryPath`,
1064
+ true,
1065
+ ),
1066
+ });
1067
+ }
1068
+
1069
+ let history: EvaluationRetrievalInputsV2["history"];
1070
+ if (input.history !== undefined) {
1071
+ const historyInput = record(input.history, `${label}.history`);
1072
+ strictKeys(historyInput, ["noteIds", "query"], `${label}.history`);
1073
+ const rawNoteIds = stringList(historyInput.noteIds, `${label}.history.noteIds`, {
1074
+ allowEmpty: true,
1075
+ canonical: true,
1076
+ maximum: 100,
1077
+ });
1078
+ const noteIds = rawNoteIds.map((id, index) =>
1079
+ confinedPath(id, `${label}.history.noteIds[${index}]`));
1080
+ history = Object.freeze({
1081
+ query: boundedString(historyInput.query, `${label}.history.query`, 2_048),
1082
+ noteIds: Object.freeze(noteIds),
1083
+ });
1084
+ }
1085
+
1086
+ return Object.freeze({
1087
+ text,
1088
+ ...(context === undefined ? {} : { context }),
1089
+ ...(graph === undefined ? {} : { graph }),
1090
+ ...(history === undefined ? {} : { history }),
1091
+ ...(metadata === undefined ? {} : { metadata }),
1092
+ ...(noteId === undefined ? {} : { noteId }),
1093
+ });
1094
+ }
1095
+
1096
+ function inputLanes(inputs: EvaluationRetrievalInputsV2): readonly EvaluationInputLaneV2[] {
1097
+ return Object.freeze([
1098
+ ...(inputs.context === undefined ? [] : ["context" as const]),
1099
+ ...(inputs.graph === undefined ? [] : ["graph" as const]),
1100
+ ...(inputs.history === undefined ? [] : ["history" as const]),
1101
+ ...(inputs.metadata === undefined ? [] : ["metadata" as const]),
1102
+ ...(inputs.noteId === undefined ? [] : ["noteId" as const]),
1103
+ "text" as const,
1104
+ ].toSorted());
1105
+ }
1106
+
1107
+ function parseInputOrigins(
1108
+ value: unknown,
1109
+ inputs: EvaluationRetrievalInputsV2,
1110
+ label: string,
1111
+ ): readonly EvaluationInputOriginDeclarationV2[] {
1112
+ if (!Array.isArray(value) || value.length < 1 || value.length > 6) {
1113
+ throw new TypeError(`${label} must contain from 1 through 6 entries.`);
1114
+ }
1115
+ const origins = value.map((entry, index): EvaluationInputOriginDeclarationV2 => {
1116
+ const itemLabel = `${label}[${index}]`;
1117
+ const input = record(entry, itemLabel);
1118
+ strictKeys(input, ["lane", "origin"], itemLabel);
1119
+ if (
1120
+ input.lane !== "context"
1121
+ && input.lane !== "graph"
1122
+ && input.lane !== "history"
1123
+ && input.lane !== "metadata"
1124
+ && input.lane !== "noteId"
1125
+ && input.lane !== "text"
1126
+ ) throw new TypeError(`${itemLabel}.lane is invalid.`);
1127
+ if (input.origin !== "caller" && input.origin !== "query-text") {
1128
+ throw new TypeError(`${itemLabel}.origin must be caller or query-text.`);
1129
+ }
1130
+ if (input.lane === "text" && input.origin !== "query-text") {
1131
+ throw new TypeError(`${itemLabel} must declare query-text for the text lane.`);
1132
+ }
1133
+ if (input.lane !== "text" && input.origin !== "caller") {
1134
+ throw new TypeError(`${itemLabel} must declare caller for a structured lane.`);
1135
+ }
1136
+ return Object.freeze({ lane: input.lane, origin: input.origin });
1137
+ });
1138
+ assertCanonicalOrder(origins, ({ lane }) => lane, label);
1139
+ const expected = inputLanes(inputs);
1140
+ const actual = origins.map(({ lane }) => lane);
1141
+ if (
1142
+ expected.length !== actual.length
1143
+ || expected.some((lane, index) => lane !== actual[index])
1144
+ ) throw new TypeError(`${label} must declare exactly the executable input lanes.`);
1145
+ return Object.freeze(origins);
1146
+ }
1147
+
1148
+ function validateCohortInputs(
1149
+ cohort: EvaluationCohortV2,
1150
+ inputs: EvaluationRetrievalInputsV2,
1151
+ label: string,
1152
+ ): void {
1153
+ const structured = inputLanes(inputs).filter((lane) => lane !== "text");
1154
+ if (cohort === "text-only" && structured.length > 0) {
1155
+ throw new TypeError(`${label} text-only queries may expose only inputs.text.`);
1156
+ }
1157
+ if (cohort === "caller-seeded" && structured.length === 0) {
1158
+ throw new TypeError(`${label} caller-seeded queries require a structured executable lane.`);
1159
+ }
1160
+ }
1161
+
1162
+ function validatePrimaryLaneInput(
1163
+ lane: EvaluationLaneIdV2,
1164
+ inputs: EvaluationRetrievalInputsV2,
1165
+ label: string,
1166
+ ): void {
1167
+ const present = lane === "metadata"
1168
+ ? inputs.metadata !== undefined
1169
+ : lane === "graph"
1170
+ ? inputs.graph !== undefined
1171
+ : lane === "path-context"
1172
+ ? inputs.context !== undefined
1173
+ : lane === "git"
1174
+ ? inputs.history !== undefined
1175
+ : lane === "note"
1176
+ ? inputs.noteId !== undefined
1177
+ : true;
1178
+ if (!present) throw new TypeError(`${label} primary lane ${lane} has no executable input.`);
1179
+ }
1180
+
1181
+ function parseSourceFamily(value: unknown, index: number): EvaluationSourceFamilyV2 {
1182
+ const label = `sourceFamilies[${index}]`;
1183
+ const input = record(value, label);
1184
+ strictKeys(input, ["familyAssignmentSha256", "id", "sourceClass", "trustClass"], label);
1185
+ if (
1186
+ input.sourceClass !== "authored-note"
1187
+ && input.sourceClass !== "captured-source"
1188
+ && input.sourceClass !== "git-history"
1189
+ && input.sourceClass !== "repository-file"
1190
+ ) throw new TypeError(`${label}.sourceClass is invalid.`);
1191
+ if (typeof input.trustClass !== "string" || !trustClasses.has(input.trustClass as EvaluationTrustClassV2)) {
1192
+ throw new TypeError(`${label}.trustClass is invalid.`);
1193
+ }
1194
+ const compatibleTrust = EVALUATION_SOURCE_TRUST_COMPATIBILITY_V2[input.sourceClass];
1195
+ if (!(compatibleTrust as readonly EvaluationTrustClassV2[]).includes(
1196
+ input.trustClass as EvaluationTrustClassV2,
1197
+ )) {
1198
+ throw new TypeError(`${label} sourceClass and trustClass are incompatible.`);
1199
+ }
1200
+ return Object.freeze({
1201
+ id: boundedString(input.id, `${label}.id`, 256),
1202
+ sourceClass: input.sourceClass,
1203
+ trustClass: input.trustClass as EvaluationTrustClassV2,
1204
+ ...(input.familyAssignmentSha256 === undefined
1205
+ ? {}
1206
+ : {
1207
+ familyAssignmentSha256: requireSha256(
1208
+ input.familyAssignmentSha256,
1209
+ `${label}.familyAssignmentSha256`,
1210
+ ),
1211
+ }),
1212
+ });
1213
+ }
1214
+
1215
+ function parseDocument(value: unknown, index: number): EvaluationDocumentV2 {
1216
+ const label = `documents[${index}]`;
1217
+ const input = record(value, label);
1218
+ strictKeys(input, ["id", "sourceFamilyId", "sourcePath", "trustClass"], label);
1219
+ if (typeof input.trustClass !== "string" || !trustClasses.has(input.trustClass as EvaluationTrustClassV2)) {
1220
+ throw new TypeError(`${label}.trustClass is invalid.`);
1221
+ }
1222
+ return Object.freeze({
1223
+ id: confinedPath(input.id, `${label}.id`),
1224
+ sourcePath: confinedPath(input.sourcePath, `${label}.sourcePath`),
1225
+ sourceFamilyId: boundedString(input.sourceFamilyId, `${label}.sourceFamilyId`, 256),
1226
+ trustClass: input.trustClass as EvaluationTrustClassV2,
1227
+ });
1228
+ }
1229
+
1230
+ function parseLineRange(value: unknown, label: string): EvaluationLineRangeV2 {
1231
+ const input = record(value, label);
1232
+ strictKeys(input, ["end", "start"], label);
1233
+ const start = safeInteger(input.start, `${label}.start`, 1, 10_000_000);
1234
+ const end = safeInteger(input.end, `${label}.end`, 1, 10_000_000);
1235
+ if (end < start) throw new TypeError(`${label}.end must not precede start.`);
1236
+ return Object.freeze({ start, end });
1237
+ }
1238
+
1239
+ function parseEvidenceUnit(value: unknown, index: number): EvaluationEvidenceUnitV2 {
1240
+ const label = `evidenceUnits[${index}]`;
1241
+ const input = record(value, label);
1242
+ strictKeys(
1243
+ input,
1244
+ [
1245
+ "documentId",
1246
+ "headingPath",
1247
+ "id",
1248
+ "lineRange",
1249
+ "sourceFamilyId",
1250
+ "sourcePage",
1251
+ "sourcePath",
1252
+ "trustClass",
1253
+ ],
1254
+ label,
1255
+ );
1256
+ if (typeof input.trustClass !== "string" || !trustClasses.has(input.trustClass as EvaluationTrustClassV2)) {
1257
+ throw new TypeError(`${label}.trustClass is invalid.`);
1258
+ }
1259
+ const sourcePage = input.sourcePage === undefined
1260
+ ? undefined
1261
+ : safeInteger(input.sourcePage, `${label}.sourcePage`, 1, 1_000_000);
1262
+ return Object.freeze({
1263
+ id: evidenceUnitId(input.id, `${label}.id`),
1264
+ documentId: confinedPath(input.documentId, `${label}.documentId`),
1265
+ sourceFamilyId: boundedString(input.sourceFamilyId, `${label}.sourceFamilyId`, 256),
1266
+ trustClass: input.trustClass as EvaluationTrustClassV2,
1267
+ sourcePath: confinedPath(input.sourcePath, `${label}.sourcePath`),
1268
+ lineRange: parseLineRange(input.lineRange, `${label}.lineRange`),
1269
+ headingPath: parseHeadingPath(input.headingPath, `${label}.headingPath`),
1270
+ ...(sourcePage === undefined ? {} : { sourcePage }),
1271
+ });
1272
+ }
1273
+
1274
+ function parseRelevance(value: unknown, label: string): 0 | 1 | 2 | 3 {
1275
+ if (value !== 0 && value !== 1 && value !== 2 && value !== 3) {
1276
+ throw new TypeError(`${label} must be an integer from 0 through 3.`);
1277
+ }
1278
+ return value;
1279
+ }
1280
+
1281
+ function parseDocumentJudgments(
1282
+ value: unknown,
1283
+ label: string,
1284
+ ): readonly EvaluationDocumentJudgmentV2[] {
1285
+ if (!Array.isArray(value) || value.length > MAX_EVALUATION_V2_JUDGMENTS_PER_QUERY) {
1286
+ throw new TypeError(`${label} must have at most ${MAX_EVALUATION_V2_JUDGMENTS_PER_QUERY} entries.`);
1287
+ }
1288
+ const judgments = value.map((entry, index): EvaluationDocumentJudgmentV2 => {
1289
+ const itemLabel = `${label}[${index}]`;
1290
+ const input = record(entry, itemLabel);
1291
+ strictKeys(input, ["documentId", "relevance"], itemLabel);
1292
+ return Object.freeze({
1293
+ documentId: confinedPath(input.documentId, `${itemLabel}.documentId`),
1294
+ relevance: parseRelevance(input.relevance, `${itemLabel}.relevance`),
1295
+ });
1296
+ });
1297
+ assertCanonicalOrder(judgments, ({ documentId }) => documentId, label);
1298
+ return Object.freeze(judgments);
1299
+ }
1300
+
1301
+ function parseEvidenceUnitJudgments(
1302
+ value: unknown,
1303
+ label: string,
1304
+ ): readonly EvaluationEvidenceUnitJudgmentV2[] {
1305
+ if (!Array.isArray(value) || value.length > MAX_EVALUATION_V2_JUDGMENTS_PER_QUERY) {
1306
+ throw new TypeError(`${label} must have at most ${MAX_EVALUATION_V2_JUDGMENTS_PER_QUERY} entries.`);
1307
+ }
1308
+ const judgments = value.map((entry, index): EvaluationEvidenceUnitJudgmentV2 => {
1309
+ const itemLabel = `${label}[${index}]`;
1310
+ const input = record(entry, itemLabel);
1311
+ strictKeys(input, ["evidenceUnitId", "relevance"], itemLabel);
1312
+ return Object.freeze({
1313
+ evidenceUnitId: evidenceUnitId(input.evidenceUnitId, `${itemLabel}.evidenceUnitId`),
1314
+ relevance: parseRelevance(input.relevance, `${itemLabel}.relevance`),
1315
+ });
1316
+ });
1317
+ assertCanonicalOrder(judgments, ({ evidenceUnitId }) => evidenceUnitId, label);
1318
+ return Object.freeze(judgments);
1319
+ }
1320
+
1321
+ function parseNuggets(value: unknown, label: string): readonly EvaluationAtomicNuggetV2[] {
1322
+ if (
1323
+ !Array.isArray(value)
1324
+ || value.length < 1
1325
+ || value.length > MAX_EVALUATION_V2_NUGGETS_PER_QUERY
1326
+ ) {
1327
+ throw new TypeError(
1328
+ `${label} must contain from 1 through ${MAX_EVALUATION_V2_NUGGETS_PER_QUERY} entries.`,
1329
+ );
1330
+ }
1331
+ const nuggets = value.map((entry, index): EvaluationAtomicNuggetV2 => {
1332
+ const itemLabel = `${label}[${index}]`;
1333
+ const input = record(entry, itemLabel);
1334
+ strictKeys(input, ["acceptableSupportSets", "id", "required", "text"], itemLabel);
1335
+ if (typeof input.required !== "boolean") {
1336
+ throw new TypeError(`${itemLabel}.required must be boolean.`);
1337
+ }
1338
+ if (
1339
+ !Array.isArray(input.acceptableSupportSets)
1340
+ || input.acceptableSupportSets.length > MAX_EVALUATION_V2_SUPPORT_SETS_PER_NUGGET
1341
+ ) {
1342
+ throw new TypeError(
1343
+ `${itemLabel}.acceptableSupportSets must have at most ${MAX_EVALUATION_V2_SUPPORT_SETS_PER_NUGGET} entries.`,
1344
+ );
1345
+ }
1346
+ const acceptableSupportSets = input.acceptableSupportSets.map(
1347
+ (supportEntry, supportIndex): EvaluationAcceptableSupportSetV2 => {
1348
+ const supportLabel = `${itemLabel}.acceptableSupportSets[${supportIndex}]`;
1349
+ const supportInput = record(supportEntry, supportLabel);
1350
+ strictKeys(supportInput, ["evidenceUnitIds", "id"], supportLabel);
1351
+ return Object.freeze({
1352
+ id: boundedString(supportInput.id, `${supportLabel}.id`, 256),
1353
+ evidenceUnitIds: Object.freeze(stringList(
1354
+ supportInput.evidenceUnitIds,
1355
+ `${supportLabel}.evidenceUnitIds`,
1356
+ { canonical: true, maximum: 100 },
1357
+ ).map((id, evidenceIndex) => evidenceUnitId(
1358
+ id,
1359
+ `${supportLabel}.evidenceUnitIds[${evidenceIndex}]`,
1360
+ ))),
1361
+ });
1362
+ },
1363
+ );
1364
+ assertCanonicalOrder(acceptableSupportSets, ({ id }) => id, `${itemLabel}.acceptableSupportSets`);
1365
+ return Object.freeze({
1366
+ id: boundedString(input.id, `${itemLabel}.id`, 256),
1367
+ text: boundedString(input.text, `${itemLabel}.text`, 4_096),
1368
+ required: input.required,
1369
+ acceptableSupportSets: Object.freeze(acceptableSupportSets),
1370
+ });
1371
+ });
1372
+ assertCanonicalOrder(nuggets, ({ id }) => id, label);
1373
+ if (!nuggets.some(({ required }) => required)) {
1374
+ throw new TypeError(`${label} must contain at least one required nugget.`);
1375
+ }
1376
+ return Object.freeze(nuggets);
1377
+ }
1378
+
1379
+ function parseGold(value: unknown, label: string): EvaluationGoldJudgmentV2 {
1380
+ const input = record(value, label);
1381
+ strictKeys(input, ["documents", "evidenceUnits", "nuggets"], label);
1382
+ return Object.freeze({
1383
+ documents: parseDocumentJudgments(input.documents, `${label}.documents`),
1384
+ evidenceUnits: parseEvidenceUnitJudgments(input.evidenceUnits, `${label}.evidenceUnits`),
1385
+ nuggets: parseNuggets(input.nuggets, `${label}.nuggets`),
1386
+ });
1387
+ }
1388
+
1389
+ function parseRawAssessments(
1390
+ value: unknown,
1391
+ queryIndex: number,
1392
+ finalNuggets: readonly EvaluationAtomicNuggetV2[],
1393
+ ): readonly EvaluationRawAssessorJudgmentV2[] {
1394
+ const label = `queries[${queryIndex}].rawAssessments`;
1395
+ if (!Array.isArray(value) || value.length < 1 || value.length > 10) {
1396
+ throw new TypeError(`${label} must contain from 1 through 10 entries.`);
1397
+ }
1398
+ const assessments = value.map((entry, index): EvaluationRawAssessorJudgmentV2 => {
1399
+ const itemLabel = `${label}[${index}]`;
1400
+ const input = record(entry, itemLabel);
1401
+ strictKeys(
1402
+ input,
1403
+ ["assessorId", "documents", "evidenceUnits", "expectedSupport", "nuggets"],
1404
+ itemLabel,
1405
+ );
1406
+ if (input.expectedSupport !== "supported" && input.expectedSupport !== "insufficient") {
1407
+ throw new TypeError(`${itemLabel}.expectedSupport must be insufficient or supported.`);
1408
+ }
1409
+ if (!Array.isArray(input.nuggets) || input.nuggets.length > MAX_EVALUATION_V2_NUGGETS_PER_QUERY) {
1410
+ throw new TypeError(`${itemLabel}.nuggets has too many entries.`);
1411
+ }
1412
+ const nuggets = input.nuggets.map((nuggetEntry, nuggetIndex) => {
1413
+ const nuggetLabel = `${itemLabel}.nuggets[${nuggetIndex}]`;
1414
+ const nuggetInput = record(nuggetEntry, nuggetLabel);
1415
+ strictKeys(nuggetInput, ["acceptableSupportSetIds", "nuggetId", "required"], nuggetLabel);
1416
+ const nuggetId = boundedString(nuggetInput.nuggetId, `${nuggetLabel}.nuggetId`, 256);
1417
+ const finalRequired = finalNuggets.find(({ id }) => id === nuggetId)?.required;
1418
+ if (nuggetInput.required !== undefined && typeof nuggetInput.required !== "boolean") {
1419
+ throw new TypeError(`${nuggetLabel}.required must be boolean.`);
1420
+ }
1421
+ return Object.freeze({
1422
+ nuggetId,
1423
+ required: nuggetInput.required ?? finalRequired ?? true,
1424
+ acceptableSupportSetIds: stringList(
1425
+ nuggetInput.acceptableSupportSetIds,
1426
+ `${nuggetLabel}.acceptableSupportSetIds`,
1427
+ { allowEmpty: true, canonical: true, maximum: MAX_EVALUATION_V2_SUPPORT_SETS_PER_NUGGET },
1428
+ ),
1429
+ });
1430
+ });
1431
+ assertCanonicalOrder(nuggets, ({ nuggetId }) => nuggetId, `${itemLabel}.nuggets`);
1432
+ return Object.freeze({
1433
+ assessorId: boundedString(input.assessorId, `${itemLabel}.assessorId`, 256),
1434
+ expectedSupport: input.expectedSupport,
1435
+ documents: parseDocumentJudgments(input.documents, `${itemLabel}.documents`),
1436
+ evidenceUnits: parseEvidenceUnitJudgments(input.evidenceUnits, `${itemLabel}.evidenceUnits`),
1437
+ nuggets: Object.freeze(nuggets),
1438
+ });
1439
+ });
1440
+ assertCanonicalOrder(assessments, ({ assessorId }) => assessorId, label);
1441
+ return Object.freeze(assessments);
1442
+ }
1443
+
1444
+ function parseAdjudication(value: unknown, queryIndex: number): EvaluationAdjudicationV2 {
1445
+ const label = `queries[${queryIndex}].adjudication`;
1446
+ const input = record(value, label);
1447
+ strictKeys(input, ["adjudicatorId", "rationale", "status"], label);
1448
+ if (input.status === "single-assessor" || input.status === "agreed") {
1449
+ if (input.adjudicatorId !== undefined || input.rationale !== undefined) {
1450
+ throw new TypeError(`${label} may name an adjudicator only when status is resolved.`);
1451
+ }
1452
+ return Object.freeze({ status: input.status });
1453
+ }
1454
+ if (input.status !== "resolved") {
1455
+ throw new TypeError(`${label}.status must be agreed, resolved, or single-assessor.`);
1456
+ }
1457
+ return Object.freeze({
1458
+ status: "resolved",
1459
+ adjudicatorId: boundedString(input.adjudicatorId, `${label}.adjudicatorId`, 256),
1460
+ rationale: boundedString(input.rationale, `${label}.rationale`, 4_096),
1461
+ });
1462
+ }
1463
+
1464
+ function finalNuggetRows(gold: EvaluationGoldJudgmentV2): EvaluationRawAssessorJudgmentV2["nuggets"] {
1465
+ return Object.freeze(gold.nuggets.map((nugget) => Object.freeze({
1466
+ nuggetId: nugget.id,
1467
+ required: nugget.required,
1468
+ acceptableSupportSetIds: Object.freeze(
1469
+ nugget.acceptableSupportSets.map(({ id }) => id),
1470
+ ),
1471
+ })));
1472
+ }
1473
+
1474
+ function judgmentSignature(
1475
+ judgment: Omit<EvaluationRawAssessorJudgmentV2, "assessorId">,
1476
+ ): string {
1477
+ return canonicalJson(judgment);
1478
+ }
1479
+
1480
+ function finalJudgmentSignature(query: EvaluationQueryV2): string {
1481
+ return judgmentSignature({
1482
+ expectedSupport: query.expectedSupport,
1483
+ documents: query.gold.documents,
1484
+ evidenceUnits: query.gold.evidenceUnits,
1485
+ nuggets: finalNuggetRows(query.gold),
1486
+ });
1487
+ }
1488
+
1489
+ function rawJudgmentSignature(judgment: EvaluationRawAssessorJudgmentV2): string {
1490
+ return judgmentSignature({
1491
+ expectedSupport: judgment.expectedSupport,
1492
+ documents: judgment.documents,
1493
+ evidenceUnits: judgment.evidenceUnits,
1494
+ nuggets: judgment.nuggets,
1495
+ });
1496
+ }
1497
+
1498
+ function hasCompleteSupport(
1499
+ expectedSupportSets: readonly { readonly id: string; readonly evidenceUnitIds: readonly string[] }[],
1500
+ selectedSupportSetIds: ReadonlySet<string>,
1501
+ evidenceGrades: ReadonlyMap<string, number>,
1502
+ ): boolean {
1503
+ return expectedSupportSets.some((supportSet) =>
1504
+ selectedSupportSetIds.has(supportSet.id)
1505
+ && supportSet.evidenceUnitIds.every((id) => (evidenceGrades.get(id) ?? 0) > 0));
1506
+ }
1507
+
1508
+ function hasCompleteRequiredNuggetCoverage(
1509
+ nuggets: readonly EvaluationAtomicNuggetV2[],
1510
+ selected: ReadonlyMap<string, ReadonlySet<string>>,
1511
+ evidenceGrades: ReadonlyMap<string, number>,
1512
+ ): boolean {
1513
+ return nuggets.filter(({ required }) => required).every((nugget) =>
1514
+ hasCompleteSupport(
1515
+ nugget.acceptableSupportSets,
1516
+ selected.get(nugget.id) ?? new Set<string>(),
1517
+ evidenceGrades,
1518
+ ));
1519
+ }
1520
+
1521
+ function validateGoldReferences(
1522
+ query: EvaluationQueryV2,
1523
+ documents: ReadonlyMap<string, EvaluationDocumentV2>,
1524
+ evidenceUnits: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
1525
+ assessorIds: ReadonlySet<string>,
1526
+ label: string,
1527
+ ): void {
1528
+ const documentGrades = new Map(query.gold.documents.map((row) => [row.documentId, row.relevance]));
1529
+ const evidenceGrades = new Map(query.gold.evidenceUnits.map((row) => [row.evidenceUnitId, row.relevance]));
1530
+ for (const row of query.gold.documents) {
1531
+ if (!documents.has(row.documentId)) {
1532
+ throw new TypeError(`${label}.gold.documents references unknown document ${row.documentId}.`);
1533
+ }
1534
+ }
1535
+ for (const row of query.gold.evidenceUnits) {
1536
+ const unit = evidenceUnits.get(row.evidenceUnitId);
1537
+ if (unit === undefined) {
1538
+ throw new TypeError(`${label}.gold.evidenceUnits references unknown unit ${row.evidenceUnitId}.`);
1539
+ }
1540
+ if (!documentGrades.has(unit.documentId)) {
1541
+ throw new TypeError(`${label}.gold must judge the document containing ${row.evidenceUnitId}.`);
1542
+ }
1543
+ if (row.relevance > 0 && (documentGrades.get(unit.documentId) ?? 0) === 0) {
1544
+ throw new TypeError(`${label}.gold cannot place relevant evidence in an irrelevant document.`);
1545
+ }
1546
+ }
1547
+ const supportSetIds = new Set<string>();
1548
+ for (const nugget of query.gold.nuggets) {
1549
+ for (const supportSet of nugget.acceptableSupportSets) {
1550
+ if (supportSetIds.has(supportSet.id)) {
1551
+ throw new TypeError(`${label}.gold support-set IDs must be unique across nuggets.`);
1552
+ }
1553
+ supportSetIds.add(supportSet.id);
1554
+ for (const unitId of supportSet.evidenceUnitIds) {
1555
+ if (!evidenceUnits.has(unitId)) {
1556
+ throw new TypeError(`${label}.gold support set references unknown unit ${unitId}.`);
1557
+ }
1558
+ if ((evidenceGrades.get(unitId) ?? 0) === 0) {
1559
+ throw new TypeError(`${label}.gold support set ${supportSet.id} must use positively judged units.`);
1560
+ }
1561
+ }
1562
+ }
1563
+ }
1564
+ const finalSelection = new Map(finalNuggetRows(query.gold).map((row) => [
1565
+ row.nuggetId,
1566
+ new Set(row.acceptableSupportSetIds),
1567
+ ]));
1568
+ const complete = hasCompleteRequiredNuggetCoverage(query.gold.nuggets, finalSelection, evidenceGrades);
1569
+ if (query.expectedSupport === "supported" && !complete) {
1570
+ throw new TypeError(`${label} is supported but has no complete acceptable support for every required nugget.`);
1571
+ }
1572
+ if (query.expectedSupport === "insufficient" && complete) {
1573
+ throw new TypeError(`${label} is insufficient but contains complete acceptable support for all required nuggets.`);
1574
+ }
1575
+ if (query.expectedSupport === "supported" && query.negativeSubtype !== undefined) {
1576
+ throw new TypeError(`${label}.negativeSubtype is forbidden for supported queries.`);
1577
+ }
1578
+ if (query.expectedSupport === "insufficient" && query.negativeSubtype === undefined) {
1579
+ throw new TypeError(`${label}.negativeSubtype is required for insufficient queries.`);
1580
+ }
1581
+
1582
+ const finalSignature = finalJudgmentSignature(query);
1583
+ const rawSignatures = query.rawAssessments.map(rawJudgmentSignature);
1584
+ for (const raw of query.rawAssessments) {
1585
+ if (!assessorIds.has(raw.assessorId)) {
1586
+ throw new TypeError(`${label}.rawAssessments names undeclared assessor ${raw.assessorId}.`);
1587
+ }
1588
+ if (canonicalJson(raw.documents.map(({ documentId }) => documentId))
1589
+ !== canonicalJson(query.gold.documents.map(({ documentId }) => documentId))) {
1590
+ throw new TypeError(`${label} raw assessors must judge the complete final document pool.`);
1591
+ }
1592
+ if (canonicalJson(raw.evidenceUnits.map(({ evidenceUnitId }) => evidenceUnitId))
1593
+ !== canonicalJson(query.gold.evidenceUnits.map(({ evidenceUnitId }) => evidenceUnitId))) {
1594
+ throw new TypeError(`${label} raw assessors must judge the complete final evidence-unit pool.`);
1595
+ }
1596
+ if (canonicalJson(raw.nuggets.map(({ nuggetId }) => nuggetId))
1597
+ !== canonicalJson(query.gold.nuggets.map(({ id }) => id))) {
1598
+ throw new TypeError(`${label} raw assessors must judge every final nugget.`);
1599
+ }
1600
+ const rawEvidenceGrades = new Map(raw.evidenceUnits.map((row) => [row.evidenceUnitId, row.relevance]));
1601
+ if (!raw.nuggets.some(({ required }) => required)) {
1602
+ throw new TypeError(`${label} raw assessments must require at least one final nugget.`);
1603
+ }
1604
+ const rawSelection = new Map(raw.nuggets.map((row) => [
1605
+ row.nuggetId,
1606
+ new Set(row.acceptableSupportSetIds),
1607
+ ]));
1608
+ for (const row of raw.nuggets) {
1609
+ const nugget = query.gold.nuggets.find(({ id }) => id === row.nuggetId);
1610
+ const allowed = new Set(nugget?.acceptableSupportSets.map(({ id }) => id) ?? []);
1611
+ for (const id of row.acceptableSupportSetIds) {
1612
+ if (!allowed.has(id)) {
1613
+ throw new TypeError(`${label} raw assessment references unknown support set ${id}.`);
1614
+ }
1615
+ }
1616
+ }
1617
+ const rawRequiredNuggets = query.gold.nuggets.map((nugget) => Object.freeze({
1618
+ ...nugget,
1619
+ required: raw.nuggets.find(({ nuggetId }) => nuggetId === nugget.id)?.required ?? nugget.required,
1620
+ }));
1621
+ const rawComplete = hasCompleteRequiredNuggetCoverage(rawRequiredNuggets, rawSelection, rawEvidenceGrades);
1622
+ if ((raw.expectedSupport === "supported") !== rawComplete) {
1623
+ throw new TypeError(`${label} raw assessment support state contradicts its support-set coverage.`);
1624
+ }
1625
+ }
1626
+
1627
+ const distinct = new Set(rawSignatures);
1628
+ if (query.rawAssessments.length === 1) {
1629
+ if (query.adjudication.status !== "single-assessor" || rawSignatures[0] !== finalSignature) {
1630
+ throw new TypeError(`${label} single assessment requires explicit single-assessor adjudication and matching final gold.`);
1631
+ }
1632
+ return;
1633
+ }
1634
+ if (distinct.size === 1) {
1635
+ if (query.adjudication.status !== "agreed" || rawSignatures[0] !== finalSignature) {
1636
+ throw new TypeError(`${label} matching independent assessments require explicit agreed adjudication and matching final gold.`);
1637
+ }
1638
+ return;
1639
+ }
1640
+ if (query.adjudication.status !== "resolved") {
1641
+ throw new TypeError(`${label} assessment disagreement requires explicit resolved adjudication.`);
1642
+ }
1643
+ if (!assessorIds.has(query.adjudication.adjudicatorId)) {
1644
+ throw new TypeError(`${label}.adjudication.adjudicatorId must name a declared assessor.`);
1645
+ }
1646
+ }
1647
+
1648
+ function parseQuery(value: unknown, index: number): EvaluationQueryV2 {
1649
+ const label = `queries[${index}]`;
1650
+ const input = record(value, label);
1651
+ strictKeys(
1652
+ input,
1653
+ [
1654
+ "adjudication",
1655
+ "cohort",
1656
+ "expectedSupport",
1657
+ "gold",
1658
+ "id",
1659
+ "inputOrigins",
1660
+ "inputs",
1661
+ "negativeSubtype",
1662
+ "primaryLane",
1663
+ "primaryStratum",
1664
+ "rawAssessments",
1665
+ "split",
1666
+ "strata",
1667
+ "text",
1668
+ ],
1669
+ label,
1670
+ );
1671
+ if (input.split !== "development" && input.split !== "test") {
1672
+ throw new TypeError(`${label}.split must be development or test.`);
1673
+ }
1674
+ if (input.cohort !== "caller-seeded" && input.cohort !== "text-only") {
1675
+ throw new TypeError(`${label}.cohort must be caller-seeded or text-only.`);
1676
+ }
1677
+ if (input.expectedSupport !== "supported" && input.expectedSupport !== "insufficient") {
1678
+ throw new TypeError(`${label}.expectedSupport must be insufficient or supported.`);
1679
+ }
1680
+ if (typeof input.primaryLane !== "string" || !lanes.has(input.primaryLane as EvaluationLaneIdV2)) {
1681
+ throw new TypeError(`${label}.primaryLane is invalid.`);
1682
+ }
1683
+ const negativeSubtype = input.negativeSubtype;
1684
+ if (
1685
+ negativeSubtype !== undefined
1686
+ && negativeSubtype !== "boundary-near-miss"
1687
+ && negativeSubtype !== "conflicting-evidence"
1688
+ && negativeSubtype !== "missing-required-support"
1689
+ && negativeSubtype !== "stale-only"
1690
+ && negativeSubtype !== "topical-near-miss"
1691
+ && negativeSubtype !== "unknown-entity"
1692
+ ) throw new TypeError(`${label}.negativeSubtype is invalid.`);
1693
+ if (!Array.isArray(input.strata) || input.strata.length < 1 || input.strata.length > strata.size) {
1694
+ throw new TypeError(`${label}.strata must be a non-empty bounded array.`);
1695
+ }
1696
+ const parsedStrata = input.strata.map((entry, stratumIndex) => {
1697
+ if (typeof entry !== "string" || !strata.has(entry as EvaluationStratumV2)) {
1698
+ throw new TypeError(`${label}.strata[${stratumIndex}] is invalid.`);
1699
+ }
1700
+ return entry as EvaluationStratumV2;
1701
+ });
1702
+ if (new Set(parsedStrata).size !== parsedStrata.length) {
1703
+ throw new TypeError(`${label}.strata must not contain duplicates.`);
1704
+ }
1705
+ const sortedStrata = parsedStrata.toSorted();
1706
+ if (parsedStrata.some((entry, stratumIndex) => entry !== sortedStrata[stratumIndex])) {
1707
+ throw new TypeError(`${label}.strata must be in canonical order.`);
1708
+ }
1709
+ if (
1710
+ typeof input.primaryStratum !== "string"
1711
+ || !strata.has(input.primaryStratum as EvaluationStratumV2)
1712
+ || !parsedStrata.includes(input.primaryStratum as EvaluationStratumV2)
1713
+ ) {
1714
+ throw new TypeError(`${label}.primaryStratum must name one of the query strata.`);
1715
+ }
1716
+ const inputs = parseRetrievalInputsV2(input.inputs, `${label}.inputs`);
1717
+ validateCohortInputs(input.cohort, inputs, label);
1718
+ validatePrimaryLaneInput(input.primaryLane as EvaluationLaneIdV2, inputs, label);
1719
+ const gold = parseGold(input.gold, `${label}.gold`);
1720
+ return Object.freeze({
1721
+ id: boundedString(input.id, `${label}.id`, 256),
1722
+ text: boundedString(input.text, `${label}.text`),
1723
+ split: input.split,
1724
+ cohort: input.cohort,
1725
+ strata: Object.freeze(parsedStrata),
1726
+ primaryStratum: input.primaryStratum as EvaluationStratumV2,
1727
+ expectedSupport: input.expectedSupport,
1728
+ primaryLane: input.primaryLane as EvaluationLaneIdV2,
1729
+ ...(negativeSubtype === undefined ? {} : { negativeSubtype }),
1730
+ inputs,
1731
+ inputOrigins: parseInputOrigins(input.inputOrigins, inputs, `${label}.inputOrigins`),
1732
+ gold,
1733
+ rawAssessments: parseRawAssessments(input.rawAssessments, index, gold.nuggets),
1734
+ adjudication: parseAdjudication(input.adjudication, index),
1735
+ });
1736
+ }
1737
+
1738
+ function parseAssessor(value: unknown, index: number): EvaluationAssessorV2 {
1739
+ const label = `assessment.assessors[${index}]`;
1740
+ const input = record(value, label);
1741
+ strictKeys(input, ["affiliation", "displayName", "id"], label);
1742
+ const displayName = optionalBoundedString(input.displayName, `${label}.displayName`);
1743
+ const affiliation = optionalBoundedString(input.affiliation, `${label}.affiliation`);
1744
+ return Object.freeze({
1745
+ id: canonicalId(input.id, `${label}.id`),
1746
+ ...(displayName === undefined ? {} : { displayName }),
1747
+ ...(affiliation === undefined ? {} : { affiliation }),
1748
+ });
1749
+ }
1750
+
1751
+ function parseMeasurementProfile(value: unknown, index: number): EvaluationMeasurementProfileV2 {
1752
+ const label = `measurementProfiles[${index}]`;
1753
+ const input = record(value, label);
1754
+ strictKeys(input, ["cacheState", "concurrency", "id", "operation", "repetitions", "scope"], label);
1755
+ if (
1756
+ input.operation !== "cold-index"
1757
+ && input.operation !== "four-reader-query"
1758
+ && input.operation !== "incremental-update"
1759
+ && input.operation !== "packing"
1760
+ && input.operation !== "warm-query"
1761
+ ) throw new TypeError(`${label}.operation is invalid.`);
1762
+ if (input.scope !== "query" && input.scope !== "retriever") {
1763
+ throw new TypeError(`${label}.scope must be query or retriever.`);
1764
+ }
1765
+ if (
1766
+ input.cacheState !== "changed-generation"
1767
+ && input.cacheState !== "cold"
1768
+ && input.cacheState !== "not-applicable"
1769
+ && input.cacheState !== "warm"
1770
+ ) throw new TypeError(`${label}.cacheState is invalid.`);
1771
+ const concurrency = safeInteger(input.concurrency, `${label}.concurrency`, 1, 64);
1772
+ const repetitions = safeInteger(input.repetitions, `${label}.repetitions`, 1, 100);
1773
+ const expected = {
1774
+ "cold-index": { scope: "retriever", cacheState: "cold", concurrency: 1 },
1775
+ "four-reader-query": { scope: "query", cacheState: "warm", concurrency: 4 },
1776
+ "incremental-update": { scope: "retriever", cacheState: "changed-generation", concurrency: 1 },
1777
+ packing: { scope: "query", cacheState: "warm", concurrency: 1 },
1778
+ "warm-query": { scope: "query", cacheState: "warm", concurrency: 1 },
1779
+ } as const;
1780
+ const required = expected[input.operation];
1781
+ if (input.scope !== required.scope || input.cacheState !== required.cacheState || concurrency !== required.concurrency) {
1782
+ throw new TypeError(`${label} does not match the fixed ${input.operation} operation profile.`);
1783
+ }
1784
+ return Object.freeze({
1785
+ id: canonicalId(input.id, `${label}.id`),
1786
+ operation: input.operation,
1787
+ scope: input.scope,
1788
+ cacheState: input.cacheState,
1789
+ concurrency,
1790
+ repetitions,
1791
+ });
1792
+ }
1793
+
1794
+ function parseExperiment(value: unknown): EvaluationExperimentV2 {
1795
+ const input = record(value, "experiment");
1796
+ strictKeys(input, ["environment", "protocol"], "experiment");
1797
+ const protocolInput = record(input.protocol, "experiment.protocol");
1798
+ strictKeys(
1799
+ protocolInput,
1800
+ ["contextCeilings", "minimumUsefulEffects", "nonInferiorityMargins", "pairedPower"],
1801
+ "experiment.protocol",
1802
+ );
1803
+ if (
1804
+ !Array.isArray(protocolInput.minimumUsefulEffects)
1805
+ || protocolInput.minimumUsefulEffects.length < 1
1806
+ || protocolInput.minimumUsefulEffects.length > minimumUsefulEffectMetrics.size * 2
1807
+ ) {
1808
+ throw new TypeError("experiment.protocol.minimumUsefulEffects must be a non-empty bounded array.");
1809
+ }
1810
+ const minimumUsefulEffects = protocolInput.minimumUsefulEffects.map((entry, index) => {
1811
+ const label = `experiment.protocol.minimumUsefulEffects[${index}]`;
1812
+ const effectInput = record(entry, label);
1813
+ strictKeys(effectInput, ["cohort", "metric", "minimumAbsoluteDifference"], label);
1814
+ if (
1815
+ typeof effectInput.metric !== "string"
1816
+ || !minimumUsefulEffectMetrics.has(effectInput.metric as EvaluationMinimumUsefulEffectMetricV2)
1817
+ ) throw new TypeError(`${label}.metric is invalid.`);
1818
+ if (effectInput.cohort !== "caller-seeded" && effectInput.cohort !== "text-only") {
1819
+ throw new TypeError(`${label}.cohort must be caller-seeded or text-only.`);
1820
+ }
1821
+ const minimumAbsoluteDifference = positiveNumber(
1822
+ effectInput.minimumAbsoluteDifference,
1823
+ `${label}.minimumAbsoluteDifference`,
1824
+ );
1825
+ if (minimumAbsoluteDifference > 1) {
1826
+ throw new TypeError(`${label}.minimumAbsoluteDifference must not exceed one.`);
1827
+ }
1828
+ return Object.freeze({
1829
+ metric: effectInput.metric as EvaluationMinimumUsefulEffectMetricV2,
1830
+ cohort: effectInput.cohort,
1831
+ minimumAbsoluteDifference,
1832
+ });
1833
+ });
1834
+ assertCanonicalOrder(
1835
+ minimumUsefulEffects,
1836
+ ({ metric, cohort }) => `${metric}:${cohort}`,
1837
+ "experiment.protocol.minimumUsefulEffects",
1838
+ );
1839
+
1840
+ if (
1841
+ !Array.isArray(protocolInput.nonInferiorityMargins)
1842
+ || protocolInput.nonInferiorityMargins.length < 1
1843
+ || protocolInput.nonInferiorityMargins.length > nonInferiorityMetrics.size
1844
+ ) {
1845
+ throw new TypeError("experiment.protocol.nonInferiorityMargins must be a non-empty bounded array.");
1846
+ }
1847
+ const nonInferiorityMargins = protocolInput.nonInferiorityMargins.map((entry, index) => {
1848
+ const label = `experiment.protocol.nonInferiorityMargins[${index}]`;
1849
+ const marginInput = record(entry, label);
1850
+ strictKeys(
1851
+ marginInput,
1852
+ ["maximumAbsoluteRegression", "maximumRelativeRegression", "metric"],
1853
+ label,
1854
+ );
1855
+ if (
1856
+ typeof marginInput.metric !== "string"
1857
+ || !nonInferiorityMetrics.has(marginInput.metric as EvaluationNonInferiorityMetricV2)
1858
+ ) throw new TypeError(`${label}.metric is invalid.`);
1859
+ const maximumAbsoluteRegression = nonnegativeNumber(
1860
+ marginInput.maximumAbsoluteRegression,
1861
+ `${label}.maximumAbsoluteRegression`,
1862
+ );
1863
+ const maximumRelativeRegression = nonnegativeNumber(
1864
+ marginInput.maximumRelativeRegression,
1865
+ `${label}.maximumRelativeRegression`,
1866
+ );
1867
+ const boundedQuality = !marginInput.metric.endsWith("-p95-ms");
1868
+ if (boundedQuality && maximumAbsoluteRegression > 1) {
1869
+ throw new TypeError(`${label}.maximumAbsoluteRegression must not exceed one for quality metrics.`);
1870
+ }
1871
+ if (maximumRelativeRegression > 10) {
1872
+ throw new TypeError(`${label}.maximumRelativeRegression must not exceed ten.`);
1873
+ }
1874
+ if (
1875
+ !boundedQuality
1876
+ && maximumAbsoluteRegression === 0
1877
+ && maximumRelativeRegression === 0
1878
+ ) {
1879
+ throw new TypeError(`${label} must declare a non-zero latency margin.`);
1880
+ }
1881
+ return Object.freeze({
1882
+ metric: marginInput.metric as EvaluationNonInferiorityMetricV2,
1883
+ maximumAbsoluteRegression,
1884
+ maximumRelativeRegression,
1885
+ });
1886
+ });
1887
+ assertCanonicalOrder(
1888
+ nonInferiorityMargins,
1889
+ ({ metric }) => metric,
1890
+ "experiment.protocol.nonInferiorityMargins",
1891
+ );
1892
+
1893
+ const pairedPowerInput = record(
1894
+ protocolInput.pairedPower,
1895
+ "experiment.protocol.pairedPower",
1896
+ );
1897
+ strictKeys(
1898
+ pairedPowerInput,
1899
+ [
1900
+ "alpha",
1901
+ "assumedDiscordantRate",
1902
+ "assumedEffect",
1903
+ "minimumUsefulEffect",
1904
+ "requiredPairs",
1905
+ "targetPower",
1906
+ ],
1907
+ "experiment.protocol.pairedPower",
1908
+ );
1909
+ const pairedPowerWithoutCount = Object.freeze({
1910
+ alpha: positiveNumber(pairedPowerInput.alpha, "experiment.protocol.pairedPower.alpha"),
1911
+ targetPower: positiveNumber(
1912
+ pairedPowerInput.targetPower,
1913
+ "experiment.protocol.pairedPower.targetPower",
1914
+ ),
1915
+ assumedDiscordantRate: positiveNumber(
1916
+ pairedPowerInput.assumedDiscordantRate,
1917
+ "experiment.protocol.pairedPower.assumedDiscordantRate",
1918
+ ),
1919
+ assumedEffect: positiveNumber(
1920
+ pairedPowerInput.assumedEffect,
1921
+ "experiment.protocol.pairedPower.assumedEffect",
1922
+ ),
1923
+ minimumUsefulEffect: nonnegativeNumber(
1924
+ pairedPowerInput.minimumUsefulEffect,
1925
+ "experiment.protocol.pairedPower.minimumUsefulEffect",
1926
+ ),
1927
+ });
1928
+ const expectedRequiredPairs = requiredPairedObservationsV2(pairedPowerWithoutCount);
1929
+ const requiredPairs = safeInteger(
1930
+ pairedPowerInput.requiredPairs,
1931
+ "experiment.protocol.pairedPower.requiredPairs",
1932
+ 1,
1933
+ MAX_EVALUATION_V2_QUERIES,
1934
+ );
1935
+ if (requiredPairs !== expectedRequiredPairs) {
1936
+ throw new TypeError(
1937
+ `experiment.protocol.pairedPower.requiredPairs must equal the derived count ${expectedRequiredPairs}.`,
1938
+ );
1939
+ }
1940
+ const pairedPower = Object.freeze({ ...pairedPowerWithoutCount, requiredPairs });
1941
+
1942
+ const ceilingsInput = record(protocolInput.contextCeilings, "experiment.protocol.contextCeilings");
1943
+ strictKeys(ceilingsInput, ["readerTokens", "utf8Bytes"], "experiment.protocol.contextCeilings");
1944
+ const contextCeilings = Object.freeze({
1945
+ utf8Bytes: safeInteger(
1946
+ ceilingsInput.utf8Bytes,
1947
+ "experiment.protocol.contextCeilings.utf8Bytes",
1948
+ 1,
1949
+ 1_000_000_000,
1950
+ ),
1951
+ readerTokens: safeInteger(
1952
+ ceilingsInput.readerTokens,
1953
+ "experiment.protocol.contextCeilings.readerTokens",
1954
+ 1,
1955
+ 1_000_000_000,
1956
+ ),
1957
+ });
1958
+
1959
+ const environmentInput = record(input.environment, "experiment.environment");
1960
+ strictKeys(
1961
+ environmentInput,
1962
+ [
1963
+ "cache",
1964
+ "fourReaderBatch",
1965
+ "hardware",
1966
+ "incrementalMutation",
1967
+ "localModel",
1968
+ "runtime",
1969
+ "tokenizer",
1970
+ ],
1971
+ "experiment.environment",
1972
+ );
1973
+ const tokenizerInput = record(environmentInput.tokenizer, "experiment.environment.tokenizer");
1974
+ strictKeys(tokenizerInput, ["id", "sha256"], "experiment.environment.tokenizer");
1975
+ const runtimeInput = record(environmentInput.runtime, "experiment.environment.runtime");
1976
+ strictKeys(runtimeInput, ["id", "sha256"], "experiment.environment.runtime");
1977
+ const hardwareInput = record(environmentInput.hardware, "experiment.environment.hardware");
1978
+ strictKeys(hardwareInput, ["id"], "experiment.environment.hardware");
1979
+ const cacheInput = record(environmentInput.cache, "experiment.environment.cache");
1980
+ strictKeys(cacheInput, ["fingerprintSha256", "preparation"], "experiment.environment.cache");
1981
+ const fourReaderBatchInput = record(
1982
+ environmentInput.fourReaderBatch,
1983
+ "experiment.environment.fourReaderBatch",
1984
+ );
1985
+ strictKeys(fourReaderBatchInput, ["id", "sha256"], "experiment.environment.fourReaderBatch");
1986
+ const incrementalMutationInput = record(
1987
+ environmentInput.incrementalMutation,
1988
+ "experiment.environment.incrementalMutation",
1989
+ );
1990
+ strictKeys(
1991
+ incrementalMutationInput,
1992
+ ["appendUtf8Sha256", "expectedPostMutationSha256", "sourcePath"],
1993
+ "experiment.environment.incrementalMutation",
1994
+ );
1995
+ const localModelInput = record(environmentInput.localModel, "experiment.environment.localModel");
1996
+ if (localModelInput.kind !== "none" && localModelInput.kind !== "model") {
1997
+ throw new TypeError("experiment.environment.localModel.kind must be model or none.");
1998
+ }
1999
+ strictKeys(
2000
+ localModelInput,
2001
+ localModelInput.kind === "none" ? ["kind"] : ["id", "kind", "sha256"],
2002
+ "experiment.environment.localModel",
2003
+ );
2004
+ const localModel: EvaluationExperimentV2["environment"]["localModel"] = localModelInput.kind === "none"
2005
+ ? Object.freeze({ kind: "none" })
2006
+ : Object.freeze({
2007
+ kind: "model",
2008
+ id: bridgeString(localModelInput.id, "experiment.environment.localModel.id", 512),
2009
+ sha256: requireSha256(localModelInput.sha256, "experiment.environment.localModel.sha256"),
2010
+ });
2011
+
2012
+ return Object.freeze({
2013
+ protocol: Object.freeze({
2014
+ minimumUsefulEffects: Object.freeze(minimumUsefulEffects),
2015
+ nonInferiorityMargins: Object.freeze(nonInferiorityMargins),
2016
+ pairedPower,
2017
+ contextCeilings,
2018
+ }),
2019
+ environment: Object.freeze({
2020
+ tokenizer: Object.freeze({
2021
+ id: bridgeString(tokenizerInput.id, "experiment.environment.tokenizer.id", 512),
2022
+ sha256: requireSha256(tokenizerInput.sha256, "experiment.environment.tokenizer.sha256"),
2023
+ }),
2024
+ runtime: Object.freeze({
2025
+ id: bridgeString(runtimeInput.id, "experiment.environment.runtime.id", 512),
2026
+ sha256: requireSha256(runtimeInput.sha256, "experiment.environment.runtime.sha256"),
2027
+ }),
2028
+ hardware: Object.freeze({
2029
+ id: bridgeString(hardwareInput.id, "experiment.environment.hardware.id", 1_024),
2030
+ }),
2031
+ localModel,
2032
+ cache: Object.freeze({
2033
+ preparation: bridgeString(cacheInput.preparation, "experiment.environment.cache.preparation", 2_048),
2034
+ fingerprintSha256: requireSha256(
2035
+ cacheInput.fingerprintSha256,
2036
+ "experiment.environment.cache.fingerprintSha256",
2037
+ ),
2038
+ }),
2039
+ fourReaderBatch: Object.freeze({
2040
+ id: bridgeString(fourReaderBatchInput.id, "experiment.environment.fourReaderBatch.id", 512),
2041
+ sha256: requireSha256(
2042
+ fourReaderBatchInput.sha256,
2043
+ "experiment.environment.fourReaderBatch.sha256",
2044
+ ),
2045
+ }),
2046
+ incrementalMutation: Object.freeze({
2047
+ sourcePath: confinedPath(
2048
+ incrementalMutationInput.sourcePath,
2049
+ "experiment.environment.incrementalMutation.sourcePath",
2050
+ ),
2051
+ appendUtf8Sha256: requireSha256(
2052
+ incrementalMutationInput.appendUtf8Sha256,
2053
+ "experiment.environment.incrementalMutation.appendUtf8Sha256",
2054
+ ),
2055
+ expectedPostMutationSha256: requireSha256(
2056
+ incrementalMutationInput.expectedPostMutationSha256,
2057
+ "experiment.environment.incrementalMutation.expectedPostMutationSha256",
2058
+ ),
2059
+ }),
2060
+ }),
2061
+ });
2062
+ }
2063
+
2064
+ function parseConfiguration(value: unknown, label: string): Readonly<Record<string, string | number | boolean | null>> {
2065
+ const input = record(value, label);
2066
+ if (Object.keys(input).length > 64) throw new TypeError(`${label} may have at most 64 fields.`);
2067
+ const output: Record<string, string | number | boolean | null> = {};
2068
+ for (const rawKey of Object.keys(input).toSorted()) {
2069
+ const key = canonicalId(rawKey, `${label} key`);
2070
+ const candidate = input[rawKey];
2071
+ if (
2072
+ candidate !== null
2073
+ && typeof candidate !== "boolean"
2074
+ && typeof candidate !== "number"
2075
+ && typeof candidate !== "string"
2076
+ ) throw new TypeError(`${label}.${key} must be a JSON scalar.`);
2077
+ if (typeof candidate === "number" && !Number.isFinite(candidate)) {
2078
+ throw new TypeError(`${label}.${key} must be finite.`);
2079
+ }
2080
+ output[key] = typeof candidate === "string" ? boundedString(candidate, `${label}.${key}`) : candidate;
2081
+ }
2082
+ return Object.freeze(output);
2083
+ }
2084
+
2085
+ function parseRetrieverDescriptor(value: unknown, index: number): EvaluationRetrieverDescriptorV2 {
2086
+ const label = `retrievers[${index}]`;
2087
+ const input = record(value, label);
2088
+ strictKeys(input, ["configuration", "id", "implementationSha256", "lanes", "role", "version"], label);
2089
+ if (input.role !== "ablation" && input.role !== "baseline" && input.role !== "candidate") {
2090
+ throw new TypeError(`${label}.role is invalid.`);
2091
+ }
2092
+ if (!Array.isArray(input.lanes) || input.lanes.length < 1 || input.lanes.length > lanes.size) {
2093
+ throw new TypeError(`${label}.lanes must be a non-empty bounded array.`);
2094
+ }
2095
+ const parsedLanes = input.lanes.map((lane, laneIndex) => {
2096
+ if (typeof lane !== "string" || !lanes.has(lane as EvaluationLaneIdV2)) {
2097
+ throw new TypeError(`${label}.lanes[${laneIndex}] is invalid.`);
2098
+ }
2099
+ return lane as EvaluationLaneIdV2;
2100
+ });
2101
+ const sortedLanes = parsedLanes.toSorted();
2102
+ if (new Set(parsedLanes).size !== parsedLanes.length || parsedLanes.some((lane, index) => lane !== sortedLanes[index])) {
2103
+ throw new TypeError(`${label}.lanes must be unique and in canonical order.`);
2104
+ }
2105
+ return Object.freeze({
2106
+ id: canonicalId(input.id, `${label}.id`),
2107
+ role: input.role,
2108
+ version: boundedString(input.version, `${label}.version`, 512),
2109
+ implementationSha256: requireSha256(input.implementationSha256, `${label}.implementationSha256`),
2110
+ lanes: Object.freeze(parsedLanes),
2111
+ configuration: parseConfiguration(input.configuration, `${label}.configuration`),
2112
+ });
2113
+ }
2114
+
2115
+ function parseCandidateLock(value: unknown): EvaluationCandidateLockV2 {
2116
+ const label = "candidateLock";
2117
+ const input = record(value, label);
2118
+ strictKeys(input, ["baselineRetrieverId", "candidateRetrieverIds", "descriptorDigests"], label);
2119
+ const candidateRetrieverIds = stringList(input.candidateRetrieverIds, `${label}.candidateRetrieverIds`, {
2120
+ canonical: true,
2121
+ maximum: 32,
2122
+ }).map((id, index) => canonicalId(id, `${label}.candidateRetrieverIds[${index}]`));
2123
+ if (!Array.isArray(input.descriptorDigests) || input.descriptorDigests.length < 1 || input.descriptorDigests.length > 64) {
2124
+ throw new TypeError(`${label}.descriptorDigests must contain from 1 through 64 entries.`);
2125
+ }
2126
+ const descriptorDigests = input.descriptorDigests.map((entry, index) => {
2127
+ const itemLabel = `${label}.descriptorDigests[${index}]`;
2128
+ const digestInput = record(entry, itemLabel);
2129
+ strictKeys(digestInput, ["retrieverId", "sha256"], itemLabel);
2130
+ return Object.freeze({
2131
+ retrieverId: canonicalId(digestInput.retrieverId, `${itemLabel}.retrieverId`),
2132
+ sha256: requireSha256(digestInput.sha256, `${itemLabel}.sha256`),
2133
+ });
2134
+ });
2135
+ assertCanonicalOrder(descriptorDigests, ({ retrieverId }) => retrieverId, `${label}.descriptorDigests`);
2136
+ return Object.freeze({
2137
+ baselineRetrieverId: canonicalId(input.baselineRetrieverId, `${label}.baselineRetrieverId`),
2138
+ candidateRetrieverIds: Object.freeze(candidateRetrieverIds),
2139
+ descriptorDigests: Object.freeze(descriptorDigests),
2140
+ });
2141
+ }
2142
+
2143
+ export function evaluationRetrieverDescriptorDigestV2(descriptor: EvaluationRetrieverDescriptorV2): string {
2144
+ return sha256(descriptor);
2145
+ }
2146
+
2147
+ export function evaluationCandidateLockDigestV2(lock: EvaluationCandidateLockV2): string {
2148
+ return sha256(lock);
2149
+ }
2150
+
2151
+ function corpusCommitmentPayload(value: RetrievalEvaluationCorpusV2): unknown {
2152
+ return {
2153
+ schemaVersion: value.schemaVersion,
2154
+ id: value.id,
2155
+ description: value.description,
2156
+ manifest: {
2157
+ protocol: value.manifest.protocol,
2158
+ sealedAt: value.manifest.sealedAt,
2159
+ buildContractSha256: value.manifest.buildContractSha256,
2160
+ },
2161
+ frozen: value.frozen,
2162
+ assessment: value.assessment,
2163
+ experiment: value.experiment,
2164
+ sourceFamilies: value.sourceFamilies,
2165
+ documents: value.documents,
2166
+ evidenceUnits: value.evidenceUnits,
2167
+ measurementProfiles: value.measurementProfiles,
2168
+ retrievers: value.retrievers,
2169
+ candidateLock: value.candidateLock,
2170
+ queries: value.queries,
2171
+ };
2172
+ }
2173
+
2174
+ export function evaluationCorpusDigestV2(corpus: RetrievalEvaluationCorpusV2): string {
2175
+ return sha256(corpusCommitmentPayload(corpus));
2176
+ }
2177
+
2178
+ /** Git-object commitment of the corpus's canonical JSON representation. */
2179
+ export function evaluationCorpusGitBlobCommitmentV2(
2180
+ corpus: RetrievalEvaluationCorpusV2,
2181
+ objectFormat: "sha1" | "sha256" = "sha1",
2182
+ ): string {
2183
+ const bytes = Buffer.from(canonicalJson(corpus), "utf8");
2184
+ return createHash(objectFormat)
2185
+ .update(`blob ${bytes.byteLength}\0`, "utf8")
2186
+ .update(bytes)
2187
+ .digest("hex");
2188
+ }
2189
+
2190
+ function parseManifest(value: unknown): RetrievalEvaluationCorpusV2["manifest"] {
2191
+ const input = record(value, "manifest");
2192
+ strictKeys(
2193
+ input,
2194
+ ["buildContractSha256", "candidateLockSha256", "corpusSha256", "protocol", "sealedAt"],
2195
+ "manifest",
2196
+ );
2197
+ if (input.protocol !== RETRIEVAL_EVALUATION_V2_PROTOCOL) {
2198
+ throw new TypeError(`manifest.protocol must be ${RETRIEVAL_EVALUATION_V2_PROTOCOL}.`);
2199
+ }
2200
+ const timestamp = new Date(input.sealedAt as string);
2201
+ if (typeof input.sealedAt !== "string" || Number.isNaN(timestamp.valueOf()) || timestamp.toISOString() !== input.sealedAt) {
2202
+ throw new TypeError("manifest.sealedAt must be a canonical ISO timestamp.");
2203
+ }
2204
+ return Object.freeze({
2205
+ protocol: RETRIEVAL_EVALUATION_V2_PROTOCOL,
2206
+ sealedAt: input.sealedAt,
2207
+ corpusSha256: requireSha256(input.corpusSha256, "manifest.corpusSha256"),
2208
+ candidateLockSha256: requireSha256(input.candidateLockSha256, "manifest.candidateLockSha256"),
2209
+ buildContractSha256: requireSha256(input.buildContractSha256, "manifest.buildContractSha256"),
2210
+ });
2211
+ }
2212
+
2213
+ function validateDescriptorLock(corpus: RetrievalEvaluationCorpusV2): void {
2214
+ const descriptorIds = corpus.retrievers.map(({ id }) => id);
2215
+ const lockIds = corpus.candidateLock.descriptorDigests.map(({ retrieverId }) => retrieverId);
2216
+ if (descriptorIds.length !== lockIds.length || descriptorIds.some((id, index) => id !== lockIds[index])) {
2217
+ throw new TypeError("candidateLock must commit to every retriever descriptor exactly once.");
2218
+ }
2219
+ const descriptors = new Map(corpus.retrievers.map((descriptor) => [descriptor.id, descriptor]));
2220
+ for (const locked of corpus.candidateLock.descriptorDigests) {
2221
+ const descriptor = descriptors.get(locked.retrieverId);
2222
+ if (descriptor === undefined || evaluationRetrieverDescriptorDigestV2(descriptor) !== locked.sha256) {
2223
+ throw new TypeError(`candidateLock descriptor digest does not match ${locked.retrieverId}.`);
2224
+ }
2225
+ }
2226
+ if (descriptors.get(corpus.candidateLock.baselineRetrieverId)?.role !== "baseline") {
2227
+ throw new TypeError("candidateLock.baselineRetrieverId must name a baseline descriptor.");
2228
+ }
2229
+ const candidates = corpus.retrievers.filter(({ role }) => role === "candidate").map(({ id }) => id);
2230
+ if (candidates.length !== corpus.candidateLock.candidateRetrieverIds.length || candidates.some((id, index) => id !== corpus.candidateLock.candidateRetrieverIds[index])) {
2231
+ throw new TypeError("candidateLock.candidateRetrieverIds must exactly name candidate descriptors.");
2232
+ }
2233
+ if (evaluationCandidateLockDigestV2(corpus.candidateLock) !== corpus.manifest.candidateLockSha256) {
2234
+ throw new TypeError("manifest.candidateLockSha256 does not match candidateLock.");
2235
+ }
2236
+ }
2237
+
2238
+ export function parseRetrievalEvaluationCorpusV2(
2239
+ inputValue: unknown,
2240
+ options: EvaluationCorpusParseOptionsV2,
2241
+ ): RetrievalEvaluationCorpusV2 {
2242
+ if (options === undefined || typeof options !== "object" || options === null) {
2243
+ throw new TypeError("evaluation v2 corpus parsing requires an explicit promotion claim.");
2244
+ }
2245
+ const parsedOptions = record(options, "evaluation v2 corpus parse options");
2246
+ if (parsedOptions.claimPromotion !== true && parsedOptions.claimPromotion !== false) {
2247
+ throw new TypeError("evaluation v2 corpus parsing requires claimPromotion true or false.");
2248
+ }
2249
+ strictKeys(
2250
+ parsedOptions,
2251
+ parsedOptions.claimPromotion ? ["claimPromotion", "expectedSeal"] : ["claimPromotion"],
2252
+ "evaluation v2 corpus parse options",
2253
+ );
2254
+ const input = record(inputValue, "evaluation v2 corpus");
2255
+ strictKeys(input, [
2256
+ "assessment", "candidateLock", "description", "documents", "evidenceUnits", "experiment", "frozen", "id",
2257
+ "manifest", "measurementProfiles", "queries", "retrievers", "schemaVersion", "sourceFamilies",
2258
+ ], "evaluation v2 corpus");
2259
+ if (input.schemaVersion !== RETRIEVAL_EVALUATION_V2_SCHEMA_VERSION) {
2260
+ throw new TypeError(`evaluation v2 corpus schemaVersion must be ${RETRIEVAL_EVALUATION_V2_SCHEMA_VERSION}.`);
2261
+ }
2262
+ const frozenInput = record(input.frozen, "frozen");
2263
+ strictKeys(frozenInput, ["repositoryCommit", "vaultRoot", "vaultTree"], "frozen");
2264
+ if (typeof frozenInput.repositoryCommit !== "string" || !gitObjectPattern.test(frozenInput.repositoryCommit)) {
2265
+ throw new TypeError("frozen.repositoryCommit must be a lowercase Git object ID.");
2266
+ }
2267
+ if (typeof frozenInput.vaultTree !== "string" || !gitObjectPattern.test(frozenInput.vaultTree)) {
2268
+ throw new TypeError("frozen.vaultTree must be a lowercase Git object ID.");
2269
+ }
2270
+ const assessmentInput = record(input.assessment, "assessment");
2271
+ strictKeys(assessmentInput, ["assessors", "rubricVersion"], "assessment");
2272
+ if (!Array.isArray(assessmentInput.assessors) || assessmentInput.assessors.length < 1 || assessmentInput.assessors.length > 100) {
2273
+ throw new TypeError("assessment.assessors must contain from 1 through 100 entries.");
2274
+ }
2275
+ const assessors = assessmentInput.assessors.map(parseAssessor);
2276
+ assertCanonicalOrder(assessors, ({ id }) => id, "assessment.assessors");
2277
+
2278
+ if (!Array.isArray(input.sourceFamilies) || input.sourceFamilies.length < 1 || input.sourceFamilies.length > MAX_EVALUATION_V2_DOCUMENTS) {
2279
+ throw new TypeError("sourceFamilies must be a non-empty bounded array.");
2280
+ }
2281
+ const sourceFamilies = input.sourceFamilies.map(parseSourceFamily);
2282
+ assertCanonicalOrder(sourceFamilies, ({ id }) => id, "sourceFamilies");
2283
+ if (!Array.isArray(input.documents) || input.documents.length < 1 || input.documents.length > MAX_EVALUATION_V2_DOCUMENTS) {
2284
+ throw new TypeError("documents must be a non-empty bounded array.");
2285
+ }
2286
+ const documents = input.documents.map(parseDocument);
2287
+ assertCanonicalOrder(documents, ({ id }) => id, "documents");
2288
+ if (!Array.isArray(input.evidenceUnits) || input.evidenceUnits.length < 1 || input.evidenceUnits.length > MAX_EVALUATION_V2_EVIDENCE_UNITS) {
2289
+ throw new TypeError("evidenceUnits must be a non-empty bounded array.");
2290
+ }
2291
+ const evidenceUnits = input.evidenceUnits.map(parseEvidenceUnit);
2292
+ assertCanonicalOrder(evidenceUnits, ({ id }) => id, "evidenceUnits");
2293
+ if (!Array.isArray(input.measurementProfiles) || input.measurementProfiles.length < 1 || input.measurementProfiles.length > 32) {
2294
+ throw new TypeError("measurementProfiles must contain from 1 through 32 entries.");
2295
+ }
2296
+ const measurementProfiles = input.measurementProfiles.map(parseMeasurementProfile);
2297
+ assertCanonicalOrder(measurementProfiles, ({ id }) => id, "measurementProfiles");
2298
+ if (new Set(measurementProfiles.map(({ operation }) => operation)).size !== measurementProfiles.length) {
2299
+ throw new TypeError("measurementProfiles must not repeat an operation.");
2300
+ }
2301
+ if (!Array.isArray(input.retrievers) || input.retrievers.length < 1 || input.retrievers.length > 64) {
2302
+ throw new TypeError("retrievers must contain from 1 through 64 entries.");
2303
+ }
2304
+ const retrievers = input.retrievers.map(parseRetrieverDescriptor);
2305
+ assertCanonicalOrder(retrievers, ({ id }) => id, "retrievers");
2306
+ if (!Array.isArray(input.queries) || input.queries.length < 1 || input.queries.length > MAX_EVALUATION_V2_QUERIES) {
2307
+ throw new TypeError(`queries must contain from 1 through ${MAX_EVALUATION_V2_QUERIES} entries.`);
2308
+ }
2309
+ const queries = input.queries.map(parseQuery);
2310
+ assertCanonicalOrder(queries, ({ id }) => id, "queries");
2311
+ const nuggetIds = queries.flatMap(({ gold }) => gold.nuggets.map(({ id }) => id));
2312
+ if (new Set(nuggetIds).size !== nuggetIds.length) {
2313
+ throw new TypeError("query nuggets must use corpus-wide unique IDs.");
2314
+ }
2315
+ const supportSetIds = queries.flatMap(({ gold }) =>
2316
+ gold.nuggets.flatMap(({ acceptableSupportSets }) => acceptableSupportSets.map(({ id }) => id)));
2317
+ if (new Set(supportSetIds).size !== supportSetIds.length) {
2318
+ throw new TypeError("acceptable support sets must use corpus-wide unique IDs.");
2319
+ }
2320
+
2321
+ const corpus: RetrievalEvaluationCorpusV2 = Object.freeze({
2322
+ schemaVersion: 2,
2323
+ id: canonicalId(input.id, "id"),
2324
+ description: boundedString(input.description, "description"),
2325
+ manifest: parseManifest(input.manifest),
2326
+ frozen: Object.freeze({
2327
+ repositoryCommit: frozenInput.repositoryCommit,
2328
+ vaultTree: frozenInput.vaultTree,
2329
+ vaultRoot: confinedPath(frozenInput.vaultRoot, "frozen.vaultRoot"),
2330
+ }),
2331
+ assessment: Object.freeze({
2332
+ rubricVersion: boundedString(assessmentInput.rubricVersion, "assessment.rubricVersion", 256),
2333
+ assessors: Object.freeze(assessors),
2334
+ }),
2335
+ experiment: parseExperiment(input.experiment),
2336
+ sourceFamilies: Object.freeze(sourceFamilies),
2337
+ documents: Object.freeze(documents),
2338
+ evidenceUnits: Object.freeze(evidenceUnits),
2339
+ measurementProfiles: Object.freeze(measurementProfiles),
2340
+ retrievers: Object.freeze(retrievers),
2341
+ candidateLock: parseCandidateLock(input.candidateLock),
2342
+ queries: Object.freeze(queries),
2343
+ });
2344
+
2345
+ const familyById = new Map(sourceFamilies.map((family) => [family.id, family]));
2346
+ const documentById = new Map(documents.map((document) => [document.id, document]));
2347
+ const evidenceById = new Map(evidenceUnits.map((unit) => [unit.id, unit]));
2348
+ const assessorIds = new Set(assessors.map(({ id }) => id));
2349
+ const documentBySourcePath = new Map<string, string>();
2350
+ for (const document of documents) {
2351
+ const family = familyById.get(document.sourceFamilyId);
2352
+ if (family === undefined) throw new TypeError(`document ${document.id} references an unknown source family.`);
2353
+ if (family.trustClass !== document.trustClass) {
2354
+ throw new TypeError(`document ${document.id} trust declaration disagrees with its source family.`);
2355
+ }
2356
+ const previousDocumentId = documentBySourcePath.get(document.sourcePath);
2357
+ if (previousDocumentId !== undefined) {
2358
+ throw new TypeError(
2359
+ `document source path ${document.sourcePath} is already bound to ${previousDocumentId}.`,
2360
+ );
2361
+ }
2362
+ documentBySourcePath.set(document.sourcePath, document.id);
2363
+ }
2364
+ const occupiedRanges = new Set<string>();
2365
+ const sourceIdentityByPath = new Map<string, { readonly documentId: string; readonly sourceFamilyId: string }>();
2366
+ for (const unit of evidenceUnits) {
2367
+ const document = documentById.get(unit.documentId);
2368
+ const family = familyById.get(unit.sourceFamilyId);
2369
+ if (document === undefined) throw new TypeError(`evidence unit ${unit.id} references an unknown document.`);
2370
+ if (family === undefined) throw new TypeError(`evidence unit ${unit.id} references an unknown source family.`);
2371
+ if (document.sourceFamilyId !== unit.sourceFamilyId || document.trustClass !== unit.trustClass || family.trustClass !== unit.trustClass) {
2372
+ throw new TypeError(`evidence unit ${unit.id} has inconsistent source-family or trust declarations.`);
2373
+ }
2374
+ if (document.sourcePath !== unit.sourcePath) {
2375
+ throw new TypeError(
2376
+ `evidence unit ${unit.id} source path disagrees with document ${document.id}.`,
2377
+ );
2378
+ }
2379
+ const previousSourceIdentity = sourceIdentityByPath.get(unit.sourcePath);
2380
+ if (
2381
+ previousSourceIdentity !== undefined
2382
+ && (
2383
+ previousSourceIdentity.documentId !== unit.documentId
2384
+ || previousSourceIdentity.sourceFamilyId !== unit.sourceFamilyId
2385
+ )
2386
+ ) {
2387
+ throw new TypeError(
2388
+ `source path ${unit.sourcePath} must belong to exactly one document and source family.`,
2389
+ );
2390
+ }
2391
+ sourceIdentityByPath.set(unit.sourcePath, {
2392
+ documentId: unit.documentId,
2393
+ sourceFamilyId: unit.sourceFamilyId,
2394
+ });
2395
+ const rangeKey = `${unit.sourcePath}\0${unit.lineRange.start}\0${unit.lineRange.end}`;
2396
+ if (occupiedRanges.has(rangeKey)) throw new TypeError(`evidence unit ${unit.id} repeats a canonical source range.`);
2397
+ occupiedRanges.add(rangeKey);
2398
+ }
2399
+ for (const query of queries) {
2400
+ validateGoldReferences(query, documentById, evidenceById, assessorIds, `query ${query.id}`);
2401
+ }
2402
+ validateDescriptorLock(corpus);
2403
+ if (evaluationCorpusDigestV2(corpus) !== corpus.manifest.corpusSha256) {
2404
+ throw new TypeError("manifest.corpusSha256 does not match the sealed evaluation corpus.");
2405
+ }
2406
+ if (options.claimPromotion) {
2407
+ const expectedSeal = record(options.expectedSeal, "promotion expected seal");
2408
+ strictKeys(expectedSeal, ["expectedCorpusSha256", "expectedGitBlob"], "promotion expected seal");
2409
+ const hasCorpusSha256 = expectedSeal.expectedCorpusSha256 !== undefined;
2410
+ const hasGitBlob = expectedSeal.expectedGitBlob !== undefined;
2411
+ if (hasCorpusSha256 === hasGitBlob) {
2412
+ throw new TypeError("promotion expected seal must supply exactly one corpus digest or Git blob commitment.");
2413
+ }
2414
+ if (hasCorpusSha256) {
2415
+ const expected = requireSha256(
2416
+ expectedSeal.expectedCorpusSha256,
2417
+ "promotion expected seal.expectedCorpusSha256",
2418
+ );
2419
+ if (expected !== evaluationCorpusDigestV2(corpus)) {
2420
+ throw new TypeError("promotion corpus does not match the independently supplied corpus digest.");
2421
+ }
2422
+ } else {
2423
+ if (typeof expectedSeal.expectedGitBlob !== "string" || !gitObjectPattern.test(expectedSeal.expectedGitBlob)) {
2424
+ throw new TypeError("promotion expected seal.expectedGitBlob must be a lowercase Git object ID.");
2425
+ }
2426
+ const objectFormat = expectedSeal.expectedGitBlob.length === 40 ? "sha1" : "sha256";
2427
+ if (evaluationCorpusGitBlobCommitmentV2(corpus, objectFormat) !== expectedSeal.expectedGitBlob) {
2428
+ throw new TypeError("promotion corpus does not match the independently supplied Git blob commitment.");
2429
+ }
2430
+ }
2431
+ validatePromotionCorpusLayoutV2(corpus);
2432
+ }
2433
+ return corpus;
2434
+ }
2435
+
2436
+ export function assertEvaluationRetrieverLockedV2(
2437
+ corpus: RetrievalEvaluationCorpusV2,
2438
+ descriptor: EvaluationRetrieverDescriptorV2,
2439
+ ): void {
2440
+ const locked = corpus.candidateLock.descriptorDigests.find(({ retrieverId }) => retrieverId === descriptor.id);
2441
+ const declared = corpus.retrievers.find(({ id }) => id === descriptor.id);
2442
+ const digest = evaluationRetrieverDescriptorDigestV2(descriptor);
2443
+ if (
2444
+ locked === undefined
2445
+ || declared === undefined
2446
+ || locked.sha256 !== digest
2447
+ || evaluationRetrieverDescriptorDigestV2(declared) !== digest
2448
+ ) throw new TypeError(`Retriever descriptor ${descriptor.id} is not committed by the sealed suite.`);
2449
+ }
2450
+
2451
+ export function projectEvaluationExecutionQueryV2(
2452
+ query: Pick<EvaluationQueryV2, "cohort" | "inputs">,
2453
+ ): EvaluationExecutionQueryV2 {
2454
+ const cohort = query.cohort;
2455
+ if (cohort !== "caller-seeded" && cohort !== "text-only") {
2456
+ throw new TypeError("execution query cohort is invalid.");
2457
+ }
2458
+ const inputs = parseRetrievalInputsV2(query.inputs, "execution query inputs");
2459
+ validateCohortInputs(cohort, inputs, "execution query");
2460
+ return cohort === "text-only"
2461
+ ? Object.freeze({ inputs: Object.freeze({ text: inputs.text }) })
2462
+ : Object.freeze({ inputs });
2463
+ }
2464
+
2465
+ export function createEvaluationExecutionRequestV2(options: {
2466
+ readonly corpus: RetrievalEvaluationCorpusV2;
2467
+ readonly query: Pick<EvaluationQueryV2, "cohort" | "inputs">;
2468
+ readonly descriptor: EvaluationRetrieverDescriptorV2;
2469
+ readonly limit: number;
2470
+ readonly signal: AbortSignal;
2471
+ }): EvaluationExecutionRequestV2 {
2472
+ assertEvaluationRetrieverLockedV2(options.corpus, options.descriptor);
2473
+ return Object.freeze({
2474
+ corpus: options.corpus.frozen,
2475
+ query: projectEvaluationExecutionQueryV2(options.query),
2476
+ limit: safeInteger(options.limit, "execution limit", 1, MAX_EVALUATION_V2_RESULTS_PER_LANE),
2477
+ signal: options.signal,
2478
+ });
2479
+ }
2480
+
2481
+ function referencedFamilyIds(
2482
+ query: EvaluationQueryV2,
2483
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
2484
+ documentById: ReadonlyMap<string, EvaluationDocumentV2>,
2485
+ ): ReadonlySet<string> {
2486
+ const families = new Set<string>();
2487
+ for (const { documentId } of query.gold.documents) {
2488
+ const familyId = documentById.get(documentId)?.sourceFamilyId;
2489
+ if (familyId !== undefined) families.add(familyId);
2490
+ }
2491
+ for (const { evidenceUnitId } of query.gold.evidenceUnits) {
2492
+ const familyId = evidenceById.get(evidenceUnitId)?.sourceFamilyId;
2493
+ if (familyId !== undefined) families.add(familyId);
2494
+ }
2495
+ return families;
2496
+ }
2497
+
2498
+ /**
2499
+ * Assign each query to the connected component of every causal source family it
2500
+ * judges. Reviewed families may contain multiple provenance/trust subfamilies;
2501
+ * their shared assignment commitment keeps those dependent records in one
2502
+ * statistical cluster. Legacy/unreviewed corpora fall back to the provenance
2503
+ * family ID.
2504
+ */
2505
+ export function evaluationSourceFamilyClusterIdsV2(
2506
+ queries: readonly EvaluationQueryV2[],
2507
+ documents: readonly EvaluationDocumentV2[],
2508
+ evidenceUnits: readonly EvaluationEvidenceUnitV2[],
2509
+ sourceFamilies: readonly EvaluationSourceFamilyV2[] = [],
2510
+ ): ReadonlyMap<string, string> {
2511
+ const documentById = new Map(documents.map((document) => [document.id, document]));
2512
+ const evidenceById = new Map(evidenceUnits.map((unit) => [unit.id, unit]));
2513
+ const clusterKeyByFamilyId = new Map(sourceFamilies.map((family) => [
2514
+ family.id,
2515
+ family.familyAssignmentSha256 === undefined
2516
+ ? family.id
2517
+ : `family-assignment:${family.familyAssignmentSha256}`,
2518
+ ]));
2519
+ const familiesByQuery = new Map(queries.map((query) => {
2520
+ const families = [...referencedFamilyIds(query, evidenceById, documentById)]
2521
+ .map((familyId) => clusterKeyByFamilyId.get(familyId) ?? familyId)
2522
+ .toSorted();
2523
+ if (families.length === 0) throw new TypeError(`Query ${query.id} has no sealed source-family cluster.`);
2524
+ return [query.id, families] as const;
2525
+ }));
2526
+ const parent = new Map<string, string>();
2527
+ const find = (id: string): string => {
2528
+ const existing = parent.get(id);
2529
+ if (existing === undefined) {
2530
+ parent.set(id, id);
2531
+ return id;
2532
+ }
2533
+ if (existing === id) return id;
2534
+ const root = find(existing);
2535
+ parent.set(id, root);
2536
+ return root;
2537
+ };
2538
+ const union = (left: string, right: string): void => {
2539
+ const leftRoot = find(left);
2540
+ const rightRoot = find(right);
2541
+ if (leftRoot === rightRoot) return;
2542
+ const root = leftRoot < rightRoot ? leftRoot : rightRoot;
2543
+ parent.set(root === leftRoot ? rightRoot : leftRoot, root);
2544
+ };
2545
+ for (const families of familiesByQuery.values()) {
2546
+ const first = families[0];
2547
+ if (first === undefined) continue;
2548
+ for (const family of families.slice(1)) union(first, family);
2549
+ }
2550
+ return new Map([...familiesByQuery].map(([queryId, families]) => {
2551
+ const first = families[0];
2552
+ if (first === undefined) throw new TypeError(`Query ${queryId} has no source family.`);
2553
+ return [queryId, find(first)] as const;
2554
+ }));
2555
+ }
2556
+
2557
+ function requireOpaqueIds(corpus: RetrievalEvaluationCorpusV2): void {
2558
+ for (const query of corpus.queries) {
2559
+ if (!opaquePatterns.query.test(query.id)) {
2560
+ throw new TypeError(`promotion query ID ${query.id} must be opaque and canonical.`);
2561
+ }
2562
+ for (const nugget of query.gold.nuggets) {
2563
+ if (!opaquePatterns.nugget.test(nugget.id)) {
2564
+ throw new TypeError(`promotion nugget ID ${nugget.id} must be opaque and canonical.`);
2565
+ }
2566
+ for (const supportSet of nugget.acceptableSupportSets) {
2567
+ if (!opaquePatterns.supportSet.test(supportSet.id)) {
2568
+ throw new TypeError(`promotion support-set ID ${supportSet.id} must be opaque and canonical.`);
2569
+ }
2570
+ }
2571
+ }
2572
+ }
2573
+ for (const family of corpus.sourceFamilies) {
2574
+ if (!opaquePatterns.sourceFamily.test(family.id)) {
2575
+ throw new TypeError(`promotion source-family ID ${family.id} must be opaque and canonical.`);
2576
+ }
2577
+ }
2578
+ for (const unit of corpus.evidenceUnits) {
2579
+ if (!opaquePatterns.evidenceUnit.test(unit.id)) {
2580
+ throw new TypeError(`promotion evidence-unit ID ${unit.id} must be opaque and canonical.`);
2581
+ }
2582
+ }
2583
+ }
2584
+
2585
+ function validatePromotionCorpusLayoutV2(corpus: RetrievalEvaluationCorpusV2): void {
2586
+ if (corpus.queries.length !== PROMOTION_EVALUATION_QUERY_COUNT_V2) {
2587
+ throw new TypeError(`promotion corpus must contain exactly ${PROMOTION_EVALUATION_QUERY_COUNT_V2} queries.`);
2588
+ }
2589
+ const development = corpus.queries.filter(({ split }) => split === "development");
2590
+ const test = corpus.queries.filter(({ split }) => split === "test");
2591
+ if (development.length !== PROMOTION_DEVELOPMENT_QUERY_COUNT_V2 || test.length !== PROMOTION_TEST_QUERY_COUNT_V2) {
2592
+ throw new TypeError("promotion corpus must contain exactly 48 development and 120 test queries.");
2593
+ }
2594
+ for (const cohort of ["caller-seeded", "text-only"] as const) {
2595
+ if (development.filter((query) => query.cohort === cohort).length !== 24) {
2596
+ throw new TypeError(`promotion development split must contain exactly 24 ${cohort} queries.`);
2597
+ }
2598
+ for (const expectedSupport of ["supported", "insufficient"] as const) {
2599
+ if (development.filter((query) =>
2600
+ query.cohort === cohort && query.expectedSupport === expectedSupport).length !== 12) {
2601
+ throw new TypeError(
2602
+ `promotion development ${cohort} cohort must contain exactly 12 ${expectedSupport} queries.`,
2603
+ );
2604
+ }
2605
+ }
2606
+ }
2607
+ if (
2608
+ test.filter(({ expectedSupport }) => expectedSupport === "supported").length !== PROMOTION_TEST_SUPPORTED_COUNT_V2
2609
+ || test.filter(({ expectedSupport }) => expectedSupport === "insufficient").length !== PROMOTION_TEST_INSUFFICIENT_COUNT_V2
2610
+ ) throw new TypeError("promotion test split must contain exactly 80 supported and 40 insufficient queries.");
2611
+ for (const cohort of ["caller-seeded", "text-only"] as const) {
2612
+ if (corpus.queries.filter((query) => query.cohort === cohort).length !== PROMOTION_COHORT_COUNT_V2) {
2613
+ throw new TypeError(`promotion corpus must contain exactly 84 ${cohort} queries.`);
2614
+ }
2615
+ if (test.filter((query) => query.cohort === cohort).length !== PROMOTION_TEST_COHORT_COUNT_V2) {
2616
+ throw new TypeError(`promotion test split must contain exactly 60 ${cohort} queries.`);
2617
+ }
2618
+ if (
2619
+ test.filter((query) => query.cohort === cohort && query.expectedSupport === "supported").length !== 40
2620
+ || test.filter((query) => query.cohort === cohort && query.expectedSupport === "insufficient").length !== 20
2621
+ ) {
2622
+ throw new TypeError(
2623
+ `promotion test ${cohort} cohort must contain exactly 40 supported and 20 insufficient queries.`,
2624
+ );
2625
+ }
2626
+ }
2627
+ for (const [stratum, minimum] of Object.entries(PROMOTION_ACCEPTANCE_STRATUM_MINIMA_V2)) {
2628
+ if (test.filter((query) => query.primaryStratum === stratum).length < minimum) {
2629
+ throw new TypeError(
2630
+ `promotion test split requires at least ${minimum} primary ${stratum} queries.`,
2631
+ );
2632
+ }
2633
+ for (const cohort of ["caller-seeded", "text-only"] as const) {
2634
+ const cohortMinimum = PROMOTION_ACCEPTANCE_STRATUM_COHORT_MINIMA_V2[
2635
+ stratum as keyof typeof PROMOTION_ACCEPTANCE_STRATUM_COHORT_MINIMA_V2
2636
+ ];
2637
+ if (
2638
+ test.filter((query) => query.cohort === cohort && query.primaryStratum === stratum).length
2639
+ < cohortMinimum
2640
+ ) {
2641
+ throw new TypeError(
2642
+ `promotion test ${cohort} cohort requires at least ${cohortMinimum} primary ${stratum} queries.`,
2643
+ );
2644
+ }
2645
+ }
2646
+ }
2647
+ for (const query of test) {
2648
+ if (
2649
+ (query.expectedSupport === "insufficient")
2650
+ !== query.strata.includes("no-answer-near-miss")
2651
+ ) {
2652
+ throw new TypeError(
2653
+ "promotion test insufficient queries must be explicitly stratified as no-answer near misses, and supported queries must not be.",
2654
+ );
2655
+ }
2656
+ }
2657
+ for (const [inputLane, minimum] of Object.entries(PROMOTION_CRITICAL_INPUT_MINIMA_V2)) {
2658
+ if (test.filter((query) => query.inputs[inputLane as keyof EvaluationRetrievalInputsV2] !== undefined).length < minimum) {
2659
+ throw new TypeError(`promotion test split requires at least ${minimum} executable ${inputLane} lane inputs.`);
2660
+ }
2661
+ }
2662
+ const dual = corpus.queries.filter(({ rawAssessments }) => rawAssessments.length >= 2);
2663
+ if (dual.length < PROMOTION_DUAL_ASSESSMENT_MINIMUM_V2) {
2664
+ throw new TypeError("promotion corpus requires independent dual assessment for at least 25 percent of queries.");
2665
+ }
2666
+ const promotionStrata = [
2667
+ ...Object.keys(PROMOTION_ACCEPTANCE_STRATUM_MINIMA_V2),
2668
+ "no-answer-near-miss",
2669
+ ] as EvaluationStratumV2[];
2670
+ for (const stratum of promotionStrata) {
2671
+ for (const cohort of ["caller-seeded", "text-only"] as const) {
2672
+ const cell = test.filter((query) =>
2673
+ query.cohort === cohort
2674
+ && (stratum === "no-answer-near-miss"
2675
+ ? query.strata.includes(stratum)
2676
+ : query.primaryStratum === stratum));
2677
+ const minimumDual = Math.max(
2678
+ PROMOTION_STRATUM_COHORT_DUAL_MINIMUM_V2,
2679
+ Math.ceil(cell.length * PROMOTION_STRATUM_COHORT_DUAL_FRACTION_V2),
2680
+ );
2681
+ if (cell.filter(({ rawAssessments }) => rawAssessments.length >= 2).length < minimumDual) {
2682
+ throw new TypeError(
2683
+ `promotion test ${cohort} ${stratum} stratum requires at least ${minimumDual} independently dual-assessed queries.`,
2684
+ );
2685
+ }
2686
+ }
2687
+ }
2688
+
2689
+ const effects = corpus.experiment.protocol.minimumUsefulEffects;
2690
+ if (
2691
+ effects.length !== 1
2692
+ || effects[0]?.metric !== "nugget-coverage"
2693
+ || effects[0].cohort !== "caller-seeded"
2694
+ ) {
2695
+ throw new TypeError(
2696
+ "promotion experiment must declare exactly one caller-seeded nugget-coverage minimum useful effect.",
2697
+ );
2698
+ }
2699
+ const marginMetrics = corpus.experiment.protocol.nonInferiorityMargins.map(({ metric }) => metric);
2700
+ if (
2701
+ marginMetrics.length !== promotionNonInferiorityMetrics.size
2702
+ || [...promotionNonInferiorityMetrics].some((metric) => !marginMetrics.includes(metric))
2703
+ ) {
2704
+ throw new TypeError("promotion experiment must predeclare every required metric-specific non-inferiority margin.");
2705
+ }
2706
+ const pairedPower = corpus.experiment.protocol.pairedPower;
2707
+ if (
2708
+ pairedPower.alpha !== 0.05
2709
+ || pairedPower.targetPower < 0.8
2710
+ || pairedPower.minimumUsefulEffect !== effects[0].minimumAbsoluteDifference
2711
+ ) {
2712
+ throw new TypeError(
2713
+ "promotion paired-power design must use one-sided alpha 0.05, at least 80 percent power, and the primary MUE threshold.",
2714
+ );
2715
+ }
2716
+ const eligiblePrimaryPairs = test.filter((query) =>
2717
+ query.cohort === "caller-seeded" && query.expectedSupport === "supported").length;
2718
+ if (eligiblePrimaryPairs < pairedPower.requiredPairs) {
2719
+ throw new TypeError(
2720
+ `promotion primary effect has ${eligiblePrimaryPairs} eligible pairs but its prospective design requires ${pairedPower.requiredPairs}.`,
2721
+ );
2722
+ }
2723
+ const testClusterIds = evaluationSourceFamilyClusterIdsV2(
2724
+ test,
2725
+ corpus.documents,
2726
+ corpus.evidenceUnits,
2727
+ corpus.sourceFamilies,
2728
+ );
2729
+ const eligiblePrimaryClusters = new Set(test
2730
+ .filter((query) => query.cohort === "caller-seeded" && query.expectedSupport === "supported")
2731
+ .map((query) => testClusterIds.get(query.id))).size;
2732
+ if (eligiblePrimaryClusters < pairedPower.requiredPairs) {
2733
+ throw new TypeError(
2734
+ `promotion primary effect has ${eligiblePrimaryClusters} independent source-family clusters but its prospective design requires ${pairedPower.requiredPairs}.`,
2735
+ );
2736
+ }
2737
+ for (const descriptor of corpus.retrievers) {
2738
+ if (Object.keys(descriptor.configuration).length === 0) {
2739
+ throw new TypeError(`promotion retriever descriptor ${descriptor.id} must have a non-empty configuration.`);
2740
+ }
2741
+ }
2742
+
2743
+ const evidenceById = new Map(corpus.evidenceUnits.map((unit) => [unit.id, unit]));
2744
+ const documentById = new Map(corpus.documents.map((document) => [document.id, document]));
2745
+ const sourceFamilyById = new Map(corpus.sourceFamilies.map((family) => [family.id, family]));
2746
+ const familySplits = new Map<string, EvaluationSplitV2>();
2747
+ for (const query of corpus.queries) {
2748
+ for (const familyId of referencedFamilyIds(query, evidenceById, documentById)) {
2749
+ const family = sourceFamilyById.get(familyId);
2750
+ if (family?.familyAssignmentSha256 === undefined) {
2751
+ throw new TypeError(
2752
+ `promotion source family ${familyId} is referenced by a query but lacks an independently reviewed family-assignment commitment.`,
2753
+ );
2754
+ }
2755
+ const familyClusterId = `family-assignment:${family.familyAssignmentSha256}`;
2756
+ const previous = familySplits.get(familyClusterId);
2757
+ if (previous !== undefined && previous !== query.split) {
2758
+ throw new TypeError(`source-family assignment ${family.familyAssignmentSha256} crosses development and test splits.`);
2759
+ }
2760
+ familySplits.set(familyClusterId, query.split);
2761
+ }
2762
+ }
2763
+ // The frozen catalog intentionally contains retrieval distractors that are not qrels.
2764
+ // Every judgment remains registry-bound above; catalog membership does not imply a grade.
2765
+ requireOpaqueIds(corpus);
2766
+ const requiredOperations = new Set<EvaluationMeasurementOperationV2>([
2767
+ "cold-index", "four-reader-query", "incremental-update", "packing", "warm-query",
2768
+ ]);
2769
+ for (const profile of corpus.measurementProfiles) {
2770
+ requiredOperations.delete(profile.operation);
2771
+ if (profile.repetitions < 3) {
2772
+ throw new TypeError(`promotion measurement profile ${profile.id} requires at least three repetitions.`);
2773
+ }
2774
+ }
2775
+ if (requiredOperations.size > 0) {
2776
+ throw new TypeError(`promotion corpus is missing measurement profiles: ${[...requiredOperations].toSorted().join(", ")}.`);
2777
+ }
2778
+ }
2779
+
2780
+ /** Validate every promotion-design invariant before an independent seal is anchored. */
2781
+ export function validatePromotionCorpusDesignV2(
2782
+ input: RetrievalEvaluationCorpusV2,
2783
+ ): RetrievalEvaluationCorpusV2 {
2784
+ const corpus = parseRetrievalEvaluationCorpusV2(input, { claimPromotion: false });
2785
+ validatePromotionCorpusLayoutV2(corpus);
2786
+ return corpus;
2787
+ }
2788
+
2789
+ export function validatePromotionCorpusV2(
2790
+ input: unknown,
2791
+ expectedSeal: EvaluationExternalCorpusSealV2,
2792
+ ): RetrievalEvaluationCorpusV2 {
2793
+ return parseRetrievalEvaluationCorpusV2(input, {
2794
+ claimPromotion: true,
2795
+ expectedSeal,
2796
+ });
2797
+ }
2798
+
2799
+ function parseEvidenceLocator(
2800
+ value: unknown,
2801
+ label: string,
2802
+ documentId: string,
2803
+ laneId: EvaluationLaneIdV2,
2804
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
2805
+ familyById: ReadonlyMap<string, EvaluationSourceFamilyV2>,
2806
+ ): EvaluationEvidenceLocatorV2 {
2807
+ const input = record(value, label);
2808
+ strictKeys(input, [
2809
+ "evidenceUnitId",
2810
+ "headingPath",
2811
+ "lineRange",
2812
+ "sourceClass",
2813
+ "sourceFamilyId",
2814
+ "sourcePage",
2815
+ "sourcePath",
2816
+ "trustClass",
2817
+ ], label);
2818
+ const parsedEvidenceUnitId = evidenceUnitId(input.evidenceUnitId, `${label}.evidenceUnitId`);
2819
+ const unit = evidenceById.get(parsedEvidenceUnitId);
2820
+ if (unit === undefined) throw new TypeError(`${label} references unknown evidence unit ${parsedEvidenceUnitId}.`);
2821
+ if (unit.documentId !== documentId && laneId !== "graph") {
2822
+ throw new TypeError(`${label} evidence unit ${parsedEvidenceUnitId} belongs to a different document.`);
2823
+ }
2824
+ const family = familyById.get(unit.sourceFamilyId);
2825
+ if (family === undefined) throw new TypeError(`${label} evidence unit has an unknown source family.`);
2826
+ if (
2827
+ input.sourceFamilyId !== unit.sourceFamilyId
2828
+ || input.sourceClass !== family.sourceClass
2829
+ || input.trustClass !== unit.trustClass
2830
+ ) {
2831
+ throw new TypeError(`${label} source-family, source-class, or trust declaration is not registry-bound.`);
2832
+ }
2833
+ const sourcePage = input.sourcePage === undefined
2834
+ ? undefined
2835
+ : safeInteger(input.sourcePage, `${label}.sourcePage`, 1, 1_000_000);
2836
+ const parsed: EvaluationEvidenceLocatorV2 = Object.freeze({
2837
+ evidenceUnitId: parsedEvidenceUnitId,
2838
+ sourceFamilyId: boundedString(input.sourceFamilyId, `${label}.sourceFamilyId`, 256),
2839
+ sourceClass: input.sourceClass as EvaluationSourceClassV2,
2840
+ trustClass: input.trustClass as EvaluationTrustClassV2,
2841
+ sourcePath: confinedPath(input.sourcePath, `${label}.sourcePath`),
2842
+ lineRange: parseLineRange(input.lineRange, `${label}.lineRange`),
2843
+ headingPath: parseHeadingPath(input.headingPath, `${label}.headingPath`),
2844
+ ...(sourcePage === undefined ? {} : { sourcePage }),
2845
+ });
2846
+ const expected: EvaluationEvidenceLocatorV2 = {
2847
+ evidenceUnitId: unit.id,
2848
+ sourceFamilyId: unit.sourceFamilyId,
2849
+ sourceClass: family.sourceClass,
2850
+ trustClass: unit.trustClass,
2851
+ sourcePath: unit.sourcePath,
2852
+ lineRange: unit.lineRange,
2853
+ headingPath: unit.headingPath,
2854
+ ...(unit.sourcePage === undefined ? {} : { sourcePage: unit.sourcePage }),
2855
+ };
2856
+ if (canonicalJson(parsed) !== canonicalJson(expected)) {
2857
+ throw new TypeError(`${label} must exactly match its frozen registry evidence unit.`);
2858
+ }
2859
+ return parsed;
2860
+ }
2861
+
2862
+ function parseEvidenceLocators(
2863
+ value: unknown,
2864
+ label: string,
2865
+ documentId: string,
2866
+ laneId: EvaluationLaneIdV2,
2867
+ evidenceUnitIds: readonly string[],
2868
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
2869
+ familyById: ReadonlyMap<string, EvaluationSourceFamilyV2>,
2870
+ ): readonly EvaluationEvidenceLocatorV2[] {
2871
+ if (!Array.isArray(value) || value.length > 100) {
2872
+ throw new TypeError(`${label} must contain at most 100 per-unit locators.`);
2873
+ }
2874
+ const parsed = value.map((entry, index) => parseEvidenceLocator(
2875
+ entry,
2876
+ `${label}[${index}]`,
2877
+ documentId,
2878
+ laneId,
2879
+ evidenceById,
2880
+ familyById,
2881
+ ));
2882
+ assertCanonicalOrder(parsed, ({ evidenceUnitId: id }) => id, label);
2883
+ if (
2884
+ parsed.length !== evidenceUnitIds.length
2885
+ || parsed.some(({ evidenceUnitId: id }, index) => id !== evidenceUnitIds[index])
2886
+ ) {
2887
+ throw new TypeError(`${label} must preserve a one-to-one locator association for every evidence unit.`);
2888
+ }
2889
+ return Object.freeze(parsed);
2890
+ }
2891
+
2892
+ function parseRankedCandidate(
2893
+ value: unknown,
2894
+ label: string,
2895
+ laneId: EvaluationLaneIdV2,
2896
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
2897
+ documentById: ReadonlyMap<string, EvaluationDocumentV2>,
2898
+ familyById: ReadonlyMap<string, EvaluationSourceFamilyV2>,
2899
+ ): EvaluationRankedCandidateV2 {
2900
+ const input = record(value, label);
2901
+ strictKeys(input, ["documentId", "evidenceUnitIds", "provenance", "rank", "score"], label);
2902
+ const documentId = confinedPath(input.documentId, `${label}.documentId`);
2903
+ if (!documentById.has(documentId)) throw new TypeError(`${label} references unknown document ${documentId}.`);
2904
+ const evidenceUnitIds = Object.freeze(stringList(input.evidenceUnitIds, `${label}.evidenceUnitIds`, {
2905
+ allowEmpty: true,
2906
+ canonical: true,
2907
+ maximum: 100,
2908
+ }).map((id, evidenceIndex) => evidenceUnitId(id, `${label}.evidenceUnitIds[${evidenceIndex}]`)));
2909
+ for (const id of evidenceUnitIds) {
2910
+ const unit = evidenceById.get(id);
2911
+ if (unit === undefined) throw new TypeError(`${label} references unknown evidence unit ${id}.`);
2912
+ if (unit.documentId !== documentId && laneId !== "graph") {
2913
+ throw new TypeError(`${label} evidence unit ${id} belongs to a different document.`);
2914
+ }
2915
+ }
2916
+ if (input.score !== undefined && (typeof input.score !== "number" || !Number.isFinite(input.score))) {
2917
+ throw new TypeError(`${label}.score must be finite.`);
2918
+ }
2919
+ return Object.freeze({
2920
+ documentId,
2921
+ evidenceUnitIds,
2922
+ rank: safeInteger(input.rank, `${label}.rank`, 1, MAX_EVALUATION_V2_RESULTS_PER_LANE),
2923
+ ...(input.score === undefined ? {} : { score: input.score }),
2924
+ provenance: parseEvidenceLocators(
2925
+ input.provenance,
2926
+ `${label}.provenance`,
2927
+ documentId,
2928
+ laneId,
2929
+ evidenceUnitIds,
2930
+ evidenceById,
2931
+ familyById,
2932
+ ),
2933
+ });
2934
+ }
2935
+
2936
+ function parseLaneOutcome(
2937
+ value: unknown,
2938
+ index: number,
2939
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
2940
+ documentById: ReadonlyMap<string, EvaluationDocumentV2>,
2941
+ familyById: ReadonlyMap<string, EvaluationSourceFamilyV2>,
2942
+ ): EvaluationLaneOutcomeV2 {
2943
+ const label = `sample.trace.laneOutcomes[${index}]`;
2944
+ const input = record(value, label);
2945
+ strictKeys(input, ["applicability", "laneId", "rawRanking", "reasonCodes", "status"], label);
2946
+ if (typeof input.laneId !== "string" || !lanes.has(input.laneId as EvaluationLaneIdV2)) {
2947
+ throw new TypeError(`${label}.laneId is invalid.`);
2948
+ }
2949
+ if (input.applicability !== "applied" && input.applicability !== "skipped") {
2950
+ throw new TypeError(`${label}.applicability is invalid.`);
2951
+ }
2952
+ if (input.status !== "degraded" && input.status !== "ready" && input.status !== "unavailable") {
2953
+ throw new TypeError(`${label}.status is invalid.`);
2954
+ }
2955
+ const reasonCodes = stringList(input.reasonCodes, `${label}.reasonCodes`, {
2956
+ allowEmpty: true,
2957
+ canonical: true,
2958
+ maximum: 100,
2959
+ }).map((code, reasonIndex) => canonicalId(code, `${label}.reasonCodes[${reasonIndex}]`));
2960
+ if (!Array.isArray(input.rawRanking) || input.rawRanking.length > MAX_EVALUATION_V2_RESULTS_PER_LANE) {
2961
+ throw new TypeError(`${label}.rawRanking has too many entries.`);
2962
+ }
2963
+ const rawRanking = input.rawRanking.map((entry, rankIndex) =>
2964
+ parseRankedCandidate(
2965
+ entry,
2966
+ `${label}.rawRanking[${rankIndex}]`,
2967
+ input.laneId as EvaluationLaneIdV2,
2968
+ evidenceById,
2969
+ documentById,
2970
+ familyById,
2971
+ ));
2972
+ if (rawRanking.some(({ rank }, rankIndex) => rank !== rankIndex + 1)) {
2973
+ throw new TypeError(`${label}.rawRanking ranks must be contiguous and canonical.`);
2974
+ }
2975
+ if (new Set(rawRanking.map(({ documentId }) => documentId)).size !== rawRanking.length) {
2976
+ throw new TypeError(`${label}.rawRanking must not repeat a document.`);
2977
+ }
2978
+ if ((input.applicability === "skipped" || input.status === "unavailable") && rawRanking.length > 0) {
2979
+ throw new TypeError(`${label} skipped or unavailable lanes may not contain a raw ranking.`);
2980
+ }
2981
+ if (
2982
+ (input.applicability === "skipped" || input.status === "degraded" || input.status === "unavailable")
2983
+ && reasonCodes.length === 0
2984
+ ) {
2985
+ throw new TypeError(`${label} skipped, degraded, or unavailable lanes require a reason code.`);
2986
+ }
2987
+ if (input.applicability === "skipped" && input.status === "degraded") {
2988
+ throw new TypeError(`${label} skipped lanes cannot have degraded status.`);
2989
+ }
2990
+ return Object.freeze({
2991
+ laneId: input.laneId as EvaluationLaneIdV2,
2992
+ applicability: input.applicability,
2993
+ status: input.status,
2994
+ reasonCodes: Object.freeze(reasonCodes),
2995
+ rawRanking: Object.freeze(rawRanking),
2996
+ });
2997
+ }
2998
+
2999
+ const candidateReasons = new Set<EvaluationCandidateReasonV2>([
3000
+ "appended", "boundary", "deduplicated", "missing-provenance", "output-limit", "primary",
3001
+ "primary-retain-limit", "trust", "unsupported",
3002
+ ]);
3003
+
3004
+ function parseCandidateDecision(
3005
+ value: unknown,
3006
+ index: number,
3007
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
3008
+ documentById: ReadonlyMap<string, EvaluationDocumentV2>,
3009
+ familyById: ReadonlyMap<string, EvaluationSourceFamilyV2>,
3010
+ ): EvaluationCandidateDecisionV2 {
3011
+ const label = `sample.trace.candidateDecisions[${index}]`;
3012
+ const input = record(value, label);
3013
+ strictKeys(input, [
3014
+ "disposition", "documentId", "evidenceUnitIds", "laneId", "outputRank", "provenance",
3015
+ "reasonCodes", "sourceRank",
3016
+ ], label);
3017
+ if (typeof input.laneId !== "string" || !lanes.has(input.laneId as EvaluationLaneIdV2)) {
3018
+ throw new TypeError(`${label}.laneId is invalid.`);
3019
+ }
3020
+ if (input.disposition !== "accepted" && input.disposition !== "excluded") {
3021
+ throw new TypeError(`${label}.disposition is invalid.`);
3022
+ }
3023
+ const documentId = confinedPath(input.documentId, `${label}.documentId`);
3024
+ if (!documentById.has(documentId)) throw new TypeError(`${label} references unknown document ${documentId}.`);
3025
+ const evidenceUnitIds = Object.freeze(stringList(input.evidenceUnitIds, `${label}.evidenceUnitIds`, {
3026
+ allowEmpty: true,
3027
+ canonical: true,
3028
+ maximum: 100,
3029
+ }).map((id, evidenceIndex) => evidenceUnitId(id, `${label}.evidenceUnitIds[${evidenceIndex}]`)));
3030
+ for (const id of evidenceUnitIds) {
3031
+ const unit = evidenceById.get(id);
3032
+ if (unit === undefined) throw new TypeError(`${label} references unknown evidence unit ${id}.`);
3033
+ if (unit.documentId !== documentId && input.laneId !== "graph") {
3034
+ throw new TypeError(`${label} evidence unit ${id} belongs to another document.`);
3035
+ }
3036
+ }
3037
+ if (!Array.isArray(input.reasonCodes) || input.reasonCodes.length < 1 || input.reasonCodes.length > 20) {
3038
+ throw new TypeError(`${label}.reasonCodes must be a non-empty bounded array.`);
3039
+ }
3040
+ const reasonCodes = input.reasonCodes.map((reason, reasonIndex) => {
3041
+ if (typeof reason !== "string" || !candidateReasons.has(reason as EvaluationCandidateReasonV2)) {
3042
+ throw new TypeError(`${label}.reasonCodes[${reasonIndex}] is invalid.`);
3043
+ }
3044
+ return reason as EvaluationCandidateReasonV2;
3045
+ });
3046
+ if (new Set(reasonCodes).size !== reasonCodes.length || reasonCodes.some((reason, reasonIndex) => reason !== reasonCodes.toSorted()[reasonIndex])) {
3047
+ throw new TypeError(`${label}.reasonCodes must be unique and in canonical order.`);
3048
+ }
3049
+ const outputRank = input.outputRank === undefined
3050
+ ? undefined
3051
+ : safeInteger(input.outputRank, `${label}.outputRank`, 1, MAX_EVALUATION_V2_RESULTS_PER_LANE);
3052
+ if ((input.disposition === "accepted") !== (outputRank !== undefined)) {
3053
+ throw new TypeError(`${label} accepted decisions require outputRank and excluded decisions forbid it.`);
3054
+ }
3055
+ const provenance = parseEvidenceLocators(
3056
+ input.provenance,
3057
+ `${label}.provenance`,
3058
+ documentId,
3059
+ input.laneId as EvaluationLaneIdV2,
3060
+ evidenceUnitIds,
3061
+ evidenceById,
3062
+ familyById,
3063
+ );
3064
+ if (input.disposition === "accepted" && (evidenceUnitIds.length === 0 || provenance.length === 0)) {
3065
+ throw new TypeError(`${label} accepted decisions require registry-bound evidence-unit provenance.`);
3066
+ }
3067
+ const allowedReasons = input.disposition === "accepted"
3068
+ ? new Set<EvaluationCandidateReasonV2>(["appended", "primary"])
3069
+ : new Set<EvaluationCandidateReasonV2>([
3070
+ "boundary",
3071
+ "deduplicated",
3072
+ "missing-provenance",
3073
+ "output-limit",
3074
+ "primary-retain-limit",
3075
+ "trust",
3076
+ "unsupported",
3077
+ ]);
3078
+ if (reasonCodes.some((reason) => !allowedReasons.has(reason))) {
3079
+ throw new TypeError(`${label}.reasonCodes contradict the candidate disposition.`);
3080
+ }
3081
+ if (input.disposition === "accepted" && reasonCodes.length !== 1) {
3082
+ throw new TypeError(`${label} accepted decisions require exactly one acceptance reason.`);
3083
+ }
3084
+ if (reasonCodes.includes("missing-provenance") && provenance.length > 0) {
3085
+ throw new TypeError(`${label} cannot report missing-provenance with registry-bound provenance.`);
3086
+ }
3087
+ if (reasonCodes.includes("missing-provenance") && reasonCodes.length !== 1) {
3088
+ throw new TypeError(`${label} missing-provenance must be the sole exclusion reason.`);
3089
+ }
3090
+ return Object.freeze({
3091
+ documentId,
3092
+ evidenceUnitIds,
3093
+ laneId: input.laneId as EvaluationLaneIdV2,
3094
+ sourceRank: safeInteger(input.sourceRank, `${label}.sourceRank`, 1, MAX_EVALUATION_V2_RESULTS_PER_LANE),
3095
+ disposition: input.disposition,
3096
+ reasonCodes: Object.freeze(reasonCodes),
3097
+ ...(outputRank === undefined ? {} : { outputRank }),
3098
+ provenance,
3099
+ });
3100
+ }
3101
+
3102
+ function parseTrace(
3103
+ value: unknown,
3104
+ descriptor: EvaluationRetrieverDescriptorV2,
3105
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
3106
+ documentById: ReadonlyMap<string, EvaluationDocumentV2>,
3107
+ familyById: ReadonlyMap<string, EvaluationSourceFamilyV2>,
3108
+ ): EvaluationRetrieverTraceV2 {
3109
+ const input = record(value, "sample.trace");
3110
+ strictKeys(input, ["candidateDecisions", "laneOutcomes"], "sample.trace");
3111
+ if (!Array.isArray(input.laneOutcomes) || input.laneOutcomes.length > lanes.size) {
3112
+ throw new TypeError("sample.trace.laneOutcomes is invalid.");
3113
+ }
3114
+ const laneOutcomes = input.laneOutcomes.map((entry, index) => parseLaneOutcome(
3115
+ entry,
3116
+ index,
3117
+ evidenceById,
3118
+ documentById,
3119
+ familyById,
3120
+ ));
3121
+ const actualLanes = laneOutcomes.map(({ laneId }) => laneId);
3122
+ if (actualLanes.length !== descriptor.lanes.length || actualLanes.some((lane, index) => lane !== descriptor.lanes[index])) {
3123
+ throw new TypeError("sample.trace must report every locked descriptor lane in canonical order.");
3124
+ }
3125
+ if (!Array.isArray(input.candidateDecisions) || input.candidateDecisions.length > MAX_EVALUATION_V2_TRACE_DECISIONS) {
3126
+ throw new TypeError("sample.trace.candidateDecisions has too many entries.");
3127
+ }
3128
+ const candidateDecisions = input.candidateDecisions.map((entry, index) =>
3129
+ parseCandidateDecision(entry, index, evidenceById, documentById, familyById));
3130
+ const laneOrder = new Map(descriptor.lanes.map((lane, index) => [lane, index]));
3131
+ for (let index = 1; index < candidateDecisions.length; index += 1) {
3132
+ const previous = candidateDecisions[index - 1];
3133
+ const current = candidateDecisions[index];
3134
+ if (previous === undefined || current === undefined) continue;
3135
+ const comparison = (laneOrder.get(previous.laneId) ?? 0) - (laneOrder.get(current.laneId) ?? 0)
3136
+ || previous.sourceRank - current.sourceRank
3137
+ || previous.documentId.localeCompare(current.documentId);
3138
+ if (comparison > 0) throw new TypeError("sample.trace.candidateDecisions must be in canonical lane/rank/document order.");
3139
+ }
3140
+ const decisionKeys = candidateDecisions.map((decision) =>
3141
+ `${decision.laneId}\0${decision.sourceRank}\0${decision.documentId}`);
3142
+ if (new Set(decisionKeys).size !== decisionKeys.length) {
3143
+ throw new TypeError("sample.trace.candidateDecisions must not repeat a lane/rank/document decision.");
3144
+ }
3145
+ const rawRankingByKey = new Map<string, EvaluationRankedCandidateV2>(laneOutcomes.flatMap(
3146
+ (outcome) => outcome.rawRanking.map((candidate) => [
3147
+ `${outcome.laneId}\0${candidate.rank}\0${candidate.documentId}`,
3148
+ candidate,
3149
+ ] as const),
3150
+ ));
3151
+ if (candidateDecisions.length !== rawRankingByKey.size) {
3152
+ throw new TypeError("sample.trace must make exactly one candidate decision for every raw-ranking row.");
3153
+ }
3154
+ for (const decision of candidateDecisions) {
3155
+ if (!laneOrder.has(decision.laneId)) {
3156
+ throw new TypeError(`sample.trace decision lane ${decision.laneId} is not in the locked descriptor.`);
3157
+ }
3158
+ const key = `${decision.laneId}\0${decision.sourceRank}\0${decision.documentId}`;
3159
+ const ranked = rawRankingByKey.get(key);
3160
+ if (ranked === undefined) {
3161
+ throw new TypeError("sample.trace candidate decision does not join to its raw-ranking lane/rank/document row.");
3162
+ }
3163
+ if (
3164
+ canonicalJson(decision.evidenceUnitIds) !== canonicalJson(ranked.evidenceUnitIds)
3165
+ || canonicalJson(decision.provenance) !== canonicalJson(ranked.provenance)
3166
+ ) {
3167
+ throw new TypeError("sample.trace candidate decision evidence and provenance must match its raw-ranking row.");
3168
+ }
3169
+ }
3170
+ const accepted = candidateDecisions.filter(({ disposition }) => disposition === "accepted");
3171
+ const outputRanks = accepted.map(({ outputRank }) => outputRank as number)
3172
+ .toSorted((left, right) => left - right);
3173
+ if (new Set(outputRanks).size !== outputRanks.length) {
3174
+ throw new TypeError("sample.trace accepted output ranks must be unique.");
3175
+ }
3176
+ if (outputRanks.some((rank, index) => rank !== index + 1)) {
3177
+ throw new TypeError("sample.trace accepted output ranks must be contiguous.");
3178
+ }
3179
+ if (new Set(accepted.map(({ documentId }) => documentId)).size !== accepted.length) {
3180
+ throw new TypeError("sample.trace accepted output documents must be unique.");
3181
+ }
3182
+ return Object.freeze({
3183
+ laneOutcomes: Object.freeze(laneOutcomes),
3184
+ candidateDecisions: Object.freeze(candidateDecisions),
3185
+ });
3186
+ }
3187
+
3188
+ function hasUnpairedSurrogate(value: string): boolean {
3189
+ for (let index = 0; index < value.length; index += 1) {
3190
+ const code = value.charCodeAt(index);
3191
+ if (code >= 0xd800 && code <= 0xdbff) {
3192
+ const next = value.charCodeAt(index + 1);
3193
+ if (next < 0xdc00 || next > 0xdfff) return true;
3194
+ index += 1;
3195
+ } else if (code >= 0xdc00 && code <= 0xdfff) return true;
3196
+ }
3197
+ return false;
3198
+ }
3199
+
3200
+ function copyLaneNativeJson(
3201
+ value: unknown,
3202
+ label: string,
3203
+ depth = 0,
3204
+ ancestors = new WeakSet<object>(),
3205
+ ): unknown {
3206
+ if (value === null || typeof value === "boolean") return value;
3207
+ if (typeof value === "number") {
3208
+ if (!Number.isFinite(value)) throw new TypeError(`${label} numbers must be finite.`);
3209
+ return value;
3210
+ }
3211
+ if (typeof value === "string") {
3212
+ if (
3213
+ value.includes("\0")
3214
+ || hasUnpairedSurrogate(value)
3215
+ || value.normalize("NFC") !== value
3216
+ || Buffer.byteLength(value, "utf8") > MAX_EVALUATION_V2_RAW_EVIDENCE_STRING_BYTES
3217
+ ) throw new TypeError(`${label} strings must be bounded NFC JSON text.`);
3218
+ return value;
3219
+ }
3220
+ if (typeof value !== "object" || value === undefined) {
3221
+ throw new TypeError(`${label} must contain only JSON values.`);
3222
+ }
3223
+ if (depth >= MAX_EVALUATION_V2_RAW_EVIDENCE_DEPTH) {
3224
+ throw new TypeError(`${label} exceeds the lane-native evidence depth bound.`);
3225
+ }
3226
+ if (ancestors.has(value)) throw new TypeError(`${label} must not contain cycles.`);
3227
+ ancestors.add(value);
3228
+ try {
3229
+ if (Array.isArray(value)) {
3230
+ if (value.length > MAX_EVALUATION_V2_RAW_EVIDENCE_ARRAY_ITEMS) {
3231
+ throw new TypeError(`${label} exceeds the lane-native evidence array bound.`);
3232
+ }
3233
+ return Object.freeze(value.map((entry, index) =>
3234
+ copyLaneNativeJson(entry, `${label}[${index}]`, depth + 1, ancestors)));
3235
+ }
3236
+ const prototype = Reflect.getPrototypeOf(value);
3237
+ if (prototype !== Object.prototype && prototype !== null) {
3238
+ throw new TypeError(`${label} must contain only plain JSON objects.`);
3239
+ }
3240
+ const input = value as Readonly<Record<string, unknown>>;
3241
+ const keys = Object.keys(input);
3242
+ const ownKeys = Reflect.ownKeys(input);
3243
+ if (
3244
+ ownKeys.length !== keys.length
3245
+ || ownKeys.some((key) => typeof key !== "string")
3246
+ || keys.some((key) => {
3247
+ const descriptor = Object.getOwnPropertyDescriptor(input, key);
3248
+ return descriptor === undefined
3249
+ || descriptor.enumerable !== true
3250
+ || !("value" in descriptor);
3251
+ })
3252
+ ) throw new TypeError(`${label} must contain only enumerable JSON data properties.`);
3253
+ if (keys.length > MAX_EVALUATION_V2_RAW_EVIDENCE_OBJECT_FIELDS) {
3254
+ throw new TypeError(`${label} exceeds the lane-native evidence object-field bound.`);
3255
+ }
3256
+ const output: Record<string, unknown> = Object.create(null) as Record<string, unknown>;
3257
+ for (const key of keys.toSorted()) {
3258
+ if (
3259
+ key.includes("\0")
3260
+ || hasUnpairedSurrogate(key)
3261
+ || key.normalize("NFC") !== key
3262
+ || Buffer.byteLength(key, "utf8") > 4_096
3263
+ ) throw new TypeError(`${label} has an invalid JSON field name.`);
3264
+ output[key] = copyLaneNativeJson(input[key], `${label}.${key}`, depth + 1, ancestors);
3265
+ }
3266
+ return Object.freeze(output);
3267
+ } finally {
3268
+ ancestors.delete(value);
3269
+ }
3270
+ }
3271
+
3272
+ function parseLaneNativeEvidence(
3273
+ value: unknown,
3274
+ trace: EvaluationRetrieverTraceV2,
3275
+ ): readonly EvaluationLaneNativeEvidenceV2[] {
3276
+ if (!Array.isArray(value) || value.length > MAX_EVALUATION_V2_RAW_EVIDENCE_PER_SAMPLE) {
3277
+ throw new TypeError(
3278
+ `sample.rawEvidence must contain at most ${MAX_EVALUATION_V2_RAW_EVIDENCE_PER_SAMPLE} rows.`,
3279
+ );
3280
+ }
3281
+ const expected = trace.laneOutcomes.flatMap(({ laneId, rawRanking }) =>
3282
+ rawRanking.map(({ documentId, rank }) => ({ laneId, documentId, rank })));
3283
+ if (value.length !== expected.length) {
3284
+ throw new TypeError("sample.rawEvidence must contain exactly one row for every raw-ranking row.");
3285
+ }
3286
+ const parsed = value.map((entry, index): EvaluationLaneNativeEvidenceV2 => {
3287
+ const label = `sample.rawEvidence[${index}]`;
3288
+ const input = record(entry, label);
3289
+ strictKeys(input, ["documentId", "evidence", "laneId", "rank"], label);
3290
+ const expectedRow = expected[index];
3291
+ if (expectedRow === undefined) throw new Error("Lost expected lane-native evidence row.");
3292
+ if (
3293
+ input.laneId !== expectedRow.laneId
3294
+ || input.documentId !== expectedRow.documentId
3295
+ || input.rank !== expectedRow.rank
3296
+ ) {
3297
+ throw new TypeError(
3298
+ `${label} must join the same canonical lane, document, and rank as its raw-ranking row.`,
3299
+ );
3300
+ }
3301
+ const hasEvidence = Object.hasOwn(input, "evidence");
3302
+ const evidence = hasEvidence
3303
+ ? copyLaneNativeJson(input.evidence, `${label}.evidence`)
3304
+ : undefined;
3305
+ return Object.freeze({
3306
+ laneId: expectedRow.laneId,
3307
+ documentId: expectedRow.documentId,
3308
+ rank: expectedRow.rank,
3309
+ ...(hasEvidence ? { evidence } : {}),
3310
+ });
3311
+ });
3312
+ boundedJsonByteSize(
3313
+ parsed,
3314
+ MAX_EVALUATION_V2_RAW_EVIDENCE_BYTES_PER_SAMPLE,
3315
+ "sample.rawEvidence",
3316
+ );
3317
+ return Object.freeze(parsed);
3318
+ }
3319
+
3320
+ function parseResources(
3321
+ value: unknown,
3322
+ contextCeilings: EvaluationExperimentV2["protocol"]["contextCeilings"],
3323
+ ): EvaluationResourceAccountingV2 {
3324
+ const input = record(value, "sample.resources");
3325
+ strictKeys(input, ["cacheBytes", "embedding", "llm", "packedContext", "peakRssBytes"], "sample.resources");
3326
+ const llm = record(input.llm, "sample.resources.llm");
3327
+ strictKeys(llm, ["calls", "inputTokens", "outputTokens"], "sample.resources.llm");
3328
+ if (llm.calls !== 0 || llm.inputTokens !== 0 || llm.outputTokens !== 0) {
3329
+ throw new TypeError("evaluation memory operations require literal zero LLM calls and input/output tokens.");
3330
+ }
3331
+ const embedding = record(input.embedding, "sample.resources.embedding");
3332
+ strictKeys(
3333
+ embedding,
3334
+ ["calls", "durationMs", "durationScope", "inputTokens", "inputTokensMeasured"],
3335
+ "sample.resources.embedding",
3336
+ );
3337
+ const embeddingCalls = safeInteger(
3338
+ embedding.calls,
3339
+ "sample.resources.embedding.calls",
3340
+ 0,
3341
+ 1_000_000_000,
3342
+ );
3343
+ const embeddingInputTokens = safeInteger(
3344
+ embedding.inputTokens,
3345
+ "sample.resources.embedding.inputTokens",
3346
+ 0,
3347
+ 1_000_000_000,
3348
+ );
3349
+ const embeddingDurationMs = nonnegativeNumber(
3350
+ embedding.durationMs,
3351
+ "sample.resources.embedding.durationMs",
3352
+ );
3353
+ const inputTokensMeasured = embedding.inputTokensMeasured;
3354
+ if (Object.hasOwn(embedding, "inputTokensMeasured") && inputTokensMeasured !== false) {
3355
+ throw new TypeError("sample.resources.embedding.inputTokensMeasured must be literal false when present.");
3356
+ }
3357
+ const durationScopeValue = embedding.durationScope;
3358
+ if (
3359
+ Object.hasOwn(embedding, "durationScope")
3360
+ && durationScopeValue !== "embedding-backed-search-upper-bound"
3361
+ ) {
3362
+ throw new TypeError(
3363
+ "sample.resources.embedding.durationScope must be embedding-backed-search-upper-bound when present.",
3364
+ );
3365
+ }
3366
+ const durationScope: EvaluationResourceAccountingV2["embedding"]["durationScope"] =
3367
+ durationScopeValue === "embedding-backed-search-upper-bound"
3368
+ ? durationScopeValue
3369
+ : undefined;
3370
+ if (embeddingCalls === 0) {
3371
+ if (
3372
+ embeddingInputTokens !== 0
3373
+ || embeddingDurationMs !== 0
3374
+ || inputTokensMeasured !== undefined
3375
+ || durationScope !== undefined
3376
+ ) {
3377
+ throw new TypeError("zero-call embedding accounting must be the exact unannotated zero record.");
3378
+ }
3379
+ } else if (inputTokensMeasured === false && embeddingInputTokens !== 0) {
3380
+ throw new TypeError("unmeasured embedding input tokens must use zero only as an explicit placeholder.");
3381
+ }
3382
+ const packedContext = record(input.packedContext, "sample.resources.packedContext");
3383
+ strictKeys(packedContext, ["readerTokens", "utf8Bytes"], "sample.resources.packedContext");
3384
+ const utf8Bytes = safeInteger(
3385
+ packedContext.utf8Bytes,
3386
+ "sample.resources.packedContext.utf8Bytes",
3387
+ 0,
3388
+ 1_000_000_000,
3389
+ );
3390
+ const readerTokens = safeInteger(
3391
+ packedContext.readerTokens,
3392
+ "sample.resources.packedContext.readerTokens",
3393
+ 0,
3394
+ 1_000_000_000,
3395
+ );
3396
+ if (utf8Bytes > contextCeilings.utf8Bytes || readerTokens > contextCeilings.readerTokens) {
3397
+ throw new TypeError("sample packed context exceeds the digest-covered byte or token ceiling.");
3398
+ }
3399
+ return Object.freeze({
3400
+ llm: Object.freeze({ calls: 0, inputTokens: 0, outputTokens: 0 }),
3401
+ embedding: Object.freeze({
3402
+ calls: embeddingCalls,
3403
+ inputTokens: embeddingInputTokens,
3404
+ ...(inputTokensMeasured === false ? { inputTokensMeasured } : {}),
3405
+ durationMs: embeddingDurationMs,
3406
+ ...(durationScope === undefined ? {} : { durationScope }),
3407
+ }),
3408
+ packedContext: Object.freeze({
3409
+ utf8Bytes,
3410
+ readerTokens,
3411
+ }),
3412
+ peakRssBytes: safeInteger(input.peakRssBytes, "sample.resources.peakRssBytes", 0, Number.MAX_SAFE_INTEGER),
3413
+ cacheBytes: safeInteger(input.cacheBytes, "sample.resources.cacheBytes", 0, Number.MAX_SAFE_INTEGER),
3414
+ });
3415
+ }
3416
+
3417
+ function acceptedEvidenceOrder(trace: EvaluationRetrieverTraceV2): readonly string[] {
3418
+ const ordered: string[] = [];
3419
+ const seen = new Set<string>();
3420
+ const accepted = trace.candidateDecisions
3421
+ .filter(({ disposition }) => disposition === "accepted")
3422
+ .toSorted((left, right) => (left.outputRank ?? Number.MAX_SAFE_INTEGER)
3423
+ - (right.outputRank ?? Number.MAX_SAFE_INTEGER));
3424
+ for (const decision of accepted) {
3425
+ for (const evidenceUnitId of decision.evidenceUnitIds) {
3426
+ if (!seen.has(evidenceUnitId)) ordered.push(evidenceUnitId);
3427
+ seen.add(evidenceUnitId);
3428
+ }
3429
+ }
3430
+ return Object.freeze(ordered);
3431
+ }
3432
+
3433
+ function parsePackedContextTrace(
3434
+ value: unknown,
3435
+ profile: EvaluationMeasurementProfileV2,
3436
+ status: EvaluationRepeatedSampleV2["status"],
3437
+ trace: EvaluationRetrieverTraceV2,
3438
+ accounting: EvaluationResourceAccountingV2["packedContext"],
3439
+ ): EvaluationRepeatedSampleV2["packedContextTrace"] {
3440
+ const packingSample = profile.operation === "packing";
3441
+ if (!packingSample || status === "failed") {
3442
+ if (value !== undefined) {
3443
+ const sampleKind = packingSample ? "failed packing" : "non-packing";
3444
+ throw new TypeError(
3445
+ `sample.packedContextTrace is forbidden for ${sampleKind} samples.`,
3446
+ );
3447
+ }
3448
+ return undefined;
3449
+ }
3450
+ if (value === undefined) {
3451
+ throw new TypeError("sample.packedContextTrace is required for every nonfailed packing sample.");
3452
+ }
3453
+ const input = record(value, "sample.packedContextTrace");
3454
+ strictKeys(input, ["evidenceUnitIds", "packedBytesSha256", "truncated"], "sample.packedContextTrace");
3455
+ if (typeof input.truncated !== "boolean") {
3456
+ throw new TypeError("sample.packedContextTrace.truncated must be boolean.");
3457
+ }
3458
+ if (
3459
+ !Array.isArray(input.evidenceUnitIds)
3460
+ || input.evidenceUnitIds.length > MAX_EVALUATION_V2_PACKED_CONTEXT_EVIDENCE_UNITS
3461
+ ) {
3462
+ throw new TypeError(
3463
+ `sample.packedContextTrace.evidenceUnitIds must contain at most ${MAX_EVALUATION_V2_PACKED_CONTEXT_EVIDENCE_UNITS} entries.`,
3464
+ );
3465
+ }
3466
+ const evidenceUnitIds = input.evidenceUnitIds.map((entry, index) =>
3467
+ evidenceUnitId(entry, `sample.packedContextTrace.evidenceUnitIds[${index}]`));
3468
+ if (new Set(evidenceUnitIds).size !== evidenceUnitIds.length) {
3469
+ throw new TypeError("sample.packedContextTrace.evidenceUnitIds must not contain duplicates.");
3470
+ }
3471
+ const packedBytesSha256 = requireSha256(
3472
+ input.packedBytesSha256,
3473
+ "sample.packedContextTrace.packedBytesSha256",
3474
+ );
3475
+ const acceptedOrder = acceptedEvidenceOrder(trace);
3476
+ let acceptedCursor = 0;
3477
+ for (const packedId of evidenceUnitIds) {
3478
+ const acceptedIndex = acceptedOrder.indexOf(packedId, acceptedCursor);
3479
+ if (acceptedIndex < 0) {
3480
+ if (acceptedOrder.includes(packedId)) {
3481
+ throw new TypeError(
3482
+ "sample.packedContextTrace.evidenceUnitIds must preserve accepted output and evidence order.",
3483
+ );
3484
+ }
3485
+ throw new TypeError(
3486
+ `sample.packedContextTrace evidence unit ${packedId} is not registry-bound to an accepted trace decision.`,
3487
+ );
3488
+ }
3489
+ acceptedCursor = acceptedIndex + 1;
3490
+ }
3491
+ if (!input.truncated) {
3492
+ if (
3493
+ evidenceUnitIds.length !== acceptedOrder.length
3494
+ || evidenceUnitIds.some((id, index) => id !== acceptedOrder[index])
3495
+ ) {
3496
+ throw new TypeError(
3497
+ "A nontruncated sample.packedContextTrace must include every accepted evidence unit in order.",
3498
+ );
3499
+ }
3500
+ } else if (evidenceUnitIds.length >= acceptedOrder.length) {
3501
+ throw new TypeError(
3502
+ "A truncated sample.packedContextTrace must omit at least one accepted evidence unit.",
3503
+ );
3504
+ }
3505
+ if ((accounting.utf8Bytes === 0) !== (evidenceUnitIds.length === 0)) {
3506
+ throw new TypeError(
3507
+ "sample.packedContextTrace evidence count contradicts packed-context UTF-8 byte accounting.",
3508
+ );
3509
+ }
3510
+ if (accounting.utf8Bytes === 0 && accounting.readerTokens !== 0) {
3511
+ throw new TypeError(
3512
+ "Empty packed-context byte accounting cannot report nonzero reader tokens.",
3513
+ );
3514
+ }
3515
+ if (evidenceUnitIds.length > accounting.utf8Bytes) {
3516
+ throw new TypeError(
3517
+ "sample.packedContextTrace evidence count exceeds its packed-context UTF-8 byte count.",
3518
+ );
3519
+ }
3520
+ if (
3521
+ (accounting.utf8Bytes === 0 && packedBytesSha256 !== EMPTY_PACKED_CONTEXT_SHA256)
3522
+ || (accounting.utf8Bytes > 0 && packedBytesSha256 === EMPTY_PACKED_CONTEXT_SHA256)
3523
+ ) {
3524
+ throw new TypeError(
3525
+ "sample.packedContextTrace packed-bytes SHA-256 contradicts packed-context byte accounting.",
3526
+ );
3527
+ }
3528
+ return Object.freeze({
3529
+ evidenceUnitIds: Object.freeze(evidenceUnitIds),
3530
+ truncated: input.truncated,
3531
+ packedBytesSha256,
3532
+ });
3533
+ }
3534
+
3535
+ function parseSample(
3536
+ value: unknown,
3537
+ index: number,
3538
+ split: EvaluationSplitV2 | "all",
3539
+ descriptorById: ReadonlyMap<string, EvaluationRetrieverDescriptorV2>,
3540
+ profileById: ReadonlyMap<string, EvaluationMeasurementProfileV2>,
3541
+ queryById: ReadonlyMap<string, EvaluationQueryV2>,
3542
+ evidenceById: ReadonlyMap<string, EvaluationEvidenceUnitV2>,
3543
+ documentById: ReadonlyMap<string, EvaluationDocumentV2>,
3544
+ familyById: ReadonlyMap<string, EvaluationSourceFamilyV2>,
3545
+ experiment: EvaluationExperimentV2,
3546
+ ): EvaluationRepeatedSampleV2 {
3547
+ const label = `samples[${index}]`;
3548
+ const input = record(value, label);
3549
+ strictKeys(input, [
3550
+ "concurrencyBatchIdentity", "failure", "packedContextTrace", "profileId", "queryId", "repetition",
3551
+ "rawEvidence", "resources", "retrieverId", "status", "timings", "trace",
3552
+ ], label);
3553
+ const retrieverId = canonicalId(input.retrieverId, `${label}.retrieverId`);
3554
+ const profileId = canonicalId(input.profileId, `${label}.profileId`);
3555
+ const descriptor = descriptorById.get(retrieverId);
3556
+ const profile = profileById.get(profileId);
3557
+ if (descriptor === undefined) throw new TypeError(`${label} names unknown retriever ${retrieverId}.`);
3558
+ if (profile === undefined) throw new TypeError(`${label} names unknown profile ${profileId}.`);
3559
+ const queryId = input.queryId === undefined ? undefined : boundedString(input.queryId, `${label}.queryId`, 256);
3560
+ if ((profile.scope === "query") !== (queryId !== undefined)) {
3561
+ throw new TypeError(`${label}.queryId presence must match the measurement profile scope.`);
3562
+ }
3563
+ if (queryId !== undefined) {
3564
+ const query = queryById.get(queryId);
3565
+ if (query === undefined) throw new TypeError(`${label} names unknown query ${queryId}.`);
3566
+ if (split !== "all" && query.split !== split) throw new TypeError(`${label} query is outside the report split.`);
3567
+ }
3568
+ const repetition = safeInteger(input.repetition, `${label}.repetition`, 1, profile.repetitions);
3569
+ const concurrencyBatchIdentity = input.concurrencyBatchIdentity === undefined
3570
+ ? undefined
3571
+ : bridgeString(input.concurrencyBatchIdentity, `${label}.concurrencyBatchIdentity`, 512);
3572
+ if (profile.operation === "four-reader-query") {
3573
+ if (concurrencyBatchIdentity !== experiment.environment.fourReaderBatch.id) {
3574
+ throw new TypeError(
3575
+ `${label}.concurrencyBatchIdentity must match the digest-covered four-reader batch identity.`,
3576
+ );
3577
+ }
3578
+ } else if (concurrencyBatchIdentity !== undefined) {
3579
+ throw new TypeError(`${label}.concurrencyBatchIdentity is reserved for four-reader samples.`);
3580
+ }
3581
+ if (input.status !== "degraded" && input.status !== "failed" && input.status !== "ready" && input.status !== "unavailable") {
3582
+ throw new TypeError(`${label}.status is invalid.`);
3583
+ }
3584
+ const timingsInput = record(input.timings, `${label}.timings`);
3585
+ strictKeys(timingsInput, ["elapsedMs", "indexMs", "packingMs", "queryMs", "updateMs"], `${label}.timings`);
3586
+ const timings = Object.freeze({
3587
+ elapsedMs: nonnegativeNumber(timingsInput.elapsedMs, `${label}.timings.elapsedMs`),
3588
+ indexMs: nonnegativeNumber(timingsInput.indexMs, `${label}.timings.indexMs`),
3589
+ updateMs: nonnegativeNumber(timingsInput.updateMs, `${label}.timings.updateMs`),
3590
+ queryMs: nonnegativeNumber(timingsInput.queryMs, `${label}.timings.queryMs`),
3591
+ packingMs: nonnegativeNumber(timingsInput.packingMs, `${label}.timings.packingMs`),
3592
+ });
3593
+ let failure: EvaluationRepeatedSampleV2["failure"];
3594
+ if (input.failure !== undefined) {
3595
+ const failureInput = record(input.failure, `${label}.failure`);
3596
+ strictKeys(failureInput, ["kind", "message"], `${label}.failure`);
3597
+ if (failureInput.kind !== "exception" && failureInput.kind !== "invalid-result" && failureInput.kind !== "timeout") {
3598
+ throw new TypeError(`${label}.failure.kind is invalid.`);
3599
+ }
3600
+ failure = Object.freeze({
3601
+ kind: failureInput.kind,
3602
+ message: boundedString(failureInput.message, `${label}.failure.message`, 2_000),
3603
+ });
3604
+ }
3605
+ if ((input.status === "failed") !== (failure !== undefined)) {
3606
+ throw new TypeError(`${label} failed status and failure details must occur together.`);
3607
+ }
3608
+ const trace = parseTrace(input.trace, descriptor, evidenceById, documentById, familyById);
3609
+ const rawEvidence = parseLaneNativeEvidence(input.rawEvidence, trace);
3610
+ if (input.status === "failed") {
3611
+ if (
3612
+ trace.candidateDecisions.length > 0
3613
+ || trace.laneOutcomes.some(({ rawRanking }) => rawRanking.length > 0)
3614
+ ) {
3615
+ throw new TypeError(`${label} failed samples cannot retain rankings or candidate decisions.`);
3616
+ }
3617
+ } else {
3618
+ const applied = trace.laneOutcomes.filter(({ applicability }) => applicability === "applied");
3619
+ const laneStatus: Exclude<EvaluationRepeatedSampleV2["status"], "failed"> = applied.length === 0
3620
+ || applied.every(({ status }) => status === "unavailable")
3621
+ ? "unavailable"
3622
+ : applied.some(({ status }) => status !== "ready")
3623
+ ? "degraded"
3624
+ : "ready";
3625
+ if (input.status !== laneStatus) {
3626
+ throw new TypeError(`${label} status must reconcile with its locked lane outcomes.`);
3627
+ }
3628
+ }
3629
+ const resources = parseResources(input.resources, experiment.protocol.contextCeilings);
3630
+ const packedContextTrace = parsePackedContextTrace(
3631
+ input.packedContextTrace,
3632
+ profile,
3633
+ input.status,
3634
+ trace,
3635
+ resources.packedContext,
3636
+ );
3637
+ return Object.freeze({
3638
+ retrieverId,
3639
+ profileId,
3640
+ ...(queryId === undefined ? {} : { queryId }),
3641
+ repetition,
3642
+ ...(concurrencyBatchIdentity === undefined ? {} : { concurrencyBatchIdentity }),
3643
+ status: input.status,
3644
+ timings,
3645
+ resources,
3646
+ trace,
3647
+ rawEvidence,
3648
+ ...(packedContextTrace === undefined ? {} : { packedContextTrace }),
3649
+ ...(failure === undefined ? {} : { failure }),
3650
+ });
3651
+ }
3652
+
3653
+ function sampleKey(sample: Pick<EvaluationRepeatedSampleV2, "profileId" | "queryId" | "repetition" | "retrieverId">): string {
3654
+ return `${sample.retrieverId}\0${sample.profileId}\0${sample.queryId ?? ""}\0${sample.repetition}`;
3655
+ }
3656
+
3657
+ function compareSamples(left: EvaluationRepeatedSampleV2, right: EvaluationRepeatedSampleV2): number {
3658
+ return left.retrieverId.localeCompare(right.retrieverId)
3659
+ || left.profileId.localeCompare(right.profileId)
3660
+ || (left.queryId ?? "").localeCompare(right.queryId ?? "")
3661
+ || left.repetition - right.repetition;
3662
+ }
3663
+
3664
+ function boundedJsonByteSize(value: unknown, maximum: number, label: string): number {
3665
+ const stack: unknown[] = [value];
3666
+ let bytes = 0;
3667
+ while (stack.length > 0) {
3668
+ const current = stack.pop();
3669
+ if (typeof current === "string") {
3670
+ bytes += Buffer.byteLength(JSON.stringify(current), "utf8");
3671
+ } else if (typeof current === "number") {
3672
+ bytes += Number.isFinite(current) ? Buffer.byteLength(JSON.stringify(current), "utf8") : 8;
3673
+ } else if (typeof current === "boolean" || current === null || current === undefined) {
3674
+ bytes += current === true ? 4 : 5;
3675
+ } else if (Array.isArray(current)) {
3676
+ bytes += Math.max(0, current.length - 1) + 2;
3677
+ if (bytes > maximum) throw new TypeError(`${label} exceeds its aggregate UTF-8 byte bound.`);
3678
+ for (const entry of current) stack.push(entry);
3679
+ } else if (typeof current === "object") {
3680
+ const entries = Object.entries(current as Readonly<Record<string, unknown>>);
3681
+ bytes += Math.max(0, entries.length - 1) + 2;
3682
+ if (bytes > maximum) throw new TypeError(`${label} exceeds its aggregate UTF-8 byte bound.`);
3683
+ for (const [key, entry] of entries) {
3684
+ bytes += Buffer.byteLength(JSON.stringify(key), "utf8") + 1;
3685
+ if (bytes > maximum) throw new TypeError(`${label} exceeds its aggregate UTF-8 byte bound.`);
3686
+ stack.push(entry);
3687
+ }
3688
+ } else {
3689
+ bytes += 16;
3690
+ }
3691
+ if (bytes > maximum) throw new TypeError(`${label} exceeds its aggregate UTF-8 byte bound.`);
3692
+ }
3693
+ return bytes;
3694
+ }
3695
+
3696
+ function preflightReportTraceBounds(samples: readonly unknown[]): void {
3697
+ let traceItems = 0;
3698
+ let provenanceItems = 0;
3699
+ let rawEvidenceItems = 0;
3700
+ let packedContextItems = 0;
3701
+ let traceBytes = 0;
3702
+ let provenanceBytes = 0;
3703
+ let rawEvidenceBytes = 0;
3704
+ let packedContextBytes = 0;
3705
+ for (const [sampleIndex, sampleValue] of samples.entries()) {
3706
+ const sample = record(sampleValue, `samples[${sampleIndex}]`);
3707
+ const trace = record(sample.trace, `samples[${sampleIndex}].trace`);
3708
+ const laneOutcomes = Array.isArray(trace.laneOutcomes) ? trace.laneOutcomes : [];
3709
+ const decisions = Array.isArray(trace.candidateDecisions) ? trace.candidateDecisions : [];
3710
+ traceItems += laneOutcomes.length + decisions.length;
3711
+ if (traceItems > MAX_EVALUATION_V2_REPORT_TRACE_ITEMS) {
3712
+ throw new TypeError("evaluation report exceeds the aggregate trace item bound.");
3713
+ }
3714
+ const provenanceCollections: unknown[] = [];
3715
+ for (const laneValue of laneOutcomes) {
3716
+ const lane = record(laneValue, `samples[${sampleIndex}].trace lane`);
3717
+ const rawRanking = Array.isArray(lane.rawRanking) ? lane.rawRanking : [];
3718
+ traceItems += rawRanking.length;
3719
+ if (traceItems > MAX_EVALUATION_V2_REPORT_TRACE_ITEMS) {
3720
+ throw new TypeError("evaluation report exceeds the aggregate trace item bound.");
3721
+ }
3722
+ for (const rankedValue of rawRanking) {
3723
+ const ranked = record(rankedValue, `samples[${sampleIndex}].trace raw ranking`);
3724
+ if (ranked.provenance !== undefined) provenanceCollections.push(ranked.provenance);
3725
+ }
3726
+ }
3727
+ for (const decisionValue of decisions) {
3728
+ const decision = record(decisionValue, `samples[${sampleIndex}].trace decision`);
3729
+ if (decision.provenance !== undefined) provenanceCollections.push(decision.provenance);
3730
+ }
3731
+ traceBytes += boundedJsonByteSize(
3732
+ trace,
3733
+ MAX_EVALUATION_V2_REPORT_TRACE_BYTES - traceBytes,
3734
+ "evaluation report trace",
3735
+ );
3736
+ for (const provenanceValue of provenanceCollections) {
3737
+ if (Array.isArray(provenanceValue)) provenanceItems += provenanceValue.length;
3738
+ if (provenanceItems > MAX_EVALUATION_V2_REPORT_PROVENANCE_ITEMS) {
3739
+ throw new TypeError("evaluation report exceeds the aggregate provenance item bound.");
3740
+ }
3741
+ provenanceBytes += boundedJsonByteSize(
3742
+ provenanceValue,
3743
+ MAX_EVALUATION_V2_REPORT_PROVENANCE_BYTES - provenanceBytes,
3744
+ "evaluation report provenance",
3745
+ );
3746
+ }
3747
+ const rawEvidence = Array.isArray(sample.rawEvidence) ? sample.rawEvidence : [];
3748
+ rawEvidenceItems += rawEvidence.length;
3749
+ if (rawEvidenceItems > MAX_EVALUATION_V2_REPORT_RAW_EVIDENCE_ITEMS) {
3750
+ throw new TypeError("evaluation report exceeds the aggregate lane-native evidence item bound.");
3751
+ }
3752
+ rawEvidenceBytes += boundedJsonByteSize(
3753
+ rawEvidence,
3754
+ MAX_EVALUATION_V2_REPORT_RAW_EVIDENCE_BYTES - rawEvidenceBytes,
3755
+ "evaluation report lane-native evidence",
3756
+ );
3757
+ if (sample.packedContextTrace !== undefined) {
3758
+ const packedContextTrace = record(
3759
+ sample.packedContextTrace,
3760
+ `samples[${sampleIndex}].packedContextTrace`,
3761
+ );
3762
+ if (Array.isArray(packedContextTrace.evidenceUnitIds)) {
3763
+ packedContextItems += packedContextTrace.evidenceUnitIds.length;
3764
+ }
3765
+ if (packedContextItems > MAX_EVALUATION_V2_REPORT_PACKED_CONTEXT_ITEMS) {
3766
+ throw new TypeError("evaluation report exceeds the aggregate packed-context evidence item bound.");
3767
+ }
3768
+ packedContextBytes += boundedJsonByteSize(
3769
+ packedContextTrace,
3770
+ MAX_EVALUATION_V2_REPORT_PACKED_CONTEXT_BYTES - packedContextBytes,
3771
+ "evaluation report packed-context trace",
3772
+ );
3773
+ }
3774
+ }
3775
+ }
3776
+
3777
+ function expectedSampleCardinality(
3778
+ corpus: RetrievalEvaluationCorpusV2,
3779
+ queryCount: number,
3780
+ ): number {
3781
+ let perRetriever = 0;
3782
+ for (const profile of corpus.measurementProfiles) {
3783
+ const targets = profile.scope === "query" ? queryCount : 1;
3784
+ const profileCount = targets * profile.repetitions;
3785
+ if (!Number.isSafeInteger(profileCount) || profileCount > MAX_EVALUATION_V2_SAMPLES) {
3786
+ throw new TypeError("evaluation report expected sample matrix exceeds the supported cardinality.");
3787
+ }
3788
+ perRetriever += profileCount;
3789
+ if (!Number.isSafeInteger(perRetriever) || perRetriever > MAX_EVALUATION_V2_SAMPLES) {
3790
+ throw new TypeError("evaluation report expected sample matrix exceeds the supported cardinality.");
3791
+ }
3792
+ }
3793
+ const total = perRetriever * corpus.retrievers.length;
3794
+ if (!Number.isSafeInteger(total) || total > MAX_EVALUATION_V2_SAMPLES) {
3795
+ throw new TypeError("evaluation report expected sample matrix exceeds the supported cardinality.");
3796
+ }
3797
+ return total;
3798
+ }
3799
+
3800
+ export function parseRetrievalEvaluationReportV2(
3801
+ inputValue: unknown,
3802
+ corpus: RetrievalEvaluationCorpusV2,
3803
+ ): RetrievalEvaluationReportV2 {
3804
+ const input = record(inputValue, "evaluation v2 report");
3805
+ strictKeys(input, ["candidateLockSha256", "samples", "schemaVersion", "split", "suiteSha256"], "evaluation v2 report");
3806
+ if (input.schemaVersion !== 2) throw new TypeError("evaluation v2 report schemaVersion must be 2.");
3807
+ const suiteSha256 = requireSha256(input.suiteSha256, "report.suiteSha256");
3808
+ const candidateLockSha256 = requireSha256(input.candidateLockSha256, "report.candidateLockSha256");
3809
+ if (suiteSha256 !== corpus.manifest.corpusSha256 || candidateLockSha256 !== corpus.manifest.candidateLockSha256) {
3810
+ throw new TypeError("evaluation report commitments do not match the sealed corpus and candidate lock.");
3811
+ }
3812
+ if (input.split !== "all" && input.split !== "development" && input.split !== "test") {
3813
+ throw new TypeError("evaluation report split is invalid.");
3814
+ }
3815
+ const split: EvaluationSplitV2 | "all" = input.split;
3816
+ if (!Array.isArray(input.samples) || input.samples.length < 1 || input.samples.length > MAX_EVALUATION_V2_SAMPLES) {
3817
+ throw new TypeError("evaluation report samples must be a non-empty bounded array.");
3818
+ }
3819
+ const queries = corpus.queries.filter((query) => split === "all" || query.split === split);
3820
+ const expectedCount = expectedSampleCardinality(corpus, queries.length);
3821
+ if (input.samples.length !== expectedCount) {
3822
+ throw new TypeError("evaluation report sample matrix is incomplete or contains an unexpected sample.");
3823
+ }
3824
+ preflightReportTraceBounds(input.samples);
3825
+ const descriptorById = new Map(corpus.retrievers.map((descriptor) => [descriptor.id, descriptor]));
3826
+ const profileById = new Map(corpus.measurementProfiles.map((profile) => [profile.id, profile]));
3827
+ const queryById = new Map(corpus.queries.map((query) => [query.id, query]));
3828
+ const evidenceById = new Map(corpus.evidenceUnits.map((unit) => [unit.id, unit]));
3829
+ const documentById = new Map(corpus.documents.map((document) => [document.id, document]));
3830
+ const familyById = new Map(corpus.sourceFamilies.map((family) => [family.id, family]));
3831
+ const samples = input.samples.map((sample, index) => parseSample(
3832
+ sample,
3833
+ index,
3834
+ split,
3835
+ descriptorById,
3836
+ profileById,
3837
+ queryById,
3838
+ evidenceById,
3839
+ documentById,
3840
+ familyById,
3841
+ corpus.experiment,
3842
+ ));
3843
+ if (new Set(samples.map(sampleKey)).size !== samples.length) {
3844
+ throw new TypeError("evaluation report repeats a retriever/profile/query/repetition sample.");
3845
+ }
3846
+ const sorted = samples.toSorted(compareSamples);
3847
+ if (samples.some((sample, index) => sampleKey(sample) !== sampleKey(sorted[index] as EvaluationRepeatedSampleV2))) {
3848
+ throw new TypeError("evaluation report samples must be in canonical order.");
3849
+ }
3850
+ return Object.freeze({
3851
+ schemaVersion: 2,
3852
+ suiteSha256,
3853
+ candidateLockSha256,
3854
+ split,
3855
+ samples: Object.freeze(samples),
3856
+ });
3857
+ }