@policyengine/calibration-tree 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +58 -2
- package/package.json +73 -4
- package/src/calibration-build-comparison.ts +77 -0
- package/src/calibration-comparison-blob.ts +367 -0
- package/src/calibration-comparison-bundle.ts +149 -0
- package/src/calibration-explorer.ts +202 -0
- package/src/calibration-release-locator.ts +353 -0
- package/src/calibration-tree-artifact.ts +1974 -0
- package/src/calibration-tree-blob.ts +403 -0
- package/src/calibration-tree-bundle.ts +569 -0
- package/src/calibration-tree-manifest.ts +349 -0
- package/src/calibration-tree-publication.ts +399 -0
- package/src/calibration-tree-worker.ts +637 -0
- package/src/calibration-tree.ts +1284 -0
- package/src/chronicle-metadata.ts +42 -0
- package/src/countries.ts +355 -0
- package/src/geography-labels.ts +59 -0
- package/src/hierarchy-catalog-reader.ts +297 -0
- package/src/hierarchy-target-reader.ts +243 -0
- package/src/historical-weight-summary.ts +53 -0
- package/src/latest-artifact.ts +3767 -0
- package/src/legacy-target-reader.ts +337 -0
- package/src/program-label.ts +64 -0
- package/src/publish-calibration-tree.ts +1984 -0
- package/src/reforms.ts +224 -0
- package/src/release-cut.ts +32 -0
- package/src/release-record.ts +277 -0
- package/src/source-label.ts +5 -0
- package/src/source-labels.ts +71 -0
- package/src/staging-artifact.ts +1411 -0
- package/src/staging-contract.ts +916 -0
- package/src/structured-dimension-reader.ts +137 -0
- package/src/structured-target-reader.ts +76 -0
- package/src/target-change.ts +421 -0
- package/src/target-loss-attribution-manifest.ts +436 -0
- package/src/target-loss-attribution.ts +1012 -0
- package/src/target-representation.ts +69 -0
- package/src/target-surface-matcher.ts +266 -0
- package/types/next-fetch-cache.d.ts +8 -0
|
@@ -0,0 +1,1974 @@
|
|
|
1
|
+
import {
|
|
2
|
+
buildCalibrationTree,
|
|
3
|
+
calibrationTreeMetricsFromInputs,
|
|
4
|
+
fitBandForTarget,
|
|
5
|
+
geographyLabeller,
|
|
6
|
+
normalizeCalibrationTreeTargets,
|
|
7
|
+
type CalibrationTreeGroup,
|
|
8
|
+
type CalibrationTreeMetricInput,
|
|
9
|
+
type CalibrationTreeMetrics,
|
|
10
|
+
type CalibrationTreeNode,
|
|
11
|
+
type CalibrationTreeResponse,
|
|
12
|
+
type CalibrationTreeTarget,
|
|
13
|
+
type GeographyLabeller,
|
|
14
|
+
} from "./calibration-tree";
|
|
15
|
+
import {
|
|
16
|
+
createExplorerState,
|
|
17
|
+
selectExplorerNode,
|
|
18
|
+
type CalibrationStatus,
|
|
19
|
+
type ComparisonFit,
|
|
20
|
+
type ExplorerNodeSelection,
|
|
21
|
+
type ExplorerState,
|
|
22
|
+
type FitBand,
|
|
23
|
+
} from "./calibration-explorer";
|
|
24
|
+
import { isCountry, type MicrocosmCountry } from "./countries";
|
|
25
|
+
import {
|
|
26
|
+
chronicleFactKey,
|
|
27
|
+
normalizedTargetName,
|
|
28
|
+
structuredIdentity,
|
|
29
|
+
targetRowRepresentation,
|
|
30
|
+
} from "./target-surface-matcher";
|
|
31
|
+
import type {
|
|
32
|
+
CalibrationProvenance,
|
|
33
|
+
TargetLossAttributionStatus,
|
|
34
|
+
} from "./target-loss-attribution";
|
|
35
|
+
import type {
|
|
36
|
+
TargetRepresentation,
|
|
37
|
+
TargetRowRepresentation,
|
|
38
|
+
} from "./target-representation";
|
|
39
|
+
import type {
|
|
40
|
+
TargetChangeAttributionSide,
|
|
41
|
+
TargetChangeMethodology,
|
|
42
|
+
TargetChangeMode,
|
|
43
|
+
TargetChangeRow,
|
|
44
|
+
TargetChangeSummary,
|
|
45
|
+
} from "./target-change";
|
|
46
|
+
import type { TargetMatchingSummary } from "./target-surface-matcher";
|
|
47
|
+
|
|
48
|
+
export const CALIBRATION_TREE_SCHEMA_VERSION = 6 as const;
|
|
49
|
+
export const CALIBRATION_TREE_TARGET_SUMMARY_MAX_RAW_BYTES = 8_000_000;
|
|
50
|
+
export const CALIBRATION_TREE_TARGET_DETAIL_MAX_RAW_BYTES = 4_000_000;
|
|
51
|
+
|
|
52
|
+
export type CalibrationTreeTierPart = `tier-${number}`;
|
|
53
|
+
export type CalibrationTreeTargetSummaryPart = `target-summary-${string}`;
|
|
54
|
+
export type CalibrationTreeTargetDetailsPart = `target-details-${string}`;
|
|
55
|
+
export type CalibrationTreePart =
|
|
56
|
+
| "index"
|
|
57
|
+
| CalibrationTreeTierPart
|
|
58
|
+
| "filter-index"
|
|
59
|
+
| CalibrationTreeTargetSummaryPart
|
|
60
|
+
| CalibrationTreeTargetDetailsPart;
|
|
61
|
+
|
|
62
|
+
export interface CalibrationTreeSourceArtifact {
|
|
63
|
+
path: string;
|
|
64
|
+
sha256: string;
|
|
65
|
+
hfRepo?: string;
|
|
66
|
+
hfCommitSha?: string;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface CalibrationTreeSourceArtifacts {
|
|
70
|
+
calibrationDiagnostics: CalibrationTreeSourceArtifact | null;
|
|
71
|
+
buildManifest: CalibrationTreeSourceArtifact | null;
|
|
72
|
+
releaseManifest: CalibrationTreeSourceArtifact | null;
|
|
73
|
+
demographics: CalibrationTreeSourceArtifact | null;
|
|
74
|
+
stagingRunManifest?: CalibrationTreeSourceArtifact | null;
|
|
75
|
+
stagingProgress?: CalibrationTreeSourceArtifact | null;
|
|
76
|
+
stagedDatasetReceipt?: CalibrationTreeSourceArtifact | null;
|
|
77
|
+
comparisonCurrentIndex?: CalibrationTreeSourceArtifact | null;
|
|
78
|
+
comparisonCandidateIndex?: CalibrationTreeSourceArtifact | null;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export type CalibrationTreeBuildKind = "release" | "staging" | "comparison";
|
|
82
|
+
|
|
83
|
+
export interface CalibrationTreeBuild {
|
|
84
|
+
buildArtifactId: string;
|
|
85
|
+
kind: CalibrationTreeBuildKind;
|
|
86
|
+
sourceId: string;
|
|
87
|
+
label: string;
|
|
88
|
+
releaseId: string | null;
|
|
89
|
+
hfRepo: string | null;
|
|
90
|
+
hfCommitSha: string | null;
|
|
91
|
+
createdAt: string | null;
|
|
92
|
+
sourceArtifacts: CalibrationTreeSourceArtifacts;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface CalibrationTreeTargetSummary {
|
|
96
|
+
id: string;
|
|
97
|
+
label: string;
|
|
98
|
+
metricInputs: CalibrationTreeMetricInput;
|
|
99
|
+
comparison: CalibrationTreeComparisonTarget;
|
|
100
|
+
detailSource?: {
|
|
101
|
+
currentTargetOrdinal: number | null;
|
|
102
|
+
candidateTargetOrdinal: number | null;
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface CalibrationTreeComparisonTarget {
|
|
107
|
+
normalizedBaseName: string | null;
|
|
108
|
+
chronicleFactKey: string | null;
|
|
109
|
+
structuredIdentity: string | null;
|
|
110
|
+
representation: TargetRowRepresentation;
|
|
111
|
+
row: CalibrationTreeTarget;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export interface CalibrationTreeComparisonMetadata {
|
|
115
|
+
releaseId: string;
|
|
116
|
+
calibrationProvenance?: CalibrationProvenance;
|
|
117
|
+
status: TargetLossAttributionStatus;
|
|
118
|
+
aggregate: number | null;
|
|
119
|
+
cap: number | null;
|
|
120
|
+
basisIdentifier: string | null;
|
|
121
|
+
targetRepresentation: TargetRepresentation;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export interface CalibrationTreeComparisonResultMetadata {
|
|
125
|
+
pairArtifactId: string;
|
|
126
|
+
currentBuildArtifactId: string;
|
|
127
|
+
candidateBuildArtifactId: string;
|
|
128
|
+
mode: TargetChangeMode;
|
|
129
|
+
available: boolean;
|
|
130
|
+
reason: string | null;
|
|
131
|
+
current: TargetChangeAttributionSide;
|
|
132
|
+
candidate: TargetChangeAttributionSide;
|
|
133
|
+
methodology: TargetChangeMethodology;
|
|
134
|
+
matching: TargetMatchingSummary;
|
|
135
|
+
summary: TargetChangeSummary | null;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export interface CalibrationTreePosting<T extends string | null = string> {
|
|
139
|
+
value: T;
|
|
140
|
+
targetOrdinals: number[];
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export interface CalibrationTreeFilterPostings {
|
|
144
|
+
geographyLevels: CalibrationTreePosting[];
|
|
145
|
+
geographies: CalibrationTreePosting[];
|
|
146
|
+
fitBands: CalibrationTreePosting<FitBand>[];
|
|
147
|
+
comparisonFits: CalibrationTreePosting<ComparisonFit | null>[];
|
|
148
|
+
calibrationStatuses: CalibrationTreePosting<CalibrationStatus | null>[];
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
interface CompiledCalibrationNodeBase {
|
|
152
|
+
id: string;
|
|
153
|
+
label: string;
|
|
154
|
+
selection: ExplorerNodeSelection;
|
|
155
|
+
targetOrdinals: number[];
|
|
156
|
+
metrics: CalibrationTreeMetrics;
|
|
157
|
+
authoredLabel?: boolean;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export interface CompiledCalibrationBranchNode
|
|
161
|
+
extends CompiledCalibrationNodeBase {
|
|
162
|
+
kind: Exclude<ExplorerNodeSelection["kind"], "target">;
|
|
163
|
+
nextLevelId: string;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export interface CompiledCalibrationTargetNode
|
|
167
|
+
extends CompiledCalibrationNodeBase {
|
|
168
|
+
kind: "target";
|
|
169
|
+
targetOrdinal: number;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export type CompiledCalibrationNode =
|
|
173
|
+
| CompiledCalibrationBranchNode
|
|
174
|
+
| CompiledCalibrationTargetNode;
|
|
175
|
+
|
|
176
|
+
export interface CompiledCalibrationGroup {
|
|
177
|
+
id: string;
|
|
178
|
+
label: string;
|
|
179
|
+
targetOrdinals: number[];
|
|
180
|
+
metrics: CalibrationTreeMetrics;
|
|
181
|
+
nodes: CompiledCalibrationNode[];
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export interface CompiledCalibrationLevel {
|
|
185
|
+
currentLevel: CalibrationTreeResponse["currentLevel"];
|
|
186
|
+
pathLabels: CalibrationTreeResponse["pathLabels"];
|
|
187
|
+
dimensionOrder: CalibrationTreeResponse["dimensionOrder"];
|
|
188
|
+
targetOrdinals: number[];
|
|
189
|
+
metrics: CalibrationTreeMetrics;
|
|
190
|
+
groups: CompiledCalibrationGroup[];
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
interface CalibrationTreePartIdentity {
|
|
194
|
+
schemaVersion: typeof CALIBRATION_TREE_SCHEMA_VERSION;
|
|
195
|
+
country: MicrocosmCountry;
|
|
196
|
+
buildArtifactId: string;
|
|
197
|
+
part: CalibrationTreePart;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export interface CalibrationTreePartDescriptor {
|
|
201
|
+
part: Exclude<CalibrationTreePart, "index">;
|
|
202
|
+
path: string;
|
|
203
|
+
sha256: string;
|
|
204
|
+
rawBytes: number;
|
|
205
|
+
gzipBytes: number;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export interface CalibrationTreeTierDescriptor
|
|
209
|
+
extends CalibrationTreePartDescriptor {
|
|
210
|
+
part: CalibrationTreeTierPart;
|
|
211
|
+
depth: number;
|
|
212
|
+
levelIds: string[];
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export interface CalibrationTreeTargetDetailsDescriptor
|
|
216
|
+
extends CalibrationTreePartDescriptor {
|
|
217
|
+
part: CalibrationTreeTargetDetailsPart;
|
|
218
|
+
startTargetOrdinal: number;
|
|
219
|
+
endTargetOrdinalExclusive: number;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export interface CalibrationTreeTargetSummaryDescriptor
|
|
223
|
+
extends CalibrationTreePartDescriptor {
|
|
224
|
+
part: CalibrationTreeTargetSummaryPart;
|
|
225
|
+
startTargetOrdinal: number;
|
|
226
|
+
endTargetOrdinalExclusive: number;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export interface CalibrationTreeIndexArtifact
|
|
230
|
+
extends CalibrationTreePartIdentity {
|
|
231
|
+
part: "index";
|
|
232
|
+
build: CalibrationTreeBuild;
|
|
233
|
+
targetComparison: CalibrationTreeComparisonMetadata;
|
|
234
|
+
comparison?: CalibrationTreeComparisonResultMetadata;
|
|
235
|
+
calibrationProvenance?: CalibrationProvenance;
|
|
236
|
+
lossAttributionAvailable: boolean;
|
|
237
|
+
filterOptions: CalibrationTreeResponse["filterOptions"];
|
|
238
|
+
targetCount: number;
|
|
239
|
+
maxDepth: number;
|
|
240
|
+
roots: {
|
|
241
|
+
program: string;
|
|
242
|
+
geography: string;
|
|
243
|
+
};
|
|
244
|
+
levels: Record<string, CompiledCalibrationLevel>;
|
|
245
|
+
parts: {
|
|
246
|
+
tiers: CalibrationTreeTierDescriptor[];
|
|
247
|
+
filterIndex: CalibrationTreePartDescriptor & { part: "filter-index" };
|
|
248
|
+
targetSummaries: CalibrationTreeTargetSummaryDescriptor[];
|
|
249
|
+
targetDetailStrategy: "shards" | "source-targets";
|
|
250
|
+
targetDetails: CalibrationTreeTargetDetailsDescriptor[];
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export interface CalibrationTreeTierArtifact
|
|
255
|
+
extends CalibrationTreePartIdentity {
|
|
256
|
+
part: CalibrationTreeTierPart;
|
|
257
|
+
depth: number;
|
|
258
|
+
targetCount: number;
|
|
259
|
+
levels: Record<string, CompiledCalibrationLevel>;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export interface CalibrationTreeFilterIndexArtifact
|
|
263
|
+
extends CalibrationTreePartIdentity {
|
|
264
|
+
part: "filter-index";
|
|
265
|
+
targetCount: number;
|
|
266
|
+
postings: CalibrationTreeFilterPostings;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export interface CalibrationTreeTargetSummaryArtifact
|
|
270
|
+
extends CalibrationTreePartIdentity {
|
|
271
|
+
part: CalibrationTreeTargetSummaryPart;
|
|
272
|
+
startTargetOrdinal: number;
|
|
273
|
+
endTargetOrdinalExclusive: number;
|
|
274
|
+
targets: CalibrationTreeTargetSummary[];
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
export interface CalibrationTreeTargetDetailsArtifact
|
|
278
|
+
extends CalibrationTreePartIdentity {
|
|
279
|
+
part: CalibrationTreeTargetDetailsPart;
|
|
280
|
+
startTargetOrdinal: number;
|
|
281
|
+
endTargetOrdinalExclusive: number;
|
|
282
|
+
targets: CalibrationTreeTarget[];
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export type CalibrationTreeArtifactPart =
|
|
286
|
+
| CalibrationTreeIndexArtifact
|
|
287
|
+
| CalibrationTreeTierArtifact
|
|
288
|
+
| CalibrationTreeFilterIndexArtifact
|
|
289
|
+
| CalibrationTreeTargetSummaryArtifact
|
|
290
|
+
| CalibrationTreeTargetDetailsArtifact;
|
|
291
|
+
|
|
292
|
+
export interface CalibrationTreeBundleDraft {
|
|
293
|
+
country: MicrocosmCountry;
|
|
294
|
+
build: CalibrationTreeBuild;
|
|
295
|
+
comparison: CalibrationTreeComparisonMetadata;
|
|
296
|
+
comparisonResult?: CalibrationTreeComparisonResultMetadata;
|
|
297
|
+
calibrationProvenance?: CalibrationProvenance;
|
|
298
|
+
lossAttributionAvailable: boolean;
|
|
299
|
+
filterOptions: CalibrationTreeResponse["filterOptions"];
|
|
300
|
+
roots: CalibrationTreeIndexArtifact["roots"];
|
|
301
|
+
levels: Record<string, CompiledCalibrationLevel>;
|
|
302
|
+
levelDepths: Record<string, number>;
|
|
303
|
+
targets: Array<CalibrationTreeTargetSummary & {
|
|
304
|
+
facets: {
|
|
305
|
+
geographyLevel: string;
|
|
306
|
+
geography: string;
|
|
307
|
+
fitBand: FitBand;
|
|
308
|
+
comparisonFit: ComparisonFit | null;
|
|
309
|
+
calibrationStatus: CalibrationStatus | null;
|
|
310
|
+
};
|
|
311
|
+
detail: CalibrationTreeTarget;
|
|
312
|
+
}>;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export interface LoadedCalibrationTreeBundle {
|
|
316
|
+
index: CalibrationTreeIndexArtifact;
|
|
317
|
+
tiers: CalibrationTreeTierArtifact[];
|
|
318
|
+
filterIndex?: CalibrationTreeFilterIndexArtifact;
|
|
319
|
+
targetSummaries?: CalibrationTreeTargetSummary[];
|
|
320
|
+
targetDetailShard?: CalibrationTreeTargetDetailsArtifact;
|
|
321
|
+
selectedTargetDetail?: {
|
|
322
|
+
targetOrdinal: number;
|
|
323
|
+
target: CalibrationTreeTarget;
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
export interface CalibrationTreeTargetDetailsPlan {
|
|
328
|
+
artifacts: CalibrationTreeTargetDetailsArtifact[];
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export interface CalibrationTreeTargetSummariesPlan {
|
|
332
|
+
artifacts: CalibrationTreeTargetSummaryArtifact[];
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
export interface CalibrationTreeTargetDetailSelection {
|
|
336
|
+
descriptor: CalibrationTreeTargetDetailsDescriptor;
|
|
337
|
+
offset: number;
|
|
338
|
+
targetOrdinal: number;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
export interface CalibrationTreeComparisonTargetDetailResponse {
|
|
342
|
+
schemaVersion: typeof CALIBRATION_TREE_SCHEMA_VERSION;
|
|
343
|
+
country: MicrocosmCountry;
|
|
344
|
+
buildArtifactId: string;
|
|
345
|
+
targetOrdinal: number;
|
|
346
|
+
targetId: string;
|
|
347
|
+
target: TargetChangeRow;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
export interface CompileCalibrationTreeInput {
|
|
351
|
+
country: MicrocosmCountry;
|
|
352
|
+
buildArtifactId?: string;
|
|
353
|
+
buildKind?: CalibrationTreeBuildKind;
|
|
354
|
+
sourceId?: string;
|
|
355
|
+
label?: string;
|
|
356
|
+
createdAt?: string | null;
|
|
357
|
+
releaseId: string;
|
|
358
|
+
hfRepo: string | null;
|
|
359
|
+
hfCommitSha: string | null;
|
|
360
|
+
sourceArtifacts: CalibrationTreeSourceArtifacts;
|
|
361
|
+
rows: CalibrationTreeTarget[];
|
|
362
|
+
calibrationProvenance?: CalibrationProvenance;
|
|
363
|
+
lossAttributionAvailable?: boolean;
|
|
364
|
+
comparison?: CalibrationTreeComparisonMetadata;
|
|
365
|
+
comparisonResult?: CalibrationTreeComparisonResultMetadata;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
const DEFAULT_GEOGRAPHY_LEVEL = "national";
|
|
369
|
+
const SHA256_RE = /^[0-9a-f]{64}$/;
|
|
370
|
+
const HF_COMMIT_RE = /^[0-9a-f]{40,64}$/;
|
|
371
|
+
const TIER_PART_RE = /^tier-([1-9][0-9]*)$/;
|
|
372
|
+
const TARGET_SUMMARY_PART_RE = /^target-summary-([0-9]{4})$/;
|
|
373
|
+
const TARGET_DETAILS_PART_RE = /^target-details-([0-9]{4})$/;
|
|
374
|
+
|
|
375
|
+
function targetSummaryPart(shardIndex: number): CalibrationTreeTargetSummaryPart {
|
|
376
|
+
const number = shardIndex + 1;
|
|
377
|
+
if (!Number.isSafeInteger(number) || number < 1 || number > 9_999) {
|
|
378
|
+
throw new Error("Calibration tree target-summary shard number is out of range.");
|
|
379
|
+
}
|
|
380
|
+
return `target-summary-${String(number).padStart(4, "0")}`;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function targetDetailsPart(shardIndex: number): CalibrationTreeTargetDetailsPart {
|
|
384
|
+
const number = shardIndex + 1;
|
|
385
|
+
if (!Number.isSafeInteger(number) || number < 1 || number > 9_999) {
|
|
386
|
+
throw new Error("Calibration tree target-detail shard number is out of range.");
|
|
387
|
+
}
|
|
388
|
+
return `target-details-${String(number).padStart(4, "0")}`;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function finiteNumber(value: unknown): number | null {
|
|
392
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function nonnegativeNumber(value: unknown): number | null {
|
|
396
|
+
const result = finiteNumber(value);
|
|
397
|
+
return result != null && result >= 0 ? result : null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function targetIdentifier(row: CalibrationTreeTarget, index: number): string {
|
|
401
|
+
return String(row.comparison_id ?? row.name ?? row.base_name ?? `target-${index}`);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function normalizeStatus(
|
|
405
|
+
value: CalibrationTreeTarget["calibration_status"],
|
|
406
|
+
): CalibrationStatus | null {
|
|
407
|
+
if (value === "included") return "included";
|
|
408
|
+
if (value === "skipped" || value === "not_materialized") return "skipped";
|
|
409
|
+
return null;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const COMPARISON_ROW_FIELDS = [
|
|
413
|
+
"name",
|
|
414
|
+
"base_name",
|
|
415
|
+
"target",
|
|
416
|
+
"value",
|
|
417
|
+
"final_estimate",
|
|
418
|
+
"estimate",
|
|
419
|
+
"source",
|
|
420
|
+
"source_label",
|
|
421
|
+
"variable",
|
|
422
|
+
"variable_label",
|
|
423
|
+
"variable_key",
|
|
424
|
+
"measure",
|
|
425
|
+
"source_measure_id",
|
|
426
|
+
"target_label",
|
|
427
|
+
"target_representation",
|
|
428
|
+
"dimension_adapter",
|
|
429
|
+
"dimensions",
|
|
430
|
+
"target_dimensions",
|
|
431
|
+
"chronicle",
|
|
432
|
+
"level",
|
|
433
|
+
"geography",
|
|
434
|
+
"family",
|
|
435
|
+
"breakdown",
|
|
436
|
+
"dims",
|
|
437
|
+
"calibration_status",
|
|
438
|
+
"comparison_fit",
|
|
439
|
+
"comparison_id",
|
|
440
|
+
"match_kind",
|
|
441
|
+
"current_name",
|
|
442
|
+
"candidate_name",
|
|
443
|
+
"current_representation",
|
|
444
|
+
"candidate_representation",
|
|
445
|
+
"comparison_status",
|
|
446
|
+
"current",
|
|
447
|
+
"candidate",
|
|
448
|
+
"reported_change",
|
|
449
|
+
"pooled_weight_share",
|
|
450
|
+
"shared_current_contribution",
|
|
451
|
+
"shared_candidate_contribution",
|
|
452
|
+
"shared_change",
|
|
453
|
+
"current_target_ordinal",
|
|
454
|
+
"candidate_target_ordinal",
|
|
455
|
+
"abs_relative_error",
|
|
456
|
+
"target_loss_weight",
|
|
457
|
+
"target_loss_weight_share",
|
|
458
|
+
"target_loss_scale",
|
|
459
|
+
"final_capped_scaled_error",
|
|
460
|
+
"final_loss_contribution",
|
|
461
|
+
] as const;
|
|
462
|
+
|
|
463
|
+
function comparisonReadyTarget(
|
|
464
|
+
row: CalibrationTreeTarget,
|
|
465
|
+
): CalibrationTreeComparisonTarget {
|
|
466
|
+
const compactRow = Object.fromEntries(
|
|
467
|
+
COMPARISON_ROW_FIELDS.flatMap((field) =>
|
|
468
|
+
row[field] === undefined ? [] : [[field, row[field]]],
|
|
469
|
+
),
|
|
470
|
+
) as CalibrationTreeTarget;
|
|
471
|
+
const matchingRow = row as Parameters<typeof normalizedTargetName>[0];
|
|
472
|
+
return {
|
|
473
|
+
normalizedBaseName: normalizedTargetName(matchingRow),
|
|
474
|
+
chronicleFactKey: chronicleFactKey(matchingRow),
|
|
475
|
+
structuredIdentity: structuredIdentity(matchingRow),
|
|
476
|
+
representation: targetRowRepresentation(matchingRow),
|
|
477
|
+
row: compactRow,
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function compiledTarget(
|
|
482
|
+
row: CalibrationTreeTarget,
|
|
483
|
+
index: number,
|
|
484
|
+
geographyLabel: GeographyLabeller,
|
|
485
|
+
): CalibrationTreeBundleDraft["targets"][number] {
|
|
486
|
+
const error = finiteNumber(row.abs_relative_error);
|
|
487
|
+
const hasDetailSource =
|
|
488
|
+
row.current_target_ordinal !== undefined ||
|
|
489
|
+
row.candidate_target_ordinal !== undefined;
|
|
490
|
+
const sourceOrdinal = (value: unknown, label: string): number | null => {
|
|
491
|
+
if (value == null) return null;
|
|
492
|
+
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
|
493
|
+
throw new Error(`${label} must be a non-negative integer or null.`);
|
|
494
|
+
}
|
|
495
|
+
return value as number;
|
|
496
|
+
};
|
|
497
|
+
return {
|
|
498
|
+
id: targetIdentifier(row, index),
|
|
499
|
+
label: targetIdentifier(row, index),
|
|
500
|
+
facets: {
|
|
501
|
+
geographyLevel:
|
|
502
|
+
String(row.level ?? "").trim() || DEFAULT_GEOGRAPHY_LEVEL,
|
|
503
|
+
// The geography's tree label, which the Geography filter offers and
|
|
504
|
+
// selects, so a geography that rows spell more than one way has one
|
|
505
|
+
// posting (see geographyLabeller).
|
|
506
|
+
geography: geographyLabel(row),
|
|
507
|
+
fitBand: fitBandForTarget(row),
|
|
508
|
+
comparisonFit:
|
|
509
|
+
row.comparison_fit === "improved" ||
|
|
510
|
+
row.comparison_fit === "regressed" ||
|
|
511
|
+
row.comparison_fit === "unchanged" ||
|
|
512
|
+
row.comparison_fit === "not_applicable"
|
|
513
|
+
? row.comparison_fit
|
|
514
|
+
: null,
|
|
515
|
+
calibrationStatus: normalizeStatus(row.calibration_status),
|
|
516
|
+
},
|
|
517
|
+
metricInputs: {
|
|
518
|
+
absRelativeError: error == null ? null : Math.abs(error),
|
|
519
|
+
targetLossWeightShare: nonnegativeNumber(row.target_loss_weight_share),
|
|
520
|
+
finalLossContribution: nonnegativeNumber(row.final_loss_contribution),
|
|
521
|
+
targetChange: finiteNumber(row.target_change),
|
|
522
|
+
comparisonStatus:
|
|
523
|
+
row.comparison_status === "shared" ||
|
|
524
|
+
row.comparison_status === "added" ||
|
|
525
|
+
row.comparison_status === "removed"
|
|
526
|
+
? row.comparison_status
|
|
527
|
+
: null,
|
|
528
|
+
},
|
|
529
|
+
comparison: comparisonReadyTarget(row),
|
|
530
|
+
...(hasDetailSource
|
|
531
|
+
? {
|
|
532
|
+
detailSource: {
|
|
533
|
+
currentTargetOrdinal: sourceOrdinal(
|
|
534
|
+
row.current_target_ordinal,
|
|
535
|
+
"Current comparison target ordinal",
|
|
536
|
+
),
|
|
537
|
+
candidateTargetOrdinal: sourceOrdinal(
|
|
538
|
+
row.candidate_target_ordinal,
|
|
539
|
+
"Candidate comparison target ordinal",
|
|
540
|
+
),
|
|
541
|
+
},
|
|
542
|
+
}
|
|
543
|
+
: {}),
|
|
544
|
+
detail: row,
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function uniqueSorted(indices: number[]): number[] {
|
|
549
|
+
return [...new Set(indices)].sort((left, right) => left - right);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
function sameIndices(left: number[], right: number[]): boolean {
|
|
553
|
+
return left.length === right.length &&
|
|
554
|
+
left.every((value, index) => value === right[index]);
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function semanticStateKey(state: ExplorerState): string {
|
|
558
|
+
const path = {
|
|
559
|
+
source: state.path.source,
|
|
560
|
+
program: state.path.program,
|
|
561
|
+
geography: state.path.geography,
|
|
562
|
+
dimensions: state.path.dimensions.map(({ key, label, value }) => ({
|
|
563
|
+
key,
|
|
564
|
+
label,
|
|
565
|
+
value,
|
|
566
|
+
})),
|
|
567
|
+
};
|
|
568
|
+
return path.source && path.program && path.geography
|
|
569
|
+
? JSON.stringify(path)
|
|
570
|
+
: JSON.stringify({ breakdown: state.breakdown, path });
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function targetOrdinalFor(
|
|
574
|
+
target: CalibrationTreeTarget | undefined,
|
|
575
|
+
targetOrdinals: Map<CalibrationTreeTarget, number>,
|
|
576
|
+
): number {
|
|
577
|
+
if (!target) throw new Error("Compiled target node is missing target details.");
|
|
578
|
+
const ordinal = targetOrdinals.get(target);
|
|
579
|
+
if (ordinal == null) {
|
|
580
|
+
throw new Error(`Compiled target ${String(target.name ?? target.base_name)} is unknown.`);
|
|
581
|
+
}
|
|
582
|
+
return ordinal;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
function inferredTargetRepresentation(
|
|
586
|
+
targets: CalibrationTreeTarget[],
|
|
587
|
+
): TargetRepresentation {
|
|
588
|
+
if (!targets.length) return "unknown";
|
|
589
|
+
const values = new Set(
|
|
590
|
+
targets.map((row) =>
|
|
591
|
+
targetRowRepresentation(
|
|
592
|
+
row as Parameters<typeof targetRowRepresentation>[0],
|
|
593
|
+
),
|
|
594
|
+
),
|
|
595
|
+
);
|
|
596
|
+
if (values.size > 1) return "mixed";
|
|
597
|
+
if (values.has("hierarchy")) return "hierarchy";
|
|
598
|
+
return values.has("structured") ? "structured" : "legacy";
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
export function compileCalibrationTreeBundleDraft(
|
|
602
|
+
input: CompileCalibrationTreeInput,
|
|
603
|
+
): CalibrationTreeBundleDraft {
|
|
604
|
+
if (
|
|
605
|
+
input.buildKind !== "comparison" &&
|
|
606
|
+
(input.hfCommitSha == null || !HF_COMMIT_RE.test(input.hfCommitSha))
|
|
607
|
+
) {
|
|
608
|
+
throw new Error(`Invalid Hugging Face commit SHA: ${input.hfCommitSha}`);
|
|
609
|
+
}
|
|
610
|
+
const buildArtifactId = input.buildArtifactId ??
|
|
611
|
+
input.sourceArtifacts.calibrationDiagnostics?.sha256;
|
|
612
|
+
if (!buildArtifactId) {
|
|
613
|
+
throw new Error("A calibration build artifact id is required.");
|
|
614
|
+
}
|
|
615
|
+
if (!SHA256_RE.test(buildArtifactId)) {
|
|
616
|
+
throw new Error(`Invalid calibration build artifact id: ${buildArtifactId}`);
|
|
617
|
+
}
|
|
618
|
+
const rows = normalizeCalibrationTreeTargets(input.rows);
|
|
619
|
+
const lossAttributionAvailable =
|
|
620
|
+
input.lossAttributionAvailable ??
|
|
621
|
+
(rows.length > 0 &&
|
|
622
|
+
rows.every((row) => nonnegativeNumber(row.final_loss_contribution) != null));
|
|
623
|
+
// One labeller for the geography postings and for every level of the tree.
|
|
624
|
+
const geographyLabel = geographyLabeller(rows);
|
|
625
|
+
const targets = rows.map((row, index) =>
|
|
626
|
+
compiledTarget(row, index, geographyLabel),
|
|
627
|
+
);
|
|
628
|
+
const targetOrdinals = new Map(rows.map((row, ordinal) => [row, ordinal]));
|
|
629
|
+
const levels: Record<string, CompiledCalibrationLevel> = {};
|
|
630
|
+
const stateLevels = new Map<string, string>();
|
|
631
|
+
let nextLevelNumber = 0;
|
|
632
|
+
let filterOptions: CalibrationTreeResponse["filterOptions"] | null = null;
|
|
633
|
+
|
|
634
|
+
const compileLevel = (state: ExplorerState): string => {
|
|
635
|
+
const stateKey = semanticStateKey(state);
|
|
636
|
+
const existing = stateLevels.get(stateKey);
|
|
637
|
+
if (existing) return existing;
|
|
638
|
+
|
|
639
|
+
const levelId = `level-${String(nextLevelNumber).padStart(6, "0")}`;
|
|
640
|
+
nextLevelNumber += 1;
|
|
641
|
+
stateLevels.set(stateKey, levelId);
|
|
642
|
+
|
|
643
|
+
const response = buildCalibrationTree(
|
|
644
|
+
rows,
|
|
645
|
+
state,
|
|
646
|
+
input.releaseId,
|
|
647
|
+
lossAttributionAvailable,
|
|
648
|
+
undefined,
|
|
649
|
+
geographyLabel,
|
|
650
|
+
);
|
|
651
|
+
filterOptions ??= response.filterOptions;
|
|
652
|
+
|
|
653
|
+
const groups = response.groups.map((group): CompiledCalibrationGroup => {
|
|
654
|
+
const nodes = group.nodes.map((treeNode): CompiledCalibrationNode => {
|
|
655
|
+
if (treeNode.kind === "target") {
|
|
656
|
+
const targetOrdinal = targetOrdinalFor(treeNode.target, targetOrdinals);
|
|
657
|
+
targets[targetOrdinal].id = treeNode.id;
|
|
658
|
+
targets[targetOrdinal].label = treeNode.label;
|
|
659
|
+
return {
|
|
660
|
+
id: treeNode.id,
|
|
661
|
+
label: treeNode.label,
|
|
662
|
+
kind: "target",
|
|
663
|
+
selection: treeNode.selection,
|
|
664
|
+
targetOrdinals: [targetOrdinal],
|
|
665
|
+
targetOrdinal,
|
|
666
|
+
metrics: treeNode.metrics,
|
|
667
|
+
authoredLabel: treeNode.authored_label,
|
|
668
|
+
};
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
const childState = selectExplorerNode(state, treeNode.selection);
|
|
672
|
+
if (semanticStateKey(childState) === stateKey) {
|
|
673
|
+
throw new Error(`Selection ${treeNode.id} did not advance the calibration tree.`);
|
|
674
|
+
}
|
|
675
|
+
const nextLevelId = compileLevel(childState);
|
|
676
|
+
return {
|
|
677
|
+
id: treeNode.id,
|
|
678
|
+
label: treeNode.label,
|
|
679
|
+
kind: treeNode.kind,
|
|
680
|
+
selection: treeNode.selection,
|
|
681
|
+
targetOrdinals: levels[nextLevelId].targetOrdinals,
|
|
682
|
+
nextLevelId,
|
|
683
|
+
metrics: treeNode.metrics,
|
|
684
|
+
authoredLabel: treeNode.authored_label,
|
|
685
|
+
};
|
|
686
|
+
});
|
|
687
|
+
return {
|
|
688
|
+
id: group.id,
|
|
689
|
+
label: group.label,
|
|
690
|
+
nodes,
|
|
691
|
+
targetOrdinals: uniqueSorted(nodes.flatMap((node) => node.targetOrdinals)),
|
|
692
|
+
metrics: group.metrics,
|
|
693
|
+
};
|
|
694
|
+
});
|
|
695
|
+
|
|
696
|
+
levels[levelId] = {
|
|
697
|
+
currentLevel: response.currentLevel,
|
|
698
|
+
pathLabels: response.pathLabels,
|
|
699
|
+
dimensionOrder: response.dimensionOrder,
|
|
700
|
+
groups,
|
|
701
|
+
targetOrdinals: uniqueSorted(groups.flatMap((group) => group.targetOrdinals)),
|
|
702
|
+
metrics: response.filteredMetrics,
|
|
703
|
+
};
|
|
704
|
+
return levelId;
|
|
705
|
+
};
|
|
706
|
+
|
|
707
|
+
const roots = {
|
|
708
|
+
program: compileLevel(createExplorerState()),
|
|
709
|
+
geography: compileLevel({ ...createExplorerState(), breakdown: "geography" }),
|
|
710
|
+
};
|
|
711
|
+
const levelDepths: Record<string, number> = {};
|
|
712
|
+
const queue = [
|
|
713
|
+
{ levelId: roots.program, depth: 0 },
|
|
714
|
+
{ levelId: roots.geography, depth: 0 },
|
|
715
|
+
];
|
|
716
|
+
while (queue.length) {
|
|
717
|
+
const { levelId, depth } = queue.shift()!;
|
|
718
|
+
const previousDepth = levelDepths[levelId];
|
|
719
|
+
if (previousDepth != null) {
|
|
720
|
+
if (previousDepth !== depth) {
|
|
721
|
+
throw new Error(`Calibration tree level ${levelId} is reachable at multiple depths.`);
|
|
722
|
+
}
|
|
723
|
+
continue;
|
|
724
|
+
}
|
|
725
|
+
levelDepths[levelId] = depth;
|
|
726
|
+
for (const node of levels[levelId].groups.flatMap((group) => group.nodes)) {
|
|
727
|
+
if (node.kind !== "target") {
|
|
728
|
+
queue.push({ levelId: node.nextLevelId, depth: depth + 1 });
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
if (Object.keys(levelDepths).length !== Object.keys(levels).length) {
|
|
733
|
+
throw new Error("Calibration tree contains an unreachable level.");
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
return {
|
|
737
|
+
country: input.country,
|
|
738
|
+
build: {
|
|
739
|
+
buildArtifactId,
|
|
740
|
+
kind: input.buildKind ?? "release",
|
|
741
|
+
sourceId: input.sourceId ?? input.releaseId,
|
|
742
|
+
label: input.label ?? input.releaseId,
|
|
743
|
+
releaseId: input.releaseId,
|
|
744
|
+
hfRepo: input.hfRepo,
|
|
745
|
+
hfCommitSha: input.hfCommitSha,
|
|
746
|
+
createdAt: input.createdAt ?? null,
|
|
747
|
+
sourceArtifacts: input.sourceArtifacts,
|
|
748
|
+
},
|
|
749
|
+
comparison: input.comparison ?? {
|
|
750
|
+
releaseId: input.releaseId,
|
|
751
|
+
calibrationProvenance: input.calibrationProvenance,
|
|
752
|
+
status: lossAttributionAvailable ? "derived" : "unavailable",
|
|
753
|
+
aggregate: lossAttributionAvailable
|
|
754
|
+
? targets.reduce(
|
|
755
|
+
(sum, target) =>
|
|
756
|
+
sum + (target.metricInputs.finalLossContribution ?? 0),
|
|
757
|
+
0,
|
|
758
|
+
)
|
|
759
|
+
: null,
|
|
760
|
+
cap: null,
|
|
761
|
+
basisIdentifier: null,
|
|
762
|
+
targetRepresentation: inferredTargetRepresentation(rows),
|
|
763
|
+
},
|
|
764
|
+
comparisonResult: input.comparisonResult,
|
|
765
|
+
calibrationProvenance: input.calibrationProvenance,
|
|
766
|
+
lossAttributionAvailable,
|
|
767
|
+
filterOptions: filterOptions ?? {
|
|
768
|
+
geographyLevels: [],
|
|
769
|
+
geographies: [],
|
|
770
|
+
fitBands: [],
|
|
771
|
+
comparisonFits: [],
|
|
772
|
+
calibrationStatuses: [],
|
|
773
|
+
},
|
|
774
|
+
roots,
|
|
775
|
+
levels,
|
|
776
|
+
levelDepths,
|
|
777
|
+
targets,
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
function postingValues<T extends string | null>(values: T[]): CalibrationTreePosting<T>[] {
|
|
782
|
+
const postings = new Map<T, number[]>();
|
|
783
|
+
values.forEach((value, index) => {
|
|
784
|
+
const indices = postings.get(value) ?? [];
|
|
785
|
+
indices.push(index);
|
|
786
|
+
postings.set(value, indices);
|
|
787
|
+
});
|
|
788
|
+
return [...postings.entries()]
|
|
789
|
+
.sort(([left], [right]) => String(left).localeCompare(String(right)))
|
|
790
|
+
.map(([value, targetOrdinals]) => ({ value, targetOrdinals }));
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
export function calibrationTreeFilterIndexFromDraft(
|
|
794
|
+
draft: CalibrationTreeBundleDraft,
|
|
795
|
+
): CalibrationTreeFilterIndexArtifact {
|
|
796
|
+
return {
|
|
797
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
798
|
+
country: draft.country,
|
|
799
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
800
|
+
part: "filter-index",
|
|
801
|
+
targetCount: draft.targets.length,
|
|
802
|
+
postings: {
|
|
803
|
+
geographyLevels: postingValues(
|
|
804
|
+
draft.targets.map((target) => target.facets.geographyLevel),
|
|
805
|
+
),
|
|
806
|
+
geographies: postingValues(
|
|
807
|
+
draft.targets.map((target) => target.facets.geography),
|
|
808
|
+
),
|
|
809
|
+
fitBands: postingValues(
|
|
810
|
+
draft.targets.map((target) => target.facets.fitBand),
|
|
811
|
+
),
|
|
812
|
+
comparisonFits: postingValues(
|
|
813
|
+
draft.targets.map((target) => target.facets.comparisonFit),
|
|
814
|
+
),
|
|
815
|
+
calibrationStatuses: postingValues(
|
|
816
|
+
draft.targets.map((target) => target.facets.calibrationStatus),
|
|
817
|
+
),
|
|
818
|
+
},
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
export function calibrationTreeTargetSummariesFromDraft(
|
|
823
|
+
draft: CalibrationTreeBundleDraft,
|
|
824
|
+
maxRawBytes = CALIBRATION_TREE_TARGET_SUMMARY_MAX_RAW_BYTES,
|
|
825
|
+
): CalibrationTreeTargetSummariesPlan {
|
|
826
|
+
if (!Number.isSafeInteger(maxRawBytes) || maxRawBytes <= 0) {
|
|
827
|
+
throw new Error("Calibration tree target-summary shard size must be a positive integer.");
|
|
828
|
+
}
|
|
829
|
+
const targets = draft.targets.map(({ id, label, metricInputs, comparison, detailSource }) => ({
|
|
830
|
+
id,
|
|
831
|
+
label,
|
|
832
|
+
metricInputs,
|
|
833
|
+
comparison,
|
|
834
|
+
...(detailSource ? { detailSource } : {}),
|
|
835
|
+
}));
|
|
836
|
+
const serializedTargets = targets.map(stableJson);
|
|
837
|
+
const targetBytes = serializedTargets.map(utf8Bytes);
|
|
838
|
+
const artifacts: CalibrationTreeTargetSummaryArtifact[] = [];
|
|
839
|
+
let startTargetOrdinal = 0;
|
|
840
|
+
|
|
841
|
+
while (startTargetOrdinal < targets.length) {
|
|
842
|
+
const shardIndex = artifacts.length;
|
|
843
|
+
if (shardIndex >= 9_999) {
|
|
844
|
+
throw new Error("Calibration tree target summaries require more than 9,999 shards.");
|
|
845
|
+
}
|
|
846
|
+
const part = targetSummaryPart(shardIndex);
|
|
847
|
+
let endTargetOrdinalExclusive = startTargetOrdinal;
|
|
848
|
+
let payloadBytes = 0;
|
|
849
|
+
|
|
850
|
+
while (endTargetOrdinalExclusive < targets.length) {
|
|
851
|
+
const nextPayloadBytes =
|
|
852
|
+
payloadBytes +
|
|
853
|
+
targetBytes[endTargetOrdinalExclusive] +
|
|
854
|
+
(endTargetOrdinalExclusive === startTargetOrdinal ? 0 : 1);
|
|
855
|
+
const emptyArtifact: CalibrationTreeTargetSummaryArtifact = {
|
|
856
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
857
|
+
country: draft.country,
|
|
858
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
859
|
+
part,
|
|
860
|
+
startTargetOrdinal,
|
|
861
|
+
endTargetOrdinalExclusive: endTargetOrdinalExclusive + 1,
|
|
862
|
+
targets: [],
|
|
863
|
+
};
|
|
864
|
+
const candidateBytes = utf8Bytes(`${stableJson(emptyArtifact)}\n`) + nextPayloadBytes;
|
|
865
|
+
if (candidateBytes > maxRawBytes) break;
|
|
866
|
+
payloadBytes = nextPayloadBytes;
|
|
867
|
+
endTargetOrdinalExclusive += 1;
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
if (endTargetOrdinalExclusive === startTargetOrdinal) {
|
|
871
|
+
const targetBytesWithEnvelope = utf8Bytes(`${stableJson({
|
|
872
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
873
|
+
country: draft.country,
|
|
874
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
875
|
+
part,
|
|
876
|
+
startTargetOrdinal,
|
|
877
|
+
endTargetOrdinalExclusive: startTargetOrdinal + 1,
|
|
878
|
+
targets: [targets[startTargetOrdinal]],
|
|
879
|
+
})}\n`);
|
|
880
|
+
throw new Error(
|
|
881
|
+
`Calibration target summary ordinal ${startTargetOrdinal} requires ` +
|
|
882
|
+
`${targetBytesWithEnvelope} bytes, which exceeds the ${maxRawBytes}-byte shard limit.`,
|
|
883
|
+
);
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
const artifact: CalibrationTreeTargetSummaryArtifact = {
|
|
887
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
888
|
+
country: draft.country,
|
|
889
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
890
|
+
part,
|
|
891
|
+
startTargetOrdinal,
|
|
892
|
+
endTargetOrdinalExclusive,
|
|
893
|
+
targets: targets.slice(startTargetOrdinal, endTargetOrdinalExclusive),
|
|
894
|
+
};
|
|
895
|
+
if (utf8Bytes(serializeCalibrationTreePart(artifact)) > maxRawBytes) {
|
|
896
|
+
throw new Error(`Calibration tree shard ${part} exceeded its planned byte limit.`);
|
|
897
|
+
}
|
|
898
|
+
artifacts.push(artifact);
|
|
899
|
+
startTargetOrdinal = endTargetOrdinalExclusive;
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
return { artifacts };
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
export function calibrationTreeTargetDetailsFromDraft(
|
|
906
|
+
draft: CalibrationTreeBundleDraft,
|
|
907
|
+
maxRawBytes = CALIBRATION_TREE_TARGET_DETAIL_MAX_RAW_BYTES,
|
|
908
|
+
): CalibrationTreeTargetDetailsPlan {
|
|
909
|
+
if (!Number.isSafeInteger(maxRawBytes) || maxRawBytes <= 0) {
|
|
910
|
+
throw new Error("Calibration tree target-detail shard size must be a positive integer.");
|
|
911
|
+
}
|
|
912
|
+
const serializedTargets = draft.targets.map((target) => stableJson(target.detail));
|
|
913
|
+
const targetBytes = serializedTargets.map(utf8Bytes);
|
|
914
|
+
const artifacts: CalibrationTreeTargetDetailsArtifact[] = [];
|
|
915
|
+
let startTargetOrdinal = 0;
|
|
916
|
+
|
|
917
|
+
while (startTargetOrdinal < draft.targets.length) {
|
|
918
|
+
const shardIndex = artifacts.length;
|
|
919
|
+
if (shardIndex >= 9_999) {
|
|
920
|
+
throw new Error("Calibration tree target details require more than 9,999 shards.");
|
|
921
|
+
}
|
|
922
|
+
const part = targetDetailsPart(shardIndex);
|
|
923
|
+
let endTargetOrdinalExclusive = startTargetOrdinal;
|
|
924
|
+
let payloadBytes = 0;
|
|
925
|
+
|
|
926
|
+
while (endTargetOrdinalExclusive < draft.targets.length) {
|
|
927
|
+
const nextPayloadBytes =
|
|
928
|
+
payloadBytes +
|
|
929
|
+
targetBytes[endTargetOrdinalExclusive] +
|
|
930
|
+
(endTargetOrdinalExclusive === startTargetOrdinal ? 0 : 1);
|
|
931
|
+
const emptyArtifact: CalibrationTreeTargetDetailsArtifact = {
|
|
932
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
933
|
+
country: draft.country,
|
|
934
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
935
|
+
part,
|
|
936
|
+
startTargetOrdinal,
|
|
937
|
+
endTargetOrdinalExclusive: endTargetOrdinalExclusive + 1,
|
|
938
|
+
targets: [],
|
|
939
|
+
};
|
|
940
|
+
const candidateBytes = utf8Bytes(`${stableJson(emptyArtifact)}\n`) + nextPayloadBytes;
|
|
941
|
+
if (candidateBytes > maxRawBytes) break;
|
|
942
|
+
payloadBytes = nextPayloadBytes;
|
|
943
|
+
endTargetOrdinalExclusive += 1;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
if (endTargetOrdinalExclusive === startTargetOrdinal) {
|
|
947
|
+
const targetBytesWithEnvelope = utf8Bytes(`${stableJson({
|
|
948
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
949
|
+
country: draft.country,
|
|
950
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
951
|
+
part,
|
|
952
|
+
startTargetOrdinal,
|
|
953
|
+
endTargetOrdinalExclusive: startTargetOrdinal + 1,
|
|
954
|
+
targets: [draft.targets[startTargetOrdinal].detail],
|
|
955
|
+
})}\n`);
|
|
956
|
+
throw new Error(
|
|
957
|
+
`Calibration target ordinal ${startTargetOrdinal} requires ${targetBytesWithEnvelope} bytes, ` +
|
|
958
|
+
`which exceeds the ${maxRawBytes}-byte shard limit.`,
|
|
959
|
+
);
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
const artifact: CalibrationTreeTargetDetailsArtifact = {
|
|
963
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
964
|
+
country: draft.country,
|
|
965
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
966
|
+
part,
|
|
967
|
+
startTargetOrdinal,
|
|
968
|
+
endTargetOrdinalExclusive,
|
|
969
|
+
targets: draft.targets
|
|
970
|
+
.slice(startTargetOrdinal, endTargetOrdinalExclusive)
|
|
971
|
+
.map((target) => target.detail),
|
|
972
|
+
};
|
|
973
|
+
const serialized = serializeCalibrationTreePart(artifact);
|
|
974
|
+
if (utf8Bytes(serialized) > maxRawBytes) {
|
|
975
|
+
throw new Error(`Calibration tree shard ${part} exceeded its planned byte limit.`);
|
|
976
|
+
}
|
|
977
|
+
artifacts.push(artifact);
|
|
978
|
+
startTargetOrdinal = endTargetOrdinalExclusive;
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
return { artifacts };
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
export function calibrationTreeTargetDetailSelection(
|
|
985
|
+
index: CalibrationTreeIndexArtifact,
|
|
986
|
+
targetOrdinal: number,
|
|
987
|
+
): CalibrationTreeTargetDetailSelection {
|
|
988
|
+
if (index.parts.targetDetailStrategy !== "shards") {
|
|
989
|
+
throw new Error("Calibration comparison target details resolve through source targets.");
|
|
990
|
+
}
|
|
991
|
+
const descriptor = index.parts.targetDetails.find(
|
|
992
|
+
(candidate) =>
|
|
993
|
+
targetOrdinal >= candidate.startTargetOrdinal &&
|
|
994
|
+
targetOrdinal < candidate.endTargetOrdinalExclusive,
|
|
995
|
+
);
|
|
996
|
+
if (
|
|
997
|
+
!Number.isSafeInteger(targetOrdinal) ||
|
|
998
|
+
targetOrdinal < 0 ||
|
|
999
|
+
!descriptor
|
|
1000
|
+
) {
|
|
1001
|
+
throw new Error(
|
|
1002
|
+
`Calibration target ordinal ${targetOrdinal} is not covered by a detail shard.`,
|
|
1003
|
+
);
|
|
1004
|
+
}
|
|
1005
|
+
const offset = targetOrdinal - descriptor.startTargetOrdinal;
|
|
1006
|
+
return { descriptor, offset, targetOrdinal };
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
export function calibrationTreeTiersFromDraft(
|
|
1010
|
+
draft: CalibrationTreeBundleDraft,
|
|
1011
|
+
): CalibrationTreeTierArtifact[] {
|
|
1012
|
+
const maxDepth = Math.max(0, ...Object.values(draft.levelDepths));
|
|
1013
|
+
return Array.from({ length: maxDepth }, (_, index) => index + 1).map((depth) => ({
|
|
1014
|
+
schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
|
|
1015
|
+
country: draft.country,
|
|
1016
|
+
buildArtifactId: draft.build.buildArtifactId,
|
|
1017
|
+
part: `tier-${depth}`,
|
|
1018
|
+
depth,
|
|
1019
|
+
targetCount: draft.targets.length,
|
|
1020
|
+
levels: Object.fromEntries(
|
|
1021
|
+
Object.entries(draft.levels).filter(
|
|
1022
|
+
([levelId]) => draft.levelDepths[levelId] === depth,
|
|
1023
|
+
),
|
|
1024
|
+
),
|
|
1025
|
+
}));
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
function stableValue(value: unknown): unknown {
|
|
1029
|
+
if (Array.isArray(value)) return value.map(stableValue);
|
|
1030
|
+
if (value && typeof value === "object") {
|
|
1031
|
+
return Object.fromEntries(
|
|
1032
|
+
Object.entries(value as Record<string, unknown>)
|
|
1033
|
+
.filter(([, item]) => item !== undefined)
|
|
1034
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
1035
|
+
.map(([key, item]) => [key, stableValue(item)]),
|
|
1036
|
+
);
|
|
1037
|
+
}
|
|
1038
|
+
if (typeof value === "number" && !Number.isFinite(value)) return null;
|
|
1039
|
+
return value;
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
function stableJson(value: unknown): string {
|
|
1043
|
+
return JSON.stringify(stableValue(value));
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
function utf8Bytes(value: string): number {
|
|
1047
|
+
return new TextEncoder().encode(value).byteLength;
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
export function serializeCalibrationTreePart(part: CalibrationTreeArtifactPart): string {
|
|
1051
|
+
parseCalibrationTreePart(part, part.part);
|
|
1052
|
+
return `${stableJson(part)}\n`;
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
function record(value: unknown, label: string): Record<string, unknown> {
|
|
1056
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1057
|
+
throw new Error(`${label} must be an object.`);
|
|
1058
|
+
}
|
|
1059
|
+
return value as Record<string, unknown>;
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
function validateIndices(value: unknown, targetCount: number, label: string): number[] {
|
|
1063
|
+
if (!Array.isArray(value)) throw new Error(`${label} must be an array.`);
|
|
1064
|
+
let previous = -1;
|
|
1065
|
+
for (const item of value) {
|
|
1066
|
+
if (!Number.isInteger(item) || item < 0 || item >= targetCount) {
|
|
1067
|
+
throw new Error(`${label} contains an invalid target index.`);
|
|
1068
|
+
}
|
|
1069
|
+
if (item <= previous) throw new Error(`${label} must be sorted and unique.`);
|
|
1070
|
+
previous = item;
|
|
1071
|
+
}
|
|
1072
|
+
return value as number[];
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
function stringArray(value: unknown, label: string): string[] {
|
|
1076
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
1077
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
1078
|
+
}
|
|
1079
|
+
return value as string[];
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
function validateIdentity(
|
|
1083
|
+
value: unknown,
|
|
1084
|
+
expectedPart?: CalibrationTreePart,
|
|
1085
|
+
): Record<string, unknown> {
|
|
1086
|
+
const part = record(value, "Calibration tree part");
|
|
1087
|
+
if (part.schemaVersion !== CALIBRATION_TREE_SCHEMA_VERSION) {
|
|
1088
|
+
throw new Error(`Unsupported calibration tree schema ${String(part.schemaVersion)}.`);
|
|
1089
|
+
}
|
|
1090
|
+
if (typeof part.country !== "string" || !isCountry(part.country)) {
|
|
1091
|
+
throw new Error("Calibration tree part has an unknown country.");
|
|
1092
|
+
}
|
|
1093
|
+
if (
|
|
1094
|
+
typeof part.buildArtifactId !== "string" ||
|
|
1095
|
+
!SHA256_RE.test(part.buildArtifactId)
|
|
1096
|
+
) {
|
|
1097
|
+
throw new Error("Calibration tree part has an invalid build artifact id.");
|
|
1098
|
+
}
|
|
1099
|
+
if (typeof part.part !== "string" || !isCalibrationTreePart(part.part)) {
|
|
1100
|
+
throw new Error("Calibration tree part name is invalid.");
|
|
1101
|
+
}
|
|
1102
|
+
if (expectedPart && part.part !== expectedPart) {
|
|
1103
|
+
throw new Error(`Expected calibration tree part ${expectedPart}, received ${part.part}.`);
|
|
1104
|
+
}
|
|
1105
|
+
return part;
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
function validateLevelRecords(
|
|
1109
|
+
value: unknown,
|
|
1110
|
+
targetCount: number,
|
|
1111
|
+
label: string,
|
|
1112
|
+
): Record<string, CompiledCalibrationLevel> {
|
|
1113
|
+
const levels = record(value, label);
|
|
1114
|
+
for (const [levelId, item] of Object.entries(levels)) {
|
|
1115
|
+
const level = record(item, `${label}.${levelId}`);
|
|
1116
|
+
const levelIndices = validateIndices(
|
|
1117
|
+
level.targetOrdinals,
|
|
1118
|
+
targetCount,
|
|
1119
|
+
`${levelId}.targetOrdinals`,
|
|
1120
|
+
);
|
|
1121
|
+
if (!Array.isArray(level.groups)) throw new Error(`${levelId}.groups must be an array.`);
|
|
1122
|
+
const indicesFromGroups: number[] = [];
|
|
1123
|
+
for (const [groupIndex, groupItem] of level.groups.entries()) {
|
|
1124
|
+
const group = record(groupItem, `${levelId}.groups[${groupIndex}]`);
|
|
1125
|
+
const groupIndices = validateIndices(
|
|
1126
|
+
group.targetOrdinals,
|
|
1127
|
+
targetCount,
|
|
1128
|
+
`${levelId}.groups[${groupIndex}].targetOrdinals`,
|
|
1129
|
+
);
|
|
1130
|
+
if (!Array.isArray(group.nodes)) {
|
|
1131
|
+
throw new Error(`${levelId}.groups[${groupIndex}].nodes must be an array.`);
|
|
1132
|
+
}
|
|
1133
|
+
const indicesFromNodes: number[] = [];
|
|
1134
|
+
for (const [nodeIndex, nodeItem] of group.nodes.entries()) {
|
|
1135
|
+
const node = record(nodeItem, `${levelId}.groups[${groupIndex}].nodes[${nodeIndex}]`);
|
|
1136
|
+
const indices = validateIndices(
|
|
1137
|
+
node.targetOrdinals,
|
|
1138
|
+
targetCount,
|
|
1139
|
+
`${levelId}.groups[${groupIndex}].nodes[${nodeIndex}].targetOrdinals`,
|
|
1140
|
+
);
|
|
1141
|
+
indicesFromNodes.push(...indices);
|
|
1142
|
+
if (node.kind === "target") {
|
|
1143
|
+
if (
|
|
1144
|
+
indices.length !== 1 ||
|
|
1145
|
+
!Number.isInteger(node.targetOrdinal) ||
|
|
1146
|
+
indices[0] !== node.targetOrdinal
|
|
1147
|
+
) {
|
|
1148
|
+
throw new Error(`Target node ${String(node.id)} has an invalid target ordinal.`);
|
|
1149
|
+
}
|
|
1150
|
+
} else if (typeof node.nextLevelId !== "string" || !node.nextLevelId) {
|
|
1151
|
+
throw new Error(`Branch node ${String(node.id)} has an invalid next level.`);
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
if (!sameIndices(groupIndices, uniqueSorted(indicesFromNodes))) {
|
|
1155
|
+
throw new Error(`${levelId}.groups[${groupIndex}] has inconsistent membership.`);
|
|
1156
|
+
}
|
|
1157
|
+
indicesFromGroups.push(...groupIndices);
|
|
1158
|
+
}
|
|
1159
|
+
if (!sameIndices(levelIndices, uniqueSorted(indicesFromGroups))) {
|
|
1160
|
+
throw new Error(`${levelId} has inconsistent membership.`);
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
return levels as Record<string, CompiledCalibrationLevel>;
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
function validateDescriptor(
|
|
1167
|
+
value: unknown,
|
|
1168
|
+
expectedPart: Exclude<CalibrationTreePart, "index">,
|
|
1169
|
+
country: MicrocosmCountry,
|
|
1170
|
+
buildArtifactId: string,
|
|
1171
|
+
): CalibrationTreePartDescriptor {
|
|
1172
|
+
const descriptor = record(value, `Calibration tree descriptor ${expectedPart}`);
|
|
1173
|
+
if (descriptor.part !== expectedPart) {
|
|
1174
|
+
throw new Error(`Calibration tree descriptor ${expectedPart} has the wrong part name.`);
|
|
1175
|
+
}
|
|
1176
|
+
if (
|
|
1177
|
+
descriptor.path !==
|
|
1178
|
+
calibrationTreePartPath(country, buildArtifactId, expectedPart)
|
|
1179
|
+
) {
|
|
1180
|
+
throw new Error(`Calibration tree descriptor ${expectedPart} has the wrong path.`);
|
|
1181
|
+
}
|
|
1182
|
+
if (typeof descriptor.sha256 !== "string" || !SHA256_RE.test(descriptor.sha256)) {
|
|
1183
|
+
throw new Error(`Calibration tree descriptor ${expectedPart} has an invalid hash.`);
|
|
1184
|
+
}
|
|
1185
|
+
for (const field of ["rawBytes", "gzipBytes"] as const) {
|
|
1186
|
+
if (!Number.isSafeInteger(descriptor[field]) || (descriptor[field] as number) <= 0) {
|
|
1187
|
+
throw new Error(`Calibration tree descriptor ${expectedPart} has invalid ${field}.`);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
return descriptor as unknown as CalibrationTreePartDescriptor;
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
function validateTargetShardDescriptors(options: {
|
|
1194
|
+
value: unknown;
|
|
1195
|
+
targetCount: number;
|
|
1196
|
+
country: MicrocosmCountry;
|
|
1197
|
+
buildArtifactId: string;
|
|
1198
|
+
label: "target-summary" | "target-detail";
|
|
1199
|
+
maxRawBytes: number;
|
|
1200
|
+
partForIndex: (
|
|
1201
|
+
shardIndex: number,
|
|
1202
|
+
) => CalibrationTreeTargetSummaryPart | CalibrationTreeTargetDetailsPart;
|
|
1203
|
+
}): void {
|
|
1204
|
+
if (!Array.isArray(options.value)) {
|
|
1205
|
+
throw new Error(`Calibration tree ${options.label} descriptors must be an array.`);
|
|
1206
|
+
}
|
|
1207
|
+
let nextTargetOrdinal = 0;
|
|
1208
|
+
options.value.forEach((item, shardIndex) => {
|
|
1209
|
+
const expectedPart = options.partForIndex(shardIndex);
|
|
1210
|
+
const descriptor = validateDescriptor(
|
|
1211
|
+
item,
|
|
1212
|
+
expectedPart,
|
|
1213
|
+
options.country,
|
|
1214
|
+
options.buildArtifactId,
|
|
1215
|
+
) as CalibrationTreeTargetSummaryDescriptor | CalibrationTreeTargetDetailsDescriptor;
|
|
1216
|
+
if (descriptor.rawBytes > options.maxRawBytes) {
|
|
1217
|
+
throw new Error(`Calibration tree descriptor ${expectedPart} exceeds the byte limit.`);
|
|
1218
|
+
}
|
|
1219
|
+
if (
|
|
1220
|
+
descriptor.startTargetOrdinal !== nextTargetOrdinal ||
|
|
1221
|
+
!Number.isSafeInteger(descriptor.endTargetOrdinalExclusive) ||
|
|
1222
|
+
descriptor.endTargetOrdinalExclusive <= descriptor.startTargetOrdinal ||
|
|
1223
|
+
descriptor.endTargetOrdinalExclusive > options.targetCount
|
|
1224
|
+
) {
|
|
1225
|
+
throw new Error(`Calibration tree descriptor ${expectedPart} has an invalid target range.`);
|
|
1226
|
+
}
|
|
1227
|
+
nextTargetOrdinal = descriptor.endTargetOrdinalExclusive;
|
|
1228
|
+
});
|
|
1229
|
+
if (nextTargetOrdinal !== options.targetCount) {
|
|
1230
|
+
throw new Error(
|
|
1231
|
+
`Calibration tree ${options.label} descriptors do not cover every target.`,
|
|
1232
|
+
);
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
function validateBuild(
|
|
1237
|
+
value: unknown,
|
|
1238
|
+
country: MicrocosmCountry,
|
|
1239
|
+
buildArtifactId: string,
|
|
1240
|
+
): void {
|
|
1241
|
+
const build = record(value, "Calibration tree build");
|
|
1242
|
+
if (build.buildArtifactId !== buildArtifactId) {
|
|
1243
|
+
throw new Error("Calibration tree build identity does not match its parts.");
|
|
1244
|
+
}
|
|
1245
|
+
if (!["release", "staging", "comparison"].includes(String(build.kind))) {
|
|
1246
|
+
throw new Error("Calibration tree build kind is invalid.");
|
|
1247
|
+
}
|
|
1248
|
+
for (const key of ["sourceId", "label"] as const) {
|
|
1249
|
+
if (typeof build[key] !== "string" || !build[key]) {
|
|
1250
|
+
throw new Error(`Calibration tree build ${key} is missing.`);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
if (build.releaseId != null && typeof build.releaseId !== "string") {
|
|
1254
|
+
throw new Error("Calibration tree build release id is invalid.");
|
|
1255
|
+
}
|
|
1256
|
+
if (build.hfRepo != null && (typeof build.hfRepo !== "string" || !build.hfRepo)) {
|
|
1257
|
+
throw new Error("Calibration tree build repository is invalid.");
|
|
1258
|
+
}
|
|
1259
|
+
if (
|
|
1260
|
+
build.hfCommitSha != null &&
|
|
1261
|
+
(typeof build.hfCommitSha !== "string" || !HF_COMMIT_RE.test(build.hfCommitSha))
|
|
1262
|
+
) {
|
|
1263
|
+
throw new Error("Calibration tree build commit is invalid.");
|
|
1264
|
+
}
|
|
1265
|
+
if (
|
|
1266
|
+
build.createdAt != null &&
|
|
1267
|
+
(typeof build.createdAt !== "string" || !Number.isFinite(Date.parse(build.createdAt)))
|
|
1268
|
+
) {
|
|
1269
|
+
throw new Error("Calibration tree build creation time is invalid.");
|
|
1270
|
+
}
|
|
1271
|
+
const sources = record(build.sourceArtifacts, "Calibration tree source artifacts");
|
|
1272
|
+
for (const key of [
|
|
1273
|
+
"calibrationDiagnostics",
|
|
1274
|
+
"buildManifest",
|
|
1275
|
+
"releaseManifest",
|
|
1276
|
+
"demographics",
|
|
1277
|
+
"stagingRunManifest",
|
|
1278
|
+
"stagingProgress",
|
|
1279
|
+
"stagedDatasetReceipt",
|
|
1280
|
+
"comparisonCurrentIndex",
|
|
1281
|
+
"comparisonCandidateIndex",
|
|
1282
|
+
]) {
|
|
1283
|
+
if (sources[key] == null) {
|
|
1284
|
+
if (key === "calibrationDiagnostics" && build.kind !== "comparison") {
|
|
1285
|
+
throw new Error("Calibration diagnostics source is missing.");
|
|
1286
|
+
}
|
|
1287
|
+
if (
|
|
1288
|
+
build.kind === "comparison" &&
|
|
1289
|
+
(key === "comparisonCurrentIndex" ||
|
|
1290
|
+
key === "comparisonCandidateIndex")
|
|
1291
|
+
) {
|
|
1292
|
+
throw new Error(`Calibration comparison source ${key} is missing.`);
|
|
1293
|
+
}
|
|
1294
|
+
continue;
|
|
1295
|
+
}
|
|
1296
|
+
const source = record(sources[key], `Calibration tree source ${key}`);
|
|
1297
|
+
if (
|
|
1298
|
+
typeof source.path !== "string" ||
|
|
1299
|
+
!source.path ||
|
|
1300
|
+
source.path.startsWith("/") ||
|
|
1301
|
+
source.path.split("/").includes("..")
|
|
1302
|
+
) {
|
|
1303
|
+
throw new Error(`Calibration tree source ${key} has an invalid path.`);
|
|
1304
|
+
}
|
|
1305
|
+
if (typeof source.sha256 !== "string" || !SHA256_RE.test(source.sha256)) {
|
|
1306
|
+
throw new Error(`Calibration tree source ${key} has an invalid hash.`);
|
|
1307
|
+
}
|
|
1308
|
+
if (
|
|
1309
|
+
source.hfRepo != null &&
|
|
1310
|
+
(typeof source.hfRepo !== "string" || !source.hfRepo)
|
|
1311
|
+
) {
|
|
1312
|
+
throw new Error(`Calibration tree source ${key} has an invalid repository.`);
|
|
1313
|
+
}
|
|
1314
|
+
if (
|
|
1315
|
+
source.hfCommitSha != null &&
|
|
1316
|
+
(typeof source.hfCommitSha !== "string" || !HF_COMMIT_RE.test(source.hfCommitSha))
|
|
1317
|
+
) {
|
|
1318
|
+
throw new Error(`Calibration tree source ${key} has an invalid commit.`);
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
if (!isCountry(country)) throw new Error("Calibration tree build country is invalid.");
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
export function parseCalibrationTreeIndex(value: unknown): CalibrationTreeIndexArtifact {
|
|
1325
|
+
const index = validateIdentity(value, "index");
|
|
1326
|
+
const country = index.country as MicrocosmCountry;
|
|
1327
|
+
const buildArtifactId = index.buildArtifactId as string;
|
|
1328
|
+
validateBuild(index.build, country, buildArtifactId);
|
|
1329
|
+
validateComparisonMetadata(index.targetComparison);
|
|
1330
|
+
if (index.comparison != null) {
|
|
1331
|
+
const comparison = record(index.comparison, "Calibration tree comparison result");
|
|
1332
|
+
for (const key of [
|
|
1333
|
+
"pairArtifactId",
|
|
1334
|
+
"currentBuildArtifactId",
|
|
1335
|
+
"candidateBuildArtifactId",
|
|
1336
|
+
] as const) {
|
|
1337
|
+
if (typeof comparison[key] !== "string" || !SHA256_RE.test(comparison[key])) {
|
|
1338
|
+
throw new Error(`Calibration tree comparison ${key} is invalid.`);
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
if (comparison.mode !== "reported" && comparison.mode !== "shared") {
|
|
1342
|
+
throw new Error("Calibration tree comparison mode is invalid.");
|
|
1343
|
+
}
|
|
1344
|
+
if (typeof comparison.available !== "boolean") {
|
|
1345
|
+
throw new Error("Calibration tree comparison availability is invalid.");
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
if (typeof index.lossAttributionAvailable !== "boolean") {
|
|
1349
|
+
throw new Error("Calibration tree loss-attribution availability must be boolean.");
|
|
1350
|
+
}
|
|
1351
|
+
if (!Number.isSafeInteger(index.targetCount) || (index.targetCount as number) < 0) {
|
|
1352
|
+
throw new Error("Calibration tree target count is invalid.");
|
|
1353
|
+
}
|
|
1354
|
+
const targetCount = index.targetCount as number;
|
|
1355
|
+
if (!Number.isSafeInteger(index.maxDepth) || (index.maxDepth as number) < 0) {
|
|
1356
|
+
throw new Error("Calibration tree maximum depth is invalid.");
|
|
1357
|
+
}
|
|
1358
|
+
const filters = record(index.filterOptions, "Calibration tree filter options");
|
|
1359
|
+
for (const key of [
|
|
1360
|
+
"geographyLevels",
|
|
1361
|
+
"geographies",
|
|
1362
|
+
"fitBands",
|
|
1363
|
+
"comparisonFits",
|
|
1364
|
+
"calibrationStatuses",
|
|
1365
|
+
]) {
|
|
1366
|
+
stringArray(filters[key], `Calibration tree filter option ${key}`);
|
|
1367
|
+
}
|
|
1368
|
+
const levels = validateLevelRecords(
|
|
1369
|
+
index.levels,
|
|
1370
|
+
index.targetCount as number,
|
|
1371
|
+
"Calibration tree root levels",
|
|
1372
|
+
);
|
|
1373
|
+
const roots = record(index.roots, "Calibration tree roots");
|
|
1374
|
+
for (const rootName of ["program", "geography"] as const) {
|
|
1375
|
+
if (typeof roots[rootName] !== "string" || !levels[roots[rootName] as string]) {
|
|
1376
|
+
throw new Error(`Calibration tree ${rootName} root is invalid.`);
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
const parts = record(index.parts, "Calibration tree part descriptors");
|
|
1380
|
+
if (!Array.isArray(parts.tiers)) {
|
|
1381
|
+
throw new Error("Calibration tree tier descriptors must be an array.");
|
|
1382
|
+
}
|
|
1383
|
+
const maxDepth = index.maxDepth as number;
|
|
1384
|
+
if (parts.tiers.length !== maxDepth) {
|
|
1385
|
+
throw new Error("Calibration tree tier descriptors are not contiguous.");
|
|
1386
|
+
}
|
|
1387
|
+
parts.tiers.forEach((item, tierIndex) => {
|
|
1388
|
+
const expectedDepth = tierIndex + 1;
|
|
1389
|
+
const expectedPart = `tier-${expectedDepth}` as CalibrationTreeTierPart;
|
|
1390
|
+
const descriptor = validateDescriptor(
|
|
1391
|
+
item,
|
|
1392
|
+
expectedPart,
|
|
1393
|
+
country,
|
|
1394
|
+
buildArtifactId,
|
|
1395
|
+
);
|
|
1396
|
+
const tier = descriptor as CalibrationTreeTierDescriptor;
|
|
1397
|
+
if (tier.depth !== expectedDepth) {
|
|
1398
|
+
throw new Error(`Calibration tree descriptor ${expectedPart} has the wrong depth.`);
|
|
1399
|
+
}
|
|
1400
|
+
stringArray(tier.levelIds, `Calibration tree descriptor ${expectedPart} levelIds`);
|
|
1401
|
+
});
|
|
1402
|
+
validateDescriptor(parts.filterIndex, "filter-index", country, buildArtifactId);
|
|
1403
|
+
validateTargetShardDescriptors({
|
|
1404
|
+
value: parts.targetSummaries,
|
|
1405
|
+
targetCount,
|
|
1406
|
+
country,
|
|
1407
|
+
buildArtifactId,
|
|
1408
|
+
label: "target-summary",
|
|
1409
|
+
maxRawBytes: CALIBRATION_TREE_TARGET_SUMMARY_MAX_RAW_BYTES,
|
|
1410
|
+
partForIndex: targetSummaryPart,
|
|
1411
|
+
});
|
|
1412
|
+
const expectedDetailStrategy =
|
|
1413
|
+
(index.build as CalibrationTreeBuild).kind === "comparison"
|
|
1414
|
+
? "source-targets"
|
|
1415
|
+
: "shards";
|
|
1416
|
+
if (parts.targetDetailStrategy !== expectedDetailStrategy) {
|
|
1417
|
+
throw new Error("Calibration tree target-detail strategy is inconsistent with its build.");
|
|
1418
|
+
}
|
|
1419
|
+
if (expectedDetailStrategy === "source-targets") {
|
|
1420
|
+
if (!Array.isArray(parts.targetDetails) || parts.targetDetails.length !== 0) {
|
|
1421
|
+
throw new Error("Comparison trees cannot publish target-detail shards.");
|
|
1422
|
+
}
|
|
1423
|
+
} else {
|
|
1424
|
+
validateTargetShardDescriptors({
|
|
1425
|
+
value: parts.targetDetails,
|
|
1426
|
+
targetCount,
|
|
1427
|
+
country,
|
|
1428
|
+
buildArtifactId,
|
|
1429
|
+
label: "target-detail",
|
|
1430
|
+
maxRawBytes: CALIBRATION_TREE_TARGET_DETAIL_MAX_RAW_BYTES,
|
|
1431
|
+
partForIndex: targetDetailsPart,
|
|
1432
|
+
});
|
|
1433
|
+
}
|
|
1434
|
+
return value as CalibrationTreeIndexArtifact;
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
export function parseCalibrationTreeTier(
|
|
1438
|
+
value: unknown,
|
|
1439
|
+
expectedPart?: CalibrationTreeTierPart,
|
|
1440
|
+
): CalibrationTreeTierArtifact {
|
|
1441
|
+
const tier = validateIdentity(value, expectedPart);
|
|
1442
|
+
const match = String(tier.part).match(TIER_PART_RE);
|
|
1443
|
+
if (!match) throw new Error("Calibration tree tier part name is invalid.");
|
|
1444
|
+
const depth = Number(match[1]);
|
|
1445
|
+
if (tier.depth !== depth) throw new Error(`Calibration tree tier-${depth} has the wrong depth.`);
|
|
1446
|
+
if (!Number.isSafeInteger(tier.targetCount) || (tier.targetCount as number) < 0) {
|
|
1447
|
+
throw new Error(`Calibration tree tier-${depth} has an invalid target count.`);
|
|
1448
|
+
}
|
|
1449
|
+
validateLevelRecords(
|
|
1450
|
+
tier.levels,
|
|
1451
|
+
tier.targetCount as number,
|
|
1452
|
+
`Calibration tree tier-${depth} levels`,
|
|
1453
|
+
);
|
|
1454
|
+
return value as CalibrationTreeTierArtifact;
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
function validateMetricInputs(value: unknown, label: string): void {
|
|
1458
|
+
const inputs = record(value, label);
|
|
1459
|
+
for (const key of [
|
|
1460
|
+
"absRelativeError",
|
|
1461
|
+
"targetLossWeightShare",
|
|
1462
|
+
"finalLossContribution",
|
|
1463
|
+
"targetChange",
|
|
1464
|
+
]) {
|
|
1465
|
+
if (inputs[key] != null &&
|
|
1466
|
+
(typeof inputs[key] !== "number" || !Number.isFinite(inputs[key]))) {
|
|
1467
|
+
throw new Error(`${label}.${key} must be a finite number or null.`);
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
if (
|
|
1471
|
+
inputs.comparisonStatus != null &&
|
|
1472
|
+
!["shared", "added", "removed"].includes(String(inputs.comparisonStatus))
|
|
1473
|
+
) {
|
|
1474
|
+
throw new Error(`${label}.comparisonStatus is invalid.`);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
function validateComparisonMetadata(value: unknown): void {
|
|
1479
|
+
const comparison = record(value, "Calibration tree comparison metadata");
|
|
1480
|
+
if (typeof comparison.releaseId !== "string" || !comparison.releaseId) {
|
|
1481
|
+
throw new Error("Calibration tree comparison release id is missing.");
|
|
1482
|
+
}
|
|
1483
|
+
if (
|
|
1484
|
+
!["reported", "exact_reconstructed", "derived", "unavailable"].includes(
|
|
1485
|
+
String(comparison.status),
|
|
1486
|
+
)
|
|
1487
|
+
) {
|
|
1488
|
+
throw new Error("Calibration tree comparison attribution status is invalid.");
|
|
1489
|
+
}
|
|
1490
|
+
for (const key of ["aggregate", "cap"] as const) {
|
|
1491
|
+
if (
|
|
1492
|
+
comparison[key] != null &&
|
|
1493
|
+
(typeof comparison[key] !== "number" || !Number.isFinite(comparison[key]))
|
|
1494
|
+
) {
|
|
1495
|
+
throw new Error(`Calibration tree comparison ${key} is invalid.`);
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
if (
|
|
1499
|
+
comparison.basisIdentifier != null &&
|
|
1500
|
+
typeof comparison.basisIdentifier !== "string"
|
|
1501
|
+
) {
|
|
1502
|
+
throw new Error("Calibration tree comparison basis identifier is invalid.");
|
|
1503
|
+
}
|
|
1504
|
+
if (
|
|
1505
|
+
!["legacy", "structured", "hierarchy", "mixed", "unknown"].includes(
|
|
1506
|
+
String(comparison.targetRepresentation),
|
|
1507
|
+
)
|
|
1508
|
+
) {
|
|
1509
|
+
throw new Error("Calibration tree comparison target representation is invalid.");
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
function validatePostings(
|
|
1514
|
+
value: unknown,
|
|
1515
|
+
targetCount: number,
|
|
1516
|
+
label: string,
|
|
1517
|
+
): void {
|
|
1518
|
+
if (!Array.isArray(value)) throw new Error(`${label} must be an array.`);
|
|
1519
|
+
const seen = new Set<string>();
|
|
1520
|
+
for (const [index, item] of value.entries()) {
|
|
1521
|
+
const posting = record(item, `${label}[${index}]`);
|
|
1522
|
+
if (posting.value !== null && typeof posting.value !== "string") {
|
|
1523
|
+
throw new Error(`${label}[${index}].value must be a string or null.`);
|
|
1524
|
+
}
|
|
1525
|
+
const key = posting.value === null ? "null" : `string:${posting.value}`;
|
|
1526
|
+
if (seen.has(key)) throw new Error(`${label} contains a duplicate value.`);
|
|
1527
|
+
seen.add(key);
|
|
1528
|
+
validateIndices(posting.targetOrdinals, targetCount, `${label}[${index}].targetOrdinals`);
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
export function parseCalibrationTreeFilterIndex(
|
|
1533
|
+
value: unknown,
|
|
1534
|
+
): CalibrationTreeFilterIndexArtifact {
|
|
1535
|
+
const part = validateIdentity(value, "filter-index");
|
|
1536
|
+
if (!Number.isSafeInteger(part.targetCount) || (part.targetCount as number) < 0) {
|
|
1537
|
+
throw new Error("Calibration tree filter index has an invalid target count.");
|
|
1538
|
+
}
|
|
1539
|
+
const postings = record(part.postings, "Calibration tree filter postings");
|
|
1540
|
+
for (const key of [
|
|
1541
|
+
"geographyLevels",
|
|
1542
|
+
"geographies",
|
|
1543
|
+
"fitBands",
|
|
1544
|
+
"comparisonFits",
|
|
1545
|
+
"calibrationStatuses",
|
|
1546
|
+
]) {
|
|
1547
|
+
validatePostings(
|
|
1548
|
+
postings[key],
|
|
1549
|
+
part.targetCount as number,
|
|
1550
|
+
`Calibration tree filter postings ${key}`,
|
|
1551
|
+
);
|
|
1552
|
+
}
|
|
1553
|
+
return value as CalibrationTreeFilterIndexArtifact;
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
export function parseCalibrationTreeTargetSummary(
|
|
1557
|
+
value: unknown,
|
|
1558
|
+
expectedPart?: CalibrationTreeTargetSummaryPart,
|
|
1559
|
+
): CalibrationTreeTargetSummaryArtifact {
|
|
1560
|
+
const part = validateIdentity(value, expectedPart);
|
|
1561
|
+
if (typeof part.part !== "string" || !TARGET_SUMMARY_PART_RE.test(part.part)) {
|
|
1562
|
+
throw new Error("Calibration tree target-summary part name is invalid.");
|
|
1563
|
+
}
|
|
1564
|
+
if (
|
|
1565
|
+
!Number.isSafeInteger(part.startTargetOrdinal) ||
|
|
1566
|
+
(part.startTargetOrdinal as number) < 0 ||
|
|
1567
|
+
!Number.isSafeInteger(part.endTargetOrdinalExclusive) ||
|
|
1568
|
+
(part.endTargetOrdinalExclusive as number) <= (part.startTargetOrdinal as number)
|
|
1569
|
+
) {
|
|
1570
|
+
throw new Error("Calibration tree target-summary range is invalid.");
|
|
1571
|
+
}
|
|
1572
|
+
if (!Array.isArray(part.targets)) {
|
|
1573
|
+
throw new Error("Calibration tree target summaries must be an array.");
|
|
1574
|
+
}
|
|
1575
|
+
if (
|
|
1576
|
+
part.targets.length !==
|
|
1577
|
+
(part.endTargetOrdinalExclusive as number) - (part.startTargetOrdinal as number)
|
|
1578
|
+
) {
|
|
1579
|
+
throw new Error("Calibration tree target-summary count does not match its range.");
|
|
1580
|
+
}
|
|
1581
|
+
const ids = new Set<string>();
|
|
1582
|
+
part.targets.forEach((item, index) => {
|
|
1583
|
+
const target = record(item, `Calibration tree target summary ${index}`);
|
|
1584
|
+
if (typeof target.id !== "string" || !target.id || typeof target.label !== "string") {
|
|
1585
|
+
throw new Error(`Calibration tree target summary ${index} has invalid identity fields.`);
|
|
1586
|
+
}
|
|
1587
|
+
if (ids.has(target.id)) throw new Error(`Calibration target id ${target.id} is duplicated.`);
|
|
1588
|
+
ids.add(target.id);
|
|
1589
|
+
validateMetricInputs(target.metricInputs, `Calibration tree target summary ${index}`);
|
|
1590
|
+
const comparison = record(
|
|
1591
|
+
target.comparison,
|
|
1592
|
+
`Calibration tree target summary ${index} comparison`,
|
|
1593
|
+
);
|
|
1594
|
+
for (const key of [
|
|
1595
|
+
"normalizedBaseName",
|
|
1596
|
+
"chronicleFactKey",
|
|
1597
|
+
"structuredIdentity",
|
|
1598
|
+
] as const) {
|
|
1599
|
+
if (comparison[key] != null && typeof comparison[key] !== "string") {
|
|
1600
|
+
throw new Error(
|
|
1601
|
+
`Calibration tree target summary ${index} comparison ${key} is invalid.`,
|
|
1602
|
+
);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
if (!['legacy', 'structured', 'hierarchy'].includes(String(comparison.representation))) {
|
|
1606
|
+
throw new Error(
|
|
1607
|
+
`Calibration tree target summary ${index} comparison representation is invalid.`,
|
|
1608
|
+
);
|
|
1609
|
+
}
|
|
1610
|
+
record(
|
|
1611
|
+
comparison.row,
|
|
1612
|
+
`Calibration tree target summary ${index} comparison row`,
|
|
1613
|
+
);
|
|
1614
|
+
if (target.detailSource != null) {
|
|
1615
|
+
const source = record(
|
|
1616
|
+
target.detailSource,
|
|
1617
|
+
`Calibration tree target summary ${index} detail source`,
|
|
1618
|
+
);
|
|
1619
|
+
for (const key of ["currentTargetOrdinal", "candidateTargetOrdinal"] as const) {
|
|
1620
|
+
if (
|
|
1621
|
+
source[key] != null &&
|
|
1622
|
+
(!Number.isSafeInteger(source[key]) || (source[key] as number) < 0)
|
|
1623
|
+
) {
|
|
1624
|
+
throw new Error(
|
|
1625
|
+
`Calibration tree target summary ${index} detail source ${key} is invalid.`,
|
|
1626
|
+
);
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
});
|
|
1631
|
+
return value as CalibrationTreeTargetSummaryArtifact;
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
export function parseCalibrationTreeTargetDetails(
|
|
1635
|
+
value: unknown,
|
|
1636
|
+
expectedPart?: CalibrationTreeTargetDetailsPart,
|
|
1637
|
+
): CalibrationTreeTargetDetailsArtifact {
|
|
1638
|
+
const part = validateIdentity(value, expectedPart);
|
|
1639
|
+
if (typeof part.part !== "string" || !TARGET_DETAILS_PART_RE.test(part.part)) {
|
|
1640
|
+
throw new Error("Calibration tree target-detail part name is invalid.");
|
|
1641
|
+
}
|
|
1642
|
+
if (
|
|
1643
|
+
!Number.isSafeInteger(part.startTargetOrdinal) ||
|
|
1644
|
+
(part.startTargetOrdinal as number) < 0 ||
|
|
1645
|
+
!Number.isSafeInteger(part.endTargetOrdinalExclusive) ||
|
|
1646
|
+
(part.endTargetOrdinalExclusive as number) <= (part.startTargetOrdinal as number)
|
|
1647
|
+
) {
|
|
1648
|
+
throw new Error("Calibration tree target-detail range is invalid.");
|
|
1649
|
+
}
|
|
1650
|
+
if (!Array.isArray(part.targets)) {
|
|
1651
|
+
throw new Error("Calibration tree target details must be an array.");
|
|
1652
|
+
}
|
|
1653
|
+
if (
|
|
1654
|
+
part.targets.length !==
|
|
1655
|
+
(part.endTargetOrdinalExclusive as number) - (part.startTargetOrdinal as number)
|
|
1656
|
+
) {
|
|
1657
|
+
throw new Error("Calibration tree target-detail count does not match its range.");
|
|
1658
|
+
}
|
|
1659
|
+
part.targets.forEach((item, index) => {
|
|
1660
|
+
record(item, `Calibration tree target detail ${index}`);
|
|
1661
|
+
});
|
|
1662
|
+
return value as CalibrationTreeTargetDetailsArtifact;
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
export function parseCalibrationTreeComparisonTargetDetail(
|
|
1666
|
+
value: unknown,
|
|
1667
|
+
): CalibrationTreeComparisonTargetDetailResponse {
|
|
1668
|
+
const response = record(value, "Calibration comparison target detail");
|
|
1669
|
+
if (response.schemaVersion !== CALIBRATION_TREE_SCHEMA_VERSION) {
|
|
1670
|
+
throw new Error(
|
|
1671
|
+
`Unsupported calibration tree schema ${String(response.schemaVersion)}.`,
|
|
1672
|
+
);
|
|
1673
|
+
}
|
|
1674
|
+
if (typeof response.country !== "string" || !isCountry(response.country)) {
|
|
1675
|
+
throw new Error("Calibration comparison target detail has an unknown country.");
|
|
1676
|
+
}
|
|
1677
|
+
if (
|
|
1678
|
+
typeof response.buildArtifactId !== "string" ||
|
|
1679
|
+
!SHA256_RE.test(response.buildArtifactId)
|
|
1680
|
+
) {
|
|
1681
|
+
throw new Error("Calibration comparison target detail has an invalid build id.");
|
|
1682
|
+
}
|
|
1683
|
+
if (!Number.isSafeInteger(response.targetOrdinal) || (response.targetOrdinal as number) < 0) {
|
|
1684
|
+
throw new Error("Calibration comparison target detail has an invalid target ordinal.");
|
|
1685
|
+
}
|
|
1686
|
+
if (typeof response.targetId !== "string" || !response.targetId) {
|
|
1687
|
+
throw new Error("Calibration comparison target detail has an invalid target id.");
|
|
1688
|
+
}
|
|
1689
|
+
record(response.target, "Calibration comparison target detail target");
|
|
1690
|
+
return value as CalibrationTreeComparisonTargetDetailResponse;
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
export function parseCalibrationTreePart(
|
|
1694
|
+
value: unknown,
|
|
1695
|
+
expectedPart?: CalibrationTreePart,
|
|
1696
|
+
): CalibrationTreeArtifactPart {
|
|
1697
|
+
const identity = validateIdentity(value, expectedPart);
|
|
1698
|
+
if (identity.part === "index") return parseCalibrationTreeIndex(value);
|
|
1699
|
+
if (identity.part === "filter-index") return parseCalibrationTreeFilterIndex(value);
|
|
1700
|
+
if (TARGET_SUMMARY_PART_RE.test(String(identity.part))) {
|
|
1701
|
+
return parseCalibrationTreeTargetSummary(
|
|
1702
|
+
value,
|
|
1703
|
+
identity.part as CalibrationTreeTargetSummaryPart,
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
if (TARGET_DETAILS_PART_RE.test(String(identity.part))) {
|
|
1707
|
+
return parseCalibrationTreeTargetDetails(
|
|
1708
|
+
value,
|
|
1709
|
+
identity.part as CalibrationTreeTargetDetailsPart,
|
|
1710
|
+
);
|
|
1711
|
+
}
|
|
1712
|
+
return parseCalibrationTreeTier(value, identity.part as CalibrationTreeTierPart);
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
function sameSelection(
|
|
1716
|
+
node: CompiledCalibrationNode,
|
|
1717
|
+
selection: ExplorerNodeSelection,
|
|
1718
|
+
): boolean {
|
|
1719
|
+
if (node.kind !== selection.kind) return false;
|
|
1720
|
+
if (selection.kind === "program") {
|
|
1721
|
+
return (
|
|
1722
|
+
node.selection.kind === "program" &&
|
|
1723
|
+
node.selection.source === selection.source &&
|
|
1724
|
+
node.selection.value === selection.value
|
|
1725
|
+
);
|
|
1726
|
+
}
|
|
1727
|
+
if (selection.kind === "dimension_value") {
|
|
1728
|
+
return (
|
|
1729
|
+
node.selection.kind === "dimension_value" &&
|
|
1730
|
+
node.selection.key === selection.key &&
|
|
1731
|
+
node.selection.value === selection.value
|
|
1732
|
+
);
|
|
1733
|
+
}
|
|
1734
|
+
return node.selection.value === selection.value;
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
function selectedPath(state: ExplorerState): ExplorerNodeSelection[] {
|
|
1738
|
+
const selections: ExplorerNodeSelection[] = [];
|
|
1739
|
+
if (state.breakdown === "geography" && state.path.geography) {
|
|
1740
|
+
selections.push({ kind: "geography", value: state.path.geography });
|
|
1741
|
+
}
|
|
1742
|
+
if (state.path.source && state.path.program) {
|
|
1743
|
+
selections.push({
|
|
1744
|
+
kind: "program",
|
|
1745
|
+
source: state.path.source,
|
|
1746
|
+
value: state.path.program,
|
|
1747
|
+
});
|
|
1748
|
+
}
|
|
1749
|
+
if (state.breakdown === "program" && state.path.geography) {
|
|
1750
|
+
selections.push({ kind: "geography", value: state.path.geography });
|
|
1751
|
+
}
|
|
1752
|
+
selections.push(
|
|
1753
|
+
...state.path.dimensions.map((dimension) => ({
|
|
1754
|
+
kind: "dimension_value" as const,
|
|
1755
|
+
key: dimension.key,
|
|
1756
|
+
label: dimension.label ?? dimension.key,
|
|
1757
|
+
value: dimension.value,
|
|
1758
|
+
})),
|
|
1759
|
+
);
|
|
1760
|
+
return selections;
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
function membershipForFilters(
|
|
1764
|
+
filterIndex: CalibrationTreeFilterIndexArtifact,
|
|
1765
|
+
filters: ExplorerState["filters"],
|
|
1766
|
+
): Uint8Array {
|
|
1767
|
+
const membership = new Uint8Array(filterIndex.targetCount);
|
|
1768
|
+
membership.fill(1);
|
|
1769
|
+
for (const [key, selected] of Object.entries(filters) as Array<
|
|
1770
|
+
[keyof ExplorerState["filters"], string[]]
|
|
1771
|
+
>) {
|
|
1772
|
+
if (!selected.length) continue;
|
|
1773
|
+
const category = new Uint8Array(filterIndex.targetCount);
|
|
1774
|
+
const postings = filterIndex.postings[key] as CalibrationTreePosting[];
|
|
1775
|
+
for (const posting of postings) {
|
|
1776
|
+
if (posting.value != null && selected.includes(posting.value)) {
|
|
1777
|
+
for (const targetOrdinal of posting.targetOrdinals) category[targetOrdinal] = 1;
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
for (let index = 0; index < membership.length; index += 1) {
|
|
1781
|
+
membership[index] &= category[index];
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
return membership;
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
export function calibrationTreeTargetsFromSummaries(
|
|
1788
|
+
index: CalibrationTreeIndexArtifact,
|
|
1789
|
+
summaries: CalibrationTreeTargetSummaryArtifact[],
|
|
1790
|
+
): CalibrationTreeTargetSummary[] {
|
|
1791
|
+
if (summaries.length !== index.parts.targetSummaries.length) {
|
|
1792
|
+
throw new Error("Calibration tree target-summary shards are incomplete.");
|
|
1793
|
+
}
|
|
1794
|
+
const targets: CalibrationTreeTargetSummary[] = [];
|
|
1795
|
+
const ids = new Set<string>();
|
|
1796
|
+
summaries.forEach((summary, shardIndex) => {
|
|
1797
|
+
const descriptor = index.parts.targetSummaries[shardIndex];
|
|
1798
|
+
if (
|
|
1799
|
+
!descriptor ||
|
|
1800
|
+
summary.part !== descriptor.part ||
|
|
1801
|
+
summary.country !== index.country ||
|
|
1802
|
+
summary.buildArtifactId !== index.buildArtifactId ||
|
|
1803
|
+
summary.startTargetOrdinal !== descriptor.startTargetOrdinal ||
|
|
1804
|
+
summary.endTargetOrdinalExclusive !== descriptor.endTargetOrdinalExclusive
|
|
1805
|
+
) {
|
|
1806
|
+
throw new Error(
|
|
1807
|
+
`Calibration tree target-summary shard ${shardIndex + 1} has an unexpected range or identity.`,
|
|
1808
|
+
);
|
|
1809
|
+
}
|
|
1810
|
+
for (const target of summary.targets) {
|
|
1811
|
+
if (ids.has(target.id)) {
|
|
1812
|
+
throw new Error(`Calibration target id ${target.id} is duplicated across summary shards.`);
|
|
1813
|
+
}
|
|
1814
|
+
ids.add(target.id);
|
|
1815
|
+
targets.push(target);
|
|
1816
|
+
}
|
|
1817
|
+
});
|
|
1818
|
+
if (targets.length !== index.targetCount) {
|
|
1819
|
+
throw new Error("Calibration tree target-summary shards do not match the target count.");
|
|
1820
|
+
}
|
|
1821
|
+
return targets;
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
function targetDetailForOrdinal(
|
|
1825
|
+
shard: CalibrationTreeTargetDetailsArtifact | undefined,
|
|
1826
|
+
selected: LoadedCalibrationTreeBundle["selectedTargetDetail"],
|
|
1827
|
+
targetOrdinal: number,
|
|
1828
|
+
): CalibrationTreeTarget | undefined {
|
|
1829
|
+
if (selected?.targetOrdinal === targetOrdinal) return selected.target;
|
|
1830
|
+
if (
|
|
1831
|
+
!shard ||
|
|
1832
|
+
targetOrdinal < shard.startTargetOrdinal ||
|
|
1833
|
+
targetOrdinal >= shard.endTargetOrdinalExclusive
|
|
1834
|
+
) {
|
|
1835
|
+
return undefined;
|
|
1836
|
+
}
|
|
1837
|
+
return shard.targets[targetOrdinal - shard.startTargetOrdinal];
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
function loadedLevels(bundle: LoadedCalibrationTreeBundle) {
|
|
1841
|
+
return Object.assign(
|
|
1842
|
+
{},
|
|
1843
|
+
bundle.index.levels,
|
|
1844
|
+
...bundle.tiers.map((tier) => tier.levels),
|
|
1845
|
+
) as Record<string, CompiledCalibrationLevel>;
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
export function calibrationTreeResponseFromBundle(
|
|
1849
|
+
bundle: LoadedCalibrationTreeBundle,
|
|
1850
|
+
state: ExplorerState,
|
|
1851
|
+
): CalibrationTreeResponse | null {
|
|
1852
|
+
const levels = loadedLevels(bundle);
|
|
1853
|
+
let levelId = bundle.index.roots[state.breakdown];
|
|
1854
|
+
for (const selection of selectedPath(state)) {
|
|
1855
|
+
const level = levels[levelId];
|
|
1856
|
+
if (!level) return null;
|
|
1857
|
+
const node = level.groups
|
|
1858
|
+
.flatMap((group) => group.nodes)
|
|
1859
|
+
.find((candidate) => sameSelection(candidate, selection));
|
|
1860
|
+
if (!node || node.kind === "target") {
|
|
1861
|
+
throw new Error(`Calibration tree does not contain the selected ${selection.kind}.`);
|
|
1862
|
+
}
|
|
1863
|
+
levelId = node.nextLevelId;
|
|
1864
|
+
}
|
|
1865
|
+
const level = levels[levelId];
|
|
1866
|
+
if (!level) return null;
|
|
1867
|
+
|
|
1868
|
+
const hasFilters = Object.values(state.filters).some((values) => values.length > 0);
|
|
1869
|
+
if (
|
|
1870
|
+
bundle.filterIndex &&
|
|
1871
|
+
bundle.filterIndex.targetCount !== bundle.index.targetCount
|
|
1872
|
+
) {
|
|
1873
|
+
throw new Error("Calibration tree filter index does not match the target count.");
|
|
1874
|
+
}
|
|
1875
|
+
const targets = bundle.targetSummaries;
|
|
1876
|
+
if (hasFilters && (!bundle.filterIndex || !targets)) return null;
|
|
1877
|
+
const membership = bundle.filterIndex
|
|
1878
|
+
? membershipForFilters(bundle.filterIndex, state.filters)
|
|
1879
|
+
: null;
|
|
1880
|
+
const filterIndices = (indices: number[]) =>
|
|
1881
|
+
membership ? indices.filter((index) => membership[index] === 1) : indices;
|
|
1882
|
+
const metrics = (indices: number[], fallback: CalibrationTreeMetrics) =>
|
|
1883
|
+
hasFilters && targets
|
|
1884
|
+
? calibrationTreeMetricsFromInputs(
|
|
1885
|
+
indices.map((index) => targets[index].metricInputs),
|
|
1886
|
+
)
|
|
1887
|
+
: fallback;
|
|
1888
|
+
|
|
1889
|
+
const groups: CalibrationTreeGroup[] = level.groups.flatMap((group) => {
|
|
1890
|
+
const groupIndices = filterIndices(group.targetOrdinals);
|
|
1891
|
+
if (!groupIndices.length) return [];
|
|
1892
|
+
const nodes: CalibrationTreeNode[] = group.nodes.flatMap((node) => {
|
|
1893
|
+
const nodeIndices = filterIndices(node.targetOrdinals);
|
|
1894
|
+
if (!nodeIndices.length) return [];
|
|
1895
|
+
return [{
|
|
1896
|
+
id: node.id,
|
|
1897
|
+
label: node.label,
|
|
1898
|
+
kind: node.kind,
|
|
1899
|
+
selection: node.selection,
|
|
1900
|
+
metrics: metrics(nodeIndices, node.metrics),
|
|
1901
|
+
target:
|
|
1902
|
+
node.kind === "target"
|
|
1903
|
+
? targetDetailForOrdinal(
|
|
1904
|
+
bundle.targetDetailShard,
|
|
1905
|
+
bundle.selectedTargetDetail,
|
|
1906
|
+
node.targetOrdinal,
|
|
1907
|
+
)
|
|
1908
|
+
: undefined,
|
|
1909
|
+
authored_label: node.authoredLabel,
|
|
1910
|
+
}];
|
|
1911
|
+
});
|
|
1912
|
+
if (!nodes.length) return [];
|
|
1913
|
+
return [{
|
|
1914
|
+
id: group.id,
|
|
1915
|
+
label: group.label,
|
|
1916
|
+
nodes,
|
|
1917
|
+
metrics: metrics(groupIndices, group.metrics),
|
|
1918
|
+
}];
|
|
1919
|
+
});
|
|
1920
|
+
const levelIndices = filterIndices(level.targetOrdinals);
|
|
1921
|
+
|
|
1922
|
+
return {
|
|
1923
|
+
releaseId: bundle.index.build.releaseId ?? bundle.index.build.sourceId,
|
|
1924
|
+
calibrationProvenance: bundle.index.calibrationProvenance,
|
|
1925
|
+
lossAttributionAvailable: bundle.index.lossAttributionAvailable,
|
|
1926
|
+
path: state.path,
|
|
1927
|
+
pathLabels: level.pathLabels,
|
|
1928
|
+
currentLevel: level.currentLevel,
|
|
1929
|
+
groups,
|
|
1930
|
+
dimensionOrder: level.dimensionOrder,
|
|
1931
|
+
filterOptions: bundle.index.filterOptions,
|
|
1932
|
+
filteredMetrics: metrics(levelIndices, level.metrics),
|
|
1933
|
+
};
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
export function isCalibrationTreePart(value: string): value is CalibrationTreePart {
|
|
1937
|
+
return value === "index" ||
|
|
1938
|
+
value === "filter-index" ||
|
|
1939
|
+
(TARGET_SUMMARY_PART_RE.test(value) && value !== "target-summary-0000") ||
|
|
1940
|
+
(TARGET_DETAILS_PART_RE.test(value) && value !== "target-details-0000") ||
|
|
1941
|
+
TIER_PART_RE.test(value);
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
export function calibrationTreePartDepth(part: CalibrationTreePart): number | null {
|
|
1945
|
+
const match = part.match(TIER_PART_RE);
|
|
1946
|
+
return match ? Number(match[1]) : null;
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
export function calibrationTreeBundlePrefix(
|
|
1950
|
+
country: MicrocosmCountry,
|
|
1951
|
+
buildArtifactId: string,
|
|
1952
|
+
): string {
|
|
1953
|
+
if (!SHA256_RE.test(buildArtifactId)) {
|
|
1954
|
+
throw new Error("Invalid calibration build artifact id.");
|
|
1955
|
+
}
|
|
1956
|
+
return `calibration-trees/${country}/${buildArtifactId}`;
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
export function calibrationTreePartPath(
|
|
1960
|
+
country: MicrocosmCountry,
|
|
1961
|
+
buildArtifactId: string,
|
|
1962
|
+
part: CalibrationTreePart,
|
|
1963
|
+
): string {
|
|
1964
|
+
if (!isCalibrationTreePart(part)) throw new Error("Invalid calibration tree part.");
|
|
1965
|
+
return `${calibrationTreeBundlePrefix(country, buildArtifactId)}/${part}.json.gz`;
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
export function isSha256(value: string): boolean {
|
|
1969
|
+
return SHA256_RE.test(value);
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
export function isHfCommitSha(value: string): boolean {
|
|
1973
|
+
return HF_COMMIT_RE.test(value);
|
|
1974
|
+
}
|