@policyengine/calibration-tree 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 PolicyEngine
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,59 @@
-# Temporary Holding Version
+# @policyengine/calibration-tree
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Builds, validates, reads and publishes Microcosm calibration tree artifacts
+(tree schema 6), with the release and staging readers they are built from.
+The PolicyEngine calibration dashboard reads trees with it, and the Microcosm
+collector builds them with it.
+
+## Using it
+
+The package ships TypeScript source, not compiled JavaScript. Each module is
+its own entry point, for example:
+
+```ts
+import { parseCalibrationTreeIndex } from "@policyengine/calibration-tree/calibration-tree-artifact";
+import { COUNTRY_REGISTRY } from "@policyengine/calibration-tree/countries";
+```
+
+- **Bun** runs the modules directly.
+- **Next.js** compiles them when the package is listed in `transpilePackages`
+  in `next.config.ts`.
+- **TypeScript** checks them with `moduleResolution` set to `bundler`. Some
+  modules use Bun's globals (`Bun.argv`, `import.meta.main`), so the checking
+  project needs `@types/bun`; `types/next-fetch-cache.d.ts` declares the
+  `next.revalidate` fetch option the readers pass, for projects without
+  Next.js.
+
+Plain Node.js cannot load the modules, because they are TypeScript.
+
+## The tree worker
+
+`src/calibration-tree-worker.ts` builds one release's tree from local copies
+of the files the publisher reads from Hugging Face, and writes each part's
+gzip bytes under an output directory:
+
+```bash
+bun run src/calibration-tree-worker.ts REQUEST.json OUTPUT_DIRECTORY
+```
+
+The module's opening comment describes the request. The worker writes the
+same bytes as the publisher (`src/publish-calibration-tree.ts`) for the same
+files, commit and publication time; the golden tests hold both to the trees
+recorded in `src/fixtures/calibration-tree-golden.json`. It exits 0 after
+printing a JSON description of the written parts, 1 when the request cannot be
+followed, 2 when the builder refuses the files, so another run on the same
+files cannot help, and 3 for anything else: a size limit that is not a
+positive integer, or an unexpected error such as a defect in the builder.
+
+## Development
+
+```bash
+bun install --frozen-lockfile
+bun test src/calibration-tree-worker.test.ts
+bun run typecheck
+```
+
+The source lives in
+[PolicyEngine/microcosm-collector](https://github.com/PolicyEngine/microcosm-collector)
+under `packages/calibration-tree`; its README describes how the package is
+released.

package/package.json

@@ -1,6 +1,73 @@
 {
   "name": "@policyengine/calibration-tree",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Builds, validates, reads and publishes Microcosm calibration tree artifacts (tree schema 6), with the release and staging readers they are built from.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/PolicyEngine/microcosm-collector.git",
+    "directory": "packages/calibration-tree"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "files": [
+    "src/**/*.ts",
+    "!src/**/*.test.ts",
+    "!src/test-support/**",
+    "!src/fixtures/**",
+    "types/**/*.d.ts"
+  ],
+  "exports": {
+    "./calibration-build-comparison": "./src/calibration-build-comparison.ts",
+    "./calibration-comparison-blob": "./src/calibration-comparison-blob.ts",
+    "./calibration-comparison-bundle": "./src/calibration-comparison-bundle.ts",
+    "./calibration-explorer": "./src/calibration-explorer.ts",
+    "./calibration-release-locator": "./src/calibration-release-locator.ts",
+    "./calibration-tree": "./src/calibration-tree.ts",
+    "./calibration-tree-artifact": "./src/calibration-tree-artifact.ts",
+    "./calibration-tree-blob": "./src/calibration-tree-blob.ts",
+    "./calibration-tree-bundle": "./src/calibration-tree-bundle.ts",
+    "./calibration-tree-manifest": "./src/calibration-tree-manifest.ts",
+    "./calibration-tree-worker": "./src/calibration-tree-worker.ts",
+    "./chronicle-metadata": "./src/chronicle-metadata.ts",
+    "./countries": "./src/countries.ts",
+    "./geography-labels": "./src/geography-labels.ts",
+    "./hierarchy-catalog-reader": "./src/hierarchy-catalog-reader.ts",
+    "./hierarchy-target-reader": "./src/hierarchy-target-reader.ts",
+    "./historical-weight-summary": "./src/historical-weight-summary.ts",
+    "./latest-artifact": "./src/latest-artifact.ts",
+    "./legacy-target-reader": "./src/legacy-target-reader.ts",
+    "./program-label": "./src/program-label.ts",
+    "./publish-calibration-tree": "./src/publish-calibration-tree.ts",
+    "./reforms": "./src/reforms.ts",
+    "./release-cut": "./src/release-cut.ts",
+    "./release-record": "./src/release-record.ts",
+    "./source-label": "./src/source-label.ts",
+    "./source-labels": "./src/source-labels.ts",
+    "./staging-artifact": "./src/staging-artifact.ts",
+    "./staging-contract": "./src/staging-contract.ts",
+    "./structured-dimension-reader": "./src/structured-dimension-reader.ts",
+    "./structured-target-reader": "./src/structured-target-reader.ts",
+    "./target-change": "./src/target-change.ts",
+    "./target-loss-attribution": "./src/target-loss-attribution.ts",
+    "./target-loss-attribution-manifest": "./src/target-loss-attribution-manifest.ts",
+    "./target-representation": "./src/target-representation.ts",
+    "./target-surface-matcher": "./src/target-surface-matcher.ts"
+  },
+  "scripts": {
+    "publish:calibration-tree": "bun run src/publish-calibration-tree.ts",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "worker": "bun run src/calibration-tree-worker.ts"
+  },
+  "dependencies": {
+    "@vercel/blob": "^2.6.1"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.14",
+    "fast-check": "^4.10.2",
+    "typescript": "^5.8.0"
+  }
+}

package/src/calibration-build-comparison.ts

@@ -0,0 +1,77 @@
+import type { Calibration } from "./latest-artifact";
+import type {
+  CalibrationTreeComparisonMetadata,
+  CalibrationTreeTargetSummary,
+} from "./calibration-tree-artifact";
+import type { MicrocosmCountry } from "./countries";
+import { buildTargetChangeDataset, type TargetChangeDataset } from "./target-change";
+
+export interface CalibrationTreeComparisonInput {
+  country: MicrocosmCountry;
+  comparison: CalibrationTreeComparisonMetadata;
+  targets: CalibrationTreeTargetSummary[];
+}
+
+function calibrationFromTargetSummaries(
+  input: CalibrationTreeComparisonInput,
+): Calibration {
+  const comparison = input.comparison;
+  return {
+    release_id: comparison.releaseId,
+    calibration_provenance: comparison.calibrationProvenance ?? {
+      mode: "direct",
+      dataset_release_id: comparison.releaseId,
+      calibration_source_id: comparison.releaseId,
+      diagnostics_sha256: null,
+      target_surface_sha256: null,
+      validation: { status: "not_applicable", reason: null },
+    },
+    target_schema: {
+      diagnostics_schema_version: null,
+      structured_dimensions:
+        comparison.targetRepresentation === "structured" ||
+        comparison.targetRepresentation === "hierarchy" ||
+        comparison.targetRepresentation === "mixed",
+      target_representation: comparison.targetRepresentation,
+    },
+    target_loss_attribution: {
+      status: comparison.status,
+      aggregate: comparison.aggregate,
+      historical_final_loss: null,
+      cap: comparison.cap,
+      basis_identifier: comparison.basisIdentifier,
+      basis_hash: null,
+      verification: null,
+      producer_warnings: [],
+      reason:
+        comparison.status === "unavailable"
+          ? "The immutable build does not include target-loss attribution."
+          : null,
+      targets: [],
+    },
+    rows: input.targets.map((target) => target.comparison.row),
+  } as unknown as Calibration;
+}
+
+export function buildTargetChangeDatasetFromSummaries(
+  current: CalibrationTreeComparisonInput,
+  candidate: CalibrationTreeComparisonInput,
+): TargetChangeDataset {
+  if (current.country !== candidate.country) {
+    throw new Error("Calibration builds from different countries cannot be compared.");
+  }
+  return buildTargetChangeDataset(
+    calibrationFromTargetSummaries(current),
+    calibrationFromTargetSummaries(candidate),
+  );
+}
+
+export function buildTargetChangeDatasetFromSummaryAndCalibration(
+  current: CalibrationTreeComparisonInput,
+  candidate: Calibration,
+): TargetChangeDataset {
+  return buildTargetChangeDataset(
+    calibrationFromTargetSummaries(current),
+    candidate,
+  );
+}

package/src/calibration-comparison-blob.ts

@@ -0,0 +1,367 @@
+import { createHash } from "node:crypto";
+
+import {
+  buildCalibrationComparisonBundles,
+  calibrationComparisonBuildArtifactId,
+  calibrationComparisonPairArtifactId,
+  type ComparisonBundleSource,
+} from "./calibration-comparison-bundle";
+import {
+  CALIBRATION_TREE_SCHEMA_VERSION,
+  calibrationTreeTargetDetailSelection,
+  calibrationTreeTargetsFromSummaries,
+  parseCalibrationTreeIndex,
+  parseCalibrationTreeTargetDetails,
+  parseCalibrationTreeTargetSummary,
+  type CalibrationTreeComparisonTargetDetailResponse,
+  type CalibrationTreeIndexArtifact,
+  type CalibrationTreeTargetDetailsArtifact,
+} from "./calibration-tree-artifact";
+import {
+  calibrationTreeBlobText,
+  getCalibrationTreeBlob,
+  uploadCalibrationTreeBundle,
+} from "./calibration-tree-blob";
+import type { MicrocosmCountry } from "./countries";
+import type { TargetChangeRow } from "./target-change";
+
+interface ComparisonBuildIds {
+  pairArtifactId: string;
+  reportedBuildArtifactId: string;
+  sharedBuildArtifactId: string;
+}
+
+const activeBuilds = new Map<string, Promise<ComparisonBuildIds>>();
+const ACTIVE_BUILD_LIMIT = 8;
+
+function ids(
+  country: MicrocosmCountry,
+  currentBuildArtifactId: string,
+  candidateBuildArtifactId: string,
+): ComparisonBuildIds {
+  const pairArtifactId = calibrationComparisonPairArtifactId(
+    country,
+    currentBuildArtifactId,
+    candidateBuildArtifactId,
+  );
+  return {
+    pairArtifactId,
+    reportedBuildArtifactId: calibrationComparisonBuildArtifactId(
+      pairArtifactId,
+      "reported",
+    ),
+    sharedBuildArtifactId: calibrationComparisonBuildArtifactId(
+      pairArtifactId,
+      "shared",
+    ),
+  };
+}
+
+async function textFromBlob(
+  result: Awaited<ReturnType<typeof getCalibrationTreeBlob>>,
+  label: string,
+): Promise<string> {
+  if (!result || result.statusCode !== 200 || !result.stream) {
+    throw new Error(`${label} is unavailable.`);
+  }
+  return calibrationTreeBlobText(result.stream);
+}
+
+function sha256(value: string): string {
+  return createHash("sha256").update(value, "utf8").digest("hex");
+}
+
+export async function loadCalibrationComparisonSource(
+  country: MicrocosmCountry,
+  buildArtifactId: string,
+  getBlob: typeof getCalibrationTreeBlob = getCalibrationTreeBlob,
+): Promise<ComparisonBundleSource> {
+  const indexText = await textFromBlob(
+    await getBlob({
+      country,
+      buildArtifactId,
+      part: "index",
+      consistent: true,
+    }),
+    `Calibration build ${buildArtifactId} index`,
+  );
+  const index = parseCalibrationTreeIndex(JSON.parse(indexText));
+  if (index.country !== country || index.buildArtifactId !== buildArtifactId) {
+    throw new Error(`Calibration build ${buildArtifactId} index identity is inconsistent.`);
+  }
+  if (index.build.kind === "comparison") {
+    throw new Error("A derived comparison cannot be used as a source build.");
+  }
+  const targetSummaries = await Promise.all(
+    index.parts.targetSummaries.map(async (descriptor) => {
+      const text = await textFromBlob(
+        await getBlob({
+          country,
+          buildArtifactId,
+          part: descriptor.part,
+          consistent: true,
+        }),
+        `Calibration build ${buildArtifactId} ${descriptor.part}`,
+      );
+      if (sha256(text) !== descriptor.sha256) {
+        throw new Error(
+          `Calibration build ${buildArtifactId} ${descriptor.part} digest is invalid.`,
+        );
+      }
+      return parseCalibrationTreeTargetSummary(JSON.parse(text), descriptor.part);
+    }),
+  );
+  return {
+    index,
+    indexSha256: sha256(indexText),
+    targetSummaries: calibrationTreeTargetsFromSummaries(index, targetSummaries),
+  };
+}
+
+async function loadIndex(options: {
+  country: MicrocosmCountry;
+  buildArtifactId: string;
+  expectedSha256?: string;
+  getBlob: typeof getCalibrationTreeBlob;
+}): Promise<CalibrationTreeIndexArtifact> {
+  const text = await textFromBlob(
+    await options.getBlob({
+      country: options.country,
+      buildArtifactId: options.buildArtifactId,
+      part: "index",
+      consistent: true,
+    }),
+    `Calibration build ${options.buildArtifactId} index`,
+  );
+  if (options.expectedSha256 && sha256(text) !== options.expectedSha256) {
+    throw new Error(`Calibration build ${options.buildArtifactId} index digest is invalid.`);
+  }
+  const index = parseCalibrationTreeIndex(JSON.parse(text));
+  if (
+    index.country !== options.country ||
+    index.buildArtifactId !== options.buildArtifactId
+  ) {
+    throw new Error(
+      `Calibration build ${options.buildArtifactId} index identity is inconsistent.`,
+    );
+  }
+  return index;
+}
+
+export async function loadCalibrationComparisonTargetDetail(options: {
+  country: MicrocosmCountry;
+  buildArtifactId: string;
+  targetOrdinal: number;
+  getBlob?: typeof getCalibrationTreeBlob;
+}): Promise<CalibrationTreeComparisonTargetDetailResponse> {
+  const getBlob = options.getBlob ?? getCalibrationTreeBlob;
+  const comparisonIndex = await loadIndex({
+    country: options.country,
+    buildArtifactId: options.buildArtifactId,
+    getBlob,
+  });
+  if (
+    comparisonIndex.build.kind !== "comparison" ||
+    comparisonIndex.parts.targetDetailStrategy !== "source-targets" ||
+    !comparisonIndex.comparison
+  ) {
+    throw new Error("The requested build does not use source-backed comparison details.");
+  }
+  if (
+    !Number.isSafeInteger(options.targetOrdinal) ||
+    options.targetOrdinal < 0 ||
+    options.targetOrdinal >= comparisonIndex.targetCount
+  ) {
+    throw new Error("The requested comparison target ordinal is out of range.");
+  }
+  const summaryDescriptor = comparisonIndex.parts.targetSummaries.find(
+    (descriptor) =>
+      options.targetOrdinal >= descriptor.startTargetOrdinal &&
+      options.targetOrdinal < descriptor.endTargetOrdinalExclusive,
+  );
+  if (!summaryDescriptor) {
+    throw new Error("The requested comparison target has no summary shard.");
+  }
+  const summaryText = await textFromBlob(
+    await getBlob({
+      country: options.country,
+      buildArtifactId: options.buildArtifactId,
+      part: summaryDescriptor.part,
+      consistent: true,
+    }),
+    `Calibration comparison ${options.buildArtifactId} ${summaryDescriptor.part}`,
+  );
+  if (sha256(summaryText) !== summaryDescriptor.sha256) {
+    throw new Error("The comparison target-summary digest is invalid.");
+  }
+  const summaryArtifact = parseCalibrationTreeTargetSummary(
+    JSON.parse(summaryText),
+    summaryDescriptor.part,
+  );
+  const summary = summaryArtifact.targets[
+    options.targetOrdinal - summaryDescriptor.startTargetOrdinal
+  ];
+  if (!summary?.detailSource) {
+    throw new Error("The comparison target does not identify its source targets.");
+  }
+
+  const sourceIndexes = new Map<string, Promise<CalibrationTreeIndexArtifact>>();
+  const sourceDetails = new Map<string, Promise<CalibrationTreeTargetDetailsArtifact>>();
+  const sourceIndex = (
+    buildArtifactId: string,
+    expectedSha256: string,
+  ): Promise<CalibrationTreeIndexArtifact> => {
+    const existing = sourceIndexes.get(buildArtifactId);
+    if (existing) return existing;
+    const loading = loadIndex({
+      country: options.country,
+      buildArtifactId,
+      expectedSha256,
+      getBlob,
+    });
+    sourceIndexes.set(buildArtifactId, loading);
+    return loading;
+  };
+  const sourceDetail = async (
+    buildArtifactId: string,
+    expectedIndexSha256: string,
+    targetOrdinal: number | null,
+  ) => {
+    if (targetOrdinal == null) return null;
+    const index = await sourceIndex(buildArtifactId, expectedIndexSha256);
+    if (index.build.kind === "comparison") {
+      throw new Error("Comparison target details must resolve to source builds.");
+    }
+    const selection = calibrationTreeTargetDetailSelection(index, targetOrdinal);
+    const key = `${buildArtifactId}:${selection.descriptor.part}`;
+    let loading = sourceDetails.get(key);
+    if (!loading) {
+      loading = (async () => {
+        const text = await textFromBlob(
+          await getBlob({
+            country: options.country,
+            buildArtifactId,
+            part: selection.descriptor.part,
+            consistent: true,
+          }),
+          `Calibration build ${buildArtifactId} ${selection.descriptor.part}`,
+        );
+        if (sha256(text) !== selection.descriptor.sha256) {
+          throw new Error(
+            `Calibration build ${buildArtifactId} ${selection.descriptor.part} digest is invalid.`,
+          );
+        }
+        return parseCalibrationTreeTargetDetails(
+          JSON.parse(text),
+          selection.descriptor.part,
+        );
+      })();
+      sourceDetails.set(key, loading);
+    }
+    const artifact = await loading;
+    return artifact.targets[selection.offset] ?? null;
+  };
+
+  const currentSource = comparisonIndex.build.sourceArtifacts.comparisonCurrentIndex;
+  const candidateSource = comparisonIndex.build.sourceArtifacts.comparisonCandidateIndex;
+  if (!currentSource || !candidateSource) {
+    throw new Error("The comparison build does not identify both source indexes.");
+  }
+  const { currentBuildArtifactId, candidateBuildArtifactId } = comparisonIndex.comparison;
+  const [currentDetail, candidateDetail] = await Promise.all([
+    sourceDetail(
+      currentBuildArtifactId,
+      currentSource.sha256,
+      summary.detailSource.currentTargetOrdinal,
+    ),
+    sourceDetail(
+      candidateBuildArtifactId,
+      candidateSource.sha256,
+      summary.detailSource.candidateTargetOrdinal,
+    ),
+  ]);
+  const compact = summary.comparison.row as TargetChangeRow;
+  return {
+    schemaVersion: CALIBRATION_TREE_SCHEMA_VERSION,
+    country: options.country,
+    buildArtifactId: options.buildArtifactId,
+    targetOrdinal: options.targetOrdinal,
+    targetId: summary.id,
+    target: {
+      ...compact,
+      currentDetail,
+      candidateDetail,
+    },
+  };
+}
+
+async function comparisonExists(
+  country: MicrocosmCountry,
+  buildArtifactId: string,
+): Promise<boolean> {
+  return Boolean(await getCalibrationTreeBlob({
+    country,
+    buildArtifactId,
+    part: "index",
+    consistent: true,
+  }));
+}
+
+async function buildAndPublishComparison(options: {
+  country: MicrocosmCountry;
+  currentBuildArtifactId: string;
+  candidateBuildArtifactId: string;
+  token: string;
+}): Promise<ComparisonBuildIds> {
+  const result = ids(
+    options.country,
+    options.currentBuildArtifactId,
+    options.candidateBuildArtifactId,
+  );
+  const [reportedExists, sharedExists] = await Promise.all([
+    comparisonExists(options.country, result.reportedBuildArtifactId),
+    comparisonExists(options.country, result.sharedBuildArtifactId),
+  ]);
+  if (reportedExists && sharedExists) return result;
+
+  const [current, candidate] = await Promise.all([
+    loadCalibrationComparisonSource(options.country, options.currentBuildArtifactId),
+    loadCalibrationComparisonSource(options.country, options.candidateBuildArtifactId),
+  ]);
+  const bundles = buildCalibrationComparisonBundles(current, candidate);
+  if (bundles.pairArtifactId !== result.pairArtifactId) {
+    throw new Error("Calibration comparison identity is inconsistent.");
+  }
+  await Promise.all([
+    uploadCalibrationTreeBundle({ bundle: bundles.reported, token: options.token }),
+    uploadCalibrationTreeBundle({ bundle: bundles.shared, token: options.token }),
+  ]);
+  return result;
+}
+
+export function ensureCalibrationComparison(options: {
+  country: MicrocosmCountry;
+  currentBuildArtifactId: string;
+  candidateBuildArtifactId: string;
+  token: string;
+}): Promise<ComparisonBuildIds> {
+  const key = [
+    options.country,
+    options.currentBuildArtifactId,
+    options.candidateBuildArtifactId,
+  ].join(":");
+  const existing = activeBuilds.get(key);
+  if (existing) return existing;
+  const promise = buildAndPublishComparison(options);
+  activeBuilds.set(key, promise);
+  while (activeBuilds.size > ACTIVE_BUILD_LIMIT) {
+    const oldest = activeBuilds.keys().next().value;
+    if (typeof oldest !== "string") break;
+    activeBuilds.delete(oldest);
+  }
+  void promise.then(
+    () => activeBuilds.delete(key),
+    () => activeBuilds.delete(key),
+  );
+  return promise;
+}