engineering-behavior-observatory 0.2.1

package/dist/src/aggregation.js

@@ -0,0 +1,513 @@
+import { join, resolve } from "node:path";
+import { canonicalizeMetadata, digestBytes, digestMetadata, validateArtifact } from "./artifacts.js";
+import { validateRetainedBehaviorAssertion } from "./behavior-assertions.js";
+import { effectiveReviewOutcome, isDisputedReviewOutcome, revalidateReviewSample, validateReviewHistory, } from "./human-calibration.js";
+import { assessComparisonEligibility } from "./normalization-integrity.js";
+import { readBoundedFile } from "./scheduler.js";
+import { createRetainedStructuralObservationSet } from "./structural-observations.js";
+export function aggregateEvaluation(input, policy) {
+    return aggregateValidatedEvaluation(input, policy);
+}
+export function selectAggregationAttempts(entries, policy) {
+    if (!["all-attempts", "latest-attempt-per-run"].includes(policy))
+        throw new Error("Invalid attempt selection policy.");
+    return selectAttempts(uniqueAttempts(entries).attempts, policy);
+}
+async function aggregateValidatedEvaluation(input, policy) {
+    assertPolicy(policy);
+    const { attempts, duplicates } = uniqueAttempts(input.corpusEntries);
+    const selected = selectAttempts(attempts, policy.selectedAttemptPolicy);
+    const observationSources = uniqueBy(input.observationSets, ({ document }) => `${document.runId}\0${document.attemptId}`, "observation set");
+    const observations = new Map();
+    for (const { bundleRoot, document } of observationSources.values()) {
+        assertIndexedBundle(bundleRoot, document.runId, document.attemptId, attempts);
+        const rebuilt = await createRetainedStructuralObservationSet(bundleRoot);
+        const { capabilityProfile: _capabilityProfile, ...legacyNormalization } = rebuilt.normalization;
+        const comparable = document.normalization.capabilityProfile === undefined
+            ? { ...rebuilt, normalization: legacyNormalization }
+            : rebuilt;
+        if (canonicalizeMetadata(comparable) !== canonicalizeMetadata(document))
+            throw new Error(`Structural observation set for attempt "${document.attemptId}" is stale.`);
+        observations.set(`${document.runId}\0${document.attemptId}`, rebuilt);
+    }
+    const assertionSources = uniqueBy(input.assertions, ({ document }) => assertionKey(document), "behavior assertion");
+    const assertions = new Map();
+    for (const { bundleRoot, document } of assertionSources.values()) {
+        assertIndexedBundle(bundleRoot, document.runId, document.attemptId, attempts);
+        await validateRetainedBehaviorAssertion(bundleRoot, document);
+        assertions.set(assertionKey(document), document);
+    }
+    for (const { selection } of input.calibrations)
+        await revalidateReviewSample(selection);
+    const reviewOutcomes = reviewOutcomeIndex(input.calibrations);
+    const selectedIds = new Set(selected.map(attemptKey));
+    const groups = groupAttempts(attempts, policy.groupBy).map(([dimensions, allMembers]) => {
+        const members = allMembers.filter((attempt) => selectedIds.has(attemptKey(attempt)));
+        return {
+            dimensions,
+            metrics: groupMetrics(members, attempts, observations, assertions, reviewOutcomes, dimensions),
+            behaviors: behaviorAggregates(members, assertions, reviewOutcomes),
+            variations: [
+                variation("terminal-state", members, policy.recurrence.minimumOccurrences),
+                variation("verifier-status", members, policy.recurrence.minimumOccurrences),
+            ],
+        };
+    });
+    const report = {
+        schemaVersion: "ebo.aggregation-report/v1",
+        policy: {
+            groupBy: [...policy.groupBy],
+            selectedAttemptPolicy: policy.selectedAttemptPolicy,
+            recurrence: {
+                minimumOccurrences: policy.recurrence.minimumOccurrences,
+                establishesCausality: false,
+                establishesStatisticalSignificance: false,
+            },
+        },
+        sourcePopulation: {
+            uniqueRuns: new Set(attempts.map(({ runId }) => runId)).size,
+            uniqueAttempts: attempts.length,
+            selectedAttempts: selected.length,
+            duplicateInputsIgnored: duplicates,
+        },
+        sourceLineage: {
+            requestDigest: input.lineage?.requestDigest ?? metadataDigest({ policy, comparisons: input.comparisons.map(({ eligibility: _eligibility, ...comparison }) => comparison) }),
+            corpusIndexDigest: input.lineage?.corpusIndexDigest ?? metadataDigest(input.corpusEntries),
+            manifests: attempts.map(({ runId, attemptId, manifestDigest: digest }) => ({ runId, attemptId, digest })),
+            observationSets: [...observationSources.values()].map(({ document }) => ({ runId: document.runId, attemptId: document.attemptId, digest: metadataDigest(document) })),
+            assertions: [...assertionSources.values()].map(({ document }) => ({ runId: document.runId, attemptId: document.attemptId, id: document.id, digest: metadataDigest(document) })),
+            calibrations: input.calibrations.map(({ selection, history }) => ({ selectionDigest: metadataDigest(selection), historyDigest: metadataDigest(history) })),
+            comparisonGates: input.comparisons.flatMap(({ eligibility }) => eligibility.map(({ request, report }) => ({
+                requestDigest: metadataDigest(request),
+                reportDigest: metadataDigest(report),
+            }))),
+        },
+        groups,
+        comparisons: input.comparisons.map((comparison) => compare(comparison, selected, observations, policy.recurrence.minimumOccurrences)),
+        limitations: [
+            "All aggregates are descriptive; no causal attribution or statistical significance is claimed.",
+            "Observational completion is a terminal-state measure, not task success.",
+            "Verifier rates include only verified attempts with an available terminal verifier result.",
+            "Human-reviewed assertion rates include only digest-bound calibration selections and validated review lineage.",
+        ],
+    };
+    assertArtifact("aggregation report", report);
+    return report;
+}
+function behaviorAggregates(selected, assertions, reviews) {
+    const selectedIds = new Set(selected.map(attemptKey));
+    const partitions = new Map();
+    for (const assertion of assertions.values()) {
+        if (!selectedIds.has(attemptKey(assertion)))
+            continue;
+        const key = canonicalizeMetadata({ behavior: assertion.behavior, rubric: assertion.rubric, evaluator: assertion.evaluator });
+        const values = partitions.get(key) ?? [];
+        values.push(assertion);
+        partitions.set(key, values);
+    }
+    return [...partitions.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([, values]) => {
+        const first = values[0];
+        const included = new Set();
+        const assessments = [];
+        const exclusions = [];
+        const outcome = (assertion) => reviews.get(assertionBindingKey(assertion)) ?? {
+            outcome: assertion.judgment.disposition === "abstained" ? "judge-abstained" : "unreviewed", disputed: false,
+        };
+        for (const attempt of selected) {
+            const candidates = values.filter((assertion) => attemptKey(assertion) === attemptKey(attempt));
+            const confirmed = candidates.filter((assertion) => assertion.judgment.disposition === "assessed"
+                && outcome(assertion).outcome === "confirmed" && !outcome(assertion).disputed);
+            const judgments = new Set(confirmed.flatMap(({ judgment }) => judgment.disposition === "assessed" ? [judgment.assessment] : []));
+            if (judgments.size === 1) {
+                assessments.push([...judgments][0]);
+                for (const assertion of confirmed)
+                    included.add(assertionKey(assertion));
+            }
+            else
+                exclusions.push(judgments.size > 1 ? "conflicting-confirmed-reruns" : candidates.length === 0 ? "dimension-assertion-missing" : "no-confirmed-assessment");
+        }
+        return {
+            behavior: structuredClone(first.behavior), rubric: structuredClone(first.rubric), evaluator: structuredClone(first.evaluator),
+            population: "confirmed-attempt-dimension",
+            assessments: ["constructive", "adverse", "mixed", "context-dependent"].map((assessment) => ({
+                assessment,
+                measurement: rateMetric(assessment, "attempt", assessments.filter((value) => value === assessment).length, assessments.length, "confirmed-attempt-dimension", exclusionCounts(exclusions, "attempt-dimension")).measurement,
+            })),
+            assertions: values.map((assertion) => ({ runId: assertion.runId, attemptId: assertion.attemptId, id: assertion.id,
+                digest: metadataDigest(assertion), reviewOutcome: outcome(assertion).outcome, disputed: outcome(assertion).disputed,
+                included: included.has(assertionKey(assertion)) })),
+        };
+    });
+}
+function groupMetrics(selected, all, observations, assertions, reviews, dimensions) {
+    const selectedIds = new Set(selected.map(attemptKey));
+    const relatedAll = all.filter((attempt) => matches(attempt, dimensions));
+    const selectionExclusions = exclusionCounts(relatedAll.filter((attempt) => !selectedIds.has(attemptKey(attempt))).map(() => "not-selected-by-attempt-policy"), "attempt");
+    const metrics = [
+        metric("attempt-count", "attempt", selected.length, relatedAll.length, "attempt", "attempt", selectionExclusions),
+        rateMetric("infrastructure-failure-rate", "attempt", selected.filter(({ failureClass }) => failureClass === "infrastructure").length, selected.length, "attempt", []),
+        rateMetric("capture-qualified-rate", "attempt", selected.filter(({ captureQualification }) => captureQualification === "qualified" || captureQualification === "qualified-with-gaps").length, selected.filter(({ captureQualification }) => captureQualification !== undefined && captureQualification !== "unavailable").length, "attempt", exclusionCounts(selected.flatMap(({ captureQualification }) => captureQualification === undefined || captureQualification === "unavailable" ? ["capture-qualification-unavailable"] : []), "attempt")),
+        rateMetric("terminal-completed-rate", "attempt", selected.filter(({ terminalState }) => terminalState === "completed").length, selected.length, "attempt", []),
+    ];
+    const verified = selected.filter(({ assessmentMode, verifierStatuses }) => assessmentMode === "verified"
+        && verifierStatuses.length === 1 && ["passed", "failed"].includes(verifierStatuses[0]));
+    metrics.push(rateMetric("verifier-pass-rate", "attempt", verified.filter(({ verifierStatuses }) => verifierStatuses[0] === "passed").length, verified.length, "verified-attempt", exclusionCounts(selected.flatMap((attempt) => attempt.assessmentMode !== "verified"
+        ? ["observational-or-unknown-assessment-mode"]
+        : attempt.verifierStatuses.length !== 1 || !["passed", "failed"].includes(attempt.verifierStatuses[0])
+            ? ["verifier-outcome-unavailable-or-infrastructure"] : []), "attempt")));
+    const selectedObservationSets = selected.flatMap((attempt) => observations.get(attemptKey(attempt)) ?? []);
+    metrics.push(rateMetric("structural-observation-set-availability-rate", "attempt", selectedObservationSets.length, selected.length, "attempt", exclusionCounts(selected.flatMap((attempt) => observations.has(attemptKey(attempt)) ? [] : ["observation-set-missing"]), "attempt")));
+    const extractorIds = [...new Set(selectedObservationSets.flatMap(({ observations: values }) => values.map(({ extractor }) => extractor.id)))].sort();
+    for (const extractorId of extractorIds) {
+        const values = selected.map((attempt) => observations.get(attemptKey(attempt))?.observations.find(({ extractor }) => extractor.id === extractorId)?.value);
+        const known = values.flatMap((value) => value?.status === "known" ? [value] : []);
+        const units = new Set(known.map(({ unit }) => unit));
+        const types = new Set(known.map(({ value }) => typeof value));
+        const exclusions = exclusionCounts(values.flatMap((value) => value?.status === "unavailable" ? [value.reason] : value === undefined ? ["observation-set-missing"] : []), "attempt");
+        if (units.size > 1 || types.size > 1) {
+            metrics.push(unavailableMetric(`structural:${extractorId}`, "attempt", "Structural observation types or units differ within the group.", selected.length, "attempt"));
+        }
+        else if (types.has("number")) {
+            metrics.push(metric(`structural:${extractorId}`, "attempt", known.reduce((sum, { value }) => sum + Number(value), 0), known.length, known[0]?.unit ?? "observation-value", "attempt-with-known-observation", exclusions));
+        }
+        else if (types.has("boolean")) {
+            metrics.push(metric(`structural:${extractorId}`, "attempt", known.filter(({ value }) => value === true).length, known.length, "attempt-with-true-observation", "attempt-with-known-observation", exclusions));
+        }
+        else {
+            for (const value of [...new Set(known.map(({ value }) => String(value)))].sort()) {
+                metrics.push(metric(`structural:${extractorId}:${value}`, "attempt", known.filter((knownValue) => knownValue.value === value).length, known.length, "attempt", "attempt-with-known-observation", exclusions));
+            }
+            if (known.length === 0)
+                metrics.push(unavailableMetric(`structural:${extractorId}`, "attempt", "The eligible denominator is empty.", 0, "attempt", exclusions));
+        }
+    }
+    const selectedAssertions = [...assertions.values()].filter(({ runId, attemptId }) => selectedIds.has(`${runId}\0${attemptId}`));
+    metrics.push(rateMetric("assertion-abstention-rate", "assertion", selectedAssertions.filter(({ judgment }) => judgment.disposition === "abstained").length, selectedAssertions.length, "assertion", []));
+    const reviewResults = selectedAssertions.map((assertion) => reviews.get(assertionBindingKey(assertion)) ?? {
+        outcome: assertion.judgment.disposition === "abstained" ? "judge-abstained" : "unreviewed",
+        disputed: false,
+    });
+    const outcomes = reviewResults.map(({ outcome }) => outcome);
+    const reviewed = outcomes.filter((outcome) => outcome === "confirmed" || outcome === "rejected" || outcome === "unresolved");
+    metrics.push(rateMetric("reviewed-assertion-confirmed-rate", "reviewed-assertion", reviewed.filter((outcome) => outcome === "confirmed").length, reviewed.length, "reviewed-assertion", exclusionCounts(outcomes.flatMap((outcome) => outcome === "unreviewed" || outcome === "judge-abstained" ? [outcome] : []), "assertion")));
+    metrics.push(rateMetric("review-unresolved-rate", "assertion", outcomes.filter((outcome) => outcome === "unresolved" || outcome === "unreviewed" || outcome === "judge-abstained").length, outcomes.length, "assertion", []));
+    metrics.push(rateMetric("review-disputed-rate", "assertion", reviewResults.filter(({ disputed }) => disputed).length, reviewResults.length, "assertion", []));
+    return metrics;
+}
+function variation(measure, attempts, recurrenceMinimum) {
+    const values = attempts.flatMap((attempt) => measure === "terminal-state"
+        ? attempt.terminalState === undefined ? [] : [attempt.terminalState]
+        : attempt.assessmentMode === "verified" && attempt.verifierStatuses.length === 1 ? [attempt.verifierStatuses[0]] : []);
+    const excluded = attempts.length - values.length;
+    const counts = new Map();
+    for (const value of values)
+        counts.set(value, (counts.get(value) ?? 0) + 1);
+    const distribution = [...counts].sort(([left], [right]) => left.localeCompare(right)).map(([value, count]) => rateMetric(`${measure}:${value}`, "attempt", count, values.length, "attempt", excluded === 0 ? [] : [{ reason: `${measure}-unavailable`, count: excluded, unit: "attempt" }]));
+    const secondMostFrequent = [...counts.values()].sort((left, right) => right - left)[1] ?? 0;
+    return {
+        measure,
+        distribution,
+        claimStatus: values.length === 0 ? "unavailable" : secondMostFrequent === 0 ? "no-variation"
+            : secondMostFrequent >= recurrenceMinimum ? "recurring-description" : "case-study",
+    };
+}
+function compare(comparison, attempts, observations, recurrenceMinimum) {
+    const gates = new Map();
+    for (const { request, report } of comparison.eligibility) {
+        const rebuilt = assessComparisonEligibility(request);
+        if (canonicalizeMetadata(rebuilt) !== canonicalizeMetadata(report)) {
+            throw new Error(`Comparison "${comparison.id}" has a stale eligibility report.`);
+        }
+        assertArtifact("comparison report", report);
+        const key = candidatePairKey(request.left, request.right);
+        const current = gates.get(key);
+        if (current !== undefined && canonicalizeMetadata(current) !== canonicalizeMetadata({ request, report })) {
+            throw new Error(`Comparison "${comparison.id}" has conflicting eligibility reports for one candidate pair.`);
+        }
+        gates.set(key, { request, report });
+    }
+    const left = attempts.filter((attempt) => matches(attempt, comparison.left));
+    const right = attempts.filter((attempt) => matches(attempt, comparison.right));
+    const exclusions = [
+        ...left.filter((attempt) => hasMissingDimension(attempt, comparison.matchBy)).map(() => "match-dimension-unavailable"),
+        ...right.filter((attempt) => hasMissingDimension(attempt, comparison.matchBy)).map(() => "match-dimension-unavailable"),
+    ];
+    const leftByMatch = indexedMatches(left.filter((attempt) => !hasMissingDimension(attempt, comparison.matchBy)), comparison.matchBy);
+    const rightByMatch = indexedMatches(right.filter((attempt) => !hasMissingDimension(attempt, comparison.matchBy)), comparison.matchBy);
+    const keys = [...new Set([...leftByMatch.keys(), ...rightByMatch.keys()])].sort();
+    const differences = [];
+    let differenceUnit;
+    for (const key of keys) {
+        const leftMatches = leftByMatch.get(key) ?? [];
+        const rightMatches = rightByMatch.get(key) ?? [];
+        if (leftMatches.length !== 1 || rightMatches.length !== 1) {
+            exclusions.push(leftMatches.length === 0 || rightMatches.length === 0 ? "unmatched-unit" : "ambiguous-matched-unit");
+            continue;
+        }
+        const gate = gates.get(candidatePairKey({ id: leftMatches[0].runId, manifestDigest: leftMatches[0].manifestDigest }, { id: rightMatches[0].runId, manifestDigest: rightMatches[0].manifestDigest }));
+        if (gate === undefined) {
+            exclusions.push("comparison-eligibility-missing");
+            continue;
+        }
+        if (gate.report.status === "unsupported") {
+            exclusions.push("comparison-eligibility-unsupported");
+            continue;
+        }
+        if (gate.report.measure !== comparison.measure) {
+            exclusions.push("comparison-measure-not-gated");
+            continue;
+        }
+        if (measureCapabilities(comparison.measure, [leftMatches[0], rightMatches[0]], observations)
+            .some((capability) => !gate.report.policy.requiredCapabilities.includes(capability))) {
+            exclusions.push("comparison-measure-capability-not-gated");
+            continue;
+        }
+        const leftCandidate = gate.request.left.id === leftMatches[0].runId ? gate.request.left : gate.request.right;
+        const rightCandidate = gate.request.left.id === rightMatches[0].runId ? gate.request.left : gate.request.right;
+        if (!candidateMatchesAttempt(leftCandidate, leftMatches[0], observations)
+            || !candidateMatchesAttempt(rightCandidate, rightMatches[0], observations)) {
+            exclusions.push("comparison-candidate-evidence-mismatch");
+            continue;
+        }
+        const leftValue = measureValue(leftMatches[0], comparison.measure, observations);
+        const rightValue = measureValue(rightMatches[0], comparison.measure, observations);
+        if (leftValue === undefined || rightValue === undefined)
+            exclusions.push("measure-unavailable");
+        else if (leftValue.unit !== rightValue.unit || differenceUnit !== undefined && differenceUnit !== leftValue.unit)
+            exclusions.push("measure-unit-mismatch");
+        else {
+            differenceUnit = leftValue.unit;
+            differences.push(rightValue.value - leftValue.value);
+        }
+    }
+    const differing = differences.filter((value) => value !== 0).length;
+    const measurement = differences.length === 0
+        ? unavailableMeasurement("No uniquely matched units have the requested measure on both sides.", 0, "matched-unit", exclusionCounts(exclusions, "matched-unit"))
+        : availableMeasurement(differences.reduce((sum, value) => sum + value, 0), differences.length, `right-minus-left-${differenceUnit}`, "matched-unit", exclusionCounts(exclusions, "matched-unit"));
+    return {
+        id: comparison.id,
+        measure: comparison.measure,
+        eligibility: comparison.eligibility.map(({ report }) => structuredClone(report)),
+        matchedDifference: measurement,
+        differingMatchedUnits: differing,
+        claimStatus: differences.length === 0 ? "unavailable" : differing === 0 ? "no-difference"
+            : differing >= recurrenceMinimum ? "recurring-description" : "case-study",
+        limitations: [
+            ...new Set(comparison.eligibility.flatMap(({ report }) => report.reasons.map(({ detail }) => detail))),
+            "Matched differences are descriptive and do not establish causality or statistical significance.",
+        ],
+    };
+}
+function candidateMatchesAttempt(candidate, attempt, observations) {
+    const exact = candidate.id === attempt.runId
+        && candidate.manifestDigest === attempt.manifestDigest
+        && candidate.task.id === attempt.taskId
+        && candidate.task.digest === attempt.taskDigest
+        && candidate.fixture.id === attempt.fixtureId
+        && candidate.fixture.digest === attempt.fixtureDigest
+        && candidate.model.id === attempt.modelId
+        && candidate.model.configurationDigest === attempt.modelConfigurationDigest
+        && candidate.harness.id === attempt.harnessId
+        && candidate.harness.version === attempt.harnessVersion
+        && candidate.harness.configurationDigest === attempt.harnessConfigurationDigest
+        && candidate.assessmentMode === attempt.assessmentMode
+        && candidate.captureProfileDigest === attempt.captureProfileDigest
+        && candidate.budgetDigest === attempt.budgetDigest
+        && candidate.toolPolicyDigest === attempt.toolPolicyDigest;
+    if (!exact)
+        return false;
+    const observationSet = observations.get(attemptKey(attempt));
+    if (observationSet?.normalization.capabilityProfile === undefined)
+        return false;
+    return candidate.adapterVersion === observationSet.normalization.adapter.version
+        && canonicalizeMetadata(candidate.capabilityProfile) === canonicalizeMetadata(observationSet.normalization.capabilityProfile);
+}
+function measureCapabilities(measure, attempts, observations) {
+    if (measure === "attempt:infrastructure-failure" || measure === "attempt:terminal-completed" || measure === "verified:verifier-passed") {
+        return ["family:outcome"];
+    }
+    if (measure.startsWith("structural:")) {
+        const extractorId = measure.slice("structural:".length);
+        return [...new Set(attempts.flatMap((attempt) => observations.get(attemptKey(attempt))?.observations
+                .find(({ extractor }) => extractor.id === extractorId)?.extractor.requiredCapabilities ?? []))];
+    }
+    throw new Error(`Unsupported aggregate comparison measure "${measure}".`);
+}
+function candidatePairKey(left, right) {
+    return canonicalizeMetadata([[left.id, left.manifestDigest], [right.id, right.manifestDigest]]
+        .sort((leftValue, rightValue) => canonicalizeMetadata(leftValue).localeCompare(canonicalizeMetadata(rightValue))));
+}
+function measureValue(attempt, measure, observations) {
+    if (measure === "attempt:infrastructure-failure")
+        return { value: attempt.failureClass === "infrastructure" ? 1 : 0, unit: "attempt" };
+    if (measure === "attempt:terminal-completed")
+        return attempt.terminalState === undefined ? undefined : { value: attempt.terminalState === "completed" ? 1 : 0, unit: "attempt" };
+    if (measure === "verified:verifier-passed")
+        return attempt.assessmentMode !== "verified" || attempt.verifierStatuses.length !== 1
+            || !["passed", "failed"].includes(attempt.verifierStatuses[0])
+            ? undefined : { value: attempt.verifierStatuses[0] === "passed" ? 1 : 0, unit: "verified-attempt" };
+    if (measure.startsWith("structural:")) {
+        const value = observations.get(attemptKey(attempt))?.observations.find(({ extractor }) => extractor.id === measure.slice("structural:".length))?.value;
+        return value?.status === "known" && (typeof value.value === "number" || typeof value.value === "boolean")
+            ? { value: typeof value.value === "boolean" ? Number(value.value) : value.value, unit: value.unit }
+            : undefined;
+    }
+    throw new Error(`Unsupported aggregate comparison measure "${measure}".`);
+}
+function reviewOutcomeIndex(calibrations) {
+    const outcomes = new Map();
+    for (const { selection, history } of calibrations) {
+        validateReviewHistory(selection, history);
+        for (const candidate of selection.candidates) {
+            const key = bindingKey(candidate.assertion.id, candidate.assertion.digest);
+            const outcome = {
+                outcome: effectiveReviewOutcome(candidate, history.decisions),
+                disputed: isDisputedReviewOutcome(candidate, history.decisions),
+            };
+            const current = outcomes.get(key);
+            if (current !== undefined && canonicalizeMetadata(current) !== canonicalizeMetadata(outcome))
+                throw new Error(`Calibration inputs conflict for assertion "${candidate.assertion.id}".`);
+            outcomes.set(key, outcome);
+        }
+    }
+    return outcomes;
+}
+function uniqueAttempts(entries) {
+    const attempts = new Map();
+    let duplicates = 0;
+    for (const entry of entries) {
+        if (entry.manifestKind !== "run" || entry.runId === undefined || entry.attemptId === undefined)
+            continue;
+        const key = attemptKey(entry);
+        const current = attempts.get(key);
+        if (current === undefined)
+            attempts.set(key, structuredClone(entry));
+        else if (canonicalizeMetadata(current) === canonicalizeMetadata(entry))
+            duplicates += 1;
+        else
+            throw new Error(`Corpus inputs conflict for run "${entry.runId}" attempt "${entry.attemptId}".`);
+    }
+    return { attempts: [...attempts.values()].sort((left, right) => attemptKey(left).localeCompare(attemptKey(right))), duplicates };
+}
+function assertIndexedBundle(bundleRoot, runId, attemptId, attempts) {
+    const indexed = attempts.filter((attempt) => attempt.runId === runId && attempt.attemptId === attemptId);
+    if (indexed.length !== 1)
+        throw new Error(`Derived source run "${runId}" attempt "${attemptId}" has no unique corpus entry.`);
+    const actual = `sha256:${digestBytes(readBoundedFile(join(resolve(bundleRoot), "manifest.json"), "Derived source manifest")).value}`;
+    if (actual !== indexed[0].manifestDigest) {
+        throw new Error(`Derived source run "${runId}" attempt "${attemptId}" does not match the indexed manifest digest.`);
+    }
+}
+function selectAttempts(attempts, policy) {
+    if (policy === "all-attempts")
+        return [...attempts];
+    const latest = new Map();
+    for (const attempt of attempts) {
+        const current = latest.get(attempt.runId);
+        if (current === undefined || (attempt.attemptNumber ?? -1) > (current.attemptNumber ?? -1))
+            latest.set(attempt.runId, attempt);
+        else if ((attempt.attemptNumber ?? -1) === (current.attemptNumber ?? -1) && attempt.attemptId !== current.attemptId) {
+            throw new Error(`Run "${attempt.runId}" has ambiguous latest attempts.`);
+        }
+    }
+    return [...latest.values()].sort((left, right) => attemptKey(left).localeCompare(attemptKey(right)));
+}
+function groupAttempts(attempts, dimensions) {
+    const groups = new Map();
+    for (const attempt of attempts) {
+        const values = Object.fromEntries(dimensions.map((dimension) => [dimension, dimensionValue(attempt, dimension) ?? "unavailable"]));
+        const key = canonicalizeMetadata(values);
+        const group = groups.get(key) ?? [values, []];
+        group[1].push(attempt);
+        groups.set(key, group);
+    }
+    return [...groups.values()].sort(([left], [right]) => canonicalizeMetadata(left).localeCompare(canonicalizeMetadata(right)));
+}
+function dimensionValue(attempt, dimension) {
+    return ({
+        task: attempt.taskId,
+        model: attempt.modelId,
+        harness: attempt.harnessId,
+        trial: attempt.trialId,
+        "capture-qualification": attempt.captureQualification,
+    })[dimension];
+}
+function matches(attempt, filters) {
+    return Object.entries(filters).every(([dimension, value]) => (dimensionValue(attempt, dimension) ?? "unavailable") === value);
+}
+function indexedMatches(attempts, dimensions) {
+    const result = new Map();
+    for (const attempt of attempts) {
+        const key = canonicalizeMetadata(dimensions.map((dimension) => dimensionValue(attempt, dimension) ?? null));
+        const values = result.get(key) ?? [];
+        values.push(attempt);
+        result.set(key, values);
+    }
+    return result;
+}
+function hasMissingDimension(attempt, dimensions) {
+    return dimensions.some((dimension) => dimensionValue(attempt, dimension) === undefined);
+}
+function metric(id, population, numerator, denominator, numeratorUnit, denominatorUnit, exclusions) {
+    const measurement = denominator === 0
+        ? unavailableMeasurement("The eligible denominator is empty.", numerator, denominatorUnit, exclusions, numeratorUnit)
+        : availableMeasurement(numerator, denominator, numeratorUnit, denominatorUnit, exclusions);
+    return { id, population, claimStatus: measurement.status === "available" ? "descriptive" : "unavailable", measurement };
+}
+function rateMetric(id, population, numerator, denominator, unit, exclusions) {
+    return metric(id, population, numerator, denominator, unit, unit, exclusions);
+}
+function unavailableMetric(id, population, reason, denominator, unit, exclusions = []) {
+    return { id, population, claimStatus: "unavailable", measurement: unavailableMeasurement(reason, 0, unit, exclusions, unit, denominator) };
+}
+function availableMeasurement(numerator, denominator, numeratorUnit, denominatorUnit, exclusions) {
+    return { status: "available", numerator: { value: numerator, unit: numeratorUnit }, denominator: { value: denominator, unit: denominatorUnit }, rate: numerator / denominator, exclusions };
+}
+function unavailableMeasurement(reason, numerator, denominatorUnit, exclusions, numeratorUnit = denominatorUnit, denominator = 0) {
+    return { status: "unavailable", numerator: { value: numerator, unit: numeratorUnit }, denominator: { value: denominator, unit: denominatorUnit }, exclusions, reason };
+}
+function exclusionCounts(reasons, unit) {
+    const counts = new Map();
+    for (const reason of reasons)
+        counts.set(reason, (counts.get(reason) ?? 0) + 1);
+    return [...counts].sort(([left], [right]) => left.localeCompare(right)).map(([reason, count]) => ({ reason, count, unit }));
+}
+function uniqueBy(values, key, label) {
+    const result = new Map();
+    for (const value of values) {
+        const identity = key(value);
+        const current = result.get(identity);
+        if (current === undefined)
+            result.set(identity, structuredClone(value));
+        else if (canonicalizeMetadata(current) !== canonicalizeMetadata(value))
+            throw new Error(`Duplicate ${label} identity has conflicting content.`);
+    }
+    return result;
+}
+function assertPolicy(policy) {
+    if (new Set(policy.groupBy).size !== policy.groupBy.length)
+        throw new Error("Aggregate grouping dimensions must be unique.");
+    if (!Number.isSafeInteger(policy.recurrence.minimumOccurrences) || policy.recurrence.minimumOccurrences < 2) {
+        throw new Error("Aggregate recurrence minimum must be an integer of at least two.");
+    }
+}
+function assertArtifact(label, value) {
+    const errors = validateArtifact(label, value);
+    if (errors.length > 0)
+        throw new Error(errors.map(({ field, message }) => `${label} ${field}: ${message}`).join("\n"));
+}
+function attemptKey(attempt) {
+    return `${attempt.runId}\0${attempt.attemptId}`;
+}
+function assertionKey(assertion) {
+    return `${assertion.runId}\0${assertion.attemptId}\0${assertion.id}`;
+}
+function assertionBindingKey(assertion) {
+    return bindingKey(assertion.id, `sha256:${digestMetadata(assertion).value}`);
+}
+function bindingKey(id, digest) {
+    return `${id}\0${digest}`;
+}
+function metadataDigest(value) {
+    return `sha256:${digestMetadata(value).value}`;
+}

package/dist/src/artifacts.d.ts

@@ -0,0 +1,36 @@
+import { type BundleRootHandle, type Digest } from "./contracts.js";
+export type ArtifactValidationError = {
+    artifact: string;
+    schemaVersion: string;
+    field: string;
+    message: string;
+};
+export type ArtifactIdentity = {
+    id: string;
+    relativePath: string;
+};
+export declare const SUPPORTED_ARTIFACT_SCHEMA_VERSIONS: string[];
+export declare function canonicalizeMetadata(value: unknown): string;
+export declare function digestBytes(bytes: Uint8Array): Digest;
+export declare function digestMetadata(metadata: unknown): Digest;
+export declare function verifyDigest(bytes: Uint8Array, expected: Digest): boolean;
+export declare function assertNoDuplicateJsonKeys(text: string): void;
+export declare function assertUniqueArtifactIdentities(identities: readonly ArtifactIdentity[]): void;
+export declare function validateArtifact(artifact: string, document: unknown): ArtifactValidationError[];
+export declare function validateRunManifestEvidence(artifact: string, manifest: unknown, bundleRoot: string): ArtifactValidationError[];
+export declare function validateExportManifest(artifact: string, exportManifest: unknown, containingManifest: unknown | undefined, bundleRoot?: string): ArtifactValidationError[];
+export declare function readVerifiedArtifact(artifactRoot: string, relativePath: string, expectedDigest: Digest, maxBytes?: number): Promise<Buffer>;
+export declare function inspectRetainedArtifact(artifactRoot: string, relativePath: string): Promise<{
+    digest: Digest;
+    sizeBytes: number;
+}>;
+export declare function writeMetadataAtomically(artifactRoot: string, relativePath: string, metadata: unknown, signal?: AbortSignal, options?: {
+    overwrite?: boolean;
+}): Promise<Digest>;
+export declare function writeArtifactAtomically(artifactRoot: string, relativePath: string, content: Uint8Array, signal?: AbortSignal, options?: {
+    overwrite?: boolean;
+}): Promise<Digest>;
+export declare function writeMetadataAtomicallyIfAbsentSync(artifactRoot: string, relativePath: string, metadata: unknown, rootHandle?: BundleRootHandle, beforeDestinationPublish?: () => void, afterDestinationPublish?: () => void): {
+    created: boolean;
+    digest: Digest;
+};