@openpond/evals 0.4.2 → 0.6.0

package/dist/rollouts.js

@@ -0,0 +1,241 @@
+import { z } from "zod";
+import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
+import { AttemptOutcomeClassSchema, FailureOwnerSchema, RewardReceiptSchema, ScoringStatusSchema, } from "./execution-contracts.js";
+import { TaskSplitSchema } from "./tasksets.js";
+import { AttemptReceiptSchema, verifyAttemptReceipt } from "./runs.js";
+export const OptimizerTrainingSampleSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.optimizerTrainingSample.v1"),
+    tokenIds: z.array(z.number().int().nonnegative()).min(2).max(32_768),
+    mask: z.array(z.boolean()).min(2).max(32_768),
+    logprobs: z.array(z.number().finite()).min(2).max(32_768),
+    temperatures: z.array(z.number().positive().finite()).min(2).max(32_768),
+    envName: z.string().trim().min(1).max(200),
+    modelRequestId: z.string().trim().min(1).max(1_000),
+    promptTokenCount: z.number().int().positive(),
+    completionTokenCount: z.number().int().positive(),
+    servedPolicyVersion: z.number().int().nonnegative(),
+})
+    .strict()
+    .superRefine((sample, context) => {
+    const length = sample.tokenIds.length;
+    for (const [name, values] of [
+        ["mask", sample.mask],
+        ["logprobs", sample.logprobs],
+        ["temperatures", sample.temperatures],
+    ]) {
+        if (values.length !== length) {
+            context.addIssue({
+                code: "custom",
+                path: [name],
+                message: `${name} must align with tokenIds`,
+            });
+        }
+    }
+    if (sample.promptTokenCount + sample.completionTokenCount !== length) {
+        context.addIssue({
+            code: "custom",
+            path: ["completionTokenCount"],
+            message: "prompt and completion token counts must span tokenIds",
+        });
+    }
+    if (sample.mask.filter((trainable) => !trainable).length !== sample.promptTokenCount) {
+        context.addIssue({
+            code: "custom",
+            path: ["mask"],
+            message: "promptTokenCount must equal the non-trainable mask count",
+        });
+    }
+    if (sample.mask.filter(Boolean).length !== sample.completionTokenCount) {
+        context.addIssue({
+            code: "custom",
+            path: ["mask"],
+            message: "completionTokenCount must equal the trainable mask count",
+        });
+    }
+});
+export const EnvironmentExecutionEvidenceSchema = z.object({
+    id: ReleaseIdSchema,
+    environmentRelease: ImmutableReleaseRefSchema,
+    status: z.enum(["completed", "failed", "timed_out", "cancelled"]),
+    startedAt: ReleaseTimestampSchema,
+    completedAt: ReleaseTimestampSchema,
+    traceRefs: z.array(ImmutableArtifactRefSchema).max(10_000),
+    metadata: MetadataSchema,
+}).strict();
+const RolloutRewardProjectionSchema = z.object({
+    receiptRef: ImmutableReleaseRefSchema,
+    status: ScoringStatusSchema,
+    value: z.number().min(0).max(1).nullable(),
+    learningEligible: z.boolean(),
+    passed: z.boolean(),
+    outcomeClass: AttemptOutcomeClassSchema,
+    failureOwner: FailureOwnerSchema.nullable(),
+    components: z.record(ReleaseIdSchema, z.number().min(0).max(1).nullable()),
+}).strict();
+const CanonicalRolloutRecordFieldsSchema = z.object({
+    schemaVersion: z.literal("openpond.canonicalRolloutRecord.v1"),
+    id: ReleaseIdSchema,
+    attemptRef: ImmutableReleaseRefSchema,
+    artifactManifestRef: ImmutableReleaseRefSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    environmentRelease: ImmutableReleaseRefSchema,
+    verifierSetRelease: ImmutableReleaseRefSchema,
+    harnessRelease: ImmutableReleaseRefSchema,
+    taskId: ReleaseIdSchema,
+    split: TaskSplitSchema,
+    model: ModelRefSchema,
+    seed: z.string().trim().min(1).max(500),
+    reward: RolloutRewardProjectionSchema,
+    traceRef: ImmutableArtifactRefSchema,
+    optimizerSample: OptimizerTrainingSampleSchema.nullable(),
+    environmentExecutions: z.array(EnvironmentExecutionEvidenceSchema).min(1).max(100_000),
+    startedAt: ReleaseTimestampSchema,
+    completedAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict();
+function validateCanonicalRolloutRecord(record, context) {
+    if (record.reward.status === "scored" && (record.reward.value === null || !record.reward.learningEligible)) {
+        context.addIssue({
+            code: "custom",
+            path: ["reward", "status"],
+            message: "A scored rollout requires a numeric, learning-eligible reward.",
+        });
+    }
+    if (record.reward.status === "unscorable" && (record.reward.value !== null || record.reward.learningEligible)) {
+        context.addIssue({
+            code: "custom",
+            path: ["reward", "status"],
+            message: "An unscorable rollout has no reward and cannot be learning-eligible.",
+        });
+    }
+}
+const CanonicalRolloutRecordBaseSchema = CanonicalRolloutRecordFieldsSchema
+    .superRefine(validateCanonicalRolloutRecord);
+export const CanonicalRolloutRecordSchema = z.object({
+    ...CanonicalRolloutRecordFieldsSchema.shape,
+    contentHash: ReleaseHashSchema,
+}).strict().superRefine(validateCanonicalRolloutRecord);
+export const RolloutQualificationSchema = z.object({
+    schemaVersion: z.literal("openpond.rolloutQualification.v1"),
+    rolloutCount: z.number().int().nonnegative(),
+    scoredCount: z.number().int().nonnegative(),
+    optimizerEligibleCount: z.number().int().nonnegative(),
+    unscorableCount: z.number().int().nonnegative(),
+    zeroRewardCount: z.number().int().nonnegative(),
+    rewardMean: z.number().min(0).max(1).nullable(),
+    rewardVariance: z.number().nonnegative().nullable(),
+    distinctRewardCount: z.number().int().nonnegative(),
+    eligibleForRl: z.boolean(),
+    reasons: z.array(z.string().trim().min(1).max(1_000)).max(100),
+}).strict();
+export function createCanonicalRolloutRecord(input) {
+    const attemptReceipt = AttemptReceiptSchema.parse(input.attemptReceipt);
+    if (!verifyAttemptReceipt(attemptReceipt)) {
+        throw new Error("Rollout Attempt Receipt failed content-hash verification.");
+    }
+    const rewardReceipt = RewardReceiptSchema.parse(input.rewardReceipt);
+    if (rewardReceipt.attemptRef.id !== attemptReceipt.id
+        || rewardReceipt.attemptRef.contentHash !== attemptReceipt.contentHash) {
+        throw new Error("Rollout Attempt Receipt does not match its Reward Receipt.");
+    }
+    if (rewardReceipt.artifactManifestRef.id !== input.artifactManifestRef.id
+        || rewardReceipt.artifactManifestRef.contentHash !== input.artifactManifestRef.contentHash) {
+        throw new Error("Rollout Artifact Manifest does not match its Reward Receipt.");
+    }
+    if (input.environmentExecutions.some((execution) => execution.environmentRelease.id !== input.environmentRelease.id
+        || execution.environmentRelease.contentHash !== input.environmentRelease.contentHash)) {
+        throw new Error("Rollout Environment execution does not match its admitted Environment Release.");
+    }
+    const content = CanonicalRolloutRecordBaseSchema.parse({
+        schemaVersion: "openpond.canonicalRolloutRecord.v1",
+        id: input.id,
+        attemptRef: rewardReceipt.attemptRef,
+        artifactManifestRef: input.artifactManifestRef,
+        tasksetRelease: input.tasksetRelease,
+        environmentRelease: input.environmentRelease,
+        verifierSetRelease: rewardReceipt.verifierSetRef,
+        harnessRelease: input.harnessRelease,
+        taskId: input.taskId,
+        split: input.split,
+        model: input.model,
+        seed: input.seed,
+        reward: {
+            receiptRef: { id: rewardReceipt.id, contentHash: rewardReceipt.contentHash },
+            status: rewardReceipt.status,
+            value: rewardReceipt.reward,
+            learningEligible: rewardReceipt.learningEligible,
+            passed: rewardReceipt.passed,
+            outcomeClass: rewardReceipt.outcomeClass,
+            failureOwner: rewardReceipt.failureOwner,
+            components: Object.fromEntries(rewardReceipt.components.map((component) => [
+                component.verifierId,
+                component.rewardContribution,
+            ])),
+        },
+        traceRef: input.traceRef,
+        optimizerSample: input.optimizerSample,
+        environmentExecutions: input.environmentExecutions,
+        startedAt: input.startedAt,
+        completedAt: input.completedAt,
+        metadata: input.metadata ?? {},
+    });
+    return CanonicalRolloutRecordSchema.parse({
+        ...content,
+        contentHash: contentHash(content),
+    });
+}
+export function verifyCanonicalRolloutRecord(value) {
+    const parsed = CanonicalRolloutRecordSchema.safeParse(value);
+    if (!parsed.success)
+        return false;
+    const { contentHash: actual, ...content } = parsed.data;
+    const canonical = CanonicalRolloutRecordBaseSchema.safeParse(content);
+    return canonical.success && contentHash(canonical.data) === actual;
+}
+export function optimizerEligibleRollouts(records) {
+    return records.filter((record) => record.reward.status === "scored"
+        && record.reward.learningEligible
+        && record.reward.value !== null
+        && record.optimizerSample !== null);
+}
+export function qualifyRolloutBatch(input) {
+    const records = input.records.map((record) => CanonicalRolloutRecordSchema.parse(record));
+    const scored = records.filter((record) => record.reward.status === "scored"
+        && record.reward.learningEligible
+        && record.reward.value !== null);
+    const optimizerEligible = optimizerEligibleRollouts(records);
+    const rewards = scored.map((record) => record.reward.value);
+    const mean = rewards.length
+        ? rewards.reduce((total, reward) => total + reward, 0) / rewards.length
+        : null;
+    const variance = mean === null
+        ? null
+        : rewards.reduce((total, reward) => total + (reward - mean) ** 2, 0) / rewards.length;
+    const distinctRewardCount = new Set(rewards.map((reward) => reward.toPrecision(12))).size;
+    const minimumScoredRollouts = input.minimumScoredRollouts ?? 2;
+    const minimumDistinctRewards = input.minimumDistinctRewards ?? 2;
+    const minimumRewardVariance = input.minimumRewardVariance ?? Number.EPSILON;
+    const reasons = [];
+    if (scored.length < minimumScoredRollouts)
+        reasons.push(`Requires at least ${minimumScoredRollouts} scored rollouts.`);
+    if (optimizerEligible.length !== scored.length)
+        reasons.push("Every scored rollout requires aligned optimizer token evidence.");
+    if (distinctRewardCount < minimumDistinctRewards)
+        reasons.push(`Requires at least ${minimumDistinctRewards} distinct reward values.`);
+    if (variance === null || variance < minimumRewardVariance)
+        reasons.push(`Reward variance must be at least ${minimumRewardVariance}.`);
+    return RolloutQualificationSchema.parse({
+        schemaVersion: "openpond.rolloutQualification.v1",
+        rolloutCount: records.length,
+        scoredCount: scored.length,
+        optimizerEligibleCount: optimizerEligible.length,
+        unscorableCount: records.length - scored.length,
+        zeroRewardCount: rewards.filter((reward) => reward === 0).length,
+        rewardMean: mean,
+        rewardVariance: variance,
+        distinctRewardCount,
+        eligibleForRl: reasons.length === 0,
+        reasons,
+    });
+}

package/dist/tasksets.js

@@ -1,6 +1,13 @@
 import { z } from "zod";
 import { CapabilityRequirementSchema, ImmutableAssetRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ToolDeclarationSchema, assertContentHash, contentHash, } from "@openpond/harness";
 export const TaskSplitSchema = z.enum(["train", "validation", "test", "frozen_eval"]);
+export const RequiredOutputContractSchema = z.object({
+    path: z.string().trim().min(1).max(2_000).refine(safeRelativePath),
+    mediaType: z.string().trim().min(1).max(200),
+    schemaRef: ImmutableAssetRefSchema.nullable().default(null),
+    maxBytes: z.number().int().positive().max(250_000_000).nullable().default(null),
+    metadata: MetadataSchema,
+}).strict();
 export const PolicyBoundarySchema = z.object({
     policyVisibleFields: z.array(ReleaseIdSchema).max(1_000).default([]),
     privilegedFields: z.array(ReleaseIdSchema).max(1_000).default([]),
@@ -60,6 +67,7 @@ export const TaskRecordSchema = z.object({
     policyVisibleContext: z.record(z.string(), z.unknown()).default({}),
     privilegedContextRef: ReleaseIdSchema.nullable(),
     artifactRefs: z.array(ImmutableAssetRefSchema).max(1_000).default([]),
+    requiredOutputs: z.array(RequiredOutputContractSchema).max(1_000).optional(),
     tags: z.array(ReleaseIdSchema).max(100).default([]),
 }).strict();
 export const TasksetReleaseContentSchema = z.object({
@@ -68,10 +76,12 @@ export const TasksetReleaseContentSchema = z.object({
     revision: z.number().int().positive(),
     policy: PolicyBoundarySchema,
     environment: EnvironmentContractSchema,
+    environmentRelease: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().optional(),
     tools: z.array(ToolDeclarationSchema).max(200),
     capabilities: z.array(CapabilityRequirementSchema).max(200),
     tasks: z.array(TaskRecordSchema).min(1).max(1_000_000),
     graders: z.array(GraderSpecSchema).min(1).max(1_000),
+    verifierSetRelease: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().optional(),
     metadata: MetadataSchema,
 }).strict();
 export const TasksetReleaseSchema = TasksetReleaseContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
@@ -92,6 +102,14 @@ export function validateTasksetRelease(input) {
     }
     const taskset = parsed.data;
     const issues = [];
+    if (Boolean(taskset.environmentRelease) !== Boolean(taskset.verifierSetRelease)) {
+        issues.push({
+            code: "execution_release_binding_incomplete",
+            severity: "error",
+            message: "A Taskset Release must bind both Environment and Verifier Set releases or neither during v2 migration.",
+            path: "environmentRelease",
+        });
+    }
     const clusterSplits = new Map();
     for (const task of taskset.tasks) {
         const splits = clusterSplits.get(task.clusterKey) ?? new Set();
@@ -147,3 +165,9 @@ export function trainingPolicyTaskViews(taskset) {
         .map(policyTaskView);
 }
 export { CapabilityRequirementSchema, ToolDeclarationSchema, } from "@openpond/harness";
+function safeRelativePath(value) {
+    const normalized = value.replaceAll("\\", "/");
+    if (!normalized || normalized.startsWith("/") || normalized.includes("\0"))
+        return false;
+    return !normalized.split("/").some((part) => !part || part === "." || part === "..");
+}

package/dist/telemetry/index.js

@@ -0,0 +1,4 @@
+export * from "../telemetry.js";
+export * from "../telemetry-catalog.js";
+export * from "../telemetry-analysis.js";
+export * from "../telemetry-bundle.js";

package/dist/telemetry-analysis.js

@@ -0,0 +1,183 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash } from "@openpond/harness";
+import { getCoreMetricDefinition, validateCoreMetricObservation } from "./telemetry-catalog.js";
+import { MetricObservationSchema, RunTelemetryEventSchema, TelemetryVisibilitySchema, } from "./telemetry.js";
+export const TelemetryCohortSchema = z.object({
+    checkpointIds: z.array(ReleaseIdSchema).max(1_000),
+    steps: z.array(z.number().int().nonnegative()).max(10_000),
+    scenarioIds: z.array(ReleaseIdSchema).max(100_000),
+    rolloutGroupIds: z.array(ReleaseIdSchema).max(100_000),
+    attemptIds: z.array(ReleaseIdSchema).max(1_000_000),
+    splits: z.array(z.string().trim().min(1).max(100)).max(32),
+    failureOwners: z.array(z.string().trim().min(1).max(100)).max(32),
+    graders: z.array(z.string().trim().min(1).max(200)).max(1_000),
+    rewardEligible: z.boolean().nullable(),
+}).strict();
+export const EvidenceReferenceSchema = z.object({
+    id: ReleaseIdSchema,
+    contentHash: ReleaseHashSchema,
+    kind: z.enum(["rollout", "attempt", "trace", "grader", "checkpoint", "artifact"]),
+    visibility: TelemetryVisibilitySchema,
+}).strict();
+export const EvidenceCompletenessSchema = z.object({
+    schemaVersion: z.literal("openpond.telemetryEvidenceCompleteness.v1"),
+    runId: ReleaseIdSchema,
+    status: z.enum(["complete", "partial", "missing"]),
+    expectedEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    observedEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    missingEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    lastSequence: z.number().int().nonnegative().nullable(),
+    sequenceGaps: z.array(z.number().int().nonnegative()).max(10_000),
+}).strict();
+export const MetricSeriesPointSchema = z.object({
+    step: z.number().int().nonnegative().nullable(),
+    observedAt: ReleaseTimestampSchema,
+    value: z.number().finite(),
+    sampleCount: z.number().int().positive(),
+}).strict();
+export const RunMetricSummarySchema = z.object({
+    schemaVersion: z.literal("openpond.runMetricSummary.v1"),
+    runId: ReleaseIdSchema,
+    metricId: ReleaseIdSchema,
+    aggregation: z.enum(["last", "sum", "mean", "min", "max", "p50", "p95"]),
+    value: z.number().finite().nullable(),
+    sampleCount: z.number().int().nonnegative(),
+    series: z.array(MetricSeriesPointSchema).max(100_000),
+}).strict();
+export function createRunTelemetryEvent(input) {
+    const idHash = contentHash({ runId: input.lineage.runId, sequence: input.sequence, type: input.type, source: input.source });
+    return RunTelemetryEventSchema.parse({
+        schemaVersion: "openpond.runTelemetryEvent.v1",
+        eventId: `telemetry-${idHash.slice(0, 32)}`,
+        ...input,
+        attributes: input.attributes ?? {},
+    });
+}
+export function createMetricObservation(input) {
+    const idHash = contentHash({ runId: input.lineage.runId, metricId: input.metricId, sequence: input.sequence });
+    return validateCoreMetricObservation(MetricObservationSchema.parse({
+        schemaVersion: "openpond.metricObservation.v1",
+        observationId: `metric-${idHash.slice(0, 32)}`,
+        ...input,
+        dimensions: input.dimensions ?? {},
+    }));
+}
+export function telemetryIdempotencyKey(item) {
+    const id = item.schemaVersion === "openpond.metricObservation.v1" ? item.observationId : item.eventId;
+    return `${item.lineage.runId}:${item.sequence}:${id}`;
+}
+export function mergeTelemetryItems(input) {
+    const accepted = new Map();
+    const sequences = new Map();
+    for (const item of [...input.events, ...input.observations]) {
+        const key = telemetryIdempotencyKey(item);
+        const existing = accepted.get(key);
+        if (existing) {
+            if (contentHash(existing) !== contentHash(item))
+                throw new Error(`Telemetry idempotency conflict: ${key}`);
+            continue;
+        }
+        const sequenceKey = `${item.lineage.runId}:${item.sequence}`;
+        const priorKey = sequences.get(sequenceKey);
+        if (priorKey && priorKey !== key)
+            throw new Error(`Telemetry sequence conflict: ${sequenceKey}`);
+        sequences.set(sequenceKey, key);
+        accepted.set(key, item);
+    }
+    const sorted = [...accepted.values()].sort((left, right) => left.sequence - right.sequence);
+    return {
+        events: sorted.filter((item) => item.schemaVersion === "openpond.runTelemetryEvent.v1"),
+        observations: sorted.filter((item) => item.schemaVersion === "openpond.metricObservation.v1"),
+    };
+}
+function matchesOptionalSet(values, candidate) {
+    return values.length === 0 || (candidate !== null && values.includes(candidate));
+}
+export function filterTelemetryCohort(input) {
+    const cohort = TelemetryCohortSchema.parse(input.cohort);
+    const lineageMatches = (lineage) => matchesOptionalSet(cohort.checkpointIds, lineage.checkpointId)
+        && matchesOptionalSet(cohort.steps, lineage.step)
+        && matchesOptionalSet(cohort.scenarioIds, lineage.scenarioId)
+        && matchesOptionalSet(cohort.rolloutGroupIds, lineage.rolloutGroupId)
+        && matchesOptionalSet(cohort.attemptIds, lineage.attemptId);
+    const attributeMatches = (values) => matchesOptionalSet(cohort.splits, typeof values.split === "string" ? values.split : null)
+        && matchesOptionalSet(cohort.failureOwners, typeof values.failureOwner === "string" ? values.failureOwner : null)
+        && matchesOptionalSet(cohort.graders, typeof values.grader === "string" ? values.grader : null)
+        && (cohort.rewardEligible === null || values.rewardEligible === cohort.rewardEligible);
+    return {
+        events: input.events.filter((event) => lineageMatches(event.lineage) && attributeMatches(event.attributes)),
+        observations: input.observations.filter((observation) => lineageMatches(observation.lineage) && attributeMatches(observation.dimensions)),
+    };
+}
+export function deriveEvidenceCompleteness(input) {
+    const events = input.events.filter((event) => event.lineage.runId === input.runId).sort((left, right) => left.sequence - right.sequence);
+    const observed = [...new Set(events.map((event) => event.type))].sort();
+    const missing = [...new Set(input.expectedEventTypes)].filter((type) => !observed.includes(type)).sort();
+    const sequenceValues = [...new Set(events.map((event) => event.sequence))].sort((left, right) => left - right);
+    const lastSequence = events.at(-1)?.sequence ?? null;
+    const sequenceGaps = [];
+    let expected = 0;
+    for (const sequence of sequenceValues) {
+        while (expected < sequence && sequenceGaps.length < 10_000)
+            sequenceGaps.push(expected++);
+        expected = sequence + 1;
+    }
+    return EvidenceCompletenessSchema.parse({
+        schemaVersion: "openpond.telemetryEvidenceCompleteness.v1",
+        runId: input.runId,
+        status: events.length === 0 ? "missing" : missing.length || sequenceGaps.length ? "partial" : "complete",
+        expectedEventTypes: [...new Set(input.expectedEventTypes)].sort(),
+        observedEventTypes: observed,
+        missingEventTypes: missing,
+        lastSequence,
+        sequenceGaps,
+    });
+}
+function aggregate(values, method) {
+    if (!values.length)
+        return null;
+    if (method === "last")
+        return values.at(-1) ?? null;
+    if (method === "sum")
+        return values.reduce((total, value) => total + value, 0);
+    if (method === "mean")
+        return values.reduce((total, value) => total + value, 0) / values.length;
+    if (method === "min")
+        return Math.min(...values);
+    if (method === "max")
+        return Math.max(...values);
+    const sorted = [...values].sort((left, right) => left - right);
+    const percentile = method === "p50" ? 0.5 : 0.95;
+    return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * percentile) - 1)] ?? null;
+}
+export function summarizeMetric(runId, metricId, observations) {
+    const definition = getCoreMetricDefinition(metricId);
+    if (!definition)
+        throw new Error(`Unknown core metric: ${metricId}`);
+    const selected = observations.filter((item) => item.lineage.runId === runId && item.metricId === metricId).sort((left, right) => left.sequence - right.sequence);
+    const buckets = new Map();
+    for (const item of selected) {
+        const key = item.lineage.step === null ? `time:${item.observedAt}` : `step:${item.lineage.step}`;
+        buckets.set(key, [...(buckets.get(key) ?? []), item]);
+    }
+    const series = [...buckets.values()].map((items) => ({
+        step: items[0]?.lineage.step ?? null,
+        observedAt: items[0]?.observedAt,
+        value: aggregate(items.map((item) => item.value), definition.aggregation),
+        sampleCount: items.length,
+    })).filter((point) => point.value !== null && point.observedAt !== undefined);
+    return RunMetricSummarySchema.parse({ schemaVersion: "openpond.runMetricSummary.v1", runId, metricId, aggregation: definition.aggregation, value: aggregate(selected.map((item) => item.value), definition.aggregation), sampleCount: selected.length, series });
+}
+export function redactTelemetryEvent(event, maximumVisibility) {
+    const rank = { policy_visible: 0, team_visible: 1, host_private: 2 };
+    if (rank[event.visibility] > rank[maximumVisibility])
+        return null;
+    return RunTelemetryEventSchema.parse(event);
+}
+export function redactTelemetryAttributes(event, deniedKeys) {
+    const denied = new Set(deniedKeys);
+    return RunTelemetryEventSchema.parse({
+        ...event,
+        attributes: Object.fromEntries(Object.entries(event.attributes).filter(([key]) => !denied.has(key))),
+    });
+}

package/dist/telemetry-bundle.js

@@ -0,0 +1,60 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash } from "@openpond/harness";
+import { MetricCatalogSchema } from "./telemetry-catalog.js";
+import { deriveEvidenceCompleteness, EvidenceCompletenessSchema, EvidenceReferenceSchema, redactTelemetryAttributes, redactTelemetryEvent } from "./telemetry-analysis.js";
+import { MetricObservationSchema, RunTelemetryEventSchema } from "./telemetry.js";
+export const TelemetryExportBundleContentSchema = z.object({
+    schemaVersion: z.literal("openpond.telemetryExportBundle.v1"),
+    id: ReleaseIdSchema,
+    runId: ReleaseIdSchema,
+    exportedAt: ReleaseTimestampSchema,
+    definitions: MetricCatalogSchema,
+    events: z.array(RunTelemetryEventSchema).max(1_000_000),
+    observations: z.array(MetricObservationSchema).max(10_000_000),
+    evidenceRefs: z.array(EvidenceReferenceSchema).max(1_000_000),
+    completeness: EvidenceCompletenessSchema,
+}).strict();
+export const TelemetryExportBundleSchema = TelemetryExportBundleContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+export function createTelemetryExportBundle(input) {
+    const content = TelemetryExportBundleContentSchema.parse(input);
+    if (content.completeness.runId !== content.runId) {
+        throw new Error("Telemetry export completeness belongs to another Run.");
+    }
+    if (content.events.some((event) => event.lineage.runId !== content.runId) || content.observations.some((observation) => observation.lineage.runId !== content.runId)) {
+        throw new Error("Telemetry export bundle contains evidence from another Run.");
+    }
+    return TelemetryExportBundleSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function verifyTelemetryExportBundle(input) {
+    const parsed = TelemetryExportBundleSchema.safeParse(input);
+    if (!parsed.success)
+        return false;
+    const { contentHash: actual, ...content } = parsed.data;
+    return contentHash(TelemetryExportBundleContentSchema.parse(content)) === actual;
+}
+export function redactTelemetryExportBundle(input) {
+    const bundle = TelemetryExportBundleSchema.parse(input.bundle);
+    if (!verifyTelemetryExportBundle(bundle))
+        throw new Error("Telemetry export bundle has an invalid content hash.");
+    const rank = { policy_visible: 0, team_visible: 1, host_private: 2 };
+    const definitions = bundle.definitions.filter((definition) => rank[definition.visibility] <= rank[input.maximumVisibility]);
+    const metricIds = new Set(definitions.map((definition) => definition.id));
+    const events = bundle.events.flatMap((event) => {
+        const visible = redactTelemetryEvent(event, input.maximumVisibility);
+        return visible ? [redactTelemetryAttributes(visible, input.deniedAttributeKeys ?? [])] : [];
+    });
+    const expectedEventTypes = bundle.completeness.expectedEventTypes.filter((type) => events.some((event) => event.type === type));
+    return createTelemetryExportBundle({
+        schemaVersion: "openpond.telemetryExportBundle.v1",
+        id: input.id,
+        runId: bundle.runId,
+        exportedAt: input.exportedAt,
+        definitions,
+        events,
+        observations: bundle.observations.filter((observation) => metricIds.has(observation.metricId)),
+        evidenceRefs: bundle.evidenceRefs.filter((reference) => rank[reference.visibility] <= rank[input.maximumVisibility]),
+        completeness: deriveEvidenceCompleteness({ runId: bundle.runId, events, expectedEventTypes: expectedEventTypes }),
+    });
+}

package/dist/telemetry-catalog.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { MetricDefinitionSchema } from "./telemetry.js";
+const definition = (input) => MetricDefinitionSchema.parse({ schemaVersion: "openpond.metricDefinition.v1", ...input });
+export const CORE_METRIC_CATALOG = [
+    definition({ id: "reward.mean", displayName: "Mean reward", description: "Mean composed reward for the cohort.", valueType: "gauge", unit: "scalar", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split", "grader"] }),
+    definition({ id: "reward.variance", displayName: "Reward variance", description: "Population variance of composed reward within a rollout group.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "reward.constant_group_rate", displayName: "Constant group rate", description: "Fraction of rollout groups with no reward variation.", valueType: "gauge", unit: "ratio", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "attempt.valid_rate", displayName: "Valid attempt rate", description: "Fraction of attempts passing deterministic validity checks.", valueType: "gauge", unit: "ratio", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split", "failureOwner"] }),
+    definition({ id: "attempt.failure_count", displayName: "Attempt failures", description: "Count of failed Attempts.", valueType: "counter", unit: "count", direction: "lower", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "failureOwner", "failureClass"] }),
+    definition({ id: "optimizer.loss", displayName: "Optimizer loss", description: "Policy optimizer loss after the step.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.learning_rate", displayName: "Learning rate", description: "Optimizer learning rate after the step.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "last", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.kl", displayName: "KL divergence", description: "Sampled KL divergence from the reference policy.", valueType: "gauge", unit: "scalar", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.entropy", displayName: "Policy entropy", description: "Observed policy entropy for trainable tokens.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.gradient_norm", displayName: "Gradient norm", description: "Gradient norm reported by the optimizer.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "max", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.clip_fraction", displayName: "Clip fraction", description: "Fraction of policy updates affected by clipping.", valueType: "gauge", unit: "ratio", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "output.duplicate_rate", displayName: "Duplicate output rate", description: "Fraction of outputs duplicated within the measured cohort.", valueType: "gauge", unit: "ratio", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "output.unique_count", displayName: "Unique outputs", description: "Distinct output count in the measured cohort.", valueType: "gauge", unit: "count", direction: "higher", aggregation: "last", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "tokens.input", displayName: "Input tokens", description: "Input tokens processed.", valueType: "counter", unit: "tokens", direction: "neutral", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "source"] }),
+    definition({ id: "tokens.output", displayName: "Output tokens", description: "Output tokens generated.", valueType: "counter", unit: "tokens", direction: "neutral", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "source"] }),
+    definition({ id: "runtime.latency_ms", displayName: "Runtime latency", description: "Wall-clock latency of the measured operation.", valueType: "distribution", unit: "milliseconds", direction: "lower", aggregation: "p95", visibility: "team_visible", boundedDimensions: ["operation", "provider"] }),
+    definition({ id: "runtime.throughput", displayName: "Token throughput", description: "Tokens processed per second.", valueType: "gauge", unit: "scalar", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["operation", "provider"] }),
+    definition({ id: "gpu.memory_bytes", displayName: "GPU memory", description: "Peak allocated GPU memory.", valueType: "gauge", unit: "bytes", direction: "neutral", aggregation: "max", visibility: "host_private", boundedDimensions: ["provider", "gpuType"] }),
+    definition({ id: "gpu.utilization", displayName: "GPU utilization", description: "Observed GPU utilization ratio.", valueType: "gauge", unit: "ratio", direction: "neutral", aggregation: "mean", visibility: "host_private", boundedDimensions: ["provider", "gpuType"] }),
+    definition({ id: "cost.usd", displayName: "Run cost", description: "Accrued hosted execution cost.", valueType: "counter", unit: "usd", direction: "lower", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["provider", "resource"] }),
+];
+const catalog = new Map(CORE_METRIC_CATALOG.map((item) => [item.id, item]));
+export function getCoreMetricDefinition(metricId) {
+    return catalog.get(metricId);
+}
+export function validateCoreMetricObservation(input) {
+    return validateMetricObservation(input, CORE_METRIC_CATALOG);
+}
+export function validateMetricObservation(input, definitions) {
+    MetricCatalogSchema.parse(definitions);
+    const metric = definitions.find((item) => item.id === input.metricId);
+    if (!metric)
+        throw new Error(`Unknown metric: ${input.metricId}`);
+    const unexpected = Object.keys(input.dimensions).filter((dimension) => !metric.boundedDimensions.includes(dimension));
+    if (unexpected.length)
+        throw new Error(`Metric ${input.metricId} has unsupported dimensions: ${unexpected.join(", ")}`);
+    if (metric.unit === "ratio" && (input.value < 0 || input.value > 1)) {
+        throw new Error(`Ratio metric ${input.metricId} must be between zero and one.`);
+    }
+    return input;
+}
+export const MetricCatalogSchema = z.array(MetricDefinitionSchema).min(1).superRefine((items, context) => {
+    if (new Set(items.map((item) => item.id)).size !== items.length) {
+        context.addIssue({ code: "custom", message: "Metric catalog contains duplicate ids." });
+    }
+});