@openpond/evals 0.5.0 → 0.7.0

package/dist/rewards.js

@@ -0,0 +1,230 @@
+import { LearningDomainError } from "./learning/errors.js";
+import { z } from "zod";
+import { ImmutableAssetRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
+import { gradeEvidence } from "./graders.js";
+import { CustomVerifierGraderSpecSchema, DeterministicGraderSpecSchema, GraderSpecSchema, HumanGraderSpecSchema, ModelJudgeGraderSpecSchema } from "./tasksets.js";
+import { assertBoundedTaskJson, validateTaskSchema } from "./task-schema.js";
+const bindingFields = { id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
+export const RewardImplementationSchema = z.union([
+    DeterministicGraderSpecSchema.omit(bindingFields),
+    ModelJudgeGraderSpecSchema.omit(bindingFields),
+    CustomVerifierGraderSpecSchema.omit(bindingFields),
+    HumanGraderSpecSchema.omit(bindingFields),
+    z.object({ kind: z.literal("learned_model"), modelVersion: ImmutableReleaseRefSchema, inputContract: ImmutableAssetRefSchema }).strict(),
+]);
+export const RewardReleaseContentSchema = z.object({
+    schemaVersion: z.literal("openpond.rewardRelease.v1"),
+    id: ReleaseIdSchema,
+    revision: z.number().int().positive(),
+    name: z.string().trim().min(1).max(500),
+    description: z.string().max(10_000),
+    implementation: RewardImplementationSchema,
+    rawScore: z.object({ minimum: z.number().finite(), maximum: z.number().finite() }).strict(),
+    assets: z.array(ImmutableAssetRefSchema).max(1_000),
+}).strict().superRefine((reward, context) => {
+    if (reward.rawScore.minimum >= reward.rawScore.maximum)
+        context.addIssue({ code: "custom", path: ["rawScore"], message: "A raw score contract requires minimum < maximum." });
+    if (reward.implementation.kind !== "learned_model" && (reward.rawScore.minimum !== 0 || reward.rawScore.maximum !== 1))
+        context.addIssue({ code: "custom", path: ["rawScore"], message: "Portable graders currently return scores in [0, 1]." });
+    if (reward.implementation.kind === "schema") {
+        const report = validateTaskSchema(reward.implementation.config.jsonSchema);
+        if (!report.valid)
+            context.addIssue({ code: "custom", path: ["implementation", "config", "jsonSchema"], message: report.issues[0].message });
+    }
+});
+export const RewardReleaseSchema = RewardReleaseContentSchema.safeExtend({ contentHash: ReleaseHashSchema }).strict();
+export const RewardReleaseRefSchema = ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict();
+export const RewardNormalizationSchema = z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("identity") }).strict(),
+    z.object({ kind: z.literal("linear"), minimum: z.number().finite(), maximum: z.number().finite(), direction: z.enum(["higher", "lower"]) }).strict(),
+]);
+export const RewardBindingSourceSchema = z.object({
+    graderId: ReleaseIdSchema,
+    reward: RewardReleaseRefSchema,
+    role: z.enum(["training", "evaluation"]),
+    normalization: RewardNormalizationSchema,
+    weight: z.number().nonnegative().max(1_000),
+    required: z.boolean(),
+    hardGate: z.boolean(),
+    privileged: z.boolean(),
+    fixtureRefs: z.array(ImmutableReleaseRefSchema).max(1_000),
+}).strict();
+export const RewardBindingContentSchema = z.object({
+    schemaVersion: z.literal("openpond.rewardBinding.v1"),
+    id: ReleaseIdSchema,
+    revision: z.number().int().positive(),
+    name: z.string().trim().min(1).max(500).optional(),
+    description: z.string().max(10_000).optional(),
+    recipeRef: RewardReleaseRefSchema.optional(),
+    sources: z.array(RewardBindingSourceSchema).min(1).max(100),
+    aggregation: z.literal("weighted_mean"),
+    unscorable: z.literal("exclude_optional_require_all_required"),
+}).strict().superRefine((binding, context) => {
+    if (new Set(binding.sources.map((source) => source.graderId)).size !== binding.sources.length)
+        context.addIssue({ code: "custom", path: ["sources"], message: "Bound grader identities must be unique." });
+    if (!binding.sources.some((source) => source.weight > 0))
+        context.addIssue({ code: "custom", path: ["sources"], message: "A binding needs at least one positively weighted source." });
+    for (const [index, source] of binding.sources.entries()) {
+        if (source.hardGate && !source.required)
+            context.addIssue({ code: "custom", path: ["sources", index], message: "A hard gate must be required." });
+        if (source.normalization.kind === "linear" && source.normalization.minimum >= source.normalization.maximum)
+            context.addIssue({ code: "custom", path: ["sources", index, "normalization"], message: "Linear normalization requires minimum < maximum." });
+    }
+});
+export const RewardBindingSchema = RewardBindingContentSchema.safeExtend({ contentHash: ReleaseHashSchema }).strict();
+export const BoundRewardResultSchema = z.object({
+    graderId: ReleaseIdSchema,
+    reward: RewardReleaseRefSchema,
+    role: z.enum(["training", "evaluation"]),
+    status: z.enum(["scored", "pending", "unavailable", "failed"]),
+    rawScore: z.number().finite().nullable(),
+    normalizedScore: z.number().min(0).max(1).nullable(),
+    passed: z.boolean().nullable(),
+    evidenceHashes: z.array(ReleaseHashSchema).max(1_000),
+    message: z.string().max(20_000).nullable(),
+}).strict();
+export const RewardCompositionContentSchema = z.object({
+    schemaVersion: z.literal("openpond.rewardComposition.v1"),
+    binding: RewardReleaseRefSchema,
+    taskHash: ReleaseHashSchema,
+    outputHash: ReleaseHashSchema,
+    results: z.array(BoundRewardResultSchema).min(1).max(100),
+    training: z.object({ status: z.enum(["scored", "unscorable", "not_configured"]), score: z.number().min(0).max(1).nullable(), passed: z.boolean().nullable() }).strict(),
+    evaluation: z.object({ status: z.enum(["scored", "unscorable", "not_configured"]), score: z.number().min(0).max(1).nullable(), passed: z.boolean().nullable() }).strict(),
+}).strict();
+export const RewardCompositionSchema = RewardCompositionContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
+export function createRewardRelease(input) {
+    assertBoundedTaskJson(input);
+    const content = RewardReleaseContentSchema.parse(input);
+    return RewardReleaseSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function createRewardBinding(input, rewards) {
+    const content = RewardBindingContentSchema.parse(input);
+    const binding = RewardBindingSchema.parse({ ...content, contentHash: contentHash(content) });
+    resolveBoundRewards(binding, rewards);
+    return binding;
+}
+export function resolveBoundRewards(input, rewards) {
+    const binding = RewardBindingSchema.parse(input);
+    assertRewardHash(binding);
+    return binding.sources.map((source) => {
+        const reward = rewards.find((candidate) => candidate.id === source.reward.id && candidate.revision === source.reward.revision && candidate.contentHash === source.reward.contentHash);
+        if (!reward)
+            throw new LearningDomainError("reward_release_missing", 422, source.graderId);
+        const parsed = RewardReleaseSchema.parse(reward);
+        assertRewardHash(parsed);
+        if (parsed.implementation.kind === "human" && source.role === "training")
+            throw new LearningDomainError("human_reward_not_executable", 422, source.graderId);
+        if (source.normalization.kind === "identity" && (parsed.rawScore.minimum < 0 || parsed.rawScore.maximum > 1))
+            throw new LearningDomainError("reward_normalization_required", 422, source.graderId);
+        if (source.normalization.kind === "linear" && (source.normalization.minimum !== parsed.rawScore.minimum || source.normalization.maximum !== parsed.rawScore.maximum))
+            throw new LearningDomainError("reward_normalization_contract_mismatch", 422, source.graderId);
+        return { source, reward: parsed };
+    });
+}
+export function compileBoundGraders(binding, rewards) {
+    return resolveBoundRewards(binding, rewards).map(({ source, reward }) => {
+        if (reward.implementation.kind === "learned_model")
+            throw new LearningDomainError("learned_reward_requires_model_execution_adapter");
+        return compileSource(source, reward);
+    });
+}
+export async function executeRewardBinding(input) {
+    const resolved = resolveBoundRewards(input.binding, input.rewards);
+    const results = [];
+    for (const { source, reward } of resolved) {
+        input.signal?.throwIfAborted();
+        const result = { graderId: source.graderId, reward: source.reward, role: source.role, status: "unavailable", rawScore: null, normalizedScore: null, passed: null, evidenceHashes: [], message: null };
+        try {
+            if (reward.implementation.kind === "human") {
+                result.status = "pending";
+                result.message = "Human review is pending.";
+            }
+            else if (reward.implementation.kind === "learned_model") {
+                if (!input.learnedReward)
+                    result.message = "Learned reward execution is not configured.";
+                else {
+                    const scored = await input.learnedReward({ reward, task: input.task, evidence: input.evidence, signal: input.signal });
+                    Object.assign(result, { status: "scored", ...scored });
+                }
+            }
+            else {
+                const [evidence] = await gradeEvidence({ task: input.task, evidence: input.evidence, graders: [compileSource(source, reward)], modelJudge: input.modelJudge, customVerifier: input.customVerifier });
+                if (!evidence)
+                    throw new LearningDomainError("reward_evidence_missing");
+                Object.assign(result, { status: evidence.score === null ? "unavailable" : "scored", rawScore: evidence.score, passed: evidence.score === null ? null : evidence.passed, evidenceHashes: [evidence.contentHash], message: evidence.feedback.join("\n") });
+            }
+            if (result.status === "scored") {
+                if (result.rawScore === null || !Number.isFinite(result.rawScore) || result.rawScore < reward.rawScore.minimum || result.rawScore > reward.rawScore.maximum)
+                    throw new LearningDomainError("reward_raw_score_outside_contract");
+                const normalization = source.normalization;
+                result.normalizedScore = normalization.kind === "identity" ? result.rawScore : (result.rawScore - normalization.minimum) / (normalization.maximum - normalization.minimum);
+                if (normalization.kind === "linear" && normalization.direction === "lower")
+                    result.normalizedScore = 1 - result.normalizedScore;
+            }
+        }
+        catch (error) {
+            input.signal?.throwIfAborted();
+            Object.assign(result, { status: "failed", rawScore: null, normalizedScore: null, passed: null, message: error instanceof Error ? error.message : "Reward execution failed." });
+        }
+        results.push(BoundRewardResultSchema.parse(result));
+    }
+    return composeBoundRewards({ binding: input.binding, taskHash: contentHash(input.task), outputHash: contentHash(input.evidence), results });
+}
+export function composeBoundRewards(input) {
+    const binding = RewardBindingSchema.parse(input.binding);
+    assertRewardHash(binding);
+    const results = input.results.map((result) => BoundRewardResultSchema.parse(result));
+    if (results.length !== binding.sources.length || new Set(results.map((result) => result.graderId)).size !== results.length)
+        throw new LearningDomainError("reward_result_set_incomplete");
+    for (const source of binding.sources) {
+        const result = results.find((candidate) => candidate.graderId === source.graderId);
+        if (!result || contentHash(result.reward) !== contentHash(source.reward) || result.role !== source.role)
+            throw new LearningDomainError("reward_result_binding_mismatch");
+        if ((result.status === "scored") !== (result.rawScore !== null && result.normalizedScore !== null && result.passed !== null))
+            throw new LearningDomainError("reward_result_score_state_invalid");
+        if (result.status !== "scored" && (result.rawScore !== null || result.normalizedScore !== null || result.passed !== null))
+            throw new LearningDomainError("unscorable_reward_contains_score");
+        if (result.status === "scored") {
+            const transform = source.normalization;
+            const normalized = transform.kind === "identity" ? result.rawScore : (result.rawScore - transform.minimum) / (transform.maximum - transform.minimum);
+            const expected = transform.kind === "linear" && transform.direction === "lower" ? 1 - normalized : normalized;
+            if (Math.abs(expected - result.normalizedScore) > 1e-12)
+                throw new LearningDomainError("reward_normalization_mismatch");
+        }
+    }
+    const content = RewardCompositionContentSchema.parse({
+        schemaVersion: "openpond.rewardComposition.v1",
+        binding: { id: binding.id, revision: binding.revision, contentHash: binding.contentHash },
+        taskHash: input.taskHash,
+        outputHash: input.outputHash,
+        results,
+        training: aggregate("training"),
+        evaluation: aggregate("evaluation"),
+    });
+    return RewardCompositionSchema.parse({ ...content, contentHash: contentHash(content) });
+    function aggregate(role) {
+        const selected = binding.sources.filter((source) => source.role === role).map((source) => ({ source, result: results.find((candidate) => candidate.graderId === source.graderId) }));
+        if (!selected.length)
+            return { status: "not_configured", score: null, passed: null };
+        if (selected.some(({ source, result }) => source.required && result.status !== "scored"))
+            return { status: "unscorable", score: null, passed: null };
+        const scored = selected.filter(({ result }) => result.status === "scored");
+        const weight = scored.reduce((sum, { source }) => sum + source.weight, 0);
+        if (weight === 0)
+            return { status: "unscorable", score: null, passed: null };
+        const gated = scored.some(({ source, result }) => source.hardGate && !result.passed);
+        return { status: "scored", score: gated ? 0 : scored.reduce((sum, { source, result }) => sum + source.weight * result.normalizedScore, 0) / weight, passed: !gated && scored.every(({ result }) => result.passed) };
+    }
+}
+export function assertRewardHash(value) {
+    assertBoundedTaskJson(value);
+    if (contentHash(withoutRewardHash(value)) !== value.contentHash)
+        throw new LearningDomainError("reward_content_hash_mismatch");
+}
+function withoutRewardHash(value) { const { contentHash: _hash, ...content } = value; return content; }
+function compileSource(source, reward) {
+    if (reward.implementation.kind === "learned_model")
+        throw new LearningDomainError("learned_reward_requires_model_execution_adapter");
+    return GraderSpecSchema.parse({ ...reward.implementation, id: source.graderId, version: String(reward.revision), weight: source.weight, hardGate: source.hardGate, rewardEligible: source.role === "training", privileged: source.privileged });
+}

package/dist/task-schema.js

@@ -0,0 +1,142 @@
+import { Ajv2020 } from "ajv/dist/2020.js";
+import { contentHash } from "@openpond/harness";
+export const TASK_JSON_SCHEMA_DIALECT = "https://json-schema.org/draft/2020-12/schema";
+export const TASK_VALUE_MAX_BYTES = 1_048_576;
+export const TASK_SCHEMA_MAX_BYTES = 32_768;
+const validators = new Map();
+const schemaMaps = new Set(["properties", "$defs", "dependentSchemas"]);
+const schemaArrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]);
+const schemaChildren = new Set(["items", "additionalProperties", "not", "if", "then", "else", "contains", "unevaluatedProperties", "unevaluatedItems", "propertyNames"]);
+const unsupported = new Set(["pattern", "patternProperties", "format", "$id", "$anchor", "$dynamicAnchor", "$dynamicRef", "$recursiveRef", "$recursiveAnchor", "$async", "$data", "contentSchema"]);
+/** Bound producer-controlled JSON before hashing, schema compilation or validation. */
+export function assertBoundedTaskJson(value, maxBytes = TASK_VALUE_MAX_BYTES) {
+    const pending = [{ value, depth: 0 }];
+    const seen = new Set();
+    let nodes = 0;
+    while (pending.length) {
+        const item = pending.pop();
+        if (item.exit) {
+            seen.delete(item.value);
+            continue;
+        }
+        if (++nodes > 100_000 || item.depth > 48)
+            throw new Error("task_json_complexity_exceeded");
+        if (item.value === null || typeof item.value === "string" || typeof item.value === "boolean")
+            continue;
+        if (typeof item.value === "number" && Number.isFinite(item.value))
+            continue;
+        if (typeof item.value !== "object")
+            throw new Error("task_json_value_invalid");
+        if (seen.has(item.value))
+            throw new Error("task_json_cycle");
+        seen.add(item.value);
+        pending.push({ ...item, exit: true });
+        const prototype = Object.getPrototypeOf(item.value);
+        if (!Array.isArray(item.value) && prototype !== Object.prototype && prototype !== null)
+            throw new Error("task_json_object_invalid");
+        if (Object.getOwnPropertySymbols(item.value).length)
+            throw new Error("task_json_symbol_invalid");
+        if (Object.hasOwn(item.value, "toJSON"))
+            throw new Error("task_json_serializer_invalid");
+        if (Array.isArray(item.value) && (item.value.length > 10_000 || Object.keys(item.value).length !== item.value.length))
+            throw new Error("task_json_array_invalid");
+        for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(item.value))) {
+            if (descriptor.get || descriptor.set)
+                throw new Error("task_json_accessor_invalid");
+            if (Array.isArray(item.value) && key !== "length" && (!/^(0|[1-9]\d*)$/u.test(key) || Number(key) >= item.value.length))
+                throw new Error("task_json_array_invalid");
+            if (descriptor.enumerable)
+                pending.push({ value: descriptor.value, depth: item.depth + 1 });
+        }
+    }
+    if (new TextEncoder().encode(JSON.stringify(value)).byteLength > maxBytes)
+        throw new Error("task_json_size_exceeded");
+}
+/** No remote resolution, custom code, data mutation, regex or format plugins. */
+export function validateTaskSchema(schema) {
+    try {
+        validator(schema);
+        return { valid: true, issues: [] };
+    }
+    catch (error) {
+        return { valid: false, issues: [{ path: "", code: "task_schema_invalid", message: error instanceof Error ? error.message : "Task schema is invalid." }] };
+    }
+}
+export function validateTaskValue(schema, value) {
+    try {
+        assertBoundedTaskJson(value);
+        const validate = validator(schema);
+        const valid = validate(value);
+        return {
+            valid,
+            issues: valid ? [] : (validate.errors ?? []).slice(0, 20).map((error) => ({
+                path: error.instancePath,
+                code: error.keyword,
+                message: error.message ?? "Value does not match the task schema.",
+            })),
+        };
+    }
+    catch (error) {
+        return { valid: false, issues: [{ path: "", code: "task_validation_unavailable", message: error instanceof Error ? error.message : "Task validation failed." }] };
+    }
+}
+function validator(schema) {
+    assertBoundedTaskJson(schema, TASK_SCHEMA_MAX_BYTES);
+    inspectSchema(schema, schema, new Set(), 0, { remaining: 2_048 });
+    const hash = contentHash(schema);
+    const cached = validators.get(hash);
+    if (cached)
+        return cached;
+    const ajv = new Ajv2020({ allErrors: false, ownProperties: true, strict: true, strictTypes: false, strictTuples: false, strictRequired: false, validateFormats: false, inlineRefs: false, loopRequired: 32, loopEnum: 32 });
+    const compiled = ajv.compile(schema);
+    if (validators.size >= 64)
+        validators.delete(validators.keys().next().value);
+    validators.set(hash, compiled);
+    return compiled;
+}
+function inspectSchema(value, root, ancestors, depth, budget) {
+    if (--budget.remaining < 0)
+        throw new Error("Task schema reference expansion exceeds the validation budget.");
+    if (typeof value === "boolean")
+        return;
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new Error("A task schema must be an object or boolean.");
+    if (depth > 24 || ancestors.has(value))
+        throw new Error("Recursive or deeply nested task schemas are not supported.");
+    const next = new Set(ancestors).add(value);
+    const schema = value;
+    if (schema.$schema !== undefined && schema.$schema !== TASK_JSON_SCHEMA_DIALECT)
+        throw new Error(`Task schemas must use ${TASK_JSON_SCHEMA_DIALECT}.`);
+    for (const [key, child] of Object.entries(schema)) {
+        if (unsupported.has(key))
+            throw new Error(`Task schema keyword ${key} is not supported by the bounded validator.`);
+        if (key === "$ref") {
+            if (typeof child !== "string" || !child.startsWith("#/$defs/"))
+                throw new Error("Task schema references must be local JSON pointers into $defs.");
+            let target = root;
+            for (const segment of child.slice(2).split("/")) {
+                if (/~(?![01])/u.test(segment))
+                    throw new Error("Invalid task schema JSON pointer.");
+                const decoded = segment.replaceAll("~1", "/").replaceAll("~0", "~");
+                if (!target || typeof target !== "object" || !Object.hasOwn(target, decoded))
+                    throw new Error(`Task schema reference ${child} is missing.`);
+                target = target[decoded];
+            }
+            inspectSchema(target, root, next, depth + 1, budget);
+        }
+        else if (schemaMaps.has(key)) {
+            if (!child || typeof child !== "object" || Array.isArray(child))
+                throw new Error(`Invalid schema map ${key}.`);
+            for (const entry of Object.values(child))
+                inspectSchema(entry, root, next, depth + 1, budget);
+        }
+        else if (schemaArrays.has(key)) {
+            if (!Array.isArray(child) || child.length > 32)
+                throw new Error(`Schema ${key} must contain at most 32 alternatives.`);
+            for (const entry of child)
+                inspectSchema(entry, root, next, depth + 1, budget);
+        }
+        else if (schemaChildren.has(key))
+            inspectSchema(child, root, next, depth + 1, budget);
+    }
+}

package/dist/tasksets.js

@@ -40,10 +40,13 @@ export const ModelJudgeGraderSpecSchema = GraderBaseSchema.extend({
     kind: z.literal("model_judge"),
     rubricRef: ImmutableAssetRefSchema,
     calibrationStatus: z.enum(["pending", "passed", "failed"]),
+    model: z.object({ providerId: ReleaseIdSchema, modelId: ReleaseIdSchema, revision: z.string().trim().min(1).max(500).nullable() }).strict().optional(),
+    temperature: z.number().min(0).max(2).optional(),
 }).strict();
 export const CustomVerifierGraderSpecSchema = GraderBaseSchema.extend({
     kind: z.literal("custom_verifier"),
     verifierRef: ImmutableAssetRefSchema,
+    exportName: z.string().trim().min(1).max(240).optional(),
     timeoutMs: z.number().int().positive().max(300_000),
     networkPolicy: z.literal("none"),
 }).strict();

package/dist/telemetry/index.js

@@ -0,0 +1,4 @@
+export * from "../telemetry.js";
+export * from "../telemetry-catalog.js";
+export * from "../telemetry-analysis.js";
+export * from "../telemetry-bundle.js";

package/dist/telemetry-analysis.js

@@ -0,0 +1,183 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash } from "@openpond/harness";
+import { getCoreMetricDefinition, validateCoreMetricObservation } from "./telemetry-catalog.js";
+import { MetricObservationSchema, RunTelemetryEventSchema, TelemetryVisibilitySchema, } from "./telemetry.js";
+export const TelemetryCohortSchema = z.object({
+    checkpointIds: z.array(ReleaseIdSchema).max(1_000),
+    steps: z.array(z.number().int().nonnegative()).max(10_000),
+    scenarioIds: z.array(ReleaseIdSchema).max(100_000),
+    rolloutGroupIds: z.array(ReleaseIdSchema).max(100_000),
+    attemptIds: z.array(ReleaseIdSchema).max(1_000_000),
+    splits: z.array(z.string().trim().min(1).max(100)).max(32),
+    failureOwners: z.array(z.string().trim().min(1).max(100)).max(32),
+    graders: z.array(z.string().trim().min(1).max(200)).max(1_000),
+    rewardEligible: z.boolean().nullable(),
+}).strict();
+export const EvidenceReferenceSchema = z.object({
+    id: ReleaseIdSchema,
+    contentHash: ReleaseHashSchema,
+    kind: z.enum(["rollout", "attempt", "trace", "grader", "checkpoint", "artifact"]),
+    visibility: TelemetryVisibilitySchema,
+}).strict();
+export const EvidenceCompletenessSchema = z.object({
+    schemaVersion: z.literal("openpond.telemetryEvidenceCompleteness.v1"),
+    runId: ReleaseIdSchema,
+    status: z.enum(["complete", "partial", "missing"]),
+    expectedEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    observedEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    missingEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    lastSequence: z.number().int().nonnegative().nullable(),
+    sequenceGaps: z.array(z.number().int().nonnegative()).max(10_000),
+}).strict();
+export const MetricSeriesPointSchema = z.object({
+    step: z.number().int().nonnegative().nullable(),
+    observedAt: ReleaseTimestampSchema,
+    value: z.number().finite(),
+    sampleCount: z.number().int().positive(),
+}).strict();
+export const RunMetricSummarySchema = z.object({
+    schemaVersion: z.literal("openpond.runMetricSummary.v1"),
+    runId: ReleaseIdSchema,
+    metricId: ReleaseIdSchema,
+    aggregation: z.enum(["last", "sum", "mean", "min", "max", "p50", "p95"]),
+    value: z.number().finite().nullable(),
+    sampleCount: z.number().int().nonnegative(),
+    series: z.array(MetricSeriesPointSchema).max(100_000),
+}).strict();
+export function createRunTelemetryEvent(input) {
+    const idHash = contentHash({ runId: input.lineage.runId, sequence: input.sequence, type: input.type, source: input.source });
+    return RunTelemetryEventSchema.parse({
+        schemaVersion: "openpond.runTelemetryEvent.v1",
+        eventId: `telemetry-${idHash.slice(0, 32)}`,
+        ...input,
+        attributes: input.attributes ?? {},
+    });
+}
+export function createMetricObservation(input) {
+    const idHash = contentHash({ runId: input.lineage.runId, metricId: input.metricId, sequence: input.sequence });
+    return validateCoreMetricObservation(MetricObservationSchema.parse({
+        schemaVersion: "openpond.metricObservation.v1",
+        observationId: `metric-${idHash.slice(0, 32)}`,
+        ...input,
+        dimensions: input.dimensions ?? {},
+    }));
+}
+export function telemetryIdempotencyKey(item) {
+    const id = item.schemaVersion === "openpond.metricObservation.v1" ? item.observationId : item.eventId;
+    return `${item.lineage.runId}:${item.sequence}:${id}`;
+}
+export function mergeTelemetryItems(input) {
+    const accepted = new Map();
+    const sequences = new Map();
+    for (const item of [...input.events, ...input.observations]) {
+        const key = telemetryIdempotencyKey(item);
+        const existing = accepted.get(key);
+        if (existing) {
+            if (contentHash(existing) !== contentHash(item))
+                throw new Error(`Telemetry idempotency conflict: ${key}`);
+            continue;
+        }
+        const sequenceKey = `${item.lineage.runId}:${item.sequence}`;
+        const priorKey = sequences.get(sequenceKey);
+        if (priorKey && priorKey !== key)
+            throw new Error(`Telemetry sequence conflict: ${sequenceKey}`);
+        sequences.set(sequenceKey, key);
+        accepted.set(key, item);
+    }
+    const sorted = [...accepted.values()].sort((left, right) => left.sequence - right.sequence);
+    return {
+        events: sorted.filter((item) => item.schemaVersion === "openpond.runTelemetryEvent.v1"),
+        observations: sorted.filter((item) => item.schemaVersion === "openpond.metricObservation.v1"),
+    };
+}
+function matchesOptionalSet(values, candidate) {
+    return values.length === 0 || (candidate !== null && values.includes(candidate));
+}
+export function filterTelemetryCohort(input) {
+    const cohort = TelemetryCohortSchema.parse(input.cohort);
+    const lineageMatches = (lineage) => matchesOptionalSet(cohort.checkpointIds, lineage.checkpointId)
+        && matchesOptionalSet(cohort.steps, lineage.step)
+        && matchesOptionalSet(cohort.scenarioIds, lineage.scenarioId)
+        && matchesOptionalSet(cohort.rolloutGroupIds, lineage.rolloutGroupId)
+        && matchesOptionalSet(cohort.attemptIds, lineage.attemptId);
+    const attributeMatches = (values) => matchesOptionalSet(cohort.splits, typeof values.split === "string" ? values.split : null)
+        && matchesOptionalSet(cohort.failureOwners, typeof values.failureOwner === "string" ? values.failureOwner : null)
+        && matchesOptionalSet(cohort.graders, typeof values.grader === "string" ? values.grader : null)
+        && (cohort.rewardEligible === null || values.rewardEligible === cohort.rewardEligible);
+    return {
+        events: input.events.filter((event) => lineageMatches(event.lineage) && attributeMatches(event.attributes)),
+        observations: input.observations.filter((observation) => lineageMatches(observation.lineage) && attributeMatches(observation.dimensions)),
+    };
+}
+export function deriveEvidenceCompleteness(input) {
+    const events = input.events.filter((event) => event.lineage.runId === input.runId).sort((left, right) => left.sequence - right.sequence);
+    const observed = [...new Set(events.map((event) => event.type))].sort();
+    const missing = [...new Set(input.expectedEventTypes)].filter((type) => !observed.includes(type)).sort();
+    const sequenceValues = [...new Set(events.map((event) => event.sequence))].sort((left, right) => left - right);
+    const lastSequence = events.at(-1)?.sequence ?? null;
+    const sequenceGaps = [];
+    let expected = 0;
+    for (const sequence of sequenceValues) {
+        while (expected < sequence && sequenceGaps.length < 10_000)
+            sequenceGaps.push(expected++);
+        expected = sequence + 1;
+    }
+    return EvidenceCompletenessSchema.parse({
+        schemaVersion: "openpond.telemetryEvidenceCompleteness.v1",
+        runId: input.runId,
+        status: events.length === 0 ? "missing" : missing.length || sequenceGaps.length ? "partial" : "complete",
+        expectedEventTypes: [...new Set(input.expectedEventTypes)].sort(),
+        observedEventTypes: observed,
+        missingEventTypes: missing,
+        lastSequence,
+        sequenceGaps,
+    });
+}
+function aggregate(values, method) {
+    if (!values.length)
+        return null;
+    if (method === "last")
+        return values.at(-1) ?? null;
+    if (method === "sum")
+        return values.reduce((total, value) => total + value, 0);
+    if (method === "mean")
+        return values.reduce((total, value) => total + value, 0) / values.length;
+    if (method === "min")
+        return Math.min(...values);
+    if (method === "max")
+        return Math.max(...values);
+    const sorted = [...values].sort((left, right) => left - right);
+    const percentile = method === "p50" ? 0.5 : 0.95;
+    return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * percentile) - 1)] ?? null;
+}
+export function summarizeMetric(runId, metricId, observations) {
+    const definition = getCoreMetricDefinition(metricId);
+    if (!definition)
+        throw new Error(`Unknown core metric: ${metricId}`);
+    const selected = observations.filter((item) => item.lineage.runId === runId && item.metricId === metricId).sort((left, right) => left.sequence - right.sequence);
+    const buckets = new Map();
+    for (const item of selected) {
+        const key = item.lineage.step === null ? `time:${item.observedAt}` : `step:${item.lineage.step}`;
+        buckets.set(key, [...(buckets.get(key) ?? []), item]);
+    }
+    const series = [...buckets.values()].map((items) => ({
+        step: items[0]?.lineage.step ?? null,
+        observedAt: items[0]?.observedAt,
+        value: aggregate(items.map((item) => item.value), definition.aggregation),
+        sampleCount: items.length,
+    })).filter((point) => point.value !== null && point.observedAt !== undefined);
+    return RunMetricSummarySchema.parse({ schemaVersion: "openpond.runMetricSummary.v1", runId, metricId, aggregation: definition.aggregation, value: aggregate(selected.map((item) => item.value), definition.aggregation), sampleCount: selected.length, series });
+}
+export function redactTelemetryEvent(event, maximumVisibility) {
+    const rank = { policy_visible: 0, team_visible: 1, host_private: 2 };
+    if (rank[event.visibility] > rank[maximumVisibility])
+        return null;
+    return RunTelemetryEventSchema.parse(event);
+}
+export function redactTelemetryAttributes(event, deniedKeys) {
+    const denied = new Set(deniedKeys);
+    return RunTelemetryEventSchema.parse({
+        ...event,
+        attributes: Object.fromEntries(Object.entries(event.attributes).filter(([key]) => !denied.has(key))),
+    });
+}

package/dist/telemetry-bundle.js

@@ -0,0 +1,60 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash } from "@openpond/harness";
+import { MetricCatalogSchema } from "./telemetry-catalog.js";
+import { deriveEvidenceCompleteness, EvidenceCompletenessSchema, EvidenceReferenceSchema, redactTelemetryAttributes, redactTelemetryEvent } from "./telemetry-analysis.js";
+import { MetricObservationSchema, RunTelemetryEventSchema } from "./telemetry.js";
+export const TelemetryExportBundleContentSchema = z.object({
+    schemaVersion: z.literal("openpond.telemetryExportBundle.v1"),
+    id: ReleaseIdSchema,
+    runId: ReleaseIdSchema,
+    exportedAt: ReleaseTimestampSchema,
+    definitions: MetricCatalogSchema,
+    events: z.array(RunTelemetryEventSchema).max(1_000_000),
+    observations: z.array(MetricObservationSchema).max(10_000_000),
+    evidenceRefs: z.array(EvidenceReferenceSchema).max(1_000_000),
+    completeness: EvidenceCompletenessSchema,
+}).strict();
+export const TelemetryExportBundleSchema = TelemetryExportBundleContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+export function createTelemetryExportBundle(input) {
+    const content = TelemetryExportBundleContentSchema.parse(input);
+    if (content.completeness.runId !== content.runId) {
+        throw new Error("Telemetry export completeness belongs to another Run.");
+    }
+    if (content.events.some((event) => event.lineage.runId !== content.runId) || content.observations.some((observation) => observation.lineage.runId !== content.runId)) {
+        throw new Error("Telemetry export bundle contains evidence from another Run.");
+    }
+    return TelemetryExportBundleSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function verifyTelemetryExportBundle(input) {
+    const parsed = TelemetryExportBundleSchema.safeParse(input);
+    if (!parsed.success)
+        return false;
+    const { contentHash: actual, ...content } = parsed.data;
+    return contentHash(TelemetryExportBundleContentSchema.parse(content)) === actual;
+}
+export function redactTelemetryExportBundle(input) {
+    const bundle = TelemetryExportBundleSchema.parse(input.bundle);
+    if (!verifyTelemetryExportBundle(bundle))
+        throw new Error("Telemetry export bundle has an invalid content hash.");
+    const rank = { policy_visible: 0, team_visible: 1, host_private: 2 };
+    const definitions = bundle.definitions.filter((definition) => rank[definition.visibility] <= rank[input.maximumVisibility]);
+    const metricIds = new Set(definitions.map((definition) => definition.id));
+    const events = bundle.events.flatMap((event) => {
+        const visible = redactTelemetryEvent(event, input.maximumVisibility);
+        return visible ? [redactTelemetryAttributes(visible, input.deniedAttributeKeys ?? [])] : [];
+    });
+    const expectedEventTypes = bundle.completeness.expectedEventTypes.filter((type) => events.some((event) => event.type === type));
+    return createTelemetryExportBundle({
+        schemaVersion: "openpond.telemetryExportBundle.v1",
+        id: input.id,
+        runId: bundle.runId,
+        exportedAt: input.exportedAt,
+        definitions,
+        events,
+        observations: bundle.observations.filter((observation) => metricIds.has(observation.metricId)),
+        evidenceRefs: bundle.evidenceRefs.filter((reference) => rank[reference.visibility] <= rank[input.maximumVisibility]),
+        completeness: deriveEvidenceCompleteness({ runId: bundle.runId, events, expectedEventTypes: expectedEventTypes }),
+    });
+}