@openpond/evals 0.5.0 → 0.6.0

package/dist/telemetry-analysis.js

@@ -0,0 +1,183 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash } from "@openpond/harness";
+import { getCoreMetricDefinition, validateCoreMetricObservation } from "./telemetry-catalog.js";
+import { MetricObservationSchema, RunTelemetryEventSchema, TelemetryVisibilitySchema, } from "./telemetry.js";
+export const TelemetryCohortSchema = z.object({
+    checkpointIds: z.array(ReleaseIdSchema).max(1_000),
+    steps: z.array(z.number().int().nonnegative()).max(10_000),
+    scenarioIds: z.array(ReleaseIdSchema).max(100_000),
+    rolloutGroupIds: z.array(ReleaseIdSchema).max(100_000),
+    attemptIds: z.array(ReleaseIdSchema).max(1_000_000),
+    splits: z.array(z.string().trim().min(1).max(100)).max(32),
+    failureOwners: z.array(z.string().trim().min(1).max(100)).max(32),
+    graders: z.array(z.string().trim().min(1).max(200)).max(1_000),
+    rewardEligible: z.boolean().nullable(),
+}).strict();
+export const EvidenceReferenceSchema = z.object({
+    id: ReleaseIdSchema,
+    contentHash: ReleaseHashSchema,
+    kind: z.enum(["rollout", "attempt", "trace", "grader", "checkpoint", "artifact"]),
+    visibility: TelemetryVisibilitySchema,
+}).strict();
+export const EvidenceCompletenessSchema = z.object({
+    schemaVersion: z.literal("openpond.telemetryEvidenceCompleteness.v1"),
+    runId: ReleaseIdSchema,
+    status: z.enum(["complete", "partial", "missing"]),
+    expectedEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    observedEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    missingEventTypes: z.array(z.string().trim().min(1).max(100)).max(100),
+    lastSequence: z.number().int().nonnegative().nullable(),
+    sequenceGaps: z.array(z.number().int().nonnegative()).max(10_000),
+}).strict();
+export const MetricSeriesPointSchema = z.object({
+    step: z.number().int().nonnegative().nullable(),
+    observedAt: ReleaseTimestampSchema,
+    value: z.number().finite(),
+    sampleCount: z.number().int().positive(),
+}).strict();
+export const RunMetricSummarySchema = z.object({
+    schemaVersion: z.literal("openpond.runMetricSummary.v1"),
+    runId: ReleaseIdSchema,
+    metricId: ReleaseIdSchema,
+    aggregation: z.enum(["last", "sum", "mean", "min", "max", "p50", "p95"]),
+    value: z.number().finite().nullable(),
+    sampleCount: z.number().int().nonnegative(),
+    series: z.array(MetricSeriesPointSchema).max(100_000),
+}).strict();
+export function createRunTelemetryEvent(input) {
+    const idHash = contentHash({ runId: input.lineage.runId, sequence: input.sequence, type: input.type, source: input.source });
+    return RunTelemetryEventSchema.parse({
+        schemaVersion: "openpond.runTelemetryEvent.v1",
+        eventId: `telemetry-${idHash.slice(0, 32)}`,
+        ...input,
+        attributes: input.attributes ?? {},
+    });
+}
+export function createMetricObservation(input) {
+    const idHash = contentHash({ runId: input.lineage.runId, metricId: input.metricId, sequence: input.sequence });
+    return validateCoreMetricObservation(MetricObservationSchema.parse({
+        schemaVersion: "openpond.metricObservation.v1",
+        observationId: `metric-${idHash.slice(0, 32)}`,
+        ...input,
+        dimensions: input.dimensions ?? {},
+    }));
+}
+export function telemetryIdempotencyKey(item) {
+    const id = item.schemaVersion === "openpond.metricObservation.v1" ? item.observationId : item.eventId;
+    return `${item.lineage.runId}:${item.sequence}:${id}`;
+}
+export function mergeTelemetryItems(input) {
+    const accepted = new Map();
+    const sequences = new Map();
+    for (const item of [...input.events, ...input.observations]) {
+        const key = telemetryIdempotencyKey(item);
+        const existing = accepted.get(key);
+        if (existing) {
+            if (contentHash(existing) !== contentHash(item))
+                throw new Error(`Telemetry idempotency conflict: ${key}`);
+            continue;
+        }
+        const sequenceKey = `${item.lineage.runId}:${item.sequence}`;
+        const priorKey = sequences.get(sequenceKey);
+        if (priorKey && priorKey !== key)
+            throw new Error(`Telemetry sequence conflict: ${sequenceKey}`);
+        sequences.set(sequenceKey, key);
+        accepted.set(key, item);
+    }
+    const sorted = [...accepted.values()].sort((left, right) => left.sequence - right.sequence);
+    return {
+        events: sorted.filter((item) => item.schemaVersion === "openpond.runTelemetryEvent.v1"),
+        observations: sorted.filter((item) => item.schemaVersion === "openpond.metricObservation.v1"),
+    };
+}
+function matchesOptionalSet(values, candidate) {
+    return values.length === 0 || (candidate !== null && values.includes(candidate));
+}
+export function filterTelemetryCohort(input) {
+    const cohort = TelemetryCohortSchema.parse(input.cohort);
+    const lineageMatches = (lineage) => matchesOptionalSet(cohort.checkpointIds, lineage.checkpointId)
+        && matchesOptionalSet(cohort.steps, lineage.step)
+        && matchesOptionalSet(cohort.scenarioIds, lineage.scenarioId)
+        && matchesOptionalSet(cohort.rolloutGroupIds, lineage.rolloutGroupId)
+        && matchesOptionalSet(cohort.attemptIds, lineage.attemptId);
+    const attributeMatches = (values) => matchesOptionalSet(cohort.splits, typeof values.split === "string" ? values.split : null)
+        && matchesOptionalSet(cohort.failureOwners, typeof values.failureOwner === "string" ? values.failureOwner : null)
+        && matchesOptionalSet(cohort.graders, typeof values.grader === "string" ? values.grader : null)
+        && (cohort.rewardEligible === null || values.rewardEligible === cohort.rewardEligible);
+    return {
+        events: input.events.filter((event) => lineageMatches(event.lineage) && attributeMatches(event.attributes)),
+        observations: input.observations.filter((observation) => lineageMatches(observation.lineage) && attributeMatches(observation.dimensions)),
+    };
+}
+export function deriveEvidenceCompleteness(input) {
+    const events = input.events.filter((event) => event.lineage.runId === input.runId).sort((left, right) => left.sequence - right.sequence);
+    const observed = [...new Set(events.map((event) => event.type))].sort();
+    const missing = [...new Set(input.expectedEventTypes)].filter((type) => !observed.includes(type)).sort();
+    const sequenceValues = [...new Set(events.map((event) => event.sequence))].sort((left, right) => left - right);
+    const lastSequence = events.at(-1)?.sequence ?? null;
+    const sequenceGaps = [];
+    let expected = 0;
+    for (const sequence of sequenceValues) {
+        while (expected < sequence && sequenceGaps.length < 10_000)
+            sequenceGaps.push(expected++);
+        expected = sequence + 1;
+    }
+    return EvidenceCompletenessSchema.parse({
+        schemaVersion: "openpond.telemetryEvidenceCompleteness.v1",
+        runId: input.runId,
+        status: events.length === 0 ? "missing" : missing.length || sequenceGaps.length ? "partial" : "complete",
+        expectedEventTypes: [...new Set(input.expectedEventTypes)].sort(),
+        observedEventTypes: observed,
+        missingEventTypes: missing,
+        lastSequence,
+        sequenceGaps,
+    });
+}
+function aggregate(values, method) {
+    if (!values.length)
+        return null;
+    if (method === "last")
+        return values.at(-1) ?? null;
+    if (method === "sum")
+        return values.reduce((total, value) => total + value, 0);
+    if (method === "mean")
+        return values.reduce((total, value) => total + value, 0) / values.length;
+    if (method === "min")
+        return Math.min(...values);
+    if (method === "max")
+        return Math.max(...values);
+    const sorted = [...values].sort((left, right) => left - right);
+    const percentile = method === "p50" ? 0.5 : 0.95;
+    return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * percentile) - 1)] ?? null;
+}
+export function summarizeMetric(runId, metricId, observations) {
+    const definition = getCoreMetricDefinition(metricId);
+    if (!definition)
+        throw new Error(`Unknown core metric: ${metricId}`);
+    const selected = observations.filter((item) => item.lineage.runId === runId && item.metricId === metricId).sort((left, right) => left.sequence - right.sequence);
+    const buckets = new Map();
+    for (const item of selected) {
+        const key = item.lineage.step === null ? `time:${item.observedAt}` : `step:${item.lineage.step}`;
+        buckets.set(key, [...(buckets.get(key) ?? []), item]);
+    }
+    const series = [...buckets.values()].map((items) => ({
+        step: items[0]?.lineage.step ?? null,
+        observedAt: items[0]?.observedAt,
+        value: aggregate(items.map((item) => item.value), definition.aggregation),
+        sampleCount: items.length,
+    })).filter((point) => point.value !== null && point.observedAt !== undefined);
+    return RunMetricSummarySchema.parse({ schemaVersion: "openpond.runMetricSummary.v1", runId, metricId, aggregation: definition.aggregation, value: aggregate(selected.map((item) => item.value), definition.aggregation), sampleCount: selected.length, series });
+}
+export function redactTelemetryEvent(event, maximumVisibility) {
+    const rank = { policy_visible: 0, team_visible: 1, host_private: 2 };
+    if (rank[event.visibility] > rank[maximumVisibility])
+        return null;
+    return RunTelemetryEventSchema.parse(event);
+}
+export function redactTelemetryAttributes(event, deniedKeys) {
+    const denied = new Set(deniedKeys);
+    return RunTelemetryEventSchema.parse({
+        ...event,
+        attributes: Object.fromEntries(Object.entries(event.attributes).filter(([key]) => !denied.has(key))),
+    });
+}

package/dist/telemetry-bundle.js

@@ -0,0 +1,60 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash } from "@openpond/harness";
+import { MetricCatalogSchema } from "./telemetry-catalog.js";
+import { deriveEvidenceCompleteness, EvidenceCompletenessSchema, EvidenceReferenceSchema, redactTelemetryAttributes, redactTelemetryEvent } from "./telemetry-analysis.js";
+import { MetricObservationSchema, RunTelemetryEventSchema } from "./telemetry.js";
+export const TelemetryExportBundleContentSchema = z.object({
+    schemaVersion: z.literal("openpond.telemetryExportBundle.v1"),
+    id: ReleaseIdSchema,
+    runId: ReleaseIdSchema,
+    exportedAt: ReleaseTimestampSchema,
+    definitions: MetricCatalogSchema,
+    events: z.array(RunTelemetryEventSchema).max(1_000_000),
+    observations: z.array(MetricObservationSchema).max(10_000_000),
+    evidenceRefs: z.array(EvidenceReferenceSchema).max(1_000_000),
+    completeness: EvidenceCompletenessSchema,
+}).strict();
+export const TelemetryExportBundleSchema = TelemetryExportBundleContentSchema.extend({
+    contentHash: ReleaseHashSchema,
+}).strict();
+export function createTelemetryExportBundle(input) {
+    const content = TelemetryExportBundleContentSchema.parse(input);
+    if (content.completeness.runId !== content.runId) {
+        throw new Error("Telemetry export completeness belongs to another Run.");
+    }
+    if (content.events.some((event) => event.lineage.runId !== content.runId) || content.observations.some((observation) => observation.lineage.runId !== content.runId)) {
+        throw new Error("Telemetry export bundle contains evidence from another Run.");
+    }
+    return TelemetryExportBundleSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function verifyTelemetryExportBundle(input) {
+    const parsed = TelemetryExportBundleSchema.safeParse(input);
+    if (!parsed.success)
+        return false;
+    const { contentHash: actual, ...content } = parsed.data;
+    return contentHash(TelemetryExportBundleContentSchema.parse(content)) === actual;
+}
+export function redactTelemetryExportBundle(input) {
+    const bundle = TelemetryExportBundleSchema.parse(input.bundle);
+    if (!verifyTelemetryExportBundle(bundle))
+        throw new Error("Telemetry export bundle has an invalid content hash.");
+    const rank = { policy_visible: 0, team_visible: 1, host_private: 2 };
+    const definitions = bundle.definitions.filter((definition) => rank[definition.visibility] <= rank[input.maximumVisibility]);
+    const metricIds = new Set(definitions.map((definition) => definition.id));
+    const events = bundle.events.flatMap((event) => {
+        const visible = redactTelemetryEvent(event, input.maximumVisibility);
+        return visible ? [redactTelemetryAttributes(visible, input.deniedAttributeKeys ?? [])] : [];
+    });
+    const expectedEventTypes = bundle.completeness.expectedEventTypes.filter((type) => events.some((event) => event.type === type));
+    return createTelemetryExportBundle({
+        schemaVersion: "openpond.telemetryExportBundle.v1",
+        id: input.id,
+        runId: bundle.runId,
+        exportedAt: input.exportedAt,
+        definitions,
+        events,
+        observations: bundle.observations.filter((observation) => metricIds.has(observation.metricId)),
+        evidenceRefs: bundle.evidenceRefs.filter((reference) => rank[reference.visibility] <= rank[input.maximumVisibility]),
+        completeness: deriveEvidenceCompleteness({ runId: bundle.runId, events, expectedEventTypes: expectedEventTypes }),
+    });
+}

package/dist/telemetry-catalog.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { MetricDefinitionSchema } from "./telemetry.js";
+const definition = (input) => MetricDefinitionSchema.parse({ schemaVersion: "openpond.metricDefinition.v1", ...input });
+export const CORE_METRIC_CATALOG = [
+    definition({ id: "reward.mean", displayName: "Mean reward", description: "Mean composed reward for the cohort.", valueType: "gauge", unit: "scalar", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split", "grader"] }),
+    definition({ id: "reward.variance", displayName: "Reward variance", description: "Population variance of composed reward within a rollout group.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "reward.constant_group_rate", displayName: "Constant group rate", description: "Fraction of rollout groups with no reward variation.", valueType: "gauge", unit: "ratio", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "attempt.valid_rate", displayName: "Valid attempt rate", description: "Fraction of attempts passing deterministic validity checks.", valueType: "gauge", unit: "ratio", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split", "failureOwner"] }),
+    definition({ id: "attempt.failure_count", displayName: "Attempt failures", description: "Count of failed Attempts.", valueType: "counter", unit: "count", direction: "lower", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "failureOwner", "failureClass"] }),
+    definition({ id: "optimizer.loss", displayName: "Optimizer loss", description: "Policy optimizer loss after the step.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.learning_rate", displayName: "Learning rate", description: "Optimizer learning rate after the step.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "last", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.kl", displayName: "KL divergence", description: "Sampled KL divergence from the reference policy.", valueType: "gauge", unit: "scalar", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.entropy", displayName: "Policy entropy", description: "Observed policy entropy for trainable tokens.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.gradient_norm", displayName: "Gradient norm", description: "Gradient norm reported by the optimizer.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "max", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.clip_fraction", displayName: "Clip fraction", description: "Fraction of policy updates affected by clipping.", valueType: "gauge", unit: "ratio", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "output.duplicate_rate", displayName: "Duplicate output rate", description: "Fraction of outputs duplicated within the measured cohort.", valueType: "gauge", unit: "ratio", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "output.unique_count", displayName: "Unique outputs", description: "Distinct output count in the measured cohort.", valueType: "gauge", unit: "count", direction: "higher", aggregation: "last", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "tokens.input", displayName: "Input tokens", description: "Input tokens processed.", valueType: "counter", unit: "tokens", direction: "neutral", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "source"] }),
+    definition({ id: "tokens.output", displayName: "Output tokens", description: "Output tokens generated.", valueType: "counter", unit: "tokens", direction: "neutral", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "source"] }),
+    definition({ id: "runtime.latency_ms", displayName: "Runtime latency", description: "Wall-clock latency of the measured operation.", valueType: "distribution", unit: "milliseconds", direction: "lower", aggregation: "p95", visibility: "team_visible", boundedDimensions: ["operation", "provider"] }),
+    definition({ id: "runtime.throughput", displayName: "Token throughput", description: "Tokens processed per second.", valueType: "gauge", unit: "scalar", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["operation", "provider"] }),
+    definition({ id: "gpu.memory_bytes", displayName: "GPU memory", description: "Peak allocated GPU memory.", valueType: "gauge", unit: "bytes", direction: "neutral", aggregation: "max", visibility: "host_private", boundedDimensions: ["provider", "gpuType"] }),
+    definition({ id: "gpu.utilization", displayName: "GPU utilization", description: "Observed GPU utilization ratio.", valueType: "gauge", unit: "ratio", direction: "neutral", aggregation: "mean", visibility: "host_private", boundedDimensions: ["provider", "gpuType"] }),
+    definition({ id: "cost.usd", displayName: "Run cost", description: "Accrued hosted execution cost.", valueType: "counter", unit: "usd", direction: "lower", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["provider", "resource"] }),
+];
+const catalog = new Map(CORE_METRIC_CATALOG.map((item) => [item.id, item]));
+export function getCoreMetricDefinition(metricId) {
+    return catalog.get(metricId);
+}
+export function validateCoreMetricObservation(input) {
+    return validateMetricObservation(input, CORE_METRIC_CATALOG);
+}
+export function validateMetricObservation(input, definitions) {
+    MetricCatalogSchema.parse(definitions);
+    const metric = definitions.find((item) => item.id === input.metricId);
+    if (!metric)
+        throw new Error(`Unknown metric: ${input.metricId}`);
+    const unexpected = Object.keys(input.dimensions).filter((dimension) => !metric.boundedDimensions.includes(dimension));
+    if (unexpected.length)
+        throw new Error(`Metric ${input.metricId} has unsupported dimensions: ${unexpected.join(", ")}`);
+    if (metric.unit === "ratio" && (input.value < 0 || input.value > 1)) {
+        throw new Error(`Ratio metric ${input.metricId} must be between zero and one.`);
+    }
+    return input;
+}
+export const MetricCatalogSchema = z.array(MetricDefinitionSchema).min(1).superRefine((items, context) => {
+    if (new Set(items.map((item) => item.id)).size !== items.length) {
+        context.addIssue({ code: "custom", message: "Metric catalog contains duplicate ids." });
+    }
+});

package/dist/telemetry.js

@@ -0,0 +1,112 @@
+import { z } from "zod";
+import { MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, } from "@openpond/harness";
+export const RUN_TELEMETRY_SCHEMA_VERSION = "openpond.runTelemetryEvent.v1";
+export const METRIC_DEFINITION_SCHEMA_VERSION = "openpond.metricDefinition.v1";
+export const METRIC_OBSERVATION_SCHEMA_VERSION = "openpond.metricObservation.v1";
+export const TelemetryVisibilitySchema = z.enum([
+    "policy_visible",
+    "team_visible",
+    "host_private",
+]);
+export const TelemetrySourceSchema = z.enum([
+    "runtime",
+    "environment",
+    "grader",
+    "optimizer",
+    "control_plane",
+    "evaluation",
+]);
+export const TelemetryEventTypeSchema = z.enum([
+    "run_started",
+    "run_state_changed",
+    "rollout_group_started",
+    "attempt_completed",
+    "grader_completed",
+    "reward_composed",
+    "optimizer_step_completed",
+    "checkpoint_committed",
+    "evaluation_completed",
+    "run_completed",
+    "run_failed",
+    "cleanup_completed",
+]);
+export const RunTelemetryLineageSchema = z.object({
+    modelProjectId: ReleaseIdSchema,
+    runId: ReleaseIdSchema,
+    modelVersionId: ReleaseIdSchema.nullable(),
+    harnessReleaseHash: ReleaseHashSchema,
+    tasksetReleaseHash: ReleaseHashSchema,
+    environmentReleaseHash: ReleaseHashSchema.nullable(),
+    checkpointId: ReleaseIdSchema.nullable(),
+    step: z.number().int().nonnegative().nullable(),
+    rolloutGroupId: ReleaseIdSchema.nullable(),
+    attemptId: ReleaseIdSchema.nullable(),
+    scenarioId: ReleaseIdSchema.nullable(),
+}).strict();
+export const RunTelemetryEventSchema = z.object({
+    schemaVersion: z.literal(RUN_TELEMETRY_SCHEMA_VERSION),
+    eventId: ReleaseIdSchema,
+    sequence: z.number().int().nonnegative(),
+    occurredAt: ReleaseTimestampSchema,
+    source: TelemetrySourceSchema,
+    type: TelemetryEventTypeSchema,
+    visibility: TelemetryVisibilitySchema,
+    lineage: RunTelemetryLineageSchema,
+    attributes: MetadataSchema,
+}).strict();
+export const MetricValueTypeSchema = z.enum(["gauge", "counter", "distribution"]);
+export const MetricUnitSchema = z.enum([
+    "ratio",
+    "count",
+    "seconds",
+    "milliseconds",
+    "tokens",
+    "bytes",
+    "usd",
+    "scalar",
+]);
+export const MetricDirectionSchema = z.enum(["higher", "lower", "neutral"]);
+export const MetricAggregationSchema = z.enum(["last", "sum", "mean", "min", "max", "p50", "p95"]);
+export const MetricDefinitionSchema = z.object({
+    schemaVersion: z.literal(METRIC_DEFINITION_SCHEMA_VERSION),
+    id: ReleaseIdSchema,
+    displayName: z.string().trim().min(1).max(200),
+    description: z.string().trim().min(1).max(2_000),
+    valueType: MetricValueTypeSchema,
+    unit: MetricUnitSchema,
+    direction: MetricDirectionSchema,
+    aggregation: MetricAggregationSchema,
+    visibility: TelemetryVisibilitySchema,
+    boundedDimensions: z.array(z.string().trim().min(1).max(100)).max(32),
+}).strict();
+export const MetricObservationSchema = z.object({
+    schemaVersion: z.literal(METRIC_OBSERVATION_SCHEMA_VERSION),
+    observationId: ReleaseIdSchema,
+    metricId: ReleaseIdSchema,
+    eventId: ReleaseIdSchema,
+    sequence: z.number().int().nonnegative(),
+    observedAt: ReleaseTimestampSchema,
+    value: z.number().finite(),
+    lineage: RunTelemetryLineageSchema,
+    dimensions: z.record(z.string().trim().min(1).max(100), z.string().max(500)),
+}).strict();
+export const RunTelemetryBatchSchema = z.object({
+    schemaVersion: z.literal("openpond.runTelemetryBatch.v1"),
+    events: z.array(RunTelemetryEventSchema).max(1_000),
+    observations: z.array(MetricObservationSchema).max(10_000),
+}).strict().superRefine((batch, context) => {
+    if (batch.events.length + batch.observations.length === 0) {
+        context.addIssue({ code: "custom", message: "Telemetry batch cannot be empty." });
+    }
+    const keys = new Set();
+    for (const item of [...batch.events, ...batch.observations]) {
+        const id = item.schemaVersion === METRIC_OBSERVATION_SCHEMA_VERSION
+            ? item.observationId
+            : item.eventId;
+        const key = `${item.lineage.runId}:${item.sequence}:${id}`;
+        if (keys.has(key)) {
+            context.addIssue({ code: "custom", message: "Telemetry batch contains a duplicate idempotency key." });
+        }
+        keys.add(key);
+    }
+});

package/dist/types/index.d.ts

@@ -6,10 +6,15 @@ export * from "./execution-contracts.js";
 export * from "./execution-receipts.js";
 export * from "./graders.js";
 export * from "./harness.js";
+export * from "./learned-preference.js";
 export * from "./runs.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
+export * from "./telemetry.js";
+export * from "./telemetry-catalog.js";
+export * from "./telemetry-analysis.js";
+export * from "./telemetry-bundle.js";
 export * from "./tasksets.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC"}