@openpond/evals 0.5.0 → 0.7.0

package/dist/types/telemetry-catalog.d.ts

@@ -0,0 +1,269 @@
+import { z } from "zod";
+import { type MetricDefinition, type MetricObservation } from "./telemetry.js";
+export declare const CORE_METRIC_CATALOG: readonly [{
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}, {
+    schemaVersion: "openpond.metricDefinition.v1";
+    id: string;
+    displayName: string;
+    description: string;
+    valueType: "gauge" | "counter" | "distribution";
+    unit: "ratio" | "count" | "seconds" | "milliseconds" | "tokens" | "bytes" | "usd" | "scalar";
+    direction: "higher" | "lower" | "neutral";
+    aggregation: "last" | "sum" | "mean" | "min" | "max" | "p50" | "p95";
+    visibility: "host_private" | "policy_visible" | "team_visible";
+    boundedDimensions: string[];
+}];
+export type CoreMetricId = (typeof CORE_METRIC_CATALOG)[number]["id"];
+export declare function getCoreMetricDefinition(metricId: string): MetricDefinition | undefined;
+export declare function validateCoreMetricObservation(input: MetricObservation): MetricObservation;
+export declare function validateMetricObservation(input: MetricObservation, definitions: readonly MetricDefinition[]): MetricObservation;
+export declare const MetricCatalogSchema: z.ZodArray<z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.metricDefinition.v1">;
+    id: z.ZodString;
+    displayName: z.ZodString;
+    description: z.ZodString;
+    valueType: z.ZodEnum<{
+        gauge: "gauge";
+        counter: "counter";
+        distribution: "distribution";
+    }>;
+    unit: z.ZodEnum<{
+        ratio: "ratio";
+        count: "count";
+        seconds: "seconds";
+        milliseconds: "milliseconds";
+        tokens: "tokens";
+        bytes: "bytes";
+        usd: "usd";
+        scalar: "scalar";
+    }>;
+    direction: z.ZodEnum<{
+        higher: "higher";
+        lower: "lower";
+        neutral: "neutral";
+    }>;
+    aggregation: z.ZodEnum<{
+        last: "last";
+        sum: "sum";
+        mean: "mean";
+        min: "min";
+        max: "max";
+        p50: "p50";
+        p95: "p95";
+    }>;
+    visibility: z.ZodEnum<{
+        host_private: "host_private";
+        policy_visible: "policy_visible";
+        team_visible: "team_visible";
+    }>;
+    boundedDimensions: z.ZodArray<z.ZodString>;
+}, z.core.$strict>>;
+//# sourceMappingURL=telemetry-catalog.d.ts.map

package/dist/types/telemetry-catalog.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"telemetry-catalog.d.ts","sourceRoot":"","sources":["../../../../src/telemetry-catalog.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAA0B,KAAK,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAKvG,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqBtB,CAAC;AAEX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;AAGtE,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAEtF;AAED,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,iBAAiB,GAAG,iBAAiB,CAEzF;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,gBAAgB,EAAE,GAAG,iBAAiB,CAU/H;AAED,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAI9B,CAAC"}

package/dist/types/telemetry.d.ts

@@ -0,0 +1,266 @@
+import { z } from "zod";
+export declare const RUN_TELEMETRY_SCHEMA_VERSION: "openpond.runTelemetryEvent.v1";
+export declare const METRIC_DEFINITION_SCHEMA_VERSION: "openpond.metricDefinition.v1";
+export declare const METRIC_OBSERVATION_SCHEMA_VERSION: "openpond.metricObservation.v1";
+export declare const TelemetryVisibilitySchema: z.ZodEnum<{
+    host_private: "host_private";
+    policy_visible: "policy_visible";
+    team_visible: "team_visible";
+}>;
+export declare const TelemetrySourceSchema: z.ZodEnum<{
+    environment: "environment";
+    evaluation: "evaluation";
+    runtime: "runtime";
+    grader: "grader";
+    optimizer: "optimizer";
+    control_plane: "control_plane";
+}>;
+export declare const TelemetryEventTypeSchema: z.ZodEnum<{
+    cleanup_completed: "cleanup_completed";
+    run_started: "run_started";
+    run_state_changed: "run_state_changed";
+    rollout_group_started: "rollout_group_started";
+    attempt_completed: "attempt_completed";
+    grader_completed: "grader_completed";
+    reward_composed: "reward_composed";
+    optimizer_step_completed: "optimizer_step_completed";
+    checkpoint_committed: "checkpoint_committed";
+    evaluation_completed: "evaluation_completed";
+    run_completed: "run_completed";
+    run_failed: "run_failed";
+}>;
+export declare const RunTelemetryLineageSchema: z.ZodObject<{
+    modelProjectId: z.ZodString;
+    runId: z.ZodString;
+    modelVersionId: z.ZodNullable<z.ZodString>;
+    harnessReleaseHash: z.ZodString;
+    tasksetReleaseHash: z.ZodString;
+    environmentReleaseHash: z.ZodNullable<z.ZodString>;
+    checkpointId: z.ZodNullable<z.ZodString>;
+    step: z.ZodNullable<z.ZodNumber>;
+    rolloutGroupId: z.ZodNullable<z.ZodString>;
+    attemptId: z.ZodNullable<z.ZodString>;
+    scenarioId: z.ZodNullable<z.ZodString>;
+}, z.core.$strict>;
+export declare const RunTelemetryEventSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.runTelemetryEvent.v1">;
+    eventId: z.ZodString;
+    sequence: z.ZodNumber;
+    occurredAt: z.ZodString;
+    source: z.ZodEnum<{
+        environment: "environment";
+        evaluation: "evaluation";
+        runtime: "runtime";
+        grader: "grader";
+        optimizer: "optimizer";
+        control_plane: "control_plane";
+    }>;
+    type: z.ZodEnum<{
+        cleanup_completed: "cleanup_completed";
+        run_started: "run_started";
+        run_state_changed: "run_state_changed";
+        rollout_group_started: "rollout_group_started";
+        attempt_completed: "attempt_completed";
+        grader_completed: "grader_completed";
+        reward_composed: "reward_composed";
+        optimizer_step_completed: "optimizer_step_completed";
+        checkpoint_committed: "checkpoint_committed";
+        evaluation_completed: "evaluation_completed";
+        run_completed: "run_completed";
+        run_failed: "run_failed";
+    }>;
+    visibility: z.ZodEnum<{
+        host_private: "host_private";
+        policy_visible: "policy_visible";
+        team_visible: "team_visible";
+    }>;
+    lineage: z.ZodObject<{
+        modelProjectId: z.ZodString;
+        runId: z.ZodString;
+        modelVersionId: z.ZodNullable<z.ZodString>;
+        harnessReleaseHash: z.ZodString;
+        tasksetReleaseHash: z.ZodString;
+        environmentReleaseHash: z.ZodNullable<z.ZodString>;
+        checkpointId: z.ZodNullable<z.ZodString>;
+        step: z.ZodNullable<z.ZodNumber>;
+        rolloutGroupId: z.ZodNullable<z.ZodString>;
+        attemptId: z.ZodNullable<z.ZodString>;
+        scenarioId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>;
+    attributes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const MetricValueTypeSchema: z.ZodEnum<{
+    gauge: "gauge";
+    counter: "counter";
+    distribution: "distribution";
+}>;
+export declare const MetricUnitSchema: z.ZodEnum<{
+    ratio: "ratio";
+    count: "count";
+    seconds: "seconds";
+    milliseconds: "milliseconds";
+    tokens: "tokens";
+    bytes: "bytes";
+    usd: "usd";
+    scalar: "scalar";
+}>;
+export declare const MetricDirectionSchema: z.ZodEnum<{
+    higher: "higher";
+    lower: "lower";
+    neutral: "neutral";
+}>;
+export declare const MetricAggregationSchema: z.ZodEnum<{
+    last: "last";
+    sum: "sum";
+    mean: "mean";
+    min: "min";
+    max: "max";
+    p50: "p50";
+    p95: "p95";
+}>;
+export declare const MetricDefinitionSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.metricDefinition.v1">;
+    id: z.ZodString;
+    displayName: z.ZodString;
+    description: z.ZodString;
+    valueType: z.ZodEnum<{
+        gauge: "gauge";
+        counter: "counter";
+        distribution: "distribution";
+    }>;
+    unit: z.ZodEnum<{
+        ratio: "ratio";
+        count: "count";
+        seconds: "seconds";
+        milliseconds: "milliseconds";
+        tokens: "tokens";
+        bytes: "bytes";
+        usd: "usd";
+        scalar: "scalar";
+    }>;
+    direction: z.ZodEnum<{
+        higher: "higher";
+        lower: "lower";
+        neutral: "neutral";
+    }>;
+    aggregation: z.ZodEnum<{
+        last: "last";
+        sum: "sum";
+        mean: "mean";
+        min: "min";
+        max: "max";
+        p50: "p50";
+        p95: "p95";
+    }>;
+    visibility: z.ZodEnum<{
+        host_private: "host_private";
+        policy_visible: "policy_visible";
+        team_visible: "team_visible";
+    }>;
+    boundedDimensions: z.ZodArray<z.ZodString>;
+}, z.core.$strict>;
+export declare const MetricObservationSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.metricObservation.v1">;
+    observationId: z.ZodString;
+    metricId: z.ZodString;
+    eventId: z.ZodString;
+    sequence: z.ZodNumber;
+    observedAt: z.ZodString;
+    value: z.ZodNumber;
+    lineage: z.ZodObject<{
+        modelProjectId: z.ZodString;
+        runId: z.ZodString;
+        modelVersionId: z.ZodNullable<z.ZodString>;
+        harnessReleaseHash: z.ZodString;
+        tasksetReleaseHash: z.ZodString;
+        environmentReleaseHash: z.ZodNullable<z.ZodString>;
+        checkpointId: z.ZodNullable<z.ZodString>;
+        step: z.ZodNullable<z.ZodNumber>;
+        rolloutGroupId: z.ZodNullable<z.ZodString>;
+        attemptId: z.ZodNullable<z.ZodString>;
+        scenarioId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>;
+    dimensions: z.ZodRecord<z.ZodString, z.ZodString>;
+}, z.core.$strict>;
+export declare const RunTelemetryBatchSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.runTelemetryBatch.v1">;
+    events: z.ZodArray<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.runTelemetryEvent.v1">;
+        eventId: z.ZodString;
+        sequence: z.ZodNumber;
+        occurredAt: z.ZodString;
+        source: z.ZodEnum<{
+            environment: "environment";
+            evaluation: "evaluation";
+            runtime: "runtime";
+            grader: "grader";
+            optimizer: "optimizer";
+            control_plane: "control_plane";
+        }>;
+        type: z.ZodEnum<{
+            cleanup_completed: "cleanup_completed";
+            run_started: "run_started";
+            run_state_changed: "run_state_changed";
+            rollout_group_started: "rollout_group_started";
+            attempt_completed: "attempt_completed";
+            grader_completed: "grader_completed";
+            reward_composed: "reward_composed";
+            optimizer_step_completed: "optimizer_step_completed";
+            checkpoint_committed: "checkpoint_committed";
+            evaluation_completed: "evaluation_completed";
+            run_completed: "run_completed";
+            run_failed: "run_failed";
+        }>;
+        visibility: z.ZodEnum<{
+            host_private: "host_private";
+            policy_visible: "policy_visible";
+            team_visible: "team_visible";
+        }>;
+        lineage: z.ZodObject<{
+            modelProjectId: z.ZodString;
+            runId: z.ZodString;
+            modelVersionId: z.ZodNullable<z.ZodString>;
+            harnessReleaseHash: z.ZodString;
+            tasksetReleaseHash: z.ZodString;
+            environmentReleaseHash: z.ZodNullable<z.ZodString>;
+            checkpointId: z.ZodNullable<z.ZodString>;
+            step: z.ZodNullable<z.ZodNumber>;
+            rolloutGroupId: z.ZodNullable<z.ZodString>;
+            attemptId: z.ZodNullable<z.ZodString>;
+            scenarioId: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>;
+        attributes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>>;
+    observations: z.ZodArray<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.metricObservation.v1">;
+        observationId: z.ZodString;
+        metricId: z.ZodString;
+        eventId: z.ZodString;
+        sequence: z.ZodNumber;
+        observedAt: z.ZodString;
+        value: z.ZodNumber;
+        lineage: z.ZodObject<{
+            modelProjectId: z.ZodString;
+            runId: z.ZodString;
+            modelVersionId: z.ZodNullable<z.ZodString>;
+            harnessReleaseHash: z.ZodString;
+            tasksetReleaseHash: z.ZodString;
+            environmentReleaseHash: z.ZodNullable<z.ZodString>;
+            checkpointId: z.ZodNullable<z.ZodString>;
+            step: z.ZodNullable<z.ZodNumber>;
+            rolloutGroupId: z.ZodNullable<z.ZodString>;
+            attemptId: z.ZodNullable<z.ZodString>;
+            scenarioId: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>;
+        dimensions: z.ZodRecord<z.ZodString, z.ZodString>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type TelemetryVisibility = z.infer<typeof TelemetryVisibilitySchema>;
+export type TelemetrySource = z.infer<typeof TelemetrySourceSchema>;
+export type TelemetryEventType = z.infer<typeof TelemetryEventTypeSchema>;
+export type RunTelemetryLineage = z.infer<typeof RunTelemetryLineageSchema>;
+export type RunTelemetryEvent = z.infer<typeof RunTelemetryEventSchema>;
+export type MetricDefinition = z.infer<typeof MetricDefinitionSchema>;
+export type MetricObservation = z.infer<typeof MetricObservationSchema>;
+export type RunTelemetryBatch = z.infer<typeof RunTelemetryBatchSchema>;
+//# sourceMappingURL=telemetry.d.ts.map

package/dist/types/telemetry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../../../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,eAAO,MAAM,4BAA4B,EAAG,+BAAwC,CAAC;AACrF,eAAO,MAAM,gCAAgC,EAAG,8BAAuC,CAAC;AACxF,eAAO,MAAM,iCAAiC,EAAG,+BAAwC,CAAC;AAE1F,eAAO,MAAM,yBAAyB;;;;EAIpC,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;;;EAOhC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;EAanC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;kBAY3B,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;EAA+C,CAAC;AAClF,eAAO,MAAM,gBAAgB;;;;;;;;;EAS3B,CAAC;AACH,eAAO,MAAM,qBAAqB;;;;EAAyC,CAAC;AAC5E,eAAO,MAAM,uBAAuB;;;;;;;;EAA8D,CAAC;AAEnG,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAWxB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAmBlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}

package/package.json

@@ -1,19 +1,43 @@
 {
   "name": "@openpond/evals",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
+  "sideEffects": false,
   "main": "./dist/index.js",
   "types": "./dist/types/index.d.ts",
   "files": [
     "dist",
+    "conformance",
+    "schemas",
     "README.md",
     "CONTRACT.md",
     "RELEASING.md",
+    "LEARNING.md",
     "LICENSE"
   ],
   "exports": {
+    "./javascript-verifier/node": {
+      "types": "./dist/types/javascript-verifier-node.d.ts",
+      "import": "./dist/javascript-verifier-node.js"
+    },
+    "./javascript-verifier": {
+      "types": "./dist/types/javascript-verifier.d.ts",
+      "import": "./dist/javascript-verifier.js"
+    },
+    "./task-schema": {
+      "types": "./dist/types/task-schema.d.ts",
+      "import": "./dist/task-schema.js"
+    },
+    "./learning": {
+      "types": "./dist/types/learning/index.d.ts",
+      "import": "./dist/learning/index.js"
+    },
+    "./rewards": {
+      "types": "./dist/types/rewards.d.ts",
+      "import": "./dist/rewards.js"
+    },
     ".": {
       "types": "./dist/types/index.d.ts",
       "import": "./dist/index.js"
@@ -42,10 +66,18 @@
       "types": "./dist/types/rollouts.d.ts",
       "import": "./dist/rollouts.js"
     },
+    "./telemetry": {
+      "types": "./dist/types/telemetry/index.d.ts",
+      "import": "./dist/telemetry/index.js"
+    },
     "./preferences": {
       "types": "./dist/types/preferences.d.ts",
       "import": "./dist/preferences.js"
     },
+    "./learned-preference": {
+      "types": "./dist/types/learned-preference.d.ts",
+      "import": "./dist/learned-preference.js"
+    },
     "./review": {
       "types": "./dist/types/review-conformance.d.ts",
       "import": "./dist/review-conformance.js"
@@ -81,14 +113,18 @@
     "node": ">=22.14.0 <25"
   },
   "scripts": {
-    "build": "tsx scripts/build.ts",
+    "build": "tsx scripts/generate-telemetry-schemas.ts && tsx scripts/generate-learning-schemas.ts && tsx scripts/build.ts",
     "typecheck": "tsc --noEmit --project tsconfig.json",
     "test": "vitest run --root ../.. --config vitest.config.ts --project evals",
     "check:api": "tsx scripts/check-public-package.ts",
     "check:consumer": "tsx scripts/verify-packed-consumer.ts",
-    "check": "pnpm run typecheck && pnpm run test && pnpm run build && pnpm run check:api && pnpm run check:consumer && npm pack --dry-run"
+    "check": "pnpm run typecheck && pnpm run test && pnpm run build && pnpm run check:api && pnpm run check:consumer && npm pack --dry-run",
+    "publish:check": "pnpm run build && pnpm run check:api && pnpm run check:consumer && npm pack --dry-run"
   },
   "dependencies": {
+    "@jitl/quickjs-singlefile-cjs-release-sync": "0.32.0",
+    "ajv": "^8.20.0",
+    "quickjs-emscripten-core": "0.32.0",
     "zod": "^4.1.13"
   },
   "peerDependencies": {
@@ -97,6 +133,7 @@
   "devDependencies": {
     "@openpond/harness": "workspace:*",
     "@types/node": "24.13.3",
+    "esbuild": "0.28.1",
     "tsx": "4.23.1",
     "typescript": "^5.9.3",
     "vitest": "4.1.10"