@openpond/evals 0.4.2 → 0.6.0

package/dist/types/telemetry.d.ts

@@ -0,0 +1,266 @@
+import { z } from "zod";
+export declare const RUN_TELEMETRY_SCHEMA_VERSION: "openpond.runTelemetryEvent.v1";
+export declare const METRIC_DEFINITION_SCHEMA_VERSION: "openpond.metricDefinition.v1";
+export declare const METRIC_OBSERVATION_SCHEMA_VERSION: "openpond.metricObservation.v1";
+export declare const TelemetryVisibilitySchema: z.ZodEnum<{
+    host_private: "host_private";
+    policy_visible: "policy_visible";
+    team_visible: "team_visible";
+}>;
+export declare const TelemetrySourceSchema: z.ZodEnum<{
+    environment: "environment";
+    evaluation: "evaluation";
+    runtime: "runtime";
+    grader: "grader";
+    optimizer: "optimizer";
+    control_plane: "control_plane";
+}>;
+export declare const TelemetryEventTypeSchema: z.ZodEnum<{
+    cleanup_completed: "cleanup_completed";
+    run_started: "run_started";
+    run_state_changed: "run_state_changed";
+    rollout_group_started: "rollout_group_started";
+    attempt_completed: "attempt_completed";
+    grader_completed: "grader_completed";
+    reward_composed: "reward_composed";
+    optimizer_step_completed: "optimizer_step_completed";
+    checkpoint_committed: "checkpoint_committed";
+    evaluation_completed: "evaluation_completed";
+    run_completed: "run_completed";
+    run_failed: "run_failed";
+}>;
+export declare const RunTelemetryLineageSchema: z.ZodObject<{
+    modelProjectId: z.ZodString;
+    runId: z.ZodString;
+    modelVersionId: z.ZodNullable<z.ZodString>;
+    harnessReleaseHash: z.ZodString;
+    tasksetReleaseHash: z.ZodString;
+    environmentReleaseHash: z.ZodNullable<z.ZodString>;
+    checkpointId: z.ZodNullable<z.ZodString>;
+    step: z.ZodNullable<z.ZodNumber>;
+    rolloutGroupId: z.ZodNullable<z.ZodString>;
+    attemptId: z.ZodNullable<z.ZodString>;
+    scenarioId: z.ZodNullable<z.ZodString>;
+}, z.core.$strict>;
+export declare const RunTelemetryEventSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.runTelemetryEvent.v1">;
+    eventId: z.ZodString;
+    sequence: z.ZodNumber;
+    occurredAt: z.ZodString;
+    source: z.ZodEnum<{
+        environment: "environment";
+        evaluation: "evaluation";
+        runtime: "runtime";
+        grader: "grader";
+        optimizer: "optimizer";
+        control_plane: "control_plane";
+    }>;
+    type: z.ZodEnum<{
+        cleanup_completed: "cleanup_completed";
+        run_started: "run_started";
+        run_state_changed: "run_state_changed";
+        rollout_group_started: "rollout_group_started";
+        attempt_completed: "attempt_completed";
+        grader_completed: "grader_completed";
+        reward_composed: "reward_composed";
+        optimizer_step_completed: "optimizer_step_completed";
+        checkpoint_committed: "checkpoint_committed";
+        evaluation_completed: "evaluation_completed";
+        run_completed: "run_completed";
+        run_failed: "run_failed";
+    }>;
+    visibility: z.ZodEnum<{
+        host_private: "host_private";
+        policy_visible: "policy_visible";
+        team_visible: "team_visible";
+    }>;
+    lineage: z.ZodObject<{
+        modelProjectId: z.ZodString;
+        runId: z.ZodString;
+        modelVersionId: z.ZodNullable<z.ZodString>;
+        harnessReleaseHash: z.ZodString;
+        tasksetReleaseHash: z.ZodString;
+        environmentReleaseHash: z.ZodNullable<z.ZodString>;
+        checkpointId: z.ZodNullable<z.ZodString>;
+        step: z.ZodNullable<z.ZodNumber>;
+        rolloutGroupId: z.ZodNullable<z.ZodString>;
+        attemptId: z.ZodNullable<z.ZodString>;
+        scenarioId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>;
+    attributes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const MetricValueTypeSchema: z.ZodEnum<{
+    gauge: "gauge";
+    counter: "counter";
+    distribution: "distribution";
+}>;
+export declare const MetricUnitSchema: z.ZodEnum<{
+    ratio: "ratio";
+    count: "count";
+    seconds: "seconds";
+    milliseconds: "milliseconds";
+    tokens: "tokens";
+    bytes: "bytes";
+    usd: "usd";
+    scalar: "scalar";
+}>;
+export declare const MetricDirectionSchema: z.ZodEnum<{
+    higher: "higher";
+    lower: "lower";
+    neutral: "neutral";
+}>;
+export declare const MetricAggregationSchema: z.ZodEnum<{
+    last: "last";
+    sum: "sum";
+    mean: "mean";
+    min: "min";
+    max: "max";
+    p50: "p50";
+    p95: "p95";
+}>;
+export declare const MetricDefinitionSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.metricDefinition.v1">;
+    id: z.ZodString;
+    displayName: z.ZodString;
+    description: z.ZodString;
+    valueType: z.ZodEnum<{
+        gauge: "gauge";
+        counter: "counter";
+        distribution: "distribution";
+    }>;
+    unit: z.ZodEnum<{
+        ratio: "ratio";
+        count: "count";
+        seconds: "seconds";
+        milliseconds: "milliseconds";
+        tokens: "tokens";
+        bytes: "bytes";
+        usd: "usd";
+        scalar: "scalar";
+    }>;
+    direction: z.ZodEnum<{
+        higher: "higher";
+        lower: "lower";
+        neutral: "neutral";
+    }>;
+    aggregation: z.ZodEnum<{
+        last: "last";
+        sum: "sum";
+        mean: "mean";
+        min: "min";
+        max: "max";
+        p50: "p50";
+        p95: "p95";
+    }>;
+    visibility: z.ZodEnum<{
+        host_private: "host_private";
+        policy_visible: "policy_visible";
+        team_visible: "team_visible";
+    }>;
+    boundedDimensions: z.ZodArray<z.ZodString>;
+}, z.core.$strict>;
+export declare const MetricObservationSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.metricObservation.v1">;
+    observationId: z.ZodString;
+    metricId: z.ZodString;
+    eventId: z.ZodString;
+    sequence: z.ZodNumber;
+    observedAt: z.ZodString;
+    value: z.ZodNumber;
+    lineage: z.ZodObject<{
+        modelProjectId: z.ZodString;
+        runId: z.ZodString;
+        modelVersionId: z.ZodNullable<z.ZodString>;
+        harnessReleaseHash: z.ZodString;
+        tasksetReleaseHash: z.ZodString;
+        environmentReleaseHash: z.ZodNullable<z.ZodString>;
+        checkpointId: z.ZodNullable<z.ZodString>;
+        step: z.ZodNullable<z.ZodNumber>;
+        rolloutGroupId: z.ZodNullable<z.ZodString>;
+        attemptId: z.ZodNullable<z.ZodString>;
+        scenarioId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>;
+    dimensions: z.ZodRecord<z.ZodString, z.ZodString>;
+}, z.core.$strict>;
+export declare const RunTelemetryBatchSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.runTelemetryBatch.v1">;
+    events: z.ZodArray<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.runTelemetryEvent.v1">;
+        eventId: z.ZodString;
+        sequence: z.ZodNumber;
+        occurredAt: z.ZodString;
+        source: z.ZodEnum<{
+            environment: "environment";
+            evaluation: "evaluation";
+            runtime: "runtime";
+            grader: "grader";
+            optimizer: "optimizer";
+            control_plane: "control_plane";
+        }>;
+        type: z.ZodEnum<{
+            cleanup_completed: "cleanup_completed";
+            run_started: "run_started";
+            run_state_changed: "run_state_changed";
+            rollout_group_started: "rollout_group_started";
+            attempt_completed: "attempt_completed";
+            grader_completed: "grader_completed";
+            reward_composed: "reward_composed";
+            optimizer_step_completed: "optimizer_step_completed";
+            checkpoint_committed: "checkpoint_committed";
+            evaluation_completed: "evaluation_completed";
+            run_completed: "run_completed";
+            run_failed: "run_failed";
+        }>;
+        visibility: z.ZodEnum<{
+            host_private: "host_private";
+            policy_visible: "policy_visible";
+            team_visible: "team_visible";
+        }>;
+        lineage: z.ZodObject<{
+            modelProjectId: z.ZodString;
+            runId: z.ZodString;
+            modelVersionId: z.ZodNullable<z.ZodString>;
+            harnessReleaseHash: z.ZodString;
+            tasksetReleaseHash: z.ZodString;
+            environmentReleaseHash: z.ZodNullable<z.ZodString>;
+            checkpointId: z.ZodNullable<z.ZodString>;
+            step: z.ZodNullable<z.ZodNumber>;
+            rolloutGroupId: z.ZodNullable<z.ZodString>;
+            attemptId: z.ZodNullable<z.ZodString>;
+            scenarioId: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>;
+        attributes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>>;
+    observations: z.ZodArray<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.metricObservation.v1">;
+        observationId: z.ZodString;
+        metricId: z.ZodString;
+        eventId: z.ZodString;
+        sequence: z.ZodNumber;
+        observedAt: z.ZodString;
+        value: z.ZodNumber;
+        lineage: z.ZodObject<{
+            modelProjectId: z.ZodString;
+            runId: z.ZodString;
+            modelVersionId: z.ZodNullable<z.ZodString>;
+            harnessReleaseHash: z.ZodString;
+            tasksetReleaseHash: z.ZodString;
+            environmentReleaseHash: z.ZodNullable<z.ZodString>;
+            checkpointId: z.ZodNullable<z.ZodString>;
+            step: z.ZodNullable<z.ZodNumber>;
+            rolloutGroupId: z.ZodNullable<z.ZodString>;
+            attemptId: z.ZodNullable<z.ZodString>;
+            scenarioId: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>;
+        dimensions: z.ZodRecord<z.ZodString, z.ZodString>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type TelemetryVisibility = z.infer<typeof TelemetryVisibilitySchema>;
+export type TelemetrySource = z.infer<typeof TelemetrySourceSchema>;
+export type TelemetryEventType = z.infer<typeof TelemetryEventTypeSchema>;
+export type RunTelemetryLineage = z.infer<typeof RunTelemetryLineageSchema>;
+export type RunTelemetryEvent = z.infer<typeof RunTelemetryEventSchema>;
+export type MetricDefinition = z.infer<typeof MetricDefinitionSchema>;
+export type MetricObservation = z.infer<typeof MetricObservationSchema>;
+export type RunTelemetryBatch = z.infer<typeof RunTelemetryBatchSchema>;
+//# sourceMappingURL=telemetry.d.ts.map

package/dist/types/telemetry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../../../src/telemetry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,eAAO,MAAM,4BAA4B,EAAG,+BAAwC,CAAC;AACrF,eAAO,MAAM,gCAAgC,EAAG,8BAAuC,CAAC;AACxF,eAAO,MAAM,iCAAiC,EAAG,+BAAwC,CAAC;AAE1F,eAAO,MAAM,yBAAyB;;;;EAIpC,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;;;EAOhC,CAAC;AAEH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;EAanC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;kBAY3B,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;EAA+C,CAAC;AAClF,eAAO,MAAM,gBAAgB;;;;;;;;;EAS3B,CAAC;AACH,eAAO,MAAM,qBAAqB;;;;EAAyC,CAAC;AAC5E,eAAO,MAAM,uBAAuB;;;;;;;;EAA8D,CAAC;AAEnG,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAWxB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;kBAUzB,CAAC;AAEZ,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAmBlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.4.2",
+  "version": "0.6.0",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
@@ -8,6 +8,8 @@
   "types": "./dist/types/index.d.ts",
   "files": [
     "dist",
+    "conformance",
+    "schemas",
     "README.md",
     "CONTRACT.md",
     "RELEASING.md",
@@ -38,6 +40,22 @@
       "types": "./dist/types/runs.d.ts",
       "import": "./dist/runs.js"
     },
+    "./rollouts": {
+      "types": "./dist/types/rollouts.d.ts",
+      "import": "./dist/rollouts.js"
+    },
+    "./telemetry": {
+      "types": "./dist/types/telemetry/index.d.ts",
+      "import": "./dist/telemetry/index.js"
+    },
+    "./preferences": {
+      "types": "./dist/types/preferences.d.ts",
+      "import": "./dist/preferences.js"
+    },
+    "./learned-preference": {
+      "types": "./dist/types/learned-preference.d.ts",
+      "import": "./dist/learned-preference.js"
+    },
     "./review": {
       "types": "./dist/types/review-conformance.d.ts",
       "import": "./dist/review-conformance.js"
@@ -73,7 +91,7 @@
     "node": ">=22.14.0 <25"
   },
   "scripts": {
-    "build": "tsx scripts/build.ts",
+    "build": "tsx scripts/generate-telemetry-schemas.ts && tsx scripts/build.ts",
     "typecheck": "tsc --noEmit --project tsconfig.json",
     "test": "vitest run --root ../.. --config vitest.config.ts --project evals",
     "check:api": "tsx scripts/check-public-package.ts",

package/schemas/telemetry/v1/evidence-completeness.schema.json

@@ -0,0 +1,82 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "openpond.telemetryEvidenceCompleteness.v1"
+    },
+    "runId": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "status": {
+      "type": "string",
+      "enum": [
+        "complete",
+        "partial",
+        "missing"
+      ]
+    },
+    "expectedEventTypes": {
+      "maxItems": 100,
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 100
+      }
+    },
+    "observedEventTypes": {
+      "maxItems": 100,
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 100
+      }
+    },
+    "missingEventTypes": {
+      "maxItems": 100,
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 100
+      }
+    },
+    "lastSequence": {
+      "anyOf": [
+        {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "sequenceGaps": {
+      "maxItems": 10000,
+      "type": "array",
+      "items": {
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 9007199254740991
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "runId",
+    "status",
+    "expectedEventTypes",
+    "observedEventTypes",
+    "missingEventTypes",
+    "lastSequence",
+    "sequenceGaps"
+  ],
+  "additionalProperties": false
+}

package/schemas/telemetry/v1/evidence-reference.schema.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "contentHash": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    },
+    "kind": {
+      "type": "string",
+      "enum": [
+        "rollout",
+        "attempt",
+        "trace",
+        "grader",
+        "checkpoint",
+        "artifact"
+      ]
+    },
+    "visibility": {
+      "type": "string",
+      "enum": [
+        "policy_visible",
+        "team_visible",
+        "host_private"
+      ]
+    }
+  },
+  "required": [
+    "id",
+    "contentHash",
+    "kind",
+    "visibility"
+  ],
+  "additionalProperties": false
+}

package/schemas/telemetry/v1/metric-definition.schema.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "openpond.metricDefinition.v1"
+    },
+    "id": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "displayName": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 200
+    },
+    "description": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 2000
+    },
+    "valueType": {
+      "type": "string",
+      "enum": [
+        "gauge",
+        "counter",
+        "distribution"
+      ]
+    },
+    "unit": {
+      "type": "string",
+      "enum": [
+        "ratio",
+        "count",
+        "seconds",
+        "milliseconds",
+        "tokens",
+        "bytes",
+        "usd",
+        "scalar"
+      ]
+    },
+    "direction": {
+      "type": "string",
+      "enum": [
+        "higher",
+        "lower",
+        "neutral"
+      ]
+    },
+    "aggregation": {
+      "type": "string",
+      "enum": [
+        "last",
+        "sum",
+        "mean",
+        "min",
+        "max",
+        "p50",
+        "p95"
+      ]
+    },
+    "visibility": {
+      "type": "string",
+      "enum": [
+        "policy_visible",
+        "team_visible",
+        "host_private"
+      ]
+    },
+    "boundedDimensions": {
+      "maxItems": 32,
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1,
+        "maxLength": 100
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "id",
+    "displayName",
+    "description",
+    "valueType",
+    "unit",
+    "direction",
+    "aggregation",
+    "visibility",
+    "boundedDimensions"
+  ],
+  "additionalProperties": false
+}