@openpond/evals 0.5.0 → 0.7.0

package/dist/telemetry-catalog.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { MetricDefinitionSchema } from "./telemetry.js";
+const definition = (input) => MetricDefinitionSchema.parse({ schemaVersion: "openpond.metricDefinition.v1", ...input });
+export const CORE_METRIC_CATALOG = [
+    definition({ id: "reward.mean", displayName: "Mean reward", description: "Mean composed reward for the cohort.", valueType: "gauge", unit: "scalar", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split", "grader"] }),
+    definition({ id: "reward.variance", displayName: "Reward variance", description: "Population variance of composed reward within a rollout group.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "reward.constant_group_rate", displayName: "Constant group rate", description: "Fraction of rollout groups with no reward variation.", valueType: "gauge", unit: "ratio", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "attempt.valid_rate", displayName: "Valid attempt rate", description: "Fraction of attempts passing deterministic validity checks.", valueType: "gauge", unit: "ratio", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split", "failureOwner"] }),
+    definition({ id: "attempt.failure_count", displayName: "Attempt failures", description: "Count of failed Attempts.", valueType: "counter", unit: "count", direction: "lower", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "failureOwner", "failureClass"] }),
+    definition({ id: "optimizer.loss", displayName: "Optimizer loss", description: "Policy optimizer loss after the step.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.learning_rate", displayName: "Learning rate", description: "Optimizer learning rate after the step.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "last", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.kl", displayName: "Policy update KL", description: "Sampled approximate KL between the frozen pre-update policy and the policy after the optimizer step.", valueType: "gauge", unit: "scalar", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.entropy", displayName: "Policy entropy", description: "Observed policy entropy for trainable tokens.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.gradient_norm", displayName: "Gradient norm", description: "Gradient norm reported by the optimizer.", valueType: "gauge", unit: "scalar", direction: "neutral", aggregation: "max", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "optimizer.clip_fraction", displayName: "Clip fraction", description: "Fraction of policy updates affected by clipping.", valueType: "gauge", unit: "ratio", direction: "neutral", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "output.duplicate_rate", displayName: "Duplicate output rate", description: "Fraction of outputs duplicated within the measured cohort.", valueType: "gauge", unit: "ratio", direction: "lower", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "output.unique_count", displayName: "Unique outputs", description: "Distinct output count in the measured cohort.", valueType: "gauge", unit: "count", direction: "higher", aggregation: "last", visibility: "team_visible", boundedDimensions: ["split"] }),
+    definition({ id: "tokens.input", displayName: "Input tokens", description: "Input tokens processed.", valueType: "counter", unit: "tokens", direction: "neutral", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "source"] }),
+    definition({ id: "tokens.output", displayName: "Output tokens", description: "Output tokens generated.", valueType: "counter", unit: "tokens", direction: "neutral", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["split", "source"] }),
+    definition({ id: "runtime.latency_ms", displayName: "Runtime latency", description: "Wall-clock latency of the measured operation.", valueType: "distribution", unit: "milliseconds", direction: "lower", aggregation: "p95", visibility: "team_visible", boundedDimensions: ["operation", "provider"] }),
+    definition({ id: "runtime.throughput", displayName: "Token throughput", description: "Tokens processed per second.", valueType: "gauge", unit: "scalar", direction: "higher", aggregation: "mean", visibility: "team_visible", boundedDimensions: ["operation", "provider"] }),
+    definition({ id: "gpu.memory_bytes", displayName: "GPU memory", description: "Peak allocated GPU memory.", valueType: "gauge", unit: "bytes", direction: "neutral", aggregation: "max", visibility: "host_private", boundedDimensions: ["provider", "gpuType"] }),
+    definition({ id: "gpu.utilization", displayName: "GPU utilization", description: "Observed GPU utilization ratio.", valueType: "gauge", unit: "ratio", direction: "neutral", aggregation: "mean", visibility: "host_private", boundedDimensions: ["provider", "gpuType"] }),
+    definition({ id: "cost.usd", displayName: "Run cost", description: "Accrued hosted execution cost.", valueType: "counter", unit: "usd", direction: "lower", aggregation: "sum", visibility: "team_visible", boundedDimensions: ["provider", "resource"] }),
+];
+const catalog = new Map(CORE_METRIC_CATALOG.map((item) => [item.id, item]));
+export function getCoreMetricDefinition(metricId) {
+    return catalog.get(metricId);
+}
+export function validateCoreMetricObservation(input) {
+    return validateMetricObservation(input, CORE_METRIC_CATALOG);
+}
+export function validateMetricObservation(input, definitions) {
+    MetricCatalogSchema.parse(definitions);
+    const metric = definitions.find((item) => item.id === input.metricId);
+    if (!metric)
+        throw new Error(`Unknown metric: ${input.metricId}`);
+    const unexpected = Object.keys(input.dimensions).filter((dimension) => !metric.boundedDimensions.includes(dimension));
+    if (unexpected.length)
+        throw new Error(`Metric ${input.metricId} has unsupported dimensions: ${unexpected.join(", ")}`);
+    if (metric.unit === "ratio" && (input.value < 0 || input.value > 1)) {
+        throw new Error(`Ratio metric ${input.metricId} must be between zero and one.`);
+    }
+    return input;
+}
+export const MetricCatalogSchema = z.array(MetricDefinitionSchema).min(1).superRefine((items, context) => {
+    if (new Set(items.map((item) => item.id)).size !== items.length) {
+        context.addIssue({ code: "custom", message: "Metric catalog contains duplicate ids." });
+    }
+});

package/dist/telemetry.js

@@ -0,0 +1,112 @@
+import { z } from "zod";
+import { MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, } from "@openpond/harness";
+export const RUN_TELEMETRY_SCHEMA_VERSION = "openpond.runTelemetryEvent.v1";
+export const METRIC_DEFINITION_SCHEMA_VERSION = "openpond.metricDefinition.v1";
+export const METRIC_OBSERVATION_SCHEMA_VERSION = "openpond.metricObservation.v1";
+export const TelemetryVisibilitySchema = z.enum([
+    "policy_visible",
+    "team_visible",
+    "host_private",
+]);
+export const TelemetrySourceSchema = z.enum([
+    "runtime",
+    "environment",
+    "grader",
+    "optimizer",
+    "control_plane",
+    "evaluation",
+]);
+export const TelemetryEventTypeSchema = z.enum([
+    "run_started",
+    "run_state_changed",
+    "rollout_group_started",
+    "attempt_completed",
+    "grader_completed",
+    "reward_composed",
+    "optimizer_step_completed",
+    "checkpoint_committed",
+    "evaluation_completed",
+    "run_completed",
+    "run_failed",
+    "cleanup_completed",
+]);
+export const RunTelemetryLineageSchema = z.object({
+    modelProjectId: ReleaseIdSchema,
+    runId: ReleaseIdSchema,
+    modelVersionId: ReleaseIdSchema.nullable(),
+    harnessReleaseHash: ReleaseHashSchema,
+    tasksetReleaseHash: ReleaseHashSchema,
+    environmentReleaseHash: ReleaseHashSchema.nullable(),
+    checkpointId: ReleaseIdSchema.nullable(),
+    step: z.number().int().nonnegative().nullable(),
+    rolloutGroupId: ReleaseIdSchema.nullable(),
+    attemptId: ReleaseIdSchema.nullable(),
+    scenarioId: ReleaseIdSchema.nullable(),
+}).strict();
+export const RunTelemetryEventSchema = z.object({
+    schemaVersion: z.literal(RUN_TELEMETRY_SCHEMA_VERSION),
+    eventId: ReleaseIdSchema,
+    sequence: z.number().int().nonnegative(),
+    occurredAt: ReleaseTimestampSchema,
+    source: TelemetrySourceSchema,
+    type: TelemetryEventTypeSchema,
+    visibility: TelemetryVisibilitySchema,
+    lineage: RunTelemetryLineageSchema,
+    attributes: MetadataSchema,
+}).strict();
+export const MetricValueTypeSchema = z.enum(["gauge", "counter", "distribution"]);
+export const MetricUnitSchema = z.enum([
+    "ratio",
+    "count",
+    "seconds",
+    "milliseconds",
+    "tokens",
+    "bytes",
+    "usd",
+    "scalar",
+]);
+export const MetricDirectionSchema = z.enum(["higher", "lower", "neutral"]);
+export const MetricAggregationSchema = z.enum(["last", "sum", "mean", "min", "max", "p50", "p95"]);
+export const MetricDefinitionSchema = z.object({
+    schemaVersion: z.literal(METRIC_DEFINITION_SCHEMA_VERSION),
+    id: ReleaseIdSchema,
+    displayName: z.string().trim().min(1).max(200),
+    description: z.string().trim().min(1).max(2_000),
+    valueType: MetricValueTypeSchema,
+    unit: MetricUnitSchema,
+    direction: MetricDirectionSchema,
+    aggregation: MetricAggregationSchema,
+    visibility: TelemetryVisibilitySchema,
+    boundedDimensions: z.array(z.string().trim().min(1).max(100)).max(32),
+}).strict();
+export const MetricObservationSchema = z.object({
+    schemaVersion: z.literal(METRIC_OBSERVATION_SCHEMA_VERSION),
+    observationId: ReleaseIdSchema,
+    metricId: ReleaseIdSchema,
+    eventId: ReleaseIdSchema,
+    sequence: z.number().int().nonnegative(),
+    observedAt: ReleaseTimestampSchema,
+    value: z.number().finite(),
+    lineage: RunTelemetryLineageSchema,
+    dimensions: z.record(z.string().trim().min(1).max(100), z.string().max(500)),
+}).strict();
+export const RunTelemetryBatchSchema = z.object({
+    schemaVersion: z.literal("openpond.runTelemetryBatch.v1"),
+    events: z.array(RunTelemetryEventSchema).max(1_000),
+    observations: z.array(MetricObservationSchema).max(10_000),
+}).strict().superRefine((batch, context) => {
+    if (batch.events.length + batch.observations.length === 0) {
+        context.addIssue({ code: "custom", message: "Telemetry batch cannot be empty." });
+    }
+    const keys = new Set();
+    for (const item of [...batch.events, ...batch.observations]) {
+        const id = item.schemaVersion === METRIC_OBSERVATION_SCHEMA_VERSION
+            ? item.observationId
+            : item.eventId;
+        const key = `${item.lineage.runId}:${item.sequence}:${id}`;
+        if (keys.has(key)) {
+            context.addIssue({ code: "custom", message: "Telemetry batch contains a duplicate idempotency key." });
+        }
+        keys.add(key);
+    }
+});

package/dist/types/builtin-benchmarks/harness-refiner.d.ts

@@ -90,6 +90,12 @@ export declare const harnessRefinerBenchmarkRelease: {
             visibility: "policy" | "verifier" | "host_private";
         };
         calibrationStatus: "pending" | "passed" | "failed";
+        model?: {
+            providerId: string;
+            modelId: string;
+            revision: string | null;
+        } | undefined;
+        temperature?: number | undefined;
     } | {
         id: string;
         version: string;
@@ -108,6 +114,7 @@ export declare const harnessRefinerBenchmarkRelease: {
         };
         timeoutMs: number;
         networkPolicy: "none";
+        exportName?: string | undefined;
     } | {
         id: string;
         version: string;

package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
+{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}

package/dist/types/conformance.d.ts

@@ -200,6 +200,12 @@ export declare const genericToolConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             };
             calibrationStatus: "pending" | "passed" | "failed";
+            model?: {
+                providerId: string;
+                modelId: string;
+                revision: string | null;
+            } | undefined;
+            temperature?: number | undefined;
         } | {
             id: string;
             version: string;
@@ -218,6 +224,7 @@ export declare const genericToolConformance: {
             };
             timeoutMs: number;
             networkPolicy: "none";
+            exportName?: string | undefined;
         } | {
             id: string;
             version: string;
@@ -489,6 +496,12 @@ export declare const marketingPortfolioConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             };
             calibrationStatus: "pending" | "passed" | "failed";
+            model?: {
+                providerId: string;
+                modelId: string;
+                revision: string | null;
+            } | undefined;
+            temperature?: number | undefined;
         } | {
             id: string;
             version: string;
@@ -507,6 +520,7 @@ export declare const marketingPortfolioConformance: {
             };
             timeoutMs: number;
             networkPolicy: "none";
+            exportName?: string | undefined;
         } | {
             id: string;
             version: string;

package/dist/types/conformance.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}
+{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}

package/dist/types/execution-contracts.d.ts

@@ -385,6 +385,12 @@ export declare const VerifierSetReleaseContentSchema: z.ZodObject<{
             passed: "passed";
             failed: "failed";
         }>;
+        model: z.ZodOptional<z.ZodObject<{
+            providerId: z.ZodString;
+            modelId: z.ZodString;
+            revision: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>>;
+        temperature: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strict>, z.ZodObject<{
         id: z.ZodString;
         version: z.ZodString;
@@ -405,6 +411,7 @@ export declare const VerifierSetReleaseContentSchema: z.ZodObject<{
                 host_private: "host_private";
             }>;
         }, z.core.$strict>;
+        exportName: z.ZodOptional<z.ZodString>;
         timeoutMs: z.ZodNumber;
         networkPolicy: z.ZodLiteral<"none">;
     }, z.core.$strict>, z.ZodObject<{
@@ -488,6 +495,12 @@ export declare const VerifierSetReleaseSchema: z.ZodObject<{
             passed: "passed";
             failed: "failed";
         }>;
+        model: z.ZodOptional<z.ZodObject<{
+            providerId: z.ZodString;
+            modelId: z.ZodString;
+            revision: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>>;
+        temperature: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strict>, z.ZodObject<{
         id: z.ZodString;
         version: z.ZodString;
@@ -508,6 +521,7 @@ export declare const VerifierSetReleaseSchema: z.ZodObject<{
                 host_private: "host_private";
             }>;
         }, z.core.$strict>;
+        exportName: z.ZodOptional<z.ZodString>;
         timeoutMs: z.ZodNumber;
         networkPolicy: z.ZodLiteral<"none">;
     }, z.core.$strict>, z.ZodObject<{

package/dist/types/execution-contracts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execution-contracts.d.ts","sourceRoot":"","sources":["../../../../src/execution-contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAcxB,eAAO,MAAM,mBAAmB;;;EAAmC,CAAC;AACpE,eAAO,MAAM,kBAAkB;;;;;;;;;EAS7B,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;EAYpC,CAAC;AAEH,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAcjC,CAAC;AACZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE1B,CAAC;AAEZ,eAAO,MAAM,8BAA8B;;;;;EAKzC,CAAC;AACH,eAAO,MAAM,8BAA8B;;;;EAIzC,CAAC;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAa7B,CAAC;AACZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAO/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAExB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAYjC,CAAC;AACZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE1B,CAAC;AAEZ,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuBvC,CAAC;AAkCH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA6D,CAAC;AACrG,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGK,CAAC;AAEtC,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}
+{"version":3,"file":"execution-contracts.d.ts","sourceRoot":"","sources":["../../../../src/execution-contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAcxB,eAAO,MAAM,mBAAmB;;;EAAmC,CAAC;AACpE,eAAO,MAAM,kBAAkB;;;;;;;;;EAS7B,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;EAYpC,CAAC;AAEH,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAcjC,CAAC;AACZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE1B,CAAC;AAEZ,eAAO,MAAM,8BAA8B;;;;;EAKzC,CAAC;AACH,eAAO,MAAM,8BAA8B;;;;EAIzC,CAAC;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAa7B,CAAC;AACZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAO/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAExB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAYjC,CAAC;AACZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE1B,CAAC;AAEZ,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuBvC,CAAC;AAkCH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA6D,CAAC;AACrG,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGK,CAAC;AAEtC,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}

package/dist/types/graders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AAkDD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGrF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA+DD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}

package/dist/types/index.d.ts

@@ -6,10 +6,18 @@ export * from "./execution-contracts.js";
 export * from "./execution-receipts.js";
 export * from "./graders.js";
 export * from "./harness.js";
+export * from "./learned-preference.js";
 export * from "./runs.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
+export * from "./telemetry.js";
+export * from "./telemetry-catalog.js";
+export * from "./telemetry-analysis.js";
+export * from "./telemetry-bundle.js";
 export * from "./tasksets.js";
+export * from "./task-schema.js";
+export * from "./rewards.js";
+export * from "./learning/index.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}

package/dist/types/javascript-verifier-contract.d.ts

@@ -0,0 +1,9 @@
+import { z } from "zod";
+export declare const JavaScriptVerifierResultSchema: z.ZodObject<{
+    score: z.ZodNumber;
+    passed: z.ZodBoolean;
+    feedback: z.ZodString;
+    evidenceRefs: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strict>;
+export type JavaScriptVerifierResult = z.infer<typeof JavaScriptVerifierResultSchema>;
+//# sourceMappingURL=javascript-verifier-contract.d.ts.map

package/dist/types/javascript-verifier-contract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript-verifier-contract.d.ts","sourceRoot":"","sources":["../../../../src/javascript-verifier-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,8BAA8B;;;;;kBAKhC,CAAC;AACZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC"}

package/dist/types/javascript-verifier-node.d.ts

@@ -0,0 +1,5 @@
+import type { executeJavaScriptVerifier } from "./javascript-verifier.js";
+import { type JavaScriptVerifierResult } from "./javascript-verifier-contract.js";
+/** Resolves/rejects only after the execution owner has stopped the worker. */
+export declare function executeJavaScriptVerifierInWorker(input: Parameters<typeof executeJavaScriptVerifier>[0]): Promise<JavaScriptVerifierResult>;
+//# sourceMappingURL=javascript-verifier-node.d.ts.map

package/dist/types/javascript-verifier-node.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript-verifier-node.d.ts","sourceRoot":"","sources":["../../../../src/javascript-verifier-node.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAC1E,OAAO,EAAkC,KAAK,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAIlH,8EAA8E;AAC9E,wBAAsB,iCAAiC,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqCjJ"}

package/dist/types/javascript-verifier-worker-source.d.ts

@@ -0,0 +1,2 @@
+/** Generated by the Evals build: trusted worker host plus the embedded interpreter. */
+export declare const javascriptVerifierWorkerSource: string;

package/dist/types/javascript-verifier-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=javascript-verifier-worker.d.ts.map

package/dist/types/javascript-verifier-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript-verifier-worker.d.ts","sourceRoot":"","sources":["../../../../src/javascript-verifier-worker.ts"],"names":[],"mappings":""}

package/dist/types/javascript-verifier.d.ts

@@ -0,0 +1,15 @@
+import { type JavaScriptVerifierResult } from "./javascript-verifier-contract.js";
+export { JavaScriptVerifierResultSchema, type JavaScriptVerifierResult } from "./javascript-verifier-contract.js";
+/**
+ * A fresh WebAssembly interpreter with no host functions, filesystem, network,
+ * timers, or module loader. Hosts should run this on a worker to remain responsive
+ * and terminate that worker when cancellation is requested.
+ */
+export declare function executeJavaScriptVerifier(input: {
+    source: string;
+    exportName?: string;
+    value: unknown;
+    timeoutMs: number;
+    signal?: AbortSignal;
+}): Promise<JavaScriptVerifierResult>;
+//# sourceMappingURL=javascript-verifier.d.ts.map

package/dist/types/javascript-verifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript-verifier.d.ts","sourceRoot":"","sources":["../../../../src/javascript-verifier.ts"],"names":[],"mappings":"AAGA,OAAO,EAAkC,KAAK,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAClH,OAAO,EAAE,8BAA8B,EAAE,KAAK,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAElH;;;;GAIG;AACH,wBAAsB,yBAAyB,CAAC,KAAK,EAAE;IACrD,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAgCpC"}