@openpond/evals 0.8.1 → 0.9.0

package/README.md

@@ -44,6 +44,42 @@ Subpath exports are available at `/harness`, `/tasksets`, `/benchmarks`, `/grade
 not a hosted client. It does not execute OpenPond Desktop or Sandbox sessions,
 resolve credentials, or persist artifacts.
 
+## Authored Taskset metrics
+
+`@openpond/evals/metrics` exports the shared `TasksetMetricPolicySchema` and
+`executeTasksetMetric`. Supply the policy from the pinned Taskset Release, its
+Run Manifest, and verified attempt receipts. The result carries the policy,
+policy hash, all input receipt references, population counts and a content hash.
+It is separate from the evaluation's ordinary mean-score accounting.
+
+The release's optional `metrics` field is the authoritative policy. For complete
+evaluations, use `aggregateTasksetEvaluationReceipts` or the Node
+`aggregateTasksetEvaluationInWorker` export. They bind the release and task IDs
+to the manifest, calculate `authoredMetric`, and attach it only to the exact
+receipt population. `meanScore` remains the ordinary mean. Hosts should capture
+and validate custom source with `assertTasksetMetricSource` before model work,
+then retain those bytes throughout the run. Complete Taskset packages require
+the declared module as a private, content-matching file.
+
+Mean score, explicit grader pass rate, weighted mean and custom aggregation use
+one population rule: nonterminal attempts, infrastructure failures, timeouts and
+cancellations are excluded. Other missing or ineligible rewards obey the
+policy's `zero` or `exclude` setting. Pass rate requires an explicit boolean
+`passed` verdict. Empty populations return `null`.
+
+Weighted means require positive `taskWeights` keyed by task ID; each attempt
+uses its task's declared weight. Missing weights are errors. Custom modules
+export `aggregate(scores)` (or the declared export) and return a finite number
+between zero and one. Exact UTF-8 source must match the policy's SHA-256 before
+execution. Module source is limited to 512 KiB and custom inputs to 4 MiB.
+
+Node hosts use `executeTasksetMetricInWorker` from `@openpond/evals/metrics/node`.
+It strips type annotations from `.ts`, `.mts` and `.cts` modules, then executes
+them in a fresh deterministic interpreter without host APIs or imports. Workers
+stop before success, failure or cancellation is returned. Other hosts pass an
+isolated executor to the portable function; authored source must never run in
+the host JavaScript context.
+
 ## Benchmarks
 
 `BenchmarkDefinition` binds a named benchmark to an immutable Taskset Release,

package/dist/index.js

@@ -8,6 +8,7 @@ export * from "./graders.js";
 export * from "./harness.js";
 export * from "./learned-preference.js";
 export * from "./runs.js";
+export * from "./metrics.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./review-conformance.js";

package/dist/metric-policy.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { assertContentHash, contentHash, ImmutableReleaseRefSchema, ReleaseHashSchema } from "@openpond/harness";
+const MetricIdSchema = z.string().trim().min(1).max(240);
+const ModulePathSchema = z.string().trim().min(1).max(1_000).refine((value) => !/[\\:\0]/.test(value) && value.split("/").every((part) => part && part !== "." && part !== ".." && !part.startsWith(".env")), "Metric modules must use a safe relative path.");
+export const TasksetMetricPolicySchema = z.object({
+    schemaVersion: z.literal("openpond.tasksetMetricPolicy.v1"),
+    primaryMetric: MetricIdSchema,
+    aggregation: z.enum(["mean_score", "pass_rate", "weighted_mean", "custom"]),
+    missingReward: z.enum(["zero", "exclude"]),
+    taskWeights: z.record(MetricIdSchema, z.number().positive().max(1_000_000)).optional(),
+    customAggregator: z.object({
+        module: ModulePathSchema,
+        exportName: z.string().trim().regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/),
+        contentHash: ReleaseHashSchema,
+        timeoutMs: z.number().int().positive().max(300_000),
+        networkPolicy: z.literal("none"),
+    }).strict().nullable(),
+}).strict().superRefine((policy, context) => {
+    if ((policy.aggregation === "custom") !== (policy.customAggregator !== null)) {
+        context.addIssue({ code: "custom", path: ["customAggregator"], message: "Only custom aggregation requires a content-hashed module." });
+    }
+    if (policy.aggregation === "weighted_mean" ? !policy.taskWeights || !Object.keys(policy.taskWeights).length : policy.taskWeights !== undefined) {
+        context.addIssue({ code: "custom", path: ["taskWeights"], message: "Only weighted means require explicit positive task weights." });
+    }
+});
+export const TasksetMetricResultContentSchema = z.object({
+    schemaVersion: z.literal("openpond.tasksetMetricResult.v1"),
+    runManifest: ImmutableReleaseRefSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    policy: TasksetMetricPolicySchema,
+    policyHash: ReleaseHashSchema,
+    receiptRefs: z.array(ImmutableReleaseRefSchema).min(1).max(1_000_000),
+    includedCount: z.number().int().nonnegative(),
+    missingRewardCount: z.number().int().nonnegative(),
+    excludedCount: z.number().int().nonnegative(),
+    value: z.number().min(0).max(1).nullable(),
+}).strict();
+export const TasksetMetricResultSchema = TasksetMetricResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
+export function assertTasksetMetricResult(input) {
+    const result = TasksetMetricResultSchema.parse(input);
+    assertContentHash(result, "Taskset metric result");
+    if (result.policyHash !== contentHash(result.policy))
+        throw new Error("Taskset metric policy hash differs from its policy.");
+    if (result.includedCount + result.excludedCount !== result.receiptRefs.length
+        || (result.includedCount === 0) !== (result.value === null)
+        || result.missingRewardCount > (result.policy.missingReward === "zero" ? result.includedCount : result.excludedCount)
+        || new Set(result.receiptRefs.map(ref => ref.id)).size !== result.receiptRefs.length) {
+        throw new Error("Taskset metric population differs from its receipt references.");
+    }
+}

package/dist/metrics-node.js

@@ -0,0 +1,16 @@
+import { stripTypeScriptTypes } from "node:module";
+import { executeJavaScriptIsolateInWorker } from "./javascript-isolate-node.js";
+import { aggregateTasksetEvaluationReceipts, executeTasksetMetric } from "./metrics.js";
+/** Verifies original source before stripping TS, then waits for worker termination
+ * on success, error or cancellation. Authored imports and host APIs are unavailable. */
+export function executeTasksetMetricInWorker(input) {
+    return executeTasksetMetric(input, executeMetric);
+}
+export function aggregateTasksetEvaluationInWorker(input) {
+    return aggregateTasksetEvaluationReceipts(input, executeMetric);
+}
+const executeMetric = ({ source, module, exportName, scores, timeoutMs, signal }) => executeJavaScriptIsolateInWorker({
+    source: /\.[cm]?ts$/.test(module) ? stripTypeScriptTypes(source, { mode: "strip" }) : source,
+    exportName, value: scores, timeoutMs, signal,
+    maxResultBytes: 1_024, deterministic: true, errorPrefix: "metric",
+});

package/dist/metrics.js

@@ -0,0 +1,105 @@
+import { assertContentHash, contentHash, sha256 } from "@openpond/harness";
+import { aggregateEvaluationReceipts, RunManifestSchema, verifyAttemptReceipt } from "./runs.js";
+import { TasksetMetricPolicySchema, TasksetMetricResultContentSchema, TasksetMetricResultSchema } from "./metric-policy.js";
+import { assertTasksetRelease } from "./tasksets.js";
+export * from "./metric-policy.js";
+/** Preflight the exact metric source before any model or tool execution. */
+export function assertTasksetMetricSource(taskset, source) {
+    assertTasksetRelease(taskset);
+    if (taskset.metrics?.customAggregator)
+        assertMetricSource(taskset.metrics.customAggregator, source);
+}
+/** Bind the declared metric to one frozen release and population, retaining
+ * ordinary mean-score accounting independently of the authored calculation. */
+export async function aggregateTasksetEvaluationReceipts(input, executor) {
+    assertTasksetMetricSource(input.taskset, input.source);
+    if (input.taskset.id !== input.manifest.tasksetRelease.id || input.taskset.contentHash !== input.manifest.tasksetRelease.contentHash)
+        throw new Error("Metric Taskset differs from the Run Manifest's pinned release.");
+    const taskIds = new Set(input.taskset.tasks.map(task => task.id));
+    if (input.receipts.some(receipt => !taskIds.has(receipt.taskId)))
+        throw new Error("Metric receipt names a task outside the pinned Taskset.");
+    const authoredMetric = input.taskset.metrics ? await executeTasksetMetric({ ...input, policy: input.taskset.metrics }, executor) : undefined;
+    return aggregateEvaluationReceipts({ id: input.id, manifest: input.manifest, receipts: input.receipts, metadata: input.metadata, authoredMetric });
+}
+/** Hosts supply the policy from the pinned Taskset, never a mutable form.
+ * Operational failures and nonterminal receipts are excluded for every policy.
+ * Missing rewards on the remaining receipts obey zero/exclude. A task's weight
+ * applies to each of its attempts; pass rate uses the grader's explicit verdict.
+ */
+export async function executeTasksetMetric(input, executor) {
+    input.signal?.throwIfAborted();
+    const policy = TasksetMetricPolicySchema.parse(input.policy);
+    const manifest = RunManifestSchema.parse(input.manifest);
+    assertContentHash(manifest, "Metric Run Manifest");
+    if (!input.receipts.length || input.receipts.length > 1_000_000)
+        throw new Error("Metric execution requires 1 to 1,000,000 attempt receipts.");
+    const ids = new Set();
+    const scores = [];
+    const weights = [];
+    let missingRewardCount = 0;
+    for (const receipt of input.receipts) {
+        input.signal?.throwIfAborted();
+        if (!verifyAttemptReceipt(receipt))
+            throw new Error(`Metric receipt ${receipt.id} has an invalid content hash.`);
+        if (receipt.runManifest.id !== manifest.id || receipt.runManifest.contentHash !== manifest.contentHash)
+            throw new Error(`Metric receipt ${receipt.id} belongs to a different Run Manifest.`);
+        if (ids.has(receipt.id))
+            throw new Error(`Metric receipt ${receipt.id} is duplicated.`);
+        ids.add(receipt.id);
+        if (!receipt.terminal || ["infrastructure_failure", "timeout", "cancelled"].includes(receipt.failureClass ?? ""))
+            continue;
+        const score = receipt.metadata.score;
+        if (typeof score === "number" && (!Number.isFinite(score) || score < 0 || score > 1))
+            throw new Error(`Metric receipt ${receipt.id} has an invalid score.`);
+        const hasReward = receipt.metadata.rewardEligible === true && typeof score === "number"
+            && (policy.aggregation !== "pass_rate" || typeof receipt.metadata.passed === "boolean");
+        if (!hasReward) {
+            missingRewardCount += 1;
+            if (policy.missingReward === "exclude")
+                continue;
+        }
+        const weight = policy.aggregation === "weighted_mean" && Object.hasOwn(policy.taskWeights, receipt.taskId) ? policy.taskWeights[receipt.taskId] : undefined;
+        if (policy.aggregation === "weighted_mean" && weight === undefined)
+            throw new Error(`Metric policy has no weight for task ${receipt.taskId}.`);
+        scores.push(hasReward ? policy.aggregation === "pass_rate" ? Number(receipt.metadata.passed) : score : 0);
+        weights.push(weight ?? 1);
+    }
+    let value = null;
+    const receiptRefs = input.receipts.map(({ id, contentHash }) => ({ id, contentHash }));
+    const includedCount = scores.length;
+    if (policy.customAggregator) {
+        assertMetricSource(policy.customAggregator, input.source);
+        if (!executor)
+            throw new Error("Custom metrics require an isolated executor.");
+        // Empty populations remain unscorable, even if authored code would return a value.
+        if (scores.length) {
+            const result = await executor({ source: input.source, module: policy.customAggregator.module, exportName: policy.customAggregator.exportName, scores, timeoutMs: policy.customAggregator.timeoutMs, signal: input.signal });
+            if (typeof result !== "number" || !Number.isFinite(result) || result < 0 || result > 1)
+                throw new Error("Custom metric must return a finite number between 0 and 1.");
+            value = result;
+        }
+    }
+    else if (scores.length) {
+        const totalWeight = weights.reduce((sum, weight) => sum + weight, 0);
+        value = scores.reduce((sum, score, index) => sum + score * (weights[index] / totalWeight), 0);
+        value = Math.max(0, Math.min(1, value));
+    }
+    input.signal?.throwIfAborted();
+    const content = TasksetMetricResultContentSchema.parse({
+        schemaVersion: "openpond.tasksetMetricResult.v1",
+        runManifest: { id: manifest.id, contentHash: manifest.contentHash },
+        tasksetRelease: manifest.tasksetRelease,
+        policy,
+        policyHash: contentHash(policy),
+        receiptRefs,
+        includedCount,
+        missingRewardCount,
+        excludedCount: receiptRefs.length - includedCount,
+        value,
+    });
+    return TasksetMetricResultSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+function assertMetricSource(aggregator, source) {
+    if (source === undefined || new TextEncoder().encode(source).byteLength > 524_288 || sha256(source) !== aggregator.contentHash)
+        throw new Error("Metric module bytes do not match the pinned content hash or exceed the source limit.");
+}

package/dist/runs.js

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { assertTasksetMetricResult, TasksetMetricResultSchema } from "./metric-policy.js";
 export const RuntimeTargetBindingSchema = z.object({
     adapterId: ReleaseIdSchema,
     placement: z.enum(["local", "remote", "colocated", "provider_native"]),
@@ -58,6 +59,7 @@ export const EvaluationResultContentSchema = z.object({
     terminalCount: z.number().int().nonnegative(),
     meanScore: z.number().min(0).max(1).nullable(),
     failureCounts: z.record(FailureClassSchema, z.number().int().nonnegative()),
+    authoredMetric: TasksetMetricResultSchema.optional(),
     metadata: MetadataSchema,
 }).strict();
 export const EvaluationResultSchema = EvaluationResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
@@ -101,6 +103,15 @@ export function aggregateEvaluationReceipts(input) {
         if (!verifyAttemptReceipt(receipt))
             throw new Error(`Attempt receipt ${receipt.id} has an invalid content hash.`);
     }
+    if (input.authoredMetric) {
+        assertTasksetMetricResult(input.authoredMetric);
+        const metric = input.authoredMetric;
+        if (metric.runManifest.id !== input.manifest.id || metric.runManifest.contentHash !== input.manifest.contentHash
+            || contentHash(metric.tasksetRelease) !== contentHash(input.manifest.tasksetRelease)
+            || contentHash(metric.receiptRefs) !== contentHash(input.receipts.map(({ id, contentHash }) => ({ id, contentHash })))) {
+            throw new Error("Authored metric does not describe this evaluation's exact manifest and receipts.");
+        }
+    }
     const scores = input.receipts.flatMap((receipt) => typeof receipt.metadata.score === "number" && Number.isFinite(receipt.metadata.score)
         ? [receipt.metadata.score]
         : []);
@@ -118,6 +129,7 @@ export function aggregateEvaluationReceipts(input) {
         terminalCount: input.receipts.filter((receipt) => receipt.terminal).length,
         meanScore: scores.length ? scores.reduce((total, score) => total + score, 0) / scores.length : null,
         failureCounts,
+        ...(input.authoredMetric ? { authoredMetric: input.authoredMetric } : {}),
         metadata: input.metadata ?? {},
     });
     return EvaluationResultSchema.parse({ ...content, contentHash: contentHash(content) });

package/dist/tasksets.js

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { CapabilityRequirementSchema, ImmutableAssetRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ToolDeclarationSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { TasksetMetricPolicySchema } from "./metric-policy.js";
 export const TaskSplitSchema = z.enum(["train", "validation", "test", "frozen_eval"]);
 export const RequiredOutputContractSchema = z.object({
     path: z.string().trim().min(1).max(2_000).refine(safeRelativePath),
@@ -85,6 +86,7 @@ export const TasksetReleaseContentSchema = z.object({
     tasks: z.array(TaskRecordSchema).min(1).max(1_000_000),
     graders: z.array(GraderSpecSchema).min(1).max(1_000),
     verifierSetRelease: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().optional(),
+    metrics: TasksetMetricPolicySchema.optional(),
     metadata: MetadataSchema,
 }).strict();
 export const TasksetReleaseSchema = TasksetReleaseContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
@@ -115,6 +117,9 @@ export function validateTasksetRelease(input) {
     }
     const clusterSplits = new Map();
     for (const task of taskset.tasks) {
+        if (taskset.metrics?.aggregation === "weighted_mean" && !Object.hasOwn(taskset.metrics.taskWeights, task.id)) {
+            issues.push({ code: "metric_task_weight_missing", severity: "error", message: `Metric policy has no weight for task ${task.id}.`, path: "metrics.taskWeights" });
+        }
         const splits = clusterSplits.get(task.clusterKey) ?? new Set();
         splits.add(task.split);
         clusterSplits.set(task.clusterKey, splits);

package/dist/types/builtin-benchmarks/harness-refiner.d.ts

@@ -143,6 +143,20 @@ export declare const harnessRefinerBenchmarkRelease: {
         id: string;
         contentHash: string;
     } | undefined;
+    metrics?: {
+        schemaVersion: "openpond.tasksetMetricPolicy.v1";
+        primaryMetric: string;
+        aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
+        missingReward: "zero" | "exclude";
+        customAggregator: {
+            module: string;
+            exportName: string;
+            contentHash: string;
+            timeoutMs: number;
+            networkPolicy: "none";
+        } | null;
+        taskWeights?: Record<string, number> | undefined;
+    } | undefined;
 };
 export declare const harnessRefinerBenchmarkAssets: Readonly<Record<string, string>>;
 //# sourceMappingURL=harness-refiner.d.ts.map

package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
+{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}

package/dist/types/conformance.d.ts

@@ -253,6 +253,20 @@ export declare const genericToolConformance: {
             id: string;
             contentHash: string;
         } | undefined;
+        metrics?: {
+            schemaVersion: "openpond.tasksetMetricPolicy.v1";
+            primaryMetric: string;
+            aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
+            missingReward: "zero" | "exclude";
+            customAggregator: {
+                module: string;
+                exportName: string;
+                contentHash: string;
+                timeoutMs: number;
+                networkPolicy: "none";
+            } | null;
+            taskWeights?: Record<string, number> | undefined;
+        } | undefined;
     };
     manifest: {
         schemaVersion: "openpond.runManifest.v1";
@@ -549,6 +563,20 @@ export declare const marketingPortfolioConformance: {
             id: string;
             contentHash: string;
         } | undefined;
+        metrics?: {
+            schemaVersion: "openpond.tasksetMetricPolicy.v1";
+            primaryMetric: string;
+            aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
+            missingReward: "zero" | "exclude";
+            customAggregator: {
+                module: string;
+                exportName: string;
+                contentHash: string;
+                timeoutMs: number;
+                networkPolicy: "none";
+            } | null;
+            taskWeights?: Record<string, number> | undefined;
+        } | undefined;
     };
     manifest: {
         schemaVersion: "openpond.runManifest.v1";

package/dist/types/conformance.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}
+{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}

package/dist/types/evidence/conformance.d.ts

@@ -32,7 +32,7 @@ export declare const completeWorkProcessTraceFixture: {
             durationMs: number | null;
             cpuTimeMs: number | null;
             memoryPeakBytes: number | null;
-            errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+            errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
         };
     }[];
     contentHash: string;
@@ -209,7 +209,7 @@ export declare const incompleteWorkProcessTraceFixture: {
             durationMs: number | null;
             cpuTimeMs: number | null;
             memoryPeakBytes: number | null;
-            errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+            errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
         };
     }[];
     contentHash: string;
@@ -415,7 +415,7 @@ export declare const workEvidenceConformance: {
                 durationMs: number | null;
                 cpuTimeMs: number | null;
                 memoryPeakBytes: number | null;
-                errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+                errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
             };
         }[];
         contentHash: string;
@@ -648,7 +648,7 @@ export declare const workEvidenceConformance: {
                 durationMs: number | null;
                 cpuTimeMs: number | null;
                 memoryPeakBytes: number | null;
-                errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+                errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
             };
         }[];
         contentHash: string;

package/dist/types/evidence/contracts.d.ts

@@ -87,9 +87,9 @@ export declare const WorkInterventionOutcomeSchema: z.ZodEnum<{
     dismissed: "dismissed";
 }>;
 export declare const WorkProcessErrorClassSchema: z.ZodEnum<{
-    validation: "validation";
-    policy: "policy";
     unknown: "unknown";
+    policy: "policy";
+    validation: "validation";
     environment: "environment";
     cancelled: "cancelled";
     timeout: "timeout";
@@ -197,9 +197,9 @@ export declare const WorkProcessStepSchema: z.ZodObject<{
         cpuTimeMs: z.ZodNullable<z.ZodNumber>;
         memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
         errorClass: z.ZodNullable<z.ZodEnum<{
-            validation: "validation";
-            policy: "policy";
             unknown: "unknown";
+            policy: "policy";
+            validation: "validation";
             environment: "environment";
             cancelled: "cancelled";
             timeout: "timeout";
@@ -315,9 +315,9 @@ export declare const WorkProcessTraceContentSchema: z.ZodObject<{
             cpuTimeMs: z.ZodNullable<z.ZodNumber>;
             memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
             errorClass: z.ZodNullable<z.ZodEnum<{
-                validation: "validation";
-                policy: "policy";
                 unknown: "unknown";
+                policy: "policy";
+                validation: "validation";
                 environment: "environment";
                 cancelled: "cancelled";
                 timeout: "timeout";
@@ -434,9 +434,9 @@ export declare const WorkProcessTraceSchema: z.ZodObject<{
             cpuTimeMs: z.ZodNullable<z.ZodNumber>;
             memoryPeakBytes: z.ZodNullable<z.ZodNumber>;
             errorClass: z.ZodNullable<z.ZodEnum<{
-                validation: "validation";
-                policy: "policy";
                 unknown: "unknown";
+                policy: "policy";
+                validation: "validation";
                 environment: "environment";
                 cancelled: "cancelled";
                 timeout: "timeout";

package/dist/types/index.d.ts

@@ -8,6 +8,7 @@ export * from "./graders.js";
 export * from "./harness.js";
 export * from "./learned-preference.js";
 export * from "./runs.js";
+export * from "./metrics.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./review-conformance.js";

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}

package/dist/types/javascript-isolate.d.ts

@@ -6,7 +6,7 @@ export interface JavaScriptIsolateInput {
     signal?: AbortSignal;
     maxResultBytes: number;
     deterministic?: boolean;
-    errorPrefix: "verifier" | "environment" | "candidate";
+    errorPrefix: "verifier" | "environment" | "candidate" | "metric";
 }
 export declare class JavaScriptIsolateExecutionError extends Error {
     constructor(message: string);

package/dist/types/javascript-isolate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"javascript-isolate.d.ts","sourceRoot":"","sources":["../../../../src/javascript-isolate.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,UAAU,GAAG,aAAa,GAAG,WAAW,CAAC;CACvD;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC5C,OAAO,EAAE,MAAM;CAC5B;AAED,kFAAkF;AAClF,wBAAsB,wBAAwB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAqC9F;AAED,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAKlF"}
+{"version":3,"file":"javascript-isolate.d.ts","sourceRoot":"","sources":["../../../../src/javascript-isolate.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,UAAU,GAAG,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;CAClE;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC5C,OAAO,EAAE,MAAM;CAC5B;AAED,kFAAkF;AAClF,wBAAsB,wBAAwB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAqC9F;AAED,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAKlF"}

package/dist/types/learning/admission.d.ts

@@ -240,8 +240,10 @@ export declare const TaskBatchPackageMetadataSchema: z.ZodObject<{
             }, z.core.$strict>>;
             temperature: z.ZodOptional<z.ZodNumber>;
         }, z.core.$strict>, z.ZodObject<{
-            kind: z.ZodLiteral<"custom_verifier">;
+            exportName: z.ZodOptional<z.ZodString>;
+            timeoutMs: z.ZodNumber;
             networkPolicy: z.ZodLiteral<"none">;
+            kind: z.ZodLiteral<"custom_verifier">;
             verifierRef: z.ZodObject<{
                 id: z.ZodString;
                 path: z.ZodString;
@@ -254,8 +256,6 @@ export declare const TaskBatchPackageMetadataSchema: z.ZodObject<{
                     host_private: "host_private";
                 }>;
             }, z.core.$strict>;
-            exportName: z.ZodOptional<z.ZodString>;
-            timeoutMs: z.ZodNumber;
         }, z.core.$strict>, z.ZodObject<{
             kind: z.ZodLiteral<"human">;
             rubricRef: z.ZodObject<{

package/dist/types/learning/contracts.d.ts

@@ -1471,8 +1471,10 @@ export declare const LearningResourceSchema: z.ZodUnion<readonly [z.ZodObject<{
         }, z.core.$strict>>;
         temperature: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strict>, z.ZodObject<{
-        kind: z.ZodLiteral<"custom_verifier">;
+        exportName: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodNumber;
         networkPolicy: z.ZodLiteral<"none">;
+        kind: z.ZodLiteral<"custom_verifier">;
         verifierRef: z.ZodObject<{
             id: z.ZodString;
             path: z.ZodString;
@@ -1485,8 +1487,6 @@ export declare const LearningResourceSchema: z.ZodUnion<readonly [z.ZodObject<{
                 host_private: "host_private";
             }>;
         }, z.core.$strict>;
-        exportName: z.ZodOptional<z.ZodString>;
-        timeoutMs: z.ZodNumber;
     }, z.core.$strict>, z.ZodObject<{
         kind: z.ZodLiteral<"human">;
         rubricRef: z.ZodObject<{

package/dist/types/learning/operations.d.ts

@@ -214,8 +214,10 @@ export declare const PublishLearningResourceCommandSchema: z.ZodDiscriminatedUni
             }, z.core.$strict>>;
             temperature: z.ZodOptional<z.ZodNumber>;
         }, z.core.$strict>, z.ZodObject<{
-            kind: z.ZodLiteral<"custom_verifier">;
+            exportName: z.ZodOptional<z.ZodString>;
+            timeoutMs: z.ZodNumber;
             networkPolicy: z.ZodLiteral<"none">;
+            kind: z.ZodLiteral<"custom_verifier">;
             verifierRef: z.ZodObject<{
                 id: z.ZodString;
                 path: z.ZodString;
@@ -228,8 +230,6 @@ export declare const PublishLearningResourceCommandSchema: z.ZodDiscriminatedUni
                     host_private: "host_private";
                 }>;
             }, z.core.$strict>;
-            exportName: z.ZodOptional<z.ZodString>;
-            timeoutMs: z.ZodNumber;
         }, z.core.$strict>, z.ZodObject<{
             kind: z.ZodLiteral<"human">;
             rubricRef: z.ZodObject<{
@@ -747,8 +747,10 @@ export declare const PublishLearningResourcesCommandSchema: z.ZodObject<{
                 }, z.core.$strict>>;
                 temperature: z.ZodOptional<z.ZodNumber>;
             }, z.core.$strict>, z.ZodObject<{
-                kind: z.ZodLiteral<"custom_verifier">;
+                exportName: z.ZodOptional<z.ZodString>;
+                timeoutMs: z.ZodNumber;
                 networkPolicy: z.ZodLiteral<"none">;
+                kind: z.ZodLiteral<"custom_verifier">;
                 verifierRef: z.ZodObject<{
                     id: z.ZodString;
                     path: z.ZodString;
@@ -761,8 +763,6 @@ export declare const PublishLearningResourcesCommandSchema: z.ZodObject<{
                         host_private: "host_private";
                     }>;
                 }, z.core.$strict>;
-                exportName: z.ZodOptional<z.ZodString>;
-                timeoutMs: z.ZodNumber;
             }, z.core.$strict>, z.ZodObject<{
                 kind: z.ZodLiteral<"human">;
                 rubricRef: z.ZodObject<{
@@ -1826,8 +1826,10 @@ export declare const LearningCommandSchema: z.ZodUnion<readonly [z.ZodObject<{
             }, z.core.$strict>>;
             temperature: z.ZodOptional<z.ZodNumber>;
         }, z.core.$strict>, z.ZodObject<{
-            kind: z.ZodLiteral<"custom_verifier">;
+            exportName: z.ZodOptional<z.ZodString>;
+            timeoutMs: z.ZodNumber;
             networkPolicy: z.ZodLiteral<"none">;
+            kind: z.ZodLiteral<"custom_verifier">;
             verifierRef: z.ZodObject<{
                 id: z.ZodString;
                 path: z.ZodString;
@@ -1840,8 +1842,6 @@ export declare const LearningCommandSchema: z.ZodUnion<readonly [z.ZodObject<{
                     host_private: "host_private";
                 }>;
             }, z.core.$strict>;
-            exportName: z.ZodOptional<z.ZodString>;
-            timeoutMs: z.ZodNumber;
         }, z.core.$strict>, z.ZodObject<{
             kind: z.ZodLiteral<"human">;
             rubricRef: z.ZodObject<{
@@ -2358,8 +2358,10 @@ export declare const LearningCommandSchema: z.ZodUnion<readonly [z.ZodObject<{
                 }, z.core.$strict>>;
                 temperature: z.ZodOptional<z.ZodNumber>;
             }, z.core.$strict>, z.ZodObject<{
-                kind: z.ZodLiteral<"custom_verifier">;
+                exportName: z.ZodOptional<z.ZodString>;
+                timeoutMs: z.ZodNumber;
                 networkPolicy: z.ZodLiteral<"none">;
+                kind: z.ZodLiteral<"custom_verifier">;
                 verifierRef: z.ZodObject<{
                     id: z.ZodString;
                     path: z.ZodString;
@@ -2372,8 +2374,6 @@ export declare const LearningCommandSchema: z.ZodUnion<readonly [z.ZodObject<{
                         host_private: "host_private";
                     }>;
                 }, z.core.$strict>;
-                exportName: z.ZodOptional<z.ZodString>;
-                timeoutMs: z.ZodNumber;
             }, z.core.$strict>, z.ZodObject<{
                 kind: z.ZodLiteral<"human">;
                 rubricRef: z.ZodObject<{