@openpond/evals 0.8.1 → 0.9.1

package/README.md

@@ -44,6 +44,53 @@ Subpath exports are available at `/harness`, `/tasksets`, `/benchmarks`, `/grade
 not a hosted client. It does not execute OpenPond Desktop or Sandbox sessions,
 resolve credentials, or persist artifacts.
 
+## Ordinary graders
+
+`evaluateDeterministicGrader` and `portableDeterministicCheck` from `/graders`
+provide the shared local/portable interpretation of ordinary checks. They retain
+explicit exact-text normalization, final-answer extraction, structured JSON,
+artifact and runtime-event requirements, and structural state comparisons.
+Native test/diff checks declare their expected output field when exported.
+Empty, unsupported or unavailable checks remain unscorable rather than earning
+a zero or a passing reward. Desktop grade components preserve null scores and
+display them as not scored.
+
+## Authored Taskset metrics
+
+`@openpond/evals/metrics` exports the shared `TasksetMetricPolicySchema` and
+`executeTasksetMetric`. Supply the policy from the pinned Taskset Release, its
+Run Manifest, and verified attempt receipts. The result carries the policy,
+policy hash, all input receipt references, population counts and a content hash.
+It is separate from the evaluation's ordinary mean-score accounting.
+
+The release's optional `metrics` field is the authoritative policy. For complete
+evaluations, use `aggregateTasksetEvaluationReceipts` or the Node
+`aggregateTasksetEvaluationInWorker` export. They bind the release and task IDs
+to the manifest, calculate `authoredMetric`, and attach it only to the exact
+receipt population. `meanScore` remains the ordinary mean. Hosts should capture
+and validate custom source with `assertTasksetMetricSource` before model work,
+then retain those bytes throughout the run. Complete Taskset packages require
+the declared module as a private, content-matching file.
+
+Mean score, explicit grader pass rate, weighted mean and custom aggregation use
+one population rule: nonterminal attempts, infrastructure failures, timeouts and
+cancellations are excluded. Other missing or ineligible rewards obey the
+policy's `zero` or `exclude` setting. Pass rate requires an explicit boolean
+`passed` verdict. Empty populations return `null`.
+
+Weighted means require positive `taskWeights` keyed by task ID; each attempt
+uses its task's declared weight. Missing weights are errors. Custom modules
+export `aggregate(scores)` (or the declared export) and return a finite number
+between zero and one. Exact UTF-8 source must match the policy's SHA-256 before
+execution. Module source is limited to 512 KiB and custom inputs to 4 MiB.
+
+Node hosts use `executeTasksetMetricInWorker` from `@openpond/evals/metrics/node`.
+It strips type annotations from `.ts`, `.mts` and `.cts` modules, then executes
+them in a fresh deterministic interpreter without host APIs or imports. Workers
+stop before success, failure or cancellation is returned. Other hosts pass an
+isolated executor to the portable function; authored source must never run in
+the host JavaScript context.
+
 ## Benchmarks
 
 `BenchmarkDefinition` binds a named benchmark to an immutable Taskset Release,

package/dist/deterministic-graders.js

@@ -0,0 +1,110 @@
+import { contentHash } from "@openpond/harness";
+import { validateTaskSchema, validateTaskValue } from "./task-schema.js";
+/** Local and portable execution interpret the same declared checks. Missing or
+ * unsupported checks are unscorable; an empty check cannot earn a reward. */
+export function evaluateDeterministicGrader(input) {
+    const { grader, task, evidence } = input;
+    const config = grader.config;
+    if (grader.kind === "content") {
+        const outputField = text(config.outputField) ?? "text";
+        const actual = text(evidence.output[outputField]);
+        if (config.operator === "exact_equals") {
+            const expected = text(config.expectedValue);
+            if (expected === null)
+                return unavailable("Exact content grading requires an expected value.");
+            const normalize = (value) => {
+                const unicode = config.normalizeUnicode === true ? value.normalize("NFC") : value;
+                return config.trimWhitespace === true ? unicode.trim() : unicode;
+            };
+            return outcome(actual !== null && normalize(actual) === normalize(expected));
+        }
+        if (config.operator !== undefined && config.operator !== "final_answer_equals_expected")
+            return unavailable("Unsupported content grader operator.");
+        if (config.operator === undefined && (config.includes !== undefined || config.excludes !== undefined)) {
+            const includes = strings(config.includes), excludes = strings(config.excludes);
+            if (!includes.length && !excludes.length)
+                return unavailable("Content grading requires a nonempty check.");
+            const value = typeof evidence.output.text === "string" ? evidence.output.text : JSON.stringify(evidence.output);
+            return outcome(includes.every(item => value.includes(item)) && excludes.every(item => !value.includes(item)));
+        }
+        const expected = text(config.expectedValue) ?? text(task.expectedOutput?.[text(config.expectedField) ?? "text"]);
+        const normalize = config.operator === "final_answer_equals_expected" ? normalizeFinalAnswer : normalizeContent;
+        if (expected === null || !normalize(expected))
+            return unavailable("Content grading requires a nonempty expected value.");
+        return outcome(actual !== null && normalize(actual) === normalize(expected));
+    }
+    if (grader.kind === "schema") {
+        if (config.operator !== undefined && config.operator !== "json_schema_subset")
+            return unavailable("Unsupported schema grader operator.");
+        const schema = config.operator === "json_schema_subset" ? config.schema : config.jsonSchema;
+        if (schema !== undefined) {
+            const checked = validateTaskSchema(schema);
+            if (!checked.valid)
+                return unavailable(checked.issues[0].message);
+            let value = evidence.output;
+            if (config.operator === "json_schema_subset") {
+                const encoded = evidence.output[text(config.jsonField) ?? "text"];
+                if (typeof encoded === "string") {
+                    try {
+                        value = JSON.parse(encoded.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, ""));
+                    }
+                    catch {
+                        return outcome(false, "The model output was not valid JSON.");
+                    }
+                }
+            }
+            const checkedValue = validateTaskValue(schema, value);
+            return outcome(checkedValue.valid, checkedValue.valid ? undefined : checkedValue.issues.map(issue => issue.message).join(" "));
+        }
+        const required = strings(config.requiredKeys);
+        if (!required.length)
+            return unavailable("Schema grading requires a JSON Schema or explicit required keys.");
+        return outcome(required.every(key => Object.hasOwn(evidence.output, key)));
+    }
+    if (grader.kind === "artifact") {
+        const contains = text(config.refIncludes);
+        if (!contains)
+            return unavailable("Artifact grading requires a declared artifact reference.");
+        return outcome(evidence.artifactRefs.some(ref => ref.includes(contains)));
+    }
+    if (grader.kind === "runtime_event") {
+        const required = strings(config.requiredEvents);
+        if (!required.length)
+            return unavailable("Runtime event grading requires declared events.");
+        return outcome(required.every(event => evidence.runtimeEventRefs.some(ref => ref.includes(event))));
+    }
+    if (config.operator === "output_field_equals") {
+        const field = text(config.outputField);
+        if (!field || !Object.hasOwn(config, "expectedValue"))
+            return unavailable("Output comparison requires a field and expected value.");
+        return outcome(Object.hasOwn(evidence.output, field) && contentHash(evidence.output[field]) === contentHash(config.expectedValue));
+    }
+    if (config.operator !== undefined)
+        return unavailable("Unsupported state grader operator.");
+    const fields = strings(config.fields);
+    const compared = fields.length ? fields : Object.keys(task.expectedOutput ?? {});
+    if (!compared.length || compared.some(field => !task.expectedOutput || !Object.hasOwn(task.expectedOutput, field)))
+        return unavailable("State grading requires expected values for every compared field.");
+    return outcome(compared.every(field => Object.hasOwn(evidence.output, field) && contentHash(evidence.output[field]) === contentHash(task.expectedOutput[field])));
+}
+/** Encode native artifact/test checks without changing their meaning in transit. */
+export function portableDeterministicCheck(grader) {
+    if (grader.kind === "file") {
+        const { pathIncludes, ...config } = grader.config;
+        return { kind: "artifact", config: { ...config, ...(pathIncludes === undefined ? {} : { refIncludes: pathIncludes }) } };
+    }
+    if (grader.kind === "diff" || grader.kind === "test")
+        return { kind: "state", config: { operator: "output_field_equals", outputField: grader.kind === "test" ? "testsPassed" : "diffAccepted", expectedValue: true } };
+    return { kind: grader.kind, config: grader.config };
+}
+function outcome(passed, feedback) { return { score: Number(passed), passed, feedback: feedback ?? (passed ? "Deterministic grader passed." : "Deterministic grader failed.") }; }
+function unavailable(feedback) { return { score: null, passed: false, feedback }; }
+function text(value) { return typeof value === "string" ? value : null; }
+function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.trim().length > 0) : []; }
+function normalizeContent(value) { return value.normalize("NFKC").trim().replace(/[,，]/g, "").replace(/[.\s]+$/g, "").replace(/\s+/g, " "); }
+function normalizeFinalAnswer(value) {
+    const boxed = [...value.matchAll(/\\boxed\{([^{}]+)\}/g)].at(-1)?.[1];
+    const hashAnswer = value.match(/####\s*([^\n\r]+)/)?.[1];
+    const label = value.match(/(?:final\s+answer|answer)\s*(?::|is|=)\s*([^\n\r]+)/i)?.[1];
+    return normalizeContent((boxed ?? hashAnswer ?? label ?? value).normalize("NFKC").trim().replace(/^\$+|\$+$/g, "").replace(/^\\\(|\\\)$/g, ""));
+}

package/dist/graders.js

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { FailureClassSchema, ReleaseHashSchema, ReleaseIdSchema, contentHash } from "@openpond/harness";
-import { validateTaskSchema, validateTaskValue } from "./task-schema.js";
+import { evaluateDeterministicGrader } from "./deterministic-graders.js";
+export { evaluateDeterministicGrader, portableDeterministicCheck } from "./deterministic-graders.js";
 export const GraderEvidenceContentSchema = z.object({
     schemaVersion: z.literal("openpond.graderEvidence.v1"),
     graderId: ReleaseIdSchema,
@@ -43,55 +44,15 @@ export async function gradeEvidence(input) {
     }));
 }
 function gradeDeterministic(grader, task, attempt) {
-    let passed = false;
-    const config = grader.config;
-    if (grader.kind === "content") {
-        const outputField = string(config.outputField) ?? "text";
-        const actual = string(attempt.output[outputField]);
-        const expected = string(config.expectedValue) ?? string(task.expectedOutput?.[string(config.expectedField) ?? "text"]);
-        if (expected === null || !normalize(expected))
-            return unavailable("Content grading requires a nonempty expected value.");
-        passed = actual !== null && expected !== null && normalize(actual) === normalize(expected);
-    }
-    else if (grader.kind === "schema") {
-        if (config.jsonSchema !== undefined) {
-            const schema = validateTaskSchema(config.jsonSchema);
-            if (!schema.valid)
-                return unavailable(schema.issues[0].message);
-            passed = validateTaskValue(config.jsonSchema, attempt.output).valid;
-        }
-        else {
-            const required = strings(config.requiredKeys);
-            if (!required.length)
-                return unavailable("Schema grading requires a JSON Schema or explicit required keys.");
-            passed = required.every((key) => Object.hasOwn(attempt.output, key));
-        }
-    }
-    else if (grader.kind === "artifact") {
-        const contains = string(config.refIncludes) ?? "";
-        if (!contains)
-            return unavailable("Artifact grading requires a declared artifact reference.");
-        passed = attempt.artifactRefs.some((ref) => ref.includes(contains));
-    }
-    else if (grader.kind === "runtime_event") {
-        const requiredEvents = strings(config.requiredEvents);
-        if (!requiredEvents.length)
-            return unavailable("Runtime event grading requires declared events.");
-        passed = requiredEvents.every((required) => attempt.runtimeEventRefs.some((ref) => ref.includes(required)));
-    }
-    else {
-        const fields = strings(config.fields);
-        const compared = fields.length ? fields : Object.keys(task.expectedOutput ?? {});
-        if (!compared.length || compared.some((field) => !task.expectedOutput || !Object.hasOwn(task.expectedOutput, field)))
-            return unavailable("State grading requires expected values for every compared field.");
-        passed = compared.every((field) => Object.hasOwn(attempt.output, field) && contentHash(attempt.output[field]) === contentHash(task.expectedOutput[field]));
-    }
+    const result = evaluateDeterministicGrader({ grader, task, evidence: attempt });
+    if (result.score === null)
+        return unavailable(result.feedback);
     return {
-        score: passed ? 1 : 0,
-        passed,
+        score: result.score,
+        passed: result.passed,
         rewardEligible: grader.rewardEligible,
-        failureClass: passed ? null : "policy_failure",
-        feedback: [passed ? "Deterministic grader passed." : "Deterministic grader failed."],
+        failureClass: result.passed ? null : "policy_failure",
+        feedback: [result.feedback],
         visibleEvidenceRefs: [...attempt.runtimeEventRefs, ...attempt.artifactRefs],
         privilegedEvidenceRefs: grader.privileged ? [task.privilegedContextRef].filter((ref) => ref !== null) : [],
     };
@@ -109,6 +70,3 @@ function evidence(grader, result) {
 function unavailable(message) {
     return { score: null, passed: false, rewardEligible: false, failureClass: "grader_failure", feedback: [message], visibleEvidenceRefs: [], privilegedEvidenceRefs: [] };
 }
-function string(value) { return typeof value === "string" ? value : null; }
-function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.trim().length > 0) : []; }
-function normalize(value) { return value.normalize("NFKC").trim().replace(/[,，]/g, "").replace(/[.\s]+$/g, "").replace(/\s+/g, " "); }

package/dist/index.js

@@ -8,6 +8,7 @@ export * from "./graders.js";
 export * from "./harness.js";
 export * from "./learned-preference.js";
 export * from "./runs.js";
+export * from "./metrics.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./review-conformance.js";

package/dist/metric-policy.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { assertContentHash, contentHash, ImmutableReleaseRefSchema, ReleaseHashSchema } from "@openpond/harness";
+const MetricIdSchema = z.string().trim().min(1).max(240);
+const ModulePathSchema = z.string().trim().min(1).max(1_000).refine((value) => !/[\\:\0]/.test(value) && value.split("/").every((part) => part && part !== "." && part !== ".." && !part.startsWith(".env")), "Metric modules must use a safe relative path.");
+export const TasksetMetricPolicySchema = z.object({
+    schemaVersion: z.literal("openpond.tasksetMetricPolicy.v1"),
+    primaryMetric: MetricIdSchema,
+    aggregation: z.enum(["mean_score", "pass_rate", "weighted_mean", "custom"]),
+    missingReward: z.enum(["zero", "exclude"]),
+    taskWeights: z.record(MetricIdSchema, z.number().positive().max(1_000_000)).optional(),
+    customAggregator: z.object({
+        module: ModulePathSchema,
+        exportName: z.string().trim().regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/),
+        contentHash: ReleaseHashSchema,
+        timeoutMs: z.number().int().positive().max(300_000),
+        networkPolicy: z.literal("none"),
+    }).strict().nullable(),
+}).strict().superRefine((policy, context) => {
+    if ((policy.aggregation === "custom") !== (policy.customAggregator !== null)) {
+        context.addIssue({ code: "custom", path: ["customAggregator"], message: "Only custom aggregation requires a content-hashed module." });
+    }
+    if (policy.aggregation === "weighted_mean" ? !policy.taskWeights || !Object.keys(policy.taskWeights).length : policy.taskWeights !== undefined) {
+        context.addIssue({ code: "custom", path: ["taskWeights"], message: "Only weighted means require explicit positive task weights." });
+    }
+});
+export const TasksetMetricResultContentSchema = z.object({
+    schemaVersion: z.literal("openpond.tasksetMetricResult.v1"),
+    runManifest: ImmutableReleaseRefSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    policy: TasksetMetricPolicySchema,
+    policyHash: ReleaseHashSchema,
+    receiptRefs: z.array(ImmutableReleaseRefSchema).min(1).max(1_000_000),
+    includedCount: z.number().int().nonnegative(),
+    missingRewardCount: z.number().int().nonnegative(),
+    excludedCount: z.number().int().nonnegative(),
+    value: z.number().min(0).max(1).nullable(),
+}).strict();
+export const TasksetMetricResultSchema = TasksetMetricResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
+export function assertTasksetMetricResult(input) {
+    const result = TasksetMetricResultSchema.parse(input);
+    assertContentHash(result, "Taskset metric result");
+    if (result.policyHash !== contentHash(result.policy))
+        throw new Error("Taskset metric policy hash differs from its policy.");
+    if (result.includedCount + result.excludedCount !== result.receiptRefs.length
+        || (result.includedCount === 0) !== (result.value === null)
+        || result.missingRewardCount > (result.policy.missingReward === "zero" ? result.includedCount : result.excludedCount)
+        || new Set(result.receiptRefs.map(ref => ref.id)).size !== result.receiptRefs.length) {
+        throw new Error("Taskset metric population differs from its receipt references.");
+    }
+}

package/dist/metrics-node.js

@@ -0,0 +1,16 @@
+import { stripTypeScriptTypes } from "node:module";
+import { executeJavaScriptIsolateInWorker } from "./javascript-isolate-node.js";
+import { aggregateTasksetEvaluationReceipts, executeTasksetMetric } from "./metrics.js";
+/** Verifies original source before stripping TS, then waits for worker termination
+ * on success, error or cancellation. Authored imports and host APIs are unavailable. */
+export function executeTasksetMetricInWorker(input) {
+    return executeTasksetMetric(input, executeMetric);
+}
+export function aggregateTasksetEvaluationInWorker(input) {
+    return aggregateTasksetEvaluationReceipts(input, executeMetric);
+}
+const executeMetric = ({ source, module, exportName, scores, timeoutMs, signal }) => executeJavaScriptIsolateInWorker({
+    source: /\.[cm]?ts$/.test(module) ? stripTypeScriptTypes(source, { mode: "strip" }) : source,
+    exportName, value: scores, timeoutMs, signal,
+    maxResultBytes: 1_024, deterministic: true, errorPrefix: "metric",
+});

package/dist/metrics.js

@@ -0,0 +1,105 @@
+import { assertContentHash, contentHash, sha256 } from "@openpond/harness";
+import { aggregateEvaluationReceipts, RunManifestSchema, verifyAttemptReceipt } from "./runs.js";
+import { TasksetMetricPolicySchema, TasksetMetricResultContentSchema, TasksetMetricResultSchema } from "./metric-policy.js";
+import { assertTasksetRelease } from "./tasksets.js";
+export * from "./metric-policy.js";
+/** Preflight the exact metric source before any model or tool execution. */
+export function assertTasksetMetricSource(taskset, source) {
+    assertTasksetRelease(taskset);
+    if (taskset.metrics?.customAggregator)
+        assertMetricSource(taskset.metrics.customAggregator, source);
+}
+/** Bind the declared metric to one frozen release and population, retaining
+ * ordinary mean-score accounting independently of the authored calculation. */
+export async function aggregateTasksetEvaluationReceipts(input, executor) {
+    assertTasksetMetricSource(input.taskset, input.source);
+    if (input.taskset.id !== input.manifest.tasksetRelease.id || input.taskset.contentHash !== input.manifest.tasksetRelease.contentHash)
+        throw new Error("Metric Taskset differs from the Run Manifest's pinned release.");
+    const taskIds = new Set(input.taskset.tasks.map(task => task.id));
+    if (input.receipts.some(receipt => !taskIds.has(receipt.taskId)))
+        throw new Error("Metric receipt names a task outside the pinned Taskset.");
+    const authoredMetric = input.taskset.metrics ? await executeTasksetMetric({ ...input, policy: input.taskset.metrics }, executor) : undefined;
+    return aggregateEvaluationReceipts({ id: input.id, manifest: input.manifest, receipts: input.receipts, metadata: input.metadata, authoredMetric });
+}
+/** Hosts supply the policy from the pinned Taskset, never a mutable form.
+ * Operational failures and nonterminal receipts are excluded for every policy.
+ * Missing rewards on the remaining receipts obey zero/exclude. A task's weight
+ * applies to each of its attempts; pass rate uses the grader's explicit verdict.
+ */
+export async function executeTasksetMetric(input, executor) {
+    input.signal?.throwIfAborted();
+    const policy = TasksetMetricPolicySchema.parse(input.policy);
+    const manifest = RunManifestSchema.parse(input.manifest);
+    assertContentHash(manifest, "Metric Run Manifest");
+    if (!input.receipts.length || input.receipts.length > 1_000_000)
+        throw new Error("Metric execution requires 1 to 1,000,000 attempt receipts.");
+    const ids = new Set();
+    const scores = [];
+    const weights = [];
+    let missingRewardCount = 0;
+    for (const receipt of input.receipts) {
+        input.signal?.throwIfAborted();
+        if (!verifyAttemptReceipt(receipt))
+            throw new Error(`Metric receipt ${receipt.id} has an invalid content hash.`);
+        if (receipt.runManifest.id !== manifest.id || receipt.runManifest.contentHash !== manifest.contentHash)
+            throw new Error(`Metric receipt ${receipt.id} belongs to a different Run Manifest.`);
+        if (ids.has(receipt.id))
+            throw new Error(`Metric receipt ${receipt.id} is duplicated.`);
+        ids.add(receipt.id);
+        if (!receipt.terminal || ["infrastructure_failure", "timeout", "cancelled"].includes(receipt.failureClass ?? ""))
+            continue;
+        const score = receipt.metadata.score;
+        if (typeof score === "number" && (!Number.isFinite(score) || score < 0 || score > 1))
+            throw new Error(`Metric receipt ${receipt.id} has an invalid score.`);
+        const hasReward = receipt.metadata.rewardEligible === true && typeof score === "number"
+            && (policy.aggregation !== "pass_rate" || typeof receipt.metadata.passed === "boolean");
+        if (!hasReward) {
+            missingRewardCount += 1;
+            if (policy.missingReward === "exclude")
+                continue;
+        }
+        const weight = policy.aggregation === "weighted_mean" && Object.hasOwn(policy.taskWeights, receipt.taskId) ? policy.taskWeights[receipt.taskId] : undefined;
+        if (policy.aggregation === "weighted_mean" && weight === undefined)
+            throw new Error(`Metric policy has no weight for task ${receipt.taskId}.`);
+        scores.push(hasReward ? policy.aggregation === "pass_rate" ? Number(receipt.metadata.passed) : score : 0);
+        weights.push(weight ?? 1);
+    }
+    let value = null;
+    const receiptRefs = input.receipts.map(({ id, contentHash }) => ({ id, contentHash }));
+    const includedCount = scores.length;
+    if (policy.customAggregator) {
+        assertMetricSource(policy.customAggregator, input.source);
+        if (!executor)
+            throw new Error("Custom metrics require an isolated executor.");
+        // Empty populations remain unscorable, even if authored code would return a value.
+        if (scores.length) {
+            const result = await executor({ source: input.source, module: policy.customAggregator.module, exportName: policy.customAggregator.exportName, scores, timeoutMs: policy.customAggregator.timeoutMs, signal: input.signal });
+            if (typeof result !== "number" || !Number.isFinite(result) || result < 0 || result > 1)
+                throw new Error("Custom metric must return a finite number between 0 and 1.");
+            value = result;
+        }
+    }
+    else if (scores.length) {
+        const totalWeight = weights.reduce((sum, weight) => sum + weight, 0);
+        value = scores.reduce((sum, score, index) => sum + score * (weights[index] / totalWeight), 0);
+        value = Math.max(0, Math.min(1, value));
+    }
+    input.signal?.throwIfAborted();
+    const content = TasksetMetricResultContentSchema.parse({
+        schemaVersion: "openpond.tasksetMetricResult.v1",
+        runManifest: { id: manifest.id, contentHash: manifest.contentHash },
+        tasksetRelease: manifest.tasksetRelease,
+        policy,
+        policyHash: contentHash(policy),
+        receiptRefs,
+        includedCount,
+        missingRewardCount,
+        excludedCount: receiptRefs.length - includedCount,
+        value,
+    });
+    return TasksetMetricResultSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+function assertMetricSource(aggregator, source) {
+    if (source === undefined || new TextEncoder().encode(source).byteLength > 524_288 || sha256(source) !== aggregator.contentHash)
+        throw new Error("Metric module bytes do not match the pinned content hash or exceed the source limit.");
+}

package/dist/runs.js

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { FailureClassSchema, ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { assertTasksetMetricResult, TasksetMetricResultSchema } from "./metric-policy.js";
 export const RuntimeTargetBindingSchema = z.object({
     adapterId: ReleaseIdSchema,
     placement: z.enum(["local", "remote", "colocated", "provider_native"]),
@@ -58,6 +59,7 @@ export const EvaluationResultContentSchema = z.object({
     terminalCount: z.number().int().nonnegative(),
     meanScore: z.number().min(0).max(1).nullable(),
     failureCounts: z.record(FailureClassSchema, z.number().int().nonnegative()),
+    authoredMetric: TasksetMetricResultSchema.optional(),
     metadata: MetadataSchema,
 }).strict();
 export const EvaluationResultSchema = EvaluationResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
@@ -101,6 +103,15 @@ export function aggregateEvaluationReceipts(input) {
         if (!verifyAttemptReceipt(receipt))
             throw new Error(`Attempt receipt ${receipt.id} has an invalid content hash.`);
     }
+    if (input.authoredMetric) {
+        assertTasksetMetricResult(input.authoredMetric);
+        const metric = input.authoredMetric;
+        if (metric.runManifest.id !== input.manifest.id || metric.runManifest.contentHash !== input.manifest.contentHash
+            || contentHash(metric.tasksetRelease) !== contentHash(input.manifest.tasksetRelease)
+            || contentHash(metric.receiptRefs) !== contentHash(input.receipts.map(({ id, contentHash }) => ({ id, contentHash })))) {
+            throw new Error("Authored metric does not describe this evaluation's exact manifest and receipts.");
+        }
+    }
     const scores = input.receipts.flatMap((receipt) => typeof receipt.metadata.score === "number" && Number.isFinite(receipt.metadata.score)
         ? [receipt.metadata.score]
         : []);
@@ -118,6 +129,7 @@ export function aggregateEvaluationReceipts(input) {
         terminalCount: input.receipts.filter((receipt) => receipt.terminal).length,
         meanScore: scores.length ? scores.reduce((total, score) => total + score, 0) / scores.length : null,
         failureCounts,
+        ...(input.authoredMetric ? { authoredMetric: input.authoredMetric } : {}),
         metadata: input.metadata ?? {},
     });
     return EvaluationResultSchema.parse({ ...content, contentHash: contentHash(content) });

package/dist/tasksets.js

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { CapabilityRequirementSchema, ImmutableAssetRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ToolDeclarationSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { TasksetMetricPolicySchema } from "./metric-policy.js";
 export const TaskSplitSchema = z.enum(["train", "validation", "test", "frozen_eval"]);
 export const RequiredOutputContractSchema = z.object({
     path: z.string().trim().min(1).max(2_000).refine(safeRelativePath),
@@ -85,6 +86,7 @@ export const TasksetReleaseContentSchema = z.object({
     tasks: z.array(TaskRecordSchema).min(1).max(1_000_000),
     graders: z.array(GraderSpecSchema).min(1).max(1_000),
     verifierSetRelease: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().optional(),
+    metrics: TasksetMetricPolicySchema.optional(),
     metadata: MetadataSchema,
 }).strict();
 export const TasksetReleaseSchema = TasksetReleaseContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
@@ -115,6 +117,9 @@ export function validateTasksetRelease(input) {
     }
     const clusterSplits = new Map();
     for (const task of taskset.tasks) {
+        if (taskset.metrics?.aggregation === "weighted_mean" && !Object.hasOwn(taskset.metrics.taskWeights, task.id)) {
+            issues.push({ code: "metric_task_weight_missing", severity: "error", message: `Metric policy has no weight for task ${task.id}.`, path: "metrics.taskWeights" });
+        }
         const splits = clusterSplits.get(task.clusterKey) ?? new Set();
         splits.add(task.split);
         clusterSplits.set(task.clusterKey, splits);

package/dist/types/builtin-benchmarks/harness-refiner.d.ts

@@ -143,6 +143,20 @@ export declare const harnessRefinerBenchmarkRelease: {
         id: string;
         contentHash: string;
     } | undefined;
+    metrics?: {
+        schemaVersion: "openpond.tasksetMetricPolicy.v1";
+        primaryMetric: string;
+        aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
+        missingReward: "zero" | "exclude";
+        customAggregator: {
+            module: string;
+            exportName: string;
+            contentHash: string;
+            timeoutMs: number;
+            networkPolicy: "none";
+        } | null;
+        taskWeights?: Record<string, number> | undefined;
+    } | undefined;
 };
 export declare const harnessRefinerBenchmarkAssets: Readonly<Record<string, string>>;
 //# sourceMappingURL=harness-refiner.d.ts.map

package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
+{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}

package/dist/types/conformance.d.ts

@@ -253,6 +253,20 @@ export declare const genericToolConformance: {
             id: string;
             contentHash: string;
         } | undefined;
+        metrics?: {
+            schemaVersion: "openpond.tasksetMetricPolicy.v1";
+            primaryMetric: string;
+            aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
+            missingReward: "zero" | "exclude";
+            customAggregator: {
+                module: string;
+                exportName: string;
+                contentHash: string;
+                timeoutMs: number;
+                networkPolicy: "none";
+            } | null;
+            taskWeights?: Record<string, number> | undefined;
+        } | undefined;
     };
     manifest: {
         schemaVersion: "openpond.runManifest.v1";
@@ -549,6 +563,20 @@ export declare const marketingPortfolioConformance: {
             id: string;
             contentHash: string;
         } | undefined;
+        metrics?: {
+            schemaVersion: "openpond.tasksetMetricPolicy.v1";
+            primaryMetric: string;
+            aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
+            missingReward: "zero" | "exclude";
+            customAggregator: {
+                module: string;
+                exportName: string;
+                contentHash: string;
+                timeoutMs: number;
+                networkPolicy: "none";
+            } | null;
+            taskWeights?: Record<string, number> | undefined;
+        } | undefined;
     };
     manifest: {
         schemaVersion: "openpond.runManifest.v1";

package/dist/types/conformance.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}
+{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}

package/dist/types/deterministic-graders.d.ts

@@ -0,0 +1,23 @@
+import type { DeterministicGraderSpec, TaskRecord } from "./tasksets.js";
+export type DeterministicGrade = {
+    score: number | null;
+    passed: boolean;
+    feedback: string;
+};
+/** Local and portable execution interpret the same declared checks. Missing or
+ * unsupported checks are unscorable; an empty check cannot earn a reward. */
+export declare function evaluateDeterministicGrader(input: {
+    grader: Pick<DeterministicGraderSpec, "kind" | "config">;
+    task: Pick<TaskRecord, "expectedOutput">;
+    evidence: {
+        output: Record<string, unknown>;
+        artifactRefs: string[];
+        runtimeEventRefs: string[];
+    };
+}): DeterministicGrade;
+/** Encode native artifact/test checks without changing their meaning in transit. */
+export declare function portableDeterministicCheck(grader: {
+    kind: DeterministicGraderSpec["kind"] | "file" | "diff" | "test";
+    config: Record<string, unknown>;
+}): Pick<DeterministicGraderSpec, "kind" | "config">;
+//# sourceMappingURL=deterministic-graders.d.ts.map

package/dist/types/deterministic-graders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic-graders.d.ts","sourceRoot":"","sources":["../../../../src/deterministic-graders.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGzE,MAAM,MAAM,kBAAkB,GAAG;IAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7F;6EAC6E;AAC7E,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,MAAM,EAAE,IAAI,CAAC,uBAAuB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC;IACzD,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;IACzC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACnG,GAAG,kBAAkB,CAoErB;AAED,oFAAoF;AACpF,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,uBAAuB,CAAC,MAAM,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAAG,IAAI,CAAC,uBAAuB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAO1M"}

package/dist/types/evidence/conformance.d.ts

@@ -32,7 +32,7 @@ export declare const completeWorkProcessTraceFixture: {
             durationMs: number | null;
             cpuTimeMs: number | null;
             memoryPeakBytes: number | null;
-            errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+            errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
         };
     }[];
     contentHash: string;
@@ -209,7 +209,7 @@ export declare const incompleteWorkProcessTraceFixture: {
             durationMs: number | null;
             cpuTimeMs: number | null;
             memoryPeakBytes: number | null;
-            errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+            errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
         };
     }[];
     contentHash: string;
@@ -415,7 +415,7 @@ export declare const workEvidenceConformance: {
                 durationMs: number | null;
                 cpuTimeMs: number | null;
                 memoryPeakBytes: number | null;
-                errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+                errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
             };
         }[];
         contentHash: string;
@@ -648,7 +648,7 @@ export declare const workEvidenceConformance: {
                 durationMs: number | null;
                 cpuTimeMs: number | null;
                 memoryPeakBytes: number | null;
-                errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
+                errorClass: "unknown" | "policy" | "validation" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
             };
         }[];
         contentHash: string;