@openpond/evals 0.9.2 → 0.10.0

package/CONTRACT.md

@@ -119,3 +119,27 @@ runtime processes.
 plus a hash-bound trace. `EvaluationRunner` selects tasks, seeds, repetitions,
 and models, invokes the executor, runs graders in their declared boundary, and
 aggregates host results. Runtime adapters stay with Desktop or Sandbox.
+
+## Taskset evaluation populations
+
+`TasksetRunManifest` freezes the complete task/seed/receipt population before
+execution. Its policy distinguishes a real model plus configuration hash from
+an authored fixture check with no model identity. Execution either pins the
+Taskset-owned environment, verifier set and policy, or an actual selected
+Harness release. The complete package hash remains pinned in either case.
+Existing `RunManifest` Harness evaluations keep their existing contract.
+
+`aggregateTasksetRunReceipts` verifies the exact Taskset and metric policy,
+requires one matching receipt per admitted member, orders receipts by the
+manifest and consumes evaluation-role evidence. Missing members, repeated
+receipt IDs, changed tasks/seeds and training-role projections are rejected.
+Operational failures remain accounted for while being excluded from metric
+scores. A nonterminal member without an operational failure cannot finalize a
+run. Hosts must project receipts from their retained execution evidence.
+
+The authored metric uses the same implementation as existing evaluations.
+Without an authored policy, a run uses the mean of eligible measured scores
+and excludes missing rewards. An empty scorable population remains null.
+`aggregateTasksetRunInWorker` from `@openpond/evals/metrics/node` verifies custom
+module bytes and uses the existing bounded isolated worker. A fixture metric
+does not qualify a model for learning or acceptance.

package/dist/index.js

@@ -9,6 +9,7 @@ export * from "./harness.js";
 export * from "./learned-preference.js";
 export * from "./runs.js";
 export * from "./metrics.js";
+export * from "./taskset-run-contract.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./review-conformance.js";

package/dist/metrics-node.js

@@ -1,6 +1,6 @@
 import { stripTypeScriptTypes } from "node:module";
 import { executeJavaScriptIsolateInWorker } from "./javascript-isolate-node.js";
-import { aggregateTasksetEvaluationReceipts, executeTasksetMetric } from "./metrics.js";
+import { aggregateTasksetEvaluationReceipts, aggregateTasksetRunReceipts, executeTasksetMetric } from "./metrics.js";
 /** Verifies original source before stripping TS, then waits for worker termination
  * on success, error or cancellation. Authored imports and host APIs are unavailable. */
 export function executeTasksetMetricInWorker(input) {
@@ -9,6 +9,9 @@ export function executeTasksetMetricInWorker(input) {
 export function aggregateTasksetEvaluationInWorker(input) {
     return aggregateTasksetEvaluationReceipts(input, executeMetric);
 }
+export function aggregateTasksetRunInWorker(input) {
+    return aggregateTasksetRunReceipts(input, executeMetric);
+}
 const executeMetric = ({ source, module, exportName, scores, timeoutMs, signal }) => executeJavaScriptIsolateInWorker({
     source: /\.[cm]?ts$/.test(module) ? stripTypeScriptTypes(source, { mode: "strip" }) : source,
     exportName, value: scores, timeoutMs, signal,

package/dist/metrics.js

@@ -2,7 +2,17 @@ import { assertContentHash, contentHash, sha256 } from "@openpond/harness";
 import { aggregateEvaluationReceipts, RunManifestSchema, verifyAttemptReceipt } from "./runs.js";
 import { TasksetMetricPolicySchema, TasksetMetricResultContentSchema, TasksetMetricResultSchema } from "./metric-policy.js";
 import { assertTasksetRelease } from "./tasksets.js";
+import { TasksetRunManifestSchema, assertTasksetRunRelease, orderTasksetRunReceipts } from "./taskset-run-contract.js";
 export * from "./metric-policy.js";
+export * from "./taskset-run-contract.js";
+/** Complete Taskset-owned, selected-Harness or fixture runs share the same
+ * metric arithmetic, with the population fixed before execution. */
+export async function aggregateTasksetRunReceipts(input, executor) {
+    input.signal?.throwIfAborted();
+    assertTasksetRunRelease(input.manifest, input.taskset);
+    assertTasksetMetricSource(input.taskset, input.source);
+    return executeTasksetMetric({ ...input, policy: input.manifest.metricPolicy, receipts: orderTasksetRunReceipts(input.manifest, input.receipts) }, executor);
+}
 /** Preflight the exact metric source before any model or tool execution. */
 export function assertTasksetMetricSource(taskset, source) {
     assertTasksetRelease(taskset);
@@ -29,8 +39,16 @@ export async function aggregateTasksetEvaluationReceipts(input, executor) {
 export async function executeTasksetMetric(input, executor) {
     input.signal?.throwIfAborted();
     const policy = TasksetMetricPolicySchema.parse(input.policy);
-    const manifest = RunManifestSchema.parse(input.manifest);
+    const manifest = input.manifest.schemaVersion === "openpond.tasksetRunManifest.v1"
+        ? TasksetRunManifestSchema.parse(input.manifest) : RunManifestSchema.parse(input.manifest);
     assertContentHash(manifest, "Metric Run Manifest");
+    if (manifest.schemaVersion === "openpond.tasksetRunManifest.v1") {
+        if (contentHash(policy) !== contentHash(manifest.metricPolicy))
+            throw new Error("Metric policy differs from the admitted run.");
+        const ordered = orderTasksetRunReceipts(manifest, input.receipts);
+        if (ordered.some((receipt, index) => receipt.id !== input.receipts[index]?.id))
+            throw new Error("Metric receipts must follow the admitted population order.");
+    }
     if (!input.receipts.length || input.receipts.length > 1_000_000)
         throw new Error("Metric execution requires 1 to 1,000,000 attempt receipts.");
     const ids = new Set();

package/dist/taskset-run-contract.js

@@ -0,0 +1,91 @@
+import { z } from "zod";
+import { assertContentHash, contentHash, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
+import { RunLimitsSchema, RuntimeTargetBindingSchema, verifyAttemptReceipt } from "./runs.js";
+import { TasksetMetricPolicySchema } from "./metric-policy.js";
+import { assertTasksetRelease } from "./tasksets.js";
+/** A fixture check owns no model identity and cannot become a model evaluation. */
+export const TasksetRunPolicySchema = z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("model"), model: ModelRefSchema, configurationHash: ReleaseHashSchema }).strict(),
+    z.object({ kind: z.literal("fixture") }).strict(),
+]);
+export const TasksetRunExecutionSchema = z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("taskset"), environmentRelease: ImmutableReleaseRefSchema, verifierSetRelease: ImmutableReleaseRefSchema, policyHash: ReleaseHashSchema }).strict(),
+    z.object({ kind: z.literal("harness"), harnessRelease: ImmutableReleaseRefSchema }).strict(),
+]);
+export const TasksetRunMemberSchema = z.object({
+    receiptId: ReleaseIdSchema,
+    taskId: ReleaseIdSchema,
+    seed: z.string().trim().min(1).max(500),
+    fixtureId: ReleaseIdSchema.nullable(),
+}).strict();
+export const TasksetRunManifestContentSchema = z.object({
+    schemaVersion: z.literal("openpond.tasksetRunManifest.v1"),
+    id: ReleaseIdSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    packageHash: ReleaseHashSchema,
+    execution: TasksetRunExecutionSchema,
+    policy: TasksetRunPolicySchema,
+    gradingRole: z.literal("evaluation"),
+    metricPolicy: TasksetMetricPolicySchema,
+    population: z.array(TasksetRunMemberSchema).min(1).max(100_000),
+    runtimeTarget: RuntimeTargetBindingSchema,
+    limits: RunLimitsSchema,
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict().superRefine((value, context) => {
+    if (new Set(value.population.map(member => member.receiptId)).size !== value.population.length)
+        context.addIssue({ code: "custom", path: ["population"], message: "Run receipt identities must be unique." });
+    if (value.population.some(member => (member.fixtureId !== null) !== (value.policy.kind === "fixture")))
+        context.addIssue({ code: "custom", path: ["population"], message: "Only fixture runs require a fixture identity for every member." });
+});
+export const TasksetRunManifestSchema = TasksetRunManifestContentSchema.safeExtend({ contentHash: ReleaseHashSchema });
+export function createTasksetRunManifest(input) {
+    const content = TasksetRunManifestContentSchema.parse(input);
+    return TasksetRunManifestSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function tasksetRunMetricPolicy(taskset) {
+    return taskset.metrics ?? {
+        schemaVersion: "openpond.tasksetMetricPolicy.v1", primaryMetric: "mean_score",
+        aggregation: "mean_score", missingReward: "exclude", customAggregator: null,
+    };
+}
+/** Admission verifies this before dispatch. Package byte verification belongs
+ * to the package loader; this validates the release and population semantics. */
+export function assertTasksetRunRelease(manifest, taskset) {
+    TasksetRunManifestSchema.parse(manifest);
+    assertContentHash(manifest, "Taskset run manifest");
+    assertTasksetRelease(taskset);
+    if (manifest.tasksetRelease.id !== taskset.id || manifest.tasksetRelease.contentHash !== taskset.contentHash)
+        throw new Error("Taskset run differs from its pinned release.");
+    if (contentHash(manifest.metricPolicy) !== contentHash(tasksetRunMetricPolicy(taskset)))
+        throw new Error("Taskset run metric differs from its pinned release.");
+    const tasks = new Set(taskset.tasks.map(task => task.id));
+    if (manifest.population.some(member => !tasks.has(member.taskId)))
+        throw new Error("Taskset run population contains an unknown task.");
+    if (manifest.execution.kind === "taskset" && (!taskset.environmentRelease || !taskset.verifierSetRelease
+        || contentHash(manifest.execution.environmentRelease) !== contentHash(taskset.environmentRelease)
+        || contentHash(manifest.execution.verifierSetRelease) !== contentHash(taskset.verifierSetRelease)
+        || manifest.execution.policyHash !== contentHash(taskset.policy)))
+        throw new Error("Taskset-owned execution differs from its pinned release.");
+}
+/** Complete runs cannot drop failed members, add convenient attempts or relabel
+ * another task/seed's evidence. Ordering is always the admitted population's. */
+export function orderTasksetRunReceipts(manifest, receipts) {
+    TasksetRunManifestSchema.parse(manifest);
+    assertContentHash(manifest, "Taskset run manifest");
+    const byId = new Map(receipts.map(receipt => [receipt.id, receipt]));
+    if (byId.size !== receipts.length || receipts.length !== manifest.population.length)
+        throw new Error("Run receipts differ from the complete admitted population.");
+    return manifest.population.map(member => {
+        const receipt = byId.get(member.receiptId);
+        if (!receipt || !verifyAttemptReceipt(receipt)
+            || receipt.runManifest.id !== manifest.id || receipt.runManifest.contentHash !== manifest.contentHash
+            || receipt.taskId !== member.taskId || receipt.seed !== member.seed)
+            throw new Error("Run receipt differs from its admitted member or manifest.");
+        if (receipt.metadata.gradingRole !== "evaluation")
+            throw new Error("Taskset run requires evaluation-role evidence.");
+        if (!receipt.terminal && !["infrastructure_failure", "timeout", "cancelled"].includes(receipt.failureClass ?? ""))
+            throw new Error("Taskset run member has no terminal accounting outcome.");
+        return receipt;
+    });
+}

package/dist/types/index.d.ts

@@ -9,6 +9,7 @@ export * from "./harness.js";
 export * from "./learned-preference.js";
 export * from "./runs.js";
 export * from "./metrics.js";
+export * from "./taskset-run-contract.js";
 export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./review-conformance.js";

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}

package/dist/types/metrics-node.d.ts

@@ -1,8 +1,9 @@
-import { type TasksetEvaluationInput, type TasksetMetricExecutionInput } from "./metrics.js";
+import { type TasksetEvaluationInput, type TasksetRunEvaluationInput, type TasksetMetricExecutionInput } from "./metrics.js";
 import type { TasksetMetricResult } from "./metric-policy.js";
 import type { EvaluationResult } from "./runs.js";
 /** Verifies original source before stripping TS, then waits for worker termination
  * on success, error or cancellation. Authored imports and host APIs are unavailable. */
 export declare function executeTasksetMetricInWorker(input: TasksetMetricExecutionInput): Promise<TasksetMetricResult>;
 export declare function aggregateTasksetEvaluationInWorker(input: TasksetEvaluationInput): Promise<EvaluationResult>;
+export declare function aggregateTasksetRunInWorker(input: TasksetRunEvaluationInput): Promise<TasksetMetricResult>;
 //# sourceMappingURL=metrics-node.d.ts.map

package/dist/types/metrics-node.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics-node.d.ts","sourceRoot":"","sources":["../../../../src/metrics-node.ts"],"names":[],"mappings":"AAEA,OAAO,EAA4D,KAAK,sBAAsB,EAAE,KAAK,2BAA2B,EAA8B,MAAM,cAAc,CAAC;AACnL,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAElD;wFACwF;AACxF,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,2BAA2B,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAE7G;AAED,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAE3G"}
+{"version":3,"file":"metrics-node.d.ts","sourceRoot":"","sources":["../../../../src/metrics-node.ts"],"names":[],"mappings":"AAEA,OAAO,EAAyF,KAAK,sBAAsB,EAAE,KAAK,yBAAyB,EAAE,KAAK,2BAA2B,EAA8B,MAAM,cAAc,CAAC;AAChP,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAElD;wFACwF;AACxF,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,2BAA2B,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAE7G;AAED,wBAAgB,kCAAkC,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAE3G;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAE1G"}

package/dist/types/metrics.d.ts

@@ -1,9 +1,11 @@
 import { type AttemptReceipt, type EvaluationResult, type RunManifest } from "./runs.js";
 import { type TasksetMetricPolicy, type TasksetMetricResult } from "./metric-policy.js";
 import { type TasksetRelease } from "./tasksets.js";
+import { type TasksetRunManifest } from "./taskset-run-contract.js";
 export * from "./metric-policy.js";
+export * from "./taskset-run-contract.js";
 export interface TasksetMetricExecutionInput {
-    manifest: RunManifest;
+    manifest: RunManifest | TasksetRunManifest;
     receipts: AttemptReceipt[];
     policy: TasksetMetricPolicy;
     /** Exact UTF-8 module bytes, verified before any compiler or executor sees them. */
@@ -19,10 +21,18 @@ export type TasksetMetricExecutor = (input: {
     signal?: AbortSignal;
 }) => Promise<unknown>;
 export interface TasksetEvaluationInput extends Omit<TasksetMetricExecutionInput, "policy"> {
+    manifest: RunManifest;
     id: string;
     taskset: TasksetRelease;
     metadata?: Record<string, unknown>;
 }
+export interface TasksetRunEvaluationInput extends Omit<TasksetMetricExecutionInput, "policy" | "manifest"> {
+    manifest: TasksetRunManifest;
+    taskset: TasksetRelease;
+}
+/** Complete Taskset-owned, selected-Harness or fixture runs share the same
+ * metric arithmetic, with the population fixed before execution. */
+export declare function aggregateTasksetRunReceipts(input: TasksetRunEvaluationInput, executor?: TasksetMetricExecutor): Promise<TasksetMetricResult>;
 /** Preflight the exact metric source before any model or tool execution. */
 export declare function assertTasksetMetricSource(taskset: TasksetRelease, source?: string): void;
 /** Bind the declared metric to one frozen release and population, retaining

package/dist/types/metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../src/metrics.ts"],"names":[],"mappings":"AACA,OAAO,EAAwE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAC/J,OAAO,EAA0F,KAAK,mBAAmB,EAAE,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAChL,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAC1E,cAAc,oBAAoB,CAAC;AAEnC,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,mBAAmB,CAAC;IAC5B,oFAAoF;IACpF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,qBAAqB,GAAG,CAAC,KAAK,EAAE;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEvB,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,2BAA2B,EAAE,QAAQ,CAAC;IACzF,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,cAAc,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,4EAA4E;AAC5E,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED;+EAC+E;AAC/E,wBAAsB,kCAAkC,CAAC,KAAK,EAAE,sBAAsB,EAAE,QAAQ,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAOnJ;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,2BAA2B,EAAE,QAAQ,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CA6D7I"}
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../src/metrics.ts"],"names":[],"mappings":"AACA,OAAO,EAAwE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAC/J,OAAO,EAA0F,KAAK,mBAAmB,EAAE,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAChL,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,EAA8E,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAChJ,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAE1C,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,WAAW,GAAG,kBAAkB,CAAC;IAC3C,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,mBAAmB,CAAC;IAC5B,oFAAoF;IACpF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,qBAAqB,GAAG,CAAC,KAAK,EAAE;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEvB,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,2BAA2B,EAAE,QAAQ,CAAC;IACzF,QAAQ,EAAE,WAAW,CAAC;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,cAAc,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,yBAA0B,SAAQ,IAAI,CAAC,2BAA2B,EAAE,QAAQ,GAAG,UAAU,CAAC;IACzG,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;CACzB;AAED;oEACoE;AACpE,wBAAsB,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,EAAE,QAAQ,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAKlJ;AAED,4EAA4E;AAC5E,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAGxF;AAED;+EAC+E;AAC/E,wBAAsB,kCAAkC,CAAC,KAAK,EAAE,sBAAsB,EAAE,QAAQ,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAOnJ;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,2BAA2B,EAAE,QAAQ,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAmE7I"}

package/dist/types/review-conformance.d.ts

@@ -62,7 +62,7 @@ export declare const harnessEvaluationReviewConformance: {
         } | null;
         classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
         triage: {
-            layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
+            layer: "model" | "tools" | "evaluation" | "runtime" | "harness" | "product" | "retrieval";
             status: "blocked" | "resolved" | "not_applicable" | "unresolved";
             reason: string;
             evidenceRefs: {
@@ -153,7 +153,7 @@ export declare const harnessEvaluationReviewConformance: {
         } | null;
         classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
         triage: {
-            layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
+            layer: "model" | "tools" | "evaluation" | "runtime" | "harness" | "product" | "retrieval";
             status: "blocked" | "resolved" | "not_applicable" | "unresolved";
             reason: string;
             evidenceRefs: {
@@ -244,7 +244,7 @@ export declare const harnessEvaluationReviewConformance: {
         } | null;
         classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
         triage: {
-            layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
+            layer: "model" | "tools" | "evaluation" | "runtime" | "harness" | "product" | "retrieval";
             status: "blocked" | "resolved" | "not_applicable" | "unresolved";
             reason: string;
             evidenceRefs: {
@@ -335,7 +335,7 @@ export declare const harnessEvaluationReviewConformance: {
         } | null;
         classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
         triage: {
-            layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
+            layer: "model" | "tools" | "evaluation" | "runtime" | "harness" | "product" | "retrieval";
             status: "blocked" | "resolved" | "not_applicable" | "unresolved";
             reason: string;
             evidenceRefs: {
@@ -580,7 +580,7 @@ export declare const harnessEvaluationReviewConformance: {
         } | null;
         classification: "runtime" | "taskset" | "no_action" | "harness_maintenance" | "product" | "model_improvement";
         triage: {
-            layer: "model" | "tools" | "evaluation" | "runtime" | "product" | "harness" | "retrieval";
+            layer: "model" | "tools" | "evaluation" | "runtime" | "harness" | "product" | "retrieval";
             status: "blocked" | "resolved" | "not_applicable" | "unresolved";
             reason: string;
             evidenceRefs: {

package/dist/types/taskset-run-contract.d.ts

@@ -0,0 +1,232 @@
+import { z } from "zod";
+import { type AttemptReceipt } from "./runs.js";
+import { type TasksetMetricPolicy } from "./metric-policy.js";
+import { type TasksetRelease } from "./tasksets.js";
+/** A fixture check owns no model identity and cannot become a model evaluation. */
+export declare const TasksetRunPolicySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"model">;
+    model: z.ZodObject<{
+        provider: z.ZodString;
+        model: z.ZodString;
+        revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    }, z.core.$strict>;
+    configurationHash: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"fixture">;
+}, z.core.$strict>], "kind">;
+export declare const TasksetRunExecutionSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"taskset">;
+    environmentRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    verifierSetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    policyHash: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"harness">;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+}, z.core.$strict>], "kind">;
+export declare const TasksetRunMemberSchema: z.ZodObject<{
+    receiptId: z.ZodString;
+    taskId: z.ZodString;
+    seed: z.ZodString;
+    fixtureId: z.ZodNullable<z.ZodString>;
+}, z.core.$strict>;
+export declare const TasksetRunManifestContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.tasksetRunManifest.v1">;
+    id: z.ZodString;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    packageHash: z.ZodString;
+    execution: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"taskset">;
+        environmentRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        verifierSetRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        policyHash: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"harness">;
+        harnessRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>], "kind">;
+    policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"model">;
+        model: z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+            revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        }, z.core.$strict>;
+        configurationHash: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"fixture">;
+    }, z.core.$strict>], "kind">;
+    gradingRole: z.ZodLiteral<"evaluation">;
+    metricPolicy: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.tasksetMetricPolicy.v1">;
+        primaryMetric: z.ZodString;
+        aggregation: z.ZodEnum<{
+            mean_score: "mean_score";
+            pass_rate: "pass_rate";
+            weighted_mean: "weighted_mean";
+            custom: "custom";
+        }>;
+        missingReward: z.ZodEnum<{
+            zero: "zero";
+            exclude: "exclude";
+        }>;
+        taskWeights: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
+        customAggregator: z.ZodNullable<z.ZodObject<{
+            module: z.ZodString;
+            exportName: z.ZodString;
+            contentHash: z.ZodString;
+            timeoutMs: z.ZodNumber;
+            networkPolicy: z.ZodLiteral<"none">;
+        }, z.core.$strict>>;
+    }, z.core.$strict>;
+    population: z.ZodArray<z.ZodObject<{
+        receiptId: z.ZodString;
+        taskId: z.ZodString;
+        seed: z.ZodString;
+        fixtureId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    runtimeTarget: z.ZodObject<{
+        adapterId: z.ZodString;
+        placement: z.ZodEnum<{
+            local: "local";
+            remote: "remote";
+            colocated: "colocated";
+            provider_native: "provider_native";
+        }>;
+        runtimeVersion: z.ZodString;
+        capabilityReceipt: z.ZodString;
+    }, z.core.$strict>;
+    limits: z.ZodObject<{
+        maxTurns: z.ZodNumber;
+        timeoutMs: z.ZodNumber;
+        maxOutputBytes: z.ZodNumber;
+        maximumSpendUsd: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const TasksetRunManifestSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.tasksetRunManifest.v1">;
+    id: z.ZodString;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    packageHash: z.ZodString;
+    execution: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"taskset">;
+        environmentRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        verifierSetRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        policyHash: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"harness">;
+        harnessRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>], "kind">;
+    policy: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"model">;
+        model: z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+            revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        }, z.core.$strict>;
+        configurationHash: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"fixture">;
+    }, z.core.$strict>], "kind">;
+    gradingRole: z.ZodLiteral<"evaluation">;
+    metricPolicy: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.tasksetMetricPolicy.v1">;
+        primaryMetric: z.ZodString;
+        aggregation: z.ZodEnum<{
+            mean_score: "mean_score";
+            pass_rate: "pass_rate";
+            weighted_mean: "weighted_mean";
+            custom: "custom";
+        }>;
+        missingReward: z.ZodEnum<{
+            zero: "zero";
+            exclude: "exclude";
+        }>;
+        taskWeights: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodNumber>>;
+        customAggregator: z.ZodNullable<z.ZodObject<{
+            module: z.ZodString;
+            exportName: z.ZodString;
+            contentHash: z.ZodString;
+            timeoutMs: z.ZodNumber;
+            networkPolicy: z.ZodLiteral<"none">;
+        }, z.core.$strict>>;
+    }, z.core.$strict>;
+    population: z.ZodArray<z.ZodObject<{
+        receiptId: z.ZodString;
+        taskId: z.ZodString;
+        seed: z.ZodString;
+        fixtureId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    runtimeTarget: z.ZodObject<{
+        adapterId: z.ZodString;
+        placement: z.ZodEnum<{
+            local: "local";
+            remote: "remote";
+            colocated: "colocated";
+            provider_native: "provider_native";
+        }>;
+        runtimeVersion: z.ZodString;
+        capabilityReceipt: z.ZodString;
+    }, z.core.$strict>;
+    limits: z.ZodObject<{
+        maxTurns: z.ZodNumber;
+        timeoutMs: z.ZodNumber;
+        maxOutputBytes: z.ZodNumber;
+        maximumSpendUsd: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type TasksetRunManifest = z.infer<typeof TasksetRunManifestSchema>;
+export declare function createTasksetRunManifest(input: z.input<typeof TasksetRunManifestContentSchema>): TasksetRunManifest;
+export declare function tasksetRunMetricPolicy(taskset: TasksetRelease): TasksetMetricPolicy;
+/** Admission verifies this before dispatch. Package byte verification belongs
+ * to the package loader; this validates the release and population semantics. */
+export declare function assertTasksetRunRelease(manifest: TasksetRunManifest, taskset: TasksetRelease): void;
+/** Complete runs cannot drop failed members, add convenient attempts or relabel
+ * another task/seed's evidence. Ordering is always the admitted population's. */
+export declare function orderTasksetRunReceipts(manifest: TasksetRunManifest, receipts: AttemptReceipt[]): AttemptReceipt[];
+//# sourceMappingURL=taskset-run-contract.d.ts.map

package/dist/types/taskset-run-contract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"taskset-run-contract.d.ts","sourceRoot":"","sources":["../../../../src/taskset-run-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnH,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,mFAAmF;AACnF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;4BAGjC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;kBAKxB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AACvH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,mBAAmB,CAKnF;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAcnG;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAclH"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.9.2",
+  "version": "0.10.0",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",
@@ -158,7 +158,7 @@
     "zod": "^4.1.13"
   },
   "peerDependencies": {
-    "@openpond/harness": "^0.2.0"
+    "@openpond/harness": "^0.2.0 || ^0.3.0"
   },
   "devDependencies": {
     "@openpond/harness": "workspace:*",