@openpond/evals 0.11.21 → 0.12.1

package/dist/native-conversations/collector-source-cache.js

@@ -12,7 +12,7 @@ export class CollectorSourceCache {
     }
     selection(connection, until, revision) {
         return contentHash([connection.source, connection.projectId, connection.teamId,
-            connection.apiBaseUrl, connection.account ?? null, connection.since, until, revision]);
+            connection.apiBaseUrl, connection.account ?? null, connection.since, connection.selectedSessionKeys ?? null, until, revision]);
     }
     read(connection, until, sessionKey, revision) {
         const row = this.store.database.prepare("SELECT value FROM source_read_cache WHERE connection_id=? AND session_key=? AND selection_hash=?").get(connection.id, sessionKey, this.selection(connection, until, revision));

package/dist/native-conversations/collector-store.js

@@ -61,6 +61,20 @@ export class CollectorStore {
                 this.database.exec("COMMIT");
                 return;
             }
+            if (prior && JSON.stringify(prior.selectedSessionKeys) !== JSON.stringify(connection.selectedSessionKeys)) {
+                const owner = JSON.parse(this.setting("owner") || "{}");
+                if (owner.pid) {
+                    let alive = true;
+                    try {
+                        process.kill(owner.pid, 0);
+                    }
+                    catch (error) {
+                        alive = error.code !== "ESRCH";
+                    }
+                    if (alive)
+                        throw new Error("Cancel or finish the running import before changing its conversation selection.");
+                }
+            }
             const identityChanged = !!prior &&
                 (prior.source.instanceId !== connection.source.instanceId ||
                     prior.projectId !== connection.projectId ||
@@ -69,7 +83,8 @@ export class CollectorStore {
                     prior.account !== connection.account);
             if (identityChanged)
                 connection = { ...connection, destinations: undefined };
-            if (prior && (identityChanged || prior.since !== connection.since)) {
+            if (prior && (identityChanged || prior.since !== connection.since ||
+                JSON.stringify(prior.selectedSessionKeys) !== JSON.stringify(connection.selectedSessionKeys))) {
                 this.database.prepare("DELETE FROM settings WHERE key=?").run(`coverage:${connection.id}`);
                 this.database.prepare("DELETE FROM coverage_sessions WHERE connection_id=?").run(connection.id);
                 // Scope changes fence queued evidence too: do not upload a wider old
@@ -81,7 +96,16 @@ export class CollectorStore {
                 this.database.prepare("DELETE FROM settings WHERE key=?").run(`syncAcknowledged:${connection.id}`);
                 connection = { ...connection, completedSyncRevision: 0 };
                 this.error(connection.id, null);
-                for (const table of ["source_scans", "checkpoints", "pending"])
+                if (!identityChanged && prior.since === connection.since) {
+                    // Preserve acknowledged boundaries when adding conversations, but
+                    // reacquire every queued boundary whose snapshot is being discarded.
+                    this.database.prepare(`DELETE FROM checkpoints WHERE connection_id=? AND EXISTS (
+            SELECT 1 FROM pending p, json_each(p.payload,'$.boundaryIds') b
+            WHERE p.connection_id=checkpoints.connection_id
+              AND json_extract(p.payload,'$.sessionKey')=checkpoints.session_key
+              AND b.value=checkpoints.boundary_id)`).run(connection.id);
+                }
+                for (const table of identityChanged || prior.since !== connection.since ? ["source_scans", "checkpoints", "pending"] : ["source_scans", "pending"])
                     this.database
                         .prepare(`DELETE FROM ${table} WHERE connection_id=?`)
                         .run(connection.id);
@@ -270,6 +294,7 @@ export class CollectorStore {
                         .get(String(row.id))?.bytes ?? 0),
                     lastAdmissionAt: this.setting(`lastAdmission:${row.id}`),
                     since: config.since,
+                    selectedConversations: config.selectedSessionKeys?.length ?? null,
                     schedule,
                     nextRunAt: schedule && config.state === "active" ? nextScheduledSync(schedule) : null,
                     lastSuccessfulSyncAt: this.setting(`lastSuccess:${row.id}`),

package/dist/native-conversations/collector.js

@@ -36,7 +36,7 @@ export async function runCollector(input) {
             }
         }
         const now = new Date();
-        selected = store.connections().filter(connection => connection.state !== "disconnected" &&
+        selected = store.connections().filter(connection => connection.state !== "disconnected" && connection.selectedSessionKeys?.length !== 0 &&
             (!input.connectionIds || input.connectionIds.includes(connection.id)) &&
             (input.trigger !== "scheduled" || (connection.state === "active" && scheduledSyncDue(store.schedule(connection.id), store.setting(`scheduledMinute:${connection.id}`), now))));
         if (input.trigger === "scheduled")
@@ -173,13 +173,14 @@ export async function runCollector(input) {
                 for (let pass = 0; pass < 3; pass++) {
                     await check();
                     const revision = connection.revision, requested = connection.requestedSyncRevision ?? 0;
+                    const selection = JSON.stringify(connection.selectedSessionKeys);
                     phase = "source";
                     new CollectorSourceCache(store).seed(connection, until, store.coverage.manifest(connection.id));
                     store.coverage.begin(connection, `${nonce}-${pass}`, run.startedAt, until);
                     const checkAcquisition = async () => {
                         await check();
                         phase = "source";
-                        if (connection.revision !== revision)
+                        if (connection.revision !== revision || JSON.stringify(connection.selectedSessionKeys) !== selection)
                             throw new CollectorScopeChangedError("Source settings changed during acquisition.");
                     };
                     try {
@@ -193,7 +194,7 @@ export async function runCollector(input) {
                     }
                     errors.set(connection.id, "source", null);
                     await drain();
-                    await input.transport.publishCoverage(connection, store.coverage.finish(connection.id, true), signal);
+                    await input.transport.publishCoverage(connection, store.coverage.finish(connection.id, connection.selectedSessionKeys === undefined), signal);
                     connection = retained(connection);
                     if (connection.revision !== revision) {
                         if (pass === 2)

package/dist/native-conversations/git-assets.js

@@ -0,0 +1,61 @@
+import { execFile } from "node:child_process";
+import { realpath } from "node:fs/promises";
+import { isAbsolute } from "node:path";
+import { promisify } from "node:util";
+import { ConnectedRepositoryContextSchema } from "../connected-evidence/contracts.js";
+import { ConnectedGitFileProofSchema, verifyConnectedGitFileProof } from "../connected-evidence/git-file-proof.js";
+const execute = promisify(execFile);
+/** Capture only recorded Git objects. A missing historical object is an explicit
+ * gap; this operation never reads the current worktree or fetches another repo. */
+export async function captureNativeGitAssetProofs(input) {
+    const context = ConnectedRepositoryContextSchema.parse(input.repositoryContext);
+    if (!isAbsolute(input.repositoryPath) || !input.paths.length || input.paths.length > 100
+        || new Set(input.paths).size !== input.paths.length)
+        throw new Error("native_git_asset_selection_invalid");
+    // Validate selections before invoking Git, including paths prohibited by the
+    // source asset policy. The remaining fields are filled from immutable objects.
+    for (const path of input.paths)
+        ConnectedGitFileProofSchema.shape.path.parse(path);
+    const repository = await realpath(input.repositoryPath);
+    const read = async (args, maxBuffer = 2 * 1024 * 1024) => {
+        input.signal?.throwIfAborted();
+        const result = await execute("git", ["-C", repository, ...args], {
+            encoding: "buffer", timeout: 15_000, maxBuffer, signal: input.signal,
+        });
+        return result.stdout;
+    };
+    const objectId = async (ref) => {
+        const hash = (await read(["rev-parse", "--verify", ref], 1_024)).toString("ascii").trim();
+        if (!new RegExp(`^[a-f0-9]{${context.commitHash.length}}$`).test(hash))
+            throw new Error("native_git_object_identity_invalid");
+        return hash;
+    };
+    const commit = await read(["cat-file", "commit", context.commitHash], 256 * 1024);
+    const cache = new Map();
+    let totalBytes = 0;
+    const proofs = [];
+    for (const path of input.paths) {
+        const directories = ["", ...path.split("/").slice(0, -1).map((_, index, parts) => parts.slice(0, index + 1).join("/"))];
+        const trees = [];
+        for (const directory of directories) {
+            const ref = directory ? `${context.commitHash}:${directory}` : `${context.commitHash}^{tree}`;
+            let tree = cache.get(ref);
+            if (!tree) {
+                const hash = await objectId(ref);
+                tree = { hash, base64: (await read(["cat-file", "tree", hash])).toString("base64") };
+                cache.set(ref, tree);
+            }
+            trees.push(tree);
+        }
+        const blob = await objectId(`${context.commitHash}:${path}`);
+        const proof = ConnectedGitFileProofSchema.parse({ schemaVersion: "openpond.connectedGitFileProof.v1", path,
+            commit: { hash: context.commitHash, base64: commit.toString("base64") }, trees,
+            contentsBase64: (await read(["cat-file", "blob", blob])).toString("base64") });
+        const verified = verifyConnectedGitFileProof(context, proof);
+        totalBytes += Buffer.byteLength(JSON.stringify(proof));
+        if (totalBytes > 16 * 1024 * 1024)
+            throw new Error("native_git_asset_capture_limit");
+        proofs.push({ proof, verified });
+    }
+    return proofs;
+}

package/dist/native-conversations/history.js

@@ -293,6 +293,13 @@ export async function readSession(source, session, input = {}) {
         const graph = claudeGraph(connectedJsonLines(file));
         if (graph.sessionId !== session.nativeSessionId)
             throw new Error("Native session changed identity.");
+        if (input.latestBranch) {
+            if (input.branchLeafId || input.expectedBranchRevision || input.branchAnchor)
+                throw new Error("Choose either the latest conversation or an explicit branch.");
+            // Desktop inspection follows the native conversation. Imports keep their
+            // existing explicit branch authority and never enable this option.
+            files = [{ ...file, text: graph.select(graph.latestLeafId()).chain.map(row => JSON.stringify(row)).join("\n") + "\n" }];
+        }
         if (input.branchAnchor) {
             if (input.branchLeafId || input.expectedBranchRevision)
                 throw new Error("Choose either a frozen branch snapshot or a followed branch anchor.");

package/dist/native-conversations/index.js

@@ -2,6 +2,7 @@
 export * from "./contracts.js";
 export * from "./discovery.js";
 export * from "./history.js";
+export * from "./selection.js";
 export * from "./collector-contracts.js";
 export * from "./collector-store.js";
 export * from "./collector.js";
@@ -10,3 +11,4 @@ export * from "./location.js";
 export * from "./collector-branches.js";
 export * from "./collector-destinations.js";
 export * from "./collector-schedule.js";
+export { captureNativeGitAssetProofs } from "./git-assets.js";

package/dist/native-conversations/selection.js

@@ -0,0 +1,57 @@
+import { contentHash } from "@openpond/harness";
+import { z } from "zod";
+import { listSessions } from "./history.js";
+export const NativeUploadSelectionSchema = z.object({
+    sourceInstanceId: z.string().min(1).max(500),
+    sessionKeys: z.array(z.string().regex(/^[a-f0-9]{64}$/)).min(1).max(10_000),
+}).strict().refine(value => new Set(value.sessionKeys).size === value.sessionKeys.length, "Conversation selection contains duplicates.");
+/** Includes the source path so duplicate native IDs cannot select a sibling file. */
+export function nativeSessionSelectionKey(session) {
+    return contentHash(JSON.stringify([session.path, session.nativeSessionId]));
+}
+export async function nativeUploadInventory(source) {
+    if (!source.available || !source.capabilities.history)
+        throw new Error("The selected conversation source is unavailable.");
+    const items = [];
+    let cursor;
+    const seen = new Set();
+    do {
+        const page = await listSessions(source, { cursor, limit: 100 });
+        items.push(...page.items);
+        if (items.length > 10_000)
+            throw new Error("Choose a source with no more than 10,000 conversations.");
+        cursor = page.nextCursor ?? undefined;
+        if (cursor && seen.has(cursor))
+            throw new Error("Conversation listing did not advance. Refresh the source.");
+        if (cursor)
+            seen.add(cursor);
+    } while (cursor);
+    return items;
+}
+export async function validateNativeUploadSelection(source, raw) {
+    const selection = NativeUploadSelectionSchema.parse(raw);
+    if (selection.sourceInstanceId !== source.instanceId)
+        throw new Error("Conversation selection belongs to another source installation.");
+    const available = new Set((await nativeUploadInventory(source)).map(nativeSessionSelectionKey));
+    if (selection.sessionKeys.some(key => !available.has(key)))
+        throw new Error("A selected conversation is no longer available. Refresh and select it again.");
+    return selection;
+}
+export function nativeUploadConversationPage(connection, inventory, input) {
+    const query = (input.query ?? "").trim().toLocaleLowerCase(), range = input.range ?? "all", offset = input.offset ?? 0;
+    if (query.length > 500 || !["day", "week", "all"].includes(range) || !Number.isInteger(offset) || offset < 0 || offset > 10_000)
+        throw new Error("Invalid conversation search or page.");
+    const since = range === "all" ? 0 : Date.now() - (range === "day" ? 1 : 7) * 86400000;
+    const items = inventory.filter(item => Date.parse(item.updatedAt) >= since &&
+        (!query || [item.title, item.cwd, item.nativeSessionId].some(text => text?.toLocaleLowerCase().includes(query))))
+        .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
+    const approved = connection.selectedSessionKeys ? new Set(connection.selectedSessionKeys) : null;
+    return {
+        total: items.length, nextOffset: offset + 50 < items.length ? offset + 50 : null,
+        items: items.slice(offset, offset + 50).map(item => ({
+            key: nativeSessionSelectionKey(item), title: item.title, cwd: item.cwd,
+            nativeSessionId: item.nativeSessionId, updatedAt: item.updatedAt, issue: item.issue ?? null,
+            approved: approved === null || approved.has(nativeSessionSelectionKey(item)),
+        })),
+    };
+}

package/dist/native-conversations/supervisor.js

@@ -164,6 +164,8 @@ export async function setCollectorSchedule(input, connectionId, schedule) {
         const connection = store.connections().find(item => item.id === connectionId);
         if (!connection || (schedule && connection.state === "disconnected"))
             throw new Error("Select a connected source before scheduling imports.");
+        if (schedule && connection.selectedSessionKeys?.length === 0)
+            throw new Error("Choose conversations to upload before enabling continual imports.");
         const owner = store.setting("scheduleOwner");
         if (owner) {
             const { pid } = JSON.parse(owner);

package/dist/prepared-process-isolation.js

@@ -0,0 +1,131 @@
+/** Staged by the private verifier owner. Untrusted policy programs run with no
+ * capabilities as a different UID, outside the sealed private verifier tree.
+ * The parent must assess returned behavior; an exit code is never correctness. */
+export const PREPARED_PROCESS_ISOLATION_SOURCE = String.raw `
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { randomInt } from "node:crypto";
+
+const MAX_BYTES = 15 * 1024 * 1024, MAX_OUTPUT = 1024 * 1024;
+const usedUsers = new Set();
+const safePath = value => typeof value === "string" && value.length > 0 && value.length <= 2000
+  && !value.startsWith("/") && !/[\\\x00-\x1f\x7f]/.test(value)
+  && value.split("/").every(part => part && part !== "." && part !== ".." && !part.startsWith(".env"));
+
+/** files contain only public original/policy bytes and a public invocation
+ * adapter. Never stage independent assertions, expected answers or checkers. */
+export async function runPolicyProgram(input) {
+  const privateRoot = path.dirname(fileURLToPath(import.meta.url));
+  if (process.getuid?.() !== 0 || !path.basename(privateRoot).startsWith(".openpond-reward-verifier-"))
+    throw new Error("policy_process_isolation_runtime_unavailable");
+  if (!Array.isArray(input.files) || input.files.length < 1 || input.files.length > 100
+    || !safePath(input.entrypoint) || !input.files.some(file => file.path === input.entrypoint)
+    || !Number.isSafeInteger(input.timeoutMs) || input.timeoutMs < 1 || input.timeoutMs > 60000
+    || !Array.isArray(input.args ?? []) || (input.args ?? []).length > 100
+    || (input.args ?? []).some(arg => typeof arg !== "string" || arg.length > 2000 || arg.includes("\0")))
+    throw new Error("policy_process_isolation_input_invalid");
+  let total = 0;
+  const paths = new Set();
+  const files = input.files.map(file => {
+    if (!safePath(file.path) || paths.has(file.path) || typeof file.base64 !== "string")
+      throw new Error("policy_process_isolation_file_invalid");
+    paths.add(file.path);
+    const bytes = Buffer.from(file.base64, "base64");
+    if (bytes.toString("base64") !== file.base64) throw new Error("policy_process_isolation_file_invalid");
+    total += bytes.length;
+    if (total > MAX_BYTES) throw new Error("policy_process_isolation_file_limit");
+    return { path: file.path, bytes };
+  });
+  // Directory access seals request JSON, checker modules, helper source and
+  // private artifacts before any untrusted program can be spawned.
+  await fs.chmod(privateRoot, 0o700);
+  const stat = await fs.stat(privateRoot);
+  if (stat.uid !== 0 || (stat.mode & 0o777) !== 0o700) throw new Error("policy_process_private_tree_not_sealed");
+  for (const executable of ["/usr/bin/setsid", "/usr/bin/setpriv", "/usr/local/bin/bun"])
+    await fs.access(executable, 1);
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "openpond-policy-"));
+  let user;
+  do { user = randomInt(1000000, 2000000); } while (usedUsers.has(user));
+  usedUsers.add(user);
+  let child, timer, timedOut = false;
+  const kill = () => {
+    if (!child) return;
+    try { process.kill(-child.pid, "SIGKILL"); } catch {}
+    try { child.kill("SIGKILL"); } catch {}
+  };
+  try {
+    await fs.chmod(directory, 0o700);
+    await fs.chown(directory, user, user);
+    for (const file of files) {
+      const target = path.join(directory, file.path);
+      const parent = path.dirname(target);
+      await fs.mkdir(parent, { recursive: true, mode: 0o700 });
+      let current = parent;
+      while (current !== directory) {
+        await fs.chmod(current, 0o700); await fs.chown(current, user, user);
+        current = path.dirname(current);
+      }
+      await fs.writeFile(target, file.bytes, { mode: 0o600, flag: "wx" });
+      await fs.chown(target, user, user);
+    }
+    child = Bun.spawn(["/usr/bin/setsid", "/usr/bin/setpriv", "--reuid=" + user, "--regid=" + user, "--clear-groups",
+      "--no-new-privs", "--bounding-set=-all", "--inh-caps=-all", "--ambient-caps=-all",
+      "/usr/local/bin/bun", path.join(directory, input.entrypoint), ...(input.args ?? [])], {
+      cwd: directory, env: { PATH: "/usr/local/bin:/usr/bin:/bin", TMPDIR: directory }, stdout: "pipe", stderr: "pipe", stdin: "ignore",
+    });
+    timer = setTimeout(() => { timedOut = true; kill(); }, input.timeoutMs);
+    const read = async stream => {
+      const reader = stream.getReader(), chunks = [];
+      let bytes = 0;
+      try {
+        while (true) {
+          const item = await reader.read(); if (item.done) break;
+          bytes += item.value.byteLength;
+          if (bytes > MAX_OUTPUT) { kill(); throw new Error("policy_process_output_limit"); }
+          chunks.push(item.value);
+        }
+        return new TextDecoder("utf8", { fatal: true }).decode(Buffer.concat(chunks));
+      } finally { await reader.cancel(); reader.releaseLock(); }
+    };
+    const [stdout, stderr, exitCode] = await Promise.all([read(child.stdout), read(child.stderr), child.exited]);
+    return { stdout, stderr, exitCode, timedOut, isolation: { uid: user, gid: user, privateTreeSealed: true } };
+  } finally {
+    if (timer) clearTimeout(timer);
+    kill();
+    // A policy child can call setsid and escape its initial process group.
+    // Drain every process with this invocation's unique real UID before the
+    // next call; namespaces cannot grant a different outside UID or privilege.
+    try { await drainPolicyProcesses(user); }
+    finally { await fs.rm(directory, { recursive: true, force: true }); }
+  }
+}
+
+async function drainPolicyProcesses(user) {
+  const deadline = Date.now() + 2000;
+  let empty = 0;
+  while (Date.now() < deadline) {
+    const entries = (await fs.readdir("/proc")).filter(value => /^\d+$/.test(value));
+    if (entries.length > 10000) throw new Error("policy_process_inventory_limit");
+    let owned = 0;
+    for (const entry of entries) {
+      let status;
+      try { status = await fs.readFile("/proc/" + entry + "/status", "utf8"); }
+      catch (error) { if (["ENOENT", "ESRCH"].includes(error.code)) continue; throw error; }
+      const ids = /^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/m.exec(status);
+      if (ids && ids.slice(1).some(value => Number(value) === user)) {
+        // Zombies cannot fork, inspect a subsequent invocation or hold files.
+        if (/^State:\s+Z/m.test(status)) continue;
+        owned++;
+        try { process.kill(Number(entry), "SIGKILL"); }
+        catch (error) { if (error.code !== "ESRCH") throw error; }
+      }
+    }
+    empty = owned === 0 ? empty + 1 : 0;
+    if (empty >= 2) return;
+    await new Promise(resolve => setTimeout(resolve, 20));
+  }
+  throw new Error("policy_process_cleanup_unconfirmed");
+}
+`;

package/dist/rewards.js

@@ -4,7 +4,7 @@ import { ImmutableAssetRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema,
 import { gradeEvidence, GraderEvidenceSchema } from "./graders.js";
 import { CustomVerifierGraderSpecSchema, DeterministicGraderSpecSchema, GraderSpecSchema, HumanGraderSpecSchema, ModelJudgeGraderSpecSchema } from "./tasksets.js";
 import { assertBoundedTaskJson, validateTaskSchema } from "./task-schema.js";
-const bindingFields = { id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
+const bindingFields = { taskIds: true, id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
 export const FeedbackKeySchema = z.string().regex(/^[a-z][a-z0-9_]{0,79}$/);
 export function feedbackKeyForReward(reward) {
     return reward.feedbackKey ?? `reward_${contentHash(reward.id).slice(0, 24)}`;
@@ -49,6 +49,7 @@ export const RewardNormalizationSchema = z.discriminatedUnion("kind", [
 ]);
 export const RewardBindingSourceSchema = z.object({
     graderId: ReleaseIdSchema,
+    taskIds: z.array(ReleaseIdSchema).min(1).max(10_000).refine(ids => new Set(ids).size === ids.length, "Task scope must be unique.").optional(),
     reward: RewardReleaseRefSchema,
     role: z.enum(["training", "evaluation"]),
     normalization: RewardNormalizationSchema,
@@ -87,7 +88,7 @@ export const BoundRewardResultSchema = z.object({
     graderId: ReleaseIdSchema,
     reward: RewardReleaseRefSchema,
     role: z.enum(["training", "evaluation"]),
-    status: z.enum(["scored", "pending", "unavailable", "failed"]),
+    status: z.enum(["scored", "pending", "unavailable", "failed", "not_applicable"]),
     rawScore: z.number().finite().nullable(),
     normalizedScore: z.number().min(0).max(1).nullable(),
     passed: z.boolean().nullable(),
@@ -99,6 +100,7 @@ export const RewardCompositionContentSchema = z.object({
     schemaVersion: z.literal("openpond.rewardComposition.v1"),
     binding: RewardReleaseRefSchema,
     taskHash: ReleaseHashSchema,
+    taskId: ReleaseIdSchema.optional(),
     outputHash: ReleaseHashSchema,
     results: z.array(BoundRewardResultSchema).max(100),
     training: z.object({ status: z.enum(["scored", "pending", "unscorable", "not_configured"]), score: z.number().min(0).max(1).nullable(), passed: z.boolean().nullable() }).strict(),
@@ -149,6 +151,10 @@ export async function executeRewardBinding(input) {
     for (const { source, reward } of resolved) {
         input.signal?.throwIfAborted();
         const result = { graderId: source.graderId, reward: source.reward, role: source.role, status: "unavailable", rawScore: null, normalizedScore: null, passed: null, evidenceHashes: [], message: null };
+        if (source.taskIds && !source.taskIds.includes(input.task.id)) {
+            results.push({ ...result, status: "not_applicable" });
+            continue;
+        }
         try {
             if (reward.implementation.kind === "human") {
                 result.status = "pending";
@@ -183,11 +189,13 @@ export async function executeRewardBinding(input) {
         }
         results.push(BoundRewardResultSchema.parse(result));
     }
-    return composeBoundRewards({ binding: input.binding, taskHash: contentHash(input.task), outputHash: contentHash(input.evidence), results });
+    return composeBoundRewards({ binding: input.binding, ...(input.binding.sources.some(source => source.taskIds) ? { taskId: input.task.id } : {}), taskHash: contentHash(input.task), outputHash: contentHash(input.evidence), results });
 }
 export function composeBoundRewards(input) {
     const binding = RewardBindingSchema.parse(input.binding);
     assertRewardHash(binding);
+    if (binding.sources.some(source => source.taskIds) && !input.taskId)
+        throw new LearningDomainError("reward_task_scope_missing");
     const results = input.results.map((result) => BoundRewardResultSchema.parse(result));
     if (results.length !== binding.sources.length || new Set(results.map((result) => result.graderId)).size !== results.length)
         throw new LearningDomainError("reward_result_set_incomplete");
@@ -195,6 +203,9 @@ export function composeBoundRewards(input) {
         const result = results.find((candidate) => candidate.graderId === source.graderId);
         if (!result || contentHash(result.reward) !== contentHash(source.reward) || result.role !== source.role)
             throw new LearningDomainError("reward_result_binding_mismatch");
+        const applicable = !source.taskIds || source.taskIds.includes(input.taskId);
+        if (applicable === (result.status === "not_applicable") || !applicable && (result.graderEvidence || result.evidenceHashes.length || result.message !== null))
+            throw new LearningDomainError("reward_result_task_scope_mismatch");
         if (result.graderEvidence) {
             const { contentHash: evidenceHash, ...body } = result.graderEvidence;
             if (contentHash(body) !== evidenceHash || !result.evidenceHashes.includes(evidenceHash)
@@ -219,6 +230,7 @@ export function composeBoundRewards(input) {
         schemaVersion: "openpond.rewardComposition.v1",
         binding: { id: binding.id, revision: binding.revision, contentHash: binding.contentHash },
         taskHash: input.taskHash,
+        ...(input.taskId ? { taskId: input.taskId } : {}),
         outputHash: input.outputHash,
         results,
         training: aggregate("training"),
@@ -226,7 +238,7 @@ export function composeBoundRewards(input) {
     });
     return RewardCompositionSchema.parse({ ...content, contentHash: contentHash(content) });
     function aggregate(role) {
-        const selected = binding.sources.filter((source) => source.role === role).map((source) => ({ source, result: results.find((candidate) => candidate.graderId === source.graderId) }));
+        const selected = binding.sources.filter((source) => source.role === role && (!source.taskIds || source.taskIds.includes(input.taskId))).map((source) => ({ source, result: results.find((candidate) => candidate.graderId === source.graderId) }));
         if (!selected.length)
             return { status: "not_configured", score: null, passed: null };
         if (selected.some(({ source, result }) => source.required && result.status !== "scored" && result.status !== "pending"))
@@ -250,5 +262,5 @@ function withoutRewardHash(value) { const { contentHash: _hash, ...content } = v
 function compileSource(source, reward) {
     if (reward.implementation.kind === "learned_model")
         throw new LearningDomainError("learned_reward_requires_model_execution_adapter");
-    return GraderSpecSchema.parse({ ...reward.implementation, id: source.graderId, version: String(reward.revision), weight: source.weight, hardGate: source.hardGate, rewardEligible: source.role === "training", privileged: source.privileged });
+    return GraderSpecSchema.parse({ ...reward.implementation, ...(source.taskIds ? { taskIds: source.taskIds } : {}), id: source.graderId, version: String(reward.revision), weight: source.weight, hardGate: source.hardGate, rewardEligible: source.role === "training", privileged: source.privileged });
 }

package/dist/tasksets.js

@@ -7,6 +7,7 @@ export const RequiredOutputContractSchema = z.object({
     path: z.string().trim().min(1).max(2_000).refine(safeRelativePath),
     mediaType: z.string().trim().min(1).max(200),
     schemaRef: ImmutableAssetRefSchema.nullable().default(null),
+    submissionMode: z.literal("structured_json").optional(),
     maxBytes: z.number().int().positive().max(250_000_000).nullable().default(null),
     metadata: MetadataSchema,
 }).strict();
@@ -28,6 +29,7 @@ export const EnvironmentContractSchema = z.object({
 }).strict();
 const GraderBaseSchema = z.object({
     id: ReleaseIdSchema,
+    taskIds: z.array(ReleaseIdSchema).min(1).max(10_000).refine(ids => new Set(ids).size === ids.length, "Grader task IDs must be unique.").optional(),
     version: z.string().trim().min(1).max(100),
     weight: z.number().nonnegative().max(1_000).default(1),
     hardGate: z.boolean().default(false),
@@ -120,6 +122,11 @@ export function validateTasksetRelease(input) {
             path: "environmentRelease",
         });
     }
+    const taskIds = new Set(taskset.tasks.map(task => task.id));
+    for (const grader of taskset.graders)
+        if (grader.taskIds?.some(id => !taskIds.has(id))) {
+            issues.push({ code: "grader_task_missing", severity: "error", message: `Grader ${grader.id} refers to a task outside this release.`, path: "graders" });
+        }
     const clusterSplits = new Map();
     for (const task of taskset.tasks) {
         if (taskset.metrics?.aggregation === "weighted_mean" && !Object.hasOwn(taskset.metrics.taskWeights, task.id)) {

package/dist/types/artifact-verification.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"artifact-verification.d.ts","sourceRoot":"","sources":["../../../../src/artifact-verification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE9D,OAAO,EAGL,KAAK,gBAAgB,EAGrB,KAAK,sBAAsB,EAC5B,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,WAAW,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACpD,YAAY,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,aAAa,GAAG,IAAI,CAAC;IAC7D,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAwEnB;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,QAAQ,EAAE,gBAAgB,CAAC;CAC5B,GAAG,sBAAsB,EAAE,CAqC3B"}
+{"version":3,"file":"artifact-verification.d.ts","sourceRoot":"","sources":["../../src/artifact-verification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE9D,OAAO,EAGL,KAAK,gBAAgB,EAGrB,KAAK,sBAAsB,EAC5B,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,WAAW,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACpD,YAAY,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,aAAa,GAAG,IAAI,CAAC;IAC7D,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAwEnB;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,QAAQ,EAAE,gBAAgB,CAAC;CAC5B,GAAG,sBAAsB,EAAE,CAqC3B"}

package/dist/types/benchmarks.d.ts

@@ -2,10 +2,10 @@ import { z } from "zod";
 import { type AttemptReceipt, type EvaluationResult } from "./runs.js";
 export { harnessRefinerBenchmarkAssets, harnessRefinerBenchmarkRelease, } from "./builtin-benchmarks/harness-refiner.js";
 export declare const BenchmarkMetricSchema: z.ZodEnum<{
+    cost_usd: "cost_usd";
     foreground_tokens: "foreground_tokens";
-    success_rate: "success_rate";
     latency_ms: "latency_ms";
-    cost_usd: "cost_usd";
+    success_rate: "success_rate";
 }>;
 export declare const BenchmarkRunPhaseSchema: z.ZodEnum<{
     baseline: "baseline";
@@ -13,10 +13,10 @@ export declare const BenchmarkRunPhaseSchema: z.ZodEnum<{
 }>;
 export declare const BenchmarkProtocolSchema: z.ZodObject<{
     split: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     taskIds: z.ZodArray<z.ZodString>;
     seeds: z.ZodArray<z.ZodString>;
@@ -36,27 +36,27 @@ export declare const BenchmarkDefinitionSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     adaptationSplit: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     evaluationSplit: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     primaryMetric: z.ZodEnum<{
+        cost_usd: "cost_usd";
         foreground_tokens: "foreground_tokens";
-        success_rate: "success_rate";
         latency_ms: "latency_ms";
-        cost_usd: "cost_usd";
+        success_rate: "success_rate";
     }>;
     qualityGate: z.ZodEnum<{
-        none: "none";
-        non_regression: "non_regression";
         all_pass: "all_pass";
+        non_regression: "non_regression";
+        none: "none";
     }>;
     caseCounts: z.ZodObject<{
         adaptation: z.ZodNumber;
@@ -80,10 +80,10 @@ export declare const BenchmarkRunRequestSchema: z.ZodObject<{
     }, z.core.$strict>;
     reasoningEffort: z.ZodNullable<z.ZodString>;
     split: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     seeds: z.ZodArray<z.ZodString>;
     repetitions: z.ZodNumber;
@@ -119,10 +119,10 @@ export declare const BenchmarkRunSummaryContentSchema: z.ZodObject<{
     reasoningEffort: z.ZodNullable<z.ZodString>;
     protocol: z.ZodObject<{
         split: z.ZodEnum<{
+            frozen_eval: "frozen_eval";
+            test: "test";
             train: "train";
             validation: "validation";
-            test: "test";
-            frozen_eval: "frozen_eval";
         }>;
         taskIds: z.ZodArray<z.ZodString>;
         seeds: z.ZodArray<z.ZodString>;
@@ -175,10 +175,10 @@ export declare const BenchmarkRunSummarySchema: z.ZodObject<{
     reasoningEffort: z.ZodNullable<z.ZodString>;
     protocol: z.ZodObject<{
         split: z.ZodEnum<{
+            frozen_eval: "frozen_eval";
+            test: "test";
             train: "train";
             validation: "validation";
-            test: "test";
-            frozen_eval: "frozen_eval";
         }>;
         taskIds: z.ZodArray<z.ZodString>;
         seeds: z.ZodArray<z.ZodString>;
@@ -218,10 +218,10 @@ export declare const BenchmarkComparisonContentSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     primaryMetric: z.ZodEnum<{
+        cost_usd: "cost_usd";
         foreground_tokens: "foreground_tokens";
-        success_rate: "success_rate";
         latency_ms: "latency_ms";
-        cost_usd: "cost_usd";
+        success_rate: "success_rate";
     }>;
     qualityPassed: z.ZodBoolean;
     baselinePassRate: z.ZodNumber;
@@ -248,10 +248,10 @@ export declare const BenchmarkComparisonSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     primaryMetric: z.ZodEnum<{
+        cost_usd: "cost_usd";
         foreground_tokens: "foreground_tokens";
-        success_rate: "success_rate";
         latency_ms: "latency_ms";
-        cost_usd: "cost_usd";
+        success_rate: "success_rate";
     }>;
     qualityPassed: z.ZodBoolean;
     baselinePassRate: z.ZodNumber;