@openpond/evals 0.11.20 → 0.12.0

package/dist/native-conversations/supervisor.js

@@ -7,6 +7,8 @@ import { homedir } from "node:os";
 import { isAbsolute, join } from "node:path";
 import { contentHash } from "@openpond/harness";
 import { CollectorStore } from "./collector-store.js";
+import { validateCollectorSchedule } from "./collector-schedule.js";
+import { collectorServiceFiles } from "./collector-service-files.js";
 const execute = promisify(execFile);
 const command = async (file, args) => {
     await execute(file, args, {
@@ -15,12 +17,6 @@ const command = async (file, args) => {
         windowsHide: true,
     });
 };
-const xml = (value) => value
-    .replace(/&/gu, "&")
-    .replace(/</gu, "&lt;")
-    .replace(/>/gu, "&gt;")
-    .replace(/"/gu, "&quot;");
-const systemd = (value) => `"${value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"').replace(/%/gu, "%%").replace(/\$/gu, "$$")}"`;
 function identity(directory) {
     return `openpond-import-${contentHash(directory).slice(0, 16)}`;
 }
@@ -70,86 +66,154 @@ export async function controlCollector(directory, action, connectionId) {
         store.close();
     }
 }
-/** Per-user logon supervision. Installing does not override the retained Stop intent. */
+/** Install calendar triggers only. Registration never launches an import. */
 export async function installCollectorService(input) {
-    if (!isAbsolute(input.directory) ||
-        !isAbsolute(input.executable) ||
-        [input.directory, input.executable, ...input.args].some((value) => /[\r\n\0]/u.test(value)))
-        throw new Error("Collector service requires absolute paths and single-line arguments.");
+    // Reloading a launchd calendar also terminates its active process. Preserve
+    // the current import instead of silently cancelling it during a settings edit.
+    if (process.platform === "darwin" && (await collectorStatus(input.directory)).running)
+        throw new Error("Wait for the current import to finish, or cancel it, before changing its macOS schedule.");
+    if (!isAbsolute(input.directory) || !isAbsolute(input.executable) ||
+        [input.directory, input.executable, ...input.args].some(value => /[\r\n\0]/u.test(value)))
+        throw new Error("Collector jobs require absolute paths and single-line arguments.");
     const retainedRuntime = await retainAppImageRuntime(input);
     if (retainedRuntime)
         input = { ...input, ...retainedRuntime, environment: { ...input.environment, ELECTRON_RUN_AS_NODE: "1", APPIMAGE_EXTRACT_AND_RUN: "1" } };
     const environment = Object.entries(input.environment ?? {});
-    if (environment.some(([key, value]) => !["OPENPOND_HOME", "ELECTRON_RUN_AS_NODE", "APPIMAGE_EXTRACT_AND_RUN"].includes(key) ||
-        typeof value !== "string" ||
-        /[\r\n\0]/u.test(value)))
+    if (environment.some(([key, value]) => !["OPENPOND_HOME", "ELECTRON_RUN_AS_NODE", "APPIMAGE_EXTRACT_AND_RUN"].includes(key) || typeof value !== "string" || /[\r\n\0]/u.test(value)))
         throw new Error("Unsupported service environment setting.");
     const home = input.home ?? homedir(), name = identity(input.directory);
-    const launch = [
-        ...input.args,
-        "import",
-        "service",
-        "run",
-        "--collector-dir",
-        input.directory,
-    ];
+    const store = await CollectorStore.open(input.directory);
+    let schedules;
+    try {
+        schedules = store.connections().map(connection => store.schedule(connection.id)).filter((schedule) => schedule !== null);
+    }
+    finally {
+        store.close();
+    }
+    const gatePath = join(input.directory, "calendar-gate.cjs");
+    const files = collectorServiceFiles({ name, gatePath, executable: input.executable,
+        args: [...input.args, "import", "service", "run", "--scheduled", "--collector-dir", input.directory], environment, schedules });
+    await atomicServiceFile(gatePath, files.gate, { mode: 0o600 });
     if (process.platform === "linux") {
         const directory = join(home, ".config/systemd/user");
         await mkdir(directory, { recursive: true, mode: 0o700 });
-        await atomicServiceFile(join(directory, `${name}.service`), `[Unit]\nDescription=OpenPond conversation importer\nAfter=network-online.target\n[Service]\nType=simple\n${environment.map(([key, value]) => `Environment=${systemd(`${key}=${value}`)}`).join("\n")}\nExecStart=${[input.executable, ...launch].map(systemd).join(" ")}\nRestart=on-failure\nRestartSec=5\nUMask=0077\nNoNewPrivileges=true\n[Install]\nWantedBy=default.target\n`, { mode: 0o600 });
-        await command("systemctl", ["--user", "daemon-reload"]);
-        await command("systemctl", ["--user", "enable", `${name}.service`]);
+        // Remove the old login enablement without stopping an already owned job.
+        await optionalCommand("systemctl", ["--user", "disable", `${name}.service`]);
+        await atomicServiceFile(join(directory, `${name}.service`), files.service, { mode: 0o600 });
+        if (schedules.length) {
+            await atomicServiceFile(join(directory, `${name}.timer`), files.timer, { mode: 0o600 });
+            await command("systemctl", ["--user", "daemon-reload"]);
+            await command("systemctl", ["--user", "enable", "--now", `${name}.timer`]);
+            await command("systemctl", ["--user", "restart", `${name}.timer`]);
+        }
+        else {
+            await optionalCommand("systemctl", ["--user", "disable", "--now", `${name}.timer`]);
+            await rm(join(directory, `${name}.timer`), { force: true });
+            await command("systemctl", ["--user", "daemon-reload"]);
+        }
     }
     else if (process.platform === "darwin") {
-        const directory = join(home, "Library/LaunchAgents");
+        const directory = join(home, "Library/LaunchAgents"), file = join(directory, `${name}.plist`);
         await mkdir(directory, { recursive: true, mode: 0o700 });
-        const file = join(directory, `${name}.plist`);
-        let loaded = false;
+        await optionalCommand("launchctl", ["bootout", `gui/${process.getuid()}/${name}`]);
+        if (schedules.length) {
+            await atomicServiceFile(file, files.plist, { mode: 0o600 });
+            await command("launchctl", ["bootstrap", `gui/${process.getuid()}`, file]);
+        }
+        else
+            await rm(file, { force: true });
+    }
+    else if (process.platform === "win32") {
+        if (!schedules.length)
+            await optionalCommand("schtasks", ["/Delete", "/TN", name, "/F"]);
+        else {
+            const user = (await execute("whoami", [], { timeout: 5000, maxBuffer: 1024 })).stdout.trim();
+            const file = join(input.directory, "scheduled-task.xml"), script = join(input.directory, "start-collector.ps1");
+            await mkdir(input.directory, { recursive: true, mode: 0o700 });
+            await atomicServiceFile(script, files.powershell, { mode: 0o600 });
+            await atomicServiceFile(file, files.windowsTask(user, script), { mode: 0o600 });
+            await command("schtasks", ["/Create", "/TN", name, "/XML", file, "/F"]);
+        }
+    }
+    else
+        throw new Error("Scheduled imports need a supported OS scheduler. Use import sync manually on this OS.");
+    return { name, platform: process.platform, status: await collectorStatus(input.directory) };
+}
+// Absence is allowed only when removing an optional trigger. Installation and
+// activation failures remain errors and never claim that a schedule is enabled.
+async function optionalCommand(file, args) {
+    try {
+        await command(file, args);
+    }
+    catch (error) {
+        const result = error;
+        if (result.code === "ENOENT" || !/not (?:loaded|found|exist)|does not exist|No such|could not be found|cannot find|not been loaded/iu.test(result.stderr ?? result.message))
+            throw error;
+    }
+}
+/** Serialize OS/config updates; a failed installation restores retained intent. */
+export async function setCollectorSchedule(input, connectionId, schedule) {
+    if (schedule !== null)
+        schedule = validateCollectorSchedule(schedule);
+    if (process.platform === "darwin" && (await collectorStatus(input.directory)).running)
+        throw new Error("Wait for the current import to finish, or cancel it, before changing its macOS schedule.");
+    const store = await CollectorStore.open(input.directory), token = randomUUID();
+    let prior;
+    store.database.exec("BEGIN IMMEDIATE");
+    try {
+        const connection = store.connections().find(item => item.id === connectionId);
+        if (!connection || (schedule && connection.state === "disconnected"))
+            throw new Error("Select a connected source before scheduling imports.");
+        const owner = store.setting("scheduleOwner");
+        if (owner) {
+            const { pid } = JSON.parse(owner);
+            let alive = true;
+            try {
+                process.kill(pid, 0);
+            }
+            catch (error) {
+                alive = error.code !== "ESRCH";
+            }
+            if (alive)
+                throw new Error("Another schedule update is in progress. Try again shortly.");
+        }
+        prior = store.setting(`schedule:${connectionId}`);
+        store.set("scheduleOwner", JSON.stringify({ pid: process.pid, token }));
+        store.set(`schedule:${connectionId}`, schedule ? JSON.stringify(schedule) : "");
+        store.database.exec("COMMIT");
+    }
+    catch (error) {
+        store.database.exec("ROLLBACK");
+        store.close();
+        throw error;
+    }
+    try {
+        await installCollectorService(input);
+        return store.status();
+    }
+    catch (error) {
+        store.set(`schedule:${connectionId}`, prior ?? "");
         try {
-            await command("launchctl", ["print", `gui/${process.getuid()}/${name}`]);
-            loaded = true;
+            await installCollectorService(input);
         }
         catch {
-            /* A first installation has no loaded job. Bootstrap below reports actual setup errors. */
+            throw new Error(`Schedule installation failed and OS rollback could not be verified. Review the OS scheduler. ${error instanceof Error ? error.message : ""}`);
         }
-        if (loaded)
-            await command("launchctl", [
-                "bootout",
-                `gui/${process.getuid()}/${name}`,
-            ]);
-        await writeFile(file, `<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"><plist version="1.0"><dict><key>Label</key><string>${name}</string><key>ProgramArguments</key><array>${[input.executable, ...launch].map((value) => `<string>${xml(value)}</string>`).join("")}</array><key>EnvironmentVariables</key><dict>${environment.map(([key, value]) => `<key>${xml(key)}</key><string>${xml(value)}</string>`).join("")}</dict><key>RunAtLoad</key><true/><key>KeepAlive</key><dict><key>SuccessfulExit</key><false/></dict><key>ThrottleInterval</key><integer>5</integer></dict></plist>`, { mode: 0o600 });
-        await command("launchctl", ["bootstrap", `gui/${process.getuid()}`, file]);
+        throw error;
     }
-    else if (process.platform === "win32") {
-        const user = (await execute("whoami", [], { timeout: 5000, maxBuffer: 1024 })).stdout.trim();
-        const file = join(input.directory, "scheduled-task.xml");
-        await mkdir(input.directory, { recursive: true });
-        const quoted = (value) => `"${value.replace(/(\\*)"/gu, '$1$1\\"').replace(/(\\+)$/u, "$1$1")}"`;
-        const script = join(input.directory, "start-collector.ps1");
-        const ps = (value) => `'${value.replace(/'/gu, "''")}'`;
-        await writeFile(script, `${environment.map(([key, value]) => `$env:${key} = ${ps(value)}`).join("\n")}\n& ${[input.executable, ...launch].map(ps).join(" ")}\nexit $LASTEXITCODE\n`, { mode: 0o600 });
-        await writeFile(file, `<?xml version="1.0"?><Task version="1.4" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task"><Triggers><LogonTrigger><Enabled>true</Enabled><UserId>${xml(user)}</UserId></LogonTrigger></Triggers><Principals><Principal id="Owner"><UserId>${xml(user)}</UserId><LogonType>InteractiveToken</LogonType><RunLevel>LeastPrivilege</RunLevel></Principal></Principals><Settings><MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy><DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries><StopIfGoingOnBatteries>false</StopIfGoingOnBatteries><ExecutionTimeLimit>PT0S</ExecutionTimeLimit><RestartOnFailure><Interval>PT1M</Interval><Count>999</Count></RestartOnFailure></Settings><Actions Context="Owner"><Exec><Command>powershell.exe</Command><Arguments>${xml(["-NoProfile", "-NonInteractive", "-File", script].map(quoted).join(" "))}</Arguments></Exec></Actions></Task>`, { mode: 0o600 });
-        await command("schtasks", ["/Create", "/TN", name, "/XML", file, "/F"]);
+    finally {
+        store.set("scheduleOwner", "");
+        store.close();
     }
-    else
-        throw new Error("This OS has no qualified per-user supervisor. Run the collector in the foreground.");
-    return {
-        name,
-        platform: process.platform,
-        status: await collectorStatus(input.directory),
-    };
 }
 /** Removing supervision preserves receipts, pending uploads and retained datasets. */
 export async function uninstallCollectorService(directory, home = homedir()) {
     await controlCollector(directory, "stop");
     const name = identity(directory);
     if (process.platform === "linux") {
-        await command("systemctl", [
-            "--user",
-            "disable",
-            "--now",
-            `${name}.service`,
-        ]);
+        await optionalCommand("systemctl", ["--user", "disable", "--now", `${name}.timer`]);
+        await rm(join(home, ".config/systemd/user", `${name}.timer`), { force: true });
+        await command("systemctl", ["--user", "disable", "--now", `${name}.service`]);
         await rm(join(home, ".config/systemd/user", `${name}.service`), {
             force: true,
         });
@@ -169,30 +233,16 @@ export async function uninstallCollectorService(directory, home = homedir()) {
     }
     else
         throw new Error("No per-user supervisor is available.");
+    await rm(join(directory, "calendar-gate.cjs"), { force: true });
     if (process.platform === "linux")
         await removeAppImageRuntimes(directory);
-    return collectorStatus(directory);
-}
-export async function startCollectorService(directory) {
-    await controlCollector(directory, "start");
-    const name = identity(directory);
+    const store = await CollectorStore.open(directory);
     try {
-        if (process.platform === "linux")
-            await command("systemctl", ["--user", "restart", `${name}.service`]);
-        else if (process.platform === "darwin")
-            await command("launchctl", [
-                "kickstart",
-                "-k",
-                `gui/${process.getuid()}/${name}`,
-            ]);
-        else if (process.platform === "win32")
-            await command("schtasks", ["/Run", "/TN", name]);
-        else
-            throw new Error("No per-user supervisor is available.");
+        for (const connection of store.connections())
+            store.set(`schedule:${connection.id}`, "");
     }
-    catch (error) {
-        await controlCollector(directory, "stop");
-        throw error;
+    finally {
+        store.close();
     }
     return collectorStatus(directory);
 }

package/dist/prepared-process-isolation.js

@@ -0,0 +1,131 @@
+/** Staged by the private verifier owner. Untrusted policy programs run with no
+ * capabilities as a different UID, outside the sealed private verifier tree.
+ * The parent must assess returned behavior; an exit code is never correctness. */
+export const PREPARED_PROCESS_ISOLATION_SOURCE = String.raw `
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { randomInt } from "node:crypto";
+
+const MAX_BYTES = 15 * 1024 * 1024, MAX_OUTPUT = 1024 * 1024;
+const usedUsers = new Set();
+const safePath = value => typeof value === "string" && value.length > 0 && value.length <= 2000
+  && !value.startsWith("/") && !/[\\\x00-\x1f\x7f]/.test(value)
+  && value.split("/").every(part => part && part !== "." && part !== ".." && !part.startsWith(".env"));
+
+/** files contain only public original/policy bytes and a public invocation
+ * adapter. Never stage independent assertions, expected answers or checkers. */
+export async function runPolicyProgram(input) {
+  const privateRoot = path.dirname(fileURLToPath(import.meta.url));
+  if (process.getuid?.() !== 0 || !path.basename(privateRoot).startsWith(".openpond-reward-verifier-"))
+    throw new Error("policy_process_isolation_runtime_unavailable");
+  if (!Array.isArray(input.files) || input.files.length < 1 || input.files.length > 100
+    || !safePath(input.entrypoint) || !input.files.some(file => file.path === input.entrypoint)
+    || !Number.isSafeInteger(input.timeoutMs) || input.timeoutMs < 1 || input.timeoutMs > 60000
+    || !Array.isArray(input.args ?? []) || (input.args ?? []).length > 100
+    || (input.args ?? []).some(arg => typeof arg !== "string" || arg.length > 2000 || arg.includes("\0")))
+    throw new Error("policy_process_isolation_input_invalid");
+  let total = 0;
+  const paths = new Set();
+  const files = input.files.map(file => {
+    if (!safePath(file.path) || paths.has(file.path) || typeof file.base64 !== "string")
+      throw new Error("policy_process_isolation_file_invalid");
+    paths.add(file.path);
+    const bytes = Buffer.from(file.base64, "base64");
+    if (bytes.toString("base64") !== file.base64) throw new Error("policy_process_isolation_file_invalid");
+    total += bytes.length;
+    if (total > MAX_BYTES) throw new Error("policy_process_isolation_file_limit");
+    return { path: file.path, bytes };
+  });
+  // Directory access seals request JSON, checker modules, helper source and
+  // private artifacts before any untrusted program can be spawned.
+  await fs.chmod(privateRoot, 0o700);
+  const stat = await fs.stat(privateRoot);
+  if (stat.uid !== 0 || (stat.mode & 0o777) !== 0o700) throw new Error("policy_process_private_tree_not_sealed");
+  for (const executable of ["/usr/bin/setsid", "/usr/bin/setpriv", "/usr/local/bin/bun"])
+    await fs.access(executable, 1);
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "openpond-policy-"));
+  let user;
+  do { user = randomInt(1000000, 2000000); } while (usedUsers.has(user));
+  usedUsers.add(user);
+  let child, timer, timedOut = false;
+  const kill = () => {
+    if (!child) return;
+    try { process.kill(-child.pid, "SIGKILL"); } catch {}
+    try { child.kill("SIGKILL"); } catch {}
+  };
+  try {
+    await fs.chmod(directory, 0o700);
+    await fs.chown(directory, user, user);
+    for (const file of files) {
+      const target = path.join(directory, file.path);
+      const parent = path.dirname(target);
+      await fs.mkdir(parent, { recursive: true, mode: 0o700 });
+      let current = parent;
+      while (current !== directory) {
+        await fs.chmod(current, 0o700); await fs.chown(current, user, user);
+        current = path.dirname(current);
+      }
+      await fs.writeFile(target, file.bytes, { mode: 0o600, flag: "wx" });
+      await fs.chown(target, user, user);
+    }
+    child = Bun.spawn(["/usr/bin/setsid", "/usr/bin/setpriv", "--reuid=" + user, "--regid=" + user, "--clear-groups",
+      "--no-new-privs", "--bounding-set=-all", "--inh-caps=-all", "--ambient-caps=-all",
+      "/usr/local/bin/bun", path.join(directory, input.entrypoint), ...(input.args ?? [])], {
+      cwd: directory, env: { PATH: "/usr/local/bin:/usr/bin:/bin", TMPDIR: directory }, stdout: "pipe", stderr: "pipe", stdin: "ignore",
+    });
+    timer = setTimeout(() => { timedOut = true; kill(); }, input.timeoutMs);
+    const read = async stream => {
+      const reader = stream.getReader(), chunks = [];
+      let bytes = 0;
+      try {
+        while (true) {
+          const item = await reader.read(); if (item.done) break;
+          bytes += item.value.byteLength;
+          if (bytes > MAX_OUTPUT) { kill(); throw new Error("policy_process_output_limit"); }
+          chunks.push(item.value);
+        }
+        return new TextDecoder("utf8", { fatal: true }).decode(Buffer.concat(chunks));
+      } finally { await reader.cancel(); reader.releaseLock(); }
+    };
+    const [stdout, stderr, exitCode] = await Promise.all([read(child.stdout), read(child.stderr), child.exited]);
+    return { stdout, stderr, exitCode, timedOut, isolation: { uid: user, gid: user, privateTreeSealed: true } };
+  } finally {
+    if (timer) clearTimeout(timer);
+    kill();
+    // A policy child can call setsid and escape its initial process group.
+    // Drain every process with this invocation's unique real UID before the
+    // next call; namespaces cannot grant a different outside UID or privilege.
+    try { await drainPolicyProcesses(user); }
+    finally { await fs.rm(directory, { recursive: true, force: true }); }
+  }
+}
+
+async function drainPolicyProcesses(user) {
+  const deadline = Date.now() + 2000;
+  let empty = 0;
+  while (Date.now() < deadline) {
+    const entries = (await fs.readdir("/proc")).filter(value => /^\d+$/.test(value));
+    if (entries.length > 10000) throw new Error("policy_process_inventory_limit");
+    let owned = 0;
+    for (const entry of entries) {
+      let status;
+      try { status = await fs.readFile("/proc/" + entry + "/status", "utf8"); }
+      catch (error) { if (["ENOENT", "ESRCH"].includes(error.code)) continue; throw error; }
+      const ids = /^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/m.exec(status);
+      if (ids && ids.slice(1).some(value => Number(value) === user)) {
+        // Zombies cannot fork, inspect a subsequent invocation or hold files.
+        if (/^State:\s+Z/m.test(status)) continue;
+        owned++;
+        try { process.kill(Number(entry), "SIGKILL"); }
+        catch (error) { if (error.code !== "ESRCH") throw error; }
+      }
+    }
+    empty = owned === 0 ? empty + 1 : 0;
+    if (empty >= 2) return;
+    await new Promise(resolve => setTimeout(resolve, 20));
+  }
+  throw new Error("policy_process_cleanup_unconfirmed");
+}
+`;

package/dist/rewards.js

@@ -4,7 +4,7 @@ import { ImmutableAssetRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema,
 import { gradeEvidence, GraderEvidenceSchema } from "./graders.js";
 import { CustomVerifierGraderSpecSchema, DeterministicGraderSpecSchema, GraderSpecSchema, HumanGraderSpecSchema, ModelJudgeGraderSpecSchema } from "./tasksets.js";
 import { assertBoundedTaskJson, validateTaskSchema } from "./task-schema.js";
-const bindingFields = { id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
+const bindingFields = { taskIds: true, id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
 export const FeedbackKeySchema = z.string().regex(/^[a-z][a-z0-9_]{0,79}$/);
 export function feedbackKeyForReward(reward) {
     return reward.feedbackKey ?? `reward_${contentHash(reward.id).slice(0, 24)}`;
@@ -49,6 +49,7 @@ export const RewardNormalizationSchema = z.discriminatedUnion("kind", [
 ]);
 export const RewardBindingSourceSchema = z.object({
     graderId: ReleaseIdSchema,
+    taskIds: z.array(ReleaseIdSchema).min(1).max(10_000).refine(ids => new Set(ids).size === ids.length, "Task scope must be unique.").optional(),
     reward: RewardReleaseRefSchema,
     role: z.enum(["training", "evaluation"]),
     normalization: RewardNormalizationSchema,
@@ -87,7 +88,7 @@ export const BoundRewardResultSchema = z.object({
     graderId: ReleaseIdSchema,
     reward: RewardReleaseRefSchema,
     role: z.enum(["training", "evaluation"]),
-    status: z.enum(["scored", "pending", "unavailable", "failed"]),
+    status: z.enum(["scored", "pending", "unavailable", "failed", "not_applicable"]),
     rawScore: z.number().finite().nullable(),
     normalizedScore: z.number().min(0).max(1).nullable(),
     passed: z.boolean().nullable(),
@@ -99,6 +100,7 @@ export const RewardCompositionContentSchema = z.object({
     schemaVersion: z.literal("openpond.rewardComposition.v1"),
     binding: RewardReleaseRefSchema,
     taskHash: ReleaseHashSchema,
+    taskId: ReleaseIdSchema.optional(),
     outputHash: ReleaseHashSchema,
     results: z.array(BoundRewardResultSchema).max(100),
     training: z.object({ status: z.enum(["scored", "pending", "unscorable", "not_configured"]), score: z.number().min(0).max(1).nullable(), passed: z.boolean().nullable() }).strict(),
@@ -149,6 +151,10 @@ export async function executeRewardBinding(input) {
     for (const { source, reward } of resolved) {
         input.signal?.throwIfAborted();
         const result = { graderId: source.graderId, reward: source.reward, role: source.role, status: "unavailable", rawScore: null, normalizedScore: null, passed: null, evidenceHashes: [], message: null };
+        if (source.taskIds && !source.taskIds.includes(input.task.id)) {
+            results.push({ ...result, status: "not_applicable" });
+            continue;
+        }
         try {
             if (reward.implementation.kind === "human") {
                 result.status = "pending";
@@ -183,11 +189,13 @@ export async function executeRewardBinding(input) {
         }
         results.push(BoundRewardResultSchema.parse(result));
     }
-    return composeBoundRewards({ binding: input.binding, taskHash: contentHash(input.task), outputHash: contentHash(input.evidence), results });
+    return composeBoundRewards({ binding: input.binding, ...(input.binding.sources.some(source => source.taskIds) ? { taskId: input.task.id } : {}), taskHash: contentHash(input.task), outputHash: contentHash(input.evidence), results });
 }
 export function composeBoundRewards(input) {
     const binding = RewardBindingSchema.parse(input.binding);
     assertRewardHash(binding);
+    if (binding.sources.some(source => source.taskIds) && !input.taskId)
+        throw new LearningDomainError("reward_task_scope_missing");
     const results = input.results.map((result) => BoundRewardResultSchema.parse(result));
     if (results.length !== binding.sources.length || new Set(results.map((result) => result.graderId)).size !== results.length)
         throw new LearningDomainError("reward_result_set_incomplete");
@@ -195,6 +203,9 @@ export function composeBoundRewards(input) {
         const result = results.find((candidate) => candidate.graderId === source.graderId);
         if (!result || contentHash(result.reward) !== contentHash(source.reward) || result.role !== source.role)
             throw new LearningDomainError("reward_result_binding_mismatch");
+        const applicable = !source.taskIds || source.taskIds.includes(input.taskId);
+        if (applicable === (result.status === "not_applicable") || !applicable && (result.graderEvidence || result.evidenceHashes.length || result.message !== null))
+            throw new LearningDomainError("reward_result_task_scope_mismatch");
         if (result.graderEvidence) {
             const { contentHash: evidenceHash, ...body } = result.graderEvidence;
             if (contentHash(body) !== evidenceHash || !result.evidenceHashes.includes(evidenceHash)
@@ -219,6 +230,7 @@ export function composeBoundRewards(input) {
         schemaVersion: "openpond.rewardComposition.v1",
         binding: { id: binding.id, revision: binding.revision, contentHash: binding.contentHash },
         taskHash: input.taskHash,
+        ...(input.taskId ? { taskId: input.taskId } : {}),
         outputHash: input.outputHash,
         results,
         training: aggregate("training"),
@@ -226,7 +238,7 @@ export function composeBoundRewards(input) {
     });
     return RewardCompositionSchema.parse({ ...content, contentHash: contentHash(content) });
     function aggregate(role) {
-        const selected = binding.sources.filter((source) => source.role === role).map((source) => ({ source, result: results.find((candidate) => candidate.graderId === source.graderId) }));
+        const selected = binding.sources.filter((source) => source.role === role && (!source.taskIds || source.taskIds.includes(input.taskId))).map((source) => ({ source, result: results.find((candidate) => candidate.graderId === source.graderId) }));
         if (!selected.length)
             return { status: "not_configured", score: null, passed: null };
         if (selected.some(({ source, result }) => source.required && result.status !== "scored" && result.status !== "pending"))
@@ -250,5 +262,5 @@ function withoutRewardHash(value) { const { contentHash: _hash, ...content } = v
 function compileSource(source, reward) {
     if (reward.implementation.kind === "learned_model")
         throw new LearningDomainError("learned_reward_requires_model_execution_adapter");
-    return GraderSpecSchema.parse({ ...reward.implementation, id: source.graderId, version: String(reward.revision), weight: source.weight, hardGate: source.hardGate, rewardEligible: source.role === "training", privileged: source.privileged });
+    return GraderSpecSchema.parse({ ...reward.implementation, ...(source.taskIds ? { taskIds: source.taskIds } : {}), id: source.graderId, version: String(reward.revision), weight: source.weight, hardGate: source.hardGate, rewardEligible: source.role === "training", privileged: source.privileged });
 }

package/dist/sql-execution-node.js

@@ -1,5 +1,4 @@
 import { spawn } from "node:child_process";
-import { sqlExecutionProcessSource } from "./sql-execution-process-source.js";
 import { assertBoundedTaskJson } from "./task-schema.js";
 import { assertSqlExecutionRequest, SqlExecutionResultSchema } from "./sql-execution-contract.js";
 export * from "./sql-execution-contract.js";
@@ -9,6 +8,8 @@ export async function executeSqlInProcess(input) {
     if (!Number.isSafeInteger(input.timeoutMs) || input.timeoutMs < 1 || input.timeoutMs > 30_000)
         throw new Error("sql_timeout_invalid");
     const request = assertSqlExecutionRequest(input.request);
+    const { sqlExecutionProcessSource } = await import("./sql-execution-process-source.js");
+    input.signal?.throwIfAborted();
     const { signal } = input;
     const program = `globalThis.__openpondSqlInput = JSON.parse(${JSON.stringify(JSON.stringify(request))});\n${sqlExecutionProcessSource}`;
     return new Promise((resolve, reject) => {

package/dist/tasksets.js

@@ -28,6 +28,7 @@ export const EnvironmentContractSchema = z.object({
 }).strict();
 const GraderBaseSchema = z.object({
     id: ReleaseIdSchema,
+    taskIds: z.array(ReleaseIdSchema).min(1).max(10_000).refine(ids => new Set(ids).size === ids.length, "Grader task IDs must be unique.").optional(),
     version: z.string().trim().min(1).max(100),
     weight: z.number().nonnegative().max(1_000).default(1),
     hardGate: z.boolean().default(false),
@@ -89,7 +90,7 @@ export const TasksetReleaseContentSchema = z.object({
     tools: z.array(ToolDeclarationSchema).max(200),
     capabilities: z.array(CapabilityRequirementSchema).max(200),
     tasks: z.array(TaskRecordSchema).min(1).max(1_000_000),
-    graders: z.array(GraderSpecSchema).min(1).max(1_000),
+    graders: z.array(GraderSpecSchema).max(1_000),
     verifierSetRelease: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().optional(),
     metrics: TasksetMetricPolicySchema.optional(),
     metadata: MetadataSchema,
@@ -120,6 +121,11 @@ export function validateTasksetRelease(input) {
             path: "environmentRelease",
         });
     }
+    const taskIds = new Set(taskset.tasks.map(task => task.id));
+    for (const grader of taskset.graders)
+        if (grader.taskIds?.some(id => !taskIds.has(id))) {
+            issues.push({ code: "grader_task_missing", severity: "error", message: `Grader ${grader.id} refers to a task outside this release.`, path: "graders" });
+        }
     const clusterSplits = new Map();
     for (const task of taskset.tasks) {
         if (taskset.metrics?.aggregation === "weighted_mean" && !Object.hasOwn(taskset.metrics.taskWeights, task.id)) {

package/dist/types/builtin-benchmarks/harness-refiner.d.ts

@@ -73,6 +73,7 @@ export declare const harnessRefinerBenchmarkRelease: {
         privileged: boolean;
         kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
         config: Record<string, unknown>;
+        taskIds?: string[] | undefined;
     } | {
         id: string;
         version: string;
@@ -90,6 +91,7 @@ export declare const harnessRefinerBenchmarkRelease: {
             visibility: "policy" | "verifier" | "host_private";
         };
         calibrationStatus: "pending" | "passed" | "failed";
+        taskIds?: string[] | undefined;
         model?: {
             providerId: string;
             modelId: string;
@@ -114,6 +116,7 @@ export declare const harnessRefinerBenchmarkRelease: {
         };
         timeoutMs: number;
         networkPolicy: "none";
+        taskIds?: string[] | undefined;
         runtime?: "isolated_javascript" | "sandbox_process" | undefined;
         exportName?: string | undefined;
     } | {
@@ -133,6 +136,7 @@ export declare const harnessRefinerBenchmarkRelease: {
             visibility: "policy" | "verifier" | "host_private";
         };
         reviewerRole: string;
+        taskIds?: string[] | undefined;
         form?: {
             schemaVersion: "openpond.humanForm.v1";
             mode: "individual" | "pairwise";

package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
+{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}

package/dist/types/conformance.d.ts

@@ -183,6 +183,7 @@ export declare const genericToolConformance: {
             privileged: boolean;
             kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
             config: Record<string, unknown>;
+            taskIds?: string[] | undefined;
         } | {
             id: string;
             version: string;
@@ -200,6 +201,7 @@ export declare const genericToolConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             };
             calibrationStatus: "pending" | "passed" | "failed";
+            taskIds?: string[] | undefined;
             model?: {
                 providerId: string;
                 modelId: string;
@@ -224,6 +226,7 @@ export declare const genericToolConformance: {
             };
             timeoutMs: number;
             networkPolicy: "none";
+            taskIds?: string[] | undefined;
             runtime?: "isolated_javascript" | "sandbox_process" | undefined;
             exportName?: string | undefined;
         } | {
@@ -243,6 +246,7 @@ export declare const genericToolConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             };
             reviewerRole: string;
+            taskIds?: string[] | undefined;
             form?: {
                 schemaVersion: "openpond.humanForm.v1";
                 mode: "individual" | "pairwise";
@@ -568,6 +572,7 @@ export declare const marketingPortfolioConformance: {
             privileged: boolean;
             kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
             config: Record<string, unknown>;
+            taskIds?: string[] | undefined;
         } | {
             id: string;
             version: string;
@@ -585,6 +590,7 @@ export declare const marketingPortfolioConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             };
             calibrationStatus: "pending" | "passed" | "failed";
+            taskIds?: string[] | undefined;
             model?: {
                 providerId: string;
                 modelId: string;
@@ -609,6 +615,7 @@ export declare const marketingPortfolioConformance: {
             };
             timeoutMs: number;
             networkPolicy: "none";
+            taskIds?: string[] | undefined;
             runtime?: "isolated_javascript" | "sandbox_process" | undefined;
             exportName?: string | undefined;
         } | {
@@ -628,6 +635,7 @@ export declare const marketingPortfolioConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             };
             reviewerRole: string;
+            taskIds?: string[] | undefined;
             form?: {
                 schemaVersion: "openpond.humanForm.v1";
                 mode: "individual" | "pairwise";