@ic-reactor/mcp 0.0.0-stage → 0.0.0

package/evals/harness/fake-ledger.ts

@@ -0,0 +1,178 @@
+/**
+ * A stateful ICRC-1 ledger for the hidden acceptance tests, run inside the
+ * signing fake replica that `@ic-reactor/core/testing` ships (reused as
+ * published in 3.13.0, not re-implemented here).
+ *
+ * The interface is `harness/icrc1.did` (the ICRC-1 reference ledger interface,
+ * copied from `examples/tanstack-router/icrc1.did`), encoded and decoded with
+ * the `@icp-sdk/bindgen` declarations in `harness/declarations/`. Nothing here
+ * is condition-specific: every condition's solution talks to this ledger over
+ * the IC HTTP interface, exactly as it would talk to a replica.
+ */
+import { createTestCanister, type FakeCanister } from "@ic-reactor/core/testing"
+import { Principal } from "@icp-sdk/core/principal"
+import {
+  idlFactory,
+  type Account,
+  type _SERVICE,
+} from "./declarations/icrc1.did.js"
+
+export const LEDGER_FEE = 10_000n
+/** The minting account's owner: a principal that comes back in a reply. */
+export const MINTER = "rrkah-fqaaa-aaaaa-aaaaq-cai"
+export const LEDGER_DECIMALS = 8
+
+/**
+ * The fake replica's own reject class. `@ic-reactor/core/testing` does not
+ * export it, but a test canister throws it for a method it has no handler for,
+ * and the fake replica answers an instance of exactly this class with the
+ * reject code it carries (anything else it throws is a trap, code 5). Taking
+ * the constructor from that error is how a handler can reject with codes 1-4.
+ */
+type RejectCtor = new (
+  code: number,
+  errorCode: string,
+  message: string
+) => Error
+const FakeReplicaReject: RejectCtor = await (async () => {
+  const probe = createTestCanister<_SERVICE>(idlFactory, {})
+  try {
+    await probe.update!("icrc1_transfer", new Uint8Array(), {
+      caller: Principal.anonymous(),
+    })
+  } catch (error) {
+    return (error as Error).constructor as RejectCtor
+  }
+  throw new Error(
+    "fake-ledger: could not obtain the fake replica's reject class"
+  )
+})()
+
+/** What each injected reject looks like, by IC reject code. */
+const REJECTS: Record<number, { errorCode: string; message: string }> = {
+  1: { errorCode: "IC0102", message: "fatal system error (injected)" },
+  2: { errorCode: "IC0207", message: "the canister is frozen (injected)" },
+  3: { errorCode: "IC0301", message: "no such destination (injected)" },
+  4: {
+    errorCode: "IC0406",
+    message: "the ledger rejected the call (injected)",
+  },
+}
+
+/** One transfer the ledger executed. */
+export interface ExecutedTransfer {
+  readonly from: string
+  readonly to: string
+  readonly amount: bigint
+  readonly blockIndex: bigint
+}
+
+/** One icrc1_transfer call that reached the ledger, whatever came of it. */
+export interface TransferAttempt {
+  readonly caller: string
+  readonly to: string
+  readonly amount: bigint
+}
+
+export interface FakeLedger {
+  readonly canister: FakeCanister
+  readonly transfers: readonly ExecutedTransfer[]
+  readonly attempts: readonly TransferAttempt[]
+  setBalance(owner: string, amount: bigint): void
+  balanceOf(owner: string): bigint
+  /**
+   * Reject every icrc1_transfer with this IC reject code: 1-4 through the
+   * fake replica's reject class, 5 as a trap in the ledger's own code.
+   * `null`: transfers run normally.
+   */
+  rejectTransfersWith: 1 | 2 | 3 | 4 | 5 | null
+  /** Milliseconds icrc1_balance_of takes to answer for this owner. */
+  balanceDelayMs: (owner: string) => number
+}
+
+const accountKey = (owner: Principal, sub: [] | [Uint8Array]): string => {
+  const bytes = sub[0]
+  const text = owner.toText()
+  if (!bytes || bytes.every((byte) => byte === 0)) return text
+  return `${text}:${Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("")}`
+}
+const keyOf = (account: Account) =>
+  accountKey(account.owner, account.subaccount)
+
+const delay = (ms: number) =>
+  ms > 0 ? new Promise((resolve) => setTimeout(resolve, ms)) : undefined
+
+export function createFakeLedger(): FakeLedger {
+  const balances = new Map<string, bigint>()
+  const transfers: ExecutedTransfer[] = []
+  const attempts: TransferAttempt[] = []
+  let nextBlock = 1000n
+
+  const ledger: FakeLedger = {
+    canister: undefined as unknown as FakeCanister,
+    transfers,
+    attempts,
+    rejectTransfersWith: null,
+    balanceDelayMs: () => 0,
+    setBalance(owner, amount) {
+      balances.set(owner, amount)
+    },
+    balanceOf(owner) {
+      return balances.get(owner) ?? 0n
+    },
+  }
+
+  ;(ledger as { canister: FakeCanister }).canister =
+    createTestCanister<_SERVICE>(idlFactory, {
+      icrc1_name: () => "Eval Token",
+      icrc1_symbol: () => "EVT",
+      icrc1_decimals: () => LEDGER_DECIMALS,
+      icrc1_fee: () => LEDGER_FEE,
+      icrc1_metadata: () => [
+        ["icrc1:name", { Text: "Eval Token" }],
+        ["icrc1:symbol", { Text: "EVT" }],
+        ["icrc1:decimals", { Nat: BigInt(LEDGER_DECIMALS) }],
+        ["icrc1:fee", { Nat: LEDGER_FEE }],
+      ],
+      icrc1_total_supply: () =>
+        [...balances.values()].reduce((sum, value) => sum + value, 0n),
+      icrc1_minting_account: () => [
+        { owner: Principal.fromText(MINTER), subaccount: [] },
+      ],
+      icrc1_supported_standards: () => [
+        { name: "ICRC-1", url: "https://github.com/dfinity/ICRC-1" },
+      ],
+      icrc1_balance_of: async ([account]) => {
+        const key = keyOf(account)
+        await delay(ledger.balanceDelayMs(key))
+        return balances.get(key) ?? 0n
+      },
+      icrc1_transfer: ([arg], { caller }: { caller: Principal }) => {
+        const code = ledger.rejectTransfersWith
+        if (code !== null && code !== 5) {
+          const { errorCode, message } = REJECTS[code]
+          throw new FakeReplicaReject(code, errorCode, message)
+        }
+        const from = accountKey(caller, arg.from_subaccount)
+        const to = keyOf(arg.to)
+        attempts.push({ caller: caller.toText(), to, amount: arg.amount })
+        if (code === 5) {
+          throw new Error("the ledger trapped (injected)")
+        }
+        const fee = arg.fee[0]
+        if (fee !== undefined && fee !== LEDGER_FEE) {
+          return { Err: { BadFee: { expected_fee: LEDGER_FEE } } }
+        }
+        const balance = balances.get(from) ?? 0n
+        if (balance < arg.amount + LEDGER_FEE) {
+          return { Err: { InsufficientFunds: { balance } } }
+        }
+        balances.set(from, balance - arg.amount - LEDGER_FEE)
+        balances.set(to, (balances.get(to) ?? 0n) + arg.amount)
+        const blockIndex = nextBlock++
+        transfers.push({ from, to, amount: arg.amount, blockIndex })
+        return { Ok: blockIndex }
+      },
+    })
+  return ledger
+}

package/evals/harness/gate-plan.mjs

@@ -0,0 +1,104 @@
+// What gate.mjs scores, and which of the cells it finds empty it may skip.
+// Kept apart from the scoring so the skip rule can be tested without scoring
+// anything (harness/gate-plan.test.mjs).
+import { existsSync, readdirSync } from "node:fs"
+import { join } from "node:path"
+import {
+  CONDITIONS,
+  DEFAULT_CONDITIONS,
+  EVALS,
+  TASKS,
+  readJson,
+} from "./assemble.mjs"
+
+/**
+ * The gate's plan for `tasks`:
+ *
+ * - `cases`: one per reference-like solution
+ *   (`tasks/<task>/solutions/<condition>/reference*`, expected to pass every
+ *   test) and one per faulty solution (`tasks/<task>/faulty/<name>`, whose
+ *   `meta.json` names its condition and the tests it must fail);
+ * - `skipped`: every cell (task × condition) with no reference solution, as
+ *   `{ cell, reasons }`; `reasons` says why the cell may NOT be skipped and is
+ *   empty when it may;
+ * - `blocked`: the skipped cells with a reason, which fail the gate.
+ *
+ * A cell may not be skipped when its condition is pre-registered
+ * (`DEFAULT_CONDITIONS`: the pilots' 45 solutions include every one of their
+ * cells, so an empty one is a deletion), when its condition is in `require`
+ * (`node gate.mjs --require v4`, Addendum 3's pass rule 2), or when its task
+ * holds faulty solutions of that condition (each is its reference with one
+ * change, so finding one without a reference means a port left half done).
+ * Only the cells of a condition not yet ported, which nothing names, are
+ * skipped, and gate.mjs prints each one.
+ *
+ * Throws on a task or a `require` entry the harness does not know.
+ */
+export function gatePlan({ tasks = TASKS, require = [], root = EVALS } = {}) {
+  for (const task of tasks) {
+    if (!TASKS.includes(task)) {
+      throw new Error(
+        `--task ${JSON.stringify(task)}: not a task (${TASKS.join(", ")})`
+      )
+    }
+  }
+  for (const condition of require) {
+    if (!CONDITIONS.includes(condition)) {
+      throw new Error(
+        `--require ${JSON.stringify(condition)}: not a condition (${CONDITIONS.join(", ")})`
+      )
+    }
+  }
+  const cases = []
+  const skipped = []
+  for (const task of tasks) {
+    const faultyRoot = join(root, "tasks", task, "faulty")
+    const faulty = (existsSync(faultyRoot) ? readdirSync(faultyRoot) : [])
+      .sort()
+      .map((name) => {
+        const meta = readJson(join(faultyRoot, name, "meta.json"))
+        return {
+          task,
+          condition: meta.condition,
+          name,
+          dir: join(faultyRoot, name),
+          expectFail: meta.expectFail,
+          expectSafe: meta.safe ?? {},
+          bug: meta.bug,
+        }
+      })
+    for (const condition of CONDITIONS) {
+      // Every reference-like solution: `reference`, `reference-module-scope`, …
+      const solutions = join(root, "tasks", task, "solutions", condition)
+      const references = existsSync(solutions)
+        ? readdirSync(solutions).filter((name) => name.startsWith("reference"))
+        : []
+      for (const name of references.sort()) {
+        cases.push({
+          task,
+          condition,
+          name,
+          dir: join(solutions, name),
+          expectFail: [],
+        })
+      }
+      if (references.length > 0) continue
+      const reasons = []
+      if (DEFAULT_CONDITIONS.includes(condition))
+        reasons.push("a pre-registered condition")
+      if (require.includes(condition)) reasons.push("required by --require")
+      const orphans = faulty
+        .filter((c) => c.condition === condition)
+        .map((c) => c.name)
+      if (orphans.length > 0)
+        reasons.push(`its faulty solutions need one: ${orphans.join(", ")}`)
+      skipped.push({ cell: `${task}/${condition}`, reasons })
+    }
+    cases.push(...faulty)
+  }
+  return {
+    cases,
+    skipped,
+    blocked: skipped.filter((s) => s.reasons.length > 0),
+  }
+}

package/evals/harness/gate-plan.test.mjs

@@ -0,0 +1,243 @@
+// Tests of which empty cells the gate may skip (harness/gate-plan.mjs), on
+// seeded trees, on this tree, and through gate.mjs itself.
+//
+//   node --test harness/gate-plan.test.mjs
+import { strict as assert } from "node:assert"
+import { spawn } from "node:child_process"
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { after, beforeEach, describe, it } from "node:test"
+import { DEFAULT_CONDITIONS, EVALS } from "./assemble.mjs"
+import { gatePlan } from "./gate-plan.mjs"
+
+const TASK = "node-tool"
+
+describe("the gate's plan on a seeded tree", () => {
+  const root = mkdtempSync(join(tmpdir(), "ic-reactor-evals-gate-test-"))
+  const reference = (condition, name = "reference") =>
+    mkdirSync(join(root, "tasks", TASK, "solutions", condition, name), {
+      recursive: true,
+    })
+  const faulty = (name, condition) => {
+    const dir = join(root, "tasks", TASK, "faulty", name)
+    mkdirSync(dir, { recursive: true })
+    writeFileSync(
+      join(dir, "meta.json"),
+      JSON.stringify({ condition, expectFail: ["a_test"], bug: "one change" })
+    )
+  }
+  const plan = (require = []) => gatePlan({ tasks: [TASK], require, root })
+  const blockedCells = (p) => p.blocked.map((s) => s.cell)
+
+  beforeEach(() => {
+    rmSync(join(root, "tasks"), { recursive: true, force: true })
+    for (const condition of DEFAULT_CONDITIONS) {
+      reference(condition)
+      reference(condition, "reference-module-scope")
+    }
+    faulty("thin-some-bug", "thin")
+  })
+  after(() => rmSync(root, { recursive: true, force: true }))
+
+  it("skips a condition not yet ported when nothing names it", () => {
+    const p = plan()
+    assert.deepEqual(p.skipped, [{ cell: `${TASK}/v4`, reasons: [] }])
+    assert.deepEqual(p.blocked, [])
+    assert.equal(p.cases.length, DEFAULT_CONDITIONS.length * 2 + 1)
+    assert.deepEqual(
+      p.cases.filter((c) => c.expectFail.length > 0).map((c) => c.name),
+      ["thin-some-bug"]
+    )
+  })
+  it("refuses to skip a condition named by --require", () => {
+    assert.deepEqual(blockedCells(plan(["v4"])), [`${TASK}/v4`])
+    assert.deepEqual(plan(["v4"]).blocked[0].reasons, ["required by --require"])
+  })
+  it("refuses to skip a condition whose faulty solutions are in the task", () => {
+    faulty("v4-some-bug", "v4")
+    const p = plan()
+    assert.deepEqual(blockedCells(p), [`${TASK}/v4`])
+    assert.match(p.blocked[0].reasons.join(), /v4-some-bug/)
+  })
+  it("refuses to skip a pre-registered condition whose references are gone", () => {
+    rmSync(join(root, "tasks", TASK, "solutions", "thin-guide"), {
+      recursive: true,
+    })
+    assert.deepEqual(blockedCells(plan()), [`${TASK}/thin-guide`])
+  })
+  it("scores a required condition once its references exist", () => {
+    reference("v4")
+    faulty("v4-some-bug", "v4")
+    const p = plan(["v4"])
+    assert.deepEqual(p.skipped, [])
+    assert.ok(
+      p.cases.some((c) => c.condition === "v4" && c.name === "reference")
+    )
+    assert.ok(
+      p.cases.some((c) => c.condition === "v4" && c.name === "v4-some-bug")
+    )
+  })
+  it("refuses an unknown condition or task", () => {
+    assert.throws(() => plan(["v5"]), /--require "v5": not a condition/)
+    assert.throws(
+      () => gatePlan({ tasks: ["no-task"], root }),
+      /--task "no-task": not a task/
+    )
+  })
+})
+
+describe("the gate's plan on this tree", () => {
+  /**
+   * The v4 faulty solutions that are not ports: each catches a trap the real
+   * library leaves open where v4-proto did not, so it has no v4-proto
+   * original (conditions/v4/PORTING.md).
+   */
+  const V4_ONLY = ["node-tool/v4-anonymous-identity-sent"]
+  const isPort = (c) => !V4_ONLY.includes(`${c.task}/${c.name}`)
+
+  it("skips no cell and blocks nothing, with --require v4 too", () => {
+    for (const require of [[], ["v4"]]) {
+      const p = gatePlan({ require })
+      assert.deepEqual(p.skipped, [], `--require ${require}`)
+      assert.deepEqual(p.blocked, [], `--require ${require}`)
+    }
+  })
+  it("scores both v4 references of each task, the six ported faulty solutions and the v4-only one", () => {
+    const v4 = gatePlan({ require: ["v4"] }).cases.filter(
+      (c) => c.condition === "v4"
+    )
+    assert.deepEqual(
+      v4
+        .filter((c) => c.name.startsWith("reference"))
+        .map((c) => `${c.task}/${c.name}`)
+        .sort(),
+      [
+        "node-tool/reference",
+        "node-tool/reference-module-scope",
+        "react-wallet/reference",
+        "react-wallet/reference-module-scope",
+      ]
+    )
+    assert.deepEqual(
+      v4
+        .filter((c) => !c.name.startsWith("reference"))
+        .map((c) => `${c.task}/${c.name}`)
+        .sort(),
+      [
+        "node-tool/v4-anonymous-identity-sent",
+        "node-tool/v4-never-may-have-executed",
+        "node-tool/v4-refuses-nat64-max",
+        "react-wallet/v4-every-reject-unknown",
+        "react-wallet/v4-keep-previous-data",
+        "react-wallet/v4-retry-spread",
+        "react-wallet/v4-status-not-idle",
+      ]
+    )
+  })
+  it("expects each ported faulty solution to fail what its v4-proto original fails", () => {
+    const cases = gatePlan().cases
+    const ported = cases.filter(
+      (c) =>
+        c.condition === "v4" && !c.name.startsWith("reference") && isPort(c)
+    )
+    assert.equal(ported.length, 6)
+    for (const port of ported) {
+      const original = cases.find(
+        (c) =>
+          c.task === port.task &&
+          c.condition === "v4-proto" &&
+          c.name === port.name.replace(/^v4-/, "v4-proto-")
+      )
+      assert.ok(original, `${port.task}/${port.name} has no v4-proto original`)
+      assert.deepEqual(
+        [...port.expectFail].sort(),
+        [...original.expectFail].sort(),
+        `${port.task}/${port.name}`
+      )
+    }
+  })
+  it("has no v4-proto original for a v4-only faulty solution", () => {
+    const cases = gatePlan().cases
+    for (const id of V4_ONLY) {
+      const [task, name] = id.split("/")
+      assert.ok(
+        cases.some(
+          (c) => c.task === task && c.condition === "v4" && c.name === name
+        ),
+        `${id} is not in the plan`
+      )
+      assert.ok(
+        !cases.some(
+          (c) =>
+            c.task === task &&
+            c.condition === "v4-proto" &&
+            c.name === name.replace(/^v4-/, "v4-proto-")
+        ),
+        `${id} has a v4-proto original: it is a port`
+      )
+    }
+  })
+})
+
+describe("gate.mjs", () => {
+  /**
+   * Runs gate.mjs in a process group of its own. Here it must exit before
+   * scoring; if it starts scoring instead, the whole group (its score.mjs,
+   * tsc and vitest children) is killed after 15 s rather than left running.
+   */
+  const gate = (...args) =>
+    new Promise((resolve) => {
+      const child = spawn(
+        process.execPath,
+        [join(EVALS, "gate.mjs"), ...args],
+        { cwd: EVALS, detached: true, stdio: ["ignore", "pipe", "pipe"] }
+      )
+      let stdout = ""
+      let stderr = ""
+      child.stdout.on("data", (d) => (stdout += d))
+      child.stderr.on("data", (d) => (stderr += d))
+      const timer = setTimeout(
+        () => process.kill(-child.pid, "SIGKILL"),
+        15_000
+      )
+      child.on("close", (status) => {
+        clearTimeout(timer)
+        resolve({ status, stdout, stderr })
+      })
+    })
+
+  /**
+   * A tree in which every pre-registered cell of node-tool has its two
+   * references (empty directories: the gate must not get as far as scoring
+   * them) and the v4 cell has none, as before the v4 port.
+   */
+  const root = mkdtempSync(join(tmpdir(), "ic-reactor-evals-gate-run-"))
+  for (const condition of DEFAULT_CONDITIONS) {
+    for (const name of ["reference", "reference-module-scope"]) {
+      mkdirSync(join(root, "tasks", TASK, "solutions", condition, name), {
+        recursive: true,
+      })
+    }
+  }
+  after(() => rmSync(root, { recursive: true, force: true }))
+
+  it("exits 2 on a --require that names no condition", async () => {
+    const r = await gate("--require", "v5")
+    assert.equal(r.status, 2)
+    assert.match(r.stderr, /not a condition/)
+  })
+  it("exits 2 on a --root with no tasks directory", async () => {
+    const r = await gate("--root", join(root, "tasks", TASK))
+    assert.equal(r.status, 2)
+    assert.match(r.stderr, /has no tasks\/ directory/)
+  })
+  it("fails before scoring anything when a required cell is empty", async () => {
+    const r = await gate("--root", root, "--task", TASK, "--require", "v4")
+    assert.equal(r.status, 1, r.stdout + r.stderr)
+    assert.match(r.stdout, /^FAIL node-tool\/v4: .*required by --require$/m)
+    assert.match(r.stdout, /gate: failed before scoring/)
+    assert.doesNotMatch(r.stdout, /^(ok {2}|FAIL) node-tool\/[\w-]+\/[\w-]+: /m)
+    assert.doesNotMatch(r.stdout, /solutions behaved as expected/)
+  })
+})

package/evals/harness/icrc1.did

@@ -0,0 +1,88 @@
+type BlockIndex = nat;
+type Subaccount = blob;
+// Number of nanoseconds since the UNIX epoch in UTC timezone.
+type Timestamp = nat64;
+// Number of nanoseconds between two [Timestamp]s.
+type Duration = nat64;
+type Tokens = nat;
+
+type Account = record {
+    owner : principal;
+    subaccount : opt Subaccount;
+};
+
+type TransferArg = record {
+    from_subaccount : opt Subaccount;
+    to : Account;
+    amount : Tokens;
+    fee : opt Tokens;
+    memo : opt blob;
+    created_at_time: opt Timestamp;
+};
+
+type TransferError = variant {
+    BadFee : record { expected_fee : Tokens };
+    BadBurn : record { min_burn_amount : Tokens };
+    InsufficientFunds : record { balance : Tokens };
+    TooOld;
+    CreatedInFuture : record { ledger_time : nat64 };
+    TemporarilyUnavailable;
+    Duplicate : record { duplicate_of : BlockIndex };
+    GenericError : record { error_code : nat; message : text };
+};
+
+type TransferResult = variant {
+    Ok : BlockIndex;
+    Err : TransferError;
+};
+
+// The value returned from the [icrc1_metadata] endpoint.
+type Value = variant {
+    Nat : nat;
+    Int : int;
+    Text : text;
+    Blob : blob;
+};
+
+// The initialization parameters of the Ledger
+type InitArgs = record {
+    minting_account : Account;
+    transfer_fee : nat64;
+    token_symbol : text;
+    token_name : text;
+    metadata : vec record { text; Value };
+    initial_balances : vec record { Account; nat64 };
+    archive_options : record {
+        num_blocks_to_archive : nat64;
+        trigger_threshold : nat64;
+        max_message_size_bytes : opt nat64;
+        cycles_for_archive_creation : opt nat64;
+        node_max_memory_size_bytes : opt nat64;
+        controller_id : principal;
+    };
+};
+
+type UpgradeArgs = record {
+    metadata : opt vec record { text; Value };
+    token_symbol : opt text;
+    token_name : opt text;
+    transfer_fee : opt nat64;
+};
+
+type LedgerArg = variant {
+    Init: InitArgs;
+    Upgrade: opt UpgradeArgs;
+};
+
+service : (ledger_arg : LedgerArg) -> {
+    icrc1_name : () -> (text) query;
+    icrc1_symbol : () -> (text) query;
+    icrc1_decimals : () -> (nat8) query;
+    icrc1_metadata : () -> (vec record { text; Value }) query;
+    icrc1_total_supply : () -> (Tokens) query;
+    icrc1_fee : () -> (Tokens) query;
+    icrc1_minting_account : () -> (opt Account) query;
+    icrc1_balance_of : (Account) -> (Tokens) query;
+    icrc1_transfer : (TransferArg) -> (TransferResult);
+    icrc1_supported_standards : () -> (vec record { name : text; url : text }) query;
+}

package/evals/harness/judge.mjs

@@ -0,0 +1,63 @@
+// From per-test results to requirements, `safe` and `requirementsMet`, under
+// one prompt variant. Used by score.mjs and gate.mjs.
+//
+// A test listed in task.json `notApplicable[<variant>].tests` states a rule
+// that variant's prompt does not give the agent: it still runs and is
+// reported (under `notApplicable`, with its result) but counts toward
+// nothing; a requirement left with no applicable test is not applicable
+// either (`pass: null`).
+import { taskTests } from "./assemble.mjs"
+
+/** The tests task.json marks not applicable under a prompt variant. */
+export function notApplicableTests(spec, prompt) {
+  const names = spec.notApplicable?.[prompt]?.tests ?? []
+  const all = taskTests(spec)
+  for (const name of names) {
+    if (!all.includes(name)) {
+      throw new Error(`${spec.name}: notApplicable names unknown test ${name}`)
+    }
+  }
+  return names
+}
+
+/** `passByName`: Map test name → boolean. */
+export function judge(spec, passByName, prompt) {
+  const excluded = new Set(notApplicableTests(spec, prompt))
+  const tests = taskTests(spec).map((name) => ({
+    name,
+    pass: passByName.get(name) === true,
+    applicable: !excluded.has(name),
+  }))
+  const applicableTests = tests.filter((t) => t.applicable)
+  const requirements = Object.entries(spec.requirements).map(([name, r]) => {
+    const judged = r.tests.filter((t) => !excluded.has(t))
+    return judged.length === 0
+      ? { name, safety: r.safety, pass: null, applicable: false }
+      : {
+          name,
+          safety: r.safety,
+          pass: judged.every((t) => passByName.get(t) === true),
+          applicable: true,
+        }
+  })
+  const applicable = requirements.filter((r) => r.applicable)
+  const safetyViolations = applicable.filter((r) => r.safety && !r.pass).length
+  return {
+    prompt,
+    tests,
+    passRate: Number(
+      (
+        applicableTests.filter((t) => t.pass).length / applicableTests.length
+      ).toFixed(4)
+    ),
+    requirements,
+    notApplicable: tests
+      .filter((t) => !t.applicable)
+      .map(({ name, pass }) => ({ name, pass })),
+    safetyViolations,
+    safe: safetyViolations === 0,
+    requirementsMet: Number(
+      (applicable.filter((r) => r.pass).length / applicable.length).toFixed(4)
+    ),
+  }
+}