@ic-reactor/mcp 0.0.0-stage → 0.0.0

package/evals/harness/aggregate.test.mjs

@@ -0,0 +1,344 @@
+// Tests of the aggregator (harness/aggregate.mjs).
+//
+//   node --test harness/aggregate.test.mjs
+import { strict as assert } from "node:assert"
+import { describe, it } from "node:test"
+import {
+  aggregate,
+  newcombe,
+  pilotReport,
+  printSummary,
+  wilson,
+} from "./aggregate.mjs"
+
+const close = (a, b, eps = 0.0006) => Math.abs(a - b) < eps
+
+const record = (condition, safe, met, extra = {}) => ({
+  task: "node-tool",
+  model: "model-a",
+  condition,
+  tscClean: true,
+  safe,
+  requirementsMet: met,
+  requirements: [
+    { name: "basic_function", safety: false, pass: met === 1 },
+    { name: "no_resend", safety: true, pass: safe },
+  ],
+  agent: {
+    exit: "normal",
+    minutes: 10,
+    turns: 20,
+    tokens: { input: 1000, output: 200 },
+    costUsd: 0.5,
+  },
+  ...extra,
+})
+
+describe("intervals", () => {
+  it("Wilson matches the textbook value", () => {
+    const [l, u] = wilson(3, 6)
+    assert.ok(close(l, 0.1876) && close(u, 0.8124), `${l} ${u}`)
+  })
+  it("Newcombe matches Newcombe (1998), method 10: 56/70 − 48/80", () => {
+    const [l, u] = newcombe(56, 70, 48, 80)
+    assert.ok(close(l, 0.0524) && close(u, 0.3339), `${l} ${u}`)
+  })
+})
+
+describe("aggregate", () => {
+  const records = [
+    record("thin-guide", true, 1),
+    record("thin-guide", false, 0.5),
+    record("thin-guide", true, 1, { contaminated: true }),
+    record("thin-guide", false, 0, { harnessError: "score.mjs exit 2" }),
+    record("v4-proto", true, 1),
+    record("v4-proto", true, 1),
+    record("v4-proto", false, 0.5, { contaminated: true }),
+    record("v4-proto", true, 1, { model: "model-b" }),
+  ]
+  const summary = aggregate(records)
+
+  it("excludes harness errors everywhere and counts them", () => {
+    assert.equal(summary.harnessErrors, 1)
+    const tg = summary.main.cells.find(
+      (c) => c.condition === "thin-guide" && c.model === "model-a"
+    )
+    assert.equal(tg.harnessErrors, 1)
+    assert.equal(tg.runs, 2)
+  })
+  it("excludes contaminated runs from main and keeps them in intent-to-treat", () => {
+    assert.equal(summary.contaminated, 2)
+    const main = summary.main.cells.find(
+      (c) => c.condition === "v4-proto" && c.model === "model-a"
+    )
+    const itt = summary.intentToTreat.cells.find(
+      (c) => c.condition === "v4-proto" && c.model === "model-a"
+    )
+    assert.equal(main.runs, 2)
+    assert.equal(main.safeRate, 1)
+    assert.equal(itt.runs, 3)
+    assert.equal(itt.safeRate, 0.667)
+    assert.equal(main.contaminated, 1)
+  })
+  it("never pools models", () => {
+    const b = summary.main.cells.filter((c) => c.model === "model-b")
+    assert.equal(b.length, 1)
+    assert.equal(b[0].runs, 1)
+    assert.ok(summary.main.differences.every((d) => d.model === "model-a"))
+  })
+  it("compares within task and model, with a risk difference and its interval", () => {
+    const d = summary.main.differences.find(
+      (x) => x.comparison === "v4-proto - thin-guide"
+    )
+    assert.equal(d.safeRateDiff, 0.5)
+    assert.equal(d.safeRateDiffCI95.length, 2)
+    assert.ok(d.safeRateDiffCI95[0] < 0.5 && d.safeRateDiffCI95[1] > 0.5)
+  })
+})
+
+describe("pilot report", () => {
+  it("reports variance, cost, ceiling warnings and sample sizes", () => {
+    const records = [
+      ...Array.from({ length: 5 }, () => record("v4-proto", true, 1)),
+      ...Array.from({ length: 5 }, (_, i) =>
+        record("thin-guide", i < 3, i < 3 ? 1 : 0.5)
+      ),
+    ]
+    const report = pilotReport(records, { margin: 0.1 })
+    const v4 = report.cells.find((c) => c.condition === "v4-proto")
+    assert.ok(v4.warnings.some((w) => w.startsWith("ceiling")))
+    assert.equal(v4.minutes.mean, 10)
+    assert.equal(v4.tokens.mean, 1200)
+    const tg = report.cells.find((c) => c.condition === "thin-guide")
+    assert.equal(tg.safeRate, 0.6)
+    assert.equal(tg.requirementsMetVariance, 0.075)
+    const s = report.sampleSizes.find(
+      (x) => x.comparison === "v4-proto - thin-guide"
+    )
+    // centre 0.8, p1 0.85, p2 0.75: (1.96+0.84)^2 (0.1275+0.1875)/0.01 ≈ 248
+    assert.ok(
+      s.runsPerCellForSafeRate > 240 && s.runsPerCellForSafeRate < 255,
+      String(s.runsPerCellForSafeRate)
+    )
+  })
+})
+
+describe("effort levels", () => {
+  it("are never pooled, and compared only within one level", () => {
+    const records = [
+      record("v4-proto", true, 1, { effort: "low" }),
+      record("thin-guide", false, 0.5, { effort: "low" }),
+      record("v4-proto", true, 1, { effort: "high" }),
+    ]
+    const summary = aggregate(records)
+    const v4 = summary.main.cells.filter((c) => c.condition === "v4-proto")
+    assert.deepEqual(v4.map((c) => c.effort).sort(), ["high", "low"])
+    assert.ok(v4.every((c) => c.runs === 1))
+    assert.deepEqual(summary.effort, { low: 2, high: 1 })
+    assert.deepEqual(
+      summary.main.differences.map((d) => `${d.effort} ${d.comparison}`),
+      ["low v4-proto - thin-guide"]
+    )
+  })
+})
+
+describe("prompt variants", () => {
+  it("are never pooled, and compared only within one variant", () => {
+    const records = [
+      record("v4-proto", true, 1, { prompt: "minimal" }),
+      record("thin-guide", false, 0.5, { prompt: "minimal" }),
+      record("v4-proto", true, 1, { prompt: "explicit" }),
+      // A record from before the variants existed is explicit.
+      record("thin-guide", true, 1),
+    ]
+    const summary = aggregate(records)
+    const v4 = summary.main.cells.filter((c) => c.condition === "v4-proto")
+    assert.deepEqual(v4.map((c) => c.prompt).sort(), ["explicit", "minimal"])
+    assert.ok(v4.every((c) => c.runs === 1))
+    assert.deepEqual(summary.prompt, { minimal: 2, explicit: 2 })
+    assert.deepEqual(
+      summary.main.differences
+        .map((d) => `${d.prompt} ${d.comparison} ${d.safeRateDiff}`)
+        .sort(),
+      ["explicit v4-proto - thin-guide 0", "minimal v4-proto - thin-guide 1"]
+    )
+    const pilot = pilotReport(records, { margin: 0.15 })
+    assert.deepEqual(pilot.sampleSizes.map((s) => s.prompt).sort(), [
+      "explicit",
+      "minimal",
+    ])
+  })
+})
+
+describe("not-applicable tests", () => {
+  it("count toward nothing and are listed per variant with observed results", () => {
+    const na = (pass) => ({
+      prompt: "minimal",
+      notApplicable: [{ name: "refuses_amount_past_nat64", pass }],
+      requirements: [
+        { name: "basic_function", safety: false, pass: true, applicable: true },
+        { name: "only_na", safety: true, pass: null, applicable: false },
+      ],
+    })
+    const records = [
+      record("v4-proto", true, 1, na(false)),
+      record("v4-proto", true, 1, na(true)),
+      record("thin-guide", true, 1, { prompt: "explicit" }),
+    ]
+    const summary = aggregate(records)
+    assert.deepEqual(summary.notApplicable, {
+      minimal: {
+        "node-tool": { refuses_amount_past_nat64: { passed: 1, n: 2 } },
+      },
+    })
+    const cell = summary.main.cells.find((c) => c.prompt === "minimal")
+    assert.deepEqual(cell.notApplicable, {
+      refuses_amount_past_nat64: { passed: 1, n: 2 },
+    })
+    assert.equal(cell.fullPassRate, 1)
+    assert.deepEqual(Object.keys(cell.perRequirement), ["basic_function"])
+    let out = ""
+    printSummary(summary, (s) => (out += s))
+    assert.match(
+      out,
+      /not applicable under --prompt minimal, node-tool .*refuses_amount_past_nat64 1\/2/
+    )
+  })
+})
+
+describe("the v4 gate (PREREGISTRATION.md, Addendum 3)", () => {
+  it("compares v4 with thin-guide in the main and the intent-to-treat result", () => {
+    const minimal = (extra = {}) => ({ prompt: "minimal", ...extra })
+    const records = [
+      ...Array.from({ length: 5 }, () => record("v4", true, 1, minimal())),
+      ...Array.from({ length: 5 }, (_, i) =>
+        record("thin-guide", i < 4, i < 4 ? 1 : 0.5, minimal())
+      ),
+      record("v4", false, 0.5, minimal({ contaminated: true })),
+    ]
+    const summary = aggregate(records)
+    const diff = (result) =>
+      result.differences.find((d) => d.comparison === "v4 - thin-guide")
+    assert.equal(diff(summary.main).safeRateDiff, 0.2)
+    // 5/6 - 4/5 with the contaminated run kept.
+    assert.equal(diff(summary.intentToTreat).safeRateDiff, 0.033)
+    assert.equal(diff(summary.main).safeRateDiffCI95.length, 2)
+  })
+})
+
+describe("pooled over tasks (PREREGISTRATION.md, Addendum 4, alternative A)", () => {
+  const minimal = (task, extra = {}) => ({
+    task,
+    prompt: "minimal",
+    effort: "medium",
+    ...extra,
+  })
+  const cell = (task, condition, failures, extra = {}) =>
+    Array.from({ length: 20 }, (_, i) =>
+      record(
+        condition,
+        i >= failures,
+        i >= failures ? 1 : 0.5,
+        minimal(task, extra)
+      )
+    )
+  const pooled = (result, comparison = "v4 - thin-guide") =>
+    result.pooled.filter((d) => d.comparison === comparison)
+
+  it("pools v4 − thin-guide over both tasks: 40/40 against 40/40", () => {
+    const summary = aggregate([
+      ...cell("node-tool", "v4", 0),
+      ...cell("react-wallet", "v4", 0),
+      ...cell("node-tool", "thin-guide", 0),
+      ...cell("react-wallet", "thin-guide", 0),
+    ])
+    const [d] = pooled(summary.main)
+    assert.deepEqual(d.tasks, ["node-tool", "react-wallet"])
+    assert.deepEqual(d.counts, {
+      v4: { safe: 40, runs: 40 },
+      "thin-guide": { safe: 40, runs: 40 },
+    })
+    assert.equal(d.safeRateDiff, 0)
+    assert.deepEqual(d.safeRateDiffCI95, [-0.088, 0.088])
+    // Per task, 20/20 against 20/20 cannot clear -0.10.
+    for (const t of summary.main.differences.filter(
+      (x) => x.comparison === "v4 - thin-guide"
+    ))
+      assert.deepEqual(t.safeRateDiffCI95, [-0.161, 0.161])
+    assert.equal(pooled(summary.intentToTreat).length, 1)
+  })
+  it("excludes contaminated runs from the main pool and keeps them in intent-to-treat", () => {
+    const summary = aggregate([
+      ...cell("node-tool", "v4", 0),
+      ...cell("react-wallet", "v4", 1),
+      ...cell("node-tool", "thin-guide", 0),
+      ...cell("react-wallet", "thin-guide", 0),
+      record("v4", false, 0.5, minimal("node-tool", { contaminated: true })),
+      record("v4", true, 1, minimal("node-tool", { harnessError: "x" })),
+    ])
+    const [main] = pooled(summary.main)
+    assert.deepEqual(main.counts.v4, { safe: 39, runs: 40 })
+    assert.deepEqual(
+      main.safeRateDiffCI95,
+      newcombe(39, 40, 40, 40).map((x) => Math.round(x * 1000) / 1000)
+    )
+    assert.equal(main.safeRateDiffCI95[0], -0.129)
+    const [itt] = pooled(summary.intentToTreat)
+    assert.deepEqual(itt.counts.v4, { safe: 39, runs: 41 })
+  })
+  it("pools only tasks where both conditions ran, and never across models or prompt variants", () => {
+    const summary = aggregate([
+      ...cell("node-tool", "v4", 0),
+      ...cell("react-wallet", "v4", 0),
+      ...cell("node-tool", "thin-guide", 0),
+      // react-wallet's control ran under another model and another prompt.
+      ...cell("react-wallet", "thin-guide", 0, { model: "model-b" }),
+      ...cell("react-wallet", "thin-guide", 0, { prompt: "explicit" }),
+    ])
+    assert.deepEqual(pooled(summary.main), [])
+  })
+  it("gives no pooled row when a control cell has no usable runs", () => {
+    const v4Cells = [
+      ...cell("node-tool", "v4", 0),
+      ...cell("react-wallet", "v4", 0),
+      ...cell("node-tool", "thin-guide", 0),
+    ]
+    // Every react-wallet control run a harness error: no row in either analysis.
+    const errored = aggregate([
+      ...v4Cells,
+      ...cell("react-wallet", "thin-guide", 0, { harnessError: "x" }),
+    ])
+    assert.equal(
+      errored.main.cells.filter(
+        (c) => c.task === "react-wallet" && c.condition === "thin-guide"
+      ).length,
+      1
+    )
+    assert.deepEqual(pooled(errored.main), [])
+    assert.deepEqual(pooled(errored.intentToTreat), [])
+    // Every react-wallet control run contaminated: none in main, kept in ITT.
+    const contaminated = aggregate([
+      ...v4Cells,
+      ...cell("react-wallet", "thin-guide", 0, { contaminated: true }),
+    ])
+    assert.deepEqual(pooled(contaminated.main), [])
+    assert.deepEqual(pooled(contaminated.intentToTreat)[0].counts, {
+      v4: { safe: 40, runs: 40 },
+      "thin-guide": { safe: 40, runs: 40 },
+    })
+  })
+  it("prints the pooled comparison with its counts", () => {
+    const summary = aggregate([
+      ...cell("node-tool", "v4", 2),
+      ...cell("react-wallet", "v4", 0),
+      ...cell("node-tool", "thin-guide", 0),
+      ...cell("react-wallet", "thin-guide", 1),
+    ])
+    let out = ""
+    printSummary(summary, (s) => (out += s))
+    assert.match(
+      out,
+      /^pooled node-tool\+react-wallet\/model-a\/medium\/minimal: v4 - thin-guide: safe -0\.025 \[-0\.142, 0\.085\] \(v4 38\/40, thin-guide 39\/40\)$/m
+    )
+  })
+})

package/evals/harness/assemble.mjs

@@ -0,0 +1,216 @@
+// Builds the starter project an agent (or the scorer) sees for one
+// task × condition, from three layers that never mix:
+//
+//   tasks/<task>/starter/          identical in every condition
+//   conditions/<condition>/starter/ the condition's generated code
+//   conditions/<condition>/docs/    the condition's documentation → docs/
+//
+// plus a package.json listing the condition's dependencies and TASK.md, the
+// rendered prompt. Hidden tests, solutions and the harness never go in.
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  rmSync,
+  readFileSync,
+  writeFileSync,
+} from "node:fs"
+import { basename, dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+
+export const EVALS = join(dirname(fileURLToPath(import.meta.url)), "..")
+export const TASKS = ["node-tool", "react-wallet"]
+/** Every condition the harness knows. */
+export const CONDITIONS = ["v3", "thin", "thin-guide", "v4-proto", "v4"]
+
+/**
+ * The conditions a batch runs when none is named: the pre-registered matrix
+ * (PREREGISTRATION.md). `v4` (the real ic-reactor 4 packages, issue #786)
+ * runs only when named, as Addendum 3 does: `--condition v4 --condition
+ * thin-guide`.
+ */
+export const DEFAULT_CONDITIONS = ["v3", "thin", "thin-guide", "v4-proto"]
+
+export function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"))
+}
+
+export function taskSpec(task) {
+  if (!TASKS.includes(task))
+    throw new Error(`unknown task ${JSON.stringify(task)}`)
+  return readJson(join(EVALS, "tasks", task, "task.json"))
+}
+
+/**
+ * Every hidden test name, in requirement order. Each test belongs to exactly
+ * one requirement (task.json `requirements`); that is checked here.
+ */
+export function taskTests(spec) {
+  const names = Object.values(spec.requirements).flatMap((r) => r.tests)
+  const dup = names.filter((n, i) => names.indexOf(n) !== i)
+  if (dup.length > 0) {
+    throw new Error(
+      `${spec.name}: tests in more than one requirement: ${dup.join(", ")}`
+    )
+  }
+  return names
+}
+
+export function conditionSpec(condition) {
+  if (!CONDITIONS.includes(condition)) {
+    throw new Error(`unknown condition ${JSON.stringify(condition)}`)
+  }
+  return readJson(join(EVALS, "conditions", condition, "condition.json"))
+}
+
+/**
+ * How the agent may check its work, by run mode. `sandboxed`: the agent may
+ * run node and vitest (only ever inside an OS sandbox). `tsc-only`: the
+ * fallback when no sandbox is available. Identical across conditions.
+ */
+export const RUN_TEXT = {
+  sandboxed:
+    "You can run code: `npx vitest run` runs the tests in `test/` against a local\n" +
+    "fake Internet Computer replica with an ICRC-1 ledger (`test/support/`; start\n" +
+    "from `test/smoke.test.ts`), and `node` is available. Run `npx tsc --noEmit`\n" +
+    "before you finish; it must pass. Do not add dependencies.",
+  "tsc-only":
+    "You can type-check with `npx tsc --noEmit`; you cannot run code. Run it\n" +
+    "before you finish; it must pass. Do not add dependencies.",
+}
+
+/**
+ * The prompt variants. `explicit` states the safety rules the hidden tests
+ * check; `minimal` describes the product and its public contract (signatures,
+ * result shape, DOM test ids and states) the way a user would, and leaves the
+ * safety rules out. Same hidden tests and scoring; never pooled.
+ */
+export const PROMPT_VARIANTS = ["explicit", "minimal"]
+
+export function promptFile(task, variant = "explicit") {
+  if (!PROMPT_VARIANTS.includes(variant)) {
+    throw new Error(`unknown prompt variant ${JSON.stringify(variant)}`)
+  }
+  return join(
+    EVALS,
+    "tasks",
+    task,
+    variant === "explicit" ? "prompt.md" : `prompt.${variant}.md`
+  )
+}
+
+/**
+ * The task prompt: identical across conditions but for the {{LIBRARY}} line.
+ * {{RUN}} depends on the run mode only; the rest on the prompt variant only.
+ */
+export function renderPrompt(
+  task,
+  condition,
+  mode = "sandboxed",
+  variant = "explicit"
+) {
+  const file = promptFile(task, variant)
+  const template = readFileSync(file, "utf8")
+  for (const marker of ["{{LIBRARY}}", "{{RUN}}"]) {
+    const lines = template.split("\n").filter((line) => line.includes(marker))
+    if (lines.length !== 1 || lines[0].trim() !== marker) {
+      throw new Error(
+        `${task}/${basename(file)} must hold ${marker} alone on exactly one line`
+      )
+    }
+  }
+  if (!(mode in RUN_TEXT)) throw new Error(`unknown run mode ${mode}`)
+  return template
+    .replace("{{LIBRARY}}", conditionSpec(condition).library)
+    .replace("{{RUN}}", RUN_TEXT[mode])
+}
+
+/**
+ * The condition whose starter, dependencies and node_modules this one uses.
+ * A condition with `"base"` in condition.json (thin-guide → thin) differs
+ * from its base only in the docs its agent gets: the base's docs plus its own.
+ */
+export function baseCondition(condition) {
+  return conditionSpec(condition).base ?? condition
+}
+
+export function conditionNodeModules(condition) {
+  return join(EVALS, "conditions", baseCondition(condition), "node_modules")
+}
+
+/** The doc directories a condition's agent gets, in the order they are copied. */
+export function conditionDocDirs(condition) {
+  const dirs = [join(EVALS, "conditions", condition, "docs")]
+  const base = baseCondition(condition)
+  if (base !== condition) dirs.unshift(join(EVALS, "conditions", base, "docs"))
+  return dirs.filter((dir) => existsSync(dir))
+}
+
+/** The version of `name` in the condition's node_modules, if it is installed. */
+function installedVersion(condition, name) {
+  const file = join(conditionNodeModules(condition), name, "package.json")
+  return existsSync(file) ? readJson(file).version : undefined
+}
+
+export function assembleStarter({
+  task,
+  condition,
+  dest,
+  mode = "sandboxed",
+  prompt = "explicit",
+}) {
+  taskSpec(task)
+  const spec = conditionSpec(condition)
+  mkdirSync(dest, { recursive: true })
+  cpSync(join(EVALS, "tasks", task, "starter"), dest, { recursive: true })
+  // The public test scaffold: a fake replica with a fake ledger (identical in
+  // every condition) under test/support, next to the task's smoke test. Only
+  // where the agent may run code; the tsc-only fallback ships no tests.
+  if (mode === "sandboxed") {
+    cpSync(
+      join(EVALS, "harness", "public", "support"),
+      join(dest, "test", "support"),
+      {
+        recursive: true,
+      }
+    )
+  } else {
+    rmSync(join(dest, "test"), { recursive: true, force: true })
+  }
+  const base = baseCondition(condition)
+  const overlay = join(EVALS, "conditions", base, "starter")
+  if (existsSync(overlay)) cpSync(overlay, dest, { recursive: true })
+  for (const docs of conditionDocDirs(condition)) {
+    cpSync(docs, join(dest, "docs"), { recursive: true })
+  }
+
+  const pkg = readJson(join(EVALS, "conditions", base, "package.json"))
+  const deps = { ...pkg.dependencies }
+  for (const [name, range] of Object.entries(deps)) {
+    // A package built from this repository (v4-proto's prototype, v4's packed
+    // packages) shows the version that is installed, as npm would record it.
+    if (String(range).startsWith("workspace:"))
+      deps[name] =
+        installedVersion(condition, name) ?? spec.localVersion ?? "0.0.0"
+  }
+  writeFileSync(
+    join(dest, "package.json"),
+    JSON.stringify(
+      {
+        name: task,
+        private: true,
+        type: "module",
+        scripts: { typecheck: "tsc --noEmit" },
+        dependencies: deps,
+        devDependencies: pkg.devDependencies,
+      },
+      null,
+      2
+    ) + "\n"
+  )
+  writeFileSync(
+    join(dest, "TASK.md"),
+    renderPrompt(task, condition, mode, prompt)
+  )
+  return dest
+}

package/evals/harness/assemble.test.mjs

@@ -0,0 +1,143 @@
+// Tests of the prompt variants (harness/assemble.mjs).
+//
+//   node --test harness/assemble.test.mjs
+import { strict as assert } from "node:assert"
+import { readFileSync } from "node:fs"
+import { describe, it } from "node:test"
+import {
+  CONDITIONS,
+  PROMPT_VARIANTS,
+  RUN_TEXT,
+  TASKS,
+  conditionSpec,
+  promptFile,
+  renderPrompt,
+} from "./assemble.mjs"
+
+const template = (task, variant) =>
+  readFileSync(promptFile(task, variant), "utf8")
+
+describe("prompt parity", () => {
+  for (const task of TASKS) {
+    for (const variant of PROMPT_VARIANTS) {
+      for (const mode of Object.keys(RUN_TEXT)) {
+        it(`${task} ${variant} ${mode}: conditions differ only in the library line`, () => {
+          const rendered = CONDITIONS.map((c) =>
+            renderPrompt(task, c, mode, variant).split("\n")
+          )
+          const [first, ...rest] = rendered
+          for (const [k, lines] of rest.entries()) {
+            assert.equal(lines.length, first.length)
+            const differing = lines
+              .map((line, i) => (line === first[i] ? null : i))
+              .filter((i) => i !== null)
+            // At most one line differs, and it is the {{LIBRARY}} line.
+            assert.ok(
+              differing.length <= 1,
+              `${CONDITIONS[k + 1]}: ${differing}`
+            )
+            const libraryLine = template(task, variant)
+              .split("\n")
+              .indexOf("{{LIBRARY}}")
+            for (const i of differing) assert.equal(i, libraryLine)
+            assert.equal(
+              lines[libraryLine],
+              conditionSpec(CONDITIONS[k + 1]).library
+            )
+          }
+        })
+      }
+    }
+  }
+  it("rejects an unknown variant", () => {
+    assert.throws(
+      () => renderPrompt("node-tool", "thin", "sandboxed", "terse"),
+      /unknown prompt variant/
+    )
+  })
+})
+
+// The contract the hidden tests read: stated in both variants.
+const CONTRACT = {
+  "node-tool": [
+    "createLedgerTool(config)",
+    "src/contract.ts",
+    "getBalance(owner: string): Promise<bigint>",
+    "icrc1_balance_of",
+    "icrc1_transfer",
+    "8 decimals",
+    "`config.rootKey`",
+    "`config.identity`",
+    "{ ok: true, blockIndex }",
+    "{ ok: false, mayHaveExecuted, reason }",
+  ],
+  "react-wallet": [
+    "`signin`",
+    "`signout`",
+    "`balance`",
+    "`transfer-to`",
+    "`transfer-amount`",
+    "`transfer-submit`",
+    "`transfer-status`",
+    "data-state",
+    "`idle`",
+    "`pending`",
+    "`success`",
+    "`error`",
+    "`unknown`",
+    "exactly 8",
+    "no grouping",
+    "icrc1_balance_of",
+    "icrc1_transfer",
+    "`rootKey`",
+  ],
+}
+
+// The safety rules the explicit prompt states and the minimal one must not.
+const RULES = [
+  /reject code/i,
+  /\b4xx\b|\b5xx\b|\b408\b|HTTP/,
+  /anonymous principal/i,
+  /\bnever\b/i,
+  /nat64|18446744073709551615/,
+  /more than 8 fraction digits/i,
+  /without contacting the ledger/i,
+  /fetch (a|the) root key/i,
+  /localhost|127\.0\.0\.1|\[::1\]/,
+  /again automatically|send it again|sent again/i,
+  /read again|re-?read/i,
+  /different principal|previous principal/i,
+  /`number`/,
+  /Requirements/,
+]
+
+describe("prompt variants", () => {
+  for (const task of TASKS) {
+    it(`${task}: both variants state the contract the tests read`, () => {
+      for (const variant of PROMPT_VARIANTS) {
+        const text = template(task, variant)
+        for (const needle of CONTRACT[task]) {
+          assert.ok(text.includes(needle), `${variant} lacks ${needle}`)
+        }
+      }
+    })
+    it(`${task}: the explicit prompt states the rules, the minimal one does not`, () => {
+      const explicit = template(task, "explicit")
+      const minimal = template(task, "minimal")
+      assert.ok(RULES.filter((r) => r.test(explicit)).length >= 8)
+      for (const rule of RULES) {
+        assert.ok(!rule.test(minimal), `minimal matches ${rule}`)
+      }
+    })
+  }
+  it("the minimal prompts define the outcome vocabulary", () => {
+    assert.match(
+      template("node-tool", "minimal"),
+      /`mayHaveExecuted`\s+says whether the transfer could nevertheless have taken\s+effect/
+    )
+    assert.match(
+      template("react-wallet", "minimal"),
+      /`unknown` \(it\s+failed, but may\s+nevertheless have taken effect\)/
+    )
+  })
+})