@ic-reactor/mcp 0.0.0-stage → 0.0.0

package/evals/harness/public/support/world.ts

@@ -0,0 +1,179 @@
+// A local test world: a fake Internet Computer replica (./fake-replica.js)
+// running a small ICRC-1 ledger with the interface in ./ledger.idl.js.
+//
+// The replica answers the IC HTTP API at `world.host` through
+// `globalThis.fetch`, and signs certificates and query responses with its own
+// root key (`world.rootKey`), so an agent built with that host and root key
+// talks to it exactly as it would talk to a replica.
+//
+// One replica serves a whole test file: it is installed when this module is
+// first imported (import it before the code under test), and `createWorld()`
+// gives each test a fresh ledger and request log behind it.
+import { Cbor } from "@icp-sdk/core/agent"
+import { IDL } from "@icp-sdk/core/candid"
+import { Ed25519KeyIdentity } from "@icp-sdk/core/identity"
+import { Principal } from "@icp-sdk/core/principal"
+import { installFakeReplica, type FakeCallContext } from "./fake-replica.js"
+import { idlFactory } from "./ledger.idl.js"
+
+export const LEDGER_ID = "ryjl3-tyaaa-aaaaa-aaaba-cai"
+export const HOST = "http://127.0.0.1:4943"
+export const FEE = 10_000n
+
+export interface LoggedRequest {
+  readonly endpoint: "status" | "query" | "call" | "read_state"
+  readonly method?: string
+  readonly sender?: string
+}
+
+export interface TestWorld {
+  readonly host: string
+  readonly rootKey: Uint8Array
+  readonly canisterId: string
+  /** Set the balance, in base units, of an owner's default account. */
+  setBalance(owner: string, e8s: bigint): void
+  balanceOf(owner: string): bigint
+  /** Every request an agent sent since `createWorld()`. */
+  readonly requests: readonly LoggedRequest[]
+  /** A new random signing identity. */
+  newIdentity(): Ed25519KeyIdentity
+}
+
+interface Account {
+  owner: Principal
+  subaccount: [] | [Uint8Array]
+}
+interface TransferArg {
+  to: Account
+  amount: bigint
+  fee: [] | [bigint]
+  memo: [] | [Uint8Array]
+  from_subaccount: [] | [Uint8Array]
+  created_at_time: [] | [bigint]
+}
+
+const keyOf = (owner: Principal, sub: [] | [Uint8Array]) => {
+  const bytes = sub[0]
+  if (!bytes || bytes.every((b) => b === 0)) return owner.toText()
+  return `${owner.toText()}:${Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("")}`
+}
+
+let balances = new Map<string, bigint>()
+let requests: LoggedRequest[] = []
+let nextBlock = 0n
+
+const ledger: Record<
+  string,
+  (args: unknown[], context: FakeCallContext) => unknown
+> = {
+  icrc1_name: () => "Test Token",
+  icrc1_symbol: () => "TST",
+  icrc1_decimals: () => 8,
+  icrc1_fee: () => FEE,
+  icrc1_metadata: () => [],
+  icrc1_total_supply: () => [...balances.values()].reduce((a, b) => a + b, 0n),
+  icrc1_minting_account: () => [],
+  icrc1_supported_standards: () => [
+    { name: "ICRC-1", url: "https://github.com/dfinity/ICRC-1" },
+  ],
+  icrc1_balance_of: ([account]) => {
+    const { owner, subaccount } = account as Account
+    return balances.get(keyOf(owner, subaccount)) ?? 0n
+  },
+  icrc1_transfer: ([raw], { caller }) => {
+    const arg = raw as TransferArg
+    const from = keyOf(caller, arg.from_subaccount)
+    if (arg.fee[0] !== undefined && arg.fee[0] !== FEE) {
+      return { Err: { BadFee: { expected_fee: FEE } } }
+    }
+    const balance = balances.get(from) ?? 0n
+    if (balance < arg.amount + FEE) {
+      return { Err: { InsufficientFunds: { balance } } }
+    }
+    const to = keyOf(arg.to.owner, arg.to.subaccount)
+    balances.set(from, balance - arg.amount - FEE)
+    balances.set(to, (balances.get(to) ?? 0n) + arg.amount)
+    return { Ok: nextBlock++ }
+  },
+}
+
+const service = idlFactory({ IDL }) as IDL.ServiceClass
+const funcs = new Map<string, IDL.FuncClass>(
+  service._fields as Array<[string, IDL.FuncClass]>
+)
+
+function run(
+  kind: "query" | "update",
+  method: string,
+  arg: Uint8Array,
+  context: FakeCallContext
+) {
+  const func = funcs.get(method)
+  const impl = ledger[method]
+  if (!func || !impl) throw new Error(`the ledger has no method ${method}`)
+  if (kind === "query" && !func.annotations.includes("query")) {
+    throw new Error(`${method} is not a query`)
+  }
+  const result = impl(IDL.decode(func.argTypes, arg), context)
+  return new Uint8Array(IDL.encode(func.retTypes, [result]))
+}
+
+const original = globalThis.fetch
+const replica = installFakeReplica({
+  host: HOST,
+  canisters: {
+    [LEDGER_ID]: {
+      query: (method, arg, context) => run("query", method, arg, context),
+      update: (method, arg, context) => run("update", method, arg, context),
+    },
+  },
+})
+const fake = globalThis.fetch
+
+// Log each request in front of the replica. This `fetch` stays installed for
+// the whole file, so agents built at module scope keep working across tests.
+globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+  const url = new URL(input instanceof Request ? input.url : String(input))
+  const match =
+    /\/api\/v\d+\/(?:canister\/[^/]+\/(query|call|read_state)|(status))$/.exec(
+      url.pathname
+    )
+  if (!match) return original(input, init)
+  if (match[2]) {
+    requests.push({ endpoint: "status" })
+    return fake(input, init)
+  }
+  const body = new Uint8Array(
+    await new Response(
+      input instanceof Request ? input.body : (init?.body as BodyInit)
+    ).arrayBuffer()
+  )
+  const { content } = Cbor.decode(body) as {
+    content: { sender?: Uint8Array; method_name?: string }
+  }
+  requests.push({
+    endpoint: match[1] as LoggedRequest["endpoint"],
+    method: content.method_name,
+    sender: content.sender
+      ? Principal.fromUint8Array(content.sender).toText()
+      : undefined,
+  })
+  return fake(url, { ...init, method: "POST", body: body as BodyInit })
+}) as typeof globalThis.fetch
+
+/** A fresh ledger and request log for one test. */
+export function createWorld(): TestWorld {
+  balances = new Map()
+  requests = []
+  return {
+    host: replica.host,
+    rootKey: replica.rootKey,
+    canisterId: LEDGER_ID,
+    setBalance: (owner, e8s) => void balances.set(owner, e8s),
+    balanceOf: (owner) => balances.get(owner) ?? 0n,
+    get requests() {
+      return requests
+    },
+    newIdentity: () => Ed25519KeyIdentity.generate(),
+  }
+}

package/evals/harness/runs.mjs

@@ -0,0 +1,139 @@
+// Preparing one agent run's private directory, shared by the driver and the
+// sandbox test so the test exercises exactly what the driver builds.
+//
+//   <batch>/<task>--<condition>--<nnn>/
+//     work/   the starter (task + condition layers + docs) with a copy of the
+//             condition's shipped node_modules (.ship/<base>/node_modules),
+//             git-initialised; the agent's cwd
+//     home/   a clean HOME; CLAUDE_CONFIG_DIR is home/.claude
+//     tmp/    the agent's TMPDIR
+//
+// The batch directory lives under the OS temp dir, never inside the
+// repository, and is created 0700.
+import { spawnSync } from "node:child_process"
+import {
+  chmodSync,
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  realpathSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs"
+import { tmpdir } from "node:os"
+import { join, relative } from "node:path"
+import {
+  EVALS,
+  assembleStarter,
+  baseCondition,
+  renderPrompt,
+} from "./assemble.mjs"
+
+export const REPO = realpathSync(join(EVALS, ".."))
+
+const insideRepo = (path) => {
+  const rel = relative(REPO, realpathSync(path))
+  return !rel.startsWith("..") && !rel.startsWith("/")
+}
+
+/** A fresh 0700 batch directory under the OS temp dir. */
+export function makeBatchDir() {
+  const dir = mkdtempSync(join(tmpdir(), "ic-reactor-eval-batch-"))
+  chmodSync(dir, 0o700)
+  if (insideRepo(dir)) {
+    rmSync(dir, { recursive: true, force: true })
+    throw new Error(
+      `batch directory ${dir} is inside the repository; set TMPDIR elsewhere`
+    )
+  }
+  return realpathSync(dir)
+}
+
+export function shippedNodeModules(condition) {
+  const dir = join(EVALS, ".ship", baseCondition(condition), "node_modules")
+  if (!existsSync(dir))
+    throw new Error(`missing ${dir}: run \`node setup.mjs\``)
+  return dir
+}
+
+/**
+ * Builds one run's directory tree. `mode` is "sandboxed" (the public test
+ * scaffold ships, and the prompt says code can be run) or "tsc-only".
+ * `prompt` is the prompt variant (assemble.mjs PROMPT_VARIANTS).
+ */
+export function prepareRun({
+  batchDir,
+  task,
+  condition,
+  i,
+  mode,
+  prompt = "explicit",
+}) {
+  const runRoot = join(
+    batchDir,
+    `${task}--${condition}--${String(i).padStart(3, "0")}`
+  )
+  const work = join(runRoot, "work")
+  const home = join(runRoot, "home")
+  const tmp = join(runRoot, "tmp")
+  for (const dir of [work, join(home, ".claude"), tmp])
+    mkdirSync(dir, { recursive: true })
+  if (insideRepo(runRoot))
+    throw new Error(`run directory ${runRoot} is inside the repository`)
+  assembleStarter({ task, condition, dest: work, mode, prompt })
+  // Relative .bin links stay relative; real files everywhere else.
+  cpSync(shippedNodeModules(condition), join(work, "node_modules"), {
+    recursive: true,
+    verbatimSymlinks: true,
+  })
+  writeFileSync(join(work, ".gitignore"), "node_modules\n")
+  spawnSync(
+    "sh",
+    [
+      "-c",
+      "git init -q && git add -A && git -c user.name=eval -c user.email=eval@localhost commit -qm starter",
+    ],
+    { cwd: work, env: { PATH: process.env.PATH, HOME: home } }
+  )
+  const promptFile = join(runRoot, "prompt.md")
+  writeFileSync(promptFile, renderPrompt(task, condition, mode, prompt))
+  return { runRoot, work, home, tmp, promptFile }
+}
+
+/**
+ * The agent's environment: nothing of the host's but PATH and ONE credential
+ * (`auth`, from harness/auth.mjs `resolveAuth`); HOME,
+ * CLAUDE_CONFIG_DIR and TMPDIR inside the run. The host's ~/.claude
+ * (settings, memory, OAuth login, MCP servers) is therefore not visible.
+ */
+export function agentEnv({ home, tmp }, auth) {
+  return {
+    PATH: process.env.PATH ?? "/usr/bin:/bin",
+    HOME: home,
+    CLAUDE_CONFIG_DIR: join(home, ".claude"),
+    TMPDIR: tmp,
+    // The CLI keeps its own scratch under /tmp/claude-<uid> regardless of
+    // TMPDIR; that directory is shared with every other session on the host
+    // and the sandbox denies it. This moves it inside the run.
+    CLAUDE_CODE_TMPDIR: tmp,
+    LANG: "en_US.UTF-8",
+    ...(auth ? { [auth.name]: auth.value } : {}),
+  }
+}
+
+/**
+ * A run directory with nothing in work/: what the preflight call runs in, so
+ * it gets exactly the isolation (environment, sandbox) a real run gets.
+ */
+export function prepareBareRun({ batchDir, name = "preflight" }) {
+  const runRoot = join(batchDir, name)
+  const work = join(runRoot, "work")
+  const home = join(runRoot, "home")
+  const tmp = join(runRoot, "tmp")
+  for (const dir of [work, join(home, ".claude"), tmp])
+    mkdirSync(dir, { recursive: true })
+  if (insideRepo(runRoot))
+    throw new Error(`run directory ${runRoot} is inside the repository`)
+  return { runRoot, work, home, tmp }
+}

package/evals/harness/sandbox.mjs

@@ -0,0 +1,88 @@
+// The OS-level sandbox an agent run executes in: a macOS sandbox-exec (SBPL)
+// profile. Docker was the first choice but no Docker daemon is available on
+// the machine this was built on; see README "Leak audit".
+//
+// The profile starts from `allow default` and then
+//   - denies reading file contents (file-read-data / file-read-xattr) and all
+//     writes under every place user data lives: /Users (every home, the
+//     repository, other projects), /private/var/folders (per-user temp dirs:
+//     other runs of the batch, the scorer's directory), /private/tmp and
+//     /Volumes;
+//   - allows reading the toolchain back (the node installation, the agent
+//     CLI, anything passed as `readOnly`);
+//   - allows reading and writing the run's own directory;
+//   - denies writes everywhere else except /dev.
+// Among rules for the same operation SBPL applies the last match, so the
+// allows (which name the same operations as the denies) override them.
+// Metadata (stat) stays readable everywhere so path resolution works: a
+// denied file's existence can be observed, its contents cannot.
+import { existsSync, realpathSync, writeFileSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { spawnSync } from "node:child_process"
+
+export const SANDBOX_EXEC = "/usr/bin/sandbox-exec"
+
+/**
+ * Whether agents can run in the OS sandbox: macOS with sandbox-exec.
+ * `EVALS_NO_SANDBOX=1` makes the harness behave as on a host without it (the
+ * driver then needs `--mode tsc-only`), which is how its tests exercise that
+ * fallback on macOS.
+ */
+export function sandboxAvailable(env = process.env) {
+  return (
+    env.EVALS_NO_SANDBOX !== "1" &&
+    process.platform === "darwin" &&
+    existsSync(SANDBOX_EXEC)
+  )
+}
+
+/** The node installation this process runs on, e.g. ~/.nvm/versions/node/v24.2.0. */
+export function nodeInstallDir() {
+  return dirname(dirname(realpathSync(process.execPath)))
+}
+
+/** The agent CLI's install directory, if `name` is on PATH. */
+export function cliInstallDir(name = "claude") {
+  const found = spawnSync("sh", ["-c", `command -v ${name}`], {
+    encoding: "utf8",
+  })
+  const bin = found.stdout.trim()
+  if (found.status !== 0 || !bin) return undefined
+  return dirname(realpathSync(bin))
+}
+
+const quote = (path) => JSON.stringify(path)
+
+export function sandboxProfile({ runRoot, readOnly = [] }) {
+  const root = realpathSync(runRoot)
+  const reads = [
+    ...new Set(readOnly.filter(Boolean).map((p) => realpathSync(p))),
+  ]
+  return [
+    "(version 1)",
+    "(allow default)",
+    ";; user data: contents unreadable, nothing writable",
+    ...["/Users", "/private/var/folders", "/private/tmp", "/Volumes"].map(
+      (p) =>
+        `(deny file-read-data file-read-xattr file-write* (subpath ${quote(p)}))`
+    ),
+    ";; no writes anywhere else either",
+    '(deny file-write* (subpath "/"))',
+    '(allow file-write* (subpath "/dev"))',
+    ";; the toolchain, read-only",
+    ...reads.map(
+      (p) => `(allow file-read-data file-read-xattr (subpath ${quote(p)}))`
+    ),
+    ";; the run itself. The allow names the same operations as the deny: SBPL",
+    ";; prefers a rule for a specific operation over a wildcard (file-read*).",
+    `(allow file-read-data file-read-xattr file-write* (subpath ${quote(root)}))`,
+    "",
+  ].join("\n")
+}
+
+/** Writes the profile next to the run and returns the command prefix. */
+export function sandboxPrefix({ runRoot, readOnly }) {
+  const profile = join(runRoot, "sandbox.sb")
+  writeFileSync(profile, sandboxProfile({ runRoot, readOnly }))
+  return [SANDBOX_EXEC, "-f", profile]
+}

package/evals/harness/sandbox.test.mjs

@@ -0,0 +1,134 @@
+// Proves the agent sandbox works both ways, on a run directory built exactly
+// as the driver builds it: the toolchain works inside (tsc, the public vitest
+// smoke test), and reads outside the run — the repository's hidden tests, a
+// sibling run of the same batch, the scorer's directory, the home directory —
+// and writes into the repository are refused. Every refusal is paired with the
+// same command outside the sandbox, which must succeed, so a pass is not
+// vacuous. Skipped where sandbox-exec does not exist.
+//
+//   node --test harness/sandbox.test.mjs
+import { strict as assert } from "node:assert"
+import { spawnSync } from "node:child_process"
+import { cpSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"
+import { homedir, tmpdir } from "node:os"
+import { join } from "node:path"
+import { after, before, describe, it } from "node:test"
+import { EVALS } from "./assemble.mjs"
+import { agentEnv, makeBatchDir, prepareRun, REPO } from "./runs.mjs"
+import { nodeInstallDir, sandboxAvailable, sandboxPrefix } from "./sandbox.mjs"
+
+const skip = !sandboxAvailable() && "sandbox-exec is not available here"
+
+let batch
+let run
+let sibling
+let prefix
+const scoreDir = join(tmpdir(), "ic-reactor-evals-score")
+const hidden = join(
+  EVALS,
+  "tasks",
+  "react-wallet",
+  "hidden",
+  "react-wallet.test.ts"
+)
+
+function inSandbox(command, timeout = 180_000) {
+  return spawnSync(prefix[0], [...prefix.slice(1), "sh", "-c", command], {
+    cwd: run.work,
+    env: agentEnv(run),
+    encoding: "utf8",
+    timeout,
+  })
+}
+function outside(command) {
+  return spawnSync("sh", ["-c", command], { cwd: run.work, encoding: "utf8" })
+}
+const readCmd = (path) =>
+  `node -e 'process.stdout.write(require("fs").readFileSync(${JSON.stringify(path)}, "utf8").slice(0, 20))'`
+
+describe("the agent sandbox", { skip }, () => {
+  before(() => {
+    batch = makeBatchDir()
+    run = prepareRun({
+      batchDir: batch,
+      task: "react-wallet",
+      condition: "v4-proto",
+      i: 1,
+      mode: "sandboxed",
+    })
+    // A working solution, so the public smoke test can pass inside.
+    cpSync(
+      join(
+        EVALS,
+        "tasks",
+        "react-wallet",
+        "solutions",
+        "v4-proto",
+        "reference",
+        "src"
+      ),
+      join(run.work, "src"),
+      { recursive: true }
+    )
+    sibling = join(batch, "react-wallet--thin--002", "work")
+    mkdirSync(sibling, { recursive: true })
+    writeFileSync(join(sibling, "secret.txt"), "another run's file")
+    mkdirSync(scoreDir, { recursive: true, mode: 0o700 })
+    writeFileSync(join(scoreDir, "probe.txt"), "a scoring file")
+    prefix = sandboxPrefix({
+      runRoot: run.runRoot,
+      readOnly: [nodeInstallDir()],
+    })
+  })
+  after(() => {
+    rmSync(batch, { recursive: true, force: true })
+    rmSync(join(scoreDir, "probe.txt"), { force: true })
+  })
+
+  it("runs tsc inside the run", () => {
+    const r = inSandbox("npx tsc --noEmit")
+    assert.equal(r.status, 0, r.stdout + r.stderr)
+  })
+
+  it("runs the public vitest smoke test inside the run", () => {
+    const r = inSandbox("npx vitest run --reporter=dot", 300_000)
+    assert.equal(r.status, 0, (r.stdout + r.stderr).slice(-3000))
+  })
+
+  it("can write inside the run", () => {
+    const r = inSandbox("echo ok > scratch.txt && cat scratch.txt")
+    assert.equal(r.stdout.trim(), "ok", r.stderr)
+  })
+
+  const refusals = {
+    "the repository's hidden tests": () => readCmd(hidden),
+    "a directory listing of the repository's evals/": () =>
+      `ls ${JSON.stringify(EVALS)}`,
+    "a sibling run of the same batch": () =>
+      readCmd(join(sibling, "secret.txt")),
+    "the scorer's private directory": () =>
+      readCmd(join(scoreDir, "probe.txt")),
+    "the home directory": () => `ls ${JSON.stringify(homedir())}`,
+  }
+  for (const [what, command] of Object.entries(refusals)) {
+    it(`refuses reading ${what}`, () => {
+      const control = outside(command())
+      assert.equal(
+        control.status,
+        0,
+        `control failed, test would be vacuous: ${control.stderr}`
+      )
+      const r = inSandbox(command())
+      assert.notEqual(r.status, 0, `read succeeded: ${r.stdout}`)
+      assert.match(r.stderr, /Operation not permitted|EPERM/)
+    })
+  }
+
+  it("refuses writing into the repository", () => {
+    const target = join(EVALS, "sandbox-write-probe.txt")
+    const r = inSandbox(`echo x > ${JSON.stringify(target)}`)
+    assert.notEqual(r.status, 0)
+    assert.equal(existsSync(target), false)
+    assert.ok(REPO.length > 0)
+  })
+})