@tangle-network/agent-bench 0.3.6 → 0.3.7

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.3.7
+
+- Declare the public executable search path for Pier candidate entrypoints so the exact process contract can replay them against `@tangle-network/agent-runtime@0.102.0`.
+- Align the portable agent contract with `@tangle-network/agent-interface@0.32.0`.
+- Add the trace-driven SWE supervisor improvement loop with measured candidate worktrees and official Docker judging.
+- Settle cancelled worker trees, isolate each cell's state, serialize judges with a kernel lock, and reject unfinished processes before patch extraction or scoring.
+
 ## 0.3.6
 
 - Ship compiled ESM and declarations so Node can import the installed package without a TypeScript runtime.

package/dist/adapters.js

@@ -1,7 +1,7 @@
 import {
   ADAPTERS,
   resolveAdapter
-} from "./chunk-5SBJCB6W.js";
+} from "./chunk-PWQVGAJB.js";
 import "./chunk-2PVVP7GN.js";
 import "./chunk-VQRS7VUC.js";
 import "./chunk-Z7ML6L77.js";
@@ -15,7 +15,7 @@ import "./chunk-R36V2VP7.js";
 import "./chunk-JRWWGMK7.js";
 import "./chunk-INNOYXCP.js";
 import "./chunk-PA2ZKHJC.js";
-import "./chunk-PPYSEKFM.js";
+import "./chunk-5H5XV76F.js";
 import "./chunk-TBKU5XQI.js";
 import "./chunk-TSWPNOYM.js";
 import "./chunk-KDIKRJGB.js";

package/dist/benchmarks/humaneval.d.ts

@@ -45,9 +45,18 @@ interface CheckResult {
      *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */
     detail?: string;
 }
+interface PythonProgramResult {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+}
+/** Run arbitrary Python in the same isolated container used by the HumanEval
+ * checker. This is also the execution-tool primitive for experiments that let
+ * a model test its own snippets; model-written code must never run on the host. */
+declare function runPythonProgram(program: string, timeoutMs?: number): Promise<PythonProgramResult>;
 declare function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult>;
 /** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle
  *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */
 declare function createHumanEvalAdapter(): BenchmarkAdapter;
 
-export { type CheckResult, type HumanEvalTask, basePrompt, createHumanEvalAdapter, extractCode, loadHumanEval, runChecker };
+export { type CheckResult, type HumanEvalTask, type PythonProgramResult, basePrompt, createHumanEvalAdapter, extractCode, loadHumanEval, runChecker, runPythonProgram };

package/dist/benchmarks/humaneval.js

@@ -3,13 +3,15 @@ import {
   createHumanEvalAdapter,
   extractCode,
   loadHumanEval,
-  runChecker
-} from "../chunk-PPYSEKFM.js";
+  runChecker,
+  runPythonProgram
+} from "../chunk-5H5XV76F.js";
 export {
   basePrompt,
   createHumanEvalAdapter,
   extractCode,
   loadHumanEval,
-  runChecker
+  runChecker,
+  runPythonProgram
 };
 //# sourceMappingURL=humaneval.js.map

package/dist/{chunk-PPYSEKFM.js → chunk-5H5XV76F.js}

@@ -1,6 +1,6 @@
 // src/benchmarks/humaneval.ts
 import { execFile } from "child_process";
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
 import { tmpdir } from "os";
 import { join } from "path";
 import { gunzipSync } from "zlib";
@@ -59,10 +59,12 @@ check(${task.entryPoint})
 `;
 }
 var dockerRunSeq = 0;
-function runChecker(task, candidate) {
+function runPythonProgram(program, timeoutMs = dockerTimeoutMs) {
   const dir = mkdtempSync(join(tmpdir(), "hev-"));
-  writeFileSync(join(dir, "p.py"), buildProgram(task, candidate));
+  chmodSync(dir, 493);
+  writeFileSync(join(dir, "p.py"), program, { mode: 420 });
   const name = `hev-${process.pid}-${dockerRunSeq++}`;
+  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`;
   return new Promise((resolvePromise, reject) => {
     let settled = false;
     const cleanup = () => {
@@ -84,49 +86,98 @@ function runChecker(task, candidate) {
       cleanup();
       reject(e);
     };
-    const backstop = setTimeout(() => finish({ pass: 0 }), dockerTimeoutMs + 3e3);
+    const outerTimeoutMs = timeoutMs + 3e3;
+    const backstop = setTimeout(
+      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3e3}ms`)),
+      outerTimeoutMs + 3e3
+    );
+    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1e3));
     execFile(
       "docker",
       [
         "run",
         "--rm",
+        "--pull=never",
         "--name",
         name,
         "--network=none",
         "--cpus=1",
         "--memory=512m",
+        "--pids-limit=64",
+        "--cap-drop=ALL",
+        "--security-opt=no-new-privileges",
+        "--read-only",
+        "--tmpfs",
+        "/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777",
+        "--user",
+        "65534:65534",
         "-v",
         `${dir}:/w:ro`,
         "-w",
         "/w",
         dockerImage,
-        "python",
-        "/w/p.py"
+        "sh",
+        "-c",
+        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf "%s\\n" "$1"; exec timeout -s KILL "$2" python /w/p.py',
+        "agent-runtime-checker",
+        startMarker,
+        `${inContainerSeconds}s`
       ],
-      { timeout: dockerTimeoutMs, killSignal: "SIGKILL", maxBuffer: 4 * 1024 * 1024 },
-      (err, _stdout, stderr) => {
+      { timeout: outerTimeoutMs, killSignal: "SIGKILL", maxBuffer: 4 * 1024 * 1024 },
+      (err, stdout, stderr) => {
+        const markerLine = `${startMarker}
+`;
+        const candidateStarted = stdout?.startsWith(markerLine) === true;
+        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : stdout ?? "";
         if (err) {
           const e = err;
           if (e.code === "ENOENT") {
             fail(new Error("docker binary not found on PATH \u2014 cannot run the deployable checker"));
             return;
           }
-          if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr)) {
-            fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`));
+          if (e.killed) {
+            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`));
             return;
           }
-          if (/(unable to find image|pull access denied|manifest unknown|error response from daemon).*(pull|repository|registry)/i.test(stderr)) {
-            fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`));
+          if (!candidateStarted) {
+            if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(
+              stderr
+            )) {
+              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`));
+              return;
+            }
+            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {
+              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`));
+              return;
+            }
+            fail(
+              new Error(
+                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`
+              )
+            );
             return;
           }
-          finish({ pass: 0, detail: (stderr || "").slice(-600) || "timed out (no output)" });
+          const exitCode = typeof e.code === "number" ? e.code : 1;
+          finish({
+            exitCode,
+            stdout: candidateStdout,
+            stderr: stderr ?? ""
+          });
           return;
         }
-        finish({ pass: 1 });
+        if (!candidateStarted) {
+          fail(new Error("docker checker exited without starting the candidate"));
+          return;
+        }
+        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? "" });
       }
     );
   });
 }
+async function runChecker(task, candidate) {
+  const result = await runPythonProgram(buildProgram(task, candidate));
+  return result.exitCode === 0 ? { pass: 1 } : { pass: 0, detail: result.stderr.slice(-600) || "timed out (no output)" };
+}
 function toBenchTask(t) {
   const metadata = {
     promptHeader: t.prompt,
@@ -153,6 +204,12 @@ function createHumanEvalAdapter() {
           else resolve();
         });
       });
+      await new Promise((resolve, reject) => {
+        execFile("docker", ["image", "inspect", dockerImage], (err) => {
+          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`));
+          else resolve();
+        });
+      });
     },
     async loadTasks(opts) {
       const offset = Number(process.env.OFFSET ?? 0);
@@ -176,7 +233,8 @@ export {
   loadHumanEval,
   basePrompt,
   extractCode,
+  runPythonProgram,
   runChecker,
   createHumanEvalAdapter
 };
-//# sourceMappingURL=chunk-PPYSEKFM.js.map
+//# sourceMappingURL=chunk-5H5XV76F.js.map

package/dist/chunk-5H5XV76F.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/humaneval.ts"],"sourcesContent":["/**\n * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the\n * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout\n * through the `Supervisor` that self-corrects across rounds, vs blind\n * random@k. This is the experiment `humaneval-gate.mts` names as \"the next one\" —\n * the gate measures SELECTION over stateless single completions; this measures\n * whether observe→steer (self-correction) beats blind compute at equal k.\n *\n * Worker artifact = the model's reply (a Python function, fenced or raw). The\n * DETERMINISTIC judge runs the candidate against the task's own `test` in an\n * isolated `--network=none` python container — exit 0 = pass. No gold\n * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only\n * to self-verify the judge before spending tokens.\n *\n * The primitives (loader / extractor / Docker checker) live here and are reused by\n * `humaneval-gate.mts` — one home, no duplication.\n */\n\nimport { execFile } from 'node:child_process'\nimport { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { gunzipSync } from 'node:zlib'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst humanevalUrl = 'https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz'\nconst dockerImage = 'python:3.12-slim'\nconst dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)\n\nexport interface HumanEvalTask {\n  taskId: string\n  prompt: string\n  test: string\n  entryPoint: string\n  /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */\n  canonicalSolution?: string\n}\n\n/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch\n *  or a malformed line — a silently-short task set would poison the gate. `offset`\n *  selects a deeper slice (the later tasks are harder) so the worker has a\n *  correctable middle band rather than a saturated easy prefix. */\nexport async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {\n  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits\n  // (429) under repeated runs. Fall back to the network fetch when unset.\n  const localGz = process.env.HUMANEVAL_GZ\n  let gz: Buffer\n  if (localGz) {\n    gz = readFileSync(localGz)\n  } else {\n    const res = await fetch(humanevalUrl)\n    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)\n    gz = Buffer.from(await res.arrayBuffer())\n  }\n  const text = gunzipSync(gz).toString('utf8')\n  const tasks: HumanEvalTask[] = []\n  for (const line of text.split('\\n')) {\n    if (line.trim() === '') continue\n    const d = JSON.parse(line) as {\n      task_id?: string\n      prompt?: string\n      test?: string\n      entry_point?: string\n      canonical_solution?: string\n    }\n    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {\n      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`)\n    }\n    tasks.push({\n      taskId: d.task_id,\n      prompt: d.prompt,\n      test: d.test,\n      entryPoint: d.entry_point,\n      ...(d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}),\n    })\n  }\n  if (tasks.length === 0) throw new Error('HumanEval parsed to 0 tasks')\n  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`)\n  return tasks.slice(offset, offset + limit)\n}\n\nconst solveInstruction =\n  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'\n\nexport function basePrompt(task: HumanEvalTask): string {\n  return `${solveInstruction}\\n\\n\\`\\`\\`python\\n${task.prompt}\\`\\`\\``\n}\n\n/** Extract the function source from a model reply: prefer a fenced ```python (or\n *  bare ```) block, else fall back to the raw text. The deployable program adds the\n *  prompt header (imports + signature context), so a candidate that returns only a\n *  body still runs; a candidate that re-defines the function shadows the header. */\nexport function extractCode(reply: string): string {\n  const fenced = reply.match(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i)\n  if (fenced && typeof fenced[1] === 'string') return fenced[1].trim()\n  return reply.trim()\n}\n\n/** The deployable test program: the prompt header (imports + signature/docstring the\n *  model was given), then the candidate (its def shadows the header's stub), then the\n *  task's own check() suite and the call. No gold solution anywhere. */\nfunction buildProgram(task: HumanEvalTask, candidate: string): string {\n  return `${task.prompt}\\n${candidate}\\n\\n${task.test}\\n\\ncheck(${task.entryPoint})\\n`\n}\n\nexport interface CheckResult {\n  /** {0,1} pass-count for this candidate (1 = the check() suite passed). */\n  pass: number\n  /** On failure: the interpreter stderr tail (traceback / failing assertion). The\n   *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */\n  detail?: string\n}\n\nexport interface PythonProgramResult {\n  exitCode: number\n  stdout: string\n  stderr: string\n}\n\n/** Run one candidate's deployable test program in an isolated container:\n *  read-only root, bounded writable `/tmp`, no network, one CPU, and 512 MiB.\n *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image\n *  unavailable) is NOT a test failure — it throws so the harness fails loud rather\n *  than scoring every candidate 0 from a broken checker. */\nlet dockerRunSeq = 0\n\n/** Run arbitrary Python in the same isolated container used by the HumanEval\n * checker. This is also the execution-tool primitive for experiments that let\n * a model test its own snippets; model-written code must never run on the host. */\nexport function runPythonProgram(\n  program: string,\n  timeoutMs = dockerTimeoutMs,\n): Promise<PythonProgramResult> {\n  const dir = mkdtempSync(join(tmpdir(), 'hev-'))\n  // Rootless Docker maps container root to a subordinate host uid that cannot\n  // traverse mkdtemp's default 0700 directory. The directory contains only the\n  // disposable candidate script and is mounted read-only into the container.\n  chmodSync(dir, 0o755)\n  writeFileSync(join(dir, 'p.py'), program, { mode: 0o644 })\n  // Unique container name so we can force-reap it regardless of the docker client's state.\n  const name = `hev-${process.pid}-${dockerRunSeq++}`\n  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`\n  return new Promise<PythonProgramResult>((resolvePromise, reject) => {\n    let settled = false\n    const cleanup = () => {\n      rmSync(dir, { recursive: true, force: true })\n      // `execFile`'s `timeout` kills the docker CLIENT, not the container — a hung\n      // `python` would otherwise pin a CPU forever. Force-reap by name (fire-and-forget;\n      // the name is unique, so no reuse race).\n      execFile('docker', ['rm', '-f', name], () => {})\n    }\n    const finish = (res: PythonProgramResult) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      resolvePromise(res)\n    }\n    const fail = (e: Error) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      reject(e)\n    }\n    // Candidate timeouts are enforced inside the container. If the outer Docker\n    // client or daemon misses this larger deadline, that is infrastructure failure,\n    // not a wrong answer.\n    const outerTimeoutMs = timeoutMs + 3_000\n    const backstop = setTimeout(\n      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3_000}ms`)),\n      outerTimeoutMs + 3_000,\n    )\n    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1_000))\n    execFile(\n      'docker',\n      [\n        'run',\n        '--rm',\n        '--pull=never',\n        '--name',\n        name,\n        '--network=none',\n        '--cpus=1',\n        '--memory=512m',\n        '--pids-limit=64',\n        '--cap-drop=ALL',\n        '--security-opt=no-new-privileges',\n        '--read-only',\n        '--tmpfs',\n        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',\n        '--user',\n        '65534:65534',\n        '-v',\n        `${dir}:/w:ro`,\n        '-w',\n        '/w',\n        dockerImage,\n        'sh',\n        '-c',\n        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf \"%s\\\\n\" \"$1\"; exec timeout -s KILL \"$2\" python /w/p.py',\n        'agent-runtime-checker',\n        startMarker,\n        `${inContainerSeconds}s`,\n      ],\n      { timeout: outerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },\n      (err, stdout, stderr) => {\n        const markerLine = `${startMarker}\\n`\n        const candidateStarted = stdout?.startsWith(markerLine) === true\n        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : (stdout ?? '')\n        if (err) {\n          const e = err as Error & { killed?: boolean; code?: number | string }\n          if (e.code === 'ENOENT') {\n            fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))\n            return\n          }\n          if (e.killed) {\n            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`))\n            return\n          }\n          if (!candidateStarted) {\n            if (\n              /cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(\n                stderr,\n              )\n            ) {\n              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {\n              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            fail(\n              new Error(\n                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`,\n              ),\n            )\n            return\n          }\n          const exitCode = typeof e.code === 'number' ? e.code : 1\n          finish({\n            exitCode,\n            stdout: candidateStdout,\n            stderr: stderr ?? '',\n          })\n          return\n        }\n        if (!candidateStarted) {\n          fail(new Error('docker checker exited without starting the candidate'))\n          return\n        }\n        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? '' })\n      },\n    )\n  })\n}\n\nexport async function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {\n  const result = await runPythonProgram(buildProgram(task, candidate))\n  return result.exitCode === 0\n    ? { pass: 1 }\n    : { pass: 0, detail: result.stderr.slice(-600) || 'timed out (no output)' }\n}\n\n/** A HumanEval task carries its checker inputs in metadata so the deterministic\n *  judge can rebuild the deployable program from a `BenchTask` alone. */\ninterface HumanEvalMeta extends Record<string, unknown> {\n  promptHeader: string\n  test: string\n  entryPoint: string\n  canonicalSolution?: string\n}\n\nfunction toBenchTask(t: HumanEvalTask): BenchTask {\n  const metadata: HumanEvalMeta = {\n    promptHeader: t.prompt,\n    test: t.test,\n    entryPoint: t.entryPoint,\n    ...(t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}),\n  }\n  return { id: t.taskId, prompt: basePrompt(t), metadata }\n}\n\nfunction taskFromMeta(task: BenchTask): HumanEvalTask {\n  const m = task.metadata as HumanEvalMeta | undefined\n  if (!m?.promptHeader || !m.test || !m.entryPoint) {\n    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`)\n  }\n  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint }\n}\n\n/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle\n *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */\nexport function createHumanEvalAdapter(): BenchmarkAdapter {\n  return {\n    name: 'humaneval',\n    async preflight() {\n      // Prove the daemon and exact local image before any model call. Scoring uses\n      // --pull=never so a registry or cold pull cannot become a candidate outcome.\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {\n          if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))\n          else resolve()\n        })\n      })\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['image', 'inspect', dockerImage], (err) => {\n          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`))\n          else resolve()\n        })\n      })\n    },\n    async loadTasks(opts?: LoadOptions) {\n      const offset = Number(process.env.OFFSET ?? 0)\n      // Pull a generous window when filtering by id, else exactly `limit` from offset.\n      const all = await loadHumanEval(opts?.ids ? 164 : (opts?.limit ?? 8), offset)\n      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all\n      return picked.map(toBenchTask)\n    },\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact))\n      return { resolved: pass === 1, score: pass, detail: pass === 1 ? 'tests passed' : 'tests failed' }\n    },\n    async goldArtifact(task: BenchTask) {\n      const m = task.metadata as HumanEvalMeta | undefined\n      const sol = m?.canonicalSolution\n      // Return the COMPLETE function (signature header + canonical body), i.e. what a\n      // real worker emits — NOT the body alone. The judge runs `extractCode`, whose\n      // unfenced fallback is `reply.trim()`; trimming a body-only string strips its\n      // leading indent and breaks it, so a body-only gold fails its own judge. A full\n      // def starts at column 0, trims safely, and self-verifies.\n      return sol ? `${m!.promptHeader}${sol}` : undefined\n    },\n  }\n}\n"],"mappings":";AAkBA,SAAS,gBAAgB;AACzB,SAAS,WAAW,aAAa,cAAc,QAAQ,qBAAqB;AAC5E,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,kBAAkB;AAG3B,IAAM,eAAe;AACrB,IAAM,cAAc;AACpB,IAAM,kBAAkB,OAAO,QAAQ,IAAI,qBAAqB,GAAK;AAerE,eAAsB,cAAc,OAAe,SAAS,GAA6B;AAGvF,QAAM,UAAU,QAAQ,IAAI;AAC5B,MAAI;AACJ,MAAI,SAAS;AACX,SAAK,aAAa,OAAO;AAAA,EAC3B,OAAO;AACL,UAAM,MAAM,MAAM,MAAM,YAAY;AACpC,QAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,wBAAwB,IAAI,MAAM,KAAK,YAAY,EAAE;AAClF,SAAK,OAAO,KAAK,MAAM,IAAI,YAAY,CAAC;AAAA,EAC1C;AACA,QAAM,OAAO,WAAW,EAAE,EAAE,SAAS,MAAM;AAC3C,QAAM,QAAyB,CAAC;AAChC,aAAW,QAAQ,KAAK,MAAM,IAAI,GAAG;AACnC,QAAI,KAAK,KAAK,MAAM,GAAI;AACxB,UAAM,IAAI,KAAK,MAAM,IAAI;AAOzB,QAAI,CAAC,EAAE,WAAW,CAAC,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,EAAE,aAAa;AACxD,YAAM,IAAI,MAAM,+BAA+B,KAAK,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACrE;AACA,UAAM,KAAK;AAAA,MACT,QAAQ,EAAE;AAAA,MACV,QAAQ,EAAE;AAAA,MACV,MAAM,EAAE;AAAA,MACR,YAAY,EAAE;AAAA,MACd,GAAI,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,mBAAmB,IAAI,CAAC;AAAA,IAC5E,CAAC;AAAA,EACH;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B;AACrE,MAAI,UAAU,MAAM,OAAQ,OAAM,IAAI,MAAM,UAAU,MAAM,oBAAoB,MAAM,MAAM,EAAE;AAC9F,SAAO,MAAM,MAAM,QAAQ,SAAS,KAAK;AAC3C;AAEA,IAAM,mBACJ;AAEK,SAAS,WAAW,MAA6B;AACtD,SAAO,GAAG,gBAAgB;AAAA;AAAA;AAAA,EAAqB,KAAK,MAAM;AAC5D;AAMO,SAAS,YAAY,OAAuB;AACjD,QAAM,SAAS,MAAM,MAAM,sCAAsC;AACjE,MAAI,UAAU,OAAO,OAAO,CAAC,MAAM,SAAU,QAAO,OAAO,CAAC,EAAE,KAAK;AACnE,SAAO,MAAM,KAAK;AACpB;AAKA,SAAS,aAAa,MAAqB,WAA2B;AACpE,SAAO,GAAG,KAAK,MAAM;AAAA,EAAK,SAAS;AAAA;AAAA,EAAO,KAAK,IAAI;AAAA;AAAA,QAAa,KAAK,UAAU;AAAA;AACjF;AAqBA,IAAI,eAAe;AAKZ,SAAS,iBACd,SACA,YAAY,iBACkB;AAC9B,QAAM,MAAM,YAAY,KAAK,OAAO,GAAG,MAAM,CAAC;AAI9C,YAAU,KAAK,GAAK;AACpB,gBAAc,KAAK,KAAK,MAAM,GAAG,SAAS,EAAE,MAAM,IAAM,CAAC;AAEzD,QAAM,OAAO,OAAO,QAAQ,GAAG,IAAI,cAAc;AACjD,QAAM,cAAc,qCAAqC,IAAI;AAC7D,SAAO,IAAI,QAA6B,CAAC,gBAAgB,WAAW;AAClE,QAAI,UAAU;AACd,UAAM,UAAU,MAAM;AACpB,aAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAI5C,eAAS,UAAU,CAAC,MAAM,MAAM,IAAI,GAAG,MAAM;AAAA,MAAC,CAAC;AAAA,IACjD;AACA,UAAM,SAAS,CAAC,QAA6B;AAC3C,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,qBAAe,GAAG;AAAA,IACpB;AACA,UAAM,OAAO,CAAC,MAAa;AACzB,UAAI,QAAS;AACb,gBAAU;AACV,mBAAa,QAAQ;AACrB,cAAQ;AACR,aAAO,CAAC;AAAA,IACV;AAIA,UAAM,iBAAiB,YAAY;AACnC,UAAM,WAAW;AAAA,MACf,MAAM,KAAK,IAAI,MAAM,wCAAwC,iBAAiB,GAAK,IAAI,CAAC;AAAA,MACxF,iBAAiB;AAAA,IACnB;AACA,UAAM,qBAAqB,KAAK,IAAI,GAAG,KAAK,KAAK,YAAY,GAAK,CAAC;AACnE;AAAA,MACE;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,GAAG;AAAA,QACN;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,kBAAkB;AAAA,MACvB;AAAA,MACA,EAAE,SAAS,gBAAgB,YAAY,WAAW,WAAW,IAAI,OAAO,KAAK;AAAA,MAC7E,CAAC,KAAK,QAAQ,WAAW;AACvB,cAAM,aAAa,GAAG,WAAW;AAAA;AACjC,cAAM,mBAAmB,QAAQ,WAAW,UAAU,MAAM;AAC5D,cAAM,kBAAkB,mBAAmB,OAAO,MAAM,WAAW,MAAM,IAAK,UAAU;AACxF,YAAI,KAAK;AACP,gBAAM,IAAI;AACV,cAAI,EAAE,SAAS,UAAU;AACvB,iBAAK,IAAI,MAAM,0EAAqE,CAAC;AACrF;AAAA,UACF;AACA,cAAI,EAAE,QAAQ;AACZ,iBAAK,IAAI,MAAM,+BAA+B,cAAc,kBAAkB,CAAC;AAC/E;AAAA,UACF;AACA,cAAI,CAAC,kBAAkB;AACrB,gBACE,8GAA8G;AAAA,cAC5G;AAAA,YACF,GACA;AACA,mBAAK,IAAI,MAAM,8BAA8B,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AACpE;AAAA,YACF;AACA,gBAAI,0EAA0E,KAAK,MAAM,GAAG;AAC1F,mBAAK,IAAI,MAAM,gBAAgB,WAAW,iBAAiB,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;AAClF;AAAA,YACF;AACA;AAAA,cACE,IAAI;AAAA,gBACF,+CAA+C,OAAO,MAAM,GAAG,GAAG,KAAK,EAAE,OAAO;AAAA,cAClF;AAAA,YACF;AACA;AAAA,UACF;AACA,gBAAM,WAAW,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;AACvD,iBAAO;AAAA,YACL;AAAA,YACA,QAAQ;AAAA,YACR,QAAQ,UAAU;AAAA,UACpB,CAAC;AACD;AAAA,QACF;AACA,YAAI,CAAC,kBAAkB;AACrB,eAAK,IAAI,MAAM,sDAAsD,CAAC;AACtE;AAAA,QACF;AACA,eAAO,EAAE,UAAU,GAAG,QAAQ,iBAAiB,QAAQ,UAAU,GAAG,CAAC;AAAA,MACvE;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,eAAsB,WAAW,MAAqB,WAAyC;AAC7F,QAAM,SAAS,MAAM,iBAAiB,aAAa,MAAM,SAAS,CAAC;AACnE,SAAO,OAAO,aAAa,IACvB,EAAE,MAAM,EAAE,IACV,EAAE,MAAM,GAAG,QAAQ,OAAO,OAAO,MAAM,IAAI,KAAK,wBAAwB;AAC9E;AAWA,SAAS,YAAY,GAA6B;AAChD,QAAM,WAA0B;AAAA,IAC9B,cAAc,EAAE;AAAA,IAChB,MAAM,EAAE;AAAA,IACR,YAAY,EAAE;AAAA,IACd,GAAI,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,kBAAkB,IAAI,CAAC;AAAA,EAC1E;AACA,SAAO,EAAE,IAAI,EAAE,QAAQ,QAAQ,WAAW,CAAC,GAAG,SAAS;AACzD;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,IAAI,KAAK;AACf,MAAI,CAAC,GAAG,gBAAgB,CAAC,EAAE,QAAQ,CAAC,EAAE,YAAY;AAChD,UAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,2BAA2B;AAAA,EAC7E;AACA,SAAO,EAAE,QAAQ,KAAK,IAAI,QAAQ,EAAE,cAAc,MAAM,EAAE,MAAM,YAAY,EAAE,WAAW;AAC3F;AAIO,SAAS,yBAA2C;AACzD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,MAAM,YAAY;AAGhB,YAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,iBAAS,UAAU,CAAC,WAAW,YAAY,qBAAqB,GAAG,CAAC,QAAQ;AAC1E,cAAI,IAAK,QAAO,IAAI,MAAM,kFAAkF,CAAC;AAAA,cACxG,SAAQ;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AACD,YAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,iBAAS,UAAU,CAAC,SAAS,WAAW,WAAW,GAAG,CAAC,QAAQ;AAC7D,cAAI,IAAK,QAAO,IAAI,MAAM,iDAAiD,WAAW,EAAE,CAAC;AAAA,cACpF,SAAQ;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAAA,IACA,MAAM,UAAU,MAAoB;AAClC,YAAM,SAAS,OAAO,QAAQ,IAAI,UAAU,CAAC;AAE7C,YAAM,MAAM,MAAM,cAAc,MAAM,MAAM,MAAO,MAAM,SAAS,GAAI,MAAM;AAC5E,YAAM,SAAS,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,SAAS,EAAE,MAAM,CAAC,IAAI;AAC7E,aAAO,OAAO,IAAI,WAAW;AAAA,IAC/B;AAAA,IACA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,EAAE,KAAK,IAAI,MAAM,WAAW,aAAa,IAAI,GAAG,YAAY,QAAQ,CAAC;AAC3E,aAAO,EAAE,UAAU,SAAS,GAAG,OAAO,MAAM,QAAQ,SAAS,IAAI,iBAAiB,eAAe;AAAA,IACnG;AAAA,IACA,MAAM,aAAa,MAAiB;AAClC,YAAM,IAAI,KAAK;AACf,YAAM,MAAM,GAAG;AAMf,aAAO,MAAM,GAAG,EAAG,YAAY,GAAG,GAAG,KAAK;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}

package/dist/{chunk-5SBJCB6W.js → chunk-PWQVGAJB.js}

@@ -36,7 +36,7 @@ import {
 } from "./chunk-PA2ZKHJC.js";
 import {
   createHumanEvalAdapter
-} from "./chunk-PPYSEKFM.js";
+} from "./chunk-5H5XV76F.js";
 import {
   createMind2WebAdapter
 } from "./chunk-TBKU5XQI.js";
@@ -141,4 +141,4 @@ export {
   ADAPTERS,
   resolveAdapter
 };
-//# sourceMappingURL=chunk-5SBJCB6W.js.map
+//# sourceMappingURL=chunk-PWQVGAJB.js.map

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   ADAPTERS,
   resolveAdapter
-} from "./chunk-5SBJCB6W.js";
+} from "./chunk-PWQVGAJB.js";
 import "./chunk-2PVVP7GN.js";
 import "./chunk-VQRS7VUC.js";
 import "./chunk-Z7ML6L77.js";
@@ -24,7 +24,7 @@ import "./chunk-R36V2VP7.js";
 import "./chunk-JRWWGMK7.js";
 import "./chunk-INNOYXCP.js";
 import "./chunk-PA2ZKHJC.js";
-import "./chunk-PPYSEKFM.js";
+import "./chunk-5H5XV76F.js";
 import "./chunk-TBKU5XQI.js";
 import {
   createNoMiraclAdapter

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "type": "module",
   "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "repository": {
@@ -25,10 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.122.8",
-    "@tangle-network/agent-interface": "0.31.0",
+    "@tangle-network/agent-eval": "0.123.0",
+    "@tangle-network/agent-interface": "0.32.0",
+    "@tangle-network/agent-knowledge": "^4.1.0",
     "@tangle-network/sandbox": "^0.11.1",
-    "@tangle-network/agent-runtime": "0.97.0"
+    "@tangle-network/agent-runtime": "0.102.1"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/scripts/run-package-tests.mjs

@@ -1,5 +1,5 @@
 import { execFile } from 'node:child_process'
-import { access, readdir } from 'node:fs/promises'
+import { access, readdir, readFile } from 'node:fs/promises'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { promisify } from 'node:util'
@@ -43,14 +43,36 @@ try {
 }
 
 const tests = await collectTests(sourceDir)
-const relativeTests = tests.map((file) => path.relative(benchDir, file))
-if (relativeTests.length === 0) throw new Error('no package tests found under src/')
+if (tests.length === 0) throw new Error('no package tests found under src/')
 
-await run(process.execPath, ['--test', '--import', 'tsx', ...relativeTests], {
-  ...process.env,
-  TSX_TSCONFIG_PATH: 'tsconfig.public.json',
-})
+// Two test runtimes coexist under src/: node:test files run under `node --test`;
+// vitest files (the swe-arena suite) crash there (`vitest` APIs need the vitest
+// worker), so partition by the framework each file actually imports.
+const nodeTests = []
+const vitestTests = []
+for (const file of tests) {
+  const body = await readFile(file, 'utf8')
+  if (/from\s+['"]vitest['"]/.test(body)) vitestTests.push(file)
+  else nodeTests.push(file)
+}
+
+if (nodeTests.length > 0) {
+  await run(
+    process.execPath,
+    ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))],
+    {
+      ...process.env,
+      TSX_TSCONFIG_PATH: 'tsconfig.public.json',
+    },
+  )
+}
+
+if (vitestTests.length > 0) {
+  await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))])
+}
 
 await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
 
-console.log(`package tests passed: ${tests.length}/${tests.length} TypeScript files + Pier bridge`)
+console.log(
+  `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`,
+)

package/scripts/verify-pier-agent.mts

@@ -370,6 +370,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
       },
       instructionDelivery: { kind: 'argv-append' as const },
       cwd: { workspace: 'task' as const, path: '.' },
+      env: { PATH: { kind: 'public' as const, value: '/usr/local/bin:/usr/bin:/bin' } },
       environment: { kind: 'evaluator-task-container' as const },
       workspace: candidateWorkspace,
       isolation: {

package/src/benchmarks/humaneval.test.mts

@@ -0,0 +1,122 @@
+import assert from 'node:assert/strict'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it } from 'node:test'
+import { runPythonProgram } from './humaneval'
+
+describe('HumanEval Python isolation', () => {
+  it('runs the exact program through the resource-capped networkless container', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'humaneval-docker-test-'))
+    const fakeDocker = join(dir, 'docker')
+    const capture = join(dir, 'capture.jsonl')
+    writeFileSync(
+      fakeDocker,
+      `#!/usr/bin/env node
+const fs = require('node:fs')
+const path = require('node:path')
+const args = process.argv.slice(2)
+if (args[0] === 'rm') process.exit(0)
+if (process.env.FAKE_DOCKER_MISSING === '1') {
+  process.stderr.write('docker: Error response from daemon: No such image: python:3.12-slim')
+  process.exit(125)
+}
+const mount = args[args.indexOf('-v') + 1]
+const hostDir = mount.slice(0, -':/w:ro'.length)
+fs.appendFileSync(process.env.FAKE_DOCKER_CAPTURE, JSON.stringify({
+  args,
+  program: fs.readFileSync(path.join(hostDir, 'p.py'), 'utf8'),
+}) + '\\n')
+if (process.env.FAKE_DOCKER_NO_START === '1') {
+  process.stderr.write('docker: Error response from daemon: unable to start container process')
+  process.exit(125)
+}
+process.stdout.write(args[args.length - 2] + '\\n')
+if (process.env.FAKE_DOCKER_CANDIDATE === 'daemon-text') {
+  process.stderr.write('Cannot connect to the Docker daemon')
+  process.exit(1)
+}
+if (process.env.FAKE_DOCKER_CANDIDATE === 'exit-125') process.exit(125)
+process.stdout.write('CONTAINER_OK\\n')
+`,
+      { mode: 0o755 },
+    )
+    const originalPath = process.env.PATH
+    const originalCapture = process.env.FAKE_DOCKER_CAPTURE
+    const originalMissing = process.env.FAKE_DOCKER_MISSING
+    const originalCandidate = process.env.FAKE_DOCKER_CANDIDATE
+    const originalNoStart = process.env.FAKE_DOCKER_NO_START
+    process.env.PATH = `${dir}:${originalPath ?? ''}`
+    process.env.FAKE_DOCKER_CAPTURE = capture
+    try {
+      const program = 'print("exact bytes")\n'
+      const result = await runPythonProgram(program, 2_000)
+      assert.equal(result.exitCode, 0)
+      assert.match(result.stdout, /CONTAINER_OK/)
+
+      const firstCall = JSON.parse(readFileSync(capture, 'utf8').trim().split('\n')[0]!) as {
+        args: string[]
+        program: string
+      }
+      assert.equal(firstCall.program, program)
+      assert.deepEqual(firstCall.args.slice(0, 2), ['run', '--rm'])
+      assert.ok(firstCall.args.includes('--network=none'))
+      assert.ok(firstCall.args.includes('--cpus=1'))
+      assert.ok(firstCall.args.includes('--memory=512m'))
+      assert.ok(firstCall.args.includes('--pids-limit=64'))
+      assert.ok(firstCall.args.includes('--cap-drop=ALL'))
+      assert.ok(firstCall.args.includes('--security-opt=no-new-privileges'))
+      assert.ok(firstCall.args.includes('--pull=never'))
+      assert.ok(firstCall.args.includes('--read-only'))
+      assert.equal(
+        firstCall.args[firstCall.args.indexOf('--tmpfs') + 1],
+        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',
+      )
+      assert.equal(firstCall.args[firstCall.args.indexOf('--user') + 1], '65534:65534')
+      assert.match(firstCall.args[firstCall.args.indexOf('-v') + 1] ?? '', /:\/w:ro$/)
+      const command = firstCall.args.slice(firstCall.args.indexOf('python:3.12-slim'))
+      assert.equal(command[0], 'python:3.12-slim')
+      assert.equal(command[1], 'sh')
+      assert.equal(command[2], '-c')
+      assert.match(command[3] ?? '', /command -v timeout/)
+      assert.equal(command[4], 'agent-runtime-checker')
+      assert.match(command[5] ?? '', /^__AGENT_RUNTIME_CANDIDATE_STARTED_hev-/)
+      assert.equal(command[6], '2s')
+
+      process.env.FAKE_DOCKER_MISSING = '1'
+      await assert.rejects(
+        runPythonProgram('print("must not score")\n', 2_000),
+        /docker image python:3\.12-slim unavailable/,
+      )
+
+      delete process.env.FAKE_DOCKER_MISSING
+      process.env.FAKE_DOCKER_CANDIDATE = 'daemon-text'
+      const daemonText = await runPythonProgram('raise SystemExit(1)\n', 2_000)
+      assert.equal(daemonText.exitCode, 1)
+      assert.match(daemonText.stderr, /Cannot connect to the Docker daemon/)
+
+      process.env.FAKE_DOCKER_CANDIDATE = 'exit-125'
+      const exit125 = await runPythonProgram('raise SystemExit(125)\n', 2_000)
+      assert.equal(exit125.exitCode, 125)
+
+      delete process.env.FAKE_DOCKER_CANDIDATE
+      process.env.FAKE_DOCKER_NO_START = '1'
+      await assert.rejects(
+        runPythonProgram('print("never started")\n', 2_000),
+        /did not start the candidate/,
+      )
+    } finally {
+      if (originalPath === undefined) delete process.env.PATH
+      else process.env.PATH = originalPath
+      if (originalCapture === undefined) delete process.env.FAKE_DOCKER_CAPTURE
+      else process.env.FAKE_DOCKER_CAPTURE = originalCapture
+      if (originalMissing === undefined) delete process.env.FAKE_DOCKER_MISSING
+      else process.env.FAKE_DOCKER_MISSING = originalMissing
+      if (originalCandidate === undefined) delete process.env.FAKE_DOCKER_CANDIDATE
+      else process.env.FAKE_DOCKER_CANDIDATE = originalCandidate
+      if (originalNoStart === undefined) delete process.env.FAKE_DOCKER_NO_START
+      else process.env.FAKE_DOCKER_NO_START = originalNoStart
+      rmSync(dir, { recursive: true, force: true })
+    }
+  })
+})