@tangle-network/agent-bench 0.3.7 → 0.4.0

package/dist/{chunk-UAIOHCUK.js → chunk-SFLA7OH3.js}

@@ -1,9 +1,9 @@
 import {
   createTauBenchAdapter
-} from "./chunk-R36V2VP7.js";
+} from "./chunk-IZ5M6OAC.js";
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/tau3-banking.ts
 import { join } from "path";
@@ -24,4 +24,4 @@ function createTau3BankingAdapter() {
 export {
   createTau3BankingAdapter
 };
-//# sourceMappingURL=chunk-UAIOHCUK.js.map
+//# sourceMappingURL=chunk-SFLA7OH3.js.map

package/dist/{chunk-Y6O2OCUO.js → chunk-SHM6MRRF.js}

@@ -14,7 +14,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/t2-ragbench.ts
 import { readFile } from "fs/promises";
@@ -127,4 +127,4 @@ function createT2RagBenchAdapter() {
 export {
   createT2RagBenchAdapter
 };
-//# sourceMappingURL=chunk-Y6O2OCUO.js.map
+//# sourceMappingURL=chunk-SHM6MRRF.js.map

package/dist/{chunk-KDIKRJGB.js → chunk-SHYIRB7I.js}

@@ -14,7 +14,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/open-rag-bench.ts
 import { readFile } from "fs/promises";
@@ -117,4 +117,4 @@ function createOpenRagBenchAdapter() {
 export {
   createOpenRagBenchAdapter
 };
-//# sourceMappingURL=chunk-KDIKRJGB.js.map
+//# sourceMappingURL=chunk-SHYIRB7I.js.map

package/dist/{chunk-HHXFIHXC.js → chunk-SVR2LKYI.js}

@@ -12,7 +12,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/crag.ts
 import { readFile } from "fs/promises";
@@ -113,4 +113,4 @@ function createCragAdapter() {
 export {
   createCragAdapter
 };
-//# sourceMappingURL=chunk-HHXFIHXC.js.map
+//# sourceMappingURL=chunk-SVR2LKYI.js.map

package/dist/{chunk-PWQVGAJB.js → chunk-V7AEBY6U.js}

@@ -1,33 +1,33 @@
 import {
   createToolLlmAdapter
-} from "./chunk-2PVVP7GN.js";
+} from "./chunk-XKEFIFIC.js";
 import {
   createTrataHedgeAdapter
 } from "./chunk-VQRS7VUC.js";
 import {
   createWebArenaVerifiedAdapter
-} from "./chunk-Z7ML6L77.js";
+} from "./chunk-HWST3SED.js";
 import {
   createRagBenchAdapter
-} from "./chunk-R67DFVLO.js";
+} from "./chunk-IFVINJ4B.js";
 import {
   createSimpleQaAdapter
 } from "./chunk-SEVJPLZC.js";
 import {
   createSweBenchAdapter
-} from "./chunk-CKUVRZ2T.js";
+} from "./chunk-3U5TXJZS.js";
 import {
   createT2RagBenchAdapter
-} from "./chunk-Y6O2OCUO.js";
+} from "./chunk-SHM6MRRF.js";
 import {
   createTau2BenchAdapter
-} from "./chunk-HBSWHQNJ.js";
+} from "./chunk-RCYQEFNX.js";
 import {
   createTau3BankingAdapter
-} from "./chunk-UAIOHCUK.js";
+} from "./chunk-SFLA7OH3.js";
 import {
   createTerminalBenchAdapter
-} from "./chunk-JRWWGMK7.js";
+} from "./chunk-XYA4XSNU.js";
 import {
   createFramesAdapter
 } from "./chunk-INNOYXCP.js";
@@ -42,13 +42,13 @@ import {
 } from "./chunk-TBKU5XQI.js";
 import {
   createNoMiraclAdapter
-} from "./chunk-TSWPNOYM.js";
+} from "./chunk-NQG5XDSB.js";
 import {
   createOpenRagBenchAdapter
-} from "./chunk-KDIKRJGB.js";
+} from "./chunk-SHYIRB7I.js";
 import {
   createProgrambenchAdapter
-} from "./chunk-ZEWMTR5M.js";
+} from "./chunk-MQMRLGOG.js";
 import {
   createCadBenchAdapter
 } from "./chunk-PUIRNYI7.js";
@@ -57,35 +57,35 @@ import {
 } from "./chunk-C7T7WEK2.js";
 import {
   createCommit0Adapter
-} from "./chunk-2XU6OGEN.js";
+} from "./chunk-Z4TZ76N7.js";
 import {
   createCragAdapter
-} from "./chunk-HHXFIHXC.js";
+} from "./chunk-SVR2LKYI.js";
 import {
   createDabstepAdapter
-} from "./chunk-J3KDJNX2.js";
+} from "./chunk-RH5F53JT.js";
 import {
   createEnterpriseOpsGymAdapter
-} from "./chunk-X5YKXC6V.js";
+} from "./chunk-YSMEKBTD.js";
 import {
   createFinResearchBenchAdapter
-} from "./chunk-YCGY7UIZ.js";
+} from "./chunk-7GRVHU22.js";
 import {
   createFinsearchcompAdapter
 } from "./chunk-UPAMRDX4.js";
 import {
   createAecBenchAdapter
-} from "./chunk-ODT47UAY.js";
+} from "./chunk-K3BQGZCT.js";
 import {
   createAgentBenchAdapter
-} from "./chunk-7WSD27QQ.js";
+} from "./chunk-PB64GYIG.js";
 import {
   createAppWorldAdapter,
   createAppWorldReactAdapter
-} from "./chunk-SYDW647C.js";
+} from "./chunk-IA2FBTWC.js";
 import {
   createBfclAdapter
-} from "./chunk-IFAV6KEM.js";
+} from "./chunk-KP5KD6EN.js";
 import {
   createCadDesignAdapter
 } from "./chunk-53UPUNBZ.js";
@@ -141,4 +141,4 @@ export {
   ADAPTERS,
   resolveAdapter
 };
-//# sourceMappingURL=chunk-PWQVGAJB.js.map
+//# sourceMappingURL=chunk-V7AEBY6U.js.map

package/dist/{chunk-LRRD7NAG.js → chunk-WSKWVEQB.js}

@@ -1,6 +1,7 @@
 // src/benchmarks/_harness.ts
 import { execFile, spawn } from "child_process";
 import { createHash } from "crypto";
+import { existsSync, readFileSync } from "fs";
 import {
   cp,
   lstat,
@@ -18,7 +19,22 @@ import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "pat
 import { fileURLToPath } from "url";
 import { promisify } from "util";
 var execFileAsync = promisify(execFile);
-var benchRoot = fileURLToPath(new URL("../..", import.meta.url));
+function resolveBenchRoot(moduleUrl) {
+  let current = dirname(fileURLToPath(moduleUrl));
+  while (true) {
+    const manifestPath = join(current, "package.json");
+    if (existsSync(manifestPath)) {
+      const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+      if (manifest.name === "@tangle-network/agent-bench") return current;
+    }
+    const parent = dirname(current);
+    if (parent === current) {
+      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`);
+    }
+    current = parent;
+  }
+}
+var benchRoot = resolveBenchRoot(import.meta.url);
 function resolveBenchPython(env = process.env, root = benchRoot) {
   const configured = env.AGENT_BENCH_PYTHON;
   if (configured === void 0) return join(root, ".venv", "bin", "python");
@@ -298,4 +314,4 @@ export {
   stageFile,
   safeRunId
 };
-//# sourceMappingURL=chunk-LRRD7NAG.js.map
+//# sourceMappingURL=chunk-WSKWVEQB.js.map

package/dist/chunk-WSKWVEQB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/_harness.ts"],"sourcesContent":["/**\n * Shared code-bench harness. The \"stage the artifact → run the benchmark's own\n * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)\n * → read its JSON report → { resolved, score }\" spine that swe-bench,\n * terminal-bench, commit0, programbench and aec-bench all need. Factored out so\n * the Docker/venv/report-reading logic lives in ONE place instead of being\n * copy-pasted per adapter.\n *\n * It owns NO benchmark policy: each adapter passes the argv for its evaluator\n * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The\n * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config\n * and fail-loud diagnostics.\n *\n * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`\n * (the adapter passes the import line + the exact fix). A staged run that exits\n * nonzero throws with the captured stderr — never a fabricated score.\n */\n\nimport { execFile, spawn } from 'node:child_process'\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport {\n  cp,\n  lstat,\n  mkdir,\n  mkdtemp,\n  readFile,\n  readlink,\n  readdir,\n  rename,\n  rm,\n  writeFile,\n} from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type {\n  BenchScore,\n  JudgeArtifactFileReceipt,\n  JudgeArtifactReceipt,\n} from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Locate the package by identity because source files and compiled chunks have different depths. */\nfunction resolveBenchRoot(moduleUrl: string): string {\n  let current = dirname(fileURLToPath(moduleUrl))\n  while (true) {\n    const manifestPath = join(current, 'package.json')\n    if (existsSync(manifestPath)) {\n      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }\n      if (manifest.name === '@tangle-network/agent-bench') return current\n    }\n    const parent = dirname(current)\n    if (parent === current) {\n      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)\n    }\n    current = parent\n  }\n}\n\n/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */\nexport const benchRoot = resolveBenchRoot(import.meta.url)\n\n/** Resolve the shared interpreter without requiring an installed package to contain a venv. */\nexport function resolveBenchPython(\n  env: Readonly<{ AGENT_BENCH_PYTHON?: string }> = process.env,\n  root: string = benchRoot,\n): string {\n  const configured = env.AGENT_BENCH_PYTHON\n  if (configured === undefined) return join(root, '.venv', 'bin', 'python')\n  if (!isAbsolute(configured)) {\n    throw new Error('AGENT_BENCH_PYTHON must be an absolute path')\n  }\n  return configured\n}\n\n/** The shared interpreter every Python-backed evaluator runs through. */\nexport const venvPython = resolveBenchPython()\n\n/** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip\n *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)\n *  get their own venv and pass its python explicitly — keeping the shared one clean. */\nexport const venvPythonAt = (venvDir: string): string => join(benchRoot, venvDir, 'bin', 'python')\n/** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */\nexport const bigBuffer = 1024 * 1024 * 256\n\n/** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */\nexport function venvBin(name: string): string {\n  return join(benchRoot, '.venv', 'bin', name)\n}\n\n/**\n * Run the bench venv python with an inline script (`-c`); return stdout. Throws\n * (with stderr) on a nonzero exit — the loaders rely on this to fail loud rather\n * than parse a partial dump.\n */\nexport async function runVenvPython(\n  script: string,\n  args: string[] = [],\n  timeoutMs = 0,\n  python: string = venvPython,\n): Promise<string> {\n  const { stdout } = await execFileAsync(python, ['-c', script, ...args], {\n    maxBuffer: bigBuffer,\n    timeout: timeoutMs,\n  })\n  return stdout\n}\n\n/**\n * Preflight a python-backed harness: import the module(s) and (optionally) ping\n * Docker, all inside the bench venv. On failure THROWS the captured error joined\n * to the adapter's `fix` guidance — the contract every code-bench preflight wants.\n */\nexport async function preflightVenvImports(opts: {\n  /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */\n  modules: string[]\n  /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */\n  requireDocker?: boolean\n  /** Actionable remediation appended to the thrown message. */\n  fix: string\n  /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */\n  python?: string\n}): Promise<void> {\n  const imports = opts.modules.filter((m) => m.length > 0)\n  const lines = [...imports.map((m) => `import ${m}`)]\n  if (opts.requireDocker) lines.push('import docker', 'docker.from_env().ping()')\n  lines.push(\"print('ok')\")\n  try {\n    await runVenvPython(lines.join('\\n'), [], 0, opts.python ?? venvPython)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    throw new Error(`${msg}\\n${opts.fix}`)\n  }\n}\n\n/**\n * Run a bench-local python driver script (in the bench venv) while piping\n * `input` to its stdin, returning stdout. The driver's verdict JSON is its LAST\n * stdout line; callers parse that and inspect an `error` field (fail loud).\n *\n * Uses spawn + an explicit `stdin.end(input)` rather than promisify(execFile)'s\n * `input` option, because that option is NOT honored by async execFile — stdin\n * is left open and a driver that does `sys.stdin.read()` blocks forever. The\n * artifact-piping judges (commit0, appworld) MUST go through this.\n */\nexport function runVenvScriptStdin(\n  scriptPath: string,\n  args: string[],\n  input: string,\n  opts: { cwd?: string; timeoutMs?: number; python?: string } = {},\n): Promise<string> {\n  return new Promise<string>((resolve, reject) => {\n    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {\n      cwd: opts.cwd ?? benchRoot,\n      ...(opts.timeoutMs ? { timeout: opts.timeoutMs } : {}),\n    })\n    let stdout = ''\n    let stderr = ''\n    let bytes = 0\n    child.stdout.on('data', (c: Buffer) => {\n      bytes += c.length\n      if (bytes <= bigBuffer) stdout += c.toString('utf8')\n    })\n    child.stderr.on('data', (c: Buffer) => {\n      stderr += c.toString('utf8')\n    })\n    child.on('error', (err) => reject(err))\n    child.on('close', (code) => {\n      if (code === 0) resolve(stdout)\n      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)))\n    })\n    child.stdin.end(input)\n  })\n}\n\nexport interface StagedRunSpec {\n  /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */\n  tmpPrefix: string\n  /**\n   * Write the artifact + any harness inputs into the temp dir. Returns nothing;\n   * `argv`/`cwd` consume `dir` to point the evaluator at what was written.\n   */\n  stage(dir: string): Promise<void>\n  /** The external evaluator to spawn. `bin` defaults to the bench venv python. */\n  bin?: string\n  /** argv for the evaluator, computed from the temp `dir`. */\n  argv(dir: string): string[]\n  /** Working directory for the evaluator. Defaults to the temp `dir`. */\n  cwd?(dir: string): string\n  /** Hard timeout for the evaluator (ms); 0 = none. */\n  timeoutMs?: number\n  /**\n   * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws\n   * if the expected report is absent/malformed (fail loud — no default score).\n   */\n  parseReport(dir: string): Promise<BenchScore>\n  /**\n   * Copy the complete evaluator directory plus raw process stdout/stderr to this\n   * caller-owned directory before cleanup. The destination must not exist.\n   */\n  capture?: StagedRunCaptureSpec\n  /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */\n  keepTmp?: boolean\n}\n\nexport interface StagedRunCaptureSpec {\n  /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */\n  destination: string\n}\n\n/** A staged run failed after any requested evidence was durably retained. */\nexport class StagedJudgeError extends Error {\n  readonly judgeArtifacts?: JudgeArtifactReceipt\n\n  constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions) {\n    super(message, options)\n    this.name = 'StagedJudgeError'\n    this.judgeArtifacts = judgeArtifacts\n  }\n}\n\nfunction sha256(bytes: Uint8Array): `sha256:${string}` {\n  return `sha256:${createHash('sha256').update(bytes).digest('hex')}`\n}\n\nfunction portablePath(path: string): string {\n  return path.split(sep).join('/')\n}\n\nfunction compareText(left: string, right: string): number {\n  return left < right ? -1 : left > right ? 1 : 0\n}\n\nasync function collectArtifactFiles(\n  root: string,\n  current: string,\n): Promise<JudgeArtifactFileReceipt[]> {\n  const absolute = join(root, current)\n  const entries = await readdir(absolute, { withFileTypes: true })\n  const files: JudgeArtifactFileReceipt[] = []\n  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {\n    const relativePath = join(current, entry.name)\n    const path = join(root, relativePath)\n    if (entry.isDirectory()) {\n      files.push(...await collectArtifactFiles(root, relativePath))\n      continue\n    }\n    if (entry.isFile()) {\n      const bytes = await readFile(path)\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: bytes.byteLength,\n        sha256: sha256(bytes),\n        kind: 'file',\n      })\n      continue\n    }\n    if (entry.isSymbolicLink()) {\n      const targetBytes = await readlink(path, { encoding: 'buffer' })\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: targetBytes.byteLength,\n        sha256: sha256(targetBytes),\n        kind: 'symlink',\n      })\n      continue\n    }\n    throw new Error(`staged judge capture does not support ${relativePath}`)\n  }\n  return files\n}\n\nfunction isWithin(parent: string, candidate: string): boolean {\n  const path = relative(parent, candidate)\n  return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path))\n}\n\nasync function assertDestinationAbsent(destination: string): Promise<void> {\n  try {\n    await lstat(destination)\n  } catch (error) {\n    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return\n    throw error\n  }\n  throw new Error(`staged judge capture destination already exists: ${destination}`)\n}\n\nasync function captureStagedRun(\n  sourceDirectory: string,\n  spec: StagedRunCaptureSpec,\n  processOutput: Readonly<{ stdout: Buffer; stderr: Buffer }>,\n  evaluatorSucceeded: boolean,\n): Promise<JudgeArtifactReceipt> {\n  const source = resolve(sourceDirectory)\n  const destination = resolve(spec.destination)\n  if (isWithin(source, destination)) {\n    throw new Error('staged judge capture destination must be outside the evaluator directory')\n  }\n  await mkdir(dirname(destination), { recursive: true })\n  await assertDestinationAbsent(destination)\n  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`))\n  try {\n    await cp(source, join(staging, 'evaluator'), {\n      recursive: true,\n      errorOnExist: true,\n      force: false,\n      preserveTimestamps: true,\n      verbatimSymlinks: true,\n    })\n    await mkdir(join(staging, 'process'))\n    await writeFile(join(staging, 'process', 'stdout.bin'), processOutput.stdout)\n    await writeFile(join(staging, 'process', 'stderr.bin'), processOutput.stderr)\n\n    const files = [\n      ...await collectArtifactFiles(staging, 'evaluator'),\n      ...await collectArtifactFiles(staging, 'process'),\n    ].sort((left, right) => compareText(left.path, right.path))\n    const byteLength = files.reduce((total, file) => total + file.byteLength, 0)\n    const treeBytes = Buffer.from(\n      files\n        .map((file) => `${file.path}\\0${file.kind}\\0${file.byteLength}\\0${file.sha256}\\n`)\n        .join(''),\n      'utf8',\n    )\n    const receipt: JudgeArtifactReceipt = {\n      schema: 'agent-bench/judge-artifacts/v1',\n      directory: destination,\n      evaluatorDirectory: join(destination, 'evaluator'),\n      manifestPath: join(destination, 'receipt.json'),\n      evaluatorSucceeded,\n      files,\n      fileCount: files.length,\n      byteLength,\n      treeSha256: sha256(treeBytes),\n    }\n    await writeFile(join(staging, 'receipt.json'), `${JSON.stringify(receipt, null, 2)}\\n`)\n    await rename(staging, destination)\n    return receipt\n  } catch (error) {\n    await rm(staging, { recursive: true, force: true }).catch(() => {})\n    throw error\n  }\n}\n\nfunction processBytes(value: unknown): Buffer {\n  if (Buffer.isBuffer(value)) return value\n  if (value === undefined || value === null) return Buffer.alloc(0)\n  return Buffer.from(String(value), 'utf8')\n}\n\n/**\n * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →\n * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is\n * always removed in `finally` unless `keepTmp`.\n */\nexport async function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore> {\n  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix))\n  let stdout: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let stderr: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let evaluatorSucceeded = false\n  let score: BenchScore | undefined\n  let failure: unknown\n  try {\n    try {\n      await spec.stage(dir)\n      const bin = spec.bin ?? venvPython\n      const argv = spec.argv(dir)\n      try {\n        const output = await execFileAsync(bin, argv, {\n          cwd: spec.cwd ? spec.cwd(dir) : dir,\n          encoding: 'buffer',\n          maxBuffer: bigBuffer,\n          ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),\n        })\n        stdout = processBytes(output.stdout)\n        stderr = processBytes(output.stderr)\n        evaluatorSucceeded = true\n      } catch (err) {\n        const e = err as { stderr?: unknown; stdout?: unknown; message?: string }\n        stdout = processBytes(e.stdout)\n        stderr = processBytes(e.stderr)\n        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err))\n          .toString('utf8')\n          .slice(0, 2000)\n        throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${argv.join(' ')}):\\n${detail}`)\n      }\n      score = await spec.parseReport(dir)\n    } catch (err) {\n      failure = err\n    }\n  } finally {\n    let judgeArtifacts: JudgeArtifactReceipt | undefined\n    if (spec.capture) {\n      try {\n        judgeArtifacts = await captureStagedRun(\n          dir,\n          spec.capture,\n          { stdout, stderr },\n          evaluatorSucceeded,\n        )\n      } catch (captureError) {\n        failure = new Error(\n          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,\n          { cause: failure ?? captureError },\n        )\n      }\n    }\n    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {})\n    if (failure) {\n      throw new StagedJudgeError(\n        failure instanceof Error ? failure.message : String(failure),\n        judgeArtifacts,\n        { cause: failure },\n      )\n    }\n    if (!score) throw new StagedJudgeError('staged judge completed without a score', judgeArtifacts)\n    return judgeArtifacts ? { ...score, judgeArtifacts } : score\n  }\n}\n\n/** Read + JSON.parse a report file from a staged run; throws with the path on failure. */\nexport async function readJsonReport<T>(path: string): Promise<T> {\n  let raw: string\n  try {\n    raw = await readFile(path, 'utf8')\n  } catch (err) {\n    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`)\n  }\n  try {\n    return JSON.parse(raw) as T\n  } catch (err) {\n    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`)\n  }\n}\n\n/** Write a UTF-8 file into a staged dir (artifact / preds.json / attempt.sh / …). */\nexport async function stageFile(path: string, content: string): Promise<void> {\n  await writeFile(path, content)\n}\n\n/** Sanitize an instance id into a filesystem/run-id-safe token. */\nexport function safeRunId(prefix: string, id: string): string {\n  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, '_')\n}\n"],"mappings":";AAkBA,SAAS,UAAU,aAAa;AAChC,SAAS,kBAAkB;AAC3B,SAAS,YAAY,oBAAoB;AACzC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AACvB,SAAS,UAAU,SAAS,YAAY,MAAM,UAAU,SAAS,WAAW;AAC5E,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAO1B,IAAM,gBAAgB,UAAU,QAAQ;AAGxC,SAAS,iBAAiB,WAA2B;AACnD,MAAI,UAAU,QAAQ,cAAc,SAAS,CAAC;AAC9C,SAAO,MAAM;AACX,UAAM,eAAe,KAAK,SAAS,cAAc;AACjD,QAAI,WAAW,YAAY,GAAG;AAC5B,YAAM,WAAW,KAAK,MAAM,aAAa,cAAc,MAAM,CAAC;AAC9D,UAAI,SAAS,SAAS,8BAA+B,QAAO;AAAA,IAC9D;AACA,UAAM,SAAS,QAAQ,OAAO;AAC9B,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI,MAAM,qDAAqD,SAAS,EAAE;AAAA,IAClF;AACA,cAAU;AAAA,EACZ;AACF;AAGO,IAAM,YAAY,iBAAiB,YAAY,GAAG;AAGlD,SAAS,mBACd,MAAiD,QAAQ,KACzD,OAAe,WACP;AACR,QAAM,aAAa,IAAI;AACvB,MAAI,eAAe,OAAW,QAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;AACxE,MAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AACA,SAAO;AACT;AAGO,IAAM,aAAa,mBAAmB;AAKtC,IAAM,eAAe,CAAC,YAA4B,KAAK,WAAW,SAAS,OAAO,QAAQ;AAE1F,IAAM,YAAY,OAAO,OAAO;AAGhC,SAAS,QAAQ,MAAsB;AAC5C,SAAO,KAAK,WAAW,SAAS,OAAO,IAAI;AAC7C;AAOA,eAAsB,cACpB,QACA,OAAiB,CAAC,GAClB,YAAY,GACZ,SAAiB,YACA;AACjB,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,QAAQ,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IACtE,WAAW;AAAA,IACX,SAAS;AAAA,EACX,CAAC;AACD,SAAO;AACT;AAOA,eAAsB,qBAAqB,MASzB;AAChB,QAAM,UAAU,KAAK,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AACvD,QAAM,QAAQ,CAAC,GAAG,QAAQ,IAAI,CAAC,MAAM,UAAU,CAAC,EAAE,CAAC;AACnD,MAAI,KAAK,cAAe,OAAM,KAAK,iBAAiB,0BAA0B;AAC9E,QAAM,KAAK,aAAa;AACxB,MAAI;AACF,UAAM,cAAc,MAAM,KAAK,IAAI,GAAG,CAAC,GAAG,GAAG,KAAK,UAAU,UAAU;AAAA,EACxE,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI,MAAM,GAAG,GAAG;AAAA,EAAK,KAAK,GAAG,EAAE;AAAA,EACvC;AACF;AAYO,SAAS,mBACd,YACA,MACA,OACA,OAA8D,CAAC,GAC9C;AACjB,SAAO,IAAI,QAAgB,CAACA,UAAS,WAAW;AAC9C,UAAM,QAAQ,MAAM,KAAK,UAAU,YAAY,CAAC,YAAY,GAAG,IAAI,GAAG;AAAA,MACpE,KAAK,KAAK,OAAO;AAAA,MACjB,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,IACtD,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,QAAQ;AACZ,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,eAAS,EAAE;AACX,UAAI,SAAS,UAAW,WAAU,EAAE,SAAS,MAAM;AAAA,IACrD,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,gBAAU,EAAE,SAAS,MAAM;AAAA,IAC7B,CAAC;AACD,UAAM,GAAG,SAAS,CAAC,QAAQ,OAAO,GAAG,CAAC;AACtC,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,UAAI,SAAS,EAAG,CAAAA,SAAQ,MAAM;AAAA,UACzB,QAAO,IAAI,OAAO,UAAU,UAAU,QAAQ,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC;AAAA,IAC5E,CAAC;AACD,UAAM,MAAM,IAAI,KAAK;AAAA,EACvB,CAAC;AACH;AAsCO,IAAM,mBAAN,cAA+B,MAAM;AAAA,EACjC;AAAA,EAET,YAAY,SAAiB,gBAAuC,SAAwB;AAC1F,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO;AACZ,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,SAAS,OAAO,OAAuC;AACrD,SAAO,UAAU,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK,CAAC;AACnE;AAEA,SAAS,aAAa,MAAsB;AAC1C,SAAO,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG;AACjC;AAEA,SAAS,YAAY,MAAc,OAAuB;AACxD,SAAO,OAAO,QAAQ,KAAK,OAAO,QAAQ,IAAI;AAChD;AAEA,eAAe,qBACb,MACA,SACqC;AACrC,QAAM,WAAW,KAAK,MAAM,OAAO;AACnC,QAAM,UAAU,MAAM,QAAQ,UAAU,EAAE,eAAe,KAAK,CAAC;AAC/D,QAAM,QAAoC,CAAC;AAC3C,aAAW,SAAS,QAAQ,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG;AACrF,UAAM,eAAe,KAAK,SAAS,MAAM,IAAI;AAC7C,UAAM,OAAO,KAAK,MAAM,YAAY;AACpC,QAAI,MAAM,YAAY,GAAG;AACvB,YAAM,KAAK,GAAG,MAAM,qBAAqB,MAAM,YAAY,CAAC;AAC5D;AAAA,IACF;AACA,QAAI,MAAM,OAAO,GAAG;AAClB,YAAM,QAAQ,MAAM,SAAS,IAAI;AACjC,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,MAAM;AAAA,QAClB,QAAQ,OAAO,KAAK;AAAA,QACpB,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,QAAI,MAAM,eAAe,GAAG;AAC1B,YAAM,cAAc,MAAM,SAAS,MAAM,EAAE,UAAU,SAAS,CAAC;AAC/D,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,YAAY;AAAA,QACxB,QAAQ,OAAO,WAAW;AAAA,QAC1B,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,UAAM,IAAI,MAAM,yCAAyC,YAAY,EAAE;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAgB,WAA4B;AAC5D,QAAM,OAAO,SAAS,QAAQ,SAAS;AACvC,SAAO,SAAS,MAAO,CAAC,KAAK,WAAW,KAAK,GAAG,EAAE,KAAK,SAAS,QAAQ,CAAC,WAAW,IAAI;AAC1F;AAEA,eAAe,wBAAwB,aAAoC;AACzE,MAAI;AACF,UAAM,MAAM,WAAW;AAAA,EACzB,SAAS,OAAO;AACd,QAAK,MAAgC,SAAS,SAAU;AACxD,UAAM;AAAA,EACR;AACA,QAAM,IAAI,MAAM,oDAAoD,WAAW,EAAE;AACnF;AAEA,eAAe,iBACb,iBACA,MACA,eACA,oBAC+B;AAC/B,QAAM,SAAS,QAAQ,eAAe;AACtC,QAAM,cAAc,QAAQ,KAAK,WAAW;AAC5C,MAAI,SAAS,QAAQ,WAAW,GAAG;AACjC,UAAM,IAAI,MAAM,0EAA0E;AAAA,EAC5F;AACA,QAAM,MAAM,QAAQ,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,QAAM,wBAAwB,WAAW;AACzC,QAAM,UAAU,MAAM,QAAQ,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,CAAC,WAAW,CAAC;AAC9F,MAAI;AACF,UAAM,GAAG,QAAQ,KAAK,SAAS,WAAW,GAAG;AAAA,MAC3C,WAAW;AAAA,MACX,cAAc;AAAA,MACd,OAAO;AAAA,MACP,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB,CAAC;AACD,UAAM,MAAM,KAAK,SAAS,SAAS,CAAC;AACpC,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAC5E,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAE5E,UAAM,QAAQ;AAAA,MACZ,GAAG,MAAM,qBAAqB,SAAS,WAAW;AAAA,MAClD,GAAG,MAAM,qBAAqB,SAAS,SAAS;AAAA,IAClD,EAAE,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC;AAC1D,UAAM,aAAa,MAAM,OAAO,CAAC,OAAO,SAAS,QAAQ,KAAK,YAAY,CAAC;AAC3E,UAAM,YAAY,OAAO;AAAA,MACvB,MACG,IAAI,CAAC,SAAS,GAAG,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,UAAU,KAAK,KAAK,MAAM;AAAA,CAAI,EAChF,KAAK,EAAE;AAAA,MACV;AAAA,IACF;AACA,UAAM,UAAgC;AAAA,MACpC,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,oBAAoB,KAAK,aAAa,WAAW;AAAA,MACjD,cAAc,KAAK,aAAa,cAAc;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB;AAAA,MACA,YAAY,OAAO,SAAS;AAAA,IAC9B;AACA,UAAM,UAAU,KAAK,SAAS,cAAc,GAAG,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,CAAI;AACtF,UAAM,OAAO,SAAS,WAAW;AACjC,WAAO;AAAA,EACT,SAAS,OAAO;AACd,UAAM,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAClE,UAAM;AAAA,EACR;AACF;AAEA,SAAS,aAAa,OAAwB;AAC5C,MAAI,OAAO,SAAS,KAAK,EAAG,QAAO;AACnC,MAAI,UAAU,UAAa,UAAU,KAAM,QAAO,OAAO,MAAM,CAAC;AAChE,SAAO,OAAO,KAAK,OAAO,KAAK,GAAG,MAAM;AAC1C;AAOA,eAAsB,eAAe,MAA0C;AAC7E,QAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,KAAK,SAAS,CAAC;AACxD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,qBAAqB;AACzB,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,QAAI;AACF,YAAM,KAAK,MAAM,GAAG;AACpB,YAAM,MAAM,KAAK,OAAO;AACxB,YAAM,OAAO,KAAK,KAAK,GAAG;AAC1B,UAAI;AACF,cAAM,SAAS,MAAM,cAAc,KAAK,MAAM;AAAA,UAC5C,KAAK,KAAK,MAAM,KAAK,IAAI,GAAG,IAAI;AAAA,UAChC,UAAU;AAAA,UACV,WAAW;AAAA,UACX,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,QACtD,CAAC;AACD,iBAAS,aAAa,OAAO,MAAM;AACnC,iBAAS,aAAa,OAAO,MAAM;AACnC,6BAAqB;AAAA,MACvB,SAAS,KAAK;AACZ,cAAM,IAAI;AACV,iBAAS,aAAa,EAAE,MAAM;AAC9B,iBAAS,aAAa,EAAE,MAAM;AAC9B,cAAM,SAAS,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG,CAAC,EACzE,SAAS,MAAM,EACf,MAAM,GAAG,GAAI;AAChB,cAAM,IAAI,MAAM,GAAG,KAAK,UAAU,QAAQ,MAAM,EAAE,CAAC,sBAAsB,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC;AAAA,EAAO,MAAM,EAAE;AAAA,MAC/G;AACA,cAAQ,MAAM,KAAK,YAAY,GAAG;AAAA,IACpC,SAAS,KAAK;AACZ,gBAAU;AAAA,IACZ;AAAA,EACF,UAAE;AACA,QAAI;AACJ,QAAI,KAAK,SAAS;AAChB,UAAI;AACF,yBAAiB,MAAM;AAAA,UACrB;AAAA,UACA,KAAK;AAAA,UACL,EAAE,QAAQ,OAAO;AAAA,UACjB;AAAA,QACF;AAAA,MACF,SAAS,cAAc;AACrB,kBAAU,IAAI;AAAA,UACZ,uDAAuD,wBAAwB,QAAQ,aAAa,UAAU,YAAY;AAAA,UAC1H,EAAE,OAAO,WAAW,aAAa;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,KAAK,QAAS,OAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACjF,QAAI,SAAS;AACX,YAAM,IAAI;AAAA,QACR,mBAAmB,QAAQ,QAAQ,UAAU,OAAO,OAAO;AAAA,QAC3D;AAAA,QACA,EAAE,OAAO,QAAQ;AAAA,MACnB;AAAA,IACF;AACA,QAAI,CAAC,MAAO,OAAM,IAAI,iBAAiB,0CAA0C,cAAc;AAC/F,WAAO,iBAAiB,EAAE,GAAG,OAAO,eAAe,IAAI;AAAA,EACzD;AACF;AAGA,eAAsB,eAAkB,MAA0B;AAChE,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,SAAS,MAAM,MAAM;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,gCAAgC,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACtG;AACA,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,4BAA4B,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,EACjG;AACF;AAGA,eAAsB,UAAU,MAAc,SAAgC;AAC5E,QAAM,UAAU,MAAM,OAAO;AAC/B;AAGO,SAAS,UAAU,QAAgB,IAAoB;AAC5D,SAAO,GAAG,MAAM,IAAI,EAAE,GAAG,QAAQ,oBAAoB,GAAG;AAC1D;","names":["resolve"]}

package/dist/{chunk-2PVVP7GN.js → chunk-XKEFIFIC.js}

@@ -1,6 +1,6 @@
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/toollm.ts
 import { readFile } from "fs/promises";
@@ -194,4 +194,4 @@ export {
   toollmOutput,
   createToolLlmAdapter
 };
-//# sourceMappingURL=chunk-2PVVP7GN.js.map
+//# sourceMappingURL=chunk-XKEFIFIC.js.map

package/dist/{chunk-JRWWGMK7.js → chunk-XYA4XSNU.js}

@@ -7,7 +7,7 @@ import {
   safeRunId,
   stageFile,
   venvPythonAt
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/terminal-bench.ts
 import { join } from "path";
@@ -145,4 +145,4 @@ function createTerminalBenchAdapter() {
 export {
   createTerminalBenchAdapter
 };
-//# sourceMappingURL=chunk-JRWWGMK7.js.map
+//# sourceMappingURL=chunk-XYA4XSNU.js.map

package/dist/{chunk-X5YKXC6V.js → chunk-YSMEKBTD.js}

@@ -1,7 +1,7 @@
 import {
   benchRoot,
   runVenvScriptStdin
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/enterpriseops-gym.ts
 import { mkdir, readFile, rm, writeFile } from "fs/promises";
@@ -208,4 +208,4 @@ export {
   enterpriseOpsTranscriptOutput,
   createEnterpriseOpsGymAdapter
 };
-//# sourceMappingURL=chunk-X5YKXC6V.js.map
+//# sourceMappingURL=chunk-YSMEKBTD.js.map

package/dist/{chunk-2XU6OGEN.js → chunk-Z4TZ76N7.js}

@@ -3,7 +3,7 @@ import {
   preflightVenvImports,
   runVenvScriptStdin,
   venvPythonAt
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/commit0.ts
 import { join } from "path";
@@ -167,4 +167,4 @@ export {
   commit0DiffOutput,
   createCommit0Adapter
 };
-//# sourceMappingURL=chunk-2XU6OGEN.js.map
+//# sourceMappingURL=chunk-Z4TZ76N7.js.map

package/dist/index.js

@@ -1,47 +1,47 @@
 import {
   ADAPTERS,
   resolveAdapter
-} from "./chunk-PWQVGAJB.js";
-import "./chunk-2PVVP7GN.js";
+} from "./chunk-V7AEBY6U.js";
+import "./chunk-XKEFIFIC.js";
 import "./chunk-VQRS7VUC.js";
-import "./chunk-Z7ML6L77.js";
+import "./chunk-HWST3SED.js";
 import {
   createRagBenchAdapter
-} from "./chunk-R67DFVLO.js";
+} from "./chunk-IFVINJ4B.js";
 import "./chunk-SEVJPLZC.js";
 import {
   createSweBenchAdapter,
   scoreSweReport,
   sweEvaluationArgv,
   swePatchOutput
-} from "./chunk-CKUVRZ2T.js";
+} from "./chunk-3U5TXJZS.js";
 import {
   createT2RagBenchAdapter
-} from "./chunk-Y6O2OCUO.js";
-import "./chunk-HBSWHQNJ.js";
-import "./chunk-UAIOHCUK.js";
-import "./chunk-R36V2VP7.js";
-import "./chunk-JRWWGMK7.js";
+} from "./chunk-SHM6MRRF.js";
+import "./chunk-RCYQEFNX.js";
+import "./chunk-SFLA7OH3.js";
+import "./chunk-IZ5M6OAC.js";
+import "./chunk-XYA4XSNU.js";
 import "./chunk-INNOYXCP.js";
 import "./chunk-PA2ZKHJC.js";
 import "./chunk-5H5XV76F.js";
 import "./chunk-TBKU5XQI.js";
 import {
   createNoMiraclAdapter
-} from "./chunk-TSWPNOYM.js";
+} from "./chunk-NQG5XDSB.js";
 import {
   createOpenRagBenchAdapter
-} from "./chunk-KDIKRJGB.js";
-import "./chunk-ZEWMTR5M.js";
+} from "./chunk-SHYIRB7I.js";
+import "./chunk-MQMRLGOG.js";
 import "./chunk-PUIRNYI7.js";
 import "./chunk-C7T7WEK2.js";
 import {
   runRefineLoop
 } from "./chunk-JTHWEDEW.js";
-import "./chunk-2XU6OGEN.js";
+import "./chunk-Z4TZ76N7.js";
 import {
   createCragAdapter
-} from "./chunk-HHXFIHXC.js";
+} from "./chunk-SVR2LKYI.js";
 import {
   FINAL_ANSWER_SENTINEL,
   answerScoreToBenchScore,
@@ -54,18 +54,18 @@ import {
   scoreAnswerArtifact,
   tokenF1
 } from "./chunk-X3BTXCJ4.js";
-import "./chunk-J3KDJNX2.js";
-import "./chunk-X5YKXC6V.js";
-import "./chunk-YCGY7UIZ.js";
+import "./chunk-RH5F53JT.js";
+import "./chunk-YSMEKBTD.js";
+import "./chunk-7GRVHU22.js";
 import "./chunk-UPAMRDX4.js";
-import "./chunk-ODT47UAY.js";
-import "./chunk-7WSD27QQ.js";
-import "./chunk-SYDW647C.js";
-import "./chunk-IFAV6KEM.js";
+import "./chunk-K3BQGZCT.js";
+import "./chunk-PB64GYIG.js";
+import "./chunk-IA2FBTWC.js";
+import "./chunk-KP5KD6EN.js";
 import {
   StagedJudgeError,
   runStagedJudge
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 import "./chunk-53UPUNBZ.js";
 
 // src/run-benchmarks.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.3.7",
+  "version": "0.4.0",
   "type": "module",
   "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.123.0",
+    "@tangle-network/agent-eval": "0.126.6",
     "@tangle-network/agent-interface": "0.32.0",
-    "@tangle-network/agent-knowledge": "^4.1.0",
-    "@tangle-network/sandbox": "^0.11.1",
-    "@tangle-network/agent-runtime": "0.102.1"
+    "@tangle-network/agent-knowledge": "5.0.1",
+    "@tangle-network/sandbox": "^0.12.0",
+    "@tangle-network/agent-runtime": "0.105.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/scripts/trata-hedge/README.md

@@ -13,9 +13,9 @@ sparse (1 iff all themes). No deployable ground-truth checker — it's an **orac
 - **NOT** the verifier-grounded selector gate (our HumanEval/commit0 headline). The only
   checker is the judge itself, so selecting by it = selecting by the eval metric (an
   oracle, a Goodhart trap). See `docs/results.md`.
-- **IS** admissible: (1) more-compute headroom (pass@k), (2) the **improvement loop**
-  (`selfImprove`/GEPA optimizing the analyst directive *against this judge*, held-out
-  gated — the legitimate use of a judge domain), with the honest caveat that an
+- **IS** admissible: (1) more-compute headroom (pass@k), (2) prompt optimization
+  (`improve` with agent-eval's official GEPA method, using disjoint train, selection,
+  and final-test partitions), with the honest caveat that an
   LLM-judge can be *gamed*, so a held-out lift means "higher judge score," which without
   ground truth we can't independently certify as "better analysis."
 
@@ -52,5 +52,6 @@ dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/ag
 ## Next steps
 
 1. Agentic solver: our sandbox runtime browses the corpus + cites (fair baseline).
-2. The improvement loop: `selfImprove` on the analyst directive vs this judge, held-out
-   gated, across the 102 tasks — with the judge-gaming caveat stated.
+2. Prompt optimization: run `bench/src/trata-gepa.mts` across all 102 tasks with
+   explicit train, selection, and final-test partitions, and state the judge-gaming
+   caveat with every result.

package/scripts/verify-packed-consumer.mjs

@@ -120,7 +120,7 @@ try {
   )
   await writeFile(
     path.join(consumerDir, 'index.mjs'),
-    "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\n",
+    "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n",
   )
   await writeFile(
     path.join(consumerDir, 'tsconfig.json'),

package/src/benchmarks/_harness.ts

@@ -18,6 +18,7 @@
 
 import { execFile, spawn } from 'node:child_process'
 import { createHash } from 'node:crypto'
+import { existsSync, readFileSync } from 'node:fs'
 import {
   cp,
   lstat,
@@ -42,8 +43,25 @@ import type {
 
 const execFileAsync = promisify(execFile)
 
-/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */
-export const benchRoot = fileURLToPath(new URL('../..', import.meta.url))
+/** Locate the package by identity because source files and compiled chunks have different depths. */
+function resolveBenchRoot(moduleUrl: string): string {
+  let current = dirname(fileURLToPath(moduleUrl))
+  while (true) {
+    const manifestPath = join(current, 'package.json')
+    if (existsSync(manifestPath)) {
+      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }
+      if (manifest.name === '@tangle-network/agent-bench') return current
+    }
+    const parent = dirname(current)
+    if (parent === current) {
+      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)
+    }
+    current = parent
+  }
+}
+
+/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */
+export const benchRoot = resolveBenchRoot(import.meta.url)
 
 /** Resolve the shared interpreter without requiring an installed package to contain a venv. */
 export function resolveBenchPython(

package/src/benchmarks/humaneval.test.mts

@@ -13,8 +13,8 @@ describe('HumanEval Python isolation', () => {
     writeFileSync(
       fakeDocker,
       `#!/usr/bin/env node
-const fs = require('node:fs')
-const path = require('node:path')
+import * as fs from 'node:fs'
+import * as path from 'node:path'
 const args = process.argv.slice(2)
 if (args[0] === 'rm') process.exit(0)
 if (process.env.FAKE_DOCKER_MISSING === '1') {

package/src/gate.ts

@@ -427,7 +427,7 @@ export async function runGate(opts: RunGateOptions): Promise<GateReport> {
       root: a.label,
       nodes: [],
       total: acc.get(a.label)!.spend,
-      statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0 },
+      statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 0, waiting: 0 },
     },
   }))
   const equalK = equalKOnCost(equalKArms)

package/src/hev-eval.mts

@@ -27,6 +27,7 @@ async function complete(base: string, key: string, model: string, prompt: string
 async function main(): Promise<void> {
   const key = process.env.TANGLE_API_KEY
   if (!key) throw new Error('TANGLE_API_KEY required')
+  const apiKey: string = key
   const base = process.env.ROUTER_BASE ?? 'https://api.together.xyz/v1'
   const model = process.env.WORKER_MODEL ?? 'meta-llama/Meta-Llama-3-8B-Instruct-Lite'
   const instruction = process.env.INSTRUCTION_FILE
@@ -51,8 +52,10 @@ async function main(): Promise<void> {
   let i = 0
   async function worker(): Promise<void> {
     while (i < tasks.length) {
-      const t = tasks[i++]
-      const reply = await complete(base, key, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
+      const t = tasks[i]
+      i += 1
+      if (!t) continue
+      const reply = await complete(base, apiKey, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens)
       const { pass: p } = await runChecker(t, extractCode(reply))
       if (p === 1) pass += 1
       else fails.push(t.taskId)