@tangle-network/agent-bench 0.3.7 → 0.3.8

package/dist/{chunk-KDIKRJGB.js → chunk-SHYIRB7I.js}

@@ -14,7 +14,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/open-rag-bench.ts
 import { readFile } from "fs/promises";
@@ -117,4 +117,4 @@ function createOpenRagBenchAdapter() {
 export {
   createOpenRagBenchAdapter
 };
-//# sourceMappingURL=chunk-KDIKRJGB.js.map
+//# sourceMappingURL=chunk-SHYIRB7I.js.map

package/dist/{chunk-HHXFIHXC.js → chunk-SVR2LKYI.js}

@@ -12,7 +12,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/crag.ts
 import { readFile } from "fs/promises";
@@ -113,4 +113,4 @@ function createCragAdapter() {
 export {
   createCragAdapter
 };
-//# sourceMappingURL=chunk-HHXFIHXC.js.map
+//# sourceMappingURL=chunk-SVR2LKYI.js.map

package/dist/{chunk-PWQVGAJB.js → chunk-V7AEBY6U.js}

@@ -1,33 +1,33 @@
 import {
   createToolLlmAdapter
-} from "./chunk-2PVVP7GN.js";
+} from "./chunk-XKEFIFIC.js";
 import {
   createTrataHedgeAdapter
 } from "./chunk-VQRS7VUC.js";
 import {
   createWebArenaVerifiedAdapter
-} from "./chunk-Z7ML6L77.js";
+} from "./chunk-HWST3SED.js";
 import {
   createRagBenchAdapter
-} from "./chunk-R67DFVLO.js";
+} from "./chunk-IFVINJ4B.js";
 import {
   createSimpleQaAdapter
 } from "./chunk-SEVJPLZC.js";
 import {
   createSweBenchAdapter
-} from "./chunk-CKUVRZ2T.js";
+} from "./chunk-3U5TXJZS.js";
 import {
   createT2RagBenchAdapter
-} from "./chunk-Y6O2OCUO.js";
+} from "./chunk-SHM6MRRF.js";
 import {
   createTau2BenchAdapter
-} from "./chunk-HBSWHQNJ.js";
+} from "./chunk-RCYQEFNX.js";
 import {
   createTau3BankingAdapter
-} from "./chunk-UAIOHCUK.js";
+} from "./chunk-SFLA7OH3.js";
 import {
   createTerminalBenchAdapter
-} from "./chunk-JRWWGMK7.js";
+} from "./chunk-XYA4XSNU.js";
 import {
   createFramesAdapter
 } from "./chunk-INNOYXCP.js";
@@ -42,13 +42,13 @@ import {
 } from "./chunk-TBKU5XQI.js";
 import {
   createNoMiraclAdapter
-} from "./chunk-TSWPNOYM.js";
+} from "./chunk-NQG5XDSB.js";
 import {
   createOpenRagBenchAdapter
-} from "./chunk-KDIKRJGB.js";
+} from "./chunk-SHYIRB7I.js";
 import {
   createProgrambenchAdapter
-} from "./chunk-ZEWMTR5M.js";
+} from "./chunk-MQMRLGOG.js";
 import {
   createCadBenchAdapter
 } from "./chunk-PUIRNYI7.js";
@@ -57,35 +57,35 @@ import {
 } from "./chunk-C7T7WEK2.js";
 import {
   createCommit0Adapter
-} from "./chunk-2XU6OGEN.js";
+} from "./chunk-Z4TZ76N7.js";
 import {
   createCragAdapter
-} from "./chunk-HHXFIHXC.js";
+} from "./chunk-SVR2LKYI.js";
 import {
   createDabstepAdapter
-} from "./chunk-J3KDJNX2.js";
+} from "./chunk-RH5F53JT.js";
 import {
   createEnterpriseOpsGymAdapter
-} from "./chunk-X5YKXC6V.js";
+} from "./chunk-YSMEKBTD.js";
 import {
   createFinResearchBenchAdapter
-} from "./chunk-YCGY7UIZ.js";
+} from "./chunk-7GRVHU22.js";
 import {
   createFinsearchcompAdapter
 } from "./chunk-UPAMRDX4.js";
 import {
   createAecBenchAdapter
-} from "./chunk-ODT47UAY.js";
+} from "./chunk-K3BQGZCT.js";
 import {
   createAgentBenchAdapter
-} from "./chunk-7WSD27QQ.js";
+} from "./chunk-PB64GYIG.js";
 import {
   createAppWorldAdapter,
   createAppWorldReactAdapter
-} from "./chunk-SYDW647C.js";
+} from "./chunk-IA2FBTWC.js";
 import {
   createBfclAdapter
-} from "./chunk-IFAV6KEM.js";
+} from "./chunk-KP5KD6EN.js";
 import {
   createCadDesignAdapter
 } from "./chunk-53UPUNBZ.js";
@@ -141,4 +141,4 @@ export {
   ADAPTERS,
   resolveAdapter
 };
-//# sourceMappingURL=chunk-PWQVGAJB.js.map
+//# sourceMappingURL=chunk-V7AEBY6U.js.map

package/dist/{chunk-LRRD7NAG.js → chunk-WSKWVEQB.js}

@@ -1,6 +1,7 @@
 // src/benchmarks/_harness.ts
 import { execFile, spawn } from "child_process";
 import { createHash } from "crypto";
+import { existsSync, readFileSync } from "fs";
 import {
   cp,
   lstat,
@@ -18,7 +19,22 @@ import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "pat
 import { fileURLToPath } from "url";
 import { promisify } from "util";
 var execFileAsync = promisify(execFile);
-var benchRoot = fileURLToPath(new URL("../..", import.meta.url));
+function resolveBenchRoot(moduleUrl) {
+  let current = dirname(fileURLToPath(moduleUrl));
+  while (true) {
+    const manifestPath = join(current, "package.json");
+    if (existsSync(manifestPath)) {
+      const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+      if (manifest.name === "@tangle-network/agent-bench") return current;
+    }
+    const parent = dirname(current);
+    if (parent === current) {
+      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`);
+    }
+    current = parent;
+  }
+}
+var benchRoot = resolveBenchRoot(import.meta.url);
 function resolveBenchPython(env = process.env, root = benchRoot) {
   const configured = env.AGENT_BENCH_PYTHON;
   if (configured === void 0) return join(root, ".venv", "bin", "python");
@@ -298,4 +314,4 @@ export {
   stageFile,
   safeRunId
 };
-//# sourceMappingURL=chunk-LRRD7NAG.js.map
+//# sourceMappingURL=chunk-WSKWVEQB.js.map

package/dist/chunk-WSKWVEQB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/_harness.ts"],"sourcesContent":["/**\n * Shared code-bench harness. The \"stage the artifact → run the benchmark's own\n * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)\n * → read its JSON report → { resolved, score }\" spine that swe-bench,\n * terminal-bench, commit0, programbench and aec-bench all need. Factored out so\n * the Docker/venv/report-reading logic lives in ONE place instead of being\n * copy-pasted per adapter.\n *\n * It owns NO benchmark policy: each adapter passes the argv for its evaluator\n * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The\n * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config\n * and fail-loud diagnostics.\n *\n * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`\n * (the adapter passes the import line + the exact fix). A staged run that exits\n * nonzero throws with the captured stderr — never a fabricated score.\n */\n\nimport { execFile, spawn } from 'node:child_process'\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport {\n  cp,\n  lstat,\n  mkdir,\n  mkdtemp,\n  readFile,\n  readlink,\n  readdir,\n  rename,\n  rm,\n  writeFile,\n} from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type {\n  BenchScore,\n  JudgeArtifactFileReceipt,\n  JudgeArtifactReceipt,\n} from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Locate the package by identity because source files and compiled chunks have different depths. */\nfunction resolveBenchRoot(moduleUrl: string): string {\n  let current = dirname(fileURLToPath(moduleUrl))\n  while (true) {\n    const manifestPath = join(current, 'package.json')\n    if (existsSync(manifestPath)) {\n      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }\n      if (manifest.name === '@tangle-network/agent-bench') return current\n    }\n    const parent = dirname(current)\n    if (parent === current) {\n      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)\n    }\n    current = parent\n  }\n}\n\n/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */\nexport const benchRoot = resolveBenchRoot(import.meta.url)\n\n/** Resolve the shared interpreter without requiring an installed package to contain a venv. */\nexport function resolveBenchPython(\n  env: Readonly<{ AGENT_BENCH_PYTHON?: string }> = process.env,\n  root: string = benchRoot,\n): string {\n  const configured = env.AGENT_BENCH_PYTHON\n  if (configured === undefined) return join(root, '.venv', 'bin', 'python')\n  if (!isAbsolute(configured)) {\n    throw new Error('AGENT_BENCH_PYTHON must be an absolute path')\n  }\n  return configured\n}\n\n/** The shared interpreter every Python-backed evaluator runs through. */\nexport const venvPython = resolveBenchPython()\n\n/** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip\n *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)\n *  get their own venv and pass its python explicitly — keeping the shared one clean. */\nexport const venvPythonAt = (venvDir: string): string => join(benchRoot, venvDir, 'bin', 'python')\n/** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */\nexport const bigBuffer = 1024 * 1024 * 256\n\n/** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */\nexport function venvBin(name: string): string {\n  return join(benchRoot, '.venv', 'bin', name)\n}\n\n/**\n * Run the bench venv python with an inline script (`-c`); return stdout. Throws\n * (with stderr) on a nonzero exit — the loaders rely on this to fail loud rather\n * than parse a partial dump.\n */\nexport async function runVenvPython(\n  script: string,\n  args: string[] = [],\n  timeoutMs = 0,\n  python: string = venvPython,\n): Promise<string> {\n  const { stdout } = await execFileAsync(python, ['-c', script, ...args], {\n    maxBuffer: bigBuffer,\n    timeout: timeoutMs,\n  })\n  return stdout\n}\n\n/**\n * Preflight a python-backed harness: import the module(s) and (optionally) ping\n * Docker, all inside the bench venv. On failure THROWS the captured error joined\n * to the adapter's `fix` guidance — the contract every code-bench preflight wants.\n */\nexport async function preflightVenvImports(opts: {\n  /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */\n  modules: string[]\n  /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */\n  requireDocker?: boolean\n  /** Actionable remediation appended to the thrown message. */\n  fix: string\n  /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */\n  python?: string\n}): Promise<void> {\n  const imports = opts.modules.filter((m) => m.length > 0)\n  const lines = [...imports.map((m) => `import ${m}`)]\n  if (opts.requireDocker) lines.push('import docker', 'docker.from_env().ping()')\n  lines.push(\"print('ok')\")\n  try {\n    await runVenvPython(lines.join('\\n'), [], 0, opts.python ?? venvPython)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    throw new Error(`${msg}\\n${opts.fix}`)\n  }\n}\n\n/**\n * Run a bench-local python driver script (in the bench venv) while piping\n * `input` to its stdin, returning stdout. The driver's verdict JSON is its LAST\n * stdout line; callers parse that and inspect an `error` field (fail loud).\n *\n * Uses spawn + an explicit `stdin.end(input)` rather than promisify(execFile)'s\n * `input` option, because that option is NOT honored by async execFile — stdin\n * is left open and a driver that does `sys.stdin.read()` blocks forever. The\n * artifact-piping judges (commit0, appworld) MUST go through this.\n */\nexport function runVenvScriptStdin(\n  scriptPath: string,\n  args: string[],\n  input: string,\n  opts: { cwd?: string; timeoutMs?: number; python?: string } = {},\n): Promise<string> {\n  return new Promise<string>((resolve, reject) => {\n    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {\n      cwd: opts.cwd ?? benchRoot,\n      ...(opts.timeoutMs ? { timeout: opts.timeoutMs } : {}),\n    })\n    let stdout = ''\n    let stderr = ''\n    let bytes = 0\n    child.stdout.on('data', (c: Buffer) => {\n      bytes += c.length\n      if (bytes <= bigBuffer) stdout += c.toString('utf8')\n    })\n    child.stderr.on('data', (c: Buffer) => {\n      stderr += c.toString('utf8')\n    })\n    child.on('error', (err) => reject(err))\n    child.on('close', (code) => {\n      if (code === 0) resolve(stdout)\n      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)))\n    })\n    child.stdin.end(input)\n  })\n}\n\nexport interface StagedRunSpec {\n  /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */\n  tmpPrefix: string\n  /**\n   * Write the artifact + any harness inputs into the temp dir. Returns nothing;\n   * `argv`/`cwd` consume `dir` to point the evaluator at what was written.\n   */\n  stage(dir: string): Promise<void>\n  /** The external evaluator to spawn. `bin` defaults to the bench venv python. */\n  bin?: string\n  /** argv for the evaluator, computed from the temp `dir`. */\n  argv(dir: string): string[]\n  /** Working directory for the evaluator. Defaults to the temp `dir`. */\n  cwd?(dir: string): string\n  /** Hard timeout for the evaluator (ms); 0 = none. */\n  timeoutMs?: number\n  /**\n   * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws\n   * if the expected report is absent/malformed (fail loud — no default score).\n   */\n  parseReport(dir: string): Promise<BenchScore>\n  /**\n   * Copy the complete evaluator directory plus raw process stdout/stderr to this\n   * caller-owned directory before cleanup. The destination must not exist.\n   */\n  capture?: StagedRunCaptureSpec\n  /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */\n  keepTmp?: boolean\n}\n\nexport interface StagedRunCaptureSpec {\n  /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */\n  destination: string\n}\n\n/** A staged run failed after any requested evidence was durably retained. */\nexport class StagedJudgeError extends Error {\n  readonly judgeArtifacts?: JudgeArtifactReceipt\n\n  constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions) {\n    super(message, options)\n    this.name = 'StagedJudgeError'\n    this.judgeArtifacts = judgeArtifacts\n  }\n}\n\nfunction sha256(bytes: Uint8Array): `sha256:${string}` {\n  return `sha256:${createHash('sha256').update(bytes).digest('hex')}`\n}\n\nfunction portablePath(path: string): string {\n  return path.split(sep).join('/')\n}\n\nfunction compareText(left: string, right: string): number {\n  return left < right ? -1 : left > right ? 1 : 0\n}\n\nasync function collectArtifactFiles(\n  root: string,\n  current: string,\n): Promise<JudgeArtifactFileReceipt[]> {\n  const absolute = join(root, current)\n  const entries = await readdir(absolute, { withFileTypes: true })\n  const files: JudgeArtifactFileReceipt[] = []\n  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {\n    const relativePath = join(current, entry.name)\n    const path = join(root, relativePath)\n    if (entry.isDirectory()) {\n      files.push(...await collectArtifactFiles(root, relativePath))\n      continue\n    }\n    if (entry.isFile()) {\n      const bytes = await readFile(path)\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: bytes.byteLength,\n        sha256: sha256(bytes),\n        kind: 'file',\n      })\n      continue\n    }\n    if (entry.isSymbolicLink()) {\n      const targetBytes = await readlink(path, { encoding: 'buffer' })\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: targetBytes.byteLength,\n        sha256: sha256(targetBytes),\n        kind: 'symlink',\n      })\n      continue\n    }\n    throw new Error(`staged judge capture does not support ${relativePath}`)\n  }\n  return files\n}\n\nfunction isWithin(parent: string, candidate: string): boolean {\n  const path = relative(parent, candidate)\n  return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path))\n}\n\nasync function assertDestinationAbsent(destination: string): Promise<void> {\n  try {\n    await lstat(destination)\n  } catch (error) {\n    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return\n    throw error\n  }\n  throw new Error(`staged judge capture destination already exists: ${destination}`)\n}\n\nasync function captureStagedRun(\n  sourceDirectory: string,\n  spec: StagedRunCaptureSpec,\n  processOutput: Readonly<{ stdout: Buffer; stderr: Buffer }>,\n  evaluatorSucceeded: boolean,\n): Promise<JudgeArtifactReceipt> {\n  const source = resolve(sourceDirectory)\n  const destination = resolve(spec.destination)\n  if (isWithin(source, destination)) {\n    throw new Error('staged judge capture destination must be outside the evaluator directory')\n  }\n  await mkdir(dirname(destination), { recursive: true })\n  await assertDestinationAbsent(destination)\n  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`))\n  try {\n    await cp(source, join(staging, 'evaluator'), {\n      recursive: true,\n      errorOnExist: true,\n      force: false,\n      preserveTimestamps: true,\n      verbatimSymlinks: true,\n    })\n    await mkdir(join(staging, 'process'))\n    await writeFile(join(staging, 'process', 'stdout.bin'), processOutput.stdout)\n    await writeFile(join(staging, 'process', 'stderr.bin'), processOutput.stderr)\n\n    const files = [\n      ...await collectArtifactFiles(staging, 'evaluator'),\n      ...await collectArtifactFiles(staging, 'process'),\n    ].sort((left, right) => compareText(left.path, right.path))\n    const byteLength = files.reduce((total, file) => total + file.byteLength, 0)\n    const treeBytes = Buffer.from(\n      files\n        .map((file) => `${file.path}\\0${file.kind}\\0${file.byteLength}\\0${file.sha256}\\n`)\n        .join(''),\n      'utf8',\n    )\n    const receipt: JudgeArtifactReceipt = {\n      schema: 'agent-bench/judge-artifacts/v1',\n      directory: destination,\n      evaluatorDirectory: join(destination, 'evaluator'),\n      manifestPath: join(destination, 'receipt.json'),\n      evaluatorSucceeded,\n      files,\n      fileCount: files.length,\n      byteLength,\n      treeSha256: sha256(treeBytes),\n    }\n    await writeFile(join(staging, 'receipt.json'), `${JSON.stringify(receipt, null, 2)}\\n`)\n    await rename(staging, destination)\n    return receipt\n  } catch (error) {\n    await rm(staging, { recursive: true, force: true }).catch(() => {})\n    throw error\n  }\n}\n\nfunction processBytes(value: unknown): Buffer {\n  if (Buffer.isBuffer(value)) return value\n  if (value === undefined || value === null) return Buffer.alloc(0)\n  return Buffer.from(String(value), 'utf8')\n}\n\n/**\n * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →\n * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is\n * always removed in `finally` unless `keepTmp`.\n */\nexport async function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore> {\n  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix))\n  let stdout: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let stderr: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let evaluatorSucceeded = false\n  let score: BenchScore | undefined\n  let failure: unknown\n  try {\n    try {\n      await spec.stage(dir)\n      const bin = spec.bin ?? venvPython\n      const argv = spec.argv(dir)\n      try {\n        const output = await execFileAsync(bin, argv, {\n          cwd: spec.cwd ? spec.cwd(dir) : dir,\n          encoding: 'buffer',\n          maxBuffer: bigBuffer,\n          ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),\n        })\n        stdout = processBytes(output.stdout)\n        stderr = processBytes(output.stderr)\n        evaluatorSucceeded = true\n      } catch (err) {\n        const e = err as { stderr?: unknown; stdout?: unknown; message?: string }\n        stdout = processBytes(e.stdout)\n        stderr = processBytes(e.stderr)\n        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err))\n          .toString('utf8')\n          .slice(0, 2000)\n        throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${argv.join(' ')}):\\n${detail}`)\n      }\n      score = await spec.parseReport(dir)\n    } catch (err) {\n      failure = err\n    }\n  } finally {\n    let judgeArtifacts: JudgeArtifactReceipt | undefined\n    if (spec.capture) {\n      try {\n        judgeArtifacts = await captureStagedRun(\n          dir,\n          spec.capture,\n          { stdout, stderr },\n          evaluatorSucceeded,\n        )\n      } catch (captureError) {\n        failure = new Error(\n          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,\n          { cause: failure ?? captureError },\n        )\n      }\n    }\n    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {})\n    if (failure) {\n      throw new StagedJudgeError(\n        failure instanceof Error ? failure.message : String(failure),\n        judgeArtifacts,\n        { cause: failure },\n      )\n    }\n    if (!score) throw new StagedJudgeError('staged judge completed without a score', judgeArtifacts)\n    return judgeArtifacts ? { ...score, judgeArtifacts } : score\n  }\n}\n\n/** Read + JSON.parse a report file from a staged run; throws with the path on failure. */\nexport async function readJsonReport<T>(path: string): Promise<T> {\n  let raw: string\n  try {\n    raw = await readFile(path, 'utf8')\n  } catch (err) {\n    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`)\n  }\n  try {\n    return JSON.parse(raw) as T\n  } catch (err) {\n    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`)\n  }\n}\n\n/** Write a UTF-8 file into a staged dir (artifact / preds.json / attempt.sh / …). */\nexport async function stageFile(path: string, content: string): Promise<void> {\n  await writeFile(path, content)\n}\n\n/** Sanitize an instance id into a filesystem/run-id-safe token. */\nexport function safeRunId(prefix: string, id: string): string {\n  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, '_')\n}\n"],"mappings":";AAkBA,SAAS,UAAU,aAAa;AAChC,SAAS,kBAAkB;AAC3B,SAAS,YAAY,oBAAoB;AACzC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AACvB,SAAS,UAAU,SAAS,YAAY,MAAM,UAAU,SAAS,WAAW;AAC5E,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAO1B,IAAM,gBAAgB,UAAU,QAAQ;AAGxC,SAAS,iBAAiB,WAA2B;AACnD,MAAI,UAAU,QAAQ,cAAc,SAAS,CAAC;AAC9C,SAAO,MAAM;AACX,UAAM,eAAe,KAAK,SAAS,cAAc;AACjD,QAAI,WAAW,YAAY,GAAG;AAC5B,YAAM,WAAW,KAAK,MAAM,aAAa,cAAc,MAAM,CAAC;AAC9D,UAAI,SAAS,SAAS,8BAA+B,QAAO;AAAA,IAC9D;AACA,UAAM,SAAS,QAAQ,OAAO;AAC9B,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI,MAAM,qDAAqD,SAAS,EAAE;AAAA,IAClF;AACA,cAAU;AAAA,EACZ;AACF;AAGO,IAAM,YAAY,iBAAiB,YAAY,GAAG;AAGlD,SAAS,mBACd,MAAiD,QAAQ,KACzD,OAAe,WACP;AACR,QAAM,aAAa,IAAI;AACvB,MAAI,eAAe,OAAW,QAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;AACxE,MAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AACA,SAAO;AACT;AAGO,IAAM,aAAa,mBAAmB;AAKtC,IAAM,eAAe,CAAC,YAA4B,KAAK,WAAW,SAAS,OAAO,QAAQ;AAE1F,IAAM,YAAY,OAAO,OAAO;AAGhC,SAAS,QAAQ,MAAsB;AAC5C,SAAO,KAAK,WAAW,SAAS,OAAO,IAAI;AAC7C;AAOA,eAAsB,cACpB,QACA,OAAiB,CAAC,GAClB,YAAY,GACZ,SAAiB,YACA;AACjB,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,QAAQ,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IACtE,WAAW;AAAA,IACX,SAAS;AAAA,EACX,CAAC;AACD,SAAO;AACT;AAOA,eAAsB,qBAAqB,MASzB;AAChB,QAAM,UAAU,KAAK,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AACvD,QAAM,QAAQ,CAAC,GAAG,QAAQ,IAAI,CAAC,MAAM,UAAU,CAAC,EAAE,CAAC;AACnD,MAAI,KAAK,cAAe,OAAM,KAAK,iBAAiB,0BAA0B;AAC9E,QAAM,KAAK,aAAa;AACxB,MAAI;AACF,UAAM,cAAc,MAAM,KAAK,IAAI,GAAG,CAAC,GAAG,GAAG,KAAK,UAAU,UAAU;AAAA,EACxE,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI,MAAM,GAAG,GAAG;AAAA,EAAK,KAAK,GAAG,EAAE;AAAA,EACvC;AACF;AAYO,SAAS,mBACd,YACA,MACA,OACA,OAA8D,CAAC,GAC9C;AACjB,SAAO,IAAI,QAAgB,CAACA,UAAS,WAAW;AAC9C,UAAM,QAAQ,MAAM,KAAK,UAAU,YAAY,CAAC,YAAY,GAAG,IAAI,GAAG;AAAA,MACpE,KAAK,KAAK,OAAO;AAAA,MACjB,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,IACtD,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,QAAQ;AACZ,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,eAAS,EAAE;AACX,UAAI,SAAS,UAAW,WAAU,EAAE,SAAS,MAAM;AAAA,IACrD,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,gBAAU,EAAE,SAAS,MAAM;AAAA,IAC7B,CAAC;AACD,UAAM,GAAG,SAAS,CAAC,QAAQ,OAAO,GAAG,CAAC;AACtC,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,UAAI,SAAS,EAAG,CAAAA,SAAQ,MAAM;AAAA,UACzB,QAAO,IAAI,OAAO,UAAU,UAAU,QAAQ,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC;AAAA,IAC5E,CAAC;AACD,UAAM,MAAM,IAAI,KAAK;AAAA,EACvB,CAAC;AACH;AAsCO,IAAM,mBAAN,cAA+B,MAAM;AAAA,EACjC;AAAA,EAET,YAAY,SAAiB,gBAAuC,SAAwB;AAC1F,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO;AACZ,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,SAAS,OAAO,OAAuC;AACrD,SAAO,UAAU,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK,CAAC;AACnE;AAEA,SAAS,aAAa,MAAsB;AAC1C,SAAO,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG;AACjC;AAEA,SAAS,YAAY,MAAc,OAAuB;AACxD,SAAO,OAAO,QAAQ,KAAK,OAAO,QAAQ,IAAI;AAChD;AAEA,eAAe,qBACb,MACA,SACqC;AACrC,QAAM,WAAW,KAAK,MAAM,OAAO;AACnC,QAAM,UAAU,MAAM,QAAQ,UAAU,EAAE,eAAe,KAAK,CAAC;AAC/D,QAAM,QAAoC,CAAC;AAC3C,aAAW,SAAS,QAAQ,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG;AACrF,UAAM,eAAe,KAAK,SAAS,MAAM,IAAI;AAC7C,UAAM,OAAO,KAAK,MAAM,YAAY;AACpC,QAAI,MAAM,YAAY,GAAG;AACvB,YAAM,KAAK,GAAG,MAAM,qBAAqB,MAAM,YAAY,CAAC;AAC5D;AAAA,IACF;AACA,QAAI,MAAM,OAAO,GAAG;AAClB,YAAM,QAAQ,MAAM,SAAS,IAAI;AACjC,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,MAAM;AAAA,QAClB,QAAQ,OAAO,KAAK;AAAA,QACpB,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,QAAI,MAAM,eAAe,GAAG;AAC1B,YAAM,cAAc,MAAM,SAAS,MAAM,EAAE,UAAU,SAAS,CAAC;AAC/D,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,YAAY;AAAA,QACxB,QAAQ,OAAO,WAAW;AAAA,QAC1B,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,UAAM,IAAI,MAAM,yCAAyC,YAAY,EAAE;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAgB,WAA4B;AAC5D,QAAM,OAAO,SAAS,QAAQ,SAAS;AACvC,SAAO,SAAS,MAAO,CAAC,KAAK,WAAW,KAAK,GAAG,EAAE,KAAK,SAAS,QAAQ,CAAC,WAAW,IAAI;AAC1F;AAEA,eAAe,wBAAwB,aAAoC;AACzE,MAAI;AACF,UAAM,MAAM,WAAW;AAAA,EACzB,SAAS,OAAO;AACd,QAAK,MAAgC,SAAS,SAAU;AACxD,UAAM;AAAA,EACR;AACA,QAAM,IAAI,MAAM,oDAAoD,WAAW,EAAE;AACnF;AAEA,eAAe,iBACb,iBACA,MACA,eACA,oBAC+B;AAC/B,QAAM,SAAS,QAAQ,eAAe;AACtC,QAAM,cAAc,QAAQ,KAAK,WAAW;AAC5C,MAAI,SAAS,QAAQ,WAAW,GAAG;AACjC,UAAM,IAAI,MAAM,0EAA0E;AAAA,EAC5F;AACA,QAAM,MAAM,QAAQ,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,QAAM,wBAAwB,WAAW;AACzC,QAAM,UAAU,MAAM,QAAQ,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,CAAC,WAAW,CAAC;AAC9F,MAAI;AACF,UAAM,GAAG,QAAQ,KAAK,SAAS,WAAW,GAAG;AAAA,MAC3C,WAAW;AAAA,MACX,cAAc;AAAA,MACd,OAAO;AAAA,MACP,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB,CAAC;AACD,UAAM,MAAM,KAAK,SAAS,SAAS,CAAC;AACpC,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAC5E,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAE5E,UAAM,QAAQ;AAAA,MACZ,GAAG,MAAM,qBAAqB,SAAS,WAAW;AAAA,MAClD,GAAG,MAAM,qBAAqB,SAAS,SAAS;AAAA,IAClD,EAAE,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC;AAC1D,UAAM,aAAa,MAAM,OAAO,CAAC,OAAO,SAAS,QAAQ,KAAK,YAAY,CAAC;AAC3E,UAAM,YAAY,OAAO;AAAA,MACvB,MACG,IAAI,CAAC,SAAS,GAAG,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,UAAU,KAAK,KAAK,MAAM;AAAA,CAAI,EAChF,KAAK,EAAE;AAAA,MACV;AAAA,IACF;AACA,UAAM,UAAgC;AAAA,MACpC,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,oBAAoB,KAAK,aAAa,WAAW;AAAA,MACjD,cAAc,KAAK,aAAa,cAAc;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB;AAAA,MACA,YAAY,OAAO,SAAS;AAAA,IAC9B;AACA,UAAM,UAAU,KAAK,SAAS,cAAc,GAAG,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,CAAI;AACtF,UAAM,OAAO,SAAS,WAAW;AACjC,WAAO;AAAA,EACT,SAAS,OAAO;AACd,UAAM,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAClE,UAAM;AAAA,EACR;AACF;AAEA,SAAS,aAAa,OAAwB;AAC5C,MAAI,OAAO,SAAS,KAAK,EAAG,QAAO;AACnC,MAAI,UAAU,UAAa,UAAU,KAAM,QAAO,OAAO,MAAM,CAAC;AAChE,SAAO,OAAO,KAAK,OAAO,KAAK,GAAG,MAAM;AAC1C;AAOA,eAAsB,eAAe,MAA0C;AAC7E,QAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,KAAK,SAAS,CAAC;AACxD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,qBAAqB;AACzB,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,QAAI;AACF,YAAM,KAAK,MAAM,GAAG;AACpB,YAAM,MAAM,KAAK,OAAO;AACxB,YAAM,OAAO,KAAK,KAAK,GAAG;AAC1B,UAAI;AACF,cAAM,SAAS,MAAM,cAAc,KAAK,MAAM;AAAA,UAC5C,KAAK,KAAK,MAAM,KAAK,IAAI,GAAG,IAAI;AAAA,UAChC,UAAU;AAAA,UACV,WAAW;AAAA,UACX,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,QACtD,CAAC;AACD,iBAAS,aAAa,OAAO,MAAM;AACnC,iBAAS,aAAa,OAAO,MAAM;AACnC,6BAAqB;AAAA,MACvB,SAAS,KAAK;AACZ,cAAM,IAAI;AACV,iBAAS,aAAa,EAAE,MAAM;AAC9B,iBAAS,aAAa,EAAE,MAAM;AAC9B,cAAM,SAAS,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG,CAAC,EACzE,SAAS,MAAM,EACf,MAAM,GAAG,GAAI;AAChB,cAAM,IAAI,MAAM,GAAG,KAAK,UAAU,QAAQ,MAAM,EAAE,CAAC,sBAAsB,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC;AAAA,EAAO,MAAM,EAAE;AAAA,MAC/G;AACA,cAAQ,MAAM,KAAK,YAAY,GAAG;AAAA,IACpC,SAAS,KAAK;AACZ,gBAAU;AAAA,IACZ;AAAA,EACF,UAAE;AACA,QAAI;AACJ,QAAI,KAAK,SAAS;AAChB,UAAI;AACF,yBAAiB,MAAM;AAAA,UACrB;AAAA,UACA,KAAK;AAAA,UACL,EAAE,QAAQ,OAAO;AAAA,UACjB;AAAA,QACF;AAAA,MACF,SAAS,cAAc;AACrB,kBAAU,IAAI;AAAA,UACZ,uDAAuD,wBAAwB,QAAQ,aAAa,UAAU,YAAY;AAAA,UAC1H,EAAE,OAAO,WAAW,aAAa;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,KAAK,QAAS,OAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACjF,QAAI,SAAS;AACX,YAAM,IAAI;AAAA,QACR,mBAAmB,QAAQ,QAAQ,UAAU,OAAO,OAAO;AAAA,QAC3D;AAAA,QACA,EAAE,OAAO,QAAQ;AAAA,MACnB;AAAA,IACF;AACA,QAAI,CAAC,MAAO,OAAM,IAAI,iBAAiB,0CAA0C,cAAc;AAC/F,WAAO,iBAAiB,EAAE,GAAG,OAAO,eAAe,IAAI;AAAA,EACzD;AACF;AAGA,eAAsB,eAAkB,MAA0B;AAChE,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,SAAS,MAAM,MAAM;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,gCAAgC,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACtG;AACA,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,4BAA4B,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,EACjG;AACF;AAGA,eAAsB,UAAU,MAAc,SAAgC;AAC5E,QAAM,UAAU,MAAM,OAAO;AAC/B;AAGO,SAAS,UAAU,QAAgB,IAAoB;AAC5D,SAAO,GAAG,MAAM,IAAI,EAAE,GAAG,QAAQ,oBAAoB,GAAG;AAC1D;","names":["resolve"]}

package/dist/{chunk-2PVVP7GN.js → chunk-XKEFIFIC.js}

@@ -1,6 +1,6 @@
 import {
   benchRoot
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/toollm.ts
 import { readFile } from "fs/promises";
@@ -194,4 +194,4 @@ export {
   toollmOutput,
   createToolLlmAdapter
 };
-//# sourceMappingURL=chunk-2PVVP7GN.js.map
+//# sourceMappingURL=chunk-XKEFIFIC.js.map

package/dist/{chunk-JRWWGMK7.js → chunk-XYA4XSNU.js}

@@ -7,7 +7,7 @@ import {
   safeRunId,
   stageFile,
   venvPythonAt
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/terminal-bench.ts
 import { join } from "path";
@@ -145,4 +145,4 @@ function createTerminalBenchAdapter() {
 export {
   createTerminalBenchAdapter
 };
-//# sourceMappingURL=chunk-JRWWGMK7.js.map
+//# sourceMappingURL=chunk-XYA4XSNU.js.map

package/dist/{chunk-X5YKXC6V.js → chunk-YSMEKBTD.js}

@@ -1,7 +1,7 @@
 import {
   benchRoot,
   runVenvScriptStdin
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/enterpriseops-gym.ts
 import { mkdir, readFile, rm, writeFile } from "fs/promises";
@@ -208,4 +208,4 @@ export {
   enterpriseOpsTranscriptOutput,
   createEnterpriseOpsGymAdapter
 };
-//# sourceMappingURL=chunk-X5YKXC6V.js.map
+//# sourceMappingURL=chunk-YSMEKBTD.js.map

package/dist/{chunk-2XU6OGEN.js → chunk-Z4TZ76N7.js}

@@ -3,7 +3,7 @@ import {
   preflightVenvImports,
   runVenvScriptStdin,
   venvPythonAt
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 
 // src/benchmarks/commit0.ts
 import { join } from "path";
@@ -167,4 +167,4 @@ export {
   commit0DiffOutput,
   createCommit0Adapter
 };
-//# sourceMappingURL=chunk-2XU6OGEN.js.map
+//# sourceMappingURL=chunk-Z4TZ76N7.js.map

package/dist/index.js

@@ -1,47 +1,47 @@
 import {
   ADAPTERS,
   resolveAdapter
-} from "./chunk-PWQVGAJB.js";
-import "./chunk-2PVVP7GN.js";
+} from "./chunk-V7AEBY6U.js";
+import "./chunk-XKEFIFIC.js";
 import "./chunk-VQRS7VUC.js";
-import "./chunk-Z7ML6L77.js";
+import "./chunk-HWST3SED.js";
 import {
   createRagBenchAdapter
-} from "./chunk-R67DFVLO.js";
+} from "./chunk-IFVINJ4B.js";
 import "./chunk-SEVJPLZC.js";
 import {
   createSweBenchAdapter,
   scoreSweReport,
   sweEvaluationArgv,
   swePatchOutput
-} from "./chunk-CKUVRZ2T.js";
+} from "./chunk-3U5TXJZS.js";
 import {
   createT2RagBenchAdapter
-} from "./chunk-Y6O2OCUO.js";
-import "./chunk-HBSWHQNJ.js";
-import "./chunk-UAIOHCUK.js";
-import "./chunk-R36V2VP7.js";
-import "./chunk-JRWWGMK7.js";
+} from "./chunk-SHM6MRRF.js";
+import "./chunk-RCYQEFNX.js";
+import "./chunk-SFLA7OH3.js";
+import "./chunk-IZ5M6OAC.js";
+import "./chunk-XYA4XSNU.js";
 import "./chunk-INNOYXCP.js";
 import "./chunk-PA2ZKHJC.js";
 import "./chunk-5H5XV76F.js";
 import "./chunk-TBKU5XQI.js";
 import {
   createNoMiraclAdapter
-} from "./chunk-TSWPNOYM.js";
+} from "./chunk-NQG5XDSB.js";
 import {
   createOpenRagBenchAdapter
-} from "./chunk-KDIKRJGB.js";
-import "./chunk-ZEWMTR5M.js";
+} from "./chunk-SHYIRB7I.js";
+import "./chunk-MQMRLGOG.js";
 import "./chunk-PUIRNYI7.js";
 import "./chunk-C7T7WEK2.js";
 import {
   runRefineLoop
 } from "./chunk-JTHWEDEW.js";
-import "./chunk-2XU6OGEN.js";
+import "./chunk-Z4TZ76N7.js";
 import {
   createCragAdapter
-} from "./chunk-HHXFIHXC.js";
+} from "./chunk-SVR2LKYI.js";
 import {
   FINAL_ANSWER_SENTINEL,
   answerScoreToBenchScore,
@@ -54,18 +54,18 @@ import {
   scoreAnswerArtifact,
   tokenF1
 } from "./chunk-X3BTXCJ4.js";
-import "./chunk-J3KDJNX2.js";
-import "./chunk-X5YKXC6V.js";
-import "./chunk-YCGY7UIZ.js";
+import "./chunk-RH5F53JT.js";
+import "./chunk-YSMEKBTD.js";
+import "./chunk-7GRVHU22.js";
 import "./chunk-UPAMRDX4.js";
-import "./chunk-ODT47UAY.js";
-import "./chunk-7WSD27QQ.js";
-import "./chunk-SYDW647C.js";
-import "./chunk-IFAV6KEM.js";
+import "./chunk-K3BQGZCT.js";
+import "./chunk-PB64GYIG.js";
+import "./chunk-IA2FBTWC.js";
+import "./chunk-KP5KD6EN.js";
 import {
   StagedJudgeError,
   runStagedJudge
-} from "./chunk-LRRD7NAG.js";
+} from "./chunk-WSKWVEQB.js";
 import "./chunk-53UPUNBZ.js";
 
 // src/run-benchmarks.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.3.7",
+  "version": "0.3.8",
   "type": "module",
   "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.123.0",
+    "@tangle-network/agent-eval": "0.125.0",
     "@tangle-network/agent-interface": "0.32.0",
     "@tangle-network/agent-knowledge": "^4.1.0",
-    "@tangle-network/sandbox": "^0.11.1",
-    "@tangle-network/agent-runtime": "0.102.1"
+    "@tangle-network/sandbox": "^0.12.0",
+    "@tangle-network/agent-runtime": "0.104.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/scripts/verify-packed-consumer.mjs

@@ -120,7 +120,7 @@ try {
   )
   await writeFile(
     path.join(consumerDir, 'index.mjs'),
-    "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\n",
+    "import { resolveAdapter, runBenchmarks } from '@tangle-network/agent-bench'\nimport { ADAPTERS } from '@tangle-network/agent-bench/adapters'\nimport { createCragAdapter } from '@tangle-network/agent-bench/benchmarks/crag'\n\nif (typeof resolveAdapter !== 'function' || typeof runBenchmarks !== 'function') throw new Error('root exports are not executable')\nif (typeof ADAPTERS !== 'object' || typeof createCragAdapter !== 'function') throw new Error('subpath exports are not executable')\nif (resolveAdapter('crag').name !== 'crag') throw new Error('compiled adapter registry returned the wrong adapter')\nprocess.env.TOOLLM_FIXTURES = '1'\nconst toolLlmTasks = await resolveAdapter('toollm').loadTasks({ limit: 1 })\nif (toolLlmTasks.length !== 1 || toolLlmTasks[0]?.id !== '1') throw new Error('packed ToolLLM fixture loading failed')\n",
   )
   await writeFile(
     path.join(consumerDir, 'tsconfig.json'),

package/src/benchmarks/_harness.ts

@@ -18,6 +18,7 @@
 
 import { execFile, spawn } from 'node:child_process'
 import { createHash } from 'node:crypto'
+import { existsSync, readFileSync } from 'node:fs'
 import {
   cp,
   lstat,
@@ -42,8 +43,25 @@ import type {
 
 const execFileAsync = promisify(execFile)
 
-/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */
-export const benchRoot = fileURLToPath(new URL('../..', import.meta.url))
+/** Locate the package by identity because source files and compiled chunks have different depths. */
+function resolveBenchRoot(moduleUrl: string): string {
+  let current = dirname(fileURLToPath(moduleUrl))
+  while (true) {
+    const manifestPath = join(current, 'package.json')
+    if (existsSync(manifestPath)) {
+      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }
+      if (manifest.name === '@tangle-network/agent-bench') return current
+    }
+    const parent = dirname(current)
+    if (parent === current) {
+      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)
+    }
+    current = parent
+  }
+}
+
+/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */
+export const benchRoot = resolveBenchRoot(import.meta.url)
 
 /** Resolve the shared interpreter without requiring an installed package to contain a venv. */
 export function resolveBenchPython(

package/src/benchmarks/humaneval.test.mts

@@ -13,8 +13,8 @@ describe('HumanEval Python isolation', () => {
     writeFileSync(
       fakeDocker,
       `#!/usr/bin/env node
-const fs = require('node:fs')
-const path = require('node:path')
+import * as fs from 'node:fs'
+import * as path from 'node:path'
 const args = process.argv.slice(2)
 if (args[0] === 'rm') process.exit(0)
 if (process.env.FAKE_DOCKER_MISSING === '1') {

package/src/quant-arena/README.md

@@ -0,0 +1,144 @@
+# QUANT-ARENA — a self-improving trading-strategy lab
+
+A small, fully auditable research loop: an AI strategy author writes candidate strategies, every candidate is screened for look-ahead bias, backtested walk-forward on overlapping in-sample windows against pinned benchmarks, and judged by an acceptance rule whose bar **rises with every strategy tried**.
+Every attempt — accepted, rejected, or killed for leaking — becomes a permanent row in a lab notebook.
+The final two years of data are a locked out-of-sample set that only a separate certification command may touch, once.
+
+Everything is plain TypeScript you can read in an afternoon: the backtester is one file with zero dependencies, the acceptance math is one file, the data is committed CSV.
+
+## 1. Quickstart
+
+```bash
+# from bench/ (needs node >= 20, the `claude` CLI logged in, and `uv` on PATH
+# — scoring runs in a pinned python environment, see "Two engines" below)
+npx tsx src/quant-arena/quant-loop.mts --out /tmp/quant-demo --candidates 2
+```
+
+That runs a full research campaign: 2 strategy authors x 2 candidates each — the committed capture of exactly this command cost $1.65 of model spend across 8 metered calls; backtests are free.
+No API? `--skip-llm-audit` keeps everything but the adversarial code review; the mechanical look-ahead check still runs.
+
+Run the unit tests (backtester hand-computed cases, look-ahead detection, acceptance math, window reproducibility):
+
+```bash
+npx vitest run src/quant-arena
+```
+
+## 2. What happens when you run it
+
+1. **Data loads.** ~8 years of daily bars for 11 tickers (an index `IDX` plus `S01`-`S10`) from `fixtures/data/insample/`. The final 2 years live in `fixtures/data/holdout/` and are **not** loaded — see step 8. The series are synthetic (regime-switching factor model, seeded, regenerable) because the free real-data source we checked licenses personal use only; `fixtures/data/PROVENANCE.md` has the details and how to drop in your own CSVs.
+2. **Evaluation windows are drawn.** 8 overlapping 504-day (~2-year) blocks, block-bootstrap sampled from the in-sample years with a fixed seed — every candidate in the campaign is scored on the same windows, and reruns reproduce bit-identically.
+3. **Benchmarks run.** Three pinned incumbents: buy-and-hold the index, equal-weight monthly rebalance, and a 20/100 moving-average crossover. Their per-window Sharpe ratios define the bar: "best benchmark" is the per-window maximum.
+4. **Strategy authors write code.** Each author is a Claude call with a pinned identity (one plain, one with a quant-researcher system prompt). It gets the strategy contract, the universe summary, the benchmarks' per-window Sharpes, and the current acceptance bar — and must reply with one self-contained TypeScript module exporting `onBar(ctx)`: the harness calls it once per trading day with the bars **up to that day only** (the arrays are physically sliced, so reading the future is structurally impossible), plus the strategy's current holdings and equity, and it answers with target portfolio weights or "hold". Strategies never place orders — a single shared rebalancer (`oms.ts`) turns everyone's target weights into orders under the same sizing rule, LEAN-style `(targetWeight x equity - currentPosition) / price`, long-only. Model spend is metered into a durable cost log (`cost-ledger.jsonl`) with per-call receipts.
+5. **Look-ahead screening, stage 1 (mechanical).** The candidate is re-run on data truncated at several cutoff days. Signals up to each cutoff must be bit-identical to the full-data run — any divergence proves the code read the future, and the candidate is killed with the divergence quoted.
+6. **Look-ahead screening, stage 2 (adversarial).** A second, cheap model reads the source with one job: find look-ahead — indexing past `t`, whole-series statistics feeding per-day decisions, hardcoded dates that smell like memorization. Verdict is JSON; anything but a clean verdict kills the candidate, and an unparseable reply kills it too (the rule fails closed).
+7. **Backtest and verdict.** Survivors are backtested over the whole in-sample period (next-day-open fills, 15 bps one-way costs, no shorting, no leverage) and scored per window. Two engines run: the one-file TypeScript reference engine first, as a fail-closed contract check, then the industry-standard **vectorbt** engine (a persistent python worker in a version-locked environment) produces the official numbers. A parity test suite holds the two engines to agreement on golden fixtures — exact on a no-trade book, within machine precision whenever the book holds cash, and within a documented 0.5% on fully-invested books (the engines differ only in whether fees may be financed by a slightly negative cash balance). The acceptance rule (section 3) decides. Accepted or not, the try is appended to `notebook.jsonl` with its window scores, audit evidence, code hash, and authoring cost.
+8. **Certification, later and by hand.** When you believe a winner, run it once against the untouched final 2 years:
+
+   ```bash
+   npx tsx src/quant-arena/holdout-certify.mts --strategy <path>/strategy.ts --out /tmp/quant-demo
+   ```
+
+   It backtests in-sample + out-of-sample on one axis (so lookbacks are warm), scores only the out-of-sample days against the same three benchmarks, and appends the in-sample vs out-of-sample comparison to the notebook. A second run for the same strategy hash refuses without `--force` — an out-of-sample set answers once; re-rolling it until it agrees turns it into another in-sample set.
+
+## 3. Why the acceptance rule is strict
+
+If you test enough random strategies against the same data, the best one looks brilliant by luck alone.
+Under the assumption of zero skill, the expected best Sharpe among N independent tries grows roughly like sqrt(2 ln N) — try 50 strategies and luck alone buys the winner a substantial edge.
+So the bar a candidate must clear is not fixed: it is `0.10 + 0.15 * sqrt(2 ln N)` of mean excess Sharpe, where N counts **every** candidate ever tried in the campaign, including ones killed for leaking (this is a simplified, auditable version of the Deflated Sharpe Ratio of Bailey & Lopez de Prado, Journal of Portfolio Management, 2014).
+A candidate must ALSO beat the best benchmark in at least 6 of the 8 windows, because one lucky two-year stretch should never carry a decision.
+Every try is a permanent notebook row, so N can never be quietly reset — the price of another shot at the data is a higher bar for everyone after it.
+
+## 4. Reading the notebook
+
+`notebook.jsonl` is append-only JSON lines. Three row types:
+
+- `quant-arena.baselines.v1` — the campaign header: seed, cost assumptions, the 8 windows with dates, and each benchmark's per-window Sharpe.
+- `quant-arena.candidate.v1` — one per try. The fields that matter:
+  - `nTried` — this try's position in the campaign; sets its acceptance bar.
+  - `leakAudit.truncation` / `leakAudit.llm` — both screening verdicts with evidence.
+  - `eval.perWindow` — Sharpe vs best-benchmark Sharpe for each window, with dates.
+  - `verdict` — `accepted`, `rejected-no-edge` (failed the acceptance rule), `rejected-leak`, `rejected-contract` (didn't satisfy the module contract), or `rejected-error`.
+  - `reasons` — the decision spelled out, numbers included.
+- `quant-arena.certification.v1` — the one-shot out-of-sample result, in-sample stats side by side.
+
+A real excerpt from the committed demo campaign (`fixtures/demo-campaign/`): try #4 cleared both look-ahead screens, then lost to the benchmarks in 7 of 8 windows — and after four tries the bar it would have needed had already risen to 0.35 (condensed for width):
+
+```
+{
+  "candidateId": "cand-004-quant-researcher",
+  "proposer": "quant-researcher",
+  "nTried": 4,
+  "leakAudit": {
+    "truncation": {
+      "clean": true
+    },
+    "llm": {
+      "verdict": "clean"
+    }
+  },
+  "eval": {
+    "wins": 1,
+    "requiredWins": 6,
+    "meanExcessSharpe": -0.085,
+    "threshold": 0.35,
+    "perWindow": [
+      {
+        "startDate": "2017-04-06",
+        "endDate": "2019-03-12",
+        "sharpe": 0.94,
+        "bestBaselineSharpe": 1.21,
+        "excess": -0.27
+      },
+      {
+        "startDate": "2018-04-18",
+        "endDate": "2020-03-23",
+        "sharpe": 0.56,
+        "bestBaselineSharpe": 0.61,
+        "excess": -0.05
+      },
+      {
+        "startDate": "2018-08-06",
+        "endDate": "2020-07-09",
+        "sharpe": 0.2,
+        "bestBaselineSharpe": 0.29,
+        "excess": -0.08
+      },
+      "... 5 more windows"
+    ]
+  },
+  "verdict": "rejected-no-edge",
+  "reasons": [
+    "consistency: beat the best baseline in only 1/8 windows (need 6)",
+    "multiplicity: mean excess Sharpe -0.085 < required 0.350 (bar after 4 tried candidates)"
+  ]
+}
+```
+
+## 5. Plugging in your own backtester and data
+
+Scoring goes through one narrow seam: a worker process that takes `{open prices, close prices, target-weight rows, costs, windows}` as JSON lines on stdin and answers `{per-window total return / max drawdown / Sharpe / trade count, full equity curve}` on stdout — see the protocol comment at the top of `python/vbt-worker.py` and the client in `vbt-client.ts`.
+The shipped worker is vectorbt (`Portfolio.from_orders`, target-percent sizing, shared cash, sells before buys), version-locked by `python/pyproject.toml` + `python/uv.lock`; to swap in your own engine, speak the same protocol and keep the fill model (decide at close, fill at next open, bps fees on traded dollars) or re-derive the parity fixtures in `vbt-parity.test.mts` for your model.
+Point the data loader at your own Stooq-format CSVs (one file per ticker, `IDX.csv` as the benchmark asset, an `insample/` and a `holdout/` directory).
+Keep the physical in-sample/out-of-sample split and the once-only certification rule — they are the point, not an implementation detail.
+A third engine is planned but not built: event-driven certification of a winner's order stream through Nautilus Trader (`nautilus-certify.ts` is the named stub).
+
+## Files
+
+| file | what it is |
+| --- | --- |
+| `types.ts` | the strategy contract (v2 `onBar` + the order types), including the no-look-ahead rule |
+| `driver.ts` | the incremental harness: feeds `onBar` day by day with physically truncated history; wraps old batch strategies unchanged |
+| `oms.ts` | the one shared rebalancer: target weights -> orders (strategies never place orders) |
+| `backtest.ts` | the TypeScript reference engine: next-open fills, bps costs, no shorting — zero dependencies; contract prefilter |
+| `vbt-client.ts` + `python/vbt-worker.py` | the official scorer: persistent vectorbt worker, pinned env (`python/uv.lock`), crash-safe request handling |
+| `vbt-parity.test.mts` | the two engines held to agreement on golden fixtures (prints both curves on any disagreement) |
+| `nautilus-certify.ts` | named stub for the planned event-driven certification engine (not implemented) |
+| `windows.ts` | seeded block-bootstrap evaluation windows |
+| `multiplicity.ts` | the rising acceptance bar (documented formula + citation) |
+| `leak-audit.ts` | the mechanical truncation-invariance check |
+| `quant-loop.mts` | the campaign: author -> screen -> backtest -> verdict -> notebook |
+| `holdout-certify.mts` | the once-only out-of-sample certification |
+| `strategies/` | the three pinned benchmarks |
+| `fixtures/data/` | committed daily bars + provenance; `holdout/` is the locked final 2 years |
+| `fixtures/demo-campaign/` | a real captured campaign against the v1 batch contract: notebook, authored strategies, cost receipts |
+| `fixtures/demo-campaign-v2/` | a real captured campaign against the v2 `onBar` contract (1 candidate, honestly rejected: 0/8 windows) |