@tangle-network/agent-bench 0.3.8 → 0.4.1

package/dist/{chunk-WSKWVEQB.js → chunk-EIETHPD5.js}

@@ -44,10 +44,13 @@ function resolveBenchPython(env = process.env, root = benchRoot) {
   return configured;
 }
 var venvPython = resolveBenchPython();
-var venvPythonAt = (venvDir) => join(benchRoot, venvDir, "bin", "python");
+var venvPythonAt = (venvDir) => venvBinAt(venvDir, "python");
+function venvBinAt(venvDir, name) {
+  return join(resolve(benchRoot, venvDir), "bin", name);
+}
 var bigBuffer = 1024 * 1024 * 256;
 function venvBin(name) {
-  return join(benchRoot, ".venv", "bin", name);
+  return venvBinAt(".venv", name);
 }
 async function runVenvPython(script, args = [], timeoutMs = 0, python = venvPython) {
   const { stdout } = await execFileAsync(python, ["-c", script, ...args], {
@@ -303,6 +306,7 @@ export {
   resolveBenchPython,
   venvPython,
   venvPythonAt,
+  venvBinAt,
   bigBuffer,
   venvBin,
   runVenvPython,
@@ -314,4 +318,4 @@ export {
   stageFile,
   safeRunId
 };
-//# sourceMappingURL=chunk-WSKWVEQB.js.map
+//# sourceMappingURL=chunk-EIETHPD5.js.map

package/dist/chunk-EIETHPD5.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/_harness.ts"],"sourcesContent":["/**\n * Shared code-bench harness. The \"stage the artifact → run the benchmark's own\n * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)\n * → read its JSON report → { resolved, score }\" spine that swe-bench,\n * terminal-bench, commit0, programbench and aec-bench all need. Factored out so\n * the Docker/venv/report-reading logic lives in ONE place instead of being\n * copy-pasted per adapter.\n *\n * It owns NO benchmark policy: each adapter passes the argv for its evaluator\n * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The\n * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config\n * and fail-loud diagnostics.\n *\n * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`\n * (the adapter passes the import line + the exact fix). A staged run that exits\n * nonzero throws with the captured stderr — never a fabricated score.\n */\n\nimport { execFile, spawn } from 'node:child_process'\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport {\n  cp,\n  lstat,\n  mkdir,\n  mkdtemp,\n  readFile,\n  readlink,\n  readdir,\n  rename,\n  rm,\n  writeFile,\n} from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type {\n  BenchScore,\n  JudgeArtifactFileReceipt,\n  JudgeArtifactReceipt,\n} from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Locate the package by identity because source files and compiled chunks have different depths. */\nfunction resolveBenchRoot(moduleUrl: string): string {\n  let current = dirname(fileURLToPath(moduleUrl))\n  while (true) {\n    const manifestPath = join(current, 'package.json')\n    if (existsSync(manifestPath)) {\n      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }\n      if (manifest.name === '@tangle-network/agent-bench') return current\n    }\n    const parent = dirname(current)\n    if (parent === current) {\n      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)\n    }\n    current = parent\n  }\n}\n\n/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */\nexport const benchRoot = resolveBenchRoot(import.meta.url)\n\n/** Resolve the shared interpreter without requiring an installed package to contain a venv. */\nexport function resolveBenchPython(\n  env: Readonly<{ AGENT_BENCH_PYTHON?: string }> = process.env,\n  root: string = benchRoot,\n): string {\n  const configured = env.AGENT_BENCH_PYTHON\n  if (configured === undefined) return join(root, '.venv', 'bin', 'python')\n  if (!isAbsolute(configured)) {\n    throw new Error('AGENT_BENCH_PYTHON must be an absolute path')\n  }\n  return configured\n}\n\n/** The shared interpreter every Python-backed evaluator runs through. */\nexport const venvPython = resolveBenchPython()\n\n/** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip\n *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)\n *  get their own venv and pass its python explicitly — keeping the shared one clean. */\nexport const venvPythonAt = (venvDir: string): string => venvBinAt(venvDir, 'python')\n\n/** Resolve an executable in an isolated venv. Relative venv paths are package-owned;\n * absolute paths allow installed consumers to keep large environments elsewhere. */\nexport function venvBinAt(venvDir: string, name: string): string {\n  return join(resolve(benchRoot, venvDir), 'bin', name)\n}\n\n/** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */\nexport const bigBuffer = 1024 * 1024 * 256\n\n/** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */\nexport function venvBin(name: string): string {\n  return venvBinAt('.venv', name)\n}\n\n/**\n * Run the bench venv python with an inline script (`-c`); return stdout. Throws\n * (with stderr) on a nonzero exit — the loaders rely on this to fail loud rather\n * than parse a partial dump.\n */\nexport async function runVenvPython(\n  script: string,\n  args: string[] = [],\n  timeoutMs = 0,\n  python: string = venvPython,\n): Promise<string> {\n  const { stdout } = await execFileAsync(python, ['-c', script, ...args], {\n    maxBuffer: bigBuffer,\n    timeout: timeoutMs,\n  })\n  return stdout\n}\n\n/**\n * Preflight a python-backed harness: import the module(s) and (optionally) ping\n * Docker, all inside the bench venv. On failure THROWS the captured error joined\n * to the adapter's `fix` guidance — the contract every code-bench preflight wants.\n */\nexport async function preflightVenvImports(opts: {\n  /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */\n  modules: string[]\n  /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */\n  requireDocker?: boolean\n  /** Actionable remediation appended to the thrown message. */\n  fix: string\n  /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */\n  python?: string\n}): Promise<void> {\n  const imports = opts.modules.filter((m) => m.length > 0)\n  const lines = [...imports.map((m) => `import ${m}`)]\n  if (opts.requireDocker) lines.push('import docker', 'docker.from_env().ping()')\n  lines.push(\"print('ok')\")\n  try {\n    await runVenvPython(lines.join('\\n'), [], 0, opts.python ?? venvPython)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    throw new Error(`${msg}\\n${opts.fix}`)\n  }\n}\n\n/**\n * Run a bench-local python driver script (in the bench venv) while piping\n * `input` to its stdin, returning stdout. The driver's verdict JSON is its LAST\n * stdout line; callers parse that and inspect an `error` field (fail loud).\n *\n * Uses spawn + an explicit `stdin.end(input)` rather than promisify(execFile)'s\n * `input` option, because that option is NOT honored by async execFile — stdin\n * is left open and a driver that does `sys.stdin.read()` blocks forever. The\n * artifact-piping judges (commit0, appworld) MUST go through this.\n */\nexport function runVenvScriptStdin(\n  scriptPath: string,\n  args: string[],\n  input: string,\n  opts: { cwd?: string; timeoutMs?: number; python?: string } = {},\n): Promise<string> {\n  return new Promise<string>((resolve, reject) => {\n    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {\n      cwd: opts.cwd ?? benchRoot,\n      ...(opts.timeoutMs ? { timeout: opts.timeoutMs } : {}),\n    })\n    let stdout = ''\n    let stderr = ''\n    let bytes = 0\n    child.stdout.on('data', (c: Buffer) => {\n      bytes += c.length\n      if (bytes <= bigBuffer) stdout += c.toString('utf8')\n    })\n    child.stderr.on('data', (c: Buffer) => {\n      stderr += c.toString('utf8')\n    })\n    child.on('error', (err) => reject(err))\n    child.on('close', (code) => {\n      if (code === 0) resolve(stdout)\n      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)))\n    })\n    child.stdin.end(input)\n  })\n}\n\nexport interface StagedRunSpec {\n  /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */\n  tmpPrefix: string\n  /**\n   * Write the artifact + any harness inputs into the temp dir. Returns nothing;\n   * `argv`/`cwd` consume `dir` to point the evaluator at what was written.\n   */\n  stage(dir: string): Promise<void>\n  /** The external evaluator to spawn. `bin` defaults to the bench venv python. */\n  bin?: string\n  /** argv for the evaluator, computed from the temp `dir`. */\n  argv(dir: string): string[]\n  /** Working directory for the evaluator. Defaults to the temp `dir`. */\n  cwd?(dir: string): string\n  /** Hard timeout for the evaluator (ms); 0 = none. */\n  timeoutMs?: number\n  /**\n   * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws\n   * if the expected report is absent/malformed (fail loud — no default score).\n   */\n  parseReport(dir: string): Promise<BenchScore>\n  /**\n   * Copy the complete evaluator directory plus raw process stdout/stderr to this\n   * caller-owned directory before cleanup. The destination must not exist.\n   */\n  capture?: StagedRunCaptureSpec\n  /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */\n  keepTmp?: boolean\n}\n\nexport interface StagedRunCaptureSpec {\n  /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */\n  destination: string\n}\n\n/** A staged run failed after any requested evidence was durably retained. */\nexport class StagedJudgeError extends Error {\n  readonly judgeArtifacts?: JudgeArtifactReceipt\n\n  constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions) {\n    super(message, options)\n    this.name = 'StagedJudgeError'\n    this.judgeArtifacts = judgeArtifacts\n  }\n}\n\nfunction sha256(bytes: Uint8Array): `sha256:${string}` {\n  return `sha256:${createHash('sha256').update(bytes).digest('hex')}`\n}\n\nfunction portablePath(path: string): string {\n  return path.split(sep).join('/')\n}\n\nfunction compareText(left: string, right: string): number {\n  return left < right ? -1 : left > right ? 1 : 0\n}\n\nasync function collectArtifactFiles(\n  root: string,\n  current: string,\n): Promise<JudgeArtifactFileReceipt[]> {\n  const absolute = join(root, current)\n  const entries = await readdir(absolute, { withFileTypes: true })\n  const files: JudgeArtifactFileReceipt[] = []\n  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {\n    const relativePath = join(current, entry.name)\n    const path = join(root, relativePath)\n    if (entry.isDirectory()) {\n      files.push(...await collectArtifactFiles(root, relativePath))\n      continue\n    }\n    if (entry.isFile()) {\n      const bytes = await readFile(path)\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: bytes.byteLength,\n        sha256: sha256(bytes),\n        kind: 'file',\n      })\n      continue\n    }\n    if (entry.isSymbolicLink()) {\n      const targetBytes = await readlink(path, { encoding: 'buffer' })\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: targetBytes.byteLength,\n        sha256: sha256(targetBytes),\n        kind: 'symlink',\n      })\n      continue\n    }\n    throw new Error(`staged judge capture does not support ${relativePath}`)\n  }\n  return files\n}\n\nfunction isWithin(parent: string, candidate: string): boolean {\n  const path = relative(parent, candidate)\n  return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path))\n}\n\nasync function assertDestinationAbsent(destination: string): Promise<void> {\n  try {\n    await lstat(destination)\n  } catch (error) {\n    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return\n    throw error\n  }\n  throw new Error(`staged judge capture destination already exists: ${destination}`)\n}\n\nasync function captureStagedRun(\n  sourceDirectory: string,\n  spec: StagedRunCaptureSpec,\n  processOutput: Readonly<{ stdout: Buffer; stderr: Buffer }>,\n  evaluatorSucceeded: boolean,\n): Promise<JudgeArtifactReceipt> {\n  const source = resolve(sourceDirectory)\n  const destination = resolve(spec.destination)\n  if (isWithin(source, destination)) {\n    throw new Error('staged judge capture destination must be outside the evaluator directory')\n  }\n  await mkdir(dirname(destination), { recursive: true })\n  await assertDestinationAbsent(destination)\n  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`))\n  try {\n    await cp(source, join(staging, 'evaluator'), {\n      recursive: true,\n      errorOnExist: true,\n      force: false,\n      preserveTimestamps: true,\n      verbatimSymlinks: true,\n    })\n    await mkdir(join(staging, 'process'))\n    await writeFile(join(staging, 'process', 'stdout.bin'), processOutput.stdout)\n    await writeFile(join(staging, 'process', 'stderr.bin'), processOutput.stderr)\n\n    const files = [\n      ...await collectArtifactFiles(staging, 'evaluator'),\n      ...await collectArtifactFiles(staging, 'process'),\n    ].sort((left, right) => compareText(left.path, right.path))\n    const byteLength = files.reduce((total, file) => total + file.byteLength, 0)\n    const treeBytes = Buffer.from(\n      files\n        .map((file) => `${file.path}\\0${file.kind}\\0${file.byteLength}\\0${file.sha256}\\n`)\n        .join(''),\n      'utf8',\n    )\n    const receipt: JudgeArtifactReceipt = {\n      schema: 'agent-bench/judge-artifacts/v1',\n      directory: destination,\n      evaluatorDirectory: join(destination, 'evaluator'),\n      manifestPath: join(destination, 'receipt.json'),\n      evaluatorSucceeded,\n      files,\n      fileCount: files.length,\n      byteLength,\n      treeSha256: sha256(treeBytes),\n    }\n    await writeFile(join(staging, 'receipt.json'), `${JSON.stringify(receipt, null, 2)}\\n`)\n    await rename(staging, destination)\n    return receipt\n  } catch (error) {\n    await rm(staging, { recursive: true, force: true }).catch(() => {})\n    throw error\n  }\n}\n\nfunction processBytes(value: unknown): Buffer {\n  if (Buffer.isBuffer(value)) return value\n  if (value === undefined || value === null) return Buffer.alloc(0)\n  return Buffer.from(String(value), 'utf8')\n}\n\n/**\n * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →\n * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is\n * always removed in `finally` unless `keepTmp`.\n */\nexport async function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore> {\n  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix))\n  let stdout: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let stderr: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let evaluatorSucceeded = false\n  let score: BenchScore | undefined\n  let failure: unknown\n  try {\n    try {\n      await spec.stage(dir)\n      const bin = spec.bin ?? venvPython\n      const argv = spec.argv(dir)\n      try {\n        const output = await execFileAsync(bin, argv, {\n          cwd: spec.cwd ? spec.cwd(dir) : dir,\n          encoding: 'buffer',\n          maxBuffer: bigBuffer,\n          ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),\n        })\n        stdout = processBytes(output.stdout)\n        stderr = processBytes(output.stderr)\n        evaluatorSucceeded = true\n      } catch (err) {\n        const e = err as { stderr?: unknown; stdout?: unknown; message?: string }\n        stdout = processBytes(e.stdout)\n        stderr = processBytes(e.stderr)\n        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err))\n          .toString('utf8')\n          .slice(0, 2000)\n        throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${argv.join(' ')}):\\n${detail}`)\n      }\n      score = await spec.parseReport(dir)\n    } catch (err) {\n      failure = err\n    }\n  } finally {\n    let judgeArtifacts: JudgeArtifactReceipt | undefined\n    if (spec.capture) {\n      try {\n        judgeArtifacts = await captureStagedRun(\n          dir,\n          spec.capture,\n          { stdout, stderr },\n          evaluatorSucceeded,\n        )\n      } catch (captureError) {\n        failure = new Error(\n          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,\n          { cause: failure ?? captureError },\n        )\n      }\n    }\n    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {})\n    if (failure) {\n      throw new StagedJudgeError(\n        failure instanceof Error ? failure.message : String(failure),\n        judgeArtifacts,\n        { cause: failure },\n      )\n    }\n    if (!score) throw new StagedJudgeError('staged judge completed without a score', judgeArtifacts)\n    return judgeArtifacts ? { ...score, judgeArtifacts } : score\n  }\n}\n\n/** Read + JSON.parse a report file from a staged run; throws with the path on failure. */\nexport async function readJsonReport<T>(path: string): Promise<T> {\n  let raw: string\n  try {\n    raw = await readFile(path, 'utf8')\n  } catch (err) {\n    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`)\n  }\n  try {\n    return JSON.parse(raw) as T\n  } catch (err) {\n    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`)\n  }\n}\n\n/** Write a UTF-8 file into a staged dir (artifact / preds.json / attempt.sh / …). */\nexport async function stageFile(path: string, content: string): Promise<void> {\n  await writeFile(path, content)\n}\n\n/** Sanitize an instance id into a filesystem/run-id-safe token. */\nexport function safeRunId(prefix: string, id: string): string {\n  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, '_')\n}\n"],"mappings":";AAkBA,SAAS,UAAU,aAAa;AAChC,SAAS,kBAAkB;AAC3B,SAAS,YAAY,oBAAoB;AACzC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AACvB,SAAS,UAAU,SAAS,YAAY,MAAM,UAAU,SAAS,WAAW;AAC5E,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAO1B,IAAM,gBAAgB,UAAU,QAAQ;AAGxC,SAAS,iBAAiB,WAA2B;AACnD,MAAI,UAAU,QAAQ,cAAc,SAAS,CAAC;AAC9C,SAAO,MAAM;AACX,UAAM,eAAe,KAAK,SAAS,cAAc;AACjD,QAAI,WAAW,YAAY,GAAG;AAC5B,YAAM,WAAW,KAAK,MAAM,aAAa,cAAc,MAAM,CAAC;AAC9D,UAAI,SAAS,SAAS,8BAA+B,QAAO;AAAA,IAC9D;AACA,UAAM,SAAS,QAAQ,OAAO;AAC9B,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI,MAAM,qDAAqD,SAAS,EAAE;AAAA,IAClF;AACA,cAAU;AAAA,EACZ;AACF;AAGO,IAAM,YAAY,iBAAiB,YAAY,GAAG;AAGlD,SAAS,mBACd,MAAiD,QAAQ,KACzD,OAAe,WACP;AACR,QAAM,aAAa,IAAI;AACvB,MAAI,eAAe,OAAW,QAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;AACxE,MAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AACA,SAAO;AACT;AAGO,IAAM,aAAa,mBAAmB;AAKtC,IAAM,eAAe,CAAC,YAA4B,UAAU,SAAS,QAAQ;AAI7E,SAAS,UAAU,SAAiB,MAAsB;AAC/D,SAAO,KAAK,QAAQ,WAAW,OAAO,GAAG,OAAO,IAAI;AACtD;AAGO,IAAM,YAAY,OAAO,OAAO;AAGhC,SAAS,QAAQ,MAAsB;AAC5C,SAAO,UAAU,SAAS,IAAI;AAChC;AAOA,eAAsB,cACpB,QACA,OAAiB,CAAC,GAClB,YAAY,GACZ,SAAiB,YACA;AACjB,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,QAAQ,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IACtE,WAAW;AAAA,IACX,SAAS;AAAA,EACX,CAAC;AACD,SAAO;AACT;AAOA,eAAsB,qBAAqB,MASzB;AAChB,QAAM,UAAU,KAAK,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AACvD,QAAM,QAAQ,CAAC,GAAG,QAAQ,IAAI,CAAC,MAAM,UAAU,CAAC,EAAE,CAAC;AACnD,MAAI,KAAK,cAAe,OAAM,KAAK,iBAAiB,0BAA0B;AAC9E,QAAM,KAAK,aAAa;AACxB,MAAI;AACF,UAAM,cAAc,MAAM,KAAK,IAAI,GAAG,CAAC,GAAG,GAAG,KAAK,UAAU,UAAU;AAAA,EACxE,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI,MAAM,GAAG,GAAG;AAAA,EAAK,KAAK,GAAG,EAAE;AAAA,EACvC;AACF;AAYO,SAAS,mBACd,YACA,MACA,OACA,OAA8D,CAAC,GAC9C;AACjB,SAAO,IAAI,QAAgB,CAACA,UAAS,WAAW;AAC9C,UAAM,QAAQ,MAAM,KAAK,UAAU,YAAY,CAAC,YAAY,GAAG,IAAI,GAAG;AAAA,MACpE,KAAK,KAAK,OAAO;AAAA,MACjB,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,IACtD,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,QAAQ;AACZ,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,eAAS,EAAE;AACX,UAAI,SAAS,UAAW,WAAU,EAAE,SAAS,MAAM;AAAA,IACrD,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,gBAAU,EAAE,SAAS,MAAM;AAAA,IAC7B,CAAC;AACD,UAAM,GAAG,SAAS,CAAC,QAAQ,OAAO,GAAG,CAAC;AACtC,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,UAAI,SAAS,EAAG,CAAAA,SAAQ,MAAM;AAAA,UACzB,QAAO,IAAI,OAAO,UAAU,UAAU,QAAQ,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC;AAAA,IAC5E,CAAC;AACD,UAAM,MAAM,IAAI,KAAK;AAAA,EACvB,CAAC;AACH;AAsCO,IAAM,mBAAN,cAA+B,MAAM;AAAA,EACjC;AAAA,EAET,YAAY,SAAiB,gBAAuC,SAAwB;AAC1F,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO;AACZ,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,SAAS,OAAO,OAAuC;AACrD,SAAO,UAAU,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK,CAAC;AACnE;AAEA,SAAS,aAAa,MAAsB;AAC1C,SAAO,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG;AACjC;AAEA,SAAS,YAAY,MAAc,OAAuB;AACxD,SAAO,OAAO,QAAQ,KAAK,OAAO,QAAQ,IAAI;AAChD;AAEA,eAAe,qBACb,MACA,SACqC;AACrC,QAAM,WAAW,KAAK,MAAM,OAAO;AACnC,QAAM,UAAU,MAAM,QAAQ,UAAU,EAAE,eAAe,KAAK,CAAC;AAC/D,QAAM,QAAoC,CAAC;AAC3C,aAAW,SAAS,QAAQ,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG;AACrF,UAAM,eAAe,KAAK,SAAS,MAAM,IAAI;AAC7C,UAAM,OAAO,KAAK,MAAM,YAAY;AACpC,QAAI,MAAM,YAAY,GAAG;AACvB,YAAM,KAAK,GAAG,MAAM,qBAAqB,MAAM,YAAY,CAAC;AAC5D;AAAA,IACF;AACA,QAAI,MAAM,OAAO,GAAG;AAClB,YAAM,QAAQ,MAAM,SAAS,IAAI;AACjC,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,MAAM;AAAA,QAClB,QAAQ,OAAO,KAAK;AAAA,QACpB,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,QAAI,MAAM,eAAe,GAAG;AAC1B,YAAM,cAAc,MAAM,SAAS,MAAM,EAAE,UAAU,SAAS,CAAC;AAC/D,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,YAAY;AAAA,QACxB,QAAQ,OAAO,WAAW;AAAA,QAC1B,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,UAAM,IAAI,MAAM,yCAAyC,YAAY,EAAE;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAgB,WAA4B;AAC5D,QAAM,OAAO,SAAS,QAAQ,SAAS;AACvC,SAAO,SAAS,MAAO,CAAC,KAAK,WAAW,KAAK,GAAG,EAAE,KAAK,SAAS,QAAQ,CAAC,WAAW,IAAI;AAC1F;AAEA,eAAe,wBAAwB,aAAoC;AACzE,MAAI;AACF,UAAM,MAAM,WAAW;AAAA,EACzB,SAAS,OAAO;AACd,QAAK,MAAgC,SAAS,SAAU;AACxD,UAAM;AAAA,EACR;AACA,QAAM,IAAI,MAAM,oDAAoD,WAAW,EAAE;AACnF;AAEA,eAAe,iBACb,iBACA,MACA,eACA,oBAC+B;AAC/B,QAAM,SAAS,QAAQ,eAAe;AACtC,QAAM,cAAc,QAAQ,KAAK,WAAW;AAC5C,MAAI,SAAS,QAAQ,WAAW,GAAG;AACjC,UAAM,IAAI,MAAM,0EAA0E;AAAA,EAC5F;AACA,QAAM,MAAM,QAAQ,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,QAAM,wBAAwB,WAAW;AACzC,QAAM,UAAU,MAAM,QAAQ,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,CAAC,WAAW,CAAC;AAC9F,MAAI;AACF,UAAM,GAAG,QAAQ,KAAK,SAAS,WAAW,GAAG;AAAA,MAC3C,WAAW;AAAA,MACX,cAAc;AAAA,MACd,OAAO;AAAA,MACP,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB,CAAC;AACD,UAAM,MAAM,KAAK,SAAS,SAAS,CAAC;AACpC,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAC5E,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAE5E,UAAM,QAAQ;AAAA,MACZ,GAAG,MAAM,qBAAqB,SAAS,WAAW;AAAA,MAClD,GAAG,MAAM,qBAAqB,SAAS,SAAS;AAAA,IAClD,EAAE,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC;AAC1D,UAAM,aAAa,MAAM,OAAO,CAAC,OAAO,SAAS,QAAQ,KAAK,YAAY,CAAC;AAC3E,UAAM,YAAY,OAAO;AAAA,MACvB,MACG,IAAI,CAAC,SAAS,GAAG,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,UAAU,KAAK,KAAK,MAAM;AAAA,CAAI,EAChF,KAAK,EAAE;AAAA,MACV;AAAA,IACF;AACA,UAAM,UAAgC;AAAA,MACpC,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,oBAAoB,KAAK,aAAa,WAAW;AAAA,MACjD,cAAc,KAAK,aAAa,cAAc;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB;AAAA,MACA,YAAY,OAAO,SAAS;AAAA,IAC9B;AACA,UAAM,UAAU,KAAK,SAAS,cAAc,GAAG,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,CAAI;AACtF,UAAM,OAAO,SAAS,WAAW;AACjC,WAAO;AAAA,EACT,SAAS,OAAO;AACd,UAAM,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAClE,UAAM;AAAA,EACR;AACF;AAEA,SAAS,aAAa,OAAwB;AAC5C,MAAI,OAAO,SAAS,KAAK,EAAG,QAAO;AACnC,MAAI,UAAU,UAAa,UAAU,KAAM,QAAO,OAAO,MAAM,CAAC;AAChE,SAAO,OAAO,KAAK,OAAO,KAAK,GAAG,MAAM;AAC1C;AAOA,eAAsB,eAAe,MAA0C;AAC7E,QAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,KAAK,SAAS,CAAC;AACxD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,qBAAqB;AACzB,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,QAAI;AACF,YAAM,KAAK,MAAM,GAAG;AACpB,YAAM,MAAM,KAAK,OAAO;AACxB,YAAM,OAAO,KAAK,KAAK,GAAG;AAC1B,UAAI;AACF,cAAM,SAAS,MAAM,cAAc,KAAK,MAAM;AAAA,UAC5C,KAAK,KAAK,MAAM,KAAK,IAAI,GAAG,IAAI;AAAA,UAChC,UAAU;AAAA,UACV,WAAW;AAAA,UACX,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,QACtD,CAAC;AACD,iBAAS,aAAa,OAAO,MAAM;AACnC,iBAAS,aAAa,OAAO,MAAM;AACnC,6BAAqB;AAAA,MACvB,SAAS,KAAK;AACZ,cAAM,IAAI;AACV,iBAAS,aAAa,EAAE,MAAM;AAC9B,iBAAS,aAAa,EAAE,MAAM;AAC9B,cAAM,SAAS,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG,CAAC,EACzE,SAAS,MAAM,EACf,MAAM,GAAG,GAAI;AAChB,cAAM,IAAI,MAAM,GAAG,KAAK,UAAU,QAAQ,MAAM,EAAE,CAAC,sBAAsB,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC;AAAA,EAAO,MAAM,EAAE;AAAA,MAC/G;AACA,cAAQ,MAAM,KAAK,YAAY,GAAG;AAAA,IACpC,SAAS,KAAK;AACZ,gBAAU;AAAA,IACZ;AAAA,EACF,UAAE;AACA,QAAI;AACJ,QAAI,KAAK,SAAS;AAChB,UAAI;AACF,yBAAiB,MAAM;AAAA,UACrB;AAAA,UACA,KAAK;AAAA,UACL,EAAE,QAAQ,OAAO;AAAA,UACjB;AAAA,QACF;AAAA,MACF,SAAS,cAAc;AACrB,kBAAU,IAAI;AAAA,UACZ,uDAAuD,wBAAwB,QAAQ,aAAa,UAAU,YAAY;AAAA,UAC1H,EAAE,OAAO,WAAW,aAAa;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,KAAK,QAAS,OAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACjF,QAAI,SAAS;AACX,YAAM,IAAI;AAAA,QACR,mBAAmB,QAAQ,QAAQ,UAAU,OAAO,OAAO;AAAA,QAC3D;AAAA,QACA,EAAE,OAAO,QAAQ;AAAA,MACnB;AAAA,IACF;AACA,QAAI,CAAC,MAAO,OAAM,IAAI,iBAAiB,0CAA0C,cAAc;AAC/F,WAAO,iBAAiB,EAAE,GAAG,OAAO,eAAe,IAAI;AAAA,EACzD;AACF;AAGA,eAAsB,eAAkB,MAA0B;AAChE,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,SAAS,MAAM,MAAM;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,gCAAgC,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACtG;AACA,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,4BAA4B,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,EACjG;AACF;AAGA,eAAsB,UAAU,MAAc,SAAgC;AAC5E,QAAM,UAAU,MAAM,OAAO;AAC/B;AAGO,SAAS,UAAU,QAAgB,IAAoB;AAC5D,SAAO,GAAG,MAAM,IAAI,EAAE,GAAG,QAAQ,oBAAoB,GAAG;AAC1D;","names":["resolve"]}

package/dist/{chunk-SHM6MRRF.js → chunk-GC2EPS6L.js}

@@ -14,7 +14,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/t2-ragbench.ts
 import { readFile } from "fs/promises";
@@ -127,4 +127,4 @@ function createT2RagBenchAdapter() {
 export {
   createT2RagBenchAdapter
 };
-//# sourceMappingURL=chunk-SHM6MRRF.js.map
+//# sourceMappingURL=chunk-GC2EPS6L.js.map

package/dist/{chunk-IZ5M6OAC.js → chunk-GCHL6XPM.js}

@@ -1,6 +1,6 @@
 import {
   runVenvPython
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/tau-bench-shared.ts
 import { readFile, stat } from "fs/promises";
@@ -166,4 +166,4 @@ export {
   tauResultsOutput,
   createTauBenchAdapter
 };
-//# sourceMappingURL=chunk-IZ5M6OAC.js.map
+//# sourceMappingURL=chunk-GCHL6XPM.js.map

package/dist/{chunk-IFVINJ4B.js → chunk-HQ5HCCKF.js}

@@ -14,7 +14,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/ragbench.ts
 import { readFile } from "fs/promises";
@@ -139,4 +139,4 @@ function createRagBenchAdapter() {
 export {
   createRagBenchAdapter
 };
-//# sourceMappingURL=chunk-IFVINJ4B.js.map
+//# sourceMappingURL=chunk-HQ5HCCKF.js.map

package/dist/{chunk-7GRVHU22.js → chunk-HVW25KSX.js}

@@ -1,6 +1,6 @@
 import {
   benchRoot
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/finresearchbench.ts
 import { readFile, stat } from "fs/promises";
@@ -205,4 +205,4 @@ function createFinResearchBenchAdapter() {
 export {
   createFinResearchBenchAdapter
 };
-//# sourceMappingURL=chunk-7GRVHU22.js.map
+//# sourceMappingURL=chunk-HVW25KSX.js.map

package/dist/{chunk-3U5TXJZS.js → chunk-J6BU3NTM.js}

@@ -5,7 +5,7 @@ import {
   runVenvPython,
   safeRunId,
   stageFile
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/swe-bench.ts
 import { join } from "path";
@@ -248,4 +248,4 @@ export {
   sweEvaluationArgv,
   createSweBenchAdapter
 };
-//# sourceMappingURL=chunk-3U5TXJZS.js.map
+//# sourceMappingURL=chunk-J6BU3NTM.js.map

package/dist/{chunk-XYA4XSNU.js → chunk-JSQOUKXS.js}

@@ -6,14 +6,15 @@ import {
   runVenvPython,
   safeRunId,
   stageFile,
+  venvBinAt,
   venvPythonAt
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/terminal-bench.ts
 import { join } from "path";
 var terminalBenchVenvDir = () => process.env.TERMINAL_BENCH_VENV ?? ".venv-terminal-bench";
 var terminalBenchPython = () => venvPythonAt(terminalBenchVenvDir());
-var terminalBenchBin = () => join(benchRoot, terminalBenchVenvDir(), "bin", "tb");
+var terminalBenchBin = () => venvBinAt(terminalBenchVenvDir(), "tb");
 var DATASET = "terminal-bench-core";
 var DATASET_VERSION = "0.1.1";
 var DATASET_REF = `${DATASET}==${DATASET_VERSION}`;
@@ -145,4 +146,4 @@ function createTerminalBenchAdapter() {
 export {
   createTerminalBenchAdapter
 };
-//# sourceMappingURL=chunk-XYA4XSNU.js.map
+//# sourceMappingURL=chunk-JSQOUKXS.js.map

package/dist/chunk-JSQOUKXS.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/terminal-bench.ts"],"sourcesContent":["/**\n * Terminal-Bench adapter. Each task = a Docker environment + an English\n * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact\n * is the shell script the agent ran to attempt the task; the judge REPLAYS that\n * script in a fresh task container via the Terminal-Bench harness (`tb run` with\n * our ScriptAgent), then the task's own verifier scores the resulting state.\n * Fully deterministic — no LLM judge, no self-authored score.\n *\n * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`\n * installed + a running Docker daemon (per-task images are built on first run).\n * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the\n * Terminal-Bench registry on first run.\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay\n * argv, and the results.json shape.\n */\n\nimport { join } from 'node:path'\nimport {\n  benchRoot,\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  venvBinAt,\n  venvPythonAt,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n// Terminal-Bench imports LiteLLM/Pydantic-2 APIs, while AppWorld pins Pydantic 1.\n// Keep it out of the shared bench .venv. Resolved at call-time so tests/runs can\n// override the env without reloading this module.\nconst terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench'\nconst terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir())\nconst terminalBenchBin = (): string => venvBinAt(terminalBenchVenvDir(), 'tb')\n\n// Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the\n// installed CLI) and is the published launch task set. name==version is what `tb\n// run -d` and `tb datasets download -d` both accept.\nconst DATASET = 'terminal-bench-core'\nconst DATASET_VERSION = '0.1.1'\nconst DATASET_REF = `${DATASET}==${DATASET_VERSION}`\n\n// Bundled fixture: when no ids/limit are given, load these. hello-world is the\n// fastest deterministic task (prebuilt python image, file-write verifier) so the\n// adapter is runnable without a large pull.\nconst FIXTURE_IDS = ['hello-world']\n\n// Import path the harness uses to load our replay agent (cwd = benchRoot).\nconst SCRIPT_AGENT = 'tb_agents.script_agent:ScriptAgent'\n\ninterface TbTaskRow {\n  id: string\n  instruction: string\n  task_dir: string\n  solution: string | null\n}\n\ninterface TbReport {\n  resolved_ids?: string[]\n  results?: Array<{ task_id: string; is_resolved: boolean | null; parser_results?: unknown }>\n}\n\n/** Enumerate dataset tasks via tb's Dataset loader (caches from the registry on\n *  first run). Reads instruction from each task.yaml and the gold solution. */\nasync function loadRows(opts: LoadOptions): Promise<TbTaskRow[]> {\n  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS)\n  const limit = opts.limit ?? null\n  const script = `\nimport json, sys\nfrom pathlib import Path\nfrom terminal_bench.dataset.dataset import Dataset\nfrom terminal_bench.handlers.trial_handler import TaskPaths\n\nreq_ids = json.loads(sys.argv[1]) if sys.argv[1] else None\nlimit = json.loads(sys.argv[2]) if sys.argv[2] else None\n\nds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)\n\nimport yaml\nout = []\nfor task_dir in ds:\n    tp = TaskPaths(task_dir)\n    cfg = yaml.safe_load(tp.task_config_path.read_text())\n    try:\n        sol = tp.solution_path.read_text()\n    except FileNotFoundError:\n        sol = None\n    out.append({\n        \"id\": task_dir.name,\n        \"instruction\": cfg[\"instruction\"],\n        \"task_dir\": str(task_dir),\n        \"solution\": sol if (tp.solution_path.suffix == \".sh\" if sol is not None else False) else None,\n    })\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(\n    script,\n    [ids ? JSON.stringify(ids) : '', limit !== null ? String(limit) : ''],\n    0,\n    terminalBenchPython(),\n  )\n  return JSON.parse(stdout) as TbTaskRow[]\n}\n\nexport function createTerminalBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'terminal-bench',\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['terminal_bench'],\n        requireDocker: true,\n        python: terminalBenchPython(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && ` +\n          `bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench ` +\n          `(an ISOLATED venv — Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; ` +\n          `override the dir with TERMINAL_BENCH_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-task images on first run). ` +\n          `The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const rows = await loadRows(opts)\n      if (rows.length === 0) {\n        throw new Error(\n          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} ` +\n            `(dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`,\n        )\n      }\n      return rows.map(\n        (r): BenchTask => ({\n          id: r.id,\n          split: DATASET_VERSION,\n          prompt: r.instruction,\n          metadata: {\n            dataset: DATASET,\n            datasetVersion: DATASET_VERSION,\n            datasetRef: DATASET_REF,\n            taskDir: r.task_dir,\n            solution: r.solution,\n            instruction: r.instruction,\n          },\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the task's solution.sh (the oracle script). solution.yaml tasks have\n      // no shell-script artifact form here, so they return undefined (cannot be\n      // verify-judged via the script-replay seam — use a .sh-solution task).\n      const sol = task.metadata?.solution\n      return typeof sol === 'string' ? sol : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', `${task.id}-${Date.now()}`)\n      return runStagedJudge({\n        tmpPrefix: 'tbench-',\n        bin: terminalBenchBin(),\n        cwd: () => benchRoot,\n        async stage(dir) {\n          await stageFile(join(dir, 'attempt.sh'), artifact)\n        },\n        // The harness builds a fresh task container, runs ScriptAgent (which replays\n        // the artifact script), then runs the task's verifier. --no-livestream keeps\n        // stdout sane; --cleanup removes the per-run images.\n        argv: (dir) => [\n          'run',\n          '-d', DATASET_REF,\n          '-t', task.id,\n          '--agent-import-path', SCRIPT_AGENT,\n          '--agent-kwarg', `script_path=${join(dir, 'attempt.sh')}`,\n          '--output-path', join(dir, 'runs'),\n          '--run-id', runId,\n          '--n-concurrent', '1',\n          '--no-livestream',\n          '--cleanup',\n        ],\n        async parseReport(dir) {\n          const report = await readJsonReport<TbReport>(join(dir, 'runs', runId, 'results.json'))\n          const resolved = (report.resolved_ids ?? []).includes(task.id)\n          const trial = report.results?.find((r) => r.task_id === task.id)\n          return {\n            resolved,\n            score: resolved ? 1 : 0,\n            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {}),\n          }\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAkBA,SAAS,YAAY;AAiBrB,IAAM,uBAAuB,MAAc,QAAQ,IAAI,uBAAuB;AAC9E,IAAM,sBAAsB,MAAc,aAAa,qBAAqB,CAAC;AAC7E,IAAM,mBAAmB,MAAc,UAAU,qBAAqB,GAAG,IAAI;AAK7E,IAAM,UAAU;AAChB,IAAM,kBAAkB;AACxB,IAAM,cAAc,GAAG,OAAO,KAAK,eAAe;AAKlD,IAAM,cAAc,CAAC,aAAa;AAGlC,IAAM,eAAe;AAgBrB,eAAe,SAAS,MAAyC;AAC/D,QAAM,MAAM,KAAK,QAAQ,KAAK,QAAQ,OAAO;AAC7C,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oBASG,KAAK,UAAU,OAAO,CAAC,aAAa,KAAK,UAAU,eAAe,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBrF,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA,CAAC,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,UAAU,OAAO,OAAO,KAAK,IAAI,EAAE;AAAA,IACpE;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO,KAAK,MAAM,MAAM;AAC1B;AAEO,SAAS,6BAA+C;AAC7D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,gBAAgB;AAAA,QAC1B,eAAe;AAAA,QACf,QAAQ,oBAAoB;AAAA,QAC5B,KACE,kCAAkC,qBAAqB,CAAC,aAC/C,qBAAqB,CAAC,gRAIxB,WAAW;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,OAAO,MAAM,SAAS,IAAI;AAChC,UAAI,KAAK,WAAW,GAAG;AACrB,cAAM,IAAI;AAAA,UACR,kDAAkD,KAAK,UAAU,IAAI,CAAC,aACxD,WAAW;AAAA,QAC3B;AAAA,MACF;AACA,aAAO,KAAK;AAAA,QACV,CAAC,OAAkB;AAAA,UACjB,IAAI,EAAE;AAAA,UACN,OAAO;AAAA,UACP,QAAQ,EAAE;AAAA,UACV,UAAU;AAAA,YACR,SAAS;AAAA,YACT,gBAAgB;AAAA,YAChB,YAAY;AAAA,YACZ,SAAS,EAAE;AAAA,YACX,UAAU,EAAE;AAAA,YACZ,aAAa,EAAE;AAAA,UACjB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,MAAM,KAAK,UAAU;AAC3B,aAAO,OAAO,QAAQ,WAAW,MAAM;AAAA,IACzC;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,QAAQ,UAAU,SAAS,GAAG,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AAC3D,aAAO,eAAe;AAAA,QACpB,WAAW;AAAA,QACX,KAAK,iBAAiB;AAAA,QACtB,KAAK,MAAM;AAAA,QACX,MAAM,MAAM,KAAK;AACf,gBAAM,UAAU,KAAK,KAAK,YAAY,GAAG,QAAQ;AAAA,QACnD;AAAA;AAAA;AAAA;AAAA,QAIA,MAAM,CAAC,QAAQ;AAAA,UACb;AAAA,UACA;AAAA,UAAM;AAAA,UACN;AAAA,UAAM,KAAK;AAAA,UACX;AAAA,UAAuB;AAAA,UACvB;AAAA,UAAiB,eAAe,KAAK,KAAK,YAAY,CAAC;AAAA,UACvD;AAAA,UAAiB,KAAK,KAAK,MAAM;AAAA,UACjC;AAAA,UAAY;AAAA,UACZ;AAAA,UAAkB;AAAA,UAClB;AAAA,UACA;AAAA,QACF;AAAA,QACA,MAAM,YAAY,KAAK;AACrB,gBAAM,SAAS,MAAM,eAAyB,KAAK,KAAK,QAAQ,OAAO,cAAc,CAAC;AACtF,gBAAM,YAAY,OAAO,gBAAgB,CAAC,GAAG,SAAS,KAAK,EAAE;AAC7D,gBAAM,QAAQ,OAAO,SAAS,KAAK,CAAC,MAAM,EAAE,YAAY,KAAK,EAAE;AAC/D,iBAAO;AAAA,YACL;AAAA,YACA,OAAO,WAAW,IAAI;AAAA,YACtB,QAAQ,KAAK,UAAU,OAAO,kBAAkB,OAAO,gBAAgB,CAAC,CAAC;AAAA,UAC3E;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/{chunk-SVR2LKYI.js → chunk-NRMGT25X.js}

@@ -12,7 +12,7 @@ import {
 } from "./chunk-X3BTXCJ4.js";
 import {
   benchRoot
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/crag.ts
 import { readFile } from "fs/promises";
@@ -113,4 +113,4 @@ function createCragAdapter() {
 export {
   createCragAdapter
 };
-//# sourceMappingURL=chunk-SVR2LKYI.js.map
+//# sourceMappingURL=chunk-NRMGT25X.js.map

package/dist/{chunk-MQMRLGOG.js → chunk-QZZEAHWJ.js}

@@ -2,7 +2,7 @@ import {
   benchRoot,
   preflightVenvImports,
   runVenvScriptStdin
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/programbench.ts
 import { join } from "path";
@@ -133,4 +133,4 @@ export {
   programbenchSubmissionOutput,
   createProgrambenchAdapter
 };
-//# sourceMappingURL=chunk-MQMRLGOG.js.map
+//# sourceMappingURL=chunk-QZZEAHWJ.js.map

package/dist/{chunk-RCYQEFNX.js → chunk-WG7TM7UV.js}

@@ -1,10 +1,10 @@
 import {
   createTauBenchAdapter,
   tauResultsOutput
-} from "./chunk-IZ5M6OAC.js";
+} from "./chunk-GCHL6XPM.js";
 import {
   benchRoot
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/tau2-bench.ts
 import { join } from "path";
@@ -27,4 +27,4 @@ export {
   tau2ResultsOutput,
   createTau2BenchAdapter
 };
-//# sourceMappingURL=chunk-RCYQEFNX.js.map
+//# sourceMappingURL=chunk-WG7TM7UV.js.map

package/dist/{chunk-HWST3SED.js → chunk-XXFF3RRD.js}

@@ -1,7 +1,7 @@
 import {
   benchRoot,
   runVenvPython
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/webarena-verified.ts
 import { access, readFile, stat } from "fs/promises";
@@ -159,4 +159,4 @@ export {
   webarenaOutputDirOutput,
   createWebArenaVerifiedAdapter
 };
-//# sourceMappingURL=chunk-HWST3SED.js.map
+//# sourceMappingURL=chunk-XXFF3RRD.js.map

package/dist/{chunk-SFLA7OH3.js → chunk-ZFNOM7WR.js}

@@ -1,9 +1,9 @@
 import {
   createTauBenchAdapter
-} from "./chunk-IZ5M6OAC.js";
+} from "./chunk-GCHL6XPM.js";
 import {
   benchRoot
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/tau3-banking.ts
 import { join } from "path";
@@ -24,4 +24,4 @@ function createTau3BankingAdapter() {
 export {
   createTau3BankingAdapter
 };
-//# sourceMappingURL=chunk-SFLA7OH3.js.map
+//# sourceMappingURL=chunk-ZFNOM7WR.js.map

package/dist/{chunk-Z4TZ76N7.js → chunk-ZNCCYTFG.js}

@@ -3,7 +3,7 @@ import {
   preflightVenvImports,
   runVenvScriptStdin,
   venvPythonAt
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 
 // src/benchmarks/commit0.ts
 import { join } from "path";
@@ -167,4 +167,4 @@ export {
   commit0DiffOutput,
   createCommit0Adapter
 };
-//# sourceMappingURL=chunk-Z4TZ76N7.js.map
+//# sourceMappingURL=chunk-ZNCCYTFG.js.map

package/dist/index.js

@@ -1,47 +1,47 @@
 import {
   ADAPTERS,
   resolveAdapter
-} from "./chunk-V7AEBY6U.js";
-import "./chunk-XKEFIFIC.js";
+} from "./chunk-EEOC6QPJ.js";
+import "./chunk-CWIOBFSP.js";
 import "./chunk-VQRS7VUC.js";
-import "./chunk-HWST3SED.js";
+import "./chunk-XXFF3RRD.js";
 import {
   createRagBenchAdapter
-} from "./chunk-IFVINJ4B.js";
+} from "./chunk-HQ5HCCKF.js";
 import "./chunk-SEVJPLZC.js";
 import {
   createSweBenchAdapter,
   scoreSweReport,
   sweEvaluationArgv,
   swePatchOutput
-} from "./chunk-3U5TXJZS.js";
+} from "./chunk-J6BU3NTM.js";
 import {
   createT2RagBenchAdapter
-} from "./chunk-SHM6MRRF.js";
-import "./chunk-RCYQEFNX.js";
-import "./chunk-SFLA7OH3.js";
-import "./chunk-IZ5M6OAC.js";
-import "./chunk-XYA4XSNU.js";
+} from "./chunk-GC2EPS6L.js";
+import "./chunk-WG7TM7UV.js";
+import "./chunk-ZFNOM7WR.js";
+import "./chunk-GCHL6XPM.js";
+import "./chunk-JSQOUKXS.js";
 import "./chunk-INNOYXCP.js";
 import "./chunk-PA2ZKHJC.js";
 import "./chunk-5H5XV76F.js";
 import "./chunk-TBKU5XQI.js";
 import {
   createNoMiraclAdapter
-} from "./chunk-NQG5XDSB.js";
+} from "./chunk-7FKBWOQT.js";
 import {
   createOpenRagBenchAdapter
-} from "./chunk-SHYIRB7I.js";
-import "./chunk-MQMRLGOG.js";
+} from "./chunk-BZY5QARD.js";
+import "./chunk-QZZEAHWJ.js";
 import "./chunk-PUIRNYI7.js";
 import "./chunk-C7T7WEK2.js";
 import {
   runRefineLoop
 } from "./chunk-JTHWEDEW.js";
-import "./chunk-Z4TZ76N7.js";
+import "./chunk-ZNCCYTFG.js";
 import {
   createCragAdapter
-} from "./chunk-SVR2LKYI.js";
+} from "./chunk-NRMGT25X.js";
 import {
   FINAL_ANSWER_SENTINEL,
   answerScoreToBenchScore,
@@ -54,18 +54,18 @@ import {
   scoreAnswerArtifact,
   tokenF1
 } from "./chunk-X3BTXCJ4.js";
-import "./chunk-RH5F53JT.js";
-import "./chunk-YSMEKBTD.js";
-import "./chunk-7GRVHU22.js";
+import "./chunk-DWALFME7.js";
+import "./chunk-5FEQDSCT.js";
+import "./chunk-HVW25KSX.js";
 import "./chunk-UPAMRDX4.js";
-import "./chunk-K3BQGZCT.js";
-import "./chunk-PB64GYIG.js";
-import "./chunk-IA2FBTWC.js";
-import "./chunk-KP5KD6EN.js";
+import "./chunk-BEN6IF2X.js";
+import "./chunk-67ACKDCX.js";
+import "./chunk-CXDUTWQE.js";
+import "./chunk-CLIKAXKH.js";
 import {
   StagedJudgeError,
   runStagedJudge
-} from "./chunk-WSKWVEQB.js";
+} from "./chunk-EIETHPD5.js";
 import "./chunk-53UPUNBZ.js";
 
 // src/run-benchmarks.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.3.8",
+  "version": "0.4.1",
   "type": "module",
   "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.125.0",
+    "@tangle-network/agent-eval": "0.126.6",
     "@tangle-network/agent-interface": "0.32.0",
-    "@tangle-network/agent-knowledge": "^4.1.0",
+    "@tangle-network/agent-knowledge": "5.0.1",
     "@tangle-network/sandbox": "^0.12.0",
-    "@tangle-network/agent-runtime": "0.104.0"
+    "@tangle-network/agent-runtime": "0.105.0"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",