@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-EEOC6QPJ.js

@@ -1,144 +0,0 @@
-import {
-  createToolLlmAdapter
-} from "./chunk-CWIOBFSP.js";
-import {
-  createTrataHedgeAdapter
-} from "./chunk-VQRS7VUC.js";
-import {
-  createWebArenaVerifiedAdapter
-} from "./chunk-XXFF3RRD.js";
-import {
-  createRagBenchAdapter
-} from "./chunk-HQ5HCCKF.js";
-import {
-  createSimpleQaAdapter
-} from "./chunk-SEVJPLZC.js";
-import {
-  createSweBenchAdapter
-} from "./chunk-J6BU3NTM.js";
-import {
-  createT2RagBenchAdapter
-} from "./chunk-GC2EPS6L.js";
-import {
-  createTau2BenchAdapter
-} from "./chunk-WG7TM7UV.js";
-import {
-  createTau3BankingAdapter
-} from "./chunk-ZFNOM7WR.js";
-import {
-  createTerminalBenchAdapter
-} from "./chunk-JSQOUKXS.js";
-import {
-  createFramesAdapter
-} from "./chunk-INNOYXCP.js";
-import {
-  createHotpotqaAdapter
-} from "./chunk-PA2ZKHJC.js";
-import {
-  createHumanEvalAdapter
-} from "./chunk-5H5XV76F.js";
-import {
-  createMind2WebAdapter
-} from "./chunk-TBKU5XQI.js";
-import {
-  createNoMiraclAdapter
-} from "./chunk-7FKBWOQT.js";
-import {
-  createOpenRagBenchAdapter
-} from "./chunk-BZY5QARD.js";
-import {
-  createProgrambenchAdapter
-} from "./chunk-QZZEAHWJ.js";
-import {
-  createCadBenchAdapter
-} from "./chunk-PUIRNYI7.js";
-import {
-  createCadGenBenchAdapter
-} from "./chunk-C7T7WEK2.js";
-import {
-  createCommit0Adapter
-} from "./chunk-ZNCCYTFG.js";
-import {
-  createCragAdapter
-} from "./chunk-NRMGT25X.js";
-import {
-  createDabstepAdapter
-} from "./chunk-DWALFME7.js";
-import {
-  createEnterpriseOpsGymAdapter
-} from "./chunk-5FEQDSCT.js";
-import {
-  createFinResearchBenchAdapter
-} from "./chunk-HVW25KSX.js";
-import {
-  createFinsearchcompAdapter
-} from "./chunk-UPAMRDX4.js";
-import {
-  createAecBenchAdapter
-} from "./chunk-BEN6IF2X.js";
-import {
-  createAgentBenchAdapter
-} from "./chunk-67ACKDCX.js";
-import {
-  createAppWorldAdapter,
-  createAppWorldReactAdapter
-} from "./chunk-CXDUTWQE.js";
-import {
-  createBfclAdapter
-} from "./chunk-CLIKAXKH.js";
-import {
-  createCadDesignAdapter
-} from "./chunk-53UPUNBZ.js";
-
-// src/adapters.ts
-var ADAPTERS = {
-  "swe-bench": createSweBenchAdapter,
-  "terminal-bench": createTerminalBenchAdapter,
-  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).
-  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge
-  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.
-  "aec-bench": createAecBenchAdapter,
-  commit0: createCommit0Adapter,
-  dabstep: createDabstepAdapter,
-  programbench: createProgrambenchAdapter,
-  "webarena-verified": createWebArenaVerifiedAdapter,
-  "tau2-bench": createTau2BenchAdapter,
-  "tau3-banking": createTau3BankingAdapter,
-  agentbench: createAgentBenchAdapter,
-  bfcl: createBfclAdapter,
-  toollm: createToolLlmAdapter,
-  appworld: createAppWorldAdapter,
-  // AppWorld's native interactive protocol — the worker is the in-engine ReAct
-  // episode (execution feedback every turn), the mode published baselines use.
-  "appworld-react": createAppWorldReactAdapter,
-  "enterpriseops-gym": createEnterpriseOpsGymAdapter,
-  "cad-design": createCadDesignAdapter,
-  cadbench: createCadBenchAdapter,
-  cadgenbench: createCadGenBenchAdapter,
-  frames: createFramesAdapter,
-  ragbench: createRagBenchAdapter,
-  crag: createCragAdapter,
-  nomiracl: createNoMiraclAdapter,
-  "open-rag-bench": createOpenRagBenchAdapter,
-  "t2-ragbench": createT2RagBenchAdapter,
-  finresearchbench: createFinResearchBenchAdapter,
-  finsearchcomp: createFinsearchcompAdapter,
-  simpleqa: createSimpleQaAdapter,
-  hotpotqa: createHotpotqaAdapter,
-  // Deployable-checker code domain: worker = router completion, judge = Docker test
-  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).
-  humaneval: createHumanEvalAdapter,
-  mind2web: createMind2WebAdapter,
-  "trata-hedge": createTrataHedgeAdapter
-};
-function resolveAdapter(key) {
-  const make = ADAPTERS[key];
-  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(", ")})`);
-  return make();
-}
-
-export {
-  ADAPTERS,
-  resolveAdapter
-};
-//# sourceMappingURL=chunk-EEOC6QPJ.js.map

package/dist/chunk-EEOC6QPJ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/adapters.ts"],"sourcesContent":["/**\n * The benchmark registry — the single source of truth for every wired benchmark.\n * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`\n * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it\n * here rather than each keeping their own copy.\n */\n\nimport { createAecBenchAdapter } from './benchmarks/aec-bench'\nimport { createAgentBenchAdapter } from './benchmarks/agentbench'\nimport { createAppWorldAdapter, createAppWorldReactAdapter } from './benchmarks/appworld'\nimport { createBfclAdapter } from './benchmarks/bfcl'\nimport { createCadBenchAdapter } from './benchmarks/cadbench'\nimport { createCadDesignAdapter } from './benchmarks/cad-design'\nimport { createCadGenBenchAdapter } from './benchmarks/cadgenbench'\nimport { createCommit0Adapter } from './benchmarks/commit0'\nimport { createCragAdapter } from './benchmarks/crag'\nimport { createDabstepAdapter } from './benchmarks/dabstep'\nimport { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'\nimport { createFinResearchBenchAdapter } from './benchmarks/finresearchbench'\nimport { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'\nimport { createFramesAdapter } from './benchmarks/frames'\nimport { createHotpotqaAdapter } from './benchmarks/hotpotqa'\nimport { createHumanEvalAdapter } from './benchmarks/humaneval'\nimport { createMind2WebAdapter } from './benchmarks/mind2web'\nimport { createNoMiraclAdapter } from './benchmarks/nomiracl'\nimport { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'\nimport { createProgrambenchAdapter } from './benchmarks/programbench'\nimport { createRagBenchAdapter } from './benchmarks/ragbench'\nimport { createSimpleQaAdapter } from './benchmarks/simpleqa'\nimport { createSweBenchAdapter } from './benchmarks/swe-bench'\nimport { createT2RagBenchAdapter } from './benchmarks/t2-ragbench'\nimport { createTau2BenchAdapter } from './benchmarks/tau2-bench'\nimport { createTau3BankingAdapter } from './benchmarks/tau3-banking'\nimport { createTerminalBenchAdapter } from './benchmarks/terminal-bench'\nimport { createToolLlmAdapter } from './benchmarks/toollm'\nimport { createTrataHedgeAdapter } from './benchmarks/trata-hedge'\nimport { createWebArenaVerifiedAdapter } from './benchmarks/webarena-verified'\nimport type { BenchmarkAdapter } from './benchmarks/types'\n\nexport const ADAPTERS: Record<string, () => BenchmarkAdapter> = {\n  'swe-bench': createSweBenchAdapter,\n  'terminal-bench': createTerminalBenchAdapter,\n  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).\n  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge\n  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.\n  'aec-bench': createAecBenchAdapter,\n  commit0: createCommit0Adapter,\n  dabstep: createDabstepAdapter,\n  programbench: createProgrambenchAdapter,\n  'webarena-verified': createWebArenaVerifiedAdapter,\n  'tau2-bench': createTau2BenchAdapter,\n  'tau3-banking': createTau3BankingAdapter,\n  agentbench: createAgentBenchAdapter,\n  bfcl: createBfclAdapter,\n  toollm: createToolLlmAdapter,\n  appworld: createAppWorldAdapter,\n  // AppWorld's native interactive protocol — the worker is the in-engine ReAct\n  // episode (execution feedback every turn), the mode published baselines use.\n  'appworld-react': createAppWorldReactAdapter,\n  'enterpriseops-gym': createEnterpriseOpsGymAdapter,\n  'cad-design': createCadDesignAdapter,\n  cadbench: createCadBenchAdapter,\n  cadgenbench: createCadGenBenchAdapter,\n  frames: createFramesAdapter,\n  ragbench: createRagBenchAdapter,\n  crag: createCragAdapter,\n  nomiracl: createNoMiraclAdapter,\n  'open-rag-bench': createOpenRagBenchAdapter,\n  't2-ragbench': createT2RagBenchAdapter,\n  finresearchbench: createFinResearchBenchAdapter,\n  finsearchcomp: createFinsearchcompAdapter,\n  simpleqa: createSimpleQaAdapter,\n  hotpotqa: createHotpotqaAdapter,\n  // Deployable-checker code domain: worker = router completion, judge = Docker test\n  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).\n  humaneval: createHumanEvalAdapter,\n  mind2web: createMind2WebAdapter,\n  'trata-hedge': createTrataHedgeAdapter,\n}\n\n/** Resolve a benchmark key to its adapter, failing loud with the known keys. */\nexport function resolveAdapter(key: string): BenchmarkAdapter {\n  const make = ADAPTERS[key]\n  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(', ')})`)\n  return make()\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCO,IAAM,WAAmD;AAAA,EAC9D,aAAa;AAAA,EACb,kBAAkB;AAAA;AAAA;AAAA;AAAA,EAIlB,aAAa;AAAA,EACb,SAAS;AAAA,EACT,SAAS;AAAA,EACT,cAAc;AAAA,EACd,qBAAqB;AAAA,EACrB,cAAc;AAAA,EACd,gBAAgB;AAAA,EAChB,YAAY;AAAA,EACZ,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,UAAU;AAAA;AAAA;AAAA,EAGV,kBAAkB;AAAA,EAClB,qBAAqB;AAAA,EACrB,cAAc;AAAA,EACd,UAAU;AAAA,EACV,aAAa;AAAA,EACb,QAAQ;AAAA,EACR,UAAU;AAAA,EACV,MAAM;AAAA,EACN,UAAU;AAAA,EACV,kBAAkB;AAAA,EAClB,eAAe;AAAA,EACf,kBAAkB;AAAA,EAClB,eAAe;AAAA,EACf,UAAU;AAAA,EACV,UAAU;AAAA;AAAA;AAAA,EAGV,WAAW;AAAA,EACX,UAAU;AAAA,EACV,eAAe;AACjB;AAGO,SAAS,eAAe,KAA+B;AAC5D,QAAM,OAAO,SAAS,GAAG;AACzB,MAAI,CAAC,KAAM,OAAM,IAAI,MAAM,qBAAqB,KAAK,UAAU,GAAG,CAAC,WAAW,OAAO,KAAK,QAAQ,EAAE,KAAK,IAAI,CAAC,GAAG;AACjH,SAAO,KAAK;AACd;","names":[]}

package/dist/chunk-EIETHPD5.js

@@ -1,321 +0,0 @@
-// src/benchmarks/_harness.ts
-import { execFile, spawn } from "child_process";
-import { createHash } from "crypto";
-import { existsSync, readFileSync } from "fs";
-import {
-  cp,
-  lstat,
-  mkdir,
-  mkdtemp,
-  readFile,
-  readlink,
-  readdir,
-  rename,
-  rm,
-  writeFile
-} from "fs/promises";
-import { tmpdir } from "os";
-import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "path";
-import { fileURLToPath } from "url";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-function resolveBenchRoot(moduleUrl) {
-  let current = dirname(fileURLToPath(moduleUrl));
-  while (true) {
-    const manifestPath = join(current, "package.json");
-    if (existsSync(manifestPath)) {
-      const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
-      if (manifest.name === "@tangle-network/agent-bench") return current;
-    }
-    const parent = dirname(current);
-    if (parent === current) {
-      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`);
-    }
-    current = parent;
-  }
-}
-var benchRoot = resolveBenchRoot(import.meta.url);
-function resolveBenchPython(env = process.env, root = benchRoot) {
-  const configured = env.AGENT_BENCH_PYTHON;
-  if (configured === void 0) return join(root, ".venv", "bin", "python");
-  if (!isAbsolute(configured)) {
-    throw new Error("AGENT_BENCH_PYTHON must be an absolute path");
-  }
-  return configured;
-}
-var venvPython = resolveBenchPython();
-var venvPythonAt = (venvDir) => venvBinAt(venvDir, "python");
-function venvBinAt(venvDir, name) {
-  return join(resolve(benchRoot, venvDir), "bin", name);
-}
-var bigBuffer = 1024 * 1024 * 256;
-function venvBin(name) {
-  return venvBinAt(".venv", name);
-}
-async function runVenvPython(script, args = [], timeoutMs = 0, python = venvPython) {
-  const { stdout } = await execFileAsync(python, ["-c", script, ...args], {
-    maxBuffer: bigBuffer,
-    timeout: timeoutMs
-  });
-  return stdout;
-}
-async function preflightVenvImports(opts) {
-  const imports = opts.modules.filter((m) => m.length > 0);
-  const lines = [...imports.map((m) => `import ${m}`)];
-  if (opts.requireDocker) lines.push("import docker", "docker.from_env().ping()");
-  lines.push("print('ok')");
-  try {
-    await runVenvPython(lines.join("\n"), [], 0, opts.python ?? venvPython);
-  } catch (err) {
-    const msg = err instanceof Error ? err.message : String(err);
-    throw new Error(`${msg}
-${opts.fix}`);
-  }
-}
-function runVenvScriptStdin(scriptPath, args, input, opts = {}) {
-  return new Promise((resolve2, reject) => {
-    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {
-      cwd: opts.cwd ?? benchRoot,
-      ...opts.timeoutMs ? { timeout: opts.timeoutMs } : {}
-    });
-    let stdout = "";
-    let stderr = "";
-    let bytes = 0;
-    child.stdout.on("data", (c) => {
-      bytes += c.length;
-      if (bytes <= bigBuffer) stdout += c.toString("utf8");
-    });
-    child.stderr.on("data", (c) => {
-      stderr += c.toString("utf8");
-    });
-    child.on("error", (err) => reject(err));
-    child.on("close", (code) => {
-      if (code === 0) resolve2(stdout);
-      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)));
-    });
-    child.stdin.end(input);
-  });
-}
-var StagedJudgeError = class extends Error {
-  judgeArtifacts;
-  constructor(message, judgeArtifacts, options) {
-    super(message, options);
-    this.name = "StagedJudgeError";
-    this.judgeArtifacts = judgeArtifacts;
-  }
-};
-function sha256(bytes) {
-  return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
-}
-function portablePath(path) {
-  return path.split(sep).join("/");
-}
-function compareText(left, right) {
-  return left < right ? -1 : left > right ? 1 : 0;
-}
-async function collectArtifactFiles(root, current) {
-  const absolute = join(root, current);
-  const entries = await readdir(absolute, { withFileTypes: true });
-  const files = [];
-  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {
-    const relativePath = join(current, entry.name);
-    const path = join(root, relativePath);
-    if (entry.isDirectory()) {
-      files.push(...await collectArtifactFiles(root, relativePath));
-      continue;
-    }
-    if (entry.isFile()) {
-      const bytes = await readFile(path);
-      files.push({
-        path: portablePath(relativePath),
-        byteLength: bytes.byteLength,
-        sha256: sha256(bytes),
-        kind: "file"
-      });
-      continue;
-    }
-    if (entry.isSymbolicLink()) {
-      const targetBytes = await readlink(path, { encoding: "buffer" });
-      files.push({
-        path: portablePath(relativePath),
-        byteLength: targetBytes.byteLength,
-        sha256: sha256(targetBytes),
-        kind: "symlink"
-      });
-      continue;
-    }
-    throw new Error(`staged judge capture does not support ${relativePath}`);
-  }
-  return files;
-}
-function isWithin(parent, candidate) {
-  const path = relative(parent, candidate);
-  return path === "" || !path.startsWith(`..${sep}`) && path !== ".." && !isAbsolute(path);
-}
-async function assertDestinationAbsent(destination) {
-  try {
-    await lstat(destination);
-  } catch (error) {
-    if (error.code === "ENOENT") return;
-    throw error;
-  }
-  throw new Error(`staged judge capture destination already exists: ${destination}`);
-}
-async function captureStagedRun(sourceDirectory, spec, processOutput, evaluatorSucceeded) {
-  const source = resolve(sourceDirectory);
-  const destination = resolve(spec.destination);
-  if (isWithin(source, destination)) {
-    throw new Error("staged judge capture destination must be outside the evaluator directory");
-  }
-  await mkdir(dirname(destination), { recursive: true });
-  await assertDestinationAbsent(destination);
-  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`));
-  try {
-    await cp(source, join(staging, "evaluator"), {
-      recursive: true,
-      errorOnExist: true,
-      force: false,
-      preserveTimestamps: true,
-      verbatimSymlinks: true
-    });
-    await mkdir(join(staging, "process"));
-    await writeFile(join(staging, "process", "stdout.bin"), processOutput.stdout);
-    await writeFile(join(staging, "process", "stderr.bin"), processOutput.stderr);
-    const files = [
-      ...await collectArtifactFiles(staging, "evaluator"),
-      ...await collectArtifactFiles(staging, "process")
-    ].sort((left, right) => compareText(left.path, right.path));
-    const byteLength = files.reduce((total, file) => total + file.byteLength, 0);
-    const treeBytes = Buffer.from(
-      files.map((file) => `${file.path}\0${file.kind}\0${file.byteLength}\0${file.sha256}
-`).join(""),
-      "utf8"
-    );
-    const receipt = {
-      schema: "agent-bench/judge-artifacts/v1",
-      directory: destination,
-      evaluatorDirectory: join(destination, "evaluator"),
-      manifestPath: join(destination, "receipt.json"),
-      evaluatorSucceeded,
-      files,
-      fileCount: files.length,
-      byteLength,
-      treeSha256: sha256(treeBytes)
-    };
-    await writeFile(join(staging, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
-`);
-    await rename(staging, destination);
-    return receipt;
-  } catch (error) {
-    await rm(staging, { recursive: true, force: true }).catch(() => {
-    });
-    throw error;
-  }
-}
-function processBytes(value) {
-  if (Buffer.isBuffer(value)) return value;
-  if (value === void 0 || value === null) return Buffer.alloc(0);
-  return Buffer.from(String(value), "utf8");
-}
-async function runStagedJudge(spec) {
-  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix));
-  let stdout = Buffer.alloc(0);
-  let stderr = Buffer.alloc(0);
-  let evaluatorSucceeded = false;
-  let score;
-  let failure;
-  try {
-    try {
-      await spec.stage(dir);
-      const bin = spec.bin ?? venvPython;
-      const argv = spec.argv(dir);
-      try {
-        const output = await execFileAsync(bin, argv, {
-          cwd: spec.cwd ? spec.cwd(dir) : dir,
-          encoding: "buffer",
-          maxBuffer: bigBuffer,
-          ...spec.timeoutMs ? { timeout: spec.timeoutMs } : {}
-        });
-        stdout = processBytes(output.stdout);
-        stderr = processBytes(output.stderr);
-        evaluatorSucceeded = true;
-      } catch (err) {
-        const e = err;
-        stdout = processBytes(e.stdout);
-        stderr = processBytes(e.stderr);
-        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err)).toString("utf8").slice(0, 2e3);
-        throw new Error(`${spec.tmpPrefix.replace(/-$/, "")} evaluator failed (${bin} ${argv.join(" ")}):
-${detail}`);
-      }
-      score = await spec.parseReport(dir);
-    } catch (err) {
-      failure = err;
-    }
-  } finally {
-    let judgeArtifacts;
-    if (spec.capture) {
-      try {
-        judgeArtifacts = await captureStagedRun(
-          dir,
-          spec.capture,
-          { stdout, stderr },
-          evaluatorSucceeded
-        );
-      } catch (captureError) {
-        failure = new Error(
-          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,
-          { cause: failure ?? captureError }
-        );
-      }
-    }
-    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {
-    });
-    if (failure) {
-      throw new StagedJudgeError(
-        failure instanceof Error ? failure.message : String(failure),
-        judgeArtifacts,
-        { cause: failure }
-      );
-    }
-    if (!score) throw new StagedJudgeError("staged judge completed without a score", judgeArtifacts);
-    return judgeArtifacts ? { ...score, judgeArtifacts } : score;
-  }
-}
-async function readJsonReport(path) {
-  let raw;
-  try {
-    raw = await readFile(path, "utf8");
-  } catch (err) {
-    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`);
-  }
-  try {
-    return JSON.parse(raw);
-  } catch (err) {
-    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`);
-  }
-}
-async function stageFile(path, content) {
-  await writeFile(path, content);
-}
-function safeRunId(prefix, id) {
-  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
-}
-
-export {
-  benchRoot,
-  resolveBenchPython,
-  venvPython,
-  venvPythonAt,
-  venvBinAt,
-  bigBuffer,
-  venvBin,
-  runVenvPython,
-  preflightVenvImports,
-  runVenvScriptStdin,
-  StagedJudgeError,
-  runStagedJudge,
-  readJsonReport,
-  stageFile,
-  safeRunId
-};
-//# sourceMappingURL=chunk-EIETHPD5.js.map

package/dist/chunk-EIETHPD5.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/_harness.ts"],"sourcesContent":["/**\n * Shared code-bench harness. The \"stage the artifact → run the benchmark's own\n * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)\n * → read its JSON report → { resolved, score }\" spine that swe-bench,\n * terminal-bench, commit0, programbench and aec-bench all need. Factored out so\n * the Docker/venv/report-reading logic lives in ONE place instead of being\n * copy-pasted per adapter.\n *\n * It owns NO benchmark policy: each adapter passes the argv for its evaluator\n * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The\n * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config\n * and fail-loud diagnostics.\n *\n * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`\n * (the adapter passes the import line + the exact fix). A staged run that exits\n * nonzero throws with the captured stderr — never a fabricated score.\n */\n\nimport { execFile, spawn } from 'node:child_process'\nimport { createHash } from 'node:crypto'\nimport { existsSync, readFileSync } from 'node:fs'\nimport {\n  cp,\n  lstat,\n  mkdir,\n  mkdtemp,\n  readFile,\n  readlink,\n  readdir,\n  rename,\n  rm,\n  writeFile,\n} from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type {\n  BenchScore,\n  JudgeArtifactFileReceipt,\n  JudgeArtifactReceipt,\n} from './types'\n\nconst execFileAsync = promisify(execFile)\n\n/** Locate the package by identity because source files and compiled chunks have different depths. */\nfunction resolveBenchRoot(moduleUrl: string): string {\n  let current = dirname(fileURLToPath(moduleUrl))\n  while (true) {\n    const manifestPath = join(current, 'package.json')\n    if (existsSync(manifestPath)) {\n      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { name?: unknown }\n      if (manifest.name === '@tangle-network/agent-bench') return current\n    }\n    const parent = dirname(current)\n    if (parent === current) {\n      throw new Error(`Unable to locate @tangle-network/agent-bench from ${moduleUrl}`)\n    }\n    current = parent\n  }\n}\n\n/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */\nexport const benchRoot = resolveBenchRoot(import.meta.url)\n\n/** Resolve the shared interpreter without requiring an installed package to contain a venv. */\nexport function resolveBenchPython(\n  env: Readonly<{ AGENT_BENCH_PYTHON?: string }> = process.env,\n  root: string = benchRoot,\n): string {\n  const configured = env.AGENT_BENCH_PYTHON\n  if (configured === undefined) return join(root, '.venv', 'bin', 'python')\n  if (!isAbsolute(configured)) {\n    throw new Error('AGENT_BENCH_PYTHON must be an absolute path')\n  }\n  return configured\n}\n\n/** The shared interpreter every Python-backed evaluator runs through. */\nexport const venvPython = resolveBenchPython()\n\n/** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip\n *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)\n *  get their own venv and pass its python explicitly — keeping the shared one clean. */\nexport const venvPythonAt = (venvDir: string): string => venvBinAt(venvDir, 'python')\n\n/** Resolve an executable in an isolated venv. Relative venv paths are package-owned;\n * absolute paths allow installed consumers to keep large environments elsewhere. */\nexport function venvBinAt(venvDir: string, name: string): string {\n  return join(resolve(benchRoot, venvDir), 'bin', name)\n}\n\n/** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */\nexport const bigBuffer = 1024 * 1024 * 256\n\n/** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */\nexport function venvBin(name: string): string {\n  return venvBinAt('.venv', name)\n}\n\n/**\n * Run the bench venv python with an inline script (`-c`); return stdout. Throws\n * (with stderr) on a nonzero exit — the loaders rely on this to fail loud rather\n * than parse a partial dump.\n */\nexport async function runVenvPython(\n  script: string,\n  args: string[] = [],\n  timeoutMs = 0,\n  python: string = venvPython,\n): Promise<string> {\n  const { stdout } = await execFileAsync(python, ['-c', script, ...args], {\n    maxBuffer: bigBuffer,\n    timeout: timeoutMs,\n  })\n  return stdout\n}\n\n/**\n * Preflight a python-backed harness: import the module(s) and (optionally) ping\n * Docker, all inside the bench venv. On failure THROWS the captured error joined\n * to the adapter's `fix` guidance — the contract every code-bench preflight wants.\n */\nexport async function preflightVenvImports(opts: {\n  /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */\n  modules: string[]\n  /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */\n  requireDocker?: boolean\n  /** Actionable remediation appended to the thrown message. */\n  fix: string\n  /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */\n  python?: string\n}): Promise<void> {\n  const imports = opts.modules.filter((m) => m.length > 0)\n  const lines = [...imports.map((m) => `import ${m}`)]\n  if (opts.requireDocker) lines.push('import docker', 'docker.from_env().ping()')\n  lines.push(\"print('ok')\")\n  try {\n    await runVenvPython(lines.join('\\n'), [], 0, opts.python ?? venvPython)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    throw new Error(`${msg}\\n${opts.fix}`)\n  }\n}\n\n/**\n * Run a bench-local python driver script (in the bench venv) while piping\n * `input` to its stdin, returning stdout. The driver's verdict JSON is its LAST\n * stdout line; callers parse that and inspect an `error` field (fail loud).\n *\n * Uses spawn + an explicit `stdin.end(input)` rather than promisify(execFile)'s\n * `input` option, because that option is NOT honored by async execFile — stdin\n * is left open and a driver that does `sys.stdin.read()` blocks forever. The\n * artifact-piping judges (commit0, appworld) MUST go through this.\n */\nexport function runVenvScriptStdin(\n  scriptPath: string,\n  args: string[],\n  input: string,\n  opts: { cwd?: string; timeoutMs?: number; python?: string } = {},\n): Promise<string> {\n  return new Promise<string>((resolve, reject) => {\n    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {\n      cwd: opts.cwd ?? benchRoot,\n      ...(opts.timeoutMs ? { timeout: opts.timeoutMs } : {}),\n    })\n    let stdout = ''\n    let stderr = ''\n    let bytes = 0\n    child.stdout.on('data', (c: Buffer) => {\n      bytes += c.length\n      if (bytes <= bigBuffer) stdout += c.toString('utf8')\n    })\n    child.stderr.on('data', (c: Buffer) => {\n      stderr += c.toString('utf8')\n    })\n    child.on('error', (err) => reject(err))\n    child.on('close', (code) => {\n      if (code === 0) resolve(stdout)\n      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)))\n    })\n    child.stdin.end(input)\n  })\n}\n\nexport interface StagedRunSpec {\n  /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */\n  tmpPrefix: string\n  /**\n   * Write the artifact + any harness inputs into the temp dir. Returns nothing;\n   * `argv`/`cwd` consume `dir` to point the evaluator at what was written.\n   */\n  stage(dir: string): Promise<void>\n  /** The external evaluator to spawn. `bin` defaults to the bench venv python. */\n  bin?: string\n  /** argv for the evaluator, computed from the temp `dir`. */\n  argv(dir: string): string[]\n  /** Working directory for the evaluator. Defaults to the temp `dir`. */\n  cwd?(dir: string): string\n  /** Hard timeout for the evaluator (ms); 0 = none. */\n  timeoutMs?: number\n  /**\n   * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws\n   * if the expected report is absent/malformed (fail loud — no default score).\n   */\n  parseReport(dir: string): Promise<BenchScore>\n  /**\n   * Copy the complete evaluator directory plus raw process stdout/stderr to this\n   * caller-owned directory before cleanup. The destination must not exist.\n   */\n  capture?: StagedRunCaptureSpec\n  /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */\n  keepTmp?: boolean\n}\n\nexport interface StagedRunCaptureSpec {\n  /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */\n  destination: string\n}\n\n/** A staged run failed after any requested evidence was durably retained. */\nexport class StagedJudgeError extends Error {\n  readonly judgeArtifacts?: JudgeArtifactReceipt\n\n  constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions) {\n    super(message, options)\n    this.name = 'StagedJudgeError'\n    this.judgeArtifacts = judgeArtifacts\n  }\n}\n\nfunction sha256(bytes: Uint8Array): `sha256:${string}` {\n  return `sha256:${createHash('sha256').update(bytes).digest('hex')}`\n}\n\nfunction portablePath(path: string): string {\n  return path.split(sep).join('/')\n}\n\nfunction compareText(left: string, right: string): number {\n  return left < right ? -1 : left > right ? 1 : 0\n}\n\nasync function collectArtifactFiles(\n  root: string,\n  current: string,\n): Promise<JudgeArtifactFileReceipt[]> {\n  const absolute = join(root, current)\n  const entries = await readdir(absolute, { withFileTypes: true })\n  const files: JudgeArtifactFileReceipt[] = []\n  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {\n    const relativePath = join(current, entry.name)\n    const path = join(root, relativePath)\n    if (entry.isDirectory()) {\n      files.push(...await collectArtifactFiles(root, relativePath))\n      continue\n    }\n    if (entry.isFile()) {\n      const bytes = await readFile(path)\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: bytes.byteLength,\n        sha256: sha256(bytes),\n        kind: 'file',\n      })\n      continue\n    }\n    if (entry.isSymbolicLink()) {\n      const targetBytes = await readlink(path, { encoding: 'buffer' })\n      files.push({\n        path: portablePath(relativePath),\n        byteLength: targetBytes.byteLength,\n        sha256: sha256(targetBytes),\n        kind: 'symlink',\n      })\n      continue\n    }\n    throw new Error(`staged judge capture does not support ${relativePath}`)\n  }\n  return files\n}\n\nfunction isWithin(parent: string, candidate: string): boolean {\n  const path = relative(parent, candidate)\n  return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path))\n}\n\nasync function assertDestinationAbsent(destination: string): Promise<void> {\n  try {\n    await lstat(destination)\n  } catch (error) {\n    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return\n    throw error\n  }\n  throw new Error(`staged judge capture destination already exists: ${destination}`)\n}\n\nasync function captureStagedRun(\n  sourceDirectory: string,\n  spec: StagedRunCaptureSpec,\n  processOutput: Readonly<{ stdout: Buffer; stderr: Buffer }>,\n  evaluatorSucceeded: boolean,\n): Promise<JudgeArtifactReceipt> {\n  const source = resolve(sourceDirectory)\n  const destination = resolve(spec.destination)\n  if (isWithin(source, destination)) {\n    throw new Error('staged judge capture destination must be outside the evaluator directory')\n  }\n  await mkdir(dirname(destination), { recursive: true })\n  await assertDestinationAbsent(destination)\n  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`))\n  try {\n    await cp(source, join(staging, 'evaluator'), {\n      recursive: true,\n      errorOnExist: true,\n      force: false,\n      preserveTimestamps: true,\n      verbatimSymlinks: true,\n    })\n    await mkdir(join(staging, 'process'))\n    await writeFile(join(staging, 'process', 'stdout.bin'), processOutput.stdout)\n    await writeFile(join(staging, 'process', 'stderr.bin'), processOutput.stderr)\n\n    const files = [\n      ...await collectArtifactFiles(staging, 'evaluator'),\n      ...await collectArtifactFiles(staging, 'process'),\n    ].sort((left, right) => compareText(left.path, right.path))\n    const byteLength = files.reduce((total, file) => total + file.byteLength, 0)\n    const treeBytes = Buffer.from(\n      files\n        .map((file) => `${file.path}\\0${file.kind}\\0${file.byteLength}\\0${file.sha256}\\n`)\n        .join(''),\n      'utf8',\n    )\n    const receipt: JudgeArtifactReceipt = {\n      schema: 'agent-bench/judge-artifacts/v1',\n      directory: destination,\n      evaluatorDirectory: join(destination, 'evaluator'),\n      manifestPath: join(destination, 'receipt.json'),\n      evaluatorSucceeded,\n      files,\n      fileCount: files.length,\n      byteLength,\n      treeSha256: sha256(treeBytes),\n    }\n    await writeFile(join(staging, 'receipt.json'), `${JSON.stringify(receipt, null, 2)}\\n`)\n    await rename(staging, destination)\n    return receipt\n  } catch (error) {\n    await rm(staging, { recursive: true, force: true }).catch(() => {})\n    throw error\n  }\n}\n\nfunction processBytes(value: unknown): Buffer {\n  if (Buffer.isBuffer(value)) return value\n  if (value === undefined || value === null) return Buffer.alloc(0)\n  return Buffer.from(String(value), 'utf8')\n}\n\n/**\n * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →\n * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is\n * always removed in `finally` unless `keepTmp`.\n */\nexport async function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore> {\n  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix))\n  let stdout: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let stderr: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n  let evaluatorSucceeded = false\n  let score: BenchScore | undefined\n  let failure: unknown\n  try {\n    try {\n      await spec.stage(dir)\n      const bin = spec.bin ?? venvPython\n      const argv = spec.argv(dir)\n      try {\n        const output = await execFileAsync(bin, argv, {\n          cwd: spec.cwd ? spec.cwd(dir) : dir,\n          encoding: 'buffer',\n          maxBuffer: bigBuffer,\n          ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),\n        })\n        stdout = processBytes(output.stdout)\n        stderr = processBytes(output.stderr)\n        evaluatorSucceeded = true\n      } catch (err) {\n        const e = err as { stderr?: unknown; stdout?: unknown; message?: string }\n        stdout = processBytes(e.stdout)\n        stderr = processBytes(e.stderr)\n        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err))\n          .toString('utf8')\n          .slice(0, 2000)\n        throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${argv.join(' ')}):\\n${detail}`)\n      }\n      score = await spec.parseReport(dir)\n    } catch (err) {\n      failure = err\n    }\n  } finally {\n    let judgeArtifacts: JudgeArtifactReceipt | undefined\n    if (spec.capture) {\n      try {\n        judgeArtifacts = await captureStagedRun(\n          dir,\n          spec.capture,\n          { stdout, stderr },\n          evaluatorSucceeded,\n        )\n      } catch (captureError) {\n        failure = new Error(\n          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,\n          { cause: failure ?? captureError },\n        )\n      }\n    }\n    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {})\n    if (failure) {\n      throw new StagedJudgeError(\n        failure instanceof Error ? failure.message : String(failure),\n        judgeArtifacts,\n        { cause: failure },\n      )\n    }\n    if (!score) throw new StagedJudgeError('staged judge completed without a score', judgeArtifacts)\n    return judgeArtifacts ? { ...score, judgeArtifacts } : score\n  }\n}\n\n/** Read + JSON.parse a report file from a staged run; throws with the path on failure. */\nexport async function readJsonReport<T>(path: string): Promise<T> {\n  let raw: string\n  try {\n    raw = await readFile(path, 'utf8')\n  } catch (err) {\n    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`)\n  }\n  try {\n    return JSON.parse(raw) as T\n  } catch (err) {\n    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`)\n  }\n}\n\n/** Write a UTF-8 file into a staged dir (artifact / preds.json / attempt.sh / …). */\nexport async function stageFile(path: string, content: string): Promise<void> {\n  await writeFile(path, content)\n}\n\n/** Sanitize an instance id into a filesystem/run-id-safe token. */\nexport function safeRunId(prefix: string, id: string): string {\n  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, '_')\n}\n"],"mappings":";AAkBA,SAAS,UAAU,aAAa;AAChC,SAAS,kBAAkB;AAC3B,SAAS,YAAY,oBAAoB;AACzC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AACvB,SAAS,UAAU,SAAS,YAAY,MAAM,UAAU,SAAS,WAAW;AAC5E,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAO1B,IAAM,gBAAgB,UAAU,QAAQ;AAGxC,SAAS,iBAAiB,WAA2B;AACnD,MAAI,UAAU,QAAQ,cAAc,SAAS,CAAC;AAC9C,SAAO,MAAM;AACX,UAAM,eAAe,KAAK,SAAS,cAAc;AACjD,QAAI,WAAW,YAAY,GAAG;AAC5B,YAAM,WAAW,KAAK,MAAM,aAAa,cAAc,MAAM,CAAC;AAC9D,UAAI,SAAS,SAAS,8BAA+B,QAAO;AAAA,IAC9D;AACA,UAAM,SAAS,QAAQ,OAAO;AAC9B,QAAI,WAAW,SAAS;AACtB,YAAM,IAAI,MAAM,qDAAqD,SAAS,EAAE;AAAA,IAClF;AACA,cAAU;AAAA,EACZ;AACF;AAGO,IAAM,YAAY,iBAAiB,YAAY,GAAG;AAGlD,SAAS,mBACd,MAAiD,QAAQ,KACzD,OAAe,WACP;AACR,QAAM,aAAa,IAAI;AACvB,MAAI,eAAe,OAAW,QAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;AACxE,MAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AACA,SAAO;AACT;AAGO,IAAM,aAAa,mBAAmB;AAKtC,IAAM,eAAe,CAAC,YAA4B,UAAU,SAAS,QAAQ;AAI7E,SAAS,UAAU,SAAiB,MAAsB;AAC/D,SAAO,KAAK,QAAQ,WAAW,OAAO,GAAG,OAAO,IAAI;AACtD;AAGO,IAAM,YAAY,OAAO,OAAO;AAGhC,SAAS,QAAQ,MAAsB;AAC5C,SAAO,UAAU,SAAS,IAAI;AAChC;AAOA,eAAsB,cACpB,QACA,OAAiB,CAAC,GAClB,YAAY,GACZ,SAAiB,YACA;AACjB,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,QAAQ,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IACtE,WAAW;AAAA,IACX,SAAS;AAAA,EACX,CAAC;AACD,SAAO;AACT;AAOA,eAAsB,qBAAqB,MASzB;AAChB,QAAM,UAAU,KAAK,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AACvD,QAAM,QAAQ,CAAC,GAAG,QAAQ,IAAI,CAAC,MAAM,UAAU,CAAC,EAAE,CAAC;AACnD,MAAI,KAAK,cAAe,OAAM,KAAK,iBAAiB,0BAA0B;AAC9E,QAAM,KAAK,aAAa;AACxB,MAAI;AACF,UAAM,cAAc,MAAM,KAAK,IAAI,GAAG,CAAC,GAAG,GAAG,KAAK,UAAU,UAAU;AAAA,EACxE,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI,MAAM,GAAG,GAAG;AAAA,EAAK,KAAK,GAAG,EAAE;AAAA,EACvC;AACF;AAYO,SAAS,mBACd,YACA,MACA,OACA,OAA8D,CAAC,GAC9C;AACjB,SAAO,IAAI,QAAgB,CAACA,UAAS,WAAW;AAC9C,UAAM,QAAQ,MAAM,KAAK,UAAU,YAAY,CAAC,YAAY,GAAG,IAAI,GAAG;AAAA,MACpE,KAAK,KAAK,OAAO;AAAA,MACjB,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,IACtD,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,QAAQ;AACZ,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,eAAS,EAAE;AACX,UAAI,SAAS,UAAW,WAAU,EAAE,SAAS,MAAM;AAAA,IACrD,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,gBAAU,EAAE,SAAS,MAAM;AAAA,IAC7B,CAAC;AACD,UAAM,GAAG,SAAS,CAAC,QAAQ,OAAO,GAAG,CAAC;AACtC,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,UAAI,SAAS,EAAG,CAAAA,SAAQ,MAAM;AAAA,UACzB,QAAO,IAAI,OAAO,UAAU,UAAU,QAAQ,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC;AAAA,IAC5E,CAAC;AACD,UAAM,MAAM,IAAI,KAAK;AAAA,EACvB,CAAC;AACH;AAsCO,IAAM,mBAAN,cAA+B,MAAM;AAAA,EACjC;AAAA,EAET,YAAY,SAAiB,gBAAuC,SAAwB;AAC1F,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO;AACZ,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,SAAS,OAAO,OAAuC;AACrD,SAAO,UAAU,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK,CAAC;AACnE;AAEA,SAAS,aAAa,MAAsB;AAC1C,SAAO,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG;AACjC;AAEA,SAAS,YAAY,MAAc,OAAuB;AACxD,SAAO,OAAO,QAAQ,KAAK,OAAO,QAAQ,IAAI;AAChD;AAEA,eAAe,qBACb,MACA,SACqC;AACrC,QAAM,WAAW,KAAK,MAAM,OAAO;AACnC,QAAM,UAAU,MAAM,QAAQ,UAAU,EAAE,eAAe,KAAK,CAAC;AAC/D,QAAM,QAAoC,CAAC;AAC3C,aAAW,SAAS,QAAQ,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG;AACrF,UAAM,eAAe,KAAK,SAAS,MAAM,IAAI;AAC7C,UAAM,OAAO,KAAK,MAAM,YAAY;AACpC,QAAI,MAAM,YAAY,GAAG;AACvB,YAAM,KAAK,GAAG,MAAM,qBAAqB,MAAM,YAAY,CAAC;AAC5D;AAAA,IACF;AACA,QAAI,MAAM,OAAO,GAAG;AAClB,YAAM,QAAQ,MAAM,SAAS,IAAI;AACjC,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,MAAM;AAAA,QAClB,QAAQ,OAAO,KAAK;AAAA,QACpB,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,QAAI,MAAM,eAAe,GAAG;AAC1B,YAAM,cAAc,MAAM,SAAS,MAAM,EAAE,UAAU,SAAS,CAAC;AAC/D,YAAM,KAAK;AAAA,QACT,MAAM,aAAa,YAAY;AAAA,QAC/B,YAAY,YAAY;AAAA,QACxB,QAAQ,OAAO,WAAW;AAAA,QAC1B,MAAM;AAAA,MACR,CAAC;AACD;AAAA,IACF;AACA,UAAM,IAAI,MAAM,yCAAyC,YAAY,EAAE;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,SAAS,QAAgB,WAA4B;AAC5D,QAAM,OAAO,SAAS,QAAQ,SAAS;AACvC,SAAO,SAAS,MAAO,CAAC,KAAK,WAAW,KAAK,GAAG,EAAE,KAAK,SAAS,QAAQ,CAAC,WAAW,IAAI;AAC1F;AAEA,eAAe,wBAAwB,aAAoC;AACzE,MAAI;AACF,UAAM,MAAM,WAAW;AAAA,EACzB,SAAS,OAAO;AACd,QAAK,MAAgC,SAAS,SAAU;AACxD,UAAM;AAAA,EACR;AACA,QAAM,IAAI,MAAM,oDAAoD,WAAW,EAAE;AACnF;AAEA,eAAe,iBACb,iBACA,MACA,eACA,oBAC+B;AAC/B,QAAM,SAAS,QAAQ,eAAe;AACtC,QAAM,cAAc,QAAQ,KAAK,WAAW;AAC5C,MAAI,SAAS,QAAQ,WAAW,GAAG;AACjC,UAAM,IAAI,MAAM,0EAA0E;AAAA,EAC5F;AACA,QAAM,MAAM,QAAQ,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AACrD,QAAM,wBAAwB,WAAW;AACzC,QAAM,UAAU,MAAM,QAAQ,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,CAAC,WAAW,CAAC;AAC9F,MAAI;AACF,UAAM,GAAG,QAAQ,KAAK,SAAS,WAAW,GAAG;AAAA,MAC3C,WAAW;AAAA,MACX,cAAc;AAAA,MACd,OAAO;AAAA,MACP,oBAAoB;AAAA,MACpB,kBAAkB;AAAA,IACpB,CAAC;AACD,UAAM,MAAM,KAAK,SAAS,SAAS,CAAC;AACpC,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAC5E,UAAM,UAAU,KAAK,SAAS,WAAW,YAAY,GAAG,cAAc,MAAM;AAE5E,UAAM,QAAQ;AAAA,MACZ,GAAG,MAAM,qBAAqB,SAAS,WAAW;AAAA,MAClD,GAAG,MAAM,qBAAqB,SAAS,SAAS;AAAA,IAClD,EAAE,KAAK,CAAC,MAAM,UAAU,YAAY,KAAK,MAAM,MAAM,IAAI,CAAC;AAC1D,UAAM,aAAa,MAAM,OAAO,CAAC,OAAO,SAAS,QAAQ,KAAK,YAAY,CAAC;AAC3E,UAAM,YAAY,OAAO;AAAA,MACvB,MACG,IAAI,CAAC,SAAS,GAAG,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,KAAK,UAAU,KAAK,KAAK,MAAM;AAAA,CAAI,EAChF,KAAK,EAAE;AAAA,MACV;AAAA,IACF;AACA,UAAM,UAAgC;AAAA,MACpC,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,oBAAoB,KAAK,aAAa,WAAW;AAAA,MACjD,cAAc,KAAK,aAAa,cAAc;AAAA,MAC9C;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB;AAAA,MACA,YAAY,OAAO,SAAS;AAAA,IAC9B;AACA,UAAM,UAAU,KAAK,SAAS,cAAc,GAAG,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,CAAI;AACtF,UAAM,OAAO,SAAS,WAAW;AACjC,WAAO;AAAA,EACT,SAAS,OAAO;AACd,UAAM,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAClE,UAAM;AAAA,EACR;AACF;AAEA,SAAS,aAAa,OAAwB;AAC5C,MAAI,OAAO,SAAS,KAAK,EAAG,QAAO;AACnC,MAAI,UAAU,UAAa,UAAU,KAAM,QAAO,OAAO,MAAM,CAAC;AAChE,SAAO,OAAO,KAAK,OAAO,KAAK,GAAG,MAAM;AAC1C;AAOA,eAAsB,eAAe,MAA0C;AAC7E,QAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,KAAK,SAAS,CAAC;AACxD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,SAAkC,OAAO,MAAM,CAAC;AACpD,MAAI,qBAAqB;AACzB,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,QAAI;AACF,YAAM,KAAK,MAAM,GAAG;AACpB,YAAM,MAAM,KAAK,OAAO;AACxB,YAAM,OAAO,KAAK,KAAK,GAAG;AAC1B,UAAI;AACF,cAAM,SAAS,MAAM,cAAc,KAAK,MAAM;AAAA,UAC5C,KAAK,KAAK,MAAM,KAAK,IAAI,GAAG,IAAI;AAAA,UAChC,UAAU;AAAA,UACV,WAAW;AAAA,UACX,GAAI,KAAK,YAAY,EAAE,SAAS,KAAK,UAAU,IAAI,CAAC;AAAA,QACtD,CAAC;AACD,iBAAS,aAAa,OAAO,MAAM;AACnC,iBAAS,aAAa,OAAO,MAAM;AACnC,6BAAqB;AAAA,MACvB,SAAS,KAAK;AACZ,cAAM,IAAI;AACV,iBAAS,aAAa,EAAE,MAAM;AAC9B,iBAAS,aAAa,EAAE,MAAM;AAC9B,cAAM,SAAS,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG,CAAC,EACzE,SAAS,MAAM,EACf,MAAM,GAAG,GAAI;AAChB,cAAM,IAAI,MAAM,GAAG,KAAK,UAAU,QAAQ,MAAM,EAAE,CAAC,sBAAsB,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC;AAAA,EAAO,MAAM,EAAE;AAAA,MAC/G;AACA,cAAQ,MAAM,KAAK,YAAY,GAAG;AAAA,IACpC,SAAS,KAAK;AACZ,gBAAU;AAAA,IACZ;AAAA,EACF,UAAE;AACA,QAAI;AACJ,QAAI,KAAK,SAAS;AAChB,UAAI;AACF,yBAAiB,MAAM;AAAA,UACrB;AAAA,UACA,KAAK;AAAA,UACL,EAAE,QAAQ,OAAO;AAAA,UACjB;AAAA,QACF;AAAA,MACF,SAAS,cAAc;AACrB,kBAAU,IAAI;AAAA,UACZ,uDAAuD,wBAAwB,QAAQ,aAAa,UAAU,YAAY;AAAA,UAC1H,EAAE,OAAO,WAAW,aAAa;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,KAAK,QAAS,OAAM,GAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACjF,QAAI,SAAS;AACX,YAAM,IAAI;AAAA,QACR,mBAAmB,QAAQ,QAAQ,UAAU,OAAO,OAAO;AAAA,QAC3D;AAAA,QACA,EAAE,OAAO,QAAQ;AAAA,MACnB;AAAA,IACF;AACA,QAAI,CAAC,MAAO,OAAM,IAAI,iBAAiB,0CAA0C,cAAc;AAC/F,WAAO,iBAAiB,EAAE,GAAG,OAAO,eAAe,IAAI;AAAA,EACzD;AACF;AAGA,eAAsB,eAAkB,MAA0B;AAChE,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,SAAS,MAAM,MAAM;AAAA,EACnC,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,gCAAgC,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACtG;AACA,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,4BAA4B,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,EACjG;AACF;AAGA,eAAsB,UAAU,MAAc,SAAgC;AAC5E,QAAM,UAAU,MAAM,OAAO;AAC/B;AAGO,SAAS,UAAU,QAAgB,IAAoB;AAC5D,SAAO,GAAG,MAAM,IAAI,EAAE,GAAG,QAAQ,oBAAoB,GAAG;AAC1D;","names":["resolve"]}

package/dist/chunk-GC2EPS6L.js

@@ -1,130 +0,0 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/t2-ragbench.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "t2-ragbench.json");
-var dataFile = () => process.env.T2_RAGBENCH_DATA_FILE;
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`);
-  const query = firstString(raw, ["question", "query", "prompt"]);
-  const goldAnswers = allStrings(raw, [
-    "program_answer",
-    "original_answer",
-    "answer",
-    "answers",
-    "reference",
-    "reference_answer",
-    "gold"
-  ]);
-  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`);
-  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`);
-  const baseContexts = contextsFrom(raw.context).length > 0 ? contextsFrom(raw.context) : contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.chunks).length > 0 ? contextsFrom(raw.chunks) : contextsFrom(raw.passages);
-  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text);
-  const contexts = table ? [...baseContexts, { id: "table", title: "Table", text: table }] : baseContexts;
-  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? "unknown";
-  const documentId = stringFrom(raw.context_id) ?? stringFrom(raw.document_id) ?? stringFrom(raw.doc_id) ?? stringFrom(raw.file_name) ?? "unknown";
-  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`;
-  const meta = {
-    benchmark: "t2-ragbench",
-    query,
-    goldAnswers,
-    contexts,
-    subset,
-    documentId
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? subset,
-    prompt: [
-      "Answer this T2-RAGBench text-and-table financial question.",
-      "Do the required numerical reasoning from the supplied context before giving the final value.",
-      "End with a single final line: `FINAL ANSWER: <answer>`.",
-      "",
-      `Question: ${query}`,
-      `Document: ${documentId}`,
-      `Subset: ${subset}`,
-      contexts.length > 0 ? `
-Context:
-${contextBlock(contexts)}` : void 0
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldAnswers)) {
-    throw new Error(`T2-RAGBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "T2-RAGBench");
-}
-function createT2RagBenchAdapter() {
-  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === "1";
-  return {
-    name: "t2-ragbench",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "T2-RAGBench");
-    },
-    async goldArtifact(task) {
-      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 });
-      return answerScoreToBenchScore(score, {
-        benchmark: meta.benchmark,
-        subset: meta.subset,
-        documentId: meta.documentId,
-        contextCount: meta.contexts.length
-      });
-    }
-  };
-}
-
-export {
-  createT2RagBenchAdapter
-};
-//# sourceMappingURL=chunk-GC2EPS6L.js.map