@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/chunk-JRWWGMK7.js

@@ -0,0 +1,148 @@
+import {
+  benchRoot,
+  preflightVenvImports,
+  readJsonReport,
+  runStagedJudge,
+  runVenvPython,
+  safeRunId,
+  stageFile,
+  venvPythonAt
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/terminal-bench.ts
+import { join } from "path";
+var terminalBenchVenvDir = () => process.env.TERMINAL_BENCH_VENV ?? ".venv-terminal-bench";
+var terminalBenchPython = () => venvPythonAt(terminalBenchVenvDir());
+var terminalBenchBin = () => join(benchRoot, terminalBenchVenvDir(), "bin", "tb");
+var DATASET = "terminal-bench-core";
+var DATASET_VERSION = "0.1.1";
+var DATASET_REF = `${DATASET}==${DATASET_VERSION}`;
+var FIXTURE_IDS = ["hello-world"];
+var SCRIPT_AGENT = "tb_agents.script_agent:ScriptAgent";
+async function loadRows(opts) {
+  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS);
+  const limit = opts.limit ?? null;
+  const script = `
+import json, sys
+from pathlib import Path
+from terminal_bench.dataset.dataset import Dataset
+from terminal_bench.handlers.trial_handler import TaskPaths
+
+req_ids = json.loads(sys.argv[1]) if sys.argv[1] else None
+limit = json.loads(sys.argv[2]) if sys.argv[2] else None
+
+ds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)
+
+import yaml
+out = []
+for task_dir in ds:
+    tp = TaskPaths(task_dir)
+    cfg = yaml.safe_load(tp.task_config_path.read_text())
+    try:
+        sol = tp.solution_path.read_text()
+    except FileNotFoundError:
+        sol = None
+    out.append({
+        "id": task_dir.name,
+        "instruction": cfg["instruction"],
+        "task_dir": str(task_dir),
+        "solution": sol if (tp.solution_path.suffix == ".sh" if sol is not None else False) else None,
+    })
+print(json.dumps(out))
+`;
+  const stdout = await runVenvPython(
+    script,
+    [ids ? JSON.stringify(ids) : "", limit !== null ? String(limit) : ""],
+    0,
+    terminalBenchPython()
+  );
+  return JSON.parse(stdout);
+}
+function createTerminalBenchAdapter() {
+  return {
+    name: "terminal-bench",
+    async preflight() {
+      await preflightVenvImports({
+        modules: ["terminal_bench"],
+        requireDocker: true,
+        python: terminalBenchPython(),
+        fix: `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench (an ISOLATED venv \u2014 Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; override the dir with TERMINAL_BENCH_VENV) ; (2) ensure the Docker daemon is running (the judge builds per-task images on first run). The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`
+      });
+    },
+    async loadTasks(opts = {}) {
+      const rows = await loadRows(opts);
+      if (rows.length === 0) {
+        throw new Error(
+          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} (dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`
+        );
+      }
+      return rows.map(
+        (r) => ({
+          id: r.id,
+          split: DATASET_VERSION,
+          prompt: r.instruction,
+          metadata: {
+            dataset: DATASET,
+            datasetVersion: DATASET_VERSION,
+            datasetRef: DATASET_REF,
+            taskDir: r.task_dir,
+            solution: r.solution,
+            instruction: r.instruction
+          }
+        })
+      );
+    },
+    async goldArtifact(task) {
+      const sol = task.metadata?.solution;
+      return typeof sol === "string" ? sol : void 0;
+    },
+    async judge(task, artifact) {
+      const runId = safeRunId("bench", `${task.id}-${Date.now()}`);
+      return runStagedJudge({
+        tmpPrefix: "tbench-",
+        bin: terminalBenchBin(),
+        cwd: () => benchRoot,
+        async stage(dir) {
+          await stageFile(join(dir, "attempt.sh"), artifact);
+        },
+        // The harness builds a fresh task container, runs ScriptAgent (which replays
+        // the artifact script), then runs the task's verifier. --no-livestream keeps
+        // stdout sane; --cleanup removes the per-run images.
+        argv: (dir) => [
+          "run",
+          "-d",
+          DATASET_REF,
+          "-t",
+          task.id,
+          "--agent-import-path",
+          SCRIPT_AGENT,
+          "--agent-kwarg",
+          `script_path=${join(dir, "attempt.sh")}`,
+          "--output-path",
+          join(dir, "runs"),
+          "--run-id",
+          runId,
+          "--n-concurrent",
+          "1",
+          "--no-livestream",
+          "--cleanup"
+        ],
+        async parseReport(dir) {
+          const report = await readJsonReport(join(dir, "runs", runId, "results.json"));
+          const resolved = (report.resolved_ids ?? []).includes(task.id);
+          const trial = report.results?.find((r) => r.task_id === task.id);
+          return {
+            resolved,
+            score: resolved ? 1 : 0,
+            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {})
+          };
+        }
+      });
+    }
+  };
+}
+
+export {
+  createTerminalBenchAdapter
+};
+//# sourceMappingURL=chunk-JRWWGMK7.js.map

package/dist/chunk-JRWWGMK7.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/terminal-bench.ts"],"sourcesContent":["/**\n * Terminal-Bench adapter. Each task = a Docker environment + an English\n * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact\n * is the shell script the agent ran to attempt the task; the judge REPLAYS that\n * script in a fresh task container via the Terminal-Bench harness (`tb run` with\n * our ScriptAgent), then the task's own verifier scores the resulting state.\n * Fully deterministic — no LLM judge, no self-authored score.\n *\n * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`\n * installed + a running Docker daemon (per-task images are built on first run).\n * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the\n * Terminal-Bench registry on first run.\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay\n * argv, and the results.json shape.\n */\n\nimport { join } from 'node:path'\nimport {\n  benchRoot,\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  venvPythonAt,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n// Terminal-Bench imports LiteLLM/Pydantic-2 APIs, while AppWorld pins Pydantic 1.\n// Keep it out of the shared bench .venv. Resolved at call-time so tests/runs can\n// override the env without reloading this module.\nconst terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench'\nconst terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir())\nconst terminalBenchBin = (): string => join(benchRoot, terminalBenchVenvDir(), 'bin', 'tb')\n\n// Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the\n// installed CLI) and is the published launch task set. name==version is what `tb\n// run -d` and `tb datasets download -d` both accept.\nconst DATASET = 'terminal-bench-core'\nconst DATASET_VERSION = '0.1.1'\nconst DATASET_REF = `${DATASET}==${DATASET_VERSION}`\n\n// Bundled fixture: when no ids/limit are given, load these. hello-world is the\n// fastest deterministic task (prebuilt python image, file-write verifier) so the\n// adapter is runnable without a large pull.\nconst FIXTURE_IDS = ['hello-world']\n\n// Import path the harness uses to load our replay agent (cwd = benchRoot).\nconst SCRIPT_AGENT = 'tb_agents.script_agent:ScriptAgent'\n\ninterface TbTaskRow {\n  id: string\n  instruction: string\n  task_dir: string\n  solution: string | null\n}\n\ninterface TbReport {\n  resolved_ids?: string[]\n  results?: Array<{ task_id: string; is_resolved: boolean | null; parser_results?: unknown }>\n}\n\n/** Enumerate dataset tasks via tb's Dataset loader (caches from the registry on\n *  first run). Reads instruction from each task.yaml and the gold solution. */\nasync function loadRows(opts: LoadOptions): Promise<TbTaskRow[]> {\n  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS)\n  const limit = opts.limit ?? null\n  const script = `\nimport json, sys\nfrom pathlib import Path\nfrom terminal_bench.dataset.dataset import Dataset\nfrom terminal_bench.handlers.trial_handler import TaskPaths\n\nreq_ids = json.loads(sys.argv[1]) if sys.argv[1] else None\nlimit = json.loads(sys.argv[2]) if sys.argv[2] else None\n\nds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)\n\nimport yaml\nout = []\nfor task_dir in ds:\n    tp = TaskPaths(task_dir)\n    cfg = yaml.safe_load(tp.task_config_path.read_text())\n    try:\n        sol = tp.solution_path.read_text()\n    except FileNotFoundError:\n        sol = None\n    out.append({\n        \"id\": task_dir.name,\n        \"instruction\": cfg[\"instruction\"],\n        \"task_dir\": str(task_dir),\n        \"solution\": sol if (tp.solution_path.suffix == \".sh\" if sol is not None else False) else None,\n    })\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(\n    script,\n    [ids ? JSON.stringify(ids) : '', limit !== null ? String(limit) : ''],\n    0,\n    terminalBenchPython(),\n  )\n  return JSON.parse(stdout) as TbTaskRow[]\n}\n\nexport function createTerminalBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'terminal-bench',\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['terminal_bench'],\n        requireDocker: true,\n        python: terminalBenchPython(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && ` +\n          `bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench ` +\n          `(an ISOLATED venv — Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; ` +\n          `override the dir with TERMINAL_BENCH_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-task images on first run). ` +\n          `The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const rows = await loadRows(opts)\n      if (rows.length === 0) {\n        throw new Error(\n          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} ` +\n            `(dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`,\n        )\n      }\n      return rows.map(\n        (r): BenchTask => ({\n          id: r.id,\n          split: DATASET_VERSION,\n          prompt: r.instruction,\n          metadata: {\n            dataset: DATASET,\n            datasetVersion: DATASET_VERSION,\n            datasetRef: DATASET_REF,\n            taskDir: r.task_dir,\n            solution: r.solution,\n            instruction: r.instruction,\n          },\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the task's solution.sh (the oracle script). solution.yaml tasks have\n      // no shell-script artifact form here, so they return undefined (cannot be\n      // verify-judged via the script-replay seam — use a .sh-solution task).\n      const sol = task.metadata?.solution\n      return typeof sol === 'string' ? sol : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', `${task.id}-${Date.now()}`)\n      return runStagedJudge({\n        tmpPrefix: 'tbench-',\n        bin: terminalBenchBin(),\n        cwd: () => benchRoot,\n        async stage(dir) {\n          await stageFile(join(dir, 'attempt.sh'), artifact)\n        },\n        // The harness builds a fresh task container, runs ScriptAgent (which replays\n        // the artifact script), then runs the task's verifier. --no-livestream keeps\n        // stdout sane; --cleanup removes the per-run images.\n        argv: (dir) => [\n          'run',\n          '-d', DATASET_REF,\n          '-t', task.id,\n          '--agent-import-path', SCRIPT_AGENT,\n          '--agent-kwarg', `script_path=${join(dir, 'attempt.sh')}`,\n          '--output-path', join(dir, 'runs'),\n          '--run-id', runId,\n          '--n-concurrent', '1',\n          '--no-livestream',\n          '--cleanup',\n        ],\n        async parseReport(dir) {\n          const report = await readJsonReport<TbReport>(join(dir, 'runs', runId, 'results.json'))\n          const resolved = (report.resolved_ids ?? []).includes(task.id)\n          const trial = report.results?.find((r) => r.task_id === task.id)\n          return {\n            resolved,\n            score: resolved ? 1 : 0,\n            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {}),\n          }\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAkBA,SAAS,YAAY;AAgBrB,IAAM,uBAAuB,MAAc,QAAQ,IAAI,uBAAuB;AAC9E,IAAM,sBAAsB,MAAc,aAAa,qBAAqB,CAAC;AAC7E,IAAM,mBAAmB,MAAc,KAAK,WAAW,qBAAqB,GAAG,OAAO,IAAI;AAK1F,IAAM,UAAU;AAChB,IAAM,kBAAkB;AACxB,IAAM,cAAc,GAAG,OAAO,KAAK,eAAe;AAKlD,IAAM,cAAc,CAAC,aAAa;AAGlC,IAAM,eAAe;AAgBrB,eAAe,SAAS,MAAyC;AAC/D,QAAM,MAAM,KAAK,QAAQ,KAAK,QAAQ,OAAO;AAC7C,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oBASG,KAAK,UAAU,OAAO,CAAC,aAAa,KAAK,UAAU,eAAe,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBrF,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA,CAAC,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,UAAU,OAAO,OAAO,KAAK,IAAI,EAAE;AAAA,IACpE;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO,KAAK,MAAM,MAAM;AAC1B;AAEO,SAAS,6BAA+C;AAC7D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,gBAAgB;AAAA,QAC1B,eAAe;AAAA,QACf,QAAQ,oBAAoB;AAAA,QAC5B,KACE,kCAAkC,qBAAqB,CAAC,aAC/C,qBAAqB,CAAC,gRAIxB,WAAW;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,OAAO,MAAM,SAAS,IAAI;AAChC,UAAI,KAAK,WAAW,GAAG;AACrB,cAAM,IAAI;AAAA,UACR,kDAAkD,KAAK,UAAU,IAAI,CAAC,aACxD,WAAW;AAAA,QAC3B;AAAA,MACF;AACA,aAAO,KAAK;AAAA,QACV,CAAC,OAAkB;AAAA,UACjB,IAAI,EAAE;AAAA,UACN,OAAO;AAAA,UACP,QAAQ,EAAE;AAAA,UACV,UAAU;AAAA,YACR,SAAS;AAAA,YACT,gBAAgB;AAAA,YAChB,YAAY;AAAA,YACZ,SAAS,EAAE;AAAA,YACX,UAAU,EAAE;AAAA,YACZ,aAAa,EAAE;AAAA,UACjB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,MAAM,KAAK,UAAU;AAC3B,aAAO,OAAO,QAAQ,WAAW,MAAM;AAAA,IACzC;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,QAAQ,UAAU,SAAS,GAAG,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AAC3D,aAAO,eAAe;AAAA,QACpB,WAAW;AAAA,QACX,KAAK,iBAAiB;AAAA,QACtB,KAAK,MAAM;AAAA,QACX,MAAM,MAAM,KAAK;AACf,gBAAM,UAAU,KAAK,KAAK,YAAY,GAAG,QAAQ;AAAA,QACnD;AAAA;AAAA;AAAA;AAAA,QAIA,MAAM,CAAC,QAAQ;AAAA,UACb;AAAA,UACA;AAAA,UAAM;AAAA,UACN;AAAA,UAAM,KAAK;AAAA,UACX;AAAA,UAAuB;AAAA,UACvB;AAAA,UAAiB,eAAe,KAAK,KAAK,YAAY,CAAC;AAAA,UACvD;AAAA,UAAiB,KAAK,KAAK,MAAM;AAAA,UACjC;AAAA,UAAY;AAAA,UACZ;AAAA,UAAkB;AAAA,UAClB;AAAA,UACA;AAAA,QACF;AAAA,QACA,MAAM,YAAY,KAAK;AACrB,gBAAM,SAAS,MAAM,eAAyB,KAAK,KAAK,QAAQ,OAAO,cAAc,CAAC;AACtF,gBAAM,YAAY,OAAO,gBAAgB,CAAC,GAAG,SAAS,KAAK,EAAE;AAC7D,gBAAM,QAAQ,OAAO,SAAS,KAAK,CAAC,MAAM,EAAE,YAAY,KAAK,EAAE;AAC/D,iBAAO;AAAA,YACL;AAAA,YACA,OAAO,WAAW,IAAI;AAAA,YACtB,QAAQ,KAAK,UAAU,OAAO,kBAAkB,OAAO,gBAAgB,CAAC,CAAC;AAAA,UAC3E;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-JTHWEDEW.js

@@ -0,0 +1,32 @@
+// src/refine-loop.ts
+var defaultDecide = (history) => history[history.length - 1]?.verdict?.valid === true;
+async function runRefineLoop(spec) {
+  const rounds = Math.max(1, spec.rounds);
+  const decide = spec.decide ?? defaultDecide;
+  let ctxSet = false;
+  let ctx = void 0;
+  try {
+    if (spec.setup) {
+      ctx = await spec.setup();
+      ctxSet = true;
+    }
+    const history = [];
+    for (let r = 1; r <= rounds; r += 1) {
+      const prompt = spec.prompt(r, history, ctx);
+      const { artifact, note } = await spec.runShot(prompt, r, ctx);
+      const verdict = spec.judge ? await spec.judge(artifact, r) : void 0;
+      history.push({ round: r, prompt, artifact, verdict, note });
+      if (decide(history)) break;
+    }
+    const blind = history[0];
+    const final = history[history.length - 1];
+    return { rounds: history, blind, final, resolved: history.some((h) => h.verdict?.valid === true) };
+  } finally {
+    if (ctxSet && spec.teardown) await spec.teardown(ctx);
+  }
+}
+
+export {
+  runRefineLoop
+};
+//# sourceMappingURL=chunk-JTHWEDEW.js.map

package/dist/chunk-JTHWEDEW.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/refine-loop.ts"],"sourcesContent":["/**\n * runRefineLoop — the ONE shared k-shot loop (see docs/architecture.md §1, §12).\n *\n * It replaces the seven hand-rolled `for (round 1..k) { shot → judge → decide →\n * carry-forward }` copies across the workers (~700 LOC of duplicated skeleton).\n * The worker is an OPAQUE `runShot`; this loop owns iteration, carry-forward\n * steering, optional per-round judging + early stop, and round bookkeeping.\n *\n * Two carry-forward channels, both first-class (the extraction found both in the\n * wild and they must stay pluggable):\n *   - EXECUTION CONTEXT (`Ctx`): created once by `setup`, threaded to every shot,\n *     torn down by `teardown`. This is how filesystem/session state carries — a\n *     cloned repo whose edits persist (SWE refine), a shared sandbox box whose\n *     session persists (sandbox research), or a scratch dir.\n *   - PROMPT (`prompt(round, history, ctx)`): how textual state carries — round 1\n *     is the blind prompt; rounds 2+ fold prior rounds' artifacts + a directive.\n *\n * Judging is OPTIONAL: workers that run all k rounds and let the orchestrator\n * judge omit `judge`/`decide` (the loop runs to budget); a loop that should stop\n * on the first valid answer wires `judge` (default `decide` = stop-on-valid).\n *\n * This is the inference-timescale instance of the spine's atom: `runShot` is the\n * worker Agent's `act→Output`; `prompt` is the driver Agent's `act→steer`. It is\n * deliberately corpus-agnostic — callers map `RefineLoopResult` to a RunRecord.\n */\n\n/** Minimal per-round verdict — `valid` gates early-stop, `score` is informational. */\nexport interface RoundVerdict {\n  valid: boolean\n  score?: number\n}\n\nexport interface RoundRecord<Artifact> {\n  /** 1-based round index. */\n  round: number\n  prompt: string\n  artifact: Artifact\n  /** Present only when a `judge` is wired. */\n  verdict?: RoundVerdict\n  /** Non-fatal note (e.g. a liveness backstop fired this round). */\n  note?: string\n}\n\nexport interface RefineLoopSpec<Artifact, Ctx = void> {\n  /** Max shots. Always ≥ 1. */\n  rounds: number\n  /** Build the per-task execution context once (clone repo / create box / mkdtemp). */\n  setup?: () => Promise<Ctx>\n  /** Round `r`'s prompt. `r === 1` is the blind prompt; `r > 1` carries `history`. */\n  prompt: (round: number, history: ReadonlyArray<RoundRecord<Artifact>>, ctx: Ctx) => string\n  /** Run ONE shot — the opaque worker (local spawn / sandbox stream / router chat). */\n  runShot: (\n    prompt: string,\n    round: number,\n    ctx: Ctx,\n  ) => Promise<{ artifact: Artifact; note?: string }>\n  /** Optional per-round judge. Omit to run all `rounds` (orchestrator judges later). */\n  judge?: (artifact: Artifact, round: number) => Promise<RoundVerdict>\n  /** Stop after this round? Default: stop once a round's verdict is valid (else run to budget). */\n  decide?: (history: ReadonlyArray<RoundRecord<Artifact>>) => boolean\n  /** Always runs (even on throw) when `setup` ran. */\n  teardown?: (ctx: Ctx) => Promise<void>\n}\n\nexport interface RefineLoopResult<Artifact> {\n  /** Every executed round in order. */\n  rounds: RoundRecord<Artifact>[]\n  /** Round 1 — the blind artifact. */\n  blind: RoundRecord<Artifact>\n  /** The last executed round. */\n  final: RoundRecord<Artifact>\n  /** True iff any round was judged valid (only meaningful when `judge` is wired). */\n  resolved: boolean\n}\n\nconst defaultDecide = <Artifact>(history: ReadonlyArray<RoundRecord<Artifact>>): boolean =>\n  history[history.length - 1]?.verdict?.valid === true\n\nexport async function runRefineLoop<Artifact, Ctx = void>(\n  spec: RefineLoopSpec<Artifact, Ctx>,\n): Promise<RefineLoopResult<Artifact>> {\n  const rounds = Math.max(1, spec.rounds)\n  const decide = spec.decide ?? defaultDecide\n  let ctxSet = false\n  let ctx = undefined as Ctx\n  try {\n    if (spec.setup) {\n      ctx = await spec.setup()\n      ctxSet = true\n    }\n    const history: RoundRecord<Artifact>[] = []\n    for (let r = 1; r <= rounds; r += 1) {\n      const prompt = spec.prompt(r, history, ctx)\n      const { artifact, note } = await spec.runShot(prompt, r, ctx)\n      const verdict = spec.judge ? await spec.judge(artifact, r) : undefined\n      history.push({ round: r, prompt, artifact, verdict, note })\n      if (decide(history)) break\n    }\n    // The loop runs ≥ 1 round, so history is non-empty.\n    const blind = history[0] as RoundRecord<Artifact>\n    const final = history[history.length - 1] as RoundRecord<Artifact>\n    return { rounds: history, blind, final, resolved: history.some((h) => h.verdict?.valid === true) }\n  } finally {\n    if (ctxSet && spec.teardown) await spec.teardown(ctx)\n  }\n}\n"],"mappings":";AA2EA,IAAM,gBAAgB,CAAW,YAC/B,QAAQ,QAAQ,SAAS,CAAC,GAAG,SAAS,UAAU;AAElD,eAAsB,cACpB,MACqC;AACrC,QAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM;AACtC,QAAM,SAAS,KAAK,UAAU;AAC9B,MAAI,SAAS;AACb,MAAI,MAAM;AACV,MAAI;AACF,QAAI,KAAK,OAAO;AACd,YAAM,MAAM,KAAK,MAAM;AACvB,eAAS;AAAA,IACX;AACA,UAAM,UAAmC,CAAC;AAC1C,aAAS,IAAI,GAAG,KAAK,QAAQ,KAAK,GAAG;AACnC,YAAM,SAAS,KAAK,OAAO,GAAG,SAAS,GAAG;AAC1C,YAAM,EAAE,UAAU,KAAK,IAAI,MAAM,KAAK,QAAQ,QAAQ,GAAG,GAAG;AAC5D,YAAM,UAAU,KAAK,QAAQ,MAAM,KAAK,MAAM,UAAU,CAAC,IAAI;AAC7D,cAAQ,KAAK,EAAE,OAAO,GAAG,QAAQ,UAAU,SAAS,KAAK,CAAC;AAC1D,UAAI,OAAO,OAAO,EAAG;AAAA,IACvB;AAEA,UAAM,QAAQ,QAAQ,CAAC;AACvB,UAAM,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AACxC,WAAO,EAAE,QAAQ,SAAS,OAAO,OAAO,UAAU,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,UAAU,IAAI,EAAE;AAAA,EACnG,UAAE;AACA,QAAI,UAAU,KAAK,SAAU,OAAM,KAAK,SAAS,GAAG;AAAA,EACtD;AACF;","names":[]}

package/dist/chunk-KDIKRJGB.js

@@ -0,0 +1,120 @@
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringFrom
+} from "./chunk-X3BTXCJ4.js";
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/open-rag-bench.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "open-rag-bench.json");
+var dataFile = () => process.env.OPEN_RAG_BENCH_DATA_FILE;
+function rowToTask(raw, index) {
+  if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`);
+  const query = firstString(raw, ["question", "query", "prompt"]);
+  const goldAnswers = allStrings(raw, ["answer", "answers", "reference", "reference_answer", "gold"]);
+  if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`);
+  if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`);
+  const contexts = contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.chunks).length > 0 ? contextsFrom(raw.chunks) : contextsFrom(raw.pages);
+  const documentId = stringFrom(raw.document_id) ?? stringFrom(raw.doc_id) ?? stringFrom(raw.pdf_id) ?? stringFrom(raw.file_name) ?? stringFrom(raw.filename) ?? "unknown";
+  const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? "unknown";
+  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}`;
+  const meta = {
+    benchmark: "open-rag-bench",
+    query,
+    goldAnswers,
+    contexts,
+    documentId,
+    modality
+  };
+  return {
+    id,
+    split: stringFrom(raw.split) ?? modality,
+    prompt: [
+      "Answer this Open RAG Bench PDF question using the supplied document context.",
+      "Use tables, text, and image-derived notes when present.",
+      "End with a single final line: `FINAL ANSWER: <answer>`.",
+      "",
+      `Question: ${query}`,
+      `Document: ${documentId}`,
+      `Modality: ${modality}`,
+      contexts.length > 0 ? `
+Document context:
+${contextBlock(contexts)}` : void 0
+    ].filter(Boolean).join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.goldAnswers)) {
+    throw new Error(`Open RAG Bench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+async function loadRows(path) {
+  const rows = await readJsonRows(path);
+  if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`);
+  return rows;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
+  return selectTasks(rows.map(rowToTask), opts, "Open RAG Bench");
+}
+function createOpenRagBenchAdapter() {
+  const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === "1";
+  return {
+    name: "open-rag-bench",
+    output: ragAnswerOutput,
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8");
+        return;
+      }
+      const path = dataFile();
+      if (!path) {
+        throw new Error(
+          "OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing."
+        );
+      }
+      await loadRows(path);
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const path = dataFile();
+      if (!path) throw new Error("OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks");
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, "Open RAG Bench");
+    },
+    async goldArtifact(task) {
+      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const score = scoreAnswerArtifact(artifact, meta.goldAnswers);
+      return answerScoreToBenchScore(score, {
+        benchmark: meta.benchmark,
+        documentId: meta.documentId,
+        modality: meta.modality,
+        contextCount: meta.contexts.length
+      });
+    }
+  };
+}
+
+export {
+  createOpenRagBenchAdapter
+};
+//# sourceMappingURL=chunk-KDIKRJGB.js.map

package/dist/chunk-KDIKRJGB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/open-rag-bench.ts"],"sourcesContent":["/**\n * Open RAG Bench adapter.\n *\n * This targets Vectara-style Open RAG Bench exports over PDF-derived text,\n * table, and image contexts. The deterministic judge scores final-answer\n * agreement and surfaces modality/document metadata for diagnostics.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'open-rag-bench.json')\n\ninterface OpenRagBenchMeta {\n  benchmark: 'open-rag-bench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  documentId: string\n  modality: string\n}\n\nconst dataFile = (): string | undefined => process.env.OPEN_RAG_BENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'prompt'])\n  const goldAnswers = allStrings(raw, ['answer', 'answers', 'reference', 'reference_answer', 'gold'])\n  if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`)\n  const contexts =\n    contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.chunks).length > 0\n        ? contextsFrom(raw.chunks)\n        : contextsFrom(raw.pages)\n  const documentId =\n    stringFrom(raw.document_id) ??\n    stringFrom(raw.doc_id) ??\n    stringFrom(raw.pdf_id) ??\n    stringFrom(raw.file_name) ??\n    stringFrom(raw.filename) ??\n    'unknown'\n  const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? 'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}`\n  const meta: OpenRagBenchMeta = {\n    benchmark: 'open-rag-bench',\n    query,\n    goldAnswers,\n    contexts,\n    documentId,\n    modality,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? modality,\n    prompt: [\n      'Answer this Open RAG Bench PDF question using the supplied document context.',\n      'Use tables, text, and image-derived notes when present.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Document: ${documentId}`,\n      `Modality: ${modality}`,\n      contexts.length > 0 ? `\\nDocument context:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): OpenRagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`Open RAG Bench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as OpenRagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'Open RAG Bench')\n}\n\nexport function createOpenRagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === '1'\n\n  return {\n    name: 'open-rag-bench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'Open RAG Bench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        documentId: meta.documentId,\n        modality: meta.modality,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAQA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,qBAAqB;AAWlE,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,sBAAsB,KAAK,oBAAoB;AACnF,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,QAAQ,CAAC;AAC9D,QAAM,cAAc,WAAW,KAAK,CAAC,UAAU,WAAW,aAAa,oBAAoB,MAAM,CAAC;AAClG,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,sBAAsB,KAAK,yBAAyB;AAChF,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,sBAAsB,KAAK,iBAAiB;AAC1F,QAAM,WACJ,aAAa,IAAI,QAAQ,EAAE,SAAS,IAChC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,MAAM,EAAE,SAAS,IAChC,aAAa,IAAI,MAAM,IACvB,aAAa,IAAI,KAAK;AAC9B,QAAM,aACJ,WAAW,IAAI,WAAW,KAC1B,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,SAAS,KACxB,WAAW,IAAI,QAAQ,KACvB;AACF,QAAM,WAAW,WAAW,IAAI,QAAQ,KAAK,WAAW,IAAI,eAAe,KAAK;AAChF,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,kBAAkB,KAAK;AACpF,QAAM,OAAyB;AAAA,IAC7B,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,aAAa,UAAU;AAAA,MACvB,aAAa,QAAQ;AAAA,MACrB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAwB,aAAa,QAAQ,CAAC,KAAK;AAAA,IAC3E,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAmC;AACnD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,uBAAuB,KAAK,EAAE,wDAAmD;AAAA,EACnG;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,8BAA8B,IAAI,EAAE;AAC3E,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,6DAAwD,KAAK,MAAM,mBAAmB;AACnG,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,gBAAgB;AAChE;AAEO,SAAS,4BAA8C;AAC5D,QAAM,eAAe,QAAQ,IAAI,4BAA4B;AAE7D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,mEAAmE;AAC9F,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,gBAAgB;AAAA,IAClF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,WAAW;AAC5D,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,YAAY,KAAK;AAAA,QACjB,UAAU,KAAK;AAAA,QACf,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-LRRD7NAG.js

@@ -0,0 +1,301 @@
+// src/benchmarks/_harness.ts
+import { execFile, spawn } from "child_process";
+import { createHash } from "crypto";
+import {
+  cp,
+  lstat,
+  mkdir,
+  mkdtemp,
+  readFile,
+  readlink,
+  readdir,
+  rename,
+  rm,
+  writeFile
+} from "fs/promises";
+import { tmpdir } from "os";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "path";
+import { fileURLToPath } from "url";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+var benchRoot = fileURLToPath(new URL("../..", import.meta.url));
+function resolveBenchPython(env = process.env, root = benchRoot) {
+  const configured = env.AGENT_BENCH_PYTHON;
+  if (configured === void 0) return join(root, ".venv", "bin", "python");
+  if (!isAbsolute(configured)) {
+    throw new Error("AGENT_BENCH_PYTHON must be an absolute path");
+  }
+  return configured;
+}
+var venvPython = resolveBenchPython();
+var venvPythonAt = (venvDir) => join(benchRoot, venvDir, "bin", "python");
+var bigBuffer = 1024 * 1024 * 256;
+function venvBin(name) {
+  return join(benchRoot, ".venv", "bin", name);
+}
+async function runVenvPython(script, args = [], timeoutMs = 0, python = venvPython) {
+  const { stdout } = await execFileAsync(python, ["-c", script, ...args], {
+    maxBuffer: bigBuffer,
+    timeout: timeoutMs
+  });
+  return stdout;
+}
+async function preflightVenvImports(opts) {
+  const imports = opts.modules.filter((m) => m.length > 0);
+  const lines = [...imports.map((m) => `import ${m}`)];
+  if (opts.requireDocker) lines.push("import docker", "docker.from_env().ping()");
+  lines.push("print('ok')");
+  try {
+    await runVenvPython(lines.join("\n"), [], 0, opts.python ?? venvPython);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new Error(`${msg}
+${opts.fix}`);
+  }
+}
+function runVenvScriptStdin(scriptPath, args, input, opts = {}) {
+  return new Promise((resolve2, reject) => {
+    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {
+      cwd: opts.cwd ?? benchRoot,
+      ...opts.timeoutMs ? { timeout: opts.timeoutMs } : {}
+    });
+    let stdout = "";
+    let stderr = "";
+    let bytes = 0;
+    child.stdout.on("data", (c) => {
+      bytes += c.length;
+      if (bytes <= bigBuffer) stdout += c.toString("utf8");
+    });
+    child.stderr.on("data", (c) => {
+      stderr += c.toString("utf8");
+    });
+    child.on("error", (err) => reject(err));
+    child.on("close", (code) => {
+      if (code === 0) resolve2(stdout);
+      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)));
+    });
+    child.stdin.end(input);
+  });
+}
+var StagedJudgeError = class extends Error {
+  judgeArtifacts;
+  constructor(message, judgeArtifacts, options) {
+    super(message, options);
+    this.name = "StagedJudgeError";
+    this.judgeArtifacts = judgeArtifacts;
+  }
+};
+function sha256(bytes) {
+  return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+}
+function portablePath(path) {
+  return path.split(sep).join("/");
+}
+function compareText(left, right) {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+async function collectArtifactFiles(root, current) {
+  const absolute = join(root, current);
+  const entries = await readdir(absolute, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries.sort((left, right) => compareText(left.name, right.name))) {
+    const relativePath = join(current, entry.name);
+    const path = join(root, relativePath);
+    if (entry.isDirectory()) {
+      files.push(...await collectArtifactFiles(root, relativePath));
+      continue;
+    }
+    if (entry.isFile()) {
+      const bytes = await readFile(path);
+      files.push({
+        path: portablePath(relativePath),
+        byteLength: bytes.byteLength,
+        sha256: sha256(bytes),
+        kind: "file"
+      });
+      continue;
+    }
+    if (entry.isSymbolicLink()) {
+      const targetBytes = await readlink(path, { encoding: "buffer" });
+      files.push({
+        path: portablePath(relativePath),
+        byteLength: targetBytes.byteLength,
+        sha256: sha256(targetBytes),
+        kind: "symlink"
+      });
+      continue;
+    }
+    throw new Error(`staged judge capture does not support ${relativePath}`);
+  }
+  return files;
+}
+function isWithin(parent, candidate) {
+  const path = relative(parent, candidate);
+  return path === "" || !path.startsWith(`..${sep}`) && path !== ".." && !isAbsolute(path);
+}
+async function assertDestinationAbsent(destination) {
+  try {
+    await lstat(destination);
+  } catch (error) {
+    if (error.code === "ENOENT") return;
+    throw error;
+  }
+  throw new Error(`staged judge capture destination already exists: ${destination}`);
+}
+async function captureStagedRun(sourceDirectory, spec, processOutput, evaluatorSucceeded) {
+  const source = resolve(sourceDirectory);
+  const destination = resolve(spec.destination);
+  if (isWithin(source, destination)) {
+    throw new Error("staged judge capture destination must be outside the evaluator directory");
+  }
+  await mkdir(dirname(destination), { recursive: true });
+  await assertDestinationAbsent(destination);
+  const staging = await mkdtemp(join(dirname(destination), `.${basename(destination)}-capture-`));
+  try {
+    await cp(source, join(staging, "evaluator"), {
+      recursive: true,
+      errorOnExist: true,
+      force: false,
+      preserveTimestamps: true,
+      verbatimSymlinks: true
+    });
+    await mkdir(join(staging, "process"));
+    await writeFile(join(staging, "process", "stdout.bin"), processOutput.stdout);
+    await writeFile(join(staging, "process", "stderr.bin"), processOutput.stderr);
+    const files = [
+      ...await collectArtifactFiles(staging, "evaluator"),
+      ...await collectArtifactFiles(staging, "process")
+    ].sort((left, right) => compareText(left.path, right.path));
+    const byteLength = files.reduce((total, file) => total + file.byteLength, 0);
+    const treeBytes = Buffer.from(
+      files.map((file) => `${file.path}\0${file.kind}\0${file.byteLength}\0${file.sha256}
+`).join(""),
+      "utf8"
+    );
+    const receipt = {
+      schema: "agent-bench/judge-artifacts/v1",
+      directory: destination,
+      evaluatorDirectory: join(destination, "evaluator"),
+      manifestPath: join(destination, "receipt.json"),
+      evaluatorSucceeded,
+      files,
+      fileCount: files.length,
+      byteLength,
+      treeSha256: sha256(treeBytes)
+    };
+    await writeFile(join(staging, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
+`);
+    await rename(staging, destination);
+    return receipt;
+  } catch (error) {
+    await rm(staging, { recursive: true, force: true }).catch(() => {
+    });
+    throw error;
+  }
+}
+function processBytes(value) {
+  if (Buffer.isBuffer(value)) return value;
+  if (value === void 0 || value === null) return Buffer.alloc(0);
+  return Buffer.from(String(value), "utf8");
+}
+async function runStagedJudge(spec) {
+  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix));
+  let stdout = Buffer.alloc(0);
+  let stderr = Buffer.alloc(0);
+  let evaluatorSucceeded = false;
+  let score;
+  let failure;
+  try {
+    try {
+      await spec.stage(dir);
+      const bin = spec.bin ?? venvPython;
+      const argv = spec.argv(dir);
+      try {
+        const output = await execFileAsync(bin, argv, {
+          cwd: spec.cwd ? spec.cwd(dir) : dir,
+          encoding: "buffer",
+          maxBuffer: bigBuffer,
+          ...spec.timeoutMs ? { timeout: spec.timeoutMs } : {}
+        });
+        stdout = processBytes(output.stdout);
+        stderr = processBytes(output.stderr);
+        evaluatorSucceeded = true;
+      } catch (err) {
+        const e = err;
+        stdout = processBytes(e.stdout);
+        stderr = processBytes(e.stderr);
+        const detail = processBytes(e.stderr ?? e.stdout ?? e.message ?? String(err)).toString("utf8").slice(0, 2e3);
+        throw new Error(`${spec.tmpPrefix.replace(/-$/, "")} evaluator failed (${bin} ${argv.join(" ")}):
+${detail}`);
+      }
+      score = await spec.parseReport(dir);
+    } catch (err) {
+      failure = err;
+    }
+  } finally {
+    let judgeArtifacts;
+    if (spec.capture) {
+      try {
+        judgeArtifacts = await captureStagedRun(
+          dir,
+          spec.capture,
+          { stdout, stderr },
+          evaluatorSucceeded
+        );
+      } catch (captureError) {
+        failure = new Error(
+          `staged judge failed to capture evaluator artifacts: ${captureError instanceof Error ? captureError.message : captureError}`,
+          { cause: failure ?? captureError }
+        );
+      }
+    }
+    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {
+    });
+    if (failure) {
+      throw new StagedJudgeError(
+        failure instanceof Error ? failure.message : String(failure),
+        judgeArtifacts,
+        { cause: failure }
+      );
+    }
+    if (!score) throw new StagedJudgeError("staged judge completed without a score", judgeArtifacts);
+    return judgeArtifacts ? { ...score, judgeArtifacts } : score;
+  }
+}
+async function readJsonReport(path) {
+  let raw;
+  try {
+    raw = await readFile(path, "utf8");
+  } catch (err) {
+    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`);
+  }
+  try {
+    return JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`);
+  }
+}
+async function stageFile(path, content) {
+  await writeFile(path, content);
+}
+function safeRunId(prefix, id) {
+  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
+}
+
+export {
+  benchRoot,
+  resolveBenchPython,
+  venvPython,
+  venvPythonAt,
+  bigBuffer,
+  venvBin,
+  runVenvPython,
+  preflightVenvImports,
+  runVenvScriptStdin,
+  StagedJudgeError,
+  runStagedJudge,
+  readJsonReport,
+  stageFile,
+  safeRunId
+};
+//# sourceMappingURL=chunk-LRRD7NAG.js.map