@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-J6BU3NTM.js

@@ -1,251 +0,0 @@
-import {
-  preflightVenvImports,
-  readJsonReport,
-  runStagedJudge,
-  runVenvPython,
-  safeRunId,
-  stageFile
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/swe-bench.ts
-import { join } from "path";
-var SWE_REPO_DIR = "/work";
-var swePatchOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)];
-    const last = fences.at(-1)?.[1];
-    return (last ?? text).trim();
-  }
-};
-var DATASET = "princeton-nlp/SWE-bench_Verified";
-var SWE_CACHE_LEVELS = /* @__PURE__ */ new Set(["none", "base", "env", "instance"]);
-function scorerNamespace() {
-  const namespace = process.env.SWEBENCH_NAMESPACE ?? "swebench";
-  if (namespace !== "swebench" && namespace !== "none") {
-    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got "${namespace}"`);
-  }
-  return namespace;
-}
-var TEST_FILE_EXCLUDES = [
-  "':(exclude,glob)**/tests/**'",
-  "':(exclude,glob)**/test/**'",
-  "':(exclude,glob)test_*.py'",
-  "':(exclude,glob)**/test_*.py'",
-  "':(exclude,glob)*_test.py'",
-  "':(exclude,glob)**/*_test.py'",
-  "':(exclude,glob)conftest.py'",
-  "':(exclude,glob)**/conftest.py'"
-].join(" ");
-function stringIds(report, key) {
-  const value = report[key];
-  if (value === void 0) return [];
-  if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) {
-    throw new Error(`swe-bench: malformed ${key}`);
-  }
-  return value;
-}
-function scoreSweReport(taskId, value) {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    throw new Error("swe-bench: report must be an object");
-  }
-  const report = value;
-  const statusIds = {
-    resolved: stringIds(report, "resolved_ids"),
-    unresolved: stringIds(report, "unresolved_ids"),
-    emptyPatch: stringIds(report, "empty_patch_ids"),
-    completed: stringIds(report, "completed_ids"),
-    incomplete: stringIds(report, "incomplete_ids"),
-    error: stringIds(report, "error_ids")
-  };
-  const submitted = stringIds(report, "submitted_ids");
-  const mentioned = Object.values(statusIds).flat();
-  if (mentioned.some((id) => id !== taskId) || submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId)) {
-    throw new Error(`swe-bench: report identity mismatch for ${taskId}`);
-  }
-  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {
-    throw new Error(`swe-bench: evaluator failed for ${taskId}`);
-  }
-  const outcomes = [
-    statusIds.resolved.includes(taskId),
-    statusIds.unresolved.includes(taskId),
-    statusIds.emptyPatch.includes(taskId)
-  ];
-  if (outcomes.filter(Boolean).length !== 1) {
-    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`);
-  }
-  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {
-    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`);
-  }
-  const resolved = outcomes[0];
-  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) };
-}
-function sweEvaluationArgv(args) {
-  return [
-    "-m",
-    "swebench.harness.run_evaluation",
-    "--dataset_name",
-    DATASET,
-    "--predictions_path",
-    args.predictionsPath,
-    "--run_id",
-    args.runId,
-    "--instance_ids",
-    args.instanceId,
-    "--max_workers",
-    "1",
-    "--namespace",
-    args.namespace ?? scorerNamespace(),
-    "--cache_level",
-    args.cacheLevel
-  ];
-}
-function shellQuote(value) {
-  return `'${value.replace(/'/g, `'\\''`)}'`;
-}
-function sweMetadata(task) {
-  const repo = String(task.metadata?.repo ?? "");
-  const base = String(task.metadata?.base_commit ?? "");
-  if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) {
-    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`);
-  }
-  if (!/^[0-9a-f]{7,40}$/i.test(base)) {
-    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`);
-  }
-  return { repo, base };
-}
-function createSweBenchAdapter(options = {}) {
-  if (options.timeoutMs !== void 0 && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)) throw new Error("swe-bench: timeoutMs must be a positive integer");
-  const cacheLevel = options.cacheLevel ?? "env";
-  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error("swe-bench: invalid cacheLevel");
-  if (options.captureEvaluatorArtifacts !== void 0 && typeof options.captureEvaluatorArtifacts !== "function") throw new Error("swe-bench: captureEvaluatorArtifacts must be a function");
-  let attemptSequence = 0;
-  return {
-    name: "swe-bench-verified",
-    output: swePatchOutput,
-    // Extract the patch from repo STATE, not printed text: stage every edit the
-    // agent made in the cloned repo and diff it against the checked-out
-    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold
-    // `test_patch` itself, so an agent edit to a test would collide on apply. The
-    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold
-    // patch format the swebench judge's `git apply` expects.
-    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the
-    // agent only edits (the harness owns the checkout — a stochastic model can't be
-    // trusted to clone to an exact path). `--quiet` keeps the exec output small.
-    boxSetup(task) {
-      const { repo, base } = sweMetadata(task);
-      return {
-        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`
-      };
-    },
-    boxExtract() {
-      return {
-        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`
-      };
-    },
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["swebench"],
-        requireDocker: true,
-        fix: `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; (2) ensure the Docker daemon is running (the judge builds per-instance images).`
-      });
-    },
-    async loadTasks(opts = {}) {
-      const limit = opts.limit ?? 10;
-      const split = opts.split ?? "test";
-      const script = `
-import json, sys
-from datasets import load_dataset
-ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})
-ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
-out = []
-for r in ds:
-    if ids is not None and r["instance_id"] not in ids:
-        continue
-    out.append({
-        "instance_id": r["instance_id"], "repo": r["repo"], "base_commit": r["base_commit"],
-        "problem_statement": r["problem_statement"], "patch": r["patch"], "test_patch": r["test_patch"],
-        "FAIL_TO_PASS": r["FAIL_TO_PASS"], "PASS_TO_PASS": r["PASS_TO_PASS"],
-        "version": r.get("version"), "environment_setup_commit": r.get("environment_setup_commit"),
-    })
-    if ids is None and len(out) >= ${limit}:
-        break
-print(json.dumps(out))
-`;
-      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ""]);
-      const rows = JSON.parse(stdout);
-      return rows.map(
-        (r) => ({
-          id: String(r.instance_id),
-          split,
-          prompt: [
-            `Repository: ${r.repo} @ ${r.base_commit}`,
-            "",
-            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
-            "",
-            "Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files \u2014 the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.",
-            "Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff \u2014 the harness reads your committed edits directly from the repo.",
-            "",
-            "--- Issue ---",
-            String(r.problem_statement ?? "")
-          ].join("\n"),
-          metadata: r
-        })
-      );
-    },
-    async goldArtifact(task) {
-      const gold = task.metadata?.patch;
-      return typeof gold === "string" ? gold : void 0;
-    },
-    async judge(task, artifact) {
-      const runId = safeRunId("bench", task.id);
-      const capture = options.captureEvaluatorArtifacts?.({
-        taskId: task.id,
-        runId,
-        attemptSequence: ++attemptSequence
-      });
-      return runStagedJudge({
-        tmpPrefix: "swebench-",
-        ...options.timeoutMs === void 0 ? {} : { timeoutMs: options.timeoutMs },
-        ...capture === void 0 ? {} : { capture },
-        // Debug: retain the staged dir (holds swebench's per-instance apply/run
-        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.
-        ...process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {},
-        async stage(dir) {
-          await stageFile(
-            join(dir, "preds.json"),
-            JSON.stringify([
-              { instance_id: task.id, model_name_or_path: "agent-runtime-bench", model_patch: artifact }
-            ])
-          );
-        },
-        // The official evaluation harness. Pulls/builds the instance image, applies
-        // the patch, runs the test spec, writes a per-run report JSON in cwd.
-        argv: (dir) => sweEvaluationArgv({
-          predictionsPath: join(dir, "preds.json"),
-          runId,
-          instanceId: task.id,
-          cacheLevel,
-          namespace: scorerNamespace()
-        }),
-        async parseReport(dir) {
-          const report = await readJsonReport(join(dir, `agent-runtime-bench.${runId}.json`));
-          return scoreSweReport(task.id, report);
-        }
-      });
-    }
-  };
-}
-
-export {
-  swePatchOutput,
-  scoreSweReport,
-  sweEvaluationArgv,
-  createSweBenchAdapter
-};
-//# sourceMappingURL=chunk-J6BU3NTM.js.map

package/dist/chunk-J6BU3NTM.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/**\n * Fixed in-box path the agent clones the instance repo into. It is the SINGLE\n * source of truth shared by the prompt template (which tells the agent to clone\n * here) and `boxExtract` (which runs `git diff` here after the shot) — so the\n * harness always knows exactly where the agent's edits live, for any instance.\n */\nconst SWE_REPO_DIR = '/work'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n    '--namespace', args.namespace ?? scorerNamespace(),\n    '--cache_level', args.cacheLevel,\n  ]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgv({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;AAcA,SAAS,YAAY;AAmBrB,IAAM,eAAe;AAWd,IAAM,iBAAwC;AAAA,EACnD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AAGA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,uCAAuC,CAAC;AACzE,UAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC;AAC9B,YAAQ,QAAQ,MAAM,KAAK;AAAA,EAC7B;AACF;AAEA,IAAM,UAAU;AAsBhB,IAAM,mBAAmB,oBAAI,IAAwB,CAAC,QAAQ,QAAQ,OAAO,UAAU,CAAC;AAExF,SAAS,kBAAuC;AAC9C,QAAM,YAAY,QAAQ,IAAI,sBAAsB;AACpD,MAAI,cAAc,cAAc,cAAc,QAAQ;AACpD,UAAM,IAAI,MAAM,kDAAkD,SAAS,GAAG;AAAA,EAChF;AACA,SAAO;AACT;AACA,IAAM,qBAAqB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;AAClF,QAAM,QAAQ,OAAO,GAAG;AACxB,MAAI,UAAU,OAAW,QAAO,CAAC;AACjC,MAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,KAAK,CAAC,UAAU,OAAO,UAAU,QAAQ,GAAG;AAC7E,UAAM,IAAI,MAAM,wBAAwB,GAAG,EAAE;AAAA,EAC/C;AACA,SAAO;AACT;AAGO,SAAS,eAAe,QAAgB,OAA4B;AACzE,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AAC/D,UAAM,IAAI,MAAM,qCAAqC;AAAA,EACvD;AACA,QAAM,SAAS;AACf,QAAM,YAAY;AAAA,IAChB,UAAU,UAAU,QAAQ,cAAc;AAAA,IAC1C,YAAY,UAAU,QAAQ,gBAAgB;AAAA,IAC9C,YAAY,UAAU,QAAQ,iBAAiB;AAAA,IAC/C,WAAW,UAAU,QAAQ,eAAe;AAAA,IAC5C,YAAY,UAAU,QAAQ,gBAAgB;AAAA,IAC9C,OAAO,UAAU,QAAQ,WAAW;AAAA,EACtC;AACA,QAAM,YAAY,UAAU,QAAQ,eAAe;AACnD,QAAM,YAAY,OAAO,OAAO,SAAS,EAAE,KAAK;AAChD,MACE,UAAU,KAAK,CAAC,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,CAAC,MAAM,SACxE;AACA,UAAM,IAAI,MAAM,2CAA2C,MAAM,EAAE;AAAA,EACrE;AACA,MAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAAG;AAC7E,UAAM,IAAI,MAAM,mCAAmC,MAAM,EAAE;AAAA,EAC7D;AACA,QAAM,WAAW;AAAA,IACf,UAAU,SAAS,SAAS,MAAM;AAAA,IAClC,UAAU,WAAW,SAAS,MAAM;AAAA,IACpC,UAAU,WAAW,SAAS,MAAM;AAAA,EACtC;AACA,MAAI,SAAS,OAAO,OAAO,EAAE,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,+CAA+C,MAAM,EAAE;AAAA,EACzE;AACA,OAAK,SAAS,CAAC,KAAK,SAAS,CAAC,MAAM,CAAC,UAAU,UAAU,SAAS,MAAM,GAAG;AACzE,UAAM,IAAI,MAAM,sDAAsD,MAAM,EAAE;AAAA,EAChF;AACA,QAAM,WAAW,SAAS,CAAC;AAC3B,SAAO,EAAE,UAAU,OAAO,WAAW,IAAI,GAAG,QAAQ,KAAK,UAAU,MAAM,EAAE;AAC7E;AAEO,SAAS,kBAAkB,MAMrB;AACX,SAAO;AAAA,IACL;AAAA,IAAM;AAAA,IACN;AAAA,IAAkB;AAAA,IAClB;AAAA,IAAsB,KAAK;AAAA,IAC3B;AAAA,IAAY,KAAK;AAAA,IACjB;AAAA,IAAkB,KAAK;AAAA,IACvB;AAAA,IAAiB;AAAA,IACjB;AAAA,IAAe,KAAK,aAAa,gBAAgB;AAAA,IACjD;AAAA,IAAiB,KAAK;AAAA,EACxB;AACF;AAEA,SAAS,WAAW,OAAuB;AACzC,SAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,CAAC;AACzC;AAEA,SAAS,YAAY,MAAiD;AACpE,QAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;AAC7C,QAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;AACpD,MAAI,CAAC,qCAAqC,KAAK,IAAI,GAAG;AACpD,UAAM,IAAI,MAAM,wCAAwC,KAAK,EAAE,KAAK,IAAI,EAAE;AAAA,EAC5E;AACA,MAAI,CAAC,oBAAoB,KAAK,IAAI,GAAG;AACnC,UAAM,IAAI,MAAM,+CAA+C,KAAK,EAAE,KAAK,IAAI,EAAE;AAAA,EACnF;AACA,SAAO,EAAE,MAAM,KAAK;AACtB;AAEO,SAAS,sBAAsB,UAAkC,CAAC,GAAqB;AAC5F,MACE,QAAQ,cAAc,WAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,GACrE,OAAM,IAAI,MAAM,iDAAiD;AACnE,QAAM,aAAa,QAAQ,cAAc;AACzC,MAAI,CAAC,iBAAiB,IAAI,UAAU,EAAG,OAAM,IAAI,MAAM,+BAA+B;AACtF,MACE,QAAQ,8BAA8B,UACnC,OAAO,QAAQ,8BAA8B,WAChD,OAAM,IAAI,MAAM,yDAAyD;AAC3E,MAAI,kBAAkB;AACtB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAWR,SAAS,MAAM;AACb,YAAM,EAAE,MAAM,KAAK,IAAI,YAAY,IAAI;AACvC,aAAO;AAAA,QACL,SAAS,UAAU,WAAW,YAAY,CAAC,yBAAyB,WAAW,sBAAsB,IAAI,EAAE,CAAC,IAAI,WAAW,YAAY,CAAC,cAAc,WAAW,YAAY,CAAC,qBAAqB,WAAW,IAAI,CAAC;AAAA,MACrN;AAAA,IACF;AAAA,IACA,aAAa;AACX,aAAO;AAAA,QACL,SAAS,UAAU,WAAW,YAAY,CAAC,qBAAqB,WAAW,YAAY,CAAC,uBAAuB,kBAAkB;AAAA,MACnI;AAAA,IACF;AAAA,IAEA,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,UAAU;AAAA,QACpB,eAAe;AAAA,QACf,KACE;AAAA,MAEJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,QAAQ,KAAK,SAAS;AAE5B,YAAM,SAAS;AAAA;AAAA;AAAA,oBAGD,KAAK,UAAU,OAAO,CAAC,WAAW,KAAK,UAAU,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qCAYtC,KAAK;AAAA;AAAA;AAAA;AAIpC,YAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;AACrF,YAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,aAAO,KAAK;AAAA,QACV,CAAC,OAAkB;AAAA,UACjB,IAAI,OAAO,EAAE,WAAW;AAAA,UACxB;AAAA,UACA,QAAQ;AAAA,YACN,eAAe,EAAE,IAAI,MAAM,EAAE,WAAW;AAAA,YACxC;AAAA,YACA,uCAAuC,YAAY,2BAA2B,EAAE,WAAW,+BAA+B,YAAY;AAAA,YACtI;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,EAAE,qBAAqB,EAAE;AAAA,UAClC,EAAE,KAAK,IAAI;AAAA,UACX,UAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,KAAK,UAAU;AAC5B,aAAO,OAAO,SAAS,WAAW,OAAO;AAAA,IAC3C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;AACxC,YAAM,UAAU,QAAQ,4BAA4B;AAAA,QAClD,QAAQ,KAAK;AAAA,QACb;AAAA,QACA,iBAAiB,EAAE;AAAA,MACrB,CAAC;AACD,aAAO,eAAe;AAAA,QACpB,WAAW;AAAA,QACX,GAAI,QAAQ,cAAc,SAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;AAAA,QAC1E,GAAI,YAAY,SAAY,CAAC,IAAI,EAAE,QAAQ;AAAA;AAAA;AAAA,QAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;AAAA,QACzD,MAAM,MAAM,KAAK;AACf,gBAAM;AAAA,YACJ,KAAK,KAAK,YAAY;AAAA,YACtB,KAAK,UAAU;AAAA,cACb,EAAE,aAAa,KAAK,IAAI,oBAAoB,uBAAuB,aAAa,SAAS;AAAA,YAC3F,CAAC;AAAA,UACH;AAAA,QACF;AAAA;AAAA;AAAA,QAGA,MAAM,CAAC,QAAQ,kBAAkB;AAAA,UAC/B,iBAAiB,KAAK,KAAK,YAAY;AAAA,UACvC;AAAA,UACA,YAAY,KAAK;AAAA,UACjB;AAAA,UACA,WAAW,gBAAgB;AAAA,QAC7B,CAAC;AAAA,QACD,MAAM,YAAY,KAAK;AAErB,gBAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,KAAK,OAAO,CAAC;AAC7F,iBAAO,eAAe,KAAK,IAAI,MAAM;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-JSQOUKXS.js

@@ -1,149 +0,0 @@
-import {
-  benchRoot,
-  preflightVenvImports,
-  readJsonReport,
-  runStagedJudge,
-  runVenvPython,
-  safeRunId,
-  stageFile,
-  venvBinAt,
-  venvPythonAt
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/terminal-bench.ts
-import { join } from "path";
-var terminalBenchVenvDir = () => process.env.TERMINAL_BENCH_VENV ?? ".venv-terminal-bench";
-var terminalBenchPython = () => venvPythonAt(terminalBenchVenvDir());
-var terminalBenchBin = () => venvBinAt(terminalBenchVenvDir(), "tb");
-var DATASET = "terminal-bench-core";
-var DATASET_VERSION = "0.1.1";
-var DATASET_REF = `${DATASET}==${DATASET_VERSION}`;
-var FIXTURE_IDS = ["hello-world"];
-var SCRIPT_AGENT = "tb_agents.script_agent:ScriptAgent";
-async function loadRows(opts) {
-  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS);
-  const limit = opts.limit ?? null;
-  const script = `
-import json, sys
-from pathlib import Path
-from terminal_bench.dataset.dataset import Dataset
-from terminal_bench.handlers.trial_handler import TaskPaths
-
-req_ids = json.loads(sys.argv[1]) if sys.argv[1] else None
-limit = json.loads(sys.argv[2]) if sys.argv[2] else None
-
-ds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)
-
-import yaml
-out = []
-for task_dir in ds:
-    tp = TaskPaths(task_dir)
-    cfg = yaml.safe_load(tp.task_config_path.read_text())
-    try:
-        sol = tp.solution_path.read_text()
-    except FileNotFoundError:
-        sol = None
-    out.append({
-        "id": task_dir.name,
-        "instruction": cfg["instruction"],
-        "task_dir": str(task_dir),
-        "solution": sol if (tp.solution_path.suffix == ".sh" if sol is not None else False) else None,
-    })
-print(json.dumps(out))
-`;
-  const stdout = await runVenvPython(
-    script,
-    [ids ? JSON.stringify(ids) : "", limit !== null ? String(limit) : ""],
-    0,
-    terminalBenchPython()
-  );
-  return JSON.parse(stdout);
-}
-function createTerminalBenchAdapter() {
-  return {
-    name: "terminal-bench",
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["terminal_bench"],
-        requireDocker: true,
-        python: terminalBenchPython(),
-        fix: `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench (an ISOLATED venv \u2014 Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; override the dir with TERMINAL_BENCH_VENV) ; (2) ensure the Docker daemon is running (the judge builds per-task images on first run). The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`
-      });
-    },
-    async loadTasks(opts = {}) {
-      const rows = await loadRows(opts);
-      if (rows.length === 0) {
-        throw new Error(
-          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} (dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`
-        );
-      }
-      return rows.map(
-        (r) => ({
-          id: r.id,
-          split: DATASET_VERSION,
-          prompt: r.instruction,
-          metadata: {
-            dataset: DATASET,
-            datasetVersion: DATASET_VERSION,
-            datasetRef: DATASET_REF,
-            taskDir: r.task_dir,
-            solution: r.solution,
-            instruction: r.instruction
-          }
-        })
-      );
-    },
-    async goldArtifact(task) {
-      const sol = task.metadata?.solution;
-      return typeof sol === "string" ? sol : void 0;
-    },
-    async judge(task, artifact) {
-      const runId = safeRunId("bench", `${task.id}-${Date.now()}`);
-      return runStagedJudge({
-        tmpPrefix: "tbench-",
-        bin: terminalBenchBin(),
-        cwd: () => benchRoot,
-        async stage(dir) {
-          await stageFile(join(dir, "attempt.sh"), artifact);
-        },
-        // The harness builds a fresh task container, runs ScriptAgent (which replays
-        // the artifact script), then runs the task's verifier. --no-livestream keeps
-        // stdout sane; --cleanup removes the per-run images.
-        argv: (dir) => [
-          "run",
-          "-d",
-          DATASET_REF,
-          "-t",
-          task.id,
-          "--agent-import-path",
-          SCRIPT_AGENT,
-          "--agent-kwarg",
-          `script_path=${join(dir, "attempt.sh")}`,
-          "--output-path",
-          join(dir, "runs"),
-          "--run-id",
-          runId,
-          "--n-concurrent",
-          "1",
-          "--no-livestream",
-          "--cleanup"
-        ],
-        async parseReport(dir) {
-          const report = await readJsonReport(join(dir, "runs", runId, "results.json"));
-          const resolved = (report.resolved_ids ?? []).includes(task.id);
-          const trial = report.results?.find((r) => r.task_id === task.id);
-          return {
-            resolved,
-            score: resolved ? 1 : 0,
-            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {})
-          };
-        }
-      });
-    }
-  };
-}
-
-export {
-  createTerminalBenchAdapter
-};
-//# sourceMappingURL=chunk-JSQOUKXS.js.map

package/dist/chunk-JSQOUKXS.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/terminal-bench.ts"],"sourcesContent":["/**\n * Terminal-Bench adapter. Each task = a Docker environment + an English\n * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact\n * is the shell script the agent ran to attempt the task; the judge REPLAYS that\n * script in a fresh task container via the Terminal-Bench harness (`tb run` with\n * our ScriptAgent), then the task's own verifier scores the resulting state.\n * Fully deterministic — no LLM judge, no self-authored score.\n *\n * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`\n * installed + a running Docker daemon (per-task images are built on first run).\n * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the\n * Terminal-Bench registry on first run.\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay\n * argv, and the results.json shape.\n */\n\nimport { join } from 'node:path'\nimport {\n  benchRoot,\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  venvBinAt,\n  venvPythonAt,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n// Terminal-Bench imports LiteLLM/Pydantic-2 APIs, while AppWorld pins Pydantic 1.\n// Keep it out of the shared bench .venv. Resolved at call-time so tests/runs can\n// override the env without reloading this module.\nconst terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench'\nconst terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir())\nconst terminalBenchBin = (): string => venvBinAt(terminalBenchVenvDir(), 'tb')\n\n// Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the\n// installed CLI) and is the published launch task set. name==version is what `tb\n// run -d` and `tb datasets download -d` both accept.\nconst DATASET = 'terminal-bench-core'\nconst DATASET_VERSION = '0.1.1'\nconst DATASET_REF = `${DATASET}==${DATASET_VERSION}`\n\n// Bundled fixture: when no ids/limit are given, load these. hello-world is the\n// fastest deterministic task (prebuilt python image, file-write verifier) so the\n// adapter is runnable without a large pull.\nconst FIXTURE_IDS = ['hello-world']\n\n// Import path the harness uses to load our replay agent (cwd = benchRoot).\nconst SCRIPT_AGENT = 'tb_agents.script_agent:ScriptAgent'\n\ninterface TbTaskRow {\n  id: string\n  instruction: string\n  task_dir: string\n  solution: string | null\n}\n\ninterface TbReport {\n  resolved_ids?: string[]\n  results?: Array<{ task_id: string; is_resolved: boolean | null; parser_results?: unknown }>\n}\n\n/** Enumerate dataset tasks via tb's Dataset loader (caches from the registry on\n *  first run). Reads instruction from each task.yaml and the gold solution. */\nasync function loadRows(opts: LoadOptions): Promise<TbTaskRow[]> {\n  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS)\n  const limit = opts.limit ?? null\n  const script = `\nimport json, sys\nfrom pathlib import Path\nfrom terminal_bench.dataset.dataset import Dataset\nfrom terminal_bench.handlers.trial_handler import TaskPaths\n\nreq_ids = json.loads(sys.argv[1]) if sys.argv[1] else None\nlimit = json.loads(sys.argv[2]) if sys.argv[2] else None\n\nds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)\n\nimport yaml\nout = []\nfor task_dir in ds:\n    tp = TaskPaths(task_dir)\n    cfg = yaml.safe_load(tp.task_config_path.read_text())\n    try:\n        sol = tp.solution_path.read_text()\n    except FileNotFoundError:\n        sol = None\n    out.append({\n        \"id\": task_dir.name,\n        \"instruction\": cfg[\"instruction\"],\n        \"task_dir\": str(task_dir),\n        \"solution\": sol if (tp.solution_path.suffix == \".sh\" if sol is not None else False) else None,\n    })\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(\n    script,\n    [ids ? JSON.stringify(ids) : '', limit !== null ? String(limit) : ''],\n    0,\n    terminalBenchPython(),\n  )\n  return JSON.parse(stdout) as TbTaskRow[]\n}\n\nexport function createTerminalBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'terminal-bench',\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['terminal_bench'],\n        requireDocker: true,\n        python: terminalBenchPython(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && ` +\n          `bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench ` +\n          `(an ISOLATED venv — Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; ` +\n          `override the dir with TERMINAL_BENCH_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-task images on first run). ` +\n          `The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const rows = await loadRows(opts)\n      if (rows.length === 0) {\n        throw new Error(\n          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} ` +\n            `(dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`,\n        )\n      }\n      return rows.map(\n        (r): BenchTask => ({\n          id: r.id,\n          split: DATASET_VERSION,\n          prompt: r.instruction,\n          metadata: {\n            dataset: DATASET,\n            datasetVersion: DATASET_VERSION,\n            datasetRef: DATASET_REF,\n            taskDir: r.task_dir,\n            solution: r.solution,\n            instruction: r.instruction,\n          },\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the task's solution.sh (the oracle script). solution.yaml tasks have\n      // no shell-script artifact form here, so they return undefined (cannot be\n      // verify-judged via the script-replay seam — use a .sh-solution task).\n      const sol = task.metadata?.solution\n      return typeof sol === 'string' ? sol : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', `${task.id}-${Date.now()}`)\n      return runStagedJudge({\n        tmpPrefix: 'tbench-',\n        bin: terminalBenchBin(),\n        cwd: () => benchRoot,\n        async stage(dir) {\n          await stageFile(join(dir, 'attempt.sh'), artifact)\n        },\n        // The harness builds a fresh task container, runs ScriptAgent (which replays\n        // the artifact script), then runs the task's verifier. --no-livestream keeps\n        // stdout sane; --cleanup removes the per-run images.\n        argv: (dir) => [\n          'run',\n          '-d', DATASET_REF,\n          '-t', task.id,\n          '--agent-import-path', SCRIPT_AGENT,\n          '--agent-kwarg', `script_path=${join(dir, 'attempt.sh')}`,\n          '--output-path', join(dir, 'runs'),\n          '--run-id', runId,\n          '--n-concurrent', '1',\n          '--no-livestream',\n          '--cleanup',\n        ],\n        async parseReport(dir) {\n          const report = await readJsonReport<TbReport>(join(dir, 'runs', runId, 'results.json'))\n          const resolved = (report.resolved_ids ?? []).includes(task.id)\n          const trial = report.results?.find((r) => r.task_id === task.id)\n          return {\n            resolved,\n            score: resolved ? 1 : 0,\n            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {}),\n          }\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAkBA,SAAS,YAAY;AAiBrB,IAAM,uBAAuB,MAAc,QAAQ,IAAI,uBAAuB;AAC9E,IAAM,sBAAsB,MAAc,aAAa,qBAAqB,CAAC;AAC7E,IAAM,mBAAmB,MAAc,UAAU,qBAAqB,GAAG,IAAI;AAK7E,IAAM,UAAU;AAChB,IAAM,kBAAkB;AACxB,IAAM,cAAc,GAAG,OAAO,KAAK,eAAe;AAKlD,IAAM,cAAc,CAAC,aAAa;AAGlC,IAAM,eAAe;AAgBrB,eAAe,SAAS,MAAyC;AAC/D,QAAM,MAAM,KAAK,QAAQ,KAAK,QAAQ,OAAO;AAC7C,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oBASG,KAAK,UAAU,OAAO,CAAC,aAAa,KAAK,UAAU,eAAe,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBrF,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA,CAAC,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,UAAU,OAAO,OAAO,KAAK,IAAI,EAAE;AAAA,IACpE;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO,KAAK,MAAM,MAAM;AAC1B;AAEO,SAAS,6BAA+C;AAC7D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,gBAAgB;AAAA,QAC1B,eAAe;AAAA,QACf,QAAQ,oBAAoB;AAAA,QAC5B,KACE,kCAAkC,qBAAqB,CAAC,aAC/C,qBAAqB,CAAC,gRAIxB,WAAW;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,OAAO,MAAM,SAAS,IAAI;AAChC,UAAI,KAAK,WAAW,GAAG;AACrB,cAAM,IAAI;AAAA,UACR,kDAAkD,KAAK,UAAU,IAAI,CAAC,aACxD,WAAW;AAAA,QAC3B;AAAA,MACF;AACA,aAAO,KAAK;AAAA,QACV,CAAC,OAAkB;AAAA,UACjB,IAAI,EAAE;AAAA,UACN,OAAO;AAAA,UACP,QAAQ,EAAE;AAAA,UACV,UAAU;AAAA,YACR,SAAS;AAAA,YACT,gBAAgB;AAAA,YAChB,YAAY;AAAA,YACZ,SAAS,EAAE;AAAA,YACX,UAAU,EAAE;AAAA,YACZ,aAAa,EAAE;AAAA,UACjB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,MAAM,KAAK,UAAU;AAC3B,aAAO,OAAO,QAAQ,WAAW,MAAM;AAAA,IACzC;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,QAAQ,UAAU,SAAS,GAAG,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AAC3D,aAAO,eAAe;AAAA,QACpB,WAAW;AAAA,QACX,KAAK,iBAAiB;AAAA,QACtB,KAAK,MAAM;AAAA,QACX,MAAM,MAAM,KAAK;AACf,gBAAM,UAAU,KAAK,KAAK,YAAY,GAAG,QAAQ;AAAA,QACnD;AAAA;AAAA;AAAA;AAAA,QAIA,MAAM,CAAC,QAAQ;AAAA,UACb;AAAA,UACA;AAAA,UAAM;AAAA,UACN;AAAA,UAAM,KAAK;AAAA,UACX;AAAA,UAAuB;AAAA,UACvB;AAAA,UAAiB,eAAe,KAAK,KAAK,YAAY,CAAC;AAAA,UACvD;AAAA,UAAiB,KAAK,KAAK,MAAM;AAAA,UACjC;AAAA,UAAY;AAAA,UACZ;AAAA,UAAkB;AAAA,UAClB;AAAA,UACA;AAAA,QACF;AAAA,QACA,MAAM,YAAY,KAAK;AACrB,gBAAM,SAAS,MAAM,eAAyB,KAAK,KAAK,QAAQ,OAAO,cAAc,CAAC;AACtF,gBAAM,YAAY,OAAO,gBAAgB,CAAC,GAAG,SAAS,KAAK,EAAE;AAC7D,gBAAM,QAAQ,OAAO,SAAS,KAAK,CAAC,MAAM,EAAE,YAAY,KAAK,EAAE;AAC/D,iBAAO;AAAA,YACL;AAAA,YACA,OAAO,WAAW,IAAI;AAAA,YACtB,QAAQ,KAAK,UAAU,OAAO,kBAAkB,OAAO,gBAAgB,CAAC,CAAC;AAAA,UAC3E;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-JTHWEDEW.js

@@ -1,32 +0,0 @@
-// src/refine-loop.ts
-var defaultDecide = (history) => history[history.length - 1]?.verdict?.valid === true;
-async function runRefineLoop(spec) {
-  const rounds = Math.max(1, spec.rounds);
-  const decide = spec.decide ?? defaultDecide;
-  let ctxSet = false;
-  let ctx = void 0;
-  try {
-    if (spec.setup) {
-      ctx = await spec.setup();
-      ctxSet = true;
-    }
-    const history = [];
-    for (let r = 1; r <= rounds; r += 1) {
-      const prompt = spec.prompt(r, history, ctx);
-      const { artifact, note } = await spec.runShot(prompt, r, ctx);
-      const verdict = spec.judge ? await spec.judge(artifact, r) : void 0;
-      history.push({ round: r, prompt, artifact, verdict, note });
-      if (decide(history)) break;
-    }
-    const blind = history[0];
-    const final = history[history.length - 1];
-    return { rounds: history, blind, final, resolved: history.some((h) => h.verdict?.valid === true) };
-  } finally {
-    if (ctxSet && spec.teardown) await spec.teardown(ctx);
-  }
-}
-
-export {
-  runRefineLoop
-};
-//# sourceMappingURL=chunk-JTHWEDEW.js.map

package/dist/chunk-JTHWEDEW.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/refine-loop.ts"],"sourcesContent":["/**\n * runRefineLoop — the ONE shared k-shot loop (see docs/architecture.md §1, §12).\n *\n * It replaces the seven hand-rolled `for (round 1..k) { shot → judge → decide →\n * carry-forward }` copies across the workers (~700 LOC of duplicated skeleton).\n * The worker is an OPAQUE `runShot`; this loop owns iteration, carry-forward\n * steering, optional per-round judging + early stop, and round bookkeeping.\n *\n * Two carry-forward channels, both first-class (the extraction found both in the\n * wild and they must stay pluggable):\n *   - EXECUTION CONTEXT (`Ctx`): created once by `setup`, threaded to every shot,\n *     torn down by `teardown`. This is how filesystem/session state carries — a\n *     cloned repo whose edits persist (SWE refine), a shared sandbox box whose\n *     session persists (sandbox research), or a scratch dir.\n *   - PROMPT (`prompt(round, history, ctx)`): how textual state carries — round 1\n *     is the blind prompt; rounds 2+ fold prior rounds' artifacts + a directive.\n *\n * Judging is OPTIONAL: workers that run all k rounds and let the orchestrator\n * judge omit `judge`/`decide` (the loop runs to budget); a loop that should stop\n * on the first valid answer wires `judge` (default `decide` = stop-on-valid).\n *\n * This is the inference-timescale instance of the spine's atom: `runShot` is the\n * worker Agent's `act→Output`; `prompt` is the driver Agent's `act→steer`. It is\n * deliberately corpus-agnostic — callers map `RefineLoopResult` to a RunRecord.\n */\n\n/** Minimal per-round verdict — `valid` gates early-stop, `score` is informational. */\nexport interface RoundVerdict {\n  valid: boolean\n  score?: number\n}\n\nexport interface RoundRecord<Artifact> {\n  /** 1-based round index. */\n  round: number\n  prompt: string\n  artifact: Artifact\n  /** Present only when a `judge` is wired. */\n  verdict?: RoundVerdict\n  /** Non-fatal note (e.g. a liveness backstop fired this round). */\n  note?: string\n}\n\nexport interface RefineLoopSpec<Artifact, Ctx = void> {\n  /** Max shots. Always ≥ 1. */\n  rounds: number\n  /** Build the per-task execution context once (clone repo / create box / mkdtemp). */\n  setup?: () => Promise<Ctx>\n  /** Round `r`'s prompt. `r === 1` is the blind prompt; `r > 1` carries `history`. */\n  prompt: (round: number, history: ReadonlyArray<RoundRecord<Artifact>>, ctx: Ctx) => string\n  /** Run ONE shot — the opaque worker (local spawn / sandbox stream / router chat). */\n  runShot: (\n    prompt: string,\n    round: number,\n    ctx: Ctx,\n  ) => Promise<{ artifact: Artifact; note?: string }>\n  /** Optional per-round judge. Omit to run all `rounds` (orchestrator judges later). */\n  judge?: (artifact: Artifact, round: number) => Promise<RoundVerdict>\n  /** Stop after this round? Default: stop once a round's verdict is valid (else run to budget). */\n  decide?: (history: ReadonlyArray<RoundRecord<Artifact>>) => boolean\n  /** Always runs (even on throw) when `setup` ran. */\n  teardown?: (ctx: Ctx) => Promise<void>\n}\n\nexport interface RefineLoopResult<Artifact> {\n  /** Every executed round in order. */\n  rounds: RoundRecord<Artifact>[]\n  /** Round 1 — the blind artifact. */\n  blind: RoundRecord<Artifact>\n  /** The last executed round. */\n  final: RoundRecord<Artifact>\n  /** True iff any round was judged valid (only meaningful when `judge` is wired). */\n  resolved: boolean\n}\n\nconst defaultDecide = <Artifact>(history: ReadonlyArray<RoundRecord<Artifact>>): boolean =>\n  history[history.length - 1]?.verdict?.valid === true\n\nexport async function runRefineLoop<Artifact, Ctx = void>(\n  spec: RefineLoopSpec<Artifact, Ctx>,\n): Promise<RefineLoopResult<Artifact>> {\n  const rounds = Math.max(1, spec.rounds)\n  const decide = spec.decide ?? defaultDecide\n  let ctxSet = false\n  let ctx = undefined as Ctx\n  try {\n    if (spec.setup) {\n      ctx = await spec.setup()\n      ctxSet = true\n    }\n    const history: RoundRecord<Artifact>[] = []\n    for (let r = 1; r <= rounds; r += 1) {\n      const prompt = spec.prompt(r, history, ctx)\n      const { artifact, note } = await spec.runShot(prompt, r, ctx)\n      const verdict = spec.judge ? await spec.judge(artifact, r) : undefined\n      history.push({ round: r, prompt, artifact, verdict, note })\n      if (decide(history)) break\n    }\n    // The loop runs ≥ 1 round, so history is non-empty.\n    const blind = history[0] as RoundRecord<Artifact>\n    const final = history[history.length - 1] as RoundRecord<Artifact>\n    return { rounds: history, blind, final, resolved: history.some((h) => h.verdict?.valid === true) }\n  } finally {\n    if (ctxSet && spec.teardown) await spec.teardown(ctx)\n  }\n}\n"],"mappings":";AA2EA,IAAM,gBAAgB,CAAW,YAC/B,QAAQ,QAAQ,SAAS,CAAC,GAAG,SAAS,UAAU;AAElD,eAAsB,cACpB,MACqC;AACrC,QAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM;AACtC,QAAM,SAAS,KAAK,UAAU;AAC9B,MAAI,SAAS;AACb,MAAI,MAAM;AACV,MAAI;AACF,QAAI,KAAK,OAAO;AACd,YAAM,MAAM,KAAK,MAAM;AACvB,eAAS;AAAA,IACX;AACA,UAAM,UAAmC,CAAC;AAC1C,aAAS,IAAI,GAAG,KAAK,QAAQ,KAAK,GAAG;AACnC,YAAM,SAAS,KAAK,OAAO,GAAG,SAAS,GAAG;AAC1C,YAAM,EAAE,UAAU,KAAK,IAAI,MAAM,KAAK,QAAQ,QAAQ,GAAG,GAAG;AAC5D,YAAM,UAAU,KAAK,QAAQ,MAAM,KAAK,MAAM,UAAU,CAAC,IAAI;AAC7D,cAAQ,KAAK,EAAE,OAAO,GAAG,QAAQ,UAAU,SAAS,KAAK,CAAC;AAC1D,UAAI,OAAO,OAAO,EAAG;AAAA,IACvB;AAEA,UAAM,QAAQ,QAAQ,CAAC;AACvB,UAAM,QAAQ,QAAQ,QAAQ,SAAS,CAAC;AACxC,WAAO,EAAE,QAAQ,SAAS,OAAO,OAAO,UAAU,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,UAAU,IAAI,EAAE;AAAA,EACnG,UAAE;AACA,QAAI,UAAU,KAAK,SAAU,OAAM,KAAK,SAAS,GAAG;AAAA,EACtD;AACF;","names":[]}

package/dist/chunk-NRMGT25X.js

@@ -1,116 +0,0 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/crag.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "crag.json");
-var dataFile = () => process.env.CRAG_DATA_FILE;
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`CRAG row ${index} must be an object`);
-  const query = firstString(raw, ["query", "question", "prompt"]);
-  const goldAnswers = allStrings(raw, ["answer", "answers", "gold", "gold_answer", "expected_answer"]);
-  if (!query) throw new Error(`CRAG row ${index} missing query`);
-  if (goldAnswers.length === 0) throw new Error(`CRAG row ${index} missing gold answer`);
-  const domain = stringFrom(raw.domain) ?? "unknown";
-  const questionType = stringFrom(raw.question_type) ?? stringFrom(raw.questionType) ?? "unknown";
-  const dynamism = stringFrom(raw.static_or_dynamic) ?? stringFrom(raw.dynamism) ?? "unknown";
-  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `crag-${index}`;
-  const meta = {
-    benchmark: "crag",
-    query,
-    goldAnswers,
-    domain,
-    questionType,
-    dynamism
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? domain,
-    prompt: [
-      "Answer this CRAG factual question.",
-      "Return a concise answer and do not guess when the evidence is insufficient.",
-      "End with a single final line: `FINAL ANSWER: <answer>`.",
-      "",
-      `Question: ${query}`,
-      `Domain: ${domain}`,
-      `Question type: ${questionType}`,
-      `Dynamism: ${dynamism}`
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldAnswers)) {
-    throw new Error(`CRAG task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`CRAG: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[crag] CRAG_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "CRAG");
-}
-function createCragAdapter() {
-  const fixturesMode = process.env.CRAG_FIXTURES === "1";
-  return {
-    name: "crag",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "CRAG_DATA_FILE is required. Fix: export facebookresearch/CRAG rows to JSONL and set CRAG_DATA_FILE=/path/to/crag.jsonl, or set CRAG_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("CRAG_DATA_FILE is required to load CRAG tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "CRAG");
-    },
-    async goldArtifact(task) {
-      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const score = scoreAnswerArtifact(artifact, meta.goldAnswers);
-      return answerScoreToBenchScore(score, {
-        benchmark: meta.benchmark,
-        domain: meta.domain,
-        questionType: meta.questionType,
-        dynamism: meta.dynamism
-      });
-    }
-  };
-}
-
-export {
-  createCragAdapter
-};
-//# sourceMappingURL=chunk-NRMGT25X.js.map

package/dist/chunk-NRMGT25X.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/crag.ts"],"sourcesContent":["/**\n * CRAG adapter (Comprehensive RAG Benchmark).\n *\n * Live mode expects an official or compatible CRAG JSON/JSONL export. The\n * adapter preserves CRAG domain/type/dynamism tags in metadata and scores final\n * answers deterministically against the provided gold answer list.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'crag.json')\n\ninterface CragMeta {\n  benchmark: 'crag'\n  query: string\n  goldAnswers: string[]\n  domain: string\n  questionType: string\n  dynamism: string\n}\n\nconst dataFile = (): string | undefined => process.env.CRAG_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`CRAG row ${index} must be an object`)\n  const query = firstString(raw, ['query', 'question', 'prompt'])\n  const goldAnswers = allStrings(raw, ['answer', 'answers', 'gold', 'gold_answer', 'expected_answer'])\n  if (!query) throw new Error(`CRAG row ${index} missing query`)\n  if (goldAnswers.length === 0) throw new Error(`CRAG row ${index} missing gold answer`)\n  const domain = stringFrom(raw.domain) ?? 'unknown'\n  const questionType = stringFrom(raw.question_type) ?? stringFrom(raw.questionType) ?? 'unknown'\n  const dynamism = stringFrom(raw.static_or_dynamic) ?? stringFrom(raw.dynamism) ?? 'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `crag-${index}`\n  const meta: CragMeta = {\n    benchmark: 'crag',\n    query,\n    goldAnswers,\n    domain,\n    questionType,\n    dynamism,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? domain,\n    prompt: [\n      'Answer this CRAG factual question.',\n      'Return a concise answer and do not guess when the evidence is insufficient.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Domain: ${domain}`,\n      `Question type: ${questionType}`,\n      `Dynamism: ${dynamism}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): CragMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`CRAG task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as CragMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`CRAG: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[crag] CRAG_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'CRAG')\n}\n\nexport function createCragAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.CRAG_FIXTURES === '1'\n\n  return {\n    name: 'crag',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'CRAG_DATA_FILE is required. Fix: export facebookresearch/CRAG rows to JSONL and set CRAG_DATA_FILE=/path/to/crag.jsonl, or set CRAG_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('CRAG_DATA_FILE is required to load CRAG tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'CRAG')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        domain: meta.domain,\n        questionType: meta.questionType,\n        dynamism: meta.dynamism,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAQA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAgBrB,IAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AAWxD,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,YAAY,KAAK,oBAAoB;AACzE,QAAM,QAAQ,YAAY,KAAK,CAAC,SAAS,YAAY,QAAQ,CAAC;AAC9D,QAAM,cAAc,WAAW,KAAK,CAAC,UAAU,WAAW,QAAQ,eAAe,iBAAiB,CAAC;AACnG,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,YAAY,KAAK,gBAAgB;AAC7D,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,YAAY,KAAK,sBAAsB;AACrF,QAAM,SAAS,WAAW,IAAI,MAAM,KAAK;AACzC,QAAM,eAAe,WAAW,IAAI,aAAa,KAAK,WAAW,IAAI,YAAY,KAAK;AACtF,QAAM,WAAW,WAAW,IAAI,iBAAiB,KAAK,WAAW,IAAI,QAAQ,KAAK;AAClF,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,QAAQ,KAAK;AAC1E,QAAM,OAAiB;AAAA,IACrB,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,WAAW,MAAM;AAAA,MACjB,kBAAkB,YAAY;AAAA,MAC9B,aAAa,QAAQ;AAAA,IACvB,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA2B;AAC3C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,aAAa,KAAK,EAAE,wDAAmD;AAAA,EACzF;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,oBAAoB,IAAI,EAAE;AACjE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,yCAAoC,KAAK,MAAM,mBAAmB;AAC/E,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,MAAM;AACtD;AAEO,SAAS,oBAAsC;AACpD,QAAM,eAAe,QAAQ,IAAI,kBAAkB;AAEnD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,+CAA+C;AAC1E,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,MAAM;AAAA,IACxE;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,WAAW;AAC5D,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,QAAQ,KAAK;AAAA,QACb,cAAc,KAAK;AAAA,QACnB,UAAU,KAAK;AAAA,MACjB,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}