@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/benchmarks/swe-bench.js

@@ -1,14 +1,235 @@
-import {
-  createSweBenchAdapter,
-  scoreSweReport,
-  sweEvaluationArgv,
-  swePatchOutput
-} from "../chunk-J6BU3NTM.js";
-import "../chunk-EIETHPD5.js";
-export {
-  createSweBenchAdapter,
-  scoreSweReport,
-  sweEvaluationArgv,
-  swePatchOutput
-};
+import { preflightVenvImports, readJsonReport, runStagedJudge, runVenvPython, safeRunId, stageFile } from "./_harness.js";
+import { join } from "node:path";
+//#region src/benchmarks/swe-bench.ts
+/**
+* SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =
+* the official `swebench` harness: apply the patch in the instance's Docker
+* image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic
+* — no LLM judge.
+*
+* Requires: the bench `.venv` with `swebench` installed + a running Docker
+* daemon (per-instance images are pulled/built on first run).
+*
+* Process/Docker/report plumbing is shared via ./_harness; this file owns the
+* SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the
+* predictions-file → run_evaluation argv → report-shape mapping.
+*/
+/**
+* Fixed in-box path the agent clones the instance repo into. It is the SINGLE
+* source of truth shared by the prompt template (which tells the agent to clone
+* here) and `boxExtract` (which runs `git diff` here after the shot) — so the
+* harness always knows exactly where the agent's edits live, for any instance.
+*/
+const SWE_REPO_DIR = "/work";
+/**
+* The SWE deliverable's FALLBACK parser, from the agent's event STREAM.
+*
+* The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's
+* actual edits, read from the cloned repo's STATE inside the box (standard
+* SWE-bench practice). This event-stream parse only runs when that diff is empty
+* — a model that edited the source correctly but never printed a fenced diff (the
+* exact failure this replaces) still scores off its real changes, not its prose.
+*/
+const swePatchOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return ([...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+} };
+const DATASET = "princeton-nlp/SWE-bench_Verified";
+const SWE_CACHE_LEVELS = /* @__PURE__ */ new Set([
+	"none",
+	"base",
+	"env",
+	"instance"
+]);
+function scorerNamespace() {
+	const namespace = process.env.SWEBENCH_NAMESPACE ?? "swebench";
+	if (namespace !== "swebench" && namespace !== "none") throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got "${namespace}"`);
+	return namespace;
+}
+const TEST_FILE_EXCLUDES = [
+	"':(exclude,glob)**/tests/**'",
+	"':(exclude,glob)**/test/**'",
+	"':(exclude,glob)test_*.py'",
+	"':(exclude,glob)**/test_*.py'",
+	"':(exclude,glob)*_test.py'",
+	"':(exclude,glob)**/*_test.py'",
+	"':(exclude,glob)conftest.py'",
+	"':(exclude,glob)**/conftest.py'"
+].join(" ");
+function stringIds(report, key) {
+	const value = report[key];
+	if (value === void 0) return [];
+	if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) throw new Error(`swe-bench: malformed ${key}`);
+	return value;
+}
+/** Convert one official report into a score without turning evaluator failures into agent failures. */
+function scoreSweReport(taskId, value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("swe-bench: report must be an object");
+	const report = value;
+	const statusIds = {
+		resolved: stringIds(report, "resolved_ids"),
+		unresolved: stringIds(report, "unresolved_ids"),
+		emptyPatch: stringIds(report, "empty_patch_ids"),
+		completed: stringIds(report, "completed_ids"),
+		incomplete: stringIds(report, "incomplete_ids"),
+		error: stringIds(report, "error_ids")
+	};
+	const submitted = stringIds(report, "submitted_ids");
+	if (Object.values(statusIds).flat().some((id) => id !== taskId) || submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId)) throw new Error(`swe-bench: report identity mismatch for ${taskId}`);
+	if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) throw new Error(`swe-bench: evaluator failed for ${taskId}`);
+	const outcomes = [
+		statusIds.resolved.includes(taskId),
+		statusIds.unresolved.includes(taskId),
+		statusIds.emptyPatch.includes(taskId)
+	];
+	if (outcomes.filter(Boolean).length !== 1) throw new Error(`swe-bench: report has no unique outcome for ${taskId}`);
+	if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`);
+	const resolved = outcomes[0];
+	return {
+		resolved,
+		score: resolved ? 1 : 0,
+		detail: JSON.stringify(report)
+	};
+}
+function sweEvaluationArgv(args) {
+	return [
+		"-m",
+		"swebench.harness.run_evaluation",
+		"--dataset_name",
+		DATASET,
+		"--predictions_path",
+		args.predictionsPath,
+		"--run_id",
+		args.runId,
+		"--instance_ids",
+		args.instanceId,
+		"--max_workers",
+		"1",
+		"--namespace",
+		args.namespace ?? scorerNamespace(),
+		"--cache_level",
+		args.cacheLevel
+	];
+}
+function shellQuote(value) {
+	return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+function sweMetadata(task) {
+	const repo = String(task.metadata?.repo ?? "");
+	const base = String(task.metadata?.base_commit ?? "");
+	if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`);
+	if (!/^[0-9a-f]{7,40}$/i.test(base)) throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`);
+	return {
+		repo,
+		base
+	};
+}
+function createSweBenchAdapter(options = {}) {
+	if (options.timeoutMs !== void 0 && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)) throw new Error("swe-bench: timeoutMs must be a positive integer");
+	const cacheLevel = options.cacheLevel ?? "env";
+	if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error("swe-bench: invalid cacheLevel");
+	if (options.captureEvaluatorArtifacts !== void 0 && typeof options.captureEvaluatorArtifacts !== "function") throw new Error("swe-bench: captureEvaluatorArtifacts must be a function");
+	let attemptSequence = 0;
+	return {
+		name: "swe-bench-verified",
+		output: swePatchOutput,
+		boxSetup(task) {
+			const { repo, base } = sweMetadata(task);
+			return { command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}` };
+		},
+		boxExtract() {
+			return { command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}` };
+		},
+		async preflight() {
+			await preflightVenvImports({
+				modules: ["swebench"],
+				requireDocker: true,
+				fix: "Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; (2) ensure the Docker daemon is running (the judge builds per-instance images)."
+			});
+		},
+		async loadTasks(opts = {}) {
+			const limit = opts.limit ?? 10;
+			const split = opts.split ?? "test";
+			const stdout = await runVenvPython(`
+import json, sys
+from datasets import load_dataset
+ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})
+ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
+out = []
+for r in ds:
+    if ids is not None and r["instance_id"] not in ids:
+        continue
+    out.append({
+        "instance_id": r["instance_id"], "repo": r["repo"], "base_commit": r["base_commit"],
+        "problem_statement": r["problem_statement"], "patch": r["patch"], "test_patch": r["test_patch"],
+        "FAIL_TO_PASS": r["FAIL_TO_PASS"], "PASS_TO_PASS": r["PASS_TO_PASS"],
+        "version": r.get("version"), "environment_setup_commit": r.get("environment_setup_commit"),
+    })
+    if ids is None and len(out) >= ${limit}:
+        break
+print(json.dumps(out))
+`, [opts.ids ? JSON.stringify(opts.ids) : ""]);
+			return JSON.parse(stdout).map((r) => ({
+				id: String(r.instance_id),
+				split,
+				prompt: [
+					`Repository: ${r.repo} @ ${r.base_commit}`,
+					"",
+					`The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
+					"",
+					"Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.",
+					"Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.",
+					"",
+					"--- Issue ---",
+					String(r.problem_statement ?? "")
+				].join("\n"),
+				metadata: r
+			}));
+		},
+		async goldArtifact(task) {
+			const gold = task.metadata?.patch;
+			return typeof gold === "string" ? gold : void 0;
+		},
+		async judge(task, artifact) {
+			const runId = safeRunId("bench", task.id);
+			const capture = options.captureEvaluatorArtifacts?.({
+				taskId: task.id,
+				runId,
+				attemptSequence: ++attemptSequence
+			});
+			return runStagedJudge({
+				tmpPrefix: "swebench-",
+				...options.timeoutMs === void 0 ? {} : { timeoutMs: options.timeoutMs },
+				...capture === void 0 ? {} : { capture },
+				...process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {},
+				async stage(dir) {
+					await stageFile(join(dir, "preds.json"), JSON.stringify([{
+						instance_id: task.id,
+						model_name_or_path: "agent-runtime-bench",
+						model_patch: artifact
+					}]));
+				},
+				argv: (dir) => sweEvaluationArgv({
+					predictionsPath: join(dir, "preds.json"),
+					runId,
+					instanceId: task.id,
+					cacheLevel,
+					namespace: scorerNamespace()
+				}),
+				async parseReport(dir) {
+					const report = await readJsonReport(join(dir, `agent-runtime-bench.${runId}.json`));
+					return scoreSweReport(task.id, report);
+				}
+			});
+		}
+	};
+}
+//#endregion
+export { createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput };
+
 //# sourceMappingURL=swe-bench.js.map

package/dist/benchmarks/swe-bench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"swe-bench.js","names":[],"sources":["../../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/**\n * Fixed in-box path the agent clones the instance repo into. It is the SINGLE\n * source of truth shared by the prompt template (which tells the agent to clone\n * here) and `boxExtract` (which runs `git diff` here after the shot) — so the\n * harness always knows exactly where the agent's edits live, for any instance.\n */\nconst SWE_REPO_DIR = '/work'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n    '--namespace', args.namespace ?? scorerNamespace(),\n    '--cache_level', args.cacheLevel,\n  ]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgv({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAiCA,MAAM,eAAe;;;;;;;;;;AAWrB,MAAa,iBAAwC,EACnD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAKA,QADa,CADG,GAAG,KAAK,SAAS,uCAAuC,CACtD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MACb,KAAA,CAAM,KAAK;AAC7B,EACF;AAEA,MAAM,UAAU;AAsBhB,MAAM,mCAAmB,IAAI,IAAwB;CAAC;CAAQ;CAAQ;CAAO;AAAU,CAAC;AAExF,SAAS,kBAAuC;CAC9C,MAAM,YAAY,QAAQ,IAAI,sBAAsB;CACpD,IAAI,cAAc,cAAc,cAAc,QAC5C,MAAM,IAAI,MAAM,kDAAkD,UAAU,EAAE;CAEhF,OAAO;AACT;AACA,MAAM,qBAAqB;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;CAClF,MAAM,QAAQ,OAAO;CACrB,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;CACjC,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAM,UAAU,OAAO,UAAU,QAAQ,GAC1E,MAAM,IAAI,MAAM,wBAAwB,KAAK;CAE/C,OAAO;AACT;;AAGA,SAAgB,eAAe,QAAgB,OAA4B;CACzE,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAC5D,MAAM,IAAI,MAAM,qCAAqC;CAEvD,MAAM,SAAS;CACf,MAAM,YAAY;EAChB,UAAU,UAAU,QAAQ,cAAc;EAC1C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,YAAY,UAAU,QAAQ,iBAAiB;EAC/C,WAAW,UAAU,QAAQ,eAAe;EAC5C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,OAAO,UAAU,QAAQ,WAAW;CACtC;CACA,MAAM,YAAY,UAAU,QAAQ,eAAe;CAEnD,IADkB,OAAO,OAAO,SAAS,CAAC,CAAC,KAEjC,CAAC,CAAC,MAAM,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,OAAO,SAExE,MAAM,IAAI,MAAM,2CAA2C,QAAQ;CAErE,IAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAC1E,MAAM,IAAI,MAAM,mCAAmC,QAAQ;CAE7D,MAAM,WAAW;EACf,UAAU,SAAS,SAAS,MAAM;EAClC,UAAU,WAAW,SAAS,MAAM;EACpC,UAAU,WAAW,SAAS,MAAM;CACtC;CACA,IAAI,SAAS,OAAO,OAAO,CAAC,CAAC,WAAW,GACtC,MAAM,IAAI,MAAM,+CAA+C,QAAQ;CAEzE,KAAK,SAAS,MAAM,SAAS,OAAO,CAAC,UAAU,UAAU,SAAS,MAAM,GACtE,MAAM,IAAI,MAAM,sDAAsD,QAAQ;CAEhF,MAAM,WAAW,SAAS;CAC1B,OAAO;EAAE;EAAU,OAAO,WAAW,IAAI;EAAG,QAAQ,KAAK,UAAU,MAAM;CAAE;AAC7E;AAEA,SAAgB,kBAAkB,MAMrB;CACX,OAAO;EACL;EAAM;EACN;EAAkB;EAClB;EAAsB,KAAK;EAC3B;EAAY,KAAK;EACjB;EAAkB,KAAK;EACvB;EAAiB;EACjB;EAAe,KAAK,aAAa,gBAAgB;EACjD;EAAiB,KAAK;CACxB;AACF;AAEA,SAAS,WAAW,OAAuB;CACzC,OAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,EAAE;AAC1C;AAEA,SAAS,YAAY,MAAiD;CACpE,MAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;CAC7C,MAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;CACpD,IAAI,CAAC,qCAAqC,KAAK,IAAI,GACjD,MAAM,IAAI,MAAM,wCAAwC,KAAK,GAAG,IAAI,MAAM;CAE5E,IAAI,CAAC,oBAAoB,KAAK,IAAI,GAChC,MAAM,IAAI,MAAM,+CAA+C,KAAK,GAAG,IAAI,MAAM;CAEnF,OAAO;EAAE;EAAM;CAAK;AACtB;AAEA,SAAgB,sBAAsB,UAAkC,CAAC,GAAqB;CAC5F,IACE,QAAQ,cAAc,KAAA,MAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,IACrE,MAAM,IAAI,MAAM,iDAAiD;CACnE,MAAM,aAAa,QAAQ,cAAc;CACzC,IAAI,CAAC,iBAAiB,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,+BAA+B;CACtF,IACE,QAAQ,8BAA8B,KAAA,KACnC,OAAO,QAAQ,8BAA8B,YAChD,MAAM,IAAI,MAAM,yDAAyD;CAC3E,IAAI,kBAAkB;CACtB,OAAO;EACL,MAAM;EACN,QAAQ;EAWR,SAAS,MAAM;GACb,MAAM,EAAE,MAAM,SAAS,YAAY,IAAI;GACvC,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,wBAAwB,WAAW,sBAAsB,MAAM,EAAE,GAAG,WAAW,YAAY,EAAE,aAAa,WAAW,YAAY,EAAE,oBAAoB,WAAW,IAAI,IACpN;EACF;EACA,aAAa;GACX,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,oBAAoB,WAAW,YAAY,EAAE,sBAAsB,qBACjH;EACF;EAEA,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAEJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,MAAM,QAAQ,KAAK,SAAS;GAqB5B,MAAM,SAAS,MAAM,cAAc;;;oBAhBrB,KAAK,UAAU,OAAO,EAAE,UAAU,KAAK,UAAU,KAAK,EAAE;;;;;;;;;;;;qCAYvC,MAAM;;;GAIM,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAErF,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KACT,OAAkB;IACjB,IAAI,OAAO,EAAE,WAAW;IACxB;IACA,QAAQ;KACN,eAAe,EAAE,KAAK,KAAK,EAAE;KAC7B;KACA,uCAAuC,aAAa,0BAA0B,EAAE,YAAY,8BAA8B,aAAa;KACvI;KACA;KACA;KACA;KACA;KACA,OAAO,EAAE,qBAAqB,EAAE;IAClC,CAAC,CAAC,KAAK,IAAI;IACX,UAAU;GACZ,EACF;EACF;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,KAAK,UAAU;GAC5B,OAAO,OAAO,SAAS,WAAW,OAAO,KAAA;EAC3C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;GACxC,MAAM,UAAU,QAAQ,4BAA4B;IAClD,QAAQ,KAAK;IACb;IACA,iBAAiB,EAAE;GACrB,CAAC;GACD,OAAO,eAAe;IACpB,WAAW;IACX,GAAI,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;IAC1E,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;IAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;IACzD,MAAM,MAAM,KAAK;KACf,MAAM,UACJ,KAAK,KAAK,YAAY,GACtB,KAAK,UAAU,CACb;MAAE,aAAa,KAAK;MAAI,oBAAoB;MAAuB,aAAa;KAAS,CAC3F,CAAC,CACH;IACF;IAGA,OAAO,QAAQ,kBAAkB;KAC/B,iBAAiB,KAAK,KAAK,YAAY;KACvC;KACA,YAAY,KAAK;KACjB;KACA,WAAW,gBAAgB;IAC7B,CAAC;IACD,MAAM,YAAY,KAAK;KAErB,MAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,MAAM,MAAM,CAAC;KAC7F,OAAO,eAAe,KAAK,IAAI,MAAM;IACvC;GACF,CAAC;EACH;CACF;AACF"}

package/dist/benchmarks/t2-ragbench.d.ts

@@ -1,14 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * T2-RAGBench adapter.
- *
- * T2-RAGBench stresses text+table retrieval and numerical reasoning over
- * financial documents. The judge uses the shared deterministic answer scorer
- * with numeric tolerance enabled by default.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/t2-ragbench.d.ts
 declare function createT2RagBenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createT2RagBenchAdapter };
+//# sourceMappingURL=t2-ragbench.d.ts.map

package/dist/benchmarks/t2-ragbench.js

@@ -1,9 +1,119 @@
-import {
-  createT2RagBenchAdapter
-} from "../chunk-GC2EPS6L.js";
-import "../chunk-X3BTXCJ4.js";
-import "../chunk-EIETHPD5.js";
-export {
-  createT2RagBenchAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { FINAL_ANSWER_SENTINEL, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringFrom } from "./rag-shared.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/t2-ragbench.ts
+/**
+* T2-RAGBench adapter.
+*
+* T2-RAGBench stresses text+table retrieval and numerical reasoning over
+* financial documents. The judge uses the shared deterministic answer scorer
+* with numeric tolerance enabled by default.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "t2-ragbench.json");
+const dataFile = () => process.env.T2_RAGBENCH_DATA_FILE;
+function rowToTask(raw, index) {
+	if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`);
+	const query = firstString(raw, [
+		"question",
+		"query",
+		"prompt"
+	]);
+	const goldAnswers = allStrings(raw, [
+		"program_answer",
+		"original_answer",
+		"answer",
+		"answers",
+		"reference",
+		"reference_answer",
+		"gold"
+	]);
+	if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`);
+	if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`);
+	const baseContexts = contextsFrom(raw.context).length > 0 ? contextsFrom(raw.context) : contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.chunks).length > 0 ? contextsFrom(raw.chunks) : contextsFrom(raw.passages);
+	const table = stringFrom(raw.table) ?? stringFrom(raw.table_text);
+	const contexts = table ? [...baseContexts, {
+		id: "table",
+		title: "Table",
+		text: table
+	}] : baseContexts;
+	const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? "unknown";
+	const documentId = stringFrom(raw.context_id) ?? stringFrom(raw.document_id) ?? stringFrom(raw.doc_id) ?? stringFrom(raw.file_name) ?? "unknown";
+	const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`;
+	const meta = {
+		benchmark: "t2-ragbench",
+		query,
+		goldAnswers,
+		contexts,
+		subset,
+		documentId
+	};
+	return {
+		id,
+		split: stringFrom(raw.split) ?? subset,
+		prompt: [
+			"Answer this T2-RAGBench text-and-table financial question.",
+			"Do the required numerical reasoning from the supplied context before giving the final value.",
+			"End with a single final line: `FINAL ANSWER: <answer>`.",
+			"",
+			`Question: ${query}`,
+			`Document: ${documentId}`,
+			`Subset: ${subset}`,
+			contexts.length > 0 ? `\nContext:\n${contextBlock(contexts)}` : void 0
+		].filter(Boolean).join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.goldAnswers)) throw new Error(`T2-RAGBench task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+async function loadRows(path) {
+	const rows = await readJsonRows(path);
+	if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`);
+	return rows;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`);
+	return selectTasks(rows.map(rowToTask), opts, "T2-RAGBench");
+}
+function createT2RagBenchAdapter() {
+	const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === "1";
+	return {
+		name: "t2-ragbench",
+		output: ragAnswerOutput,
+		async preflight() {
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8");
+				return;
+			}
+			const path = dataFile();
+			if (!path) throw new Error("T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing.");
+			await loadRows(path);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const path = dataFile();
+			if (!path) throw new Error("T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks");
+			return selectTasks((await loadRows(path)).map(rowToTask), opts, "T2-RAGBench");
+		},
+		async goldArtifact(task) {
+			return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			return answerScoreToBenchScore(scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: .01 }), {
+				benchmark: meta.benchmark,
+				subset: meta.subset,
+				documentId: meta.documentId,
+				contextCount: meta.contexts.length
+			});
+		}
+	};
+}
+//#endregion
+export { createT2RagBenchAdapter };
+
 //# sourceMappingURL=t2-ragbench.js.map

package/dist/benchmarks/t2-ragbench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"t2-ragbench.js","names":[],"sources":["../../src/benchmarks/t2-ragbench.ts"],"sourcesContent":["/**\n * T2-RAGBench adapter.\n *\n * T2-RAGBench stresses text+table retrieval and numerical reasoning over\n * financial documents. The judge uses the shared deterministic answer scorer\n * with numeric tolerance enabled by default.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 't2-ragbench.json')\n\ninterface T2RagBenchMeta {\n  benchmark: 't2-ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  subset: string\n  documentId: string\n}\n\nconst dataFile = (): string | undefined => process.env.T2_RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'program_answer',\n    'original_answer',\n    'answer',\n    'answers',\n    'reference',\n    'reference_answer',\n    'gold',\n  ])\n  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`)\n  const baseContexts =\n    contextsFrom(raw.context).length > 0\n      ? contextsFrom(raw.context)\n      : contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.chunks).length > 0\n        ? contextsFrom(raw.chunks)\n        : contextsFrom(raw.passages)\n  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text)\n  const contexts = table\n    ? [...baseContexts, { id: 'table', title: 'Table', text: table }]\n    : baseContexts\n  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? 'unknown'\n  const documentId =\n    stringFrom(raw.context_id) ??\n    stringFrom(raw.document_id) ??\n    stringFrom(raw.doc_id) ??\n    stringFrom(raw.file_name) ??\n    'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`\n  const meta: T2RagBenchMeta = {\n    benchmark: 't2-ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    subset,\n    documentId,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? subset,\n    prompt: [\n      'Answer this T2-RAGBench text-and-table financial question.',\n      'Do the required numerical reasoning from the supplied context before giving the final value.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Document: ${documentId}`,\n      `Subset: ${subset}`,\n      contexts.length > 0 ? `\\nContext:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): T2RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`T2-RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as T2RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'T2-RAGBench')\n}\n\nexport function createT2RagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 't2-ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'T2-RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 })\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        subset: meta.subset,\n        documentId: meta.documentId,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA4BA,MAAM,WAAW,KAAK,WAAW,YAAY,kBAAkB;AAW/D,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;CACzD,IAAI,CAAC,SAAS,GAAG,GAAG,MAAM,IAAI,MAAM,mBAAmB,MAAM,mBAAmB;CAChF,MAAM,QAAQ,YAAY,KAAK;EAAC;EAAY;EAAS;CAAQ,CAAC;CAC9D,MAAM,cAAc,WAAW,KAAK;EAClC;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC;CACD,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,mBAAmB,MAAM,wBAAwB;CAC7E,IAAI,YAAY,WAAW,GAAG,MAAM,IAAI,MAAM,mBAAmB,MAAM,gBAAgB;CACvF,MAAM,eACJ,aAAa,IAAI,OAAO,CAAC,CAAC,SAAS,IAC/B,aAAa,IAAI,OAAO,IACxB,aAAa,IAAI,QAAQ,CAAC,CAAC,SAAS,IACpC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,MAAM,CAAC,CAAC,SAAS,IAChC,aAAa,IAAI,MAAM,IACvB,aAAa,IAAI,QAAQ;CACjC,MAAM,QAAQ,WAAW,IAAI,KAAK,KAAK,WAAW,IAAI,UAAU;CAChE,MAAM,WAAW,QACb,CAAC,GAAG,cAAc;EAAE,IAAI;EAAS,OAAO;EAAS,MAAM;CAAM,CAAC,IAC9D;CACJ,MAAM,SAAS,WAAW,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,KAAK;CACpE,MAAM,aACJ,WAAW,IAAI,UAAU,KACzB,WAAW,IAAI,WAAW,KAC1B,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,SAAS,KACxB;CACF,MAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,QAAQ,KAAK,eAAe;CACnG,MAAM,OAAuB;EAC3B,WAAW;EACX;EACA;EACA;EACA;EACA;CACF;CACA,OAAO;EACL;EACA,OAAO,WAAW,IAAI,KAAK,KAAK;EAChC,QAAQ;GACN;GACA;GACA;GACA;GACA,aAAa;GACb,aAAa;GACb,WAAW;GACX,SAAS,SAAS,IAAI,eAAe,aAAa,QAAQ,MAAM,KAAA;EAClE,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiC;CACjD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GACtC,MAAM,IAAI,MAAM,oBAAoB,KAAK,GAAG,kDAAkD;CAEhG,OAAO;AACT;AAEA,eAAe,SAAS,MAAkC;CACxD,MAAM,OAAO,MAAM,aAAa,IAAI;CACpC,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,2BAA2B,MAAM;CACxE,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,kDAAkD,KAAK,OAAO,kBAAkB;CAC7F,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,aAAa;AAC7D;AAEA,SAAgB,0BAA4C;CAC1D,MAAM,eAAe,QAAQ,IAAI,yBAAyB;CAE1D,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM;IAC/B;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,wLACF;GAEF,MAAM,SAAS,IAAI;EACrB;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,6DAA6D;GACxF,OAAO,aAAa,MAAM,SAAS,IAAI,EAAA,CAAG,IAAI,SAAS,GAAG,MAAM,aAAa;EAC/E;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,GAAG,sBAAsB,GAAG,SAAS,IAAI,CAAC,CAAC,YAAY,MAAM;EACtE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAE1B,OAAO,wBADO,oBAAoB,UAAU,KAAK,aAAa,EAAE,kBAAkB,IAAK,CACpD,GAAG;IACpC,WAAW,KAAK;IAChB,QAAQ,KAAK;IACb,YAAY,KAAK;IACjB,cAAc,KAAK,SAAS;GAC9B,CAAC;EACH;CACF;AACF"}

package/dist/benchmarks/tau-bench-shared.d.ts

@@ -1,26 +1,18 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * Shared tau-bench adapter spine.
- *
- * tau2 and tau3 live in the same upstream repository/package namespace today:
- * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation
- * for task loading and reward recomputation so the domain/version adapters only
- * choose env names, default domain, and fixture file.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/tau-bench-shared.d.ts
 interface TauBenchConfig {
-    name: string;
-    fixturePath: string;
-    fixturesEnv: string;
-    dirEnv: string;
-    domainEnv: string;
-    defaultDomain: string;
-    installHint: string;
-    taskIntro: string;
+  name: string;
+  fixturePath: string;
+  fixturesEnv: string;
+  dirEnv: string;
+  domainEnv: string;
+  defaultDomain: string;
+  installHint: string;
+  taskIntro: string;
 }
 declare const tauResultsOutput: OutputAdapter<string>;
 declare function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter;
-
-export { type TauBenchConfig, createTauBenchAdapter, tauResultsOutput };
+//#endregion
+export { TauBenchConfig, createTauBenchAdapter, tauResultsOutput };
+//# sourceMappingURL=tau-bench-shared.d.ts.map