@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/commit0.d.ts

@@ -1,31 +1,9 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.
- * Each record is a Python repo stubbed at `base_commit` (public function bodies
- * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker
- * artifact = a unified diff that fills in `src_dir`. Judge = the official
- * `commit0` test harness: it stages the starter repo, applies the worker's diff,
- * builds the library's deps and runs `pytest`, writing a per-repo pytest-json
- * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the
- * macro-averaged unit-test pass-rate the leaderboard reports). Fully
- * deterministic — no LLM judge.
- *
- * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its
- * implementation as a fenced ```diff against the stubbed repo — same deliverable
- * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the
- * real `commit0` harness on a local Docker backend (NOT reimplemented here).
- *
- * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed
- * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +
- * a Docker daemon (`--backend local`). For offline/CI dataset listing set
- * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)
- * — judging still needs the harness + Docker and fails loud, never a fabricated score.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/commit0.d.ts
 /** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */
 declare const commit0DiffOutput: OutputAdapter<string>;
 declare function createCommit0Adapter(): BenchmarkAdapter;
-
+//#endregion
 export { commit0DiffOutput, createCommit0Adapter };
+//# sourceMappingURL=commit0.d.ts.map

package/dist/benchmarks/commit0.js

@@ -1,10 +1,188 @@
-import {
-  commit0DiffOutput,
-  createCommit0Adapter
-} from "../chunk-Z4TZ76N7.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  commit0DiffOutput,
-  createCommit0Adapter
-};
+import { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPythonAt } from "./_harness.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/commit0.ts
+/**
+* Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.
+* Each record is a Python repo stubbed at `base_commit` (public function bodies
+* emptied to `pass`) + the full test suite + a natural-language spec URL. Worker
+* artifact = a unified diff that fills in `src_dir`. Judge = the official
+* `commit0` test harness: it stages the starter repo, applies the worker's diff,
+* builds the library's deps and runs `pytest`, writing a per-repo pytest-json
+* `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the
+* macro-averaged unit-test pass-rate the leaderboard reports). Fully
+* deterministic — no LLM judge.
+*
+* The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its
+* implementation as a fenced ```diff against the stubbed repo — same deliverable
+* shape as swe-bench. The expensive per-repo clone+build+test is delegated to the
+* real `commit0` harness on a local Docker backend (NOT reimplemented here).
+*
+* Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed
+* (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +
+* a Docker daemon (`--backend local`). For offline/CI dataset listing set
+* COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)
+* — judging still needs the harness + Docker and fails loud, never a fabricated score.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "commit0.json");
+const commit0VenvDir = () => process.env.COMMIT0_VENV ?? ".venv-commit0";
+const commit0Python = () => venvPythonAt(commit0VenvDir());
+const DATASET = "wentingzhao/commit0_combined";
+const DATASET_SPLIT = "test";
+const ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`;
+/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */
+const commit0DiffOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	const body = ([...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+	return body.length > 0 ? `${body}\n` : body;
+} };
+function rowToTask(row) {
+	const meta = {
+		instanceId: row.instance_id,
+		repo: row.repo,
+		originalRepo: row.original_repo,
+		baseCommit: row.base_commit,
+		referenceCommit: row.reference_commit,
+		srcDir: row.src_dir,
+		testDir: row.test.test_dir,
+		testCmd: row.test.test_cmd,
+		specification: row.setup.specification
+	};
+	return {
+		id: row.instance_id,
+		split: DATASET_SPLIT,
+		prompt: [
+			`Clone https://github.com/${row.repo} into /work, then \`cd /work && git checkout ${row.base_commit}\`.`,
+			`This is the STUBBED library: the public functions/classes under \`${row.src_dir}\` have empty bodies (\`pass\`/\`...\`).`,
+			`Fill in COMPLETE implementations under \`${row.src_dir}\` so the existing test suite under \`${row.test.test_dir}\` passes. Read those tests and the spec to learn the required behavior.`,
+			`Specification / docs: ${row.setup.specification}`,
+			"Do NOT edit the test files — the evaluation re-runs the existing tests on a fresh clone. Implement only the source.",
+			`When done, from /work run EXACTLY: \`git add -A && git diff --cached -- ${row.src_dir}\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \`\`\`diff … \`\`\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`
+		].join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.instanceId !== "string" || typeof md.referenceCommit !== "string" || typeof md.srcDir !== "string") throw new Error(`commit0 task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function selectRows(rows, opts) {
+	let tasks = rows.map(rowToTask);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((t) => want.has(t.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	return tasks;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[commit0] COMMIT0_FIXTURES=1 — loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`);
+	return selectRows(rows, opts);
+}
+/** Pull real rows from the HF rows server (paged). limit caps the pull; ids
+*  filtered client-side. Throws on a non-OK response (fail loud). */
+async function fetchRows(opts) {
+	const target = opts.ids ? opts.ids.length * 4 : opts.limit ?? 16;
+	const rows = [];
+	const want = opts.ids ? new Set(opts.ids) : null;
+	const page = 100;
+	for (let offset = 0; offset < 64 && rows.length < target; offset += page) {
+		const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`);
+		if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`);
+		const got = (await res.json()).rows ?? [];
+		if (got.length === 0) break;
+		for (const r of got) {
+			if (want && !want.has(r.row.instance_id)) continue;
+			rows.push(r.row);
+		}
+		if (got.length < page) break;
+	}
+	if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`);
+	return rows;
+}
+/**
+* Run the official commit0 harness for one repo over the worker's diff. The
+* harness clones base_commit, applies the diff into src_dir, installs deps and
+* runs pytest, then writes a per-repo pytest-json report. We read that report and
+* compute (passed + xfail) / total — the leaderboard's per-repo pass-rate.
+*
+* This is the expensive, Docker-backed boundary; it is DELEGATED to `commit0`,
+* not reimplemented. The driver script lives in scripts/commit0_judge.py so the
+* harness call + report parse are one auditable python entrypoint.
+*/
+async function runHarness(meta, artifact) {
+	const judge = join(benchRoot, "scripts", "commit0_judge.py");
+	let stdout;
+	try {
+		stdout = await runVenvScriptStdin(judge, [
+			"--dataset",
+			DATASET,
+			"--split",
+			DATASET_SPLIT,
+			"--instance",
+			meta.instanceId,
+			"--src-dir",
+			meta.srcDir
+		], artifact, {
+			cwd: benchRoot,
+			python: commit0Python()
+		});
+	} catch (err) {
+		const e = err;
+		throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`);
+	}
+	const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+	if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`);
+	if (typeof report.passed !== "number" || typeof report.total !== "number") throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`);
+	if (report.total <= 0) throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`);
+	const score = report.passed / report.total;
+	return {
+		resolved: report.passed === report.total,
+		score,
+		detail: JSON.stringify({
+			instanceId: meta.instanceId,
+			passed: report.passed,
+			total: report.total
+		})
+	};
+}
+function createCommit0Adapter() {
+	const fixturesMode = process.env.COMMIT0_FIXTURES === "1";
+	return {
+		name: "commit0",
+		output: commit0DiffOutput,
+		async preflight() {
+			await preflightVenvImports({
+				modules: ["commit0"],
+				requireDocker: true,
+				python: commit0Python(),
+				fix: `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets (an ISOLATED venv — commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; (2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`
+			});
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			let rows;
+			try {
+				rows = await fetchRows(opts);
+			} catch (err) {
+				console.warn(`[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`);
+				return loadFixtures(opts);
+			}
+			return selectRows(rows, opts);
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			return runHarness(readMeta(task), artifact);
+		}
+	};
+}
+//#endregion
+export { commit0DiffOutput, createCommit0Adapter };
+
 //# sourceMappingURL=commit0.js.map

package/dist/benchmarks/commit0.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"commit0.js","names":[],"sources":["../../src/benchmarks/commit0.ts"],"sourcesContent":["/**\n * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.\n * Each record is a Python repo stubbed at `base_commit` (public function bodies\n * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker\n * artifact = a unified diff that fills in `src_dir`. Judge = the official\n * `commit0` test harness: it stages the starter repo, applies the worker's diff,\n * builds the library's deps and runs `pytest`, writing a per-repo pytest-json\n * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the\n * macro-averaged unit-test pass-rate the leaderboard reports). Fully\n * deterministic — no LLM judge.\n *\n * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its\n * implementation as a fenced ```diff against the stubbed repo — same deliverable\n * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the\n * real `commit0` harness on a local Docker backend (NOT reimplemented here).\n *\n * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed\n * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +\n * a Docker daemon (`--backend local`). For offline/CI dataset listing set\n * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)\n * — judging still needs the harness + Docker and fails loud, never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPythonAt } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'commit0.json')\n\n// commit0's pip deps (pydantic/sqlalchemy v1, modal, …) conflict with the shared\n// bench .venv, so its harness runs in an ISOLATED venv. Override with COMMIT0_VENV.\n// Resolved at call-time so the env is honored at run-time (not frozen at import).\nconst commit0VenvDir = (): string => process.env.COMMIT0_VENV ?? '.venv-commit0'\nconst commit0Python = (): string => venvPythonAt(commit0VenvDir())\n\nconst DATASET = 'wentingzhao/commit0_combined'\nconst DATASET_SPLIT = 'test'\n// HF rows server — columnar pull with no `datasets` install required for listing.\nconst ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`\n\n/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */\nexport const commit0DiffOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const body = (fences.at(-1)?.[1] ?? text).trim()\n    // `git apply` rejects a patch that is not newline-terminated (\"corrupt patch at\n    // line N+1\"); the .trim() above strips the final newline, so restore exactly one.\n    return body.length > 0 ? `${body}\\n` : body\n  },\n}\n\ninterface Commit0Setup {\n  install: string\n  packages: string[] | null\n  pip_packages: string[] | null\n  pre_install: string[] | null\n  python: string\n  specification: string\n}\n\ninterface Commit0Row {\n  instance_id: string\n  repo: string\n  original_repo: string\n  base_commit: string\n  reference_commit: string\n  setup: Commit0Setup\n  test: { test_cmd: string; test_dir: string }\n  src_dir: string\n}\n\ninterface Commit0Meta {\n  instanceId: string\n  repo: string\n  originalRepo: string\n  baseCommit: string\n  referenceCommit: string\n  srcDir: string\n  testDir: string\n  testCmd: string\n  specification: string\n}\n\nfunction rowToTask(row: Commit0Row): BenchTask {\n  const meta: Commit0Meta = {\n    instanceId: row.instance_id,\n    repo: row.repo,\n    originalRepo: row.original_repo,\n    baseCommit: row.base_commit,\n    referenceCommit: row.reference_commit,\n    srcDir: row.src_dir,\n    testDir: row.test.test_dir,\n    testCmd: row.test.test_cmd,\n    specification: row.setup.specification,\n  }\n  return {\n    id: row.instance_id,\n    split: DATASET_SPLIT,\n    prompt: [\n      `Clone https://github.com/${row.repo} into /work, then \\`cd /work && git checkout ${row.base_commit}\\`.`,\n      `This is the STUBBED library: the public functions/classes under \\`${row.src_dir}\\` have empty bodies (\\`pass\\`/\\`...\\`).`,\n      `Fill in COMPLETE implementations under \\`${row.src_dir}\\` so the existing test suite under \\`${row.test.test_dir}\\` passes. Read those tests and the spec to learn the required behavior.`,\n      `Specification / docs: ${row.setup.specification}`,\n      'Do NOT edit the test files — the evaluation re-runs the existing tests on a fresh clone. Implement only the source.',\n      `When done, from /work run EXACTLY: \\`git add -A && git diff --cached -- ${row.src_dir}\\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \\`\\`\\`diff … \\`\\`\\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): Commit0Meta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string' || typeof md.referenceCommit !== 'string' || typeof md.srcDir !== 'string') {\n    throw new Error(`commit0 task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as Commit0Meta\n}\n\nfunction selectRows(rows: Commit0Row[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as Commit0Row[]\n  console.warn(\n    `[commit0] COMMIT0_FIXTURES=1 — loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`,\n  )\n  return selectRows(rows, opts)\n}\n\n/** Pull real rows from the HF rows server (paged). limit caps the pull; ids\n *  filtered client-side. Throws on a non-OK response (fail loud). */\nasync function fetchRows(opts: LoadOptions): Promise<Commit0Row[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: Commit0Row[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  for (let offset = 0; offset < 64 && rows.length < target; offset += page) {\n    const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`)\n    if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    const body = (await res.json()) as { rows?: Array<{ row: Commit0Row }>; num_rows_total?: number }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.instance_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`)\n  return rows\n}\n\n/**\n * Run the official commit0 harness for one repo over the worker's diff. The\n * harness clones base_commit, applies the diff into src_dir, installs deps and\n * runs pytest, then writes a per-repo pytest-json report. We read that report and\n * compute (passed + xfail) / total — the leaderboard's per-repo pass-rate.\n *\n * This is the expensive, Docker-backed boundary; it is DELEGATED to `commit0`,\n * not reimplemented. The driver script lives in scripts/commit0_judge.py so the\n * harness call + report parse are one auditable python entrypoint.\n */\nasync function runHarness(meta: Commit0Meta, artifact: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'commit0_judge.py')\n  let stdout: string\n  try {\n    // The worker's diff is piped to the driver's stdin (shared stdin-aware runner —\n    // execFile's `input` option is not honored async and hangs the reader).\n    stdout = await runVenvScriptStdin(\n      judge,\n      ['--dataset', DATASET, '--split', DATASET_SPLIT, '--instance', meta.instanceId, '--src-dir', meta.srcDir],\n      artifact,\n      { cwd: benchRoot, python: commit0Python() },\n    )\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  // total=0 means the harness MEASURED NOTHING (collection error with no declared\n  // test ids) — an unmeasured attempt, not a 0% one. Throw so the caller excludes\n  // it as infra instead of recording a fabricated zero.\n  if (report.total <= 0) {\n    throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`)\n  }\n  const score = report.passed / report.total\n  return {\n    resolved: report.passed === report.total,\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\nexport function createCommit0Adapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.COMMIT0_FIXTURES === '1'\n\n  return {\n    name: 'commit0',\n    output: commit0DiffOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['commit0'],\n        requireDocker: true,\n        python: commit0Python(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets ` +\n          `(an ISOLATED venv — commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). ` +\n          `Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let rows: Commit0Row[]\n      try {\n        rows = await fetchRows(opts)\n      } catch (err) {\n        console.warn(\n          `[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRows(rows, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the reference_commit's src_dir, which the commit0 harness\n      // checks out by ref — not expressible as a portable diff string without\n      // cloning. verify-judge against this adapter requires the live harness, so\n      // we return undefined (no offline gold-diff). Judge correctness is proven by\n      // running the harness on a real solve, not by a synthetic gold patch.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,MAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAK3D,MAAM,uBAA+B,QAAQ,IAAI,gBAAgB;AACjE,MAAM,sBAA8B,aAAa,eAAe,CAAC;AAEjE,MAAM,UAAU;AAChB,MAAM,gBAAgB;AAEtB,MAAM,WAAW,uDAAuD,mBAAmB,OAAO,EAAE,wBAAwB;;AAG5H,MAAa,oBAA2C,EACtD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,MAAM,QAAQ,CADE,GAAG,KAAK,SAAS,uCAAuC,CACrD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAG/C,OAAO,KAAK,SAAS,IAAI,GAAG,KAAK,MAAM;AACzC,EACF;AAkCA,SAAS,UAAU,KAA4B;CAC7C,MAAM,OAAoB;EACxB,YAAY,IAAI;EAChB,MAAM,IAAI;EACV,cAAc,IAAI;EAClB,YAAY,IAAI;EAChB,iBAAiB,IAAI;EACrB,QAAQ,IAAI;EACZ,SAAS,IAAI,KAAK;EAClB,SAAS,IAAI,KAAK;EAClB,eAAe,IAAI,MAAM;CAC3B;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ;GACN,4BAA4B,IAAI,KAAK,+CAA+C,IAAI,YAAY;GACpG,qEAAqE,IAAI,QAAQ;GACjF,4CAA4C,IAAI,QAAQ,wCAAwC,IAAI,KAAK,SAAS;GAClH,yBAAyB,IAAI,MAAM;GACnC;GACA,2EAA2E,IAAI,QAAQ,uKAAuK,IAAI,YAAY;EAChR,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA8B;CAC9C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,eAAe,YAAY,OAAO,GAAG,oBAAoB,YAAY,OAAO,GAAG,WAAW,UAC7G,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kDAAkD;CAE5F,OAAO;AACT;AAEA,SAAS,WAAW,MAAoB,MAAgC;CACtE,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KACN,0CAA0C,KAAK,OAAO,4BAA4B,SAAS,eAC7F;CACA,OAAO,WAAW,MAAM,IAAI;AAC9B;;;AAIA,eAAe,UAAU,MAA0C;CACjE,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;CAC/D,MAAM,OAAqB,CAAC;CAC5B,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;CAC5C,MAAM,OAAO;CACb,KAAK,IAAI,SAAS,GAAG,SAAS,MAAM,KAAK,SAAS,QAAQ,UAAU,MAAM;EACxE,MAAM,MAAM,MAAM,MAAM,GAAG,SAAS,UAAU,OAAO,UAAU,MAAM;EACrE,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,qBAAqB,IAAI,OAAO,WAAW,OAAO,MAAM,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EAEtH,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,CAAC;EAC1B,IAAI,IAAI,WAAW,GAAG;EACtB,KAAK,MAAM,KAAK,KAAK;GACnB,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,WAAW,GAAG;GAC1C,KAAK,KAAK,EAAE,GAAG;EACjB;EACA,IAAI,IAAI,SAAS,MAAM;CACzB;CACA,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,4BAA4B,KAAK,UAAU,IAAI,EAAE,QAAQ,SAAS;CACzG,OAAO;AACT;;;;;;;;;;;AAYA,eAAe,WAAW,MAAmB,UAAuC;CAClF,MAAM,QAAQ,KAAK,WAAW,WAAW,kBAAkB;CAC3D,IAAI;CACJ,IAAI;EAGF,SAAS,MAAM,mBACb,OACA;GAAC;GAAa;GAAS;GAAW;GAAe;GAAc,KAAK;GAAY;GAAa,KAAK;EAAM,GACxG,UACA;GAAE,KAAK;GAAW,QAAQ,cAAc;EAAE,CAC5C;CACF,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,8BAA8B,KAAK,WAAW,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC/G;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAKlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,6BAA6B,KAAK,WAAW,IAAI,OAAO,OAAO;CACjG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,6CAA6C,OAAO,MAAM,GAAG,GAAG,GAAG;CAKrF,IAAI,OAAO,SAAS,GAClB,MAAM,IAAI,MAAM,uCAAuC,KAAK,WAAW,cAAc,OAAO,MAAM,GAAG,GAAG,GAAG;CAE7G,MAAM,QAAQ,OAAO,SAAS,OAAO;CACrC,OAAO;EACL,UAAU,OAAO,WAAW,OAAO;EACnC;EACA,QAAQ,KAAK,UAAU;GAAE,YAAY,KAAK;GAAY,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACpG;AACF;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,qBAAqB;CAEtD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,SAAS;IACnB,eAAe;IACf,QAAQ,cAAc;IACtB,KACE,kCAAkC,eAAe,EAAE,YAAY,eAAe,EAAE;GAIpF,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,UAAU,IAAI;GAC7B,SAAS,KAAK;IACZ,QAAQ,KACN,qCAAqC,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAA2C,UAC3H;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,WAAW,MAAM,IAAI;EAC9B;EAEA,MAAM,eAAe,CAOrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,WADM,SAAS,IACD,GAAG,QAAQ;EAClC;CACF;AACF"}

package/dist/benchmarks/crag.d.ts

@@ -1,14 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * CRAG adapter (Comprehensive RAG Benchmark).
- *
- * Live mode expects an official or compatible CRAG JSON/JSONL export. The
- * adapter preserves CRAG domain/type/dynamism tags in metadata and scores final
- * answers deterministically against the provided gold answer list.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/crag.d.ts
 declare function createCragAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createCragAdapter };
+//# sourceMappingURL=crag.d.ts.map

package/dist/benchmarks/crag.js

@@ -1,9 +1,111 @@
-import {
-  createCragAdapter
-} from "../chunk-SVR2LKYI.js";
-import "../chunk-X3BTXCJ4.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createCragAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { FINAL_ANSWER_SENTINEL, allStrings, answerScoreToBenchScore, firstString, isObject, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringFrom } from "./rag-shared.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/crag.ts
+/**
+* CRAG adapter (Comprehensive RAG Benchmark).
+*
+* Live mode expects an official or compatible CRAG JSON/JSONL export. The
+* adapter preserves CRAG domain/type/dynamism tags in metadata and scores final
+* answers deterministically against the provided gold answer list.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "crag.json");
+const dataFile = () => process.env.CRAG_DATA_FILE;
+function rowToTask(raw, index) {
+	if (!isObject(raw)) throw new Error(`CRAG row ${index} must be an object`);
+	const query = firstString(raw, [
+		"query",
+		"question",
+		"prompt"
+	]);
+	const goldAnswers = allStrings(raw, [
+		"answer",
+		"answers",
+		"gold",
+		"gold_answer",
+		"expected_answer"
+	]);
+	if (!query) throw new Error(`CRAG row ${index} missing query`);
+	if (goldAnswers.length === 0) throw new Error(`CRAG row ${index} missing gold answer`);
+	const domain = stringFrom(raw.domain) ?? "unknown";
+	const questionType = stringFrom(raw.question_type) ?? stringFrom(raw.questionType) ?? "unknown";
+	const dynamism = stringFrom(raw.static_or_dynamic) ?? stringFrom(raw.dynamism) ?? "unknown";
+	const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `crag-${index}`;
+	const meta = {
+		benchmark: "crag",
+		query,
+		goldAnswers,
+		domain,
+		questionType,
+		dynamism
+	};
+	return {
+		id,
+		split: stringFrom(raw.split) ?? domain,
+		prompt: [
+			"Answer this CRAG factual question.",
+			"Return a concise answer and do not guess when the evidence is insufficient.",
+			"End with a single final line: `FINAL ANSWER: <answer>`.",
+			"",
+			`Question: ${query}`,
+			`Domain: ${domain}`,
+			`Question type: ${questionType}`,
+			`Dynamism: ${dynamism}`
+		].join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.goldAnswers)) throw new Error(`CRAG task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+async function loadRows(path) {
+	const rows = await readJsonRows(path);
+	if (rows.length === 0) throw new Error(`CRAG: no rows in ${path}`);
+	return rows;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[crag] CRAG_FIXTURES=1 — loading ${rows.length} adapter fixtures`);
+	return selectTasks(rows.map(rowToTask), opts, "CRAG");
+}
+function createCragAdapter() {
+	const fixturesMode = process.env.CRAG_FIXTURES === "1";
+	return {
+		name: "crag",
+		output: ragAnswerOutput,
+		async preflight() {
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8");
+				return;
+			}
+			const path = dataFile();
+			if (!path) throw new Error("CRAG_DATA_FILE is required. Fix: export facebookresearch/CRAG rows to JSONL and set CRAG_DATA_FILE=/path/to/crag.jsonl, or set CRAG_FIXTURES=1 for adapter plumbing.");
+			await loadRows(path);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const path = dataFile();
+			if (!path) throw new Error("CRAG_DATA_FILE is required to load CRAG tasks");
+			return selectTasks((await loadRows(path)).map(rowToTask), opts, "CRAG");
+		},
+		async goldArtifact(task) {
+			return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			return answerScoreToBenchScore(scoreAnswerArtifact(artifact, meta.goldAnswers), {
+				benchmark: meta.benchmark,
+				domain: meta.domain,
+				questionType: meta.questionType,
+				dynamism: meta.dynamism
+			});
+		}
+	};
+}
+//#endregion
+export { createCragAdapter };
+
 //# sourceMappingURL=crag.js.map

package/dist/benchmarks/crag.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"crag.js","names":[],"sources":["../../src/benchmarks/crag.ts"],"sourcesContent":["/**\n * CRAG adapter (Comprehensive RAG Benchmark).\n *\n * Live mode expects an official or compatible CRAG JSON/JSONL export. The\n * adapter preserves CRAG domain/type/dynamism tags in metadata and scores final\n * answers deterministically against the provided gold answer list.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'crag.json')\n\ninterface CragMeta {\n  benchmark: 'crag'\n  query: string\n  goldAnswers: string[]\n  domain: string\n  questionType: string\n  dynamism: string\n}\n\nconst dataFile = (): string | undefined => process.env.CRAG_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`CRAG row ${index} must be an object`)\n  const query = firstString(raw, ['query', 'question', 'prompt'])\n  const goldAnswers = allStrings(raw, ['answer', 'answers', 'gold', 'gold_answer', 'expected_answer'])\n  if (!query) throw new Error(`CRAG row ${index} missing query`)\n  if (goldAnswers.length === 0) throw new Error(`CRAG row ${index} missing gold answer`)\n  const domain = stringFrom(raw.domain) ?? 'unknown'\n  const questionType = stringFrom(raw.question_type) ?? stringFrom(raw.questionType) ?? 'unknown'\n  const dynamism = stringFrom(raw.static_or_dynamic) ?? stringFrom(raw.dynamism) ?? 'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `crag-${index}`\n  const meta: CragMeta = {\n    benchmark: 'crag',\n    query,\n    goldAnswers,\n    domain,\n    questionType,\n    dynamism,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? domain,\n    prompt: [\n      'Answer this CRAG factual question.',\n      'Return a concise answer and do not guess when the evidence is insufficient.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Domain: ${domain}`,\n      `Question type: ${questionType}`,\n      `Dynamism: ${dynamism}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): CragMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`CRAG task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as CragMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`CRAG: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[crag] CRAG_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'CRAG')\n}\n\nexport function createCragAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.CRAG_FIXTURES === '1'\n\n  return {\n    name: 'crag',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'CRAG_DATA_FILE is required. Fix: export facebookresearch/CRAG rows to JSONL and set CRAG_DATA_FILE=/path/to/crag.jsonl, or set CRAG_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('CRAG_DATA_FILE is required to load CRAG tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'CRAG')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        domain: meta.domain,\n        questionType: meta.questionType,\n        dynamism: meta.dynamism,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAyBA,MAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AAWxD,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;CACzD,IAAI,CAAC,SAAS,GAAG,GAAG,MAAM,IAAI,MAAM,YAAY,MAAM,mBAAmB;CACzE,MAAM,QAAQ,YAAY,KAAK;EAAC;EAAS;EAAY;CAAQ,CAAC;CAC9D,MAAM,cAAc,WAAW,KAAK;EAAC;EAAU;EAAW;EAAQ;EAAe;CAAiB,CAAC;CACnG,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,YAAY,MAAM,eAAe;CAC7D,IAAI,YAAY,WAAW,GAAG,MAAM,IAAI,MAAM,YAAY,MAAM,qBAAqB;CACrF,MAAM,SAAS,WAAW,IAAI,MAAM,KAAK;CACzC,MAAM,eAAe,WAAW,IAAI,aAAa,KAAK,WAAW,IAAI,YAAY,KAAK;CACtF,MAAM,WAAW,WAAW,IAAI,iBAAiB,KAAK,WAAW,IAAI,QAAQ,KAAK;CAClF,MAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,QAAQ;CACrE,MAAM,OAAiB;EACrB,WAAW;EACX;EACA;EACA;EACA;EACA;CACF;CACA,OAAO;EACL;EACA,OAAO,WAAW,IAAI,KAAK,KAAK;EAChC,QAAQ;GACN;GACA;GACA;GACA;GACA,aAAa;GACb,WAAW;GACX,kBAAkB;GAClB,aAAa;EACf,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA2B;CAC3C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GACtC,MAAM,IAAI,MAAM,aAAa,KAAK,GAAG,kDAAkD;CAEzF,OAAO;AACT;AAEA,eAAe,SAAS,MAAkC;CACxD,MAAM,OAAO,MAAM,aAAa,IAAI;CACpC,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,oBAAoB,MAAM;CACjE,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,oCAAoC,KAAK,OAAO,kBAAkB;CAC/E,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,MAAM;AACtD;AAEA,SAAgB,oBAAsC;CACpD,MAAM,eAAe,QAAQ,IAAI,kBAAkB;CAEnD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM;IAC/B;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,sKACF;GAEF,MAAM,SAAS,IAAI;EACrB;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,+CAA+C;GAC1E,OAAO,aAAa,MAAM,SAAS,IAAI,EAAA,CAAG,IAAI,SAAS,GAAG,MAAM,MAAM;EACxE;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,GAAG,sBAAsB,GAAG,SAAS,IAAI,CAAC,CAAC,YAAY,MAAM;EACtE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAE1B,OAAO,wBADO,oBAAoB,UAAU,KAAK,WACd,GAAG;IACpC,WAAW,KAAK;IAChB,QAAQ,KAAK;IACb,cAAc,KAAK;IACnB,UAAU,KAAK;GACjB,CAAC;EACH;CACF;AACF"}

package/dist/benchmarks/dabstep.d.ts

@@ -1,18 +1,8 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic
- * payment files. Worker artifact = final answer text. Judge = the official
- * DABStep `grade.py` normalization/matching function. No LLM judge.
- *
- * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that
- * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter
- * exposes `metadata.resourceRoot` so runners can mount the benchmark files into
- * AgentProfile.resources.files; it does not paste the dataset into prompt text.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/dabstep.d.ts
 declare const dabstepAnswerOutput: OutputAdapter<string>;
 declare function createDabstepAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createDabstepAdapter, dabstepAnswerOutput };
+//# sourceMappingURL=dabstep.d.ts.map

package/dist/benchmarks/dabstep.js

@@ -1,10 +1,178 @@
-import {
-  createDabstepAdapter,
-  dabstepAnswerOutput
-} from "../chunk-RH5F53JT.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createDabstepAdapter,
-  dabstepAnswerOutput
-};
+import { benchRoot, runVenvPython, runVenvScriptStdin } from "./_harness.js";
+import { access, readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/dabstep.ts
+/**
+* DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic
+* payment files. Worker artifact = final answer text. Judge = the official
+* DABStep `grade.py` normalization/matching function. No LLM judge.
+*
+* Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that
+* includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter
+* exposes `metadata.resourceRoot` so runners can mount the benchmark files into
+* AgentProfile.resources.files; it does not paste the dataset into prompt text.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "dabstep.json");
+const DEFAULT_SPLIT = "easy";
+const dabstepDir = () => process.env.DABSTEP_DIR;
+const gradeFile = (dir) => join(dir, "grade.py");
+const resourceRoot = (dir) => join(dir, "files");
+async function assertFile(path, label) {
+	try {
+		await access(path);
+	} catch (err) {
+		throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+	}
+}
+async function assertOfficialFiles(dir, split) {
+	await assertFile(join(dir, "dataset.csv"), "released dataset.csv");
+	await assertFile(join(dir, "splits", `${split}.txt`), `${split} split file`);
+	await assertFile(gradeFile(dir), "official grade.py");
+	const files = resourceRoot(dir);
+	try {
+		if (!(await stat(files)).isDirectory()) throw new Error("not a directory");
+	} catch (err) {
+		throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`);
+	}
+}
+const dabstepAnswerOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return ([...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+} };
+function rowToTask(row, split, dir) {
+	const meta = {
+		taskId: row.task_id,
+		split,
+		golds: row.all_golds_by_task,
+		...dir ? { resourceRoot: resourceRoot(dir) } : {}
+	};
+	return {
+		id: String(row.task_id),
+		split,
+		prompt: [
+			"Solve this DABStep data-analysis task using the mounted payment files.",
+			"Use code or shell commands as needed, then return only the final answer.",
+			"",
+			row.instructions
+		].join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.taskId !== "number" || !Array.isArray(md.golds)) throw new Error(`dabstep task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function selectRows(rows, opts, split, dir) {
+	let tasks = rows.map((row) => rowToTask(row, split, dir));
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((task) => want.has(task.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`);
+	return tasks;
+}
+async function loadFixtures(opts, split) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[dabstep] DABSTEP_FIXTURES=1 — loading ${rows.length} adapter fixtures from ${FIXTURES}`);
+	return selectRows(rows, opts, split);
+}
+async function loadOfficialTasks(dir, opts, split) {
+	const stdout = await runVenvPython(`
+import ast, csv, json, sys
+from pathlib import Path
+
+root = Path(sys.argv[1])
+split = sys.argv[2]
+limit = None if sys.argv[3] == "" else int(sys.argv[3])
+ids = set(json.loads(sys.argv[4]))
+dataset = root / "dataset.csv"
+split_file = root / "splits" / f"{split}.txt"
+if not dataset.exists():
+    raise SystemExit(f"missing official DABStep dataset.csv at {dataset}")
+if not split_file.exists():
+    raise SystemExit(f"missing official DABStep split file at {split_file}")
+split_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}
+out = []
+with dataset.open(newline="") as f:
+    for row in csv.DictReader(f):
+        task_id = int(row["task_id"])
+        if task_id not in split_ids:
+            continue
+        if ids and str(task_id) not in ids:
+            continue
+        out.append({
+            "task_id": task_id,
+            "instructions": f"{row['question']}\\n{row['guidelines']}",
+            "all_golds_by_task": ast.literal_eval(str(row["all_golds_by_task"])),
+        })
+        if limit is not None and len(out) >= limit:
+            break
+if not out:
+    raise SystemExit(f"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}")
+print(json.dumps(out))
+`, [
+		dir,
+		split,
+		opts.limit === void 0 ? "" : String(opts.limit),
+		JSON.stringify(opts.ids ?? [])
+	]);
+	return selectRows(JSON.parse(stdout), opts, split, dir);
+}
+function createDabstepAdapter() {
+	const fixturesMode = process.env.DABSTEP_FIXTURES === "1";
+	return {
+		name: "dabstep",
+		output: dabstepAnswerOutput,
+		async preflight() {
+			if (fixturesMode) return;
+			const dir = dabstepDir();
+			if (!dir) throw new Error("DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.");
+			await assertOfficialFiles(dir, DEFAULT_SPLIT);
+			await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT);
+		},
+		async loadTasks(opts = {}) {
+			const split = opts.split ?? DEFAULT_SPLIT;
+			if (fixturesMode) return loadFixtures(opts, split);
+			const dir = dabstepDir();
+			if (!dir) throw new Error("DABSTEP_DIR is required to load official DABStep tasks");
+			return loadOfficialTasks(dir, opts, split);
+		},
+		async goldArtifact(task) {
+			const first = readMeta(task).golds[0];
+			if (!first) return void 0;
+			const value = first.value;
+			return value === void 0 ? void 0 : String(value);
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const dir = dabstepDir();
+			if (!dir) throw new Error("DABSTEP_DIR is required to judge DABStep tasks with the official grade.py");
+			const stdout = await runVenvScriptStdin(join(benchRoot, "scripts", "dabstep_judge.py"), ["--grade-file", gradeFile(dir)], JSON.stringify({
+				prediction: artifact,
+				golds: meta.golds
+			}), { cwd: benchRoot });
+			const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+			if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`);
+			const score = typeof report.score === "number" ? report.score : 0;
+			return {
+				resolved: report.correct === true,
+				score,
+				detail: JSON.stringify({
+					taskId: meta.taskId,
+					split: meta.split,
+					correct: report.correct
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createDabstepAdapter, dabstepAnswerOutput };
+
 //# sourceMappingURL=dabstep.js.map