@tangle-network/agent-bench 0.4.1 → 0.4.4

package/dist/benchmarks/open-rag-bench.js

@@ -1,9 +1,111 @@
-import {
-  createOpenRagBenchAdapter
-} from "../chunk-BZY5QARD.js";
-import "../chunk-X3BTXCJ4.js";
-import "../chunk-EIETHPD5.js";
-export {
-  createOpenRagBenchAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { FINAL_ANSWER_SENTINEL, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringFrom } from "./rag-shared.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/open-rag-bench.ts
+/**
+* Open RAG Bench adapter.
+*
+* This targets Vectara-style Open RAG Bench exports over PDF-derived text,
+* table, and image contexts. The deterministic judge scores final-answer
+* agreement and surfaces modality/document metadata for diagnostics.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "open-rag-bench.json");
+const dataFile = () => process.env.OPEN_RAG_BENCH_DATA_FILE;
+function rowToTask(raw, index) {
+	if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`);
+	const query = firstString(raw, [
+		"question",
+		"query",
+		"prompt"
+	]);
+	const goldAnswers = allStrings(raw, [
+		"answer",
+		"answers",
+		"reference",
+		"reference_answer",
+		"gold"
+	]);
+	if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`);
+	if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`);
+	const contexts = contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.chunks).length > 0 ? contextsFrom(raw.chunks) : contextsFrom(raw.pages);
+	const documentId = stringFrom(raw.document_id) ?? stringFrom(raw.doc_id) ?? stringFrom(raw.pdf_id) ?? stringFrom(raw.file_name) ?? stringFrom(raw.filename) ?? "unknown";
+	const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? "unknown";
+	const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}`;
+	const meta = {
+		benchmark: "open-rag-bench",
+		query,
+		goldAnswers,
+		contexts,
+		documentId,
+		modality
+	};
+	return {
+		id,
+		split: stringFrom(raw.split) ?? modality,
+		prompt: [
+			"Answer this Open RAG Bench PDF question using the supplied document context.",
+			"Use tables, text, and image-derived notes when present.",
+			"End with a single final line: `FINAL ANSWER: <answer>`.",
+			"",
+			`Question: ${query}`,
+			`Document: ${documentId}`,
+			`Modality: ${modality}`,
+			contexts.length > 0 ? `\nDocument context:\n${contextBlock(contexts)}` : void 0
+		].filter(Boolean).join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.goldAnswers)) throw new Error(`Open RAG Bench task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+async function loadRows(path) {
+	const rows = await readJsonRows(path);
+	if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`);
+	return rows;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`);
+	return selectTasks(rows.map(rowToTask), opts, "Open RAG Bench");
+}
+function createOpenRagBenchAdapter() {
+	const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === "1";
+	return {
+		name: "open-rag-bench",
+		output: ragAnswerOutput,
+		async preflight() {
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8");
+				return;
+			}
+			const path = dataFile();
+			if (!path) throw new Error("OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing.");
+			await loadRows(path);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const path = dataFile();
+			if (!path) throw new Error("OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks");
+			return selectTasks((await loadRows(path)).map(rowToTask), opts, "Open RAG Bench");
+		},
+		async goldArtifact(task) {
+			return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			return answerScoreToBenchScore(scoreAnswerArtifact(artifact, meta.goldAnswers), {
+				benchmark: meta.benchmark,
+				documentId: meta.documentId,
+				modality: meta.modality,
+				contextCount: meta.contexts.length
+			});
+		}
+	};
+}
+//#endregion
+export { createOpenRagBenchAdapter };
+
 //# sourceMappingURL=open-rag-bench.js.map

package/dist/benchmarks/open-rag-bench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"open-rag-bench.js","names":[],"sources":["../../src/benchmarks/open-rag-bench.ts"],"sourcesContent":["/**\n * Open RAG Bench adapter.\n *\n * This targets Vectara-style Open RAG Bench exports over PDF-derived text,\n * table, and image contexts. The deterministic judge scores final-answer\n * agreement and surfaces modality/document metadata for diagnostics.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'open-rag-bench.json')\n\ninterface OpenRagBenchMeta {\n  benchmark: 'open-rag-bench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  documentId: string\n  modality: string\n}\n\nconst dataFile = (): string | undefined => process.env.OPEN_RAG_BENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'prompt'])\n  const goldAnswers = allStrings(raw, ['answer', 'answers', 'reference', 'reference_answer', 'gold'])\n  if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`)\n  const contexts =\n    contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.chunks).length > 0\n        ? contextsFrom(raw.chunks)\n        : contextsFrom(raw.pages)\n  const documentId =\n    stringFrom(raw.document_id) ??\n    stringFrom(raw.doc_id) ??\n    stringFrom(raw.pdf_id) ??\n    stringFrom(raw.file_name) ??\n    stringFrom(raw.filename) ??\n    'unknown'\n  const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? 'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}`\n  const meta: OpenRagBenchMeta = {\n    benchmark: 'open-rag-bench',\n    query,\n    goldAnswers,\n    contexts,\n    documentId,\n    modality,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? modality,\n    prompt: [\n      'Answer this Open RAG Bench PDF question using the supplied document context.',\n      'Use tables, text, and image-derived notes when present.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Document: ${documentId}`,\n      `Modality: ${modality}`,\n      contexts.length > 0 ? `\\nDocument context:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): OpenRagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`Open RAG Bench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as OpenRagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'Open RAG Bench')\n}\n\nexport function createOpenRagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === '1'\n\n  return {\n    name: 'open-rag-bench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'Open RAG Bench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        documentId: meta.documentId,\n        modality: meta.modality,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA4BA,MAAM,WAAW,KAAK,WAAW,YAAY,qBAAqB;AAWlE,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;CACzD,IAAI,CAAC,SAAS,GAAG,GAAG,MAAM,IAAI,MAAM,sBAAsB,MAAM,mBAAmB;CACnF,MAAM,QAAQ,YAAY,KAAK;EAAC;EAAY;EAAS;CAAQ,CAAC;CAC9D,MAAM,cAAc,WAAW,KAAK;EAAC;EAAU;EAAW;EAAa;EAAoB;CAAM,CAAC;CAClG,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,sBAAsB,MAAM,wBAAwB;CAChF,IAAI,YAAY,WAAW,GAAG,MAAM,IAAI,MAAM,sBAAsB,MAAM,gBAAgB;CAC1F,MAAM,WACJ,aAAa,IAAI,QAAQ,CAAC,CAAC,SAAS,IAChC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,MAAM,CAAC,CAAC,SAAS,IAChC,aAAa,IAAI,MAAM,IACvB,aAAa,IAAI,KAAK;CAC9B,MAAM,aACJ,WAAW,IAAI,WAAW,KAC1B,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,SAAS,KACxB,WAAW,IAAI,QAAQ,KACvB;CACF,MAAM,WAAW,WAAW,IAAI,QAAQ,KAAK,WAAW,IAAI,eAAe,KAAK;CAChF,MAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,kBAAkB;CAC/E,MAAM,OAAyB;EAC7B,WAAW;EACX;EACA;EACA;EACA;EACA;CACF;CACA,OAAO;EACL;EACA,OAAO,WAAW,IAAI,KAAK,KAAK;EAChC,QAAQ;GACN;GACA;GACA;GACA;GACA,aAAa;GACb,aAAa;GACb,aAAa;GACb,SAAS,SAAS,IAAI,wBAAwB,aAAa,QAAQ,MAAM,KAAA;EAC3E,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAmC;CACnD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GACtC,MAAM,IAAI,MAAM,uBAAuB,KAAK,GAAG,kDAAkD;CAEnG,OAAO;AACT;AAEA,eAAe,SAAS,MAAkC;CACxD,MAAM,OAAO,MAAM,aAAa,IAAI;CACpC,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,8BAA8B,MAAM;CAC3E,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,wDAAwD,KAAK,OAAO,kBAAkB;CACnG,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,gBAAgB;AAChE;AAEA,SAAgB,4BAA8C;CAC5D,MAAM,eAAe,QAAQ,IAAI,4BAA4B;CAE7D,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM;IAC/B;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,+MACF;GAEF,MAAM,SAAS,IAAI;EACrB;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,mEAAmE;GAC9F,OAAO,aAAa,MAAM,SAAS,IAAI,EAAA,CAAG,IAAI,SAAS,GAAG,MAAM,gBAAgB;EAClF;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,GAAG,sBAAsB,GAAG,SAAS,IAAI,CAAC,CAAC,YAAY,MAAM;EACtE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAE1B,OAAO,wBADO,oBAAoB,UAAU,KAAK,WACd,GAAG;IACpC,WAAW,KAAK;IAChB,YAAY,KAAK;IACjB,UAAU,KAAK;IACf,cAAc,KAAK,SAAS;GAC9B,CAAC;EACH;CACF;AACF"}

package/dist/benchmarks/programbench.d.ts

@@ -1,31 +1,6 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom
- * reverse-engineering: rebuild a black-box executable's behavior from scratch.
- * The agent is given only the gold `./executable` (run-only) + stripped docs and
- * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`
- * with identical observable behavior. Judge = the official `programbench`
- * harness: it extracts the submission, runs compile.sh in the per-task cleanroom
- * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via
- * `programbench blob sync`). Score = fraction of non-ignored tests passed
- * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests
- * pass. Fully deterministic — no LLM judge.
- *
- * OutputAdapter is stream-only, so the worker emits its codebase as fenced
- * `path:`-prefixed file blocks (including compile.sh); the adapter materializes
- * those into submission.tar.gz. Test execution is delegated to the real
- * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT
- * reimplemented here.
- *
- * Requires for a live run: the bench `.venv` with `programbench` installed +
- * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for
- * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load
- * the committed instance ids (bench/fixtures/programbench.json); judging still
- * needs the harness + Docker and fails loud without them — never a fabricated score.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/programbench.d.ts
 /**
  * Parse the worker stream into the submission text the driver materializes: the
  * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to
@@ -34,5 +9,6 @@ import { BenchmarkAdapter } from './types.js';
  */
 declare const programbenchSubmissionOutput: OutputAdapter<string>;
 declare function createProgrambenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createProgrambenchAdapter, programbenchSubmissionOutput };
+//# sourceMappingURL=programbench.d.ts.map

package/dist/benchmarks/programbench.js

@@ -1,10 +1,162 @@
-import {
-  createProgrambenchAdapter,
-  programbenchSubmissionOutput
-} from "../chunk-QZZEAHWJ.js";
-import "../chunk-EIETHPD5.js";
-export {
-  createProgrambenchAdapter,
-  programbenchSubmissionOutput
-};
+import { benchRoot, preflightVenvImports, runVenvScriptStdin } from "./_harness.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/programbench.ts
+/**
+* ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom
+* reverse-engineering: rebuild a black-box executable's behavior from scratch.
+* The agent is given only the gold `./executable` (run-only) + stripped docs and
+* must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`
+* with identical observable behavior. Judge = the official `programbench`
+* harness: it extracts the submission, runs compile.sh in the per-task cleanroom
+* Docker image, then runs the HIDDEN behavioral pytest suites (pulled via
+* `programbench blob sync`). Score = fraction of non-ignored tests passed
+* (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests
+* pass. Fully deterministic — no LLM judge.
+*
+* OutputAdapter is stream-only, so the worker emits its codebase as fenced
+* `path:`-prefixed file blocks (including compile.sh); the adapter materializes
+* those into submission.tar.gz. Test execution is delegated to the real
+* `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT
+* reimplemented here.
+*
+* Requires for a live run: the bench `.venv` with `programbench` installed +
+* Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for
+* the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load
+* the committed instance ids (bench/fixtures/programbench.json); judging still
+* needs the harness + Docker and fails loud without them — never a fabricated score.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "programbench.json");
+const TESTS_DATASET = "programbench/ProgramBench-Tests";
+const TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`;
+const PROMPT = [
+	"This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.",
+	"Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.",
+	"You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.",
+	"Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence."
+].join("\n");
+function fixtureToTask(f) {
+	const meta = {
+		instanceId: f.instance_id,
+		imageName: f.image_name
+	};
+	return {
+		id: f.instance_id,
+		prompt: PROMPT,
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.instanceId !== "string") throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`);
+	return md;
+}
+function selectTasks(fixtures, opts) {
+	let tasks = fixtures.map(fixtureToTask);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((t) => want.has(t.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	return tasks;
+}
+/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */
+function imageName(instanceId) {
+	return instanceId.split(".")[0].replace("__", "/");
+}
+async function loadFixtures(opts) {
+	const fixtures = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`);
+	return selectTasks(fixtures, opts);
+}
+/** Enumerate real instance ids from the HF Tests dataset tree (one dir per
+*  instance). Throws on a non-OK response (fail loud). */
+async function fetchInstanceIds() {
+	const res = await fetch(TREE_API);
+	if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`);
+	const ids = (await res.json()).filter((e) => e.type === "directory" && e.path.includes("__")).map((e) => e.path);
+	if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`);
+	return ids.map((id) => ({
+		instance_id: id,
+		image_name: imageName(id)
+	}));
+}
+/**
+* Run the official programbench harness for one instance over the worker's
+* file-manifest submission. The driver builds the run-dir layout (one folder
+* holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,
+* then parses <instance_id>.eval.json applying the tests.json ignore mask via
+* `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.
+*/
+async function runHarness(meta, submission) {
+	const judge = join(benchRoot, "scripts", "programbench_judge.py");
+	let stdout;
+	try {
+		stdout = await runVenvScriptStdin(judge, ["--instance", meta.instanceId], submission, { cwd: benchRoot });
+	} catch (err) {
+		const e = err;
+		throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`);
+	}
+	const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+	if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`);
+	if (typeof report.passed !== "number" || typeof report.total !== "number") throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`);
+	const score = report.total > 0 ? report.passed / report.total : 0;
+	return {
+		resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),
+		score,
+		detail: JSON.stringify({
+			instanceId: meta.instanceId,
+			passed: report.passed,
+			total: report.total
+		})
+	};
+}
+/**
+* Parse the worker stream into the submission text the driver materializes: the
+* concatenation of every ```path:<p>``` fenced block. Passed through verbatim to
+* the python driver, which tars it. Empty when the worker emitted no file block
+* (fail-closed → the harness scores a missing compile.sh as 0).
+*/
+const programbenchSubmissionOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	const blocks = [...text.matchAll(/```path:([^\n`]+)\n([\s\S]*?)```/g)];
+	if (blocks.length === 0) return "";
+	return blocks.map((m) => `===FILE:${m[1].trim()}===\n${m[2]}`).join("\n");
+} };
+function createProgrambenchAdapter() {
+	const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === "1";
+	return {
+		name: "programbench",
+		output: programbenchSubmissionOutput,
+		async preflight() {
+			await preflightVenvImports({
+				modules: ["programbench"],
+				requireDocker: true,
+				fix: "Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; (2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; (3) HF access for the hidden test blobs (the driver runs `programbench blob sync`). Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline."
+			});
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			let fixtures;
+			try {
+				fixtures = await fetchInstanceIds();
+			} catch (err) {
+				console.warn(`[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`);
+				return loadFixtures(opts);
+			}
+			return selectTasks(fixtures, opts);
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			return runHarness(readMeta(task), artifact);
+		}
+	};
+}
+//#endregion
+export { createProgrambenchAdapter, programbenchSubmissionOutput };
+
 //# sourceMappingURL=programbench.js.map

package/dist/benchmarks/programbench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"programbench.js","names":[],"sources":["../../src/benchmarks/programbench.ts"],"sourcesContent":["/**\n * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom\n * reverse-engineering: rebuild a black-box executable's behavior from scratch.\n * The agent is given only the gold `./executable` (run-only) + stripped docs and\n * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`\n * with identical observable behavior. Judge = the official `programbench`\n * harness: it extracts the submission, runs compile.sh in the per-task cleanroom\n * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via\n * `programbench blob sync`). Score = fraction of non-ignored tests passed\n * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests\n * pass. Fully deterministic — no LLM judge.\n *\n * OutputAdapter is stream-only, so the worker emits its codebase as fenced\n * `path:`-prefixed file blocks (including compile.sh); the adapter materializes\n * those into submission.tar.gz. Test execution is delegated to the real\n * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT\n * reimplemented here.\n *\n * Requires for a live run: the bench `.venv` with `programbench` installed +\n * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for\n * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load\n * the committed instance ids (bench/fixtures/programbench.json); judging still\n * needs the harness + Docker and fails loud without them — never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'programbench.json')\n\nconst TESTS_DATASET = 'programbench/ProgramBench-Tests'\nconst TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`\n\ninterface PbFixture {\n  instance_id: string\n  image_name: string\n  note?: string\n}\n\ninterface PbMeta {\n  instanceId: string\n  imageName: string\n}\n\nconst PROMPT = [\n  'This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.',\n  'Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.',\n  'You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.',\n  'Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence.',\n].join('\\n')\n\nfunction fixtureToTask(f: PbFixture): BenchTask {\n  const meta: PbMeta = { instanceId: f.instance_id, imageName: f.image_name }\n  return {\n    id: f.instance_id,\n    prompt: PROMPT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): PbMeta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string') {\n    throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`)\n  }\n  return md as unknown as PbMeta\n}\n\nfunction selectTasks(fixtures: PbFixture[], opts: LoadOptions): BenchTask[] {\n  let tasks = fixtures.map(fixtureToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\n/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */\nfunction imageName(instanceId: string): string {\n  return instanceId.split('.')[0].replace('__', '/')\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixtures = JSON.parse(await readFile(FIXTURES, 'utf8')) as PbFixture[]\n  console.warn(\n    `[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`,\n  )\n  return selectTasks(fixtures, opts)\n}\n\n/** Enumerate real instance ids from the HF Tests dataset tree (one dir per\n *  instance). Throws on a non-OK response (fail loud). */\nasync function fetchInstanceIds(): Promise<PbFixture[]> {\n  const res = await fetch(TREE_API)\n  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`)\n  const tree = (await res.json()) as Array<{ path: string; type: string }>\n  const ids = tree.filter((e) => e.type === 'directory' && e.path.includes('__')).map((e) => e.path)\n  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`)\n  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }))\n}\n\n/**\n * Run the official programbench harness for one instance over the worker's\n * file-manifest submission. The driver builds the run-dir layout (one folder\n * holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,\n * then parses <instance_id>.eval.json applying the tests.json ignore mask via\n * `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.\n */\nasync function runHarness(meta: PbMeta, submission: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'programbench_judge.py')\n  let stdout: string\n  try {\n    // The submission manifest is piped to the driver's stdin via the shared\n    // stdin-aware runner — execFile's `input` option is not honored async and\n    // hangs the driver's sys.stdin.read() forever.\n    stdout = await runVenvScriptStdin(judge, ['--instance', meta.instanceId], submission, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    resolved?: boolean\n    error?: string\n  }\n  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passed / report.total : 0\n  return {\n    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\n/**\n * Parse the worker stream into the submission text the driver materializes: the\n * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to\n * the python driver, which tars it. Empty when the worker emitted no file block\n * (fail-closed → the harness scores a missing compile.sh as 0).\n */\nexport const programbenchSubmissionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const blocks = [...text.matchAll(/```path:([^\\n`]+)\\n([\\s\\S]*?)```/g)]\n    if (blocks.length === 0) return ''\n    // Re-serialize in the same path:<p> envelope the driver splits on.\n    return blocks.map((m) => `===FILE:${m[1].trim()}===\\n${m[2]}`).join('\\n')\n  },\n}\n\nexport function createProgrambenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === '1'\n\n  return {\n    name: 'programbench',\n    output: programbenchSubmissionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['programbench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; ` +\n          `(2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; ` +\n          `(3) HF access for the hidden test blobs (the driver runs \\`programbench blob sync\\`). ` +\n          `Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let fixtures: PbFixture[]\n      try {\n        fixtures = await fetchInstanceIds()\n      } catch (err) {\n        console.warn(\n          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectTasks(fixtures, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the original repo's source (stripped from the task image),\n      // not redistributable as a portable submission string. Judge correctness is\n      // proven by running the real harness on a real solve, not a synthetic gold.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,MAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEhE,MAAM,gBAAgB;AACtB,MAAM,WAAW,uCAAuC,cAAc;AAatE,MAAM,SAAS;CACb;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,cAAc,GAAyB;CAC9C,MAAM,OAAe;EAAE,YAAY,EAAE;EAAa,WAAW,EAAE;CAAW;CAC1E,OAAO;EACL,IAAI,EAAE;EACN,QAAQ;EACR,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAyB;CACzC,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,eAAe,UAClC,MAAM,IAAI,MAAM,qBAAqB,KAAK,GAAG,6DAA6D;CAE5G,OAAO;AACT;AAEA,SAAS,YAAY,UAAuB,MAAgC;CAC1E,IAAI,QAAQ,SAAS,IAAI,aAAa;CACtC,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;;AAGA,SAAS,UAAU,YAA4B;CAC7C,OAAO,WAAW,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,MAAM,GAAG;AACnD;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,WAAW,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KACN,oDAAoD,SAAS,OAAO,+BAA+B,SAAS,oBAC9G;CACA,OAAO,YAAY,UAAU,IAAI;AACnC;;;AAIA,eAAe,mBAAyC;CACtD,MAAM,MAAM,MAAM,MAAM,QAAQ;CAChC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,0BAA0B,IAAI,OAAO,IAAI,UAAU;CAEhF,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,MAAM,EAAE,SAAS,eAAe,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,IAAI;CACjG,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,qCAAqC,cAAc,MAAM;CAC/F,OAAO,IAAI,KAAK,QAAQ;EAAE,aAAa;EAAI,YAAY,UAAU,EAAE;CAAE,EAAE;AACzE;;;;;;;;AASA,eAAe,WAAW,MAAc,YAAyC;CAC/E,MAAM,QAAQ,KAAK,WAAW,WAAW,uBAAuB;CAChE,IAAI;CACJ,IAAI;EAIF,SAAS,MAAM,mBAAmB,OAAO,CAAC,cAAc,KAAK,UAAU,GAAG,YAAY,EAAE,KAAK,UAAU,CAAC;CAC1G,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,mCAAmC,KAAK,WAAW,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CACpH;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAMlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,kCAAkC,KAAK,WAAW,IAAI,OAAO,OAAO;CACtG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,kDAAkD,OAAO,MAAM,GAAG,GAAG,GAAG;CAE1F,MAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;CAChE,OAAO;EACL,UAAU,OAAO,aAAa,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;EAC3E;EACA,QAAQ,KAAK,UAAU;GAAE,YAAY,KAAK;GAAY,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACpG;AACF;;;;;;;AAQA,MAAa,+BAAsD,EACjE,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,SAAS,mCAAmC,CAAC;CACrE,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,OAAO,OAAO,KAAK,MAAM,WAAW,EAAE,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI;AAC1E,EACF;AAEA,SAAgB,4BAA8C;CAC5D,MAAM,eAAe,QAAQ,IAAI,0BAA0B;CAE3D,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,cAAc;IACxB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,WAAW,MAAM,iBAAiB;GACpC,SAAS,KAAK;IACZ,QAAQ,KACN,0CAA0C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAA2C,UAChI;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,YAAY,UAAU,IAAI;EACnC;EAEA,MAAM,eAAe,CAKrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,WADM,SAAS,IACD,GAAG,QAAQ;EAClC;CACF;AACF"}

package/dist/benchmarks/rag-shared.d.ts

@@ -1,23 +1,23 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchScore, BenchTask, LoadOptions } from './types.js';
-
+import { BenchScore, BenchTask, LoadOptions } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/rag-shared.d.ts
 declare const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
 interface RagContext {
-    id: string;
-    text: string;
-    title?: string;
-    source?: string;
-    relevant?: boolean;
+  id: string;
+  text: string;
+  title?: string;
+  source?: string;
+  relevant?: boolean;
 }
 interface RagAnswerScore {
-    resolved: boolean;
-    score: number;
-    finalAnswer: string;
-    bestGold: string | null;
-    exact: boolean;
-    numeric: boolean;
-    f1: number;
-    threshold: number;
+  resolved: boolean;
+  score: number;
+  finalAnswer: string;
+  bestGold: string | null;
+  exact: boolean;
+  numeric: boolean;
+  f1: number;
+  threshold: number;
 }
 declare const ragAnswerOutput: OutputAdapter<string>;
 declare function parseFinalAnswer(artifact: string): string;
@@ -25,8 +25,8 @@ declare function parseCitations(artifact: string): string[];
 declare function normalizeAnswer(input: string): string;
 declare function tokenF1(candidate: string, gold: string): number;
 declare function scoreAnswerArtifact(artifact: string, golds: readonly string[], options?: {
-    threshold?: number;
-    numericTolerance?: number;
+  threshold?: number;
+  numericTolerance?: number;
 }): RagAnswerScore;
 declare function answerScoreToBenchScore(score: RagAnswerScore, detail: Record<string, unknown>): BenchScore;
 declare function readJsonRows(path: string): Promise<unknown[]>;
@@ -38,5 +38,6 @@ declare function allStrings(row: Record<string, unknown>, keys: readonly string[
 declare function contextsFrom(value: unknown): RagContext[];
 declare function contextBlock(contexts: readonly RagContext[]): string;
 declare function isObject(value: unknown): value is Record<string, unknown>;
-
-export { FINAL_ANSWER_SENTINEL, type RagAnswerScore, type RagContext, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, normalizeAnswer, parseCitations, parseFinalAnswer, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringArrayFrom, stringFrom, tokenF1 };
+//#endregion
+export { FINAL_ANSWER_SENTINEL, RagAnswerScore, RagContext, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, normalizeAnswer, parseCitations, parseFinalAnswer, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringArrayFrom, stringFrom, tokenF1 };
+//# sourceMappingURL=rag-shared.d.ts.map