@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/hotpotqa.js

@@ -1,15 +1,251 @@
-import {
-  createHotpotqaAdapter,
-  exactMatch,
-  normalizeAnswer,
-  parseFinalAnswer,
-  tokenF1
-} from "../chunk-PA2ZKHJC.js";
-export {
-  createHotpotqaAdapter,
-  exactMatch,
-  normalizeAnswer,
-  parseFinalAnswer,
-  tokenF1
-};
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { execFile } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+//#region src/benchmarks/hotpotqa.ts
+/**
+* HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').
+* Multi-hop factoid QA. Worker artifact = a single free-text final answer string.
+*
+* Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no
+* LLM. Both the predicted final answer and the gold are normalized (lowercase,
+* strip articles a/an/the, strip punctuation, collapse whitespace), then scored
+* by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS
+* (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the
+* suite a judge that needs no model tokens at all.
+*
+* metadata carries the gold answer + supporting_facts (the title/sent_id pairs of
+* the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,
+* never part of the score.
+*
+* Requires for a live run: the bench `.venv` with `datasets` installed + network
+* to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the
+* committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.
+*/
+const execFileAsync = promisify(execFile);
+const BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+const PY = join(BENCH_ROOT, ".venv", "bin", "python");
+const FIXTURES = join(BENCH_ROOT, "fixtures", "hotpotqa.json");
+const DATASET = "hotpotqa/hotpot_qa";
+const DATASET_CONFIG = "distractor";
+const DATASET_SPLIT = "validation";
+const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+/** Default F1 pass threshold; HotpotQA's leaderboard reports EM and F1 separately. */
+const DEFAULT_F1_PASS = .6;
+/** Worker contract appended to every task prompt. The judge keys off the sentinel. */
+const WORKER_CONTRACT = [
+	"",
+	"Answer this multi-hop question. Reason across the facts you need, then commit to a single short answer.",
+	`End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
+	"The answer after the sentinel must be the bare value only (no explanation on that line)."
+].join("\n");
+/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */
+async function py(script, args = []) {
+	const { stdout } = await execFileAsync(PY, [
+		"-c",
+		script,
+		...args
+	], { maxBuffer: 1024 * 1024 * 256 });
+	return stdout;
+}
+const ARTICLES = /* @__PURE__ */ new Set([
+	"a",
+	"an",
+	"the"
+]);
+/**
+* The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,
+* drop articles (a/an/the), collapse whitespace. Token comparisons run on the
+* output of this exactly as the published evaluator does.
+*/
+function normalizeAnswer(input) {
+	return input.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((t) => t.length > 0).filter((t) => !ARTICLES.has(t)).join(" ").trim();
+}
+/** Normalized-token list (the F1 bag-of-tokens unit). */
+function answerTokens(input) {
+	const n = normalizeAnswer(input);
+	return n.length === 0 ? [] : n.split(" ");
+}
+/** Exact match on the normalized strings. */
+function exactMatch(prediction, gold) {
+	return normalizeAnswer(prediction) === normalizeAnswer(gold);
+}
+/**
+* Token-level F1, the SQuAD/HotpotQA definition. Multiset (bag) intersection of
+* normalized tokens. Mirrors the published evaluator's special-case handling of
+* yes/no/noanswer and empty bags: if either side is empty, F1 is 1 only when both
+* are empty, else 0.
+*/
+function tokenF1(prediction, gold) {
+	const predTokens = answerTokens(prediction);
+	const goldTokens = answerTokens(gold);
+	if (predTokens.length === 0 || goldTokens.length === 0) return predTokens.length === 0 && goldTokens.length === 0 ? 1 : 0;
+	const goldCounts = /* @__PURE__ */ new Map();
+	for (const t of goldTokens) goldCounts.set(t, (goldCounts.get(t) ?? 0) + 1);
+	let common = 0;
+	for (const t of predTokens) {
+		const left = goldCounts.get(t);
+		if (left !== void 0 && left > 0) {
+			common += 1;
+			goldCounts.set(t, left - 1);
+		}
+	}
+	if (common === 0) return 0;
+	const precision = common / predTokens.length;
+	const recall = common / goldTokens.length;
+	return 2 * precision * recall / (precision + recall);
+}
+/** Read the configured F1 pass threshold; fail loud on a malformed override. */
+function f1PassThreshold() {
+	const raw = process.env.HOTPOTQA_F1_PASS;
+	if (raw === void 0 || raw.length === 0) return DEFAULT_F1_PASS;
+	const v = Number(raw);
+	if (!Number.isFinite(v) || v < 0 || v > 1) throw new Error(`HOTPOTQA_F1_PASS must be a number in [0,1], got ${JSON.stringify(raw)}`);
+	return v;
+}
+/**
+* Parse the worker artifact into the final answer string.
+* Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+* fall back to the trimmed last non-empty line. Returns '' when nothing is
+* parseable (fail-closed — never guess), which judge() counts as resolved=false.
+*/
+function parseFinalAnswer(artifact) {
+	const lines = artifact.split(/\r?\n/);
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const line = lines[i] ?? "";
+		const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
+		if (idx !== -1) return line.slice(idx + 13).trim();
+	}
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const t = (lines[i] ?? "").trim();
+		if (t.length > 0) return t;
+	}
+	return "";
+}
+function rowToTask(row) {
+	const meta = {
+		gold: row.answer,
+		supportingFacts: row.supporting_facts ?? {
+			title: [],
+			sent_id: []
+		},
+		type: row.type ?? "",
+		level: row.level ?? "",
+		rawQuestion: row.question
+	};
+	return {
+		id: `hotpotqa-${row.id}`,
+		split: DATASET_SPLIT,
+		prompt: row.question + WORKER_CONTRACT,
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.gold !== "string") throw new Error(`HotpotQA task ${task.id} missing metadata.gold — loadTasks did not populate it`);
+	return md;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[hotpotqa] HOTPOTQA_FIXTURES=1 — loading ${rows.length} committed fixtures from ${FIXTURES} (no HF download)`);
+	let tasks = rows.map(rowToTask);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((t) => want.has(t.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	return tasks;
+}
+function createHotpotqaAdapter() {
+	const fixturesMode = process.env.HOTPOTQA_FIXTURES === "1";
+	f1PassThreshold();
+	return {
+		name: "hotpotqa",
+		async preflight() {
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8").catch((err) => {
+					throw new Error(`HOTPOTQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+				});
+				return;
+			}
+			try {
+				await py(`from datasets import load_dataset
+load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
+print('ok')`);
+			} catch (err) {
+				const msg = err instanceof Error ? err.message : String(err);
+				throw new Error(`hotpotqa preflight failed: ${msg}\nFix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; (2) ensure network access to Hugging Face for ${DATASET} (${DATASET_CONFIG}) ; or set HOTPOTQA_FIXTURES=1 to run against the committed fixtures offline.`);
+			}
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const limit = opts.limit ?? 10;
+			const stdout = await py(`
+import json, sys
+from datasets import load_dataset
+ds = load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})
+ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
+out = []
+for r in ds:
+    rid = f"hotpotqa-{r['id']}"
+    if ids is not None and rid not in ids:
+        continue
+    sf = r.get('supporting_facts', {}) or {}
+    out.append({
+        "id": r["id"],
+        "question": r.get("question", ""),
+        "answer": r.get("answer", ""),
+        "type": str(r.get("type", "")),
+        "level": str(r.get("level", "")),
+        "supporting_facts": {
+            "title": list(sf.get("title", [])),
+            "sent_id": [int(x) for x in sf.get("sent_id", [])],
+        },
+    })
+    if ids is None and len(out) >= ${limit}:
+        break
+print(json.dumps(out))
+`, [opts.ids ? JSON.stringify(opts.ids) : ""]);
+			return JSON.parse(stdout).map(rowToTask);
+		},
+		async goldArtifact(task) {
+			const meta = readMeta(task);
+			return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const finalAnswer = parseFinalAnswer(artifact);
+			if (finalAnswer.length === 0) return {
+				resolved: false,
+				score: 0,
+				detail: JSON.stringify({
+					reason: "no parseable answer",
+					em: false,
+					f1: 0,
+					normalizedGold: normalizeAnswer(meta.gold)
+				})
+			};
+			const em = exactMatch(finalAnswer, meta.gold);
+			const f1 = tokenF1(finalAnswer, meta.gold);
+			const threshold = f1PassThreshold();
+			return {
+				resolved: em || f1 >= threshold,
+				score: f1,
+				detail: JSON.stringify({
+					em,
+					f1,
+					threshold,
+					normalizedAnswer: normalizeAnswer(finalAnswer),
+					normalizedGold: normalizeAnswer(meta.gold),
+					type: meta.type,
+					level: meta.level,
+					supportingFacts: meta.supportingFacts
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createHotpotqaAdapter, exactMatch, normalizeAnswer, parseFinalAnswer, tokenF1 };
+
 //# sourceMappingURL=hotpotqa.js.map

package/dist/benchmarks/hotpotqa.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"hotpotqa.js","names":[],"sources":["../../src/benchmarks/hotpotqa.ts"],"sourcesContent":["/**\n * HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').\n * Multi-hop factoid QA. Worker artifact = a single free-text final answer string.\n *\n * Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no\n * LLM. Both the predicted final answer and the gold are normalized (lowercase,\n * strip articles a/an/the, strip punctuation, collapse whitespace), then scored\n * by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS\n * (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the\n * suite a judge that needs no model tokens at all.\n *\n * metadata carries the gold answer + supporting_facts (the title/sent_id pairs of\n * the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,\n * never part of the score.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed + network\n * to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the\n * committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'hotpotqa.json')\n\nconst DATASET = 'hotpotqa/hotpot_qa'\nconst DATASET_CONFIG = 'distractor'\nconst DATASET_SPLIT = 'validation'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Default F1 pass threshold; HotpotQA's leaderboard reports EM and F1 separately. */\nconst DEFAULT_F1_PASS = 0.6\n\n/** Worker contract appended to every task prompt. The judge keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Answer this multi-hop question. Reason across the facts you need, then commit to a single short answer.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\ninterface SupportingFacts {\n  title: string[]\n  sent_id: number[]\n}\n\ninterface HotpotRow {\n  id: string\n  question: string\n  answer: string\n  type: string\n  level: string\n  supporting_facts: SupportingFacts\n}\n\ninterface HotpotMeta {\n  gold: string\n  supportingFacts: SupportingFacts\n  type: string\n  level: string\n  rawQuestion: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\n\n/**\n * The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,\n * drop articles (a/an/the), collapse whitespace. Token comparisons run on the\n * output of this exactly as the published evaluator does.\n */\nexport function normalizeAnswer(input: string): string {\n  const lower = input.toLowerCase()\n  // strip punctuation: keep word chars + whitespace only\n  const noPunct = lower.replace(/[^\\w\\s]/g, ' ')\n  const tokens = noPunct\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Normalized-token list (the F1 bag-of-tokens unit). */\nfunction answerTokens(input: string): string[] {\n  const n = normalizeAnswer(input)\n  return n.length === 0 ? [] : n.split(' ')\n}\n\n/** Exact match on the normalized strings. */\nexport function exactMatch(prediction: string, gold: string): boolean {\n  return normalizeAnswer(prediction) === normalizeAnswer(gold)\n}\n\n/**\n * Token-level F1, the SQuAD/HotpotQA definition. Multiset (bag) intersection of\n * normalized tokens. Mirrors the published evaluator's special-case handling of\n * yes/no/noanswer and empty bags: if either side is empty, F1 is 1 only when both\n * are empty, else 0.\n */\nexport function tokenF1(prediction: string, gold: string): number {\n  const predTokens = answerTokens(prediction)\n  const goldTokens = answerTokens(gold)\n  if (predTokens.length === 0 || goldTokens.length === 0) {\n    return predTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const goldCounts = new Map<string, number>()\n  for (const t of goldTokens) goldCounts.set(t, (goldCounts.get(t) ?? 0) + 1)\n  let common = 0\n  for (const t of predTokens) {\n    const left = goldCounts.get(t)\n    if (left !== undefined && left > 0) {\n      common += 1\n      goldCounts.set(t, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / predTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\n/** Read the configured F1 pass threshold; fail loud on a malformed override. */\nfunction f1PassThreshold(): number {\n  const raw = process.env.HOTPOTQA_F1_PASS\n  if (raw === undefined || raw.length === 0) return DEFAULT_F1_PASS\n  const v = Number(raw)\n  if (!Number.isFinite(v) || v < 0 || v > 1) {\n    throw new Error(`HOTPOTQA_F1_PASS must be a number in [0,1], got ${JSON.stringify(raw)}`)\n  }\n  return v\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the trimmed last non-empty line. Returns '' when nothing is\n * parseable (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\nfunction rowToTask(row: HotpotRow): BenchTask {\n  const meta: HotpotMeta = {\n    gold: row.answer,\n    supportingFacts: row.supporting_facts ?? { title: [], sent_id: [] },\n    type: row.type ?? '',\n    level: row.level ?? '',\n    rawQuestion: row.question,\n  }\n  return {\n    id: `hotpotqa-${row.id}`,\n    split: DATASET_SPLIT,\n    prompt: row.question + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): HotpotMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`HotpotQA task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as HotpotMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as HotpotRow[]\n  console.warn(\n    `[hotpotqa] HOTPOTQA_FIXTURES=1 — loading ${rows.length} committed fixtures from ${FIXTURES} (no HF download)`,\n  )\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createHotpotqaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.HOTPOTQA_FIXTURES === '1'\n  // Validate the threshold at construction so a malformed env fails before any run.\n  f1PassThreshold()\n\n  return {\n    name: 'hotpotqa',\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `HOTPOTQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `hotpotqa preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} (${DATASET_CONFIG}) ; ` +\n            `or set HOTPOTQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, ${JSON.stringify(DATASET_CONFIG)}, split=${JSON.stringify(DATASET_SPLIT)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    rid = f\"hotpotqa-{r['id']}\"\n    if ids is not None and rid not in ids:\n        continue\n    sf = r.get('supporting_facts', {}) or {}\n    out.append({\n        \"id\": r[\"id\"],\n        \"question\": r.get(\"question\", \"\"),\n        \"answer\": r.get(\"answer\", \"\"),\n        \"type\": str(r.get(\"type\", \"\")),\n        \"level\": str(r.get(\"level\", \"\")),\n        \"supporting_facts\": {\n            \"title\": list(sf.get(\"title\", [])),\n            \"sent_id\": [int(x) for x in sf.get(\"sent_id\", [])],\n        },\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as HotpotRow[]\n      return rows.map(rowToTask)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            reason: 'no parseable answer',\n            em: false,\n            f1: 0,\n            normalizedGold: normalizeAnswer(meta.gold),\n          }),\n        }\n      }\n\n      const em = exactMatch(finalAnswer, meta.gold)\n      const f1 = tokenF1(finalAnswer, meta.gold)\n      const threshold = f1PassThreshold()\n      const resolved = em || f1 >= threshold\n      return {\n        resolved,\n        score: f1,\n        detail: JSON.stringify({\n          em,\n          f1,\n          threshold,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          type: meta.type,\n          level: meta.level,\n          supportingFacts: meta.supportingFacts,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,MAAM,UAAU;AAChB,MAAM,iBAAiB;AACvB,MAAM,gBAAgB;AACtB,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;;AAGxB,MAAM,kBAAkB;CACtB;CACA;CACA,iDAAiD,sBAAsB;CACvE;AACF,CAAC,CAAC,KAAK,IAAI;;AAyBX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;AAEA,MAAM,2BAAW,IAAI,IAAI;CAAC;CAAK;CAAM;AAAK,CAAC;;;;;;AAO3C,SAAgB,gBAAgB,OAAuB;CAQrD,OAPc,MAAM,YAEA,CAAC,CAAC,QAAQ,YAAY,GACrB,CAAC,CACnB,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC,CAC3B,QAAQ,MAAM,CAAC,SAAS,IAAI,CAAC,CACpB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;AAC/B;;AAGA,SAAS,aAAa,OAAyB;CAC7C,MAAM,IAAI,gBAAgB,KAAK;CAC/B,OAAO,EAAE,WAAW,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG;AAC1C;;AAGA,SAAgB,WAAW,YAAoB,MAAuB;CACpE,OAAO,gBAAgB,UAAU,MAAM,gBAAgB,IAAI;AAC7D;;;;;;;AAQA,SAAgB,QAAQ,YAAoB,MAAsB;CAChE,MAAM,aAAa,aAAa,UAAU;CAC1C,MAAM,aAAa,aAAa,IAAI;CACpC,IAAI,WAAW,WAAW,KAAK,WAAW,WAAW,GACnD,OAAO,WAAW,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;CAElE,MAAM,6BAAa,IAAI,IAAoB;CAC3C,KAAK,MAAM,KAAK,YAAY,WAAW,IAAI,IAAI,WAAW,IAAI,CAAC,KAAK,KAAK,CAAC;CAC1E,IAAI,SAAS;CACb,KAAK,MAAM,KAAK,YAAY;EAC1B,MAAM,OAAO,WAAW,IAAI,CAAC;EAC7B,IAAI,SAAS,KAAA,KAAa,OAAO,GAAG;GAClC,UAAU;GACV,WAAW,IAAI,GAAG,OAAO,CAAC;EAC5B;CACF;CACA,IAAI,WAAW,GAAG,OAAO;CACzB,MAAM,YAAY,SAAS,WAAW;CACtC,MAAM,SAAS,SAAS,WAAW;CACnC,OAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;;AAGA,SAAS,kBAA0B;CACjC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,QAAQ,KAAA,KAAa,IAAI,WAAW,GAAG,OAAO;CAClD,MAAM,IAAI,OAAO,GAAG;CACpB,IAAI,CAAC,OAAO,SAAS,CAAC,KAAK,IAAI,KAAK,IAAI,GACtC,MAAM,IAAI,MAAM,mDAAmD,KAAK,UAAU,GAAG,GAAG;CAE1F,OAAO;AACT;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;AAEA,SAAS,UAAU,KAA2B;CAC5C,MAAM,OAAmB;EACvB,MAAM,IAAI;EACV,iBAAiB,IAAI,oBAAoB;GAAE,OAAO,CAAC;GAAG,SAAS,CAAC;EAAE;EAClE,MAAM,IAAI,QAAQ;EAClB,OAAO,IAAI,SAAS;EACpB,aAAa,IAAI;CACnB;CACA,OAAO;EACL,IAAI,YAAY,IAAI;EACpB,OAAO;EACP,QAAQ,IAAI,WAAW;EACvB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA6B;CAC7C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAC5B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,uDAAuD;CAElG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KACN,4CAA4C,KAAK,OAAO,2BAA2B,SAAS,kBAC9F;CACA,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,gBAAgB;CAEhB,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,2BAA2B,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC1F;IACF,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,OAAO,EAAE,IAAI,KAAK,UAAU,cAAc,EAAE,UAAU,KAAK,UAAU,aAAa,EAAE;YAE1G;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,8BAA8B,IAAI,iIAEiB,QAAQ,IAAI,eAAe,8EAEhF;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,QAAQ,KAAK,SAAS;GA2B5B,MAAM,SAAS,MAAM,GAAG;;;oBAvBV,KAAK,UAAU,OAAO,EAAE,IAAI,KAAK,UAAU,cAAc,EAAE,UAAU,KAAK,UAAU,aAAa,EAAE;;;;;;;;;;;;;;;;;;;qCAmBlF,MAAM;;;GAIL,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAE1E,OADa,KAAK,MAAM,MACd,CAAC,CAAC,IAAI,SAAS;EAC3B;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAE7C,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,QAAQ;KACR,IAAI;KACJ,IAAI;KACJ,gBAAgB,gBAAgB,KAAK,IAAI;IAC3C,CAAC;GACH;GAGF,MAAM,KAAK,WAAW,aAAa,KAAK,IAAI;GAC5C,MAAM,KAAK,QAAQ,aAAa,KAAK,IAAI;GACzC,MAAM,YAAY,gBAAgB;GAElC,OAAO;IACL,UAFe,MAAM,MAAM;IAG3B,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB;KACA;KACA;KACA,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,iBAAiB,KAAK;IACxB,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/humaneval.d.ts

@@ -1,31 +1,12 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the
- * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout
- * through the `Supervisor` that self-corrects across rounds, vs blind
- * random@k. This is the experiment `humaneval-gate.mts` names as "the next one" —
- * the gate measures SELECTION over stateless single completions; this measures
- * whether observe→steer (self-correction) beats blind compute at equal k.
- *
- * Worker artifact = the model's reply (a Python function, fenced or raw). The
- * DETERMINISTIC judge runs the candidate against the task's own `test` in an
- * isolated `--network=none` python container — exit 0 = pass. No gold
- * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only
- * to self-verify the judge before spending tokens.
- *
- * The primitives (loader / extractor / Docker checker) live here and are reused by
- * `humaneval-gate.mts` — one home, no duplication.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/humaneval.d.ts
 interface HumanEvalTask {
-    taskId: string;
-    prompt: string;
-    test: string;
-    entryPoint: string;
-    /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */
-    canonicalSolution?: string;
+  taskId: string;
+  prompt: string;
+  test: string;
+  entryPoint: string;
+  /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */
+  canonicalSolution?: string;
 }
 /** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch
  *  or a malformed line — a silently-short task set would poison the gate. `offset`
@@ -39,16 +20,16 @@ declare function basePrompt(task: HumanEvalTask): string;
  *  body still runs; a candidate that re-defines the function shadows the header. */
 declare function extractCode(reply: string): string;
 interface CheckResult {
-    /** {0,1} pass-count for this candidate (1 = the check() suite passed). */
-    pass: number;
-    /** On failure: the interpreter stderr tail (traceback / failing assertion). The
-     *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */
-    detail?: string;
+  /** {0,1} pass-count for this candidate (1 = the check() suite passed). */
+  pass: number;
+  /** On failure: the interpreter stderr tail (traceback / failing assertion). The
+   *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */
+  detail?: string;
 }
 interface PythonProgramResult {
-    exitCode: number;
-    stdout: string;
-    stderr: string;
+  exitCode: number;
+  stdout: string;
+  stderr: string;
 }
 /** Run arbitrary Python in the same isolated container used by the HumanEval
  * checker. This is also the execution-tool primitive for experiments that let
@@ -58,5 +39,6 @@ declare function runChecker(task: HumanEvalTask, candidate: string): Promise<Che
 /** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle
  *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */
 declare function createHumanEvalAdapter(): BenchmarkAdapter;
-
-export { type CheckResult, type HumanEvalTask, type PythonProgramResult, basePrompt, createHumanEvalAdapter, extractCode, loadHumanEval, runChecker, runPythonProgram };
+//#endregion
+export { CheckResult, HumanEvalTask, PythonProgramResult, basePrompt, createHumanEvalAdapter, extractCode, loadHumanEval, runChecker, runPythonProgram };
+//# sourceMappingURL=humaneval.d.ts.map

package/dist/benchmarks/humaneval.js

@@ -1,17 +1,280 @@
-import {
-  basePrompt,
-  createHumanEvalAdapter,
-  extractCode,
-  loadHumanEval,
-  runChecker,
-  runPythonProgram
-} from "../chunk-5H5XV76F.js";
-export {
-  basePrompt,
-  createHumanEvalAdapter,
-  extractCode,
-  loadHumanEval,
-  runChecker,
-  runPythonProgram
-};
+import { join } from "node:path";
+import { execFile } from "node:child_process";
+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { gunzipSync } from "node:zlib";
+//#region src/benchmarks/humaneval.ts
+/**
+* HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the
+* gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout
+* through the `Supervisor` that self-corrects across rounds, vs blind
+* random@k. This is the experiment `humaneval-gate.mts` names as "the next one" —
+* the gate measures SELECTION over stateless single completions; this measures
+* whether observe→steer (self-correction) beats blind compute at equal k.
+*
+* Worker artifact = the model's reply (a Python function, fenced or raw). The
+* DETERMINISTIC judge runs the candidate against the task's own `test` in an
+* isolated `--network=none` python container — exit 0 = pass. No gold
+* `canonical_solution` is ever shown to the model; `goldArtifact` returns it only
+* to self-verify the judge before spending tokens.
+*
+* The primitives (loader / extractor / Docker checker) live here and are reused by
+* `humaneval-gate.mts` — one home, no duplication.
+*/
+const humanevalUrl = "https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz";
+const dockerImage = "python:3.12-slim";
+const dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 2e4);
+/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch
+*  or a malformed line — a silently-short task set would poison the gate. `offset`
+*  selects a deeper slice (the later tasks are harder) so the worker has a
+*  correctable middle band rather than a saturated easy prefix. */
+async function loadHumanEval(limit, offset = 0) {
+	const localGz = process.env.HUMANEVAL_GZ;
+	let gz;
+	if (localGz) gz = readFileSync(localGz);
+	else {
+		const res = await fetch(humanevalUrl);
+		if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`);
+		gz = Buffer.from(await res.arrayBuffer());
+	}
+	const text = gunzipSync(gz).toString("utf8");
+	const tasks = [];
+	for (const line of text.split("\n")) {
+		if (line.trim() === "") continue;
+		const d = JSON.parse(line);
+		if (!d.task_id || !d.prompt || !d.test || !d.entry_point) throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`);
+		tasks.push({
+			taskId: d.task_id,
+			prompt: d.prompt,
+			test: d.test,
+			entryPoint: d.entry_point,
+			...d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}
+		});
+	}
+	if (tasks.length === 0) throw new Error("HumanEval parsed to 0 tasks");
+	if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`);
+	return tasks.slice(offset, offset + limit);
+}
+const solveInstruction = "Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.";
+function basePrompt(task) {
+	return `${solveInstruction}\n\n\`\`\`python\n${task.prompt}\`\`\``;
+}
+/** Extract the function source from a model reply: prefer a fenced ```python (or
+*  bare ```) block, else fall back to the raw text. The deployable program adds the
+*  prompt header (imports + signature context), so a candidate that returns only a
+*  body still runs; a candidate that re-defines the function shadows the header. */
+function extractCode(reply) {
+	const fenced = reply.match(/```(?:python|py)?\s*\n([\s\S]*?)```/i);
+	if (fenced && typeof fenced[1] === "string") return fenced[1].trim();
+	return reply.trim();
+}
+/** The deployable test program: the prompt header (imports + signature/docstring the
+*  model was given), then the candidate (its def shadows the header's stub), then the
+*  task's own check() suite and the call. No gold solution anywhere. */
+function buildProgram(task, candidate) {
+	return `${task.prompt}\n${candidate}\n\n${task.test}\n\ncheck(${task.entryPoint})\n`;
+}
+/** Run one candidate's deployable test program in an isolated container:
+*  read-only root, bounded writable `/tmp`, no network, one CPU, and 512 MiB.
+*  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image
+*  unavailable) is NOT a test failure — it throws so the harness fails loud rather
+*  than scoring every candidate 0 from a broken checker. */
+let dockerRunSeq = 0;
+/** Run arbitrary Python in the same isolated container used by the HumanEval
+* checker. This is also the execution-tool primitive for experiments that let
+* a model test its own snippets; model-written code must never run on the host. */
+function runPythonProgram(program, timeoutMs = dockerTimeoutMs) {
+	const dir = mkdtempSync(join(tmpdir(), "hev-"));
+	chmodSync(dir, 493);
+	writeFileSync(join(dir, "p.py"), program, { mode: 420 });
+	const name = `hev-${process.pid}-${dockerRunSeq++}`;
+	const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`;
+	return new Promise((resolvePromise, reject) => {
+		let settled = false;
+		const cleanup = () => {
+			rmSync(dir, {
+				recursive: true,
+				force: true
+			});
+			execFile("docker", [
+				"rm",
+				"-f",
+				name
+			], () => {});
+		};
+		const finish = (res) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(backstop);
+			cleanup();
+			resolvePromise(res);
+		};
+		const fail = (e) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(backstop);
+			cleanup();
+			reject(e);
+		};
+		const outerTimeoutMs = timeoutMs + 3e3;
+		const backstop = setTimeout(() => fail(/* @__PURE__ */ new Error(`docker checker did not return within ${outerTimeoutMs + 3e3}ms`)), outerTimeoutMs + 3e3);
+		const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1e3));
+		execFile("docker", [
+			"run",
+			"--rm",
+			"--pull=never",
+			"--name",
+			name,
+			"--network=none",
+			"--cpus=1",
+			"--memory=512m",
+			"--pids-limit=64",
+			"--cap-drop=ALL",
+			"--security-opt=no-new-privileges",
+			"--read-only",
+			"--tmpfs",
+			"/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777",
+			"--user",
+			"65534:65534",
+			"-v",
+			`${dir}:/w:ro`,
+			"-w",
+			"/w",
+			dockerImage,
+			"sh",
+			"-c",
+			"command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf \"%s\\n\" \"$1\"; exec timeout -s KILL \"$2\" python /w/p.py",
+			"agent-runtime-checker",
+			startMarker,
+			`${inContainerSeconds}s`
+		], {
+			timeout: outerTimeoutMs,
+			killSignal: "SIGKILL",
+			maxBuffer: 4 * 1024 * 1024
+		}, (err, stdout, stderr) => {
+			const markerLine = `${startMarker}\n`;
+			const candidateStarted = stdout?.startsWith(markerLine) === true;
+			const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : stdout ?? "";
+			if (err) {
+				const e = err;
+				if (e.code === "ENOENT") {
+					fail(/* @__PURE__ */ new Error("docker binary not found on PATH — cannot run the deployable checker"));
+					return;
+				}
+				if (e.killed) {
+					fail(/* @__PURE__ */ new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`));
+					return;
+				}
+				if (!candidateStarted) {
+					if (/cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(stderr)) {
+						fail(/* @__PURE__ */ new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`));
+						return;
+					}
+					if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {
+						fail(/* @__PURE__ */ new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`));
+						return;
+					}
+					fail(/* @__PURE__ */ new Error(`docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`));
+					return;
+				}
+				const exitCode = typeof e.code === "number" ? e.code : 1;
+				finish({
+					exitCode,
+					stdout: candidateStdout,
+					stderr: stderr ?? ""
+				});
+				return;
+			}
+			if (!candidateStarted) {
+				fail(/* @__PURE__ */ new Error("docker checker exited without starting the candidate"));
+				return;
+			}
+			finish({
+				exitCode: 0,
+				stdout: candidateStdout,
+				stderr: stderr ?? ""
+			});
+		});
+	});
+}
+async function runChecker(task, candidate) {
+	const result = await runPythonProgram(buildProgram(task, candidate));
+	return result.exitCode === 0 ? { pass: 1 } : {
+		pass: 0,
+		detail: result.stderr.slice(-600) || "timed out (no output)"
+	};
+}
+function toBenchTask(t) {
+	const metadata = {
+		promptHeader: t.prompt,
+		test: t.test,
+		entryPoint: t.entryPoint,
+		...t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}
+	};
+	return {
+		id: t.taskId,
+		prompt: basePrompt(t),
+		metadata
+	};
+}
+function taskFromMeta(task) {
+	const m = task.metadata;
+	if (!m?.promptHeader || !m.test || !m.entryPoint) throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`);
+	return {
+		taskId: task.id,
+		prompt: m.promptHeader,
+		test: m.test,
+		entryPoint: m.entryPoint
+	};
+}
+/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle
+*  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */
+function createHumanEvalAdapter() {
+	return {
+		name: "humaneval",
+		async preflight() {
+			await new Promise((resolve, reject) => {
+				execFile("docker", [
+					"version",
+					"--format",
+					"{{.Server.Version}}"
+				], (err) => {
+					if (err) reject(/* @__PURE__ */ new Error("HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)"));
+					else resolve();
+				});
+			});
+			await new Promise((resolve, reject) => {
+				execFile("docker", [
+					"image",
+					"inspect",
+					dockerImage
+				], (err) => {
+					if (err) reject(/* @__PURE__ */ new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`));
+					else resolve();
+				});
+			});
+		},
+		async loadTasks(opts) {
+			const offset = Number(process.env.OFFSET ?? 0);
+			const all = await loadHumanEval(opts?.ids ? 164 : opts?.limit ?? 8, offset);
+			return (opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all).map(toBenchTask);
+		},
+		async judge(task, artifact) {
+			const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact));
+			return {
+				resolved: pass === 1,
+				score: pass,
+				detail: pass === 1 ? "tests passed" : "tests failed"
+			};
+		},
+		async goldArtifact(task) {
+			const m = task.metadata;
+			const sol = m?.canonicalSolution;
+			return sol ? `${m.promptHeader}${sol}` : void 0;
+		}
+	};
+}
+//#endregion
+export { basePrompt, createHumanEvalAdapter, extractCode, loadHumanEval, runChecker, runPythonProgram };
+
 //# sourceMappingURL=humaneval.js.map