@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/frames.js

@@ -1,13 +1,410 @@
-import {
-  createFramesAdapter,
-  normalizeAnswer,
-  parseCitations,
-  parseFinalAnswer
-} from "../chunk-INNOYXCP.js";
-export {
-  createFramesAdapter,
-  normalizeAnswer,
-  parseCitations,
-  parseFinalAnswer
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { execFile } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+//#region src/benchmarks/frames.ts
+/**
+* FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text
+* final answer string (optionally preceded by a citations block). FRAMES ships,
+* per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs
+* needed to answer) — the gold-citation column is what lets the loop's critic
+* check citation coverage deterministically.
+*
+* Judge is two-tier, deterministic-first:
+*   Tier 1 — normalized exact / token-boundary containment match (no model tokens).
+*   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,
+*            temperature 0), fired ONLY when Tier 1 misses, to absorb the
+*            paraphrase/alias equivalence FRAMES intends to allow.
+*
+* score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge
+* resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced
+* in `detail` for the critic, never a hard pass criterion.
+*
+* Requires for a live run: the bench `.venv` with `datasets` installed and a
+* JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to
+* load the committed fixtures (bench/fixtures/frames.json) — no HF download.
+*/
+const execFileAsync = promisify(execFile);
+const BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+const PY = join(BENCH_ROOT, ".venv", "bin", "python");
+const FIXTURES = join(BENCH_ROOT, "fixtures", "frames.json");
+const DATASET = "google/frames-benchmark";
+/** Pin the dataset revision: ids are derived from row order (no native id column),
+*  so a reorder would break opts.ids selection + scorecard joins. */
+const DATASET_REVISION = "main";
+const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+/** Worker contract appended to every task prompt. Tier-1 parsing keys off the sentinel. */
+const WORKER_CONTRACT = [
+	"",
+	"Research the question using live web sources and answer it.",
+	"Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.",
+	`End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
+	"The answer after the sentinel must be the bare value only (no explanation on that line)."
+].join("\n");
+/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */
+async function py(script, args = []) {
+	const { stdout } = await execFileAsync(PY, [
+		"-c",
+		script,
+		...args
+	], { maxBuffer: 1024 * 1024 * 256 });
+	return stdout;
+}
+const ARTICLES = /* @__PURE__ */ new Set([
+	"a",
+	"an",
+	"the"
+]);
+const UNIT_WORDS = /* @__PURE__ */ new Set([
+	"years",
+	"year",
+	"months",
+	"month",
+	"days",
+	"day",
+	"people",
+	"percent",
+	"dollars",
+	"meters",
+	"metres",
+	"kilometers",
+	"kilometres",
+	"miles",
+	"km",
+	"m"
+]);
+const WRITTEN_NUMBERS = {
+	zero: 0,
+	one: 1,
+	two: 2,
+	three: 3,
+	four: 4,
+	five: 5,
+	six: 6,
+	seven: 7,
+	eight: 8,
+	nine: 9,
+	ten: 10,
+	eleven: 11,
+	twelve: 12,
+	thirteen: 13,
+	fourteen: 14,
+	fifteen: 15,
+	sixteen: 16,
+	seventeen: 17,
+	eighteen: 18,
+	nineteen: 19,
+	twenty: 20,
+	thirty: 30,
+	forty: 40,
+	fifty: 50,
+	sixty: 60,
+	seventy: 70,
+	eighty: 80,
+	ninety: 90,
+	hundred: 100,
+	thousand: 1e3,
+	million: 1e6,
+	billion: 1e9
 };
+const MONTHS = {
+	january: "01",
+	february: "02",
+	march: "03",
+	april: "04",
+	may: "05",
+	june: "06",
+	july: "07",
+	august: "08",
+	september: "09",
+	october: "10",
+	november: "11",
+	december: "12"
+};
+const SCALES = /* @__PURE__ */ new Set([
+	"hundred",
+	"thousand",
+	"million",
+	"billion"
+]);
+/** Fold a run of written-number words into one integer ("twenty eight"→28, "two hundred"→200). */
+function composeWrittenRun(words) {
+	let total = 0;
+	let current = 0;
+	for (const w of words) {
+		const v = WRITTEN_NUMBERS[w];
+		if (v === void 0) continue;
+		if (w === "hundred") current = (current === 0 ? 1 : current) * 100;
+		else if (SCALES.has(w)) {
+			current = (current === 0 ? 1 : current) * v;
+			total += current;
+			current = 0;
+		} else current += v;
+	}
+	return total + current;
+}
+/** Canonicalize "1,234" / "1.2 million" / written numerals to a single numeric token. */
+function canonicalizeNumbers(s) {
+	let out = s.replace(/(\d),(?=\d{3}\b)/g, "$1");
+	out = out.replace(/\b(\d+(?:\.\d+)?)\s+(hundred|thousand|million|billion)\b/g, (_m, num, scaleWord) => {
+		const scale = WRITTEN_NUMBERS[scaleWord] ?? 1;
+		return String(Number(num) * scale);
+	});
+	out = out.replace(/\b[a-z]+(?:[\s-]+[a-z]+)*\b/g, (run) => {
+		const words = run.split(/[\s-]+/);
+		if (!words.every((w) => w in WRITTEN_NUMBERS)) return run;
+		return String(composeWrittenRun(words));
+	});
+	return out;
+}
+/** "March 3, 1879" / "3 March 1879" → "1879-03-03"; year-only stays as the year. */
+function canonicalizeDates(s) {
+	let out = s.replace(/\b(january|february|march|april|may|june|july|august|september|october|november|december)\s+(\d{1,2}),?\s+(\d{4})\b/g, (_m, mon, day, year) => `${year}-${MONTHS[mon]}-${day.padStart(2, "0")}`);
+	out = out.replace(/\b(\d{1,2})\s+(january|february|march|april|may|june|july|august|september|october|november|december)\s+(\d{4})\b/g, (_m, day, mon, year) => `${year}-${MONTHS[mon]}-${day.padStart(2, "0")}`);
+	return out;
+}
+/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */
+function normalizeAnswer(input) {
+	let s = input.toLowerCase();
+	s = canonicalizeDates(s);
+	s = canonicalizeNumbers(s);
+	s = s.replace(/[^\w\s-]/g, " ");
+	return s.split(/\s+/).filter((t) => t.length > 0).filter((t) => !ARTICLES.has(t)).filter((t) => !UNIT_WORDS.has(t)).join(" ").trim();
+}
+/** Tier-1 match: normalized equality OR normalized gold as a token-boundary substring. */
+function tier1Match(candidate, gold) {
+	const nc = normalizeAnswer(candidate);
+	const ng = normalizeAnswer(gold);
+	if (ng.length === 0) return false;
+	if (nc === ng) return true;
+	return new RegExp(`(^|\\s)${ng.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(\\s|$)`).test(nc);
+}
+/**
+* Parse the worker artifact into the final answer string.
+* Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+* fall back to the last non-empty line. Returns '' when nothing is parseable
+* (fail-closed — never guess), which judge() counts as resolved=false.
+*/
+function parseFinalAnswer(artifact) {
+	const lines = artifact.split(/\r?\n/);
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const line = lines[i] ?? "";
+		const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
+		if (idx !== -1) return line.slice(idx + 13).trim();
+	}
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const t = (lines[i] ?? "").trim();
+		if (t.length > 0) return t;
+	}
+	return "";
+}
+/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */
+function parseCitations(artifact) {
+	const urls = /* @__PURE__ */ new Set();
+	for (const m of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) urls.add(m[0].replace(/[.,;]+$/, ""));
+	return [...urls];
+}
+/** Fraction of gold Wikipedia article slugs touched by the candidate's citations. SOFT signal. */
+function citationCoverage(citations, goldSources) {
+	if (goldSources.length === 0) return 1;
+	const slug = (u) => {
+		const m = u.match(/\/wiki\/([^#?]+)/);
+		return m ? decodeURIComponent(m[1] ?? "").toLowerCase() : u.toLowerCase();
+	};
+	const got = new Set(citations.map(slug));
+	let hit = 0;
+	for (const g of goldSources) if (got.has(slug(g))) hit += 1;
+	return hit / goldSources.length;
+}
+const JUDGE_PROMPT = (question, gold, candidate) => [
+	"You are a strict answer-equivalence checker for a factual question-answering benchmark.",
+	"Decide ONLY whether the candidate answer is semantically equivalent to the gold answer for this question.",
+	"It is CORRECT iff it contains the same factual value as the gold answer. Extra correct detail is fine.",
+	"A different value, a missing value, or a wrong value is INCORRECT.",
+	"",
+	`Question: ${question}`,
+	`Gold answer: ${gold}`,
+	`Candidate answer: ${candidate}`,
+	"",
+	"Respond with ONLY a fenced JSON block and nothing else:",
+	"```json",
+	"{\"verdict\": \"correct\" | \"incorrect\"}",
+	"```"
+].join("\n");
+function judgeRouter() {
+	const model = process.env.JUDGE_MODEL;
+	if (!model) throw new Error("JUDGE_MODEL is required for the FRAMES Tier-2 equivalence judge");
+	const key = process.env.TANGLE_API_KEY;
+	if (!key) throw new Error("TANGLE_API_KEY is required for the FRAMES Tier-2 judge");
+	return {
+		baseUrl: process.env.JUDGE_ROUTER_BASE ?? process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
+		key,
+		model
+	};
+}
+/** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */
+async function tier2Judge(question, gold, candidate, router) {
+	const res = await fetch(`${router.baseUrl}/chat/completions`, {
+		method: "POST",
+		headers: {
+			"content-type": "application/json",
+			authorization: `Bearer ${router.key}`
+		},
+		body: JSON.stringify({
+			model: router.model,
+			temperature: 0,
+			seed: 0,
+			messages: [{
+				role: "user",
+				content: JUDGE_PROMPT(question, gold, candidate)
+			}]
+		})
+	});
+	if (!res.ok) throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+	const body = await res.json();
+	const content = body.choices?.[0]?.message?.content;
+	if (typeof content !== "string") throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
+	const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
+	let parsed;
+	try {
+		parsed = JSON.parse(raw);
+	} catch {
+		throw new Error(`FRAMES Tier-2 judge produced unparseable output (no JSON verdict): ${content.slice(0, 300)}`);
+	}
+	if (parsed.verdict === "correct") return true;
+	if (parsed.verdict === "incorrect") return false;
+	throw new Error(`FRAMES Tier-2 judge verdict not in {correct,incorrect}: ${JSON.stringify(parsed).slice(0, 200)}`);
+}
+/** FRAMES wiki_links is a python-repr list string, e.g. "['https://…', 'https://…']". */
+function parseWikiLinks(raw) {
+	if (!raw) return [];
+	const out = [];
+	for (const m of raw.matchAll(/https?:\/\/[^\s'"\]]+/g)) out.push(m[0]);
+	return out;
+}
+function rowToTask(row, index) {
+	const goldSources = parseWikiLinks(row.wiki_links);
+	const meta = {
+		gold: row.Answer,
+		goldSources,
+		reasoningTypes: row.reasoning_types ?? "",
+		rawPrompt: row.Prompt
+	};
+	return {
+		id: `frames-${index}`,
+		split: "test",
+		prompt: row.Prompt + WORKER_CONTRACT,
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.gold !== "string") throw new Error(`FRAMES task ${task.id} missing metadata.gold — loadTasks did not populate it`);
+	return md;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.log(`[frames] FRAMES_FIXTURES=1 — loading ${rows.length} committed fixtures (no HF download)`);
+	let tasks = rows.map(rowToTask);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((t) => want.has(t.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	return tasks;
+}
+function createFramesAdapter() {
+	const fixturesMode = process.env.FRAMES_FIXTURES === "1";
+	return {
+		name: "frames",
+		async preflight() {
+			judgeRouter();
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8").catch((err) => {
+					throw new Error(`FRAMES_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+				});
+				return;
+			}
+			try {
+				await py(`from datasets import load_dataset
+load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})
+print('ok')`);
+			} catch (err) {
+				const msg = err instanceof Error ? err.message : String(err);
+				throw new Error(`frames preflight failed: ${msg}\nFix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; (2) ensure network access to Hugging Face for ${DATASET} ; or set FRAMES_FIXTURES=1 to run against the committed fixtures offline.`);
+			}
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const limit = opts.limit ?? 10;
+			const stdout = await py(`
+import json, sys
+from datasets import load_dataset
+ds = load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})
+ids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None
+out = []
+for i, r in enumerate(ds):
+    rid = f"frames-{i}"
+    if ids is not None and rid not in ids:
+        continue
+    out.append({
+        "_index": i,
+        "Prompt": r.get("Prompt", ""),
+        "Answer": r.get("Answer", ""),
+        "wiki_links": str(r.get("wiki_links", "")),
+        "reasoning_types": str(r.get("reasoning_types", "")),
+    })
+    if ids is None and len(out) >= ${limit}:
+        break
+print(json.dumps(out))
+`, [opts.ids ? JSON.stringify(opts.ids) : ""]);
+			return JSON.parse(stdout).map((r) => rowToTask(r, r._index));
+		},
+		async goldArtifact(task) {
+			const meta = readMeta(task);
+			return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const finalAnswer = parseFinalAnswer(artifact);
+			const coverage = citationCoverage(parseCitations(artifact), meta.goldSources);
+			if (finalAnswer.length === 0) return {
+				resolved: false,
+				score: 0,
+				detail: JSON.stringify({
+					tier: "none",
+					reason: "no parseable answer",
+					normalizedGold: normalizeAnswer(meta.gold),
+					citationCoverage: coverage
+				})
+			};
+			if (tier1Match(finalAnswer, meta.gold)) return {
+				resolved: true,
+				score: 1,
+				detail: JSON.stringify({
+					tier: 1,
+					normalizedAnswer: normalizeAnswer(finalAnswer),
+					normalizedGold: normalizeAnswer(meta.gold),
+					citationCoverage: coverage
+				})
+			};
+			const verdict = await tier2Judge(meta.rawPrompt, meta.gold, finalAnswer, judgeRouter());
+			return {
+				resolved: verdict,
+				score: verdict ? 1 : 0,
+				detail: JSON.stringify({
+					tier: 2,
+					normalizedAnswer: normalizeAnswer(finalAnswer),
+					normalizedGold: normalizeAnswer(meta.gold),
+					judgeVerdict: verdict ? "correct" : "incorrect",
+					citationCoverage: coverage
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createFramesAdapter, normalizeAnswer, parseCitations, parseFinalAnswer };
+
 //# sourceMappingURL=frames.js.map

package/dist/benchmarks/frames.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"frames.js","names":[],"sources":["../../src/benchmarks/frames.ts"],"sourcesContent":["/**\n * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text\n * final answer string (optionally preceded by a citations block). FRAMES ships,\n * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs\n * needed to answer) — the gold-citation column is what lets the loop's critic\n * check citation coverage deterministically.\n *\n * Judge is two-tier, deterministic-first:\n *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).\n *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,\n *            temperature 0), fired ONLY when Tier 1 misses, to absorb the\n *            paraphrase/alias equivalence FRAMES intends to allow.\n *\n * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge\n * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced\n * in `detail` for the critic, never a hard pass criterion.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed and a\n * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to\n * load the committed fixtures (bench/fixtures/frames.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'frames.json')\n\nconst DATASET = 'google/frames-benchmark'\n/** Pin the dataset revision: ids are derived from row order (no native id column),\n *  so a reorder would break opts.ids selection + scorecard joins. */\nconst DATASET_REVISION = 'main'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Tier-1 parsing keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\n/**\n * Typed seam for the future dynamic-topology research worker. The benchmark\n * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the\n * loop worker decodes its agent runs into a {@link ResearchAnswer} and serializes\n * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — judge resolves strictly against this. */\n  gold: string\n  /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */\n  goldSources: string[]\n  /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */\n  reasoningTypes: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\ninterface FramesRow {\n  Prompt: string\n  Answer: string\n  wiki_links: string\n  reasoning_types: string\n}\n\ninterface FramesMeta {\n  gold: string\n  goldSources: string[]\n  reasoningTypes: string\n  rawPrompt: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\nconst UNIT_WORDS = new Set([\n  'years',\n  'year',\n  'months',\n  'month',\n  'days',\n  'day',\n  'people',\n  'percent',\n  'dollars',\n  'meters',\n  'metres',\n  'kilometers',\n  'kilometres',\n  'miles',\n  'km',\n  'm',\n])\n\nconst WRITTEN_NUMBERS: Record<string, number> = {\n  zero: 0,\n  one: 1,\n  two: 2,\n  three: 3,\n  four: 4,\n  five: 5,\n  six: 6,\n  seven: 7,\n  eight: 8,\n  nine: 9,\n  ten: 10,\n  eleven: 11,\n  twelve: 12,\n  thirteen: 13,\n  fourteen: 14,\n  fifteen: 15,\n  sixteen: 16,\n  seventeen: 17,\n  eighteen: 18,\n  nineteen: 19,\n  twenty: 20,\n  thirty: 30,\n  forty: 40,\n  fifty: 50,\n  sixty: 60,\n  seventy: 70,\n  eighty: 80,\n  ninety: 90,\n  hundred: 100,\n  thousand: 1000,\n  million: 1_000_000,\n  billion: 1_000_000_000,\n}\n\nconst MONTHS: Record<string, string> = {\n  january: '01',\n  february: '02',\n  march: '03',\n  april: '04',\n  may: '05',\n  june: '06',\n  july: '07',\n  august: '08',\n  september: '09',\n  october: '10',\n  november: '11',\n  december: '12',\n}\n\nconst SCALES = new Set(['hundred', 'thousand', 'million', 'billion'])\n\n/** Fold a run of written-number words into one integer (\"twenty eight\"→28, \"two hundred\"→200). */\nfunction composeWrittenRun(words: string[]): number {\n  let total = 0\n  let current = 0\n  for (const w of words) {\n    const v = WRITTEN_NUMBERS[w]\n    if (v === undefined) continue\n    if (w === 'hundred') {\n      current = (current === 0 ? 1 : current) * 100\n    } else if (SCALES.has(w)) {\n      current = (current === 0 ? 1 : current) * v\n      total += current\n      current = 0\n    } else {\n      current += v\n    }\n  }\n  return total + current\n}\n\n/** Canonicalize \"1,234\" / \"1.2 million\" / written numerals to a single numeric token. */\nfunction canonicalizeNumbers(s: string): string {\n  // \"1,234,567\" → \"1234567\"\n  let out = s.replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n  // \"1.2 million\" / \"3 billion\" → expanded integer\n  out = out.replace(/\\b(\\d+(?:\\.\\d+)?)\\s+(hundred|thousand|million|billion)\\b/g, (_m, num: string, scaleWord: string) => {\n    const scale = WRITTEN_NUMBERS[scaleWord] ?? 1\n    return String(Number(num) * scale)\n  })\n  // fold contiguous runs of written-number words into one integer\n  out = out.replace(/\\b[a-z]+(?:[\\s-]+[a-z]+)*\\b/g, (run) => {\n    const words = run.split(/[\\s-]+/)\n    if (!words.every((w) => w in WRITTEN_NUMBERS)) return run\n    return String(composeWrittenRun(words))\n  })\n  return out\n}\n\n/** \"March 3, 1879\" / \"3 March 1879\" → \"1879-03-03\"; year-only stays as the year. */\nfunction canonicalizeDates(s: string): string {\n  let out = s.replace(\n    /\\b(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{1,2}),?\\s+(\\d{4})\\b/g,\n    (_m, mon: string, day: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  out = out.replace(\n    /\\b(\\d{1,2})\\s+(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{4})\\b/g,\n    (_m, day: string, mon: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  return out\n}\n\n/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */\nexport function normalizeAnswer(input: string): string {\n  let s = input.toLowerCase()\n  s = canonicalizeDates(s)\n  s = canonicalizeNumbers(s)\n  // strip punctuation (keep alphanumerics, ISO-date hyphens collapse to space-free below)\n  s = s.replace(/[^\\w\\s-]/g, ' ')\n  // tokenize, drop articles + trailing unit words, keep order\n  const tokens = s\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n    .filter((t) => !UNIT_WORDS.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Tier-1 match: normalized equality OR normalized gold as a token-boundary substring. */\nfunction tier1Match(candidate: string, gold: string): boolean {\n  const nc = normalizeAnswer(candidate)\n  const ng = normalizeAnswer(gold)\n  if (ng.length === 0) return false\n  if (nc === ng) return true\n  // token-boundary containment: gold appears as a whole-token run inside candidate\n  const re = new RegExp(`(^|\\\\s)${ng.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}(\\\\s|$)`)\n  return re.test(nc)\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/** Fraction of gold Wikipedia article slugs touched by the candidate's citations. SOFT signal. */\nfunction citationCoverage(citations: string[], goldSources: string[]): number {\n  if (goldSources.length === 0) return 1\n  const slug = (u: string) => {\n    const m = u.match(/\\/wiki\\/([^#?]+)/)\n    return m ? decodeURIComponent(m[1] ?? '').toLowerCase() : u.toLowerCase()\n  }\n  const got = new Set(citations.map(slug))\n  let hit = 0\n  for (const g of goldSources) if (got.has(slug(g))) hit += 1\n  return hit / goldSources.length\n}\n\nconst JUDGE_PROMPT = (question: string, gold: string, candidate: string): string =>\n  [\n    'You are a strict answer-equivalence checker for a factual question-answering benchmark.',\n    'Decide ONLY whether the candidate answer is semantically equivalent to the gold answer for this question.',\n    'It is CORRECT iff it contains the same factual value as the gold answer. Extra correct detail is fine.',\n    'A different value, a missing value, or a wrong value is INCORRECT.',\n    '',\n    `Question: ${question}`,\n    `Gold answer: ${gold}`,\n    `Candidate answer: ${candidate}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"verdict\": \"correct\" | \"incorrect\"}',\n    '```',\n  ].join('\\n')\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const model = process.env.JUDGE_MODEL\n  if (!model) throw new Error('JUDGE_MODEL is required for the FRAMES Tier-2 equivalence judge')\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FRAMES Tier-2 judge')\n  const baseUrl = process.env.JUDGE_ROUTER_BASE ?? process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function tier2Judge(\n  question: string,\n  gold: string,\n  candidate: string,\n  router: JudgeRouter,\n): Promise<boolean> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      seed: 0,\n      messages: [{ role: 'user', content: JUDGE_PROMPT(question, gold, candidate) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { verdict?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { verdict?: unknown }\n  } catch {\n    throw new Error(`FRAMES Tier-2 judge produced unparseable output (no JSON verdict): ${content.slice(0, 300)}`)\n  }\n  if (parsed.verdict === 'correct') return true\n  if (parsed.verdict === 'incorrect') return false\n  throw new Error(`FRAMES Tier-2 judge verdict not in {correct,incorrect}: ${JSON.stringify(parsed).slice(0, 200)}`)\n}\n\n/** FRAMES wiki_links is a python-repr list string, e.g. \"['https://…', 'https://…']\". */\nfunction parseWikiLinks(raw: string): string[] {\n  if (!raw) return []\n  const out: string[] = []\n  for (const m of raw.matchAll(/https?:\\/\\/[^\\s'\"\\]]+/g)) out.push(m[0])\n  return out\n}\n\nfunction rowToTask(row: FramesRow, index: number): BenchTask {\n  const goldSources = parseWikiLinks(row.wiki_links)\n  const meta: FramesMeta = {\n    gold: row.Answer,\n    goldSources,\n    reasoningTypes: row.reasoning_types ?? '',\n    rawPrompt: row.Prompt,\n  }\n  return {\n    id: `frames-${index}`,\n    split: 'test',\n    prompt: row.Prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FramesMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`FRAMES task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as FramesMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as FramesRow[]\n  console.log(`[frames] FRAMES_FIXTURES=1 — loading ${rows.length} committed fixtures (no HF download)`)\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createFramesAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FRAMES_FIXTURES === '1'\n\n  return {\n    name: 'frames',\n\n    async preflight() {\n      // Tier-2 judge router config must be present in both modes — the loop's\n      // citation-coverage stop gate is meaningless without the equivalence judge.\n      judgeRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`FRAMES_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `frames preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} ; ` +\n            `or set FRAMES_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor i, r in enumerate(ds):\n    rid = f\"frames-{i}\"\n    if ids is not None and rid not in ids:\n        continue\n    out.append({\n        \"_index\": i,\n        \"Prompt\": r.get(\"Prompt\", \"\"),\n        \"Answer\": r.get(\"Answer\", \"\"),\n        \"wiki_links\": str(r.get(\"wiki_links\", \"\")),\n        \"reasoning_types\": str(r.get(\"reasoning_types\", \"\")),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<FramesRow & { _index: number }>\n      return rows.map((r) => rowToTask(r, r._index))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold Answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes (Tier-1 short-circuits with no model tokens).\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n      const coverage = citationCoverage(citations, meta.goldSources)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            tier: 'none',\n            reason: 'no parseable answer',\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 1 — free, deterministic.\n      if (tier1Match(finalAnswer, meta.gold)) {\n        return {\n          resolved: true,\n          score: 1,\n          detail: JSON.stringify({\n            tier: 1,\n            normalizedAnswer: normalizeAnswer(finalAnswer),\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 2 — constrained LLM equivalence gate (fail loud on unparseable output).\n      const verdict = await tier2Judge(meta.rawPrompt, meta.gold, finalAnswer, judgeRouter())\n      return {\n        resolved: verdict,\n        score: verdict ? 1 : 0,\n        detail: JSON.stringify({\n          tier: 2,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          judgeVerdict: verdict ? 'correct' : 'incorrect',\n          citationCoverage: coverage,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,aAAa;AAE3D,MAAM,UAAU;;;AAGhB,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;AACF,CAAC,CAAC,KAAK,IAAI;;AAuCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;AAEA,MAAM,2BAAW,IAAI,IAAI;CAAC;CAAK;CAAM;AAAK,CAAC;AAC3C,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,kBAA0C;CAC9C,MAAM;CACN,KAAK;CACL,KAAK;CACL,OAAO;CACP,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,OAAO;CACP,MAAM;CACN,KAAK;CACL,QAAQ;CACR,QAAQ;CACR,UAAU;CACV,UAAU;CACV,SAAS;CACT,SAAS;CACT,WAAW;CACX,UAAU;CACV,UAAU;CACV,QAAQ;CACR,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,SAAS;CACT,UAAU;CACV,SAAS;CACT,SAAS;AACX;AAEA,MAAM,SAAiC;CACrC,SAAS;CACT,UAAU;CACV,OAAO;CACP,OAAO;CACP,KAAK;CACL,MAAM;CACN,MAAM;CACN,QAAQ;CACR,WAAW;CACX,SAAS;CACT,UAAU;CACV,UAAU;AACZ;AAEA,MAAM,yBAAS,IAAI,IAAI;CAAC;CAAW;CAAY;CAAW;AAAS,CAAC;;AAGpE,SAAS,kBAAkB,OAAyB;CAClD,IAAI,QAAQ;CACZ,IAAI,UAAU;CACd,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,gBAAgB;EAC1B,IAAI,MAAM,KAAA,GAAW;EACrB,IAAI,MAAM,WACR,WAAW,YAAY,IAAI,IAAI,WAAW;OACrC,IAAI,OAAO,IAAI,CAAC,GAAG;GACxB,WAAW,YAAY,IAAI,IAAI,WAAW;GAC1C,SAAS;GACT,UAAU;EACZ,OACE,WAAW;CAEf;CACA,OAAO,QAAQ;AACjB;;AAGA,SAAS,oBAAoB,GAAmB;CAE9C,IAAI,MAAM,EAAE,QAAQ,qBAAqB,IAAI;CAE7C,MAAM,IAAI,QAAQ,8DAA8D,IAAI,KAAa,cAAsB;EACrH,MAAM,QAAQ,gBAAgB,cAAc;EAC5C,OAAO,OAAO,OAAO,GAAG,IAAI,KAAK;CACnC,CAAC;CAED,MAAM,IAAI,QAAQ,iCAAiC,QAAQ;EACzD,MAAM,QAAQ,IAAI,MAAM,QAAQ;EAChC,IAAI,CAAC,MAAM,OAAO,MAAM,KAAK,eAAe,GAAG,OAAO;EACtD,OAAO,OAAO,kBAAkB,KAAK,CAAC;CACxC,CAAC;CACD,OAAO;AACT;;AAGA,SAAS,kBAAkB,GAAmB;CAC5C,IAAI,MAAM,EAAE,QACV,yHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,MAAM,IAAI,QACR,uHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,OAAO;AACT;;AAGA,SAAgB,gBAAgB,OAAuB;CACrD,IAAI,IAAI,MAAM,YAAY;CAC1B,IAAI,kBAAkB,CAAC;CACvB,IAAI,oBAAoB,CAAC;CAEzB,IAAI,EAAE,QAAQ,aAAa,GAAG;CAO9B,OALe,EACZ,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC,CAC3B,QAAQ,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAC/B,QAAQ,MAAM,CAAC,WAAW,IAAI,CAAC,CACtB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;AAC/B;;AAGA,SAAS,WAAW,WAAmB,MAAuB;CAC5D,MAAM,KAAK,gBAAgB,SAAS;CACpC,MAAM,KAAK,gBAAgB,IAAI;CAC/B,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,IAAI,OAAO,IAAI,OAAO;CAGtB,OAAO,IADQ,OAAO,UAAU,GAAG,QAAQ,uBAAuB,MAAM,EAAE,QAClE,CAAC,CAAC,KAAK,EAAE;AACnB;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;AAGA,SAAS,iBAAiB,WAAqB,aAA+B;CAC5E,IAAI,YAAY,WAAW,GAAG,OAAO;CACrC,MAAM,QAAQ,MAAc;EAC1B,MAAM,IAAI,EAAE,MAAM,kBAAkB;EACpC,OAAO,IAAI,mBAAmB,EAAE,MAAM,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE,YAAY;CAC1E;CACA,MAAM,MAAM,IAAI,IAAI,UAAU,IAAI,IAAI,CAAC;CACvC,IAAI,MAAM;CACV,KAAK,MAAM,KAAK,aAAa,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,GAAG,OAAO;CAC1D,OAAO,MAAM,YAAY;AAC3B;AAEA,MAAM,gBAAgB,UAAkB,MAAc,cACpD;CACE;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,cAA2B;CAClC,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,iEAAiE;CAC7F,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;CAElF,OAAO;EAAE,SADO,QAAQ,IAAI,qBAAqB,QAAQ,IAAI,eAAe;EAC1D;EAAK;CAAM;AAC/B;;AAGA,eAAe,WACb,UACA,MACA,WACA,QACkB;CAClB,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,MAAM;GACN,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS,aAAa,UAAU,MAAM,SAAS;GAAE,CAAC;EAC/E,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,4BAA4B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,oDAAoD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAE1G,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,sEAAsE,QAAQ,MAAM,GAAG,GAAG,GAAG;CAC/G;CACA,IAAI,OAAO,YAAY,WAAW,OAAO;CACzC,IAAI,OAAO,YAAY,aAAa,OAAO;CAC3C,MAAM,IAAI,MAAM,2DAA2D,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;AACnH;;AAGA,SAAS,eAAe,KAAuB;CAC7C,IAAI,CAAC,KAAK,OAAO,CAAC;CAClB,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,KAAK,IAAI,SAAS,wBAAwB,GAAG,IAAI,KAAK,EAAE,EAAE;CACrE,OAAO;AACT;AAEA,SAAS,UAAU,KAAgB,OAA0B;CAC3D,MAAM,cAAc,eAAe,IAAI,UAAU;CACjD,MAAM,OAAmB;EACvB,MAAM,IAAI;EACV;EACA,gBAAgB,IAAI,mBAAmB;EACvC,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,UAAU;EACd,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA6B;CAC7C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAC5B,MAAM,IAAI,MAAM,eAAe,KAAK,GAAG,uDAAuD;CAEhG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,IAAI,wCAAwC,KAAK,OAAO,qCAAqC;CACrG,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,SAAgB,sBAAwC;CACtD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,yBAAyB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC7G,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;YAE3F;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,4BAA4B,IAAI,iIAEmB,QAAQ,2EAE7D;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,QAAQ,KAAK,SAAS;GAsB5B,MAAM,SAAS,MAAM,GAAG;;;oBAlBV,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;;;;;;;;;;;;;;qCAcnE,MAAM;;;GAIL,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAE1E,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KAAK,MAAM,UAAU,GAAG,EAAE,MAAM,CAAC;EAC/C;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAE7C,MAAM,WAAW,iBADC,eAAe,QACS,GAAG,KAAK,WAAW;GAE7D,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,QAAQ;KACR,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,IAAI,WAAW,aAAa,KAAK,IAAI,GACnC,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,MAAM,UAAU,MAAM,WAAW,KAAK,WAAW,KAAK,MAAM,aAAa,YAAY,CAAC;GACtF,OAAO;IACL,UAAU;IACV,OAAO,UAAU,IAAI;IACrB,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,cAAc,UAAU,YAAY;KACpC,kBAAkB;IACpB,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/hotpotqa.d.ts

@@ -1,26 +1,5 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * HotpotQA adapter (hotpotqa/hotpot_qa, config 'distractor', split 'validation').
- * Multi-hop factoid QA. Worker artifact = a single free-text final answer string.
- *
- * Judge is the official HotpotQA / SQuAD-style metric, FULLY DETERMINISTIC — no
- * LLM. Both the predicted final answer and the gold are normalized (lowercase,
- * strip articles a/an/the, strip punctuation, collapse whitespace), then scored
- * by exact-match (EM) and token-level F1. resolved = EM OR F1 >= HOTPOTQA_F1_PASS
- * (default 0.6); score = the F1 (0..1). detail carries em + f1. This gives the
- * suite a judge that needs no model tokens at all.
- *
- * metadata carries the gold answer + supporting_facts (the title/sent_id pairs of
- * the gold supporting sentences) — a SOFT retrieval signal for the loop's critic,
- * never part of the score.
- *
- * Requires for a live run: the bench `.venv` with `datasets` installed + network
- * to Hugging Face. For offline/CI verification set HOTPOTQA_FIXTURES=1 to load the
- * committed fixtures (bench/fixtures/hotpotqa.json) — no HF download.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/hotpotqa.d.ts
 /**
  * The official HotpotQA / SQuAD `normalize_answer`: lowercase, strip punctuation,
  * drop articles (a/an/the), collapse whitespace. Token comparisons run on the
@@ -44,5 +23,6 @@ declare function tokenF1(prediction: string, gold: string): number;
  */
 declare function parseFinalAnswer(artifact: string): string;
 declare function createHotpotqaAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createHotpotqaAdapter, exactMatch, normalizeAnswer, parseFinalAnswer, tokenF1 };
+//# sourceMappingURL=hotpotqa.d.ts.map