@tangle-network/agent-bench 0.4.1 → 0.4.4

package/dist/benchmarks/simpleqa.js

@@ -1,11 +1,294 @@
-import {
-  createSimpleQaAdapter,
-  parseCitations,
-  parseFinalAnswer
-} from "../chunk-SEVJPLZC.js";
-export {
-  createSimpleQaAdapter,
-  parseCitations,
-  parseFinalAnswer
-};
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { execFile } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+//#region src/benchmarks/simpleqa.ts
+/**
+* SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact
+* = a single free-text final answer string (optionally preceded by a CITATIONS:
+* block). Each item ships a short gold `answer` plus metadata (topic,
+* answer_type, source urls).
+*
+* Judge is the official SimpleQA grader — an LLM classifier that maps
+* (question, gold target, predicted answer) to exactly one of:
+*   A = CORRECT       — fully contains the gold, no contradiction
+*   B = INCORRECT     — contradicts / contains a different factual value
+*   C = NOT_ATTEMPTED — hedged, non-committal, or no value given
+* resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter
+* (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from
+* INCORRECT so the scorecard can separate abstention from error.
+*
+* There is no deterministic tier: SimpleQA's rubric (containment + abstention)
+* is the grader's job by design, so judge() always calls the pinned grader model
+* (temperature 0) and fails loud on unparseable grader output. The final-answer
+* extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').
+*
+* Requires for a live run: the bench `.venv` with `datasets`/`requests` not
+* needed — the test set is a single public CSV fetched over HTTP — plus a
+* grader key (TANGLE_API_KEY). For offline/CI verification set
+* SIMPLEQA_FIXTURES=1 to load the committed fixtures
+* (bench/fixtures/simpleqa.json) — no network.
+*/
+const execFileAsync = promisify(execFile);
+const BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+const PY = join(BENCH_ROOT, ".venv", "bin", "python");
+const FIXTURES = join(BENCH_ROOT, "fixtures", "simpleqa.json");
+const DATASET_URL = "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv";
+const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */
+const WORKER_CONTRACT = [
+	"",
+	"Research the question using live web sources and answer it with a short, specific factual value.",
+	"Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.",
+	`End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
+	"The answer after the sentinel must be the bare value only (no explanation on that line).",
+	"If you do not know the answer, state that you do not know rather than guessing."
+].join("\n");
+/** Run the bench venv python with a script; return stdout (throws on nonzero). */
+async function py(script, args = []) {
+	const { stdout } = await execFileAsync(PY, [
+		"-c",
+		script,
+		...args
+	], { maxBuffer: 1024 * 1024 * 256 });
+	return stdout;
+}
+/**
+* Parse the worker artifact into the final answer string.
+* Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);
+* fall back to the last non-empty line. Returns '' when nothing is parseable
+* (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.
+*/
+function parseFinalAnswer(artifact) {
+	const lines = artifact.split(/\r?\n/);
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const line = lines[i] ?? "";
+		const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
+		if (idx !== -1) return line.slice(idx + 13).trim();
+	}
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const t = (lines[i] ?? "").trim();
+		if (t.length > 0) return t;
+	}
+	return "";
+}
+/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */
+function parseCitations(artifact) {
+	const urls = /* @__PURE__ */ new Set();
+	for (const m of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) urls.add(m[0].replace(/[.,;]+$/, ""));
+	return [...urls];
+}
+/**
+* The official SimpleQA grader prompt (faithful paraphrase of OpenAI's
+* simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the
+* gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted
+* answer must fully contain the gold value with no contradicting statement;
+* hedged or non-committal answers are NOT_ATTEMPTED.
+*/
+const GRADER_PROMPT = (question, gold, predicted) => [
+	"Your job is to grade a predicted answer to a factual question against the single gold target answer.",
+	"Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.",
+	"First, follow these grading rules carefully.",
+	"",
+	"CORRECT:",
+	"  The predicted answer fully contains the gold target without contradicting it.",
+	"  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.",
+	"  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.",
+	"  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.",
+	"  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.",
+	"",
+	"INCORRECT:",
+	"  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.",
+	"  A prediction that states a different factual value than the gold is INCORRECT.",
+	"",
+	"NOT_ATTEMPTED:",
+	"  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,",
+	"  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.",
+	"",
+	"Also note the following rules:",
+	"  Do not punish a prediction for omitting information that would not change whether the gold target is contained.",
+	"  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.",
+	"",
+	`Question: ${question}`,
+	`Gold target: ${gold}`,
+	`Predicted answer: ${predicted}`,
+	"",
+	"Respond with ONLY a fenced JSON block and nothing else:",
+	"```json",
+	"{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}",
+	"```"
+].join("\n");
+function graderRouter() {
+	const key = process.env.TANGLE_API_KEY;
+	if (!key) throw new Error("TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)");
+	const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
+	return {
+		baseUrl: process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
+		key,
+		model
+	};
+}
+/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */
+async function gradeAnswer(question, gold, predicted, router) {
+	const res = await fetch(`${router.baseUrl}/chat/completions`, {
+		method: "POST",
+		headers: {
+			"content-type": "application/json",
+			authorization: `Bearer ${router.key}`
+		},
+		body: JSON.stringify({
+			model: router.model,
+			temperature: 0,
+			messages: [{
+				role: "user",
+				content: GRADER_PROMPT(question, gold, predicted)
+			}]
+		})
+	});
+	if (!res.ok) throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+	const body = await res.json();
+	const content = body.choices?.[0]?.message?.content;
+	if (typeof content !== "string") throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
+	const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
+	let parsed;
+	try {
+		parsed = JSON.parse(raw);
+	} catch {
+		throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`);
+	}
+	if (parsed.grade === "CORRECT" || parsed.grade === "INCORRECT" || parsed.grade === "NOT_ATTEMPTED") return parsed.grade;
+	throw new Error(`SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`);
+}
+function rowToTask(row, index) {
+	const meta = {
+		gold: row.answer,
+		goldSources: row.urls,
+		topic: row.topic,
+		answerType: row.answer_type,
+		question: row.problem
+	};
+	return {
+		id: `simpleqa-${index}`,
+		split: "test",
+		prompt: row.problem + WORKER_CONTRACT,
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.gold !== "string" || typeof md.question !== "string") throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`);
+	return md;
+}
+function selectTasks(tasks, opts) {
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		return tasks.filter((t) => want.has(t.id));
+	}
+	if (opts.limit !== void 0) return tasks.slice(0, opts.limit);
+	return tasks;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`);
+	return selectTasks(rows.map(rowToTask), opts);
+}
+/**
+* Load the live test set: fetch the public CSV via the bench venv python, parse
+* with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for
+* the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape
+* the fixtures use, so loadTasks/judge share one parse path.
+*/
+async function loadLive(opts) {
+	const stdout = await py(`
+import csv, ast, json, io, sys
+from urllib.request import urlopen
+with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
+    text = resp.read().decode('utf-8')
+rows = list(csv.DictReader(io.StringIO(text)))
+out = []
+for r in rows:
+    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}
+    out.append({
+        'problem': r.get('problem', ''),
+        'answer': r.get('answer', ''),
+        'topic': str(meta.get('topic', '')),
+        'answer_type': str(meta.get('answer_type', '')),
+        'urls': list(meta.get('urls', [])),
+    })
+print(json.dumps(out))
+`);
+	return selectTasks(JSON.parse(stdout).map(rowToTask), opts);
+}
+function createSimpleQaAdapter() {
+	const fixturesMode = process.env.SIMPLEQA_FIXTURES === "1";
+	return {
+		name: "simpleqa",
+		async preflight() {
+			graderRouter();
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8").catch((err) => {
+					throw new Error(`SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+				});
+				return;
+			}
+			try {
+				await py(`from urllib.request import urlopen
+with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
+    head = resp.read(64).decode('utf-8', 'replace')
+assert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]
+print('ok')`);
+			} catch (err) {
+				const msg = err instanceof Error ? err.message : String(err);
+				throw new Error(`simpleqa preflight failed: ${msg}\nFix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; (2) ensure network access to ${DATASET_URL} ; or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`);
+			}
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			return loadLive(opts);
+		},
+		async goldArtifact(task) {
+			const meta = readMeta(task);
+			return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const finalAnswer = parseFinalAnswer(artifact);
+			const citations = parseCitations(artifact);
+			if (finalAnswer.length === 0) return {
+				resolved: false,
+				score: 0,
+				detail: JSON.stringify({
+					grade: "NOT_ATTEMPTED",
+					gradeLetter: "C",
+					reason: "no parseable answer",
+					gold: meta.gold,
+					topic: meta.topic,
+					answerType: meta.answerType,
+					citationCount: citations.length
+				})
+			};
+			const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter());
+			const gradeLetter = grade === "CORRECT" ? "A" : grade === "INCORRECT" ? "B" : "C";
+			const resolved = grade === "CORRECT";
+			return {
+				resolved,
+				score: resolved ? 1 : 0,
+				detail: JSON.stringify({
+					grade,
+					gradeLetter,
+					gold: meta.gold,
+					predicted: finalAnswer,
+					topic: meta.topic,
+					answerType: meta.answerType,
+					citationCount: citations.length
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createSimpleQaAdapter, parseCitations, parseFinalAnswer };
+
 //# sourceMappingURL=simpleqa.js.map

package/dist/benchmarks/simpleqa.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"simpleqa.js","names":[],"sources":["../../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,MAAM,cACJ;AACF,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;;;;;;;AASA,MAAM,iBAAiB,UAAkB,MAAc,cACrD;CACE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,eAA6B;CACpC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6EAA6E;CACvG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;CAChB,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS,cAAc,UAAU,MAAM,SAAS;GAAE,CAAC;EAChF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,wBAAwB,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE3F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,gDAAgD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAEtG,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,GAAG;CACzG;CACA,IAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBACjF,OAAO,OAAO;CAEhB,MAAM,IAAI,MACR,mEAAmE,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GACxG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;CAC7D,MAAM,OAAqB;EACzB,MAAM,IAAI;EACV,aAAa,IAAI;EACjB,OAAO,IAAI;EACX,YAAY,IAAI;EAChB,UAAU,IAAI;CAChB;CACA,OAAO;EACL,IAAI,YAAY;EAChB,OAAO;EACP,QAAQ,IAAI,UAAU;EACtB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAC/D,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,gEAAgE;CAE3G,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,OAAO,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC3C;CACA,IAAI,KAAK,UAAU,KAAA,GAAW,OAAO,MAAM,MAAM,GAAG,KAAK,KAAK;CAC9D,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,4CAA4C,KAAK,OAAO,iCAAiC;CACtG,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;;;;;;;AAQA,eAAe,SAAS,MAAyC;CAmB/D,MAAM,SAAS,MAAM,GAAG;;;eAfX,KAAK,UAAU,WAAW,EAAE;;;;;;;;;;;;;;CAeX;CAE9B,OAAO,YADM,KAAK,MAAM,MACF,CAAC,CAAC,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,aAAa;GACb,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,2BAA2B,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC1F;IACF,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,WAAW,EAAE;;;YAInC;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,8BAA8B,IAAI,uGAEA,YAAY,6EAEhD;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,OAAO,SAAS,IAAI;EACtB;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAC7C,MAAM,YAAY,eAAe,QAAQ;GAEzC,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,OAAO;KACP,aAAa;KACb,QAAQ;KACR,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;GAGF,MAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;GACrF,MAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;GAC9E,MAAM,WAAW,UAAU;GAC3B,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KACrB;KACA;KACA,MAAM,KAAK;KACX,WAAW;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/swe-bench.d.ts

@@ -1,21 +1,7 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { StagedRunCaptureSpec } from './_harness.js';
-import { BenchmarkAdapter, BenchScore } from './types.js';
-
-/**
- * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =
- * the official `swebench` harness: apply the patch in the instance's Docker
- * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic
- * — no LLM judge.
- *
- * Requires: the bench `.venv` with `swebench` installed + a running Docker
- * daemon (per-instance images are pulled/built on first run).
- *
- * Process/Docker/report plumbing is shared via ./_harness; this file owns the
- * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the
- * predictions-file → run_evaluation argv → report-shape mapping.
- */
-
+import { BenchScore, BenchmarkAdapter } from "./types.js";
+import { StagedRunCaptureSpec } from "./_harness.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/swe-bench.d.ts
 /**
  * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.
  *
@@ -28,29 +14,30 @@ import { BenchmarkAdapter, BenchScore } from './types.js';
 declare const swePatchOutput: OutputAdapter<string>;
 type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance';
 interface SweBenchArtifactCaptureContext {
-    readonly taskId: string;
-    readonly runId: string;
-    /** One-based sequence unique within this adapter instance. */
-    readonly attemptSequence: number;
+  readonly taskId: string;
+  readonly runId: string;
+  /** One-based sequence unique within this adapter instance. */
+  readonly attemptSequence: number;
 }
 interface SweBenchAdapterOptions {
-    readonly timeoutMs?: number;
-    readonly cacheLevel?: SweBenchCacheLevel;
-    /**
-     * Return a unique destination for any attempt whose complete official
-     * evaluator directory and process logs should be retained.
-     */
-    readonly captureEvaluatorArtifacts?: (context: SweBenchArtifactCaptureContext) => StagedRunCaptureSpec | undefined;
+  readonly timeoutMs?: number;
+  readonly cacheLevel?: SweBenchCacheLevel;
+  /**
+   * Return a unique destination for any attempt whose complete official
+   * evaluator directory and process logs should be retained.
+   */
+  readonly captureEvaluatorArtifacts?: (context: SweBenchArtifactCaptureContext) => StagedRunCaptureSpec | undefined;
 }
 /** Convert one official report into a score without turning evaluator failures into agent failures. */
 declare function scoreSweReport(taskId: string, value: unknown): BenchScore;
 declare function sweEvaluationArgv(args: {
-    readonly predictionsPath: string;
-    readonly runId: string;
-    readonly instanceId: string;
-    readonly cacheLevel: SweBenchCacheLevel;
-    readonly namespace?: 'swebench' | 'none';
+  readonly predictionsPath: string;
+  readonly runId: string;
+  readonly instanceId: string;
+  readonly cacheLevel: SweBenchCacheLevel;
+  readonly namespace?: 'swebench' | 'none';
 }): string[];
 declare function createSweBenchAdapter(options?: SweBenchAdapterOptions): BenchmarkAdapter;
-
-export { type SweBenchAdapterOptions, type SweBenchArtifactCaptureContext, type SweBenchCacheLevel, createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput };
+//#endregion
+export { SweBenchAdapterOptions, SweBenchArtifactCaptureContext, SweBenchCacheLevel, createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput };
+//# sourceMappingURL=swe-bench.d.ts.map