@tangle-network/agent-bench 0.7.0 → 0.8.0

package/dist/benchmarks/finsearchcomp.js

@@ -1,3 +1,4 @@
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -138,28 +139,21 @@ function parseJudgeOutput(content) {
 }
 /** Run the record's own judge via the router. Fail loud on transport/parse errors. */
 async function runRecordJudge(meta, response, router) {
-	const res = await fetch(`${router.baseUrl}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${router.key}`
-		},
-		body: JSON.stringify({
-			model: router.model,
-			temperature: 0,
-			messages: [{
-				role: "system",
-				content: meta.judgeSystemPrompt
-			}, {
-				role: "user",
-				content: fillJudgePrompt(meta, response)
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-	const body = await res.json();
-	const content = body.choices?.[0]?.message?.content;
-	if (typeof content !== "string") throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const content = (await runBenchRouterTurn({
+		routerBaseUrl: router.baseUrl,
+		routerKey: router.key,
+		profile: {
+			name: "finsearchcomp-judge",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: router.model,
+				metadata: { temperature: 0 }
+			},
+			prompt: { systemPrompt: meta.judgeSystemPrompt }
+		}
+	}, fillJudgePrompt(meta, response))).finalText;
+	if (!content) throw new Error("FinSearchComp judge returned no message content");
 	const { resolved, score, raw } = parseJudgeOutput(content);
 	return {
 		resolved,

package/dist/benchmarks/finsearchcomp.js.map

@@ -1 +1 @@
-{"version":3,"file":"finsearchcomp.js","names":[],"sources":["../../src/benchmarks/finsearchcomp.ts"],"sourcesContent":["/**\n * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +\n * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).\n *\n * Each record is SELF-CONTAINED and carries its OWN judge: a per-record\n * `judge_system_prompt` + a `judge_prompt_template` with {prompt},\n * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),\n * and {response} (the student answer) placeholders. We faithfully REPLICATE the\n * benchmark's judge — fill the template, run it under the record's system prompt\n * via the pinned router (temperature 0), and parse the JSON the judge emits. The\n * judge returns an `answer_score` field; 1/positive = resolved, else not.\n * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).\n *\n * There is NO deterministic Tier-1 path here: the gold answer is prose with\n * region-specific units, embedded tolerances (\"允许1%的误差\"), and multi-point\n * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,\n * so the judge IS the score — fail loud on unparseable judge output (never\n * default to resolved).\n *\n * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)\n * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)\n * needs a live market snapshot / akshare ground truth to fill the judge's\n * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.\n * See {@link T1Seam}.\n *\n * Requires for a live run: network access to the GitHub-hosted dataset JSON and\n * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back\n * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit\n * console.warn — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'finsearchcomp.json')\n\nconst DATASET_URL =\n  'https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json'\n\n/** Task-type prefix on prompt_id, e.g. \"(T2)Simple_Historical_Lookup_001\". */\ntype TaskType = 'T1' | 'T2' | 'T3'\n/** Only these are self-contained (gold + own judge, no external ground truth). */\nconst SUPPORTED_TYPES: ReadonlySet<TaskType> = new Set<TaskType>(['T2', 'T3'])\n\n/**\n * Worker contract appended to every task prompt. The judge is paraphrase- and\n * keyword-aware (\"答案为\", \"the answer is\"), so we do not impose a sentinel; we\n * pass the whole artifact through as {response}. We DO ask the worker to lead\n * with its final value so a truncated artifact still carries the answer.\n */\nconst WORKER_CONTRACT = [\n  '',\n  'Research this financial question using live web/market sources and answer it.',\n  'State your final answer explicitly, with the exact units and precision the question requests.',\n  'Respect any tolerance the question implies; show the value you settled on, not just intermediate work.',\n].join('\\n')\n\n/**\n * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a\n * live market snapshot the judge consumes via a {ground_truth} template slot;\n * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a\n * resolver that fetches the akshare/market value for the record at judge time\n * and filling {ground_truth}. Until then T1 is excluded from loadTasks.\n */\nexport interface T1Seam {\n  promptId: string\n  /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */\n  resolveGroundTruth(promptId: string): Promise<string>\n}\n\ninterface FinSearchRecord {\n  prompt_id: string\n  prompt: string\n  response_reference: string\n  judge_prompt_template: string\n  judge_system_prompt: string\n  label: string\n}\n\ninterface FinSearchMeta {\n  promptId: string\n  label: string\n  taskType: TaskType\n  region: string\n  responseReference: string\n  judgePromptTemplate: string\n  judgeSystemPrompt: string\n  rawPrompt: string\n}\n\n/** \"(T2)Simple_Historical_Lookup_001\" → \"T2\"; throws on an unknown/missing prefix. */\nfunction parseTaskType(promptId: string): TaskType {\n  const m = promptId.match(/^\\((T[123])\\)/)\n  if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`)\n  return m[1] as TaskType\n}\n\n/** \"Simple_Historical_Lookup(Greater China)\" → \"Greater China\"; '' when absent. */\nfunction parseRegion(label: string): string {\n  const m = label.match(/\\(([^()]+)\\)\\s*$/)\n  return m ? m[1].trim() : ''\n}\n\nfunction recordToTask(rec: FinSearchRecord): BenchTask {\n  const taskType = parseTaskType(rec.prompt_id)\n  const meta: FinSearchMeta = {\n    promptId: rec.prompt_id,\n    label: rec.label,\n    taskType,\n    region: parseRegion(rec.label),\n    responseReference: rec.response_reference,\n    judgePromptTemplate: rec.judge_prompt_template,\n    judgeSystemPrompt: rec.judge_system_prompt,\n    rawPrompt: rec.prompt,\n  }\n  return {\n    id: rec.prompt_id,\n    split: taskType,\n    prompt: rec.prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinSearchMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.responseReference !== 'string' ||\n    typeof md.judgePromptTemplate !== 'string' ||\n    typeof md.judgeSystemPrompt !== 'string' ||\n    typeof md.rawPrompt !== 'string'\n  ) {\n    throw new Error(`FinSearchComp task ${task.id} missing judge metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinSearchMeta\n}\n\n/** Fill the per-record judge_prompt_template. {ground_truth} is T1-only and unused for T2/T3. */\nfunction fillJudgePrompt(meta: FinSearchMeta, response: string): string {\n  return meta.judgePromptTemplate\n    .replaceAll('{prompt}', meta.rawPrompt)\n    .replaceAll('{response_reference}', meta.responseReference)\n    .replaceAll('{response}', response)\n}\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/**\n * The judge emits a JSON object with an `answer_score`. In this dataset it is a\n * flat integer ({\"answer_score\": 1}); we also accept the nested-array form\n * ([[1]] → read [0][0]) the upstream scorer can produce. 1/positive = resolved.\n * A \"null\" sentinel (judge could not obtain ground truth) is NOT a pass.\n */\nfunction readAnswerScore(value: unknown): { resolved: boolean; score: number; raw: unknown } {\n  let v: unknown = value\n  // unwrap nested-array form: [[s]] / [s]\n  while (Array.isArray(v)) v = v[0]\n  if (v === null || v === undefined || v === 'null') {\n    return { resolved: false, score: 0, raw: value }\n  }\n  const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v) : NaN\n  if (!Number.isFinite(n)) {\n    throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`)\n  }\n  // Their scale: 1 = correct, 0 = wrong. Treat any positive as resolved; clamp to 0..1.\n  const resolved = n >= 1\n  const score = n <= 0 ? 0 : n >= 1 ? 1 : n\n  return { resolved, score, raw: value }\n}\n\n/** Extract the first JSON object carrying `answer_score` from judge output; throw if none. */\nfunction parseJudgeOutput(content: string): { resolved: boolean; score: number; raw: unknown } {\n  // Prefer a fenced block, then fall back to scanning for a {...} with answer_score.\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[^{}]*\"answer_score\"[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const c of candidates) {\n    let parsed: { answer_score?: unknown }\n    try {\n      parsed = JSON.parse(c) as { answer_score?: unknown }\n    } catch {\n      continue\n    }\n    if ('answer_score' in parsed) return readAnswerScore(parsed.answer_score)\n  }\n  throw new Error(\n    `FinSearchComp judge produced no parseable {\"answer_score\": …}: ${content.slice(0, 400)}`,\n  )\n}\n\n/** Run the record's own judge via the router. Fail loud on transport/parse errors. */\nasync function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise<BenchScore> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillJudgePrompt(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const { resolved, score, raw } = parseJudgeOutput(content)\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      taskType: meta.taskType,\n      region: meta.region,\n      answerScore: raw,\n      judgeModel: router.model,\n    }),\n  }\n}\n\nfunction selectRecords(records: FinSearchRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records\n    .filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id)))\n    .map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = balanceByType(tasks, opts.limit)\n  }\n  return tasks\n}\n\n/** Round-robin across task types (T2/T3) so a `limit` pulls a balanced mix\n *  rather than slicing the type-ordered dataset's first N (which yields all\n *  T2). Deterministic — preserves dataset order within each type. */\nfunction balanceByType(tasks: BenchTask[], limit: number): BenchTask[] {\n  const queues = new Map<TaskType, BenchTask[]>()\n  for (const t of tasks) {\n    const k = parseTaskType(t.id)\n    const q = queues.get(k)\n    if (q) q.push(t)\n    else queues.set(k, [t])\n  }\n  const lanes = [...queues.values()]\n  const out: BenchTask[] = []\n  for (let i = 0; out.length < limit; i++) {\n    const before = out.length\n    for (const lane of lanes) {\n      if (i < lane.length) out.push(lane[i])\n      if (out.length >= limit) break\n    }\n    if (out.length === before) break // all lanes exhausted\n  }\n  return out\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as FinSearchRecord[]\n  return selectRecords(records, opts)\n}\n\nasync function fetchDataset(): Promise<FinSearchRecord[]> {\n  const res = await fetch(DATASET_URL)\n  if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`)\n  const data = (await res.json()) as FinSearchRecord[]\n  if (!Array.isArray(data) || data.length === 0) {\n    throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`)\n  }\n  return data\n}\n\nexport function createFinsearchcompAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === '1'\n\n  return {\n    name: 'finsearchcomp',\n\n    async preflight() {\n      // The per-record LLM judge is the arbiter in both modes — its router\n      // config must exist or no score can be produced.\n      judgeRouter()\n      console.warn(\n        '[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live ' +\n          'market snapshot / akshare ground truth for the judge {ground_truth} slot. ' +\n          'loadTasks supports T2 + T3 only; wire T1Seam to add T1.',\n      )\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      const res = await fetch(DATASET_URL, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 ` +\n            `to run against the committed fixtures offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. ` +\n            `Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let records: FinSearchRecord[]\n      try {\n        records = await fetchDataset()\n      } catch (err) {\n        console.warn(\n          `[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); ` +\n            `falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRecords(records, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the record's response_reference fed back as the student\n      // answer, so verify-judge proves gold→resolved through the SAME per-record\n      // judge the real artifact takes. The reference may carry tolerance/criteria\n      // prose; the judge is built to read the value out of it.\n      const meta = readMeta(task)\n      return meta.responseReference\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const response = artifact.trim()\n      if (response.length === 0) {\n        // Fail-closed — the benchmark's own judges score an empty answer 0.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ taskType: meta.taskType, region: meta.region, reason: 'empty answer' }),\n        }\n      }\n      return runRecordJudge(meta, response, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,MAAM,WAAW,KADE,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAClC,GAAG,YAAY,oBAAoB;AAElE,MAAM,cACJ;;AAKF,MAAM,kCAAyC,IAAI,IAAc,CAAC,MAAM,IAAI,CAAC;;;;;;;AAQ7E,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AAoCX,SAAS,cAAc,UAA4B;CACjD,MAAM,IAAI,SAAS,MAAM,eAAe;CACxC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,qDAAqD,KAAK,UAAU,QAAQ,GAAG;CACvG,OAAO,EAAE;AACX;;AAGA,SAAS,YAAY,OAAuB;CAC1C,MAAM,IAAI,MAAM,MAAM,kBAAkB;CACxC,OAAO,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI;AAC3B;AAEA,SAAS,aAAa,KAAiC;CACrD,MAAM,WAAW,cAAc,IAAI,SAAS;CAC5C,MAAM,OAAsB;EAC1B,UAAU,IAAI;EACd,OAAO,IAAI;EACX;EACA,QAAQ,YAAY,IAAI,KAAK;EAC7B,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,mBAAmB,IAAI;EACvB,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,wBAAwB,YAClC,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,cAAc,UAExB,MAAM,IAAI,MAAM,sBAAsB,KAAK,GAAG,wDAAwD;CAExG,OAAO;AACT;;AAGA,SAAS,gBAAgB,MAAqB,UAA0B;CACtE,OAAO,KAAK,oBACT,WAAW,YAAY,KAAK,SAAS,CAAC,CACtC,WAAW,wBAAwB,KAAK,iBAAiB,CAAC,CAC1D,WAAW,cAAc,QAAQ;AACtC;AAQA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,uEAAuE;CACjG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;;;;;;AAQA,SAAS,gBAAgB,OAAoE;CAC3F,IAAI,IAAa;CAEjB,OAAO,MAAM,QAAQ,CAAC,GAAG,IAAI,EAAE;CAC/B,IAAI,MAAM,QAAQ,MAAM,KAAA,KAAa,MAAM,QACzC,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,KAAK;CAAM;CAEjD,MAAM,IAAI,OAAO,MAAM,WAAW,IAAI,OAAO,MAAM,WAAW,OAAO,CAAC,IAAI;CAC1E,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,MAAM,iDAAiD,KAAK,UAAU,KAAK,GAAG;CAK1F,OAAO;EAAE,UAFQ,KAAK;EAEH,OADL,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI;EACd,KAAK;CAAM;AACvC;;AAGA,SAAS,iBAAiB,SAAqE;CAE7F,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,mCAAmC,GAAG,WAAW,KAAK,EAAE,EAAE;CAC3F,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,KAAK,YAAY;EAC1B,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,CAAC;EACvB,QAAQ;GACN;EACF;EACA,IAAI,kBAAkB,QAAQ,OAAO,gBAAgB,OAAO,YAAY;CAC1E;CACA,MAAM,IAAI,MACR,kEAAkE,QAAQ,MAAM,GAAG,GAAG,GACxF;AACF;;AAGA,eAAe,eAAe,MAAqB,UAAkB,QAA0C;CAC7G,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,KAAK;GAAkB,GAClD;IAAE,MAAM;IAAQ,SAAS,gBAAgB,MAAM,QAAQ;GAAE,CAC3D;EACF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,4BAA4B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,oDAAoD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAE1G,MAAM,EAAE,UAAU,OAAO,QAAQ,iBAAiB,OAAO;CACzD,OAAO;EACL;EACA;EACA,QAAQ,KAAK,UAAU;GACrB,UAAU,KAAK;GACf,QAAQ,KAAK;GACb,aAAa;GACb,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAEA,SAAS,cAAc,SAA4B,MAAgC;CACjF,IAAI,QAAQ,QACT,QAAQ,MAAM,gBAAgB,IAAI,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,CAC9D,IAAI,YAAY;CACnB,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,MAAM,EAAE,UAAU,KAAK,KAAK;CAClE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,cAAc,OAAO,KAAK,KAAK;CAEzC,OAAO;AACT;;;;AAKA,SAAS,cAAc,OAAoB,OAA4B;CACrE,MAAM,yBAAS,IAAI,IAA2B;CAC9C,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,cAAc,EAAE,EAAE;EAC5B,MAAM,IAAI,OAAO,IAAI,CAAC;EACtB,IAAI,GAAG,EAAE,KAAK,CAAC;OACV,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC;CACxB;CACA,MAAM,QAAQ,CAAC,GAAG,OAAO,OAAO,CAAC;CACjC,MAAM,MAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,OAAO,KAAK;EACvC,MAAM,SAAS,IAAI;EACnB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,IAAI,KAAK,QAAQ,IAAI,KAAK,KAAK,EAAE;GACrC,IAAI,IAAI,UAAU,OAAO;EAC3B;EACA,IAAI,IAAI,WAAW,QAAQ;CAC7B;CACA,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CAEnE,OAAO,cADS,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAC/B,GAAG,IAAI;AACpC;AAEA,eAAe,eAA2C;CACxD,MAAM,MAAM,MAAM,MAAM,WAAW;CACnC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,8BAA8B,IAAI,OAAO,YAAY,aAAa;CAC/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,IAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAC1C,MAAM,IAAI,MAAM,gDAAgD,aAAa;CAE/E,OAAO;AACT;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,eAAe,QAAQ,IAAI,2BAA2B;CAE5D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,QAAQ,KACN,oNAGF;GACA,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,gCAAgC,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC/F;IACF,CAAC;IACD;GACF;GACA,MAAM,MAAM,MAAM,MAAM,aAAa,EAAE,QAAQ,OAAO,CAAC,CAAC,CAAC,OAAO,QAAQ;IACtE,MAAM,IAAI,MACR,2CAA2C,YAAY,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,0IAGtG;GACF,CAAC;GACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MACR,yCAAyC,IAAI,OAAO,OAAO,YAAY,8EAEzE;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,aAAa;GAC/B,SAAS,KAAK;IACZ,QAAQ,KACN,8CAA8C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAC5C,UAC7C;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,cAAc,SAAS,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAMlC,OADa,SAAS,IACZ,CAAC,CAAC;EACd;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,WAAW,SAAS,KAAK;GAC/B,IAAI,SAAS,WAAW,GAEtB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,UAAU,KAAK;KAAU,QAAQ,KAAK;KAAQ,QAAQ;IAAe,CAAC;GACjG;GAEF,OAAO,eAAe,MAAM,UAAU,YAAY,CAAC;EACrD;CACF;AACF"}
+{"version":3,"file":"finsearchcomp.js","names":[],"sources":["../../src/benchmarks/finsearchcomp.ts"],"sourcesContent":["/**\n * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +\n * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).\n *\n * Each record is SELF-CONTAINED and carries its OWN judge: a per-record\n * `judge_system_prompt` + a `judge_prompt_template` with {prompt},\n * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),\n * and {response} (the student answer) placeholders. We faithfully REPLICATE the\n * benchmark's judge — fill the template, run it under the record's system prompt\n * via the pinned router (temperature 0), and parse the JSON the judge emits. The\n * judge returns an `answer_score` field; 1/positive = resolved, else not.\n * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).\n *\n * There is NO deterministic Tier-1 path here: the gold answer is prose with\n * region-specific units, embedded tolerances (\"允许1%的误差\"), and multi-point\n * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,\n * so the judge IS the score — fail loud on unparseable judge output (never\n * default to resolved).\n *\n * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)\n * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)\n * needs a live market snapshot / akshare ground truth to fill the judge's\n * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.\n * See {@link T1Seam}.\n *\n * Requires for a live run: network access to the GitHub-hosted dataset JSON and\n * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back\n * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit\n * console.warn — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'finsearchcomp.json')\n\nconst DATASET_URL =\n  'https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json'\n\n/** Task-type prefix on prompt_id, e.g. \"(T2)Simple_Historical_Lookup_001\". */\ntype TaskType = 'T1' | 'T2' | 'T3'\n/** Only these are self-contained (gold + own judge, no external ground truth). */\nconst SUPPORTED_TYPES: ReadonlySet<TaskType> = new Set<TaskType>(['T2', 'T3'])\n\n/**\n * Worker contract appended to every task prompt. The judge is paraphrase- and\n * keyword-aware (\"答案为\", \"the answer is\"), so we do not impose a sentinel; we\n * pass the whole artifact through as {response}. We DO ask the worker to lead\n * with its final value so a truncated artifact still carries the answer.\n */\nconst WORKER_CONTRACT = [\n  '',\n  'Research this financial question using live web/market sources and answer it.',\n  'State your final answer explicitly, with the exact units and precision the question requests.',\n  'Respect any tolerance the question implies; show the value you settled on, not just intermediate work.',\n].join('\\n')\n\n/**\n * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a\n * live market snapshot the judge consumes via a {ground_truth} template slot;\n * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a\n * resolver that fetches the akshare/market value for the record at judge time\n * and filling {ground_truth}. Until then T1 is excluded from loadTasks.\n */\nexport interface T1Seam {\n  promptId: string\n  /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */\n  resolveGroundTruth(promptId: string): Promise<string>\n}\n\ninterface FinSearchRecord {\n  prompt_id: string\n  prompt: string\n  response_reference: string\n  judge_prompt_template: string\n  judge_system_prompt: string\n  label: string\n}\n\ninterface FinSearchMeta {\n  promptId: string\n  label: string\n  taskType: TaskType\n  region: string\n  responseReference: string\n  judgePromptTemplate: string\n  judgeSystemPrompt: string\n  rawPrompt: string\n}\n\n/** \"(T2)Simple_Historical_Lookup_001\" → \"T2\"; throws on an unknown/missing prefix. */\nfunction parseTaskType(promptId: string): TaskType {\n  const m = promptId.match(/^\\((T[123])\\)/)\n  if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`)\n  return m[1] as TaskType\n}\n\n/** \"Simple_Historical_Lookup(Greater China)\" → \"Greater China\"; '' when absent. */\nfunction parseRegion(label: string): string {\n  const m = label.match(/\\(([^()]+)\\)\\s*$/)\n  return m ? m[1].trim() : ''\n}\n\nfunction recordToTask(rec: FinSearchRecord): BenchTask {\n  const taskType = parseTaskType(rec.prompt_id)\n  const meta: FinSearchMeta = {\n    promptId: rec.prompt_id,\n    label: rec.label,\n    taskType,\n    region: parseRegion(rec.label),\n    responseReference: rec.response_reference,\n    judgePromptTemplate: rec.judge_prompt_template,\n    judgeSystemPrompt: rec.judge_system_prompt,\n    rawPrompt: rec.prompt,\n  }\n  return {\n    id: rec.prompt_id,\n    split: taskType,\n    prompt: rec.prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinSearchMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.responseReference !== 'string' ||\n    typeof md.judgePromptTemplate !== 'string' ||\n    typeof md.judgeSystemPrompt !== 'string' ||\n    typeof md.rawPrompt !== 'string'\n  ) {\n    throw new Error(`FinSearchComp task ${task.id} missing judge metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinSearchMeta\n}\n\n/** Fill the per-record judge_prompt_template. {ground_truth} is T1-only and unused for T2/T3. */\nfunction fillJudgePrompt(meta: FinSearchMeta, response: string): string {\n  return meta.judgePromptTemplate\n    .replaceAll('{prompt}', meta.rawPrompt)\n    .replaceAll('{response_reference}', meta.responseReference)\n    .replaceAll('{response}', response)\n}\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/**\n * The judge emits a JSON object with an `answer_score`. In this dataset it is a\n * flat integer ({\"answer_score\": 1}); we also accept the nested-array form\n * ([[1]] → read [0][0]) the upstream scorer can produce. 1/positive = resolved.\n * A \"null\" sentinel (judge could not obtain ground truth) is NOT a pass.\n */\nfunction readAnswerScore(value: unknown): { resolved: boolean; score: number; raw: unknown } {\n  let v: unknown = value\n  // unwrap nested-array form: [[s]] / [s]\n  while (Array.isArray(v)) v = v[0]\n  if (v === null || v === undefined || v === 'null') {\n    return { resolved: false, score: 0, raw: value }\n  }\n  const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v) : NaN\n  if (!Number.isFinite(n)) {\n    throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`)\n  }\n  // Their scale: 1 = correct, 0 = wrong. Treat any positive as resolved; clamp to 0..1.\n  const resolved = n >= 1\n  const score = n <= 0 ? 0 : n >= 1 ? 1 : n\n  return { resolved, score, raw: value }\n}\n\n/** Extract the first JSON object carrying `answer_score` from judge output; throw if none. */\nfunction parseJudgeOutput(content: string): { resolved: boolean; score: number; raw: unknown } {\n  // Prefer a fenced block, then fall back to scanning for a {...} with answer_score.\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[^{}]*\"answer_score\"[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const c of candidates) {\n    let parsed: { answer_score?: unknown }\n    try {\n      parsed = JSON.parse(c) as { answer_score?: unknown }\n    } catch {\n      continue\n    }\n    if ('answer_score' in parsed) return readAnswerScore(parsed.answer_score)\n  }\n  throw new Error(\n    `FinSearchComp judge produced no parseable {\"answer_score\": …}: ${content.slice(0, 400)}`,\n  )\n}\n\n/** Run the record's own judge via the router. Fail loud on transport/parse errors. */\nasync function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise<BenchScore> {\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'finsearchcomp-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0 },\n        },\n        prompt: { systemPrompt: meta.judgeSystemPrompt },\n      },\n    },\n    fillJudgePrompt(meta, response),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('FinSearchComp judge returned no message content')\n  const { resolved, score, raw } = parseJudgeOutput(content)\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      taskType: meta.taskType,\n      region: meta.region,\n      answerScore: raw,\n      judgeModel: router.model,\n    }),\n  }\n}\n\nfunction selectRecords(records: FinSearchRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records\n    .filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id)))\n    .map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = balanceByType(tasks, opts.limit)\n  }\n  return tasks\n}\n\n/** Round-robin across task types (T2/T3) so a `limit` pulls a balanced mix\n *  rather than slicing the type-ordered dataset's first N (which yields all\n *  T2). Deterministic — preserves dataset order within each type. */\nfunction balanceByType(tasks: BenchTask[], limit: number): BenchTask[] {\n  const queues = new Map<TaskType, BenchTask[]>()\n  for (const t of tasks) {\n    const k = parseTaskType(t.id)\n    const q = queues.get(k)\n    if (q) q.push(t)\n    else queues.set(k, [t])\n  }\n  const lanes = [...queues.values()]\n  const out: BenchTask[] = []\n  for (let i = 0; out.length < limit; i++) {\n    const before = out.length\n    for (const lane of lanes) {\n      if (i < lane.length) out.push(lane[i])\n      if (out.length >= limit) break\n    }\n    if (out.length === before) break // all lanes exhausted\n  }\n  return out\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as FinSearchRecord[]\n  return selectRecords(records, opts)\n}\n\nasync function fetchDataset(): Promise<FinSearchRecord[]> {\n  const res = await fetch(DATASET_URL)\n  if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`)\n  const data = (await res.json()) as FinSearchRecord[]\n  if (!Array.isArray(data) || data.length === 0) {\n    throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`)\n  }\n  return data\n}\n\nexport function createFinsearchcompAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === '1'\n\n  return {\n    name: 'finsearchcomp',\n\n    async preflight() {\n      // The per-record LLM judge is the arbiter in both modes — its router\n      // config must exist or no score can be produced.\n      judgeRouter()\n      console.warn(\n        '[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live ' +\n          'market snapshot / akshare ground truth for the judge {ground_truth} slot. ' +\n          'loadTasks supports T2 + T3 only; wire T1Seam to add T1.',\n      )\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      const res = await fetch(DATASET_URL, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 ` +\n            `to run against the committed fixtures offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. ` +\n            `Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let records: FinSearchRecord[]\n      try {\n        records = await fetchDataset()\n      } catch (err) {\n        console.warn(\n          `[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); ` +\n            `falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRecords(records, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the record's response_reference fed back as the student\n      // answer, so verify-judge proves gold→resolved through the SAME per-record\n      // judge the real artifact takes. The reference may carry tolerance/criteria\n      // prose; the judge is built to read the value out of it.\n      const meta = readMeta(task)\n      return meta.responseReference\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const response = artifact.trim()\n      if (response.length === 0) {\n        // Fail-closed — the benchmark's own judges score an empty answer 0.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ taskType: meta.taskType, region: meta.region, reason: 'empty answer' }),\n        }\n      }\n      return runRecordJudge(meta, response, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsCA,MAAM,WAAW,KADE,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAClC,GAAG,YAAY,oBAAoB;AAElE,MAAM,cACJ;;AAKF,MAAM,kCAAyC,IAAI,IAAc,CAAC,MAAM,IAAI,CAAC;;;;;;;AAQ7E,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AAoCX,SAAS,cAAc,UAA4B;CACjD,MAAM,IAAI,SAAS,MAAM,eAAe;CACxC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,qDAAqD,KAAK,UAAU,QAAQ,GAAG;CACvG,OAAO,EAAE;AACX;;AAGA,SAAS,YAAY,OAAuB;CAC1C,MAAM,IAAI,MAAM,MAAM,kBAAkB;CACxC,OAAO,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI;AAC3B;AAEA,SAAS,aAAa,KAAiC;CACrD,MAAM,WAAW,cAAc,IAAI,SAAS;CAC5C,MAAM,OAAsB;EAC1B,UAAU,IAAI;EACd,OAAO,IAAI;EACX;EACA,QAAQ,YAAY,IAAI,KAAK;EAC7B,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,mBAAmB,IAAI;EACvB,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,wBAAwB,YAClC,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,cAAc,UAExB,MAAM,IAAI,MAAM,sBAAsB,KAAK,GAAG,wDAAwD;CAExG,OAAO;AACT;;AAGA,SAAS,gBAAgB,MAAqB,UAA0B;CACtE,OAAO,KAAK,oBACT,WAAW,YAAY,KAAK,SAAS,CAAC,CACtC,WAAW,wBAAwB,KAAK,iBAAiB,CAAC,CAC1D,WAAW,cAAc,QAAQ;AACtC;AAQA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,uEAAuE;CACjG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;;;;;;AAQA,SAAS,gBAAgB,OAAoE;CAC3F,IAAI,IAAa;CAEjB,OAAO,MAAM,QAAQ,CAAC,GAAG,IAAI,EAAE;CAC/B,IAAI,MAAM,QAAQ,MAAM,KAAA,KAAa,MAAM,QACzC,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,KAAK;CAAM;CAEjD,MAAM,IAAI,OAAO,MAAM,WAAW,IAAI,OAAO,MAAM,WAAW,OAAO,CAAC,IAAI;CAC1E,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,MAAM,iDAAiD,KAAK,UAAU,KAAK,GAAG;CAK1F,OAAO;EAAE,UAFQ,KAAK;EAEH,OADL,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI;EACd,KAAK;CAAM;AACvC;;AAGA,SAAS,iBAAiB,SAAqE;CAE7F,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,mCAAmC,GAAG,WAAW,KAAK,EAAE,EAAE;CAC3F,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,KAAK,YAAY;EAC1B,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,CAAC;EACvB,QAAQ;GACN;EACF;EACA,IAAI,kBAAkB,QAAQ,OAAO,gBAAgB,OAAO,YAAY;CAC1E;CACA,MAAM,IAAI,MACR,kEAAkE,QAAQ,MAAM,GAAG,GAAG,GACxF;AACF;;AAGA,eAAe,eAAe,MAAqB,UAAkB,QAA0C;CAkB7G,MAAM,WAAU,MAjBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU,EAAE,aAAa,EAAE;GAC7B;GACA,QAAQ,EAAE,cAAc,KAAK,kBAAkB;EACjD;CACF,GACA,gBAAgB,MAAM,QAAQ,CAChC,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,iDAAiD;CAC/E,MAAM,EAAE,UAAU,OAAO,QAAQ,iBAAiB,OAAO;CACzD,OAAO;EACL;EACA;EACA,QAAQ,KAAK,UAAU;GACrB,UAAU,KAAK;GACf,QAAQ,KAAK;GACb,aAAa;GACb,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAEA,SAAS,cAAc,SAA4B,MAAgC;CACjF,IAAI,QAAQ,QACT,QAAQ,MAAM,gBAAgB,IAAI,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,CAC9D,IAAI,YAAY;CACnB,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,MAAM,EAAE,UAAU,KAAK,KAAK;CAClE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,cAAc,OAAO,KAAK,KAAK;CAEzC,OAAO;AACT;;;;AAKA,SAAS,cAAc,OAAoB,OAA4B;CACrE,MAAM,yBAAS,IAAI,IAA2B;CAC9C,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,cAAc,EAAE,EAAE;EAC5B,MAAM,IAAI,OAAO,IAAI,CAAC;EACtB,IAAI,GAAG,EAAE,KAAK,CAAC;OACV,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC;CACxB;CACA,MAAM,QAAQ,CAAC,GAAG,OAAO,OAAO,CAAC;CACjC,MAAM,MAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,OAAO,KAAK;EACvC,MAAM,SAAS,IAAI;EACnB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,IAAI,KAAK,QAAQ,IAAI,KAAK,KAAK,EAAE;GACrC,IAAI,IAAI,UAAU,OAAO;EAC3B;EACA,IAAI,IAAI,WAAW,QAAQ;CAC7B;CACA,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CAEnE,OAAO,cADS,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAC/B,GAAG,IAAI;AACpC;AAEA,eAAe,eAA2C;CACxD,MAAM,MAAM,MAAM,MAAM,WAAW;CACnC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,8BAA8B,IAAI,OAAO,YAAY,aAAa;CAC/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,IAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAC1C,MAAM,IAAI,MAAM,gDAAgD,aAAa;CAE/E,OAAO;AACT;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,eAAe,QAAQ,IAAI,2BAA2B;CAE5D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,QAAQ,KACN,oNAGF;GACA,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,gCAAgC,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC/F;IACF,CAAC;IACD;GACF;GACA,MAAM,MAAM,MAAM,MAAM,aAAa,EAAE,QAAQ,OAAO,CAAC,CAAC,CAAC,OAAO,QAAQ;IACtE,MAAM,IAAI,MACR,2CAA2C,YAAY,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,0IAGtG;GACF,CAAC;GACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MACR,yCAAyC,IAAI,OAAO,OAAO,YAAY,8EAEzE;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,aAAa;GAC/B,SAAS,KAAK;IACZ,QAAQ,KACN,8CAA8C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAC5C,UAC7C;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,cAAc,SAAS,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAMlC,OADa,SAAS,IACZ,CAAC,CAAC;EACd;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,WAAW,SAAS,KAAK;GAC/B,IAAI,SAAS,WAAW,GAEtB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,UAAU,KAAK;KAAU,QAAQ,KAAK;KAAQ,QAAQ;IAAe,CAAC;GACjG;GAEF,OAAO,eAAe,MAAM,UAAU,YAAY,CAAC;EACrD;CACF;AACF"}

package/dist/benchmarks/frames.js

@@ -1,3 +1,4 @@
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
@@ -245,26 +246,23 @@ function judgeRouter() {
 }
 /** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */
 async function tier2Judge(question, gold, candidate, router) {
-	const res = await fetch(`${router.baseUrl}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${router.key}`
-		},
-		body: JSON.stringify({
-			model: router.model,
-			temperature: 0,
-			seed: 0,
-			messages: [{
-				role: "user",
-				content: JUDGE_PROMPT(question, gold, candidate)
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-	const body = await res.json();
-	const content = body.choices?.[0]?.message?.content;
-	if (typeof content !== "string") throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const content = (await runBenchRouterTurn({
+		routerBaseUrl: router.baseUrl,
+		routerKey: router.key,
+		profile: {
+			name: "frames-equivalence-judge",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: router.model,
+				metadata: {
+					temperature: 0,
+					seed: 0
+				}
+			}
+		}
+	}, JUDGE_PROMPT(question, gold, candidate))).finalText;
+	if (!content) throw new Error("FRAMES Tier-2 judge returned no message content");
 	const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
 	const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
 	let parsed;

package/dist/benchmarks/frames.js.map

@@ -1 +1 @@
-{"version":3,"file":"frames.js","names":[],"sources":["../../src/benchmarks/frames.ts"],"sourcesContent":["/**\n * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text\n * final answer string (optionally preceded by a citations block). FRAMES ships,\n * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs\n * needed to answer) — the gold-citation column is what lets the loop's critic\n * check citation coverage deterministically.\n *\n * Judge is two-tier, deterministic-first:\n *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).\n *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,\n *            temperature 0), fired ONLY when Tier 1 misses, to absorb the\n *            paraphrase/alias equivalence FRAMES intends to allow.\n *\n * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge\n * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced\n * in `detail` for the critic, never a hard pass criterion.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed and a\n * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to\n * load the committed fixtures (bench/fixtures/frames.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'frames.json')\n\nconst DATASET = 'google/frames-benchmark'\n/** Pin the dataset revision: ids are derived from row order (no native id column),\n *  so a reorder would break opts.ids selection + scorecard joins. */\nconst DATASET_REVISION = 'main'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Tier-1 parsing keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\n/**\n * Typed seam for the future dynamic-topology research worker. The benchmark\n * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the\n * loop worker decodes its agent runs into a {@link ResearchAnswer} and serializes\n * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — judge resolves strictly against this. */\n  gold: string\n  /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */\n  goldSources: string[]\n  /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */\n  reasoningTypes: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\ninterface FramesRow {\n  Prompt: string\n  Answer: string\n  wiki_links: string\n  reasoning_types: string\n}\n\ninterface FramesMeta {\n  gold: string\n  goldSources: string[]\n  reasoningTypes: string\n  rawPrompt: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\nconst UNIT_WORDS = new Set([\n  'years',\n  'year',\n  'months',\n  'month',\n  'days',\n  'day',\n  'people',\n  'percent',\n  'dollars',\n  'meters',\n  'metres',\n  'kilometers',\n  'kilometres',\n  'miles',\n  'km',\n  'm',\n])\n\nconst WRITTEN_NUMBERS: Record<string, number> = {\n  zero: 0,\n  one: 1,\n  two: 2,\n  three: 3,\n  four: 4,\n  five: 5,\n  six: 6,\n  seven: 7,\n  eight: 8,\n  nine: 9,\n  ten: 10,\n  eleven: 11,\n  twelve: 12,\n  thirteen: 13,\n  fourteen: 14,\n  fifteen: 15,\n  sixteen: 16,\n  seventeen: 17,\n  eighteen: 18,\n  nineteen: 19,\n  twenty: 20,\n  thirty: 30,\n  forty: 40,\n  fifty: 50,\n  sixty: 60,\n  seventy: 70,\n  eighty: 80,\n  ninety: 90,\n  hundred: 100,\n  thousand: 1000,\n  million: 1_000_000,\n  billion: 1_000_000_000,\n}\n\nconst MONTHS: Record<string, string> = {\n  january: '01',\n  february: '02',\n  march: '03',\n  april: '04',\n  may: '05',\n  june: '06',\n  july: '07',\n  august: '08',\n  september: '09',\n  october: '10',\n  november: '11',\n  december: '12',\n}\n\nconst SCALES = new Set(['hundred', 'thousand', 'million', 'billion'])\n\n/** Fold a run of written-number words into one integer (\"twenty eight\"→28, \"two hundred\"→200). */\nfunction composeWrittenRun(words: string[]): number {\n  let total = 0\n  let current = 0\n  for (const w of words) {\n    const v = WRITTEN_NUMBERS[w]\n    if (v === undefined) continue\n    if (w === 'hundred') {\n      current = (current === 0 ? 1 : current) * 100\n    } else if (SCALES.has(w)) {\n      current = (current === 0 ? 1 : current) * v\n      total += current\n      current = 0\n    } else {\n      current += v\n    }\n  }\n  return total + current\n}\n\n/** Canonicalize \"1,234\" / \"1.2 million\" / written numerals to a single numeric token. */\nfunction canonicalizeNumbers(s: string): string {\n  // \"1,234,567\" → \"1234567\"\n  let out = s.replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n  // \"1.2 million\" / \"3 billion\" → expanded integer\n  out = out.replace(/\\b(\\d+(?:\\.\\d+)?)\\s+(hundred|thousand|million|billion)\\b/g, (_m, num: string, scaleWord: string) => {\n    const scale = WRITTEN_NUMBERS[scaleWord] ?? 1\n    return String(Number(num) * scale)\n  })\n  // fold contiguous runs of written-number words into one integer\n  out = out.replace(/\\b[a-z]+(?:[\\s-]+[a-z]+)*\\b/g, (run) => {\n    const words = run.split(/[\\s-]+/)\n    if (!words.every((w) => w in WRITTEN_NUMBERS)) return run\n    return String(composeWrittenRun(words))\n  })\n  return out\n}\n\n/** \"March 3, 1879\" / \"3 March 1879\" → \"1879-03-03\"; year-only stays as the year. */\nfunction canonicalizeDates(s: string): string {\n  let out = s.replace(\n    /\\b(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{1,2}),?\\s+(\\d{4})\\b/g,\n    (_m, mon: string, day: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  out = out.replace(\n    /\\b(\\d{1,2})\\s+(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{4})\\b/g,\n    (_m, day: string, mon: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  return out\n}\n\n/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */\nexport function normalizeAnswer(input: string): string {\n  let s = input.toLowerCase()\n  s = canonicalizeDates(s)\n  s = canonicalizeNumbers(s)\n  // strip punctuation (keep alphanumerics, ISO-date hyphens collapse to space-free below)\n  s = s.replace(/[^\\w\\s-]/g, ' ')\n  // tokenize, drop articles + trailing unit words, keep order\n  const tokens = s\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n    .filter((t) => !UNIT_WORDS.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Tier-1 match: normalized equality OR normalized gold as a token-boundary substring. */\nfunction tier1Match(candidate: string, gold: string): boolean {\n  const nc = normalizeAnswer(candidate)\n  const ng = normalizeAnswer(gold)\n  if (ng.length === 0) return false\n  if (nc === ng) return true\n  // token-boundary containment: gold appears as a whole-token run inside candidate\n  const re = new RegExp(`(^|\\\\s)${ng.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}(\\\\s|$)`)\n  return re.test(nc)\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/** Fraction of gold Wikipedia article slugs touched by the candidate's citations. SOFT signal. */\nfunction citationCoverage(citations: string[], goldSources: string[]): number {\n  if (goldSources.length === 0) return 1\n  const slug = (u: string) => {\n    const m = u.match(/\\/wiki\\/([^#?]+)/)\n    return m ? decodeURIComponent(m[1] ?? '').toLowerCase() : u.toLowerCase()\n  }\n  const got = new Set(citations.map(slug))\n  let hit = 0\n  for (const g of goldSources) if (got.has(slug(g))) hit += 1\n  return hit / goldSources.length\n}\n\nconst JUDGE_PROMPT = (question: string, gold: string, candidate: string): string =>\n  [\n    'You are a strict answer-equivalence checker for a factual question-answering benchmark.',\n    'Decide ONLY whether the candidate answer is semantically equivalent to the gold answer for this question.',\n    'It is CORRECT iff it contains the same factual value as the gold answer. Extra correct detail is fine.',\n    'A different value, a missing value, or a wrong value is INCORRECT.',\n    '',\n    `Question: ${question}`,\n    `Gold answer: ${gold}`,\n    `Candidate answer: ${candidate}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"verdict\": \"correct\" | \"incorrect\"}',\n    '```',\n  ].join('\\n')\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const model = process.env.JUDGE_MODEL\n  if (!model) throw new Error('JUDGE_MODEL is required for the FRAMES Tier-2 equivalence judge')\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FRAMES Tier-2 judge')\n  const baseUrl = process.env.JUDGE_ROUTER_BASE ?? process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function tier2Judge(\n  question: string,\n  gold: string,\n  candidate: string,\n  router: JudgeRouter,\n): Promise<boolean> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      seed: 0,\n      messages: [{ role: 'user', content: JUDGE_PROMPT(question, gold, candidate) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { verdict?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { verdict?: unknown }\n  } catch {\n    throw new Error(`FRAMES Tier-2 judge produced unparseable output (no JSON verdict): ${content.slice(0, 300)}`)\n  }\n  if (parsed.verdict === 'correct') return true\n  if (parsed.verdict === 'incorrect') return false\n  throw new Error(`FRAMES Tier-2 judge verdict not in {correct,incorrect}: ${JSON.stringify(parsed).slice(0, 200)}`)\n}\n\n/** FRAMES wiki_links is a python-repr list string, e.g. \"['https://…', 'https://…']\". */\nfunction parseWikiLinks(raw: string): string[] {\n  if (!raw) return []\n  const out: string[] = []\n  for (const m of raw.matchAll(/https?:\\/\\/[^\\s'\"\\]]+/g)) out.push(m[0])\n  return out\n}\n\nfunction rowToTask(row: FramesRow, index: number): BenchTask {\n  const goldSources = parseWikiLinks(row.wiki_links)\n  const meta: FramesMeta = {\n    gold: row.Answer,\n    goldSources,\n    reasoningTypes: row.reasoning_types ?? '',\n    rawPrompt: row.Prompt,\n  }\n  return {\n    id: `frames-${index}`,\n    split: 'test',\n    prompt: row.Prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FramesMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`FRAMES task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as FramesMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as FramesRow[]\n  console.log(`[frames] FRAMES_FIXTURES=1 — loading ${rows.length} committed fixtures (no HF download)`)\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createFramesAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FRAMES_FIXTURES === '1'\n\n  return {\n    name: 'frames',\n\n    async preflight() {\n      // Tier-2 judge router config must be present in both modes — the loop's\n      // citation-coverage stop gate is meaningless without the equivalence judge.\n      judgeRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`FRAMES_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `frames preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} ; ` +\n            `or set FRAMES_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor i, r in enumerate(ds):\n    rid = f\"frames-{i}\"\n    if ids is not None and rid not in ids:\n        continue\n    out.append({\n        \"_index\": i,\n        \"Prompt\": r.get(\"Prompt\", \"\"),\n        \"Answer\": r.get(\"Answer\", \"\"),\n        \"wiki_links\": str(r.get(\"wiki_links\", \"\")),\n        \"reasoning_types\": str(r.get(\"reasoning_types\", \"\")),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<FramesRow & { _index: number }>\n      return rows.map((r) => rowToTask(r, r._index))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold Answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes (Tier-1 short-circuits with no model tokens).\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n      const coverage = citationCoverage(citations, meta.goldSources)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            tier: 'none',\n            reason: 'no parseable answer',\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 1 — free, deterministic.\n      if (tier1Match(finalAnswer, meta.gold)) {\n        return {\n          resolved: true,\n          score: 1,\n          detail: JSON.stringify({\n            tier: 1,\n            normalizedAnswer: normalizeAnswer(finalAnswer),\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 2 — constrained LLM equivalence gate (fail loud on unparseable output).\n      const verdict = await tier2Judge(meta.rawPrompt, meta.gold, finalAnswer, judgeRouter())\n      return {\n        resolved: verdict,\n        score: verdict ? 1 : 0,\n        detail: JSON.stringify({\n          tier: 2,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          judgeVerdict: verdict ? 'correct' : 'incorrect',\n          citationCoverage: coverage,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,aAAa;AAE3D,MAAM,UAAU;;;AAGhB,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;AACF,CAAC,CAAC,KAAK,IAAI;;AAuCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;AAEA,MAAM,2BAAW,IAAI,IAAI;CAAC;CAAK;CAAM;AAAK,CAAC;AAC3C,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,kBAA0C;CAC9C,MAAM;CACN,KAAK;CACL,KAAK;CACL,OAAO;CACP,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,OAAO;CACP,MAAM;CACN,KAAK;CACL,QAAQ;CACR,QAAQ;CACR,UAAU;CACV,UAAU;CACV,SAAS;CACT,SAAS;CACT,WAAW;CACX,UAAU;CACV,UAAU;CACV,QAAQ;CACR,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,SAAS;CACT,UAAU;CACV,SAAS;CACT,SAAS;AACX;AAEA,MAAM,SAAiC;CACrC,SAAS;CACT,UAAU;CACV,OAAO;CACP,OAAO;CACP,KAAK;CACL,MAAM;CACN,MAAM;CACN,QAAQ;CACR,WAAW;CACX,SAAS;CACT,UAAU;CACV,UAAU;AACZ;AAEA,MAAM,yBAAS,IAAI,IAAI;CAAC;CAAW;CAAY;CAAW;AAAS,CAAC;;AAGpE,SAAS,kBAAkB,OAAyB;CAClD,IAAI,QAAQ;CACZ,IAAI,UAAU;CACd,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,gBAAgB;EAC1B,IAAI,MAAM,KAAA,GAAW;EACrB,IAAI,MAAM,WACR,WAAW,YAAY,IAAI,IAAI,WAAW;OACrC,IAAI,OAAO,IAAI,CAAC,GAAG;GACxB,WAAW,YAAY,IAAI,IAAI,WAAW;GAC1C,SAAS;GACT,UAAU;EACZ,OACE,WAAW;CAEf;CACA,OAAO,QAAQ;AACjB;;AAGA,SAAS,oBAAoB,GAAmB;CAE9C,IAAI,MAAM,EAAE,QAAQ,qBAAqB,IAAI;CAE7C,MAAM,IAAI,QAAQ,8DAA8D,IAAI,KAAa,cAAsB;EACrH,MAAM,QAAQ,gBAAgB,cAAc;EAC5C,OAAO,OAAO,OAAO,GAAG,IAAI,KAAK;CACnC,CAAC;CAED,MAAM,IAAI,QAAQ,iCAAiC,QAAQ;EACzD,MAAM,QAAQ,IAAI,MAAM,QAAQ;EAChC,IAAI,CAAC,MAAM,OAAO,MAAM,KAAK,eAAe,GAAG,OAAO;EACtD,OAAO,OAAO,kBAAkB,KAAK,CAAC;CACxC,CAAC;CACD,OAAO;AACT;;AAGA,SAAS,kBAAkB,GAAmB;CAC5C,IAAI,MAAM,EAAE,QACV,yHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,MAAM,IAAI,QACR,uHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,OAAO;AACT;;AAGA,SAAgB,gBAAgB,OAAuB;CACrD,IAAI,IAAI,MAAM,YAAY;CAC1B,IAAI,kBAAkB,CAAC;CACvB,IAAI,oBAAoB,CAAC;CAEzB,IAAI,EAAE,QAAQ,aAAa,GAAG;CAO9B,OALe,EACZ,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC,CAC3B,QAAQ,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAC/B,QAAQ,MAAM,CAAC,WAAW,IAAI,CAAC,CACtB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;AAC/B;;AAGA,SAAS,WAAW,WAAmB,MAAuB;CAC5D,MAAM,KAAK,gBAAgB,SAAS;CACpC,MAAM,KAAK,gBAAgB,IAAI;CAC/B,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,IAAI,OAAO,IAAI,OAAO;CAGtB,OAAO,IADQ,OAAO,UAAU,GAAG,QAAQ,uBAAuB,MAAM,EAAE,QAClE,CAAC,CAAC,KAAK,EAAE;AACnB;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;AAGA,SAAS,iBAAiB,WAAqB,aAA+B;CAC5E,IAAI,YAAY,WAAW,GAAG,OAAO;CACrC,MAAM,QAAQ,MAAc;EAC1B,MAAM,IAAI,EAAE,MAAM,kBAAkB;EACpC,OAAO,IAAI,mBAAmB,EAAE,MAAM,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE,YAAY;CAC1E;CACA,MAAM,MAAM,IAAI,IAAI,UAAU,IAAI,IAAI,CAAC;CACvC,IAAI,MAAM;CACV,KAAK,MAAM,KAAK,aAAa,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,GAAG,OAAO;CAC1D,OAAO,MAAM,YAAY;AAC3B;AAEA,MAAM,gBAAgB,UAAkB,MAAc,cACpD;CACE;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,cAA2B;CAClC,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,iEAAiE;CAC7F,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;CAElF,OAAO;EAAE,SADO,QAAQ,IAAI,qBAAqB,QAAQ,IAAI,eAAe;EAC1D;EAAK;CAAM;AAC/B;;AAGA,eAAe,WACb,UACA,MACA,WACA,QACkB;CAClB,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,MAAM;GACN,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS,aAAa,UAAU,MAAM,SAAS;GAAE,CAAC;EAC/E,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,4BAA4B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,oDAAoD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAE1G,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,sEAAsE,QAAQ,MAAM,GAAG,GAAG,GAAG;CAC/G;CACA,IAAI,OAAO,YAAY,WAAW,OAAO;CACzC,IAAI,OAAO,YAAY,aAAa,OAAO;CAC3C,MAAM,IAAI,MAAM,2DAA2D,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;AACnH;;AAGA,SAAS,eAAe,KAAuB;CAC7C,IAAI,CAAC,KAAK,OAAO,CAAC;CAClB,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,KAAK,IAAI,SAAS,wBAAwB,GAAG,IAAI,KAAK,EAAE,EAAE;CACrE,OAAO;AACT;AAEA,SAAS,UAAU,KAAgB,OAA0B;CAC3D,MAAM,cAAc,eAAe,IAAI,UAAU;CACjD,MAAM,OAAmB;EACvB,MAAM,IAAI;EACV;EACA,gBAAgB,IAAI,mBAAmB;EACvC,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,UAAU;EACd,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA6B;CAC7C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAC5B,MAAM,IAAI,MAAM,eAAe,KAAK,GAAG,uDAAuD;CAEhG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,IAAI,wCAAwC,KAAK,OAAO,qCAAqC;CACrG,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,SAAgB,sBAAwC;CACtD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,yBAAyB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC7G,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;YAE3F;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,4BAA4B,IAAI,iIAEmB,QAAQ,2EAE7D;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,QAAQ,KAAK,SAAS;GAsB5B,MAAM,SAAS,MAAM,GAAG;;;oBAlBV,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;;;;;;;;;;;;;;qCAcnE,MAAM;;;GAIL,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAE1E,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KAAK,MAAM,UAAU,GAAG,EAAE,MAAM,CAAC;EAC/C;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAE7C,MAAM,WAAW,iBADC,eAAe,QACS,GAAG,KAAK,WAAW;GAE7D,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,QAAQ;KACR,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,IAAI,WAAW,aAAa,KAAK,IAAI,GACnC,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,MAAM,UAAU,MAAM,WAAW,KAAK,WAAW,KAAK,MAAM,aAAa,YAAY,CAAC;GACtF,OAAO;IACL,UAAU;IACV,OAAO,UAAU,IAAI;IACrB,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,cAAc,UAAU,YAAY;KACpC,kBAAkB;IACpB,CAAC;GACH;EACF;CACF;AACF"}
+{"version":3,"file":"frames.js","names":[],"sources":["../../src/benchmarks/frames.ts"],"sourcesContent":["/**\n * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text\n * final answer string (optionally preceded by a citations block). FRAMES ships,\n * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs\n * needed to answer) — the gold-citation column is what lets the loop's critic\n * check citation coverage deterministically.\n *\n * Judge is two-tier, deterministic-first:\n *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).\n *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,\n *            temperature 0), fired ONLY when Tier 1 misses, to absorb the\n *            paraphrase/alias equivalence FRAMES intends to allow.\n *\n * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge\n * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced\n * in `detail` for the critic, never a hard pass criterion.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed and a\n * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to\n * load the committed fixtures (bench/fixtures/frames.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'frames.json')\n\nconst DATASET = 'google/frames-benchmark'\n/** Pin the dataset revision: ids are derived from row order (no native id column),\n *  so a reorder would break opts.ids selection + scorecard joins. */\nconst DATASET_REVISION = 'main'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Tier-1 parsing keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\n/**\n * Typed seam for the future dynamic-topology research worker. The benchmark\n * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the\n * loop worker decodes its agent runs into a {@link ResearchAnswer} and serializes\n * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — judge resolves strictly against this. */\n  gold: string\n  /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */\n  goldSources: string[]\n  /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */\n  reasoningTypes: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\ninterface FramesRow {\n  Prompt: string\n  Answer: string\n  wiki_links: string\n  reasoning_types: string\n}\n\ninterface FramesMeta {\n  gold: string\n  goldSources: string[]\n  reasoningTypes: string\n  rawPrompt: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\nconst UNIT_WORDS = new Set([\n  'years',\n  'year',\n  'months',\n  'month',\n  'days',\n  'day',\n  'people',\n  'percent',\n  'dollars',\n  'meters',\n  'metres',\n  'kilometers',\n  'kilometres',\n  'miles',\n  'km',\n  'm',\n])\n\nconst WRITTEN_NUMBERS: Record<string, number> = {\n  zero: 0,\n  one: 1,\n  two: 2,\n  three: 3,\n  four: 4,\n  five: 5,\n  six: 6,\n  seven: 7,\n  eight: 8,\n  nine: 9,\n  ten: 10,\n  eleven: 11,\n  twelve: 12,\n  thirteen: 13,\n  fourteen: 14,\n  fifteen: 15,\n  sixteen: 16,\n  seventeen: 17,\n  eighteen: 18,\n  nineteen: 19,\n  twenty: 20,\n  thirty: 30,\n  forty: 40,\n  fifty: 50,\n  sixty: 60,\n  seventy: 70,\n  eighty: 80,\n  ninety: 90,\n  hundred: 100,\n  thousand: 1000,\n  million: 1_000_000,\n  billion: 1_000_000_000,\n}\n\nconst MONTHS: Record<string, string> = {\n  january: '01',\n  february: '02',\n  march: '03',\n  april: '04',\n  may: '05',\n  june: '06',\n  july: '07',\n  august: '08',\n  september: '09',\n  october: '10',\n  november: '11',\n  december: '12',\n}\n\nconst SCALES = new Set(['hundred', 'thousand', 'million', 'billion'])\n\n/** Fold a run of written-number words into one integer (\"twenty eight\"→28, \"two hundred\"→200). */\nfunction composeWrittenRun(words: string[]): number {\n  let total = 0\n  let current = 0\n  for (const w of words) {\n    const v = WRITTEN_NUMBERS[w]\n    if (v === undefined) continue\n    if (w === 'hundred') {\n      current = (current === 0 ? 1 : current) * 100\n    } else if (SCALES.has(w)) {\n      current = (current === 0 ? 1 : current) * v\n      total += current\n      current = 0\n    } else {\n      current += v\n    }\n  }\n  return total + current\n}\n\n/** Canonicalize \"1,234\" / \"1.2 million\" / written numerals to a single numeric token. */\nfunction canonicalizeNumbers(s: string): string {\n  // \"1,234,567\" → \"1234567\"\n  let out = s.replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n  // \"1.2 million\" / \"3 billion\" → expanded integer\n  out = out.replace(/\\b(\\d+(?:\\.\\d+)?)\\s+(hundred|thousand|million|billion)\\b/g, (_m, num: string, scaleWord: string) => {\n    const scale = WRITTEN_NUMBERS[scaleWord] ?? 1\n    return String(Number(num) * scale)\n  })\n  // fold contiguous runs of written-number words into one integer\n  out = out.replace(/\\b[a-z]+(?:[\\s-]+[a-z]+)*\\b/g, (run) => {\n    const words = run.split(/[\\s-]+/)\n    if (!words.every((w) => w in WRITTEN_NUMBERS)) return run\n    return String(composeWrittenRun(words))\n  })\n  return out\n}\n\n/** \"March 3, 1879\" / \"3 March 1879\" → \"1879-03-03\"; year-only stays as the year. */\nfunction canonicalizeDates(s: string): string {\n  let out = s.replace(\n    /\\b(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{1,2}),?\\s+(\\d{4})\\b/g,\n    (_m, mon: string, day: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  out = out.replace(\n    /\\b(\\d{1,2})\\s+(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{4})\\b/g,\n    (_m, day: string, mon: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  return out\n}\n\n/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */\nexport function normalizeAnswer(input: string): string {\n  let s = input.toLowerCase()\n  s = canonicalizeDates(s)\n  s = canonicalizeNumbers(s)\n  // strip punctuation (keep alphanumerics, ISO-date hyphens collapse to space-free below)\n  s = s.replace(/[^\\w\\s-]/g, ' ')\n  // tokenize, drop articles + trailing unit words, keep order\n  const tokens = s\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n    .filter((t) => !UNIT_WORDS.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Tier-1 match: normalized equality OR normalized gold as a token-boundary substring. */\nfunction tier1Match(candidate: string, gold: string): boolean {\n  const nc = normalizeAnswer(candidate)\n  const ng = normalizeAnswer(gold)\n  if (ng.length === 0) return false\n  if (nc === ng) return true\n  // token-boundary containment: gold appears as a whole-token run inside candidate\n  const re = new RegExp(`(^|\\\\s)${ng.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}(\\\\s|$)`)\n  return re.test(nc)\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/** Fraction of gold Wikipedia article slugs touched by the candidate's citations. SOFT signal. */\nfunction citationCoverage(citations: string[], goldSources: string[]): number {\n  if (goldSources.length === 0) return 1\n  const slug = (u: string) => {\n    const m = u.match(/\\/wiki\\/([^#?]+)/)\n    return m ? decodeURIComponent(m[1] ?? '').toLowerCase() : u.toLowerCase()\n  }\n  const got = new Set(citations.map(slug))\n  let hit = 0\n  for (const g of goldSources) if (got.has(slug(g))) hit += 1\n  return hit / goldSources.length\n}\n\nconst JUDGE_PROMPT = (question: string, gold: string, candidate: string): string =>\n  [\n    'You are a strict answer-equivalence checker for a factual question-answering benchmark.',\n    'Decide ONLY whether the candidate answer is semantically equivalent to the gold answer for this question.',\n    'It is CORRECT iff it contains the same factual value as the gold answer. Extra correct detail is fine.',\n    'A different value, a missing value, or a wrong value is INCORRECT.',\n    '',\n    `Question: ${question}`,\n    `Gold answer: ${gold}`,\n    `Candidate answer: ${candidate}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"verdict\": \"correct\" | \"incorrect\"}',\n    '```',\n  ].join('\\n')\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const model = process.env.JUDGE_MODEL\n  if (!model) throw new Error('JUDGE_MODEL is required for the FRAMES Tier-2 equivalence judge')\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FRAMES Tier-2 judge')\n  const baseUrl = process.env.JUDGE_ROUTER_BASE ?? process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function tier2Judge(\n  question: string,\n  gold: string,\n  candidate: string,\n  router: JudgeRouter,\n): Promise<boolean> {\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'frames-equivalence-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0, seed: 0 },\n        },\n      },\n    },\n    JUDGE_PROMPT(question, gold, candidate),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('FRAMES Tier-2 judge returned no message content')\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { verdict?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { verdict?: unknown }\n  } catch {\n    throw new Error(`FRAMES Tier-2 judge produced unparseable output (no JSON verdict): ${content.slice(0, 300)}`)\n  }\n  if (parsed.verdict === 'correct') return true\n  if (parsed.verdict === 'incorrect') return false\n  throw new Error(`FRAMES Tier-2 judge verdict not in {correct,incorrect}: ${JSON.stringify(parsed).slice(0, 200)}`)\n}\n\n/** FRAMES wiki_links is a python-repr list string, e.g. \"['https://…', 'https://…']\". */\nfunction parseWikiLinks(raw: string): string[] {\n  if (!raw) return []\n  const out: string[] = []\n  for (const m of raw.matchAll(/https?:\\/\\/[^\\s'\"\\]]+/g)) out.push(m[0])\n  return out\n}\n\nfunction rowToTask(row: FramesRow, index: number): BenchTask {\n  const goldSources = parseWikiLinks(row.wiki_links)\n  const meta: FramesMeta = {\n    gold: row.Answer,\n    goldSources,\n    reasoningTypes: row.reasoning_types ?? '',\n    rawPrompt: row.Prompt,\n  }\n  return {\n    id: `frames-${index}`,\n    split: 'test',\n    prompt: row.Prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FramesMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`FRAMES task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as FramesMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as FramesRow[]\n  console.log(`[frames] FRAMES_FIXTURES=1 — loading ${rows.length} committed fixtures (no HF download)`)\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createFramesAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FRAMES_FIXTURES === '1'\n\n  return {\n    name: 'frames',\n\n    async preflight() {\n      // Tier-2 judge router config must be present in both modes — the loop's\n      // citation-coverage stop gate is meaningless without the equivalence judge.\n      judgeRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`FRAMES_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `frames preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} ; ` +\n            `or set FRAMES_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor i, r in enumerate(ds):\n    rid = f\"frames-{i}\"\n    if ids is not None and rid not in ids:\n        continue\n    out.append({\n        \"_index\": i,\n        \"Prompt\": r.get(\"Prompt\", \"\"),\n        \"Answer\": r.get(\"Answer\", \"\"),\n        \"wiki_links\": str(r.get(\"wiki_links\", \"\")),\n        \"reasoning_types\": str(r.get(\"reasoning_types\", \"\")),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<FramesRow & { _index: number }>\n      return rows.map((r) => rowToTask(r, r._index))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold Answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes (Tier-1 short-circuits with no model tokens).\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n      const coverage = citationCoverage(citations, meta.goldSources)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            tier: 'none',\n            reason: 'no parseable answer',\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 1 — free, deterministic.\n      if (tier1Match(finalAnswer, meta.gold)) {\n        return {\n          resolved: true,\n          score: 1,\n          detail: JSON.stringify({\n            tier: 1,\n            normalizedAnswer: normalizeAnswer(finalAnswer),\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 2 — constrained LLM equivalence gate (fail loud on unparseable output).\n      const verdict = await tier2Judge(meta.rawPrompt, meta.gold, finalAnswer, judgeRouter())\n      return {\n        resolved: verdict,\n        score: verdict ? 1 : 0,\n        detail: JSON.stringify({\n          tier: 2,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          judgeVerdict: verdict ? 'correct' : 'incorrect',\n          citationCoverage: coverage,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,aAAa;AAE3D,MAAM,UAAU;;;AAGhB,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;AACF,CAAC,CAAC,KAAK,IAAI;;AAuCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;AAEA,MAAM,2BAAW,IAAI,IAAI;CAAC;CAAK;CAAM;AAAK,CAAC;AAC3C,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,kBAA0C;CAC9C,MAAM;CACN,KAAK;CACL,KAAK;CACL,OAAO;CACP,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,OAAO;CACP,MAAM;CACN,KAAK;CACL,QAAQ;CACR,QAAQ;CACR,UAAU;CACV,UAAU;CACV,SAAS;CACT,SAAS;CACT,WAAW;CACX,UAAU;CACV,UAAU;CACV,QAAQ;CACR,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,SAAS;CACT,UAAU;CACV,SAAS;CACT,SAAS;AACX;AAEA,MAAM,SAAiC;CACrC,SAAS;CACT,UAAU;CACV,OAAO;CACP,OAAO;CACP,KAAK;CACL,MAAM;CACN,MAAM;CACN,QAAQ;CACR,WAAW;CACX,SAAS;CACT,UAAU;CACV,UAAU;AACZ;AAEA,MAAM,yBAAS,IAAI,IAAI;CAAC;CAAW;CAAY;CAAW;AAAS,CAAC;;AAGpE,SAAS,kBAAkB,OAAyB;CAClD,IAAI,QAAQ;CACZ,IAAI,UAAU;CACd,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,gBAAgB;EAC1B,IAAI,MAAM,KAAA,GAAW;EACrB,IAAI,MAAM,WACR,WAAW,YAAY,IAAI,IAAI,WAAW;OACrC,IAAI,OAAO,IAAI,CAAC,GAAG;GACxB,WAAW,YAAY,IAAI,IAAI,WAAW;GAC1C,SAAS;GACT,UAAU;EACZ,OACE,WAAW;CAEf;CACA,OAAO,QAAQ;AACjB;;AAGA,SAAS,oBAAoB,GAAmB;CAE9C,IAAI,MAAM,EAAE,QAAQ,qBAAqB,IAAI;CAE7C,MAAM,IAAI,QAAQ,8DAA8D,IAAI,KAAa,cAAsB;EACrH,MAAM,QAAQ,gBAAgB,cAAc;EAC5C,OAAO,OAAO,OAAO,GAAG,IAAI,KAAK;CACnC,CAAC;CAED,MAAM,IAAI,QAAQ,iCAAiC,QAAQ;EACzD,MAAM,QAAQ,IAAI,MAAM,QAAQ;EAChC,IAAI,CAAC,MAAM,OAAO,MAAM,KAAK,eAAe,GAAG,OAAO;EACtD,OAAO,OAAO,kBAAkB,KAAK,CAAC;CACxC,CAAC;CACD,OAAO;AACT;;AAGA,SAAS,kBAAkB,GAAmB;CAC5C,IAAI,MAAM,EAAE,QACV,yHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,MAAM,IAAI,QACR,uHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,OAAO;AACT;;AAGA,SAAgB,gBAAgB,OAAuB;CACrD,IAAI,IAAI,MAAM,YAAY;CAC1B,IAAI,kBAAkB,CAAC;CACvB,IAAI,oBAAoB,CAAC;CAEzB,IAAI,EAAE,QAAQ,aAAa,GAAG;CAO9B,OALe,EACZ,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC,CAC3B,QAAQ,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAC/B,QAAQ,MAAM,CAAC,WAAW,IAAI,CAAC,CACtB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;AAC/B;;AAGA,SAAS,WAAW,WAAmB,MAAuB;CAC5D,MAAM,KAAK,gBAAgB,SAAS;CACpC,MAAM,KAAK,gBAAgB,IAAI;CAC/B,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,IAAI,OAAO,IAAI,OAAO;CAGtB,OAAO,IADQ,OAAO,UAAU,GAAG,QAAQ,uBAAuB,MAAM,EAAE,QAClE,CAAC,CAAC,KAAK,EAAE;AACnB;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;AAGA,SAAS,iBAAiB,WAAqB,aAA+B;CAC5E,IAAI,YAAY,WAAW,GAAG,OAAO;CACrC,MAAM,QAAQ,MAAc;EAC1B,MAAM,IAAI,EAAE,MAAM,kBAAkB;EACpC,OAAO,IAAI,mBAAmB,EAAE,MAAM,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE,YAAY;CAC1E;CACA,MAAM,MAAM,IAAI,IAAI,UAAU,IAAI,IAAI,CAAC;CACvC,IAAI,MAAM;CACV,KAAK,MAAM,KAAK,aAAa,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,GAAG,OAAO;CAC1D,OAAO,MAAM,YAAY;AAC3B;AAEA,MAAM,gBAAgB,UAAkB,MAAc,cACpD;CACE;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,cAA2B;CAClC,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,iEAAiE;CAC7F,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;CAElF,OAAO;EAAE,SADO,QAAQ,IAAI,qBAAqB,QAAQ,IAAI,eAAe;EAC1D;EAAK;CAAM;AAC/B;;AAGA,eAAe,WACb,UACA,MACA,WACA,QACkB;CAiBlB,MAAM,WAAU,MAhBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU;KAAE,aAAa;KAAG,MAAM;IAAE;GACtC;EACF;CACF,GACA,aAAa,UAAU,MAAM,SAAS,CACxC,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,iDAAiD;CAC/E,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,sEAAsE,QAAQ,MAAM,GAAG,GAAG,GAAG;CAC/G;CACA,IAAI,OAAO,YAAY,WAAW,OAAO;CACzC,IAAI,OAAO,YAAY,aAAa,OAAO;CAC3C,MAAM,IAAI,MAAM,2DAA2D,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;AACnH;;AAGA,SAAS,eAAe,KAAuB;CAC7C,IAAI,CAAC,KAAK,OAAO,CAAC;CAClB,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,KAAK,IAAI,SAAS,wBAAwB,GAAG,IAAI,KAAK,EAAE,EAAE;CACrE,OAAO;AACT;AAEA,SAAS,UAAU,KAAgB,OAA0B;CAC3D,MAAM,cAAc,eAAe,IAAI,UAAU;CACjD,MAAM,OAAmB;EACvB,MAAM,IAAI;EACV;EACA,gBAAgB,IAAI,mBAAmB;EACvC,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,UAAU;EACd,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA6B;CAC7C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAC5B,MAAM,IAAI,MAAM,eAAe,KAAK,GAAG,uDAAuD;CAEhG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,IAAI,wCAAwC,KAAK,OAAO,qCAAqC;CACrG,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,SAAgB,sBAAwC;CACtD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,yBAAyB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC7G,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;YAE3F;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,4BAA4B,IAAI,iIAEmB,QAAQ,2EAE7D;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,QAAQ,KAAK,SAAS;GAsB5B,MAAM,SAAS,MAAM,GAAG;;;oBAlBV,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;;;;;;;;;;;;;;qCAcnE,MAAM;;;GAIL,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAE1E,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KAAK,MAAM,UAAU,GAAG,EAAE,MAAM,CAAC;EAC/C;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAE7C,MAAM,WAAW,iBADC,eAAe,QACS,GAAG,KAAK,WAAW;GAE7D,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,QAAQ;KACR,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,IAAI,WAAW,aAAa,KAAK,IAAI,GACnC,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,MAAM,UAAU,MAAM,WAAW,KAAK,WAAW,KAAK,MAAM,aAAa,YAAY,CAAC;GACtF,OAAO;IACL,UAAU;IACV,OAAO,UAAU,IAAI;IACrB,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,cAAc,UAAU,YAAY;KACpC,kBAAkB;IACpB,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/simpleqa.js

@@ -1,3 +1,4 @@
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
@@ -131,25 +132,20 @@ function graderRouter() {
 }
 /** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */
 async function gradeAnswer(question, gold, predicted, router) {
-	const res = await fetch(`${router.baseUrl}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${router.key}`
-		},
-		body: JSON.stringify({
-			model: router.model,
-			temperature: 0,
-			messages: [{
-				role: "user",
-				content: GRADER_PROMPT(question, gold, predicted)
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-	const body = await res.json();
-	const content = body.choices?.[0]?.message?.content;
-	if (typeof content !== "string") throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const content = (await runBenchRouterTurn({
+		routerBaseUrl: router.baseUrl,
+		routerKey: router.key,
+		profile: {
+			name: "simpleqa-grader",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: router.model,
+				metadata: { temperature: 0 }
+			}
+		}
+	}, GRADER_PROMPT(question, gold, predicted))).finalText;
+	if (!content) throw new Error("SimpleQA grader returned no message content");
 	const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
 	const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
 	let parsed;

package/dist/benchmarks/simpleqa.js.map

@@ -1 +1 @@
-{"version":3,"file":"simpleqa.js","names":[],"sources":["../../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,MAAM,cACJ;AACF,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;;;;;;;AASA,MAAM,iBAAiB,UAAkB,MAAc,cACrD;CACE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,eAA6B;CACpC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6EAA6E;CACvG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;CAChB,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS,cAAc,UAAU,MAAM,SAAS;GAAE,CAAC;EAChF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,wBAAwB,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE3F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,gDAAgD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAEtG,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,GAAG;CACzG;CACA,IAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBACjF,OAAO,OAAO;CAEhB,MAAM,IAAI,MACR,mEAAmE,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GACxG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;CAC7D,MAAM,OAAqB;EACzB,MAAM,IAAI;EACV,aAAa,IAAI;EACjB,OAAO,IAAI;EACX,YAAY,IAAI;EAChB,UAAU,IAAI;CAChB;CACA,OAAO;EACL,IAAI,YAAY;EAChB,OAAO;EACP,QAAQ,IAAI,UAAU;EACtB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAC/D,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,gEAAgE;CAE3G,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,OAAO,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC3C;CACA,IAAI,KAAK,UAAU,KAAA,GAAW,OAAO,MAAM,MAAM,GAAG,KAAK,KAAK;CAC9D,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,4CAA4C,KAAK,OAAO,iCAAiC;CACtG,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;;;;;;;AAQA,eAAe,SAAS,MAAyC;CAmB/D,MAAM,SAAS,MAAM,GAAG;;;eAfX,KAAK,UAAU,WAAW,EAAE;;;;;;;;;;;;;;CAeX;CAE9B,OAAO,YADM,KAAK,MAAM,MACF,CAAC,CAAC,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,aAAa;GACb,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,2BAA2B,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC1F;IACF,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,WAAW,EAAE;;;YAInC;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,8BAA8B,IAAI,uGAEA,YAAY,6EAEhD;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,OAAO,SAAS,IAAI;EACtB;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAC7C,MAAM,YAAY,eAAe,QAAQ;GAEzC,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,OAAO;KACP,aAAa;KACb,QAAQ;KACR,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;GAGF,MAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;GACrF,MAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;GAC9E,MAAM,WAAW,UAAU;GAC3B,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KACrB;KACA;KACA,MAAM,KAAK;KACX,WAAW;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;EACF;CACF;AACF"}
+{"version":3,"file":"simpleqa.js","names":[],"sources":["../../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'simpleqa-grader',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0 },\n        },\n      },\n    },\n    GRADER_PROMPT(question, gold, predicted),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('SimpleQA grader returned no message content')\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmCA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,MAAM,cACJ;AACF,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;;;;;;;AASA,MAAM,iBAAiB,UAAkB,MAAc,cACrD;CACE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,eAA6B;CACpC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6EAA6E;CACvG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;CAiBhB,MAAM,WAAU,MAhBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU,EAAE,aAAa,EAAE;GAC7B;EACF;CACF,GACA,cAAc,UAAU,MAAM,SAAS,CACzC,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,6CAA6C;CAC3E,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,GAAG;CACzG;CACA,IAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBACjF,OAAO,OAAO;CAEhB,MAAM,IAAI,MACR,mEAAmE,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GACxG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;CAC7D,MAAM,OAAqB;EACzB,MAAM,IAAI;EACV,aAAa,IAAI;EACjB,OAAO,IAAI;EACX,YAAY,IAAI;EAChB,UAAU,IAAI;CAChB;CACA,OAAO;EACL,IAAI,YAAY;EAChB,OAAO;EACP,QAAQ,IAAI,UAAU;EACtB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAC/D,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,gEAAgE;CAE3G,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,OAAO,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC3C;CACA,IAAI,KAAK,UAAU,KAAA,GAAW,OAAO,MAAM,MAAM,GAAG,KAAK,KAAK;CAC9D,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,4CAA4C,KAAK,OAAO,iCAAiC;CACtG,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;;;;;;;AAQA,eAAe,SAAS,MAAyC;CAmB/D,MAAM,SAAS,MAAM,GAAG;;;eAfX,KAAK,UAAU,WAAW,EAAE;;;;;;;;;;;;;;CAeX;CAE9B,OAAO,YADM,KAAK,MAAM,MACF,CAAC,CAAC,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,aAAa;GACb,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,2BAA2B,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC1F;IACF,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,WAAW,EAAE;;;YAInC;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,8BAA8B,IAAI,uGAEA,YAAY,6EAEhD;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,OAAO,SAAS,IAAI;EACtB;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAC7C,MAAM,YAAY,eAAe,QAAQ;GAEzC,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,OAAO;KACP,aAAa;KACb,QAAQ;KACR,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;GAGF,MAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;GACrF,MAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;GAC9E,MAAM,WAAW,UAAU;GAC3B,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KACrB;KACA;KACA,MAAM,KAAK;KACX,WAAW;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/trata-hedge.js

@@ -1,3 +1,4 @@
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { join } from "node:path";
 import { readFileSync, readdirSync, statSync } from "node:fs";
 //#region src/benchmarks/trata-hedge.ts
@@ -144,31 +145,27 @@ function parseJsonFallback(raw) {
 	return null;
 }
 async function callJudge(router, prompt, maxAttempts = 2) {
-	for (let i = 0; i < maxAttempts; i++) {
-		const res = await fetch(`${router.baseUrl}/chat/completions`, {
-			method: "POST",
-			headers: {
-				"content-type": "application/json",
-				authorization: `Bearer ${router.key}`
-			},
-			body: JSON.stringify({
-				model: router.model,
-				temperature: 0,
-				max_tokens: 16384,
-				messages: [{
-					role: "user",
-					content: prompt
-				}]
-			})
-		});
-		if (!res.ok) {
-			if (i < maxAttempts - 1) continue;
-			throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-		}
-		const content = (await res.json()).choices?.[0]?.message?.content;
-		if (typeof content !== "string") continue;
+	for (let i = 0; i < maxAttempts; i++) try {
+		const content = (await runBenchRouterTurn({
+			routerBaseUrl: router.baseUrl,
+			routerKey: router.key,
+			profile: {
+				name: "trata-hedge-judge",
+				harness: "cli-base",
+				model: {
+					provider: "tangle-router",
+					default: router.model,
+					metadata: {
+						temperature: 0,
+						maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 16384)
+					}
+				}
+			}
+		}, prompt)).finalText;
 		const parsed = parseJsonFallback(content);
 		if (parsed !== null) return parsed;
+	} catch (error) {
+		if (i + 1 === maxAttempts) throw error;
 	}
 	return null;
 }