@tangle-network/agent-bench 0.7.1 → 0.8.1

package/dist/benchmarks/finresearchbench.js.map

@@ -1 +1 @@
-{"version":3,"file":"finresearchbench.js","names":[],"sources":["../../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillTemplate(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAcA,MAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,eAAgE;CACvE,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,kEAAkE;CAC5F,OAAO;EACL,SAAS,QAAQ,IAAI,eAAe;EACpC;EACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;CAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;CACxE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,6BAA6B,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;CACrD,MAAM,UAAU,IAAI,KAAK;CACzB,IAAI,QAAQ,WAAW,GAAG,GAAG,OAAO,KAAK,MAAM,OAAO;CACtD,OAAO,QACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;CAC1F,MAAM,UAAU,QACb,QAAQ,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,CAAC,CACvE,KAAK,QAAQ,IAAI,EAAE;CACtB,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,mCAAmC,OAAO,sCAAsC,QAAQ,OAAO,GAAG,QAAQ,OAAO,WAAW,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,kHAE7J;AAEJ;AAEA,SAAS,UAAU,KAAwB,cAAkC;CAC3E,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,OAAwB;EAC5B,IAAI,IAAI;EACR,UAAU,IAAI,YAAY;EAC1B,UAAU,IAAI;EACd;EACA;EACA,WAAW,IAAI,cAAc;EAC7B,QAAQ,IAAI,UAAU;EACtB,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,SAAS,eAAe,4BAA4B;CACtD;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAChC,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,kDAAkD;CAErG,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;CACpG,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,YAAY,CAAC;CAC1D,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACxE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,GAAG;CACpG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KAAK,4DAA4D,QAAQ,OAAO,kBAAkB;CAC1G,OAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;CACtF,MAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;CACxD,sBAAsB,SAAS,IAAI;CACnC,OAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;CACrE,MAAM,WAAW,KAAK;CACtB,IAAI,CAAC,UAAU,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,+BAA+B;CAC/F,OAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,CAAC,CACvC,WAAW,cAAc,QAAQ,CAAC,CAClC,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,CAAC,CACnE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;CACzE,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,eAAe,GAAG,WAAW,KAAK,EAAE,EAAE;CACvE,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,aAAa,YAAY;EAClC,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,SAAS;EAC/B,QAAQ;GACN;EACF;EACA,MAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;EACjF,MAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;EAClF,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG;EACzB,IAAI,IAAI,GAAG,MAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,GAAG;EAChG,MAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;EAClD,IAAI,QAAQ,GAAG,MAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,GAAG;EAChH,OAAO;GAAE;GAAO,KAAK;EAAO;CAC9B;CACA,MAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,GAAG;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;CAC5F,IAAI,CAAC,KAAK,mBAAmB,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,6BAA6B;CAC3G,MAAM,SAAS,aAAa;CAC5B,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,KAAK;GAAkB,GAClD;IAAE,MAAM;IAAQ,SAAS,aAAa,MAAM,QAAQ;GAAE,CACxD;EACF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAC7G,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UAAU,MAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAC5I,MAAM,EAAE,OAAO,QAAQ,gBAAgB,OAAO;CAC9C,OAAO;EACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,EAAG;EAC5E;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;GAAU,YAAY,OAAO;GAAO;EAAI,CAAC;CAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;CAC5C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;CACzE,MAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;CACzE,MAAM,WAAW,cAAc,QAAQ;CACvC,MAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;CACnE,OAAO;EACL,UAAU,UAAU;EACpB;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;EAAS,CAAC;CAC3E;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,8BAA8B;CAE/D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,eAAe,UAAU,cAAc;IAC7C;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,4NACF;GAEF,aAAa;GACb,MAAM,eAAe,MAAM,WAAW;GACtC,MAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,sEAAsE;GACjG,OAAO,kBAAkB,MAAM,IAAI;EACrC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,mBAAmB,KAAK,mBAAmB,KAAA;EACzD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,cAAc,OAAO,aAAa,MAAM,QAAQ;GACpD,IAAI,SAAS,KAAK,CAAC,CAAC,WAAW,GAC7B,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,SAAS,KAAK;KAAS,UAAU,KAAK;KAAU,QAAQ;IAAe,CAAC;GACnG;GAEF,OAAO,iBAAiB,MAAM,QAAQ;EACxC;CACF;AACF"}
+{"version":3,"file":"finresearchbench.js","names":[],"sources":["../../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { runBenchRouterTurn } from '../router-turn'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'finresearchbench-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0 },\n        },\n        prompt: { systemPrompt: meta.judgeSystemPrompt },\n      },\n    },\n    fillTemplate(meta, response),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('FinResearchBench judge returned no message content')\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,eAAgE;CACvE,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,kEAAkE;CAC5F,OAAO;EACL,SAAS,QAAQ,IAAI,eAAe;EACpC;EACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;CAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;CACxE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,6BAA6B,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;CACrD,MAAM,UAAU,IAAI,KAAK;CACzB,IAAI,QAAQ,WAAW,GAAG,GAAG,OAAO,KAAK,MAAM,OAAO;CACtD,OAAO,QACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;CAC1F,MAAM,UAAU,QACb,QAAQ,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,CAAC,CACvE,KAAK,QAAQ,IAAI,EAAE;CACtB,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,mCAAmC,OAAO,sCAAsC,QAAQ,OAAO,GAAG,QAAQ,OAAO,WAAW,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,kHAE7J;AAEJ;AAEA,SAAS,UAAU,KAAwB,cAAkC;CAC3E,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,OAAwB;EAC5B,IAAI,IAAI;EACR,UAAU,IAAI,YAAY;EAC1B,UAAU,IAAI;EACd;EACA;EACA,WAAW,IAAI,cAAc;EAC7B,QAAQ,IAAI,UAAU;EACtB,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,SAAS,eAAe,4BAA4B;CACtD;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAChC,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,kDAAkD;CAErG,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;CACpG,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,YAAY,CAAC;CAC1D,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACxE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,GAAG;CACpG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KAAK,4DAA4D,QAAQ,OAAO,kBAAkB;CAC1G,OAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;CACtF,MAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;CACxD,sBAAsB,SAAS,IAAI;CACnC,OAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;CACrE,MAAM,WAAW,KAAK;CACtB,IAAI,CAAC,UAAU,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,+BAA+B;CAC/F,OAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,CAAC,CACvC,WAAW,cAAc,QAAQ,CAAC,CAClC,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,CAAC,CACnE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;CACzE,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,eAAe,GAAG,WAAW,KAAK,EAAE,EAAE;CACvE,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,aAAa,YAAY;EAClC,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,SAAS;EAC/B,QAAQ;GACN;EACF;EACA,MAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;EACjF,MAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;EAClF,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG;EACzB,IAAI,IAAI,GAAG,MAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,GAAG;EAChG,MAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;EAClD,IAAI,QAAQ,GAAG,MAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,GAAG;EAChH,OAAO;GAAE;GAAO,KAAK;EAAO;CAC9B;CACA,MAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,GAAG;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;CAC5F,IAAI,CAAC,KAAK,mBAAmB,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,6BAA6B;CAC3G,MAAM,SAAS,aAAa;CAkB5B,MAAM,WAAU,MAjBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU,EAAE,aAAa,EAAE;GAC7B;GACA,QAAQ,EAAE,cAAc,KAAK,kBAAkB;EACjD;CACF,GACA,aAAa,MAAM,QAAQ,CAC7B,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,oDAAoD;CAClF,MAAM,EAAE,OAAO,QAAQ,gBAAgB,OAAO;CAC9C,OAAO;EACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,EAAG;EAC5E;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;GAAU,YAAY,OAAO;GAAO;EAAI,CAAC;CAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;CAC5C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;CACzE,MAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;CACzE,MAAM,WAAW,cAAc,QAAQ;CACvC,MAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;CACnE,OAAO;EACL,UAAU,UAAU;EACpB;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;EAAS,CAAC;CAC3E;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,8BAA8B;CAE/D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,eAAe,UAAU,cAAc;IAC7C;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,4NACF;GAEF,aAAa;GACb,MAAM,eAAe,MAAM,WAAW;GACtC,MAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,sEAAsE;GACjG,OAAO,kBAAkB,MAAM,IAAI;EACrC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,mBAAmB,KAAK,mBAAmB,KAAA;EACzD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,cAAc,OAAO,aAAa,MAAM,QAAQ;GACpD,IAAI,SAAS,KAAK,CAAC,CAAC,WAAW,GAC7B,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,SAAS,KAAK;KAAS,UAAU,KAAK;KAAU,QAAQ;IAAe,CAAC;GACnG;GAEF,OAAO,iBAAiB,MAAM,QAAQ;EACxC;CACF;AACF"}

package/dist/benchmarks/finsearchcomp.js

@@ -1,3 +1,4 @@
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -138,28 +139,21 @@ function parseJudgeOutput(content) {
 }
 /** Run the record's own judge via the router. Fail loud on transport/parse errors. */
 async function runRecordJudge(meta, response, router) {
-	const res = await fetch(`${router.baseUrl}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${router.key}`
-		},
-		body: JSON.stringify({
-			model: router.model,
-			temperature: 0,
-			messages: [{
-				role: "system",
-				content: meta.judgeSystemPrompt
-			}, {
-				role: "user",
-				content: fillJudgePrompt(meta, response)
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-	const body = await res.json();
-	const content = body.choices?.[0]?.message?.content;
-	if (typeof content !== "string") throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const content = (await runBenchRouterTurn({
+		routerBaseUrl: router.baseUrl,
+		routerKey: router.key,
+		profile: {
+			name: "finsearchcomp-judge",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: router.model,
+				metadata: { temperature: 0 }
+			},
+			prompt: { systemPrompt: meta.judgeSystemPrompt }
+		}
+	}, fillJudgePrompt(meta, response))).finalText;
+	if (!content) throw new Error("FinSearchComp judge returned no message content");
 	const { resolved, score, raw } = parseJudgeOutput(content);
 	return {
 		resolved,

package/dist/benchmarks/finsearchcomp.js.map

@@ -1 +1 @@
-{"version":3,"file":"finsearchcomp.js","names":[],"sources":["../../src/benchmarks/finsearchcomp.ts"],"sourcesContent":["/**\n * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +\n * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).\n *\n * Each record is SELF-CONTAINED and carries its OWN judge: a per-record\n * `judge_system_prompt` + a `judge_prompt_template` with {prompt},\n * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),\n * and {response} (the student answer) placeholders. We faithfully REPLICATE the\n * benchmark's judge — fill the template, run it under the record's system prompt\n * via the pinned router (temperature 0), and parse the JSON the judge emits. The\n * judge returns an `answer_score` field; 1/positive = resolved, else not.\n * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).\n *\n * There is NO deterministic Tier-1 path here: the gold answer is prose with\n * region-specific units, embedded tolerances (\"允许1%的误差\"), and multi-point\n * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,\n * so the judge IS the score — fail loud on unparseable judge output (never\n * default to resolved).\n *\n * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)\n * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)\n * needs a live market snapshot / akshare ground truth to fill the judge's\n * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.\n * See {@link T1Seam}.\n *\n * Requires for a live run: network access to the GitHub-hosted dataset JSON and\n * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back\n * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit\n * console.warn — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'finsearchcomp.json')\n\nconst DATASET_URL =\n  'https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json'\n\n/** Task-type prefix on prompt_id, e.g. \"(T2)Simple_Historical_Lookup_001\". */\ntype TaskType = 'T1' | 'T2' | 'T3'\n/** Only these are self-contained (gold + own judge, no external ground truth). */\nconst SUPPORTED_TYPES: ReadonlySet<TaskType> = new Set<TaskType>(['T2', 'T3'])\n\n/**\n * Worker contract appended to every task prompt. The judge is paraphrase- and\n * keyword-aware (\"答案为\", \"the answer is\"), so we do not impose a sentinel; we\n * pass the whole artifact through as {response}. We DO ask the worker to lead\n * with its final value so a truncated artifact still carries the answer.\n */\nconst WORKER_CONTRACT = [\n  '',\n  'Research this financial question using live web/market sources and answer it.',\n  'State your final answer explicitly, with the exact units and precision the question requests.',\n  'Respect any tolerance the question implies; show the value you settled on, not just intermediate work.',\n].join('\\n')\n\n/**\n * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a\n * live market snapshot the judge consumes via a {ground_truth} template slot;\n * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a\n * resolver that fetches the akshare/market value for the record at judge time\n * and filling {ground_truth}. Until then T1 is excluded from loadTasks.\n */\nexport interface T1Seam {\n  promptId: string\n  /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */\n  resolveGroundTruth(promptId: string): Promise<string>\n}\n\ninterface FinSearchRecord {\n  prompt_id: string\n  prompt: string\n  response_reference: string\n  judge_prompt_template: string\n  judge_system_prompt: string\n  label: string\n}\n\ninterface FinSearchMeta {\n  promptId: string\n  label: string\n  taskType: TaskType\n  region: string\n  responseReference: string\n  judgePromptTemplate: string\n  judgeSystemPrompt: string\n  rawPrompt: string\n}\n\n/** \"(T2)Simple_Historical_Lookup_001\" → \"T2\"; throws on an unknown/missing prefix. */\nfunction parseTaskType(promptId: string): TaskType {\n  const m = promptId.match(/^\\((T[123])\\)/)\n  if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`)\n  return m[1] as TaskType\n}\n\n/** \"Simple_Historical_Lookup(Greater China)\" → \"Greater China\"; '' when absent. */\nfunction parseRegion(label: string): string {\n  const m = label.match(/\\(([^()]+)\\)\\s*$/)\n  return m ? m[1].trim() : ''\n}\n\nfunction recordToTask(rec: FinSearchRecord): BenchTask {\n  const taskType = parseTaskType(rec.prompt_id)\n  const meta: FinSearchMeta = {\n    promptId: rec.prompt_id,\n    label: rec.label,\n    taskType,\n    region: parseRegion(rec.label),\n    responseReference: rec.response_reference,\n    judgePromptTemplate: rec.judge_prompt_template,\n    judgeSystemPrompt: rec.judge_system_prompt,\n    rawPrompt: rec.prompt,\n  }\n  return {\n    id: rec.prompt_id,\n    split: taskType,\n    prompt: rec.prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinSearchMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.responseReference !== 'string' ||\n    typeof md.judgePromptTemplate !== 'string' ||\n    typeof md.judgeSystemPrompt !== 'string' ||\n    typeof md.rawPrompt !== 'string'\n  ) {\n    throw new Error(`FinSearchComp task ${task.id} missing judge metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinSearchMeta\n}\n\n/** Fill the per-record judge_prompt_template. {ground_truth} is T1-only and unused for T2/T3. */\nfunction fillJudgePrompt(meta: FinSearchMeta, response: string): string {\n  return meta.judgePromptTemplate\n    .replaceAll('{prompt}', meta.rawPrompt)\n    .replaceAll('{response_reference}', meta.responseReference)\n    .replaceAll('{response}', response)\n}\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/**\n * The judge emits a JSON object with an `answer_score`. In this dataset it is a\n * flat integer ({\"answer_score\": 1}); we also accept the nested-array form\n * ([[1]] → read [0][0]) the upstream scorer can produce. 1/positive = resolved.\n * A \"null\" sentinel (judge could not obtain ground truth) is NOT a pass.\n */\nfunction readAnswerScore(value: unknown): { resolved: boolean; score: number; raw: unknown } {\n  let v: unknown = value\n  // unwrap nested-array form: [[s]] / [s]\n  while (Array.isArray(v)) v = v[0]\n  if (v === null || v === undefined || v === 'null') {\n    return { resolved: false, score: 0, raw: value }\n  }\n  const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v) : NaN\n  if (!Number.isFinite(n)) {\n    throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`)\n  }\n  // Their scale: 1 = correct, 0 = wrong. Treat any positive as resolved; clamp to 0..1.\n  const resolved = n >= 1\n  const score = n <= 0 ? 0 : n >= 1 ? 1 : n\n  return { resolved, score, raw: value }\n}\n\n/** Extract the first JSON object carrying `answer_score` from judge output; throw if none. */\nfunction parseJudgeOutput(content: string): { resolved: boolean; score: number; raw: unknown } {\n  // Prefer a fenced block, then fall back to scanning for a {...} with answer_score.\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[^{}]*\"answer_score\"[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const c of candidates) {\n    let parsed: { answer_score?: unknown }\n    try {\n      parsed = JSON.parse(c) as { answer_score?: unknown }\n    } catch {\n      continue\n    }\n    if ('answer_score' in parsed) return readAnswerScore(parsed.answer_score)\n  }\n  throw new Error(\n    `FinSearchComp judge produced no parseable {\"answer_score\": …}: ${content.slice(0, 400)}`,\n  )\n}\n\n/** Run the record's own judge via the router. Fail loud on transport/parse errors. */\nasync function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise<BenchScore> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillJudgePrompt(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const { resolved, score, raw } = parseJudgeOutput(content)\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      taskType: meta.taskType,\n      region: meta.region,\n      answerScore: raw,\n      judgeModel: router.model,\n    }),\n  }\n}\n\nfunction selectRecords(records: FinSearchRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records\n    .filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id)))\n    .map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = balanceByType(tasks, opts.limit)\n  }\n  return tasks\n}\n\n/** Round-robin across task types (T2/T3) so a `limit` pulls a balanced mix\n *  rather than slicing the type-ordered dataset's first N (which yields all\n *  T2). Deterministic — preserves dataset order within each type. */\nfunction balanceByType(tasks: BenchTask[], limit: number): BenchTask[] {\n  const queues = new Map<TaskType, BenchTask[]>()\n  for (const t of tasks) {\n    const k = parseTaskType(t.id)\n    const q = queues.get(k)\n    if (q) q.push(t)\n    else queues.set(k, [t])\n  }\n  const lanes = [...queues.values()]\n  const out: BenchTask[] = []\n  for (let i = 0; out.length < limit; i++) {\n    const before = out.length\n    for (const lane of lanes) {\n      if (i < lane.length) out.push(lane[i])\n      if (out.length >= limit) break\n    }\n    if (out.length === before) break // all lanes exhausted\n  }\n  return out\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as FinSearchRecord[]\n  return selectRecords(records, opts)\n}\n\nasync function fetchDataset(): Promise<FinSearchRecord[]> {\n  const res = await fetch(DATASET_URL)\n  if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`)\n  const data = (await res.json()) as FinSearchRecord[]\n  if (!Array.isArray(data) || data.length === 0) {\n    throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`)\n  }\n  return data\n}\n\nexport function createFinsearchcompAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === '1'\n\n  return {\n    name: 'finsearchcomp',\n\n    async preflight() {\n      // The per-record LLM judge is the arbiter in both modes — its router\n      // config must exist or no score can be produced.\n      judgeRouter()\n      console.warn(\n        '[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live ' +\n          'market snapshot / akshare ground truth for the judge {ground_truth} slot. ' +\n          'loadTasks supports T2 + T3 only; wire T1Seam to add T1.',\n      )\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      const res = await fetch(DATASET_URL, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 ` +\n            `to run against the committed fixtures offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. ` +\n            `Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let records: FinSearchRecord[]\n      try {\n        records = await fetchDataset()\n      } catch (err) {\n        console.warn(\n          `[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); ` +\n            `falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRecords(records, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the record's response_reference fed back as the student\n      // answer, so verify-judge proves gold→resolved through the SAME per-record\n      // judge the real artifact takes. The reference may carry tolerance/criteria\n      // prose; the judge is built to read the value out of it.\n      const meta = readMeta(task)\n      return meta.responseReference\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const response = artifact.trim()\n      if (response.length === 0) {\n        // Fail-closed — the benchmark's own judges score an empty answer 0.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ taskType: meta.taskType, region: meta.region, reason: 'empty answer' }),\n        }\n      }\n      return runRecordJudge(meta, response, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,MAAM,WAAW,KADE,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAClC,GAAG,YAAY,oBAAoB;AAElE,MAAM,cACJ;;AAKF,MAAM,kCAAyC,IAAI,IAAc,CAAC,MAAM,IAAI,CAAC;;;;;;;AAQ7E,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AAoCX,SAAS,cAAc,UAA4B;CACjD,MAAM,IAAI,SAAS,MAAM,eAAe;CACxC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,qDAAqD,KAAK,UAAU,QAAQ,GAAG;CACvG,OAAO,EAAE;AACX;;AAGA,SAAS,YAAY,OAAuB;CAC1C,MAAM,IAAI,MAAM,MAAM,kBAAkB;CACxC,OAAO,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI;AAC3B;AAEA,SAAS,aAAa,KAAiC;CACrD,MAAM,WAAW,cAAc,IAAI,SAAS;CAC5C,MAAM,OAAsB;EAC1B,UAAU,IAAI;EACd,OAAO,IAAI;EACX;EACA,QAAQ,YAAY,IAAI,KAAK;EAC7B,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,mBAAmB,IAAI;EACvB,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,wBAAwB,YAClC,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,cAAc,UAExB,MAAM,IAAI,MAAM,sBAAsB,KAAK,GAAG,wDAAwD;CAExG,OAAO;AACT;;AAGA,SAAS,gBAAgB,MAAqB,UAA0B;CACtE,OAAO,KAAK,oBACT,WAAW,YAAY,KAAK,SAAS,CAAC,CACtC,WAAW,wBAAwB,KAAK,iBAAiB,CAAC,CAC1D,WAAW,cAAc,QAAQ;AACtC;AAQA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,uEAAuE;CACjG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;;;;;;AAQA,SAAS,gBAAgB,OAAoE;CAC3F,IAAI,IAAa;CAEjB,OAAO,MAAM,QAAQ,CAAC,GAAG,IAAI,EAAE;CAC/B,IAAI,MAAM,QAAQ,MAAM,KAAA,KAAa,MAAM,QACzC,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,KAAK;CAAM;CAEjD,MAAM,IAAI,OAAO,MAAM,WAAW,IAAI,OAAO,MAAM,WAAW,OAAO,CAAC,IAAI;CAC1E,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,MAAM,iDAAiD,KAAK,UAAU,KAAK,GAAG;CAK1F,OAAO;EAAE,UAFQ,KAAK;EAEH,OADL,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI;EACd,KAAK;CAAM;AACvC;;AAGA,SAAS,iBAAiB,SAAqE;CAE7F,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,mCAAmC,GAAG,WAAW,KAAK,EAAE,EAAE;CAC3F,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,KAAK,YAAY;EAC1B,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,CAAC;EACvB,QAAQ;GACN;EACF;EACA,IAAI,kBAAkB,QAAQ,OAAO,gBAAgB,OAAO,YAAY;CAC1E;CACA,MAAM,IAAI,MACR,kEAAkE,QAAQ,MAAM,GAAG,GAAG,GACxF;AACF;;AAGA,eAAe,eAAe,MAAqB,UAAkB,QAA0C;CAC7G,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,KAAK;GAAkB,GAClD;IAAE,MAAM;IAAQ,SAAS,gBAAgB,MAAM,QAAQ;GAAE,CAC3D;EACF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,4BAA4B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,oDAAoD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAE1G,MAAM,EAAE,UAAU,OAAO,QAAQ,iBAAiB,OAAO;CACzD,OAAO;EACL;EACA;EACA,QAAQ,KAAK,UAAU;GACrB,UAAU,KAAK;GACf,QAAQ,KAAK;GACb,aAAa;GACb,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAEA,SAAS,cAAc,SAA4B,MAAgC;CACjF,IAAI,QAAQ,QACT,QAAQ,MAAM,gBAAgB,IAAI,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,CAC9D,IAAI,YAAY;CACnB,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,MAAM,EAAE,UAAU,KAAK,KAAK;CAClE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,cAAc,OAAO,KAAK,KAAK;CAEzC,OAAO;AACT;;;;AAKA,SAAS,cAAc,OAAoB,OAA4B;CACrE,MAAM,yBAAS,IAAI,IAA2B;CAC9C,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,cAAc,EAAE,EAAE;EAC5B,MAAM,IAAI,OAAO,IAAI,CAAC;EACtB,IAAI,GAAG,EAAE,KAAK,CAAC;OACV,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC;CACxB;CACA,MAAM,QAAQ,CAAC,GAAG,OAAO,OAAO,CAAC;CACjC,MAAM,MAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,OAAO,KAAK;EACvC,MAAM,SAAS,IAAI;EACnB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,IAAI,KAAK,QAAQ,IAAI,KAAK,KAAK,EAAE;GACrC,IAAI,IAAI,UAAU,OAAO;EAC3B;EACA,IAAI,IAAI,WAAW,QAAQ;CAC7B;CACA,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CAEnE,OAAO,cADS,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAC/B,GAAG,IAAI;AACpC;AAEA,eAAe,eAA2C;CACxD,MAAM,MAAM,MAAM,MAAM,WAAW;CACnC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,8BAA8B,IAAI,OAAO,YAAY,aAAa;CAC/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,IAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAC1C,MAAM,IAAI,MAAM,gDAAgD,aAAa;CAE/E,OAAO;AACT;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,eAAe,QAAQ,IAAI,2BAA2B;CAE5D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,QAAQ,KACN,oNAGF;GACA,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,gCAAgC,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC/F;IACF,CAAC;IACD;GACF;GACA,MAAM,MAAM,MAAM,MAAM,aAAa,EAAE,QAAQ,OAAO,CAAC,CAAC,CAAC,OAAO,QAAQ;IACtE,MAAM,IAAI,MACR,2CAA2C,YAAY,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,0IAGtG;GACF,CAAC;GACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MACR,yCAAyC,IAAI,OAAO,OAAO,YAAY,8EAEzE;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,aAAa;GAC/B,SAAS,KAAK;IACZ,QAAQ,KACN,8CAA8C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAC5C,UAC7C;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,cAAc,SAAS,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAMlC,OADa,SAAS,IACZ,CAAC,CAAC;EACd;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,WAAW,SAAS,KAAK;GAC/B,IAAI,SAAS,WAAW,GAEtB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,UAAU,KAAK;KAAU,QAAQ,KAAK;KAAQ,QAAQ;IAAe,CAAC;GACjG;GAEF,OAAO,eAAe,MAAM,UAAU,YAAY,CAAC;EACrD;CACF;AACF"}
+{"version":3,"file":"finsearchcomp.js","names":[],"sources":["../../src/benchmarks/finsearchcomp.ts"],"sourcesContent":["/**\n * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +\n * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).\n *\n * Each record is SELF-CONTAINED and carries its OWN judge: a per-record\n * `judge_system_prompt` + a `judge_prompt_template` with {prompt},\n * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),\n * and {response} (the student answer) placeholders. We faithfully REPLICATE the\n * benchmark's judge — fill the template, run it under the record's system prompt\n * via the pinned router (temperature 0), and parse the JSON the judge emits. The\n * judge returns an `answer_score` field; 1/positive = resolved, else not.\n * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).\n *\n * There is NO deterministic Tier-1 path here: the gold answer is prose with\n * region-specific units, embedded tolerances (\"允许1%的误差\"), and multi-point\n * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,\n * so the judge IS the score — fail loud on unparseable judge output (never\n * default to resolved).\n *\n * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)\n * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)\n * needs a live market snapshot / akshare ground truth to fill the judge's\n * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.\n * See {@link T1Seam}.\n *\n * Requires for a live run: network access to the GitHub-hosted dataset JSON and\n * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back\n * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit\n * console.warn — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'finsearchcomp.json')\n\nconst DATASET_URL =\n  'https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json'\n\n/** Task-type prefix on prompt_id, e.g. \"(T2)Simple_Historical_Lookup_001\". */\ntype TaskType = 'T1' | 'T2' | 'T3'\n/** Only these are self-contained (gold + own judge, no external ground truth). */\nconst SUPPORTED_TYPES: ReadonlySet<TaskType> = new Set<TaskType>(['T2', 'T3'])\n\n/**\n * Worker contract appended to every task prompt. The judge is paraphrase- and\n * keyword-aware (\"答案为\", \"the answer is\"), so we do not impose a sentinel; we\n * pass the whole artifact through as {response}. We DO ask the worker to lead\n * with its final value so a truncated artifact still carries the answer.\n */\nconst WORKER_CONTRACT = [\n  '',\n  'Research this financial question using live web/market sources and answer it.',\n  'State your final answer explicitly, with the exact units and precision the question requests.',\n  'Respect any tolerance the question implies; show the value you settled on, not just intermediate work.',\n].join('\\n')\n\n/**\n * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a\n * live market snapshot the judge consumes via a {ground_truth} template slot;\n * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a\n * resolver that fetches the akshare/market value for the record at judge time\n * and filling {ground_truth}. Until then T1 is excluded from loadTasks.\n */\nexport interface T1Seam {\n  promptId: string\n  /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */\n  resolveGroundTruth(promptId: string): Promise<string>\n}\n\ninterface FinSearchRecord {\n  prompt_id: string\n  prompt: string\n  response_reference: string\n  judge_prompt_template: string\n  judge_system_prompt: string\n  label: string\n}\n\ninterface FinSearchMeta {\n  promptId: string\n  label: string\n  taskType: TaskType\n  region: string\n  responseReference: string\n  judgePromptTemplate: string\n  judgeSystemPrompt: string\n  rawPrompt: string\n}\n\n/** \"(T2)Simple_Historical_Lookup_001\" → \"T2\"; throws on an unknown/missing prefix. */\nfunction parseTaskType(promptId: string): TaskType {\n  const m = promptId.match(/^\\((T[123])\\)/)\n  if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`)\n  return m[1] as TaskType\n}\n\n/** \"Simple_Historical_Lookup(Greater China)\" → \"Greater China\"; '' when absent. */\nfunction parseRegion(label: string): string {\n  const m = label.match(/\\(([^()]+)\\)\\s*$/)\n  return m ? m[1].trim() : ''\n}\n\nfunction recordToTask(rec: FinSearchRecord): BenchTask {\n  const taskType = parseTaskType(rec.prompt_id)\n  const meta: FinSearchMeta = {\n    promptId: rec.prompt_id,\n    label: rec.label,\n    taskType,\n    region: parseRegion(rec.label),\n    responseReference: rec.response_reference,\n    judgePromptTemplate: rec.judge_prompt_template,\n    judgeSystemPrompt: rec.judge_system_prompt,\n    rawPrompt: rec.prompt,\n  }\n  return {\n    id: rec.prompt_id,\n    split: taskType,\n    prompt: rec.prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinSearchMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.responseReference !== 'string' ||\n    typeof md.judgePromptTemplate !== 'string' ||\n    typeof md.judgeSystemPrompt !== 'string' ||\n    typeof md.rawPrompt !== 'string'\n  ) {\n    throw new Error(`FinSearchComp task ${task.id} missing judge metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinSearchMeta\n}\n\n/** Fill the per-record judge_prompt_template. {ground_truth} is T1-only and unused for T2/T3. */\nfunction fillJudgePrompt(meta: FinSearchMeta, response: string): string {\n  return meta.judgePromptTemplate\n    .replaceAll('{prompt}', meta.rawPrompt)\n    .replaceAll('{response_reference}', meta.responseReference)\n    .replaceAll('{response}', response)\n}\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/**\n * The judge emits a JSON object with an `answer_score`. In this dataset it is a\n * flat integer ({\"answer_score\": 1}); we also accept the nested-array form\n * ([[1]] → read [0][0]) the upstream scorer can produce. 1/positive = resolved.\n * A \"null\" sentinel (judge could not obtain ground truth) is NOT a pass.\n */\nfunction readAnswerScore(value: unknown): { resolved: boolean; score: number; raw: unknown } {\n  let v: unknown = value\n  // unwrap nested-array form: [[s]] / [s]\n  while (Array.isArray(v)) v = v[0]\n  if (v === null || v === undefined || v === 'null') {\n    return { resolved: false, score: 0, raw: value }\n  }\n  const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v) : NaN\n  if (!Number.isFinite(n)) {\n    throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`)\n  }\n  // Their scale: 1 = correct, 0 = wrong. Treat any positive as resolved; clamp to 0..1.\n  const resolved = n >= 1\n  const score = n <= 0 ? 0 : n >= 1 ? 1 : n\n  return { resolved, score, raw: value }\n}\n\n/** Extract the first JSON object carrying `answer_score` from judge output; throw if none. */\nfunction parseJudgeOutput(content: string): { resolved: boolean; score: number; raw: unknown } {\n  // Prefer a fenced block, then fall back to scanning for a {...} with answer_score.\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[^{}]*\"answer_score\"[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const c of candidates) {\n    let parsed: { answer_score?: unknown }\n    try {\n      parsed = JSON.parse(c) as { answer_score?: unknown }\n    } catch {\n      continue\n    }\n    if ('answer_score' in parsed) return readAnswerScore(parsed.answer_score)\n  }\n  throw new Error(\n    `FinSearchComp judge produced no parseable {\"answer_score\": …}: ${content.slice(0, 400)}`,\n  )\n}\n\n/** Run the record's own judge via the router. Fail loud on transport/parse errors. */\nasync function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise<BenchScore> {\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'finsearchcomp-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0 },\n        },\n        prompt: { systemPrompt: meta.judgeSystemPrompt },\n      },\n    },\n    fillJudgePrompt(meta, response),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('FinSearchComp judge returned no message content')\n  const { resolved, score, raw } = parseJudgeOutput(content)\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      taskType: meta.taskType,\n      region: meta.region,\n      answerScore: raw,\n      judgeModel: router.model,\n    }),\n  }\n}\n\nfunction selectRecords(records: FinSearchRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records\n    .filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id)))\n    .map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = balanceByType(tasks, opts.limit)\n  }\n  return tasks\n}\n\n/** Round-robin across task types (T2/T3) so a `limit` pulls a balanced mix\n *  rather than slicing the type-ordered dataset's first N (which yields all\n *  T2). Deterministic — preserves dataset order within each type. */\nfunction balanceByType(tasks: BenchTask[], limit: number): BenchTask[] {\n  const queues = new Map<TaskType, BenchTask[]>()\n  for (const t of tasks) {\n    const k = parseTaskType(t.id)\n    const q = queues.get(k)\n    if (q) q.push(t)\n    else queues.set(k, [t])\n  }\n  const lanes = [...queues.values()]\n  const out: BenchTask[] = []\n  for (let i = 0; out.length < limit; i++) {\n    const before = out.length\n    for (const lane of lanes) {\n      if (i < lane.length) out.push(lane[i])\n      if (out.length >= limit) break\n    }\n    if (out.length === before) break // all lanes exhausted\n  }\n  return out\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as FinSearchRecord[]\n  return selectRecords(records, opts)\n}\n\nasync function fetchDataset(): Promise<FinSearchRecord[]> {\n  const res = await fetch(DATASET_URL)\n  if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`)\n  const data = (await res.json()) as FinSearchRecord[]\n  if (!Array.isArray(data) || data.length === 0) {\n    throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`)\n  }\n  return data\n}\n\nexport function createFinsearchcompAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === '1'\n\n  return {\n    name: 'finsearchcomp',\n\n    async preflight() {\n      // The per-record LLM judge is the arbiter in both modes — its router\n      // config must exist or no score can be produced.\n      judgeRouter()\n      console.warn(\n        '[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live ' +\n          'market snapshot / akshare ground truth for the judge {ground_truth} slot. ' +\n          'loadTasks supports T2 + T3 only; wire T1Seam to add T1.',\n      )\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      const res = await fetch(DATASET_URL, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 ` +\n            `to run against the committed fixtures offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. ` +\n            `Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let records: FinSearchRecord[]\n      try {\n        records = await fetchDataset()\n      } catch (err) {\n        console.warn(\n          `[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); ` +\n            `falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRecords(records, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the record's response_reference fed back as the student\n      // answer, so verify-judge proves gold→resolved through the SAME per-record\n      // judge the real artifact takes. The reference may carry tolerance/criteria\n      // prose; the judge is built to read the value out of it.\n      const meta = readMeta(task)\n      return meta.responseReference\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const response = artifact.trim()\n      if (response.length === 0) {\n        // Fail-closed — the benchmark's own judges score an empty answer 0.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ taskType: meta.taskType, region: meta.region, reason: 'empty answer' }),\n        }\n      }\n      return runRecordJudge(meta, response, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsCA,MAAM,WAAW,KADE,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAClC,GAAG,YAAY,oBAAoB;AAElE,MAAM,cACJ;;AAKF,MAAM,kCAAyC,IAAI,IAAc,CAAC,MAAM,IAAI,CAAC;;;;;;;AAQ7E,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AAoCX,SAAS,cAAc,UAA4B;CACjD,MAAM,IAAI,SAAS,MAAM,eAAe;CACxC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,qDAAqD,KAAK,UAAU,QAAQ,GAAG;CACvG,OAAO,EAAE;AACX;;AAGA,SAAS,YAAY,OAAuB;CAC1C,MAAM,IAAI,MAAM,MAAM,kBAAkB;CACxC,OAAO,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI;AAC3B;AAEA,SAAS,aAAa,KAAiC;CACrD,MAAM,WAAW,cAAc,IAAI,SAAS;CAC5C,MAAM,OAAsB;EAC1B,UAAU,IAAI;EACd,OAAO,IAAI;EACX;EACA,QAAQ,YAAY,IAAI,KAAK;EAC7B,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,mBAAmB,IAAI;EACvB,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,wBAAwB,YAClC,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,cAAc,UAExB,MAAM,IAAI,MAAM,sBAAsB,KAAK,GAAG,wDAAwD;CAExG,OAAO;AACT;;AAGA,SAAS,gBAAgB,MAAqB,UAA0B;CACtE,OAAO,KAAK,oBACT,WAAW,YAAY,KAAK,SAAS,CAAC,CACtC,WAAW,wBAAwB,KAAK,iBAAiB,CAAC,CAC1D,WAAW,cAAc,QAAQ;AACtC;AAQA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,uEAAuE;CACjG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;;;;;;AAQA,SAAS,gBAAgB,OAAoE;CAC3F,IAAI,IAAa;CAEjB,OAAO,MAAM,QAAQ,CAAC,GAAG,IAAI,EAAE;CAC/B,IAAI,MAAM,QAAQ,MAAM,KAAA,KAAa,MAAM,QACzC,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,KAAK;CAAM;CAEjD,MAAM,IAAI,OAAO,MAAM,WAAW,IAAI,OAAO,MAAM,WAAW,OAAO,CAAC,IAAI;CAC1E,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,MAAM,iDAAiD,KAAK,UAAU,KAAK,GAAG;CAK1F,OAAO;EAAE,UAFQ,KAAK;EAEH,OADL,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI;EACd,KAAK;CAAM;AACvC;;AAGA,SAAS,iBAAiB,SAAqE;CAE7F,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,mCAAmC,GAAG,WAAW,KAAK,EAAE,EAAE;CAC3F,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,KAAK,YAAY;EAC1B,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,CAAC;EACvB,QAAQ;GACN;EACF;EACA,IAAI,kBAAkB,QAAQ,OAAO,gBAAgB,OAAO,YAAY;CAC1E;CACA,MAAM,IAAI,MACR,kEAAkE,QAAQ,MAAM,GAAG,GAAG,GACxF;AACF;;AAGA,eAAe,eAAe,MAAqB,UAAkB,QAA0C;CAkB7G,MAAM,WAAU,MAjBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU,EAAE,aAAa,EAAE;GAC7B;GACA,QAAQ,EAAE,cAAc,KAAK,kBAAkB;EACjD;CACF,GACA,gBAAgB,MAAM,QAAQ,CAChC,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,iDAAiD;CAC/E,MAAM,EAAE,UAAU,OAAO,QAAQ,iBAAiB,OAAO;CACzD,OAAO;EACL;EACA;EACA,QAAQ,KAAK,UAAU;GACrB,UAAU,KAAK;GACf,QAAQ,KAAK;GACb,aAAa;GACb,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAEA,SAAS,cAAc,SAA4B,MAAgC;CACjF,IAAI,QAAQ,QACT,QAAQ,MAAM,gBAAgB,IAAI,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,CAC9D,IAAI,YAAY;CACnB,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,MAAM,EAAE,UAAU,KAAK,KAAK;CAClE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,cAAc,OAAO,KAAK,KAAK;CAEzC,OAAO;AACT;;;;AAKA,SAAS,cAAc,OAAoB,OAA4B;CACrE,MAAM,yBAAS,IAAI,IAA2B;CAC9C,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,cAAc,EAAE,EAAE;EAC5B,MAAM,IAAI,OAAO,IAAI,CAAC;EACtB,IAAI,GAAG,EAAE,KAAK,CAAC;OACV,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC;CACxB;CACA,MAAM,QAAQ,CAAC,GAAG,OAAO,OAAO,CAAC;CACjC,MAAM,MAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,OAAO,KAAK;EACvC,MAAM,SAAS,IAAI;EACnB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,IAAI,KAAK,QAAQ,IAAI,KAAK,KAAK,EAAE;GACrC,IAAI,IAAI,UAAU,OAAO;EAC3B;EACA,IAAI,IAAI,WAAW,QAAQ;CAC7B;CACA,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CAEnE,OAAO,cADS,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAC/B,GAAG,IAAI;AACpC;AAEA,eAAe,eAA2C;CACxD,MAAM,MAAM,MAAM,MAAM,WAAW;CACnC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,8BAA8B,IAAI,OAAO,YAAY,aAAa;CAC/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,IAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAC1C,MAAM,IAAI,MAAM,gDAAgD,aAAa;CAE/E,OAAO;AACT;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,eAAe,QAAQ,IAAI,2BAA2B;CAE5D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,QAAQ,KACN,oNAGF;GACA,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,gCAAgC,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC/F;IACF,CAAC;IACD;GACF;GACA,MAAM,MAAM,MAAM,MAAM,aAAa,EAAE,QAAQ,OAAO,CAAC,CAAC,CAAC,OAAO,QAAQ;IACtE,MAAM,IAAI,MACR,2CAA2C,YAAY,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,0IAGtG;GACF,CAAC;GACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MACR,yCAAyC,IAAI,OAAO,OAAO,YAAY,8EAEzE;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,aAAa;GAC/B,SAAS,KAAK;IACZ,QAAQ,KACN,8CAA8C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAC5C,UAC7C;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,cAAc,SAAS,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAMlC,OADa,SAAS,IACZ,CAAC,CAAC;EACd;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,WAAW,SAAS,KAAK;GAC/B,IAAI,SAAS,WAAW,GAEtB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,UAAU,KAAK;KAAU,QAAQ,KAAK;KAAQ,QAAQ;IAAe,CAAC;GACjG;GAEF,OAAO,eAAe,MAAM,UAAU,YAAY,CAAC;EACrD;CACF;AACF"}

package/dist/benchmarks/frames.js

@@ -1,3 +1,4 @@
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
@@ -245,26 +246,23 @@ function judgeRouter() {
 }
 /** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */
 async function tier2Judge(question, gold, candidate, router) {
-	const res = await fetch(`${router.baseUrl}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${router.key}`
-		},
-		body: JSON.stringify({
-			model: router.model,
-			temperature: 0,
-			seed: 0,
-			messages: [{
-				role: "user",
-				content: JUDGE_PROMPT(question, gold, candidate)
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-	const body = await res.json();
-	const content = body.choices?.[0]?.message?.content;
-	if (typeof content !== "string") throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const content = (await runBenchRouterTurn({
+		routerBaseUrl: router.baseUrl,
+		routerKey: router.key,
+		profile: {
+			name: "frames-equivalence-judge",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: router.model,
+				metadata: {
+					temperature: 0,
+					seed: 0
+				}
+			}
+		}
+	}, JUDGE_PROMPT(question, gold, candidate))).finalText;
+	if (!content) throw new Error("FRAMES Tier-2 judge returned no message content");
 	const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
 	const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
 	let parsed;

package/dist/benchmarks/frames.js.map

@@ -1 +1 @@
-{"version":3,"file":"frames.js","names":[],"sources":["../../src/benchmarks/frames.ts"],"sourcesContent":["/**\n * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text\n * final answer string (optionally preceded by a citations block). FRAMES ships,\n * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs\n * needed to answer) — the gold-citation column is what lets the loop's critic\n * check citation coverage deterministically.\n *\n * Judge is two-tier, deterministic-first:\n *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).\n *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,\n *            temperature 0), fired ONLY when Tier 1 misses, to absorb the\n *            paraphrase/alias equivalence FRAMES intends to allow.\n *\n * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge\n * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced\n * in `detail` for the critic, never a hard pass criterion.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed and a\n * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to\n * load the committed fixtures (bench/fixtures/frames.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'frames.json')\n\nconst DATASET = 'google/frames-benchmark'\n/** Pin the dataset revision: ids are derived from row order (no native id column),\n *  so a reorder would break opts.ids selection + scorecard joins. */\nconst DATASET_REVISION = 'main'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Tier-1 parsing keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\n/**\n * Typed seam for the future dynamic-topology research worker. The benchmark\n * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the\n * loop worker decodes its agent runs into a {@link ResearchAnswer} and serializes\n * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — judge resolves strictly against this. */\n  gold: string\n  /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */\n  goldSources: string[]\n  /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */\n  reasoningTypes: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\ninterface FramesRow {\n  Prompt: string\n  Answer: string\n  wiki_links: string\n  reasoning_types: string\n}\n\ninterface FramesMeta {\n  gold: string\n  goldSources: string[]\n  reasoningTypes: string\n  rawPrompt: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\nconst UNIT_WORDS = new Set([\n  'years',\n  'year',\n  'months',\n  'month',\n  'days',\n  'day',\n  'people',\n  'percent',\n  'dollars',\n  'meters',\n  'metres',\n  'kilometers',\n  'kilometres',\n  'miles',\n  'km',\n  'm',\n])\n\nconst WRITTEN_NUMBERS: Record<string, number> = {\n  zero: 0,\n  one: 1,\n  two: 2,\n  three: 3,\n  four: 4,\n  five: 5,\n  six: 6,\n  seven: 7,\n  eight: 8,\n  nine: 9,\n  ten: 10,\n  eleven: 11,\n  twelve: 12,\n  thirteen: 13,\n  fourteen: 14,\n  fifteen: 15,\n  sixteen: 16,\n  seventeen: 17,\n  eighteen: 18,\n  nineteen: 19,\n  twenty: 20,\n  thirty: 30,\n  forty: 40,\n  fifty: 50,\n  sixty: 60,\n  seventy: 70,\n  eighty: 80,\n  ninety: 90,\n  hundred: 100,\n  thousand: 1000,\n  million: 1_000_000,\n  billion: 1_000_000_000,\n}\n\nconst MONTHS: Record<string, string> = {\n  january: '01',\n  february: '02',\n  march: '03',\n  april: '04',\n  may: '05',\n  june: '06',\n  july: '07',\n  august: '08',\n  september: '09',\n  october: '10',\n  november: '11',\n  december: '12',\n}\n\nconst SCALES = new Set(['hundred', 'thousand', 'million', 'billion'])\n\n/** Fold a run of written-number words into one integer (\"twenty eight\"→28, \"two hundred\"→200). */\nfunction composeWrittenRun(words: string[]): number {\n  let total = 0\n  let current = 0\n  for (const w of words) {\n    const v = WRITTEN_NUMBERS[w]\n    if (v === undefined) continue\n    if (w === 'hundred') {\n      current = (current === 0 ? 1 : current) * 100\n    } else if (SCALES.has(w)) {\n      current = (current === 0 ? 1 : current) * v\n      total += current\n      current = 0\n    } else {\n      current += v\n    }\n  }\n  return total + current\n}\n\n/** Canonicalize \"1,234\" / \"1.2 million\" / written numerals to a single numeric token. */\nfunction canonicalizeNumbers(s: string): string {\n  // \"1,234,567\" → \"1234567\"\n  let out = s.replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n  // \"1.2 million\" / \"3 billion\" → expanded integer\n  out = out.replace(/\\b(\\d+(?:\\.\\d+)?)\\s+(hundred|thousand|million|billion)\\b/g, (_m, num: string, scaleWord: string) => {\n    const scale = WRITTEN_NUMBERS[scaleWord] ?? 1\n    return String(Number(num) * scale)\n  })\n  // fold contiguous runs of written-number words into one integer\n  out = out.replace(/\\b[a-z]+(?:[\\s-]+[a-z]+)*\\b/g, (run) => {\n    const words = run.split(/[\\s-]+/)\n    if (!words.every((w) => w in WRITTEN_NUMBERS)) return run\n    return String(composeWrittenRun(words))\n  })\n  return out\n}\n\n/** \"March 3, 1879\" / \"3 March 1879\" → \"1879-03-03\"; year-only stays as the year. */\nfunction canonicalizeDates(s: string): string {\n  let out = s.replace(\n    /\\b(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{1,2}),?\\s+(\\d{4})\\b/g,\n    (_m, mon: string, day: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  out = out.replace(\n    /\\b(\\d{1,2})\\s+(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{4})\\b/g,\n    (_m, day: string, mon: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  return out\n}\n\n/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */\nexport function normalizeAnswer(input: string): string {\n  let s = input.toLowerCase()\n  s = canonicalizeDates(s)\n  s = canonicalizeNumbers(s)\n  // strip punctuation (keep alphanumerics, ISO-date hyphens collapse to space-free below)\n  s = s.replace(/[^\\w\\s-]/g, ' ')\n  // tokenize, drop articles + trailing unit words, keep order\n  const tokens = s\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n    .filter((t) => !UNIT_WORDS.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Tier-1 match: normalized equality OR normalized gold as a token-boundary substring. */\nfunction tier1Match(candidate: string, gold: string): boolean {\n  const nc = normalizeAnswer(candidate)\n  const ng = normalizeAnswer(gold)\n  if (ng.length === 0) return false\n  if (nc === ng) return true\n  // token-boundary containment: gold appears as a whole-token run inside candidate\n  const re = new RegExp(`(^|\\\\s)${ng.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}(\\\\s|$)`)\n  return re.test(nc)\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/** Fraction of gold Wikipedia article slugs touched by the candidate's citations. SOFT signal. */\nfunction citationCoverage(citations: string[], goldSources: string[]): number {\n  if (goldSources.length === 0) return 1\n  const slug = (u: string) => {\n    const m = u.match(/\\/wiki\\/([^#?]+)/)\n    return m ? decodeURIComponent(m[1] ?? '').toLowerCase() : u.toLowerCase()\n  }\n  const got = new Set(citations.map(slug))\n  let hit = 0\n  for (const g of goldSources) if (got.has(slug(g))) hit += 1\n  return hit / goldSources.length\n}\n\nconst JUDGE_PROMPT = (question: string, gold: string, candidate: string): string =>\n  [\n    'You are a strict answer-equivalence checker for a factual question-answering benchmark.',\n    'Decide ONLY whether the candidate answer is semantically equivalent to the gold answer for this question.',\n    'It is CORRECT iff it contains the same factual value as the gold answer. Extra correct detail is fine.',\n    'A different value, a missing value, or a wrong value is INCORRECT.',\n    '',\n    `Question: ${question}`,\n    `Gold answer: ${gold}`,\n    `Candidate answer: ${candidate}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"verdict\": \"correct\" | \"incorrect\"}',\n    '```',\n  ].join('\\n')\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const model = process.env.JUDGE_MODEL\n  if (!model) throw new Error('JUDGE_MODEL is required for the FRAMES Tier-2 equivalence judge')\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FRAMES Tier-2 judge')\n  const baseUrl = process.env.JUDGE_ROUTER_BASE ?? process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function tier2Judge(\n  question: string,\n  gold: string,\n  candidate: string,\n  router: JudgeRouter,\n): Promise<boolean> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      seed: 0,\n      messages: [{ role: 'user', content: JUDGE_PROMPT(question, gold, candidate) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { verdict?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { verdict?: unknown }\n  } catch {\n    throw new Error(`FRAMES Tier-2 judge produced unparseable output (no JSON verdict): ${content.slice(0, 300)}`)\n  }\n  if (parsed.verdict === 'correct') return true\n  if (parsed.verdict === 'incorrect') return false\n  throw new Error(`FRAMES Tier-2 judge verdict not in {correct,incorrect}: ${JSON.stringify(parsed).slice(0, 200)}`)\n}\n\n/** FRAMES wiki_links is a python-repr list string, e.g. \"['https://…', 'https://…']\". */\nfunction parseWikiLinks(raw: string): string[] {\n  if (!raw) return []\n  const out: string[] = []\n  for (const m of raw.matchAll(/https?:\\/\\/[^\\s'\"\\]]+/g)) out.push(m[0])\n  return out\n}\n\nfunction rowToTask(row: FramesRow, index: number): BenchTask {\n  const goldSources = parseWikiLinks(row.wiki_links)\n  const meta: FramesMeta = {\n    gold: row.Answer,\n    goldSources,\n    reasoningTypes: row.reasoning_types ?? '',\n    rawPrompt: row.Prompt,\n  }\n  return {\n    id: `frames-${index}`,\n    split: 'test',\n    prompt: row.Prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FramesMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`FRAMES task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as FramesMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as FramesRow[]\n  console.log(`[frames] FRAMES_FIXTURES=1 — loading ${rows.length} committed fixtures (no HF download)`)\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createFramesAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FRAMES_FIXTURES === '1'\n\n  return {\n    name: 'frames',\n\n    async preflight() {\n      // Tier-2 judge router config must be present in both modes — the loop's\n      // citation-coverage stop gate is meaningless without the equivalence judge.\n      judgeRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`FRAMES_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `frames preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} ; ` +\n            `or set FRAMES_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor i, r in enumerate(ds):\n    rid = f\"frames-{i}\"\n    if ids is not None and rid not in ids:\n        continue\n    out.append({\n        \"_index\": i,\n        \"Prompt\": r.get(\"Prompt\", \"\"),\n        \"Answer\": r.get(\"Answer\", \"\"),\n        \"wiki_links\": str(r.get(\"wiki_links\", \"\")),\n        \"reasoning_types\": str(r.get(\"reasoning_types\", \"\")),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<FramesRow & { _index: number }>\n      return rows.map((r) => rowToTask(r, r._index))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold Answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes (Tier-1 short-circuits with no model tokens).\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n      const coverage = citationCoverage(citations, meta.goldSources)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            tier: 'none',\n            reason: 'no parseable answer',\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 1 — free, deterministic.\n      if (tier1Match(finalAnswer, meta.gold)) {\n        return {\n          resolved: true,\n          score: 1,\n          detail: JSON.stringify({\n            tier: 1,\n            normalizedAnswer: normalizeAnswer(finalAnswer),\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 2 — constrained LLM equivalence gate (fail loud on unparseable output).\n      const verdict = await tier2Judge(meta.rawPrompt, meta.gold, finalAnswer, judgeRouter())\n      return {\n        resolved: verdict,\n        score: verdict ? 1 : 0,\n        detail: JSON.stringify({\n          tier: 2,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          judgeVerdict: verdict ? 'correct' : 'incorrect',\n          citationCoverage: coverage,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,aAAa;AAE3D,MAAM,UAAU;;;AAGhB,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;AACF,CAAC,CAAC,KAAK,IAAI;;AAuCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;AAEA,MAAM,2BAAW,IAAI,IAAI;CAAC;CAAK;CAAM;AAAK,CAAC;AAC3C,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,kBAA0C;CAC9C,MAAM;CACN,KAAK;CACL,KAAK;CACL,OAAO;CACP,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,OAAO;CACP,MAAM;CACN,KAAK;CACL,QAAQ;CACR,QAAQ;CACR,UAAU;CACV,UAAU;CACV,SAAS;CACT,SAAS;CACT,WAAW;CACX,UAAU;CACV,UAAU;CACV,QAAQ;CACR,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,SAAS;CACT,UAAU;CACV,SAAS;CACT,SAAS;AACX;AAEA,MAAM,SAAiC;CACrC,SAAS;CACT,UAAU;CACV,OAAO;CACP,OAAO;CACP,KAAK;CACL,MAAM;CACN,MAAM;CACN,QAAQ;CACR,WAAW;CACX,SAAS;CACT,UAAU;CACV,UAAU;AACZ;AAEA,MAAM,yBAAS,IAAI,IAAI;CAAC;CAAW;CAAY;CAAW;AAAS,CAAC;;AAGpE,SAAS,kBAAkB,OAAyB;CAClD,IAAI,QAAQ;CACZ,IAAI,UAAU;CACd,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,gBAAgB;EAC1B,IAAI,MAAM,KAAA,GAAW;EACrB,IAAI,MAAM,WACR,WAAW,YAAY,IAAI,IAAI,WAAW;OACrC,IAAI,OAAO,IAAI,CAAC,GAAG;GACxB,WAAW,YAAY,IAAI,IAAI,WAAW;GAC1C,SAAS;GACT,UAAU;EACZ,OACE,WAAW;CAEf;CACA,OAAO,QAAQ;AACjB;;AAGA,SAAS,oBAAoB,GAAmB;CAE9C,IAAI,MAAM,EAAE,QAAQ,qBAAqB,IAAI;CAE7C,MAAM,IAAI,QAAQ,8DAA8D,IAAI,KAAa,cAAsB;EACrH,MAAM,QAAQ,gBAAgB,cAAc;EAC5C,OAAO,OAAO,OAAO,GAAG,IAAI,KAAK;CACnC,CAAC;CAED,MAAM,IAAI,QAAQ,iCAAiC,QAAQ;EACzD,MAAM,QAAQ,IAAI,MAAM,QAAQ;EAChC,IAAI,CAAC,MAAM,OAAO,MAAM,KAAK,eAAe,GAAG,OAAO;EACtD,OAAO,OAAO,kBAAkB,KAAK,CAAC;CACxC,CAAC;CACD,OAAO;AACT;;AAGA,SAAS,kBAAkB,GAAmB;CAC5C,IAAI,MAAM,EAAE,QACV,yHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,MAAM,IAAI,QACR,uHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,OAAO;AACT;;AAGA,SAAgB,gBAAgB,OAAuB;CACrD,IAAI,IAAI,MAAM,YAAY;CAC1B,IAAI,kBAAkB,CAAC;CACvB,IAAI,oBAAoB,CAAC;CAEzB,IAAI,EAAE,QAAQ,aAAa,GAAG;CAO9B,OALe,EACZ,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC,CAC3B,QAAQ,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAC/B,QAAQ,MAAM,CAAC,WAAW,IAAI,CAAC,CACtB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;AAC/B;;AAGA,SAAS,WAAW,WAAmB,MAAuB;CAC5D,MAAM,KAAK,gBAAgB,SAAS;CACpC,MAAM,KAAK,gBAAgB,IAAI;CAC/B,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,IAAI,OAAO,IAAI,OAAO;CAGtB,OAAO,IADQ,OAAO,UAAU,GAAG,QAAQ,uBAAuB,MAAM,EAAE,QAClE,CAAC,CAAC,KAAK,EAAE;AACnB;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;AAGA,SAAS,iBAAiB,WAAqB,aAA+B;CAC5E,IAAI,YAAY,WAAW,GAAG,OAAO;CACrC,MAAM,QAAQ,MAAc;EAC1B,MAAM,IAAI,EAAE,MAAM,kBAAkB;EACpC,OAAO,IAAI,mBAAmB,EAAE,MAAM,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE,YAAY;CAC1E;CACA,MAAM,MAAM,IAAI,IAAI,UAAU,IAAI,IAAI,CAAC;CACvC,IAAI,MAAM;CACV,KAAK,MAAM,KAAK,aAAa,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,GAAG,OAAO;CAC1D,OAAO,MAAM,YAAY;AAC3B;AAEA,MAAM,gBAAgB,UAAkB,MAAc,cACpD;CACE;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,cAA2B;CAClC,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,iEAAiE;CAC7F,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;CAElF,OAAO;EAAE,SADO,QAAQ,IAAI,qBAAqB,QAAQ,IAAI,eAAe;EAC1D;EAAK;CAAM;AAC/B;;AAGA,eAAe,WACb,UACA,MACA,WACA,QACkB;CAClB,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,MAAM;GACN,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS,aAAa,UAAU,MAAM,SAAS;GAAE,CAAC;EAC/E,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,4BAA4B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,oDAAoD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAE1G,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,sEAAsE,QAAQ,MAAM,GAAG,GAAG,GAAG;CAC/G;CACA,IAAI,OAAO,YAAY,WAAW,OAAO;CACzC,IAAI,OAAO,YAAY,aAAa,OAAO;CAC3C,MAAM,IAAI,MAAM,2DAA2D,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;AACnH;;AAGA,SAAS,eAAe,KAAuB;CAC7C,IAAI,CAAC,KAAK,OAAO,CAAC;CAClB,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,KAAK,IAAI,SAAS,wBAAwB,GAAG,IAAI,KAAK,EAAE,EAAE;CACrE,OAAO;AACT;AAEA,SAAS,UAAU,KAAgB,OAA0B;CAC3D,MAAM,cAAc,eAAe,IAAI,UAAU;CACjD,MAAM,OAAmB;EACvB,MAAM,IAAI;EACV;EACA,gBAAgB,IAAI,mBAAmB;EACvC,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,UAAU;EACd,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA6B;CAC7C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAC5B,MAAM,IAAI,MAAM,eAAe,KAAK,GAAG,uDAAuD;CAEhG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,IAAI,wCAAwC,KAAK,OAAO,qCAAqC;CACrG,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,SAAgB,sBAAwC;CACtD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,yBAAyB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC7G,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;YAE3F;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,4BAA4B,IAAI,iIAEmB,QAAQ,2EAE7D;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,QAAQ,KAAK,SAAS;GAsB5B,MAAM,SAAS,MAAM,GAAG;;;oBAlBV,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;;;;;;;;;;;;;;qCAcnE,MAAM;;;GAIL,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAE1E,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KAAK,MAAM,UAAU,GAAG,EAAE,MAAM,CAAC;EAC/C;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAE7C,MAAM,WAAW,iBADC,eAAe,QACS,GAAG,KAAK,WAAW;GAE7D,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,QAAQ;KACR,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,IAAI,WAAW,aAAa,KAAK,IAAI,GACnC,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,MAAM,UAAU,MAAM,WAAW,KAAK,WAAW,KAAK,MAAM,aAAa,YAAY,CAAC;GACtF,OAAO;IACL,UAAU;IACV,OAAO,UAAU,IAAI;IACrB,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,cAAc,UAAU,YAAY;KACpC,kBAAkB;IACpB,CAAC;GACH;EACF;CACF;AACF"}
+{"version":3,"file":"frames.js","names":[],"sources":["../../src/benchmarks/frames.ts"],"sourcesContent":["/**\n * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text\n * final answer string (optionally preceded by a citations block). FRAMES ships,\n * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs\n * needed to answer) — the gold-citation column is what lets the loop's critic\n * check citation coverage deterministically.\n *\n * Judge is two-tier, deterministic-first:\n *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).\n *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,\n *            temperature 0), fired ONLY when Tier 1 misses, to absorb the\n *            paraphrase/alias equivalence FRAMES intends to allow.\n *\n * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge\n * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced\n * in `detail` for the critic, never a hard pass criterion.\n *\n * Requires for a live run: the bench `.venv` with `datasets` installed and a\n * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to\n * load the committed fixtures (bench/fixtures/frames.json) — no HF download.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'frames.json')\n\nconst DATASET = 'google/frames-benchmark'\n/** Pin the dataset revision: ids are derived from row order (no native id column),\n *  so a reorder would break opts.ids selection + scorecard joins. */\nconst DATASET_REVISION = 'main'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Tier-1 parsing keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n].join('\\n')\n\n/**\n * Typed seam for the future dynamic-topology research worker. The benchmark\n * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the\n * loop worker decodes its agent runs into a {@link ResearchAnswer} and serializes\n * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — judge resolves strictly against this. */\n  gold: string\n  /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */\n  goldSources: string[]\n  /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */\n  reasoningTypes: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\ninterface FramesRow {\n  Prompt: string\n  Answer: string\n  wiki_links: string\n  reasoning_types: string\n}\n\ninterface FramesMeta {\n  gold: string\n  goldSources: string[]\n  reasoningTypes: string\n  rawPrompt: string\n}\n\n/** Run the bench venv python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\nconst ARTICLES = new Set(['a', 'an', 'the'])\nconst UNIT_WORDS = new Set([\n  'years',\n  'year',\n  'months',\n  'month',\n  'days',\n  'day',\n  'people',\n  'percent',\n  'dollars',\n  'meters',\n  'metres',\n  'kilometers',\n  'kilometres',\n  'miles',\n  'km',\n  'm',\n])\n\nconst WRITTEN_NUMBERS: Record<string, number> = {\n  zero: 0,\n  one: 1,\n  two: 2,\n  three: 3,\n  four: 4,\n  five: 5,\n  six: 6,\n  seven: 7,\n  eight: 8,\n  nine: 9,\n  ten: 10,\n  eleven: 11,\n  twelve: 12,\n  thirteen: 13,\n  fourteen: 14,\n  fifteen: 15,\n  sixteen: 16,\n  seventeen: 17,\n  eighteen: 18,\n  nineteen: 19,\n  twenty: 20,\n  thirty: 30,\n  forty: 40,\n  fifty: 50,\n  sixty: 60,\n  seventy: 70,\n  eighty: 80,\n  ninety: 90,\n  hundred: 100,\n  thousand: 1000,\n  million: 1_000_000,\n  billion: 1_000_000_000,\n}\n\nconst MONTHS: Record<string, string> = {\n  january: '01',\n  february: '02',\n  march: '03',\n  april: '04',\n  may: '05',\n  june: '06',\n  july: '07',\n  august: '08',\n  september: '09',\n  october: '10',\n  november: '11',\n  december: '12',\n}\n\nconst SCALES = new Set(['hundred', 'thousand', 'million', 'billion'])\n\n/** Fold a run of written-number words into one integer (\"twenty eight\"→28, \"two hundred\"→200). */\nfunction composeWrittenRun(words: string[]): number {\n  let total = 0\n  let current = 0\n  for (const w of words) {\n    const v = WRITTEN_NUMBERS[w]\n    if (v === undefined) continue\n    if (w === 'hundred') {\n      current = (current === 0 ? 1 : current) * 100\n    } else if (SCALES.has(w)) {\n      current = (current === 0 ? 1 : current) * v\n      total += current\n      current = 0\n    } else {\n      current += v\n    }\n  }\n  return total + current\n}\n\n/** Canonicalize \"1,234\" / \"1.2 million\" / written numerals to a single numeric token. */\nfunction canonicalizeNumbers(s: string): string {\n  // \"1,234,567\" → \"1234567\"\n  let out = s.replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n  // \"1.2 million\" / \"3 billion\" → expanded integer\n  out = out.replace(/\\b(\\d+(?:\\.\\d+)?)\\s+(hundred|thousand|million|billion)\\b/g, (_m, num: string, scaleWord: string) => {\n    const scale = WRITTEN_NUMBERS[scaleWord] ?? 1\n    return String(Number(num) * scale)\n  })\n  // fold contiguous runs of written-number words into one integer\n  out = out.replace(/\\b[a-z]+(?:[\\s-]+[a-z]+)*\\b/g, (run) => {\n    const words = run.split(/[\\s-]+/)\n    if (!words.every((w) => w in WRITTEN_NUMBERS)) return run\n    return String(composeWrittenRun(words))\n  })\n  return out\n}\n\n/** \"March 3, 1879\" / \"3 March 1879\" → \"1879-03-03\"; year-only stays as the year. */\nfunction canonicalizeDates(s: string): string {\n  let out = s.replace(\n    /\\b(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{1,2}),?\\s+(\\d{4})\\b/g,\n    (_m, mon: string, day: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  out = out.replace(\n    /\\b(\\d{1,2})\\s+(january|february|march|april|may|june|july|august|september|october|november|december)\\s+(\\d{4})\\b/g,\n    (_m, day: string, mon: string, year: string) => `${year}-${MONTHS[mon]}-${day.padStart(2, '0')}`,\n  )\n  return out\n}\n\n/** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */\nexport function normalizeAnswer(input: string): string {\n  let s = input.toLowerCase()\n  s = canonicalizeDates(s)\n  s = canonicalizeNumbers(s)\n  // strip punctuation (keep alphanumerics, ISO-date hyphens collapse to space-free below)\n  s = s.replace(/[^\\w\\s-]/g, ' ')\n  // tokenize, drop articles + trailing unit words, keep order\n  const tokens = s\n    .split(/\\s+/)\n    .filter((t) => t.length > 0)\n    .filter((t) => !ARTICLES.has(t))\n    .filter((t) => !UNIT_WORDS.has(t))\n  return tokens.join(' ').trim()\n}\n\n/** Tier-1 match: normalized equality OR normalized gold as a token-boundary substring. */\nfunction tier1Match(candidate: string, gold: string): boolean {\n  const nc = normalizeAnswer(candidate)\n  const ng = normalizeAnswer(gold)\n  if (ng.length === 0) return false\n  if (nc === ng) return true\n  // token-boundary containment: gold appears as a whole-token run inside candidate\n  const re = new RegExp(`(^|\\\\s)${ng.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}(\\\\s|$)`)\n  return re.test(nc)\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess), which judge() counts as resolved=false.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/** Fraction of gold Wikipedia article slugs touched by the candidate's citations. SOFT signal. */\nfunction citationCoverage(citations: string[], goldSources: string[]): number {\n  if (goldSources.length === 0) return 1\n  const slug = (u: string) => {\n    const m = u.match(/\\/wiki\\/([^#?]+)/)\n    return m ? decodeURIComponent(m[1] ?? '').toLowerCase() : u.toLowerCase()\n  }\n  const got = new Set(citations.map(slug))\n  let hit = 0\n  for (const g of goldSources) if (got.has(slug(g))) hit += 1\n  return hit / goldSources.length\n}\n\nconst JUDGE_PROMPT = (question: string, gold: string, candidate: string): string =>\n  [\n    'You are a strict answer-equivalence checker for a factual question-answering benchmark.',\n    'Decide ONLY whether the candidate answer is semantically equivalent to the gold answer for this question.',\n    'It is CORRECT iff it contains the same factual value as the gold answer. Extra correct detail is fine.',\n    'A different value, a missing value, or a wrong value is INCORRECT.',\n    '',\n    `Question: ${question}`,\n    `Gold answer: ${gold}`,\n    `Candidate answer: ${candidate}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"verdict\": \"correct\" | \"incorrect\"}',\n    '```',\n  ].join('\\n')\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const model = process.env.JUDGE_MODEL\n  if (!model) throw new Error('JUDGE_MODEL is required for the FRAMES Tier-2 equivalence judge')\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FRAMES Tier-2 judge')\n  const baseUrl = process.env.JUDGE_ROUTER_BASE ?? process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Constrained binary equivalence gate. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function tier2Judge(\n  question: string,\n  gold: string,\n  candidate: string,\n  router: JudgeRouter,\n): Promise<boolean> {\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'frames-equivalence-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0, seed: 0 },\n        },\n      },\n    },\n    JUDGE_PROMPT(question, gold, candidate),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('FRAMES Tier-2 judge returned no message content')\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { verdict?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { verdict?: unknown }\n  } catch {\n    throw new Error(`FRAMES Tier-2 judge produced unparseable output (no JSON verdict): ${content.slice(0, 300)}`)\n  }\n  if (parsed.verdict === 'correct') return true\n  if (parsed.verdict === 'incorrect') return false\n  throw new Error(`FRAMES Tier-2 judge verdict not in {correct,incorrect}: ${JSON.stringify(parsed).slice(0, 200)}`)\n}\n\n/** FRAMES wiki_links is a python-repr list string, e.g. \"['https://…', 'https://…']\". */\nfunction parseWikiLinks(raw: string): string[] {\n  if (!raw) return []\n  const out: string[] = []\n  for (const m of raw.matchAll(/https?:\\/\\/[^\\s'\"\\]]+/g)) out.push(m[0])\n  return out\n}\n\nfunction rowToTask(row: FramesRow, index: number): BenchTask {\n  const goldSources = parseWikiLinks(row.wiki_links)\n  const meta: FramesMeta = {\n    gold: row.Answer,\n    goldSources,\n    reasoningTypes: row.reasoning_types ?? '',\n    rawPrompt: row.Prompt,\n  }\n  return {\n    id: `frames-${index}`,\n    split: 'test',\n    prompt: row.Prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FramesMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string') {\n    throw new Error(`FRAMES task ${task.id} missing metadata.gold — loadTasks did not populate it`)\n  }\n  return md as unknown as FramesMeta\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as FramesRow[]\n  console.log(`[frames] FRAMES_FIXTURES=1 — loading ${rows.length} committed fixtures (no HF download)`)\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nexport function createFramesAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FRAMES_FIXTURES === '1'\n\n  return {\n    name: 'frames',\n\n    async preflight() {\n      // Tier-2 judge router config must be present in both modes — the loop's\n      // citation-coverage stop gate is meaningless without the equivalence judge.\n      judgeRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`FRAMES_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      try {\n        await py(\n          `from datasets import load_dataset\nload_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `frames preflight failed: ${msg}\\n` +\n            `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets ; ` +\n            `(2) ensure network access to Hugging Face for ${DATASET} ; ` +\n            `or set FRAMES_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const limit = opts.limit ?? 10\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split='test', revision=${JSON.stringify(DATASET_REVISION)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor i, r in enumerate(ds):\n    rid = f\"frames-{i}\"\n    if ids is not None and rid not in ids:\n        continue\n    out.append({\n        \"_index\": i,\n        \"Prompt\": r.get(\"Prompt\", \"\"),\n        \"Answer\": r.get(\"Answer\", \"\"),\n        \"wiki_links\": str(r.get(\"wiki_links\", \"\")),\n        \"reasoning_types\": str(r.get(\"reasoning_types\", \"\")),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await py(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<FramesRow & { _index: number }>\n      return rows.map((r) => rowToTask(r, r._index))\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold Answer, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes (Tier-1 short-circuits with no model tokens).\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n      const coverage = citationCoverage(citations, meta.goldSources)\n\n      if (finalAnswer.length === 0) {\n        // Fail-closed: distinguish prompt-adherence failure from a wrong answer.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            tier: 'none',\n            reason: 'no parseable answer',\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 1 — free, deterministic.\n      if (tier1Match(finalAnswer, meta.gold)) {\n        return {\n          resolved: true,\n          score: 1,\n          detail: JSON.stringify({\n            tier: 1,\n            normalizedAnswer: normalizeAnswer(finalAnswer),\n            normalizedGold: normalizeAnswer(meta.gold),\n            citationCoverage: coverage,\n          }),\n        }\n      }\n\n      // Tier 2 — constrained LLM equivalence gate (fail loud on unparseable output).\n      const verdict = await tier2Judge(meta.rawPrompt, meta.gold, finalAnswer, judgeRouter())\n      return {\n        resolved: verdict,\n        score: verdict ? 1 : 0,\n        detail: JSON.stringify({\n          tier: 2,\n          normalizedAnswer: normalizeAnswer(finalAnswer),\n          normalizedGold: normalizeAnswer(meta.gold),\n          judgeVerdict: verdict ? 'correct' : 'incorrect',\n          citationCoverage: coverage,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,aAAa;AAE3D,MAAM,UAAU;;;AAGhB,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;AACF,CAAC,CAAC,KAAK,IAAI;;AAuCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;AAEA,MAAM,2BAAW,IAAI,IAAI;CAAC;CAAK;CAAM;AAAK,CAAC;AAC3C,MAAM,6BAAa,IAAI,IAAI;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC;AAED,MAAM,kBAA0C;CAC9C,MAAM;CACN,KAAK;CACL,KAAK;CACL,OAAO;CACP,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,OAAO;CACP,MAAM;CACN,KAAK;CACL,QAAQ;CACR,QAAQ;CACR,UAAU;CACV,UAAU;CACV,SAAS;CACT,SAAS;CACT,WAAW;CACX,UAAU;CACV,UAAU;CACV,QAAQ;CACR,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,SAAS;CACT,UAAU;CACV,SAAS;CACT,SAAS;AACX;AAEA,MAAM,SAAiC;CACrC,SAAS;CACT,UAAU;CACV,OAAO;CACP,OAAO;CACP,KAAK;CACL,MAAM;CACN,MAAM;CACN,QAAQ;CACR,WAAW;CACX,SAAS;CACT,UAAU;CACV,UAAU;AACZ;AAEA,MAAM,yBAAS,IAAI,IAAI;CAAC;CAAW;CAAY;CAAW;AAAS,CAAC;;AAGpE,SAAS,kBAAkB,OAAyB;CAClD,IAAI,QAAQ;CACZ,IAAI,UAAU;CACd,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,gBAAgB;EAC1B,IAAI,MAAM,KAAA,GAAW;EACrB,IAAI,MAAM,WACR,WAAW,YAAY,IAAI,IAAI,WAAW;OACrC,IAAI,OAAO,IAAI,CAAC,GAAG;GACxB,WAAW,YAAY,IAAI,IAAI,WAAW;GAC1C,SAAS;GACT,UAAU;EACZ,OACE,WAAW;CAEf;CACA,OAAO,QAAQ;AACjB;;AAGA,SAAS,oBAAoB,GAAmB;CAE9C,IAAI,MAAM,EAAE,QAAQ,qBAAqB,IAAI;CAE7C,MAAM,IAAI,QAAQ,8DAA8D,IAAI,KAAa,cAAsB;EACrH,MAAM,QAAQ,gBAAgB,cAAc;EAC5C,OAAO,OAAO,OAAO,GAAG,IAAI,KAAK;CACnC,CAAC;CAED,MAAM,IAAI,QAAQ,iCAAiC,QAAQ;EACzD,MAAM,QAAQ,IAAI,MAAM,QAAQ;EAChC,IAAI,CAAC,MAAM,OAAO,MAAM,KAAK,eAAe,GAAG,OAAO;EACtD,OAAO,OAAO,kBAAkB,KAAK,CAAC;CACxC,CAAC;CACD,OAAO;AACT;;AAGA,SAAS,kBAAkB,GAAmB;CAC5C,IAAI,MAAM,EAAE,QACV,yHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,MAAM,IAAI,QACR,uHACC,IAAI,KAAa,KAAa,SAAiB,GAAG,KAAK,GAAG,OAAO,KAAK,GAAG,IAAI,SAAS,GAAG,GAAG,GAC/F;CACA,OAAO;AACT;;AAGA,SAAgB,gBAAgB,OAAuB;CACrD,IAAI,IAAI,MAAM,YAAY;CAC1B,IAAI,kBAAkB,CAAC;CACvB,IAAI,oBAAoB,CAAC;CAEzB,IAAI,EAAE,QAAQ,aAAa,GAAG;CAO9B,OALe,EACZ,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,EAAE,SAAS,CAAC,CAAC,CAC3B,QAAQ,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAC/B,QAAQ,MAAM,CAAC,WAAW,IAAI,CAAC,CACtB,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK;AAC/B;;AAGA,SAAS,WAAW,WAAmB,MAAuB;CAC5D,MAAM,KAAK,gBAAgB,SAAS;CACpC,MAAM,KAAK,gBAAgB,IAAI;CAC/B,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,IAAI,OAAO,IAAI,OAAO;CAGtB,OAAO,IADQ,OAAO,UAAU,GAAG,QAAQ,uBAAuB,MAAM,EAAE,QAClE,CAAC,CAAC,KAAK,EAAE;AACnB;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;AAGA,SAAS,iBAAiB,WAAqB,aAA+B;CAC5E,IAAI,YAAY,WAAW,GAAG,OAAO;CACrC,MAAM,QAAQ,MAAc;EAC1B,MAAM,IAAI,EAAE,MAAM,kBAAkB;EACpC,OAAO,IAAI,mBAAmB,EAAE,MAAM,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE,YAAY;CAC1E;CACA,MAAM,MAAM,IAAI,IAAI,UAAU,IAAI,IAAI,CAAC;CACvC,IAAI,MAAM;CACV,KAAK,MAAM,KAAK,aAAa,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,GAAG,OAAO;CAC1D,OAAO,MAAM,YAAY;AAC3B;AAEA,MAAM,gBAAgB,UAAkB,MAAc,cACpD;CACE;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,cAA2B;CAClC,MAAM,QAAQ,QAAQ,IAAI;CAC1B,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,iEAAiE;CAC7F,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;CAElF,OAAO;EAAE,SADO,QAAQ,IAAI,qBAAqB,QAAQ,IAAI,eAAe;EAC1D;EAAK;CAAM;AAC/B;;AAGA,eAAe,WACb,UACA,MACA,WACA,QACkB;CAiBlB,MAAM,WAAU,MAhBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU;KAAE,aAAa;KAAG,MAAM;IAAE;GACtC;EACF;CACF,GACA,aAAa,UAAU,MAAM,SAAS,CACxC,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,iDAAiD;CAC/E,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,sEAAsE,QAAQ,MAAM,GAAG,GAAG,GAAG;CAC/G;CACA,IAAI,OAAO,YAAY,WAAW,OAAO;CACzC,IAAI,OAAO,YAAY,aAAa,OAAO;CAC3C,MAAM,IAAI,MAAM,2DAA2D,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;AACnH;;AAGA,SAAS,eAAe,KAAuB;CAC7C,IAAI,CAAC,KAAK,OAAO,CAAC;CAClB,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,KAAK,IAAI,SAAS,wBAAwB,GAAG,IAAI,KAAK,EAAE,EAAE;CACrE,OAAO;AACT;AAEA,SAAS,UAAU,KAAgB,OAA0B;CAC3D,MAAM,cAAc,eAAe,IAAI,UAAU;CACjD,MAAM,OAAmB;EACvB,MAAM,IAAI;EACV;EACA,gBAAgB,IAAI,mBAAmB;EACvC,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,UAAU;EACd,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA6B;CAC7C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,UAC5B,MAAM,IAAI,MAAM,eAAe,KAAK,GAAG,uDAAuD;CAEhG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,IAAI,wCAAwC,KAAK,OAAO,qCAAqC;CACrG,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,SAAgB,sBAAwC;CACtD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,yBAAyB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC7G,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;YAE3F;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,4BAA4B,IAAI,iIAEmB,QAAQ,2EAE7D;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,QAAQ,KAAK,SAAS;GAsB5B,MAAM,SAAS,MAAM,GAAG;;;oBAlBV,KAAK,UAAU,OAAO,EAAE,2BAA2B,KAAK,UAAU,gBAAgB,EAAE;;;;;;;;;;;;;;qCAcnE,MAAM;;;GAIL,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAE1E,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KAAK,MAAM,UAAU,GAAG,EAAE,MAAM,CAAC;EAC/C;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAE7C,MAAM,WAAW,iBADC,eAAe,QACS,GAAG,KAAK,WAAW;GAE7D,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,QAAQ;KACR,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,IAAI,WAAW,aAAa,KAAK,IAAI,GACnC,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,kBAAkB;IACpB,CAAC;GACH;GAIF,MAAM,UAAU,MAAM,WAAW,KAAK,WAAW,KAAK,MAAM,aAAa,YAAY,CAAC;GACtF,OAAO;IACL,UAAU;IACV,OAAO,UAAU,IAAI;IACrB,QAAQ,KAAK,UAAU;KACrB,MAAM;KACN,kBAAkB,gBAAgB,WAAW;KAC7C,gBAAgB,gBAAgB,KAAK,IAAI;KACzC,cAAc,UAAU,YAAY;KACpC,kBAAkB;IACpB,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/mcad-bench.d.ts

@@ -0,0 +1,106 @@
+import { BenchmarkAdapter } from "./types.js";
+import { McadSpec } from "./mcad-tasks.js";
+//#region src/benchmarks/mcad-bench.d.ts
+interface Vec3 {
+  x: number;
+  y: number;
+  z: number;
+}
+/** One triangle as three vertices, in the STL's stated winding order. */
+type Tri = readonly [Vec3, Vec3, Vec3];
+interface McadGeometry {
+  triangles: number;
+  /** Edges NOT shared by exactly two faces. 0 iff the surface is closed + 2-manifold. */
+  openEdges: number;
+  watertight: boolean;
+  /** Zero-area faces — bad geometry even when the topology closes. */
+  degenerateFaces: number;
+  /** Enclosed volume (absolute) via the signed-tetrahedron sum. */
+  volume: number;
+  /** Connected components of the triangle-adjacency graph = disconnected bodies. */
+  solids: number;
+  bbox: {
+    min: Vec3;
+    max: Vec3;
+    size: Vec3;
+  };
+}
+/** Parse an ASCII STL into triangles by reading `vertex x y z` lines in groups of three. */
+declare function parseAsciiStl(stl: string): Tri[];
+/**
+ * Measure the closed-solid properties of a triangle soup. Pure: no I/O, no deps.
+ *
+ * `solids` is the number of connected components of the graph whose nodes are
+ * triangles and whose edges join two triangles that share a quantised mesh edge.
+ * On a watertight mesh that count IS the number of disconnected bodies (an
+ * enclosed internal cavity is its own shell and counts, which is the behaviour
+ * the specs want — a "hollow" body that is really two nested shells is not one
+ * fused solid).
+ */
+declare function measureMesh(tris: Tri[]): McadGeometry;
+/**
+ * Point-in-solid membership by ray parity, made robust three ways.
+ *
+ * ROBUSTNESS ARGUMENT. A parity test is exact except on a measure-zero set: rays
+ * that graze a triangle edge or vertex (the crossing is counted twice or zero
+ * times), rays coplanar with a face, and origins lying on the surface. Those cases
+ * are not merely rare here — they are SYSTEMATIC, because OpenSCAD emits
+ * axis-aligned meshes whose vertices land on the same round millimetre lattice the
+ * spec's probe coordinates come from, so an axis-aligned ray from a probe point
+ * hits shared edges constantly. Three defences, in order:
+ *   1. DIRECTIONS. The three fixed directions are irrational combinations, so a ray
+ *      from a lattice point cannot stay in an axis-aligned face plane and cannot
+ *      run along a lattice edge.
+ *   2. DETECTION + DETERMINISTIC RE-JITTER. Grazing is DETECTED (a barycentric
+ *      coordinate within BARY_EPS of 0, |det| below DET_EPS with the origin in the
+ *      plane, or |t| within T_EPS) rather than hoped away. A detected ray is re-cast
+ *      with a small direction perturbation drawn from a mulberry32 PRNG seeded by
+ *      the probe coordinates and ray index, so the whole judge stays deterministic:
+ *      the same mesh and the same point always take the same sequence of re-casts.
+ *   3. MAJORITY VOTE. The verdict is the majority of three independent directions,
+ *      so even an undetected miscount on one ray cannot flip the answer.
+ * A point sitting exactly ON the surface has no correct answer; every direction
+ * degenerates there and the vote falls back to whatever the jittered casts say.
+ * The specs therefore place probes with >=1 mm clearance from any surface.
+ */
+declare function pointInSolid(tris: Tri[], point: Vec3): boolean;
+interface McadCheck {
+  name: string;
+  ok: boolean;
+  measured: string;
+  expected: string;
+}
+interface McadScoring {
+  checks: McadCheck[];
+  failed: McadCheck[];
+  score: number;
+  resolved: boolean;
+}
+/** Score a measured mesh against the task's spec — one named check per assertion. */
+declare function scoreAgainstSpec(geo: McadGeometry, tris: Tri[], spec: McadSpec): McadScoring;
+/** Task metadata carried onto every `BenchTask`, so the judge needs no lookup table. */
+interface McadTaskMeta extends Record<string, unknown> {
+  spec: McadSpec;
+  source: string;
+  calibrated: boolean;
+}
+/**
+ * Strip a markdown fence if the model wrapped its answer in one. The prompt asks
+ * for bare source; a fenced answer is a formatting slip, not a geometry failure,
+ * and the geometric gate downstream is unchanged either way.
+ */
+declare function stripCodeFence(artifact: string): string;
+/** Compile OpenSCAD source and measure the resulting mesh. Judge-time only. */
+declare function compileAndMeasure(src: string): Promise<{
+  ok: true;
+  geo: McadGeometry;
+  tris: Tri[];
+  stlPath: string;
+} | {
+  ok: false;
+  detail: string;
+}>;
+declare function createMcadBenchAdapter(): BenchmarkAdapter;
+//#endregion
+export { McadCheck, McadGeometry, McadScoring, McadTaskMeta, Tri, Vec3, compileAndMeasure, createMcadBenchAdapter, measureMesh, parseAsciiStl, pointInSolid, scoreAgainstSpec, stripCodeFence };
+//# sourceMappingURL=mcad-bench.d.ts.map