@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/benchmarks/finresearchbench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"finresearchbench.js","names":[],"sources":["../../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillTemplate(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAcA,MAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,eAAgE;CACvE,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,kEAAkE;CAC5F,OAAO;EACL,SAAS,QAAQ,IAAI,eAAe;EACpC;EACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;CAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;CACxE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,6BAA6B,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;CACrD,MAAM,UAAU,IAAI,KAAK;CACzB,IAAI,QAAQ,WAAW,GAAG,GAAG,OAAO,KAAK,MAAM,OAAO;CACtD,OAAO,QACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;CAC1F,MAAM,UAAU,QACb,QAAQ,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,CAAC,CACvE,KAAK,QAAQ,IAAI,EAAE;CACtB,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,mCAAmC,OAAO,sCAAsC,QAAQ,OAAO,GAAG,QAAQ,OAAO,WAAW,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,kHAE7J;AAEJ;AAEA,SAAS,UAAU,KAAwB,cAAkC;CAC3E,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,OAAwB;EAC5B,IAAI,IAAI;EACR,UAAU,IAAI,YAAY;EAC1B,UAAU,IAAI;EACd;EACA;EACA,WAAW,IAAI,cAAc;EAC7B,QAAQ,IAAI,UAAU;EACtB,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,SAAS,eAAe,4BAA4B;CACtD;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAChC,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,kDAAkD;CAErG,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;CACpG,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,YAAY,CAAC;CAC1D,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACxE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,GAAG;CACpG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KAAK,4DAA4D,QAAQ,OAAO,kBAAkB;CAC1G,OAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;CACtF,MAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;CACxD,sBAAsB,SAAS,IAAI;CACnC,OAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;CACrE,MAAM,WAAW,KAAK;CACtB,IAAI,CAAC,UAAU,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,+BAA+B;CAC/F,OAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,CAAC,CACvC,WAAW,cAAc,QAAQ,CAAC,CAClC,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,CAAC,CACnE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;CACzE,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,eAAe,GAAG,WAAW,KAAK,EAAE,EAAE;CACvE,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,aAAa,YAAY;EAClC,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,SAAS;EAC/B,QAAQ;GACN;EACF;EACA,MAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;EACjF,MAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;EAClF,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG;EACzB,IAAI,IAAI,GAAG,MAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,GAAG;EAChG,MAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;EAClD,IAAI,QAAQ,GAAG,MAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,GAAG;EAChH,OAAO;GAAE;GAAO,KAAK;EAAO;CAC9B;CACA,MAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,GAAG;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;CAC5F,IAAI,CAAC,KAAK,mBAAmB,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,6BAA6B;CAC3G,MAAM,SAAS,aAAa;CAC5B,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,KAAK;GAAkB,GAClD;IAAE,MAAM;IAAQ,SAAS,aAAa,MAAM,QAAQ;GAAE,CACxD;EACF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAC7G,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UAAU,MAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAC5I,MAAM,EAAE,OAAO,QAAQ,gBAAgB,OAAO;CAC9C,OAAO;EACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,EAAG;EAC5E;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;GAAU,YAAY,OAAO;GAAO;EAAI,CAAC;CAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;CAC5C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;CACzE,MAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;CACzE,MAAM,WAAW,cAAc,QAAQ;CACvC,MAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;CACnE,OAAO;EACL,UAAU,UAAU;EACpB;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;EAAS,CAAC;CAC3E;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,8BAA8B;CAE/D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,eAAe,UAAU,cAAc;IAC7C;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,4NACF;GAEF,aAAa;GACb,MAAM,eAAe,MAAM,WAAW;GACtC,MAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,sEAAsE;GACjG,OAAO,kBAAkB,MAAM,IAAI;EACrC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,mBAAmB,KAAK,mBAAmB,KAAA;EACzD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,cAAc,OAAO,aAAa,MAAM,QAAQ;GACpD,IAAI,SAAS,KAAK,CAAC,CAAC,WAAW,GAC7B,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,SAAS,KAAK;KAAS,UAAU,KAAK;KAAU,QAAQ;IAAe,CAAC;GACnG;GAEF,OAAO,iBAAiB,MAAM,QAAQ;EACxC;CACF;AACF"}

package/dist/benchmarks/finsearchcomp.d.ts

@@ -1,37 +1,5 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +
- * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).
- *
- * Each record is SELF-CONTAINED and carries its OWN judge: a per-record
- * `judge_system_prompt` + a `judge_prompt_template` with {prompt},
- * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),
- * and {response} (the student answer) placeholders. We faithfully REPLICATE the
- * benchmark's judge — fill the template, run it under the record's system prompt
- * via the pinned router (temperature 0), and parse the JSON the judge emits. The
- * judge returns an `answer_score` field; 1/positive = resolved, else not.
- * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).
- *
- * There is NO deterministic Tier-1 path here: the gold answer is prose with
- * region-specific units, embedded tolerances ("允许1%的误差"), and multi-point
- * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,
- * so the judge IS the score — fail loud on unparseable judge output (never
- * default to resolved).
- *
- * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)
- * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)
- * needs a live market snapshot / akshare ground truth to fill the judge's
- * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.
- * See {@link T1Seam}.
- *
- * Requires for a live run: network access to the GitHub-hosted dataset JSON and
- * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back
- * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit
- * console.warn — never a silent fallback.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/finsearchcomp.d.ts
 /**
  * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a
  * live market snapshot the judge consumes via a {ground_truth} template slot;
@@ -40,10 +8,11 @@ import '@tangle-network/agent-runtime/loops';
  * and filling {ground_truth}. Until then T1 is excluded from loadTasks.
  */
 interface T1Seam {
-    promptId: string;
-    /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */
-    resolveGroundTruth(promptId: string): Promise<string>;
+  promptId: string;
+  /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */
+  resolveGroundTruth(promptId: string): Promise<string>;
 }
 declare function createFinsearchcompAdapter(): BenchmarkAdapter;
-
-export { type T1Seam, createFinsearchcompAdapter };
+//#endregion
+export { T1Seam, createFinsearchcompAdapter };
+//# sourceMappingURL=finsearchcomp.d.ts.map

package/dist/benchmarks/finsearchcomp.js

@@ -1,7 +1,268 @@
-import {
-  createFinsearchcompAdapter
-} from "../chunk-UPAMRDX4.js";
-export {
-  createFinsearchcompAdapter
-};
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+//#region src/benchmarks/finsearchcomp.ts
+/**
+* FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +
+* reasoning. Worker artifact = the agent's free-form final answer text (zh/en).
+*
+* Each record is SELF-CONTAINED and carries its OWN judge: a per-record
+* `judge_system_prompt` + a `judge_prompt_template` with {prompt},
+* {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),
+* and {response} (the student answer) placeholders. We faithfully REPLICATE the
+* benchmark's judge — fill the template, run it under the record's system prompt
+* via the pinned router (temperature 0), and parse the JSON the judge emits. The
+* judge returns an `answer_score` field; 1/positive = resolved, else not.
+* This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).
+*
+* There is NO deterministic Tier-1 path here: the gold answer is prose with
+* region-specific units, embedded tolerances ("允许1%的误差"), and multi-point
+* scoring criteria. The benchmark intends its own LLM judge to be the arbiter,
+* so the judge IS the score — fail loud on unparseable judge output (never
+* default to resolved).
+*
+* SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)
+* are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)
+* needs a live market snapshot / akshare ground truth to fill the judge's
+* {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.
+* See {@link T1Seam}.
+*
+* Requires for a live run: network access to the GitHub-hosted dataset JSON and
+* a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back
+* to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit
+* console.warn — never a silent fallback.
+*/
+const FIXTURES = join(fileURLToPath(new URL("../..", import.meta.url)), "fixtures", "finsearchcomp.json");
+const DATASET_URL = "https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json";
+/** Only these are self-contained (gold + own judge, no external ground truth). */
+const SUPPORTED_TYPES = /* @__PURE__ */ new Set(["T2", "T3"]);
+/**
+* Worker contract appended to every task prompt. The judge is paraphrase- and
+* keyword-aware ("答案为", "the answer is"), so we do not impose a sentinel; we
+* pass the whole artifact through as {response}. We DO ask the worker to lead
+* with its final value so a truncated artifact still carries the answer.
+*/
+const WORKER_CONTRACT = [
+	"",
+	"Research this financial question using live web/market sources and answer it.",
+	"State your final answer explicitly, with the exact units and precision the question requests.",
+	"Respect any tolerance the question implies; show the value you settled on, not just intermediate work."
+].join("\n");
+/** "(T2)Simple_Historical_Lookup_001" → "T2"; throws on an unknown/missing prefix. */
+function parseTaskType(promptId) {
+	const m = promptId.match(/^\((T[123])\)/);
+	if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`);
+	return m[1];
+}
+/** "Simple_Historical_Lookup(Greater China)" → "Greater China"; '' when absent. */
+function parseRegion(label) {
+	const m = label.match(/\(([^()]+)\)\s*$/);
+	return m ? m[1].trim() : "";
+}
+function recordToTask(rec) {
+	const taskType = parseTaskType(rec.prompt_id);
+	const meta = {
+		promptId: rec.prompt_id,
+		label: rec.label,
+		taskType,
+		region: parseRegion(rec.label),
+		responseReference: rec.response_reference,
+		judgePromptTemplate: rec.judge_prompt_template,
+		judgeSystemPrompt: rec.judge_system_prompt,
+		rawPrompt: rec.prompt
+	};
+	return {
+		id: rec.prompt_id,
+		split: taskType,
+		prompt: rec.prompt + WORKER_CONTRACT,
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.responseReference !== "string" || typeof md.judgePromptTemplate !== "string" || typeof md.judgeSystemPrompt !== "string" || typeof md.rawPrompt !== "string") throw new Error(`FinSearchComp task ${task.id} missing judge metadata — loadTasks did not populate it`);
+	return md;
+}
+/** Fill the per-record judge_prompt_template. {ground_truth} is T1-only and unused for T2/T3. */
+function fillJudgePrompt(meta, response) {
+	return meta.judgePromptTemplate.replaceAll("{prompt}", meta.rawPrompt).replaceAll("{response_reference}", meta.responseReference).replaceAll("{response}", response);
+}
+function judgeRouter() {
+	const key = process.env.TANGLE_API_KEY;
+	if (!key) throw new Error("TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge");
+	const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
+	return {
+		baseUrl: process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
+		key,
+		model
+	};
+}
+/**
+* The judge emits a JSON object with an `answer_score`. In this dataset it is a
+* flat integer ({"answer_score": 1}); we also accept the nested-array form
+* ([[1]] → read [0][0]) the upstream scorer can produce. 1/positive = resolved.
+* A "null" sentinel (judge could not obtain ground truth) is NOT a pass.
+*/
+function readAnswerScore(value) {
+	let v = value;
+	while (Array.isArray(v)) v = v[0];
+	if (v === null || v === void 0 || v === "null") return {
+		resolved: false,
+		score: 0,
+		raw: value
+	};
+	const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
+	if (!Number.isFinite(n)) throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`);
+	return {
+		resolved: n >= 1,
+		score: n <= 0 ? 0 : n >= 1 ? 1 : n,
+		raw: value
+	};
+}
+/** Extract the first JSON object carrying `answer_score` from judge output; throw if none. */
+function parseJudgeOutput(content) {
+	const candidates = [];
+	for (const m of content.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)) candidates.push(m[1].trim());
+	for (const m of content.matchAll(/\{[^{}]*"answer_score"[\s\S]*?\}/g)) candidates.push(m[0]);
+	candidates.push(content.trim());
+	for (const c of candidates) {
+		let parsed;
+		try {
+			parsed = JSON.parse(c);
+		} catch {
+			continue;
+		}
+		if ("answer_score" in parsed) return readAnswerScore(parsed.answer_score);
+	}
+	throw new Error(`FinSearchComp judge produced no parseable {"answer_score": …}: ${content.slice(0, 400)}`);
+}
+/** Run the record's own judge via the router. Fail loud on transport/parse errors. */
+async function runRecordJudge(meta, response, router) {
+	const res = await fetch(`${router.baseUrl}/chat/completions`, {
+		method: "POST",
+		headers: {
+			"content-type": "application/json",
+			authorization: `Bearer ${router.key}`
+		},
+		body: JSON.stringify({
+			model: router.model,
+			temperature: 0,
+			messages: [{
+				role: "system",
+				content: meta.judgeSystemPrompt
+			}, {
+				role: "user",
+				content: fillJudgePrompt(meta, response)
+			}]
+		})
+	});
+	if (!res.ok) throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+	const body = await res.json();
+	const content = body.choices?.[0]?.message?.content;
+	if (typeof content !== "string") throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const { resolved, score, raw } = parseJudgeOutput(content);
+	return {
+		resolved,
+		score,
+		detail: JSON.stringify({
+			taskType: meta.taskType,
+			region: meta.region,
+			answerScore: raw,
+			judgeModel: router.model
+		})
+	};
+}
+function selectRecords(records, opts) {
+	let tasks = records.filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id))).map(recordToTask);
+	if (opts.split) tasks = tasks.filter((t) => t.split === opts.split);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((t) => want.has(t.id));
+	} else if (opts.limit !== void 0) tasks = balanceByType(tasks, opts.limit);
+	return tasks;
+}
+/** Round-robin across task types (T2/T3) so a `limit` pulls a balanced mix
+*  rather than slicing the type-ordered dataset's first N (which yields all
+*  T2). Deterministic — preserves dataset order within each type. */
+function balanceByType(tasks, limit) {
+	const queues = /* @__PURE__ */ new Map();
+	for (const t of tasks) {
+		const k = parseTaskType(t.id);
+		const q = queues.get(k);
+		if (q) q.push(t);
+		else queues.set(k, [t]);
+	}
+	const lanes = [...queues.values()];
+	const out = [];
+	for (let i = 0; out.length < limit; i++) {
+		const before = out.length;
+		for (const lane of lanes) {
+			if (i < lane.length) out.push(lane[i]);
+			if (out.length >= limit) break;
+		}
+		if (out.length === before) break;
+	}
+	return out;
+}
+async function loadFixtures(opts) {
+	return selectRecords(JSON.parse(await readFile(FIXTURES, "utf8")), opts);
+}
+async function fetchDataset() {
+	const res = await fetch(DATASET_URL);
+	if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`);
+	const data = await res.json();
+	if (!Array.isArray(data) || data.length === 0) throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`);
+	return data;
+}
+function createFinsearchcompAdapter() {
+	const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === "1";
+	return {
+		name: "finsearchcomp",
+		async preflight() {
+			judgeRouter();
+			console.warn("[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live market snapshot / akshare ground truth for the judge {ground_truth} slot. loadTasks supports T2 + T3 only; wire T1Seam to add T1.");
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8").catch((err) => {
+					throw new Error(`FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+				});
+				return;
+			}
+			const res = await fetch(DATASET_URL, { method: "HEAD" }).catch((err) => {
+				throw new Error(`finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}\nFix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`);
+			});
+			if (!res.ok) throw new Error(`finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			let records;
+			try {
+				records = await fetchDataset();
+			} catch (err) {
+				console.warn(`[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`);
+				return loadFixtures(opts);
+			}
+			return selectRecords(records, opts);
+		},
+		async goldArtifact(task) {
+			return readMeta(task).responseReference;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const response = artifact.trim();
+			if (response.length === 0) return {
+				resolved: false,
+				score: 0,
+				detail: JSON.stringify({
+					taskType: meta.taskType,
+					region: meta.region,
+					reason: "empty answer"
+				})
+			};
+			return runRecordJudge(meta, response, judgeRouter());
+		}
+	};
+}
+//#endregion
+export { createFinsearchcompAdapter };
+
 //# sourceMappingURL=finsearchcomp.js.map

package/dist/benchmarks/finsearchcomp.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"finsearchcomp.js","names":[],"sources":["../../src/benchmarks/finsearchcomp.ts"],"sourcesContent":["/**\n * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +\n * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).\n *\n * Each record is SELF-CONTAINED and carries its OWN judge: a per-record\n * `judge_system_prompt` + a `judge_prompt_template` with {prompt},\n * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),\n * and {response} (the student answer) placeholders. We faithfully REPLICATE the\n * benchmark's judge — fill the template, run it under the record's system prompt\n * via the pinned router (temperature 0), and parse the JSON the judge emits. The\n * judge returns an `answer_score` field; 1/positive = resolved, else not.\n * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).\n *\n * There is NO deterministic Tier-1 path here: the gold answer is prose with\n * region-specific units, embedded tolerances (\"允许1%的误差\"), and multi-point\n * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,\n * so the judge IS the score — fail loud on unparseable judge output (never\n * default to resolved).\n *\n * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)\n * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)\n * needs a live market snapshot / akshare ground truth to fill the judge's\n * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.\n * See {@link T1Seam}.\n *\n * Requires for a live run: network access to the GitHub-hosted dataset JSON and\n * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back\n * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit\n * console.warn — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'finsearchcomp.json')\n\nconst DATASET_URL =\n  'https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json'\n\n/** Task-type prefix on prompt_id, e.g. \"(T2)Simple_Historical_Lookup_001\". */\ntype TaskType = 'T1' | 'T2' | 'T3'\n/** Only these are self-contained (gold + own judge, no external ground truth). */\nconst SUPPORTED_TYPES: ReadonlySet<TaskType> = new Set<TaskType>(['T2', 'T3'])\n\n/**\n * Worker contract appended to every task prompt. The judge is paraphrase- and\n * keyword-aware (\"答案为\", \"the answer is\"), so we do not impose a sentinel; we\n * pass the whole artifact through as {response}. We DO ask the worker to lead\n * with its final value so a truncated artifact still carries the answer.\n */\nconst WORKER_CONTRACT = [\n  '',\n  'Research this financial question using live web/market sources and answer it.',\n  'State your final answer explicitly, with the exact units and precision the question requests.',\n  'Respect any tolerance the question implies; show the value you settled on, not just intermediate work.',\n].join('\\n')\n\n/**\n * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a\n * live market snapshot the judge consumes via a {ground_truth} template slot;\n * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a\n * resolver that fetches the akshare/market value for the record at judge time\n * and filling {ground_truth}. Until then T1 is excluded from loadTasks.\n */\nexport interface T1Seam {\n  promptId: string\n  /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */\n  resolveGroundTruth(promptId: string): Promise<string>\n}\n\ninterface FinSearchRecord {\n  prompt_id: string\n  prompt: string\n  response_reference: string\n  judge_prompt_template: string\n  judge_system_prompt: string\n  label: string\n}\n\ninterface FinSearchMeta {\n  promptId: string\n  label: string\n  taskType: TaskType\n  region: string\n  responseReference: string\n  judgePromptTemplate: string\n  judgeSystemPrompt: string\n  rawPrompt: string\n}\n\n/** \"(T2)Simple_Historical_Lookup_001\" → \"T2\"; throws on an unknown/missing prefix. */\nfunction parseTaskType(promptId: string): TaskType {\n  const m = promptId.match(/^\\((T[123])\\)/)\n  if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`)\n  return m[1] as TaskType\n}\n\n/** \"Simple_Historical_Lookup(Greater China)\" → \"Greater China\"; '' when absent. */\nfunction parseRegion(label: string): string {\n  const m = label.match(/\\(([^()]+)\\)\\s*$/)\n  return m ? m[1].trim() : ''\n}\n\nfunction recordToTask(rec: FinSearchRecord): BenchTask {\n  const taskType = parseTaskType(rec.prompt_id)\n  const meta: FinSearchMeta = {\n    promptId: rec.prompt_id,\n    label: rec.label,\n    taskType,\n    region: parseRegion(rec.label),\n    responseReference: rec.response_reference,\n    judgePromptTemplate: rec.judge_prompt_template,\n    judgeSystemPrompt: rec.judge_system_prompt,\n    rawPrompt: rec.prompt,\n  }\n  return {\n    id: rec.prompt_id,\n    split: taskType,\n    prompt: rec.prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinSearchMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.responseReference !== 'string' ||\n    typeof md.judgePromptTemplate !== 'string' ||\n    typeof md.judgeSystemPrompt !== 'string' ||\n    typeof md.rawPrompt !== 'string'\n  ) {\n    throw new Error(`FinSearchComp task ${task.id} missing judge metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinSearchMeta\n}\n\n/** Fill the per-record judge_prompt_template. {ground_truth} is T1-only and unused for T2/T3. */\nfunction fillJudgePrompt(meta: FinSearchMeta, response: string): string {\n  return meta.judgePromptTemplate\n    .replaceAll('{prompt}', meta.rawPrompt)\n    .replaceAll('{response_reference}', meta.responseReference)\n    .replaceAll('{response}', response)\n}\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/**\n * The judge emits a JSON object with an `answer_score`. In this dataset it is a\n * flat integer ({\"answer_score\": 1}); we also accept the nested-array form\n * ([[1]] → read [0][0]) the upstream scorer can produce. 1/positive = resolved.\n * A \"null\" sentinel (judge could not obtain ground truth) is NOT a pass.\n */\nfunction readAnswerScore(value: unknown): { resolved: boolean; score: number; raw: unknown } {\n  let v: unknown = value\n  // unwrap nested-array form: [[s]] / [s]\n  while (Array.isArray(v)) v = v[0]\n  if (v === null || v === undefined || v === 'null') {\n    return { resolved: false, score: 0, raw: value }\n  }\n  const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v) : NaN\n  if (!Number.isFinite(n)) {\n    throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`)\n  }\n  // Their scale: 1 = correct, 0 = wrong. Treat any positive as resolved; clamp to 0..1.\n  const resolved = n >= 1\n  const score = n <= 0 ? 0 : n >= 1 ? 1 : n\n  return { resolved, score, raw: value }\n}\n\n/** Extract the first JSON object carrying `answer_score` from judge output; throw if none. */\nfunction parseJudgeOutput(content: string): { resolved: boolean; score: number; raw: unknown } {\n  // Prefer a fenced block, then fall back to scanning for a {...} with answer_score.\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[^{}]*\"answer_score\"[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const c of candidates) {\n    let parsed: { answer_score?: unknown }\n    try {\n      parsed = JSON.parse(c) as { answer_score?: unknown }\n    } catch {\n      continue\n    }\n    if ('answer_score' in parsed) return readAnswerScore(parsed.answer_score)\n  }\n  throw new Error(\n    `FinSearchComp judge produced no parseable {\"answer_score\": …}: ${content.slice(0, 400)}`,\n  )\n}\n\n/** Run the record's own judge via the router. Fail loud on transport/parse errors. */\nasync function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise<BenchScore> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillJudgePrompt(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const { resolved, score, raw } = parseJudgeOutput(content)\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      taskType: meta.taskType,\n      region: meta.region,\n      answerScore: raw,\n      judgeModel: router.model,\n    }),\n  }\n}\n\nfunction selectRecords(records: FinSearchRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records\n    .filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id)))\n    .map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = balanceByType(tasks, opts.limit)\n  }\n  return tasks\n}\n\n/** Round-robin across task types (T2/T3) so a `limit` pulls a balanced mix\n *  rather than slicing the type-ordered dataset's first N (which yields all\n *  T2). Deterministic — preserves dataset order within each type. */\nfunction balanceByType(tasks: BenchTask[], limit: number): BenchTask[] {\n  const queues = new Map<TaskType, BenchTask[]>()\n  for (const t of tasks) {\n    const k = parseTaskType(t.id)\n    const q = queues.get(k)\n    if (q) q.push(t)\n    else queues.set(k, [t])\n  }\n  const lanes = [...queues.values()]\n  const out: BenchTask[] = []\n  for (let i = 0; out.length < limit; i++) {\n    const before = out.length\n    for (const lane of lanes) {\n      if (i < lane.length) out.push(lane[i])\n      if (out.length >= limit) break\n    }\n    if (out.length === before) break // all lanes exhausted\n  }\n  return out\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as FinSearchRecord[]\n  return selectRecords(records, opts)\n}\n\nasync function fetchDataset(): Promise<FinSearchRecord[]> {\n  const res = await fetch(DATASET_URL)\n  if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`)\n  const data = (await res.json()) as FinSearchRecord[]\n  if (!Array.isArray(data) || data.length === 0) {\n    throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`)\n  }\n  return data\n}\n\nexport function createFinsearchcompAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === '1'\n\n  return {\n    name: 'finsearchcomp',\n\n    async preflight() {\n      // The per-record LLM judge is the arbiter in both modes — its router\n      // config must exist or no score can be produced.\n      judgeRouter()\n      console.warn(\n        '[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live ' +\n          'market snapshot / akshare ground truth for the judge {ground_truth} slot. ' +\n          'loadTasks supports T2 + T3 only; wire T1Seam to add T1.',\n      )\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      const res = await fetch(DATASET_URL, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 ` +\n            `to run against the committed fixtures offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. ` +\n            `Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let records: FinSearchRecord[]\n      try {\n        records = await fetchDataset()\n      } catch (err) {\n        console.warn(\n          `[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); ` +\n            `falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRecords(records, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the record's response_reference fed back as the student\n      // answer, so verify-judge proves gold→resolved through the SAME per-record\n      // judge the real artifact takes. The reference may carry tolerance/criteria\n      // prose; the judge is built to read the value out of it.\n      const meta = readMeta(task)\n      return meta.responseReference\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const response = artifact.trim()\n      if (response.length === 0) {\n        // Fail-closed — the benchmark's own judges score an empty answer 0.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ taskType: meta.taskType, region: meta.region, reason: 'empty answer' }),\n        }\n      }\n      return runRecordJudge(meta, response, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,MAAM,WAAW,KADE,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAClC,GAAG,YAAY,oBAAoB;AAElE,MAAM,cACJ;;AAKF,MAAM,kCAAyC,IAAI,IAAc,CAAC,MAAM,IAAI,CAAC;;;;;;;AAQ7E,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AAoCX,SAAS,cAAc,UAA4B;CACjD,MAAM,IAAI,SAAS,MAAM,eAAe;CACxC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,qDAAqD,KAAK,UAAU,QAAQ,GAAG;CACvG,OAAO,EAAE;AACX;;AAGA,SAAS,YAAY,OAAuB;CAC1C,MAAM,IAAI,MAAM,MAAM,kBAAkB;CACxC,OAAO,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI;AAC3B;AAEA,SAAS,aAAa,KAAiC;CACrD,MAAM,WAAW,cAAc,IAAI,SAAS;CAC5C,MAAM,OAAsB;EAC1B,UAAU,IAAI;EACd,OAAO,IAAI;EACX;EACA,QAAQ,YAAY,IAAI,KAAK;EAC7B,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,mBAAmB,IAAI;EACvB,WAAW,IAAI;CACjB;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ,IAAI,SAAS;EACrB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,wBAAwB,YAClC,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,cAAc,UAExB,MAAM,IAAI,MAAM,sBAAsB,KAAK,GAAG,wDAAwD;CAExG,OAAO;AACT;;AAGA,SAAS,gBAAgB,MAAqB,UAA0B;CACtE,OAAO,KAAK,oBACT,WAAW,YAAY,KAAK,SAAS,CAAC,CACtC,WAAW,wBAAwB,KAAK,iBAAiB,CAAC,CAC1D,WAAW,cAAc,QAAQ;AACtC;AAQA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,uEAAuE;CACjG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;;;;;;AAQA,SAAS,gBAAgB,OAAoE;CAC3F,IAAI,IAAa;CAEjB,OAAO,MAAM,QAAQ,CAAC,GAAG,IAAI,EAAE;CAC/B,IAAI,MAAM,QAAQ,MAAM,KAAA,KAAa,MAAM,QACzC,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,KAAK;CAAM;CAEjD,MAAM,IAAI,OAAO,MAAM,WAAW,IAAI,OAAO,MAAM,WAAW,OAAO,CAAC,IAAI;CAC1E,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,MAAM,iDAAiD,KAAK,UAAU,KAAK,GAAG;CAK1F,OAAO;EAAE,UAFQ,KAAK;EAEH,OADL,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI;EACd,KAAK;CAAM;AACvC;;AAGA,SAAS,iBAAiB,SAAqE;CAE7F,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,mCAAmC,GAAG,WAAW,KAAK,EAAE,EAAE;CAC3F,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,KAAK,YAAY;EAC1B,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,CAAC;EACvB,QAAQ;GACN;EACF;EACA,IAAI,kBAAkB,QAAQ,OAAO,gBAAgB,OAAO,YAAY;CAC1E;CACA,MAAM,IAAI,MACR,kEAAkE,QAAQ,MAAM,GAAG,GAAG,GACxF;AACF;;AAGA,eAAe,eAAe,MAAqB,UAAkB,QAA0C;CAC7G,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,KAAK;GAAkB,GAClD;IAAE,MAAM;IAAQ,SAAS,gBAAgB,MAAM,QAAQ;GAAE,CAC3D;EACF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,4BAA4B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,oDAAoD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAE1G,MAAM,EAAE,UAAU,OAAO,QAAQ,iBAAiB,OAAO;CACzD,OAAO;EACL;EACA;EACA,QAAQ,KAAK,UAAU;GACrB,UAAU,KAAK;GACf,QAAQ,KAAK;GACb,aAAa;GACb,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAEA,SAAS,cAAc,SAA4B,MAAgC;CACjF,IAAI,QAAQ,QACT,QAAQ,MAAM,gBAAgB,IAAI,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,CAC9D,IAAI,YAAY;CACnB,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,MAAM,EAAE,UAAU,KAAK,KAAK;CAClE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,cAAc,OAAO,KAAK,KAAK;CAEzC,OAAO;AACT;;;;AAKA,SAAS,cAAc,OAAoB,OAA4B;CACrE,MAAM,yBAAS,IAAI,IAA2B;CAC9C,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,IAAI,cAAc,EAAE,EAAE;EAC5B,MAAM,IAAI,OAAO,IAAI,CAAC;EACtB,IAAI,GAAG,EAAE,KAAK,CAAC;OACV,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC;CACxB;CACA,MAAM,QAAQ,CAAC,GAAG,OAAO,OAAO,CAAC;CACjC,MAAM,MAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,SAAS,OAAO,KAAK;EACvC,MAAM,SAAS,IAAI;EACnB,KAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,IAAI,KAAK,QAAQ,IAAI,KAAK,KAAK,EAAE;GACrC,IAAI,IAAI,UAAU,OAAO;EAC3B;EACA,IAAI,IAAI,WAAW,QAAQ;CAC7B;CACA,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CAEnE,OAAO,cADS,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAC/B,GAAG,IAAI;AACpC;AAEA,eAAe,eAA2C;CACxD,MAAM,MAAM,MAAM,MAAM,WAAW;CACnC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,8BAA8B,IAAI,OAAO,YAAY,aAAa;CAC/F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,IAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAC1C,MAAM,IAAI,MAAM,gDAAgD,aAAa;CAE/E,OAAO;AACT;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,eAAe,QAAQ,IAAI,2BAA2B;CAE5D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,YAAY;GACZ,QAAQ,KACN,oNAGF;GACA,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,gCAAgC,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC/F;IACF,CAAC;IACD;GACF;GACA,MAAM,MAAM,MAAM,MAAM,aAAa,EAAE,QAAQ,OAAO,CAAC,CAAC,CAAC,OAAO,QAAQ;IACtE,MAAM,IAAI,MACR,2CAA2C,YAAY,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,0IAGtG;GACF,CAAC;GACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MACR,yCAAyC,IAAI,OAAO,OAAO,YAAY,8EAEzE;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,aAAa;GAC/B,SAAS,KAAK;IACZ,QAAQ,KACN,8CAA8C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAC5C,UAC7C;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,cAAc,SAAS,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAMlC,OADa,SAAS,IACZ,CAAC,CAAC;EACd;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,WAAW,SAAS,KAAK;GAC/B,IAAI,SAAS,WAAW,GAEtB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,UAAU,KAAK;KAAU,QAAQ,KAAK;KAAQ,QAAQ;IAAe,CAAC;GACjG;GAEF,OAAO,eAAe,MAAM,UAAU,YAAY,CAAC;EACrD;CACF;AACF"}

package/dist/benchmarks/frames.d.ts

@@ -1,28 +1,5 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * FRAMES adapter (google/frames-benchmark). Worker artifact = a single free-text
- * final answer string (optionally preceded by a citations block). FRAMES ships,
- * per item, a short gold `Answer` PLUS gold `wiki_links` (the Wikipedia URLs
- * needed to answer) — the gold-citation column is what lets the loop's critic
- * check citation coverage deterministically.
- *
- * Judge is two-tier, deterministic-first:
- *   Tier 1 — normalized exact / token-boundary containment match (no model tokens).
- *   Tier 2 — a constrained binary equivalence gate via a pinned LLM (JUDGE_MODEL,
- *            temperature 0), fired ONLY when Tier 1 misses, to absorb the
- *            paraphrase/alias equivalence FRAMES intends to allow.
- *
- * score is binary (resolved ? 1 : 0) — FRAMES has no partial credit. The judge
- * resolves strictly on the Answer value; wiki_links are a SOFT signal surfaced
- * in `detail` for the critic, never a hard pass criterion.
- *
- * Requires for a live run: the bench `.venv` with `datasets` installed and a
- * JUDGE_MODEL router key. For offline/CI verification set FRAMES_FIXTURES=1 to
- * load the committed fixtures (bench/fixtures/frames.json) — no HF download.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/frames.d.ts
 /**
  * Typed seam for the future dynamic-topology research worker. The benchmark
  * adapter scores a plain `string` artifact (the BenchmarkAdapter contract); the
@@ -30,18 +7,18 @@ import '@tangle-network/agent-runtime/loops';
  * `finalAnswer` (+ optional `CITATIONS:` block) into that string before judging.
  */
 interface ResearchTask {
-    id: string;
-    question: string;
-    /** Gold short answer — judge resolves strictly against this. */
-    gold: string;
-    /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */
-    goldSources: string[];
-    /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */
-    reasoningTypes: string;
+  id: string;
+  question: string;
+  /** Gold short answer — judge resolves strictly against this. */
+  gold: string;
+  /** Gold Wikipedia URLs — SOFT citation-coverage signal for the critic, never a hard gate. */
+  goldSources: string[];
+  /** FRAMES reasoning_types, e.g. 'numerical | temporal' — slices the scorecard by hop-type. */
+  reasoningTypes: string;
 }
 interface ResearchAnswer {
-    finalAnswer: string;
-    citations: string[];
+  finalAnswer: string;
+  citations: string[];
 }
 /** Port of the SQuAD/GAIA normalize_answer routine, plus number/date canonicalization. */
 declare function normalizeAnswer(input: string): string;
@@ -55,5 +32,6 @@ declare function parseFinalAnswer(artifact: string): string;
 /** Extract cited URLs from a `CITATIONS:` block and any bare URLs in the artifact. */
 declare function parseCitations(artifact: string): string[];
 declare function createFramesAdapter(): BenchmarkAdapter;
-
-export { type ResearchAnswer, type ResearchTask, createFramesAdapter, normalizeAnswer, parseCitations, parseFinalAnswer };
+//#endregion
+export { ResearchAnswer, ResearchTask, createFramesAdapter, normalizeAnswer, parseCitations, parseFinalAnswer };
+//# sourceMappingURL=frames.d.ts.map