@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-GC2EPS6L.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/t2-ragbench.ts"],"sourcesContent":["/**\n * T2-RAGBench adapter.\n *\n * T2-RAGBench stresses text+table retrieval and numerical reasoning over\n * financial documents. The judge uses the shared deterministic answer scorer\n * with numeric tolerance enabled by default.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 't2-ragbench.json')\n\ninterface T2RagBenchMeta {\n  benchmark: 't2-ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  subset: string\n  documentId: string\n}\n\nconst dataFile = (): string | undefined => process.env.T2_RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'program_answer',\n    'original_answer',\n    'answer',\n    'answers',\n    'reference',\n    'reference_answer',\n    'gold',\n  ])\n  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`)\n  const baseContexts =\n    contextsFrom(raw.context).length > 0\n      ? contextsFrom(raw.context)\n      : contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.chunks).length > 0\n        ? contextsFrom(raw.chunks)\n        : contextsFrom(raw.passages)\n  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text)\n  const contexts = table\n    ? [...baseContexts, { id: 'table', title: 'Table', text: table }]\n    : baseContexts\n  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? 'unknown'\n  const documentId =\n    stringFrom(raw.context_id) ??\n    stringFrom(raw.document_id) ??\n    stringFrom(raw.doc_id) ??\n    stringFrom(raw.file_name) ??\n    'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`\n  const meta: T2RagBenchMeta = {\n    benchmark: 't2-ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    subset,\n    documentId,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? subset,\n    prompt: [\n      'Answer this T2-RAGBench text-and-table financial question.',\n      'Do the required numerical reasoning from the supplied context before giving the final value.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Document: ${documentId}`,\n      `Subset: ${subset}`,\n      contexts.length > 0 ? `\\nContext:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): T2RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`T2-RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as T2RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'T2-RAGBench')\n}\n\nexport function createT2RagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 't2-ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'T2-RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 })\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        subset: meta.subset,\n        documentId: meta.documentId,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAQA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,kBAAkB;AAW/D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,mBAAmB,KAAK,oBAAoB;AAChF,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,QAAQ,CAAC;AAC9D,QAAM,cAAc,WAAW,KAAK;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,mBAAmB,KAAK,yBAAyB;AAC7E,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,mBAAmB,KAAK,iBAAiB;AACvF,QAAM,eACJ,aAAa,IAAI,OAAO,EAAE,SAAS,IAC/B,aAAa,IAAI,OAAO,IACxB,aAAa,IAAI,QAAQ,EAAE,SAAS,IACpC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,MAAM,EAAE,SAAS,IAChC,aAAa,IAAI,MAAM,IACvB,aAAa,IAAI,QAAQ;AACjC,QAAM,QAAQ,WAAW,IAAI,KAAK,KAAK,WAAW,IAAI,UAAU;AAChE,QAAM,WAAW,QACb,CAAC,GAAG,cAAc,EAAE,IAAI,SAAS,OAAO,SAAS,MAAM,MAAM,CAAC,IAC9D;AACJ,QAAM,SAAS,WAAW,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,KAAK;AACpE,QAAM,aACJ,WAAW,IAAI,UAAU,KACzB,WAAW,IAAI,WAAW,KAC1B,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,SAAS,KACxB;AACF,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,QAAQ,KAAK,eAAe,KAAK;AACxG,QAAM,OAAuB;AAAA,IAC3B,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,aAAa,UAAU;AAAA,MACvB,WAAW,MAAM;AAAA,MACjB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAe,aAAa,QAAQ,CAAC,KAAK;AAAA,IAClE,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAiC;AACjD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,oBAAoB,KAAK,EAAE,wDAAmD;AAAA,EAChG;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,2BAA2B,IAAI,EAAE;AACxE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,uDAAkD,KAAK,MAAM,mBAAmB;AAC7F,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,aAAa;AAC7D;AAEO,SAAS,0BAA4C;AAC1D,QAAM,eAAe,QAAQ,IAAI,yBAAyB;AAE1D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,6DAA6D;AACxF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,aAAa;AAAA,IAC/E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,aAAa,EAAE,kBAAkB,KAAK,CAAC;AACxF,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,QAAQ,KAAK;AAAA,QACb,YAAY,KAAK;AAAA,QACjB,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-GCHL6XPM.js

@@ -1,169 +0,0 @@
-import {
-  runVenvPython
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/tau-bench-shared.ts
-import { readFile, stat } from "fs/promises";
-import { resolve } from "path";
-var tauResultsOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:text|path|json)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-async function assertPath(path, label, benchName) {
-  try {
-    await stat(path);
-  } catch (err) {
-    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-function benchDir(config) {
-  return process.env[config.dirEnv];
-}
-function benchDomain(config) {
-  return process.env[config.domainEnv] ?? config.defaultDomain;
-}
-function rowToTask(row, config, split) {
-  const meta = {
-    taskId: row.id,
-    domain: row.domain,
-    split,
-    userScenario: row.user_scenario,
-    description: row.description,
-    evaluationCriteria: row.evaluation_criteria
-  };
-  return {
-    id: row.id,
-    split,
-    prompt: [
-      config.taskIntro,
-      `Run this task in the official ${row.domain} domain.`,
-      "The benchmark is a simulated multi-turn user/tool conversation.",
-      "",
-      typeof row.user_scenario === "string" ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),
-      "",
-      "Return the path to the official tau results.json or trajectory file containing this task run."
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task, benchName) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "string" || typeof md.domain !== "string") {
-    throw new Error(`${benchName} task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, opts, config, split) {
-  let tasks = rows.map((row) => rowToTask(row, config, split));
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`);
-  return tasks;
-}
-async function loadFixtures(config, opts) {
-  const rows = JSON.parse(await readFile(config.fixturePath, "utf8"));
-  console.warn(`[${config.name}] ${config.fixturesEnv}=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectRows(rows, opts, config, opts.split);
-}
-async function loadOfficialTasks(config, root, opts) {
-  const domain = benchDomain(config);
-  const script = `
-import json, sys
-from pathlib import Path
-root = Path(sys.argv[1])
-domain = sys.argv[2]
-split = sys.argv[3] or None
-sys.path.insert(0, str(root / "src"))
-from tau2.registry import registry
-loader = registry.get_tasks_loader(domain)
-tasks = loader(split)
-rows = []
-for task in tasks:
-    row = task.model_dump(mode="json")
-    row["domain"] = domain
-    rows.append(row)
-print(json.dumps(rows))
-`;
-  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ""]);
-  return selectRows(JSON.parse(stdout), opts, config, opts.split);
-}
-async function scoreOfficialTrajectory(root, meta, artifactPath) {
-  const script = `
-import json, sys
-from pathlib import Path
-root = Path(sys.argv[1])
-task_id = sys.argv[2]
-artifact = Path(sys.argv[3])
-sys.path.insert(0, str(root / "src"))
-from tau2.data_model.simulation import Results
-from tau2.scripts.evaluate_trajectories import compute_simulation_rewards
-results = Results.load(artifact)
-updated = compute_simulation_rewards(results)
-scores = []
-for sim in updated.simulations:
-    if sim.task_id == task_id and sim.reward_info is not None:
-        scores.append(float(sim.reward_info.reward))
-if not scores:
-    raise SystemExit(f"no scored simulations for task_id={task_id} in {artifact}")
-print(json.dumps({"count": len(scores), "score": sum(scores) / len(scores), "scores": scores}))
-`;
-  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0);
-  return JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-}
-function createTauBenchAdapter(config) {
-  const fixturesMode = process.env[config.fixturesEnv] === "1";
-  return {
-    name: config.name,
-    output: tauResultsOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = benchDir(config);
-      if (!dir) {
-        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`);
-      }
-      await assertPath(`${dir}/src/tau2/registry.py`, "tau registry", config.name);
-      await loadOfficialTasks(config, dir, { limit: 1 });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(config, opts);
-      const dir = benchDir(config);
-      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`);
-      return loadOfficialTasks(config, dir, opts);
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const dir = benchDir(config);
-      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`);
-      const meta = readMeta(task, config.name);
-      const artifactPath = resolve(artifact.trim());
-      await assertPath(artifactPath, "tau results/trajectory artifact", config.name);
-      const report = await scoreOfficialTrajectory(dir, meta, artifactPath);
-      const score = typeof report.score === "number" ? report.score : 0;
-      return {
-        resolved: score === 1,
-        score,
-        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count })
-      };
-    }
-  };
-}
-
-export {
-  tauResultsOutput,
-  createTauBenchAdapter
-};
-//# sourceMappingURL=chunk-GCHL6XPM.js.map

package/dist/chunk-GCHL6XPM.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/tau-bench-shared.ts"],"sourcesContent":["/**\n * Shared tau-bench adapter spine.\n *\n * tau2 and tau3 live in the same upstream repository/package namespace today:\n * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation\n * for task loading and reward recomputation so the domain/version adapters only\n * choose env names, default domain, and fixture file.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nexport interface TauBenchConfig {\n  name: string\n  fixturePath: string\n  fixturesEnv: string\n  dirEnv: string\n  domainEnv: string\n  defaultDomain: string\n  installHint: string\n  taskIntro: string\n}\n\ninterface TauRow {\n  id: string\n  domain: string\n  user_scenario?: unknown\n  description?: unknown\n  evaluation_criteria?: unknown\n}\n\ninterface TauMeta {\n  taskId: string\n  domain: string\n  split?: string\n  userScenario?: unknown\n  description?: unknown\n  evaluationCriteria?: unknown\n}\n\nexport const tauResultsOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path|json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertPath(path: string, label: string, benchName: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction benchDir(config: TauBenchConfig): string | undefined {\n  return process.env[config.dirEnv]\n}\n\nfunction benchDomain(config: TauBenchConfig): string {\n  return process.env[config.domainEnv] ?? config.defaultDomain\n}\n\nfunction rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {\n  const meta: TauMeta = {\n    taskId: row.id,\n    domain: row.domain,\n    split,\n    userScenario: row.user_scenario,\n    description: row.description,\n    evaluationCriteria: row.evaluation_criteria,\n  }\n  return {\n    id: row.id,\n    split,\n    prompt: [\n      config.taskIntro,\n      `Run this task in the official ${row.domain} domain.`,\n      'The benchmark is a simulated multi-turn user/tool conversation.',\n      '',\n      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),\n      '',\n      'Return the path to the official tau results.json or trajectory file containing this task run.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask, benchName: string): TauMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {\n    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as TauMeta\n}\n\nfunction selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, config, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]\n  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts, config, opts.split)\n}\n\nasync function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const domain = benchDomain(config)\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ndomain = sys.argv[2]\nsplit = sys.argv[3] or None\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.registry import registry\nloader = registry.get_tasks_loader(domain)\ntasks = loader(split)\nrows = []\nfor task in tasks:\n    row = task.model_dump(mode=\"json\")\n    row[\"domain\"] = domain\n    rows.append(row)\nprint(json.dumps(rows))\n`\n  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])\n  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)\n}\n\nasync function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\nartifact = Path(sys.argv[3])\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.data_model.simulation import Results\nfrom tau2.scripts.evaluate_trajectories import compute_simulation_rewards\nresults = Results.load(artifact)\nupdated = compute_simulation_rewards(results)\nscores = []\nfor sim in updated.simulations:\n    if sim.task_id == task_id and sim.reward_info is not None:\n        scores.append(float(sim.reward_info.reward))\nif not scores:\n    raise SystemExit(f\"no scored simulations for task_id={task_id} in {artifact}\")\nprint(json.dumps({\"count\": len(scores), \"score\": sum(scores) / len(scores), \"scores\": scores}))\n`\n  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {\n  const fixturesMode = process.env[config.fixturesEnv] === '1'\n\n  return {\n    name: config.name,\n    output: tauResultsOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = benchDir(config)\n      if (!dir) {\n        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)\n      }\n      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)\n      await loadOfficialTasks(config, dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(config, opts)\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)\n      return loadOfficialTasks(config, dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)\n      const meta = readMeta(task, config.name)\n      const artifactPath = resolve(artifact.trim())\n      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)\n      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,UAAU,YAAY;AAC/B,SAAS,eAAe;AAiCjB,IAAM,mBAA0C;AAAA,EACrD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,2CAA2C,CAAC;AAC7E,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,WAAW,MAAc,OAAe,WAAkC;AACvF,MAAI;AACF,UAAM,KAAK,IAAI;AAAA,EACjB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,GAAG,SAAS,aAAa,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAC3G;AACF;AAEA,SAAS,SAAS,QAA4C;AAC5D,SAAO,QAAQ,IAAI,OAAO,MAAM;AAClC;AAEA,SAAS,YAAY,QAAgC;AACnD,SAAO,QAAQ,IAAI,OAAO,SAAS,KAAK,OAAO;AACjD;AAEA,SAAS,UAAU,KAAa,QAAwB,OAA2B;AACjF,QAAM,OAAgB;AAAA,IACpB,QAAQ,IAAI;AAAA,IACZ,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,cAAc,IAAI;AAAA,IAClB,aAAa,IAAI;AAAA,IACjB,oBAAoB,IAAI;AAAA,EAC1B;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,MACN,OAAO;AAAA,MACP,iCAAiC,IAAI,MAAM;AAAA,MAC3C;AAAA,MACA;AAAA,MACA,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAAgB,KAAK,UAAU,IAAI,iBAAiB,CAAC,GAAG,MAAM,CAAC;AAAA,MAC3G;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAiB,WAA4B;AAC7D,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,GAAG,WAAW,UAAU;AACzE,UAAM,IAAI,MAAM,GAAG,SAAS,SAAS,KAAK,EAAE,wDAAmD;AAAA,EACjG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAgB,MAAmB,QAAwB,OAA6B;AAC1G,MAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,UAAU,KAAK,QAAQ,KAAK,CAAC;AAC3D,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,GAAG,OAAO,IAAI,sBAAsB,KAAK,UAAU,IAAI,CAAC,EAAE;AAClG,SAAO;AACT;AAEA,eAAe,aAAa,QAAwB,MAAyC;AAC3F,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,OAAO,aAAa,MAAM,CAAC;AAClE,UAAQ,KAAK,IAAI,OAAO,IAAI,KAAK,OAAO,WAAW,qBAAgB,KAAK,MAAM,mBAAmB;AACjG,SAAO,WAAW,MAAM,MAAM,QAAQ,KAAK,KAAK;AAClD;AAEA,eAAe,kBAAkB,QAAwB,MAAc,MAAyC;AAC9G,QAAM,SAAS,YAAY,MAAM;AACjC,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,MAAM,QAAQ,KAAK,SAAS,EAAE,CAAC;AAC3E,SAAO,WAAW,KAAK,MAAM,MAAM,GAAe,MAAM,QAAQ,KAAK,KAAK;AAC5E;AAEA,eAAe,wBAAwB,MAAc,MAAe,cAAwD;AAC1H,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,MAAM,KAAK,QAAQ,YAAY,GAAG,CAAC;AAC/E,SAAO,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEO,SAAS,sBAAsB,QAA0C;AAC9E,QAAM,eAAe,QAAQ,IAAI,OAAO,WAAW,MAAM;AAEzD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,SAAS,MAAM;AAC3B,UAAI,CAAC,KAAK;AACR,cAAM,IAAI,MAAM,GAAG,OAAO,MAAM,sBAAsB,OAAO,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,WAAW,GAAG,GAAG,yBAAyB,gBAAgB,OAAO,IAAI;AAC3E,YAAM,kBAAkB,QAAQ,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IACnD;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,QAAQ,IAAI;AAClD,YAAM,MAAM,SAAS,MAAM;AAC3B,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,GAAG,OAAO,MAAM,iCAAiC,OAAO,IAAI,QAAQ;AAC9F,aAAO,kBAAkB,QAAQ,KAAK,IAAI;AAAA,IAC5C;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,MAAM,SAAS,MAAM;AAC3B,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,GAAG,OAAO,MAAM,yBAAyB,OAAO,IAAI,uBAAuB;AACrG,YAAM,OAAO,SAAS,MAAM,OAAO,IAAI;AACvC,YAAM,eAAe,QAAQ,SAAS,KAAK,CAAC;AAC5C,YAAM,WAAW,cAAc,mCAAmC,OAAO,IAAI;AAC7E,YAAM,SAAS,MAAM,wBAAwB,KAAK,MAAM,YAAY;AACpE,YAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;AAChE,aAAO;AAAA,QACL,UAAU,UAAU;AAAA,QACpB;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,KAAK,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,MAC1F;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-HQ5HCCKF.js

@@ -1,142 +0,0 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/ragbench.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "ragbench.json");
-var dataFile = () => process.env.RAGBENCH_DATA_FILE;
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`);
-  const query = firstString(raw, ["question", "query", "user_input", "prompt"]);
-  const goldAnswers = allStrings(raw, [
-    "response",
-    "responses",
-    "model_response",
-    "generated_response",
-    "reference",
-    "references",
-    "reference_answer",
-    "reference_answers",
-    "answer",
-    "answers",
-    "gold",
-    "gold_answer",
-    "ground_truth",
-    "expected_answer"
-  ]);
-  if (!query) throw new Error(`RAGBench row ${index} missing question/query`);
-  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`);
-  const contexts = contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.retrieved_contexts).length > 0 ? contextsFrom(raw.retrieved_contexts) : contextsFrom(raw.documents);
-  const traceLabels = {};
-  for (const key of [
-    "adherence",
-    "completeness",
-    "relevance",
-    "utilization",
-    "all_relevant_sentence_keys",
-    "all_utilized_sentence_keys"
-  ]) {
-    if (raw[key] !== void 0) traceLabels[key] = raw[key];
-  }
-  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset);
-  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`;
-  const meta = {
-    benchmark: "ragbench",
-    query,
-    goldAnswers,
-    contexts,
-    ...dataset ? { dataset } : {},
-    traceLabels
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? dataset ?? "ragbench",
-    prompt: [
-      "Answer this RAGBench question using the supplied retrieved context.",
-      "End with a single final line: `FINAL ANSWER: <answer>`.",
-      "",
-      `Question: ${query}`,
-      contexts.length > 0 ? `
-Retrieved context:
-${contextBlock(contexts)}` : void 0
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldAnswers)) {
-    throw new Error(`RAGBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "RAGBench");
-}
-function createRagBenchAdapter() {
-  const fixturesMode = process.env.RAGBENCH_FIXTURES === "1";
-  return {
-    name: "ragbench",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("RAGBENCH_DATA_FILE is required to load RAGBench tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "RAGBench");
-    },
-    async goldArtifact(task) {
-      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const score = scoreAnswerArtifact(artifact, meta.goldAnswers);
-      return answerScoreToBenchScore(score, {
-        benchmark: meta.benchmark,
-        dataset: meta.dataset ?? null,
-        traceLabels: meta.traceLabels,
-        contextCount: meta.contexts.length
-      });
-    }
-  };
-}
-
-export {
-  createRagBenchAdapter
-};
-//# sourceMappingURL=chunk-HQ5HCCKF.js.map

package/dist/chunk-HQ5HCCKF.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/ragbench.ts"],"sourcesContent":["/**\n * RAGBench-compatible adapter.\n *\n * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a\n * compatible table. Rows must carry a query and at least one reference answer.\n * Contexts, TRACe labels, and source metadata are preserved in task metadata\n * for diagnostics; the deterministic judge scores the worker's final answer\n * against the reference answer(s).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'ragbench.json')\n\ninterface RagBenchMeta {\n  benchmark: 'ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  dataset?: string\n  traceLabels: Record<string, unknown>\n}\n\nconst dataFile = (): string | undefined => process.env.RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'user_input', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'response',\n    'responses',\n    'model_response',\n    'generated_response',\n    'reference',\n    'references',\n    'reference_answer',\n    'reference_answers',\n    'answer',\n    'answers',\n    'gold',\n    'gold_answer',\n    'ground_truth',\n    'expected_answer',\n  ])\n  if (!query) throw new Error(`RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`)\n  const contexts =\n    contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.retrieved_contexts).length > 0\n        ? contextsFrom(raw.retrieved_contexts)\n        : contextsFrom(raw.documents)\n  const traceLabels: Record<string, unknown> = {}\n  for (const key of [\n    'adherence',\n    'completeness',\n    'relevance',\n    'utilization',\n    'all_relevant_sentence_keys',\n    'all_utilized_sentence_keys',\n  ]) {\n    if (raw[key] !== undefined) traceLabels[key] = raw[key]\n  }\n  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset)\n  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`\n  const meta: RagBenchMeta = {\n    benchmark: 'ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    ...(dataset ? { dataset } : {}),\n    traceLabels,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? dataset ?? 'ragbench',\n    prompt: [\n      'Answer this RAGBench question using the supplied retrieved context.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      contexts.length > 0 ? `\\nRetrieved context:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'RAGBench')\n}\n\nexport function createRagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 'ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('RAGBENCH_DATA_FILE is required to load RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        dataset: meta.dataset ?? null,\n        traceLabels: meta.traceLabels,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAUA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,eAAe;AAW5D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,oBAAoB;AAC7E,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,cAAc,QAAQ,CAAC;AAC5E,QAAM,cAAc,WAAW,KAAK;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,gBAAgB,KAAK,yBAAyB;AAC1E,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,2BAA2B;AAC9F,QAAM,WACJ,aAAa,IAAI,QAAQ,EAAE,SAAS,IAChC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,kBAAkB,EAAE,SAAS,IAC5C,aAAa,IAAI,kBAAkB,IACnC,aAAa,IAAI,SAAS;AAClC,QAAM,cAAuC,CAAC;AAC9C,aAAW,OAAO;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAAG;AACD,QAAI,IAAI,GAAG,MAAM,OAAW,aAAY,GAAG,IAAI,IAAI,GAAG;AAAA,EACxD;AACA,QAAM,UAAU,WAAW,IAAI,OAAO,KAAK,WAAW,IAAI,cAAc;AACxE,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,UAAU,KAAK,YAAY,KAAK;AAChF,QAAM,OAAqB;AAAA,IACzB,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;AAAA,IAC7B;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK,WAAW;AAAA,IAC3C,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAyB,aAAa,QAAQ,CAAC,KAAK;AAAA,IAC5E,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,wDAAmD;AAAA,EAC7F;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,wBAAwB,IAAI,EAAE;AACrE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,mBAAmB;AACvF,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,UAAU;AAC1D;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,uDAAuD;AAClF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,UAAU;AAAA,IAC5E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,WAAW;AAC5D,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,SAAS,KAAK,WAAW;AAAA,QACzB,aAAa,KAAK;AAAA,QAClB,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-HVW25KSX.js

@@ -1,208 +0,0 @@
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/finresearchbench.ts
-import { readFile, stat } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "finresearchbench.json");
-var dataFile = () => process.env.FINRESEARCHBENCH_DATA_FILE;
-function routerConfig() {
-  const key = process.env.TANGLE_API_KEY;
-  if (!key) throw new Error("TANGLE_API_KEY is required for FinResearchBench live LLM judging");
-  return {
-    baseUrl: process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
-    key,
-    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? "deepseek-v4-flash"
-  };
-}
-async function assertReadable(path, label) {
-  try {
-    await stat(path);
-  } catch (err) {
-    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-function readRecords(raw) {
-  const trimmed = raw.trim();
-  if (trimmed.startsWith("[")) return JSON.parse(trimmed);
-  return trimmed.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
-}
-function assertLiveJudgeFields(records, source) {
-  const missing = records.filter((row) => !row.judge_system_prompt || !row.judge_prompt_template).map((row) => row.id);
-  if (missing.length > 0) {
-    throw new Error(
-      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(", ")}. Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.`
-    );
-  }
-}
-function rowToTask(row, fixturesMode) {
-  const referenceAnswer = row.reference_answer ?? "";
-  const referenceReport = row.reference_report ?? referenceAnswer;
-  const meta = {
-    id: row.id,
-    category: row.category ?? "unknown",
-    question: row.question,
-    referenceAnswer,
-    referenceReport,
-    logicTree: row.logic_tree ?? null,
-    rubric: row.rubric ?? null,
-    judgeSystemPrompt: row.judge_system_prompt,
-    judgePromptTemplate: row.judge_prompt_template,
-    scoring: fixturesMode ? "fixture-exact-reference" : "official-logic-tree-judge"
-  };
-  return {
-    id: row.id,
-    split: row.category,
-    prompt: [
-      "Complete this FinResearchBench financial research task.",
-      "Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.",
-      "",
-      row.question
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.question !== "string") {
-    throw new Error(`FinResearchBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, opts, fixturesMode) {
-  let tasks = rows.map((row) => rowToTask(row, fixturesMode));
-  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`);
-  return tasks;
-}
-async function loadFixtures(opts) {
-  const records = readRecords(await readFile(FIXTURES, "utf8"));
-  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 \u2014 loading ${records.length} adapter fixtures`);
-  return selectRows(records, opts, true);
-}
-async function loadOfficialTasks(path, opts) {
-  const records = readRecords(await readFile(path, "utf8"));
-  assertLiveJudgeFields(records, path);
-  return selectRows(records, opts, false);
-}
-function fillTemplate(meta, response) {
-  const template = meta.judgePromptTemplate;
-  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`);
-  return template.replaceAll("{question}", meta.question).replaceAll("{response}", response).replaceAll("{reference_answer}", meta.referenceAnswer).replaceAll("{reference_report}", meta.referenceReport).replaceAll("{logic_tree}", JSON.stringify(meta.logicTree, null, 2)).replaceAll("{rubric}", JSON.stringify(meta.rubric, null, 2));
-}
-function parseJudgeScore(content) {
-  const candidates = [];
-  for (const m of content.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)) candidates.push(m[1].trim());
-  for (const m of content.matchAll(/\{[\s\S]*?\}/g)) candidates.push(m[0]);
-  candidates.push(content.trim());
-  for (const candidate of candidates) {
-    let parsed;
-    try {
-      parsed = JSON.parse(candidate);
-    } catch {
-      continue;
-    }
-    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score;
-    const n = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : NaN;
-    if (!Number.isFinite(n)) continue;
-    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`);
-    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100;
-    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`);
-    return { score, raw: parsed };
-  }
-  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`);
-}
-async function runOfficialJudge(meta, response) {
-  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`);
-  const router = routerConfig();
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: "POST",
-    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      messages: [
-        { role: "system", content: meta.judgeSystemPrompt },
-        { role: "user", content: fillTemplate(meta, response) }
-      ]
-    })
-  });
-  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-  const body = await res.json();
-  const content = body.choices?.[0]?.message?.content;
-  if (typeof content !== "string") throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
-  const { score, raw } = parseJudgeScore(content);
-  return {
-    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),
-    score,
-    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw })
-  };
-}
-function normalizeText(value) {
-  return value.toLowerCase().replace(/\s+/g, " ").trim();
-}
-function scoreFixture(meta, artifact) {
-  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport);
-  const response = normalizeText(artifact);
-  const score = answer.length > 0 && response.includes(answer) ? 1 : 0;
-  return {
-    resolved: score === 1,
-    score,
-    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category })
-  };
-}
-function createFinResearchBenchAdapter() {
-  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === "1";
-  return {
-    name: "finresearchbench",
-    async preflight() {
-      if (fixturesMode) {
-        await assertReadable(FIXTURES, "fixture file");
-        return;
-      }
-      const file = dataFile();
-      if (!file) {
-        throw new Error(
-          "FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl."
-        );
-      }
-      routerConfig();
-      await assertReadable(file, "data file");
-      await loadOfficialTasks(file, { limit: 1 });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const file = dataFile();
-      if (!file) throw new Error("FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows");
-      return loadOfficialTasks(file, opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return meta.referenceReport || meta.referenceAnswer || void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      if (fixturesMode) return scoreFixture(meta, artifact);
-      if (artifact.trim().length === 0) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: "empty answer" })
-        };
-      }
-      return runOfficialJudge(meta, artifact);
-    }
-  };
-}
-
-export {
-  createFinResearchBenchAdapter
-};
-//# sourceMappingURL=chunk-HVW25KSX.js.map

package/dist/chunk-HVW25KSX.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillTemplate(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,UAAU,YAAY;AAC/B,SAAS,YAAY;AAIrB,IAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,eAAgE;AACvE,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,kEAAkE;AAC5F,SAAO;AAAA,IACL,SAAS,QAAQ,IAAI,eAAe;AAAA,IACpC;AAAA,IACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;AAAA,EAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;AACxE,MAAI;AACF,UAAM,KAAK,IAAI;AAAA,EACjB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,6BAA6B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;AACrD,QAAM,UAAU,IAAI,KAAK;AACzB,MAAI,QAAQ,WAAW,GAAG,EAAG,QAAO,KAAK,MAAM,OAAO;AACtD,SAAO,QACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,EAChC,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;AAC1F,QAAM,UAAU,QACb,OAAO,CAAC,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,EACtE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtB,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,mCAAmC,MAAM,uCAAuC,QAAQ,MAAM,IAAI,QAAQ,MAAM,YAAY,QAAQ,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IAE5J;AAAA,EACF;AACF;AAEA,SAAS,UAAU,KAAwB,cAAkC;AAC3E,QAAM,kBAAkB,IAAI,oBAAoB;AAChD,QAAM,kBAAkB,IAAI,oBAAoB;AAChD,QAAM,OAAwB;AAAA,IAC5B,IAAI,IAAI;AAAA,IACR,UAAU,IAAI,YAAY;AAAA,IAC1B,UAAU,IAAI;AAAA,IACd;AAAA,IACA;AAAA,IACA,WAAW,IAAI,cAAc;AAAA,IAC7B,QAAQ,IAAI,UAAU;AAAA,IACtB,mBAAmB,IAAI;AAAA,IACvB,qBAAqB,IAAI;AAAA,IACzB,SAAS,eAAe,4BAA4B;AAAA,EACtD;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,IAAI;AAAA,IACN,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAkC;AAClD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAAU;AAC1C,UAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,wDAAmD;AAAA,EACrG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;AACpG,MAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,UAAU,KAAK,YAAY,CAAC;AAC1D,MAAI,KAAK,MAAO,SAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK,KAAK;AACxE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,CAAC,EAAE;AACpG,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;AAC5D,UAAQ,KAAK,iEAA4D,QAAQ,MAAM,mBAAmB;AAC1G,SAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;AACtF,QAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;AACxD,wBAAsB,SAAS,IAAI;AACnC,SAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;AACrE,QAAM,WAAW,KAAK;AACtB,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,gCAAgC;AAC/F,SAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,EACtC,WAAW,cAAc,QAAQ,EACjC,WAAW,sBAAsB,KAAK,eAAe,EACrD,WAAW,sBAAsB,KAAK,eAAe,EACrD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,EAClE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;AACzE,QAAM,aAAuB,CAAC;AAC9B,aAAW,KAAK,QAAQ,SAAS,+BAA+B,EAAG,YAAW,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC;AAC9F,aAAW,KAAK,QAAQ,SAAS,eAAe,EAAG,YAAW,KAAK,EAAE,CAAC,CAAC;AACvE,aAAW,KAAK,QAAQ,KAAK,CAAC;AAC9B,aAAW,aAAa,YAAY;AAClC,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,SAAS;AAAA,IAC/B,QAAQ;AACN;AAAA,IACF;AACA,UAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;AACjF,UAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;AAClF,QAAI,CAAC,OAAO,SAAS,CAAC,EAAG;AACzB,QAAI,IAAI,EAAG,OAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,CAAC,EAAE;AAChG,UAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;AAClD,QAAI,QAAQ,EAAG,OAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,CAAC,EAAE;AAChH,WAAO,EAAE,OAAO,KAAK,OAAO;AAAA,EAC9B;AACA,QAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,CAAC,EAAE;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;AAC5F,MAAI,CAAC,KAAK,kBAAmB,OAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,8BAA8B;AAC3G,QAAM,SAAS,aAAa;AAC5B,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,IAC5D,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,IACrF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,OAAO;AAAA,MACd,aAAa;AAAA,MACb,UAAU;AAAA,QACR,EAAE,MAAM,UAAU,SAAS,KAAK,kBAAkB;AAAA,QAClD,EAAE,MAAM,QAAQ,SAAS,aAAa,MAAM,QAAQ,EAAE;AAAA,MACxD;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAC7G,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,MAAI,OAAO,YAAY,SAAU,OAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAC5I,QAAM,EAAE,OAAO,IAAI,IAAI,gBAAgB,OAAO;AAC9C,SAAO;AAAA,IACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,GAAG;AAAA,IAC5E;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,UAAU,YAAY,OAAO,OAAO,IAAI,CAAC;AAAA,EAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;AAC5C,SAAO,MAAM,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;AACzE,QAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;AACzE,QAAM,WAAW,cAAc,QAAQ;AACvC,QAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;AACnE,SAAO;AAAA,IACL,UAAU,UAAU;AAAA,IACpB;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,SAAS,CAAC;AAAA,EAC3E;AACF;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,8BAA8B;AAE/D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,eAAe,UAAU,cAAc;AAC7C;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,mBAAa;AACb,YAAM,eAAe,MAAM,WAAW;AACtC,YAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;AAAA,IAC5C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,sEAAsE;AACjG,aAAO,kBAAkB,MAAM,IAAI;AAAA,IACrC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,KAAK,mBAAmB,KAAK,mBAAmB;AAAA,IACzD;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI,aAAc,QAAO,aAAa,MAAM,QAAQ;AACpD,UAAI,SAAS,KAAK,EAAE,WAAW,GAAG;AAChC,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,UAAU,QAAQ,eAAe,CAAC;AAAA,QACnG;AAAA,MACF;AACA,aAAO,iBAAiB,MAAM,QAAQ;AAAA,IACxC;AAAA,EACF;AACF;","names":[]}