@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/chunk-TSWPNOYM.js

@@ -0,0 +1,147 @@
+import {
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  selectTasks,
+  stringFrom
+} from "./chunk-X3BTXCJ4.js";
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/nomiracl.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "nomiracl.json");
+var dataFile = () => process.env.NOMIRACL_DATA_FILE;
+function passagesFrom(value, relevant) {
+  return contextsFrom(value).map((context) => ({ ...context, relevant }));
+}
+function answerableFrom(raw, contexts) {
+  const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant;
+  if (typeof direct === "boolean") return direct;
+  if (typeof direct === "number") return direct > 0;
+  if (typeof direct === "string") {
+    const normalized = direct.toLowerCase();
+    if (["true", "t", "relevant", "answerable", "positive", "1", "yes"].includes(normalized)) return true;
+    if (["false", "f", "non-relevant", "non_relevant", "unanswerable", "negative", "0", "no"].includes(normalized)) {
+      return false;
+    }
+  }
+  return contexts.some((context) => context.relevant === true);
+}
+function rowToTask(raw, index) {
+  if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`);
+  const query = firstString(raw, ["query", "question"]);
+  if (!query) throw new Error(`NoMIRACL row ${index} missing query`);
+  const labelledPassages = [
+    ...passagesFrom(raw.positive_passages, true),
+    ...passagesFrom(raw.negative_passages, false)
+  ];
+  const contexts = labelledPassages.length > 0 ? labelledPassages : contextsFrom(raw.passages).length > 0 ? contextsFrom(raw.passages) : contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.documents);
+  const answerable = answerableFrom(raw, contexts);
+  const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? "unknown";
+  const subset = stringFrom(raw.subset) ?? (answerable ? "relevant" : "non-relevant");
+  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}`;
+  const meta = {
+    benchmark: "nomiracl",
+    query,
+    answerable,
+    language,
+    subset,
+    contexts
+  };
+  return {
+    id,
+    split: stringFrom(raw.split) ?? language,
+    prompt: [
+      "Classify this NoMIRACL RAG relevance case.",
+      "Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.",
+      "",
+      `Query: ${query}`,
+      contexts.length > 0 ? `
+Passages:
+${contextBlock(contexts)}` : void 0
+    ].filter(Boolean).join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.answerable !== "boolean") {
+    throw new Error(`NoMIRACL task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function parseClassification(artifact) {
+  const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+  if (/\bunanswerable\b|\bnot answerable\b|\bno answer\b|\bnegative\b|\bfalse\b/.test(normalized)) {
+    return false;
+  }
+  if (/\banswerable\b|\brelevant\b|\bpositive\b|\btrue\b|\byes\b/.test(normalized)) return true;
+  return null;
+}
+async function loadRows(path) {
+  const rows = await readJsonRows(path);
+  if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`);
+  return rows;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
+  return selectTasks(rows.map(rowToTask), opts, "NoMIRACL");
+}
+function createNoMiraclAdapter() {
+  const fixturesMode = process.env.NOMIRACL_FIXTURES === "1";
+  return {
+    name: "nomiracl",
+    output: ragAnswerOutput,
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8");
+        return;
+      }
+      const path = dataFile();
+      if (!path) {
+        throw new Error(
+          "NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing."
+        );
+      }
+      await loadRows(path);
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const path = dataFile();
+      if (!path) throw new Error("NOMIRACL_DATA_FILE is required to load NoMIRACL tasks");
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, "NoMIRACL");
+    },
+    async goldArtifact(task) {
+      return readMeta(task).answerable ? "ANSWERABLE" : "UNANSWERABLE";
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const got = parseClassification(artifact);
+      const resolved = got === meta.answerable;
+      return {
+        resolved,
+        score: resolved ? 1 : 0,
+        detail: JSON.stringify({
+          benchmark: meta.benchmark,
+          language: meta.language,
+          subset: meta.subset,
+          expected: meta.answerable ? "ANSWERABLE" : "UNANSWERABLE",
+          got: got === null ? null : got ? "ANSWERABLE" : "UNANSWERABLE",
+          contextCount: meta.contexts.length
+        })
+      };
+    }
+  };
+}
+
+export {
+  createNoMiraclAdapter
+};
+//# sourceMappingURL=chunk-TSWPNOYM.js.map

package/dist/chunk-TSWPNOYM.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/nomiracl.ts"],"sourcesContent":["/**\n * NoMIRACL adapter.\n *\n * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does\n * not generate an answer here; it classifies whether the supplied passages\n * contain enough evidence to answer the query. This directly measures false\n * positive / false negative behavior for RAG abstention.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'nomiracl.json')\n\ninterface NoMiraclMeta {\n  benchmark: 'nomiracl'\n  query: string\n  answerable: boolean\n  language: string\n  subset: string\n  contexts: RagContext[]\n}\n\nconst dataFile = (): string | undefined => process.env.NOMIRACL_DATA_FILE\n\nfunction passagesFrom(value: unknown, relevant: boolean): RagContext[] {\n  return contextsFrom(value).map((context) => ({ ...context, relevant }))\n}\n\nfunction answerableFrom(raw: Record<string, unknown>, contexts: readonly RagContext[]): boolean {\n  const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant\n  if (typeof direct === 'boolean') return direct\n  if (typeof direct === 'number') return direct > 0\n  if (typeof direct === 'string') {\n    const normalized = direct.toLowerCase()\n    if (['true', 't', 'relevant', 'answerable', 'positive', '1', 'yes'].includes(normalized)) return true\n    if (['false', 'f', 'non-relevant', 'non_relevant', 'unanswerable', 'negative', '0', 'no'].includes(normalized)) {\n      return false\n    }\n  }\n  return contexts.some((context) => context.relevant === true)\n}\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`)\n  const query = firstString(raw, ['query', 'question'])\n  if (!query) throw new Error(`NoMIRACL row ${index} missing query`)\n  const labelledPassages = [\n    ...passagesFrom(raw.positive_passages, true),\n    ...passagesFrom(raw.negative_passages, false),\n  ]\n  const contexts =\n    labelledPassages.length > 0\n      ? labelledPassages\n      : contextsFrom(raw.passages).length > 0\n      ? contextsFrom(raw.passages)\n      : contextsFrom(raw.contexts).length > 0\n        ? contextsFrom(raw.contexts)\n        : contextsFrom(raw.documents)\n  const answerable = answerableFrom(raw, contexts)\n  const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? 'unknown'\n  const subset = stringFrom(raw.subset) ?? (answerable ? 'relevant' : 'non-relevant')\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}`\n  const meta: NoMiraclMeta = {\n    benchmark: 'nomiracl',\n    query,\n    answerable,\n    language,\n    subset,\n    contexts,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? language,\n    prompt: [\n      'Classify this NoMIRACL RAG relevance case.',\n      'Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.',\n      '',\n      `Query: ${query}`,\n      contexts.length > 0 ? `\\nPassages:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): NoMiraclMeta {\n  const md = task.metadata\n  if (!md || typeof md.answerable !== 'boolean') {\n    throw new Error(`NoMIRACL task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as NoMiraclMeta\n}\n\nfunction parseClassification(artifact: string): boolean | null {\n  const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()\n  if (/\\bunanswerable\\b|\\bnot answerable\\b|\\bno answer\\b|\\bnegative\\b|\\bfalse\\b/.test(normalized)) {\n    return false\n  }\n  if (/\\banswerable\\b|\\brelevant\\b|\\bpositive\\b|\\btrue\\b|\\byes\\b/.test(normalized)) return true\n  return null\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'NoMIRACL')\n}\n\nexport function createNoMiraclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.NOMIRACL_FIXTURES === '1'\n\n  return {\n    name: 'nomiracl',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('NOMIRACL_DATA_FILE is required to load NoMIRACL tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'NoMIRACL')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return readMeta(task).answerable ? 'ANSWERABLE' : 'UNANSWERABLE'\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const got = parseClassification(artifact)\n      const resolved = got === meta.answerable\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          benchmark: meta.benchmark,\n          language: meta.language,\n          subset: meta.subset,\n          expected: meta.answerable ? 'ANSWERABLE' : 'UNANSWERABLE',\n          got: got === null ? null : got ? 'ANSWERABLE' : 'UNANSWERABLE',\n          contextCount: meta.contexts.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AASA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAerB,IAAM,WAAW,KAAK,WAAW,YAAY,eAAe;AAW5D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,aAAa,OAAgB,UAAiC;AACrE,SAAO,aAAa,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,GAAG,SAAS,SAAS,EAAE;AACxE;AAEA,SAAS,eAAe,KAA8B,UAA0C;AAC9F,QAAM,SAAS,IAAI,cAAc,IAAI,iBAAiB,IAAI,cAAc,IAAI,SAAS,IAAI;AACzF,MAAI,OAAO,WAAW,UAAW,QAAO;AACxC,MAAI,OAAO,WAAW,SAAU,QAAO,SAAS;AAChD,MAAI,OAAO,WAAW,UAAU;AAC9B,UAAM,aAAa,OAAO,YAAY;AACtC,QAAI,CAAC,QAAQ,KAAK,YAAY,cAAc,YAAY,KAAK,KAAK,EAAE,SAAS,UAAU,EAAG,QAAO;AACjG,QAAI,CAAC,SAAS,KAAK,gBAAgB,gBAAgB,gBAAgB,YAAY,KAAK,IAAI,EAAE,SAAS,UAAU,GAAG;AAC9G,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO,SAAS,KAAK,CAAC,YAAY,QAAQ,aAAa,IAAI;AAC7D;AAEA,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,oBAAoB;AAC7E,QAAM,QAAQ,YAAY,KAAK,CAAC,SAAS,UAAU,CAAC;AACpD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,gBAAgB,KAAK,gBAAgB;AACjE,QAAM,mBAAmB;AAAA,IACvB,GAAG,aAAa,IAAI,mBAAmB,IAAI;AAAA,IAC3C,GAAG,aAAa,IAAI,mBAAmB,KAAK;AAAA,EAC9C;AACA,QAAM,WACJ,iBAAiB,SAAS,IACtB,mBACA,aAAa,IAAI,QAAQ,EAAE,SAAS,IACpC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,QAAQ,EAAE,SAAS,IAClC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,SAAS;AAClC,QAAM,aAAa,eAAe,KAAK,QAAQ;AAC/C,QAAM,WAAW,WAAW,IAAI,IAAI,KAAK,WAAW,IAAI,QAAQ,KAAK;AACrE,QAAM,SAAS,WAAW,IAAI,MAAM,MAAM,aAAa,aAAa;AACpE,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,YAAY,KAAK;AAC9E,QAAM,OAAqB;AAAA,IACzB,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,KAAK;AAAA,MACf,SAAS,SAAS,IAAI;AAAA;AAAA,EAAgB,aAAa,QAAQ,CAAC,KAAK;AAAA,IACnE,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,eAAe,WAAW;AAC7C,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,wDAAmD;AAAA,EAC7F;AACA,SAAO;AACT;AAEA,SAAS,oBAAoB,UAAkC;AAC7D,QAAM,aAAa,SAAS,YAAY,EAAE,QAAQ,eAAe,GAAG,EAAE,KAAK;AAC3E,MAAI,2EAA2E,KAAK,UAAU,GAAG;AAC/F,WAAO;AAAA,EACT;AACA,MAAI,4DAA4D,KAAK,UAAU,EAAG,QAAO;AACzF,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,wBAAwB,IAAI,EAAE;AACrE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,mBAAmB;AACvF,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,UAAU;AAC1D;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,uDAAuD;AAClF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,UAAU;AAAA,IAC5E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,SAAS,IAAI,EAAE,aAAa,eAAe;AAAA,IACpD;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAM,oBAAoB,QAAQ;AACxC,YAAM,WAAW,QAAQ,KAAK;AAC9B,aAAO;AAAA,QACL;AAAA,QACA,OAAO,WAAW,IAAI;AAAA,QACtB,QAAQ,KAAK,UAAU;AAAA,UACrB,WAAW,KAAK;AAAA,UAChB,UAAU,KAAK;AAAA,UACf,QAAQ,KAAK;AAAA,UACb,UAAU,KAAK,aAAa,eAAe;AAAA,UAC3C,KAAK,QAAQ,OAAO,OAAO,MAAM,eAAe;AAAA,UAChD,cAAc,KAAK,SAAS;AAAA,QAC9B,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-UAIOHCUK.js

@@ -0,0 +1,27 @@
+import {
+  createTauBenchAdapter
+} from "./chunk-R36V2VP7.js";
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/tau3-banking.ts
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "tau3-banking.json");
+function createTau3BankingAdapter() {
+  return createTauBenchAdapter({
+    name: "tau3-banking",
+    fixturePath: FIXTURES,
+    fixturesEnv: "TAU3_FIXTURES",
+    dirEnv: "TAU3_BENCH_DIR",
+    domainEnv: "TAU3_DOMAIN",
+    defaultDomain: "banking_knowledge",
+    taskIntro: "Run this tau3 banking task in the official tau3 knowledge benchmark.",
+    installHint: "clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench."
+  });
+}
+
+export {
+  createTau3BankingAdapter
+};
+//# sourceMappingURL=chunk-UAIOHCUK.js.map

package/dist/chunk-UAIOHCUK.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/tau3-banking.ts"],"sourcesContent":["/**\n * tau3-banking adapter.\n *\n * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with\n * package namespace `tau2` and a new `banking_knowledge` domain. This adapter\n * reuses the shared tau reward recomputation path and only changes the default\n * domain/env names.\n */\n\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport { createTauBenchAdapter } from './tau-bench-shared'\nimport type { BenchmarkAdapter } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'tau3-banking.json')\n\nexport function createTau3BankingAdapter(): BenchmarkAdapter {\n  return createTauBenchAdapter({\n    name: 'tau3-banking',\n    fixturePath: FIXTURES,\n    fixturesEnv: 'TAU3_FIXTURES',\n    dirEnv: 'TAU3_BENCH_DIR',\n    domainEnv: 'TAU3_DOMAIN',\n    defaultDomain: 'banking_knowledge',\n    taskIntro: 'Run this tau3 banking task in the official tau3 knowledge benchmark.',\n    installHint:\n      'clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench.',\n  })\n}\n"],"mappings":";;;;;;;;AASA,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEzD,SAAS,2BAA6C;AAC3D,SAAO,sBAAsB;AAAA,IAC3B,MAAM;AAAA,IACN,aAAa;AAAA,IACb,aAAa;AAAA,IACb,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,eAAe;AAAA,IACf,WAAW;AAAA,IACX,aACE;AAAA,EACJ,CAAC;AACH;","names":[]}

package/dist/chunk-UPAMRDX4.js

@@ -0,0 +1,233 @@
+// src/benchmarks/finsearchcomp.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+import { fileURLToPath } from "url";
+var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+var FIXTURES = join(BENCH_ROOT, "fixtures", "finsearchcomp.json");
+var DATASET_URL = "https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json";
+var SUPPORTED_TYPES = /* @__PURE__ */ new Set(["T2", "T3"]);
+var WORKER_CONTRACT = [
+  "",
+  "Research this financial question using live web/market sources and answer it.",
+  "State your final answer explicitly, with the exact units and precision the question requests.",
+  "Respect any tolerance the question implies; show the value you settled on, not just intermediate work."
+].join("\n");
+function parseTaskType(promptId) {
+  const m = promptId.match(/^\((T[123])\)/);
+  if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`);
+  return m[1];
+}
+function parseRegion(label) {
+  const m = label.match(/\(([^()]+)\)\s*$/);
+  return m ? m[1].trim() : "";
+}
+function recordToTask(rec) {
+  const taskType = parseTaskType(rec.prompt_id);
+  const meta = {
+    promptId: rec.prompt_id,
+    label: rec.label,
+    taskType,
+    region: parseRegion(rec.label),
+    responseReference: rec.response_reference,
+    judgePromptTemplate: rec.judge_prompt_template,
+    judgeSystemPrompt: rec.judge_system_prompt,
+    rawPrompt: rec.prompt
+  };
+  return {
+    id: rec.prompt_id,
+    split: taskType,
+    prompt: rec.prompt + WORKER_CONTRACT,
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.responseReference !== "string" || typeof md.judgePromptTemplate !== "string" || typeof md.judgeSystemPrompt !== "string" || typeof md.rawPrompt !== "string") {
+    throw new Error(`FinSearchComp task ${task.id} missing judge metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function fillJudgePrompt(meta, response) {
+  return meta.judgePromptTemplate.replaceAll("{prompt}", meta.rawPrompt).replaceAll("{response_reference}", meta.responseReference).replaceAll("{response}", response);
+}
+function judgeRouter() {
+  const key = process.env.TANGLE_API_KEY;
+  if (!key) throw new Error("TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge");
+  const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
+  const baseUrl = process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1";
+  return { baseUrl, key, model };
+}
+function readAnswerScore(value) {
+  let v = value;
+  while (Array.isArray(v)) v = v[0];
+  if (v === null || v === void 0 || v === "null") {
+    return { resolved: false, score: 0, raw: value };
+  }
+  const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
+  if (!Number.isFinite(n)) {
+    throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`);
+  }
+  const resolved = n >= 1;
+  const score = n <= 0 ? 0 : n >= 1 ? 1 : n;
+  return { resolved, score, raw: value };
+}
+function parseJudgeOutput(content) {
+  const candidates = [];
+  for (const m of content.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)) candidates.push(m[1].trim());
+  for (const m of content.matchAll(/\{[^{}]*"answer_score"[\s\S]*?\}/g)) candidates.push(m[0]);
+  candidates.push(content.trim());
+  for (const c of candidates) {
+    let parsed;
+    try {
+      parsed = JSON.parse(c);
+    } catch {
+      continue;
+    }
+    if ("answer_score" in parsed) return readAnswerScore(parsed.answer_score);
+  }
+  throw new Error(
+    `FinSearchComp judge produced no parseable {"answer_score": \u2026}: ${content.slice(0, 400)}`
+  );
+}
+async function runRecordJudge(meta, response, router) {
+  const res = await fetch(`${router.baseUrl}/chat/completions`, {
+    method: "POST",
+    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
+    body: JSON.stringify({
+      model: router.model,
+      temperature: 0,
+      messages: [
+        { role: "system", content: meta.judgeSystemPrompt },
+        { role: "user", content: fillJudgePrompt(meta, response) }
+      ]
+    })
+  });
+  if (!res.ok) {
+    throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+  }
+  const body = await res.json();
+  const content = body.choices?.[0]?.message?.content;
+  if (typeof content !== "string") {
+    throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+  }
+  const { resolved, score, raw } = parseJudgeOutput(content);
+  return {
+    resolved,
+    score,
+    detail: JSON.stringify({
+      taskType: meta.taskType,
+      region: meta.region,
+      answerScore: raw,
+      judgeModel: router.model
+    })
+  };
+}
+function selectRecords(records, opts) {
+  let tasks = records.filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id))).map(recordToTask);
+  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((t) => want.has(t.id));
+  } else if (opts.limit !== void 0) {
+    tasks = balanceByType(tasks, opts.limit);
+  }
+  return tasks;
+}
+function balanceByType(tasks, limit) {
+  const queues = /* @__PURE__ */ new Map();
+  for (const t of tasks) {
+    const k = parseTaskType(t.id);
+    const q = queues.get(k);
+    if (q) q.push(t);
+    else queues.set(k, [t]);
+  }
+  const lanes = [...queues.values()];
+  const out = [];
+  for (let i = 0; out.length < limit; i++) {
+    const before = out.length;
+    for (const lane of lanes) {
+      if (i < lane.length) out.push(lane[i]);
+      if (out.length >= limit) break;
+    }
+    if (out.length === before) break;
+  }
+  return out;
+}
+async function loadFixtures(opts) {
+  const records = JSON.parse(await readFile(FIXTURES, "utf8"));
+  return selectRecords(records, opts);
+}
+async function fetchDataset() {
+  const res = await fetch(DATASET_URL);
+  if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`);
+  const data = await res.json();
+  if (!Array.isArray(data) || data.length === 0) {
+    throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`);
+  }
+  return data;
+}
+function createFinsearchcompAdapter() {
+  const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === "1";
+  return {
+    name: "finsearchcomp",
+    async preflight() {
+      judgeRouter();
+      console.warn(
+        "[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live market snapshot / akshare ground truth for the judge {ground_truth} slot. loadTasks supports T2 + T3 only; wire T1Seam to add T1."
+      );
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8").catch((err) => {
+          throw new Error(
+            `FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`
+          );
+        });
+        return;
+      }
+      const res = await fetch(DATASET_URL, { method: "HEAD" }).catch((err) => {
+        throw new Error(
+          `finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}
+Fix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`
+        );
+      });
+      if (!res.ok) {
+        throw new Error(
+          `finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      let records;
+      try {
+        records = await fetchDataset();
+      } catch (err) {
+        console.warn(
+          `[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`
+        );
+        return loadFixtures(opts);
+      }
+      return selectRecords(records, opts);
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      return meta.responseReference;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const response = artifact.trim();
+      if (response.length === 0) {
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({ taskType: meta.taskType, region: meta.region, reason: "empty answer" })
+        };
+      }
+      return runRecordJudge(meta, response, judgeRouter());
+    }
+  };
+}
+
+export {
+  createFinsearchcompAdapter
+};
+//# sourceMappingURL=chunk-UPAMRDX4.js.map

package/dist/chunk-UPAMRDX4.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/finsearchcomp.ts"],"sourcesContent":["/**\n * FinSearchComp adapter (randomtutu/FinSearchComp). Agentic financial search +\n * reasoning. Worker artifact = the agent's free-form final answer text (zh/en).\n *\n * Each record is SELF-CONTAINED and carries its OWN judge: a per-record\n * `judge_system_prompt` + a `judge_prompt_template` with {prompt},\n * {response_reference} (gold answer WITH embedded tolerance/scoring-criteria),\n * and {response} (the student answer) placeholders. We faithfully REPLICATE the\n * benchmark's judge — fill the template, run it under the record's system prompt\n * via the pinned router (temperature 0), and parse the JSON the judge emits. The\n * judge returns an `answer_score` field; 1/positive = resolved, else not.\n * This matches the FinSearchComp leaderboard (Grok-4-web 68.9%).\n *\n * There is NO deterministic Tier-1 path here: the gold answer is prose with\n * region-specific units, embedded tolerances (\"允许1%的误差\"), and multi-point\n * scoring criteria. The benchmark intends its own LLM judge to be the arbiter,\n * so the judge IS the score — fail loud on unparseable judge output (never\n * default to resolved).\n *\n * SCOPE: T2 (Simple_Historical_Lookup) + T3 (Complex_Historical_Investigation)\n * are fully self-contained and supported. T1 (Time_Sensitive_Data_Fetching)\n * needs a live market snapshot / akshare ground truth to fill the judge's\n * {ground_truth} slot — it is SKIPPED in loadTasks and flagged in preflight.\n * See {@link T1Seam}.\n *\n * Requires for a live run: network access to the GitHub-hosted dataset JSON and\n * a TANGLE_API_KEY for the judge. For offline/CI verification, loadTasks falls back\n * to the committed fixtures (bench/fixtures/finsearchcomp.json) with an explicit\n * console.warn — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'finsearchcomp.json')\n\nconst DATASET_URL =\n  'https://raw.githubusercontent.com/randomtutu/FinSearchComp/main/data/finsearchcomp_data.json'\n\n/** Task-type prefix on prompt_id, e.g. \"(T2)Simple_Historical_Lookup_001\". */\ntype TaskType = 'T1' | 'T2' | 'T3'\n/** Only these are self-contained (gold + own judge, no external ground truth). */\nconst SUPPORTED_TYPES: ReadonlySet<TaskType> = new Set<TaskType>(['T2', 'T3'])\n\n/**\n * Worker contract appended to every task prompt. The judge is paraphrase- and\n * keyword-aware (\"答案为\", \"the answer is\"), so we do not impose a sentinel; we\n * pass the whole artifact through as {response}. We DO ask the worker to lead\n * with its final value so a truncated artifact still carries the answer.\n */\nconst WORKER_CONTRACT = [\n  '',\n  'Research this financial question using live web/market sources and answer it.',\n  'State your final answer explicitly, with the exact units and precision the question requests.',\n  'Respect any tolerance the question implies; show the value you settled on, not just intermediate work.',\n].join('\\n')\n\n/**\n * Typed seam for T1 (Time_Sensitive_Data_Fetching). T1 records score against a\n * live market snapshot the judge consumes via a {ground_truth} template slot;\n * the adapter does NOT fabricate that snapshot. Wiring T1 means supplying a\n * resolver that fetches the akshare/market value for the record at judge time\n * and filling {ground_truth}. Until then T1 is excluded from loadTasks.\n */\nexport interface T1Seam {\n  promptId: string\n  /** Resolver that returns the real-time ground-truth block for the {ground_truth} slot. */\n  resolveGroundTruth(promptId: string): Promise<string>\n}\n\ninterface FinSearchRecord {\n  prompt_id: string\n  prompt: string\n  response_reference: string\n  judge_prompt_template: string\n  judge_system_prompt: string\n  label: string\n}\n\ninterface FinSearchMeta {\n  promptId: string\n  label: string\n  taskType: TaskType\n  region: string\n  responseReference: string\n  judgePromptTemplate: string\n  judgeSystemPrompt: string\n  rawPrompt: string\n}\n\n/** \"(T2)Simple_Historical_Lookup_001\" → \"T2\"; throws on an unknown/missing prefix. */\nfunction parseTaskType(promptId: string): TaskType {\n  const m = promptId.match(/^\\((T[123])\\)/)\n  if (!m) throw new Error(`FinSearchComp prompt_id has no (T1|T2|T3) prefix: ${JSON.stringify(promptId)}`)\n  return m[1] as TaskType\n}\n\n/** \"Simple_Historical_Lookup(Greater China)\" → \"Greater China\"; '' when absent. */\nfunction parseRegion(label: string): string {\n  const m = label.match(/\\(([^()]+)\\)\\s*$/)\n  return m ? m[1].trim() : ''\n}\n\nfunction recordToTask(rec: FinSearchRecord): BenchTask {\n  const taskType = parseTaskType(rec.prompt_id)\n  const meta: FinSearchMeta = {\n    promptId: rec.prompt_id,\n    label: rec.label,\n    taskType,\n    region: parseRegion(rec.label),\n    responseReference: rec.response_reference,\n    judgePromptTemplate: rec.judge_prompt_template,\n    judgeSystemPrompt: rec.judge_system_prompt,\n    rawPrompt: rec.prompt,\n  }\n  return {\n    id: rec.prompt_id,\n    split: taskType,\n    prompt: rec.prompt + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinSearchMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.responseReference !== 'string' ||\n    typeof md.judgePromptTemplate !== 'string' ||\n    typeof md.judgeSystemPrompt !== 'string' ||\n    typeof md.rawPrompt !== 'string'\n  ) {\n    throw new Error(`FinSearchComp task ${task.id} missing judge metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinSearchMeta\n}\n\n/** Fill the per-record judge_prompt_template. {ground_truth} is T1-only and unused for T2/T3. */\nfunction fillJudgePrompt(meta: FinSearchMeta, response: string): string {\n  return meta.judgePromptTemplate\n    .replaceAll('{prompt}', meta.rawPrompt)\n    .replaceAll('{response_reference}', meta.responseReference)\n    .replaceAll('{response}', response)\n}\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the FinSearchComp per-record LLM judge')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/**\n * The judge emits a JSON object with an `answer_score`. In this dataset it is a\n * flat integer ({\"answer_score\": 1}); we also accept the nested-array form\n * ([[1]] → read [0][0]) the upstream scorer can produce. 1/positive = resolved.\n * A \"null\" sentinel (judge could not obtain ground truth) is NOT a pass.\n */\nfunction readAnswerScore(value: unknown): { resolved: boolean; score: number; raw: unknown } {\n  let v: unknown = value\n  // unwrap nested-array form: [[s]] / [s]\n  while (Array.isArray(v)) v = v[0]\n  if (v === null || v === undefined || v === 'null') {\n    return { resolved: false, score: 0, raw: value }\n  }\n  const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v) : NaN\n  if (!Number.isFinite(n)) {\n    throw new Error(`FinSearchComp judge answer_score not numeric: ${JSON.stringify(value)}`)\n  }\n  // Their scale: 1 = correct, 0 = wrong. Treat any positive as resolved; clamp to 0..1.\n  const resolved = n >= 1\n  const score = n <= 0 ? 0 : n >= 1 ? 1 : n\n  return { resolved, score, raw: value }\n}\n\n/** Extract the first JSON object carrying `answer_score` from judge output; throw if none. */\nfunction parseJudgeOutput(content: string): { resolved: boolean; score: number; raw: unknown } {\n  // Prefer a fenced block, then fall back to scanning for a {...} with answer_score.\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[^{}]*\"answer_score\"[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const c of candidates) {\n    let parsed: { answer_score?: unknown }\n    try {\n      parsed = JSON.parse(c) as { answer_score?: unknown }\n    } catch {\n      continue\n    }\n    if ('answer_score' in parsed) return readAnswerScore(parsed.answer_score)\n  }\n  throw new Error(\n    `FinSearchComp judge produced no parseable {\"answer_score\": …}: ${content.slice(0, 400)}`,\n  )\n}\n\n/** Run the record's own judge via the router. Fail loud on transport/parse errors. */\nasync function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise<BenchScore> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillJudgePrompt(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const { resolved, score, raw } = parseJudgeOutput(content)\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      taskType: meta.taskType,\n      region: meta.region,\n      answerScore: raw,\n      judgeModel: router.model,\n    }),\n  }\n}\n\nfunction selectRecords(records: FinSearchRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records\n    .filter((r) => SUPPORTED_TYPES.has(parseTaskType(r.prompt_id)))\n    .map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = balanceByType(tasks, opts.limit)\n  }\n  return tasks\n}\n\n/** Round-robin across task types (T2/T3) so a `limit` pulls a balanced mix\n *  rather than slicing the type-ordered dataset's first N (which yields all\n *  T2). Deterministic — preserves dataset order within each type. */\nfunction balanceByType(tasks: BenchTask[], limit: number): BenchTask[] {\n  const queues = new Map<TaskType, BenchTask[]>()\n  for (const t of tasks) {\n    const k = parseTaskType(t.id)\n    const q = queues.get(k)\n    if (q) q.push(t)\n    else queues.set(k, [t])\n  }\n  const lanes = [...queues.values()]\n  const out: BenchTask[] = []\n  for (let i = 0; out.length < limit; i++) {\n    const before = out.length\n    for (const lane of lanes) {\n      if (i < lane.length) out.push(lane[i])\n      if (out.length >= limit) break\n    }\n    if (out.length === before) break // all lanes exhausted\n  }\n  return out\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as FinSearchRecord[]\n  return selectRecords(records, opts)\n}\n\nasync function fetchDataset(): Promise<FinSearchRecord[]> {\n  const res = await fetch(DATASET_URL)\n  if (!res.ok) throw new Error(`FinSearchComp dataset HTTP ${res.status} fetching ${DATASET_URL}`)\n  const data = (await res.json()) as FinSearchRecord[]\n  if (!Array.isArray(data) || data.length === 0) {\n    throw new Error(`FinSearchComp dataset empty or not an array: ${DATASET_URL}`)\n  }\n  return data\n}\n\nexport function createFinsearchcompAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINSEARCHCOMP_FIXTURES === '1'\n\n  return {\n    name: 'finsearchcomp',\n\n    async preflight() {\n      // The per-record LLM judge is the arbiter in both modes — its router\n      // config must exist or no score can be produced.\n      judgeRouter()\n      console.warn(\n        '[finsearchcomp] T1 (Time_Sensitive_Data_Fetching) is NOT scored: it needs a live ' +\n          'market snapshot / akshare ground truth for the judge {ground_truth} slot. ' +\n          'loadTasks supports T2 + T3 only; wire T1Seam to add T1.',\n      )\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `FINSEARCHCOMP_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      const res = await fetch(DATASET_URL, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `finsearchcomp preflight failed reaching ${DATASET_URL}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set FINSEARCHCOMP_FIXTURES=1 ` +\n            `to run against the committed fixtures offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `finsearchcomp preflight: dataset HEAD ${res.status} for ${DATASET_URL}. ` +\n            `Set FINSEARCHCOMP_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let records: FinSearchRecord[]\n      try {\n        records = await fetchDataset()\n      } catch (err) {\n        console.warn(\n          `[finsearchcomp] live dataset fetch failed (${err instanceof Error ? err.message : err}); ` +\n            `falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRecords(records, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the record's response_reference fed back as the student\n      // answer, so verify-judge proves gold→resolved through the SAME per-record\n      // judge the real artifact takes. The reference may carry tolerance/criteria\n      // prose; the judge is built to read the value out of it.\n      const meta = readMeta(task)\n      return meta.responseReference\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const response = artifact.trim()\n      if (response.length === 0) {\n        // Fail-closed — the benchmark's own judges score an empty answer 0.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ taskType: meta.taskType, region: meta.region, reason: 'empty answer' }),\n        }\n      }\n      return runRecordJudge(meta, response, judgeRouter())\n    },\n  }\n}\n"],"mappings":";AA+BA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAG9B,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,WAAW,KAAK,YAAY,YAAY,oBAAoB;AAElE,IAAM,cACJ;AAKF,IAAM,kBAAyC,oBAAI,IAAc,CAAC,MAAM,IAAI,CAAC;AAQ7E,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAoCX,SAAS,cAAc,UAA4B;AACjD,QAAM,IAAI,SAAS,MAAM,eAAe;AACxC,MAAI,CAAC,EAAG,OAAM,IAAI,MAAM,qDAAqD,KAAK,UAAU,QAAQ,CAAC,EAAE;AACvG,SAAO,EAAE,CAAC;AACZ;AAGA,SAAS,YAAY,OAAuB;AAC1C,QAAM,IAAI,MAAM,MAAM,kBAAkB;AACxC,SAAO,IAAI,EAAE,CAAC,EAAE,KAAK,IAAI;AAC3B;AAEA,SAAS,aAAa,KAAiC;AACrD,QAAM,WAAW,cAAc,IAAI,SAAS;AAC5C,QAAM,OAAsB;AAAA,IAC1B,UAAU,IAAI;AAAA,IACd,OAAO,IAAI;AAAA,IACX;AAAA,IACA,QAAQ,YAAY,IAAI,KAAK;AAAA,IAC7B,mBAAmB,IAAI;AAAA,IACvB,qBAAqB,IAAI;AAAA,IACzB,mBAAmB,IAAI;AAAA,IACvB,WAAW,IAAI;AAAA,EACjB;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO;AAAA,IACP,QAAQ,IAAI,SAAS;AAAA,IACrB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAgC;AAChD,QAAM,KAAK,KAAK;AAChB,MACE,CAAC,MACD,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,wBAAwB,YAClC,OAAO,GAAG,sBAAsB,YAChC,OAAO,GAAG,cAAc,UACxB;AACA,UAAM,IAAI,MAAM,sBAAsB,KAAK,EAAE,8DAAyD;AAAA,EACxG;AACA,SAAO;AACT;AAGA,SAAS,gBAAgB,MAAqB,UAA0B;AACtE,SAAO,KAAK,oBACT,WAAW,YAAY,KAAK,SAAS,EACrC,WAAW,wBAAwB,KAAK,iBAAiB,EACzD,WAAW,cAAc,QAAQ;AACtC;AAQA,SAAS,cAA2B;AAClC,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,uEAAuE;AACjG,QAAM,QAAQ,QAAQ,IAAI,eAAe;AACzC,QAAM,UAAU,QAAQ,IAAI,eAAe;AAC3C,SAAO,EAAE,SAAS,KAAK,MAAM;AAC/B;AAQA,SAAS,gBAAgB,OAAoE;AAC3F,MAAI,IAAa;AAEjB,SAAO,MAAM,QAAQ,CAAC,EAAG,KAAI,EAAE,CAAC;AAChC,MAAI,MAAM,QAAQ,MAAM,UAAa,MAAM,QAAQ;AACjD,WAAO,EAAE,UAAU,OAAO,OAAO,GAAG,KAAK,MAAM;AAAA,EACjD;AACA,QAAM,IAAI,OAAO,MAAM,WAAW,IAAI,OAAO,MAAM,WAAW,OAAO,CAAC,IAAI;AAC1E,MAAI,CAAC,OAAO,SAAS,CAAC,GAAG;AACvB,UAAM,IAAI,MAAM,iDAAiD,KAAK,UAAU,KAAK,CAAC,EAAE;AAAA,EAC1F;AAEA,QAAM,WAAW,KAAK;AACtB,QAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI;AACxC,SAAO,EAAE,UAAU,OAAO,KAAK,MAAM;AACvC;AAGA,SAAS,iBAAiB,SAAqE;AAE7F,QAAM,aAAuB,CAAC;AAC9B,aAAW,KAAK,QAAQ,SAAS,+BAA+B,EAAG,YAAW,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC;AAC9F,aAAW,KAAK,QAAQ,SAAS,mCAAmC,EAAG,YAAW,KAAK,EAAE,CAAC,CAAC;AAC3F,aAAW,KAAK,QAAQ,KAAK,CAAC;AAC9B,aAAW,KAAK,YAAY;AAC1B,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,CAAC;AAAA,IACvB,QAAQ;AACN;AAAA,IACF;AACA,QAAI,kBAAkB,OAAQ,QAAO,gBAAgB,OAAO,YAAY;AAAA,EAC1E;AACA,QAAM,IAAI;AAAA,IACR,uEAAkE,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA,EACzF;AACF;AAGA,eAAe,eAAe,MAAqB,UAAkB,QAA0C;AAC7G,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,IAC5D,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,IACrF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,OAAO;AAAA,MACd,aAAa;AAAA,MACb,UAAU;AAAA,QACR,EAAE,MAAM,UAAU,SAAS,KAAK,kBAAkB;AAAA,QAClD,EAAE,MAAM,QAAQ,SAAS,gBAAgB,MAAM,QAAQ,EAAE;AAAA,MAC3D;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,IAAI,MAAM,4BAA4B,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC/F;AACA,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,MAAI,OAAO,YAAY,UAAU;AAC/B,UAAM,IAAI,MAAM,oDAAoD,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC1G;AACA,QAAM,EAAE,UAAU,OAAO,IAAI,IAAI,iBAAiB,OAAO;AACzD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,KAAK,UAAU;AAAA,MACrB,UAAU,KAAK;AAAA,MACf,QAAQ,KAAK;AAAA,MACb,aAAa;AAAA,MACb,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AAEA,SAAS,cAAc,SAA4B,MAAgC;AACjF,MAAI,QAAQ,QACT,OAAO,CAAC,MAAM,gBAAgB,IAAI,cAAc,EAAE,SAAS,CAAC,CAAC,EAC7D,IAAI,YAAY;AACnB,MAAI,KAAK,MAAO,SAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,KAAK;AAClE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,cAAc,OAAO,KAAK,KAAK;AAAA,EACzC;AACA,SAAO;AACT;AAKA,SAAS,cAAc,OAAoB,OAA4B;AACrE,QAAM,SAAS,oBAAI,IAA2B;AAC9C,aAAW,KAAK,OAAO;AACrB,UAAM,IAAI,cAAc,EAAE,EAAE;AAC5B,UAAM,IAAI,OAAO,IAAI,CAAC;AACtB,QAAI,EAAG,GAAE,KAAK,CAAC;AAAA,QACV,QAAO,IAAI,GAAG,CAAC,CAAC,CAAC;AAAA,EACxB;AACA,QAAM,QAAQ,CAAC,GAAG,OAAO,OAAO,CAAC;AACjC,QAAM,MAAmB,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,SAAS,OAAO,KAAK;AACvC,UAAM,SAAS,IAAI;AACnB,eAAW,QAAQ,OAAO;AACxB,UAAI,IAAI,KAAK,OAAQ,KAAI,KAAK,KAAK,CAAC,CAAC;AACrC,UAAI,IAAI,UAAU,MAAO;AAAA,IAC3B;AACA,QAAI,IAAI,WAAW,OAAQ;AAAA,EAC7B;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AAC3D,SAAO,cAAc,SAAS,IAAI;AACpC;AAEA,eAAe,eAA2C;AACxD,QAAM,MAAM,MAAM,MAAM,WAAW;AACnC,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,8BAA8B,IAAI,MAAM,aAAa,WAAW,EAAE;AAC/F,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,MAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAAG;AAC7C,UAAM,IAAI,MAAM,gDAAgD,WAAW,EAAE;AAAA,EAC/E;AACA,SAAO;AACT;AAEO,SAAS,6BAA+C;AAC7D,QAAM,eAAe,QAAQ,IAAI,2BAA2B;AAE5D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAGhB,kBAAY;AACZ,cAAQ;AAAA,QACN;AAAA,MAGF;AACA,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI;AAAA,YACR,gCAAgC,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAClG;AAAA,QACF,CAAC;AACD;AAAA,MACF;AACA,YAAM,MAAM,MAAM,MAAM,aAAa,EAAE,QAAQ,OAAO,CAAC,EAAE,MAAM,CAAC,QAAQ;AACtE,cAAM,IAAI;AAAA,UACR,2CAA2C,WAAW,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA;AAAA,QAGrG;AAAA,MACF,CAAC;AACD,UAAI,CAAC,IAAI,IAAI;AACX,cAAM,IAAI;AAAA,UACR,yCAAyC,IAAI,MAAM,QAAQ,WAAW;AAAA,QAExE;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,UAAI;AACJ,UAAI;AACF,kBAAU,MAAM,aAAa;AAAA,MAC/B,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,8CAA8C,eAAe,QAAQ,IAAI,UAAU,GAAG,4CAC3C,QAAQ;AAAA,QACrD;AACA,eAAO,aAAa,IAAI;AAAA,MAC1B;AACA,aAAO,cAAc,SAAS,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAKlC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,KAAK;AAAA,IACd;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,WAAW,SAAS,KAAK;AAC/B,UAAI,SAAS,WAAW,GAAG;AAEzB,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU,EAAE,UAAU,KAAK,UAAU,QAAQ,KAAK,QAAQ,QAAQ,eAAe,CAAC;AAAA,QACjG;AAAA,MACF;AACA,aAAO,eAAe,MAAM,UAAU,YAAY,CAAC;AAAA,IACrD;AAAA,EACF;AACF;","names":[]}