@tangle-network/agent-bench 0.1.0 → 0.3.6

package/dist/chunk-R36V2VP7.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/tau-bench-shared.ts"],"sourcesContent":["/**\n * Shared tau-bench adapter spine.\n *\n * tau2 and tau3 live in the same upstream repository/package namespace today:\n * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation\n * for task loading and reward recomputation so the domain/version adapters only\n * choose env names, default domain, and fixture file.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nexport interface TauBenchConfig {\n  name: string\n  fixturePath: string\n  fixturesEnv: string\n  dirEnv: string\n  domainEnv: string\n  defaultDomain: string\n  installHint: string\n  taskIntro: string\n}\n\ninterface TauRow {\n  id: string\n  domain: string\n  user_scenario?: unknown\n  description?: unknown\n  evaluation_criteria?: unknown\n}\n\ninterface TauMeta {\n  taskId: string\n  domain: string\n  split?: string\n  userScenario?: unknown\n  description?: unknown\n  evaluationCriteria?: unknown\n}\n\nexport const tauResultsOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path|json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertPath(path: string, label: string, benchName: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction benchDir(config: TauBenchConfig): string | undefined {\n  return process.env[config.dirEnv]\n}\n\nfunction benchDomain(config: TauBenchConfig): string {\n  return process.env[config.domainEnv] ?? config.defaultDomain\n}\n\nfunction rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {\n  const meta: TauMeta = {\n    taskId: row.id,\n    domain: row.domain,\n    split,\n    userScenario: row.user_scenario,\n    description: row.description,\n    evaluationCriteria: row.evaluation_criteria,\n  }\n  return {\n    id: row.id,\n    split,\n    prompt: [\n      config.taskIntro,\n      `Run this task in the official ${row.domain} domain.`,\n      'The benchmark is a simulated multi-turn user/tool conversation.',\n      '',\n      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),\n      '',\n      'Return the path to the official tau results.json or trajectory file containing this task run.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask, benchName: string): TauMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {\n    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as TauMeta\n}\n\nfunction selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, config, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]\n  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts, config, opts.split)\n}\n\nasync function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const domain = benchDomain(config)\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ndomain = sys.argv[2]\nsplit = sys.argv[3] or None\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.registry import registry\nloader = registry.get_tasks_loader(domain)\ntasks = loader(split)\nrows = []\nfor task in tasks:\n    row = task.model_dump(mode=\"json\")\n    row[\"domain\"] = domain\n    rows.append(row)\nprint(json.dumps(rows))\n`\n  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])\n  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)\n}\n\nasync function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\nartifact = Path(sys.argv[3])\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.data_model.simulation import Results\nfrom tau2.scripts.evaluate_trajectories import compute_simulation_rewards\nresults = Results.load(artifact)\nupdated = compute_simulation_rewards(results)\nscores = []\nfor sim in updated.simulations:\n    if sim.task_id == task_id and sim.reward_info is not None:\n        scores.append(float(sim.reward_info.reward))\nif not scores:\n    raise SystemExit(f\"no scored simulations for task_id={task_id} in {artifact}\")\nprint(json.dumps({\"count\": len(scores), \"score\": sum(scores) / len(scores), \"scores\": scores}))\n`\n  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {\n  const fixturesMode = process.env[config.fixturesEnv] === '1'\n\n  return {\n    name: config.name,\n    output: tauResultsOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = benchDir(config)\n      if (!dir) {\n        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)\n      }\n      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)\n      await loadOfficialTasks(config, dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(config, opts)\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)\n      return loadOfficialTasks(config, dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)\n      const meta = readMeta(task, config.name)\n      const artifactPath = resolve(artifact.trim())\n      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)\n      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,UAAU,YAAY;AAC/B,SAAS,eAAe;AAiCjB,IAAM,mBAA0C;AAAA,EACrD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,2CAA2C,CAAC;AAC7E,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,WAAW,MAAc,OAAe,WAAkC;AACvF,MAAI;AACF,UAAM,KAAK,IAAI;AAAA,EACjB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,GAAG,SAAS,aAAa,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAC3G;AACF;AAEA,SAAS,SAAS,QAA4C;AAC5D,SAAO,QAAQ,IAAI,OAAO,MAAM;AAClC;AAEA,SAAS,YAAY,QAAgC;AACnD,SAAO,QAAQ,IAAI,OAAO,SAAS,KAAK,OAAO;AACjD;AAEA,SAAS,UAAU,KAAa,QAAwB,OAA2B;AACjF,QAAM,OAAgB;AAAA,IACpB,QAAQ,IAAI;AAAA,IACZ,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,cAAc,IAAI;AAAA,IAClB,aAAa,IAAI;AAAA,IACjB,oBAAoB,IAAI;AAAA,EAC1B;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,MACN,OAAO;AAAA,MACP,iCAAiC,IAAI,MAAM;AAAA,MAC3C;AAAA,MACA;AAAA,MACA,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAAgB,KAAK,UAAU,IAAI,iBAAiB,CAAC,GAAG,MAAM,CAAC;AAAA,MAC3G;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAiB,WAA4B;AAC7D,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,GAAG,WAAW,UAAU;AACzE,UAAM,IAAI,MAAM,GAAG,SAAS,SAAS,KAAK,EAAE,wDAAmD;AAAA,EACjG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAgB,MAAmB,QAAwB,OAA6B;AAC1G,MAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,UAAU,KAAK,QAAQ,KAAK,CAAC;AAC3D,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,GAAG,OAAO,IAAI,sBAAsB,KAAK,UAAU,IAAI,CAAC,EAAE;AAClG,SAAO;AACT;AAEA,eAAe,aAAa,QAAwB,MAAyC;AAC3F,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,OAAO,aAAa,MAAM,CAAC;AAClE,UAAQ,KAAK,IAAI,OAAO,IAAI,KAAK,OAAO,WAAW,qBAAgB,KAAK,MAAM,mBAAmB;AACjG,SAAO,WAAW,MAAM,MAAM,QAAQ,KAAK,KAAK;AAClD;AAEA,eAAe,kBAAkB,QAAwB,MAAc,MAAyC;AAC9G,QAAM,SAAS,YAAY,MAAM;AACjC,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,MAAM,QAAQ,KAAK,SAAS,EAAE,CAAC;AAC3E,SAAO,WAAW,KAAK,MAAM,MAAM,GAAe,MAAM,QAAQ,KAAK,KAAK;AAC5E;AAEA,eAAe,wBAAwB,MAAc,MAAe,cAAwD;AAC1H,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,MAAM,KAAK,QAAQ,YAAY,GAAG,CAAC;AAC/E,SAAO,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEO,SAAS,sBAAsB,QAA0C;AAC9E,QAAM,eAAe,QAAQ,IAAI,OAAO,WAAW,MAAM;AAEzD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,SAAS,MAAM;AAC3B,UAAI,CAAC,KAAK;AACR,cAAM,IAAI,MAAM,GAAG,OAAO,MAAM,sBAAsB,OAAO,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,WAAW,GAAG,GAAG,yBAAyB,gBAAgB,OAAO,IAAI;AAC3E,YAAM,kBAAkB,QAAQ,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IACnD;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,QAAQ,IAAI;AAClD,YAAM,MAAM,SAAS,MAAM;AAC3B,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,GAAG,OAAO,MAAM,iCAAiC,OAAO,IAAI,QAAQ;AAC9F,aAAO,kBAAkB,QAAQ,KAAK,IAAI;AAAA,IAC5C;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,MAAM,SAAS,MAAM;AAC3B,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,GAAG,OAAO,MAAM,yBAAyB,OAAO,IAAI,uBAAuB;AACrG,YAAM,OAAO,SAAS,MAAM,OAAO,IAAI;AACvC,YAAM,eAAe,QAAQ,SAAS,KAAK,CAAC;AAC5C,YAAM,WAAW,cAAc,mCAAmC,OAAO,IAAI;AAC7E,YAAM,SAAS,MAAM,wBAAwB,KAAK,MAAM,YAAY;AACpE,YAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;AAChE,aAAO;AAAA,QACL,UAAU,UAAU;AAAA,QACpB;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,KAAK,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,MAC1F;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-R67DFVLO.js

@@ -0,0 +1,142 @@
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringFrom
+} from "./chunk-X3BTXCJ4.js";
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/ragbench.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "ragbench.json");
+var dataFile = () => process.env.RAGBENCH_DATA_FILE;
+function rowToTask(raw, index) {
+  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`);
+  const query = firstString(raw, ["question", "query", "user_input", "prompt"]);
+  const goldAnswers = allStrings(raw, [
+    "response",
+    "responses",
+    "model_response",
+    "generated_response",
+    "reference",
+    "references",
+    "reference_answer",
+    "reference_answers",
+    "answer",
+    "answers",
+    "gold",
+    "gold_answer",
+    "ground_truth",
+    "expected_answer"
+  ]);
+  if (!query) throw new Error(`RAGBench row ${index} missing question/query`);
+  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`);
+  const contexts = contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.retrieved_contexts).length > 0 ? contextsFrom(raw.retrieved_contexts) : contextsFrom(raw.documents);
+  const traceLabels = {};
+  for (const key of [
+    "adherence",
+    "completeness",
+    "relevance",
+    "utilization",
+    "all_relevant_sentence_keys",
+    "all_utilized_sentence_keys"
+  ]) {
+    if (raw[key] !== void 0) traceLabels[key] = raw[key];
+  }
+  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset);
+  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`;
+  const meta = {
+    benchmark: "ragbench",
+    query,
+    goldAnswers,
+    contexts,
+    ...dataset ? { dataset } : {},
+    traceLabels
+  };
+  return {
+    id,
+    split: stringFrom(raw.split) ?? dataset ?? "ragbench",
+    prompt: [
+      "Answer this RAGBench question using the supplied retrieved context.",
+      "End with a single final line: `FINAL ANSWER: <answer>`.",
+      "",
+      `Question: ${query}`,
+      contexts.length > 0 ? `
+Retrieved context:
+${contextBlock(contexts)}` : void 0
+    ].filter(Boolean).join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.goldAnswers)) {
+    throw new Error(`RAGBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+async function loadRows(path) {
+  const rows = await readJsonRows(path);
+  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`);
+  return rows;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
+  return selectTasks(rows.map(rowToTask), opts, "RAGBench");
+}
+function createRagBenchAdapter() {
+  const fixturesMode = process.env.RAGBENCH_FIXTURES === "1";
+  return {
+    name: "ragbench",
+    output: ragAnswerOutput,
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8");
+        return;
+      }
+      const path = dataFile();
+      if (!path) {
+        throw new Error(
+          "RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing."
+        );
+      }
+      await loadRows(path);
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const path = dataFile();
+      if (!path) throw new Error("RAGBENCH_DATA_FILE is required to load RAGBench tasks");
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, "RAGBench");
+    },
+    async goldArtifact(task) {
+      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const score = scoreAnswerArtifact(artifact, meta.goldAnswers);
+      return answerScoreToBenchScore(score, {
+        benchmark: meta.benchmark,
+        dataset: meta.dataset ?? null,
+        traceLabels: meta.traceLabels,
+        contextCount: meta.contexts.length
+      });
+    }
+  };
+}
+
+export {
+  createRagBenchAdapter
+};
+//# sourceMappingURL=chunk-R67DFVLO.js.map

package/dist/chunk-R67DFVLO.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/ragbench.ts"],"sourcesContent":["/**\n * RAGBench-compatible adapter.\n *\n * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a\n * compatible table. Rows must carry a query and at least one reference answer.\n * Contexts, TRACe labels, and source metadata are preserved in task metadata\n * for diagnostics; the deterministic judge scores the worker's final answer\n * against the reference answer(s).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'ragbench.json')\n\ninterface RagBenchMeta {\n  benchmark: 'ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  dataset?: string\n  traceLabels: Record<string, unknown>\n}\n\nconst dataFile = (): string | undefined => process.env.RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'user_input', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'response',\n    'responses',\n    'model_response',\n    'generated_response',\n    'reference',\n    'references',\n    'reference_answer',\n    'reference_answers',\n    'answer',\n    'answers',\n    'gold',\n    'gold_answer',\n    'ground_truth',\n    'expected_answer',\n  ])\n  if (!query) throw new Error(`RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`)\n  const contexts =\n    contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.retrieved_contexts).length > 0\n        ? contextsFrom(raw.retrieved_contexts)\n        : contextsFrom(raw.documents)\n  const traceLabels: Record<string, unknown> = {}\n  for (const key of [\n    'adherence',\n    'completeness',\n    'relevance',\n    'utilization',\n    'all_relevant_sentence_keys',\n    'all_utilized_sentence_keys',\n  ]) {\n    if (raw[key] !== undefined) traceLabels[key] = raw[key]\n  }\n  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset)\n  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`\n  const meta: RagBenchMeta = {\n    benchmark: 'ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    ...(dataset ? { dataset } : {}),\n    traceLabels,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? dataset ?? 'ragbench',\n    prompt: [\n      'Answer this RAGBench question using the supplied retrieved context.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      contexts.length > 0 ? `\\nRetrieved context:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'RAGBench')\n}\n\nexport function createRagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 'ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('RAGBENCH_DATA_FILE is required to load RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        dataset: meta.dataset ?? null,\n        traceLabels: meta.traceLabels,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAUA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,eAAe;AAW5D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,oBAAoB;AAC7E,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,cAAc,QAAQ,CAAC;AAC5E,QAAM,cAAc,WAAW,KAAK;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,gBAAgB,KAAK,yBAAyB;AAC1E,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,2BAA2B;AAC9F,QAAM,WACJ,aAAa,IAAI,QAAQ,EAAE,SAAS,IAChC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,kBAAkB,EAAE,SAAS,IAC5C,aAAa,IAAI,kBAAkB,IACnC,aAAa,IAAI,SAAS;AAClC,QAAM,cAAuC,CAAC;AAC9C,aAAW,OAAO;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAAG;AACD,QAAI,IAAI,GAAG,MAAM,OAAW,aAAY,GAAG,IAAI,IAAI,GAAG;AAAA,EACxD;AACA,QAAM,UAAU,WAAW,IAAI,OAAO,KAAK,WAAW,IAAI,cAAc;AACxE,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,UAAU,KAAK,YAAY,KAAK;AAChF,QAAM,OAAqB;AAAA,IACzB,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;AAAA,IAC7B;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK,WAAW;AAAA,IAC3C,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAyB,aAAa,QAAQ,CAAC,KAAK;AAAA,IAC5E,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,wDAAmD;AAAA,EAC7F;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,wBAAwB,IAAI,EAAE;AACrE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,mBAAmB;AACvF,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,UAAU;AAC1D;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,uDAAuD;AAClF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,UAAU;AAAA,IAC5E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,WAAW;AAC5D,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,SAAS,KAAK,WAAW;AAAA,QACzB,aAAa,KAAK;AAAA,QAClB,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-SEVJPLZC.js

@@ -0,0 +1,260 @@
+// src/benchmarks/simpleqa.ts
+import { execFile } from "child_process";
+import { readFile } from "fs/promises";
+import { join } from "path";
+import { fileURLToPath } from "url";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+var PY = join(BENCH_ROOT, ".venv", "bin", "python");
+var FIXTURES = join(BENCH_ROOT, "fixtures", "simpleqa.json");
+var DATASET_URL = "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv";
+var FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+var WORKER_CONTRACT = [
+  "",
+  "Research the question using live web sources and answer it with a short, specific factual value.",
+  "Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.",
+  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
+  "The answer after the sentinel must be the bare value only (no explanation on that line).",
+  "If you do not know the answer, state that you do not know rather than guessing."
+].join("\n");
+async function py(script, args = []) {
+  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], {
+    maxBuffer: 1024 * 1024 * 256
+  });
+  return stdout;
+}
+function parseFinalAnswer(artifact) {
+  const lines = artifact.split(/\r?\n/);
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const line = lines[i] ?? "";
+    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
+    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim();
+  }
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const t = (lines[i] ?? "").trim();
+    if (t.length > 0) return t;
+  }
+  return "";
+}
+function parseCitations(artifact) {
+  const urls = /* @__PURE__ */ new Set();
+  for (const m of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) {
+    urls.add(m[0].replace(/[.,;]+$/, ""));
+  }
+  return [...urls];
+}
+var GRADER_PROMPT = (question, gold, predicted) => [
+  "Your job is to grade a predicted answer to a factual question against the single gold target answer.",
+  "Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.",
+  "First, follow these grading rules carefully.",
+  "",
+  "CORRECT:",
+  "  The predicted answer fully contains the gold target without contradicting it.",
+  "  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.",
+  '  Hedging that still commits to the gold value (e.g. "I think it is X" where X is the gold) is CORRECT.',
+  "  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.",
+  "  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.",
+  "",
+  "INCORRECT:",
+  "  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.",
+  "  A prediction that states a different factual value than the gold is INCORRECT.",
+  "",
+  "NOT_ATTEMPTED:",
+  "  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,",
+  "  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.",
+  "",
+  "Also note the following rules:",
+  "  Do not punish a prediction for omitting information that would not change whether the gold target is contained.",
+  "  Grade ONLY whether the gold value is present and uncontradicted \u2014 not the overall quality of the response.",
+  "",
+  `Question: ${question}`,
+  `Gold target: ${gold}`,
+  `Predicted answer: ${predicted}`,
+  "",
+  "Respond with ONLY a fenced JSON block and nothing else:",
+  "```json",
+  '{"grade": "CORRECT" | "INCORRECT" | "NOT_ATTEMPTED"}',
+  "```"
+].join("\n");
+function graderRouter() {
+  const key = process.env.TANGLE_API_KEY;
+  if (!key) throw new Error("TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)");
+  const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
+  const baseUrl = process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1";
+  return { baseUrl, key, model };
+}
+async function gradeAnswer(question, gold, predicted, router) {
+  const res = await fetch(`${router.baseUrl}/chat/completions`, {
+    method: "POST",
+    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
+    body: JSON.stringify({
+      model: router.model,
+      temperature: 0,
+      messages: [{ role: "user", content: GRADER_PROMPT(question, gold, predicted) }]
+    })
+  });
+  if (!res.ok) {
+    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+  }
+  const body = await res.json();
+  const content = body.choices?.[0]?.message?.content;
+  if (typeof content !== "string") {
+    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+  }
+  const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
+  const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`);
+  }
+  if (parsed.grade === "CORRECT" || parsed.grade === "INCORRECT" || parsed.grade === "NOT_ATTEMPTED") {
+    return parsed.grade;
+  }
+  throw new Error(
+    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`
+  );
+}
+function rowToTask(row, index) {
+  const meta = {
+    gold: row.answer,
+    goldSources: row.urls,
+    topic: row.topic,
+    answerType: row.answer_type,
+    question: row.problem
+  };
+  return {
+    id: `simpleqa-${index}`,
+    split: "test",
+    prompt: row.problem + WORKER_CONTRACT,
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.gold !== "string" || typeof md.question !== "string") {
+    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectTasks(tasks, opts) {
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    return tasks.filter((t) => want.has(t.id));
+  }
+  if (opts.limit !== void 0) return tasks.slice(0, opts.limit);
+  return tasks;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 \u2014 loading ${rows.length} committed fixtures (no network)`);
+  return selectTasks(rows.map(rowToTask), opts);
+}
+async function loadLive(opts) {
+  const script = `
+import csv, ast, json, io, sys
+from urllib.request import urlopen
+with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
+    text = resp.read().decode('utf-8')
+rows = list(csv.DictReader(io.StringIO(text)))
+out = []
+for r in rows:
+    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}
+    out.append({
+        'problem': r.get('problem', ''),
+        'answer': r.get('answer', ''),
+        'topic': str(meta.get('topic', '')),
+        'answer_type': str(meta.get('answer_type', '')),
+        'urls': list(meta.get('urls', [])),
+    })
+print(json.dumps(out))
+`;
+  const stdout = await py(script);
+  const rows = JSON.parse(stdout);
+  return selectTasks(rows.map(rowToTask), opts);
+}
+function createSimpleQaAdapter() {
+  const fixturesMode = process.env.SIMPLEQA_FIXTURES === "1";
+  return {
+    name: "simpleqa",
+    async preflight() {
+      graderRouter();
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8").catch((err) => {
+          throw new Error(
+            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`
+          );
+        });
+        return;
+      }
+      try {
+        await py(
+          `from urllib.request import urlopen
+with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
+    head = resp.read(64).decode('utf-8', 'replace')
+assert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]
+print('ok')`
+        );
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `simpleqa preflight failed: ${msg}
+Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; (2) ensure network access to ${DATASET_URL} ; or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      return loadLive(opts);
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const finalAnswer = parseFinalAnswer(artifact);
+      const citations = parseCitations(artifact);
+      if (finalAnswer.length === 0) {
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({
+            grade: "NOT_ATTEMPTED",
+            gradeLetter: "C",
+            reason: "no parseable answer",
+            gold: meta.gold,
+            topic: meta.topic,
+            answerType: meta.answerType,
+            citationCount: citations.length
+          })
+        };
+      }
+      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter());
+      const gradeLetter = grade === "CORRECT" ? "A" : grade === "INCORRECT" ? "B" : "C";
+      const resolved = grade === "CORRECT";
+      return {
+        resolved,
+        score: resolved ? 1 : 0,
+        detail: JSON.stringify({
+          grade,
+          gradeLetter,
+          gold: meta.gold,
+          predicted: finalAnswer,
+          topic: meta.topic,
+          answerType: meta.answerType,
+          citationCount: citations.length
+        })
+      };
+    }
+  };
+}
+
+export {
+  parseFinalAnswer,
+  parseCitations,
+  createSimpleQaAdapter
+};
+//# sourceMappingURL=chunk-SEVJPLZC.js.map

package/dist/chunk-SEVJPLZC.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";AA2BA,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AACzB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AACxC,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,IAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,IAAM,cACJ;AACF,IAAM,wBAAwB;AAG9B,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA,iDAAiD,qBAAqB;AAAA,EACtE;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;AACtE,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,IAAI,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IAClE,WAAW,OAAO,OAAO;AAAA,EAC3B,CAAC;AACD,SAAO;AACT;AAQO,SAAS,iBAAiB,UAA0B;AACzD,QAAM,QAAQ,SAAS,MAAM,OAAO;AACpC,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,OAAO,MAAM,CAAC,KAAK;AACzB,UAAM,MAAM,KAAK,YAAY,EAAE,QAAQ,qBAAqB;AAC5D,QAAI,QAAQ,GAAI,QAAO,KAAK,MAAM,MAAM,sBAAsB,MAAM,EAAE,KAAK;AAAA,EAC7E;AACA,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,KAAK,MAAM,CAAC,KAAK,IAAI,KAAK;AAChC,QAAI,EAAE,SAAS,EAAG,QAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAGO,SAAS,eAAe,UAA4B;AACzD,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,SAAS,SAAS,yBAAyB,GAAG;AAC5D,SAAK,IAAI,EAAE,CAAC,EAAE,QAAQ,WAAW,EAAE,CAAC;AAAA,EACtC;AACA,SAAO,CAAC,GAAG,IAAI;AACjB;AASA,IAAM,gBAAgB,CAAC,UAAkB,MAAc,cACrD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa,QAAQ;AAAA,EACrB,gBAAgB,IAAI;AAAA,EACpB,qBAAqB,SAAS;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAQb,SAAS,eAA6B;AACpC,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,6EAA6E;AACvG,QAAM,QAAQ,QAAQ,IAAI,eAAe;AACzC,QAAM,UAAU,QAAQ,IAAI,eAAe;AAC3C,SAAO,EAAE,SAAS,KAAK,MAAM;AAC/B;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;AAChB,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,IAC5D,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,IACrF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,OAAO;AAAA,MACd,aAAa;AAAA,MACb,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,cAAc,UAAU,MAAM,SAAS,EAAE,CAAC;AAAA,IAChF,CAAC;AAAA,EACH,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,IAAI,MAAM,wBAAwB,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC3F;AACA,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,MAAI,OAAO,YAAY,UAAU;AAC/B,UAAM,IAAI,MAAM,gDAAgD,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACtG;AACA,QAAM,SAAS,QAAQ,MAAM,8BAA8B;AAC3D,QAAM,OAAO,SAAS,OAAO,CAAC,IAAI,UAAU,KAAK,KAAK;AACtD,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;AAAA,EACzB,QAAQ;AACN,UAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACzG;AACA,MAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBAAiB;AAClG,WAAO,OAAO;AAAA,EAChB;AACA,QAAM,IAAI;AAAA,IACR,mEAAmE,KAAK,UAAU,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC;AAAA,EACzG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;AAC7D,QAAM,OAAqB;AAAA,IACzB,MAAM,IAAI;AAAA,IACV,aAAa,IAAI;AAAA,IACjB,OAAO,IAAI;AAAA,IACX,YAAY,IAAI;AAAA,IAChB,UAAU,IAAI;AAAA,EAChB;AACA,SAAO;AAAA,IACL,IAAI,YAAY,KAAK;AAAA,IACrB,OAAO;AAAA,IACP,QAAQ,IAAI,UAAU;AAAA,IACtB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAAU;AACzE,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,sEAAiE;AAAA,EAC3G;AACA,SAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;AACvE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,WAAO,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC3C;AACA,MAAI,KAAK,UAAU,OAAW,QAAO,MAAM,MAAM,GAAG,KAAK,KAAK;AAC9D,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,kCAAkC;AACtG,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;AAQA,eAAe,SAAS,MAAyC;AAC/D,QAAM,SAAS;AAAA;AAAA;AAAA,eAGF,KAAK,UAAU,WAAW,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAexC,QAAM,SAAS,MAAM,GAAG,MAAM;AAC9B,QAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAGhB,mBAAa;AACb,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI;AAAA,YACR,2BAA2B,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAC7F;AAAA,QACF,CAAC;AACD;AAAA,MACF;AACA,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,eACK,KAAK,UAAU,WAAW,CAAC;AAAA;AAAA;AAAA;AAAA,QAIlC;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,8BAA8B,GAAG;AAAA,qGAEC,WAAW;AAAA,QAE/C;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,aAAO,SAAS,IAAI;AAAA,IACtB;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,GAAG,qBAAqB,IAAI,KAAK,IAAI;AAAA,IAC9C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,cAAc,iBAAiB,QAAQ;AAC7C,YAAM,YAAY,eAAe,QAAQ;AAEzC,UAAI,YAAY,WAAW,GAAG;AAE5B,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU;AAAA,YACrB,OAAO;AAAA,YACP,aAAa;AAAA,YACb,QAAQ;AAAA,YACR,MAAM,KAAK;AAAA,YACX,OAAO,KAAK;AAAA,YACZ,YAAY,KAAK;AAAA,YACjB,eAAe,UAAU;AAAA,UAC3B,CAAC;AAAA,QACH;AAAA,MACF;AAEA,YAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;AACrF,YAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;AAC9E,YAAM,WAAW,UAAU;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,OAAO,WAAW,IAAI;AAAA,QACtB,QAAQ,KAAK,UAAU;AAAA,UACrB;AAAA,UACA;AAAA,UACA,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,YAAY,KAAK;AAAA,UACjB,eAAe,UAAU;AAAA,QAC3B,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}