@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-RCYQEFNX.js

@@ -1,30 +0,0 @@
-import {
-  createTauBenchAdapter,
-  tauResultsOutput
-} from "./chunk-IZ5M6OAC.js";
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/tau2-bench.ts
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "tau2-bench.json");
-var tau2ResultsOutput = tauResultsOutput;
-function createTau2BenchAdapter() {
-  return createTauBenchAdapter({
-    name: "tau2-bench",
-    fixturePath: FIXTURES,
-    fixturesEnv: "TAU2_FIXTURES",
-    dirEnv: "TAU2_BENCH_DIR",
-    domainEnv: "TAU2_DOMAIN",
-    defaultDomain: "retail",
-    taskIntro: "Run this tau2 task in the official tau2 text benchmark.",
-    installHint: "clone https://github.com/sierra-research/tau2-bench, install its deps in bench/.venv, and set TAU2_BENCH_DIR=/path/to/tau2-bench."
-  });
-}
-
-export {
-  tau2ResultsOutput,
-  createTau2BenchAdapter
-};
-//# sourceMappingURL=chunk-RCYQEFNX.js.map

package/dist/chunk-RCYQEFNX.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/tau2-bench.ts"],"sourcesContent":["/**\n * tau2-bench adapter (Sierra tau2/tau-bench successor).\n *\n * Worker artifact = a tau2 `results.json`/trajectory file. Judge = tau2's own\n * reward recomputation over that trajectory. A normal final-answer transcript is\n * not a valid artifact for this benchmark.\n */\n\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport { createTauBenchAdapter, tauResultsOutput } from './tau-bench-shared'\nimport type { BenchmarkAdapter } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'tau2-bench.json')\n\nexport const tau2ResultsOutput = tauResultsOutput\n\nexport function createTau2BenchAdapter(): BenchmarkAdapter {\n  return createTauBenchAdapter({\n    name: 'tau2-bench',\n    fixturePath: FIXTURES,\n    fixturesEnv: 'TAU2_FIXTURES',\n    dirEnv: 'TAU2_BENCH_DIR',\n    domainEnv: 'TAU2_DOMAIN',\n    defaultDomain: 'retail',\n    taskIntro: 'Run this tau2 task in the official tau2 text benchmark.',\n    installHint:\n      'clone https://github.com/sierra-research/tau2-bench, install its deps in bench/.venv, and set TAU2_BENCH_DIR=/path/to/tau2-bench.',\n  })\n}\n"],"mappings":";;;;;;;;;AAQA,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,iBAAiB;AAEvD,IAAM,oBAAoB;AAE1B,SAAS,yBAA2C;AACzD,SAAO,sBAAsB;AAAA,IAC3B,MAAM;AAAA,IACN,aAAa;AAAA,IACb,aAAa;AAAA,IACb,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,eAAe;AAAA,IACf,WAAW;AAAA,IACX,aACE;AAAA,EACJ,CAAC;AACH;","names":[]}

package/dist/chunk-RH5F53JT.js

@@ -1,182 +0,0 @@
-import {
-  benchRoot,
-  runVenvPython,
-  runVenvScriptStdin
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/dabstep.ts
-import { join } from "path";
-import { access, readFile, stat } from "fs/promises";
-var FIXTURES = join(benchRoot, "fixtures", "dabstep.json");
-var DEFAULT_SPLIT = "easy";
-var dabstepDir = () => process.env.DABSTEP_DIR;
-var gradeFile = (dir) => join(dir, "grade.py");
-var resourceRoot = (dir) => join(dir, "files");
-async function assertFile(path, label) {
-  try {
-    await access(path);
-  } catch (err) {
-    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-async function assertOfficialFiles(dir, split) {
-  await assertFile(join(dir, "dataset.csv"), "released dataset.csv");
-  await assertFile(join(dir, "splits", `${split}.txt`), `${split} split file`);
-  await assertFile(gradeFile(dir), "official grade.py");
-  const files = resourceRoot(dir);
-  try {
-    const s = await stat(files);
-    if (!s.isDirectory()) throw new Error("not a directory");
-  } catch (err) {
-    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`);
-  }
-}
-var dabstepAnswerOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-function rowToTask(row, split, dir) {
-  const meta = {
-    taskId: row.task_id,
-    split,
-    golds: row.all_golds_by_task,
-    ...dir ? { resourceRoot: resourceRoot(dir) } : {}
-  };
-  return {
-    id: String(row.task_id),
-    split,
-    prompt: [
-      "Solve this DABStep data-analysis task using the mounted payment files.",
-      "Use code or shell commands as needed, then return only the final answer.",
-      "",
-      row.instructions
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "number" || !Array.isArray(md.golds)) {
-    throw new Error(`dabstep task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, opts, split, dir) {
-  let tasks = rows.map((row) => rowToTask(row, split, dir));
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`);
-  return tasks;
-}
-async function loadFixtures(opts, split) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[dabstep] DABSTEP_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures from ${FIXTURES}`);
-  return selectRows(rows, opts, split);
-}
-async function loadOfficialTasks(dir, opts, split) {
-  const script = `
-import ast, csv, json, sys
-from pathlib import Path
-
-root = Path(sys.argv[1])
-split = sys.argv[2]
-limit = None if sys.argv[3] == "" else int(sys.argv[3])
-ids = set(json.loads(sys.argv[4]))
-dataset = root / "dataset.csv"
-split_file = root / "splits" / f"{split}.txt"
-if not dataset.exists():
-    raise SystemExit(f"missing official DABStep dataset.csv at {dataset}")
-if not split_file.exists():
-    raise SystemExit(f"missing official DABStep split file at {split_file}")
-split_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}
-out = []
-with dataset.open(newline="") as f:
-    for row in csv.DictReader(f):
-        task_id = int(row["task_id"])
-        if task_id not in split_ids:
-            continue
-        if ids and str(task_id) not in ids:
-            continue
-        out.append({
-            "task_id": task_id,
-            "instructions": f"{row['question']}\\n{row['guidelines']}",
-            "all_golds_by_task": ast.literal_eval(str(row["all_golds_by_task"])),
-        })
-        if limit is not None and len(out) >= limit:
-            break
-if not out:
-    raise SystemExit(f"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}")
-print(json.dumps(out))
-`;
-  const stdout = await runVenvPython(script, [dir, split, opts.limit === void 0 ? "" : String(opts.limit), JSON.stringify(opts.ids ?? [])]);
-  return selectRows(JSON.parse(stdout), opts, split, dir);
-}
-function createDabstepAdapter() {
-  const fixturesMode = process.env.DABSTEP_FIXTURES === "1";
-  return {
-    name: "dabstep",
-    output: dabstepAnswerOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = dabstepDir();
-      if (!dir) {
-        throw new Error(
-          "DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep."
-        );
-      }
-      await assertOfficialFiles(dir, DEFAULT_SPLIT);
-      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT);
-    },
-    async loadTasks(opts = {}) {
-      const split = opts.split ?? DEFAULT_SPLIT;
-      if (fixturesMode) return loadFixtures(opts, split);
-      const dir = dabstepDir();
-      if (!dir) throw new Error("DABSTEP_DIR is required to load official DABStep tasks");
-      return loadOfficialTasks(dir, opts, split);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      const first = meta.golds[0];
-      if (!first) return void 0;
-      const value = first.value;
-      return value === void 0 ? void 0 : String(value);
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const dir = dabstepDir();
-      if (!dir) throw new Error("DABSTEP_DIR is required to judge DABStep tasks with the official grade.py");
-      const stdout = await runVenvScriptStdin(
-        join(benchRoot, "scripts", "dabstep_judge.py"),
-        ["--grade-file", gradeFile(dir)],
-        JSON.stringify({ prediction: artifact, golds: meta.golds }),
-        { cwd: benchRoot }
-      );
-      const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`);
-      const score = typeof report.score === "number" ? report.score : 0;
-      return {
-        resolved: report.correct === true,
-        score,
-        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct })
-      };
-    }
-  };
-}
-
-export {
-  dabstepAnswerOutput,
-  createDabstepAdapter
-};
-//# sourceMappingURL=chunk-RH5F53JT.js.map

package/dist/chunk-RH5F53JT.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/dabstep.ts"],"sourcesContent":["/**\n * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic\n * payment files. Worker artifact = final answer text. Judge = the official\n * DABStep `grade.py` normalization/matching function. No LLM judge.\n *\n * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that\n * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter\n * exposes `metadata.resourceRoot` so runners can mount the benchmark files into\n * AgentProfile.resources.files; it does not paste the dataset into prompt text.\n */\n\nimport { join } from 'node:path'\nimport { access, readFile, stat } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'dabstep.json')\nconst DEFAULT_SPLIT = 'easy'\n\ninterface DabstepFixtureRow {\n  task_id: number\n  instructions: string\n  all_golds_by_task: Array<Record<string, unknown>>\n}\n\ninterface DabstepMeta {\n  taskId: number\n  split: string\n  golds: Array<Record<string, unknown>>\n  resourceRoot?: string\n}\n\nconst dabstepDir = (): string | undefined => process.env.DABSTEP_DIR\nconst gradeFile = (dir: string): string => join(dir, 'grade.py')\nconst resourceRoot = (dir: string): string => join(dir, 'files')\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertOfficialFiles(dir: string, split: string): Promise<void> {\n  await assertFile(join(dir, 'dataset.csv'), 'released dataset.csv')\n  await assertFile(join(dir, 'splits', `${split}.txt`), `${split} split file`)\n  await assertFile(gradeFile(dir), 'official grade.py')\n  const files = resourceRoot(dir)\n  try {\n    const s = await stat(files)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nexport const dabstepAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction rowToTask(row: DabstepFixtureRow, split: string, dir?: string): BenchTask {\n  const meta: DabstepMeta = {\n    taskId: row.task_id,\n    split,\n    golds: row.all_golds_by_task,\n    ...(dir ? { resourceRoot: resourceRoot(dir) } : {}),\n  }\n  return {\n    id: String(row.task_id),\n    split,\n    prompt: [\n      'Solve this DABStep data-analysis task using the mounted payment files.',\n      'Use code or shell commands as needed, then return only the final answer.',\n      '',\n      row.instructions,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): DabstepMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number' || !Array.isArray(md.golds)) {\n    throw new Error(`dabstep task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as DabstepMeta\n}\n\nfunction selectRows(rows: DabstepFixtureRow[], opts: LoadOptions, split: string, dir?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, split, dir))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as DabstepFixtureRow[]\n  console.warn(`[dabstep] DABSTEP_FIXTURES=1 — loading ${rows.length} adapter fixtures from ${FIXTURES}`)\n  return selectRows(rows, opts, split)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const script = `\nimport ast, csv, json, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\nsplit = sys.argv[2]\nlimit = None if sys.argv[3] == \"\" else int(sys.argv[3])\nids = set(json.loads(sys.argv[4]))\ndataset = root / \"dataset.csv\"\nsplit_file = root / \"splits\" / f\"{split}.txt\"\nif not dataset.exists():\n    raise SystemExit(f\"missing official DABStep dataset.csv at {dataset}\")\nif not split_file.exists():\n    raise SystemExit(f\"missing official DABStep split file at {split_file}\")\nsplit_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}\nout = []\nwith dataset.open(newline=\"\") as f:\n    for row in csv.DictReader(f):\n        task_id = int(row[\"task_id\"])\n        if task_id not in split_ids:\n            continue\n        if ids and str(task_id) not in ids:\n            continue\n        out.append({\n            \"task_id\": task_id,\n            \"instructions\": f\"{row['question']}\\\\n{row['guidelines']}\",\n            \"all_golds_by_task\": ast.literal_eval(str(row[\"all_golds_by_task\"])),\n        })\n        if limit is not None and len(out) >= limit:\n            break\nif not out:\n    raise SystemExit(f\"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}\")\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(script, [dir, split, opts.limit === undefined ? '' : String(opts.limit), JSON.stringify(opts.ids ?? [])])\n  return selectRows(JSON.parse(stdout) as DabstepFixtureRow[], opts, split, dir)\n}\n\nexport function createDabstepAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.DABSTEP_FIXTURES === '1'\n\n  return {\n    name: 'dabstep',\n    output: dabstepAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = dabstepDir()\n      if (!dir) {\n        throw new Error(\n          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.',\n        )\n      }\n      await assertOfficialFiles(dir, DEFAULT_SPLIT)\n      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to load official DABStep tasks')\n      return loadOfficialTasks(dir, opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      const first = meta.golds[0]\n      if (!first) return undefined\n      const value = first.value\n      return value === undefined ? undefined : String(value)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to judge DABStep tasks with the official grade.py')\n      const stdout = await runVenvScriptStdin(\n        join(benchRoot, 'scripts', 'dabstep_judge.py'),\n        ['--grade-file', gradeFile(dir)],\n        JSON.stringify({ prediction: artifact, golds: meta.golds }),\n        { cwd: benchRoot },\n      )\n      const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as { correct?: boolean; score?: number; error?: string }\n      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: report.correct === true,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AAWA,SAAS,YAAY;AACrB,SAAS,QAAQ,UAAU,YAAY;AAKvC,IAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAC3D,IAAM,gBAAgB;AAetB,IAAM,aAAa,MAA0B,QAAQ,IAAI;AACzD,IAAM,YAAY,CAAC,QAAwB,KAAK,KAAK,UAAU;AAC/D,IAAM,eAAe,CAAC,QAAwB,KAAK,KAAK,OAAO;AAE/D,eAAe,WAAW,MAAc,OAA8B;AACpE,MAAI;AACF,UAAM,OAAO,IAAI;AAAA,EACnB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,oBAAoB,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACtG;AACF;AAEA,eAAe,oBAAoB,KAAa,OAA8B;AAC5E,QAAM,WAAW,KAAK,KAAK,aAAa,GAAG,sBAAsB;AACjE,QAAM,WAAW,KAAK,KAAK,UAAU,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,aAAa;AAC3E,QAAM,WAAW,UAAU,GAAG,GAAG,mBAAmB;AACpD,QAAM,QAAQ,aAAa,GAAG;AAC9B,MAAI;AACF,UAAM,IAAI,MAAM,KAAK,KAAK;AAC1B,QAAI,CAAC,EAAE,YAAY,EAAG,OAAM,IAAI,MAAM,iBAAiB;AAAA,EACzD,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,iDAAiD,KAAK,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACxH;AACF;AAEO,IAAM,sBAA6C;AAAA,EACxD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,wCAAwC,CAAC;AAC1E,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,SAAS,UAAU,KAAwB,OAAe,KAAyB;AACjF,QAAM,OAAoB;AAAA,IACxB,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,OAAO,IAAI;AAAA,IACX,GAAI,MAAM,EAAE,cAAc,aAAa,GAAG,EAAE,IAAI,CAAC;AAAA,EACnD;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,OAAO;AAAA,IACtB;AAAA,IACA,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,IAAI;AAAA,IACN,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA8B;AAC9C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,CAAC,MAAM,QAAQ,GAAG,KAAK,GAAG;AACpE,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wDAAmD;AAAA,EAC5F;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,OAAe,KAA2B;AAC1G,MAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,UAAU,KAAK,OAAO,GAAG,CAAC;AACxD,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,CAAC,cAAc,KAAK,EAAE;AAC9G,SAAO;AACT;AAEA,eAAe,aAAa,MAAmB,OAAqC;AAClF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,+CAA0C,KAAK,MAAM,0BAA0B,QAAQ,EAAE;AACtG,SAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,eAAe,kBAAkB,KAAa,MAAmB,OAAqC;AACpG,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkCf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,KAAK,OAAO,KAAK,UAAU,SAAY,KAAK,OAAO,KAAK,KAAK,GAAG,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAC3I,SAAO,WAAW,KAAK,MAAM,MAAM,GAA0B,MAAM,OAAO,GAAG;AAC/E;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,qBAAqB;AAEtD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,WAAW;AACvB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,oBAAoB,KAAK,aAAa;AAC5C,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,GAAG,aAAa;AAAA,IAC1D;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,QAAQ,KAAK,SAAS;AAC5B,UAAI,aAAc,QAAO,aAAa,MAAM,KAAK;AACjD,YAAM,MAAM,WAAW;AACvB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wDAAwD;AAClF,aAAO,kBAAkB,KAAK,MAAM,KAAK;AAAA,IAC3C;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,CAAC,MAAO,QAAO;AACnB,YAAM,QAAQ,MAAM;AACpB,aAAO,UAAU,SAAY,SAAY,OAAO,KAAK;AAAA,IACvD;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAM,WAAW;AACvB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,2EAA2E;AACrG,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,WAAW,WAAW,kBAAkB;AAAA,QAC7C,CAAC,gBAAgB,UAAU,GAAG,CAAC;AAAA,QAC/B,KAAK,UAAU,EAAE,YAAY,UAAU,OAAO,KAAK,MAAM,CAAC;AAAA,QAC1D,EAAE,KAAK,UAAU;AAAA,MACnB;AACA,YAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAClE,UAAI,OAAO,MAAO,OAAM,IAAI,MAAM,2BAA2B,KAAK,EAAE,KAAK,OAAO,KAAK,EAAE;AACvF,YAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;AAChE,aAAO;AAAA,QACL,UAAU,OAAO,YAAY;AAAA,QAC7B;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,OAAO,KAAK,OAAO,SAAS,OAAO,QAAQ,CAAC;AAAA,MAC5F;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-SEVJPLZC.js

@@ -1,260 +0,0 @@
-// src/benchmarks/simpleqa.ts
-import { execFile } from "child_process";
-import { readFile } from "fs/promises";
-import { join } from "path";
-import { fileURLToPath } from "url";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
-var PY = join(BENCH_ROOT, ".venv", "bin", "python");
-var FIXTURES = join(BENCH_ROOT, "fixtures", "simpleqa.json");
-var DATASET_URL = "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv";
-var FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
-var WORKER_CONTRACT = [
-  "",
-  "Research the question using live web sources and answer it with a short, specific factual value.",
-  "Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.",
-  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
-  "The answer after the sentinel must be the bare value only (no explanation on that line).",
-  "If you do not know the answer, state that you do not know rather than guessing."
-].join("\n");
-async function py(script, args = []) {
-  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], {
-    maxBuffer: 1024 * 1024 * 256
-  });
-  return stdout;
-}
-function parseFinalAnswer(artifact) {
-  const lines = artifact.split(/\r?\n/);
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const line = lines[i] ?? "";
-    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
-    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim();
-  }
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const t = (lines[i] ?? "").trim();
-    if (t.length > 0) return t;
-  }
-  return "";
-}
-function parseCitations(artifact) {
-  const urls = /* @__PURE__ */ new Set();
-  for (const m of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) {
-    urls.add(m[0].replace(/[.,;]+$/, ""));
-  }
-  return [...urls];
-}
-var GRADER_PROMPT = (question, gold, predicted) => [
-  "Your job is to grade a predicted answer to a factual question against the single gold target answer.",
-  "Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.",
-  "First, follow these grading rules carefully.",
-  "",
-  "CORRECT:",
-  "  The predicted answer fully contains the gold target without contradicting it.",
-  "  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.",
-  '  Hedging that still commits to the gold value (e.g. "I think it is X" where X is the gold) is CORRECT.',
-  "  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.",
-  "  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.",
-  "",
-  "INCORRECT:",
-  "  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.",
-  "  A prediction that states a different factual value than the gold is INCORRECT.",
-  "",
-  "NOT_ATTEMPTED:",
-  "  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,",
-  "  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.",
-  "",
-  "Also note the following rules:",
-  "  Do not punish a prediction for omitting information that would not change whether the gold target is contained.",
-  "  Grade ONLY whether the gold value is present and uncontradicted \u2014 not the overall quality of the response.",
-  "",
-  `Question: ${question}`,
-  `Gold target: ${gold}`,
-  `Predicted answer: ${predicted}`,
-  "",
-  "Respond with ONLY a fenced JSON block and nothing else:",
-  "```json",
-  '{"grade": "CORRECT" | "INCORRECT" | "NOT_ATTEMPTED"}',
-  "```"
-].join("\n");
-function graderRouter() {
-  const key = process.env.TANGLE_API_KEY;
-  if (!key) throw new Error("TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)");
-  const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
-  const baseUrl = process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1";
-  return { baseUrl, key, model };
-}
-async function gradeAnswer(question, gold, predicted, router) {
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: "POST",
-    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      messages: [{ role: "user", content: GRADER_PROMPT(question, gold, predicted) }]
-    })
-  });
-  if (!res.ok) {
-    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-  }
-  const body = await res.json();
-  const content = body.choices?.[0]?.message?.content;
-  if (typeof content !== "string") {
-    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
-  }
-  const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
-  const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch {
-    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`);
-  }
-  if (parsed.grade === "CORRECT" || parsed.grade === "INCORRECT" || parsed.grade === "NOT_ATTEMPTED") {
-    return parsed.grade;
-  }
-  throw new Error(
-    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`
-  );
-}
-function rowToTask(row, index) {
-  const meta = {
-    gold: row.answer,
-    goldSources: row.urls,
-    topic: row.topic,
-    answerType: row.answer_type,
-    question: row.problem
-  };
-  return {
-    id: `simpleqa-${index}`,
-    split: "test",
-    prompt: row.problem + WORKER_CONTRACT,
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.gold !== "string" || typeof md.question !== "string") {
-    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectTasks(tasks, opts) {
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    return tasks.filter((t) => want.has(t.id));
-  }
-  if (opts.limit !== void 0) return tasks.slice(0, opts.limit);
-  return tasks;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 \u2014 loading ${rows.length} committed fixtures (no network)`);
-  return selectTasks(rows.map(rowToTask), opts);
-}
-async function loadLive(opts) {
-  const script = `
-import csv, ast, json, io, sys
-from urllib.request import urlopen
-with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
-    text = resp.read().decode('utf-8')
-rows = list(csv.DictReader(io.StringIO(text)))
-out = []
-for r in rows:
-    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}
-    out.append({
-        'problem': r.get('problem', ''),
-        'answer': r.get('answer', ''),
-        'topic': str(meta.get('topic', '')),
-        'answer_type': str(meta.get('answer_type', '')),
-        'urls': list(meta.get('urls', [])),
-    })
-print(json.dumps(out))
-`;
-  const stdout = await py(script);
-  const rows = JSON.parse(stdout);
-  return selectTasks(rows.map(rowToTask), opts);
-}
-function createSimpleQaAdapter() {
-  const fixturesMode = process.env.SIMPLEQA_FIXTURES === "1";
-  return {
-    name: "simpleqa",
-    async preflight() {
-      graderRouter();
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8").catch((err) => {
-          throw new Error(
-            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`
-          );
-        });
-        return;
-      }
-      try {
-        await py(
-          `from urllib.request import urlopen
-with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
-    head = resp.read(64).decode('utf-8', 'replace')
-assert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]
-print('ok')`
-        );
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `simpleqa preflight failed: ${msg}
-Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; (2) ensure network access to ${DATASET_URL} ; or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      return loadLive(opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const finalAnswer = parseFinalAnswer(artifact);
-      const citations = parseCitations(artifact);
-      if (finalAnswer.length === 0) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({
-            grade: "NOT_ATTEMPTED",
-            gradeLetter: "C",
-            reason: "no parseable answer",
-            gold: meta.gold,
-            topic: meta.topic,
-            answerType: meta.answerType,
-            citationCount: citations.length
-          })
-        };
-      }
-      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter());
-      const gradeLetter = grade === "CORRECT" ? "A" : grade === "INCORRECT" ? "B" : "C";
-      const resolved = grade === "CORRECT";
-      return {
-        resolved,
-        score: resolved ? 1 : 0,
-        detail: JSON.stringify({
-          grade,
-          gradeLetter,
-          gold: meta.gold,
-          predicted: finalAnswer,
-          topic: meta.topic,
-          answerType: meta.answerType,
-          citationCount: citations.length
-        })
-      };
-    }
-  };
-}
-
-export {
-  parseFinalAnswer,
-  parseCitations,
-  createSimpleQaAdapter
-};
-//# sourceMappingURL=chunk-SEVJPLZC.js.map

package/dist/chunk-SEVJPLZC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";AA2BA,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AACzB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AACxC,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,IAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,IAAM,cACJ;AACF,IAAM,wBAAwB;AAG9B,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA,iDAAiD,qBAAqB;AAAA,EACtE;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;AACtE,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,IAAI,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IAClE,WAAW,OAAO,OAAO;AAAA,EAC3B,CAAC;AACD,SAAO;AACT;AAQO,SAAS,iBAAiB,UAA0B;AACzD,QAAM,QAAQ,SAAS,MAAM,OAAO;AACpC,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,OAAO,MAAM,CAAC,KAAK;AACzB,UAAM,MAAM,KAAK,YAAY,EAAE,QAAQ,qBAAqB;AAC5D,QAAI,QAAQ,GAAI,QAAO,KAAK,MAAM,MAAM,sBAAsB,MAAM,EAAE,KAAK;AAAA,EAC7E;AACA,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,KAAK,MAAM,CAAC,KAAK,IAAI,KAAK;AAChC,QAAI,EAAE,SAAS,EAAG,QAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAGO,SAAS,eAAe,UAA4B;AACzD,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,SAAS,SAAS,yBAAyB,GAAG;AAC5D,SAAK,IAAI,EAAE,CAAC,EAAE,QAAQ,WAAW,EAAE,CAAC;AAAA,EACtC;AACA,SAAO,CAAC,GAAG,IAAI;AACjB;AASA,IAAM,gBAAgB,CAAC,UAAkB,MAAc,cACrD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa,QAAQ;AAAA,EACrB,gBAAgB,IAAI;AAAA,EACpB,qBAAqB,SAAS;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAQb,SAAS,eAA6B;AACpC,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,6EAA6E;AACvG,QAAM,QAAQ,QAAQ,IAAI,eAAe;AACzC,QAAM,UAAU,QAAQ,IAAI,eAAe;AAC3C,SAAO,EAAE,SAAS,KAAK,MAAM;AAC/B;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;AAChB,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,IAC5D,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,IACrF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,OAAO;AAAA,MACd,aAAa;AAAA,MACb,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,cAAc,UAAU,MAAM,SAAS,EAAE,CAAC;AAAA,IAChF,CAAC;AAAA,EACH,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,IAAI,MAAM,wBAAwB,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC3F;AACA,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,MAAI,OAAO,YAAY,UAAU;AAC/B,UAAM,IAAI,MAAM,gDAAgD,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACtG;AACA,QAAM,SAAS,QAAQ,MAAM,8BAA8B;AAC3D,QAAM,OAAO,SAAS,OAAO,CAAC,IAAI,UAAU,KAAK,KAAK;AACtD,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;AAAA,EACzB,QAAQ;AACN,UAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACzG;AACA,MAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBAAiB;AAClG,WAAO,OAAO;AAAA,EAChB;AACA,QAAM,IAAI;AAAA,IACR,mEAAmE,KAAK,UAAU,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC;AAAA,EACzG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;AAC7D,QAAM,OAAqB;AAAA,IACzB,MAAM,IAAI;AAAA,IACV,aAAa,IAAI;AAAA,IACjB,OAAO,IAAI;AAAA,IACX,YAAY,IAAI;AAAA,IAChB,UAAU,IAAI;AAAA,EAChB;AACA,SAAO;AAAA,IACL,IAAI,YAAY,KAAK;AAAA,IACrB,OAAO;AAAA,IACP,QAAQ,IAAI,UAAU;AAAA,IACtB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAAU;AACzE,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,sEAAiE;AAAA,EAC3G;AACA,SAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;AACvE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,WAAO,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC3C;AACA,MAAI,KAAK,UAAU,OAAW,QAAO,MAAM,MAAM,GAAG,KAAK,KAAK;AAC9D,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,kCAAkC;AACtG,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;AAQA,eAAe,SAAS,MAAyC;AAC/D,QAAM,SAAS;AAAA;AAAA;AAAA,eAGF,KAAK,UAAU,WAAW,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAexC,QAAM,SAAS,MAAM,GAAG,MAAM;AAC9B,QAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAGhB,mBAAa;AACb,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI;AAAA,YACR,2BAA2B,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAC7F;AAAA,QACF,CAAC;AACD;AAAA,MACF;AACA,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,eACK,KAAK,UAAU,WAAW,CAAC;AAAA;AAAA;AAAA;AAAA,QAIlC;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,8BAA8B,GAAG;AAAA,qGAEC,WAAW;AAAA,QAE/C;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,aAAO,SAAS,IAAI;AAAA,IACtB;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,GAAG,qBAAqB,IAAI,KAAK,IAAI;AAAA,IAC9C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,cAAc,iBAAiB,QAAQ;AAC7C,YAAM,YAAY,eAAe,QAAQ;AAEzC,UAAI,YAAY,WAAW,GAAG;AAE5B,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU;AAAA,YACrB,OAAO;AAAA,YACP,aAAa;AAAA,YACb,QAAQ;AAAA,YACR,MAAM,KAAK;AAAA,YACX,OAAO,KAAK;AAAA,YACZ,YAAY,KAAK;AAAA,YACjB,eAAe,UAAU;AAAA,UAC3B,CAAC;AAAA,QACH;AAAA,MACF;AAEA,YAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;AACrF,YAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;AAC9E,YAAM,WAAW,UAAU;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,OAAO,WAAW,IAAI;AAAA,QACtB,QAAQ,KAAK,UAAU;AAAA,UACrB;AAAA,UACA;AAAA,UACA,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,YAAY,KAAK;AAAA,UACjB,eAAe,UAAU;AAAA,QAC3B,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-SFLA7OH3.js

@@ -1,27 +0,0 @@
-import {
-  createTauBenchAdapter
-} from "./chunk-IZ5M6OAC.js";
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/tau3-banking.ts
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "tau3-banking.json");
-function createTau3BankingAdapter() {
-  return createTauBenchAdapter({
-    name: "tau3-banking",
-    fixturePath: FIXTURES,
-    fixturesEnv: "TAU3_FIXTURES",
-    dirEnv: "TAU3_BENCH_DIR",
-    domainEnv: "TAU3_DOMAIN",
-    defaultDomain: "banking_knowledge",
-    taskIntro: "Run this tau3 banking task in the official tau3 knowledge benchmark.",
-    installHint: "clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench."
-  });
-}
-
-export {
-  createTau3BankingAdapter
-};
-//# sourceMappingURL=chunk-SFLA7OH3.js.map

package/dist/chunk-SFLA7OH3.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/tau3-banking.ts"],"sourcesContent":["/**\n * tau3-banking adapter.\n *\n * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with\n * package namespace `tau2` and a new `banking_knowledge` domain. This adapter\n * reuses the shared tau reward recomputation path and only changes the default\n * domain/env names.\n */\n\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport { createTauBenchAdapter } from './tau-bench-shared'\nimport type { BenchmarkAdapter } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'tau3-banking.json')\n\nexport function createTau3BankingAdapter(): BenchmarkAdapter {\n  return createTauBenchAdapter({\n    name: 'tau3-banking',\n    fixturePath: FIXTURES,\n    fixturesEnv: 'TAU3_FIXTURES',\n    dirEnv: 'TAU3_BENCH_DIR',\n    domainEnv: 'TAU3_DOMAIN',\n    defaultDomain: 'banking_knowledge',\n    taskIntro: 'Run this tau3 banking task in the official tau3 knowledge benchmark.',\n    installHint:\n      'clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench.',\n  })\n}\n"],"mappings":";;;;;;;;AASA,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEzD,SAAS,2BAA6C;AAC3D,SAAO,sBAAsB;AAAA,IAC3B,MAAM;AAAA,IACN,aAAa;AAAA,IACb,aAAa;AAAA,IACb,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,eAAe;AAAA,IACf,WAAW;AAAA,IACX,aACE;AAAA,EACJ,CAAC;AACH;","names":[]}