@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/dabstep.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"dabstep.js","names":[],"sources":["../../src/benchmarks/dabstep.ts"],"sourcesContent":["/**\n * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic\n * payment files. Worker artifact = final answer text. Judge = the official\n * DABStep `grade.py` normalization/matching function. No LLM judge.\n *\n * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that\n * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter\n * exposes `metadata.resourceRoot` so runners can mount the benchmark files into\n * AgentProfile.resources.files; it does not paste the dataset into prompt text.\n */\n\nimport { join } from 'node:path'\nimport { access, readFile, stat } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'dabstep.json')\nconst DEFAULT_SPLIT = 'easy'\n\ninterface DabstepFixtureRow {\n  task_id: number\n  instructions: string\n  all_golds_by_task: Array<Record<string, unknown>>\n}\n\ninterface DabstepMeta {\n  taskId: number\n  split: string\n  golds: Array<Record<string, unknown>>\n  resourceRoot?: string\n}\n\nconst dabstepDir = (): string | undefined => process.env.DABSTEP_DIR\nconst gradeFile = (dir: string): string => join(dir, 'grade.py')\nconst resourceRoot = (dir: string): string => join(dir, 'files')\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertOfficialFiles(dir: string, split: string): Promise<void> {\n  await assertFile(join(dir, 'dataset.csv'), 'released dataset.csv')\n  await assertFile(join(dir, 'splits', `${split}.txt`), `${split} split file`)\n  await assertFile(gradeFile(dir), 'official grade.py')\n  const files = resourceRoot(dir)\n  try {\n    const s = await stat(files)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nexport const dabstepAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction rowToTask(row: DabstepFixtureRow, split: string, dir?: string): BenchTask {\n  const meta: DabstepMeta = {\n    taskId: row.task_id,\n    split,\n    golds: row.all_golds_by_task,\n    ...(dir ? { resourceRoot: resourceRoot(dir) } : {}),\n  }\n  return {\n    id: String(row.task_id),\n    split,\n    prompt: [\n      'Solve this DABStep data-analysis task using the mounted payment files.',\n      'Use code or shell commands as needed, then return only the final answer.',\n      '',\n      row.instructions,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): DabstepMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number' || !Array.isArray(md.golds)) {\n    throw new Error(`dabstep task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as DabstepMeta\n}\n\nfunction selectRows(rows: DabstepFixtureRow[], opts: LoadOptions, split: string, dir?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, split, dir))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as DabstepFixtureRow[]\n  console.warn(`[dabstep] DABSTEP_FIXTURES=1 — loading ${rows.length} adapter fixtures from ${FIXTURES}`)\n  return selectRows(rows, opts, split)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const script = `\nimport ast, csv, json, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\nsplit = sys.argv[2]\nlimit = None if sys.argv[3] == \"\" else int(sys.argv[3])\nids = set(json.loads(sys.argv[4]))\ndataset = root / \"dataset.csv\"\nsplit_file = root / \"splits\" / f\"{split}.txt\"\nif not dataset.exists():\n    raise SystemExit(f\"missing official DABStep dataset.csv at {dataset}\")\nif not split_file.exists():\n    raise SystemExit(f\"missing official DABStep split file at {split_file}\")\nsplit_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}\nout = []\nwith dataset.open(newline=\"\") as f:\n    for row in csv.DictReader(f):\n        task_id = int(row[\"task_id\"])\n        if task_id not in split_ids:\n            continue\n        if ids and str(task_id) not in ids:\n            continue\n        out.append({\n            \"task_id\": task_id,\n            \"instructions\": f\"{row['question']}\\\\n{row['guidelines']}\",\n            \"all_golds_by_task\": ast.literal_eval(str(row[\"all_golds_by_task\"])),\n        })\n        if limit is not None and len(out) >= limit:\n            break\nif not out:\n    raise SystemExit(f\"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}\")\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(script, [dir, split, opts.limit === undefined ? '' : String(opts.limit), JSON.stringify(opts.ids ?? [])])\n  return selectRows(JSON.parse(stdout) as DabstepFixtureRow[], opts, split, dir)\n}\n\nexport function createDabstepAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.DABSTEP_FIXTURES === '1'\n\n  return {\n    name: 'dabstep',\n    output: dabstepAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = dabstepDir()\n      if (!dir) {\n        throw new Error(\n          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.',\n        )\n      }\n      await assertOfficialFiles(dir, DEFAULT_SPLIT)\n      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to load official DABStep tasks')\n      return loadOfficialTasks(dir, opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      const first = meta.golds[0]\n      if (!first) return undefined\n      const value = first.value\n      return value === undefined ? undefined : String(value)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to judge DABStep tasks with the official grade.py')\n      const stdout = await runVenvScriptStdin(\n        join(benchRoot, 'scripts', 'dabstep_judge.py'),\n        ['--grade-file', gradeFile(dir)],\n        JSON.stringify({ prediction: artifact, golds: meta.golds }),\n        { cwd: benchRoot },\n      )\n      const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as { correct?: boolean; score?: number; error?: string }\n      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: report.correct === true,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,MAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAC3D,MAAM,gBAAgB;AAetB,MAAM,mBAAuC,QAAQ,IAAI;AACzD,MAAM,aAAa,QAAwB,KAAK,KAAK,UAAU;AAC/D,MAAM,gBAAgB,QAAwB,KAAK,KAAK,OAAO;AAE/D,eAAe,WAAW,MAAc,OAA8B;CACpE,IAAI;EACF,MAAM,OAAO,IAAI;CACnB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,oBAAoB,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACtG;AACF;AAEA,eAAe,oBAAoB,KAAa,OAA8B;CAC5E,MAAM,WAAW,KAAK,KAAK,aAAa,GAAG,sBAAsB;CACjE,MAAM,WAAW,KAAK,KAAK,UAAU,GAAG,MAAM,KAAK,GAAG,GAAG,MAAM,YAAY;CAC3E,MAAM,WAAW,UAAU,GAAG,GAAG,mBAAmB;CACpD,MAAM,QAAQ,aAAa,GAAG;CAC9B,IAAI;EAEF,IAAI,EAAC,MADW,KAAK,KAAK,EAAA,CACnB,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;CACzD,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,iDAAiD,MAAM,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACxH;AACF;AAEA,MAAa,sBAA6C,EACxD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,wCAAwC,CAC5D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,UAAU,KAAwB,OAAe,KAAyB;CACjF,MAAM,OAAoB;EACxB,QAAQ,IAAI;EACZ;EACA,OAAO,IAAI;EACX,GAAI,MAAM,EAAE,cAAc,aAAa,GAAG,EAAE,IAAI,CAAC;CACnD;CACA,OAAO;EACL,IAAI,OAAO,IAAI,OAAO;EACtB;EACA,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA8B;CAC9C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,CAAC,MAAM,QAAQ,GAAG,KAAK,GACjE,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kDAAkD;CAE5F,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,OAAe,KAA2B;CAC1G,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,OAAO,GAAG,CAAC;CACxD,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,EAAE,aAAa,OAAO;CAC9G,OAAO;AACT;AAEA,eAAe,aAAa,MAAmB,OAAqC;CAClF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,0CAA0C,KAAK,OAAO,yBAAyB,UAAU;CACtG,OAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,eAAe,kBAAkB,KAAa,MAAmB,OAAqC;CAmCpG,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAK;EAAO,KAAK,UAAU,KAAA,IAAY,KAAK,OAAO,KAAK,KAAK;EAAG,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC;CAAC,CAAC;CAC3I,OAAO,WAAW,KAAK,MAAM,MAAM,GAA0B,MAAM,OAAO,GAAG;AAC/E;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,qBAAqB;CAEtD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KACH,MAAM,IAAI,MACR,qKACF;GAEF,MAAM,oBAAoB,KAAK,aAAa;GAC5C,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,GAAG,aAAa;EAC1D;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,IAAI,cAAc,OAAO,aAAa,MAAM,KAAK;GACjD,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;GAClF,OAAO,kBAAkB,KAAK,MAAM,KAAK;EAC3C;EAEA,MAAM,aAAa,MAAiB;GAElC,MAAM,QADO,SAAS,IACL,CAAC,CAAC,MAAM;GACzB,IAAI,CAAC,OAAO,OAAO,KAAA;GACnB,MAAM,QAAQ,MAAM;GACpB,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,OAAO,KAAK;EACvD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,2EAA2E;GACrG,MAAM,SAAS,MAAM,mBACnB,KAAK,WAAW,WAAW,kBAAkB,GAC7C,CAAC,gBAAgB,UAAU,GAAG,CAAC,GAC/B,KAAK,UAAU;IAAE,YAAY;IAAU,OAAO,KAAK;GAAM,CAAC,GAC1D,EAAE,KAAK,UAAU,CACnB;GACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;GAClE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,2BAA2B,KAAK,GAAG,IAAI,OAAO,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,OAAO,YAAY;IAC7B;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,OAAO,KAAK;KAAO,SAAS,OAAO;IAAQ,CAAC;GAC5F;EACF;CACF;AACF"}

package/dist/benchmarks/enterpriseops-gym.d.ts

@@ -1,38 +1,9 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —
- * stateful agentic planning + tool use in enterprise settings. Each record is an
- * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,
- * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`
- * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).
- * Worker artifact = the ordered tool-call transcript the agent would issue against
- * those servers, emitted as a single fenced ```json block of
- * `{ "calls": [ { "tool": ..., "arguments": {...}, "gym_name"?: ... } ] }`.
- *
- * Judge = the benchmark's OWN deterministic state-checker. The driver replays the
- * transcript against a freshly-seeded gym server (mutating its database), then runs
- * each task's `database_state` verifier — an SQL SELECT executed via the gym
- * server's /api/sql-runner endpoint, compared to `expected_value` under
- * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =
- * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;
- * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully
- * deterministic — no LLM judge.
- *
- * loadTasks enumerates the real suite from the HF rows server (config = tool-set
- * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed
- * sample (bench/fixtures/enterpriseops-gym.json) loads offline.
- *
- * Requires for a LIVE judge run: a Docker daemon with the domain gym images
- * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up
- * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge
- * fail loud with the exact pull/run/unzip step when a server is unreachable — never
- * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/enterpriseops-gym.d.ts
 /** Worker transcript = the last fenced ```json block, else the raw text. */
 declare const enterpriseOpsTranscriptOutput: OutputAdapter<string>;
 declare function createEnterpriseOpsGymAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createEnterpriseOpsGymAdapter, enterpriseOpsTranscriptOutput };
+//# sourceMappingURL=enterpriseops-gym.d.ts.map

package/dist/benchmarks/enterpriseops-gym.js

@@ -1,10 +1,237 @@
-import {
-  createEnterpriseOpsGymAdapter,
-  enterpriseOpsTranscriptOutput
-} from "../chunk-YSMEKBTD.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createEnterpriseOpsGymAdapter,
-  enterpriseOpsTranscriptOutput
-};
+import { benchRoot, runVenvScriptStdin } from "./_harness.js";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/enterpriseops-gym.ts
+/**
+* EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —
+* stateful agentic planning + tool use in enterprise settings. Each record is an
+* enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,
+* Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`
+* allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).
+* Worker artifact = the ordered tool-call transcript the agent would issue against
+* those servers, emitted as a single fenced ```json block of
+* `{ "calls": [ { "tool": ..., "arguments": {...}, "gym_name"?: ... } ] }`.
+*
+* Judge = the benchmark's OWN deterministic state-checker. The driver replays the
+* transcript against a freshly-seeded gym server (mutating its database), then runs
+* each task's `database_state` verifier — an SQL SELECT executed via the gym
+* server's /api/sql-runner endpoint, compared to `expected_value` under
+* `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =
+* (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;
+* binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully
+* deterministic — no LLM judge.
+*
+* loadTasks enumerates the real suite from the HF rows server (config = tool-set
+* MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed
+* sample (bench/fixtures/enterpriseops-gym.json) loads offline.
+*
+* Requires for a LIVE judge run: a Docker daemon with the domain gym images
+* (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up
+* on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge
+* fail loud with the exact pull/run/unzip step when a server is unreachable — never
+* a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "enterpriseops-gym.json");
+const JUDGE = join(benchRoot, "scripts", "enterpriseops_gym_judge.py");
+/** Monotonic discriminator so concurrent judge calls stage distinct task-cache files. */
+let judgeCallSeq = 0;
+const DATASET = "ServiceNow-AI/EnterpriseOps-Gym";
+/** Tool-set mode = HF config; oracle ships exact tools, plus_N adds N distractors. */
+const DEFAULT_MODE = "oracle";
+/** Domain = HF split. */
+const DEFAULT_DOMAIN = "itsm";
+const rowsApi = (mode, domain) => `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`;
+/** Worker transcript = the last fenced ```json block, else the raw text. */
+const enterpriseOpsTranscriptOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return ([...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+} };
+function asArray(v) {
+	return typeof v === "string" ? JSON.parse(v) : v;
+}
+function workerContract(tools) {
+	return [
+		"",
+		`You have exactly these tools available (call NO others): ${tools.join(", ")}.`,
+		"Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.",
+		"Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:",
+		"{ \"calls\": [ { \"tool\": \"<tool_name>\", \"arguments\": { ... } } ] }",
+		"Include one entry per tool call in execution order. Nothing after the closing fence."
+	].join("\n");
+}
+function rowToTask(row, mode) {
+	const servers = asArray(row.gym_servers_config);
+	const verifiers = asArray(row.verifiers);
+	const meta = {
+		taskId: row.task_id,
+		domain: row.domain,
+		mode,
+		selectedTools: row.selected_tools,
+		servers,
+		verifiers
+	};
+	return {
+		id: row.task_id,
+		split: row.domain,
+		prompt: [
+			row.system_prompt,
+			"",
+			row.user_prompt,
+			workerContract(row.selected_tools)
+		].join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.taskId !== "string" || !Array.isArray(md.servers) || !Array.isArray(md.verifiers) || md.verifiers.length === 0) throw new Error(`enterpriseops-gym task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function selectRows(rows, mode, opts) {
+	let tasks = rows.map((r) => rowToTask(r, mode));
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((t) => want.has(t.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	return tasks;
+}
+async function loadFixtures(mode, opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[enterpriseops-gym] EOPS_FIXTURES=1 — loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`);
+	const domain = opts.split;
+	return selectRows(domain ? rows.filter((r) => r.domain === domain) : rows, mode, opts);
+}
+/** Pull real rows from the HF rows server (paged) for one mode/domain. Throws loud on a non-OK response. */
+async function fetchRows(mode, domain, opts) {
+	const target = opts.ids ? opts.ids.length * 4 : opts.limit ?? 16;
+	const rows = [];
+	const want = opts.ids ? new Set(opts.ids) : null;
+	const page = 100;
+	const base = rowsApi(mode, domain);
+	for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {
+		const res = await fetch(`${base}&offset=${offset}&length=${page}`);
+		if (!res.ok) throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`);
+		const got = (await res.json()).rows ?? [];
+		if (got.length === 0) break;
+		for (const r of got) {
+			if (want && !want.has(r.row.task_id)) continue;
+			rows.push(r.row);
+		}
+		if (got.length < page) break;
+	}
+	if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`);
+	return rows;
+}
+/**
+* Run the benchmark's own state-checker for one task over the worker's transcript.
+* The driver replays the tool calls against the live gym server, runs each
+* database_state verifier's SQL via /api/sql-runner, and reports {passes,total}.
+* Score = passes/total (verifier_level_pass_rate); resolved = all pass
+* (overall_success_rate). This is the expensive Docker-backed boundary — delegated
+* to the python driver, not reimplemented. The transcript is piped on stdin via the
+* shared stdin-aware runner (execFile's `input` is not honored async and hangs the
+* reader).
+*/
+async function runJudge(meta, artifact) {
+	const taskJsonPath = join(benchRoot, ".eops-task-cache", `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`);
+	await writeTaskCache(taskJsonPath, meta);
+	let stdout;
+	try {
+		stdout = await runVenvScriptStdin(JUDGE, [
+			"judge",
+			"--task-json",
+			taskJsonPath
+		], artifact, { cwd: benchRoot });
+	} catch (err) {
+		const e = err;
+		throw new Error(`enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}\nFix: (1) run the ${meta.domain} gym server — docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; (2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir (the judge seeds each task's database from seed_database_file).`);
+	} finally {
+		await rm(taskJsonPath, { force: true }).catch(() => {});
+	}
+	const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+	if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`);
+	if (typeof report.passes !== "number" || typeof report.total !== "number") throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`);
+	const score = report.total > 0 ? report.passes / report.total : 0;
+	return {
+		resolved: report.success === true && report.total > 0 && report.passes === report.total,
+		score,
+		detail: JSON.stringify({
+			taskId: meta.taskId,
+			domain: meta.domain,
+			passes: report.passes,
+			total: report.total
+		})
+	};
+}
+async function writeTaskCache(path, meta) {
+	await mkdir(join(path, ".."), { recursive: true });
+	await writeFile(path, JSON.stringify({
+		task_id: meta.taskId,
+		domain: meta.domain,
+		gym_servers_config: meta.servers,
+		verifiers: meta.verifiers
+	}));
+}
+/** Ping the configured gym servers' SQL endpoint; throw loud with the docker fix if any is unreachable. */
+async function probeServers(servers, domain) {
+	for (const s of servers) {
+		const url = `${s.mcp_server_url.replace(/\/$/, "")}/api/sql-runner`;
+		try {
+			(await fetch(url, {
+				method: "POST",
+				headers: { "content-type": "application/json" },
+				body: "{}"
+			})).status;
+		} catch (err) {
+			throw new Error(`enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}\nFix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`);
+		}
+	}
+}
+function createEnterpriseOpsGymAdapter() {
+	const fixturesMode = process.env.EOPS_FIXTURES === "1";
+	const mode = process.env.EOPS_MODE ?? DEFAULT_MODE;
+	return {
+		name: "enterpriseops-gym",
+		output: enterpriseOpsTranscriptOutput,
+		async preflight() {
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8").catch((err) => {
+					throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+				});
+				return;
+			}
+			const sample = await loadFixtures(mode, {
+				split: DEFAULT_DOMAIN,
+				limit: 1
+			}).catch(() => []);
+			const servers = sample[0] ? readMeta(sample[0]).servers : [];
+			if (servers.length === 0) throw new Error("enterpriseops-gym preflight: no gym_servers_config to probe. Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.");
+			await probeServers(servers, DEFAULT_DOMAIN);
+		},
+		async loadTasks(opts = {}) {
+			const domain = opts.split ?? DEFAULT_DOMAIN;
+			if (fixturesMode) return loadFixtures(mode, opts);
+			let rows;
+			try {
+				rows = await fetchRows(mode, domain, opts);
+			} catch (err) {
+				console.warn(`[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`);
+				return loadFixtures(mode, opts);
+			}
+			return selectRows(rows, mode, opts);
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			return runJudge(readMeta(task), artifact);
+		}
+	};
+}
+//#endregion
+export { createEnterpriseOpsGymAdapter, enterpriseOpsTranscriptOutput };
+
 //# sourceMappingURL=enterpriseops-gym.js.map

package/dist/benchmarks/enterpriseops-gym.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"enterpriseops-gym.js","names":[],"sources":["../../src/benchmarks/enterpriseops-gym.ts"],"sourcesContent":["/**\n * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —\n * stateful agentic planning + tool use in enterprise settings. Each record is an\n * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,\n * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`\n * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).\n * Worker artifact = the ordered tool-call transcript the agent would issue against\n * those servers, emitted as a single fenced ```json block of\n * `{ \"calls\": [ { \"tool\": ..., \"arguments\": {...}, \"gym_name\"?: ... } ] }`.\n *\n * Judge = the benchmark's OWN deterministic state-checker. The driver replays the\n * transcript against a freshly-seeded gym server (mutating its database), then runs\n * each task's `database_state` verifier — an SQL SELECT executed via the gym\n * server's /api/sql-runner endpoint, compared to `expected_value` under\n * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =\n * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;\n * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully\n * deterministic — no LLM judge.\n *\n * loadTasks enumerates the real suite from the HF rows server (config = tool-set\n * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed\n * sample (bench/fixtures/enterpriseops-gym.json) loads offline.\n *\n * Requires for a LIVE judge run: a Docker daemon with the domain gym images\n * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up\n * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge\n * fail loud with the exact pull/run/unzip step when a server is unreachable — never\n * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.\n */\n\nimport { mkdir, readFile, rm, writeFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'enterpriseops-gym.json')\nconst JUDGE = join(benchRoot, 'scripts', 'enterpriseops_gym_judge.py')\n\n/** Monotonic discriminator so concurrent judge calls stage distinct task-cache files. */\nlet judgeCallSeq = 0\n\nconst DATASET = 'ServiceNow-AI/EnterpriseOps-Gym'\n/** Tool-set mode = HF config; oracle ships exact tools, plus_N adds N distractors. */\nconst DEFAULT_MODE = 'oracle'\n/** Domain = HF split. */\nconst DEFAULT_DOMAIN = 'itsm'\n\nconst rowsApi = (mode: string, domain: string) =>\n  `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`\n\ninterface GymServerConfig {\n  mcp_server_name: string\n  mcp_server_url: string\n  seed_database_file: string\n  context?: Record<string, string>\n  user_info?: Record<string, unknown>\n}\n\ninterface Verifier {\n  verifier_type: string\n  name: string\n  description?: string\n  gym_name: string\n  validation_config: { query: string; expected_value: unknown; comparison_type: string }\n}\n\ninterface EopsRow {\n  task_id: string\n  domain: string\n  system_prompt: string\n  user_prompt: string\n  selected_tools: string[]\n  restricted_tools: string[]\n  mcp_endpoint: string\n  number_of_runs: number\n  reset_database_between_runs: boolean\n  /** HF parquet stores these as JSON strings; fixtures store them as parsed arrays. */\n  gym_servers_config: string | GymServerConfig[]\n  verifiers: string | Verifier[]\n}\n\ninterface EopsMeta {\n  taskId: string\n  domain: string\n  mode: string\n  selectedTools: string[]\n  servers: GymServerConfig[]\n  verifiers: Verifier[]\n}\n\n/** Worker transcript = the last fenced ```json block, else the raw text. */\nexport const enterpriseOpsTranscriptOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction asArray<T>(v: string | T[]): T[] {\n  return typeof v === 'string' ? (JSON.parse(v) as T[]) : v\n}\n\nfunction workerContract(tools: string[]): string {\n  return [\n    '',\n    `You have exactly these tools available (call NO others): ${tools.join(', ')}.`,\n    'Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.',\n    'Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:',\n    '{ \"calls\": [ { \"tool\": \"<tool_name>\", \"arguments\": { ... } } ] }',\n    'Include one entry per tool call in execution order. Nothing after the closing fence.',\n  ].join('\\n')\n}\n\nfunction rowToTask(row: EopsRow, mode: string): BenchTask {\n  const servers = asArray<GymServerConfig>(row.gym_servers_config)\n  const verifiers = asArray<Verifier>(row.verifiers)\n  const meta: EopsMeta = {\n    taskId: row.task_id,\n    domain: row.domain,\n    mode,\n    selectedTools: row.selected_tools,\n    servers,\n    verifiers,\n  }\n  return {\n    id: row.task_id,\n    split: row.domain,\n    prompt: [row.system_prompt, '', row.user_prompt, workerContract(row.selected_tools)].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): EopsMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.taskId !== 'string' ||\n    !Array.isArray(md.servers) ||\n    !Array.isArray(md.verifiers) ||\n    (md.verifiers as unknown[]).length === 0\n  ) {\n    throw new Error(`enterpriseops-gym task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as EopsMeta\n}\n\nfunction selectRows(rows: EopsRow[], mode: string, opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map((r) => rowToTask(r, mode))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(mode: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as EopsRow[]\n  console.warn(\n    `[enterpriseops-gym] EOPS_FIXTURES=1 — loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`,\n  )\n  const domain = opts.split\n  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows\n  return selectRows(scoped, mode, opts)\n}\n\n/** Pull real rows from the HF rows server (paged) for one mode/domain. Throws loud on a non-OK response. */\nasync function fetchRows(mode: string, domain: string, opts: LoadOptions): Promise<EopsRow[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: EopsRow[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  const base = rowsApi(mode, domain)\n  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {\n    const res = await fetch(`${base}&offset=${offset}&length=${page}`)\n    if (!res.ok) {\n      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    }\n    const body = (await res.json()) as { rows?: Array<{ row: EopsRow }> }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.task_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`)\n  return rows\n}\n\n/**\n * Run the benchmark's own state-checker for one task over the worker's transcript.\n * The driver replays the tool calls against the live gym server, runs each\n * database_state verifier's SQL via /api/sql-runner, and reports {passes,total}.\n * Score = passes/total (verifier_level_pass_rate); resolved = all pass\n * (overall_success_rate). This is the expensive Docker-backed boundary — delegated\n * to the python driver, not reimplemented. The transcript is piped on stdin via the\n * shared stdin-aware runner (execFile's `input` is not honored async and hangs the\n * reader).\n */\nasync function runJudge(meta: EopsMeta, artifact: string): Promise<BenchScore> {\n  // Stage the full task record (servers + verifiers) so the driver has the live\n  // server URLs/contexts and the SQL it must run. The path is UNIQUE per call: the gate\n  // runs k judges for one task concurrently, so a `${taskId}.json` shared path would race.\n  const taskJsonPath = join(\n    benchRoot,\n    '.eops-task-cache',\n    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`,\n  )\n  await writeTaskCache(taskJsonPath, meta)\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(JUDGE, ['judge', '--task-json', taskJsonPath], artifact, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(\n      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}\\n` +\n        `Fix: (1) run the ${meta.domain} gym server — ` +\n        `docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `(2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir ` +\n        `(the judge seeds each task's database from seed_database_file).`,\n    )\n  } finally {\n    await rm(taskJsonPath, { force: true }).catch(() => {})\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    success?: boolean\n    passes?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`)\n  if (typeof report.passes !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passes / report.total : 0\n  return {\n    resolved: report.success === true && report.total > 0 && report.passes === report.total,\n    score,\n    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total }),\n  }\n}\n\nasync function writeTaskCache(path: string, meta: EopsMeta): Promise<void> {\n  await mkdir(join(path, '..'), { recursive: true })\n  await writeFile(\n    path,\n    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers }),\n  )\n}\n\n/** Ping the configured gym servers' SQL endpoint; throw loud with the docker fix if any is unreachable. */\nasync function probeServers(servers: GymServerConfig[], domain: string): Promise<void> {\n  for (const s of servers) {\n    const url = `${s.mcp_server_url.replace(/\\/$/, '')}/api/sql-runner`\n    try {\n      // A HEAD/empty POST just proves reachability; a real query runs in judge.\n      const res = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' })\n      // Any HTTP response (even an error status) proves the server is up. Only a\n      // transport failure (refused/ENOTFOUND) means the container is not running.\n      void res.status\n    } catch (err) {\n      throw new Error(\n        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}\\n` +\n          `Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`,\n      )\n    }\n  }\n}\n\nexport function createEnterpriseOpsGymAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.EOPS_FIXTURES === '1'\n  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE\n\n  return {\n    name: 'enterpriseops-gym',\n    output: enterpriseOpsTranscriptOutput,\n\n    async preflight() {\n      // Fixtures mode proves only that the sample file is readable; a live judge\n      // still requires running gym servers (and fails loud there if absent).\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      // Live mode: probe the gym servers for the default domain (the suite's tasks\n      // each carry their own server config; preflight verifies reachability up\n      // front so a batch fails fast with the docker fix rather than mid-run).\n      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => [])\n      const servers = sample[0] ? readMeta(sample[0]).servers : []\n      if (servers.length === 0) {\n        throw new Error(\n          `enterpriseops-gym preflight: no gym_servers_config to probe. ` +\n            `Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`,\n        )\n      }\n      await probeServers(servers, DEFAULT_DOMAIN)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const domain = opts.split ?? DEFAULT_DOMAIN\n      if (fixturesMode) return loadFixtures(mode, opts)\n      let rows: EopsRow[]\n      try {\n        rows = await fetchRows(mode, domain, opts)\n      } catch (err) {\n        console.warn(\n          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`,\n        )\n        return loadFixtures(mode, opts)\n      }\n      return selectRows(rows, mode, opts)\n    },\n\n    async goldArtifact() {\n      // The benchmark ships no portable per-task oracle transcript — the reference\n      // is the seeded final DB state the verifiers check, not a tool-call script.\n      // Judge correctness is proven by replaying a real solve against the live gym\n      // server, not by a synthetic gold transcript. Returns undefined (documented,\n      // not faked).\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCA,MAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,MAAM,QAAQ,KAAK,WAAW,WAAW,4BAA4B;;AAGrE,IAAI,eAAe;AAEnB,MAAM,UAAU;;AAEhB,MAAM,eAAe;;AAErB,MAAM,iBAAiB;AAEvB,MAAM,WAAW,MAAc,WAC7B,uDAAuD,mBAAmB,OAAO,EAAE,UAAU,mBAAmB,IAAI,EAAE,SAAS,mBAAmB,MAAM;;AA2C1J,MAAa,gCAAuD,EAClE,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,iCAAiC,CACrD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,QAAW,GAAsB;CACxC,OAAO,OAAO,MAAM,WAAY,KAAK,MAAM,CAAC,IAAY;AAC1D;AAEA,SAAS,eAAe,OAAyB;CAC/C,OAAO;EACL;EACA,4DAA4D,MAAM,KAAK,IAAI,EAAE;EAC7E;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAAc,MAAyB;CACxD,MAAM,UAAU,QAAyB,IAAI,kBAAkB;CAC/D,MAAM,YAAY,QAAkB,IAAI,SAAS;CACjD,MAAM,OAAiB;EACrB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,eAAe,IAAI;EACnB;EACA;CACF;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GAAC,IAAI;GAAe;GAAI,IAAI;GAAa,eAAe,IAAI,cAAc;EAAC,CAAC,CAAC,KAAK,IAAI;EAC9F,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA2B;CAC3C,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,WAAW,YACrB,CAAC,MAAM,QAAQ,GAAG,OAAO,KACzB,CAAC,MAAM,QAAQ,GAAG,SAAS,KAC1B,GAAG,UAAwB,WAAW,GAEvC,MAAM,IAAI,MAAM,0BAA0B,KAAK,GAAG,kDAAkD;CAEtG,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,MAAc,MAAgC;CACjF,IAAI,QAAQ,KAAK,KAAK,MAAM,UAAU,GAAG,IAAI,CAAC;CAC9C,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,eAAe,aAAa,MAAc,MAAyC;CACjF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KACN,iDAAiD,KAAK,OAAO,8BAA8B,SAAS,eACtG;CACA,MAAM,SAAS,KAAK;CAEpB,OAAO,WADQ,SAAS,KAAK,QAAQ,MAAM,EAAE,WAAW,MAAM,IAAI,MACxC,MAAM,IAAI;AACtC;;AAGA,eAAe,UAAU,MAAc,QAAgB,MAAuC;CAC5F,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;CAC/D,MAAM,OAAkB,CAAC;CACzB,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;CAC5C,MAAM,OAAO;CACb,MAAM,OAAO,QAAQ,MAAM,MAAM;CACjC,KAAK,IAAI,SAAS,GAAG,SAAS,QAAQ,KAAK,SAAS,QAAQ,UAAU,MAAM;EAC1E,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM;EACjE,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO,WAAW,OAAO,MAAM,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EAGrH,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,CAAC;EAC1B,IAAI,IAAI,WAAW,GAAG;EACtB,KAAK,MAAM,KAAK,KAAK;GACnB,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,OAAO,GAAG;GACtC,KAAK,KAAK,EAAE,GAAG;EACjB;EACA,IAAI,IAAI,SAAS,MAAM;CACzB;CACA,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,EAAE,OAAO,KAAK,GAAG,QAAQ;CACzH,OAAO;AACT;;;;;;;;;;;AAYA,eAAe,SAAS,MAAgB,UAAuC;CAI7E,MAAM,eAAe,KACnB,WACA,oBACA,GAAG,KAAK,OAAO,GAAG,QAAQ,IAAI,GAAG,eAAe,MAClD;CACA,MAAM,eAAe,cAAc,IAAI;CACvC,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,OAAO;GAAC;GAAS;GAAe;EAAY,GAAG,UAAU,EAAE,KAAK,UAAU,CAAC;CAC/G,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MACR,sCAAsC,KAAK,OAAO,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,EAAE,qBAC1E,KAAK,OAAO,yEAC4B,KAAK,OAAO,sFACK,KAAK,OAAO,sKAG7F;CACF,UAAU;EACR,MAAM,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CACxD;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAMlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,qCAAqC,KAAK,OAAO,IAAI,OAAO,OAAO;CACrG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,uDAAuD,OAAO,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;CAChE,OAAO;EACL,UAAU,OAAO,YAAY,QAAQ,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;EAClF;EACA,QAAQ,KAAK,UAAU;GAAE,QAAQ,KAAK;GAAQ,QAAQ,KAAK;GAAQ,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACjH;AACF;AAEA,eAAe,eAAe,MAAc,MAA+B;CACzE,MAAM,MAAM,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CACjD,MAAM,UACJ,MACA,KAAK,UAAU;EAAE,SAAS,KAAK;EAAQ,QAAQ,KAAK;EAAQ,oBAAoB,KAAK;EAAS,WAAW,KAAK;CAAU,CAAC,CAC3H;AACF;;AAGA,eAAe,aAAa,SAA4B,QAA+B;CACrF,KAAK,MAAM,KAAK,SAAS;EACvB,MAAM,MAAM,GAAG,EAAE,eAAe,QAAQ,OAAO,EAAE,EAAE;EACnD,IAAI;GAKF,CAAK,MAHa,MAAM,KAAK;IAAE,QAAQ;IAAQ,SAAS,EAAE,gBAAgB,mBAAmB;IAAG,MAAM;GAAK,CAAC,EAAA,CAGnG;EACX,SAAS,KAAK;GACZ,MAAM,IAAI,MACR,gCAAgC,EAAE,gBAAgB,kBAAkB,IAAI,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,kEAClD,OAAO,wFACO,OAAO,qHAE1F;EACF;CACF;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,kBAAkB;CACnD,MAAM,OAAO,QAAQ,IAAI,aAAa;CAEtC,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAGhB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,uBAAuB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC3G,CAAC;IACD;GACF;GAIA,MAAM,SAAS,MAAM,aAAa,MAAM;IAAE,OAAO;IAAgB,OAAO;GAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAC3F,MAAM,UAAU,OAAO,KAAK,SAAS,OAAO,EAAE,CAAC,CAAC,UAAU,CAAC;GAC3D,IAAI,QAAQ,WAAW,GACrB,MAAM,IAAI,MACR,wJAEF;GAEF,MAAM,aAAa,SAAS,cAAc;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,SAAS,KAAK,SAAS;GAC7B,IAAI,cAAc,OAAO,aAAa,MAAM,IAAI;GAChD,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,UAAU,MAAM,QAAQ,IAAI;GAC3C,SAAS,KAAK;IACZ,QAAQ,KACN,+CAA+C,eAAe,QAAQ,IAAI,UAAU,IAAI,yCAAyC,UACnI;IACA,OAAO,aAAa,MAAM,IAAI;GAChC;GACA,OAAO,WAAW,MAAM,MAAM,IAAI;EACpC;EAEA,MAAM,eAAe,CAOrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,SADM,SAAS,IACH,GAAG,QAAQ;EAChC;CACF;AACF"}

package/dist/benchmarks/finresearchbench.d.ts

@@ -1,15 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * FinResearchBench-compatible adapter.
- *
- * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial
- * research reports, but there is no stable public scorer package wired here.
- * Live mode therefore requires a local data export whose rows carry the official
- * judge prompt/template/logic tree. The adapter refuses to invent a judge.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/finresearchbench.d.ts
 declare function createFinResearchBenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createFinResearchBenchAdapter };
+//# sourceMappingURL=finresearchbench.d.ts.map

package/dist/benchmarks/finresearchbench.js

@@ -1,8 +1,219 @@
-import {
-  createFinResearchBenchAdapter
-} from "../chunk-7GRVHU22.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createFinResearchBenchAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/finresearchbench.ts
+/**
+* FinResearchBench-compatible adapter.
+*
+* The paper defines a logic-tree Agent-as-a-Judge benchmark for financial
+* research reports, but there is no stable public scorer package wired here.
+* Live mode therefore requires a local data export whose rows carry the official
+* judge prompt/template/logic tree. The adapter refuses to invent a judge.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "finresearchbench.json");
+const dataFile = () => process.env.FINRESEARCHBENCH_DATA_FILE;
+function routerConfig() {
+	const key = process.env.TANGLE_API_KEY;
+	if (!key) throw new Error("TANGLE_API_KEY is required for FinResearchBench live LLM judging");
+	return {
+		baseUrl: process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
+		key,
+		model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? "deepseek-v4-flash"
+	};
+}
+async function assertReadable(path, label) {
+	try {
+		await stat(path);
+	} catch (err) {
+		throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+	}
+}
+function readRecords(raw) {
+	const trimmed = raw.trim();
+	if (trimmed.startsWith("[")) return JSON.parse(trimmed);
+	return trimmed.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+}
+function assertLiveJudgeFields(records, source) {
+	const missing = records.filter((row) => !row.judge_system_prompt || !row.judge_prompt_template).map((row) => row.id);
+	if (missing.length > 0) throw new Error(`FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(", ")}. Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.`);
+}
+function rowToTask(row, fixturesMode) {
+	const referenceAnswer = row.reference_answer ?? "";
+	const referenceReport = row.reference_report ?? referenceAnswer;
+	const meta = {
+		id: row.id,
+		category: row.category ?? "unknown",
+		question: row.question,
+		referenceAnswer,
+		referenceReport,
+		logicTree: row.logic_tree ?? null,
+		rubric: row.rubric ?? null,
+		judgeSystemPrompt: row.judge_system_prompt,
+		judgePromptTemplate: row.judge_prompt_template,
+		scoring: fixturesMode ? "fixture-exact-reference" : "official-logic-tree-judge"
+	};
+	return {
+		id: row.id,
+		split: row.category,
+		prompt: [
+			"Complete this FinResearchBench financial research task.",
+			"Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.",
+			"",
+			row.question
+		].join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.question !== "string") throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function selectRows(rows, opts, fixturesMode) {
+	let tasks = rows.map((row) => rowToTask(row, fixturesMode));
+	if (opts.split) tasks = tasks.filter((task) => task.split === opts.split);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((task) => want.has(task.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`);
+	return tasks;
+}
+async function loadFixtures(opts) {
+	const records = readRecords(await readFile(FIXTURES, "utf8"));
+	console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`);
+	return selectRows(records, opts, true);
+}
+async function loadOfficialTasks(path, opts) {
+	const records = readRecords(await readFile(path, "utf8"));
+	assertLiveJudgeFields(records, path);
+	return selectRows(records, opts, false);
+}
+function fillTemplate(meta, response) {
+	const template = meta.judgePromptTemplate;
+	if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`);
+	return template.replaceAll("{question}", meta.question).replaceAll("{response}", response).replaceAll("{reference_answer}", meta.referenceAnswer).replaceAll("{reference_report}", meta.referenceReport).replaceAll("{logic_tree}", JSON.stringify(meta.logicTree, null, 2)).replaceAll("{rubric}", JSON.stringify(meta.rubric, null, 2));
+}
+function parseJudgeScore(content) {
+	const candidates = [];
+	for (const m of content.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)) candidates.push(m[1].trim());
+	for (const m of content.matchAll(/\{[\s\S]*?\}/g)) candidates.push(m[0]);
+	candidates.push(content.trim());
+	for (const candidate of candidates) {
+		let parsed;
+		try {
+			parsed = JSON.parse(candidate);
+		} catch {
+			continue;
+		}
+		const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score;
+		const n = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : NaN;
+		if (!Number.isFinite(n)) continue;
+		if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`);
+		const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100;
+		if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`);
+		return {
+			score,
+			raw: parsed
+		};
+	}
+	throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`);
+}
+async function runOfficialJudge(meta, response) {
+	if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`);
+	const router = routerConfig();
+	const res = await fetch(`${router.baseUrl}/chat/completions`, {
+		method: "POST",
+		headers: {
+			"content-type": "application/json",
+			authorization: `Bearer ${router.key}`
+		},
+		body: JSON.stringify({
+			model: router.model,
+			temperature: 0,
+			messages: [{
+				role: "system",
+				content: meta.judgeSystemPrompt
+			}, {
+				role: "user",
+				content: fillTemplate(meta, response)
+			}]
+		})
+	});
+	if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+	const body = await res.json();
+	const content = body.choices?.[0]?.message?.content;
+	if (typeof content !== "string") throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const { score, raw } = parseJudgeScore(content);
+	return {
+		resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? .8),
+		score,
+		detail: JSON.stringify({
+			scoring: meta.scoring,
+			category: meta.category,
+			judgeModel: router.model,
+			raw
+		})
+	};
+}
+function normalizeText(value) {
+	return value.toLowerCase().replace(/\s+/g, " ").trim();
+}
+function scoreFixture(meta, artifact) {
+	const answer = normalizeText(meta.referenceAnswer || meta.referenceReport);
+	const response = normalizeText(artifact);
+	const score = answer.length > 0 && response.includes(answer) ? 1 : 0;
+	return {
+		resolved: score === 1,
+		score,
+		detail: JSON.stringify({
+			scoring: meta.scoring,
+			category: meta.category
+		})
+	};
+}
+function createFinResearchBenchAdapter() {
+	const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === "1";
+	return {
+		name: "finresearchbench",
+		async preflight() {
+			if (fixturesMode) {
+				await assertReadable(FIXTURES, "fixture file");
+				return;
+			}
+			const file = dataFile();
+			if (!file) throw new Error("FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.");
+			routerConfig();
+			await assertReadable(file, "data file");
+			await loadOfficialTasks(file, { limit: 1 });
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const file = dataFile();
+			if (!file) throw new Error("FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows");
+			return loadOfficialTasks(file, opts);
+		},
+		async goldArtifact(task) {
+			const meta = readMeta(task);
+			return meta.referenceReport || meta.referenceAnswer || void 0;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			if (fixturesMode) return scoreFixture(meta, artifact);
+			if (artifact.trim().length === 0) return {
+				resolved: false,
+				score: 0,
+				detail: JSON.stringify({
+					scoring: meta.scoring,
+					category: meta.category,
+					reason: "empty answer"
+				})
+			};
+			return runOfficialJudge(meta, artifact);
+		}
+	};
+}
+//#endregion
+export { createFinResearchBenchAdapter };
+
 //# sourceMappingURL=finresearchbench.js.map