@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-XKEFIFIC.js

@@ -1,197 +0,0 @@
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/toollm.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "toollm.json");
-var DEFAULT_QUERY_REL = join("data_example", "instruction", "G1_query.json");
-var toolbenchDir = () => process.env.TOOLBENCH_DIR;
-var queryFile = (dir) => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL);
-var toollmOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    return text.trim();
-  }
-};
-function rowToTask(row) {
-  const relevantApis = normalizeApiPairs(row["relevant APIs"] ?? []);
-  const meta = {
-    queryId: String(row.query_id),
-    apiList: row.api_list ?? [],
-    relevantApis,
-    deterministicJudge: "api-selection"
-  };
-  return {
-    id: String(row.query_id),
-    prompt: [
-      "Solve this ToolLLM/ToolBench API-use task.",
-      "Use only the listed APIs/tools and return the completed tool-use trace plus final answer.",
-      "",
-      `Query: ${row.query}`,
-      "",
-      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,
-      "",
-      'Return the APIs you used as JSON: {"api_calls":[{"tool_name":"...","api_name":"..."}]}.'
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function normalizeApiPart(value) {
-  return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
-}
-function apiKey(pair) {
-  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`;
-}
-function normalizeApiPairs(value) {
-  if (!Array.isArray(value)) return [];
-  const out = [];
-  for (const item of value) {
-    if (Array.isArray(item) && typeof item[0] === "string" && typeof item[1] === "string") {
-      out.push([item[0], item[1]]);
-    } else if (item && typeof item === "object" && typeof item.tool_name === "string" && typeof item.api_name === "string") {
-      out.push([item.tool_name, item.api_name]);
-    }
-  }
-  return out;
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.relevantApis)) {
-    throw new Error(`ToolLLM task ${task.id} missing metadata \u2014 loadTasks did not populate deterministic API-selection labels`);
-  }
-  return md;
-}
-function selectRows(rows, opts) {
-  let tasks = rows.map(rowToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`);
-  return tasks;
-}
-function assertDeterministicSubset(tasks, source) {
-  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id);
-  if (missing.length > 0) {
-    throw new Error(
-      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(", ")}. Use a ToolBench query file that includes "relevant APIs" labels, or do not score ToolLLM in agent-bench.`
-    );
-  }
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[toollm] TOOLLM_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  const tasks = selectRows(rows, opts);
-  assertDeterministicSubset(tasks, FIXTURES);
-  return tasks;
-}
-async function loadOfficialTasks(dir, opts) {
-  const source = queryFile(dir);
-  const tasks = selectRows(JSON.parse(await readFile(source, "utf8")), opts);
-  assertDeterministicSubset(tasks, source);
-  return tasks;
-}
-function extractJsonBlock(text) {
-  const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)];
-  const raw = (fences.at(-1)?.[1] ?? text).trim();
-  try {
-    return JSON.parse(raw);
-  } catch {
-    return void 0;
-  }
-}
-function extractCalledApis(text, expected) {
-  const parsed = extractJsonBlock(text);
-  if (parsed && typeof parsed === "object") {
-    const raw = parsed;
-    const fromApiCalls = normalizeApiPairs(raw.api_calls);
-    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: "structured-json" };
-    const fromCalls = normalizeApiPairs(raw.calls);
-    if (fromCalls.length > 0) return { pairs: fromCalls, source: "structured-json" };
-  }
-  const lower = text.toLowerCase();
-  return {
-    pairs: expected.filter(([tool, api]) => {
-      const toolNeedle = normalizeApiPart(tool);
-      const apiNeedle = normalizeApiPart(api);
-      const compactText = lower.replace(/[^a-z0-9]+/g, "");
-      return compactText.includes(`${toolNeedle}${apiNeedle}`) || lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase());
-    }),
-    source: "text-mention"
-  };
-}
-function scoreApiSelection(task, artifact) {
-  const meta = readMeta(task);
-  if (meta.relevantApis.length === 0) {
-    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`);
-  }
-  const expected = new Set(meta.relevantApis.map(apiKey));
-  const extracted = extractCalledApis(artifact, meta.relevantApis);
-  const calledPairs = extracted.pairs;
-  const called = new Set(calledPairs.map(apiKey));
-  const truePositives = [...called].filter((key) => expected.has(key)).length;
-  const precision = called.size === 0 ? 0 : truePositives / called.size;
-  const recall = truePositives / expected.size;
-  const score = expected.size === 0 ? 0 : recall;
-  const resolved = extracted.source === "structured-json" && recall === 1 && precision === 1;
-  return {
-    resolved,
-    score,
-    detail: JSON.stringify({
-      scoring: "api-selection-only",
-      extractionSource: extracted.source,
-      queryId: meta.queryId,
-      expected: meta.relevantApis,
-      called: calledPairs,
-      precision,
-      recall,
-      fullToolEvalScore: null
-    })
-  };
-}
-function createToolLlmAdapter() {
-  const fixturesMode = process.env.TOOLLM_FIXTURES === "1";
-  return {
-    name: "toollm",
-    output: toollmOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = toolbenchDir();
-      if (!dir) {
-        throw new Error("TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.");
-      }
-      await loadOfficialTasks(dir, { limit: 1 });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const dir = toolbenchDir();
-      if (!dir) throw new Error("TOOLBENCH_DIR is required to load ToolLLM tasks");
-      return loadOfficialTasks(dir, opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      if (meta.relevantApis.length === 0) return void 0;
-      return JSON.stringify({
-        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name }))
-      }, null, 2);
-    },
-    async judge(task, artifact) {
-      return scoreApiSelection(task, artifact);
-    }
-  };
-}
-
-export {
-  toollmOutput,
-  createToolLlmAdapter
-};
-//# sourceMappingURL=chunk-XKEFIFIC.js.map

package/dist/chunk-XKEFIFIC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/toollm.ts"],"sourcesContent":["/**\n * ToolLLM/ToolBench adapter.\n *\n * ToolBench task loading is useful for breadth, but the official ToolEval pass\n * rate evaluator is LLM-based and stochastic. This adapter therefore scores\n * only ToolBench's deterministic API-selection labels (`relevant APIs`). It\n * never records a full ToolEval pass-rate score.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')\nconst DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')\n\ninterface ToolApi {\n  category_name?: string\n  tool_name: string\n  api_name: string\n  api_description?: string\n  required_parameters?: unknown[]\n  optional_parameters?: unknown[]\n  method?: string\n}\n\ninterface ToolBenchRow {\n  query_id: number | string\n  query: string\n  api_list?: ToolApi[]\n  'relevant APIs'?: Array<[string, string]>\n}\n\ninterface ToolBenchMeta {\n  queryId: string\n  apiList: ToolApi[]\n  relevantApis: Array<[string, string]>\n  deterministicJudge: 'api-selection'\n}\n\nconst toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR\nconst queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)\n\nexport const toollmOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nfunction rowToTask(row: ToolBenchRow): BenchTask {\n  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])\n  const meta: ToolBenchMeta = {\n    queryId: String(row.query_id),\n    apiList: row.api_list ?? [],\n    relevantApis,\n    deterministicJudge: 'api-selection',\n  }\n  return {\n    id: String(row.query_id),\n    prompt: [\n      'Solve this ToolLLM/ToolBench API-use task.',\n      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',\n      '',\n      `Query: ${row.query}`,\n      '',\n      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,\n      '',\n      'Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction normalizeApiPart(value: string): string {\n  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')\n}\n\nfunction apiKey(pair: readonly [string, string]): string {\n  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`\n}\n\nfunction normalizeApiPairs(value: unknown): Array<[string, string]> {\n  if (!Array.isArray(value)) return []\n  const out: Array<[string, string]> = []\n  for (const item of value) {\n    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {\n      out.push([item[0], item[1]])\n    } else if (\n      item && typeof item === 'object'\n      && typeof (item as { tool_name?: unknown }).tool_name === 'string'\n      && typeof (item as { api_name?: unknown }).api_name === 'string'\n    ) {\n      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])\n    }\n  }\n  return out\n}\n\nfunction readMeta(task: BenchTask): ToolBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.relevantApis)) {\n    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)\n  }\n  return md as unknown as ToolBenchMeta\n}\n\nfunction selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nfunction assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {\n  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a ToolBench query file that includes \"relevant APIs\" labels, or do not score ToolLLM in agent-bench.',\n    )\n  }\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]\n  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  const tasks = selectRows(rows, opts)\n  assertDeterministicSubset(tasks, FIXTURES)\n  return tasks\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const source = queryFile(dir)\n  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)\n  assertDeterministicSubset(tasks, source)\n  return tasks\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\ninterface ExtractedApis {\n  pairs: Array<[string, string]>\n  source: 'structured-json' | 'text-mention'\n}\n\nfunction extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {\n  const parsed = extractJsonBlock(text)\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    const fromApiCalls = normalizeApiPairs(raw.api_calls)\n    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }\n    const fromCalls = normalizeApiPairs(raw.calls)\n    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }\n  }\n\n  const lower = text.toLowerCase()\n  return {\n    pairs: expected.filter(([tool, api]) => {\n      const toolNeedle = normalizeApiPart(tool)\n      const apiNeedle = normalizeApiPart(api)\n      const compactText = lower.replace(/[^a-z0-9]+/g, '')\n      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))\n    }),\n    source: 'text-mention',\n  }\n}\n\nfunction scoreApiSelection(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  if (meta.relevantApis.length === 0) {\n    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)\n  }\n  const expected = new Set(meta.relevantApis.map(apiKey))\n  const extracted = extractCalledApis(artifact, meta.relevantApis)\n  const calledPairs = extracted.pairs\n  const called = new Set(calledPairs.map(apiKey))\n  const truePositives = [...called].filter((key) => expected.has(key)).length\n  const precision = called.size === 0 ? 0 : truePositives / called.size\n  const recall = truePositives / expected.size\n  const score = expected.size === 0 ? 0 : recall\n  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      scoring: 'api-selection-only',\n      extractionSource: extracted.source,\n      queryId: meta.queryId,\n      expected: meta.relevantApis,\n      called: calledPairs,\n      precision,\n      recall,\n      fullToolEvalScore: null,\n    }),\n  }\n}\n\nexport function createToolLlmAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'\n\n  return {\n    name: 'toollm',\n    output: toollmOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = toolbenchDir()\n      if (!dir) {\n        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')\n      }\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = toolbenchDir()\n      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      if (meta.relevantApis.length === 0) return undefined\n      return JSON.stringify({\n        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreApiSelection(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,aAAa;AAC1D,IAAM,oBAAoB,KAAK,gBAAgB,eAAe,eAAe;AA0B7E,IAAM,eAAe,MAA0B,QAAQ,IAAI;AAC3D,IAAM,YAAY,CAAC,QAAwB,QAAQ,IAAI,qBAAqB,KAAK,KAAK,iBAAiB;AAEhG,IAAM,eAAsC;AAAA,EACjD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,UAAU,KAA8B;AAC/C,QAAM,eAAe,kBAAkB,IAAI,eAAe,KAAK,CAAC,CAAC;AACjE,QAAM,OAAsB;AAAA,IAC1B,SAAS,OAAO,IAAI,QAAQ;AAAA,IAC5B,SAAS,IAAI,YAAY,CAAC;AAAA,IAC1B;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,QAAQ;AAAA,IACvB,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,IAAI,KAAK;AAAA,MACnB;AAAA,MACA,mBAAmB,KAAK,UAAU,IAAI,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,MAC9D;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,iBAAiB,OAAuB;AAC/C,SAAO,MAAM,YAAY,EAAE,QAAQ,eAAe,EAAE;AACtD;AAEA,SAAS,OAAO,MAAyC;AACvD,SAAO,GAAG,iBAAiB,KAAK,CAAC,CAAC,CAAC,IAAI,iBAAiB,KAAK,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,kBAAkB,OAAyC;AAClE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,MAA+B,CAAC;AACtC,aAAW,QAAQ,OAAO;AACxB,QAAI,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAK,CAAC,MAAM,YAAY,OAAO,KAAK,CAAC,MAAM,UAAU;AACrF,UAAI,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAAA,IAC7B,WACE,QAAQ,OAAO,SAAS,YACrB,OAAQ,KAAiC,cAAc,YACvD,OAAQ,KAAgC,aAAa,UACxD;AACA,UAAI,KAAK,CAAE,KAA+B,WAAY,KAA8B,QAAQ,CAAC;AAAA,IAC/F;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,SAAS,MAAgC;AAChD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,YAAY,GAAG;AAC1C,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wFAAmF;AAAA,EAC5H;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAsB,MAAgC;AACxE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,CAAC,EAAE;AAC3F,SAAO;AACT;AAEA,SAAS,0BAA0B,OAA6B,QAAsB;AACpF,QAAM,UAAU,MAAM,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,aAAa,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,KAAK,EAAE;AACtG,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,0DAA0D,QAAQ,MAAM,IAAI,MAAM,MAAM,iBAAiB,MAAM,KAAK,QAAQ,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IAEpJ;AAAA,EACF;AACF;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,6CAAwC,KAAK,MAAM,mBAAmB;AACnF,QAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,4BAA0B,OAAO,QAAQ;AACzC,SAAO;AACT;AAEA,eAAe,kBAAkB,KAAa,MAAyC;AACrF,QAAM,SAAS,UAAU,GAAG;AAC5B,QAAM,QAAQ,WAAW,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,CAAC,GAAqB,IAAI;AAC3F,4BAA0B,OAAO,MAAM;AACvC,SAAO;AACT;AAEA,SAAS,iBAAiB,MAAuB;AAC/C,QAAM,SAAS,CAAC,GAAG,KAAK,SAAS,iCAAiC,CAAC;AACnE,QAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAC9C,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAOA,SAAS,kBAAkB,MAAc,UAAsD;AAC7F,QAAM,SAAS,iBAAiB,IAAI;AACpC,MAAI,UAAU,OAAO,WAAW,UAAU;AACxC,UAAM,MAAM;AACZ,UAAM,eAAe,kBAAkB,IAAI,SAAS;AACpD,QAAI,aAAa,SAAS,EAAG,QAAO,EAAE,OAAO,cAAc,QAAQ,kBAAkB;AACrF,UAAM,YAAY,kBAAkB,IAAI,KAAK;AAC7C,QAAI,UAAU,SAAS,EAAG,QAAO,EAAE,OAAO,WAAW,QAAQ,kBAAkB;AAAA,EACjF;AAEA,QAAM,QAAQ,KAAK,YAAY;AAC/B,SAAO;AAAA,IACL,OAAO,SAAS,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM;AACtC,YAAM,aAAa,iBAAiB,IAAI;AACxC,YAAM,YAAY,iBAAiB,GAAG;AACtC,YAAM,cAAc,MAAM,QAAQ,eAAe,EAAE;AACnD,aAAO,YAAY,SAAS,GAAG,UAAU,GAAG,SAAS,EAAE,KAAM,MAAM,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC;AAAA,IACrI,CAAC;AAAA,IACD,QAAQ;AAAA,EACV;AACF;AAEA,SAAS,kBAAkB,MAAiB,UAA8B;AACxE,QAAM,OAAO,SAAS,IAAI;AAC1B,MAAI,KAAK,aAAa,WAAW,GAAG;AAClC,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,+DAA+D;AAAA,EACxG;AACA,QAAM,WAAW,IAAI,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC;AACtD,QAAM,YAAY,kBAAkB,UAAU,KAAK,YAAY;AAC/D,QAAM,cAAc,UAAU;AAC9B,QAAM,SAAS,IAAI,IAAI,YAAY,IAAI,MAAM,CAAC;AAC9C,QAAM,gBAAgB,CAAC,GAAG,MAAM,EAAE,OAAO,CAAC,QAAQ,SAAS,IAAI,GAAG,CAAC,EAAE;AACrE,QAAM,YAAY,OAAO,SAAS,IAAI,IAAI,gBAAgB,OAAO;AACjE,QAAM,SAAS,gBAAgB,SAAS;AACxC,QAAM,QAAQ,SAAS,SAAS,IAAI,IAAI;AACxC,QAAM,WAAW,UAAU,WAAW,qBAAqB,WAAW,KAAK,cAAc;AACzF,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,KAAK,UAAU;AAAA,MACrB,SAAS;AAAA,MACT,kBAAkB,UAAU;AAAA,MAC5B,SAAS,KAAK;AAAA,MACd,UAAU,KAAK;AAAA,MACf,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,MACA,mBAAmB;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,oBAAoB;AAErD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI,MAAM,sHAAsH;AAAA,MACxI;AACA,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,iDAAiD;AAC3E,aAAO,kBAAkB,KAAK,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI,KAAK,aAAa,WAAW,EAAG,QAAO;AAC3C,aAAO,KAAK,UAAU;AAAA,QACpB,WAAW,KAAK,aAAa,IAAI,CAAC,CAAC,WAAW,QAAQ,OAAO,EAAE,WAAW,SAAS,EAAE;AAAA,MACvF,GAAG,MAAM,CAAC;AAAA,IACZ;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,aAAO,kBAAkB,MAAM,QAAQ;AAAA,IACzC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-XYA4XSNU.js

@@ -1,148 +0,0 @@
-import {
-  benchRoot,
-  preflightVenvImports,
-  readJsonReport,
-  runStagedJudge,
-  runVenvPython,
-  safeRunId,
-  stageFile,
-  venvPythonAt
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/terminal-bench.ts
-import { join } from "path";
-var terminalBenchVenvDir = () => process.env.TERMINAL_BENCH_VENV ?? ".venv-terminal-bench";
-var terminalBenchPython = () => venvPythonAt(terminalBenchVenvDir());
-var terminalBenchBin = () => join(benchRoot, terminalBenchVenvDir(), "bin", "tb");
-var DATASET = "terminal-bench-core";
-var DATASET_VERSION = "0.1.1";
-var DATASET_REF = `${DATASET}==${DATASET_VERSION}`;
-var FIXTURE_IDS = ["hello-world"];
-var SCRIPT_AGENT = "tb_agents.script_agent:ScriptAgent";
-async function loadRows(opts) {
-  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS);
-  const limit = opts.limit ?? null;
-  const script = `
-import json, sys
-from pathlib import Path
-from terminal_bench.dataset.dataset import Dataset
-from terminal_bench.handlers.trial_handler import TaskPaths
-
-req_ids = json.loads(sys.argv[1]) if sys.argv[1] else None
-limit = json.loads(sys.argv[2]) if sys.argv[2] else None
-
-ds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)
-
-import yaml
-out = []
-for task_dir in ds:
-    tp = TaskPaths(task_dir)
-    cfg = yaml.safe_load(tp.task_config_path.read_text())
-    try:
-        sol = tp.solution_path.read_text()
-    except FileNotFoundError:
-        sol = None
-    out.append({
-        "id": task_dir.name,
-        "instruction": cfg["instruction"],
-        "task_dir": str(task_dir),
-        "solution": sol if (tp.solution_path.suffix == ".sh" if sol is not None else False) else None,
-    })
-print(json.dumps(out))
-`;
-  const stdout = await runVenvPython(
-    script,
-    [ids ? JSON.stringify(ids) : "", limit !== null ? String(limit) : ""],
-    0,
-    terminalBenchPython()
-  );
-  return JSON.parse(stdout);
-}
-function createTerminalBenchAdapter() {
-  return {
-    name: "terminal-bench",
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["terminal_bench"],
-        requireDocker: true,
-        python: terminalBenchPython(),
-        fix: `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench (an ISOLATED venv \u2014 Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; override the dir with TERMINAL_BENCH_VENV) ; (2) ensure the Docker daemon is running (the judge builds per-task images on first run). The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`
-      });
-    },
-    async loadTasks(opts = {}) {
-      const rows = await loadRows(opts);
-      if (rows.length === 0) {
-        throw new Error(
-          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} (dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`
-        );
-      }
-      return rows.map(
-        (r) => ({
-          id: r.id,
-          split: DATASET_VERSION,
-          prompt: r.instruction,
-          metadata: {
-            dataset: DATASET,
-            datasetVersion: DATASET_VERSION,
-            datasetRef: DATASET_REF,
-            taskDir: r.task_dir,
-            solution: r.solution,
-            instruction: r.instruction
-          }
-        })
-      );
-    },
-    async goldArtifact(task) {
-      const sol = task.metadata?.solution;
-      return typeof sol === "string" ? sol : void 0;
-    },
-    async judge(task, artifact) {
-      const runId = safeRunId("bench", `${task.id}-${Date.now()}`);
-      return runStagedJudge({
-        tmpPrefix: "tbench-",
-        bin: terminalBenchBin(),
-        cwd: () => benchRoot,
-        async stage(dir) {
-          await stageFile(join(dir, "attempt.sh"), artifact);
-        },
-        // The harness builds a fresh task container, runs ScriptAgent (which replays
-        // the artifact script), then runs the task's verifier. --no-livestream keeps
-        // stdout sane; --cleanup removes the per-run images.
-        argv: (dir) => [
-          "run",
-          "-d",
-          DATASET_REF,
-          "-t",
-          task.id,
-          "--agent-import-path",
-          SCRIPT_AGENT,
-          "--agent-kwarg",
-          `script_path=${join(dir, "attempt.sh")}`,
-          "--output-path",
-          join(dir, "runs"),
-          "--run-id",
-          runId,
-          "--n-concurrent",
-          "1",
-          "--no-livestream",
-          "--cleanup"
-        ],
-        async parseReport(dir) {
-          const report = await readJsonReport(join(dir, "runs", runId, "results.json"));
-          const resolved = (report.resolved_ids ?? []).includes(task.id);
-          const trial = report.results?.find((r) => r.task_id === task.id);
-          return {
-            resolved,
-            score: resolved ? 1 : 0,
-            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {})
-          };
-        }
-      });
-    }
-  };
-}
-
-export {
-  createTerminalBenchAdapter
-};
-//# sourceMappingURL=chunk-XYA4XSNU.js.map

package/dist/chunk-XYA4XSNU.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/terminal-bench.ts"],"sourcesContent":["/**\n * Terminal-Bench adapter. Each task = a Docker environment + an English\n * instruction + a per-task verifier (run-tests.sh + pytest). The worker artifact\n * is the shell script the agent ran to attempt the task; the judge REPLAYS that\n * script in a fresh task container via the Terminal-Bench harness (`tb run` with\n * our ScriptAgent), then the task's own verifier scores the resulting state.\n * Fully deterministic — no LLM judge, no self-authored score.\n *\n * Requires: an isolated bench `.venv-terminal-bench` with `terminal-bench`\n * installed + a running Docker daemon (per-task images are built on first run).\n * Override with TERMINAL_BENCH_VENV. loadTasks caches the dataset from the\n * Terminal-Bench registry on first run.\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * Terminal-Bench-specific pieces: the Dataset enumeration, the ScriptAgent replay\n * argv, and the results.json shape.\n */\n\nimport { join } from 'node:path'\nimport {\n  benchRoot,\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  venvPythonAt,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n// Terminal-Bench imports LiteLLM/Pydantic-2 APIs, while AppWorld pins Pydantic 1.\n// Keep it out of the shared bench .venv. Resolved at call-time so tests/runs can\n// override the env without reloading this module.\nconst terminalBenchVenvDir = (): string => process.env.TERMINAL_BENCH_VENV ?? '.venv-terminal-bench'\nconst terminalBenchPython = (): string => venvPythonAt(terminalBenchVenvDir())\nconst terminalBenchBin = (): string => join(benchRoot, terminalBenchVenvDir(), 'bin', 'tb')\n\n// Pinned dataset: the 0.1.1 core set is patched for terminal-bench >=0.2.4 (the\n// installed CLI) and is the published launch task set. name==version is what `tb\n// run -d` and `tb datasets download -d` both accept.\nconst DATASET = 'terminal-bench-core'\nconst DATASET_VERSION = '0.1.1'\nconst DATASET_REF = `${DATASET}==${DATASET_VERSION}`\n\n// Bundled fixture: when no ids/limit are given, load these. hello-world is the\n// fastest deterministic task (prebuilt python image, file-write verifier) so the\n// adapter is runnable without a large pull.\nconst FIXTURE_IDS = ['hello-world']\n\n// Import path the harness uses to load our replay agent (cwd = benchRoot).\nconst SCRIPT_AGENT = 'tb_agents.script_agent:ScriptAgent'\n\ninterface TbTaskRow {\n  id: string\n  instruction: string\n  task_dir: string\n  solution: string | null\n}\n\ninterface TbReport {\n  resolved_ids?: string[]\n  results?: Array<{ task_id: string; is_resolved: boolean | null; parser_results?: unknown }>\n}\n\n/** Enumerate dataset tasks via tb's Dataset loader (caches from the registry on\n *  first run). Reads instruction from each task.yaml and the gold solution. */\nasync function loadRows(opts: LoadOptions): Promise<TbTaskRow[]> {\n  const ids = opts.ids ?? (opts.limit ? null : FIXTURE_IDS)\n  const limit = opts.limit ?? null\n  const script = `\nimport json, sys\nfrom pathlib import Path\nfrom terminal_bench.dataset.dataset import Dataset\nfrom terminal_bench.handlers.trial_handler import TaskPaths\n\nreq_ids = json.loads(sys.argv[1]) if sys.argv[1] else None\nlimit = json.loads(sys.argv[2]) if sys.argv[2] else None\n\nds = Dataset(name=${JSON.stringify(DATASET)}, version=${JSON.stringify(DATASET_VERSION)}, task_ids=req_ids, n_tasks=limit)\n\nimport yaml\nout = []\nfor task_dir in ds:\n    tp = TaskPaths(task_dir)\n    cfg = yaml.safe_load(tp.task_config_path.read_text())\n    try:\n        sol = tp.solution_path.read_text()\n    except FileNotFoundError:\n        sol = None\n    out.append({\n        \"id\": task_dir.name,\n        \"instruction\": cfg[\"instruction\"],\n        \"task_dir\": str(task_dir),\n        \"solution\": sol if (tp.solution_path.suffix == \".sh\" if sol is not None else False) else None,\n    })\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(\n    script,\n    [ids ? JSON.stringify(ids) : '', limit !== null ? String(limit) : ''],\n    0,\n    terminalBenchPython(),\n  )\n  return JSON.parse(stdout) as TbTaskRow[]\n}\n\nexport function createTerminalBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'terminal-bench',\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['terminal_bench'],\n        requireDocker: true,\n        python: terminalBenchPython(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${terminalBenchVenvDir()} && ` +\n          `bench/${terminalBenchVenvDir()}/bin/pip install terminal-bench ` +\n          `(an ISOLATED venv — Terminal-Bench/LiteLLM require Pydantic 2 while AppWorld pins Pydantic 1; ` +\n          `override the dir with TERMINAL_BENCH_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-task images on first run). ` +\n          `The ${DATASET_REF} dataset is cached from the Terminal-Bench registry on first loadTasks.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const rows = await loadRows(opts)\n      if (rows.length === 0) {\n        throw new Error(\n          `terminal-bench loadTasks returned no tasks for ${JSON.stringify(opts)} ` +\n            `(dataset ${DATASET_REF}). Check the requested ids exist in the dataset.`,\n        )\n      }\n      return rows.map(\n        (r): BenchTask => ({\n          id: r.id,\n          split: DATASET_VERSION,\n          prompt: r.instruction,\n          metadata: {\n            dataset: DATASET,\n            datasetVersion: DATASET_VERSION,\n            datasetRef: DATASET_REF,\n            taskDir: r.task_dir,\n            solution: r.solution,\n            instruction: r.instruction,\n          },\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the task's solution.sh (the oracle script). solution.yaml tasks have\n      // no shell-script artifact form here, so they return undefined (cannot be\n      // verify-judged via the script-replay seam — use a .sh-solution task).\n      const sol = task.metadata?.solution\n      return typeof sol === 'string' ? sol : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', `${task.id}-${Date.now()}`)\n      return runStagedJudge({\n        tmpPrefix: 'tbench-',\n        bin: terminalBenchBin(),\n        cwd: () => benchRoot,\n        async stage(dir) {\n          await stageFile(join(dir, 'attempt.sh'), artifact)\n        },\n        // The harness builds a fresh task container, runs ScriptAgent (which replays\n        // the artifact script), then runs the task's verifier. --no-livestream keeps\n        // stdout sane; --cleanup removes the per-run images.\n        argv: (dir) => [\n          'run',\n          '-d', DATASET_REF,\n          '-t', task.id,\n          '--agent-import-path', SCRIPT_AGENT,\n          '--agent-kwarg', `script_path=${join(dir, 'attempt.sh')}`,\n          '--output-path', join(dir, 'runs'),\n          '--run-id', runId,\n          '--n-concurrent', '1',\n          '--no-livestream',\n          '--cleanup',\n        ],\n        async parseReport(dir) {\n          const report = await readJsonReport<TbReport>(join(dir, 'runs', runId, 'results.json'))\n          const resolved = (report.resolved_ids ?? []).includes(task.id)\n          const trial = report.results?.find((r) => r.task_id === task.id)\n          return {\n            resolved,\n            score: resolved ? 1 : 0,\n            detail: JSON.stringify(trial?.parser_results ?? report.resolved_ids ?? {}),\n          }\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAkBA,SAAS,YAAY;AAgBrB,IAAM,uBAAuB,MAAc,QAAQ,IAAI,uBAAuB;AAC9E,IAAM,sBAAsB,MAAc,aAAa,qBAAqB,CAAC;AAC7E,IAAM,mBAAmB,MAAc,KAAK,WAAW,qBAAqB,GAAG,OAAO,IAAI;AAK1F,IAAM,UAAU;AAChB,IAAM,kBAAkB;AACxB,IAAM,cAAc,GAAG,OAAO,KAAK,eAAe;AAKlD,IAAM,cAAc,CAAC,aAAa;AAGlC,IAAM,eAAe;AAgBrB,eAAe,SAAS,MAAyC;AAC/D,QAAM,MAAM,KAAK,QAAQ,KAAK,QAAQ,OAAO;AAC7C,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oBASG,KAAK,UAAU,OAAO,CAAC,aAAa,KAAK,UAAU,eAAe,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBrF,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA,CAAC,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,UAAU,OAAO,OAAO,KAAK,IAAI,EAAE;AAAA,IACpE;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO,KAAK,MAAM,MAAM;AAC1B;AAEO,SAAS,6BAA+C;AAC7D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,gBAAgB;AAAA,QAC1B,eAAe;AAAA,QACf,QAAQ,oBAAoB;AAAA,QAC5B,KACE,kCAAkC,qBAAqB,CAAC,aAC/C,qBAAqB,CAAC,gRAIxB,WAAW;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,OAAO,MAAM,SAAS,IAAI;AAChC,UAAI,KAAK,WAAW,GAAG;AACrB,cAAM,IAAI;AAAA,UACR,kDAAkD,KAAK,UAAU,IAAI,CAAC,aACxD,WAAW;AAAA,QAC3B;AAAA,MACF;AACA,aAAO,KAAK;AAAA,QACV,CAAC,OAAkB;AAAA,UACjB,IAAI,EAAE;AAAA,UACN,OAAO;AAAA,UACP,QAAQ,EAAE;AAAA,UACV,UAAU;AAAA,YACR,SAAS;AAAA,YACT,gBAAgB;AAAA,YAChB,YAAY;AAAA,YACZ,SAAS,EAAE;AAAA,YACX,UAAU,EAAE;AAAA,YACZ,aAAa,EAAE;AAAA,UACjB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,MAAM,KAAK,UAAU;AAC3B,aAAO,OAAO,QAAQ,WAAW,MAAM;AAAA,IACzC;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,QAAQ,UAAU,SAAS,GAAG,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AAC3D,aAAO,eAAe;AAAA,QACpB,WAAW;AAAA,QACX,KAAK,iBAAiB;AAAA,QACtB,KAAK,MAAM;AAAA,QACX,MAAM,MAAM,KAAK;AACf,gBAAM,UAAU,KAAK,KAAK,YAAY,GAAG,QAAQ;AAAA,QACnD;AAAA;AAAA;AAAA;AAAA,QAIA,MAAM,CAAC,QAAQ;AAAA,UACb;AAAA,UACA;AAAA,UAAM;AAAA,UACN;AAAA,UAAM,KAAK;AAAA,UACX;AAAA,UAAuB;AAAA,UACvB;AAAA,UAAiB,eAAe,KAAK,KAAK,YAAY,CAAC;AAAA,UACvD;AAAA,UAAiB,KAAK,KAAK,MAAM;AAAA,UACjC;AAAA,UAAY;AAAA,UACZ;AAAA,UAAkB;AAAA,UAClB;AAAA,UACA;AAAA,QACF;AAAA,QACA,MAAM,YAAY,KAAK;AACrB,gBAAM,SAAS,MAAM,eAAyB,KAAK,KAAK,QAAQ,OAAO,cAAc,CAAC;AACtF,gBAAM,YAAY,OAAO,gBAAgB,CAAC,GAAG,SAAS,KAAK,EAAE;AAC7D,gBAAM,QAAQ,OAAO,SAAS,KAAK,CAAC,MAAM,EAAE,YAAY,KAAK,EAAE;AAC/D,iBAAO;AAAA,YACL;AAAA,YACA,OAAO,WAAW,IAAI;AAAA,YACtB,QAAQ,KAAK,UAAU,OAAO,kBAAkB,OAAO,gBAAgB,CAAC,CAAC;AAAA,UAC3E;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-YSMEKBTD.js

@@ -1,211 +0,0 @@
-import {
-  benchRoot,
-  runVenvScriptStdin
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/enterpriseops-gym.ts
-import { mkdir, readFile, rm, writeFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "enterpriseops-gym.json");
-var JUDGE = join(benchRoot, "scripts", "enterpriseops_gym_judge.py");
-var judgeCallSeq = 0;
-var DATASET = "ServiceNow-AI/EnterpriseOps-Gym";
-var DEFAULT_MODE = "oracle";
-var DEFAULT_DOMAIN = "itsm";
-var rowsApi = (mode, domain) => `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`;
-var enterpriseOpsTranscriptOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-function asArray(v) {
-  return typeof v === "string" ? JSON.parse(v) : v;
-}
-function workerContract(tools) {
-  return [
-    "",
-    `You have exactly these tools available (call NO others): ${tools.join(", ")}.`,
-    "Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.",
-    "Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:",
-    '{ "calls": [ { "tool": "<tool_name>", "arguments": { ... } } ] }',
-    "Include one entry per tool call in execution order. Nothing after the closing fence."
-  ].join("\n");
-}
-function rowToTask(row, mode) {
-  const servers = asArray(row.gym_servers_config);
-  const verifiers = asArray(row.verifiers);
-  const meta = {
-    taskId: row.task_id,
-    domain: row.domain,
-    mode,
-    selectedTools: row.selected_tools,
-    servers,
-    verifiers
-  };
-  return {
-    id: row.task_id,
-    split: row.domain,
-    prompt: [row.system_prompt, "", row.user_prompt, workerContract(row.selected_tools)].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "string" || !Array.isArray(md.servers) || !Array.isArray(md.verifiers) || md.verifiers.length === 0) {
-    throw new Error(`enterpriseops-gym task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, mode, opts) {
-  let tasks = rows.map((r) => rowToTask(r, mode));
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((t) => want.has(t.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  return tasks;
-}
-async function loadFixtures(mode, opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(
-    `[enterpriseops-gym] EOPS_FIXTURES=1 \u2014 loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`
-  );
-  const domain = opts.split;
-  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows;
-  return selectRows(scoped, mode, opts);
-}
-async function fetchRows(mode, domain, opts) {
-  const target = opts.ids ? opts.ids.length * 4 : opts.limit ?? 16;
-  const rows = [];
-  const want = opts.ids ? new Set(opts.ids) : null;
-  const page = 100;
-  const base = rowsApi(mode, domain);
-  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {
-    const res = await fetch(`${base}&offset=${offset}&length=${page}`);
-    if (!res.ok) {
-      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`);
-    }
-    const body = await res.json();
-    const got = body.rows ?? [];
-    if (got.length === 0) break;
-    for (const r of got) {
-      if (want && !want.has(r.row.task_id)) continue;
-      rows.push(r.row);
-    }
-    if (got.length < page) break;
-  }
-  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`);
-  return rows;
-}
-async function runJudge(meta, artifact) {
-  const taskJsonPath = join(
-    benchRoot,
-    ".eops-task-cache",
-    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`
-  );
-  await writeTaskCache(taskJsonPath, meta);
-  let stdout;
-  try {
-    stdout = await runVenvScriptStdin(JUDGE, ["judge", "--task-json", taskJsonPath], artifact, { cwd: benchRoot });
-  } catch (err) {
-    const e = err;
-    throw new Error(
-      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}
-Fix: (1) run the ${meta.domain} gym server \u2014 docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; (2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir (the judge seeds each task's database from seed_database_file).`
-    );
-  } finally {
-    await rm(taskJsonPath, { force: true }).catch(() => {
-    });
-  }
-  const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`);
-  if (typeof report.passes !== "number" || typeof report.total !== "number") {
-    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`);
-  }
-  const score = report.total > 0 ? report.passes / report.total : 0;
-  return {
-    resolved: report.success === true && report.total > 0 && report.passes === report.total,
-    score,
-    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total })
-  };
-}
-async function writeTaskCache(path, meta) {
-  await mkdir(join(path, ".."), { recursive: true });
-  await writeFile(
-    path,
-    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers })
-  );
-}
-async function probeServers(servers, domain) {
-  for (const s of servers) {
-    const url = `${s.mcp_server_url.replace(/\/$/, "")}/api/sql-runner`;
-    try {
-      const res = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
-      void res.status;
-    } catch (err) {
-      throw new Error(
-        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}
-Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`
-      );
-    }
-  }
-}
-function createEnterpriseOpsGymAdapter() {
-  const fixturesMode = process.env.EOPS_FIXTURES === "1";
-  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE;
-  return {
-    name: "enterpriseops-gym",
-    output: enterpriseOpsTranscriptOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8").catch((err) => {
-          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
-        });
-        return;
-      }
-      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => []);
-      const servers = sample[0] ? readMeta(sample[0]).servers : [];
-      if (servers.length === 0) {
-        throw new Error(
-          `enterpriseops-gym preflight: no gym_servers_config to probe. Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`
-        );
-      }
-      await probeServers(servers, DEFAULT_DOMAIN);
-    },
-    async loadTasks(opts = {}) {
-      const domain = opts.split ?? DEFAULT_DOMAIN;
-      if (fixturesMode) return loadFixtures(mode, opts);
-      let rows;
-      try {
-        rows = await fetchRows(mode, domain, opts);
-      } catch (err) {
-        console.warn(
-          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`
-        );
-        return loadFixtures(mode, opts);
-      }
-      return selectRows(rows, mode, opts);
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      return runJudge(meta, artifact);
-    }
-  };
-}
-
-export {
-  enterpriseOpsTranscriptOutput,
-  createEnterpriseOpsGymAdapter
-};
-//# sourceMappingURL=chunk-YSMEKBTD.js.map

package/dist/chunk-YSMEKBTD.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/enterpriseops-gym.ts"],"sourcesContent":["/**\n * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —\n * stateful agentic planning + tool use in enterprise settings. Each record is an\n * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,\n * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`\n * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).\n * Worker artifact = the ordered tool-call transcript the agent would issue against\n * those servers, emitted as a single fenced ```json block of\n * `{ \"calls\": [ { \"tool\": ..., \"arguments\": {...}, \"gym_name\"?: ... } ] }`.\n *\n * Judge = the benchmark's OWN deterministic state-checker. The driver replays the\n * transcript against a freshly-seeded gym server (mutating its database), then runs\n * each task's `database_state` verifier — an SQL SELECT executed via the gym\n * server's /api/sql-runner endpoint, compared to `expected_value` under\n * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =\n * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;\n * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully\n * deterministic — no LLM judge.\n *\n * loadTasks enumerates the real suite from the HF rows server (config = tool-set\n * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed\n * sample (bench/fixtures/enterpriseops-gym.json) loads offline.\n *\n * Requires for a LIVE judge run: a Docker daemon with the domain gym images\n * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up\n * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge\n * fail loud with the exact pull/run/unzip step when a server is unreachable — never\n * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.\n */\n\nimport { mkdir, readFile, rm, writeFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'enterpriseops-gym.json')\nconst JUDGE = join(benchRoot, 'scripts', 'enterpriseops_gym_judge.py')\n\n/** Monotonic discriminator so concurrent judge calls stage distinct task-cache files. */\nlet judgeCallSeq = 0\n\nconst DATASET = 'ServiceNow-AI/EnterpriseOps-Gym'\n/** Tool-set mode = HF config; oracle ships exact tools, plus_N adds N distractors. */\nconst DEFAULT_MODE = 'oracle'\n/** Domain = HF split. */\nconst DEFAULT_DOMAIN = 'itsm'\n\nconst rowsApi = (mode: string, domain: string) =>\n  `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`\n\ninterface GymServerConfig {\n  mcp_server_name: string\n  mcp_server_url: string\n  seed_database_file: string\n  context?: Record<string, string>\n  user_info?: Record<string, unknown>\n}\n\ninterface Verifier {\n  verifier_type: string\n  name: string\n  description?: string\n  gym_name: string\n  validation_config: { query: string; expected_value: unknown; comparison_type: string }\n}\n\ninterface EopsRow {\n  task_id: string\n  domain: string\n  system_prompt: string\n  user_prompt: string\n  selected_tools: string[]\n  restricted_tools: string[]\n  mcp_endpoint: string\n  number_of_runs: number\n  reset_database_between_runs: boolean\n  /** HF parquet stores these as JSON strings; fixtures store them as parsed arrays. */\n  gym_servers_config: string | GymServerConfig[]\n  verifiers: string | Verifier[]\n}\n\ninterface EopsMeta {\n  taskId: string\n  domain: string\n  mode: string\n  selectedTools: string[]\n  servers: GymServerConfig[]\n  verifiers: Verifier[]\n}\n\n/** Worker transcript = the last fenced ```json block, else the raw text. */\nexport const enterpriseOpsTranscriptOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction asArray<T>(v: string | T[]): T[] {\n  return typeof v === 'string' ? (JSON.parse(v) as T[]) : v\n}\n\nfunction workerContract(tools: string[]): string {\n  return [\n    '',\n    `You have exactly these tools available (call NO others): ${tools.join(', ')}.`,\n    'Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.',\n    'Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:',\n    '{ \"calls\": [ { \"tool\": \"<tool_name>\", \"arguments\": { ... } } ] }',\n    'Include one entry per tool call in execution order. Nothing after the closing fence.',\n  ].join('\\n')\n}\n\nfunction rowToTask(row: EopsRow, mode: string): BenchTask {\n  const servers = asArray<GymServerConfig>(row.gym_servers_config)\n  const verifiers = asArray<Verifier>(row.verifiers)\n  const meta: EopsMeta = {\n    taskId: row.task_id,\n    domain: row.domain,\n    mode,\n    selectedTools: row.selected_tools,\n    servers,\n    verifiers,\n  }\n  return {\n    id: row.task_id,\n    split: row.domain,\n    prompt: [row.system_prompt, '', row.user_prompt, workerContract(row.selected_tools)].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): EopsMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.taskId !== 'string' ||\n    !Array.isArray(md.servers) ||\n    !Array.isArray(md.verifiers) ||\n    (md.verifiers as unknown[]).length === 0\n  ) {\n    throw new Error(`enterpriseops-gym task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as EopsMeta\n}\n\nfunction selectRows(rows: EopsRow[], mode: string, opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map((r) => rowToTask(r, mode))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(mode: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as EopsRow[]\n  console.warn(\n    `[enterpriseops-gym] EOPS_FIXTURES=1 — loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`,\n  )\n  const domain = opts.split\n  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows\n  return selectRows(scoped, mode, opts)\n}\n\n/** Pull real rows from the HF rows server (paged) for one mode/domain. Throws loud on a non-OK response. */\nasync function fetchRows(mode: string, domain: string, opts: LoadOptions): Promise<EopsRow[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: EopsRow[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  const base = rowsApi(mode, domain)\n  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {\n    const res = await fetch(`${base}&offset=${offset}&length=${page}`)\n    if (!res.ok) {\n      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    }\n    const body = (await res.json()) as { rows?: Array<{ row: EopsRow }> }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.task_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`)\n  return rows\n}\n\n/**\n * Run the benchmark's own state-checker for one task over the worker's transcript.\n * The driver replays the tool calls against the live gym server, runs each\n * database_state verifier's SQL via /api/sql-runner, and reports {passes,total}.\n * Score = passes/total (verifier_level_pass_rate); resolved = all pass\n * (overall_success_rate). This is the expensive Docker-backed boundary — delegated\n * to the python driver, not reimplemented. The transcript is piped on stdin via the\n * shared stdin-aware runner (execFile's `input` is not honored async and hangs the\n * reader).\n */\nasync function runJudge(meta: EopsMeta, artifact: string): Promise<BenchScore> {\n  // Stage the full task record (servers + verifiers) so the driver has the live\n  // server URLs/contexts and the SQL it must run. The path is UNIQUE per call: the gate\n  // runs k judges for one task concurrently, so a `${taskId}.json` shared path would race.\n  const taskJsonPath = join(\n    benchRoot,\n    '.eops-task-cache',\n    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`,\n  )\n  await writeTaskCache(taskJsonPath, meta)\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(JUDGE, ['judge', '--task-json', taskJsonPath], artifact, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(\n      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}\\n` +\n        `Fix: (1) run the ${meta.domain} gym server — ` +\n        `docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `(2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir ` +\n        `(the judge seeds each task's database from seed_database_file).`,\n    )\n  } finally {\n    await rm(taskJsonPath, { force: true }).catch(() => {})\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    success?: boolean\n    passes?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`)\n  if (typeof report.passes !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passes / report.total : 0\n  return {\n    resolved: report.success === true && report.total > 0 && report.passes === report.total,\n    score,\n    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total }),\n  }\n}\n\nasync function writeTaskCache(path: string, meta: EopsMeta): Promise<void> {\n  await mkdir(join(path, '..'), { recursive: true })\n  await writeFile(\n    path,\n    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers }),\n  )\n}\n\n/** Ping the configured gym servers' SQL endpoint; throw loud with the docker fix if any is unreachable. */\nasync function probeServers(servers: GymServerConfig[], domain: string): Promise<void> {\n  for (const s of servers) {\n    const url = `${s.mcp_server_url.replace(/\\/$/, '')}/api/sql-runner`\n    try {\n      // A HEAD/empty POST just proves reachability; a real query runs in judge.\n      const res = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' })\n      // Any HTTP response (even an error status) proves the server is up. Only a\n      // transport failure (refused/ENOTFOUND) means the container is not running.\n      void res.status\n    } catch (err) {\n      throw new Error(\n        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}\\n` +\n          `Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`,\n      )\n    }\n  }\n}\n\nexport function createEnterpriseOpsGymAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.EOPS_FIXTURES === '1'\n  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE\n\n  return {\n    name: 'enterpriseops-gym',\n    output: enterpriseOpsTranscriptOutput,\n\n    async preflight() {\n      // Fixtures mode proves only that the sample file is readable; a live judge\n      // still requires running gym servers (and fails loud there if absent).\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      // Live mode: probe the gym servers for the default domain (the suite's tasks\n      // each carry their own server config; preflight verifies reachability up\n      // front so a batch fails fast with the docker fix rather than mid-run).\n      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => [])\n      const servers = sample[0] ? readMeta(sample[0]).servers : []\n      if (servers.length === 0) {\n        throw new Error(\n          `enterpriseops-gym preflight: no gym_servers_config to probe. ` +\n            `Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`,\n        )\n      }\n      await probeServers(servers, DEFAULT_DOMAIN)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const domain = opts.split ?? DEFAULT_DOMAIN\n      if (fixturesMode) return loadFixtures(mode, opts)\n      let rows: EopsRow[]\n      try {\n        rows = await fetchRows(mode, domain, opts)\n      } catch (err) {\n        console.warn(\n          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`,\n        )\n        return loadFixtures(mode, opts)\n      }\n      return selectRows(rows, mode, opts)\n    },\n\n    async goldArtifact() {\n      // The benchmark ships no portable per-task oracle transcript — the reference\n      // is the seeded final DB state the verifiers check, not a tool-call script.\n      // Judge correctness is proven by replaying a real solve against the live gym\n      // server, not by a synthetic gold transcript. Returns undefined (documented,\n      // not faked).\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;AA8BA,SAAS,OAAO,UAAU,IAAI,iBAAiB;AAC/C,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,IAAM,QAAQ,KAAK,WAAW,WAAW,4BAA4B;AAGrE,IAAI,eAAe;AAEnB,IAAM,UAAU;AAEhB,IAAM,eAAe;AAErB,IAAM,iBAAiB;AAEvB,IAAM,UAAU,CAAC,MAAc,WAC7B,uDAAuD,mBAAmB,OAAO,CAAC,WAAW,mBAAmB,IAAI,CAAC,UAAU,mBAAmB,MAAM,CAAC;AA2CpJ,IAAM,gCAAuD;AAAA,EAClE,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,iCAAiC,CAAC;AACnE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,SAAS,QAAW,GAAsB;AACxC,SAAO,OAAO,MAAM,WAAY,KAAK,MAAM,CAAC,IAAY;AAC1D;AAEA,SAAS,eAAe,OAAyB;AAC/C,SAAO;AAAA,IACL;AAAA,IACA,4DAA4D,MAAM,KAAK,IAAI,CAAC;AAAA,IAC5E;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAAc,MAAyB;AACxD,QAAM,UAAU,QAAyB,IAAI,kBAAkB;AAC/D,QAAM,YAAY,QAAkB,IAAI,SAAS;AACjD,QAAM,OAAiB;AAAA,IACrB,QAAQ,IAAI;AAAA,IACZ,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,eAAe,IAAI;AAAA,IACnB;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,QAAQ,CAAC,IAAI,eAAe,IAAI,IAAI,aAAa,eAAe,IAAI,cAAc,CAAC,EAAE,KAAK,IAAI;AAAA,IAC9F,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA2B;AAC3C,QAAM,KAAK,KAAK;AAChB,MACE,CAAC,MACD,OAAO,GAAG,WAAW,YACrB,CAAC,MAAM,QAAQ,GAAG,OAAO,KACzB,CAAC,MAAM,QAAQ,GAAG,SAAS,KAC1B,GAAG,UAAwB,WAAW,GACvC;AACA,UAAM,IAAI,MAAM,0BAA0B,KAAK,EAAE,wDAAmD;AAAA,EACtG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAiB,MAAc,MAAgC;AACjF,MAAI,QAAQ,KAAK,IAAI,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAC9C,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAc,MAAyC;AACjF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ;AAAA,IACN,sDAAiD,KAAK,MAAM,+BAA+B,QAAQ;AAAA,EACrG;AACA,QAAM,SAAS,KAAK;AACpB,QAAM,SAAS,SAAS,KAAK,OAAO,CAAC,MAAM,EAAE,WAAW,MAAM,IAAI;AAClE,SAAO,WAAW,QAAQ,MAAM,IAAI;AACtC;AAGA,eAAe,UAAU,MAAc,QAAgB,MAAuC;AAC5F,QAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;AAC/D,QAAM,OAAkB,CAAC;AACzB,QAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;AAC5C,QAAM,OAAO;AACb,QAAM,OAAO,QAAQ,MAAM,MAAM;AACjC,WAAS,SAAS,GAAG,SAAS,QAAQ,KAAK,SAAS,QAAQ,UAAU,MAAM;AAC1E,UAAM,MAAM,MAAM,MAAM,GAAG,IAAI,WAAW,MAAM,WAAW,IAAI,EAAE;AACjE,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,YAAY,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACrH;AACA,UAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,UAAM,MAAM,KAAK,QAAQ,CAAC;AAC1B,QAAI,IAAI,WAAW,EAAG;AACtB,eAAW,KAAK,KAAK;AACnB,UAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,OAAO,EAAG;AACtC,WAAK,KAAK,EAAE,GAAG;AAAA,IACjB;AACA,QAAI,IAAI,SAAS,KAAM;AAAA,EACzB;AACA,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,CAAC,QAAQ,IAAI,IAAI,MAAM,EAAE;AACzH,SAAO;AACT;AAYA,eAAe,SAAS,MAAgB,UAAuC;AAI7E,QAAM,eAAe;AAAA,IACnB;AAAA,IACA;AAAA,IACA,GAAG,KAAK,MAAM,IAAI,QAAQ,GAAG,IAAI,cAAc;AAAA,EACjD;AACA,QAAM,eAAe,cAAc,IAAI;AACvC,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,mBAAmB,OAAO,CAAC,SAAS,eAAe,YAAY,GAAG,UAAU,EAAE,KAAK,UAAU,CAAC;AAAA,EAC/G,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI;AAAA,MACR,sCAAsC,KAAK,MAAM,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC;AAAA,mBACzE,KAAK,MAAM,+EAC6B,KAAK,MAAM,uFACM,KAAK,MAAM;AAAA,IAG5F;AAAA,EACF,UAAE;AACA,UAAM,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACxD;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAMlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,qCAAqC,KAAK,MAAM,KAAK,OAAO,KAAK,EAAE;AACrG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,uDAAuD,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC/F;AACA,QAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;AAChE,SAAO;AAAA,IACL,UAAU,OAAO,YAAY,QAAQ,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;AAAA,IAClF;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,KAAK,QAAQ,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACjH;AACF;AAEA,eAAe,eAAe,MAAc,MAA+B;AACzE,QAAM,MAAM,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,QAAM;AAAA,IACJ;AAAA,IACA,KAAK,UAAU,EAAE,SAAS,KAAK,QAAQ,QAAQ,KAAK,QAAQ,oBAAoB,KAAK,SAAS,WAAW,KAAK,UAAU,CAAC;AAAA,EAC3H;AACF;AAGA,eAAe,aAAa,SAA4B,QAA+B;AACrF,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,GAAG,EAAE,eAAe,QAAQ,OAAO,EAAE,CAAC;AAClD,QAAI;AAEF,YAAM,MAAM,MAAM,MAAM,KAAK,EAAE,QAAQ,QAAQ,SAAS,EAAE,gBAAgB,mBAAmB,GAAG,MAAM,KAAK,CAAC;AAG5G,WAAK,IAAI;AAAA,IACX,SAAS,KAAK;AACZ,YAAM,IAAI;AAAA,QACR,gCAAgC,EAAE,eAAe,mBAAmB,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,gEACjD,MAAM,yFACQ,MAAM;AAAA,MAEzF;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,kBAAkB;AACnD,QAAM,OAAO,QAAQ,IAAI,aAAa;AAEtC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAGhB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI,MAAM,uBAAuB,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,QAC3G,CAAC;AACD;AAAA,MACF;AAIA,YAAM,SAAS,MAAM,aAAa,MAAM,EAAE,OAAO,gBAAgB,OAAO,EAAE,CAAC,EAAE,MAAM,MAAM,CAAC,CAAC;AAC3F,YAAM,UAAU,OAAO,CAAC,IAAI,SAAS,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC;AAC3D,UAAI,QAAQ,WAAW,GAAG;AACxB,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,YAAM,aAAa,SAAS,cAAc;AAAA,IAC5C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,SAAS,KAAK,SAAS;AAC7B,UAAI,aAAc,QAAO,aAAa,MAAM,IAAI;AAChD,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,MAC3C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,+CAA+C,eAAe,QAAQ,IAAI,UAAU,GAAG,0CAA0C,QAAQ;AAAA,QAC3I;AACA,eAAO,aAAa,MAAM,IAAI;AAAA,MAChC;AACA,aAAO,WAAW,MAAM,MAAM,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,eAAe;AAMnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,SAAS,MAAM,QAAQ;AAAA,IAChC;AAAA,EACF;AACF;","names":[]}