@tangle-network/agent-bench 0.3.5 → 0.3.7

package/dist/chunk-2PVVP7GN.js

@@ -0,0 +1,197 @@
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/toollm.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "toollm.json");
+var DEFAULT_QUERY_REL = join("data_example", "instruction", "G1_query.json");
+var toolbenchDir = () => process.env.TOOLBENCH_DIR;
+var queryFile = (dir) => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL);
+var toollmOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    return text.trim();
+  }
+};
+function rowToTask(row) {
+  const relevantApis = normalizeApiPairs(row["relevant APIs"] ?? []);
+  const meta = {
+    queryId: String(row.query_id),
+    apiList: row.api_list ?? [],
+    relevantApis,
+    deterministicJudge: "api-selection"
+  };
+  return {
+    id: String(row.query_id),
+    prompt: [
+      "Solve this ToolLLM/ToolBench API-use task.",
+      "Use only the listed APIs/tools and return the completed tool-use trace plus final answer.",
+      "",
+      `Query: ${row.query}`,
+      "",
+      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,
+      "",
+      'Return the APIs you used as JSON: {"api_calls":[{"tool_name":"...","api_name":"..."}]}.'
+    ].join("\n"),
+    metadata: meta
+  };
+}
+function normalizeApiPart(value) {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
+}
+function apiKey(pair) {
+  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`;
+}
+function normalizeApiPairs(value) {
+  if (!Array.isArray(value)) return [];
+  const out = [];
+  for (const item of value) {
+    if (Array.isArray(item) && typeof item[0] === "string" && typeof item[1] === "string") {
+      out.push([item[0], item[1]]);
+    } else if (item && typeof item === "object" && typeof item.tool_name === "string" && typeof item.api_name === "string") {
+      out.push([item.tool_name, item.api_name]);
+    }
+  }
+  return out;
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.relevantApis)) {
+    throw new Error(`ToolLLM task ${task.id} missing metadata \u2014 loadTasks did not populate deterministic API-selection labels`);
+  }
+  return md;
+}
+function selectRows(rows, opts) {
+  let tasks = rows.map(rowToTask);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((task) => want.has(task.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`);
+  return tasks;
+}
+function assertDeterministicSubset(tasks, source) {
+  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id);
+  if (missing.length > 0) {
+    throw new Error(
+      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(", ")}. Use a ToolBench query file that includes "relevant APIs" labels, or do not score ToolLLM in agent-bench.`
+    );
+  }
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[toollm] TOOLLM_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
+  const tasks = selectRows(rows, opts);
+  assertDeterministicSubset(tasks, FIXTURES);
+  return tasks;
+}
+async function loadOfficialTasks(dir, opts) {
+  const source = queryFile(dir);
+  const tasks = selectRows(JSON.parse(await readFile(source, "utf8")), opts);
+  assertDeterministicSubset(tasks, source);
+  return tasks;
+}
+function extractJsonBlock(text) {
+  const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)];
+  const raw = (fences.at(-1)?.[1] ?? text).trim();
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return void 0;
+  }
+}
+function extractCalledApis(text, expected) {
+  const parsed = extractJsonBlock(text);
+  if (parsed && typeof parsed === "object") {
+    const raw = parsed;
+    const fromApiCalls = normalizeApiPairs(raw.api_calls);
+    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: "structured-json" };
+    const fromCalls = normalizeApiPairs(raw.calls);
+    if (fromCalls.length > 0) return { pairs: fromCalls, source: "structured-json" };
+  }
+  const lower = text.toLowerCase();
+  return {
+    pairs: expected.filter(([tool, api]) => {
+      const toolNeedle = normalizeApiPart(tool);
+      const apiNeedle = normalizeApiPart(api);
+      const compactText = lower.replace(/[^a-z0-9]+/g, "");
+      return compactText.includes(`${toolNeedle}${apiNeedle}`) || lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase());
+    }),
+    source: "text-mention"
+  };
+}
+function scoreApiSelection(task, artifact) {
+  const meta = readMeta(task);
+  if (meta.relevantApis.length === 0) {
+    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`);
+  }
+  const expected = new Set(meta.relevantApis.map(apiKey));
+  const extracted = extractCalledApis(artifact, meta.relevantApis);
+  const calledPairs = extracted.pairs;
+  const called = new Set(calledPairs.map(apiKey));
+  const truePositives = [...called].filter((key) => expected.has(key)).length;
+  const precision = called.size === 0 ? 0 : truePositives / called.size;
+  const recall = truePositives / expected.size;
+  const score = expected.size === 0 ? 0 : recall;
+  const resolved = extracted.source === "structured-json" && recall === 1 && precision === 1;
+  return {
+    resolved,
+    score,
+    detail: JSON.stringify({
+      scoring: "api-selection-only",
+      extractionSource: extracted.source,
+      queryId: meta.queryId,
+      expected: meta.relevantApis,
+      called: calledPairs,
+      precision,
+      recall,
+      fullToolEvalScore: null
+    })
+  };
+}
+function createToolLlmAdapter() {
+  const fixturesMode = process.env.TOOLLM_FIXTURES === "1";
+  return {
+    name: "toollm",
+    output: toollmOutput,
+    async preflight() {
+      if (fixturesMode) return;
+      const dir = toolbenchDir();
+      if (!dir) {
+        throw new Error("TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.");
+      }
+      await loadOfficialTasks(dir, { limit: 1 });
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const dir = toolbenchDir();
+      if (!dir) throw new Error("TOOLBENCH_DIR is required to load ToolLLM tasks");
+      return loadOfficialTasks(dir, opts);
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      if (meta.relevantApis.length === 0) return void 0;
+      return JSON.stringify({
+        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name }))
+      }, null, 2);
+    },
+    async judge(task, artifact) {
+      return scoreApiSelection(task, artifact);
+    }
+  };
+}
+
+export {
+  toollmOutput,
+  createToolLlmAdapter
+};
+//# sourceMappingURL=chunk-2PVVP7GN.js.map

package/dist/chunk-2PVVP7GN.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/toollm.ts"],"sourcesContent":["/**\n * ToolLLM/ToolBench adapter.\n *\n * ToolBench task loading is useful for breadth, but the official ToolEval pass\n * rate evaluator is LLM-based and stochastic. This adapter therefore scores\n * only ToolBench's deterministic API-selection labels (`relevant APIs`). It\n * never records a full ToolEval pass-rate score.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')\nconst DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')\n\ninterface ToolApi {\n  category_name?: string\n  tool_name: string\n  api_name: string\n  api_description?: string\n  required_parameters?: unknown[]\n  optional_parameters?: unknown[]\n  method?: string\n}\n\ninterface ToolBenchRow {\n  query_id: number | string\n  query: string\n  api_list?: ToolApi[]\n  'relevant APIs'?: Array<[string, string]>\n}\n\ninterface ToolBenchMeta {\n  queryId: string\n  apiList: ToolApi[]\n  relevantApis: Array<[string, string]>\n  deterministicJudge: 'api-selection'\n}\n\nconst toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR\nconst queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)\n\nexport const toollmOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nfunction rowToTask(row: ToolBenchRow): BenchTask {\n  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])\n  const meta: ToolBenchMeta = {\n    queryId: String(row.query_id),\n    apiList: row.api_list ?? [],\n    relevantApis,\n    deterministicJudge: 'api-selection',\n  }\n  return {\n    id: String(row.query_id),\n    prompt: [\n      'Solve this ToolLLM/ToolBench API-use task.',\n      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',\n      '',\n      `Query: ${row.query}`,\n      '',\n      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,\n      '',\n      'Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction normalizeApiPart(value: string): string {\n  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')\n}\n\nfunction apiKey(pair: readonly [string, string]): string {\n  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`\n}\n\nfunction normalizeApiPairs(value: unknown): Array<[string, string]> {\n  if (!Array.isArray(value)) return []\n  const out: Array<[string, string]> = []\n  for (const item of value) {\n    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {\n      out.push([item[0], item[1]])\n    } else if (\n      item && typeof item === 'object'\n      && typeof (item as { tool_name?: unknown }).tool_name === 'string'\n      && typeof (item as { api_name?: unknown }).api_name === 'string'\n    ) {\n      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])\n    }\n  }\n  return out\n}\n\nfunction readMeta(task: BenchTask): ToolBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.relevantApis)) {\n    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)\n  }\n  return md as unknown as ToolBenchMeta\n}\n\nfunction selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nfunction assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {\n  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a ToolBench query file that includes \"relevant APIs\" labels, or do not score ToolLLM in agent-bench.',\n    )\n  }\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]\n  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  const tasks = selectRows(rows, opts)\n  assertDeterministicSubset(tasks, FIXTURES)\n  return tasks\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const source = queryFile(dir)\n  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)\n  assertDeterministicSubset(tasks, source)\n  return tasks\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\ninterface ExtractedApis {\n  pairs: Array<[string, string]>\n  source: 'structured-json' | 'text-mention'\n}\n\nfunction extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {\n  const parsed = extractJsonBlock(text)\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    const fromApiCalls = normalizeApiPairs(raw.api_calls)\n    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }\n    const fromCalls = normalizeApiPairs(raw.calls)\n    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }\n  }\n\n  const lower = text.toLowerCase()\n  return {\n    pairs: expected.filter(([tool, api]) => {\n      const toolNeedle = normalizeApiPart(tool)\n      const apiNeedle = normalizeApiPart(api)\n      const compactText = lower.replace(/[^a-z0-9]+/g, '')\n      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))\n    }),\n    source: 'text-mention',\n  }\n}\n\nfunction scoreApiSelection(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  if (meta.relevantApis.length === 0) {\n    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)\n  }\n  const expected = new Set(meta.relevantApis.map(apiKey))\n  const extracted = extractCalledApis(artifact, meta.relevantApis)\n  const calledPairs = extracted.pairs\n  const called = new Set(calledPairs.map(apiKey))\n  const truePositives = [...called].filter((key) => expected.has(key)).length\n  const precision = called.size === 0 ? 0 : truePositives / called.size\n  const recall = truePositives / expected.size\n  const score = expected.size === 0 ? 0 : recall\n  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      scoring: 'api-selection-only',\n      extractionSource: extracted.source,\n      queryId: meta.queryId,\n      expected: meta.relevantApis,\n      called: calledPairs,\n      precision,\n      recall,\n      fullToolEvalScore: null,\n    }),\n  }\n}\n\nexport function createToolLlmAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'\n\n  return {\n    name: 'toollm',\n    output: toollmOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = toolbenchDir()\n      if (!dir) {\n        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')\n      }\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = toolbenchDir()\n      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      if (meta.relevantApis.length === 0) return undefined\n      return JSON.stringify({\n        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreApiSelection(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,aAAa;AAC1D,IAAM,oBAAoB,KAAK,gBAAgB,eAAe,eAAe;AA0B7E,IAAM,eAAe,MAA0B,QAAQ,IAAI;AAC3D,IAAM,YAAY,CAAC,QAAwB,QAAQ,IAAI,qBAAqB,KAAK,KAAK,iBAAiB;AAEhG,IAAM,eAAsC;AAAA,EACjD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,UAAU,KAA8B;AAC/C,QAAM,eAAe,kBAAkB,IAAI,eAAe,KAAK,CAAC,CAAC;AACjE,QAAM,OAAsB;AAAA,IAC1B,SAAS,OAAO,IAAI,QAAQ;AAAA,IAC5B,SAAS,IAAI,YAAY,CAAC;AAAA,IAC1B;AAAA,IACA,oBAAoB;AAAA,EACtB;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,QAAQ;AAAA,IACvB,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,IAAI,KAAK;AAAA,MACnB;AAAA,MACA,mBAAmB,KAAK,UAAU,IAAI,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,MAC9D;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,iBAAiB,OAAuB;AAC/C,SAAO,MAAM,YAAY,EAAE,QAAQ,eAAe,EAAE;AACtD;AAEA,SAAS,OAAO,MAAyC;AACvD,SAAO,GAAG,iBAAiB,KAAK,CAAC,CAAC,CAAC,IAAI,iBAAiB,KAAK,CAAC,CAAC,CAAC;AAClE;AAEA,SAAS,kBAAkB,OAAyC;AAClE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,MAA+B,CAAC;AACtC,aAAW,QAAQ,OAAO;AACxB,QAAI,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAK,CAAC,MAAM,YAAY,OAAO,KAAK,CAAC,MAAM,UAAU;AACrF,UAAI,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAAA,IAC7B,WACE,QAAQ,OAAO,SAAS,YACrB,OAAQ,KAAiC,cAAc,YACvD,OAAQ,KAAgC,aAAa,UACxD;AACA,UAAI,KAAK,CAAE,KAA+B,WAAY,KAA8B,QAAQ,CAAC;AAAA,IAC/F;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,SAAS,MAAgC;AAChD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,YAAY,GAAG;AAC1C,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wFAAmF;AAAA,EAC5H;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAsB,MAAgC;AACxE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,CAAC,EAAE;AAC3F,SAAO;AACT;AAEA,SAAS,0BAA0B,OAA6B,QAAsB;AACpF,QAAM,UAAU,MAAM,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,aAAa,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,KAAK,EAAE;AACtG,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,0DAA0D,QAAQ,MAAM,IAAI,MAAM,MAAM,iBAAiB,MAAM,KAAK,QAAQ,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IAEpJ;AAAA,EACF;AACF;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,6CAAwC,KAAK,MAAM,mBAAmB;AACnF,QAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,4BAA0B,OAAO,QAAQ;AACzC,SAAO;AACT;AAEA,eAAe,kBAAkB,KAAa,MAAyC;AACrF,QAAM,SAAS,UAAU,GAAG;AAC5B,QAAM,QAAQ,WAAW,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,CAAC,GAAqB,IAAI;AAC3F,4BAA0B,OAAO,MAAM;AACvC,SAAO;AACT;AAEA,SAAS,iBAAiB,MAAuB;AAC/C,QAAM,SAAS,CAAC,GAAG,KAAK,SAAS,iCAAiC,CAAC;AACnE,QAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAC9C,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAOA,SAAS,kBAAkB,MAAc,UAAsD;AAC7F,QAAM,SAAS,iBAAiB,IAAI;AACpC,MAAI,UAAU,OAAO,WAAW,UAAU;AACxC,UAAM,MAAM;AACZ,UAAM,eAAe,kBAAkB,IAAI,SAAS;AACpD,QAAI,aAAa,SAAS,EAAG,QAAO,EAAE,OAAO,cAAc,QAAQ,kBAAkB;AACrF,UAAM,YAAY,kBAAkB,IAAI,KAAK;AAC7C,QAAI,UAAU,SAAS,EAAG,QAAO,EAAE,OAAO,WAAW,QAAQ,kBAAkB;AAAA,EACjF;AAEA,QAAM,QAAQ,KAAK,YAAY;AAC/B,SAAO;AAAA,IACL,OAAO,SAAS,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM;AACtC,YAAM,aAAa,iBAAiB,IAAI;AACxC,YAAM,YAAY,iBAAiB,GAAG;AACtC,YAAM,cAAc,MAAM,QAAQ,eAAe,EAAE;AACnD,aAAO,YAAY,SAAS,GAAG,UAAU,GAAG,SAAS,EAAE,KAAM,MAAM,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC;AAAA,IACrI,CAAC;AAAA,IACD,QAAQ;AAAA,EACV;AACF;AAEA,SAAS,kBAAkB,MAAiB,UAA8B;AACxE,QAAM,OAAO,SAAS,IAAI;AAC1B,MAAI,KAAK,aAAa,WAAW,GAAG;AAClC,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,+DAA+D;AAAA,EACxG;AACA,QAAM,WAAW,IAAI,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC;AACtD,QAAM,YAAY,kBAAkB,UAAU,KAAK,YAAY;AAC/D,QAAM,cAAc,UAAU;AAC9B,QAAM,SAAS,IAAI,IAAI,YAAY,IAAI,MAAM,CAAC;AAC9C,QAAM,gBAAgB,CAAC,GAAG,MAAM,EAAE,OAAO,CAAC,QAAQ,SAAS,IAAI,GAAG,CAAC,EAAE;AACrE,QAAM,YAAY,OAAO,SAAS,IAAI,IAAI,gBAAgB,OAAO;AACjE,QAAM,SAAS,gBAAgB,SAAS;AACxC,QAAM,QAAQ,SAAS,SAAS,IAAI,IAAI;AACxC,QAAM,WAAW,UAAU,WAAW,qBAAqB,WAAW,KAAK,cAAc;AACzF,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,KAAK,UAAU;AAAA,MACrB,SAAS;AAAA,MACT,kBAAkB,UAAU;AAAA,MAC5B,SAAS,KAAK;AAAA,MACd,UAAU,KAAK;AAAA,MACf,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,MACA,mBAAmB;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,oBAAoB;AAErD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI,MAAM,sHAAsH;AAAA,MACxI;AACA,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,aAAa;AACzB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,iDAAiD;AAC3E,aAAO,kBAAkB,KAAK,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI,KAAK,aAAa,WAAW,EAAG,QAAO;AAC3C,aAAO,KAAK,UAAU;AAAA,QACpB,WAAW,KAAK,aAAa,IAAI,CAAC,CAAC,WAAW,QAAQ,OAAO,EAAE,WAAW,SAAS,EAAE;AAAA,MACvF,GAAG,MAAM,CAAC;AAAA,IACZ;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,aAAO,kBAAkB,MAAM,QAAQ;AAAA,IACzC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-2XU6OGEN.js

@@ -0,0 +1,170 @@
+import {
+  benchRoot,
+  preflightVenvImports,
+  runVenvScriptStdin,
+  venvPythonAt
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/commit0.ts
+import { join } from "path";
+import { readFile } from "fs/promises";
+var FIXTURES = join(benchRoot, "fixtures", "commit0.json");
+var commit0VenvDir = () => process.env.COMMIT0_VENV ?? ".venv-commit0";
+var commit0Python = () => venvPythonAt(commit0VenvDir());
+var DATASET = "wentingzhao/commit0_combined";
+var DATASET_SPLIT = "test";
+var ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`;
+var commit0DiffOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:diff|patch)?\s*\n([\s\S]*?)```/g)];
+    const body = (fences.at(-1)?.[1] ?? text).trim();
+    return body.length > 0 ? `${body}
+` : body;
+  }
+};
+function rowToTask(row) {
+  const meta = {
+    instanceId: row.instance_id,
+    repo: row.repo,
+    originalRepo: row.original_repo,
+    baseCommit: row.base_commit,
+    referenceCommit: row.reference_commit,
+    srcDir: row.src_dir,
+    testDir: row.test.test_dir,
+    testCmd: row.test.test_cmd,
+    specification: row.setup.specification
+  };
+  return {
+    id: row.instance_id,
+    split: DATASET_SPLIT,
+    prompt: [
+      `Clone https://github.com/${row.repo} into /work, then \`cd /work && git checkout ${row.base_commit}\`.`,
+      `This is the STUBBED library: the public functions/classes under \`${row.src_dir}\` have empty bodies (\`pass\`/\`...\`).`,
+      `Fill in COMPLETE implementations under \`${row.src_dir}\` so the existing test suite under \`${row.test.test_dir}\` passes. Read those tests and the spec to learn the required behavior.`,
+      `Specification / docs: ${row.setup.specification}`,
+      "Do NOT edit the test files \u2014 the evaluation re-runs the existing tests on a fresh clone. Implement only the source.",
+      `When done, from /work run EXACTLY: \`git add -A && git diff --cached -- ${row.src_dir}\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \`\`\`diff \u2026 \`\`\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`
+    ].join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.instanceId !== "string" || typeof md.referenceCommit !== "string" || typeof md.srcDir !== "string") {
+    throw new Error(`commit0 task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectRows(rows, opts) {
+  let tasks = rows.map(rowToTask);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((t) => want.has(t.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  return tasks;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(
+    `[commit0] COMMIT0_FIXTURES=1 \u2014 loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`
+  );
+  return selectRows(rows, opts);
+}
+async function fetchRows(opts) {
+  const target = opts.ids ? opts.ids.length * 4 : opts.limit ?? 16;
+  const rows = [];
+  const want = opts.ids ? new Set(opts.ids) : null;
+  const page = 100;
+  for (let offset = 0; offset < 64 && rows.length < target; offset += page) {
+    const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`);
+    if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`);
+    const body = await res.json();
+    const got = body.rows ?? [];
+    if (got.length === 0) break;
+    for (const r of got) {
+      if (want && !want.has(r.row.instance_id)) continue;
+      rows.push(r.row);
+    }
+    if (got.length < page) break;
+  }
+  if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`);
+  return rows;
+}
+async function runHarness(meta, artifact) {
+  const judge = join(benchRoot, "scripts", "commit0_judge.py");
+  let stdout;
+  try {
+    stdout = await runVenvScriptStdin(
+      judge,
+      ["--dataset", DATASET, "--split", DATASET_SPLIT, "--instance", meta.instanceId, "--src-dir", meta.srcDir],
+      artifact,
+      { cwd: benchRoot, python: commit0Python() }
+    );
+  } catch (err) {
+    const e = err;
+    throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`);
+  }
+  const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+  if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`);
+  if (typeof report.passed !== "number" || typeof report.total !== "number") {
+    throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`);
+  }
+  if (report.total <= 0) {
+    throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`);
+  }
+  const score = report.passed / report.total;
+  return {
+    resolved: report.passed === report.total,
+    score,
+    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total })
+  };
+}
+function createCommit0Adapter() {
+  const fixturesMode = process.env.COMMIT0_FIXTURES === "1";
+  return {
+    name: "commit0",
+    output: commit0DiffOutput,
+    async preflight() {
+      await preflightVenvImports({
+        modules: ["commit0"],
+        requireDocker: true,
+        python: commit0Python(),
+        fix: `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets (an ISOLATED venv \u2014 commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; (2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`
+      });
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      let rows;
+      try {
+        rows = await fetchRows(opts);
+      } catch (err) {
+        console.warn(
+          `[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`
+        );
+        return loadFixtures(opts);
+      }
+      return selectRows(rows, opts);
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      return runHarness(meta, artifact);
+    }
+  };
+}
+
+export {
+  commit0DiffOutput,
+  createCommit0Adapter
+};
+//# sourceMappingURL=chunk-2XU6OGEN.js.map

package/dist/chunk-2XU6OGEN.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/commit0.ts"],"sourcesContent":["/**\n * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.\n * Each record is a Python repo stubbed at `base_commit` (public function bodies\n * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker\n * artifact = a unified diff that fills in `src_dir`. Judge = the official\n * `commit0` test harness: it stages the starter repo, applies the worker's diff,\n * builds the library's deps and runs `pytest`, writing a per-repo pytest-json\n * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the\n * macro-averaged unit-test pass-rate the leaderboard reports). Fully\n * deterministic — no LLM judge.\n *\n * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its\n * implementation as a fenced ```diff against the stubbed repo — same deliverable\n * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the\n * real `commit0` harness on a local Docker backend (NOT reimplemented here).\n *\n * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed\n * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +\n * a Docker daemon (`--backend local`). For offline/CI dataset listing set\n * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)\n * — judging still needs the harness + Docker and fails loud, never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPythonAt } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'commit0.json')\n\n// commit0's pip deps (pydantic/sqlalchemy v1, modal, …) conflict with the shared\n// bench .venv, so its harness runs in an ISOLATED venv. Override with COMMIT0_VENV.\n// Resolved at call-time so the env is honored at run-time (not frozen at import).\nconst commit0VenvDir = (): string => process.env.COMMIT0_VENV ?? '.venv-commit0'\nconst commit0Python = (): string => venvPythonAt(commit0VenvDir())\n\nconst DATASET = 'wentingzhao/commit0_combined'\nconst DATASET_SPLIT = 'test'\n// HF rows server — columnar pull with no `datasets` install required for listing.\nconst ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`\n\n/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */\nexport const commit0DiffOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const body = (fences.at(-1)?.[1] ?? text).trim()\n    // `git apply` rejects a patch that is not newline-terminated (\"corrupt patch at\n    // line N+1\"); the .trim() above strips the final newline, so restore exactly one.\n    return body.length > 0 ? `${body}\\n` : body\n  },\n}\n\ninterface Commit0Setup {\n  install: string\n  packages: string[] | null\n  pip_packages: string[] | null\n  pre_install: string[] | null\n  python: string\n  specification: string\n}\n\ninterface Commit0Row {\n  instance_id: string\n  repo: string\n  original_repo: string\n  base_commit: string\n  reference_commit: string\n  setup: Commit0Setup\n  test: { test_cmd: string; test_dir: string }\n  src_dir: string\n}\n\ninterface Commit0Meta {\n  instanceId: string\n  repo: string\n  originalRepo: string\n  baseCommit: string\n  referenceCommit: string\n  srcDir: string\n  testDir: string\n  testCmd: string\n  specification: string\n}\n\nfunction rowToTask(row: Commit0Row): BenchTask {\n  const meta: Commit0Meta = {\n    instanceId: row.instance_id,\n    repo: row.repo,\n    originalRepo: row.original_repo,\n    baseCommit: row.base_commit,\n    referenceCommit: row.reference_commit,\n    srcDir: row.src_dir,\n    testDir: row.test.test_dir,\n    testCmd: row.test.test_cmd,\n    specification: row.setup.specification,\n  }\n  return {\n    id: row.instance_id,\n    split: DATASET_SPLIT,\n    prompt: [\n      `Clone https://github.com/${row.repo} into /work, then \\`cd /work && git checkout ${row.base_commit}\\`.`,\n      `This is the STUBBED library: the public functions/classes under \\`${row.src_dir}\\` have empty bodies (\\`pass\\`/\\`...\\`).`,\n      `Fill in COMPLETE implementations under \\`${row.src_dir}\\` so the existing test suite under \\`${row.test.test_dir}\\` passes. Read those tests and the spec to learn the required behavior.`,\n      `Specification / docs: ${row.setup.specification}`,\n      'Do NOT edit the test files — the evaluation re-runs the existing tests on a fresh clone. Implement only the source.',\n      `When done, from /work run EXACTLY: \\`git add -A && git diff --cached -- ${row.src_dir}\\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \\`\\`\\`diff … \\`\\`\\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): Commit0Meta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string' || typeof md.referenceCommit !== 'string' || typeof md.srcDir !== 'string') {\n    throw new Error(`commit0 task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as Commit0Meta\n}\n\nfunction selectRows(rows: Commit0Row[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as Commit0Row[]\n  console.warn(\n    `[commit0] COMMIT0_FIXTURES=1 — loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`,\n  )\n  return selectRows(rows, opts)\n}\n\n/** Pull real rows from the HF rows server (paged). limit caps the pull; ids\n *  filtered client-side. Throws on a non-OK response (fail loud). */\nasync function fetchRows(opts: LoadOptions): Promise<Commit0Row[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: Commit0Row[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  for (let offset = 0; offset < 64 && rows.length < target; offset += page) {\n    const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`)\n    if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    const body = (await res.json()) as { rows?: Array<{ row: Commit0Row }>; num_rows_total?: number }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.instance_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`)\n  return rows\n}\n\n/**\n * Run the official commit0 harness for one repo over the worker's diff. The\n * harness clones base_commit, applies the diff into src_dir, installs deps and\n * runs pytest, then writes a per-repo pytest-json report. We read that report and\n * compute (passed + xfail) / total — the leaderboard's per-repo pass-rate.\n *\n * This is the expensive, Docker-backed boundary; it is DELEGATED to `commit0`,\n * not reimplemented. The driver script lives in scripts/commit0_judge.py so the\n * harness call + report parse are one auditable python entrypoint.\n */\nasync function runHarness(meta: Commit0Meta, artifact: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'commit0_judge.py')\n  let stdout: string\n  try {\n    // The worker's diff is piped to the driver's stdin (shared stdin-aware runner —\n    // execFile's `input` option is not honored async and hangs the reader).\n    stdout = await runVenvScriptStdin(\n      judge,\n      ['--dataset', DATASET, '--split', DATASET_SPLIT, '--instance', meta.instanceId, '--src-dir', meta.srcDir],\n      artifact,\n      { cwd: benchRoot, python: commit0Python() },\n    )\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  // total=0 means the harness MEASURED NOTHING (collection error with no declared\n  // test ids) — an unmeasured attempt, not a 0% one. Throw so the caller excludes\n  // it as infra instead of recording a fabricated zero.\n  if (report.total <= 0) {\n    throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`)\n  }\n  const score = report.passed / report.total\n  return {\n    resolved: report.passed === report.total,\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\nexport function createCommit0Adapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.COMMIT0_FIXTURES === '1'\n\n  return {\n    name: 'commit0',\n    output: commit0DiffOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['commit0'],\n        requireDocker: true,\n        python: commit0Python(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets ` +\n          `(an ISOLATED venv — commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). ` +\n          `Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let rows: Commit0Row[]\n      try {\n        rows = await fetchRows(opts)\n      } catch (err) {\n        console.warn(\n          `[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRows(rows, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the reference_commit's src_dir, which the commit0 harness\n      // checks out by ref — not expressible as a portable diff string without\n      // cloning. verify-judge against this adapter requires the live harness, so\n      // we return undefined (no offline gold-diff). Judge correctness is proven by\n      // running the harness on a real solve, not by a synthetic gold patch.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;AAuBA,SAAS,YAAY;AACrB,SAAS,gBAAgB;AAKzB,IAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAK3D,IAAM,iBAAiB,MAAc,QAAQ,IAAI,gBAAgB;AACjE,IAAM,gBAAgB,MAAc,aAAa,eAAe,CAAC;AAEjE,IAAM,UAAU;AAChB,IAAM,gBAAgB;AAEtB,IAAM,WAAW,uDAAuD,mBAAmB,OAAO,CAAC,yBAAyB,aAAa;AAGlI,IAAM,oBAA2C;AAAA,EACtD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,uCAAuC,CAAC;AACzE,UAAM,QAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAG/C,WAAO,KAAK,SAAS,IAAI,GAAG,IAAI;AAAA,IAAO;AAAA,EACzC;AACF;AAkCA,SAAS,UAAU,KAA4B;AAC7C,QAAM,OAAoB;AAAA,IACxB,YAAY,IAAI;AAAA,IAChB,MAAM,IAAI;AAAA,IACV,cAAc,IAAI;AAAA,IAClB,YAAY,IAAI;AAAA,IAChB,iBAAiB,IAAI;AAAA,IACrB,QAAQ,IAAI;AAAA,IACZ,SAAS,IAAI,KAAK;AAAA,IAClB,SAAS,IAAI,KAAK;AAAA,IAClB,eAAe,IAAI,MAAM;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO;AAAA,IACP,QAAQ;AAAA,MACN,4BAA4B,IAAI,IAAI,gDAAgD,IAAI,WAAW;AAAA,MACnG,qEAAqE,IAAI,OAAO;AAAA,MAChF,4CAA4C,IAAI,OAAO,yCAAyC,IAAI,KAAK,QAAQ;AAAA,MACjH,yBAAyB,IAAI,MAAM,aAAa;AAAA,MAChD;AAAA,MACA,2EAA2E,IAAI,OAAO,6KAAwK,IAAI,WAAW;AAAA,IAC/Q,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA8B;AAC9C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,eAAe,YAAY,OAAO,GAAG,oBAAoB,YAAY,OAAO,GAAG,WAAW,UAAU;AACvH,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wDAAmD;AAAA,EAC5F;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAoB,MAAgC;AACtE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ;AAAA,IACN,+CAA0C,KAAK,MAAM,6BAA6B,QAAQ;AAAA,EAC5F;AACA,SAAO,WAAW,MAAM,IAAI;AAC9B;AAIA,eAAe,UAAU,MAA0C;AACjE,QAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;AAC/D,QAAM,OAAqB,CAAC;AAC5B,QAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;AAC5C,QAAM,OAAO;AACb,WAAS,SAAS,GAAG,SAAS,MAAM,KAAK,SAAS,QAAQ,UAAU,MAAM;AACxE,UAAM,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,MAAM,WAAW,IAAI,EAAE;AACrE,QAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,qBAAqB,IAAI,MAAM,YAAY,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AACtH,UAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,UAAM,MAAM,KAAK,QAAQ,CAAC;AAC1B,QAAI,IAAI,WAAW,EAAG;AACtB,eAAW,KAAK,KAAK;AACnB,UAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,WAAW,EAAG;AAC1C,WAAK,KAAK,EAAE,GAAG;AAAA,IACjB;AACA,QAAI,IAAI,SAAS,KAAM;AAAA,EACzB;AACA,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B,KAAK,UAAU,IAAI,CAAC,SAAS,OAAO,EAAE;AACzG,SAAO;AACT;AAYA,eAAe,WAAW,MAAmB,UAAuC;AAClF,QAAM,QAAQ,KAAK,WAAW,WAAW,kBAAkB;AAC3D,MAAI;AACJ,MAAI;AAGF,aAAS,MAAM;AAAA,MACb;AAAA,MACA,CAAC,aAAa,SAAS,WAAW,eAAe,cAAc,KAAK,YAAY,aAAa,KAAK,MAAM;AAAA,MACxG;AAAA,MACA,EAAE,KAAK,WAAW,QAAQ,cAAc,EAAE;AAAA,IAC5C;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,8BAA8B,KAAK,UAAU,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EAC/G;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAKlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,KAAK,OAAO,KAAK,EAAE;AACjG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,6CAA6C,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACrF;AAIA,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,IAAI,MAAM,uCAAuC,KAAK,UAAU,eAAe,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC7G;AACA,QAAM,QAAQ,OAAO,SAAS,OAAO;AACrC,SAAO;AAAA,IACL,UAAU,OAAO,WAAW,OAAO;AAAA,IACnC;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,YAAY,KAAK,YAAY,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACpG;AACF;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,qBAAqB;AAEtD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,SAAS;AAAA,QACnB,eAAe;AAAA,QACf,QAAQ,cAAc;AAAA,QACtB,KACE,kCAAkC,eAAe,CAAC,aAAa,eAAe,CAAC;AAAA,MAInF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,UAAU,IAAI;AAAA,MAC7B,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,qCAAqC,eAAe,QAAQ,IAAI,UAAU,GAAG,4CAA4C,QAAQ;AAAA,QACnI;AACA,eAAO,aAAa,IAAI;AAAA,MAC1B;AACA,aAAO,WAAW,MAAM,IAAI;AAAA,IAC9B;AAAA,IAEA,MAAM,eAAe;AAMnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,WAAW,MAAM,QAAQ;AAAA,IAClC;AAAA,EACF;AACF;","names":[]}