@tangle-network/agent-bench 0.4.7 → 0.4.8

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.4.8
+
+- Consume Runtime 0.109.0 through the canonical `./kernel` entrypoint.
+- Align Bench with Eval 0.135.1, Interface 0.36.0, Knowledge 6.1.10, Materialize 0.9.2, and Sandbox 0.15.2.
+
 ## 0.4.7
 
 - Consume Runtime 0.108.1 with Eval 0.134.2, Interface 0.36.0, Knowledge 6.1.8, Materialize 0.9.2, and Sandbox 0.15.2.

package/HARNESS.md

@@ -4,7 +4,7 @@ If you're an agent picking this up: read this page, then run `pnpm help` + `pnpm
 do NOT re-derive the harness from source. This map is SHORT on purpose; if it disagrees
 with the code, the code wins — fix this page in the same turn (the anti-rediscovery law).
 Verified against source 2026-07-18 · agent-eval pinned `0.122.8`. The CANONICAL surface is now
-the published optimization suite (`@tangle-network/agent-runtime/loops`): `Environment` +
+the published optimization suite (`@tangle-network/agent-runtime/kernel`): `Environment` +
 `Strategy`/`defineStrategy` + `runBenchmark` — see the section below FIRST. The recursive
 diverse-vs-blind gate runs through the keystone (`gate-cli.mts` → `runGate`);
 the offline selector replay (`corpus-replay.mts` / `corpus-report.mts`) gates the legacy corpora.

package/dist/adapters.js

@@ -2,9 +2,9 @@ import { createAecBenchAdapter } from "./benchmarks/aec-bench.js";
 import { createAgentBenchAdapter } from "./benchmarks/agentbench.js";
 import { createAppWorldAdapter, createAppWorldReactAdapter } from "./benchmarks/appworld.js";
 import { createBfclAdapter } from "./benchmarks/bfcl.js";
-import { t as createCadBenchAdapter } from "./cadbench-DpQWZHp4.js";
+import { t as createCadBenchAdapter } from "./cadbench-BrpwOU6A.js";
 import { createCadDesignAdapter } from "./benchmarks/cad-design.js";
-import { t as createCadGenBenchAdapter } from "./cadgenbench-DRhczfsG.js";
+import { t as createCadGenBenchAdapter } from "./cadgenbench-DF7hYHdl.js";
 import { createCommit0Adapter } from "./benchmarks/commit0.js";
 import { createCragAdapter } from "./benchmarks/crag.js";
 import { createDabstepAdapter } from "./benchmarks/dabstep.js";

package/dist/benchmarks/agentbench.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/agentbench.d.ts
 declare const agentbenchAnswerOutput: OutputAdapter<string>;
 declare function createAgentBenchAdapter(): BenchmarkAdapter;

package/dist/benchmarks/agentbench.js.map

@@ -1 +1 @@
-{"version":3,"file":"agentbench.js","names":[],"sources":["../../src/benchmarks/agentbench.ts"],"sourcesContent":["/**\n * AgentBench deterministic subset adapter.\n *\n * This targets AgentBench DBBench rows only: question + table + published label.\n * It does not wrap AgentBench's controller protocol or the non-deterministic game\n * environments. Worker artifact = final answer text. Judge = exact match against\n * the official DBBench label list after light whitespace/case normalization.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'agentbench-dbbench.json')\nconst DEFAULT_SPLIT = 'dev'\n\ninterface AgentBenchDbRow {\n  description: string\n  label: string[]\n  table?: {\n    table_name?: string\n    table_info?: {\n      columns?: Array<{ name: string; type?: string }>\n      rows?: unknown[][]\n    }\n  }\n}\n\ninterface AgentBenchMeta {\n  labels: string[]\n  split: string\n  subset: 'dbbench'\n  table?: AgentBenchDbRow['table']\n}\n\nconst agentbenchDir = (): string | undefined => process.env.AGENTBENCH_DIR\n\nexport const agentbenchAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction normalizeAnswer(value: string): string {\n  return value\n    .toLowerCase()\n    .replace(/\\s+/g, ' ')\n    .replace(/^[\"'`]+|[\"'`]+$/g, '')\n    .trim()\n}\n\nfunction rowToTask(row: AgentBenchDbRow, index: number, split: string): BenchTask {\n  const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ''}`).join(', ')\n  const sampleRows = row.table?.table_info?.rows?.slice(0, 40)\n  const meta: AgentBenchMeta = {\n    labels: row.label,\n    split,\n    subset: 'dbbench',\n    table: row.table,\n  }\n  return {\n    id: `dbbench-${split}-${index}`,\n    split,\n    prompt: [\n      'Answer this AgentBench DBBench question using the table below.',\n      'Return only the answer value.',\n      '',\n      `Question: ${row.description}`,\n      row.table?.table_name ? `Table: ${row.table.table_name}` : undefined,\n      columns ? `Columns: ${columns}` : undefined,\n      sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): AgentBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.labels)) {\n    throw new Error(`agentbench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as AgentBenchMeta\n}\n\nfunction selectRows(rows: AgentBenchDbRow[], opts: LoadOptions, split: string): BenchTask[] {\n  let tasks = rows.map((row, index) => rowToTask(row, index, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadJsonl(path: string): Promise<AgentBenchDbRow[]> {\n  const raw = await readFile(path, 'utf8')\n  return raw\n    .split('\\n')\n    .map((line) => line.trim())\n    .filter(Boolean)\n    .map((line) => JSON.parse(line) as AgentBenchDbRow)\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as AgentBenchDbRow[]\n  console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 — loading ${rows.length} DBBench adapter fixtures`)\n  return selectRows(rows, opts, split)\n}\n\nexport function createAgentBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.AGENTBENCH_FIXTURES === '1'\n\n  return {\n    name: 'agentbench',\n    output: agentbenchAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = agentbenchDir()\n      if (!dir) {\n        throw new Error('AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.')\n      }\n      await loadJsonl(join(dir, 'data', 'dbbench', `${DEFAULT_SPLIT}.jsonl`))\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = agentbenchDir()\n      if (!dir) throw new Error('AGENTBENCH_DIR is required to load AgentBench DBBench tasks')\n      return selectRows(await loadJsonl(join(dir, 'data', 'dbbench', `${split}.jsonl`)), opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return readMeta(task).labels[0]\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const got = normalizeAnswer(artifact)\n      const expected = meta.labels.map(normalizeAnswer)\n      const resolved = expected.includes(got)\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({ subset: meta.subset, split: meta.split, expected: meta.labels, got: artifact }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,yBAAyB;AACtE,MAAM,gBAAgB;AAqBtB,MAAM,sBAA0C,QAAQ,IAAI;AAE5D,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,wCAAwC,CAC5D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,gBAAgB,OAAuB;CAC9C,OAAO,MACJ,YAAY,CAAC,CACb,QAAQ,QAAQ,GAAG,CAAC,CACpB,QAAQ,oBAAoB,EAAE,CAAC,CAC/B,KAAK;AACV;AAEA,SAAS,UAAU,KAAsB,OAAe,OAA0B;CAChF,MAAM,UAAU,IAAI,OAAO,YAAY,SAAS,KAAK,MAAM,GAAG,EAAE,OAAO,EAAE,OAAO,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI;CAChH,MAAM,aAAa,IAAI,OAAO,YAAY,MAAM,MAAM,GAAG,EAAE;CAC3D,MAAM,OAAuB;EAC3B,QAAQ,IAAI;EACZ;EACA,QAAQ;EACR,OAAO,IAAI;CACb;CACA,OAAO;EACL,IAAI,WAAW,MAAM,GAAG;EACxB;EACA,QAAQ;GACN;GACA;GACA;GACA,aAAa,IAAI;GACjB,IAAI,OAAO,aAAa,UAAU,IAAI,MAAM,eAAe,KAAA;GAC3D,UAAU,YAAY,YAAY,KAAA;GAClC,aAAa,cAAc,KAAK,UAAU,UAAU,MAAM,KAAA;EAC5D,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiC;CACjD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,GACjC,MAAM,IAAI,MAAM,mBAAmB,KAAK,GAAG,kDAAkD;CAE/F,OAAO;AACT;AAEA,SAAS,WAAW,MAAyB,MAAmB,OAA4B;CAC1F,IAAI,QAAQ,KAAK,KAAK,KAAK,UAAU,UAAU,KAAK,OAAO,KAAK,CAAC;CACjE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,wCAAwC,KAAK,UAAU,IAAI,GAAG;CACtG,OAAO;AACT;AAEA,eAAe,UAAU,MAA0C;CAEjE,QAAO,MADW,SAAS,MAAM,MAAM,EAAA,CAEpC,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,KAAK,SAAS,KAAK,MAAM,IAAI,CAAoB;AACtD;AAEA,eAAe,aAAa,MAAmB,OAAqC;CAClF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,gDAAgD,KAAK,OAAO,0BAA0B;CACnG,OAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,SAAgB,0BAA4C;CAC1D,MAAM,eAAe,QAAQ,IAAI,wBAAwB;CAEzD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,cAAc;GAC1B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,wHAAwH;GAE1I,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,cAAc,OAAO,CAAC;EACxE;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,IAAI,cAAc,OAAO,aAAa,MAAM,KAAK;GACjD,MAAM,MAAM,cAAc;GAC1B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6DAA6D;GACvF,OAAO,WAAW,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,MAAM,KAAK;EAChG;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,SAAS,IAAI,CAAC,CAAC,OAAO;EAC/B;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAM,gBAAgB,QAAQ;GAEpC,MAAM,WADW,KAAK,OAAO,IAAI,eACT,CAAC,CAAC,SAAS,GAAG;GACtC,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,OAAO,KAAK;KAAO,UAAU,KAAK;KAAQ,KAAK;IAAS,CAAC;GACzG;EACF;CACF;AACF"}
+{"version":3,"file":"agentbench.js","names":[],"sources":["../../src/benchmarks/agentbench.ts"],"sourcesContent":["/**\n * AgentBench deterministic subset adapter.\n *\n * This targets AgentBench DBBench rows only: question + table + published label.\n * It does not wrap AgentBench's controller protocol or the non-deterministic game\n * environments. Worker artifact = final answer text. Judge = exact match against\n * the official DBBench label list after light whitespace/case normalization.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'agentbench-dbbench.json')\nconst DEFAULT_SPLIT = 'dev'\n\ninterface AgentBenchDbRow {\n  description: string\n  label: string[]\n  table?: {\n    table_name?: string\n    table_info?: {\n      columns?: Array<{ name: string; type?: string }>\n      rows?: unknown[][]\n    }\n  }\n}\n\ninterface AgentBenchMeta {\n  labels: string[]\n  split: string\n  subset: 'dbbench'\n  table?: AgentBenchDbRow['table']\n}\n\nconst agentbenchDir = (): string | undefined => process.env.AGENTBENCH_DIR\n\nexport const agentbenchAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction normalizeAnswer(value: string): string {\n  return value\n    .toLowerCase()\n    .replace(/\\s+/g, ' ')\n    .replace(/^[\"'`]+|[\"'`]+$/g, '')\n    .trim()\n}\n\nfunction rowToTask(row: AgentBenchDbRow, index: number, split: string): BenchTask {\n  const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ''}`).join(', ')\n  const sampleRows = row.table?.table_info?.rows?.slice(0, 40)\n  const meta: AgentBenchMeta = {\n    labels: row.label,\n    split,\n    subset: 'dbbench',\n    table: row.table,\n  }\n  return {\n    id: `dbbench-${split}-${index}`,\n    split,\n    prompt: [\n      'Answer this AgentBench DBBench question using the table below.',\n      'Return only the answer value.',\n      '',\n      `Question: ${row.description}`,\n      row.table?.table_name ? `Table: ${row.table.table_name}` : undefined,\n      columns ? `Columns: ${columns}` : undefined,\n      sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): AgentBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.labels)) {\n    throw new Error(`agentbench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as AgentBenchMeta\n}\n\nfunction selectRows(rows: AgentBenchDbRow[], opts: LoadOptions, split: string): BenchTask[] {\n  let tasks = rows.map((row, index) => rowToTask(row, index, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadJsonl(path: string): Promise<AgentBenchDbRow[]> {\n  const raw = await readFile(path, 'utf8')\n  return raw\n    .split('\\n')\n    .map((line) => line.trim())\n    .filter(Boolean)\n    .map((line) => JSON.parse(line) as AgentBenchDbRow)\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as AgentBenchDbRow[]\n  console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 — loading ${rows.length} DBBench adapter fixtures`)\n  return selectRows(rows, opts, split)\n}\n\nexport function createAgentBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.AGENTBENCH_FIXTURES === '1'\n\n  return {\n    name: 'agentbench',\n    output: agentbenchAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = agentbenchDir()\n      if (!dir) {\n        throw new Error('AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.')\n      }\n      await loadJsonl(join(dir, 'data', 'dbbench', `${DEFAULT_SPLIT}.jsonl`))\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = agentbenchDir()\n      if (!dir) throw new Error('AGENTBENCH_DIR is required to load AgentBench DBBench tasks')\n      return selectRows(await loadJsonl(join(dir, 'data', 'dbbench', `${split}.jsonl`)), opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return readMeta(task).labels[0]\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const got = normalizeAnswer(artifact)\n      const expected = meta.labels.map(normalizeAnswer)\n      const resolved = expected.includes(got)\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({ subset: meta.subset, split: meta.split, expected: meta.labels, got: artifact }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,yBAAyB;AACtE,MAAM,gBAAgB;AAqBtB,MAAM,sBAA0C,QAAQ,IAAI;AAE5D,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,wCAAwC,CAC5D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,gBAAgB,OAAuB;CAC9C,OAAO,MACJ,YAAY,CAAC,CACb,QAAQ,QAAQ,GAAG,CAAC,CACpB,QAAQ,oBAAoB,EAAE,CAAC,CAC/B,KAAK;AACV;AAEA,SAAS,UAAU,KAAsB,OAAe,OAA0B;CAChF,MAAM,UAAU,IAAI,OAAO,YAAY,SAAS,KAAK,MAAM,GAAG,EAAE,OAAO,EAAE,OAAO,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI;CAChH,MAAM,aAAa,IAAI,OAAO,YAAY,MAAM,MAAM,GAAG,EAAE;CAC3D,MAAM,OAAuB;EAC3B,QAAQ,IAAI;EACZ;EACA,QAAQ;EACR,OAAO,IAAI;CACb;CACA,OAAO;EACL,IAAI,WAAW,MAAM,GAAG;EACxB;EACA,QAAQ;GACN;GACA;GACA;GACA,aAAa,IAAI;GACjB,IAAI,OAAO,aAAa,UAAU,IAAI,MAAM,eAAe,KAAA;GAC3D,UAAU,YAAY,YAAY,KAAA;GAClC,aAAa,cAAc,KAAK,UAAU,UAAU,MAAM,KAAA;EAC5D,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiC;CACjD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,GACjC,MAAM,IAAI,MAAM,mBAAmB,KAAK,GAAG,kDAAkD;CAE/F,OAAO;AACT;AAEA,SAAS,WAAW,MAAyB,MAAmB,OAA4B;CAC1F,IAAI,QAAQ,KAAK,KAAK,KAAK,UAAU,UAAU,KAAK,OAAO,KAAK,CAAC;CACjE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,wCAAwC,KAAK,UAAU,IAAI,GAAG;CACtG,OAAO;AACT;AAEA,eAAe,UAAU,MAA0C;CAEjE,QAAO,MADW,SAAS,MAAM,MAAM,EAAA,CAEpC,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,KAAK,SAAS,KAAK,MAAM,IAAI,CAAoB;AACtD;AAEA,eAAe,aAAa,MAAmB,OAAqC;CAClF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,gDAAgD,KAAK,OAAO,0BAA0B;CACnG,OAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,SAAgB,0BAA4C;CAC1D,MAAM,eAAe,QAAQ,IAAI,wBAAwB;CAEzD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,cAAc;GAC1B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,wHAAwH;GAE1I,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,cAAc,OAAO,CAAC;EACxE;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,IAAI,cAAc,OAAO,aAAa,MAAM,KAAK;GACjD,MAAM,MAAM,cAAc;GAC1B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6DAA6D;GACvF,OAAO,WAAW,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,MAAM,KAAK;EAChG;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,SAAS,IAAI,CAAC,CAAC,OAAO;EAC/B;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAM,gBAAgB,QAAQ;GAEpC,MAAM,WADW,KAAK,OAAO,IAAI,eACT,CAAC,CAAC,SAAS,GAAG;GACtC,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,OAAO,KAAK;KAAO,UAAU,KAAK;KAAQ,KAAK;IAAS,CAAC;GACzG;EACF;CACF;AACF"}

package/dist/benchmarks/appworld.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/appworld.d.ts
 /** Worker solution code = the last fenced ```python block, else the raw text. */
 declare const appworldSolutionOutput: OutputAdapter<string>;

package/dist/benchmarks/appworld.js

@@ -2,7 +2,7 @@ import { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from
 import { join } from "node:path";
 import { spawn } from "node:child_process";
 import { createInterface } from "node:readline";
-import { routerToolLoop } from "@tangle-network/agent-runtime/loops";
+import { routerToolLoop } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/appworld.ts
 /**
 * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's

package/dist/benchmarks/appworld.js.map

@@ -1 +1 @@
-{"version":3,"file":"appworld.js","names":[],"sources":["../../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\ninterface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  transcript?: string\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))\n      }\n      pending.push((l) => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        resolve(l)\n      })\n      child.once('exit', onExit)\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    child.stdin.end()\n    child.kill('SIGTERM')\n  }\n}\n\n/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string) {\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const loop = await routerToolLoop(\n              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },\n              system,\n              `Task: ${instruction}`,\n              [EXECUTE_TOOL],\n              async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                return `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n              },\n              { maxTurns },\n            )\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = loop.toolTrace\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            return {\n              ...verdict,\n              turns: loop.turns,\n              input_tokens: loop.usage.input,\n              output_tokens: loop.usage.output,\n              transcript,\n            } satisfies ReactResult\n          })\n          // Real usage from the episode — flat llm_call so the kernel meters it.\n          if (out.input_tokens || out.output_tokens) {\n            yield {\n              type: 'llm_call',\n              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },\n            }\n          }\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;;AAG9D,MAAM,gBAAgB;;AAQtB,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,yDAAyD;CAEpG,OAAO;AACT;;;;;;;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;CAClE,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;CAC3E,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC5G;CACA,MAAM,OAAO,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;CACjD,MAAM,SAAS,KAAK,MAAM,IAAI;CAC9B,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,0BAA0B,OAAO,OAAO;CAC1E,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAC5D,MAAM,QAAQ,KAAK,SAAS;GAO5B,MAAM,SAAQ,MANK,OAAO;IACxB;IACA;IAAW;IACX,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;IAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;GAClD,CAAC,EAAA,CACiB,SAAS,CAAC;GAC5B,IAAI,MAAM,WAAW,GACnB,MAAM,IAAI,MAAM,kDAAkD,MAAM,GAAG,KAAK,UAAU,IAAI,GAAG;GAEnG,OAAO,MAAM,KACV,OAAkB;IACjB,IAAI,EAAE;IACN;IACA,QAAQ,EAAE,cAAc;IACxB,UAAU;KAAE,QAAQ,EAAE;KAAS;IAAM;GACvC,EACF;EACF;EAEA,MAAM,eAAe,CAQrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAO,MAAM,OAAO;IAAC;IAAY;IAAa,KAAK;IAAQ;IAAW,KAAK;GAAK,GAAG,QAAQ;GAOjG,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI,SAAS;GAI3B,MAAM,QAAQ,IAAI,aAAa,SAAS;GACxC,MAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;GAG3C,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA;KACA;KACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;IACxC,CAAC;GACH;EACF;CACF;AACF;AAmCA,MAAM,eAAe;AAErB,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,MAAM,eAAyB;CAC7B,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY,EAAE,MAAM;IAAE,MAAM;IAAU,aAAa;GAA2C,EAAE;GAChG,UAAU,CAAC,MAAM;EACnB;CACF;AACF;;AAGA,eAAe,iBACb,QACA,OACA,IACY;CACZ,MAAM,QAAQ,MAAM,YAAY;EAAC;EAAQ;EAAW;EAAa;EAAQ;EAAW;CAAK,GAAG,EAC1F,KAAK,UACP,CAAC;CACD,MAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;CAClD,MAAM,UAAyC,CAAC;CAChD,MAAM,UAAoB,CAAC;CAC3B,GAAG,GAAG,SAAS,MAAM;EACnB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK,CAAC;OACX,QAAQ,KAAK,CAAC;CACrB,CAAC;CACD,IAAI,SAAS;CACb,MAAM,OAAO,GAAG,SAAS,MAAc;EACrC,UAAU,EAAE,SAAS,MAAM;CAC7B,CAAC;CACD,MAAM,YAAY,cAChB,IAAI,SAAS,SAAS,WAAW;EAC/B,MAAM,cAAc,QAAQ,MAAM;EAClC,IAAI,gBAAgB,KAAA,GAAW,OAAO,QAAQ,WAAW;EACzD,MAAM,IAAI,iBACF,uBAAO,IAAI,MAAM,oCAAoC,UAAU,cAAc,OAAO,MAAM,IAAI,GAAG,CAAC,GACxG,SACF;EAGA,MAAM,UAAU,SAA8B;GAC5C,aAAa,CAAC;GACd,uBAAO,IAAI,MAAM,4BAA4B,KAAK,aAAa,OAAO,MAAM,IAAI,GAAG,CAAC;EACtF;EACA,QAAQ,MAAM,MAAM;GAClB,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,QAAQ,CAAC;EACX,CAAC;EACD,MAAM,KAAK,QAAQ,MAAM;CAC3B,CAAC;CACH,IAAI;EACF,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;EAChD,IAAI,CAAC,MAAM,OAAO,MAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,iBAAiB;EACvG,MAAM,OAAO,OAAO,QAAmE;GACrF,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG;GAC5C,MAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;GAC9C,IAAI,OAAO,IAAI,UAAU,UAAU,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO;GAC7F,OAAO;EACT;EACA,OAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;CAC/C,UAAU;EACR,MAAM,MAAM,IAAI;EAChB,MAAM,KAAK,SAAS;CACtB;AACF;;AAGA,SAAgB,uBAAuB,KAK3B;CACV,MAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;CACzE,IAAI,MAAM;CACV,OAAO,EACL,MAAM,SAAS;EAEb,OAAO;GACL,IAAA,qBAF8B;GAG9B,OAAO,aAAa,QAAgB;IAClC,MAAM,IAAI,OAAO,MAAM,YAAY;IACnC,IAAI,CAAC,GACH,MAAM,IAAI,MACR,wFAAwF,OAAO,MAAM,GAAG,GAAG,GAC7G;IAEF,MAAM,GAAG,QAAQ,SAAS;IAC1B,MAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,CAAC,CAAC,KAAK;IACxD,MAAM,MAAM,MAAM,iBAAiB,QAAkB,OAAiB,OAAO,MAAM,gBAAgB;KACjG,MAAM,SAAS,YAAY,GAAG,eAAe,MAAM,cAAc;KACjE,MAAM,OAAO,MAAM,eACjB;MAAE,eAAe,IAAI;MAAe,WAAW,IAAI;MAAW,OAAO,IAAI;KAAM,GAC/E,QACA,SAAS,eACT,CAAC,YAAY,GACb,OAAO,MAAM,SAAS;MACpB,IAAI,SAAS,kBAAkB,OAAO,uBAAuB;MAC7D,MAAM,MAAM,MAAM,KAAK;OAAE,IAAI;OAAW,MAAM,OAAO,KAAK,QAAQ,EAAE;MAAE,CAAC;MACvE,MAAM,OAAO,IAAI,mBAAmB;MACpC,OAAO,GAAG,OAAO,IAAI,UAAU,EAAE,IAAI,OAAO,2FAA2F;KACzI,GACA,EAAE,SAAS,CACb;KACA,MAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;KAC9C,MAAM,aAAa,KAAK,UACrB,MAAM,EAAE,CAAC,CACT,KAAK,MAAM,UAAU,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,aAAa,EAAE,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC,CAChF,KAAK,SAAS,CAAC,CACf,MAAM,GAAG,IAAI;KAChB,OAAO;MACL,GAAG;MACH,OAAO,KAAK;MACZ,cAAc,KAAK,MAAM;MACzB,eAAe,KAAK,MAAM;MAC1B;KACF;IACF,CAAC;IAED,IAAI,IAAI,gBAAgB,IAAI,eAC1B,MAAM;KACJ,MAAM;KACN,MAAM;MAAE,UAAU,IAAI,gBAAgB;MAAG,WAAW,IAAI,iBAAiB;MAAG,OAAO,IAAI;KAAM;IAC/F;IAEF,MAAM;KAAE,MAAM;KAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE;IAAE;GACnE;GACA,MAAM,SAAS,CAAC;EAClB;CACF,EACF;AACF;;AAGA,MAAM,qBAA4C,EAChD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EAEvB,MAAM,KADK,IAA2C,KAAA,EACzC;EACb,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO;AACT,EACF;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,OAAO,sBAAsB;CACnC,OAAO;EACL,MAAM;EACN,QAAQ;EACR,iBAAiB,KAAK,UAAU;EAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,QAAO,MADa,KAAK,UAAU,IAAI,EAAA,CAC1B,KAAK,MAAM;IACtB,MAAM,OAAO,SAAS,CAAC;IACvB,OAAO;KACL,GAAG;KAGH,QAAQ,mBAAmB,KAAK,OAAO,GAAG,KAAK,MAAM;IACvD;GACF,CAAC;EACH;EAEA,oBAAoB,QAAQ,QAAQ,KAAA,CAAS;EAE7C,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,QAAQ;GAC3B,QAAQ;IACN,MAAM,IAAI,MACR,6EAA6E,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GACrH;GACF;GACA,IAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAC/D,MAAM,IAAI,MACR,sEAAsE,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GAC9G;GAEF,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI;GAClB,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;IACpC,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA,OAAO,IAAI,SAAS;KACpB;KACA,OAAO,IAAI;KACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;KACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;IACzE,CAAC;GACH;EACF;EAEA,aAAa,MAAM,uBAAuB,CAAC;CAC7C;AACF"}
+{"version":3,"file":"appworld.js","names":[],"sources":["../../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\ninterface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  transcript?: string\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))\n      }\n      pending.push((l) => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        resolve(l)\n      })\n      child.once('exit', onExit)\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    child.stdin.end()\n    child.kill('SIGTERM')\n  }\n}\n\n/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string) {\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const loop = await routerToolLoop(\n              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },\n              system,\n              `Task: ${instruction}`,\n              [EXECUTE_TOOL],\n              async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                return `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n              },\n              { maxTurns },\n            )\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = loop.toolTrace\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            return {\n              ...verdict,\n              turns: loop.turns,\n              input_tokens: loop.usage.input,\n              output_tokens: loop.usage.output,\n              transcript,\n            } satisfies ReactResult\n          })\n          // Real usage from the episode — flat llm_call so the kernel meters it.\n          if (out.input_tokens || out.output_tokens) {\n            yield {\n              type: 'llm_call',\n              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },\n            }\n          }\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;;AAG9D,MAAM,gBAAgB;;AAQtB,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,yDAAyD;CAEpG,OAAO;AACT;;;;;;;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;CAClE,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;CAC3E,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC5G;CACA,MAAM,OAAO,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;CACjD,MAAM,SAAS,KAAK,MAAM,IAAI;CAC9B,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,0BAA0B,OAAO,OAAO;CAC1E,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAC5D,MAAM,QAAQ,KAAK,SAAS;GAO5B,MAAM,SAAQ,MANK,OAAO;IACxB;IACA;IAAW;IACX,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;IAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;GAClD,CAAC,EAAA,CACiB,SAAS,CAAC;GAC5B,IAAI,MAAM,WAAW,GACnB,MAAM,IAAI,MAAM,kDAAkD,MAAM,GAAG,KAAK,UAAU,IAAI,GAAG;GAEnG,OAAO,MAAM,KACV,OAAkB;IACjB,IAAI,EAAE;IACN;IACA,QAAQ,EAAE,cAAc;IACxB,UAAU;KAAE,QAAQ,EAAE;KAAS;IAAM;GACvC,EACF;EACF;EAEA,MAAM,eAAe,CAQrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAO,MAAM,OAAO;IAAC;IAAY;IAAa,KAAK;IAAQ;IAAW,KAAK;GAAK,GAAG,QAAQ;GAOjG,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI,SAAS;GAI3B,MAAM,QAAQ,IAAI,aAAa,SAAS;GACxC,MAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;GAG3C,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA;KACA;KACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;IACxC,CAAC;GACH;EACF;CACF;AACF;AAmCA,MAAM,eAAe;AAErB,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,MAAM,eAAyB;CAC7B,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY,EAAE,MAAM;IAAE,MAAM;IAAU,aAAa;GAA2C,EAAE;GAChG,UAAU,CAAC,MAAM;EACnB;CACF;AACF;;AAGA,eAAe,iBACb,QACA,OACA,IACY;CACZ,MAAM,QAAQ,MAAM,YAAY;EAAC;EAAQ;EAAW;EAAa;EAAQ;EAAW;CAAK,GAAG,EAC1F,KAAK,UACP,CAAC;CACD,MAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;CAClD,MAAM,UAAyC,CAAC;CAChD,MAAM,UAAoB,CAAC;CAC3B,GAAG,GAAG,SAAS,MAAM;EACnB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK,CAAC;OACX,QAAQ,KAAK,CAAC;CACrB,CAAC;CACD,IAAI,SAAS;CACb,MAAM,OAAO,GAAG,SAAS,MAAc;EACrC,UAAU,EAAE,SAAS,MAAM;CAC7B,CAAC;CACD,MAAM,YAAY,cAChB,IAAI,SAAS,SAAS,WAAW;EAC/B,MAAM,cAAc,QAAQ,MAAM;EAClC,IAAI,gBAAgB,KAAA,GAAW,OAAO,QAAQ,WAAW;EACzD,MAAM,IAAI,iBACF,uBAAO,IAAI,MAAM,oCAAoC,UAAU,cAAc,OAAO,MAAM,IAAI,GAAG,CAAC,GACxG,SACF;EAGA,MAAM,UAAU,SAA8B;GAC5C,aAAa,CAAC;GACd,uBAAO,IAAI,MAAM,4BAA4B,KAAK,aAAa,OAAO,MAAM,IAAI,GAAG,CAAC;EACtF;EACA,QAAQ,MAAM,MAAM;GAClB,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,QAAQ,CAAC;EACX,CAAC;EACD,MAAM,KAAK,QAAQ,MAAM;CAC3B,CAAC;CACH,IAAI;EACF,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;EAChD,IAAI,CAAC,MAAM,OAAO,MAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,iBAAiB;EACvG,MAAM,OAAO,OAAO,QAAmE;GACrF,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG;GAC5C,MAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;GAC9C,IAAI,OAAO,IAAI,UAAU,UAAU,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO;GAC7F,OAAO;EACT;EACA,OAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;CAC/C,UAAU;EACR,MAAM,MAAM,IAAI;EAChB,MAAM,KAAK,SAAS;CACtB;AACF;;AAGA,SAAgB,uBAAuB,KAK3B;CACV,MAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;CACzE,IAAI,MAAM;CACV,OAAO,EACL,MAAM,SAAS;EAEb,OAAO;GACL,IAAA,qBAF8B;GAG9B,OAAO,aAAa,QAAgB;IAClC,MAAM,IAAI,OAAO,MAAM,YAAY;IACnC,IAAI,CAAC,GACH,MAAM,IAAI,MACR,wFAAwF,OAAO,MAAM,GAAG,GAAG,GAC7G;IAEF,MAAM,GAAG,QAAQ,SAAS;IAC1B,MAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,CAAC,CAAC,KAAK;IACxD,MAAM,MAAM,MAAM,iBAAiB,QAAkB,OAAiB,OAAO,MAAM,gBAAgB;KACjG,MAAM,SAAS,YAAY,GAAG,eAAe,MAAM,cAAc;KACjE,MAAM,OAAO,MAAM,eACjB;MAAE,eAAe,IAAI;MAAe,WAAW,IAAI;MAAW,OAAO,IAAI;KAAM,GAC/E,QACA,SAAS,eACT,CAAC,YAAY,GACb,OAAO,MAAM,SAAS;MACpB,IAAI,SAAS,kBAAkB,OAAO,uBAAuB;MAC7D,MAAM,MAAM,MAAM,KAAK;OAAE,IAAI;OAAW,MAAM,OAAO,KAAK,QAAQ,EAAE;MAAE,CAAC;MACvE,MAAM,OAAO,IAAI,mBAAmB;MACpC,OAAO,GAAG,OAAO,IAAI,UAAU,EAAE,IAAI,OAAO,2FAA2F;KACzI,GACA,EAAE,SAAS,CACb;KACA,MAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;KAC9C,MAAM,aAAa,KAAK,UACrB,MAAM,EAAE,CAAC,CACT,KAAK,MAAM,UAAU,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,aAAa,EAAE,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC,CAChF,KAAK,SAAS,CAAC,CACf,MAAM,GAAG,IAAI;KAChB,OAAO;MACL,GAAG;MACH,OAAO,KAAK;MACZ,cAAc,KAAK,MAAM;MACzB,eAAe,KAAK,MAAM;MAC1B;KACF;IACF,CAAC;IAED,IAAI,IAAI,gBAAgB,IAAI,eAC1B,MAAM;KACJ,MAAM;KACN,MAAM;MAAE,UAAU,IAAI,gBAAgB;MAAG,WAAW,IAAI,iBAAiB;MAAG,OAAO,IAAI;KAAM;IAC/F;IAEF,MAAM;KAAE,MAAM;KAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE;IAAE;GACnE;GACA,MAAM,SAAS,CAAC;EAClB;CACF,EACF;AACF;;AAGA,MAAM,qBAA4C,EAChD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EAEvB,MAAM,KADK,IAA2C,KAAA,EACzC;EACb,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO;AACT,EACF;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,OAAO,sBAAsB;CACnC,OAAO;EACL,MAAM;EACN,QAAQ;EACR,iBAAiB,KAAK,UAAU;EAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,QAAO,MADa,KAAK,UAAU,IAAI,EAAA,CAC1B,KAAK,MAAM;IACtB,MAAM,OAAO,SAAS,CAAC;IACvB,OAAO;KACL,GAAG;KAGH,QAAQ,mBAAmB,KAAK,OAAO,GAAG,KAAK,MAAM;IACvD;GACF,CAAC;EACH;EAEA,oBAAoB,QAAQ,QAAQ,KAAA,CAAS;EAE7C,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,QAAQ;GAC3B,QAAQ;IACN,MAAM,IAAI,MACR,6EAA6E,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GACrH;GACF;GACA,IAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAC/D,MAAM,IAAI,MACR,sEAAsE,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GAC9G;GAEF,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI;GAClB,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;IACpC,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA,OAAO,IAAI,SAAS;KACpB;KACA,OAAO,IAAI;KACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;KACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;IACzE,CAAC;GACH;EACF;EAEA,aAAa,MAAM,uBAAuB,CAAC;CAC7C;AACF"}

package/dist/benchmarks/bfcl.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/bfcl.d.ts
 declare const bfclOutput: OutputAdapter<string>;
 declare function createBfclAdapter(): BenchmarkAdapter;

package/dist/benchmarks/bfcl.js.map

@@ -1 +1 @@
-{"version":3,"file":"bfcl.js","names":[],"sources":["../../src/benchmarks/bfcl.ts"],"sourcesContent":["/**\n * Berkeley Function Calling Leaderboard adapter.\n *\n * Scope: deterministic function-call ground-truth categories from the official\n * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic\n * web-search/memory categories and BFCL's own model-response harness remain\n * upstream responsibilities. The adapter loads official JSONL rows plus their\n * `possible_answer` file and scores structured function-call artifacts against\n * allowed function/argument values.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'bfcl.json')\nconst DEFAULT_CATEGORY = 'BFCL_v4_simple_python'\n\ninterface BfclRow {\n  id: string\n  question: unknown\n  function: unknown\n}\n\ninterface BfclAnswerRow {\n  id: string\n  ground_truth: unknown\n}\n\ninterface BfclAllowedCall {\n  name: string\n  arguments: Record<string, unknown[]>\n}\n\ninterface BfclMeta {\n  rowId: string\n  category: string\n  functions: unknown\n  expected: BfclAllowedCall[]\n  scoring: 'bfcl-ground-truth-subset'\n}\n\ninterface BfclActualCall {\n  name: string\n  arguments: Record<string, unknown>\n}\n\nconst bfclDir = (): string | undefined => process.env.BFCL_DIR\nconst bfclCategory = (): string => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY\n\nexport const bfclOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nasync function assertPath(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readJsonl(raw: string): unknown[] {\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nfunction taskFile(dir: string, category: string): string {\n  return process.env.BFCL_DATA_FILE ?? join(dir, 'bfcl_eval', 'data', `${category}.json`)\n}\n\nfunction answerFile(dir: string, category: string): string {\n  return process.env.BFCL_ANSWER_FILE ?? join(dir, 'bfcl_eval', 'data', 'possible_answer', `${category}.json`)\n}\n\nfunction questionText(question: unknown): string {\n  if (typeof question === 'string') return question\n  if (!Array.isArray(question)) return JSON.stringify(question, null, 2)\n  const turns: string[] = []\n  for (const item of question.flat(3)) {\n    if (item && typeof item === 'object') {\n      const role = typeof (item as { role?: unknown }).role === 'string' ? (item as { role: string }).role : 'user'\n      const content = (item as { content?: unknown }).content\n      if (typeof content === 'string') turns.push(`${role}: ${content}`)\n    }\n  }\n  return turns.length > 0 ? turns.join('\\n') : JSON.stringify(question, null, 2)\n}\n\nfunction normalizeScalar(value: unknown): string {\n  if (typeof value === 'number') return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)))\n  if (typeof value === 'string') return value.trim().toLowerCase()\n  return JSON.stringify(value)\n}\n\nfunction normalizeAllowed(value: unknown): unknown[] {\n  return Array.isArray(value) ? value : [value]\n}\n\nfunction groundTruthToCalls(value: unknown): BfclAllowedCall[] {\n  if (!Array.isArray(value)) return []\n  const out: BfclAllowedCall[] = []\n  for (const item of value) {\n    if (!item || typeof item !== 'object') continue\n    for (const [name, args] of Object.entries(item as Record<string, unknown>)) {\n      if (!args || typeof args !== 'object' || Array.isArray(args)) continue\n      const normalized: Record<string, unknown[]> = {}\n      for (const [argName, allowed] of Object.entries(args as Record<string, unknown>)) {\n        normalized[argName] = normalizeAllowed(allowed)\n      }\n      out.push({ name, arguments: normalized })\n    }\n  }\n  return out\n}\n\nfunction rowToTask(row: BfclRow, answer: BfclAnswerRow, category: string): BenchTask {\n  const expected = groundTruthToCalls(answer.ground_truth)\n  if (expected.length === 0) {\n    throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`)\n  }\n  const meta: BfclMeta = {\n    rowId: row.id,\n    category,\n    functions: row.function,\n    expected,\n    scoring: 'bfcl-ground-truth-subset',\n  }\n  return {\n    id: row.id,\n    split: category,\n    prompt: [\n      'Solve this BFCL function-calling task.',\n      'Return only JSON: {\"function_calls\":[{\"name\":\"...\",\"arguments\":{...}}]}.',\n      '',\n      questionText(row.question),\n      '',\n      `Available functions: ${JSON.stringify(row.function, null, 2)}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): BfclMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.expected)) {\n    throw new Error(`BFCL task ${task.id} missing expected calls — loadTasks did not populate metadata`)\n  }\n  return md as unknown as BfclMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  let out = tasks\n  if (opts.split) out = out.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    out = out.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    out = out.slice(0, opts.limit)\n  }\n  if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`)\n  return out\n}\n\nfunction buildTasks(rows: BfclRow[], answers: BfclAnswerRow[], category: string, opts: LoadOptions): BenchTask[] {\n  const byId = new Map(answers.map((answer) => [answer.id, answer]))\n  const tasks = rows.map((row) => {\n    const answer = byId.get(row.id)\n    if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`)\n    return rowToTask(row, answer, category)\n  })\n  return selectTasks(tasks, opts)\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixture = JSON.parse(await readFile(FIXTURES, 'utf8')) as { rows: BfclRow[]; answers: BfclAnswerRow[]; category: string }\n  console.warn(`[bfcl] BFCL_FIXTURES=1 — loading ${fixture.rows.length} adapter fixtures`)\n  return buildTasks(fixture.rows, fixture.answers, fixture.category, opts)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const category = bfclCategory()\n  const dataPath = taskFile(dir, category)\n  const answersPath = answerFile(dir, category)\n  const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, 'utf8'), readFile(answersPath, 'utf8')])\n  const rows = readJsonl(rawRows) as BfclRow[]\n  const answers = readJsonl(rawAnswers) as BfclAnswerRow[]\n  return buildTasks(rows, answers, category, opts)\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\nfunction normalizeArguments(value: unknown): Record<string, unknown> {\n  if (typeof value === 'string') {\n    try {\n      return normalizeArguments(JSON.parse(value))\n    } catch {\n      return {}\n    }\n  }\n  if (!value || typeof value !== 'object' || Array.isArray(value)) return {}\n  return value as Record<string, unknown>\n}\n\nfunction callFromObject(value: unknown): BfclActualCall | undefined {\n  if (!value || typeof value !== 'object') return undefined\n  const raw = value as Record<string, unknown>\n  if (raw.function && typeof raw.function === 'object') {\n    const fn = raw.function as Record<string, unknown>\n    if (typeof fn.name === 'string') return { name: fn.name, arguments: normalizeArguments(fn.arguments) }\n  }\n  const name =\n    typeof raw.name === 'string' ? raw.name\n      : typeof raw.function_name === 'string' ? raw.function_name\n        : typeof raw.tool_name === 'string' ? raw.tool_name\n          : undefined\n  if (!name) return undefined\n  return { name, arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters) }\n}\n\nfunction extractActualCalls(artifact: string): BfclActualCall[] {\n  const parsed = extractJsonBlock(artifact)\n  if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    for (const key of ['function_calls', 'tool_calls', 'calls']) {\n      if (Array.isArray(raw[key])) {\n        return raw[key].map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n      }\n    }\n    const single = callFromObject(raw)\n    if (single) return [single]\n  }\n  return []\n}\n\nfunction argMatches(expectedValues: unknown[], actual: unknown): boolean {\n  return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual))\n}\n\nfunction callMatches(expected: BfclAllowedCall, actual: BfclActualCall): boolean {\n  if (expected.name !== actual.name) return false\n  for (const [argName, allowed] of Object.entries(expected.arguments)) {\n    if (!(argName in actual.arguments)) {\n      if (allowed.some((value) => value === '' || value === null || value === undefined)) continue\n      return false\n    }\n    if (!argMatches(allowed, actual.arguments[argName])) return false\n  }\n  return true\n}\n\nfunction scoreCalls(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  const actual = extractActualCalls(artifact)\n  const used = new Set<number>()\n  let matched = 0\n  for (const expected of meta.expected) {\n    const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call))\n    if (index >= 0) {\n      used.add(index)\n      matched += 1\n    }\n  }\n  const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length\n  const precision = actual.length === 0 ? 0 : matched / actual.length\n  const score = recall\n  return {\n    resolved: recall === 1 && precision === 1,\n    score,\n    detail: JSON.stringify({\n      scoring: meta.scoring,\n      category: meta.category,\n      expected: meta.expected,\n      actual,\n      precision,\n      recall,\n      fullBfclLeaderboardScore: null,\n    }),\n  }\n}\n\nexport function createBfclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.BFCL_FIXTURES === '1'\n\n  return {\n    name: 'bfcl',\n    output: bfclOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = bfclDir()\n      if (!dir) {\n        throw new Error(\n          'BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python.',\n        )\n      }\n      const category = bfclCategory()\n      await assertPath(taskFile(dir, category), 'BFCL task JSONL')\n      await assertPath(answerFile(dir, category), 'BFCL possible_answer JSONL')\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = bfclDir()\n      if (!dir) throw new Error('BFCL_DIR is required to load official BFCL rows')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return JSON.stringify({\n        function_calls: meta.expected.map((call) => ({\n          name: call.name,\n          arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]])),\n        })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreCalls(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,MAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AACxD,MAAM,mBAAmB;AA+BzB,MAAM,gBAAoC,QAAQ,IAAI;AACtD,MAAM,qBAA6B,QAAQ,IAAI,iBAAiB;AAEhE,MAAa,aAAoC,EAC/C,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,eAAe,WAAW,MAAc,OAA8B;CACpE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,iBAAiB,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACnG;AACF;AAEA,SAAS,UAAU,KAAwB;CACzC,OAAO,IACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAS,SAAS,KAAa,UAA0B;CACvD,OAAO,QAAQ,IAAI,kBAAkB,KAAK,KAAK,aAAa,QAAQ,GAAG,SAAS,MAAM;AACxF;AAEA,SAAS,WAAW,KAAa,UAA0B;CACzD,OAAO,QAAQ,IAAI,oBAAoB,KAAK,KAAK,aAAa,QAAQ,mBAAmB,GAAG,SAAS,MAAM;AAC7G;AAEA,SAAS,aAAa,UAA2B;CAC/C,IAAI,OAAO,aAAa,UAAU,OAAO;CACzC,IAAI,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO,KAAK,UAAU,UAAU,MAAM,CAAC;CACrE,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,QAAQ,SAAS,KAAK,CAAC,GAChC,IAAI,QAAQ,OAAO,SAAS,UAAU;EACpC,MAAM,OAAO,OAAQ,KAA4B,SAAS,WAAY,KAA0B,OAAO;EACvG,MAAM,UAAW,KAA+B;EAChD,IAAI,OAAO,YAAY,UAAU,MAAM,KAAK,GAAG,KAAK,IAAI,SAAS;CACnE;CAEF,OAAO,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,UAAU,UAAU,MAAM,CAAC;AAC/E;AAEA,SAAS,gBAAgB,OAAwB;CAC/C,IAAI,OAAO,UAAU,UAAU,OAAO,OAAO,UAAU,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,OAAO,MAAM,QAAQ,CAAC,CAAC,CAAC;CAC/G,IAAI,OAAO,UAAU,UAAU,OAAO,MAAM,KAAK,CAAC,CAAC,YAAY;CAC/D,OAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAS,iBAAiB,OAA2B;CACnD,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAC9C;AAEA,SAAS,mBAAmB,OAAmC;CAC7D,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,MAAM,MAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;EACvC,KAAK,MAAM,CAAC,MAAM,SAAS,OAAO,QAAQ,IAA+B,GAAG;GAC1E,IAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;GAC9D,MAAM,aAAwC,CAAC;GAC/C,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,IAA+B,GAC7E,WAAW,WAAW,iBAAiB,OAAO;GAEhD,IAAI,KAAK;IAAE;IAAM,WAAW;GAAW,CAAC;EAC1C;CACF;CACA,OAAO;AACT;AAEA,SAAS,UAAU,KAAc,QAAuB,UAA6B;CACnF,MAAM,WAAW,mBAAmB,OAAO,YAAY;CACvD,IAAI,SAAS,WAAW,GACtB,MAAM,IAAI,MAAM,QAAQ,IAAI,GAAG,0DAA0D;CAE3F,MAAM,OAAiB;EACrB,OAAO,IAAI;EACX;EACA,WAAW,IAAI;EACf;EACA,SAAS;CACX;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ;GACN;GACA;GACA;GACA,aAAa,IAAI,QAAQ;GACzB;GACA,wBAAwB,KAAK,UAAU,IAAI,UAAU,MAAM,CAAC;EAC9D,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA2B;CAC3C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,QAAQ,GACnC,MAAM,IAAI,MAAM,aAAa,KAAK,GAAG,8DAA8D;CAErG,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,MAAM;CACV,IAAI,KAAK,OAAO,MAAM,IAAI,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACpE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,MAAM,IAAI,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAC9C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;CAE/B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,0BAA0B,KAAK,UAAU,IAAI,GAAG;CACtF,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,SAA0B,UAAkB,MAAgC;CAC/G,MAAM,OAAO,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;CAMjE,OAAO,YALO,KAAK,KAAK,QAAQ;EAC9B,MAAM,SAAS,KAAK,IAAI,IAAI,EAAE;EAC9B,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,QAAQ,SAAS,gCAAgC,IAAI,IAAI;EACtF,OAAO,UAAU,KAAK,QAAQ,QAAQ;CACxC,CACuB,GAAG,IAAI;AAChC;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CAC3D,QAAQ,KAAK,oCAAoC,QAAQ,KAAK,OAAO,kBAAkB;CACvF,OAAO,WAAW,QAAQ,MAAM,QAAQ,SAAS,QAAQ,UAAU,IAAI;AACzE;AAEA,eAAe,kBAAkB,KAAa,MAAyC;CACrF,MAAM,WAAW,aAAa;CAC9B,MAAM,WAAW,SAAS,KAAK,QAAQ;CACvC,MAAM,cAAc,WAAW,KAAK,QAAQ;CAC5C,MAAM,CAAC,SAAS,cAAc,MAAM,QAAQ,IAAI,CAAC,SAAS,UAAU,MAAM,GAAG,SAAS,aAAa,MAAM,CAAC,CAAC;CAG3G,OAAO,WAFM,UAAU,OAEF,GADL,UAAU,UACI,GAAG,UAAU,IAAI;AACjD;AAEA,SAAS,iBAAiB,MAAuB;CAE/C,MAAM,OAAO,CADG,GAAG,KAAK,SAAS,+BAA+B,CAC9C,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAC9C,IAAI;EACF,OAAO,KAAK,MAAM,GAAG;CACvB,QAAQ;EACN;CACF;AACF;AAEA,SAAS,mBAAmB,OAAyC;CACnE,IAAI,OAAO,UAAU,UACnB,IAAI;EACF,OAAO,mBAAmB,KAAK,MAAM,KAAK,CAAC;CAC7C,QAAQ;EACN,OAAO,CAAC;CACV;CAEF,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACzE,OAAO;AACT;AAEA,SAAS,eAAe,OAA4C;CAClE,IAAI,CAAC,SAAS,OAAO,UAAU,UAAU,OAAO,KAAA;CAChD,MAAM,MAAM;CACZ,IAAI,IAAI,YAAY,OAAO,IAAI,aAAa,UAAU;EACpD,MAAM,KAAK,IAAI;EACf,IAAI,OAAO,GAAG,SAAS,UAAU,OAAO;GAAE,MAAM,GAAG;GAAM,WAAW,mBAAmB,GAAG,SAAS;EAAE;CACvG;CACA,MAAM,OACJ,OAAO,IAAI,SAAS,WAAW,IAAI,OAC/B,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAC1C,OAAO,IAAI,cAAc,WAAW,IAAI,YACtC,KAAA;CACV,IAAI,CAAC,MAAM,OAAO,KAAA;CAClB,OAAO;EAAE;EAAM,WAAW,mBAAmB,IAAI,aAAa,IAAI,QAAQ,IAAI,UAAU;CAAE;AAC5F;AAEA,SAAS,mBAAmB,UAAoC;CAC9D,MAAM,SAAS,iBAAiB,QAAQ;CACxC,IAAI,MAAM,QAAQ,MAAM,GAAG,OAAO,OAAO,IAAI,cAAc,CAAC,CAAC,QAAQ,SAAiC,QAAQ,IAAI,CAAC;CACnH,IAAI,UAAU,OAAO,WAAW,UAAU;EACxC,MAAM,MAAM;EACZ,KAAK,MAAM,OAAO;GAAC;GAAkB;GAAc;EAAO,GACxD,IAAI,MAAM,QAAQ,IAAI,IAAI,GACxB,OAAO,IAAI,IAAI,CAAC,IAAI,cAAc,CAAC,CAAC,QAAQ,SAAiC,QAAQ,IAAI,CAAC;EAG9F,MAAM,SAAS,eAAe,GAAG;EACjC,IAAI,QAAQ,OAAO,CAAC,MAAM;CAC5B;CACA,OAAO,CAAC;AACV;AAEA,SAAS,WAAW,gBAA2B,QAA0B;CACvE,OAAO,eAAe,MAAM,aAAa,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM,CAAC;AAChG;AAEA,SAAS,YAAY,UAA2B,QAAiC;CAC/E,IAAI,SAAS,SAAS,OAAO,MAAM,OAAO;CAC1C,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,SAAS,SAAS,GAAG;EACnE,IAAI,EAAE,WAAW,OAAO,YAAY;GAClC,IAAI,QAAQ,MAAM,UAAU,UAAU,MAAM,UAAU,QAAQ,UAAU,KAAA,CAAS,GAAG;GACpF,OAAO;EACT;EACA,IAAI,CAAC,WAAW,SAAS,OAAO,UAAU,QAAQ,GAAG,OAAO;CAC9D;CACA,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,UAA8B;CACjE,MAAM,OAAO,SAAS,IAAI;CAC1B,MAAM,SAAS,mBAAmB,QAAQ;CAC1C,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,UAAU;CACd,KAAK,MAAM,YAAY,KAAK,UAAU;EACpC,MAAM,QAAQ,OAAO,WAAW,MAAM,MAAM,CAAC,KAAK,IAAI,CAAC,KAAK,YAAY,UAAU,IAAI,CAAC;EACvF,IAAI,SAAS,GAAG;GACd,KAAK,IAAI,KAAK;GACd,WAAW;EACb;CACF;CACA,MAAM,SAAS,KAAK,SAAS,WAAW,IAAI,IAAI,UAAU,KAAK,SAAS;CACxE,MAAM,YAAY,OAAO,WAAW,IAAI,IAAI,UAAU,OAAO;CAE7D,OAAO;EACL,UAAU,WAAW,KAAK,cAAc;EACxC,OAAA;EACA,QAAQ,KAAK,UAAU;GACrB,SAAS,KAAK;GACd,UAAU,KAAK;GACf,UAAU,KAAK;GACf;GACA;GACA;GACA,0BAA0B;EAC5B,CAAC;CACH;AACF;AAEA,SAAgB,oBAAsC;CACpD,MAAM,eAAe,QAAQ,IAAI,kBAAkB;CAEnD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,QAAQ;GACpB,IAAI,CAAC,KACH,MAAM,IAAI,MACR,qMACF;GAEF,MAAM,WAAW,aAAa;GAC9B,MAAM,WAAW,SAAS,KAAK,QAAQ,GAAG,iBAAiB;GAC3D,MAAM,WAAW,WAAW,KAAK,QAAQ,GAAG,4BAA4B;GACxE,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;EAC3C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,MAAM,QAAQ;GACpB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,iDAAiD;GAC3E,OAAO,kBAAkB,KAAK,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,UAAU,EACpB,gBAAgB,KAAK,SAAS,KAAK,UAAU;IAC3C,MAAM,KAAK;IACX,WAAW,OAAO,YAAY,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,YAAY,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;GACvG,EAAE,EACJ,GAAG,MAAM,CAAC;EACZ;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,OAAO,WAAW,MAAM,QAAQ;EAClC;CACF;AACF"}
+{"version":3,"file":"bfcl.js","names":[],"sources":["../../src/benchmarks/bfcl.ts"],"sourcesContent":["/**\n * Berkeley Function Calling Leaderboard adapter.\n *\n * Scope: deterministic function-call ground-truth categories from the official\n * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic\n * web-search/memory categories and BFCL's own model-response harness remain\n * upstream responsibilities. The adapter loads official JSONL rows plus their\n * `possible_answer` file and scores structured function-call artifacts against\n * allowed function/argument values.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'bfcl.json')\nconst DEFAULT_CATEGORY = 'BFCL_v4_simple_python'\n\ninterface BfclRow {\n  id: string\n  question: unknown\n  function: unknown\n}\n\ninterface BfclAnswerRow {\n  id: string\n  ground_truth: unknown\n}\n\ninterface BfclAllowedCall {\n  name: string\n  arguments: Record<string, unknown[]>\n}\n\ninterface BfclMeta {\n  rowId: string\n  category: string\n  functions: unknown\n  expected: BfclAllowedCall[]\n  scoring: 'bfcl-ground-truth-subset'\n}\n\ninterface BfclActualCall {\n  name: string\n  arguments: Record<string, unknown>\n}\n\nconst bfclDir = (): string | undefined => process.env.BFCL_DIR\nconst bfclCategory = (): string => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY\n\nexport const bfclOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nasync function assertPath(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readJsonl(raw: string): unknown[] {\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nfunction taskFile(dir: string, category: string): string {\n  return process.env.BFCL_DATA_FILE ?? join(dir, 'bfcl_eval', 'data', `${category}.json`)\n}\n\nfunction answerFile(dir: string, category: string): string {\n  return process.env.BFCL_ANSWER_FILE ?? join(dir, 'bfcl_eval', 'data', 'possible_answer', `${category}.json`)\n}\n\nfunction questionText(question: unknown): string {\n  if (typeof question === 'string') return question\n  if (!Array.isArray(question)) return JSON.stringify(question, null, 2)\n  const turns: string[] = []\n  for (const item of question.flat(3)) {\n    if (item && typeof item === 'object') {\n      const role = typeof (item as { role?: unknown }).role === 'string' ? (item as { role: string }).role : 'user'\n      const content = (item as { content?: unknown }).content\n      if (typeof content === 'string') turns.push(`${role}: ${content}`)\n    }\n  }\n  return turns.length > 0 ? turns.join('\\n') : JSON.stringify(question, null, 2)\n}\n\nfunction normalizeScalar(value: unknown): string {\n  if (typeof value === 'number') return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)))\n  if (typeof value === 'string') return value.trim().toLowerCase()\n  return JSON.stringify(value)\n}\n\nfunction normalizeAllowed(value: unknown): unknown[] {\n  return Array.isArray(value) ? value : [value]\n}\n\nfunction groundTruthToCalls(value: unknown): BfclAllowedCall[] {\n  if (!Array.isArray(value)) return []\n  const out: BfclAllowedCall[] = []\n  for (const item of value) {\n    if (!item || typeof item !== 'object') continue\n    for (const [name, args] of Object.entries(item as Record<string, unknown>)) {\n      if (!args || typeof args !== 'object' || Array.isArray(args)) continue\n      const normalized: Record<string, unknown[]> = {}\n      for (const [argName, allowed] of Object.entries(args as Record<string, unknown>)) {\n        normalized[argName] = normalizeAllowed(allowed)\n      }\n      out.push({ name, arguments: normalized })\n    }\n  }\n  return out\n}\n\nfunction rowToTask(row: BfclRow, answer: BfclAnswerRow, category: string): BenchTask {\n  const expected = groundTruthToCalls(answer.ground_truth)\n  if (expected.length === 0) {\n    throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`)\n  }\n  const meta: BfclMeta = {\n    rowId: row.id,\n    category,\n    functions: row.function,\n    expected,\n    scoring: 'bfcl-ground-truth-subset',\n  }\n  return {\n    id: row.id,\n    split: category,\n    prompt: [\n      'Solve this BFCL function-calling task.',\n      'Return only JSON: {\"function_calls\":[{\"name\":\"...\",\"arguments\":{...}}]}.',\n      '',\n      questionText(row.question),\n      '',\n      `Available functions: ${JSON.stringify(row.function, null, 2)}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): BfclMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.expected)) {\n    throw new Error(`BFCL task ${task.id} missing expected calls — loadTasks did not populate metadata`)\n  }\n  return md as unknown as BfclMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  let out = tasks\n  if (opts.split) out = out.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    out = out.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    out = out.slice(0, opts.limit)\n  }\n  if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`)\n  return out\n}\n\nfunction buildTasks(rows: BfclRow[], answers: BfclAnswerRow[], category: string, opts: LoadOptions): BenchTask[] {\n  const byId = new Map(answers.map((answer) => [answer.id, answer]))\n  const tasks = rows.map((row) => {\n    const answer = byId.get(row.id)\n    if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`)\n    return rowToTask(row, answer, category)\n  })\n  return selectTasks(tasks, opts)\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixture = JSON.parse(await readFile(FIXTURES, 'utf8')) as { rows: BfclRow[]; answers: BfclAnswerRow[]; category: string }\n  console.warn(`[bfcl] BFCL_FIXTURES=1 — loading ${fixture.rows.length} adapter fixtures`)\n  return buildTasks(fixture.rows, fixture.answers, fixture.category, opts)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const category = bfclCategory()\n  const dataPath = taskFile(dir, category)\n  const answersPath = answerFile(dir, category)\n  const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, 'utf8'), readFile(answersPath, 'utf8')])\n  const rows = readJsonl(rawRows) as BfclRow[]\n  const answers = readJsonl(rawAnswers) as BfclAnswerRow[]\n  return buildTasks(rows, answers, category, opts)\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\nfunction normalizeArguments(value: unknown): Record<string, unknown> {\n  if (typeof value === 'string') {\n    try {\n      return normalizeArguments(JSON.parse(value))\n    } catch {\n      return {}\n    }\n  }\n  if (!value || typeof value !== 'object' || Array.isArray(value)) return {}\n  return value as Record<string, unknown>\n}\n\nfunction callFromObject(value: unknown): BfclActualCall | undefined {\n  if (!value || typeof value !== 'object') return undefined\n  const raw = value as Record<string, unknown>\n  if (raw.function && typeof raw.function === 'object') {\n    const fn = raw.function as Record<string, unknown>\n    if (typeof fn.name === 'string') return { name: fn.name, arguments: normalizeArguments(fn.arguments) }\n  }\n  const name =\n    typeof raw.name === 'string' ? raw.name\n      : typeof raw.function_name === 'string' ? raw.function_name\n        : typeof raw.tool_name === 'string' ? raw.tool_name\n          : undefined\n  if (!name) return undefined\n  return { name, arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters) }\n}\n\nfunction extractActualCalls(artifact: string): BfclActualCall[] {\n  const parsed = extractJsonBlock(artifact)\n  if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    for (const key of ['function_calls', 'tool_calls', 'calls']) {\n      if (Array.isArray(raw[key])) {\n        return raw[key].map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n      }\n    }\n    const single = callFromObject(raw)\n    if (single) return [single]\n  }\n  return []\n}\n\nfunction argMatches(expectedValues: unknown[], actual: unknown): boolean {\n  return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual))\n}\n\nfunction callMatches(expected: BfclAllowedCall, actual: BfclActualCall): boolean {\n  if (expected.name !== actual.name) return false\n  for (const [argName, allowed] of Object.entries(expected.arguments)) {\n    if (!(argName in actual.arguments)) {\n      if (allowed.some((value) => value === '' || value === null || value === undefined)) continue\n      return false\n    }\n    if (!argMatches(allowed, actual.arguments[argName])) return false\n  }\n  return true\n}\n\nfunction scoreCalls(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  const actual = extractActualCalls(artifact)\n  const used = new Set<number>()\n  let matched = 0\n  for (const expected of meta.expected) {\n    const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call))\n    if (index >= 0) {\n      used.add(index)\n      matched += 1\n    }\n  }\n  const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length\n  const precision = actual.length === 0 ? 0 : matched / actual.length\n  const score = recall\n  return {\n    resolved: recall === 1 && precision === 1,\n    score,\n    detail: JSON.stringify({\n      scoring: meta.scoring,\n      category: meta.category,\n      expected: meta.expected,\n      actual,\n      precision,\n      recall,\n      fullBfclLeaderboardScore: null,\n    }),\n  }\n}\n\nexport function createBfclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.BFCL_FIXTURES === '1'\n\n  return {\n    name: 'bfcl',\n    output: bfclOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = bfclDir()\n      if (!dir) {\n        throw new Error(\n          'BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python.',\n        )\n      }\n      const category = bfclCategory()\n      await assertPath(taskFile(dir, category), 'BFCL task JSONL')\n      await assertPath(answerFile(dir, category), 'BFCL possible_answer JSONL')\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = bfclDir()\n      if (!dir) throw new Error('BFCL_DIR is required to load official BFCL rows')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return JSON.stringify({\n        function_calls: meta.expected.map((call) => ({\n          name: call.name,\n          arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]])),\n        })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreCalls(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,MAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AACxD,MAAM,mBAAmB;AA+BzB,MAAM,gBAAoC,QAAQ,IAAI;AACtD,MAAM,qBAA6B,QAAQ,IAAI,iBAAiB;AAEhE,MAAa,aAAoC,EAC/C,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,eAAe,WAAW,MAAc,OAA8B;CACpE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,iBAAiB,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACnG;AACF;AAEA,SAAS,UAAU,KAAwB;CACzC,OAAO,IACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAS,SAAS,KAAa,UAA0B;CACvD,OAAO,QAAQ,IAAI,kBAAkB,KAAK,KAAK,aAAa,QAAQ,GAAG,SAAS,MAAM;AACxF;AAEA,SAAS,WAAW,KAAa,UAA0B;CACzD,OAAO,QAAQ,IAAI,oBAAoB,KAAK,KAAK,aAAa,QAAQ,mBAAmB,GAAG,SAAS,MAAM;AAC7G;AAEA,SAAS,aAAa,UAA2B;CAC/C,IAAI,OAAO,aAAa,UAAU,OAAO;CACzC,IAAI,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO,KAAK,UAAU,UAAU,MAAM,CAAC;CACrE,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,QAAQ,SAAS,KAAK,CAAC,GAChC,IAAI,QAAQ,OAAO,SAAS,UAAU;EACpC,MAAM,OAAO,OAAQ,KAA4B,SAAS,WAAY,KAA0B,OAAO;EACvG,MAAM,UAAW,KAA+B;EAChD,IAAI,OAAO,YAAY,UAAU,MAAM,KAAK,GAAG,KAAK,IAAI,SAAS;CACnE;CAEF,OAAO,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,UAAU,UAAU,MAAM,CAAC;AAC/E;AAEA,SAAS,gBAAgB,OAAwB;CAC/C,IAAI,OAAO,UAAU,UAAU,OAAO,OAAO,UAAU,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,OAAO,MAAM,QAAQ,CAAC,CAAC,CAAC;CAC/G,IAAI,OAAO,UAAU,UAAU,OAAO,MAAM,KAAK,CAAC,CAAC,YAAY;CAC/D,OAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAS,iBAAiB,OAA2B;CACnD,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAC9C;AAEA,SAAS,mBAAmB,OAAmC;CAC7D,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,MAAM,MAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;EACvC,KAAK,MAAM,CAAC,MAAM,SAAS,OAAO,QAAQ,IAA+B,GAAG;GAC1E,IAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;GAC9D,MAAM,aAAwC,CAAC;GAC/C,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,IAA+B,GAC7E,WAAW,WAAW,iBAAiB,OAAO;GAEhD,IAAI,KAAK;IAAE;IAAM,WAAW;GAAW,CAAC;EAC1C;CACF;CACA,OAAO;AACT;AAEA,SAAS,UAAU,KAAc,QAAuB,UAA6B;CACnF,MAAM,WAAW,mBAAmB,OAAO,YAAY;CACvD,IAAI,SAAS,WAAW,GACtB,MAAM,IAAI,MAAM,QAAQ,IAAI,GAAG,0DAA0D;CAE3F,MAAM,OAAiB;EACrB,OAAO,IAAI;EACX;EACA,WAAW,IAAI;EACf;EACA,SAAS;CACX;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ;GACN;GACA;GACA;GACA,aAAa,IAAI,QAAQ;GACzB;GACA,wBAAwB,KAAK,UAAU,IAAI,UAAU,MAAM,CAAC;EAC9D,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA2B;CAC3C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,QAAQ,GACnC,MAAM,IAAI,MAAM,aAAa,KAAK,GAAG,8DAA8D;CAErG,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,MAAM;CACV,IAAI,KAAK,OAAO,MAAM,IAAI,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACpE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,MAAM,IAAI,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAC9C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;CAE/B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,0BAA0B,KAAK,UAAU,IAAI,GAAG;CACtF,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,SAA0B,UAAkB,MAAgC;CAC/G,MAAM,OAAO,IAAI,IAAI,QAAQ,KAAK,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;CAMjE,OAAO,YALO,KAAK,KAAK,QAAQ;EAC9B,MAAM,SAAS,KAAK,IAAI,IAAI,EAAE;EAC9B,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,QAAQ,SAAS,gCAAgC,IAAI,IAAI;EACtF,OAAO,UAAU,KAAK,QAAQ,QAAQ;CACxC,CACuB,GAAG,IAAI;AAChC;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CAC3D,QAAQ,KAAK,oCAAoC,QAAQ,KAAK,OAAO,kBAAkB;CACvF,OAAO,WAAW,QAAQ,MAAM,QAAQ,SAAS,QAAQ,UAAU,IAAI;AACzE;AAEA,eAAe,kBAAkB,KAAa,MAAyC;CACrF,MAAM,WAAW,aAAa;CAC9B,MAAM,WAAW,SAAS,KAAK,QAAQ;CACvC,MAAM,cAAc,WAAW,KAAK,QAAQ;CAC5C,MAAM,CAAC,SAAS,cAAc,MAAM,QAAQ,IAAI,CAAC,SAAS,UAAU,MAAM,GAAG,SAAS,aAAa,MAAM,CAAC,CAAC;CAG3G,OAAO,WAFM,UAAU,OAEF,GADL,UAAU,UACI,GAAG,UAAU,IAAI;AACjD;AAEA,SAAS,iBAAiB,MAAuB;CAE/C,MAAM,OAAO,CADG,GAAG,KAAK,SAAS,+BAA+B,CAC9C,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAC9C,IAAI;EACF,OAAO,KAAK,MAAM,GAAG;CACvB,QAAQ;EACN;CACF;AACF;AAEA,SAAS,mBAAmB,OAAyC;CACnE,IAAI,OAAO,UAAU,UACnB,IAAI;EACF,OAAO,mBAAmB,KAAK,MAAM,KAAK,CAAC;CAC7C,QAAQ;EACN,OAAO,CAAC;CACV;CAEF,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACzE,OAAO;AACT;AAEA,SAAS,eAAe,OAA4C;CAClE,IAAI,CAAC,SAAS,OAAO,UAAU,UAAU,OAAO,KAAA;CAChD,MAAM,MAAM;CACZ,IAAI,IAAI,YAAY,OAAO,IAAI,aAAa,UAAU;EACpD,MAAM,KAAK,IAAI;EACf,IAAI,OAAO,GAAG,SAAS,UAAU,OAAO;GAAE,MAAM,GAAG;GAAM,WAAW,mBAAmB,GAAG,SAAS;EAAE;CACvG;CACA,MAAM,OACJ,OAAO,IAAI,SAAS,WAAW,IAAI,OAC/B,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAC1C,OAAO,IAAI,cAAc,WAAW,IAAI,YACtC,KAAA;CACV,IAAI,CAAC,MAAM,OAAO,KAAA;CAClB,OAAO;EAAE;EAAM,WAAW,mBAAmB,IAAI,aAAa,IAAI,QAAQ,IAAI,UAAU;CAAE;AAC5F;AAEA,SAAS,mBAAmB,UAAoC;CAC9D,MAAM,SAAS,iBAAiB,QAAQ;CACxC,IAAI,MAAM,QAAQ,MAAM,GAAG,OAAO,OAAO,IAAI,cAAc,CAAC,CAAC,QAAQ,SAAiC,QAAQ,IAAI,CAAC;CACnH,IAAI,UAAU,OAAO,WAAW,UAAU;EACxC,MAAM,MAAM;EACZ,KAAK,MAAM,OAAO;GAAC;GAAkB;GAAc;EAAO,GACxD,IAAI,MAAM,QAAQ,IAAI,IAAI,GACxB,OAAO,IAAI,IAAI,CAAC,IAAI,cAAc,CAAC,CAAC,QAAQ,SAAiC,QAAQ,IAAI,CAAC;EAG9F,MAAM,SAAS,eAAe,GAAG;EACjC,IAAI,QAAQ,OAAO,CAAC,MAAM;CAC5B;CACA,OAAO,CAAC;AACV;AAEA,SAAS,WAAW,gBAA2B,QAA0B;CACvE,OAAO,eAAe,MAAM,aAAa,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM,CAAC;AAChG;AAEA,SAAS,YAAY,UAA2B,QAAiC;CAC/E,IAAI,SAAS,SAAS,OAAO,MAAM,OAAO;CAC1C,KAAK,MAAM,CAAC,SAAS,YAAY,OAAO,QAAQ,SAAS,SAAS,GAAG;EACnE,IAAI,EAAE,WAAW,OAAO,YAAY;GAClC,IAAI,QAAQ,MAAM,UAAU,UAAU,MAAM,UAAU,QAAQ,UAAU,KAAA,CAAS,GAAG;GACpF,OAAO;EACT;EACA,IAAI,CAAC,WAAW,SAAS,OAAO,UAAU,QAAQ,GAAG,OAAO;CAC9D;CACA,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,UAA8B;CACjE,MAAM,OAAO,SAAS,IAAI;CAC1B,MAAM,SAAS,mBAAmB,QAAQ;CAC1C,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,UAAU;CACd,KAAK,MAAM,YAAY,KAAK,UAAU;EACpC,MAAM,QAAQ,OAAO,WAAW,MAAM,MAAM,CAAC,KAAK,IAAI,CAAC,KAAK,YAAY,UAAU,IAAI,CAAC;EACvF,IAAI,SAAS,GAAG;GACd,KAAK,IAAI,KAAK;GACd,WAAW;EACb;CACF;CACA,MAAM,SAAS,KAAK,SAAS,WAAW,IAAI,IAAI,UAAU,KAAK,SAAS;CACxE,MAAM,YAAY,OAAO,WAAW,IAAI,IAAI,UAAU,OAAO;CAE7D,OAAO;EACL,UAAU,WAAW,KAAK,cAAc;EACxC,OAAA;EACA,QAAQ,KAAK,UAAU;GACrB,SAAS,KAAK;GACd,UAAU,KAAK;GACf,UAAU,KAAK;GACf;GACA;GACA;GACA,0BAA0B;EAC5B,CAAC;CACH;AACF;AAEA,SAAgB,oBAAsC;CACpD,MAAM,eAAe,QAAQ,IAAI,kBAAkB;CAEnD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,QAAQ;GACpB,IAAI,CAAC,KACH,MAAM,IAAI,MACR,qMACF;GAEF,MAAM,WAAW,aAAa;GAC9B,MAAM,WAAW,SAAS,KAAK,QAAQ,GAAG,iBAAiB;GAC3D,MAAM,WAAW,WAAW,KAAK,QAAQ,GAAG,4BAA4B;GACxE,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;EAC3C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,MAAM,QAAQ;GACpB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,iDAAiD;GAC3E,OAAO,kBAAkB,KAAK,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,UAAU,EACpB,gBAAgB,KAAK,SAAS,KAAK,UAAU;IAC3C,MAAM,KAAK;IACX,WAAW,OAAO,YAAY,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,YAAY,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;GACvG,EAAE,EACJ,GAAG,MAAM,CAAC;EACZ;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,OAAO,WAAW,MAAM,QAAQ;EAClC;CACF;AACF"}

package/dist/benchmarks/cadbench.js

@@ -1,2 +1,2 @@
-import { t as createCadBenchAdapter } from "../cadbench-DpQWZHp4.js";
+import { t as createCadBenchAdapter } from "../cadbench-BrpwOU6A.js";
 export { createCadBenchAdapter };

package/dist/benchmarks/cadgenbench.js

@@ -1,2 +1,2 @@
-import { t as createCadGenBenchAdapter } from "../cadgenbench-DRhczfsG.js";
+import { t as createCadGenBenchAdapter } from "../cadgenbench-DF7hYHdl.js";
 export { createCadGenBenchAdapter };

package/dist/benchmarks/commit0.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/commit0.d.ts
 /** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */
 declare const commit0DiffOutput: OutputAdapter<string>;

package/dist/benchmarks/commit0.js.map

@@ -1 +1 @@
-{"version":3,"file":"commit0.js","names":[],"sources":["../../src/benchmarks/commit0.ts"],"sourcesContent":["/**\n * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.\n * Each record is a Python repo stubbed at `base_commit` (public function bodies\n * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker\n * artifact = a unified diff that fills in `src_dir`. Judge = the official\n * `commit0` test harness: it stages the starter repo, applies the worker's diff,\n * builds the library's deps and runs `pytest`, writing a per-repo pytest-json\n * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the\n * macro-averaged unit-test pass-rate the leaderboard reports). Fully\n * deterministic — no LLM judge.\n *\n * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its\n * implementation as a fenced ```diff against the stubbed repo — same deliverable\n * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the\n * real `commit0` harness on a local Docker backend (NOT reimplemented here).\n *\n * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed\n * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +\n * a Docker daemon (`--backend local`). For offline/CI dataset listing set\n * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)\n * — judging still needs the harness + Docker and fails loud, never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPythonAt } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'commit0.json')\n\n// commit0's pip deps (pydantic/sqlalchemy v1, modal, …) conflict with the shared\n// bench .venv, so its harness runs in an ISOLATED venv. Override with COMMIT0_VENV.\n// Resolved at call-time so the env is honored at run-time (not frozen at import).\nconst commit0VenvDir = (): string => process.env.COMMIT0_VENV ?? '.venv-commit0'\nconst commit0Python = (): string => venvPythonAt(commit0VenvDir())\n\nconst DATASET = 'wentingzhao/commit0_combined'\nconst DATASET_SPLIT = 'test'\n// HF rows server — columnar pull with no `datasets` install required for listing.\nconst ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`\n\n/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */\nexport const commit0DiffOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const body = (fences.at(-1)?.[1] ?? text).trim()\n    // `git apply` rejects a patch that is not newline-terminated (\"corrupt patch at\n    // line N+1\"); the .trim() above strips the final newline, so restore exactly one.\n    return body.length > 0 ? `${body}\\n` : body\n  },\n}\n\ninterface Commit0Setup {\n  install: string\n  packages: string[] | null\n  pip_packages: string[] | null\n  pre_install: string[] | null\n  python: string\n  specification: string\n}\n\ninterface Commit0Row {\n  instance_id: string\n  repo: string\n  original_repo: string\n  base_commit: string\n  reference_commit: string\n  setup: Commit0Setup\n  test: { test_cmd: string; test_dir: string }\n  src_dir: string\n}\n\ninterface Commit0Meta {\n  instanceId: string\n  repo: string\n  originalRepo: string\n  baseCommit: string\n  referenceCommit: string\n  srcDir: string\n  testDir: string\n  testCmd: string\n  specification: string\n}\n\nfunction rowToTask(row: Commit0Row): BenchTask {\n  const meta: Commit0Meta = {\n    instanceId: row.instance_id,\n    repo: row.repo,\n    originalRepo: row.original_repo,\n    baseCommit: row.base_commit,\n    referenceCommit: row.reference_commit,\n    srcDir: row.src_dir,\n    testDir: row.test.test_dir,\n    testCmd: row.test.test_cmd,\n    specification: row.setup.specification,\n  }\n  return {\n    id: row.instance_id,\n    split: DATASET_SPLIT,\n    prompt: [\n      `Clone https://github.com/${row.repo} into /work, then \\`cd /work && git checkout ${row.base_commit}\\`.`,\n      `This is the STUBBED library: the public functions/classes under \\`${row.src_dir}\\` have empty bodies (\\`pass\\`/\\`...\\`).`,\n      `Fill in COMPLETE implementations under \\`${row.src_dir}\\` so the existing test suite under \\`${row.test.test_dir}\\` passes. Read those tests and the spec to learn the required behavior.`,\n      `Specification / docs: ${row.setup.specification}`,\n      'Do NOT edit the test files — the evaluation re-runs the existing tests on a fresh clone. Implement only the source.',\n      `When done, from /work run EXACTLY: \\`git add -A && git diff --cached -- ${row.src_dir}\\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \\`\\`\\`diff … \\`\\`\\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): Commit0Meta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string' || typeof md.referenceCommit !== 'string' || typeof md.srcDir !== 'string') {\n    throw new Error(`commit0 task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as Commit0Meta\n}\n\nfunction selectRows(rows: Commit0Row[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as Commit0Row[]\n  console.warn(\n    `[commit0] COMMIT0_FIXTURES=1 — loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`,\n  )\n  return selectRows(rows, opts)\n}\n\n/** Pull real rows from the HF rows server (paged). limit caps the pull; ids\n *  filtered client-side. Throws on a non-OK response (fail loud). */\nasync function fetchRows(opts: LoadOptions): Promise<Commit0Row[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: Commit0Row[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  for (let offset = 0; offset < 64 && rows.length < target; offset += page) {\n    const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`)\n    if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    const body = (await res.json()) as { rows?: Array<{ row: Commit0Row }>; num_rows_total?: number }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.instance_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`)\n  return rows\n}\n\n/**\n * Run the official commit0 harness for one repo over the worker's diff. The\n * harness clones base_commit, applies the diff into src_dir, installs deps and\n * runs pytest, then writes a per-repo pytest-json report. We read that report and\n * compute (passed + xfail) / total — the leaderboard's per-repo pass-rate.\n *\n * This is the expensive, Docker-backed boundary; it is DELEGATED to `commit0`,\n * not reimplemented. The driver script lives in scripts/commit0_judge.py so the\n * harness call + report parse are one auditable python entrypoint.\n */\nasync function runHarness(meta: Commit0Meta, artifact: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'commit0_judge.py')\n  let stdout: string\n  try {\n    // The worker's diff is piped to the driver's stdin (shared stdin-aware runner —\n    // execFile's `input` option is not honored async and hangs the reader).\n    stdout = await runVenvScriptStdin(\n      judge,\n      ['--dataset', DATASET, '--split', DATASET_SPLIT, '--instance', meta.instanceId, '--src-dir', meta.srcDir],\n      artifact,\n      { cwd: benchRoot, python: commit0Python() },\n    )\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  // total=0 means the harness MEASURED NOTHING (collection error with no declared\n  // test ids) — an unmeasured attempt, not a 0% one. Throw so the caller excludes\n  // it as infra instead of recording a fabricated zero.\n  if (report.total <= 0) {\n    throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`)\n  }\n  const score = report.passed / report.total\n  return {\n    resolved: report.passed === report.total,\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\nexport function createCommit0Adapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.COMMIT0_FIXTURES === '1'\n\n  return {\n    name: 'commit0',\n    output: commit0DiffOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['commit0'],\n        requireDocker: true,\n        python: commit0Python(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets ` +\n          `(an ISOLATED venv — commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). ` +\n          `Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let rows: Commit0Row[]\n      try {\n        rows = await fetchRows(opts)\n      } catch (err) {\n        console.warn(\n          `[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRows(rows, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the reference_commit's src_dir, which the commit0 harness\n      // checks out by ref — not expressible as a portable diff string without\n      // cloning. verify-judge against this adapter requires the live harness, so\n      // we return undefined (no offline gold-diff). Judge correctness is proven by\n      // running the harness on a real solve, not by a synthetic gold patch.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,MAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAK3D,MAAM,uBAA+B,QAAQ,IAAI,gBAAgB;AACjE,MAAM,sBAA8B,aAAa,eAAe,CAAC;AAEjE,MAAM,UAAU;AAChB,MAAM,gBAAgB;AAEtB,MAAM,WAAW,uDAAuD,mBAAmB,OAAO,EAAE,wBAAwB;;AAG5H,MAAa,oBAA2C,EACtD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,MAAM,QAAQ,CADE,GAAG,KAAK,SAAS,uCAAuC,CACrD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAG/C,OAAO,KAAK,SAAS,IAAI,GAAG,KAAK,MAAM;AACzC,EACF;AAkCA,SAAS,UAAU,KAA4B;CAC7C,MAAM,OAAoB;EACxB,YAAY,IAAI;EAChB,MAAM,IAAI;EACV,cAAc,IAAI;EAClB,YAAY,IAAI;EAChB,iBAAiB,IAAI;EACrB,QAAQ,IAAI;EACZ,SAAS,IAAI,KAAK;EAClB,SAAS,IAAI,KAAK;EAClB,eAAe,IAAI,MAAM;CAC3B;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ;GACN,4BAA4B,IAAI,KAAK,+CAA+C,IAAI,YAAY;GACpG,qEAAqE,IAAI,QAAQ;GACjF,4CAA4C,IAAI,QAAQ,wCAAwC,IAAI,KAAK,SAAS;GAClH,yBAAyB,IAAI,MAAM;GACnC;GACA,2EAA2E,IAAI,QAAQ,uKAAuK,IAAI,YAAY;EAChR,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA8B;CAC9C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,eAAe,YAAY,OAAO,GAAG,oBAAoB,YAAY,OAAO,GAAG,WAAW,UAC7G,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kDAAkD;CAE5F,OAAO;AACT;AAEA,SAAS,WAAW,MAAoB,MAAgC;CACtE,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KACN,0CAA0C,KAAK,OAAO,4BAA4B,SAAS,eAC7F;CACA,OAAO,WAAW,MAAM,IAAI;AAC9B;;;AAIA,eAAe,UAAU,MAA0C;CACjE,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;CAC/D,MAAM,OAAqB,CAAC;CAC5B,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;CAC5C,MAAM,OAAO;CACb,KAAK,IAAI,SAAS,GAAG,SAAS,MAAM,KAAK,SAAS,QAAQ,UAAU,MAAM;EACxE,MAAM,MAAM,MAAM,MAAM,GAAG,SAAS,UAAU,OAAO,UAAU,MAAM;EACrE,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,qBAAqB,IAAI,OAAO,WAAW,OAAO,MAAM,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EAEtH,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,CAAC;EAC1B,IAAI,IAAI,WAAW,GAAG;EACtB,KAAK,MAAM,KAAK,KAAK;GACnB,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,WAAW,GAAG;GAC1C,KAAK,KAAK,EAAE,GAAG;EACjB;EACA,IAAI,IAAI,SAAS,MAAM;CACzB;CACA,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,4BAA4B,KAAK,UAAU,IAAI,EAAE,QAAQ,SAAS;CACzG,OAAO;AACT;;;;;;;;;;;AAYA,eAAe,WAAW,MAAmB,UAAuC;CAClF,MAAM,QAAQ,KAAK,WAAW,WAAW,kBAAkB;CAC3D,IAAI;CACJ,IAAI;EAGF,SAAS,MAAM,mBACb,OACA;GAAC;GAAa;GAAS;GAAW;GAAe;GAAc,KAAK;GAAY;GAAa,KAAK;EAAM,GACxG,UACA;GAAE,KAAK;GAAW,QAAQ,cAAc;EAAE,CAC5C;CACF,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,8BAA8B,KAAK,WAAW,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC/G;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAKlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,6BAA6B,KAAK,WAAW,IAAI,OAAO,OAAO;CACjG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,6CAA6C,OAAO,MAAM,GAAG,GAAG,GAAG;CAKrF,IAAI,OAAO,SAAS,GAClB,MAAM,IAAI,MAAM,uCAAuC,KAAK,WAAW,cAAc,OAAO,MAAM,GAAG,GAAG,GAAG;CAE7G,MAAM,QAAQ,OAAO,SAAS,OAAO;CACrC,OAAO;EACL,UAAU,OAAO,WAAW,OAAO;EACnC;EACA,QAAQ,KAAK,UAAU;GAAE,YAAY,KAAK;GAAY,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACpG;AACF;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,qBAAqB;CAEtD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,SAAS;IACnB,eAAe;IACf,QAAQ,cAAc;IACtB,KACE,kCAAkC,eAAe,EAAE,YAAY,eAAe,EAAE;GAIpF,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,UAAU,IAAI;GAC7B,SAAS,KAAK;IACZ,QAAQ,KACN,qCAAqC,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAA2C,UAC3H;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,WAAW,MAAM,IAAI;EAC9B;EAEA,MAAM,eAAe,CAOrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,WADM,SAAS,IACD,GAAG,QAAQ;EAClC;CACF;AACF"}
+{"version":3,"file":"commit0.js","names":[],"sources":["../../src/benchmarks/commit0.ts"],"sourcesContent":["/**\n * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.\n * Each record is a Python repo stubbed at `base_commit` (public function bodies\n * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker\n * artifact = a unified diff that fills in `src_dir`. Judge = the official\n * `commit0` test harness: it stages the starter repo, applies the worker's diff,\n * builds the library's deps and runs `pytest`, writing a per-repo pytest-json\n * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the\n * macro-averaged unit-test pass-rate the leaderboard reports). Fully\n * deterministic — no LLM judge.\n *\n * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its\n * implementation as a fenced ```diff against the stubbed repo — same deliverable\n * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the\n * real `commit0` harness on a local Docker backend (NOT reimplemented here).\n *\n * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed\n * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +\n * a Docker daemon (`--backend local`). For offline/CI dataset listing set\n * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)\n * — judging still needs the harness + Docker and fails loud, never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPythonAt } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'commit0.json')\n\n// commit0's pip deps (pydantic/sqlalchemy v1, modal, …) conflict with the shared\n// bench .venv, so its harness runs in an ISOLATED venv. Override with COMMIT0_VENV.\n// Resolved at call-time so the env is honored at run-time (not frozen at import).\nconst commit0VenvDir = (): string => process.env.COMMIT0_VENV ?? '.venv-commit0'\nconst commit0Python = (): string => venvPythonAt(commit0VenvDir())\n\nconst DATASET = 'wentingzhao/commit0_combined'\nconst DATASET_SPLIT = 'test'\n// HF rows server — columnar pull with no `datasets` install required for listing.\nconst ROWS_API = `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=default&split=${DATASET_SPLIT}`\n\n/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */\nexport const commit0DiffOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const body = (fences.at(-1)?.[1] ?? text).trim()\n    // `git apply` rejects a patch that is not newline-terminated (\"corrupt patch at\n    // line N+1\"); the .trim() above strips the final newline, so restore exactly one.\n    return body.length > 0 ? `${body}\\n` : body\n  },\n}\n\ninterface Commit0Setup {\n  install: string\n  packages: string[] | null\n  pip_packages: string[] | null\n  pre_install: string[] | null\n  python: string\n  specification: string\n}\n\ninterface Commit0Row {\n  instance_id: string\n  repo: string\n  original_repo: string\n  base_commit: string\n  reference_commit: string\n  setup: Commit0Setup\n  test: { test_cmd: string; test_dir: string }\n  src_dir: string\n}\n\ninterface Commit0Meta {\n  instanceId: string\n  repo: string\n  originalRepo: string\n  baseCommit: string\n  referenceCommit: string\n  srcDir: string\n  testDir: string\n  testCmd: string\n  specification: string\n}\n\nfunction rowToTask(row: Commit0Row): BenchTask {\n  const meta: Commit0Meta = {\n    instanceId: row.instance_id,\n    repo: row.repo,\n    originalRepo: row.original_repo,\n    baseCommit: row.base_commit,\n    referenceCommit: row.reference_commit,\n    srcDir: row.src_dir,\n    testDir: row.test.test_dir,\n    testCmd: row.test.test_cmd,\n    specification: row.setup.specification,\n  }\n  return {\n    id: row.instance_id,\n    split: DATASET_SPLIT,\n    prompt: [\n      `Clone https://github.com/${row.repo} into /work, then \\`cd /work && git checkout ${row.base_commit}\\`.`,\n      `This is the STUBBED library: the public functions/classes under \\`${row.src_dir}\\` have empty bodies (\\`pass\\`/\\`...\\`).`,\n      `Fill in COMPLETE implementations under \\`${row.src_dir}\\` so the existing test suite under \\`${row.test.test_dir}\\` passes. Read those tests and the spec to learn the required behavior.`,\n      `Specification / docs: ${row.setup.specification}`,\n      'Do NOT edit the test files — the evaluation re-runs the existing tests on a fresh clone. Implement only the source.',\n      `When done, from /work run EXACTLY: \\`git add -A && git diff --cached -- ${row.src_dir}\\` and END your reply with its COMPLETE output as the LAST thing, fenced exactly as \\`\\`\\`diff … \\`\\`\\` (nothing after the closing fence). That fenced diff (against ${row.base_commit}) is the only deliverable.`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): Commit0Meta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string' || typeof md.referenceCommit !== 'string' || typeof md.srcDir !== 'string') {\n    throw new Error(`commit0 task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as Commit0Meta\n}\n\nfunction selectRows(rows: Commit0Row[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as Commit0Row[]\n  console.warn(\n    `[commit0] COMMIT0_FIXTURES=1 — loading ${rows.length} committed lite rows from ${FIXTURES} (no HF fetch)`,\n  )\n  return selectRows(rows, opts)\n}\n\n/** Pull real rows from the HF rows server (paged). limit caps the pull; ids\n *  filtered client-side. Throws on a non-OK response (fail loud). */\nasync function fetchRows(opts: LoadOptions): Promise<Commit0Row[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: Commit0Row[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  for (let offset = 0; offset < 64 && rows.length < target; offset += page) {\n    const res = await fetch(`${ROWS_API}&offset=${offset}&length=${page}`)\n    if (!res.ok) throw new Error(`commit0 rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    const body = (await res.json()) as { rows?: Array<{ row: Commit0Row }>; num_rows_total?: number }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.instance_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`commit0: no rows matched ${JSON.stringify(opts)} from ${DATASET}`)\n  return rows\n}\n\n/**\n * Run the official commit0 harness for one repo over the worker's diff. The\n * harness clones base_commit, applies the diff into src_dir, installs deps and\n * runs pytest, then writes a per-repo pytest-json report. We read that report and\n * compute (passed + xfail) / total — the leaderboard's per-repo pass-rate.\n *\n * This is the expensive, Docker-backed boundary; it is DELEGATED to `commit0`,\n * not reimplemented. The driver script lives in scripts/commit0_judge.py so the\n * harness call + report parse are one auditable python entrypoint.\n */\nasync function runHarness(meta: Commit0Meta, artifact: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'commit0_judge.py')\n  let stdout: string\n  try {\n    // The worker's diff is piped to the driver's stdin (shared stdin-aware runner —\n    // execFile's `input` option is not honored async and hangs the reader).\n    stdout = await runVenvScriptStdin(\n      judge,\n      ['--dataset', DATASET, '--split', DATASET_SPLIT, '--instance', meta.instanceId, '--src-dir', meta.srcDir],\n      artifact,\n      { cwd: benchRoot, python: commit0Python() },\n    )\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`commit0 harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`commit0 harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`commit0 judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  // total=0 means the harness MEASURED NOTHING (collection error with no declared\n  // test ids) — an unmeasured attempt, not a 0% one. Throw so the caller excludes\n  // it as infra instead of recording a fabricated zero.\n  if (report.total <= 0) {\n    throw new Error(`commit0 judge measured no tests for ${meta.instanceId} (total=0): ${stdout.slice(0, 400)}`)\n  }\n  const score = report.passed / report.total\n  return {\n    resolved: report.passed === report.total,\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\nexport function createCommit0Adapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.COMMIT0_FIXTURES === '1'\n\n  return {\n    name: 'commit0',\n    output: commit0DiffOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['commit0'],\n        requireDocker: true,\n        python: commit0Python(),\n        fix:\n          `Fix: (1) python3 -m venv bench/${commit0VenvDir()} && bench/${commit0VenvDir()}/bin/pip install commit0 datasets ` +\n          `(an ISOLATED venv — commit0's deps conflict with the shared bench/.venv; override the dir with COMMIT0_VENV) ; ` +\n          `(2) ensure the Docker daemon is running (commit0 --backend local builds per-repo images). ` +\n          `Dataset rows come from the HF rows server; set COMMIT0_FIXTURES=1 to list the committed lite rows offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let rows: Commit0Row[]\n      try {\n        rows = await fetchRows(opts)\n      } catch (err) {\n        console.warn(\n          `[commit0] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectRows(rows, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the reference_commit's src_dir, which the commit0 harness\n      // checks out by ref — not expressible as a portable diff string without\n      // cloning. verify-judge against this adapter requires the live harness, so\n      // we return undefined (no offline gold-diff). Judge correctness is proven by\n      // running the harness on a real solve, not by a synthetic gold patch.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,MAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAK3D,MAAM,uBAA+B,QAAQ,IAAI,gBAAgB;AACjE,MAAM,sBAA8B,aAAa,eAAe,CAAC;AAEjE,MAAM,UAAU;AAChB,MAAM,gBAAgB;AAEtB,MAAM,WAAW,uDAAuD,mBAAmB,OAAO,EAAE,wBAAwB;;AAG5H,MAAa,oBAA2C,EACtD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,MAAM,QAAQ,CADE,GAAG,KAAK,SAAS,uCAAuC,CACrD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAG/C,OAAO,KAAK,SAAS,IAAI,GAAG,KAAK,MAAM;AACzC,EACF;AAkCA,SAAS,UAAU,KAA4B;CAC7C,MAAM,OAAoB;EACxB,YAAY,IAAI;EAChB,MAAM,IAAI;EACV,cAAc,IAAI;EAClB,YAAY,IAAI;EAChB,iBAAiB,IAAI;EACrB,QAAQ,IAAI;EACZ,SAAS,IAAI,KAAK;EAClB,SAAS,IAAI,KAAK;EAClB,eAAe,IAAI,MAAM;CAC3B;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO;EACP,QAAQ;GACN,4BAA4B,IAAI,KAAK,+CAA+C,IAAI,YAAY;GACpG,qEAAqE,IAAI,QAAQ;GACjF,4CAA4C,IAAI,QAAQ,wCAAwC,IAAI,KAAK,SAAS;GAClH,yBAAyB,IAAI,MAAM;GACnC;GACA,2EAA2E,IAAI,QAAQ,uKAAuK,IAAI,YAAY;EAChR,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA8B;CAC9C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,eAAe,YAAY,OAAO,GAAG,oBAAoB,YAAY,OAAO,GAAG,WAAW,UAC7G,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kDAAkD;CAE5F,OAAO;AACT;AAEA,SAAS,WAAW,MAAoB,MAAgC;CACtE,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KACN,0CAA0C,KAAK,OAAO,4BAA4B,SAAS,eAC7F;CACA,OAAO,WAAW,MAAM,IAAI;AAC9B;;;AAIA,eAAe,UAAU,MAA0C;CACjE,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;CAC/D,MAAM,OAAqB,CAAC;CAC5B,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;CAC5C,MAAM,OAAO;CACb,KAAK,IAAI,SAAS,GAAG,SAAS,MAAM,KAAK,SAAS,QAAQ,UAAU,MAAM;EACxE,MAAM,MAAM,MAAM,MAAM,GAAG,SAAS,UAAU,OAAO,UAAU,MAAM;EACrE,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,qBAAqB,IAAI,OAAO,WAAW,OAAO,MAAM,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EAEtH,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,CAAC;EAC1B,IAAI,IAAI,WAAW,GAAG;EACtB,KAAK,MAAM,KAAK,KAAK;GACnB,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,WAAW,GAAG;GAC1C,KAAK,KAAK,EAAE,GAAG;EACjB;EACA,IAAI,IAAI,SAAS,MAAM;CACzB;CACA,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,4BAA4B,KAAK,UAAU,IAAI,EAAE,QAAQ,SAAS;CACzG,OAAO;AACT;;;;;;;;;;;AAYA,eAAe,WAAW,MAAmB,UAAuC;CAClF,MAAM,QAAQ,KAAK,WAAW,WAAW,kBAAkB;CAC3D,IAAI;CACJ,IAAI;EAGF,SAAS,MAAM,mBACb,OACA;GAAC;GAAa;GAAS;GAAW;GAAe;GAAc,KAAK;GAAY;GAAa,KAAK;EAAM,GACxG,UACA;GAAE,KAAK;GAAW,QAAQ,cAAc;EAAE,CAC5C;CACF,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,8BAA8B,KAAK,WAAW,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC/G;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAKlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,6BAA6B,KAAK,WAAW,IAAI,OAAO,OAAO;CACjG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,6CAA6C,OAAO,MAAM,GAAG,GAAG,GAAG;CAKrF,IAAI,OAAO,SAAS,GAClB,MAAM,IAAI,MAAM,uCAAuC,KAAK,WAAW,cAAc,OAAO,MAAM,GAAG,GAAG,GAAG;CAE7G,MAAM,QAAQ,OAAO,SAAS,OAAO;CACrC,OAAO;EACL,UAAU,OAAO,WAAW,OAAO;EACnC;EACA,QAAQ,KAAK,UAAU;GAAE,YAAY,KAAK;GAAY,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACpG;AACF;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,qBAAqB;CAEtD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,SAAS;IACnB,eAAe;IACf,QAAQ,cAAc;IACtB,KACE,kCAAkC,eAAe,EAAE,YAAY,eAAe,EAAE;GAIpF,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,UAAU,IAAI;GAC7B,SAAS,KAAK;IACZ,QAAQ,KACN,qCAAqC,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAA2C,UAC3H;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,WAAW,MAAM,IAAI;EAC9B;EAEA,MAAM,eAAe,CAOrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,WADM,SAAS,IACD,GAAG,QAAQ;EAClC;CACF;AACF"}

package/dist/benchmarks/dabstep.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/dabstep.d.ts
 declare const dabstepAnswerOutput: OutputAdapter<string>;
 declare function createDabstepAdapter(): BenchmarkAdapter;

package/dist/benchmarks/dabstep.js.map

@@ -1 +1 @@
-{"version":3,"file":"dabstep.js","names":[],"sources":["../../src/benchmarks/dabstep.ts"],"sourcesContent":["/**\n * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic\n * payment files. Worker artifact = final answer text. Judge = the official\n * DABStep `grade.py` normalization/matching function. No LLM judge.\n *\n * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that\n * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter\n * exposes `metadata.resourceRoot` so runners can mount the benchmark files into\n * AgentProfile.resources.files; it does not paste the dataset into prompt text.\n */\n\nimport { join } from 'node:path'\nimport { access, readFile, stat } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'dabstep.json')\nconst DEFAULT_SPLIT = 'easy'\n\ninterface DabstepFixtureRow {\n  task_id: number\n  instructions: string\n  all_golds_by_task: Array<Record<string, unknown>>\n}\n\ninterface DabstepMeta {\n  taskId: number\n  split: string\n  golds: Array<Record<string, unknown>>\n  resourceRoot?: string\n}\n\nconst dabstepDir = (): string | undefined => process.env.DABSTEP_DIR\nconst gradeFile = (dir: string): string => join(dir, 'grade.py')\nconst resourceRoot = (dir: string): string => join(dir, 'files')\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertOfficialFiles(dir: string, split: string): Promise<void> {\n  await assertFile(join(dir, 'dataset.csv'), 'released dataset.csv')\n  await assertFile(join(dir, 'splits', `${split}.txt`), `${split} split file`)\n  await assertFile(gradeFile(dir), 'official grade.py')\n  const files = resourceRoot(dir)\n  try {\n    const s = await stat(files)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nexport const dabstepAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction rowToTask(row: DabstepFixtureRow, split: string, dir?: string): BenchTask {\n  const meta: DabstepMeta = {\n    taskId: row.task_id,\n    split,\n    golds: row.all_golds_by_task,\n    ...(dir ? { resourceRoot: resourceRoot(dir) } : {}),\n  }\n  return {\n    id: String(row.task_id),\n    split,\n    prompt: [\n      'Solve this DABStep data-analysis task using the mounted payment files.',\n      'Use code or shell commands as needed, then return only the final answer.',\n      '',\n      row.instructions,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): DabstepMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number' || !Array.isArray(md.golds)) {\n    throw new Error(`dabstep task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as DabstepMeta\n}\n\nfunction selectRows(rows: DabstepFixtureRow[], opts: LoadOptions, split: string, dir?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, split, dir))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as DabstepFixtureRow[]\n  console.warn(`[dabstep] DABSTEP_FIXTURES=1 — loading ${rows.length} adapter fixtures from ${FIXTURES}`)\n  return selectRows(rows, opts, split)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const script = `\nimport ast, csv, json, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\nsplit = sys.argv[2]\nlimit = None if sys.argv[3] == \"\" else int(sys.argv[3])\nids = set(json.loads(sys.argv[4]))\ndataset = root / \"dataset.csv\"\nsplit_file = root / \"splits\" / f\"{split}.txt\"\nif not dataset.exists():\n    raise SystemExit(f\"missing official DABStep dataset.csv at {dataset}\")\nif not split_file.exists():\n    raise SystemExit(f\"missing official DABStep split file at {split_file}\")\nsplit_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}\nout = []\nwith dataset.open(newline=\"\") as f:\n    for row in csv.DictReader(f):\n        task_id = int(row[\"task_id\"])\n        if task_id not in split_ids:\n            continue\n        if ids and str(task_id) not in ids:\n            continue\n        out.append({\n            \"task_id\": task_id,\n            \"instructions\": f\"{row['question']}\\\\n{row['guidelines']}\",\n            \"all_golds_by_task\": ast.literal_eval(str(row[\"all_golds_by_task\"])),\n        })\n        if limit is not None and len(out) >= limit:\n            break\nif not out:\n    raise SystemExit(f\"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}\")\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(script, [dir, split, opts.limit === undefined ? '' : String(opts.limit), JSON.stringify(opts.ids ?? [])])\n  return selectRows(JSON.parse(stdout) as DabstepFixtureRow[], opts, split, dir)\n}\n\nexport function createDabstepAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.DABSTEP_FIXTURES === '1'\n\n  return {\n    name: 'dabstep',\n    output: dabstepAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = dabstepDir()\n      if (!dir) {\n        throw new Error(\n          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.',\n        )\n      }\n      await assertOfficialFiles(dir, DEFAULT_SPLIT)\n      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to load official DABStep tasks')\n      return loadOfficialTasks(dir, opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      const first = meta.golds[0]\n      if (!first) return undefined\n      const value = first.value\n      return value === undefined ? undefined : String(value)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to judge DABStep tasks with the official grade.py')\n      const stdout = await runVenvScriptStdin(\n        join(benchRoot, 'scripts', 'dabstep_judge.py'),\n        ['--grade-file', gradeFile(dir)],\n        JSON.stringify({ prediction: artifact, golds: meta.golds }),\n        { cwd: benchRoot },\n      )\n      const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as { correct?: boolean; score?: number; error?: string }\n      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: report.correct === true,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,MAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAC3D,MAAM,gBAAgB;AAetB,MAAM,mBAAuC,QAAQ,IAAI;AACzD,MAAM,aAAa,QAAwB,KAAK,KAAK,UAAU;AAC/D,MAAM,gBAAgB,QAAwB,KAAK,KAAK,OAAO;AAE/D,eAAe,WAAW,MAAc,OAA8B;CACpE,IAAI;EACF,MAAM,OAAO,IAAI;CACnB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,oBAAoB,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACtG;AACF;AAEA,eAAe,oBAAoB,KAAa,OAA8B;CAC5E,MAAM,WAAW,KAAK,KAAK,aAAa,GAAG,sBAAsB;CACjE,MAAM,WAAW,KAAK,KAAK,UAAU,GAAG,MAAM,KAAK,GAAG,GAAG,MAAM,YAAY;CAC3E,MAAM,WAAW,UAAU,GAAG,GAAG,mBAAmB;CACpD,MAAM,QAAQ,aAAa,GAAG;CAC9B,IAAI;EAEF,IAAI,EAAC,MADW,KAAK,KAAK,EAAA,CACnB,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;CACzD,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,iDAAiD,MAAM,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACxH;AACF;AAEA,MAAa,sBAA6C,EACxD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,wCAAwC,CAC5D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,UAAU,KAAwB,OAAe,KAAyB;CACjF,MAAM,OAAoB;EACxB,QAAQ,IAAI;EACZ;EACA,OAAO,IAAI;EACX,GAAI,MAAM,EAAE,cAAc,aAAa,GAAG,EAAE,IAAI,CAAC;CACnD;CACA,OAAO;EACL,IAAI,OAAO,IAAI,OAAO;EACtB;EACA,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA8B;CAC9C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,CAAC,MAAM,QAAQ,GAAG,KAAK,GACjE,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kDAAkD;CAE5F,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,OAAe,KAA2B;CAC1G,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,OAAO,GAAG,CAAC;CACxD,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,EAAE,aAAa,OAAO;CAC9G,OAAO;AACT;AAEA,eAAe,aAAa,MAAmB,OAAqC;CAClF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,0CAA0C,KAAK,OAAO,yBAAyB,UAAU;CACtG,OAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,eAAe,kBAAkB,KAAa,MAAmB,OAAqC;CAmCpG,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAK;EAAO,KAAK,UAAU,KAAA,IAAY,KAAK,OAAO,KAAK,KAAK;EAAG,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC;CAAC,CAAC;CAC3I,OAAO,WAAW,KAAK,MAAM,MAAM,GAA0B,MAAM,OAAO,GAAG;AAC/E;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,qBAAqB;CAEtD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KACH,MAAM,IAAI,MACR,qKACF;GAEF,MAAM,oBAAoB,KAAK,aAAa;GAC5C,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,GAAG,aAAa;EAC1D;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,IAAI,cAAc,OAAO,aAAa,MAAM,KAAK;GACjD,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;GAClF,OAAO,kBAAkB,KAAK,MAAM,KAAK;EAC3C;EAEA,MAAM,aAAa,MAAiB;GAElC,MAAM,QADO,SAAS,IACL,CAAC,CAAC,MAAM;GACzB,IAAI,CAAC,OAAO,OAAO,KAAA;GACnB,MAAM,QAAQ,MAAM;GACpB,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,OAAO,KAAK;EACvD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,2EAA2E;GACrG,MAAM,SAAS,MAAM,mBACnB,KAAK,WAAW,WAAW,kBAAkB,GAC7C,CAAC,gBAAgB,UAAU,GAAG,CAAC,GAC/B,KAAK,UAAU;IAAE,YAAY;IAAU,OAAO,KAAK;GAAM,CAAC,GAC1D,EAAE,KAAK,UAAU,CACnB;GACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;GAClE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,2BAA2B,KAAK,GAAG,IAAI,OAAO,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,OAAO,YAAY;IAC7B;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,OAAO,KAAK;KAAO,SAAS,OAAO;IAAQ,CAAC;GAC5F;EACF;CACF;AACF"}
+{"version":3,"file":"dabstep.js","names":[],"sources":["../../src/benchmarks/dabstep.ts"],"sourcesContent":["/**\n * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic\n * payment files. Worker artifact = final answer text. Judge = the official\n * DABStep `grade.py` normalization/matching function. No LLM judge.\n *\n * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that\n * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter\n * exposes `metadata.resourceRoot` so runners can mount the benchmark files into\n * AgentProfile.resources.files; it does not paste the dataset into prompt text.\n */\n\nimport { join } from 'node:path'\nimport { access, readFile, stat } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, runVenvPython, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'dabstep.json')\nconst DEFAULT_SPLIT = 'easy'\n\ninterface DabstepFixtureRow {\n  task_id: number\n  instructions: string\n  all_golds_by_task: Array<Record<string, unknown>>\n}\n\ninterface DabstepMeta {\n  taskId: number\n  split: string\n  golds: Array<Record<string, unknown>>\n  resourceRoot?: string\n}\n\nconst dabstepDir = (): string | undefined => process.env.DABSTEP_DIR\nconst gradeFile = (dir: string): string => join(dir, 'grade.py')\nconst resourceRoot = (dir: string): string => join(dir, 'files')\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertOfficialFiles(dir: string, split: string): Promise<void> {\n  await assertFile(join(dir, 'dataset.csv'), 'released dataset.csv')\n  await assertFile(join(dir, 'splits', `${split}.txt`), `${split} split file`)\n  await assertFile(gradeFile(dir), 'official grade.py')\n  const files = resourceRoot(dir)\n  try {\n    const s = await stat(files)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nexport const dabstepAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction rowToTask(row: DabstepFixtureRow, split: string, dir?: string): BenchTask {\n  const meta: DabstepMeta = {\n    taskId: row.task_id,\n    split,\n    golds: row.all_golds_by_task,\n    ...(dir ? { resourceRoot: resourceRoot(dir) } : {}),\n  }\n  return {\n    id: String(row.task_id),\n    split,\n    prompt: [\n      'Solve this DABStep data-analysis task using the mounted payment files.',\n      'Use code or shell commands as needed, then return only the final answer.',\n      '',\n      row.instructions,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): DabstepMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number' || !Array.isArray(md.golds)) {\n    throw new Error(`dabstep task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as DabstepMeta\n}\n\nfunction selectRows(rows: DabstepFixtureRow[], opts: LoadOptions, split: string, dir?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, split, dir))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as DabstepFixtureRow[]\n  console.warn(`[dabstep] DABSTEP_FIXTURES=1 — loading ${rows.length} adapter fixtures from ${FIXTURES}`)\n  return selectRows(rows, opts, split)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const script = `\nimport ast, csv, json, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\nsplit = sys.argv[2]\nlimit = None if sys.argv[3] == \"\" else int(sys.argv[3])\nids = set(json.loads(sys.argv[4]))\ndataset = root / \"dataset.csv\"\nsplit_file = root / \"splits\" / f\"{split}.txt\"\nif not dataset.exists():\n    raise SystemExit(f\"missing official DABStep dataset.csv at {dataset}\")\nif not split_file.exists():\n    raise SystemExit(f\"missing official DABStep split file at {split_file}\")\nsplit_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}\nout = []\nwith dataset.open(newline=\"\") as f:\n    for row in csv.DictReader(f):\n        task_id = int(row[\"task_id\"])\n        if task_id not in split_ids:\n            continue\n        if ids and str(task_id) not in ids:\n            continue\n        out.append({\n            \"task_id\": task_id,\n            \"instructions\": f\"{row['question']}\\\\n{row['guidelines']}\",\n            \"all_golds_by_task\": ast.literal_eval(str(row[\"all_golds_by_task\"])),\n        })\n        if limit is not None and len(out) >= limit:\n            break\nif not out:\n    raise SystemExit(f\"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}\")\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(script, [dir, split, opts.limit === undefined ? '' : String(opts.limit), JSON.stringify(opts.ids ?? [])])\n  return selectRows(JSON.parse(stdout) as DabstepFixtureRow[], opts, split, dir)\n}\n\nexport function createDabstepAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.DABSTEP_FIXTURES === '1'\n\n  return {\n    name: 'dabstep',\n    output: dabstepAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = dabstepDir()\n      if (!dir) {\n        throw new Error(\n          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.',\n        )\n      }\n      await assertOfficialFiles(dir, DEFAULT_SPLIT)\n      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to load official DABStep tasks')\n      return loadOfficialTasks(dir, opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      const first = meta.golds[0]\n      if (!first) return undefined\n      const value = first.value\n      return value === undefined ? undefined : String(value)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to judge DABStep tasks with the official grade.py')\n      const stdout = await runVenvScriptStdin(\n        join(benchRoot, 'scripts', 'dabstep_judge.py'),\n        ['--grade-file', gradeFile(dir)],\n        JSON.stringify({ prediction: artifact, golds: meta.golds }),\n        { cwd: benchRoot },\n      )\n      const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as { correct?: boolean; score?: number; error?: string }\n      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: report.correct === true,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAiBA,MAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAC3D,MAAM,gBAAgB;AAetB,MAAM,mBAAuC,QAAQ,IAAI;AACzD,MAAM,aAAa,QAAwB,KAAK,KAAK,UAAU;AAC/D,MAAM,gBAAgB,QAAwB,KAAK,KAAK,OAAO;AAE/D,eAAe,WAAW,MAAc,OAA8B;CACpE,IAAI;EACF,MAAM,OAAO,IAAI;CACnB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,oBAAoB,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACtG;AACF;AAEA,eAAe,oBAAoB,KAAa,OAA8B;CAC5E,MAAM,WAAW,KAAK,KAAK,aAAa,GAAG,sBAAsB;CACjE,MAAM,WAAW,KAAK,KAAK,UAAU,GAAG,MAAM,KAAK,GAAG,GAAG,MAAM,YAAY;CAC3E,MAAM,WAAW,UAAU,GAAG,GAAG,mBAAmB;CACpD,MAAM,QAAQ,aAAa,GAAG;CAC9B,IAAI;EAEF,IAAI,EAAC,MADW,KAAK,KAAK,EAAA,CACnB,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;CACzD,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,iDAAiD,MAAM,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CACxH;AACF;AAEA,MAAa,sBAA6C,EACxD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,wCAAwC,CAC5D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,UAAU,KAAwB,OAAe,KAAyB;CACjF,MAAM,OAAoB;EACxB,QAAQ,IAAI;EACZ;EACA,OAAO,IAAI;EACX,GAAI,MAAM,EAAE,cAAc,aAAa,GAAG,EAAE,IAAI,CAAC;CACnD;CACA,OAAO;EACL,IAAI,OAAO,IAAI,OAAO;EACtB;EACA,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA8B;CAC9C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,CAAC,MAAM,QAAQ,GAAG,KAAK,GACjE,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kDAAkD;CAE5F,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,OAAe,KAA2B;CAC1G,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,OAAO,GAAG,CAAC;CACxD,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,EAAE,aAAa,OAAO;CAC9G,OAAO;AACT;AAEA,eAAe,aAAa,MAAmB,OAAqC;CAClF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,0CAA0C,KAAK,OAAO,yBAAyB,UAAU;CACtG,OAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,eAAe,kBAAkB,KAAa,MAAmB,OAAqC;CAmCpG,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAK;EAAO,KAAK,UAAU,KAAA,IAAY,KAAK,OAAO,KAAK,KAAK;EAAG,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC;CAAC,CAAC;CAC3I,OAAO,WAAW,KAAK,MAAM,MAAM,GAA0B,MAAM,OAAO,GAAG;AAC/E;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,qBAAqB;CAEtD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KACH,MAAM,IAAI,MACR,qKACF;GAEF,MAAM,oBAAoB,KAAK,aAAa;GAC5C,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,GAAG,aAAa;EAC1D;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,IAAI,cAAc,OAAO,aAAa,MAAM,KAAK;GACjD,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wDAAwD;GAClF,OAAO,kBAAkB,KAAK,MAAM,KAAK;EAC3C;EAEA,MAAM,aAAa,MAAiB;GAElC,MAAM,QADO,SAAS,IACL,CAAC,CAAC,MAAM;GACzB,IAAI,CAAC,OAAO,OAAO,KAAA;GACnB,MAAM,QAAQ,MAAM;GACpB,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,OAAO,KAAK;EACvD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAM,WAAW;GACvB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,2EAA2E;GACrG,MAAM,SAAS,MAAM,mBACnB,KAAK,WAAW,WAAW,kBAAkB,GAC7C,CAAC,gBAAgB,UAAU,GAAG,CAAC,GAC/B,KAAK,UAAU;IAAE,YAAY;IAAU,OAAO,KAAK;GAAM,CAAC,GAC1D,EAAE,KAAK,UAAU,CACnB;GACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;GAClE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,2BAA2B,KAAK,GAAG,IAAI,OAAO,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,OAAO,YAAY;IAC7B;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,OAAO,KAAK;KAAO,SAAS,OAAO;IAAQ,CAAC;GAC5F;EACF;CACF;AACF"}

package/dist/benchmarks/enterpriseops-gym.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/enterpriseops-gym.d.ts
 /** Worker transcript = the last fenced ```json block, else the raw text. */
 declare const enterpriseOpsTranscriptOutput: OutputAdapter<string>;