@tangle-network/agent-bench 0.4.7 → 0.4.8

package/dist/benchmarks/enterpriseops-gym.js.map

@@ -1 +1 @@
-{"version":3,"file":"enterpriseops-gym.js","names":[],"sources":["../../src/benchmarks/enterpriseops-gym.ts"],"sourcesContent":["/**\n * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —\n * stateful agentic planning + tool use in enterprise settings. Each record is an\n * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,\n * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`\n * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).\n * Worker artifact = the ordered tool-call transcript the agent would issue against\n * those servers, emitted as a single fenced ```json block of\n * `{ \"calls\": [ { \"tool\": ..., \"arguments\": {...}, \"gym_name\"?: ... } ] }`.\n *\n * Judge = the benchmark's OWN deterministic state-checker. The driver replays the\n * transcript against a freshly-seeded gym server (mutating its database), then runs\n * each task's `database_state` verifier — an SQL SELECT executed via the gym\n * server's /api/sql-runner endpoint, compared to `expected_value` under\n * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =\n * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;\n * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully\n * deterministic — no LLM judge.\n *\n * loadTasks enumerates the real suite from the HF rows server (config = tool-set\n * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed\n * sample (bench/fixtures/enterpriseops-gym.json) loads offline.\n *\n * Requires for a LIVE judge run: a Docker daemon with the domain gym images\n * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up\n * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge\n * fail loud with the exact pull/run/unzip step when a server is unreachable — never\n * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.\n */\n\nimport { mkdir, readFile, rm, writeFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'enterpriseops-gym.json')\nconst JUDGE = join(benchRoot, 'scripts', 'enterpriseops_gym_judge.py')\n\n/** Monotonic discriminator so concurrent judge calls stage distinct task-cache files. */\nlet judgeCallSeq = 0\n\nconst DATASET = 'ServiceNow-AI/EnterpriseOps-Gym'\n/** Tool-set mode = HF config; oracle ships exact tools, plus_N adds N distractors. */\nconst DEFAULT_MODE = 'oracle'\n/** Domain = HF split. */\nconst DEFAULT_DOMAIN = 'itsm'\n\nconst rowsApi = (mode: string, domain: string) =>\n  `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`\n\ninterface GymServerConfig {\n  mcp_server_name: string\n  mcp_server_url: string\n  seed_database_file: string\n  context?: Record<string, string>\n  user_info?: Record<string, unknown>\n}\n\ninterface Verifier {\n  verifier_type: string\n  name: string\n  description?: string\n  gym_name: string\n  validation_config: { query: string; expected_value: unknown; comparison_type: string }\n}\n\ninterface EopsRow {\n  task_id: string\n  domain: string\n  system_prompt: string\n  user_prompt: string\n  selected_tools: string[]\n  restricted_tools: string[]\n  mcp_endpoint: string\n  number_of_runs: number\n  reset_database_between_runs: boolean\n  /** HF parquet stores these as JSON strings; fixtures store them as parsed arrays. */\n  gym_servers_config: string | GymServerConfig[]\n  verifiers: string | Verifier[]\n}\n\ninterface EopsMeta {\n  taskId: string\n  domain: string\n  mode: string\n  selectedTools: string[]\n  servers: GymServerConfig[]\n  verifiers: Verifier[]\n}\n\n/** Worker transcript = the last fenced ```json block, else the raw text. */\nexport const enterpriseOpsTranscriptOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction asArray<T>(v: string | T[]): T[] {\n  return typeof v === 'string' ? (JSON.parse(v) as T[]) : v\n}\n\nfunction workerContract(tools: string[]): string {\n  return [\n    '',\n    `You have exactly these tools available (call NO others): ${tools.join(', ')}.`,\n    'Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.',\n    'Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:',\n    '{ \"calls\": [ { \"tool\": \"<tool_name>\", \"arguments\": { ... } } ] }',\n    'Include one entry per tool call in execution order. Nothing after the closing fence.',\n  ].join('\\n')\n}\n\nfunction rowToTask(row: EopsRow, mode: string): BenchTask {\n  const servers = asArray<GymServerConfig>(row.gym_servers_config)\n  const verifiers = asArray<Verifier>(row.verifiers)\n  const meta: EopsMeta = {\n    taskId: row.task_id,\n    domain: row.domain,\n    mode,\n    selectedTools: row.selected_tools,\n    servers,\n    verifiers,\n  }\n  return {\n    id: row.task_id,\n    split: row.domain,\n    prompt: [row.system_prompt, '', row.user_prompt, workerContract(row.selected_tools)].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): EopsMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.taskId !== 'string' ||\n    !Array.isArray(md.servers) ||\n    !Array.isArray(md.verifiers) ||\n    (md.verifiers as unknown[]).length === 0\n  ) {\n    throw new Error(`enterpriseops-gym task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as EopsMeta\n}\n\nfunction selectRows(rows: EopsRow[], mode: string, opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map((r) => rowToTask(r, mode))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(mode: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as EopsRow[]\n  console.warn(\n    `[enterpriseops-gym] EOPS_FIXTURES=1 — loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`,\n  )\n  const domain = opts.split\n  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows\n  return selectRows(scoped, mode, opts)\n}\n\n/** Pull real rows from the HF rows server (paged) for one mode/domain. Throws loud on a non-OK response. */\nasync function fetchRows(mode: string, domain: string, opts: LoadOptions): Promise<EopsRow[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: EopsRow[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  const base = rowsApi(mode, domain)\n  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {\n    const res = await fetch(`${base}&offset=${offset}&length=${page}`)\n    if (!res.ok) {\n      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    }\n    const body = (await res.json()) as { rows?: Array<{ row: EopsRow }> }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.task_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`)\n  return rows\n}\n\n/**\n * Run the benchmark's own state-checker for one task over the worker's transcript.\n * The driver replays the tool calls against the live gym server, runs each\n * database_state verifier's SQL via /api/sql-runner, and reports {passes,total}.\n * Score = passes/total (verifier_level_pass_rate); resolved = all pass\n * (overall_success_rate). This is the expensive Docker-backed boundary — delegated\n * to the python driver, not reimplemented. The transcript is piped on stdin via the\n * shared stdin-aware runner (execFile's `input` is not honored async and hangs the\n * reader).\n */\nasync function runJudge(meta: EopsMeta, artifact: string): Promise<BenchScore> {\n  // Stage the full task record (servers + verifiers) so the driver has the live\n  // server URLs/contexts and the SQL it must run. The path is UNIQUE per call: the gate\n  // runs k judges for one task concurrently, so a `${taskId}.json` shared path would race.\n  const taskJsonPath = join(\n    benchRoot,\n    '.eops-task-cache',\n    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`,\n  )\n  await writeTaskCache(taskJsonPath, meta)\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(JUDGE, ['judge', '--task-json', taskJsonPath], artifact, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(\n      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}\\n` +\n        `Fix: (1) run the ${meta.domain} gym server — ` +\n        `docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `(2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir ` +\n        `(the judge seeds each task's database from seed_database_file).`,\n    )\n  } finally {\n    await rm(taskJsonPath, { force: true }).catch(() => {})\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    success?: boolean\n    passes?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`)\n  if (typeof report.passes !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passes / report.total : 0\n  return {\n    resolved: report.success === true && report.total > 0 && report.passes === report.total,\n    score,\n    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total }),\n  }\n}\n\nasync function writeTaskCache(path: string, meta: EopsMeta): Promise<void> {\n  await mkdir(join(path, '..'), { recursive: true })\n  await writeFile(\n    path,\n    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers }),\n  )\n}\n\n/** Ping the configured gym servers' SQL endpoint; throw loud with the docker fix if any is unreachable. */\nasync function probeServers(servers: GymServerConfig[], domain: string): Promise<void> {\n  for (const s of servers) {\n    const url = `${s.mcp_server_url.replace(/\\/$/, '')}/api/sql-runner`\n    try {\n      // A HEAD/empty POST just proves reachability; a real query runs in judge.\n      const res = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' })\n      // Any HTTP response (even an error status) proves the server is up. Only a\n      // transport failure (refused/ENOTFOUND) means the container is not running.\n      void res.status\n    } catch (err) {\n      throw new Error(\n        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}\\n` +\n          `Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`,\n      )\n    }\n  }\n}\n\nexport function createEnterpriseOpsGymAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.EOPS_FIXTURES === '1'\n  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE\n\n  return {\n    name: 'enterpriseops-gym',\n    output: enterpriseOpsTranscriptOutput,\n\n    async preflight() {\n      // Fixtures mode proves only that the sample file is readable; a live judge\n      // still requires running gym servers (and fails loud there if absent).\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      // Live mode: probe the gym servers for the default domain (the suite's tasks\n      // each carry their own server config; preflight verifies reachability up\n      // front so a batch fails fast with the docker fix rather than mid-run).\n      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => [])\n      const servers = sample[0] ? readMeta(sample[0]).servers : []\n      if (servers.length === 0) {\n        throw new Error(\n          `enterpriseops-gym preflight: no gym_servers_config to probe. ` +\n            `Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`,\n        )\n      }\n      await probeServers(servers, DEFAULT_DOMAIN)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const domain = opts.split ?? DEFAULT_DOMAIN\n      if (fixturesMode) return loadFixtures(mode, opts)\n      let rows: EopsRow[]\n      try {\n        rows = await fetchRows(mode, domain, opts)\n      } catch (err) {\n        console.warn(\n          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`,\n        )\n        return loadFixtures(mode, opts)\n      }\n      return selectRows(rows, mode, opts)\n    },\n\n    async goldArtifact() {\n      // The benchmark ships no portable per-task oracle transcript — the reference\n      // is the seeded final DB state the verifiers check, not a tool-call script.\n      // Judge correctness is proven by replaying a real solve against the live gym\n      // server, not by a synthetic gold transcript. Returns undefined (documented,\n      // not faked).\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCA,MAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,MAAM,QAAQ,KAAK,WAAW,WAAW,4BAA4B;;AAGrE,IAAI,eAAe;AAEnB,MAAM,UAAU;;AAEhB,MAAM,eAAe;;AAErB,MAAM,iBAAiB;AAEvB,MAAM,WAAW,MAAc,WAC7B,uDAAuD,mBAAmB,OAAO,EAAE,UAAU,mBAAmB,IAAI,EAAE,SAAS,mBAAmB,MAAM;;AA2C1J,MAAa,gCAAuD,EAClE,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,iCAAiC,CACrD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,QAAW,GAAsB;CACxC,OAAO,OAAO,MAAM,WAAY,KAAK,MAAM,CAAC,IAAY;AAC1D;AAEA,SAAS,eAAe,OAAyB;CAC/C,OAAO;EACL;EACA,4DAA4D,MAAM,KAAK,IAAI,EAAE;EAC7E;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAAc,MAAyB;CACxD,MAAM,UAAU,QAAyB,IAAI,kBAAkB;CAC/D,MAAM,YAAY,QAAkB,IAAI,SAAS;CACjD,MAAM,OAAiB;EACrB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,eAAe,IAAI;EACnB;EACA;CACF;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GAAC,IAAI;GAAe;GAAI,IAAI;GAAa,eAAe,IAAI,cAAc;EAAC,CAAC,CAAC,KAAK,IAAI;EAC9F,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA2B;CAC3C,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,WAAW,YACrB,CAAC,MAAM,QAAQ,GAAG,OAAO,KACzB,CAAC,MAAM,QAAQ,GAAG,SAAS,KAC1B,GAAG,UAAwB,WAAW,GAEvC,MAAM,IAAI,MAAM,0BAA0B,KAAK,GAAG,kDAAkD;CAEtG,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,MAAc,MAAgC;CACjF,IAAI,QAAQ,KAAK,KAAK,MAAM,UAAU,GAAG,IAAI,CAAC;CAC9C,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,eAAe,aAAa,MAAc,MAAyC;CACjF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KACN,iDAAiD,KAAK,OAAO,8BAA8B,SAAS,eACtG;CACA,MAAM,SAAS,KAAK;CAEpB,OAAO,WADQ,SAAS,KAAK,QAAQ,MAAM,EAAE,WAAW,MAAM,IAAI,MACxC,MAAM,IAAI;AACtC;;AAGA,eAAe,UAAU,MAAc,QAAgB,MAAuC;CAC5F,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;CAC/D,MAAM,OAAkB,CAAC;CACzB,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;CAC5C,MAAM,OAAO;CACb,MAAM,OAAO,QAAQ,MAAM,MAAM;CACjC,KAAK,IAAI,SAAS,GAAG,SAAS,QAAQ,KAAK,SAAS,QAAQ,UAAU,MAAM;EAC1E,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM;EACjE,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO,WAAW,OAAO,MAAM,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EAGrH,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,CAAC;EAC1B,IAAI,IAAI,WAAW,GAAG;EACtB,KAAK,MAAM,KAAK,KAAK;GACnB,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,OAAO,GAAG;GACtC,KAAK,KAAK,EAAE,GAAG;EACjB;EACA,IAAI,IAAI,SAAS,MAAM;CACzB;CACA,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,EAAE,OAAO,KAAK,GAAG,QAAQ;CACzH,OAAO;AACT;;;;;;;;;;;AAYA,eAAe,SAAS,MAAgB,UAAuC;CAI7E,MAAM,eAAe,KACnB,WACA,oBACA,GAAG,KAAK,OAAO,GAAG,QAAQ,IAAI,GAAG,eAAe,MAClD;CACA,MAAM,eAAe,cAAc,IAAI;CACvC,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,OAAO;GAAC;GAAS;GAAe;EAAY,GAAG,UAAU,EAAE,KAAK,UAAU,CAAC;CAC/G,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MACR,sCAAsC,KAAK,OAAO,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,EAAE,qBAC1E,KAAK,OAAO,yEAC4B,KAAK,OAAO,sFACK,KAAK,OAAO,sKAG7F;CACF,UAAU;EACR,MAAM,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CACxD;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAMlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,qCAAqC,KAAK,OAAO,IAAI,OAAO,OAAO;CACrG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,uDAAuD,OAAO,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;CAChE,OAAO;EACL,UAAU,OAAO,YAAY,QAAQ,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;EAClF;EACA,QAAQ,KAAK,UAAU;GAAE,QAAQ,KAAK;GAAQ,QAAQ,KAAK;GAAQ,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACjH;AACF;AAEA,eAAe,eAAe,MAAc,MAA+B;CACzE,MAAM,MAAM,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CACjD,MAAM,UACJ,MACA,KAAK,UAAU;EAAE,SAAS,KAAK;EAAQ,QAAQ,KAAK;EAAQ,oBAAoB,KAAK;EAAS,WAAW,KAAK;CAAU,CAAC,CAC3H;AACF;;AAGA,eAAe,aAAa,SAA4B,QAA+B;CACrF,KAAK,MAAM,KAAK,SAAS;EACvB,MAAM,MAAM,GAAG,EAAE,eAAe,QAAQ,OAAO,EAAE,EAAE;EACnD,IAAI;GAKF,CAAK,MAHa,MAAM,KAAK;IAAE,QAAQ;IAAQ,SAAS,EAAE,gBAAgB,mBAAmB;IAAG,MAAM;GAAK,CAAC,EAAA,CAGnG;EACX,SAAS,KAAK;GACZ,MAAM,IAAI,MACR,gCAAgC,EAAE,gBAAgB,kBAAkB,IAAI,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,kEAClD,OAAO,wFACO,OAAO,qHAE1F;EACF;CACF;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,kBAAkB;CACnD,MAAM,OAAO,QAAQ,IAAI,aAAa;CAEtC,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAGhB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,uBAAuB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC3G,CAAC;IACD;GACF;GAIA,MAAM,SAAS,MAAM,aAAa,MAAM;IAAE,OAAO;IAAgB,OAAO;GAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAC3F,MAAM,UAAU,OAAO,KAAK,SAAS,OAAO,EAAE,CAAC,CAAC,UAAU,CAAC;GAC3D,IAAI,QAAQ,WAAW,GACrB,MAAM,IAAI,MACR,wJAEF;GAEF,MAAM,aAAa,SAAS,cAAc;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,SAAS,KAAK,SAAS;GAC7B,IAAI,cAAc,OAAO,aAAa,MAAM,IAAI;GAChD,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,UAAU,MAAM,QAAQ,IAAI;GAC3C,SAAS,KAAK;IACZ,QAAQ,KACN,+CAA+C,eAAe,QAAQ,IAAI,UAAU,IAAI,yCAAyC,UACnI;IACA,OAAO,aAAa,MAAM,IAAI;GAChC;GACA,OAAO,WAAW,MAAM,MAAM,IAAI;EACpC;EAEA,MAAM,eAAe,CAOrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,SADM,SAAS,IACH,GAAG,QAAQ;EAChC;CACF;AACF"}
+{"version":3,"file":"enterpriseops-gym.js","names":[],"sources":["../../src/benchmarks/enterpriseops-gym.ts"],"sourcesContent":["/**\n * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —\n * stateful agentic planning + tool use in enterprise settings. Each record is an\n * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,\n * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`\n * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).\n * Worker artifact = the ordered tool-call transcript the agent would issue against\n * those servers, emitted as a single fenced ```json block of\n * `{ \"calls\": [ { \"tool\": ..., \"arguments\": {...}, \"gym_name\"?: ... } ] }`.\n *\n * Judge = the benchmark's OWN deterministic state-checker. The driver replays the\n * transcript against a freshly-seeded gym server (mutating its database), then runs\n * each task's `database_state` verifier — an SQL SELECT executed via the gym\n * server's /api/sql-runner endpoint, compared to `expected_value` under\n * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =\n * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;\n * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully\n * deterministic — no LLM judge.\n *\n * loadTasks enumerates the real suite from the HF rows server (config = tool-set\n * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed\n * sample (bench/fixtures/enterpriseops-gym.json) loads offline.\n *\n * Requires for a LIVE judge run: a Docker daemon with the domain gym images\n * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up\n * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge\n * fail loud with the exact pull/run/unzip step when a server is unreachable — never\n * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.\n */\n\nimport { mkdir, readFile, rm, writeFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'enterpriseops-gym.json')\nconst JUDGE = join(benchRoot, 'scripts', 'enterpriseops_gym_judge.py')\n\n/** Monotonic discriminator so concurrent judge calls stage distinct task-cache files. */\nlet judgeCallSeq = 0\n\nconst DATASET = 'ServiceNow-AI/EnterpriseOps-Gym'\n/** Tool-set mode = HF config; oracle ships exact tools, plus_N adds N distractors. */\nconst DEFAULT_MODE = 'oracle'\n/** Domain = HF split. */\nconst DEFAULT_DOMAIN = 'itsm'\n\nconst rowsApi = (mode: string, domain: string) =>\n  `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`\n\ninterface GymServerConfig {\n  mcp_server_name: string\n  mcp_server_url: string\n  seed_database_file: string\n  context?: Record<string, string>\n  user_info?: Record<string, unknown>\n}\n\ninterface Verifier {\n  verifier_type: string\n  name: string\n  description?: string\n  gym_name: string\n  validation_config: { query: string; expected_value: unknown; comparison_type: string }\n}\n\ninterface EopsRow {\n  task_id: string\n  domain: string\n  system_prompt: string\n  user_prompt: string\n  selected_tools: string[]\n  restricted_tools: string[]\n  mcp_endpoint: string\n  number_of_runs: number\n  reset_database_between_runs: boolean\n  /** HF parquet stores these as JSON strings; fixtures store them as parsed arrays. */\n  gym_servers_config: string | GymServerConfig[]\n  verifiers: string | Verifier[]\n}\n\ninterface EopsMeta {\n  taskId: string\n  domain: string\n  mode: string\n  selectedTools: string[]\n  servers: GymServerConfig[]\n  verifiers: Verifier[]\n}\n\n/** Worker transcript = the last fenced ```json block, else the raw text. */\nexport const enterpriseOpsTranscriptOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction asArray<T>(v: string | T[]): T[] {\n  return typeof v === 'string' ? (JSON.parse(v) as T[]) : v\n}\n\nfunction workerContract(tools: string[]): string {\n  return [\n    '',\n    `You have exactly these tools available (call NO others): ${tools.join(', ')}.`,\n    'Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.',\n    'Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:',\n    '{ \"calls\": [ { \"tool\": \"<tool_name>\", \"arguments\": { ... } } ] }',\n    'Include one entry per tool call in execution order. Nothing after the closing fence.',\n  ].join('\\n')\n}\n\nfunction rowToTask(row: EopsRow, mode: string): BenchTask {\n  const servers = asArray<GymServerConfig>(row.gym_servers_config)\n  const verifiers = asArray<Verifier>(row.verifiers)\n  const meta: EopsMeta = {\n    taskId: row.task_id,\n    domain: row.domain,\n    mode,\n    selectedTools: row.selected_tools,\n    servers,\n    verifiers,\n  }\n  return {\n    id: row.task_id,\n    split: row.domain,\n    prompt: [row.system_prompt, '', row.user_prompt, workerContract(row.selected_tools)].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): EopsMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.taskId !== 'string' ||\n    !Array.isArray(md.servers) ||\n    !Array.isArray(md.verifiers) ||\n    (md.verifiers as unknown[]).length === 0\n  ) {\n    throw new Error(`enterpriseops-gym task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as EopsMeta\n}\n\nfunction selectRows(rows: EopsRow[], mode: string, opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map((r) => rowToTask(r, mode))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(mode: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as EopsRow[]\n  console.warn(\n    `[enterpriseops-gym] EOPS_FIXTURES=1 — loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`,\n  )\n  const domain = opts.split\n  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows\n  return selectRows(scoped, mode, opts)\n}\n\n/** Pull real rows from the HF rows server (paged) for one mode/domain. Throws loud on a non-OK response. */\nasync function fetchRows(mode: string, domain: string, opts: LoadOptions): Promise<EopsRow[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: EopsRow[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  const base = rowsApi(mode, domain)\n  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {\n    const res = await fetch(`${base}&offset=${offset}&length=${page}`)\n    if (!res.ok) {\n      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    }\n    const body = (await res.json()) as { rows?: Array<{ row: EopsRow }> }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.task_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`)\n  return rows\n}\n\n/**\n * Run the benchmark's own state-checker for one task over the worker's transcript.\n * The driver replays the tool calls against the live gym server, runs each\n * database_state verifier's SQL via /api/sql-runner, and reports {passes,total}.\n * Score = passes/total (verifier_level_pass_rate); resolved = all pass\n * (overall_success_rate). This is the expensive Docker-backed boundary — delegated\n * to the python driver, not reimplemented. The transcript is piped on stdin via the\n * shared stdin-aware runner (execFile's `input` is not honored async and hangs the\n * reader).\n */\nasync function runJudge(meta: EopsMeta, artifact: string): Promise<BenchScore> {\n  // Stage the full task record (servers + verifiers) so the driver has the live\n  // server URLs/contexts and the SQL it must run. The path is UNIQUE per call: the gate\n  // runs k judges for one task concurrently, so a `${taskId}.json` shared path would race.\n  const taskJsonPath = join(\n    benchRoot,\n    '.eops-task-cache',\n    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`,\n  )\n  await writeTaskCache(taskJsonPath, meta)\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(JUDGE, ['judge', '--task-json', taskJsonPath], artifact, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(\n      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}\\n` +\n        `Fix: (1) run the ${meta.domain} gym server — ` +\n        `docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `(2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir ` +\n        `(the judge seeds each task's database from seed_database_file).`,\n    )\n  } finally {\n    await rm(taskJsonPath, { force: true }).catch(() => {})\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    success?: boolean\n    passes?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`)\n  if (typeof report.passes !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passes / report.total : 0\n  return {\n    resolved: report.success === true && report.total > 0 && report.passes === report.total,\n    score,\n    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total }),\n  }\n}\n\nasync function writeTaskCache(path: string, meta: EopsMeta): Promise<void> {\n  await mkdir(join(path, '..'), { recursive: true })\n  await writeFile(\n    path,\n    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers }),\n  )\n}\n\n/** Ping the configured gym servers' SQL endpoint; throw loud with the docker fix if any is unreachable. */\nasync function probeServers(servers: GymServerConfig[], domain: string): Promise<void> {\n  for (const s of servers) {\n    const url = `${s.mcp_server_url.replace(/\\/$/, '')}/api/sql-runner`\n    try {\n      // A HEAD/empty POST just proves reachability; a real query runs in judge.\n      const res = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' })\n      // Any HTTP response (even an error status) proves the server is up. Only a\n      // transport failure (refused/ENOTFOUND) means the container is not running.\n      void res.status\n    } catch (err) {\n      throw new Error(\n        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}\\n` +\n          `Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`,\n      )\n    }\n  }\n}\n\nexport function createEnterpriseOpsGymAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.EOPS_FIXTURES === '1'\n  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE\n\n  return {\n    name: 'enterpriseops-gym',\n    output: enterpriseOpsTranscriptOutput,\n\n    async preflight() {\n      // Fixtures mode proves only that the sample file is readable; a live judge\n      // still requires running gym servers (and fails loud there if absent).\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      // Live mode: probe the gym servers for the default domain (the suite's tasks\n      // each carry their own server config; preflight verifies reachability up\n      // front so a batch fails fast with the docker fix rather than mid-run).\n      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => [])\n      const servers = sample[0] ? readMeta(sample[0]).servers : []\n      if (servers.length === 0) {\n        throw new Error(\n          `enterpriseops-gym preflight: no gym_servers_config to probe. ` +\n            `Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`,\n        )\n      }\n      await probeServers(servers, DEFAULT_DOMAIN)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const domain = opts.split ?? DEFAULT_DOMAIN\n      if (fixturesMode) return loadFixtures(mode, opts)\n      let rows: EopsRow[]\n      try {\n        rows = await fetchRows(mode, domain, opts)\n      } catch (err) {\n        console.warn(\n          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`,\n        )\n        return loadFixtures(mode, opts)\n      }\n      return selectRows(rows, mode, opts)\n    },\n\n    async goldArtifact() {\n      // The benchmark ships no portable per-task oracle transcript — the reference\n      // is the seeded final DB state the verifiers check, not a tool-call script.\n      // Judge correctness is proven by replaying a real solve against the live gym\n      // server, not by a synthetic gold transcript. Returns undefined (documented,\n      // not faked).\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoCA,MAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,MAAM,QAAQ,KAAK,WAAW,WAAW,4BAA4B;;AAGrE,IAAI,eAAe;AAEnB,MAAM,UAAU;;AAEhB,MAAM,eAAe;;AAErB,MAAM,iBAAiB;AAEvB,MAAM,WAAW,MAAc,WAC7B,uDAAuD,mBAAmB,OAAO,EAAE,UAAU,mBAAmB,IAAI,EAAE,SAAS,mBAAmB,MAAM;;AA2C1J,MAAa,gCAAuD,EAClE,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,iCAAiC,CACrD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,QAAW,GAAsB;CACxC,OAAO,OAAO,MAAM,WAAY,KAAK,MAAM,CAAC,IAAY;AAC1D;AAEA,SAAS,eAAe,OAAyB;CAC/C,OAAO;EACL;EACA,4DAA4D,MAAM,KAAK,IAAI,EAAE;EAC7E;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAAc,MAAyB;CACxD,MAAM,UAAU,QAAyB,IAAI,kBAAkB;CAC/D,MAAM,YAAY,QAAkB,IAAI,SAAS;CACjD,MAAM,OAAiB;EACrB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,eAAe,IAAI;EACnB;EACA;CACF;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GAAC,IAAI;GAAe;GAAI,IAAI;GAAa,eAAe,IAAI,cAAc;EAAC,CAAC,CAAC,KAAK,IAAI;EAC9F,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA2B;CAC3C,MAAM,KAAK,KAAK;CAChB,IACE,CAAC,MACD,OAAO,GAAG,WAAW,YACrB,CAAC,MAAM,QAAQ,GAAG,OAAO,KACzB,CAAC,MAAM,QAAQ,GAAG,SAAS,KAC1B,GAAG,UAAwB,WAAW,GAEvC,MAAM,IAAI,MAAM,0BAA0B,KAAK,GAAG,kDAAkD;CAEtG,OAAO;AACT;AAEA,SAAS,WAAW,MAAiB,MAAc,MAAgC;CACjF,IAAI,QAAQ,KAAK,KAAK,MAAM,UAAU,GAAG,IAAI,CAAC;CAC9C,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,eAAe,aAAa,MAAc,MAAyC;CACjF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KACN,iDAAiD,KAAK,OAAO,8BAA8B,SAAS,eACtG;CACA,MAAM,SAAS,KAAK;CAEpB,OAAO,WADQ,SAAS,KAAK,QAAQ,MAAM,EAAE,WAAW,MAAM,IAAI,MACxC,MAAM,IAAI;AACtC;;AAGA,eAAe,UAAU,MAAc,QAAgB,MAAuC;CAC5F,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;CAC/D,MAAM,OAAkB,CAAC;CACzB,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;CAC5C,MAAM,OAAO;CACb,MAAM,OAAO,QAAQ,MAAM,MAAM;CACjC,KAAK,IAAI,SAAS,GAAG,SAAS,QAAQ,KAAK,SAAS,QAAQ,UAAU,MAAM;EAC1E,MAAM,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM;EACjE,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO,WAAW,OAAO,MAAM,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EAGrH,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,CAAC;EAC1B,IAAI,IAAI,WAAW,GAAG;EACtB,KAAK,MAAM,KAAK,KAAK;GACnB,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,OAAO,GAAG;GACtC,KAAK,KAAK,EAAE,GAAG;EACjB;EACA,IAAI,IAAI,SAAS,MAAM;CACzB;CACA,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,EAAE,OAAO,KAAK,GAAG,QAAQ;CACzH,OAAO;AACT;;;;;;;;;;;AAYA,eAAe,SAAS,MAAgB,UAAuC;CAI7E,MAAM,eAAe,KACnB,WACA,oBACA,GAAG,KAAK,OAAO,GAAG,QAAQ,IAAI,GAAG,eAAe,MAClD;CACA,MAAM,eAAe,cAAc,IAAI;CACvC,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,OAAO;GAAC;GAAS;GAAe;EAAY,GAAG,UAAU,EAAE,KAAK,UAAU,CAAC;CAC/G,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MACR,sCAAsC,KAAK,OAAO,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,EAAE,qBAC1E,KAAK,OAAO,yEAC4B,KAAK,OAAO,sFACK,KAAK,OAAO,sKAG7F;CACF,UAAU;EACR,MAAM,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CACxD;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAMlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,qCAAqC,KAAK,OAAO,IAAI,OAAO,OAAO;CACrG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,uDAAuD,OAAO,MAAM,GAAG,GAAG,GAAG;CAE/F,MAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;CAChE,OAAO;EACL,UAAU,OAAO,YAAY,QAAQ,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;EAClF;EACA,QAAQ,KAAK,UAAU;GAAE,QAAQ,KAAK;GAAQ,QAAQ,KAAK;GAAQ,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACjH;AACF;AAEA,eAAe,eAAe,MAAc,MAA+B;CACzE,MAAM,MAAM,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;CACjD,MAAM,UACJ,MACA,KAAK,UAAU;EAAE,SAAS,KAAK;EAAQ,QAAQ,KAAK;EAAQ,oBAAoB,KAAK;EAAS,WAAW,KAAK;CAAU,CAAC,CAC3H;AACF;;AAGA,eAAe,aAAa,SAA4B,QAA+B;CACrF,KAAK,MAAM,KAAK,SAAS;EACvB,MAAM,MAAM,GAAG,EAAE,eAAe,QAAQ,OAAO,EAAE,EAAE;EACnD,IAAI;GAKF,CAAK,MAHa,MAAM,KAAK;IAAE,QAAQ;IAAQ,SAAS,EAAE,gBAAgB,mBAAmB;IAAG,MAAM;GAAK,CAAC,EAAA,CAGnG;EACX,SAAS,KAAK;GACZ,MAAM,IAAI,MACR,gCAAgC,EAAE,gBAAgB,kBAAkB,IAAI,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,kEAClD,OAAO,wFACO,OAAO,qHAE1F;EACF;CACF;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,kBAAkB;CACnD,MAAM,OAAO,QAAQ,IAAI,aAAa;CAEtC,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAGhB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,uBAAuB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC3G,CAAC;IACD;GACF;GAIA,MAAM,SAAS,MAAM,aAAa,MAAM;IAAE,OAAO;IAAgB,OAAO;GAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAC3F,MAAM,UAAU,OAAO,KAAK,SAAS,OAAO,EAAE,CAAC,CAAC,UAAU,CAAC;GAC3D,IAAI,QAAQ,WAAW,GACrB,MAAM,IAAI,MACR,wJAEF;GAEF,MAAM,aAAa,SAAS,cAAc;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,SAAS,KAAK,SAAS;GAC7B,IAAI,cAAc,OAAO,aAAa,MAAM,IAAI;GAChD,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,UAAU,MAAM,QAAQ,IAAI;GAC3C,SAAS,KAAK;IACZ,QAAQ,KACN,+CAA+C,eAAe,QAAQ,IAAI,UAAU,IAAI,yCAAyC,UACnI;IACA,OAAO,aAAa,MAAM,IAAI;GAChC;GACA,OAAO,WAAW,MAAM,MAAM,IAAI;EACpC;EAEA,MAAM,eAAe,CAOrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,SADM,SAAS,IACH,GAAG,QAAQ;EAChC;CACF;AACF"}

package/dist/benchmarks/programbench.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/programbench.d.ts
 /**
  * Parse the worker stream into the submission text the driver materializes: the

package/dist/benchmarks/programbench.js.map

@@ -1 +1 @@
-{"version":3,"file":"programbench.js","names":[],"sources":["../../src/benchmarks/programbench.ts"],"sourcesContent":["/**\n * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom\n * reverse-engineering: rebuild a black-box executable's behavior from scratch.\n * The agent is given only the gold `./executable` (run-only) + stripped docs and\n * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`\n * with identical observable behavior. Judge = the official `programbench`\n * harness: it extracts the submission, runs compile.sh in the per-task cleanroom\n * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via\n * `programbench blob sync`). Score = fraction of non-ignored tests passed\n * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests\n * pass. Fully deterministic — no LLM judge.\n *\n * OutputAdapter is stream-only, so the worker emits its codebase as fenced\n * `path:`-prefixed file blocks (including compile.sh); the adapter materializes\n * those into submission.tar.gz. Test execution is delegated to the real\n * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT\n * reimplemented here.\n *\n * Requires for a live run: the bench `.venv` with `programbench` installed +\n * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for\n * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load\n * the committed instance ids (bench/fixtures/programbench.json); judging still\n * needs the harness + Docker and fails loud without them — never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'programbench.json')\n\nconst TESTS_DATASET = 'programbench/ProgramBench-Tests'\nconst TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`\n\ninterface PbFixture {\n  instance_id: string\n  image_name: string\n  note?: string\n}\n\ninterface PbMeta {\n  instanceId: string\n  imageName: string\n}\n\nconst PROMPT = [\n  'This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.',\n  'Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.',\n  'You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.',\n  'Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence.',\n].join('\\n')\n\nfunction fixtureToTask(f: PbFixture): BenchTask {\n  const meta: PbMeta = { instanceId: f.instance_id, imageName: f.image_name }\n  return {\n    id: f.instance_id,\n    prompt: PROMPT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): PbMeta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string') {\n    throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`)\n  }\n  return md as unknown as PbMeta\n}\n\nfunction selectTasks(fixtures: PbFixture[], opts: LoadOptions): BenchTask[] {\n  let tasks = fixtures.map(fixtureToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\n/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */\nfunction imageName(instanceId: string): string {\n  return instanceId.split('.')[0].replace('__', '/')\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixtures = JSON.parse(await readFile(FIXTURES, 'utf8')) as PbFixture[]\n  console.warn(\n    `[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`,\n  )\n  return selectTasks(fixtures, opts)\n}\n\n/** Enumerate real instance ids from the HF Tests dataset tree (one dir per\n *  instance). Throws on a non-OK response (fail loud). */\nasync function fetchInstanceIds(): Promise<PbFixture[]> {\n  const res = await fetch(TREE_API)\n  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`)\n  const tree = (await res.json()) as Array<{ path: string; type: string }>\n  const ids = tree.filter((e) => e.type === 'directory' && e.path.includes('__')).map((e) => e.path)\n  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`)\n  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }))\n}\n\n/**\n * Run the official programbench harness for one instance over the worker's\n * file-manifest submission. The driver builds the run-dir layout (one folder\n * holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,\n * then parses <instance_id>.eval.json applying the tests.json ignore mask via\n * `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.\n */\nasync function runHarness(meta: PbMeta, submission: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'programbench_judge.py')\n  let stdout: string\n  try {\n    // The submission manifest is piped to the driver's stdin via the shared\n    // stdin-aware runner — execFile's `input` option is not honored async and\n    // hangs the driver's sys.stdin.read() forever.\n    stdout = await runVenvScriptStdin(judge, ['--instance', meta.instanceId], submission, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    resolved?: boolean\n    error?: string\n  }\n  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passed / report.total : 0\n  return {\n    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\n/**\n * Parse the worker stream into the submission text the driver materializes: the\n * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to\n * the python driver, which tars it. Empty when the worker emitted no file block\n * (fail-closed → the harness scores a missing compile.sh as 0).\n */\nexport const programbenchSubmissionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const blocks = [...text.matchAll(/```path:([^\\n`]+)\\n([\\s\\S]*?)```/g)]\n    if (blocks.length === 0) return ''\n    // Re-serialize in the same path:<p> envelope the driver splits on.\n    return blocks.map((m) => `===FILE:${m[1].trim()}===\\n${m[2]}`).join('\\n')\n  },\n}\n\nexport function createProgrambenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === '1'\n\n  return {\n    name: 'programbench',\n    output: programbenchSubmissionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['programbench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; ` +\n          `(2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; ` +\n          `(3) HF access for the hidden test blobs (the driver runs \\`programbench blob sync\\`). ` +\n          `Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let fixtures: PbFixture[]\n      try {\n        fixtures = await fetchInstanceIds()\n      } catch (err) {\n        console.warn(\n          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectTasks(fixtures, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the original repo's source (stripped from the task image),\n      // not redistributable as a portable submission string. Judge correctness is\n      // proven by running the real harness on a real solve, not a synthetic gold.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,MAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEhE,MAAM,gBAAgB;AACtB,MAAM,WAAW,uCAAuC,cAAc;AAatE,MAAM,SAAS;CACb;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,cAAc,GAAyB;CAC9C,MAAM,OAAe;EAAE,YAAY,EAAE;EAAa,WAAW,EAAE;CAAW;CAC1E,OAAO;EACL,IAAI,EAAE;EACN,QAAQ;EACR,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAyB;CACzC,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,eAAe,UAClC,MAAM,IAAI,MAAM,qBAAqB,KAAK,GAAG,6DAA6D;CAE5G,OAAO;AACT;AAEA,SAAS,YAAY,UAAuB,MAAgC;CAC1E,IAAI,QAAQ,SAAS,IAAI,aAAa;CACtC,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;;AAGA,SAAS,UAAU,YAA4B;CAC7C,OAAO,WAAW,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,MAAM,GAAG;AACnD;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,WAAW,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KACN,oDAAoD,SAAS,OAAO,+BAA+B,SAAS,oBAC9G;CACA,OAAO,YAAY,UAAU,IAAI;AACnC;;;AAIA,eAAe,mBAAyC;CACtD,MAAM,MAAM,MAAM,MAAM,QAAQ;CAChC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,0BAA0B,IAAI,OAAO,IAAI,UAAU;CAEhF,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,MAAM,EAAE,SAAS,eAAe,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,IAAI;CACjG,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,qCAAqC,cAAc,MAAM;CAC/F,OAAO,IAAI,KAAK,QAAQ;EAAE,aAAa;EAAI,YAAY,UAAU,EAAE;CAAE,EAAE;AACzE;;;;;;;;AASA,eAAe,WAAW,MAAc,YAAyC;CAC/E,MAAM,QAAQ,KAAK,WAAW,WAAW,uBAAuB;CAChE,IAAI;CACJ,IAAI;EAIF,SAAS,MAAM,mBAAmB,OAAO,CAAC,cAAc,KAAK,UAAU,GAAG,YAAY,EAAE,KAAK,UAAU,CAAC;CAC1G,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,mCAAmC,KAAK,WAAW,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CACpH;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAMlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,kCAAkC,KAAK,WAAW,IAAI,OAAO,OAAO;CACtG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,kDAAkD,OAAO,MAAM,GAAG,GAAG,GAAG;CAE1F,MAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;CAChE,OAAO;EACL,UAAU,OAAO,aAAa,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;EAC3E;EACA,QAAQ,KAAK,UAAU;GAAE,YAAY,KAAK;GAAY,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACpG;AACF;;;;;;;AAQA,MAAa,+BAAsD,EACjE,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,SAAS,mCAAmC,CAAC;CACrE,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,OAAO,OAAO,KAAK,MAAM,WAAW,EAAE,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI;AAC1E,EACF;AAEA,SAAgB,4BAA8C;CAC5D,MAAM,eAAe,QAAQ,IAAI,0BAA0B;CAE3D,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,cAAc;IACxB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,WAAW,MAAM,iBAAiB;GACpC,SAAS,KAAK;IACZ,QAAQ,KACN,0CAA0C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAA2C,UAChI;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,YAAY,UAAU,IAAI;EACnC;EAEA,MAAM,eAAe,CAKrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,WADM,SAAS,IACD,GAAG,QAAQ;EAClC;CACF;AACF"}
+{"version":3,"file":"programbench.js","names":[],"sources":["../../src/benchmarks/programbench.ts"],"sourcesContent":["/**\n * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom\n * reverse-engineering: rebuild a black-box executable's behavior from scratch.\n * The agent is given only the gold `./executable` (run-only) + stripped docs and\n * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`\n * with identical observable behavior. Judge = the official `programbench`\n * harness: it extracts the submission, runs compile.sh in the per-task cleanroom\n * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via\n * `programbench blob sync`). Score = fraction of non-ignored tests passed\n * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests\n * pass. Fully deterministic — no LLM judge.\n *\n * OutputAdapter is stream-only, so the worker emits its codebase as fenced\n * `path:`-prefixed file blocks (including compile.sh); the adapter materializes\n * those into submission.tar.gz. Test execution is delegated to the real\n * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT\n * reimplemented here.\n *\n * Requires for a live run: the bench `.venv` with `programbench` installed +\n * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for\n * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load\n * the committed instance ids (bench/fixtures/programbench.json); judging still\n * needs the harness + Docker and fails loud without them — never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'programbench.json')\n\nconst TESTS_DATASET = 'programbench/ProgramBench-Tests'\nconst TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`\n\ninterface PbFixture {\n  instance_id: string\n  image_name: string\n  note?: string\n}\n\ninterface PbMeta {\n  instanceId: string\n  imageName: string\n}\n\nconst PROMPT = [\n  'This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.',\n  'Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.',\n  'You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.',\n  'Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence.',\n].join('\\n')\n\nfunction fixtureToTask(f: PbFixture): BenchTask {\n  const meta: PbMeta = { instanceId: f.instance_id, imageName: f.image_name }\n  return {\n    id: f.instance_id,\n    prompt: PROMPT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): PbMeta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string') {\n    throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`)\n  }\n  return md as unknown as PbMeta\n}\n\nfunction selectTasks(fixtures: PbFixture[], opts: LoadOptions): BenchTask[] {\n  let tasks = fixtures.map(fixtureToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\n/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */\nfunction imageName(instanceId: string): string {\n  return instanceId.split('.')[0].replace('__', '/')\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixtures = JSON.parse(await readFile(FIXTURES, 'utf8')) as PbFixture[]\n  console.warn(\n    `[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`,\n  )\n  return selectTasks(fixtures, opts)\n}\n\n/** Enumerate real instance ids from the HF Tests dataset tree (one dir per\n *  instance). Throws on a non-OK response (fail loud). */\nasync function fetchInstanceIds(): Promise<PbFixture[]> {\n  const res = await fetch(TREE_API)\n  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`)\n  const tree = (await res.json()) as Array<{ path: string; type: string }>\n  const ids = tree.filter((e) => e.type === 'directory' && e.path.includes('__')).map((e) => e.path)\n  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`)\n  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }))\n}\n\n/**\n * Run the official programbench harness for one instance over the worker's\n * file-manifest submission. The driver builds the run-dir layout (one folder\n * holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,\n * then parses <instance_id>.eval.json applying the tests.json ignore mask via\n * `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.\n */\nasync function runHarness(meta: PbMeta, submission: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'programbench_judge.py')\n  let stdout: string\n  try {\n    // The submission manifest is piped to the driver's stdin via the shared\n    // stdin-aware runner — execFile's `input` option is not honored async and\n    // hangs the driver's sys.stdin.read() forever.\n    stdout = await runVenvScriptStdin(judge, ['--instance', meta.instanceId], submission, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    resolved?: boolean\n    error?: string\n  }\n  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passed / report.total : 0\n  return {\n    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\n/**\n * Parse the worker stream into the submission text the driver materializes: the\n * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to\n * the python driver, which tars it. Empty when the worker emitted no file block\n * (fail-closed → the harness scores a missing compile.sh as 0).\n */\nexport const programbenchSubmissionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const blocks = [...text.matchAll(/```path:([^\\n`]+)\\n([\\s\\S]*?)```/g)]\n    if (blocks.length === 0) return ''\n    // Re-serialize in the same path:<p> envelope the driver splits on.\n    return blocks.map((m) => `===FILE:${m[1].trim()}===\\n${m[2]}`).join('\\n')\n  },\n}\n\nexport function createProgrambenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === '1'\n\n  return {\n    name: 'programbench',\n    output: programbenchSubmissionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['programbench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; ` +\n          `(2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; ` +\n          `(3) HF access for the hidden test blobs (the driver runs \\`programbench blob sync\\`). ` +\n          `Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let fixtures: PbFixture[]\n      try {\n        fixtures = await fetchInstanceIds()\n      } catch (err) {\n        console.warn(\n          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectTasks(fixtures, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the original repo's source (stripped from the task image),\n      // not redistributable as a portable submission string. Judge correctness is\n      // proven by running the real harness on a real solve, not a synthetic gold.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,MAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEhE,MAAM,gBAAgB;AACtB,MAAM,WAAW,uCAAuC,cAAc;AAatE,MAAM,SAAS;CACb;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,cAAc,GAAyB;CAC9C,MAAM,OAAe;EAAE,YAAY,EAAE;EAAa,WAAW,EAAE;CAAW;CAC1E,OAAO;EACL,IAAI,EAAE;EACN,QAAQ;EACR,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAyB;CACzC,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,eAAe,UAClC,MAAM,IAAI,MAAM,qBAAqB,KAAK,GAAG,6DAA6D;CAE5G,OAAO;AACT;AAEA,SAAS,YAAY,UAAuB,MAAgC;CAC1E,IAAI,QAAQ,SAAS,IAAI,aAAa;CACtC,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;;AAGA,SAAS,UAAU,YAA4B;CAC7C,OAAO,WAAW,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,MAAM,GAAG;AACnD;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,WAAW,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KACN,oDAAoD,SAAS,OAAO,+BAA+B,SAAS,oBAC9G;CACA,OAAO,YAAY,UAAU,IAAI;AACnC;;;AAIA,eAAe,mBAAyC;CACtD,MAAM,MAAM,MAAM,MAAM,QAAQ;CAChC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,0BAA0B,IAAI,OAAO,IAAI,UAAU;CAEhF,MAAM,OAAM,MADQ,IAAI,KAAK,EAAA,CACZ,QAAQ,MAAM,EAAE,SAAS,eAAe,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,IAAI;CACjG,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,qCAAqC,cAAc,MAAM;CAC/F,OAAO,IAAI,KAAK,QAAQ;EAAE,aAAa;EAAI,YAAY,UAAU,EAAE;CAAE,EAAE;AACzE;;;;;;;;AASA,eAAe,WAAW,MAAc,YAAyC;CAC/E,MAAM,QAAQ,KAAK,WAAW,WAAW,uBAAuB;CAChE,IAAI;CACJ,IAAI;EAIF,SAAS,MAAM,mBAAmB,OAAO,CAAC,cAAc,KAAK,UAAU,GAAG,YAAY,EAAE,KAAK,UAAU,CAAC;CAC1G,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,mCAAmC,KAAK,WAAW,KAAK,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CACpH;CACA,MAAM,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;CAMlE,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,kCAAkC,KAAK,WAAW,IAAI,OAAO,OAAO;CACtG,IAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAC/D,MAAM,IAAI,MAAM,kDAAkD,OAAO,MAAM,GAAG,GAAG,GAAG;CAE1F,MAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;CAChE,OAAO;EACL,UAAU,OAAO,aAAa,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;EAC3E;EACA,QAAQ,KAAK,UAAU;GAAE,YAAY,KAAK;GAAY,QAAQ,OAAO;GAAQ,OAAO,OAAO;EAAM,CAAC;CACpG;AACF;;;;;;;AAQA,MAAa,+BAAsD,EACjE,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,SAAS,mCAAmC,CAAC;CACrE,IAAI,OAAO,WAAW,GAAG,OAAO;CAEhC,OAAO,OAAO,KAAK,MAAM,WAAW,EAAE,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI;AAC1E,EACF;AAEA,SAAgB,4BAA8C;CAC5D,MAAM,eAAe,QAAQ,IAAI,0BAA0B;CAE3D,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,cAAc;IACxB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;GACJ,IAAI;IACF,WAAW,MAAM,iBAAiB;GACpC,SAAS,KAAK;IACZ,QAAQ,KACN,0CAA0C,eAAe,QAAQ,IAAI,UAAU,IAAI,2CAA2C,UAChI;IACA,OAAO,aAAa,IAAI;GAC1B;GACA,OAAO,YAAY,UAAU,IAAI;EACnC;EAEA,MAAM,eAAe,CAKrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,OAAO,WADM,SAAS,IACD,GAAG,QAAQ;EAClC;CACF;AACF"}

package/dist/benchmarks/rag-shared.d.ts

@@ -1,5 +1,5 @@
 import { BenchScore, BenchTask, LoadOptions } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/rag-shared.d.ts
 declare const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
 interface RagContext {

package/dist/benchmarks/rag-shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"rag-shared.js","names":[],"sources":["../../src/benchmarks/rag-shared.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport type { BenchScore, BenchTask, LoadOptions } from './types'\n\nexport const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\nexport interface RagContext {\n  id: string\n  text: string\n  title?: string\n  source?: string\n  relevant?: boolean\n}\n\nexport interface RagAnswerScore {\n  resolved: boolean\n  score: number\n  finalAnswer: string\n  bestGold: string | null\n  exact: boolean\n  numeric: boolean\n  f1: number\n  threshold: number\n}\n\nexport const ragAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const trimmed = (lines[i] ?? '').trim()\n    if (trimmed.length > 0) return trimmed\n  }\n  return ''\n}\n\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const match of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(match[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\nexport function normalizeAnswer(input: string): string {\n  return input\n    .toLowerCase()\n    .replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n    .replace(/[^\\p{L}\\p{N}\\s.-]+/gu, ' ')\n    .split(/\\s+/)\n    .filter((token) => token.length > 0)\n    .filter((token) => !['a', 'an', 'the'].includes(token))\n    .join(' ')\n    .trim()\n}\n\nfunction tokens(input: string): string[] {\n  const normalized = normalizeAnswer(input)\n  return normalized.length === 0 ? [] : normalized.split(/\\s+/)\n}\n\nexport function tokenF1(candidate: string, gold: string): number {\n  const candidateTokens = tokens(candidate)\n  const goldTokens = tokens(gold)\n  if (candidateTokens.length === 0 || goldTokens.length === 0) {\n    return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const counts = new Map<string, number>()\n  for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1)\n  let common = 0\n  for (const token of candidateTokens) {\n    const left = counts.get(token)\n    if (left !== undefined && left > 0) {\n      common += 1\n      counts.set(token, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / candidateTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\nfunction exactOrContains(candidate: string, gold: string): boolean {\n  const normalizedCandidate = normalizeAnswer(candidate)\n  const normalizedGold = normalizeAnswer(gold)\n  if (normalizedGold.length === 0) return false\n  if (normalizedCandidate === normalizedGold) return true\n  const escaped = normalizedGold.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n  return new RegExp(`(^|\\\\s)${escaped}(\\\\s|$)`).test(normalizedCandidate)\n}\n\nfunction numbers(input: string): number[] {\n  return [...input.replace(/,/g, '').matchAll(/-?\\d+(?:\\.\\d+)?/g)]\n    .map((match) => Number(match[0]))\n    .filter(Number.isFinite)\n}\n\nfunction numericMatch(candidate: string, gold: string, relativeTolerance: number): boolean {\n  const got = numbers(candidate)\n  const want = numbers(gold)\n  if (got.length === 0 || want.length === 0) return false\n  return want.some((expected) =>\n    got.some((actual) => {\n      const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance)\n      return Math.abs(actual - expected) <= tolerance\n    }),\n  )\n}\n\nexport function scoreAnswerArtifact(\n  artifact: string,\n  golds: readonly string[],\n  options: { threshold?: number; numericTolerance?: number } = {},\n): RagAnswerScore {\n  const finalAnswer = parseFinalAnswer(artifact)\n  const threshold = options.threshold ?? 0.72\n  const numericTolerance = options.numericTolerance ?? 0.01\n  let best: RagAnswerScore = {\n    resolved: false,\n    score: 0,\n    finalAnswer,\n    bestGold: null,\n    exact: false,\n    numeric: false,\n    f1: 0,\n    threshold,\n  }\n  if (finalAnswer.length === 0 || golds.length === 0) return best\n\n  for (const gold of golds) {\n    const exact = exactOrContains(finalAnswer, gold)\n    const numeric = numericMatch(finalAnswer, gold, numericTolerance)\n    const f1 = tokenF1(finalAnswer, gold)\n    const resolved = exact || numeric || f1 >= threshold\n    const score = exact || numeric ? 1 : f1\n    if (score > best.score || (resolved && !best.resolved)) {\n      best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold }\n    }\n  }\n  return best\n}\n\nexport function answerScoreToBenchScore(\n  score: RagAnswerScore,\n  detail: Record<string, unknown>,\n): BenchScore {\n  return {\n    resolved: score.resolved,\n    score: score.score,\n    detail: JSON.stringify({\n      ...detail,\n      finalAnswer: score.finalAnswer,\n      bestGold: score.bestGold,\n      exact: score.exact,\n      numeric: score.numeric,\n      f1: score.f1,\n      threshold: score.threshold,\n    }),\n  }\n}\n\nexport async function readJsonRows(path: string): Promise<unknown[]> {\n  const raw = (await readFile(path, 'utf8')).trim()\n  if (raw.length === 0) return []\n  if (raw.startsWith('[')) {\n    const parsed = JSON.parse(raw) as unknown\n    if (Array.isArray(parsed)) return parsed\n    throw new Error(`${path} must contain a JSON array when it starts with [`)\n  }\n  if (raw.startsWith('{')) {\n    try {\n      const parsed = JSON.parse(raw) as unknown\n      if (isObject(parsed)) {\n        for (const key of ['rows', 'data', 'examples', 'items']) {\n          const value = parsed[key]\n          if (Array.isArray(value)) return value\n        }\n        return [parsed]\n      }\n      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`)\n    } catch (err) {\n      if (!raw.includes('\\n')) throw err\n    }\n  }\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nexport function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[] {\n  let selected = tasks\n  if (opts.split) selected = selected.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const ids = new Set(opts.ids)\n    selected = selected.filter((task) => ids.has(task.id))\n  } else if (opts.limit !== undefined) {\n    selected = selected.slice(0, opts.limit)\n  }\n  if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`)\n  return selected\n}\n\nexport function stringFrom(value: unknown): string | undefined {\n  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined\n}\n\nexport function stringArrayFrom(value: unknown): string[] {\n  if (typeof value === 'string' && value.trim().length > 0) return [value.trim()]\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry) => {\n    if (typeof entry === 'string' && entry.trim().length > 0) return [entry.trim()]\n    if (isObject(entry)) {\n      const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text)\n      return text ? [text] : []\n    }\n    return []\n  })\n}\n\nexport function firstString(row: Record<string, unknown>, keys: readonly string[]): string {\n  for (const key of keys) {\n    const value = stringFrom(row[key])\n    if (value) return value\n  }\n  return ''\n}\n\nexport function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[] {\n  const out: string[] = []\n  for (const key of keys) out.push(...stringArrayFrom(row[key]))\n  return [...new Set(out)]\n}\n\nexport function contextsFrom(value: unknown): RagContext[] {\n  if (typeof value === 'string' && value.trim().length > 0) {\n    return [{ id: 'ctx-1', text: value.trim() }]\n  }\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry, index): RagContext[] => {\n    if (typeof entry === 'string' && entry.trim().length > 0) {\n      return [{ id: `ctx-${index + 1}`, text: entry.trim() }]\n    }\n    if (Array.isArray(entry)) {\n      const [first, second] = entry\n      if (typeof first === 'string' && typeof second === 'string') {\n        return [{ id: first, text: second.trim() }]\n      }\n      return entry.flatMap((nested, nestedIndex) =>\n        contextsFrom([nested]).map((ctx) => ({\n          ...ctx,\n          id: ctx.id.startsWith('ctx-') ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id,\n        })),\n      )\n    }\n    if (!isObject(entry)) return []\n    const text =\n      stringFrom(entry.text) ??\n      stringFrom(entry.context) ??\n      stringFrom(entry.content) ??\n      stringFrom(entry.passage) ??\n      stringFrom(entry.document) ??\n      stringFrom(entry.page_content) ??\n      stringFrom(entry.chunk)\n    if (!text) return []\n    const id =\n      stringFrom(entry.id) ??\n      stringFrom(entry.docid) ??\n      stringFrom(entry.doc_id) ??\n      stringFrom(entry.document_id) ??\n      `ctx-${index + 1}`\n    const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance\n    const relevant =\n      typeof relevantRaw === 'boolean'\n        ? relevantRaw\n        : typeof relevantRaw === 'number'\n          ? relevantRaw > 0\n          : undefined\n    return [\n      {\n        id,\n        text,\n        ...(stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {}),\n        ...(stringFrom(entry.source) ?? stringFrom(entry.url)\n          ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) }\n          : {}),\n        ...(relevant !== undefined ? { relevant } : {}),\n      },\n    ]\n  })\n}\n\nexport function contextBlock(contexts: readonly RagContext[]): string {\n  if (contexts.length === 0) return ''\n  return contexts\n    .map((ctx, index) =>\n      [\n        `[${index + 1}] ${ctx.title ?? ctx.id}`,\n        ctx.source ? `Source: ${ctx.source}` : undefined,\n        ctx.text,\n      ]\n        .filter(Boolean)\n        .join('\\n'),\n    )\n    .join('\\n\\n')\n}\n\nexport function isObject(value: unknown): value is Record<string, unknown> {\n  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)\n}\n"],"mappings":";;AAIA,MAAa,wBAAwB;AAqBrC,MAAa,kBAAyC,EACpD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,WAAW,MAAM,MAAM,GAAA,CAAI,KAAK;EACtC,IAAI,QAAQ,SAAS,GAAG,OAAO;CACjC;CACA,OAAO;AACT;AAEA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,SAAS,SAAS,SAAS,yBAAyB,GAC7D,KAAK,IAAI,MAAM,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAE1C,OAAO,CAAC,GAAG,IAAI;AACjB;AAEA,SAAgB,gBAAgB,OAAuB;CACrD,OAAO,MACJ,YAAY,CAAC,CACb,QAAQ,qBAAqB,IAAI,CAAC,CAClC,QAAQ,wBAAwB,GAAG,CAAC,CACpC,MAAM,KAAK,CAAC,CACZ,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,QAAQ,UAAU,CAAC;EAAC;EAAK;EAAM;CAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CACtD,KAAK,GAAG,CAAC,CACT,KAAK;AACV;AAEA,SAAS,OAAO,OAAyB;CACvC,MAAM,aAAa,gBAAgB,KAAK;CACxC,OAAO,WAAW,WAAW,IAAI,CAAC,IAAI,WAAW,MAAM,KAAK;AAC9D;AAEA,SAAgB,QAAQ,WAAmB,MAAsB;CAC/D,MAAM,kBAAkB,OAAO,SAAS;CACxC,MAAM,aAAa,OAAO,IAAI;CAC9B,IAAI,gBAAgB,WAAW,KAAK,WAAW,WAAW,GACxD,OAAO,gBAAgB,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;CAEvE,MAAM,yBAAS,IAAI,IAAoB;CACvC,KAAK,MAAM,SAAS,YAAY,OAAO,IAAI,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CAC9E,IAAI,SAAS;CACb,KAAK,MAAM,SAAS,iBAAiB;EACnC,MAAM,OAAO,OAAO,IAAI,KAAK;EAC7B,IAAI,SAAS,KAAA,KAAa,OAAO,GAAG;GAClC,UAAU;GACV,OAAO,IAAI,OAAO,OAAO,CAAC;EAC5B;CACF;CACA,IAAI,WAAW,GAAG,OAAO;CACzB,MAAM,YAAY,SAAS,gBAAgB;CAC3C,MAAM,SAAS,SAAS,WAAW;CACnC,OAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;AAEA,SAAS,gBAAgB,WAAmB,MAAuB;CACjE,MAAM,sBAAsB,gBAAgB,SAAS;CACrD,MAAM,iBAAiB,gBAAgB,IAAI;CAC3C,IAAI,eAAe,WAAW,GAAG,OAAO;CACxC,IAAI,wBAAwB,gBAAgB,OAAO;CACnD,MAAM,UAAU,eAAe,QAAQ,uBAAuB,MAAM;CACpE,OAAO,IAAI,OAAO,UAAU,QAAQ,QAAQ,CAAC,CAAC,KAAK,mBAAmB;AACxE;AAEA,SAAS,QAAQ,OAAyB;CACxC,OAAO,CAAC,GAAG,MAAM,QAAQ,MAAM,EAAE,CAAC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAC7D,KAAK,UAAU,OAAO,MAAM,EAAE,CAAC,CAAC,CAChC,OAAO,OAAO,QAAQ;AAC3B;AAEA,SAAS,aAAa,WAAmB,MAAc,mBAAoC;CACzF,MAAM,MAAM,QAAQ,SAAS;CAC7B,MAAM,OAAO,QAAQ,IAAI;CACzB,IAAI,IAAI,WAAW,KAAK,KAAK,WAAW,GAAG,OAAO;CAClD,OAAO,KAAK,MAAM,aAChB,IAAI,MAAM,WAAW;EACnB,MAAM,YAAY,KAAK,IAAI,KAAK,IAAI,QAAQ,IAAI,mBAAmB,iBAAiB;EACpF,OAAO,KAAK,IAAI,SAAS,QAAQ,KAAK;CACxC,CAAC,CACH;AACF;AAEA,SAAgB,oBACd,UACA,OACA,UAA6D,CAAC,GAC9C;CAChB,MAAM,cAAc,iBAAiB,QAAQ;CAC7C,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,mBAAmB,QAAQ,oBAAoB;CACrD,IAAI,OAAuB;EACzB,UAAU;EACV,OAAO;EACP;EACA,UAAU;EACV,OAAO;EACP,SAAS;EACT,IAAI;EACJ;CACF;CACA,IAAI,YAAY,WAAW,KAAK,MAAM,WAAW,GAAG,OAAO;CAE3D,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,QAAQ,gBAAgB,aAAa,IAAI;EAC/C,MAAM,UAAU,aAAa,aAAa,MAAM,gBAAgB;EAChE,MAAM,KAAK,QAAQ,aAAa,IAAI;EACpC,MAAM,WAAW,SAAS,WAAW,MAAM;EAC3C,MAAM,QAAQ,SAAS,UAAU,IAAI;EACrC,IAAI,QAAQ,KAAK,SAAU,YAAY,CAAC,KAAK,UAC3C,OAAO;GAAE;GAAU;GAAO;GAAa,UAAU;GAAM;GAAO;GAAS;GAAI;EAAU;CAEzF;CACA,OAAO;AACT;AAEA,SAAgB,wBACd,OACA,QACY;CACZ,OAAO;EACL,UAAU,MAAM;EAChB,OAAO,MAAM;EACb,QAAQ,KAAK,UAAU;GACrB,GAAG;GACH,aAAa,MAAM;GACnB,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,IAAI,MAAM;GACV,WAAW,MAAM;EACnB,CAAC;CACH;AACF;AAEA,eAAsB,aAAa,MAAkC;CACnE,MAAM,OAAO,MAAM,SAAS,MAAM,MAAM,EAAA,CAAG,KAAK;CAChD,IAAI,IAAI,WAAW,GAAG,OAAO,CAAC;CAC9B,IAAI,IAAI,WAAW,GAAG,GAAG;EACvB,MAAM,SAAS,KAAK,MAAM,GAAG;EAC7B,IAAI,MAAM,QAAQ,MAAM,GAAG,OAAO;EAClC,MAAM,IAAI,MAAM,GAAG,KAAK,iDAAiD;CAC3E;CACA,IAAI,IAAI,WAAW,GAAG,GACpB,IAAI;EACF,MAAM,SAAS,KAAK,MAAM,GAAG;EAC7B,IAAI,SAAS,MAAM,GAAG;GACpB,KAAK,MAAM,OAAO;IAAC;IAAQ;IAAQ;IAAY;GAAO,GAAG;IACvD,MAAM,QAAQ,OAAO;IACrB,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO;GACnC;GACA,OAAO,CAAC,MAAM;EAChB;EACA,MAAM,IAAI,MAAM,GAAG,KAAK,gGAAgG;CAC1H,SAAS,KAAK;EACZ,IAAI,CAAC,IAAI,SAAS,IAAI,GAAG,MAAM;CACjC;CAEF,OAAO,IACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAgB,YAAY,OAAoB,MAAmB,OAA4B;CAC7F,IAAI,WAAW;CACf,IAAI,KAAK,OAAO,WAAW,SAAS,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CAC9E,IAAI,KAAK,KAAK;EACZ,MAAM,MAAM,IAAI,IAAI,KAAK,GAAG;EAC5B,WAAW,SAAS,QAAQ,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;CACvD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,WAAW,SAAS,MAAM,GAAG,KAAK,KAAK;CAEzC,IAAI,SAAS,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,MAAM,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAC/F,OAAO;AACT;AAEA,SAAgB,WAAW,OAAoC;CAC7D,OAAO,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,IAAI,MAAM,KAAK,IAAI,KAAA;AAC/E;AAEA,SAAgB,gBAAgB,OAA0B;CACxD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC;CAC9E,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,SAAS,UAAU;EAC9B,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC;EAC9E,IAAI,SAAS,KAAK,GAAG;GACnB,MAAM,OAAO,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,MAAM,IAAI;GACzF,OAAO,OAAO,CAAC,IAAI,IAAI,CAAC;EAC1B;EACA,OAAO,CAAC;CACV,CAAC;AACH;AAEA,SAAgB,YAAY,KAA8B,MAAiC;CACzF,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,WAAW,IAAI,IAAI;EACjC,IAAI,OAAO,OAAO;CACpB;CACA,OAAO;AACT;AAEA,SAAgB,WAAW,KAA8B,MAAmC;CAC1F,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC;CAC7D,OAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAEA,SAAgB,aAAa,OAA8B;CACzD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GACrD,OAAO,CAAC;EAAE,IAAI;EAAS,MAAM,MAAM,KAAK;CAAE,CAAC;CAE7C,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,SAAS,OAAO,UAAwB;EACnD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GACrD,OAAO,CAAC;GAAE,IAAI,OAAO,QAAQ;GAAK,MAAM,MAAM,KAAK;EAAE,CAAC;EAExD,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxB,MAAM,CAAC,OAAO,UAAU;GACxB,IAAI,OAAO,UAAU,YAAY,OAAO,WAAW,UACjD,OAAO,CAAC;IAAE,IAAI;IAAO,MAAM,OAAO,KAAK;GAAE,CAAC;GAE5C,OAAO,MAAM,SAAS,QAAQ,gBAC5B,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,SAAS;IACnC,GAAG;IACH,IAAI,IAAI,GAAG,WAAW,MAAM,IAAI,OAAO,QAAQ,EAAE,GAAG,cAAc,MAAM,IAAI;GAC9E,EAAE,CACJ;EACF;EACA,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO,CAAC;EAC9B,MAAM,OACJ,WAAW,MAAM,IAAI,KACrB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,QAAQ,KACzB,WAAW,MAAM,YAAY,KAC7B,WAAW,MAAM,KAAK;EACxB,IAAI,CAAC,MAAM,OAAO,CAAC;EACnB,MAAM,KACJ,WAAW,MAAM,EAAE,KACnB,WAAW,MAAM,KAAK,KACtB,WAAW,MAAM,MAAM,KACvB,WAAW,MAAM,WAAW,KAC5B,OAAO,QAAQ;EACjB,MAAM,cAAc,MAAM,YAAY,MAAM,eAAe,MAAM,SAAS,MAAM;EAChF,MAAM,WACJ,OAAO,gBAAgB,YACnB,cACA,OAAO,gBAAgB,WACrB,cAAc,IACd,KAAA;EACR,OAAO,CACL;GACE;GACA;GACA,GAAI,WAAW,MAAM,KAAK,IAAI,EAAE,OAAO,WAAW,MAAM,KAAK,EAAE,IAAI,CAAC;GACpE,GAAI,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,IAChD,EAAE,QAAQ,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,EAAE,IAC5D,CAAC;GACL,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;EAC/C,CACF;CACF,CAAC;AACH;AAEA,SAAgB,aAAa,UAAyC;CACpE,IAAI,SAAS,WAAW,GAAG,OAAO;CAClC,OAAO,SACJ,KAAK,KAAK,UACT;EACE,IAAI,QAAQ,EAAE,IAAI,IAAI,SAAS,IAAI;EACnC,IAAI,SAAS,WAAW,IAAI,WAAW,KAAA;EACvC,IAAI;CACN,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI,CACd,CAAC,CACA,KAAK,MAAM;AAChB;AAEA,SAAgB,SAAS,OAAkD;CACzE,OAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK;AAC5E"}
+{"version":3,"file":"rag-shared.js","names":[],"sources":["../../src/benchmarks/rag-shared.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport type { BenchScore, BenchTask, LoadOptions } from './types'\n\nexport const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\nexport interface RagContext {\n  id: string\n  text: string\n  title?: string\n  source?: string\n  relevant?: boolean\n}\n\nexport interface RagAnswerScore {\n  resolved: boolean\n  score: number\n  finalAnswer: string\n  bestGold: string | null\n  exact: boolean\n  numeric: boolean\n  f1: number\n  threshold: number\n}\n\nexport const ragAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const trimmed = (lines[i] ?? '').trim()\n    if (trimmed.length > 0) return trimmed\n  }\n  return ''\n}\n\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const match of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(match[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\nexport function normalizeAnswer(input: string): string {\n  return input\n    .toLowerCase()\n    .replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n    .replace(/[^\\p{L}\\p{N}\\s.-]+/gu, ' ')\n    .split(/\\s+/)\n    .filter((token) => token.length > 0)\n    .filter((token) => !['a', 'an', 'the'].includes(token))\n    .join(' ')\n    .trim()\n}\n\nfunction tokens(input: string): string[] {\n  const normalized = normalizeAnswer(input)\n  return normalized.length === 0 ? [] : normalized.split(/\\s+/)\n}\n\nexport function tokenF1(candidate: string, gold: string): number {\n  const candidateTokens = tokens(candidate)\n  const goldTokens = tokens(gold)\n  if (candidateTokens.length === 0 || goldTokens.length === 0) {\n    return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const counts = new Map<string, number>()\n  for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1)\n  let common = 0\n  for (const token of candidateTokens) {\n    const left = counts.get(token)\n    if (left !== undefined && left > 0) {\n      common += 1\n      counts.set(token, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / candidateTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\nfunction exactOrContains(candidate: string, gold: string): boolean {\n  const normalizedCandidate = normalizeAnswer(candidate)\n  const normalizedGold = normalizeAnswer(gold)\n  if (normalizedGold.length === 0) return false\n  if (normalizedCandidate === normalizedGold) return true\n  const escaped = normalizedGold.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n  return new RegExp(`(^|\\\\s)${escaped}(\\\\s|$)`).test(normalizedCandidate)\n}\n\nfunction numbers(input: string): number[] {\n  return [...input.replace(/,/g, '').matchAll(/-?\\d+(?:\\.\\d+)?/g)]\n    .map((match) => Number(match[0]))\n    .filter(Number.isFinite)\n}\n\nfunction numericMatch(candidate: string, gold: string, relativeTolerance: number): boolean {\n  const got = numbers(candidate)\n  const want = numbers(gold)\n  if (got.length === 0 || want.length === 0) return false\n  return want.some((expected) =>\n    got.some((actual) => {\n      const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance)\n      return Math.abs(actual - expected) <= tolerance\n    }),\n  )\n}\n\nexport function scoreAnswerArtifact(\n  artifact: string,\n  golds: readonly string[],\n  options: { threshold?: number; numericTolerance?: number } = {},\n): RagAnswerScore {\n  const finalAnswer = parseFinalAnswer(artifact)\n  const threshold = options.threshold ?? 0.72\n  const numericTolerance = options.numericTolerance ?? 0.01\n  let best: RagAnswerScore = {\n    resolved: false,\n    score: 0,\n    finalAnswer,\n    bestGold: null,\n    exact: false,\n    numeric: false,\n    f1: 0,\n    threshold,\n  }\n  if (finalAnswer.length === 0 || golds.length === 0) return best\n\n  for (const gold of golds) {\n    const exact = exactOrContains(finalAnswer, gold)\n    const numeric = numericMatch(finalAnswer, gold, numericTolerance)\n    const f1 = tokenF1(finalAnswer, gold)\n    const resolved = exact || numeric || f1 >= threshold\n    const score = exact || numeric ? 1 : f1\n    if (score > best.score || (resolved && !best.resolved)) {\n      best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold }\n    }\n  }\n  return best\n}\n\nexport function answerScoreToBenchScore(\n  score: RagAnswerScore,\n  detail: Record<string, unknown>,\n): BenchScore {\n  return {\n    resolved: score.resolved,\n    score: score.score,\n    detail: JSON.stringify({\n      ...detail,\n      finalAnswer: score.finalAnswer,\n      bestGold: score.bestGold,\n      exact: score.exact,\n      numeric: score.numeric,\n      f1: score.f1,\n      threshold: score.threshold,\n    }),\n  }\n}\n\nexport async function readJsonRows(path: string): Promise<unknown[]> {\n  const raw = (await readFile(path, 'utf8')).trim()\n  if (raw.length === 0) return []\n  if (raw.startsWith('[')) {\n    const parsed = JSON.parse(raw) as unknown\n    if (Array.isArray(parsed)) return parsed\n    throw new Error(`${path} must contain a JSON array when it starts with [`)\n  }\n  if (raw.startsWith('{')) {\n    try {\n      const parsed = JSON.parse(raw) as unknown\n      if (isObject(parsed)) {\n        for (const key of ['rows', 'data', 'examples', 'items']) {\n          const value = parsed[key]\n          if (Array.isArray(value)) return value\n        }\n        return [parsed]\n      }\n      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`)\n    } catch (err) {\n      if (!raw.includes('\\n')) throw err\n    }\n  }\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nexport function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[] {\n  let selected = tasks\n  if (opts.split) selected = selected.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const ids = new Set(opts.ids)\n    selected = selected.filter((task) => ids.has(task.id))\n  } else if (opts.limit !== undefined) {\n    selected = selected.slice(0, opts.limit)\n  }\n  if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`)\n  return selected\n}\n\nexport function stringFrom(value: unknown): string | undefined {\n  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined\n}\n\nexport function stringArrayFrom(value: unknown): string[] {\n  if (typeof value === 'string' && value.trim().length > 0) return [value.trim()]\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry) => {\n    if (typeof entry === 'string' && entry.trim().length > 0) return [entry.trim()]\n    if (isObject(entry)) {\n      const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text)\n      return text ? [text] : []\n    }\n    return []\n  })\n}\n\nexport function firstString(row: Record<string, unknown>, keys: readonly string[]): string {\n  for (const key of keys) {\n    const value = stringFrom(row[key])\n    if (value) return value\n  }\n  return ''\n}\n\nexport function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[] {\n  const out: string[] = []\n  for (const key of keys) out.push(...stringArrayFrom(row[key]))\n  return [...new Set(out)]\n}\n\nexport function contextsFrom(value: unknown): RagContext[] {\n  if (typeof value === 'string' && value.trim().length > 0) {\n    return [{ id: 'ctx-1', text: value.trim() }]\n  }\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry, index): RagContext[] => {\n    if (typeof entry === 'string' && entry.trim().length > 0) {\n      return [{ id: `ctx-${index + 1}`, text: entry.trim() }]\n    }\n    if (Array.isArray(entry)) {\n      const [first, second] = entry\n      if (typeof first === 'string' && typeof second === 'string') {\n        return [{ id: first, text: second.trim() }]\n      }\n      return entry.flatMap((nested, nestedIndex) =>\n        contextsFrom([nested]).map((ctx) => ({\n          ...ctx,\n          id: ctx.id.startsWith('ctx-') ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id,\n        })),\n      )\n    }\n    if (!isObject(entry)) return []\n    const text =\n      stringFrom(entry.text) ??\n      stringFrom(entry.context) ??\n      stringFrom(entry.content) ??\n      stringFrom(entry.passage) ??\n      stringFrom(entry.document) ??\n      stringFrom(entry.page_content) ??\n      stringFrom(entry.chunk)\n    if (!text) return []\n    const id =\n      stringFrom(entry.id) ??\n      stringFrom(entry.docid) ??\n      stringFrom(entry.doc_id) ??\n      stringFrom(entry.document_id) ??\n      `ctx-${index + 1}`\n    const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance\n    const relevant =\n      typeof relevantRaw === 'boolean'\n        ? relevantRaw\n        : typeof relevantRaw === 'number'\n          ? relevantRaw > 0\n          : undefined\n    return [\n      {\n        id,\n        text,\n        ...(stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {}),\n        ...(stringFrom(entry.source) ?? stringFrom(entry.url)\n          ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) }\n          : {}),\n        ...(relevant !== undefined ? { relevant } : {}),\n      },\n    ]\n  })\n}\n\nexport function contextBlock(contexts: readonly RagContext[]): string {\n  if (contexts.length === 0) return ''\n  return contexts\n    .map((ctx, index) =>\n      [\n        `[${index + 1}] ${ctx.title ?? ctx.id}`,\n        ctx.source ? `Source: ${ctx.source}` : undefined,\n        ctx.text,\n      ]\n        .filter(Boolean)\n        .join('\\n'),\n    )\n    .join('\\n\\n')\n}\n\nexport function isObject(value: unknown): value is Record<string, unknown> {\n  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)\n}\n"],"mappings":";;AAIA,MAAa,wBAAwB;AAqBrC,MAAa,kBAAyC,EACpD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,WAAW,MAAM,MAAM,GAAA,CAAI,KAAK;EACtC,IAAI,QAAQ,SAAS,GAAG,OAAO;CACjC;CACA,OAAO;AACT;AAEA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,SAAS,SAAS,SAAS,yBAAyB,GAC7D,KAAK,IAAI,MAAM,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAE1C,OAAO,CAAC,GAAG,IAAI;AACjB;AAEA,SAAgB,gBAAgB,OAAuB;CACrD,OAAO,MACJ,YAAY,CAAC,CACb,QAAQ,qBAAqB,IAAI,CAAC,CAClC,QAAQ,wBAAwB,GAAG,CAAC,CACpC,MAAM,KAAK,CAAC,CACZ,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,QAAQ,UAAU,CAAC;EAAC;EAAK;EAAM;CAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CACtD,KAAK,GAAG,CAAC,CACT,KAAK;AACV;AAEA,SAAS,OAAO,OAAyB;CACvC,MAAM,aAAa,gBAAgB,KAAK;CACxC,OAAO,WAAW,WAAW,IAAI,CAAC,IAAI,WAAW,MAAM,KAAK;AAC9D;AAEA,SAAgB,QAAQ,WAAmB,MAAsB;CAC/D,MAAM,kBAAkB,OAAO,SAAS;CACxC,MAAM,aAAa,OAAO,IAAI;CAC9B,IAAI,gBAAgB,WAAW,KAAK,WAAW,WAAW,GACxD,OAAO,gBAAgB,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;CAEvE,MAAM,yBAAS,IAAI,IAAoB;CACvC,KAAK,MAAM,SAAS,YAAY,OAAO,IAAI,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CAC9E,IAAI,SAAS;CACb,KAAK,MAAM,SAAS,iBAAiB;EACnC,MAAM,OAAO,OAAO,IAAI,KAAK;EAC7B,IAAI,SAAS,KAAA,KAAa,OAAO,GAAG;GAClC,UAAU;GACV,OAAO,IAAI,OAAO,OAAO,CAAC;EAC5B;CACF;CACA,IAAI,WAAW,GAAG,OAAO;CACzB,MAAM,YAAY,SAAS,gBAAgB;CAC3C,MAAM,SAAS,SAAS,WAAW;CACnC,OAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;AAEA,SAAS,gBAAgB,WAAmB,MAAuB;CACjE,MAAM,sBAAsB,gBAAgB,SAAS;CACrD,MAAM,iBAAiB,gBAAgB,IAAI;CAC3C,IAAI,eAAe,WAAW,GAAG,OAAO;CACxC,IAAI,wBAAwB,gBAAgB,OAAO;CACnD,MAAM,UAAU,eAAe,QAAQ,uBAAuB,MAAM;CACpE,OAAO,IAAI,OAAO,UAAU,QAAQ,QAAQ,CAAC,CAAC,KAAK,mBAAmB;AACxE;AAEA,SAAS,QAAQ,OAAyB;CACxC,OAAO,CAAC,GAAG,MAAM,QAAQ,MAAM,EAAE,CAAC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAC7D,KAAK,UAAU,OAAO,MAAM,EAAE,CAAC,CAAC,CAChC,OAAO,OAAO,QAAQ;AAC3B;AAEA,SAAS,aAAa,WAAmB,MAAc,mBAAoC;CACzF,MAAM,MAAM,QAAQ,SAAS;CAC7B,MAAM,OAAO,QAAQ,IAAI;CACzB,IAAI,IAAI,WAAW,KAAK,KAAK,WAAW,GAAG,OAAO;CAClD,OAAO,KAAK,MAAM,aAChB,IAAI,MAAM,WAAW;EACnB,MAAM,YAAY,KAAK,IAAI,KAAK,IAAI,QAAQ,IAAI,mBAAmB,iBAAiB;EACpF,OAAO,KAAK,IAAI,SAAS,QAAQ,KAAK;CACxC,CAAC,CACH;AACF;AAEA,SAAgB,oBACd,UACA,OACA,UAA6D,CAAC,GAC9C;CAChB,MAAM,cAAc,iBAAiB,QAAQ;CAC7C,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,mBAAmB,QAAQ,oBAAoB;CACrD,IAAI,OAAuB;EACzB,UAAU;EACV,OAAO;EACP;EACA,UAAU;EACV,OAAO;EACP,SAAS;EACT,IAAI;EACJ;CACF;CACA,IAAI,YAAY,WAAW,KAAK,MAAM,WAAW,GAAG,OAAO;CAE3D,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,QAAQ,gBAAgB,aAAa,IAAI;EAC/C,MAAM,UAAU,aAAa,aAAa,MAAM,gBAAgB;EAChE,MAAM,KAAK,QAAQ,aAAa,IAAI;EACpC,MAAM,WAAW,SAAS,WAAW,MAAM;EAC3C,MAAM,QAAQ,SAAS,UAAU,IAAI;EACrC,IAAI,QAAQ,KAAK,SAAU,YAAY,CAAC,KAAK,UAC3C,OAAO;GAAE;GAAU;GAAO;GAAa,UAAU;GAAM;GAAO;GAAS;GAAI;EAAU;CAEzF;CACA,OAAO;AACT;AAEA,SAAgB,wBACd,OACA,QACY;CACZ,OAAO;EACL,UAAU,MAAM;EAChB,OAAO,MAAM;EACb,QAAQ,KAAK,UAAU;GACrB,GAAG;GACH,aAAa,MAAM;GACnB,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,IAAI,MAAM;GACV,WAAW,MAAM;EACnB,CAAC;CACH;AACF;AAEA,eAAsB,aAAa,MAAkC;CACnE,MAAM,OAAO,MAAM,SAAS,MAAM,MAAM,EAAA,CAAG,KAAK;CAChD,IAAI,IAAI,WAAW,GAAG,OAAO,CAAC;CAC9B,IAAI,IAAI,WAAW,GAAG,GAAG;EACvB,MAAM,SAAS,KAAK,MAAM,GAAG;EAC7B,IAAI,MAAM,QAAQ,MAAM,GAAG,OAAO;EAClC,MAAM,IAAI,MAAM,GAAG,KAAK,iDAAiD;CAC3E;CACA,IAAI,IAAI,WAAW,GAAG,GACpB,IAAI;EACF,MAAM,SAAS,KAAK,MAAM,GAAG;EAC7B,IAAI,SAAS,MAAM,GAAG;GACpB,KAAK,MAAM,OAAO;IAAC;IAAQ;IAAQ;IAAY;GAAO,GAAG;IACvD,MAAM,QAAQ,OAAO;IACrB,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO;GACnC;GACA,OAAO,CAAC,MAAM;EAChB;EACA,MAAM,IAAI,MAAM,GAAG,KAAK,gGAAgG;CAC1H,SAAS,KAAK;EACZ,IAAI,CAAC,IAAI,SAAS,IAAI,GAAG,MAAM;CACjC;CAEF,OAAO,IACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAgB,YAAY,OAAoB,MAAmB,OAA4B;CAC7F,IAAI,WAAW;CACf,IAAI,KAAK,OAAO,WAAW,SAAS,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CAC9E,IAAI,KAAK,KAAK;EACZ,MAAM,MAAM,IAAI,IAAI,KAAK,GAAG;EAC5B,WAAW,SAAS,QAAQ,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;CACvD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,WAAW,SAAS,MAAM,GAAG,KAAK,KAAK;CAEzC,IAAI,SAAS,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,MAAM,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAC/F,OAAO;AACT;AAEA,SAAgB,WAAW,OAAoC;CAC7D,OAAO,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,IAAI,MAAM,KAAK,IAAI,KAAA;AAC/E;AAEA,SAAgB,gBAAgB,OAA0B;CACxD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC;CAC9E,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,SAAS,UAAU;EAC9B,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC;EAC9E,IAAI,SAAS,KAAK,GAAG;GACnB,MAAM,OAAO,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,MAAM,IAAI;GACzF,OAAO,OAAO,CAAC,IAAI,IAAI,CAAC;EAC1B;EACA,OAAO,CAAC;CACV,CAAC;AACH;AAEA,SAAgB,YAAY,KAA8B,MAAiC;CACzF,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,WAAW,IAAI,IAAI;EACjC,IAAI,OAAO,OAAO;CACpB;CACA,OAAO;AACT;AAEA,SAAgB,WAAW,KAA8B,MAAmC;CAC1F,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC;CAC7D,OAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAEA,SAAgB,aAAa,OAA8B;CACzD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GACrD,OAAO,CAAC;EAAE,IAAI;EAAS,MAAM,MAAM,KAAK;CAAE,CAAC;CAE7C,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,SAAS,OAAO,UAAwB;EACnD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GACrD,OAAO,CAAC;GAAE,IAAI,OAAO,QAAQ;GAAK,MAAM,MAAM,KAAK;EAAE,CAAC;EAExD,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxB,MAAM,CAAC,OAAO,UAAU;GACxB,IAAI,OAAO,UAAU,YAAY,OAAO,WAAW,UACjD,OAAO,CAAC;IAAE,IAAI;IAAO,MAAM,OAAO,KAAK;GAAE,CAAC;GAE5C,OAAO,MAAM,SAAS,QAAQ,gBAC5B,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,SAAS;IACnC,GAAG;IACH,IAAI,IAAI,GAAG,WAAW,MAAM,IAAI,OAAO,QAAQ,EAAE,GAAG,cAAc,MAAM,IAAI;GAC9E,EAAE,CACJ;EACF;EACA,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO,CAAC;EAC9B,MAAM,OACJ,WAAW,MAAM,IAAI,KACrB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,QAAQ,KACzB,WAAW,MAAM,YAAY,KAC7B,WAAW,MAAM,KAAK;EACxB,IAAI,CAAC,MAAM,OAAO,CAAC;EACnB,MAAM,KACJ,WAAW,MAAM,EAAE,KACnB,WAAW,MAAM,KAAK,KACtB,WAAW,MAAM,MAAM,KACvB,WAAW,MAAM,WAAW,KAC5B,OAAO,QAAQ;EACjB,MAAM,cAAc,MAAM,YAAY,MAAM,eAAe,MAAM,SAAS,MAAM;EAChF,MAAM,WACJ,OAAO,gBAAgB,YACnB,cACA,OAAO,gBAAgB,WACrB,cAAc,IACd,KAAA;EACR,OAAO,CACL;GACE;GACA;GACA,GAAI,WAAW,MAAM,KAAK,IAAI,EAAE,OAAO,WAAW,MAAM,KAAK,EAAE,IAAI,CAAC;GACpE,GAAI,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,IAChD,EAAE,QAAQ,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,EAAE,IAC5D,CAAC;GACL,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;EAC/C,CACF;CACF,CAAC;AACH;AAEA,SAAgB,aAAa,UAAyC;CACpE,IAAI,SAAS,WAAW,GAAG,OAAO;CAClC,OAAO,SACJ,KAAK,KAAK,UACT;EACE,IAAI,QAAQ,EAAE,IAAI,IAAI,SAAS,IAAI;EACnC,IAAI,SAAS,WAAW,IAAI,WAAW,KAAA;EACvC,IAAI;CACN,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI,CACd,CAAC,CACA,KAAK,MAAM;AAChB;AAEA,SAAgB,SAAS,OAAkD;CACzE,OAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK;AAC5E"}

package/dist/benchmarks/swe-bench.d.ts

@@ -1,6 +1,6 @@
 import { BenchScore, BenchmarkAdapter } from "./types.js";
 import { StagedRunCaptureSpec } from "./_harness.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/swe-bench.d.ts
 /**
  * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.

package/dist/benchmarks/swe-bench.js.map

@@ -1 +1 @@
-{"version":3,"file":"swe-bench.js","names":[],"sources":["../../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/**\n * Fixed in-box path the agent clones the instance repo into. It is the SINGLE\n * source of truth shared by the prompt template (which tells the agent to clone\n * here) and `boxExtract` (which runs `git diff` here after the shot) — so the\n * harness always knows exactly where the agent's edits live, for any instance.\n */\nconst SWE_REPO_DIR = '/work'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n    '--namespace', args.namespace ?? scorerNamespace(),\n    '--cache_level', args.cacheLevel,\n  ]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgv({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAiCA,MAAM,eAAe;;;;;;;;;;AAWrB,MAAa,iBAAwC,EACnD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAKA,QADa,CADG,GAAG,KAAK,SAAS,uCAAuC,CACtD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MACb,KAAA,CAAM,KAAK;AAC7B,EACF;AAEA,MAAM,UAAU;AAsBhB,MAAM,mCAAmB,IAAI,IAAwB;CAAC;CAAQ;CAAQ;CAAO;AAAU,CAAC;AAExF,SAAS,kBAAuC;CAC9C,MAAM,YAAY,QAAQ,IAAI,sBAAsB;CACpD,IAAI,cAAc,cAAc,cAAc,QAC5C,MAAM,IAAI,MAAM,kDAAkD,UAAU,EAAE;CAEhF,OAAO;AACT;AACA,MAAM,qBAAqB;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;CAClF,MAAM,QAAQ,OAAO;CACrB,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;CACjC,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAM,UAAU,OAAO,UAAU,QAAQ,GAC1E,MAAM,IAAI,MAAM,wBAAwB,KAAK;CAE/C,OAAO;AACT;;AAGA,SAAgB,eAAe,QAAgB,OAA4B;CACzE,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAC5D,MAAM,IAAI,MAAM,qCAAqC;CAEvD,MAAM,SAAS;CACf,MAAM,YAAY;EAChB,UAAU,UAAU,QAAQ,cAAc;EAC1C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,YAAY,UAAU,QAAQ,iBAAiB;EAC/C,WAAW,UAAU,QAAQ,eAAe;EAC5C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,OAAO,UAAU,QAAQ,WAAW;CACtC;CACA,MAAM,YAAY,UAAU,QAAQ,eAAe;CAEnD,IADkB,OAAO,OAAO,SAAS,CAAC,CAAC,KAEjC,CAAC,CAAC,MAAM,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,OAAO,SAExE,MAAM,IAAI,MAAM,2CAA2C,QAAQ;CAErE,IAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAC1E,MAAM,IAAI,MAAM,mCAAmC,QAAQ;CAE7D,MAAM,WAAW;EACf,UAAU,SAAS,SAAS,MAAM;EAClC,UAAU,WAAW,SAAS,MAAM;EACpC,UAAU,WAAW,SAAS,MAAM;CACtC;CACA,IAAI,SAAS,OAAO,OAAO,CAAC,CAAC,WAAW,GACtC,MAAM,IAAI,MAAM,+CAA+C,QAAQ;CAEzE,KAAK,SAAS,MAAM,SAAS,OAAO,CAAC,UAAU,UAAU,SAAS,MAAM,GACtE,MAAM,IAAI,MAAM,sDAAsD,QAAQ;CAEhF,MAAM,WAAW,SAAS;CAC1B,OAAO;EAAE;EAAU,OAAO,WAAW,IAAI;EAAG,QAAQ,KAAK,UAAU,MAAM;CAAE;AAC7E;AAEA,SAAgB,kBAAkB,MAMrB;CACX,OAAO;EACL;EAAM;EACN;EAAkB;EAClB;EAAsB,KAAK;EAC3B;EAAY,KAAK;EACjB;EAAkB,KAAK;EACvB;EAAiB;EACjB;EAAe,KAAK,aAAa,gBAAgB;EACjD;EAAiB,KAAK;CACxB;AACF;AAEA,SAAS,WAAW,OAAuB;CACzC,OAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,EAAE;AAC1C;AAEA,SAAS,YAAY,MAAiD;CACpE,MAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;CAC7C,MAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;CACpD,IAAI,CAAC,qCAAqC,KAAK,IAAI,GACjD,MAAM,IAAI,MAAM,wCAAwC,KAAK,GAAG,IAAI,MAAM;CAE5E,IAAI,CAAC,oBAAoB,KAAK,IAAI,GAChC,MAAM,IAAI,MAAM,+CAA+C,KAAK,GAAG,IAAI,MAAM;CAEnF,OAAO;EAAE;EAAM;CAAK;AACtB;AAEA,SAAgB,sBAAsB,UAAkC,CAAC,GAAqB;CAC5F,IACE,QAAQ,cAAc,KAAA,MAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,IACrE,MAAM,IAAI,MAAM,iDAAiD;CACnE,MAAM,aAAa,QAAQ,cAAc;CACzC,IAAI,CAAC,iBAAiB,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,+BAA+B;CACtF,IACE,QAAQ,8BAA8B,KAAA,KACnC,OAAO,QAAQ,8BAA8B,YAChD,MAAM,IAAI,MAAM,yDAAyD;CAC3E,IAAI,kBAAkB;CACtB,OAAO;EACL,MAAM;EACN,QAAQ;EAWR,SAAS,MAAM;GACb,MAAM,EAAE,MAAM,SAAS,YAAY,IAAI;GACvC,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,wBAAwB,WAAW,sBAAsB,MAAM,EAAE,GAAG,WAAW,YAAY,EAAE,aAAa,WAAW,YAAY,EAAE,oBAAoB,WAAW,IAAI,IACpN;EACF;EACA,aAAa;GACX,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,oBAAoB,WAAW,YAAY,EAAE,sBAAsB,qBACjH;EACF;EAEA,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAEJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,MAAM,QAAQ,KAAK,SAAS;GAqB5B,MAAM,SAAS,MAAM,cAAc;;;oBAhBrB,KAAK,UAAU,OAAO,EAAE,UAAU,KAAK,UAAU,KAAK,EAAE;;;;;;;;;;;;qCAYvC,MAAM;;;GAIM,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAErF,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KACT,OAAkB;IACjB,IAAI,OAAO,EAAE,WAAW;IACxB;IACA,QAAQ;KACN,eAAe,EAAE,KAAK,KAAK,EAAE;KAC7B;KACA,uCAAuC,aAAa,0BAA0B,EAAE,YAAY,8BAA8B,aAAa;KACvI;KACA;KACA;KACA;KACA;KACA,OAAO,EAAE,qBAAqB,EAAE;IAClC,CAAC,CAAC,KAAK,IAAI;IACX,UAAU;GACZ,EACF;EACF;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,KAAK,UAAU;GAC5B,OAAO,OAAO,SAAS,WAAW,OAAO,KAAA;EAC3C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;GACxC,MAAM,UAAU,QAAQ,4BAA4B;IAClD,QAAQ,KAAK;IACb;IACA,iBAAiB,EAAE;GACrB,CAAC;GACD,OAAO,eAAe;IACpB,WAAW;IACX,GAAI,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;IAC1E,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;IAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;IACzD,MAAM,MAAM,KAAK;KACf,MAAM,UACJ,KAAK,KAAK,YAAY,GACtB,KAAK,UAAU,CACb;MAAE,aAAa,KAAK;MAAI,oBAAoB;MAAuB,aAAa;KAAS,CAC3F,CAAC,CACH;IACF;IAGA,OAAO,QAAQ,kBAAkB;KAC/B,iBAAiB,KAAK,KAAK,YAAY;KACvC;KACA,YAAY,KAAK;KACjB;KACA,WAAW,gBAAgB;IAC7B,CAAC;IACD,MAAM,YAAY,KAAK;KAErB,MAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,MAAM,MAAM,CAAC;KAC7F,OAAO,eAAe,KAAK,IAAI,MAAM;IACvC;GACF,CAAC;EACH;CACF;AACF"}
+{"version":3,"file":"swe-bench.js","names":[],"sources":["../../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/**\n * Fixed in-box path the agent clones the instance repo into. It is the SINGLE\n * source of truth shared by the prompt template (which tells the agent to clone\n * here) and `boxExtract` (which runs `git diff` here after the shot) — so the\n * harness always knows exactly where the agent's edits live, for any instance.\n */\nconst SWE_REPO_DIR = '/work'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n    '--namespace', args.namespace ?? scorerNamespace(),\n    '--cache_level', args.cacheLevel,\n  ]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgv({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAiCA,MAAM,eAAe;;;;;;;;;;AAWrB,MAAa,iBAAwC,EACnD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAKA,QADa,CADG,GAAG,KAAK,SAAS,uCAAuC,CACtD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MACb,KAAA,CAAM,KAAK;AAC7B,EACF;AAEA,MAAM,UAAU;AAsBhB,MAAM,mCAAmB,IAAI,IAAwB;CAAC;CAAQ;CAAQ;CAAO;AAAU,CAAC;AAExF,SAAS,kBAAuC;CAC9C,MAAM,YAAY,QAAQ,IAAI,sBAAsB;CACpD,IAAI,cAAc,cAAc,cAAc,QAC5C,MAAM,IAAI,MAAM,kDAAkD,UAAU,EAAE;CAEhF,OAAO;AACT;AACA,MAAM,qBAAqB;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;CAClF,MAAM,QAAQ,OAAO;CACrB,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;CACjC,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAM,UAAU,OAAO,UAAU,QAAQ,GAC1E,MAAM,IAAI,MAAM,wBAAwB,KAAK;CAE/C,OAAO;AACT;;AAGA,SAAgB,eAAe,QAAgB,OAA4B;CACzE,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAC5D,MAAM,IAAI,MAAM,qCAAqC;CAEvD,MAAM,SAAS;CACf,MAAM,YAAY;EAChB,UAAU,UAAU,QAAQ,cAAc;EAC1C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,YAAY,UAAU,QAAQ,iBAAiB;EAC/C,WAAW,UAAU,QAAQ,eAAe;EAC5C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,OAAO,UAAU,QAAQ,WAAW;CACtC;CACA,MAAM,YAAY,UAAU,QAAQ,eAAe;CAEnD,IADkB,OAAO,OAAO,SAAS,CAAC,CAAC,KAEjC,CAAC,CAAC,MAAM,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,OAAO,SAExE,MAAM,IAAI,MAAM,2CAA2C,QAAQ;CAErE,IAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAC1E,MAAM,IAAI,MAAM,mCAAmC,QAAQ;CAE7D,MAAM,WAAW;EACf,UAAU,SAAS,SAAS,MAAM;EAClC,UAAU,WAAW,SAAS,MAAM;EACpC,UAAU,WAAW,SAAS,MAAM;CACtC;CACA,IAAI,SAAS,OAAO,OAAO,CAAC,CAAC,WAAW,GACtC,MAAM,IAAI,MAAM,+CAA+C,QAAQ;CAEzE,KAAK,SAAS,MAAM,SAAS,OAAO,CAAC,UAAU,UAAU,SAAS,MAAM,GACtE,MAAM,IAAI,MAAM,sDAAsD,QAAQ;CAEhF,MAAM,WAAW,SAAS;CAC1B,OAAO;EAAE;EAAU,OAAO,WAAW,IAAI;EAAG,QAAQ,KAAK,UAAU,MAAM;CAAE;AAC7E;AAEA,SAAgB,kBAAkB,MAMrB;CACX,OAAO;EACL;EAAM;EACN;EAAkB;EAClB;EAAsB,KAAK;EAC3B;EAAY,KAAK;EACjB;EAAkB,KAAK;EACvB;EAAiB;EACjB;EAAe,KAAK,aAAa,gBAAgB;EACjD;EAAiB,KAAK;CACxB;AACF;AAEA,SAAS,WAAW,OAAuB;CACzC,OAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,EAAE;AAC1C;AAEA,SAAS,YAAY,MAAiD;CACpE,MAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;CAC7C,MAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;CACpD,IAAI,CAAC,qCAAqC,KAAK,IAAI,GACjD,MAAM,IAAI,MAAM,wCAAwC,KAAK,GAAG,IAAI,MAAM;CAE5E,IAAI,CAAC,oBAAoB,KAAK,IAAI,GAChC,MAAM,IAAI,MAAM,+CAA+C,KAAK,GAAG,IAAI,MAAM;CAEnF,OAAO;EAAE;EAAM;CAAK;AACtB;AAEA,SAAgB,sBAAsB,UAAkC,CAAC,GAAqB;CAC5F,IACE,QAAQ,cAAc,KAAA,MAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,IACrE,MAAM,IAAI,MAAM,iDAAiD;CACnE,MAAM,aAAa,QAAQ,cAAc;CACzC,IAAI,CAAC,iBAAiB,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,+BAA+B;CACtF,IACE,QAAQ,8BAA8B,KAAA,KACnC,OAAO,QAAQ,8BAA8B,YAChD,MAAM,IAAI,MAAM,yDAAyD;CAC3E,IAAI,kBAAkB;CACtB,OAAO;EACL,MAAM;EACN,QAAQ;EAWR,SAAS,MAAM;GACb,MAAM,EAAE,MAAM,SAAS,YAAY,IAAI;GACvC,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,wBAAwB,WAAW,sBAAsB,MAAM,EAAE,GAAG,WAAW,YAAY,EAAE,aAAa,WAAW,YAAY,EAAE,oBAAoB,WAAW,IAAI,IACpN;EACF;EACA,aAAa;GACX,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,oBAAoB,WAAW,YAAY,EAAE,sBAAsB,qBACjH;EACF;EAEA,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAEJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,MAAM,QAAQ,KAAK,SAAS;GAqB5B,MAAM,SAAS,MAAM,cAAc;;;oBAhBrB,KAAK,UAAU,OAAO,EAAE,UAAU,KAAK,UAAU,KAAK,EAAE;;;;;;;;;;;;qCAYvC,MAAM;;;GAIM,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAErF,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KACT,OAAkB;IACjB,IAAI,OAAO,EAAE,WAAW;IACxB;IACA,QAAQ;KACN,eAAe,EAAE,KAAK,KAAK,EAAE;KAC7B;KACA,uCAAuC,aAAa,0BAA0B,EAAE,YAAY,8BAA8B,aAAa;KACvI;KACA;KACA;KACA;KACA;KACA,OAAO,EAAE,qBAAqB,EAAE;IAClC,CAAC,CAAC,KAAK,IAAI;IACX,UAAU;GACZ,EACF;EACF;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,KAAK,UAAU;GAC5B,OAAO,OAAO,SAAS,WAAW,OAAO,KAAA;EAC3C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;GACxC,MAAM,UAAU,QAAQ,4BAA4B;IAClD,QAAQ,KAAK;IACb;IACA,iBAAiB,EAAE;GACrB,CAAC;GACD,OAAO,eAAe;IACpB,WAAW;IACX,GAAI,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;IAC1E,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;IAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;IACzD,MAAM,MAAM,KAAK;KACf,MAAM,UACJ,KAAK,KAAK,YAAY,GACtB,KAAK,UAAU,CACb;MAAE,aAAa,KAAK;MAAI,oBAAoB;MAAuB,aAAa;KAAS,CAC3F,CAAC,CACH;IACF;IAGA,OAAO,QAAQ,kBAAkB;KAC/B,iBAAiB,KAAK,KAAK,YAAY;KACvC;KACA,YAAY,KAAK;KACjB;KACA,WAAW,gBAAgB;IAC7B,CAAC;IACD,MAAM,YAAY,KAAK;KAErB,MAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,MAAM,MAAM,CAAC;KAC7F,OAAO,eAAe,KAAK,IAAI,MAAM;IACvC;GACF,CAAC;EACH;CACF;AACF"}

package/dist/benchmarks/tau-bench-shared.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/tau-bench-shared.d.ts
 interface TauBenchConfig {
   name: string;

package/dist/benchmarks/tau-bench-shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"tau-bench-shared.js","names":[],"sources":["../../src/benchmarks/tau-bench-shared.ts"],"sourcesContent":["/**\n * Shared tau-bench adapter spine.\n *\n * tau2 and tau3 live in the same upstream repository/package namespace today:\n * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation\n * for task loading and reward recomputation so the domain/version adapters only\n * choose env names, default domain, and fixture file.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nexport interface TauBenchConfig {\n  name: string\n  fixturePath: string\n  fixturesEnv: string\n  dirEnv: string\n  domainEnv: string\n  defaultDomain: string\n  installHint: string\n  taskIntro: string\n}\n\ninterface TauRow {\n  id: string\n  domain: string\n  user_scenario?: unknown\n  description?: unknown\n  evaluation_criteria?: unknown\n}\n\ninterface TauMeta {\n  taskId: string\n  domain: string\n  split?: string\n  userScenario?: unknown\n  description?: unknown\n  evaluationCriteria?: unknown\n}\n\nexport const tauResultsOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path|json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertPath(path: string, label: string, benchName: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction benchDir(config: TauBenchConfig): string | undefined {\n  return process.env[config.dirEnv]\n}\n\nfunction benchDomain(config: TauBenchConfig): string {\n  return process.env[config.domainEnv] ?? config.defaultDomain\n}\n\nfunction rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {\n  const meta: TauMeta = {\n    taskId: row.id,\n    domain: row.domain,\n    split,\n    userScenario: row.user_scenario,\n    description: row.description,\n    evaluationCriteria: row.evaluation_criteria,\n  }\n  return {\n    id: row.id,\n    split,\n    prompt: [\n      config.taskIntro,\n      `Run this task in the official ${row.domain} domain.`,\n      'The benchmark is a simulated multi-turn user/tool conversation.',\n      '',\n      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),\n      '',\n      'Return the path to the official tau results.json or trajectory file containing this task run.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask, benchName: string): TauMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {\n    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as TauMeta\n}\n\nfunction selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, config, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]\n  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts, config, opts.split)\n}\n\nasync function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const domain = benchDomain(config)\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ndomain = sys.argv[2]\nsplit = sys.argv[3] or None\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.registry import registry\nloader = registry.get_tasks_loader(domain)\ntasks = loader(split)\nrows = []\nfor task in tasks:\n    row = task.model_dump(mode=\"json\")\n    row[\"domain\"] = domain\n    rows.append(row)\nprint(json.dumps(rows))\n`\n  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])\n  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)\n}\n\nasync function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\nartifact = Path(sys.argv[3])\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.data_model.simulation import Results\nfrom tau2.scripts.evaluate_trajectories import compute_simulation_rewards\nresults = Results.load(artifact)\nupdated = compute_simulation_rewards(results)\nscores = []\nfor sim in updated.simulations:\n    if sim.task_id == task_id and sim.reward_info is not None:\n        scores.append(float(sim.reward_info.reward))\nif not scores:\n    raise SystemExit(f\"no scored simulations for task_id={task_id} in {artifact}\")\nprint(json.dumps({\"count\": len(scores), \"score\": sum(scores) / len(scores), \"scores\": scores}))\n`\n  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {\n  const fixturesMode = process.env[config.fixturesEnv] === '1'\n\n  return {\n    name: config.name,\n    output: tauResultsOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = benchDir(config)\n      if (!dir) {\n        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)\n      }\n      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)\n      await loadOfficialTasks(config, dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(config, opts)\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)\n      return loadOfficialTasks(config, dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)\n      const meta = readMeta(task, config.name)\n      const artifactPath = resolve(artifact.trim())\n      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)\n      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA2CA,MAAa,mBAA0C,EACrD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,2CAA2C,CAC/D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,eAAe,WAAW,MAAc,OAAe,WAAkC;CACvF,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,GAAG,UAAU,YAAY,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC3G;AACF;AAEA,SAAS,SAAS,QAA4C;CAC5D,OAAO,QAAQ,IAAI,OAAO;AAC5B;AAEA,SAAS,YAAY,QAAgC;CACnD,OAAO,QAAQ,IAAI,OAAO,cAAc,OAAO;AACjD;AAEA,SAAS,UAAU,KAAa,QAAwB,OAA2B;CACjF,MAAM,OAAgB;EACpB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,cAAc,IAAI;EAClB,aAAa,IAAI;EACjB,oBAAoB,IAAI;CAC1B;CACA,OAAO;EACL,IAAI,IAAI;EACR;EACA,QAAQ;GACN,OAAO;GACP,iCAAiC,IAAI,OAAO;GAC5C;GACA;GACA,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAAgB,KAAK,UAAU,IAAI,iBAAiB,CAAC,GAAG,MAAM,CAAC;GAC3G;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiB,WAA4B;CAC7D,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,GAAG,WAAW,UAC/D,MAAM,IAAI,MAAM,GAAG,UAAU,QAAQ,KAAK,GAAG,kDAAkD;CAEjG,OAAO;AACT;AAEA,SAAS,WAAW,MAAgB,MAAmB,QAAwB,OAA6B;CAC1G,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,QAAQ,KAAK,CAAC;CAC3D,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,OAAO,KAAK,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAClG,OAAO;AACT;AAEA,eAAe,aAAa,QAAwB,MAAyC;CAC3F,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,OAAO,aAAa,MAAM,CAAC;CAClE,QAAQ,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,YAAY,eAAe,KAAK,OAAO,kBAAkB;CACjG,OAAO,WAAW,MAAM,MAAM,QAAQ,KAAK,KAAK;AAClD;AAEA,eAAe,kBAAkB,QAAwB,MAAc,MAAyC;CAmB9G,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;GAAQ;EAAC;EAlB7B,YAAY,MAkB4B;EAAG,KAAK,SAAS;CAAE,CAAC;CAC3E,OAAO,WAAW,KAAK,MAAM,MAAM,GAAe,MAAM,QAAQ,KAAK,KAAK;AAC5E;AAEA,eAAe,wBAAwB,MAAc,MAAe,cAAwD;CAoB1H,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAM,KAAK;EAAQ;CAAY,GAAG,CAAC;CAC/E,OAAO,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEA,SAAgB,sBAAsB,QAA0C;CAC9E,MAAM,eAAe,QAAQ,IAAI,OAAO,iBAAiB;CAEzD,OAAO;EACL,MAAM,OAAO;EACb,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,qBAAqB,OAAO,aAAa;GAE5E,MAAM,WAAW,GAAG,IAAI,wBAAwB,gBAAgB,OAAO,IAAI;GAC3E,MAAM,kBAAkB,QAAQ,KAAK,EAAE,OAAO,EAAE,CAAC;EACnD;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,QAAQ,IAAI;GAClD,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,gCAAgC,OAAO,KAAK,OAAO;GAC9F,OAAO,kBAAkB,QAAQ,KAAK,IAAI;EAC5C;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,wBAAwB,OAAO,KAAK,sBAAsB;GACrG,MAAM,OAAO,SAAS,MAAM,OAAO,IAAI;GACvC,MAAM,eAAe,QAAQ,SAAS,KAAK,CAAC;GAC5C,MAAM,WAAW,cAAc,mCAAmC,OAAO,IAAI;GAC7E,MAAM,SAAS,MAAM,wBAAwB,KAAK,MAAM,YAAY;GACpE,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,UAAU;IACpB;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,QAAQ,KAAK;KAAQ,OAAO,OAAO;IAAM,CAAC;GAC1F;EACF;CACF;AACF"}
+{"version":3,"file":"tau-bench-shared.js","names":[],"sources":["../../src/benchmarks/tau-bench-shared.ts"],"sourcesContent":["/**\n * Shared tau-bench adapter spine.\n *\n * tau2 and tau3 live in the same upstream repository/package namespace today:\n * `sierra-research/tau2-bench`, Python package `tau2`. Keep one implementation\n * for task loading and reward recomputation so the domain/version adapters only\n * choose env names, default domain, and fixture file.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nexport interface TauBenchConfig {\n  name: string\n  fixturePath: string\n  fixturesEnv: string\n  dirEnv: string\n  domainEnv: string\n  defaultDomain: string\n  installHint: string\n  taskIntro: string\n}\n\ninterface TauRow {\n  id: string\n  domain: string\n  user_scenario?: unknown\n  description?: unknown\n  evaluation_criteria?: unknown\n}\n\ninterface TauMeta {\n  taskId: string\n  domain: string\n  split?: string\n  userScenario?: unknown\n  description?: unknown\n  evaluationCriteria?: unknown\n}\n\nexport const tauResultsOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path|json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertPath(path: string, label: string, benchName: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`${benchName}: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction benchDir(config: TauBenchConfig): string | undefined {\n  return process.env[config.dirEnv]\n}\n\nfunction benchDomain(config: TauBenchConfig): string {\n  return process.env[config.domainEnv] ?? config.defaultDomain\n}\n\nfunction rowToTask(row: TauRow, config: TauBenchConfig, split?: string): BenchTask {\n  const meta: TauMeta = {\n    taskId: row.id,\n    domain: row.domain,\n    split,\n    userScenario: row.user_scenario,\n    description: row.description,\n    evaluationCriteria: row.evaluation_criteria,\n  }\n  return {\n    id: row.id,\n    split,\n    prompt: [\n      config.taskIntro,\n      `Run this task in the official ${row.domain} domain.`,\n      'The benchmark is a simulated multi-turn user/tool conversation.',\n      '',\n      typeof row.user_scenario === 'string' ? row.user_scenario : JSON.stringify(row.user_scenario ?? {}, null, 2),\n      '',\n      'Return the path to the official tau results.json or trajectory file containing this task run.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask, benchName: string): TauMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string' || typeof md.domain !== 'string') {\n    throw new Error(`${benchName} task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as TauMeta\n}\n\nfunction selectRows(rows: TauRow[], opts: LoadOptions, config: TauBenchConfig, split?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, config, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`${config.name}: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(config: TauBenchConfig, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(config.fixturePath, 'utf8')) as TauRow[]\n  console.warn(`[${config.name}] ${config.fixturesEnv}=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts, config, opts.split)\n}\n\nasync function loadOfficialTasks(config: TauBenchConfig, root: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const domain = benchDomain(config)\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ndomain = sys.argv[2]\nsplit = sys.argv[3] or None\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.registry import registry\nloader = registry.get_tasks_loader(domain)\ntasks = loader(split)\nrows = []\nfor task in tasks:\n    row = task.model_dump(mode=\"json\")\n    row[\"domain\"] = domain\n    rows.append(row)\nprint(json.dumps(rows))\n`\n  const stdout = await runVenvPython(script, [root, domain, opts.split ?? ''])\n  return selectRows(JSON.parse(stdout) as TauRow[], opts, config, opts.split)\n}\n\nasync function scoreOfficialTrajectory(root: string, meta: TauMeta, artifactPath: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, sys\nfrom pathlib import Path\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\nartifact = Path(sys.argv[3])\nsys.path.insert(0, str(root / \"src\"))\nfrom tau2.data_model.simulation import Results\nfrom tau2.scripts.evaluate_trajectories import compute_simulation_rewards\nresults = Results.load(artifact)\nupdated = compute_simulation_rewards(results)\nscores = []\nfor sim in updated.simulations:\n    if sim.task_id == task_id and sim.reward_info is not None:\n        scores.append(float(sim.reward_info.reward))\nif not scores:\n    raise SystemExit(f\"no scored simulations for task_id={task_id} in {artifact}\")\nprint(json.dumps({\"count\": len(scores), \"score\": sum(scores) / len(scores), \"scores\": scores}))\n`\n  const stdout = await runVenvPython(script, [root, meta.taskId, artifactPath], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createTauBenchAdapter(config: TauBenchConfig): BenchmarkAdapter {\n  const fixturesMode = process.env[config.fixturesEnv] === '1'\n\n  return {\n    name: config.name,\n    output: tauResultsOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = benchDir(config)\n      if (!dir) {\n        throw new Error(`${config.dirEnv} is required. Fix: ${config.installHint}`)\n      }\n      await assertPath(`${dir}/src/tau2/registry.py`, 'tau registry', config.name)\n      await loadOfficialTasks(config, dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(config, opts)\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to load official ${config.name} tasks`)\n      return loadOfficialTasks(config, dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = benchDir(config)\n      if (!dir) throw new Error(`${config.dirEnv} is required to judge ${config.name} trajectory artifacts`)\n      const meta = readMeta(task, config.name)\n      const artifactPath = resolve(artifact.trim())\n      await assertPath(artifactPath, 'tau results/trajectory artifact', config.name)\n      const report = await scoreOfficialTrajectory(dir, meta, artifactPath)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, count: report.count }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AA2CA,MAAa,mBAA0C,EACrD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,2CAA2C,CAC/D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,eAAe,WAAW,MAAc,OAAe,WAAkC;CACvF,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,GAAG,UAAU,YAAY,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC3G;AACF;AAEA,SAAS,SAAS,QAA4C;CAC5D,OAAO,QAAQ,IAAI,OAAO;AAC5B;AAEA,SAAS,YAAY,QAAgC;CACnD,OAAO,QAAQ,IAAI,OAAO,cAAc,OAAO;AACjD;AAEA,SAAS,UAAU,KAAa,QAAwB,OAA2B;CACjF,MAAM,OAAgB;EACpB,QAAQ,IAAI;EACZ,QAAQ,IAAI;EACZ;EACA,cAAc,IAAI;EAClB,aAAa,IAAI;EACjB,oBAAoB,IAAI;CAC1B;CACA,OAAO;EACL,IAAI,IAAI;EACR;EACA,QAAQ;GACN,OAAO;GACP,iCAAiC,IAAI,OAAO;GAC5C;GACA;GACA,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAAgB,KAAK,UAAU,IAAI,iBAAiB,CAAC,GAAG,MAAM,CAAC;GAC3G;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiB,WAA4B;CAC7D,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,GAAG,WAAW,UAC/D,MAAM,IAAI,MAAM,GAAG,UAAU,QAAQ,KAAK,GAAG,kDAAkD;CAEjG,OAAO;AACT;AAEA,SAAS,WAAW,MAAgB,MAAmB,QAAwB,OAA6B;CAC1G,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,QAAQ,KAAK,CAAC;CAC3D,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,OAAO,KAAK,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAClG,OAAO;AACT;AAEA,eAAe,aAAa,QAAwB,MAAyC;CAC3F,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,OAAO,aAAa,MAAM,CAAC;CAClE,QAAQ,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,YAAY,eAAe,KAAK,OAAO,kBAAkB;CACjG,OAAO,WAAW,MAAM,MAAM,QAAQ,KAAK,KAAK;AAClD;AAEA,eAAe,kBAAkB,QAAwB,MAAc,MAAyC;CAmB9G,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;GAAQ;EAAC;EAlB7B,YAAY,MAkB4B;EAAG,KAAK,SAAS;CAAE,CAAC;CAC3E,OAAO,WAAW,KAAK,MAAM,MAAM,GAAe,MAAM,QAAQ,KAAK,KAAK;AAC5E;AAEA,eAAe,wBAAwB,MAAc,MAAe,cAAwD;CAoB1H,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAM,KAAK;EAAQ;CAAY,GAAG,CAAC;CAC/E,OAAO,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEA,SAAgB,sBAAsB,QAA0C;CAC9E,MAAM,eAAe,QAAQ,IAAI,OAAO,iBAAiB;CAEzD,OAAO;EACL,MAAM,OAAO;EACb,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,qBAAqB,OAAO,aAAa;GAE5E,MAAM,WAAW,GAAG,IAAI,wBAAwB,gBAAgB,OAAO,IAAI;GAC3E,MAAM,kBAAkB,QAAQ,KAAK,EAAE,OAAO,EAAE,CAAC;EACnD;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,QAAQ,IAAI;GAClD,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,gCAAgC,OAAO,KAAK,OAAO;GAC9F,OAAO,kBAAkB,QAAQ,KAAK,IAAI;EAC5C;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,MAAM,SAAS,MAAM;GAC3B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,GAAG,OAAO,OAAO,wBAAwB,OAAO,KAAK,sBAAsB;GACrG,MAAM,OAAO,SAAS,MAAM,OAAO,IAAI;GACvC,MAAM,eAAe,QAAQ,SAAS,KAAK,CAAC;GAC5C,MAAM,WAAW,cAAc,mCAAmC,OAAO,IAAI;GAC7E,MAAM,SAAS,MAAM,wBAAwB,KAAK,MAAM,YAAY;GACpE,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,OAAO;IACL,UAAU,UAAU;IACpB;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,QAAQ,KAAK;KAAQ,OAAO,OAAO;IAAM,CAAC;GAC1F;EACF;CACF;AACF"}

package/dist/benchmarks/tau2-bench.d.ts

@@ -1,6 +1,6 @@
 import { BenchmarkAdapter } from "./types.js";
 //#region src/benchmarks/tau2-bench.d.ts
-declare const tau2ResultsOutput: import("@tangle-network/agent-runtime/loops").OutputAdapter<string>;
+declare const tau2ResultsOutput: import("@tangle-network/agent-runtime/kernel").OutputAdapter<string>;
 declare function createTau2BenchAdapter(): BenchmarkAdapter;
 //#endregion
 export { createTau2BenchAdapter, tau2ResultsOutput };

package/dist/benchmarks/toollm.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/toollm.d.ts
 declare const toollmOutput: OutputAdapter<string>;
 declare function createToolLlmAdapter(): BenchmarkAdapter;

package/dist/benchmarks/toollm.js.map

@@ -1 +1 @@
-{"version":3,"file":"toollm.js","names":[],"sources":["../../src/benchmarks/toollm.ts"],"sourcesContent":["/**\n * ToolLLM/ToolBench adapter.\n *\n * ToolBench task loading is useful for breadth, but the official ToolEval pass\n * rate evaluator is LLM-based and stochastic. This adapter therefore scores\n * only ToolBench's deterministic API-selection labels (`relevant APIs`). It\n * never records a full ToolEval pass-rate score.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')\nconst DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')\n\ninterface ToolApi {\n  category_name?: string\n  tool_name: string\n  api_name: string\n  api_description?: string\n  required_parameters?: unknown[]\n  optional_parameters?: unknown[]\n  method?: string\n}\n\ninterface ToolBenchRow {\n  query_id: number | string\n  query: string\n  api_list?: ToolApi[]\n  'relevant APIs'?: Array<[string, string]>\n}\n\ninterface ToolBenchMeta {\n  queryId: string\n  apiList: ToolApi[]\n  relevantApis: Array<[string, string]>\n  deterministicJudge: 'api-selection'\n}\n\nconst toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR\nconst queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)\n\nexport const toollmOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nfunction rowToTask(row: ToolBenchRow): BenchTask {\n  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])\n  const meta: ToolBenchMeta = {\n    queryId: String(row.query_id),\n    apiList: row.api_list ?? [],\n    relevantApis,\n    deterministicJudge: 'api-selection',\n  }\n  return {\n    id: String(row.query_id),\n    prompt: [\n      'Solve this ToolLLM/ToolBench API-use task.',\n      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',\n      '',\n      `Query: ${row.query}`,\n      '',\n      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,\n      '',\n      'Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction normalizeApiPart(value: string): string {\n  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')\n}\n\nfunction apiKey(pair: readonly [string, string]): string {\n  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`\n}\n\nfunction normalizeApiPairs(value: unknown): Array<[string, string]> {\n  if (!Array.isArray(value)) return []\n  const out: Array<[string, string]> = []\n  for (const item of value) {\n    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {\n      out.push([item[0], item[1]])\n    } else if (\n      item && typeof item === 'object'\n      && typeof (item as { tool_name?: unknown }).tool_name === 'string'\n      && typeof (item as { api_name?: unknown }).api_name === 'string'\n    ) {\n      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])\n    }\n  }\n  return out\n}\n\nfunction readMeta(task: BenchTask): ToolBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.relevantApis)) {\n    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)\n  }\n  return md as unknown as ToolBenchMeta\n}\n\nfunction selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nfunction assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {\n  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a ToolBench query file that includes \"relevant APIs\" labels, or do not score ToolLLM in agent-bench.',\n    )\n  }\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]\n  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  const tasks = selectRows(rows, opts)\n  assertDeterministicSubset(tasks, FIXTURES)\n  return tasks\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const source = queryFile(dir)\n  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)\n  assertDeterministicSubset(tasks, source)\n  return tasks\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\ninterface ExtractedApis {\n  pairs: Array<[string, string]>\n  source: 'structured-json' | 'text-mention'\n}\n\nfunction extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {\n  const parsed = extractJsonBlock(text)\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    const fromApiCalls = normalizeApiPairs(raw.api_calls)\n    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }\n    const fromCalls = normalizeApiPairs(raw.calls)\n    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }\n  }\n\n  const lower = text.toLowerCase()\n  return {\n    pairs: expected.filter(([tool, api]) => {\n      const toolNeedle = normalizeApiPart(tool)\n      const apiNeedle = normalizeApiPart(api)\n      const compactText = lower.replace(/[^a-z0-9]+/g, '')\n      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))\n    }),\n    source: 'text-mention',\n  }\n}\n\nfunction scoreApiSelection(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  if (meta.relevantApis.length === 0) {\n    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)\n  }\n  const expected = new Set(meta.relevantApis.map(apiKey))\n  const extracted = extractCalledApis(artifact, meta.relevantApis)\n  const calledPairs = extracted.pairs\n  const called = new Set(calledPairs.map(apiKey))\n  const truePositives = [...called].filter((key) => expected.has(key)).length\n  const precision = called.size === 0 ? 0 : truePositives / called.size\n  const recall = truePositives / expected.size\n  const score = expected.size === 0 ? 0 : recall\n  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      scoring: 'api-selection-only',\n      extractionSource: extracted.source,\n      queryId: meta.queryId,\n      expected: meta.relevantApis,\n      called: calledPairs,\n      precision,\n      recall,\n      fullToolEvalScore: null,\n    }),\n  }\n}\n\nexport function createToolLlmAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'\n\n  return {\n    name: 'toollm',\n    output: toollmOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = toolbenchDir()\n      if (!dir) {\n        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')\n      }\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = toolbenchDir()\n      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      if (meta.relevantApis.length === 0) return undefined\n      return JSON.stringify({\n        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreApiSelection(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,aAAa;AAC1D,MAAM,oBAAoB,KAAK,gBAAgB,eAAe,eAAe;AA0B7E,MAAM,qBAAyC,QAAQ,IAAI;AAC3D,MAAM,aAAa,QAAwB,QAAQ,IAAI,qBAAqB,KAAK,KAAK,iBAAiB;AAEvG,MAAa,eAAsC,EACjD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,SAAS,UAAU,KAA8B;CAC/C,MAAM,eAAe,kBAAkB,IAAI,oBAAoB,CAAC,CAAC;CACjE,MAAM,OAAsB;EAC1B,SAAS,OAAO,IAAI,QAAQ;EAC5B,SAAS,IAAI,YAAY,CAAC;EAC1B;EACA,oBAAoB;CACtB;CACA,OAAO;EACL,IAAI,OAAO,IAAI,QAAQ;EACvB,QAAQ;GACN;GACA;GACA;GACA,UAAU,IAAI;GACd;GACA,mBAAmB,KAAK,UAAU,IAAI,YAAY,CAAC,GAAG,MAAM,CAAC;GAC7D;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,iBAAiB,OAAuB;CAC/C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,eAAe,EAAE;AACtD;AAEA,SAAS,OAAO,MAAyC;CACvD,OAAO,GAAG,iBAAiB,KAAK,EAAE,EAAE,GAAG,iBAAiB,KAAK,EAAE;AACjE;AAEA,SAAS,kBAAkB,OAAyC;CAClE,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,MAAM,MAA+B,CAAC;CACtC,KAAK,MAAM,QAAQ,OACjB,IAAI,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAK,OAAO,YAAY,OAAO,KAAK,OAAO,UAC3E,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;MACtB,IACL,QAAQ,OAAO,SAAS,YACrB,OAAQ,KAAiC,cAAc,YACvD,OAAQ,KAAgC,aAAa,UAExD,IAAI,KAAK,CAAE,KAA+B,WAAY,KAA8B,QAAQ,CAAC;CAGjG,OAAO;AACT;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,YAAY,GACvC,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kFAAkF;CAE5H,OAAO;AACT;AAEA,SAAS,WAAW,MAAsB,MAAgC;CACxE,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,GAAG;CAC3F,OAAO;AACT;AAEA,SAAS,0BAA0B,OAA6B,QAAsB;CACpF,MAAM,UAAU,MAAM,QAAQ,SAAS,SAAS,IAAI,CAAC,CAAC,aAAa,WAAW,CAAC,CAAC,CAAC,KAAK,SAAS,KAAK,EAAE;CACtG,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,0DAA0D,QAAQ,OAAO,GAAG,MAAM,OAAO,gBAAgB,OAAO,IAAI,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,2GAErJ;AAEJ;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,wCAAwC,KAAK,OAAO,kBAAkB;CACnF,MAAM,QAAQ,WAAW,MAAM,IAAI;CACnC,0BAA0B,OAAO,QAAQ;CACzC,OAAO;AACT;AAEA,eAAe,kBAAkB,KAAa,MAAyC;CACrF,MAAM,SAAS,UAAU,GAAG;CAC5B,MAAM,QAAQ,WAAW,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,CAAC,GAAqB,IAAI;CAC3F,0BAA0B,OAAO,MAAM;CACvC,OAAO;AACT;AAEA,SAAS,iBAAiB,MAAuB;CAE/C,MAAM,OAAO,CADG,GAAG,KAAK,SAAS,iCAAiC,CAChD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAC9C,IAAI;EACF,OAAO,KAAK,MAAM,GAAG;CACvB,QAAQ;EACN;CACF;AACF;AAOA,SAAS,kBAAkB,MAAc,UAAsD;CAC7F,MAAM,SAAS,iBAAiB,IAAI;CACpC,IAAI,UAAU,OAAO,WAAW,UAAU;EACxC,MAAM,MAAM;EACZ,MAAM,eAAe,kBAAkB,IAAI,SAAS;EACpD,IAAI,aAAa,SAAS,GAAG,OAAO;GAAE,OAAO;GAAc,QAAQ;EAAkB;EACrF,MAAM,YAAY,kBAAkB,IAAI,KAAK;EAC7C,IAAI,UAAU,SAAS,GAAG,OAAO;GAAE,OAAO;GAAW,QAAQ;EAAkB;CACjF;CAEA,MAAM,QAAQ,KAAK,YAAY;CAC/B,OAAO;EACL,OAAO,SAAS,QAAQ,CAAC,MAAM,SAAS;GACtC,MAAM,aAAa,iBAAiB,IAAI;GACxC,MAAM,YAAY,iBAAiB,GAAG;GAEtC,OADoB,MAAM,QAAQ,eAAe,EAChC,CAAC,CAAC,SAAS,GAAG,aAAa,WAAW,KAAM,MAAM,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC;EACrI,CAAC;EACD,QAAQ;CACV;AACF;AAEA,SAAS,kBAAkB,MAAiB,UAA8B;CACxE,MAAM,OAAO,SAAS,IAAI;CAC1B,IAAI,KAAK,aAAa,WAAW,GAC/B,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,8DAA8D;CAExG,MAAM,WAAW,IAAI,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC;CACtD,MAAM,YAAY,kBAAkB,UAAU,KAAK,YAAY;CAC/D,MAAM,cAAc,UAAU;CAC9B,MAAM,SAAS,IAAI,IAAI,YAAY,IAAI,MAAM,CAAC;CAC9C,MAAM,gBAAgB,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,QAAQ,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC;CACrE,MAAM,YAAY,OAAO,SAAS,IAAI,IAAI,gBAAgB,OAAO;CACjE,MAAM,SAAS,gBAAgB,SAAS;CACxC,MAAM,QAAQ,SAAS,SAAS,IAAI,IAAI;CAExC,OAAO;EACL,UAFe,UAAU,WAAW,qBAAqB,WAAW,KAAK,cAAc;EAGvF;EACA,QAAQ,KAAK,UAAU;GACrB,SAAS;GACT,kBAAkB,UAAU;GAC5B,SAAS,KAAK;GACd,UAAU,KAAK;GACf,QAAQ;GACR;GACA;GACA,mBAAmB;EACrB,CAAC;CACH;AACF;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,aAAa;GACzB,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,sHAAsH;GAExI,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;EAC3C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,MAAM,aAAa;GACzB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,iDAAiD;GAC3E,OAAO,kBAAkB,KAAK,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,KAAK,aAAa,WAAW,GAAG,OAAO,KAAA;GAC3C,OAAO,KAAK,UAAU,EACpB,WAAW,KAAK,aAAa,KAAK,CAAC,WAAW,eAAe;IAAE;IAAW;GAAS,EAAE,EACvF,GAAG,MAAM,CAAC;EACZ;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,OAAO,kBAAkB,MAAM,QAAQ;EACzC;CACF;AACF"}
+{"version":3,"file":"toollm.js","names":[],"sources":["../../src/benchmarks/toollm.ts"],"sourcesContent":["/**\n * ToolLLM/ToolBench adapter.\n *\n * ToolBench task loading is useful for breadth, but the official ToolEval pass\n * rate evaluator is LLM-based and stochastic. This adapter therefore scores\n * only ToolBench's deterministic API-selection labels (`relevant APIs`). It\n * never records a full ToolEval pass-rate score.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'toollm.json')\nconst DEFAULT_QUERY_REL = join('data_example', 'instruction', 'G1_query.json')\n\ninterface ToolApi {\n  category_name?: string\n  tool_name: string\n  api_name: string\n  api_description?: string\n  required_parameters?: unknown[]\n  optional_parameters?: unknown[]\n  method?: string\n}\n\ninterface ToolBenchRow {\n  query_id: number | string\n  query: string\n  api_list?: ToolApi[]\n  'relevant APIs'?: Array<[string, string]>\n}\n\ninterface ToolBenchMeta {\n  queryId: string\n  apiList: ToolApi[]\n  relevantApis: Array<[string, string]>\n  deterministicJudge: 'api-selection'\n}\n\nconst toolbenchDir = (): string | undefined => process.env.TOOLBENCH_DIR\nconst queryFile = (dir: string): string => process.env.TOOLLM_QUERY_FILE ?? join(dir, DEFAULT_QUERY_REL)\n\nexport const toollmOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nfunction rowToTask(row: ToolBenchRow): BenchTask {\n  const relevantApis = normalizeApiPairs(row['relevant APIs'] ?? [])\n  const meta: ToolBenchMeta = {\n    queryId: String(row.query_id),\n    apiList: row.api_list ?? [],\n    relevantApis,\n    deterministicJudge: 'api-selection',\n  }\n  return {\n    id: String(row.query_id),\n    prompt: [\n      'Solve this ToolLLM/ToolBench API-use task.',\n      'Use only the listed APIs/tools and return the completed tool-use trace plus final answer.',\n      '',\n      `Query: ${row.query}`,\n      '',\n      `Available APIs: ${JSON.stringify(row.api_list ?? [], null, 2)}`,\n      '',\n      'Return the APIs you used as JSON: {\"api_calls\":[{\"tool_name\":\"...\",\"api_name\":\"...\"}]}.',\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction normalizeApiPart(value: string): string {\n  return value.toLowerCase().replace(/[^a-z0-9]+/g, '')\n}\n\nfunction apiKey(pair: readonly [string, string]): string {\n  return `${normalizeApiPart(pair[0])}.${normalizeApiPart(pair[1])}`\n}\n\nfunction normalizeApiPairs(value: unknown): Array<[string, string]> {\n  if (!Array.isArray(value)) return []\n  const out: Array<[string, string]> = []\n  for (const item of value) {\n    if (Array.isArray(item) && typeof item[0] === 'string' && typeof item[1] === 'string') {\n      out.push([item[0], item[1]])\n    } else if (\n      item && typeof item === 'object'\n      && typeof (item as { tool_name?: unknown }).tool_name === 'string'\n      && typeof (item as { api_name?: unknown }).api_name === 'string'\n    ) {\n      out.push([(item as { tool_name: string }).tool_name, (item as { api_name: string }).api_name])\n    }\n  }\n  return out\n}\n\nfunction readMeta(task: BenchTask): ToolBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.relevantApis)) {\n    throw new Error(`ToolLLM task ${task.id} missing metadata — loadTasks did not populate deterministic API-selection labels`)\n  }\n  return md as unknown as ToolBenchMeta\n}\n\nfunction selectRows(rows: ToolBenchRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`ToolLLM: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nfunction assertDeterministicSubset(tasks: readonly BenchTask[], source: string): void {\n  const missing = tasks.filter((task) => readMeta(task).relevantApis.length === 0).map((task) => task.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `ToolLLM deterministic API-selection labels missing for ${missing.length}/${tasks.length} task(s) from ${source}: ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a ToolBench query file that includes \"relevant APIs\" labels, or do not score ToolLLM in agent-bench.',\n    )\n  }\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as ToolBenchRow[]\n  console.warn(`[toollm] TOOLLM_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  const tasks = selectRows(rows, opts)\n  assertDeterministicSubset(tasks, FIXTURES)\n  return tasks\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const source = queryFile(dir)\n  const tasks = selectRows(JSON.parse(await readFile(source, 'utf8')) as ToolBenchRow[], opts)\n  assertDeterministicSubset(tasks, source)\n  return tasks\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\ninterface ExtractedApis {\n  pairs: Array<[string, string]>\n  source: 'structured-json' | 'text-mention'\n}\n\nfunction extractCalledApis(text: string, expected: readonly [string, string][]): ExtractedApis {\n  const parsed = extractJsonBlock(text)\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    const fromApiCalls = normalizeApiPairs(raw.api_calls)\n    if (fromApiCalls.length > 0) return { pairs: fromApiCalls, source: 'structured-json' }\n    const fromCalls = normalizeApiPairs(raw.calls)\n    if (fromCalls.length > 0) return { pairs: fromCalls, source: 'structured-json' }\n  }\n\n  const lower = text.toLowerCase()\n  return {\n    pairs: expected.filter(([tool, api]) => {\n      const toolNeedle = normalizeApiPart(tool)\n      const apiNeedle = normalizeApiPart(api)\n      const compactText = lower.replace(/[^a-z0-9]+/g, '')\n      return compactText.includes(`${toolNeedle}${apiNeedle}`) || (lower.includes(tool.toLowerCase()) && lower.includes(api.toLowerCase()))\n    }),\n    source: 'text-mention',\n  }\n}\n\nfunction scoreApiSelection(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  if (meta.relevantApis.length === 0) {\n    throw new Error(`ToolLLM task ${task.id} has no deterministic API-selection labels; refusing to score`)\n  }\n  const expected = new Set(meta.relevantApis.map(apiKey))\n  const extracted = extractCalledApis(artifact, meta.relevantApis)\n  const calledPairs = extracted.pairs\n  const called = new Set(calledPairs.map(apiKey))\n  const truePositives = [...called].filter((key) => expected.has(key)).length\n  const precision = called.size === 0 ? 0 : truePositives / called.size\n  const recall = truePositives / expected.size\n  const score = expected.size === 0 ? 0 : recall\n  const resolved = extracted.source === 'structured-json' && recall === 1 && precision === 1\n  return {\n    resolved,\n    score,\n    detail: JSON.stringify({\n      scoring: 'api-selection-only',\n      extractionSource: extracted.source,\n      queryId: meta.queryId,\n      expected: meta.relevantApis,\n      called: calledPairs,\n      precision,\n      recall,\n      fullToolEvalScore: null,\n    }),\n  }\n}\n\nexport function createToolLlmAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.TOOLLM_FIXTURES === '1'\n\n  return {\n    name: 'toollm',\n    output: toollmOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = toolbenchDir()\n      if (!dir) {\n        throw new Error('TOOLBENCH_DIR is required. Fix: clone https://github.com/OpenBMB/ToolBench and set TOOLBENCH_DIR=/path/to/ToolBench.')\n      }\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = toolbenchDir()\n      if (!dir) throw new Error('TOOLBENCH_DIR is required to load ToolLLM tasks')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      if (meta.relevantApis.length === 0) return undefined\n      return JSON.stringify({\n        api_calls: meta.relevantApis.map(([tool_name, api_name]) => ({ tool_name, api_name })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreApiSelection(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,aAAa;AAC1D,MAAM,oBAAoB,KAAK,gBAAgB,eAAe,eAAe;AA0B7E,MAAM,qBAAyC,QAAQ,IAAI;AAC3D,MAAM,aAAa,QAAwB,QAAQ,IAAI,qBAAqB,KAAK,KAAK,iBAAiB;AAEvG,MAAa,eAAsC,EACjD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,SAAS,UAAU,KAA8B;CAC/C,MAAM,eAAe,kBAAkB,IAAI,oBAAoB,CAAC,CAAC;CACjE,MAAM,OAAsB;EAC1B,SAAS,OAAO,IAAI,QAAQ;EAC5B,SAAS,IAAI,YAAY,CAAC;EAC1B;EACA,oBAAoB;CACtB;CACA,OAAO;EACL,IAAI,OAAO,IAAI,QAAQ;EACvB,QAAQ;GACN;GACA;GACA;GACA,UAAU,IAAI;GACd;GACA,mBAAmB,KAAK,UAAU,IAAI,YAAY,CAAC,GAAG,MAAM,CAAC;GAC7D;GACA;EACF,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,iBAAiB,OAAuB;CAC/C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,eAAe,EAAE;AACtD;AAEA,SAAS,OAAO,MAAyC;CACvD,OAAO,GAAG,iBAAiB,KAAK,EAAE,EAAE,GAAG,iBAAiB,KAAK,EAAE;AACjE;AAEA,SAAS,kBAAkB,OAAyC;CAClE,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,MAAM,MAA+B,CAAC;CACtC,KAAK,MAAM,QAAQ,OACjB,IAAI,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAK,OAAO,YAAY,OAAO,KAAK,OAAO,UAC3E,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;MACtB,IACL,QAAQ,OAAO,SAAS,YACrB,OAAQ,KAAiC,cAAc,YACvD,OAAQ,KAAgC,aAAa,UAExD,IAAI,KAAK,CAAE,KAA+B,WAAY,KAA8B,QAAQ,CAAC;CAGjG,OAAO;AACT;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,YAAY,GACvC,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,kFAAkF;CAE5H,OAAO;AACT;AAEA,SAAS,WAAW,MAAsB,MAAgC;CACxE,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,GAAG;CAC3F,OAAO;AACT;AAEA,SAAS,0BAA0B,OAA6B,QAAsB;CACpF,MAAM,UAAU,MAAM,QAAQ,SAAS,SAAS,IAAI,CAAC,CAAC,aAAa,WAAW,CAAC,CAAC,CAAC,KAAK,SAAS,KAAK,EAAE;CACtG,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,0DAA0D,QAAQ,OAAO,GAAG,MAAM,OAAO,gBAAgB,OAAO,IAAI,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,2GAErJ;AAEJ;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,wCAAwC,KAAK,OAAO,kBAAkB;CACnF,MAAM,QAAQ,WAAW,MAAM,IAAI;CACnC,0BAA0B,OAAO,QAAQ;CACzC,OAAO;AACT;AAEA,eAAe,kBAAkB,KAAa,MAAyC;CACrF,MAAM,SAAS,UAAU,GAAG;CAC5B,MAAM,QAAQ,WAAW,KAAK,MAAM,MAAM,SAAS,QAAQ,MAAM,CAAC,GAAqB,IAAI;CAC3F,0BAA0B,OAAO,MAAM;CACvC,OAAO;AACT;AAEA,SAAS,iBAAiB,MAAuB;CAE/C,MAAM,OAAO,CADG,GAAG,KAAK,SAAS,iCAAiC,CAChD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;CAC9C,IAAI;EACF,OAAO,KAAK,MAAM,GAAG;CACvB,QAAQ;EACN;CACF;AACF;AAOA,SAAS,kBAAkB,MAAc,UAAsD;CAC7F,MAAM,SAAS,iBAAiB,IAAI;CACpC,IAAI,UAAU,OAAO,WAAW,UAAU;EACxC,MAAM,MAAM;EACZ,MAAM,eAAe,kBAAkB,IAAI,SAAS;EACpD,IAAI,aAAa,SAAS,GAAG,OAAO;GAAE,OAAO;GAAc,QAAQ;EAAkB;EACrF,MAAM,YAAY,kBAAkB,IAAI,KAAK;EAC7C,IAAI,UAAU,SAAS,GAAG,OAAO;GAAE,OAAO;GAAW,QAAQ;EAAkB;CACjF;CAEA,MAAM,QAAQ,KAAK,YAAY;CAC/B,OAAO;EACL,OAAO,SAAS,QAAQ,CAAC,MAAM,SAAS;GACtC,MAAM,aAAa,iBAAiB,IAAI;GACxC,MAAM,YAAY,iBAAiB,GAAG;GAEtC,OADoB,MAAM,QAAQ,eAAe,EAChC,CAAC,CAAC,SAAS,GAAG,aAAa,WAAW,KAAM,MAAM,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC;EACrI,CAAC;EACD,QAAQ;CACV;AACF;AAEA,SAAS,kBAAkB,MAAiB,UAA8B;CACxE,MAAM,OAAO,SAAS,IAAI;CAC1B,IAAI,KAAK,aAAa,WAAW,GAC/B,MAAM,IAAI,MAAM,gBAAgB,KAAK,GAAG,8DAA8D;CAExG,MAAM,WAAW,IAAI,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC;CACtD,MAAM,YAAY,kBAAkB,UAAU,KAAK,YAAY;CAC/D,MAAM,cAAc,UAAU;CAC9B,MAAM,SAAS,IAAI,IAAI,YAAY,IAAI,MAAM,CAAC;CAC9C,MAAM,gBAAgB,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,QAAQ,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC;CACrE,MAAM,YAAY,OAAO,SAAS,IAAI,IAAI,gBAAgB,OAAO;CACjE,MAAM,SAAS,gBAAgB,SAAS;CACxC,MAAM,QAAQ,SAAS,SAAS,IAAI,IAAI;CAExC,OAAO;EACL,UAFe,UAAU,WAAW,qBAAqB,WAAW,KAAK,cAAc;EAGvF;EACA,QAAQ,KAAK,UAAU;GACrB,SAAS;GACT,kBAAkB,UAAU;GAC5B,SAAS,KAAK;GACd,UAAU,KAAK;GACf,QAAQ;GACR;GACA;GACA,mBAAmB;EACrB,CAAC;CACH;AACF;AAEA,SAAgB,uBAAyC;CACvD,MAAM,eAAe,QAAQ,IAAI,oBAAoB;CAErD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,aAAa;GACzB,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,sHAAsH;GAExI,MAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;EAC3C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,MAAM,aAAa;GACzB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,iDAAiD;GAC3E,OAAO,kBAAkB,KAAK,IAAI;EACpC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,KAAK,aAAa,WAAW,GAAG,OAAO,KAAA;GAC3C,OAAO,KAAK,UAAU,EACpB,WAAW,KAAK,aAAa,KAAK,CAAC,WAAW,eAAe;IAAE;IAAW;GAAS,EAAE,EACvF,GAAG,MAAM,CAAC;EACZ;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,OAAO,kBAAkB,MAAM,QAAQ;EACzC;CACF;AACF"}

package/dist/benchmarks/types.d.ts

@@ -1,4 +1,4 @@
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/types.d.ts
 interface BenchTask {
   /** Stable benchmark instance id. */

package/dist/benchmarks/webarena-verified.d.ts

@@ -1,5 +1,5 @@
 import { BenchmarkAdapter } from "./types.js";
-import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/webarena-verified.d.ts
 declare const webarenaOutputDirOutput: OutputAdapter<string>;
 declare function createWebArenaVerifiedAdapter(): BenchmarkAdapter;