@tangle-network/agent-bench 0.4.1 → 0.4.4

package/dist/benchmarks/trata-hedge.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"trata-hedge.js","names":[],"sources":["../../src/benchmarks/trata-hedge.ts"],"sourcesContent":["/**\n * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial\n * analysis tasks across 6 domains (private equity, managed care, industrials,\n * vertical SaaS, REITs, insurance). Each task bundles real earnings-call\n * transcripts, financial statements, press releases, and SEC filings under\n * `environment/data/`; the agent must produce a grounded analysis citing those\n * files.\n *\n * Harbor architecture: the original benchmark runs each task in a Docker\n * container with file-read tools. This adapter skips Harbor and embeds the data\n * files directly in the worker prompt so any router backend can score it. Large\n * files (> 30 KB) are truncated to fit within model context limits. The judge\n * reads the actual files from disk for citation verification.\n *\n * Judge: a 3-stage LLM cascade faithful to grade.py —\n *   Task 1: hallucination check (cited-file context + agent answer)\n *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)\n *   Task 3: synthesis check (ground truth + agent answer)\n * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,\n * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;\n * resolved = score 4 (sparse reward).\n *\n * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.\n * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench\n */\n\nimport { readdirSync, readFileSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench'\nconst MAX_FILE_BYTES = 8_000\n// Sandbox API rejects messages > 100K chars; instruction + suffix adds ~500 chars overhead.\nconst MAX_DATA_CHARS = 85_000\nconst WORKER_SUFFIX = [\n  '',\n  'The data files above are your only source. Cite each claim with the filename that supports it.',\n  'Write your complete analysis to the text block that starts with \"ANALYSIS:\" on a line by itself.',\n  'Take a clear position on the topic. Every factual claim must name the file it comes from.',\n].join('\\n')\n\n// Judge model: Gemini 2.5 Pro via router (matches gemini-3.1-pro-preview tier used by grade.py).\nconst JUDGE_MODEL = 'gemini-2.5-pro'\n\ninterface TrataTaskMeta {\n  taskDir: string\n  taskId: string\n  dataDir: string\n  groundTruth: string\n  gradingTask1: string\n  gradingTask2: string\n  gradingTask3: string\n  /** [(label, n_moves)] in rubric order. */\n  themeMoveCounts: Array<{ label: string; nMoves: number }>\n}\n\n// ── file reading helpers ──────────────────────────────────────────────────────\n\nfunction readText(p: string, maxBytes = MAX_FILE_BYTES): string {\n  try {\n    const buf = readFileSync(p)\n    if (buf.length <= maxBytes) return buf.toString('utf8')\n    return buf.subarray(0, maxBytes).toString('utf8') + `\\n[… truncated at ${maxBytes} bytes …]`\n  } catch {\n    return `[unreadable: ${p}]`\n  }\n}\n\nfunction listFilesRec(dir: string): string[] {\n  const out: string[] = []\n  try {\n    for (const name of readdirSync(dir)) {\n      const full = join(dir, name)\n      try {\n        if (statSync(full).isDirectory()) out.push(...listFilesRec(full))\n        else out.push(full)\n      } catch {\n        // skip unreadable entries\n      }\n    }\n  } catch {\n    // dir absent\n  }\n  return out.sort()\n}\n\nfunction buildDataBlock(dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  if (files.length === 0) return '(no data files found)'\n  // Prioritise: company profiles → earnings calls → primary-ticker financials →\n  // sec filings → press releases → peers. Stop once total chars hit MAX_DATA_CHARS.\n  const priority = (f: string) => {\n    if (f.includes('company_profiles')) return 0\n    if (f.includes('earnings_call')) return 1\n    if (/financials\\/(income|cash_flow|balance)/.test(f) && !/bam_|bx_|kkr_|ares_|cg_|owl_|fsk_|cat_|de_|tdg_|car_/.test(f)) return 2\n    if (f.includes('sec_filings')) return 3\n    if (f.includes('investor_pres')) return 4\n    if (f.includes('press_release')) return 5\n    return 6\n  }\n  const sorted = [...files].sort((a, b) => priority(a) - priority(b) || a.localeCompare(b))\n  const blocks: string[] = []\n  let total = 0\n  for (const f of sorted) {\n    const rel = f.slice(dataDir.length + 1)\n    const content = readText(f)\n    const chunk = `\\n=== FILE: data/${rel} ===\\n${content}`\n    if (total + chunk.length > MAX_DATA_CHARS) {\n      blocks.push(`\\n[… ${sorted.length - blocks.length} more files omitted to stay within context limit …]`)\n      break\n    }\n    blocks.push(chunk)\n    total += chunk.length\n  }\n  return blocks.join('')\n}\n\nfunction buildCitedDataBlock(answerText: string, dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  const cited: string[] = []\n  for (const f of files) {\n    const rel = f.slice(dataDir.length + 1)\n    const name = rel.split('/').pop() ?? ''\n    if (answerText.includes(rel) || answerText.includes(name)) cited.push(f)\n  }\n  if (cited.length === 0) {\n    return '(The agent did not cite any data files, or no cited file could be located.)'\n  }\n  const blocks: string[] = []\n  for (const f of cited) {\n    const rel = f.slice(dataDir.length + 1)\n    blocks.push(`\\n=== FILE: data/${rel} ===\\n${readText(f)}`)\n  }\n  return blocks.join('')\n}\n\n// ── ground-truth parser ───────────────────────────────────────────────────────\n\nfunction parseThemeMoveCounts(groundTruth: string): Array<{ label: string; nMoves: number }> {\n  const lines = groundTruth.split('\\n')\n  const themeStarts = lines.reduce<number[]>((acc, l, i) => {\n    if (/^\\d+\\./.test(l.trim())) acc.push(i)\n    return acc\n  }, [])\n  return themeStarts.map((start, k) => {\n    const end = k + 1 < themeStarts.length ? themeStarts[k + 1] : lines.length\n    const block = lines.slice(start, end).join('\\n')\n    const label = lines[start].replace(/^\\d+\\.\\s*/, '').trim()\n    const nMoves = (block.match(/^\\s*\\[[a-z]\\]\\s+/gm) ?? []).length\n    return { label, nMoves }\n  })\n}\n\nfunction coverageThreshold(nMoves: number): number {\n  return Math.max(1, Math.min(nMoves - 1, 3))\n}\n\n// ── router judge calls ────────────────────────────────────────────────────────\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the Trata hedge-bench judge')\n  const model = process.env.JUDGE_MODEL ?? JUDGE_MODEL\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\nfunction parseJsonFallback(raw: string): unknown {\n  let s = raw.trim()\n  if (s.startsWith('```')) s = s.split('\\n').slice(1).join('\\n')\n  if (s.endsWith('```')) s = s.slice(0, s.lastIndexOf('```'))\n  s = s.replace(/,(\\s*[}\\]])/g, '$1').trim()\n  try {\n    return JSON.parse(s)\n  } catch {\n    try {\n      const m = s.match(/\\{[\\s\\S]*\\}/)\n      if (m) return JSON.parse(m[0])\n    } catch {\n      // fallthrough\n    }\n  }\n  return null\n}\n\nasync function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise<unknown> {\n  for (let i = 0; i < maxAttempts; i++) {\n    const res = await fetch(`${router.baseUrl}/chat/completions`, {\n      method: 'POST',\n      headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n      body: JSON.stringify({\n        model: router.model,\n        temperature: 0,\n        max_tokens: 16384,\n        messages: [{ role: 'user', content: prompt }],\n      }),\n    })\n    if (!res.ok) {\n      if (i < maxAttempts - 1) continue\n      throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n    }\n    const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n    const content = body.choices?.[0]?.message?.content\n    if (typeof content !== 'string') continue\n    const parsed = parseJsonFallback(content)\n    if (parsed !== null) return parsed\n  }\n  return null\n}\n\ninterface Task1Result {\n  hallucinations_detected?: boolean | string\n  unverifiable_claims?: Array<{ claim: string; cited_file?: string; issue?: string; evidence_check?: string }>\n}\n\ninterface Task2ThemeResult {\n  label?: string\n  moves_hit?: string[]\n  moves_missed?: string[]\n  moves_tainted?: string[]\n  move_reasoning?: Record<string, string>\n}\n\ninterface Task2Result {\n  themes?: Task2ThemeResult[]\n}\n\ninterface Task3Result {\n  synthesis_found?: string | null\n}\n\nasync function task1HallucinationCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task1Result> {\n  const citedBlock = buildCitedDataBlock(answer, meta.dataDir)\n  const dataContext =\n    'The block below contains the contents of the data files the agent cited in their answer. ' +\n    'Treat this as the ground truth for factual verification.\\n\\n' +\n    `<cited_data>\\n${citedBlock}\\n</cited_data>`\n  const prompt = meta.gradingTask1\n    .replace('{data_context}', dataContext)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task1Result) ?? { hallucinations_detected: false, unverifiable_claims: [] }\n}\n\nasync function task2PerMoveCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  flaggedClaims: Task1Result['unverifiable_claims'],\n  router: JudgeRouter,\n): Promise<Task2Result> {\n  const claimsBlock =\n    flaggedClaims && flaggedClaims.length > 0\n      ? JSON.stringify(flaggedClaims, null, 2)\n      : '(No claims were flagged as hallucinations.)'\n  const prompt = meta.gradingTask2\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n    .replace('{flagged_claims_block}', claimsBlock)\n    .replace('{num_themes}', String(meta.themeMoveCounts.length))\n  const result = await callJudge(router, prompt)\n  return (result as Task2Result) ?? { themes: [] }\n}\n\nasync function task3SynthesisCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task3Result> {\n  const prompt = meta.gradingTask3\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task3Result) ?? { synthesis_found: null }\n}\n\nasync function runTriataJudge(meta: TrataTaskMeta, answer: string, router: JudgeRouter): Promise<BenchScore> {\n  if (!answer.trim()) {\n    return { resolved: false, score: 0, detail: JSON.stringify({ error: 'empty answer' }) }\n  }\n\n  // Stage 1: hallucination check (serial — feeds into stage 2).\n  const t1 = await task1HallucinationCheck(meta, answer, router)\n  const flaggedClaims = t1.unverifiable_claims ?? []\n\n  // Stages 2 + 3 in parallel.\n  const [t2, t3] = await Promise.all([\n    task2PerMoveCheck(meta, answer, flaggedClaims, router),\n    task3SynthesisCheck(meta, answer, router),\n  ])\n\n  const judgeThemes = t2.themes ?? []\n  const themesHit: string[] = []\n  const themesMissed: string[] = []\n\n  for (let i = 0; i < meta.themeMoveCounts.length; i++) {\n    const { label, nMoves } = meta.themeMoveCounts[i]\n    const judge = judgeThemes[i] ?? {}\n    const movesHit = (judge.moves_hit ?? []).map((m: string) => m.replace(/[\\[\\]]/g, ''))\n    const tainted = new Set((judge.moves_tainted ?? []).map((m: string) => m.replace(/[\\[\\]]/g, '')))\n    const validHit = movesHit.filter((m: string) => !tainted.has(m))\n    const threshold = coverageThreshold(nMoves)\n    if (validHit.length >= threshold) themesHit.push(label)\n    else themesMissed.push(label)\n  }\n\n  const numThemes = meta.themeMoveCounts.length\n  const synth = t3.synthesis_found\n  const hasSynthesis = typeof synth === 'string' && synth.length > 0 && synth !== 'null'\n\n  let rawScore: number\n  if (numThemes === 0) rawScore = 0\n  else if (themesHit.length === numThemes && hasSynthesis) rawScore = 4\n  else if (themesHit.length === numThemes) rawScore = 3\n  else if (themesHit.length >= 2) rawScore = 2\n  else if (themesHit.length >= 1) rawScore = 1\n  else rawScore = 0\n\n  const hallRaw = t1.hallucinations_detected\n  const hallucinations = hallRaw === true || hallRaw === 'true'\n\n  return {\n    resolved: rawScore === 4,\n    score: numThemes > 0 ? rawScore / 4 : 0,\n    detail: JSON.stringify({\n      rawScore,\n      themesHit,\n      themesMissed,\n      hallucinations,\n      unverifiableClaims: flaggedClaims.length,\n      synthesis: synth ?? null,\n      judgeModel: router.model,\n    }),\n  }\n}\n\n// ── task loading ──────────────────────────────────────────────────────────────\n\nfunction benchRoot(): string {\n  return process.env.TRATA_BENCH_ROOT ?? DEFAULT_BENCH_ROOT\n}\n\nfunction loadTask(taskId: string): { task: BenchTask; meta: TrataTaskMeta } {\n  const taskDir = join(benchRoot(), 'environments', taskId)\n  const dataDir = join(taskDir, 'environment', 'data')\n  const testsDir = join(taskDir, 'tests')\n\n  const instruction = readText(join(taskDir, 'instruction.md'), Infinity)\n  const dataBlock = buildDataBlock(dataDir)\n  const groundTruth = readText(join(testsDir, 'ground_truth.txt'), Infinity)\n  const gradingTask1 = readText(join(testsDir, 'grading_prompt_task1.md'), Infinity)\n  const gradingTask2 = readText(join(testsDir, 'grading_prompt_task2.md'), Infinity)\n  const gradingTask3 = readText(join(testsDir, 'grading_prompt_task3.md'), Infinity)\n\n  const themeMoveCounts = parseThemeMoveCounts(groundTruth)\n\n  // Rewrite the instruction so the agent knows data is inline (no /app/data/ FS).\n  const prompt = [\n    instruction\n      .replace(\n        \"You are a financial analyst with access to the data in `/app/data/`.\",\n        'You are a financial analyst. The relevant financial data files are provided inline below.',\n      )\n      .replace(/Write your full analysis to `\\/app\\/answer\\.txt`\\.?/, 'Write your full analysis below.')\n      .replace(/`\\/app\\/data\\/`/g, 'the data files below'),\n    '',\n    '## Data files (inline)',\n    dataBlock,\n    WORKER_SUFFIX,\n  ].join('\\n')\n\n  const meta: TrataTaskMeta = {\n    taskDir,\n    taskId,\n    dataDir,\n    groundTruth,\n    gradingTask1,\n    gradingTask2,\n    gradingTask3,\n    themeMoveCounts,\n  }\n\n  // Store meta in the BenchTask so the judge can access it without re-reading.\n  const task: BenchTask = {\n    id: taskId,\n    prompt,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n\n  return { task, meta }\n}\n\nfunction readMeta(task: BenchTask): TrataTaskMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskDir !== 'string' || typeof md.groundTruth !== 'string') {\n    throw new Error(`Trata task ${task.id} missing judge metadata — was it loaded by this adapter?`)\n  }\n  return md as unknown as TrataTaskMeta\n}\n\n/** List all task ids from the environments directory. */\nfunction listTaskIds(root: string): string[] {\n  try {\n    return readdirSync(join(root, 'environments'))\n      .filter((name) => {\n        try {\n          return statSync(join(root, 'environments', name)).isDirectory()\n        } catch {\n          return false\n        }\n      })\n      .sort()\n  } catch (err) {\n    throw new Error(\n      `Trata benchmark environments not found at ${join(root, 'environments')}: ` +\n        `${err instanceof Error ? err.message : err}.\\n` +\n        `Clone the repo: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n    )\n  }\n}\n\n// ── extract agent analysis from artifact ─────────────────────────────────────\n\nfunction extractAnalysis(artifact: string): string {\n  // Model may write \"ANALYSIS:\" as its very first token (no leading newline) or\n  // after some preamble. Both are valid; fail-closed on absence.\n  if (artifact.startsWith('ANALYSIS:')) return artifact.slice('ANALYSIS:'.length).trim()\n  const marker = artifact.indexOf('\\nANALYSIS:')\n  if (marker !== -1) return artifact.slice(marker + '\\nANALYSIS:'.length).trim()\n  return ''\n}\n\n// ── adapter factory ───────────────────────────────────────────────────────────\n\nexport function createTrataHedgeAdapter(): BenchmarkAdapter {\n  return {\n    name: 'trata-hedge',\n\n    async preflight() {\n      const root = benchRoot()\n      const envDir = join(root, 'environments')\n      try {\n        const stat = statSync(envDir)\n        if (!stat.isDirectory()) throw new Error('not a directory')\n      } catch {\n        throw new Error(\n          `Trata hedge-bench not found at ${envDir}.\\n` +\n            `Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n        )\n      }\n      // Validate judge router connectivity.\n      judgeRouter()\n      const ids = listTaskIds(root)\n      if (ids.length === 0) throw new Error(`No task directories found under ${envDir}`)\n      console.warn(\n        `[trata-hedge] ${ids.length} tasks loaded from ${root}. ` +\n          `Worker model reads embedded data inline (Harbor/Docker not required). ` +\n          `Judge uses ${process.env.JUDGE_MODEL ?? JUDGE_MODEL} via Tangle router.`,\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const root = benchRoot()\n      let ids = listTaskIds(root)\n      if (opts.ids) {\n        const want = new Set(opts.ids)\n        ids = ids.filter((id) => want.has(id))\n      } else if (opts.limit !== undefined) {\n        ids = ids.slice(0, opts.limit)\n      }\n      return ids.map((id) => loadTask(id).task)\n    },\n\n    async goldArtifact(_task: BenchTask): Promise<string | undefined> {\n      // No single gold answer: the score comes from rubric theme coverage, not\n      // string-match against a reference response. Return undefined — the bench\n      // harness will skip judge self-verification for this adapter.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const answer = extractAnalysis(artifact)\n      return runTriataJudge(meta, answer, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAM,qBAAqB;AAC3B,MAAM,iBAAiB;AAEvB,MAAM,iBAAiB;AACvB,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAGX,MAAM,cAAc;AAgBpB,SAAS,SAAS,GAAW,WAAW,gBAAwB;CAC9D,IAAI;EACF,MAAM,MAAM,aAAa,CAAC;EAC1B,IAAI,IAAI,UAAU,UAAU,OAAO,IAAI,SAAS,MAAM;EACtD,OAAO,IAAI,SAAS,GAAG,QAAQ,CAAC,CAAC,SAAS,MAAM,IAAI,qBAAqB,SAAS;CACpF,QAAQ;EACN,OAAO,gBAAgB,EAAE;CAC3B;AACF;AAEA,SAAS,aAAa,KAAuB;CAC3C,MAAM,MAAgB,CAAC;CACvB,IAAI;EACF,KAAK,MAAM,QAAQ,YAAY,GAAG,GAAG;GACnC,MAAM,OAAO,KAAK,KAAK,IAAI;GAC3B,IAAI;IACF,IAAI,SAAS,IAAI,CAAC,CAAC,YAAY,GAAG,IAAI,KAAK,GAAG,aAAa,IAAI,CAAC;SAC3D,IAAI,KAAK,IAAI;GACpB,QAAQ,CAER;EACF;CACF,QAAQ,CAER;CACA,OAAO,IAAI,KAAK;AAClB;AAEA,SAAS,eAAe,SAAyB;CAC/C,MAAM,QAAQ,aAAa,OAAO;CAClC,IAAI,MAAM,WAAW,GAAG,OAAO;CAG/B,MAAM,YAAY,MAAc;EAC9B,IAAI,EAAE,SAAS,kBAAkB,GAAG,OAAO;EAC3C,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,yCAAyC,KAAK,CAAC,KAAK,CAAC,uDAAuD,KAAK,CAAC,GAAG,OAAO;EAChI,IAAI,EAAE,SAAS,aAAa,GAAG,OAAO;EACtC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,OAAO;CACT;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;CACxF,MAAM,SAAmB,CAAC;CAC1B,IAAI,QAAQ;CACZ,KAAK,MAAM,KAAK,QAAQ;EAGtB,MAAM,QAAQ,oBAFF,EAAE,MAAM,QAAQ,SAAS,CAED,EAAE,QADtB,SAAS,CAC2B;EACpD,IAAI,QAAQ,MAAM,SAAS,gBAAgB;GACzC,OAAO,KAAK,QAAQ,OAAO,SAAS,OAAO,OAAO,oDAAoD;GACtG;EACF;EACA,OAAO,KAAK,KAAK;EACjB,SAAS,MAAM;CACjB;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAEA,SAAS,oBAAoB,YAAoB,SAAyB;CACxE,MAAM,QAAQ,aAAa,OAAO;CAClC,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;EACrC,IAAI,WAAW,SAAS,GAAG,KAAK,WAAW,SAAS,IAAI,GAAG,MAAM,KAAK,CAAC;CACzE;CACA,IAAI,MAAM,WAAW,GACnB,OAAO;CAET,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,OAAO,KAAK,oBAAoB,IAAI,QAAQ,SAAS,CAAC,GAAG;CAC3D;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAIA,SAAS,qBAAqB,aAA+D;CAC3F,MAAM,QAAQ,YAAY,MAAM,IAAI;CACpC,MAAM,cAAc,MAAM,QAAkB,KAAK,GAAG,MAAM;EACxD,IAAI,SAAS,KAAK,EAAE,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC;EACvC,OAAO;CACT,GAAG,CAAC,CAAC;CACL,OAAO,YAAY,KAAK,OAAO,MAAM;EACnC,MAAM,MAAM,IAAI,IAAI,YAAY,SAAS,YAAY,IAAI,KAAK,MAAM;EACpE,MAAM,QAAQ,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI;EAG/C,OAAO;GAAE,OAFK,MAAM,MAAM,CAAC,QAAQ,aAAa,EAAE,CAAC,CAAC,KAEvC;GAAG,SADA,MAAM,MAAM,oBAAoB,KAAK,CAAC,EAAA,CAAG;EAClC;CACzB,CAAC;AACH;AAEA,SAAS,kBAAkB,QAAwB;CACjD,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,GAAG,CAAC,CAAC;AAC5C;AAUA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,4DAA4D;CACtF,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;AAEA,SAAS,kBAAkB,KAAsB;CAC/C,IAAI,IAAI,IAAI,KAAK;CACjB,IAAI,EAAE,WAAW,KAAK,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI;CAC7D,IAAI,EAAE,SAAS,KAAK,GAAG,IAAI,EAAE,MAAM,GAAG,EAAE,YAAY,KAAK,CAAC;CAC1D,IAAI,EAAE,QAAQ,gBAAgB,IAAI,CAAC,CAAC,KAAK;CACzC,IAAI;EACF,OAAO,KAAK,MAAM,CAAC;CACrB,QAAQ;EACN,IAAI;GACF,MAAM,IAAI,EAAE,MAAM,aAAa;GAC/B,IAAI,GAAG,OAAO,KAAK,MAAM,EAAE,EAAE;EAC/B,QAAQ,CAER;CACF;CACA,OAAO;AACT;AAEA,eAAe,UAAU,QAAqB,QAAgB,cAAc,GAAqB;CAC/F,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;EACpC,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;GAC5D,QAAQ;GACR,SAAS;IAAE,gBAAgB;IAAoB,eAAe,UAAU,OAAO;GAAM;GACrF,MAAM,KAAK,UAAU;IACnB,OAAO,OAAO;IACd,aAAa;IACb,YAAY;IACZ,UAAU,CAAC;KAAE,MAAM;KAAQ,SAAS;IAAO,CAAC;GAC9C,CAAC;EACH,CAAC;EACD,IAAI,CAAC,IAAI,IAAI;GACX,IAAI,IAAI,cAAc,GAAG;GACzB,MAAM,IAAI,MAAM,oBAAoB,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EACvF;EAEA,MAAM,WAAU,MADI,IAAI,KAAK,EAAA,CACR,UAAU,EAAE,EAAE,SAAS;EAC5C,IAAI,OAAO,YAAY,UAAU;EACjC,MAAM,SAAS,kBAAkB,OAAO;EACxC,IAAI,WAAW,MAAM,OAAO;CAC9B;CACA,OAAO;AACT;AAuBA,eAAe,wBACb,MACA,QACA,QACsB;CAEtB,MAAM,cACJ;;gBAFiB,oBAAoB,QAAQ,KAAK,OAIxB,EAAE;CAK9B,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,WAAW,CAAC,CACtC,QAAQ,kBAAkB,MACe,CAAC,KACX;EAAE,yBAAyB;EAAO,qBAAqB,CAAC;CAAE;AAC9F;AAEA,eAAe,kBACb,MACA,QACA,eACA,QACsB;CACtB,MAAM,cACJ,iBAAiB,cAAc,SAAS,IACpC,KAAK,UAAU,eAAe,MAAM,CAAC,IACrC;CAON,OAAQ,MADa,UAAU,QALhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MAAM,CAAC,CACjC,QAAQ,0BAA0B,WAAW,CAAC,CAC9C,QAAQ,gBAAgB,OAAO,KAAK,gBAAgB,MAAM,CACjB,CAAC,KACX,EAAE,QAAQ,CAAC,EAAE;AACjD;AAEA,eAAe,oBACb,MACA,QACA,QACsB;CAKtB,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MACe,CAAC,KACX,EAAE,iBAAiB,KAAK;AAC5D;AAEA,eAAe,eAAe,MAAqB,QAAgB,QAA0C;CAC3G,IAAI,CAAC,OAAO,KAAK,GACf,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,QAAQ,KAAK,UAAU,EAAE,OAAO,eAAe,CAAC;CAAE;CAIxF,MAAM,KAAK,MAAM,wBAAwB,MAAM,QAAQ,MAAM;CAC7D,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;CAGjD,MAAM,CAAC,IAAI,MAAM,MAAM,QAAQ,IAAI,CACjC,kBAAkB,MAAM,QAAQ,eAAe,MAAM,GACrD,oBAAoB,MAAM,QAAQ,MAAM,CAC1C,CAAC;CAED,MAAM,cAAc,GAAG,UAAU,CAAC;CAClC,MAAM,YAAsB,CAAC;CAC7B,MAAM,eAAyB,CAAC;CAEhC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,gBAAgB,QAAQ,KAAK;EACpD,MAAM,EAAE,OAAO,WAAW,KAAK,gBAAgB;EAC/C,MAAM,QAAQ,YAAY,MAAM,CAAC;EACjC,MAAM,YAAY,MAAM,aAAa,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC;EACpF,MAAM,UAAU,IAAI,KAAK,MAAM,iBAAiB,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC,CAAC;EAChG,MAAM,WAAW,SAAS,QAAQ,MAAc,CAAC,QAAQ,IAAI,CAAC,CAAC;EAC/D,MAAM,YAAY,kBAAkB,MAAM;EAC1C,IAAI,SAAS,UAAU,WAAW,UAAU,KAAK,KAAK;OACjD,aAAa,KAAK,KAAK;CAC9B;CAEA,MAAM,YAAY,KAAK,gBAAgB;CACvC,MAAM,QAAQ,GAAG;CACjB,MAAM,eAAe,OAAO,UAAU,YAAY,MAAM,SAAS,KAAK,UAAU;CAEhF,IAAI;CACJ,IAAI,cAAc,GAAG,WAAW;MAC3B,IAAI,UAAU,WAAW,aAAa,cAAc,WAAW;MAC/D,IAAI,UAAU,WAAW,WAAW,WAAW;MAC/C,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,WAAW;CAEhB,MAAM,UAAU,GAAG;CACnB,MAAM,iBAAiB,YAAY,QAAQ,YAAY;CAEvD,OAAO;EACL,UAAU,aAAa;EACvB,OAAO,YAAY,IAAI,WAAW,IAAI;EACtC,QAAQ,KAAK,UAAU;GACrB;GACA;GACA;GACA;GACA,oBAAoB,cAAc;GAClC,WAAW,SAAS;GACpB,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAIA,SAAS,YAAoB;CAC3B,OAAO,QAAQ,IAAI,oBAAoB;AACzC;AAEA,SAAS,SAAS,QAA0D;CAC1E,MAAM,UAAU,KAAK,UAAU,GAAG,gBAAgB,MAAM;CACxD,MAAM,UAAU,KAAK,SAAS,eAAe,MAAM;CACnD,MAAM,WAAW,KAAK,SAAS,OAAO;CAEtC,MAAM,cAAc,SAAS,KAAK,SAAS,gBAAgB,GAAG,QAAQ;CACtE,MAAM,YAAY,eAAe,OAAO;CACxC,MAAM,cAAc,SAAS,KAAK,UAAU,kBAAkB,GAAG,QAAQ;CACzE,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CAEjF,MAAM,kBAAkB,qBAAqB,WAAW;CAGxD,MAAM,SAAS;EACb,YACG,QACC,wEACA,2FACF,CAAC,CACA,QAAQ,uDAAuD,iCAAiC,CAAC,CACjG,QAAQ,oBAAoB,sBAAsB;EACrD;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CAEX,MAAM,OAAsB;EAC1B;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF;CASA,OAAO;EAAE,MAAA;GALP,IAAI;GACJ;GACA,UAAU;EAGA;EAAG;CAAK;AACtB;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,YAAY,YAAY,OAAO,GAAG,gBAAgB,UACrE,MAAM,IAAI,MAAM,cAAc,KAAK,GAAG,yDAAyD;CAEjG,OAAO;AACT;;AAGA,SAAS,YAAY,MAAwB;CAC3C,IAAI;EACF,OAAO,YAAY,KAAK,MAAM,cAAc,CAAC,CAAC,CAC3C,QAAQ,SAAS;GAChB,IAAI;IACF,OAAO,SAAS,KAAK,MAAM,gBAAgB,IAAI,CAAC,CAAC,CAAC,YAAY;GAChE,QAAQ;IACN,OAAO;GACT;EACF,CAAC,CAAC,CACD,KAAK;CACV,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,6CAA6C,KAAK,MAAM,cAAc,EAAE,IACnE,eAAe,QAAQ,IAAI,UAAU,IAAI,wFAC0C,MAC1F;CACF;AACF;AAIA,SAAS,gBAAgB,UAA0B;CAGjD,IAAI,SAAS,WAAW,WAAW,GAAG,OAAO,SAAS,MAAM,CAAkB,CAAC,CAAC,KAAK;CACrF,MAAM,SAAS,SAAS,QAAQ,aAAa;CAC7C,IAAI,WAAW,IAAI,OAAO,SAAS,MAAM,SAAS,EAAoB,CAAC,CAAC,KAAK;CAC7E,OAAO;AACT;AAIA,SAAgB,0BAA4C;CAC1D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,OAAO,UAAU;GACvB,MAAM,SAAS,KAAK,MAAM,cAAc;GACxC,IAAI;IAEF,IAAI,CADS,SAAS,MACd,CAAC,CAAC,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;GAC5D,QAAQ;IACN,MAAM,IAAI,MACR,kCAAkC,OAAO,+EACsC,MACjF;GACF;GAEA,YAAY;GACZ,MAAM,MAAM,YAAY,IAAI;GAC5B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,mCAAmC,QAAQ;GACjF,QAAQ,KACN,iBAAiB,IAAI,OAAO,qBAAqB,KAAK,qFAEtC,QAAQ,IAAI,eAAe,YAAY,oBACzD;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,IAAI,MAAM,YADG,UACY,CAAC;GAC1B,IAAI,KAAK,KAAK;IACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;IAC7B,MAAM,IAAI,QAAQ,OAAO,KAAK,IAAI,EAAE,CAAC;GACvC,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;GAE/B,OAAO,IAAI,KAAK,OAAO,SAAS,EAAE,CAAC,CAAC,IAAI;EAC1C;EAEA,MAAM,aAAa,OAA+C,CAKlE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAGlE,OAAO,eAFM,SAAS,IAEG,GADV,gBAAgB,QACE,GAAG,YAAY,CAAC;EACnD;CACF;AACF"}

package/dist/benchmarks/types.d.ts

@@ -1,107 +1,98 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-
-/**
- * BenchmarkAdapter — the seam every external benchmark implements so the
- * agent-runtime loop can be A/B'd (blind vs steering vs steering+memory) against
- * a REAL, DETERMINISTIC judge. The worker produces an `artifact` (a patch, a
- * command transcript, a final state); the benchmark's own harness scores it.
- *
- * The point: no self-authored judge, no invented score noise. The number comes
- * from the benchmark's published evaluation harness.
- */
-
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/types.d.ts
 interface BenchTask {
-    /** Stable benchmark instance id. */
-    id: string;
-    /** The task statement handed to the worker agent. */
-    prompt: string;
-    split?: string;
-    /** Benchmark-specific fields the worker/judge need (repo, base_commit, gold, …). */
-    metadata?: Record<string, unknown>;
+  /** Stable benchmark instance id. */
+  id: string;
+  /** The task statement handed to the worker agent. */
+  prompt: string;
+  split?: string;
+  /** Benchmark-specific fields the worker/judge need (repo, base_commit, gold, …). */
+  metadata?: Record<string, unknown>;
 }
 interface JudgeArtifactFileReceipt {
-    /** POSIX path relative to the capture directory. */
-    path: string;
-    /** Exact byte length of the retained file or symbolic-link target. */
-    byteLength: number;
-    /** SHA-256 over the retained file bytes or UTF-8 symbolic-link target. */
-    sha256: `sha256:${string}`;
-    kind: 'file' | 'symlink';
+  /** POSIX path relative to the capture directory. */
+  path: string;
+  /** Exact byte length of the retained file or symbolic-link target. */
+  byteLength: number;
+  /** SHA-256 over the retained file bytes or UTF-8 symbolic-link target. */
+  sha256: `sha256:${string}`;
+  kind: 'file' | 'symlink';
 }
 /** Durable evidence written before a staged evaluator's temporary directory is removed. */
 interface JudgeArtifactReceipt {
-    schema: 'agent-bench/judge-artifacts/v1';
-    /** Absolute directory containing `evaluator/`, `process/`, and `receipt.json`. */
-    directory: string;
-    /** Exact copy of the evaluator working directory. */
-    evaluatorDirectory: string;
-    manifestPath: string;
-    evaluatorSucceeded: boolean;
-    files: JudgeArtifactFileReceipt[];
-    fileCount: number;
-    byteLength: number;
-    /** SHA-256 over every sorted path, kind, byte length, and content hash. */
-    treeSha256: `sha256:${string}`;
+  schema: 'agent-bench/judge-artifacts/v1';
+  /** Absolute directory containing `evaluator/`, `process/`, and `receipt.json`. */
+  directory: string;
+  /** Exact copy of the evaluator working directory. */
+  evaluatorDirectory: string;
+  manifestPath: string;
+  evaluatorSucceeded: boolean;
+  files: JudgeArtifactFileReceipt[];
+  fileCount: number;
+  byteLength: number;
+  /** SHA-256 over every sorted path, kind, byte length, and content hash. */
+  treeSha256: `sha256:${string}`;
 }
 interface BenchScore {
-    /** Did the deterministic judge pass (tests resolved / state correct)? */
-    resolved: boolean;
-    /** 0..1 — 1 = fully resolved; partial credit where the harness supports it. */
-    score: number;
-    detail?: string;
-    /** Present only when the caller explicitly requested durable judge evidence. */
-    judgeArtifacts?: JudgeArtifactReceipt;
+  /** Did the deterministic judge pass (tests resolved / state correct)? */
+  resolved: boolean;
+  /** 0..1 — 1 = fully resolved; partial credit where the harness supports it. */
+  score: number;
+  detail?: string;
+  /** Present only when the caller explicitly requested durable judge evidence. */
+  judgeArtifacts?: JudgeArtifactReceipt;
 }
 interface LoadOptions {
-    limit?: number;
-    split?: string;
-    ids?: string[];
+  limit?: number;
+  split?: string;
+  ids?: string[];
 }
 interface BenchmarkAdapter {
-    readonly name: string;
-    /** Throw with actionable guidance when the harness/judge isn't installed/runnable. */
-    preflight(): Promise<void>;
-    loadTasks(opts?: LoadOptions): Promise<BenchTask[]>;
-    /** DETERMINISTIC judge: score the worker's produced artifact for a task. */
-    judge(task: BenchTask, artifact: string): Promise<BenchScore>;
-    /** Gold/oracle artifact — lets us self-verify the judge before spending model tokens. */
-    goldArtifact(task: BenchTask): Promise<string | undefined>;
-    /** How to extract the judged artifact from a run's event stream. Optional —
-     *  defaults to the agent's final answer text (the research/QA case). SWE sets
-     *  it to a patch parser. This is `benchmark = adapter` owning its deliverable,
-     *  so the gate runner (`runGate` / `runBenchmark`) needs no
-     *  per-benchmark branching. */
-    output?: OutputAdapter<string>;
-    /** Post-shot deliverable extraction from the box FILESYSTEM, not the event stream.
-     *  When set, the shot runner execs `command` in the STILL-ALIVE box after the agent
-     *  turn drains and uses its stdout as the judged artifact — the durable way to capture
-     *  a git diff of the agent's in-box edits (standard SWE-bench practice: SWE-agent /
-     *  OpenHands read the diff from repo STATE), instead of hoping the model printed a
-     *  fenced diff in its reply. Empty stdout ⇒ the runner falls back to `output` (the
-     *  event-stream parse). `cwd` defaults to the box root. */
-    boxExtract?(task: BenchTask): {
-        command: string;
-        cwd?: string;
-    };
-    /** Optional workspace pre-stage run in the box BEFORE the agent shot (same
-     *  session as `boxExtract`). For repo-state benchmarks (SWE-bench) this clones
-     *  the instance repo at `base_commit` into a fixed path so the agent only edits
-     *  — the harness owns the checkout, not the (stochastic) model. A non-zero exit
-     *  fails the shot loud rather than letting the agent run against an empty box. */
-    boxSetup?(task: BenchTask): {
-        command: string;
-        cwd?: string;
-    };
-    /** Benchmark-owned worker leaf. Set when the benchmark's native protocol IS the
-     *  worker (e.g. AppWorld's interactive ReAct episode runs inside the engine,
-     *  not as a chat completion) — the experiment uses this instead of the
-     *  BACKEND-selected client; the steer still flows through the per-round prompt.
-     *  Typed loosely to avoid a runtime import cycle; the harness casts it. */
-    leafClient?: (cfg: {
-        model: string;
-        routerBaseUrl: string;
-        routerKey: string;
-    }) => unknown;
+  readonly name: string;
+  /** Throw with actionable guidance when the harness/judge isn't installed/runnable. */
+  preflight(): Promise<void>;
+  loadTasks(opts?: LoadOptions): Promise<BenchTask[]>;
+  /** DETERMINISTIC judge: score the worker's produced artifact for a task. */
+  judge(task: BenchTask, artifact: string): Promise<BenchScore>;
+  /** Gold/oracle artifact — lets us self-verify the judge before spending model tokens. */
+  goldArtifact(task: BenchTask): Promise<string | undefined>;
+  /** How to extract the judged artifact from a run's event stream. Optional —
+   *  defaults to the agent's final answer text (the research/QA case). SWE sets
+   *  it to a patch parser. This is `benchmark = adapter` owning its deliverable,
+   *  so the gate runner (`runGate` / `runBenchmark`) needs no
+   *  per-benchmark branching. */
+  output?: OutputAdapter<string>;
+  /** Post-shot deliverable extraction from the box FILESYSTEM, not the event stream.
+   *  When set, the shot runner execs `command` in the STILL-ALIVE box after the agent
+   *  turn drains and uses its stdout as the judged artifact — the durable way to capture
+   *  a git diff of the agent's in-box edits (standard SWE-bench practice: SWE-agent /
+   *  OpenHands read the diff from repo STATE), instead of hoping the model printed a
+   *  fenced diff in its reply. Empty stdout ⇒ the runner falls back to `output` (the
+   *  event-stream parse). `cwd` defaults to the box root. */
+  boxExtract?(task: BenchTask): {
+    command: string;
+    cwd?: string;
+  };
+  /** Optional workspace pre-stage run in the box BEFORE the agent shot (same
+   *  session as `boxExtract`). For repo-state benchmarks (SWE-bench) this clones
+   *  the instance repo at `base_commit` into a fixed path so the agent only edits
+   *  — the harness owns the checkout, not the (stochastic) model. A non-zero exit
+   *  fails the shot loud rather than letting the agent run against an empty box. */
+  boxSetup?(task: BenchTask): {
+    command: string;
+    cwd?: string;
+  };
+  /** Benchmark-owned worker leaf. Set when the benchmark's native protocol IS the
+   *  worker (e.g. AppWorld's interactive ReAct episode runs inside the engine,
+   *  not as a chat completion) — the experiment uses this instead of the
+   *  BACKEND-selected client; the steer still flows through the per-round prompt.
+   *  Typed loosely to avoid a runtime import cycle; the harness casts it. */
+  leafClient?: (cfg: {
+    model: string;
+    routerBaseUrl: string;
+    routerKey: string;
+  }) => unknown;
 }
-
-export type { BenchScore, BenchTask, BenchmarkAdapter, JudgeArtifactFileReceipt, JudgeArtifactReceipt, LoadOptions };
+//#endregion
+export { BenchScore, BenchTask, BenchmarkAdapter, JudgeArtifactFileReceipt, JudgeArtifactReceipt, LoadOptions };
+//# sourceMappingURL=types.d.ts.map

package/dist/benchmarks/types.js

@@ -1 +1 @@
-//# sourceMappingURL=types.js.map
+export {};

package/dist/benchmarks/webarena-verified.d.ts

@@ -1,16 +1,8 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * WebArena-Verified adapter (ServiceNow/webarena-verified).
- *
- * Worker artifact = a WebArena-Verified run output directory, not final chat text.
- * Judge = the official `webarena_verified eval-tasks` evaluator over that output
- * directory. The adapter refuses to score a plain answer so we do not turn a DOM
- * benchmark into a fake text benchmark.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/webarena-verified.d.ts
 declare const webarenaOutputDirOutput: OutputAdapter<string>;
 declare function createWebArenaVerifiedAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createWebArenaVerifiedAdapter, webarenaOutputDirOutput };
+//# sourceMappingURL=webarena-verified.d.ts.map

package/dist/benchmarks/webarena-verified.js

@@ -1,10 +1,153 @@
-import {
-  createWebArenaVerifiedAdapter,
-  webarenaOutputDirOutput
-} from "../chunk-XXFF3RRD.js";
-import "../chunk-EIETHPD5.js";
-export {
-  createWebArenaVerifiedAdapter,
-  webarenaOutputDirOutput
-};
+import { benchRoot, runVenvPython } from "./_harness.js";
+import { access, readFile, stat } from "node:fs/promises";
+import { join, resolve } from "node:path";
+//#region src/benchmarks/webarena-verified.ts
+/**
+* WebArena-Verified adapter (ServiceNow/webarena-verified).
+*
+* Worker artifact = a WebArena-Verified run output directory, not final chat text.
+* Judge = the official `webarena_verified eval-tasks` evaluator over that output
+* directory. The adapter refuses to score a plain answer so we do not turn a DOM
+* benchmark into a fake text benchmark.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "webarena-verified.json");
+const DATASET_REL = join("assets", "dataset", "webarena-verified.json");
+const webarenaDir = () => process.env.WEBARENA_VERIFIED_DIR;
+const webarenaOutputDirOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return ([...text.matchAll(/```(?:text|path)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+} };
+async function assertFile(path, label) {
+	try {
+		await access(path);
+	} catch (err) {
+		throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+	}
+}
+async function assertDir(path, label) {
+	try {
+		if (!(await stat(path)).isDirectory()) throw new Error("not a directory");
+	} catch (err) {
+		throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+	}
+}
+function rowToTask(row) {
+	const meta = {
+		taskId: row.task_id,
+		intentTemplateId: row.intent_template_id,
+		sites: row.sites ?? [],
+		startUrls: row.start_urls ?? [],
+		revision: row.revision,
+		eval: row.eval
+	};
+	return {
+		id: String(row.task_id),
+		prompt: [
+			"Run this WebArena-Verified browser task in the official environment.",
+			`Goal: ${row.intent}`,
+			row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(", ")}` : void 0,
+			row.sites?.length ? `Sites: ${row.sites.join(", ")}` : void 0,
+			"",
+			"Return the path to the official WebArena-Verified run output directory for this task.",
+			"The judge expects that directory to contain the task response and network trace files."
+		].filter(Boolean).join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.taskId !== "number") throw new Error(`webarena-verified task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function selectRows(rows, opts) {
+	let tasks = rows.map(rowToTask);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((task) => want.has(task.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`);
+	return tasks;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 — loading ${rows.length} adapter fixtures`);
+	return selectRows(rows, opts);
+}
+async function loadOfficialRows(dir, opts) {
+	return selectRows(JSON.parse(await readFile(join(dir, DATASET_REL), "utf8")), opts);
+}
+async function runOfficialEval(root, taskId, outputDir) {
+	const stdout = await runVenvPython(`
+import json, os, subprocess, sys
+from pathlib import Path
+
+root = Path(sys.argv[1])
+task_id = sys.argv[2]
+output_dir = Path(sys.argv[3])
+env = os.environ.copy()
+env["PYTHONPATH"] = str(root / "src") + os.pathsep + env.get("PYTHONPATH", "")
+cmd = [sys.executable, "-m", "webarena_verified", "eval-tasks", "--task-ids", task_id, "--output-dir", str(output_dir)]
+proc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)
+if proc.returncode != 0:
+    raise SystemExit((proc.stderr or proc.stdout or f"exit {proc.returncode}")[:2000])
+result_path = output_dir / task_id / "eval_result.json"
+if not result_path.exists():
+    raise SystemExit(f"official evaluator did not write {result_path}")
+print(json.dumps(json.loads(result_path.read_text())))
+`, [
+		root,
+		String(taskId),
+		outputDir
+	], 0);
+	return JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+}
+function createWebArenaVerifiedAdapter() {
+	const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === "1";
+	return {
+		name: "webarena-verified",
+		output: webarenaOutputDirOutput,
+		async preflight() {
+			if (fixturesMode) return;
+			const dir = webarenaDir();
+			if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified.");
+			await assertFile(join(dir, DATASET_REL), "official dataset");
+			await assertFile(join(dir, "src", "webarena_verified", "__main__.py"), "official CLI module");
+			await runVenvPython("import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print(\"ok\")", [join(dir, "src")]);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const dir = webarenaDir();
+			if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks");
+			return loadOfficialRows(dir, opts);
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			const dir = webarenaDir();
+			if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts");
+			const outputDir = resolve(artifact.trim());
+			await assertDir(outputDir, "run output directory");
+			const meta = readMeta(task);
+			const report = await runOfficialEval(dir, meta.taskId, outputDir);
+			const score = typeof report.score === "number" ? report.score : 0;
+			const status = typeof report.status === "string" ? report.status : "unknown";
+			return {
+				resolved: score === 1,
+				score,
+				detail: JSON.stringify({
+					taskId: meta.taskId,
+					status,
+					outputDir
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createWebArenaVerifiedAdapter, webarenaOutputDirOutput };
+
 //# sourceMappingURL=webarena-verified.js.map

package/dist/benchmarks/webarena-verified.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"webarena-verified.js","names":[],"sources":["../../src/benchmarks/webarena-verified.ts"],"sourcesContent":["/**\n * WebArena-Verified adapter (ServiceNow/webarena-verified).\n *\n * Worker artifact = a WebArena-Verified run output directory, not final chat text.\n * Judge = the official `webarena_verified eval-tasks` evaluator over that output\n * directory. The adapter refuses to score a plain answer so we do not turn a DOM\n * benchmark into a fake text benchmark.\n */\n\nimport { access, readFile, stat } from 'node:fs/promises'\nimport { join, resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'webarena-verified.json')\nconst DATASET_REL = join('assets', 'dataset', 'webarena-verified.json')\n\ninterface WebArenaRow {\n  task_id: number\n  intent: string\n  intent_template_id?: number\n  sites?: string[]\n  start_urls?: string[]\n  eval?: unknown[]\n  revision?: number\n}\n\ninterface WebArenaMeta {\n  taskId: number\n  intentTemplateId?: number\n  sites: string[]\n  startUrls: string[]\n  revision?: number\n  eval?: unknown[]\n}\n\nconst webarenaDir = (): string | undefined => process.env.WEBARENA_VERIFIED_DIR\n\nexport const webarenaOutputDirOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertDir(path: string, label: string): Promise<void> {\n  try {\n    const s = await stat(path)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction rowToTask(row: WebArenaRow): BenchTask {\n  const meta: WebArenaMeta = {\n    taskId: row.task_id,\n    intentTemplateId: row.intent_template_id,\n    sites: row.sites ?? [],\n    startUrls: row.start_urls ?? [],\n    revision: row.revision,\n    eval: row.eval,\n  }\n  return {\n    id: String(row.task_id),\n    prompt: [\n      'Run this WebArena-Verified browser task in the official environment.',\n      `Goal: ${row.intent}`,\n      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(', ')}` : undefined,\n      row.sites?.length ? `Sites: ${row.sites.join(', ')}` : undefined,\n      '',\n      'Return the path to the official WebArena-Verified run output directory for this task.',\n      'The judge expects that directory to contain the task response and network trace files.',\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): WebArenaMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number') {\n    throw new Error(`webarena-verified task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as WebArenaMeta\n}\n\nfunction selectRows(rows: WebArenaRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as WebArenaRow[]\n  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts)\n}\n\nasync function loadOfficialRows(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), 'utf8')) as WebArenaRow[]\n  return selectRows(rows, opts)\n}\n\nasync function runOfficialEval(root: string, taskId: number, outputDir: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, os, subprocess, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\noutput_dir = Path(sys.argv[3])\nenv = os.environ.copy()\nenv[\"PYTHONPATH\"] = str(root / \"src\") + os.pathsep + env.get(\"PYTHONPATH\", \"\")\ncmd = [sys.executable, \"-m\", \"webarena_verified\", \"eval-tasks\", \"--task-ids\", task_id, \"--output-dir\", str(output_dir)]\nproc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)\nif proc.returncode != 0:\n    raise SystemExit((proc.stderr or proc.stdout or f\"exit {proc.returncode}\")[:2000])\nresult_path = output_dir / task_id / \"eval_result.json\"\nif not result_path.exists():\n    raise SystemExit(f\"official evaluator did not write {result_path}\")\nprint(json.dumps(json.loads(result_path.read_text())))\n`\n  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createWebArenaVerifiedAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === '1'\n\n  return {\n    name: 'webarena-verified',\n    output: webarenaOutputDirOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = webarenaDir()\n      if (!dir) {\n        throw new Error(\n          'WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified.',\n        )\n      }\n      await assertFile(join(dir, DATASET_REL), 'official dataset')\n      await assertFile(join(dir, 'src', 'webarena_verified', '__main__.py'), 'official CLI module')\n      await runVenvPython(\n        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print(\"ok\")',\n        [join(dir, 'src')],\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks')\n      return loadOfficialRows(dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts')\n      const outputDir = resolve(artifact.trim())\n      await assertDir(outputDir, 'run output directory')\n      const meta = readMeta(task)\n      const report = await runOfficialEval(dir, meta.taskId, outputDir)\n      const score = typeof report.score === 'number' ? report.score : 0\n      const status = typeof report.status === 'string' ? report.status : 'unknown'\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,MAAM,cAAc,KAAK,UAAU,WAAW,wBAAwB;AAqBtE,MAAM,oBAAwC,QAAQ,IAAI;AAE1D,MAAa,0BAAiD,EAC5D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,eAAe,WAAW,MAAc,OAA8B;CACpE,IAAI;EACF,MAAM,OAAO,IAAI;CACnB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,8BAA8B,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAChH;AACF;AAEA,eAAe,UAAU,MAAc,OAA8B;CACnE,IAAI;EAEF,IAAI,EAAC,MADW,KAAK,IAAI,EAAA,CAClB,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;CACzD,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,8BAA8B,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAChH;AACF;AAEA,SAAS,UAAU,KAA6B;CAC9C,MAAM,OAAqB;EACzB,QAAQ,IAAI;EACZ,kBAAkB,IAAI;EACtB,OAAO,IAAI,SAAS,CAAC;EACrB,WAAW,IAAI,cAAc,CAAC;EAC9B,UAAU,IAAI;EACd,MAAM,IAAI;CACZ;CACA,OAAO;EACL,IAAI,OAAO,IAAI,OAAO;EACtB,QAAQ;GACN;GACA,SAAS,IAAI;GACb,IAAI,YAAY,SAAS,wBAAwB,IAAI,WAAW,KAAK,IAAI,MAAM,KAAA;GAC/E,IAAI,OAAO,SAAS,UAAU,IAAI,MAAM,KAAK,IAAI,MAAM,KAAA;GACvD;GACA;GACA;EACF,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,0BAA0B,KAAK,GAAG,kDAAkD;CAEtG,OAAO;AACT;AAEA,SAAS,WAAW,MAAqB,MAAgC;CACvE,IAAI,QAAQ,KAAK,IAAI,SAAS;CAC9B,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,uCAAuC,KAAK,UAAU,IAAI,GAAG;CACrG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,8DAA8D,KAAK,OAAO,kBAAkB;CACzG,OAAO,WAAW,MAAM,IAAI;AAC9B;AAEA,eAAe,iBAAiB,KAAa,MAAyC;CAEpF,OAAO,WADM,KAAK,MAAM,MAAM,SAAS,KAAK,KAAK,WAAW,GAAG,MAAM,CAChD,GAAG,IAAI;AAC9B;AAEA,eAAe,gBAAgB,MAAc,QAAgB,WAAqD;CAmBhH,MAAM,SAAS,MAAM,cAAc;;;;;;;;;;;;;;;;;GAAQ;EAAC;EAAM,OAAO,MAAM;EAAG;CAAS,GAAG,CAAC;CAC/E,OAAO,KAAK,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,+BAA+B;CAEhE,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,YAAY;GACxB,IAAI,CAAC,KACH,MAAM,IAAI,MACR,2LACF;GAEF,MAAM,WAAW,KAAK,KAAK,WAAW,GAAG,kBAAkB;GAC3D,MAAM,WAAW,KAAK,KAAK,OAAO,qBAAqB,aAAa,GAAG,qBAAqB;GAC5F,MAAM,cACJ,wFACA,CAAC,KAAK,KAAK,KAAK,CAAC,CACnB;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,MAAM,YAAY;GACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,4EAA4E;GACtG,OAAO,iBAAiB,KAAK,IAAI;EACnC;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,MAAM,YAAY;GACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,wEAAwE;GAClG,MAAM,YAAY,QAAQ,SAAS,KAAK,CAAC;GACzC,MAAM,UAAU,WAAW,sBAAsB;GACjD,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK,QAAQ,SAAS;GAChE,MAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;GAChE,MAAM,SAAS,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS;GACnE,OAAO;IACL,UAAU,UAAU;IACpB;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ;KAAQ;IAAU,CAAC;GACnE;EACF;CACF;AACF"}