@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-VQRS7VUC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/trata-hedge.ts"],"sourcesContent":["/**\n * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial\n * analysis tasks across 6 domains (private equity, managed care, industrials,\n * vertical SaaS, REITs, insurance). Each task bundles real earnings-call\n * transcripts, financial statements, press releases, and SEC filings under\n * `environment/data/`; the agent must produce a grounded analysis citing those\n * files.\n *\n * Harbor architecture: the original benchmark runs each task in a Docker\n * container with file-read tools. This adapter skips Harbor and embeds the data\n * files directly in the worker prompt so any router backend can score it. Large\n * files (> 30 KB) are truncated to fit within model context limits. The judge\n * reads the actual files from disk for citation verification.\n *\n * Judge: a 3-stage LLM cascade faithful to grade.py —\n *   Task 1: hallucination check (cited-file context + agent answer)\n *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)\n *   Task 3: synthesis check (ground truth + agent answer)\n * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,\n * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;\n * resolved = score 4 (sparse reward).\n *\n * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.\n * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench\n */\n\nimport { readdirSync, readFileSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench'\nconst MAX_FILE_BYTES = 8_000\n// Sandbox API rejects messages > 100K chars; instruction + suffix adds ~500 chars overhead.\nconst MAX_DATA_CHARS = 85_000\nconst WORKER_SUFFIX = [\n  '',\n  'The data files above are your only source. Cite each claim with the filename that supports it.',\n  'Write your complete analysis to the text block that starts with \"ANALYSIS:\" on a line by itself.',\n  'Take a clear position on the topic. Every factual claim must name the file it comes from.',\n].join('\\n')\n\n// Judge model: Gemini 2.5 Pro via router (matches gemini-3.1-pro-preview tier used by grade.py).\nconst JUDGE_MODEL = 'gemini-2.5-pro'\n\ninterface TrataTaskMeta {\n  taskDir: string\n  taskId: string\n  dataDir: string\n  groundTruth: string\n  gradingTask1: string\n  gradingTask2: string\n  gradingTask3: string\n  /** [(label, n_moves)] in rubric order. */\n  themeMoveCounts: Array<{ label: string; nMoves: number }>\n}\n\n// ── file reading helpers ──────────────────────────────────────────────────────\n\nfunction readText(p: string, maxBytes = MAX_FILE_BYTES): string {\n  try {\n    const buf = readFileSync(p)\n    if (buf.length <= maxBytes) return buf.toString('utf8')\n    return buf.subarray(0, maxBytes).toString('utf8') + `\\n[… truncated at ${maxBytes} bytes …]`\n  } catch {\n    return `[unreadable: ${p}]`\n  }\n}\n\nfunction listFilesRec(dir: string): string[] {\n  const out: string[] = []\n  try {\n    for (const name of readdirSync(dir)) {\n      const full = join(dir, name)\n      try {\n        if (statSync(full).isDirectory()) out.push(...listFilesRec(full))\n        else out.push(full)\n      } catch {\n        // skip unreadable entries\n      }\n    }\n  } catch {\n    // dir absent\n  }\n  return out.sort()\n}\n\nfunction buildDataBlock(dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  if (files.length === 0) return '(no data files found)'\n  // Prioritise: company profiles → earnings calls → primary-ticker financials →\n  // sec filings → press releases → peers. Stop once total chars hit MAX_DATA_CHARS.\n  const priority = (f: string) => {\n    if (f.includes('company_profiles')) return 0\n    if (f.includes('earnings_call')) return 1\n    if (/financials\\/(income|cash_flow|balance)/.test(f) && !/bam_|bx_|kkr_|ares_|cg_|owl_|fsk_|cat_|de_|tdg_|car_/.test(f)) return 2\n    if (f.includes('sec_filings')) return 3\n    if (f.includes('investor_pres')) return 4\n    if (f.includes('press_release')) return 5\n    return 6\n  }\n  const sorted = [...files].sort((a, b) => priority(a) - priority(b) || a.localeCompare(b))\n  const blocks: string[] = []\n  let total = 0\n  for (const f of sorted) {\n    const rel = f.slice(dataDir.length + 1)\n    const content = readText(f)\n    const chunk = `\\n=== FILE: data/${rel} ===\\n${content}`\n    if (total + chunk.length > MAX_DATA_CHARS) {\n      blocks.push(`\\n[… ${sorted.length - blocks.length} more files omitted to stay within context limit …]`)\n      break\n    }\n    blocks.push(chunk)\n    total += chunk.length\n  }\n  return blocks.join('')\n}\n\nfunction buildCitedDataBlock(answerText: string, dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  const cited: string[] = []\n  for (const f of files) {\n    const rel = f.slice(dataDir.length + 1)\n    const name = rel.split('/').pop() ?? ''\n    if (answerText.includes(rel) || answerText.includes(name)) cited.push(f)\n  }\n  if (cited.length === 0) {\n    return '(The agent did not cite any data files, or no cited file could be located.)'\n  }\n  const blocks: string[] = []\n  for (const f of cited) {\n    const rel = f.slice(dataDir.length + 1)\n    blocks.push(`\\n=== FILE: data/${rel} ===\\n${readText(f)}`)\n  }\n  return blocks.join('')\n}\n\n// ── ground-truth parser ───────────────────────────────────────────────────────\n\nfunction parseThemeMoveCounts(groundTruth: string): Array<{ label: string; nMoves: number }> {\n  const lines = groundTruth.split('\\n')\n  const themeStarts = lines.reduce<number[]>((acc, l, i) => {\n    if (/^\\d+\\./.test(l.trim())) acc.push(i)\n    return acc\n  }, [])\n  return themeStarts.map((start, k) => {\n    const end = k + 1 < themeStarts.length ? themeStarts[k + 1] : lines.length\n    const block = lines.slice(start, end).join('\\n')\n    const label = lines[start].replace(/^\\d+\\.\\s*/, '').trim()\n    const nMoves = (block.match(/^\\s*\\[[a-z]\\]\\s+/gm) ?? []).length\n    return { label, nMoves }\n  })\n}\n\nfunction coverageThreshold(nMoves: number): number {\n  return Math.max(1, Math.min(nMoves - 1, 3))\n}\n\n// ── router judge calls ────────────────────────────────────────────────────────\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the Trata hedge-bench judge')\n  const model = process.env.JUDGE_MODEL ?? JUDGE_MODEL\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\nfunction parseJsonFallback(raw: string): unknown {\n  let s = raw.trim()\n  if (s.startsWith('```')) s = s.split('\\n').slice(1).join('\\n')\n  if (s.endsWith('```')) s = s.slice(0, s.lastIndexOf('```'))\n  s = s.replace(/,(\\s*[}\\]])/g, '$1').trim()\n  try {\n    return JSON.parse(s)\n  } catch {\n    try {\n      const m = s.match(/\\{[\\s\\S]*\\}/)\n      if (m) return JSON.parse(m[0])\n    } catch {\n      // fallthrough\n    }\n  }\n  return null\n}\n\nasync function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise<unknown> {\n  for (let i = 0; i < maxAttempts; i++) {\n    const res = await fetch(`${router.baseUrl}/chat/completions`, {\n      method: 'POST',\n      headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n      body: JSON.stringify({\n        model: router.model,\n        temperature: 0,\n        max_tokens: 16384,\n        messages: [{ role: 'user', content: prompt }],\n      }),\n    })\n    if (!res.ok) {\n      if (i < maxAttempts - 1) continue\n      throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n    }\n    const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n    const content = body.choices?.[0]?.message?.content\n    if (typeof content !== 'string') continue\n    const parsed = parseJsonFallback(content)\n    if (parsed !== null) return parsed\n  }\n  return null\n}\n\ninterface Task1Result {\n  hallucinations_detected?: boolean | string\n  unverifiable_claims?: Array<{ claim: string; cited_file?: string; issue?: string; evidence_check?: string }>\n}\n\ninterface Task2ThemeResult {\n  label?: string\n  moves_hit?: string[]\n  moves_missed?: string[]\n  moves_tainted?: string[]\n  move_reasoning?: Record<string, string>\n}\n\ninterface Task2Result {\n  themes?: Task2ThemeResult[]\n}\n\ninterface Task3Result {\n  synthesis_found?: string | null\n}\n\nasync function task1HallucinationCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task1Result> {\n  const citedBlock = buildCitedDataBlock(answer, meta.dataDir)\n  const dataContext =\n    'The block below contains the contents of the data files the agent cited in their answer. ' +\n    'Treat this as the ground truth for factual verification.\\n\\n' +\n    `<cited_data>\\n${citedBlock}\\n</cited_data>`\n  const prompt = meta.gradingTask1\n    .replace('{data_context}', dataContext)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task1Result) ?? { hallucinations_detected: false, unverifiable_claims: [] }\n}\n\nasync function task2PerMoveCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  flaggedClaims: Task1Result['unverifiable_claims'],\n  router: JudgeRouter,\n): Promise<Task2Result> {\n  const claimsBlock =\n    flaggedClaims && flaggedClaims.length > 0\n      ? JSON.stringify(flaggedClaims, null, 2)\n      : '(No claims were flagged as hallucinations.)'\n  const prompt = meta.gradingTask2\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n    .replace('{flagged_claims_block}', claimsBlock)\n    .replace('{num_themes}', String(meta.themeMoveCounts.length))\n  const result = await callJudge(router, prompt)\n  return (result as Task2Result) ?? { themes: [] }\n}\n\nasync function task3SynthesisCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task3Result> {\n  const prompt = meta.gradingTask3\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task3Result) ?? { synthesis_found: null }\n}\n\nasync function runTriataJudge(meta: TrataTaskMeta, answer: string, router: JudgeRouter): Promise<BenchScore> {\n  if (!answer.trim()) {\n    return { resolved: false, score: 0, detail: JSON.stringify({ error: 'empty answer' }) }\n  }\n\n  // Stage 1: hallucination check (serial — feeds into stage 2).\n  const t1 = await task1HallucinationCheck(meta, answer, router)\n  const flaggedClaims = t1.unverifiable_claims ?? []\n\n  // Stages 2 + 3 in parallel.\n  const [t2, t3] = await Promise.all([\n    task2PerMoveCheck(meta, answer, flaggedClaims, router),\n    task3SynthesisCheck(meta, answer, router),\n  ])\n\n  const judgeThemes = t2.themes ?? []\n  const themesHit: string[] = []\n  const themesMissed: string[] = []\n\n  for (let i = 0; i < meta.themeMoveCounts.length; i++) {\n    const { label, nMoves } = meta.themeMoveCounts[i]\n    const judge = judgeThemes[i] ?? {}\n    const movesHit = (judge.moves_hit ?? []).map((m: string) => m.replace(/[\\[\\]]/g, ''))\n    const tainted = new Set((judge.moves_tainted ?? []).map((m: string) => m.replace(/[\\[\\]]/g, '')))\n    const validHit = movesHit.filter((m: string) => !tainted.has(m))\n    const threshold = coverageThreshold(nMoves)\n    if (validHit.length >= threshold) themesHit.push(label)\n    else themesMissed.push(label)\n  }\n\n  const numThemes = meta.themeMoveCounts.length\n  const synth = t3.synthesis_found\n  const hasSynthesis = typeof synth === 'string' && synth.length > 0 && synth !== 'null'\n\n  let rawScore: number\n  if (numThemes === 0) rawScore = 0\n  else if (themesHit.length === numThemes && hasSynthesis) rawScore = 4\n  else if (themesHit.length === numThemes) rawScore = 3\n  else if (themesHit.length >= 2) rawScore = 2\n  else if (themesHit.length >= 1) rawScore = 1\n  else rawScore = 0\n\n  const hallRaw = t1.hallucinations_detected\n  const hallucinations = hallRaw === true || hallRaw === 'true'\n\n  return {\n    resolved: rawScore === 4,\n    score: numThemes > 0 ? rawScore / 4 : 0,\n    detail: JSON.stringify({\n      rawScore,\n      themesHit,\n      themesMissed,\n      hallucinations,\n      unverifiableClaims: flaggedClaims.length,\n      synthesis: synth ?? null,\n      judgeModel: router.model,\n    }),\n  }\n}\n\n// ── task loading ──────────────────────────────────────────────────────────────\n\nfunction benchRoot(): string {\n  return process.env.TRATA_BENCH_ROOT ?? DEFAULT_BENCH_ROOT\n}\n\nfunction loadTask(taskId: string): { task: BenchTask; meta: TrataTaskMeta } {\n  const taskDir = join(benchRoot(), 'environments', taskId)\n  const dataDir = join(taskDir, 'environment', 'data')\n  const testsDir = join(taskDir, 'tests')\n\n  const instruction = readText(join(taskDir, 'instruction.md'), Infinity)\n  const dataBlock = buildDataBlock(dataDir)\n  const groundTruth = readText(join(testsDir, 'ground_truth.txt'), Infinity)\n  const gradingTask1 = readText(join(testsDir, 'grading_prompt_task1.md'), Infinity)\n  const gradingTask2 = readText(join(testsDir, 'grading_prompt_task2.md'), Infinity)\n  const gradingTask3 = readText(join(testsDir, 'grading_prompt_task3.md'), Infinity)\n\n  const themeMoveCounts = parseThemeMoveCounts(groundTruth)\n\n  // Rewrite the instruction so the agent knows data is inline (no /app/data/ FS).\n  const prompt = [\n    instruction\n      .replace(\n        \"You are a financial analyst with access to the data in `/app/data/`.\",\n        'You are a financial analyst. The relevant financial data files are provided inline below.',\n      )\n      .replace(/Write your full analysis to `\\/app\\/answer\\.txt`\\.?/, 'Write your full analysis below.')\n      .replace(/`\\/app\\/data\\/`/g, 'the data files below'),\n    '',\n    '## Data files (inline)',\n    dataBlock,\n    WORKER_SUFFIX,\n  ].join('\\n')\n\n  const meta: TrataTaskMeta = {\n    taskDir,\n    taskId,\n    dataDir,\n    groundTruth,\n    gradingTask1,\n    gradingTask2,\n    gradingTask3,\n    themeMoveCounts,\n  }\n\n  // Store meta in the BenchTask so the judge can access it without re-reading.\n  const task: BenchTask = {\n    id: taskId,\n    prompt,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n\n  return { task, meta }\n}\n\nfunction readMeta(task: BenchTask): TrataTaskMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskDir !== 'string' || typeof md.groundTruth !== 'string') {\n    throw new Error(`Trata task ${task.id} missing judge metadata — was it loaded by this adapter?`)\n  }\n  return md as unknown as TrataTaskMeta\n}\n\n/** List all task ids from the environments directory. */\nfunction listTaskIds(root: string): string[] {\n  try {\n    return readdirSync(join(root, 'environments'))\n      .filter((name) => {\n        try {\n          return statSync(join(root, 'environments', name)).isDirectory()\n        } catch {\n          return false\n        }\n      })\n      .sort()\n  } catch (err) {\n    throw new Error(\n      `Trata benchmark environments not found at ${join(root, 'environments')}: ` +\n        `${err instanceof Error ? err.message : err}.\\n` +\n        `Clone the repo: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n    )\n  }\n}\n\n// ── extract agent analysis from artifact ─────────────────────────────────────\n\nfunction extractAnalysis(artifact: string): string {\n  // Model may write \"ANALYSIS:\" as its very first token (no leading newline) or\n  // after some preamble. Both are valid; fail-closed on absence.\n  if (artifact.startsWith('ANALYSIS:')) return artifact.slice('ANALYSIS:'.length).trim()\n  const marker = artifact.indexOf('\\nANALYSIS:')\n  if (marker !== -1) return artifact.slice(marker + '\\nANALYSIS:'.length).trim()\n  return ''\n}\n\n// ── adapter factory ───────────────────────────────────────────────────────────\n\nexport function createTrataHedgeAdapter(): BenchmarkAdapter {\n  return {\n    name: 'trata-hedge',\n\n    async preflight() {\n      const root = benchRoot()\n      const envDir = join(root, 'environments')\n      try {\n        const stat = statSync(envDir)\n        if (!stat.isDirectory()) throw new Error('not a directory')\n      } catch {\n        throw new Error(\n          `Trata hedge-bench not found at ${envDir}.\\n` +\n            `Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n        )\n      }\n      // Validate judge router connectivity.\n      judgeRouter()\n      const ids = listTaskIds(root)\n      if (ids.length === 0) throw new Error(`No task directories found under ${envDir}`)\n      console.warn(\n        `[trata-hedge] ${ids.length} tasks loaded from ${root}. ` +\n          `Worker model reads embedded data inline (Harbor/Docker not required). ` +\n          `Judge uses ${process.env.JUDGE_MODEL ?? JUDGE_MODEL} via Tangle router.`,\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const root = benchRoot()\n      let ids = listTaskIds(root)\n      if (opts.ids) {\n        const want = new Set(opts.ids)\n        ids = ids.filter((id) => want.has(id))\n      } else if (opts.limit !== undefined) {\n        ids = ids.slice(0, opts.limit)\n      }\n      return ids.map((id) => loadTask(id).task)\n    },\n\n    async goldArtifact(_task: BenchTask): Promise<string | undefined> {\n      // No single gold answer: the score comes from rubric theme coverage, not\n      // string-match against a reference response. Return undefined — the bench\n      // harness will skip judge self-verification for this adapter.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const answer = extractAnalysis(artifact)\n      return runTriataJudge(meta, answer, judgeRouter())\n    },\n  }\n}\n"],"mappings":";AA0BA,SAAS,aAAa,cAAc,gBAAgB;AACpD,SAAS,YAAY;AAGrB,IAAM,qBAAqB;AAC3B,IAAM,iBAAiB;AAEvB,IAAM,iBAAiB;AACvB,IAAM,gBAAgB;AAAA,EACpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAGX,IAAM,cAAc;AAgBpB,SAAS,SAAS,GAAW,WAAW,gBAAwB;AAC9D,MAAI;AACF,UAAM,MAAM,aAAa,CAAC;AAC1B,QAAI,IAAI,UAAU,SAAU,QAAO,IAAI,SAAS,MAAM;AACtD,WAAO,IAAI,SAAS,GAAG,QAAQ,EAAE,SAAS,MAAM,IAAI;AAAA,uBAAqB,QAAQ;AAAA,EACnF,QAAQ;AACN,WAAO,gBAAgB,CAAC;AAAA,EAC1B;AACF;AAEA,SAAS,aAAa,KAAuB;AAC3C,QAAM,MAAgB,CAAC;AACvB,MAAI;AACF,eAAW,QAAQ,YAAY,GAAG,GAAG;AACnC,YAAM,OAAO,KAAK,KAAK,IAAI;AAC3B,UAAI;AACF,YAAI,SAAS,IAAI,EAAE,YAAY,EAAG,KAAI,KAAK,GAAG,aAAa,IAAI,CAAC;AAAA,YAC3D,KAAI,KAAK,IAAI;AAAA,MACpB,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF,QAAQ;AAAA,EAER;AACA,SAAO,IAAI,KAAK;AAClB;AAEA,SAAS,eAAe,SAAyB;AAC/C,QAAM,QAAQ,aAAa,OAAO;AAClC,MAAI,MAAM,WAAW,EAAG,QAAO;AAG/B,QAAM,WAAW,CAAC,MAAc;AAC9B,QAAI,EAAE,SAAS,kBAAkB,EAAG,QAAO;AAC3C,QAAI,EAAE,SAAS,eAAe,EAAG,QAAO;AACxC,QAAI,yCAAyC,KAAK,CAAC,KAAK,CAAC,uDAAuD,KAAK,CAAC,EAAG,QAAO;AAChI,QAAI,EAAE,SAAS,aAAa,EAAG,QAAO;AACtC,QAAI,EAAE,SAAS,eAAe,EAAG,QAAO;AACxC,QAAI,EAAE,SAAS,eAAe,EAAG,QAAO;AACxC,WAAO;AAAA,EACT;AACA,QAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;AACxF,QAAM,SAAmB,CAAC;AAC1B,MAAI,QAAQ;AACZ,aAAW,KAAK,QAAQ;AACtB,UAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;AACtC,UAAM,UAAU,SAAS,CAAC;AAC1B,UAAM,QAAQ;AAAA,iBAAoB,GAAG;AAAA,EAAS,OAAO;AACrD,QAAI,QAAQ,MAAM,SAAS,gBAAgB;AACzC,aAAO,KAAK;AAAA,UAAQ,OAAO,SAAS,OAAO,MAAM,0DAAqD;AACtG;AAAA,IACF;AACA,WAAO,KAAK,KAAK;AACjB,aAAS,MAAM;AAAA,EACjB;AACA,SAAO,OAAO,KAAK,EAAE;AACvB;AAEA,SAAS,oBAAoB,YAAoB,SAAyB;AACxE,QAAM,QAAQ,aAAa,OAAO;AAClC,QAAM,QAAkB,CAAC;AACzB,aAAW,KAAK,OAAO;AACrB,UAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;AACtC,UAAM,OAAO,IAAI,MAAM,GAAG,EAAE,IAAI,KAAK;AACrC,QAAI,WAAW,SAAS,GAAG,KAAK,WAAW,SAAS,IAAI,EAAG,OAAM,KAAK,CAAC;AAAA,EACzE;AACA,MAAI,MAAM,WAAW,GAAG;AACtB,WAAO;AAAA,EACT;AACA,QAAM,SAAmB,CAAC;AAC1B,aAAW,KAAK,OAAO;AACrB,UAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;AACtC,WAAO,KAAK;AAAA,iBAAoB,GAAG;AAAA,EAAS,SAAS,CAAC,CAAC,EAAE;AAAA,EAC3D;AACA,SAAO,OAAO,KAAK,EAAE;AACvB;AAIA,SAAS,qBAAqB,aAA+D;AAC3F,QAAM,QAAQ,YAAY,MAAM,IAAI;AACpC,QAAM,cAAc,MAAM,OAAiB,CAAC,KAAK,GAAG,MAAM;AACxD,QAAI,SAAS,KAAK,EAAE,KAAK,CAAC,EAAG,KAAI,KAAK,CAAC;AACvC,WAAO;AAAA,EACT,GAAG,CAAC,CAAC;AACL,SAAO,YAAY,IAAI,CAAC,OAAO,MAAM;AACnC,UAAM,MAAM,IAAI,IAAI,YAAY,SAAS,YAAY,IAAI,CAAC,IAAI,MAAM;AACpE,UAAM,QAAQ,MAAM,MAAM,OAAO,GAAG,EAAE,KAAK,IAAI;AAC/C,UAAM,QAAQ,MAAM,KAAK,EAAE,QAAQ,aAAa,EAAE,EAAE,KAAK;AACzD,UAAM,UAAU,MAAM,MAAM,oBAAoB,KAAK,CAAC,GAAG;AACzD,WAAO,EAAE,OAAO,OAAO;AAAA,EACzB,CAAC;AACH;AAEA,SAAS,kBAAkB,QAAwB;AACjD,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,GAAG,CAAC,CAAC;AAC5C;AAUA,SAAS,cAA2B;AAClC,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,4DAA4D;AACtF,QAAM,QAAQ,QAAQ,IAAI,eAAe;AACzC,QAAM,UAAU,QAAQ,IAAI,eAAe;AAC3C,SAAO,EAAE,SAAS,KAAK,MAAM;AAC/B;AAEA,SAAS,kBAAkB,KAAsB;AAC/C,MAAI,IAAI,IAAI,KAAK;AACjB,MAAI,EAAE,WAAW,KAAK,EAAG,KAAI,EAAE,MAAM,IAAI,EAAE,MAAM,CAAC,EAAE,KAAK,IAAI;AAC7D,MAAI,EAAE,SAAS,KAAK,EAAG,KAAI,EAAE,MAAM,GAAG,EAAE,YAAY,KAAK,CAAC;AAC1D,MAAI,EAAE,QAAQ,gBAAgB,IAAI,EAAE,KAAK;AACzC,MAAI;AACF,WAAO,KAAK,MAAM,CAAC;AAAA,EACrB,QAAQ;AACN,QAAI;AACF,YAAM,IAAI,EAAE,MAAM,aAAa;AAC/B,UAAI,EAAG,QAAO,KAAK,MAAM,EAAE,CAAC,CAAC;AAAA,IAC/B,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAe,UAAU,QAAqB,QAAgB,cAAc,GAAqB;AAC/F,WAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,UAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,MAC5D,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,MACrF,MAAM,KAAK,UAAU;AAAA,QACnB,OAAO,OAAO;AAAA,QACd,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,MAC9C,CAAC;AAAA,IACH,CAAC;AACD,QAAI,CAAC,IAAI,IAAI;AACX,UAAI,IAAI,cAAc,EAAG;AACzB,YAAM,IAAI,MAAM,oBAAoB,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACvF;AACA,UAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,UAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,QAAI,OAAO,YAAY,SAAU;AACjC,UAAM,SAAS,kBAAkB,OAAO;AACxC,QAAI,WAAW,KAAM,QAAO;AAAA,EAC9B;AACA,SAAO;AACT;AAuBA,eAAe,wBACb,MACA,QACA,QACsB;AACtB,QAAM,aAAa,oBAAoB,QAAQ,KAAK,OAAO;AAC3D,QAAM,cACJ;AAAA;AAAA;AAAA,EAEiB,UAAU;AAAA;AAC7B,QAAM,SAAS,KAAK,aACjB,QAAQ,kBAAkB,WAAW,EACrC,QAAQ,kBAAkB,MAAM;AACnC,QAAM,SAAS,MAAM,UAAU,QAAQ,MAAM;AAC7C,SAAQ,UAA0B,EAAE,yBAAyB,OAAO,qBAAqB,CAAC,EAAE;AAC9F;AAEA,eAAe,kBACb,MACA,QACA,eACA,QACsB;AACtB,QAAM,cACJ,iBAAiB,cAAc,SAAS,IACpC,KAAK,UAAU,eAAe,MAAM,CAAC,IACrC;AACN,QAAM,SAAS,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,EAC1C,QAAQ,kBAAkB,MAAM,EAChC,QAAQ,0BAA0B,WAAW,EAC7C,QAAQ,gBAAgB,OAAO,KAAK,gBAAgB,MAAM,CAAC;AAC9D,QAAM,SAAS,MAAM,UAAU,QAAQ,MAAM;AAC7C,SAAQ,UAA0B,EAAE,QAAQ,CAAC,EAAE;AACjD;AAEA,eAAe,oBACb,MACA,QACA,QACsB;AACtB,QAAM,SAAS,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,EAC1C,QAAQ,kBAAkB,MAAM;AACnC,QAAM,SAAS,MAAM,UAAU,QAAQ,MAAM;AAC7C,SAAQ,UAA0B,EAAE,iBAAiB,KAAK;AAC5D;AAEA,eAAe,eAAe,MAAqB,QAAgB,QAA0C;AAC3G,MAAI,CAAC,OAAO,KAAK,GAAG;AAClB,WAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,KAAK,UAAU,EAAE,OAAO,eAAe,CAAC,EAAE;AAAA,EACxF;AAGA,QAAM,KAAK,MAAM,wBAAwB,MAAM,QAAQ,MAAM;AAC7D,QAAM,gBAAgB,GAAG,uBAAuB,CAAC;AAGjD,QAAM,CAAC,IAAI,EAAE,IAAI,MAAM,QAAQ,IAAI;AAAA,IACjC,kBAAkB,MAAM,QAAQ,eAAe,MAAM;AAAA,IACrD,oBAAoB,MAAM,QAAQ,MAAM;AAAA,EAC1C,CAAC;AAED,QAAM,cAAc,GAAG,UAAU,CAAC;AAClC,QAAM,YAAsB,CAAC;AAC7B,QAAM,eAAyB,CAAC;AAEhC,WAAS,IAAI,GAAG,IAAI,KAAK,gBAAgB,QAAQ,KAAK;AACpD,UAAM,EAAE,OAAO,OAAO,IAAI,KAAK,gBAAgB,CAAC;AAChD,UAAM,QAAQ,YAAY,CAAC,KAAK,CAAC;AACjC,UAAM,YAAY,MAAM,aAAa,CAAC,GAAG,IAAI,CAAC,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC;AACpF,UAAM,UAAU,IAAI,KAAK,MAAM,iBAAiB,CAAC,GAAG,IAAI,CAAC,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC,CAAC;AAChG,UAAM,WAAW,SAAS,OAAO,CAAC,MAAc,CAAC,QAAQ,IAAI,CAAC,CAAC;AAC/D,UAAM,YAAY,kBAAkB,MAAM;AAC1C,QAAI,SAAS,UAAU,UAAW,WAAU,KAAK,KAAK;AAAA,QACjD,cAAa,KAAK,KAAK;AAAA,EAC9B;AAEA,QAAM,YAAY,KAAK,gBAAgB;AACvC,QAAM,QAAQ,GAAG;AACjB,QAAM,eAAe,OAAO,UAAU,YAAY,MAAM,SAAS,KAAK,UAAU;AAEhF,MAAI;AACJ,MAAI,cAAc,EAAG,YAAW;AAAA,WACvB,UAAU,WAAW,aAAa,aAAc,YAAW;AAAA,WAC3D,UAAU,WAAW,UAAW,YAAW;AAAA,WAC3C,UAAU,UAAU,EAAG,YAAW;AAAA,WAClC,UAAU,UAAU,EAAG,YAAW;AAAA,MACtC,YAAW;AAEhB,QAAM,UAAU,GAAG;AACnB,QAAM,iBAAiB,YAAY,QAAQ,YAAY;AAEvD,SAAO;AAAA,IACL,UAAU,aAAa;AAAA,IACvB,OAAO,YAAY,IAAI,WAAW,IAAI;AAAA,IACtC,QAAQ,KAAK,UAAU;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,oBAAoB,cAAc;AAAA,MAClC,WAAW,SAAS;AAAA,MACpB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AAIA,SAAS,YAAoB;AAC3B,SAAO,QAAQ,IAAI,oBAAoB;AACzC;AAEA,SAAS,SAAS,QAA0D;AAC1E,QAAM,UAAU,KAAK,UAAU,GAAG,gBAAgB,MAAM;AACxD,QAAM,UAAU,KAAK,SAAS,eAAe,MAAM;AACnD,QAAM,WAAW,KAAK,SAAS,OAAO;AAEtC,QAAM,cAAc,SAAS,KAAK,SAAS,gBAAgB,GAAG,QAAQ;AACtE,QAAM,YAAY,eAAe,OAAO;AACxC,QAAM,cAAc,SAAS,KAAK,UAAU,kBAAkB,GAAG,QAAQ;AACzE,QAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;AACjF,QAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;AACjF,QAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;AAEjF,QAAM,kBAAkB,qBAAqB,WAAW;AAGxD,QAAM,SAAS;AAAA,IACb,YACG;AAAA,MACC;AAAA,MACA;AAAA,IACF,EACC,QAAQ,uDAAuD,iCAAiC,EAChG,QAAQ,oBAAoB,sBAAsB;AAAA,IACrD;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,OAAsB;AAAA,IAC1B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAGA,QAAM,OAAkB;AAAA,IACtB,IAAI;AAAA,IACJ;AAAA,IACA,UAAU;AAAA,EACZ;AAEA,SAAO,EAAE,MAAM,KAAK;AACtB;AAEA,SAAS,SAAS,MAAgC;AAChD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,YAAY,YAAY,OAAO,GAAG,gBAAgB,UAAU;AAC/E,UAAM,IAAI,MAAM,cAAc,KAAK,EAAE,+DAA0D;AAAA,EACjG;AACA,SAAO;AACT;AAGA,SAAS,YAAY,MAAwB;AAC3C,MAAI;AACF,WAAO,YAAY,KAAK,MAAM,cAAc,CAAC,EAC1C,OAAO,CAAC,SAAS;AAChB,UAAI;AACF,eAAO,SAAS,KAAK,MAAM,gBAAgB,IAAI,CAAC,EAAE,YAAY;AAAA,MAChE,QAAQ;AACN,eAAO;AAAA,MACT;AAAA,IACF,CAAC,EACA,KAAK;AAAA,EACV,SAAS,KAAK;AACZ,UAAM,IAAI;AAAA,MACR,6CAA6C,KAAK,MAAM,cAAc,CAAC,KAClE,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,qFAC2C,IAAI;AAAA,IAC9F;AAAA,EACF;AACF;AAIA,SAAS,gBAAgB,UAA0B;AAGjD,MAAI,SAAS,WAAW,WAAW,EAAG,QAAO,SAAS,MAAM,YAAY,MAAM,EAAE,KAAK;AACrF,QAAM,SAAS,SAAS,QAAQ,aAAa;AAC7C,MAAI,WAAW,GAAI,QAAO,SAAS,MAAM,SAAS,cAAc,MAAM,EAAE,KAAK;AAC7E,SAAO;AACT;AAIO,SAAS,0BAA4C;AAC1D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,OAAO,UAAU;AACvB,YAAM,SAAS,KAAK,MAAM,cAAc;AACxC,UAAI;AACF,cAAM,OAAO,SAAS,MAAM;AAC5B,YAAI,CAAC,KAAK,YAAY,EAAG,OAAM,IAAI,MAAM,iBAAiB;AAAA,MAC5D,QAAQ;AACN,cAAM,IAAI;AAAA,UACR,kCAAkC,MAAM;AAAA,4EACuC,IAAI;AAAA,QACrF;AAAA,MACF;AAEA,kBAAY;AACZ,YAAM,MAAM,YAAY,IAAI;AAC5B,UAAI,IAAI,WAAW,EAAG,OAAM,IAAI,MAAM,mCAAmC,MAAM,EAAE;AACjF,cAAQ;AAAA,QACN,iBAAiB,IAAI,MAAM,sBAAsB,IAAI,sFAErC,QAAQ,IAAI,eAAe,WAAW;AAAA,MACxD;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;AAC5D,YAAM,OAAO,UAAU;AACvB,UAAI,MAAM,YAAY,IAAI;AAC1B,UAAI,KAAK,KAAK;AACZ,cAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,cAAM,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;AAAA,MACvC,WAAW,KAAK,UAAU,QAAW;AACnC,cAAM,IAAI,MAAM,GAAG,KAAK,KAAK;AAAA,MAC/B;AACA,aAAO,IAAI,IAAI,CAAC,OAAO,SAAS,EAAE,EAAE,IAAI;AAAA,IAC1C;AAAA,IAEA,MAAM,aAAa,OAA+C;AAIhE,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,SAAS,gBAAgB,QAAQ;AACvC,aAAO,eAAe,MAAM,QAAQ,YAAY,CAAC;AAAA,IACnD;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-WG7TM7UV.js

@@ -1,30 +0,0 @@
-import {
-  createTauBenchAdapter,
-  tauResultsOutput
-} from "./chunk-GCHL6XPM.js";
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/tau2-bench.ts
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "tau2-bench.json");
-var tau2ResultsOutput = tauResultsOutput;
-function createTau2BenchAdapter() {
-  return createTauBenchAdapter({
-    name: "tau2-bench",
-    fixturePath: FIXTURES,
-    fixturesEnv: "TAU2_FIXTURES",
-    dirEnv: "TAU2_BENCH_DIR",
-    domainEnv: "TAU2_DOMAIN",
-    defaultDomain: "retail",
-    taskIntro: "Run this tau2 task in the official tau2 text benchmark.",
-    installHint: "clone https://github.com/sierra-research/tau2-bench, install its deps in bench/.venv, and set TAU2_BENCH_DIR=/path/to/tau2-bench."
-  });
-}
-
-export {
-  tau2ResultsOutput,
-  createTau2BenchAdapter
-};
-//# sourceMappingURL=chunk-WG7TM7UV.js.map

package/dist/chunk-WG7TM7UV.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/tau2-bench.ts"],"sourcesContent":["/**\n * tau2-bench adapter (Sierra tau2/tau-bench successor).\n *\n * Worker artifact = a tau2 `results.json`/trajectory file. Judge = tau2's own\n * reward recomputation over that trajectory. A normal final-answer transcript is\n * not a valid artifact for this benchmark.\n */\n\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport { createTauBenchAdapter, tauResultsOutput } from './tau-bench-shared'\nimport type { BenchmarkAdapter } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'tau2-bench.json')\n\nexport const tau2ResultsOutput = tauResultsOutput\n\nexport function createTau2BenchAdapter(): BenchmarkAdapter {\n  return createTauBenchAdapter({\n    name: 'tau2-bench',\n    fixturePath: FIXTURES,\n    fixturesEnv: 'TAU2_FIXTURES',\n    dirEnv: 'TAU2_BENCH_DIR',\n    domainEnv: 'TAU2_DOMAIN',\n    defaultDomain: 'retail',\n    taskIntro: 'Run this tau2 task in the official tau2 text benchmark.',\n    installHint:\n      'clone https://github.com/sierra-research/tau2-bench, install its deps in bench/.venv, and set TAU2_BENCH_DIR=/path/to/tau2-bench.',\n  })\n}\n"],"mappings":";;;;;;;;;AAQA,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,iBAAiB;AAEvD,IAAM,oBAAoB;AAE1B,SAAS,yBAA2C;AACzD,SAAO,sBAAsB;AAAA,IAC3B,MAAM;AAAA,IACN,aAAa;AAAA,IACb,aAAa;AAAA,IACb,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,eAAe;AAAA,IACf,WAAW;AAAA,IACX,aACE;AAAA,EACJ,CAAC;AACH;","names":[]}

package/dist/chunk-X3BTXCJ4.js

@@ -1,262 +0,0 @@
-// src/benchmarks/rag-shared.ts
-import { readFile } from "fs/promises";
-var FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
-var ragAnswerOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    return text.trim();
-  }
-};
-function parseFinalAnswer(artifact) {
-  const lines = artifact.split(/\r?\n/);
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const line = lines[i] ?? "";
-    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
-    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim();
-  }
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const trimmed = (lines[i] ?? "").trim();
-    if (trimmed.length > 0) return trimmed;
-  }
-  return "";
-}
-function parseCitations(artifact) {
-  const urls = /* @__PURE__ */ new Set();
-  for (const match of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) {
-    urls.add(match[0].replace(/[.,;]+$/, ""));
-  }
-  return [...urls];
-}
-function normalizeAnswer(input) {
-  return input.toLowerCase().replace(/(\d),(?=\d{3}\b)/g, "$1").replace(/[^\p{L}\p{N}\s.-]+/gu, " ").split(/\s+/).filter((token) => token.length > 0).filter((token) => !["a", "an", "the"].includes(token)).join(" ").trim();
-}
-function tokens(input) {
-  const normalized = normalizeAnswer(input);
-  return normalized.length === 0 ? [] : normalized.split(/\s+/);
-}
-function tokenF1(candidate, gold) {
-  const candidateTokens = tokens(candidate);
-  const goldTokens = tokens(gold);
-  if (candidateTokens.length === 0 || goldTokens.length === 0) {
-    return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0;
-  }
-  const counts = /* @__PURE__ */ new Map();
-  for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1);
-  let common = 0;
-  for (const token of candidateTokens) {
-    const left = counts.get(token);
-    if (left !== void 0 && left > 0) {
-      common += 1;
-      counts.set(token, left - 1);
-    }
-  }
-  if (common === 0) return 0;
-  const precision = common / candidateTokens.length;
-  const recall = common / goldTokens.length;
-  return 2 * precision * recall / (precision + recall);
-}
-function exactOrContains(candidate, gold) {
-  const normalizedCandidate = normalizeAnswer(candidate);
-  const normalizedGold = normalizeAnswer(gold);
-  if (normalizedGold.length === 0) return false;
-  if (normalizedCandidate === normalizedGold) return true;
-  const escaped = normalizedGold.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  return new RegExp(`(^|\\s)${escaped}(\\s|$)`).test(normalizedCandidate);
-}
-function numbers(input) {
-  return [...input.replace(/,/g, "").matchAll(/-?\d+(?:\.\d+)?/g)].map((match) => Number(match[0])).filter(Number.isFinite);
-}
-function numericMatch(candidate, gold, relativeTolerance) {
-  const got = numbers(candidate);
-  const want = numbers(gold);
-  if (got.length === 0 || want.length === 0) return false;
-  return want.some(
-    (expected) => got.some((actual) => {
-      const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance);
-      return Math.abs(actual - expected) <= tolerance;
-    })
-  );
-}
-function scoreAnswerArtifact(artifact, golds, options = {}) {
-  const finalAnswer = parseFinalAnswer(artifact);
-  const threshold = options.threshold ?? 0.72;
-  const numericTolerance = options.numericTolerance ?? 0.01;
-  let best = {
-    resolved: false,
-    score: 0,
-    finalAnswer,
-    bestGold: null,
-    exact: false,
-    numeric: false,
-    f1: 0,
-    threshold
-  };
-  if (finalAnswer.length === 0 || golds.length === 0) return best;
-  for (const gold of golds) {
-    const exact = exactOrContains(finalAnswer, gold);
-    const numeric = numericMatch(finalAnswer, gold, numericTolerance);
-    const f1 = tokenF1(finalAnswer, gold);
-    const resolved = exact || numeric || f1 >= threshold;
-    const score = exact || numeric ? 1 : f1;
-    if (score > best.score || resolved && !best.resolved) {
-      best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold };
-    }
-  }
-  return best;
-}
-function answerScoreToBenchScore(score, detail) {
-  return {
-    resolved: score.resolved,
-    score: score.score,
-    detail: JSON.stringify({
-      ...detail,
-      finalAnswer: score.finalAnswer,
-      bestGold: score.bestGold,
-      exact: score.exact,
-      numeric: score.numeric,
-      f1: score.f1,
-      threshold: score.threshold
-    })
-  };
-}
-async function readJsonRows(path) {
-  const raw = (await readFile(path, "utf8")).trim();
-  if (raw.length === 0) return [];
-  if (raw.startsWith("[")) {
-    const parsed = JSON.parse(raw);
-    if (Array.isArray(parsed)) return parsed;
-    throw new Error(`${path} must contain a JSON array when it starts with [`);
-  }
-  if (raw.startsWith("{")) {
-    try {
-      const parsed = JSON.parse(raw);
-      if (isObject(parsed)) {
-        for (const key of ["rows", "data", "examples", "items"]) {
-          const value = parsed[key];
-          if (Array.isArray(value)) return value;
-        }
-        return [parsed];
-      }
-      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`);
-    } catch (err) {
-      if (!raw.includes("\n")) throw err;
-    }
-  }
-  return raw.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
-}
-function selectTasks(tasks, opts, label) {
-  let selected = tasks;
-  if (opts.split) selected = selected.filter((task) => task.split === opts.split);
-  if (opts.ids) {
-    const ids = new Set(opts.ids);
-    selected = selected.filter((task) => ids.has(task.id));
-  } else if (opts.limit !== void 0) {
-    selected = selected.slice(0, opts.limit);
-  }
-  if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`);
-  return selected;
-}
-function stringFrom(value) {
-  return typeof value === "string" && value.trim().length > 0 ? value.trim() : void 0;
-}
-function stringArrayFrom(value) {
-  if (typeof value === "string" && value.trim().length > 0) return [value.trim()];
-  if (!Array.isArray(value)) return [];
-  return value.flatMap((entry) => {
-    if (typeof entry === "string" && entry.trim().length > 0) return [entry.trim()];
-    if (isObject(entry)) {
-      const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text);
-      return text ? [text] : [];
-    }
-    return [];
-  });
-}
-function firstString(row, keys) {
-  for (const key of keys) {
-    const value = stringFrom(row[key]);
-    if (value) return value;
-  }
-  return "";
-}
-function allStrings(row, keys) {
-  const out = [];
-  for (const key of keys) out.push(...stringArrayFrom(row[key]));
-  return [...new Set(out)];
-}
-function contextsFrom(value) {
-  if (typeof value === "string" && value.trim().length > 0) {
-    return [{ id: "ctx-1", text: value.trim() }];
-  }
-  if (!Array.isArray(value)) return [];
-  return value.flatMap((entry, index) => {
-    if (typeof entry === "string" && entry.trim().length > 0) {
-      return [{ id: `ctx-${index + 1}`, text: entry.trim() }];
-    }
-    if (Array.isArray(entry)) {
-      const [first, second] = entry;
-      if (typeof first === "string" && typeof second === "string") {
-        return [{ id: first, text: second.trim() }];
-      }
-      return entry.flatMap(
-        (nested, nestedIndex) => contextsFrom([nested]).map((ctx) => ({
-          ...ctx,
-          id: ctx.id.startsWith("ctx-") ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id
-        }))
-      );
-    }
-    if (!isObject(entry)) return [];
-    const text = stringFrom(entry.text) ?? stringFrom(entry.context) ?? stringFrom(entry.content) ?? stringFrom(entry.passage) ?? stringFrom(entry.document) ?? stringFrom(entry.page_content) ?? stringFrom(entry.chunk);
-    if (!text) return [];
-    const id = stringFrom(entry.id) ?? stringFrom(entry.docid) ?? stringFrom(entry.doc_id) ?? stringFrom(entry.document_id) ?? `ctx-${index + 1}`;
-    const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance;
-    const relevant = typeof relevantRaw === "boolean" ? relevantRaw : typeof relevantRaw === "number" ? relevantRaw > 0 : void 0;
-    return [
-      {
-        id,
-        text,
-        ...stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {},
-        ...stringFrom(entry.source) ?? stringFrom(entry.url) ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) } : {},
-        ...relevant !== void 0 ? { relevant } : {}
-      }
-    ];
-  });
-}
-function contextBlock(contexts) {
-  if (contexts.length === 0) return "";
-  return contexts.map(
-    (ctx, index) => [
-      `[${index + 1}] ${ctx.title ?? ctx.id}`,
-      ctx.source ? `Source: ${ctx.source}` : void 0,
-      ctx.text
-    ].filter(Boolean).join("\n")
-  ).join("\n\n");
-}
-function isObject(value) {
-  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
-}
-
-export {
-  FINAL_ANSWER_SENTINEL,
-  ragAnswerOutput,
-  parseFinalAnswer,
-  parseCitations,
-  normalizeAnswer,
-  tokenF1,
-  scoreAnswerArtifact,
-  answerScoreToBenchScore,
-  readJsonRows,
-  selectTasks,
-  stringFrom,
-  stringArrayFrom,
-  firstString,
-  allStrings,
-  contextsFrom,
-  contextBlock,
-  isObject
-};
-//# sourceMappingURL=chunk-X3BTXCJ4.js.map

package/dist/chunk-X3BTXCJ4.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/rag-shared.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport type { BenchScore, BenchTask, LoadOptions } from './types'\n\nexport const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\nexport interface RagContext {\n  id: string\n  text: string\n  title?: string\n  source?: string\n  relevant?: boolean\n}\n\nexport interface RagAnswerScore {\n  resolved: boolean\n  score: number\n  finalAnswer: string\n  bestGold: string | null\n  exact: boolean\n  numeric: boolean\n  f1: number\n  threshold: number\n}\n\nexport const ragAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const trimmed = (lines[i] ?? '').trim()\n    if (trimmed.length > 0) return trimmed\n  }\n  return ''\n}\n\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const match of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(match[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\nexport function normalizeAnswer(input: string): string {\n  return input\n    .toLowerCase()\n    .replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n    .replace(/[^\\p{L}\\p{N}\\s.-]+/gu, ' ')\n    .split(/\\s+/)\n    .filter((token) => token.length > 0)\n    .filter((token) => !['a', 'an', 'the'].includes(token))\n    .join(' ')\n    .trim()\n}\n\nfunction tokens(input: string): string[] {\n  const normalized = normalizeAnswer(input)\n  return normalized.length === 0 ? [] : normalized.split(/\\s+/)\n}\n\nexport function tokenF1(candidate: string, gold: string): number {\n  const candidateTokens = tokens(candidate)\n  const goldTokens = tokens(gold)\n  if (candidateTokens.length === 0 || goldTokens.length === 0) {\n    return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const counts = new Map<string, number>()\n  for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1)\n  let common = 0\n  for (const token of candidateTokens) {\n    const left = counts.get(token)\n    if (left !== undefined && left > 0) {\n      common += 1\n      counts.set(token, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / candidateTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\nfunction exactOrContains(candidate: string, gold: string): boolean {\n  const normalizedCandidate = normalizeAnswer(candidate)\n  const normalizedGold = normalizeAnswer(gold)\n  if (normalizedGold.length === 0) return false\n  if (normalizedCandidate === normalizedGold) return true\n  const escaped = normalizedGold.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n  return new RegExp(`(^|\\\\s)${escaped}(\\\\s|$)`).test(normalizedCandidate)\n}\n\nfunction numbers(input: string): number[] {\n  return [...input.replace(/,/g, '').matchAll(/-?\\d+(?:\\.\\d+)?/g)]\n    .map((match) => Number(match[0]))\n    .filter(Number.isFinite)\n}\n\nfunction numericMatch(candidate: string, gold: string, relativeTolerance: number): boolean {\n  const got = numbers(candidate)\n  const want = numbers(gold)\n  if (got.length === 0 || want.length === 0) return false\n  return want.some((expected) =>\n    got.some((actual) => {\n      const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance)\n      return Math.abs(actual - expected) <= tolerance\n    }),\n  )\n}\n\nexport function scoreAnswerArtifact(\n  artifact: string,\n  golds: readonly string[],\n  options: { threshold?: number; numericTolerance?: number } = {},\n): RagAnswerScore {\n  const finalAnswer = parseFinalAnswer(artifact)\n  const threshold = options.threshold ?? 0.72\n  const numericTolerance = options.numericTolerance ?? 0.01\n  let best: RagAnswerScore = {\n    resolved: false,\n    score: 0,\n    finalAnswer,\n    bestGold: null,\n    exact: false,\n    numeric: false,\n    f1: 0,\n    threshold,\n  }\n  if (finalAnswer.length === 0 || golds.length === 0) return best\n\n  for (const gold of golds) {\n    const exact = exactOrContains(finalAnswer, gold)\n    const numeric = numericMatch(finalAnswer, gold, numericTolerance)\n    const f1 = tokenF1(finalAnswer, gold)\n    const resolved = exact || numeric || f1 >= threshold\n    const score = exact || numeric ? 1 : f1\n    if (score > best.score || (resolved && !best.resolved)) {\n      best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold }\n    }\n  }\n  return best\n}\n\nexport function answerScoreToBenchScore(\n  score: RagAnswerScore,\n  detail: Record<string, unknown>,\n): BenchScore {\n  return {\n    resolved: score.resolved,\n    score: score.score,\n    detail: JSON.stringify({\n      ...detail,\n      finalAnswer: score.finalAnswer,\n      bestGold: score.bestGold,\n      exact: score.exact,\n      numeric: score.numeric,\n      f1: score.f1,\n      threshold: score.threshold,\n    }),\n  }\n}\n\nexport async function readJsonRows(path: string): Promise<unknown[]> {\n  const raw = (await readFile(path, 'utf8')).trim()\n  if (raw.length === 0) return []\n  if (raw.startsWith('[')) {\n    const parsed = JSON.parse(raw) as unknown\n    if (Array.isArray(parsed)) return parsed\n    throw new Error(`${path} must contain a JSON array when it starts with [`)\n  }\n  if (raw.startsWith('{')) {\n    try {\n      const parsed = JSON.parse(raw) as unknown\n      if (isObject(parsed)) {\n        for (const key of ['rows', 'data', 'examples', 'items']) {\n          const value = parsed[key]\n          if (Array.isArray(value)) return value\n        }\n        return [parsed]\n      }\n      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`)\n    } catch (err) {\n      if (!raw.includes('\\n')) throw err\n    }\n  }\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nexport function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[] {\n  let selected = tasks\n  if (opts.split) selected = selected.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const ids = new Set(opts.ids)\n    selected = selected.filter((task) => ids.has(task.id))\n  } else if (opts.limit !== undefined) {\n    selected = selected.slice(0, opts.limit)\n  }\n  if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`)\n  return selected\n}\n\nexport function stringFrom(value: unknown): string | undefined {\n  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined\n}\n\nexport function stringArrayFrom(value: unknown): string[] {\n  if (typeof value === 'string' && value.trim().length > 0) return [value.trim()]\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry) => {\n    if (typeof entry === 'string' && entry.trim().length > 0) return [entry.trim()]\n    if (isObject(entry)) {\n      const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text)\n      return text ? [text] : []\n    }\n    return []\n  })\n}\n\nexport function firstString(row: Record<string, unknown>, keys: readonly string[]): string {\n  for (const key of keys) {\n    const value = stringFrom(row[key])\n    if (value) return value\n  }\n  return ''\n}\n\nexport function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[] {\n  const out: string[] = []\n  for (const key of keys) out.push(...stringArrayFrom(row[key]))\n  return [...new Set(out)]\n}\n\nexport function contextsFrom(value: unknown): RagContext[] {\n  if (typeof value === 'string' && value.trim().length > 0) {\n    return [{ id: 'ctx-1', text: value.trim() }]\n  }\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry, index): RagContext[] => {\n    if (typeof entry === 'string' && entry.trim().length > 0) {\n      return [{ id: `ctx-${index + 1}`, text: entry.trim() }]\n    }\n    if (Array.isArray(entry)) {\n      const [first, second] = entry\n      if (typeof first === 'string' && typeof second === 'string') {\n        return [{ id: first, text: second.trim() }]\n      }\n      return entry.flatMap((nested, nestedIndex) =>\n        contextsFrom([nested]).map((ctx) => ({\n          ...ctx,\n          id: ctx.id.startsWith('ctx-') ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id,\n        })),\n      )\n    }\n    if (!isObject(entry)) return []\n    const text =\n      stringFrom(entry.text) ??\n      stringFrom(entry.context) ??\n      stringFrom(entry.content) ??\n      stringFrom(entry.passage) ??\n      stringFrom(entry.document) ??\n      stringFrom(entry.page_content) ??\n      stringFrom(entry.chunk)\n    if (!text) return []\n    const id =\n      stringFrom(entry.id) ??\n      stringFrom(entry.docid) ??\n      stringFrom(entry.doc_id) ??\n      stringFrom(entry.document_id) ??\n      `ctx-${index + 1}`\n    const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance\n    const relevant =\n      typeof relevantRaw === 'boolean'\n        ? relevantRaw\n        : typeof relevantRaw === 'number'\n          ? relevantRaw > 0\n          : undefined\n    return [\n      {\n        id,\n        text,\n        ...(stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {}),\n        ...(stringFrom(entry.source) ?? stringFrom(entry.url)\n          ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) }\n          : {}),\n        ...(relevant !== undefined ? { relevant } : {}),\n      },\n    ]\n  })\n}\n\nexport function contextBlock(contexts: readonly RagContext[]): string {\n  if (contexts.length === 0) return ''\n  return contexts\n    .map((ctx, index) =>\n      [\n        `[${index + 1}] ${ctx.title ?? ctx.id}`,\n        ctx.source ? `Source: ${ctx.source}` : undefined,\n        ctx.text,\n      ]\n        .filter(Boolean)\n        .join('\\n'),\n    )\n    .join('\\n\\n')\n}\n\nexport function isObject(value: unknown): value is Record<string, unknown> {\n  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AAIlB,IAAM,wBAAwB;AAqB9B,IAAM,kBAAyC;AAAA,EACpD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEO,SAAS,iBAAiB,UAA0B;AACzD,QAAM,QAAQ,SAAS,MAAM,OAAO;AACpC,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,OAAO,MAAM,CAAC,KAAK;AACzB,UAAM,MAAM,KAAK,YAAY,EAAE,QAAQ,qBAAqB;AAC5D,QAAI,QAAQ,GAAI,QAAO,KAAK,MAAM,MAAM,sBAAsB,MAAM,EAAE,KAAK;AAAA,EAC7E;AACA,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,WAAW,MAAM,CAAC,KAAK,IAAI,KAAK;AACtC,QAAI,QAAQ,SAAS,EAAG,QAAO;AAAA,EACjC;AACA,SAAO;AACT;AAEO,SAAS,eAAe,UAA4B;AACzD,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,SAAS,SAAS,SAAS,yBAAyB,GAAG;AAChE,SAAK,IAAI,MAAM,CAAC,EAAE,QAAQ,WAAW,EAAE,CAAC;AAAA,EAC1C;AACA,SAAO,CAAC,GAAG,IAAI;AACjB;AAEO,SAAS,gBAAgB,OAAuB;AACrD,SAAO,MACJ,YAAY,EACZ,QAAQ,qBAAqB,IAAI,EACjC,QAAQ,wBAAwB,GAAG,EACnC,MAAM,KAAK,EACX,OAAO,CAAC,UAAU,MAAM,SAAS,CAAC,EAClC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC,EACrD,KAAK,GAAG,EACR,KAAK;AACV;AAEA,SAAS,OAAO,OAAyB;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,WAAW,WAAW,IAAI,CAAC,IAAI,WAAW,MAAM,KAAK;AAC9D;AAEO,SAAS,QAAQ,WAAmB,MAAsB;AAC/D,QAAM,kBAAkB,OAAO,SAAS;AACxC,QAAM,aAAa,OAAO,IAAI;AAC9B,MAAI,gBAAgB,WAAW,KAAK,WAAW,WAAW,GAAG;AAC3D,WAAO,gBAAgB,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;AAAA,EACvE;AACA,QAAM,SAAS,oBAAI,IAAoB;AACvC,aAAW,SAAS,WAAY,QAAO,IAAI,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;AAC9E,MAAI,SAAS;AACb,aAAW,SAAS,iBAAiB;AACnC,UAAM,OAAO,OAAO,IAAI,KAAK;AAC7B,QAAI,SAAS,UAAa,OAAO,GAAG;AAClC,gBAAU;AACV,aAAO,IAAI,OAAO,OAAO,CAAC;AAAA,IAC5B;AAAA,EACF;AACA,MAAI,WAAW,EAAG,QAAO;AACzB,QAAM,YAAY,SAAS,gBAAgB;AAC3C,QAAM,SAAS,SAAS,WAAW;AACnC,SAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;AAEA,SAAS,gBAAgB,WAAmB,MAAuB;AACjE,QAAM,sBAAsB,gBAAgB,SAAS;AACrD,QAAM,iBAAiB,gBAAgB,IAAI;AAC3C,MAAI,eAAe,WAAW,EAAG,QAAO;AACxC,MAAI,wBAAwB,eAAgB,QAAO;AACnD,QAAM,UAAU,eAAe,QAAQ,uBAAuB,MAAM;AACpE,SAAO,IAAI,OAAO,UAAU,OAAO,SAAS,EAAE,KAAK,mBAAmB;AACxE;AAEA,SAAS,QAAQ,OAAyB;AACxC,SAAO,CAAC,GAAG,MAAM,QAAQ,MAAM,EAAE,EAAE,SAAS,kBAAkB,CAAC,EAC5D,IAAI,CAAC,UAAU,OAAO,MAAM,CAAC,CAAC,CAAC,EAC/B,OAAO,OAAO,QAAQ;AAC3B;AAEA,SAAS,aAAa,WAAmB,MAAc,mBAAoC;AACzF,QAAM,MAAM,QAAQ,SAAS;AAC7B,QAAM,OAAO,QAAQ,IAAI;AACzB,MAAI,IAAI,WAAW,KAAK,KAAK,WAAW,EAAG,QAAO;AAClD,SAAO,KAAK;AAAA,IAAK,CAAC,aAChB,IAAI,KAAK,CAAC,WAAW;AACnB,YAAM,YAAY,KAAK,IAAI,KAAK,IAAI,QAAQ,IAAI,mBAAmB,iBAAiB;AACpF,aAAO,KAAK,IAAI,SAAS,QAAQ,KAAK;AAAA,IACxC,CAAC;AAAA,EACH;AACF;AAEO,SAAS,oBACd,UACA,OACA,UAA6D,CAAC,GAC9C;AAChB,QAAM,cAAc,iBAAiB,QAAQ;AAC7C,QAAM,YAAY,QAAQ,aAAa;AACvC,QAAM,mBAAmB,QAAQ,oBAAoB;AACrD,MAAI,OAAuB;AAAA,IACzB,UAAU;AAAA,IACV,OAAO;AAAA,IACP;AAAA,IACA,UAAU;AAAA,IACV,OAAO;AAAA,IACP,SAAS;AAAA,IACT,IAAI;AAAA,IACJ;AAAA,EACF;AACA,MAAI,YAAY,WAAW,KAAK,MAAM,WAAW,EAAG,QAAO;AAE3D,aAAW,QAAQ,OAAO;AACxB,UAAM,QAAQ,gBAAgB,aAAa,IAAI;AAC/C,UAAM,UAAU,aAAa,aAAa,MAAM,gBAAgB;AAChE,UAAM,KAAK,QAAQ,aAAa,IAAI;AACpC,UAAM,WAAW,SAAS,WAAW,MAAM;AAC3C,UAAM,QAAQ,SAAS,UAAU,IAAI;AACrC,QAAI,QAAQ,KAAK,SAAU,YAAY,CAAC,KAAK,UAAW;AACtD,aAAO,EAAE,UAAU,OAAO,aAAa,UAAU,MAAM,OAAO,SAAS,IAAI,UAAU;AAAA,IACvF;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,wBACd,OACA,QACY;AACZ,SAAO;AAAA,IACL,UAAU,MAAM;AAAA,IAChB,OAAO,MAAM;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,MACrB,GAAG;AAAA,MACH,aAAa,MAAM;AAAA,MACnB,UAAU,MAAM;AAAA,MAChB,OAAO,MAAM;AAAA,MACb,SAAS,MAAM;AAAA,MACf,IAAI,MAAM;AAAA,MACV,WAAW,MAAM;AAAA,IACnB,CAAC;AAAA,EACH;AACF;AAEA,eAAsB,aAAa,MAAkC;AACnE,QAAM,OAAO,MAAM,SAAS,MAAM,MAAM,GAAG,KAAK;AAChD,MAAI,IAAI,WAAW,EAAG,QAAO,CAAC;AAC9B,MAAI,IAAI,WAAW,GAAG,GAAG;AACvB,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,QAAI,MAAM,QAAQ,MAAM,EAAG,QAAO;AAClC,UAAM,IAAI,MAAM,GAAG,IAAI,kDAAkD;AAAA,EAC3E;AACA,MAAI,IAAI,WAAW,GAAG,GAAG;AACvB,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,UAAI,SAAS,MAAM,GAAG;AACpB,mBAAW,OAAO,CAAC,QAAQ,QAAQ,YAAY,OAAO,GAAG;AACvD,gBAAM,QAAQ,OAAO,GAAG;AACxB,cAAI,MAAM,QAAQ,KAAK,EAAG,QAAO;AAAA,QACnC;AACA,eAAO,CAAC,MAAM;AAAA,MAChB;AACA,YAAM,IAAI,MAAM,GAAG,IAAI,iGAAiG;AAAA,IAC1H,SAAS,KAAK;AACZ,UAAI,CAAC,IAAI,SAAS,IAAI,EAAG,OAAM;AAAA,IACjC;AAAA,EACF;AACA,SAAO,IACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,EAChC,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEO,SAAS,YAAY,OAAoB,MAAmB,OAA4B;AAC7F,MAAI,WAAW;AACf,MAAI,KAAK,MAAO,YAAW,SAAS,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK,KAAK;AAC9E,MAAI,KAAK,KAAK;AACZ,UAAM,MAAM,IAAI,IAAI,KAAK,GAAG;AAC5B,eAAW,SAAS,OAAO,CAAC,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;AAAA,EACvD,WAAW,KAAK,UAAU,QAAW;AACnC,eAAW,SAAS,MAAM,GAAG,KAAK,KAAK;AAAA,EACzC;AACA,MAAI,SAAS,WAAW,EAAG,OAAM,IAAI,MAAM,GAAG,KAAK,sBAAsB,KAAK,UAAU,IAAI,CAAC,EAAE;AAC/F,SAAO;AACT;AAEO,SAAS,WAAW,OAAoC;AAC7D,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,MAAM,KAAK,IAAI;AAC/E;AAEO,SAAS,gBAAgB,OAA0B;AACxD,MAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,EAAG,QAAO,CAAC,MAAM,KAAK,CAAC;AAC9E,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM,QAAQ,CAAC,UAAU;AAC9B,QAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,EAAG,QAAO,CAAC,MAAM,KAAK,CAAC;AAC9E,QAAI,SAAS,KAAK,GAAG;AACnB,YAAM,OAAO,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,MAAM,IAAI;AACzF,aAAO,OAAO,CAAC,IAAI,IAAI,CAAC;AAAA,IAC1B;AACA,WAAO,CAAC;AAAA,EACV,CAAC;AACH;AAEO,SAAS,YAAY,KAA8B,MAAiC;AACzF,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,WAAW,IAAI,GAAG,CAAC;AACjC,QAAI,MAAO,QAAO;AAAA,EACpB;AACA,SAAO;AACT;AAEO,SAAS,WAAW,KAA8B,MAAmC;AAC1F,QAAM,MAAgB,CAAC;AACvB,aAAW,OAAO,KAAM,KAAI,KAAK,GAAG,gBAAgB,IAAI,GAAG,CAAC,CAAC;AAC7D,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAEO,SAAS,aAAa,OAA8B;AACzD,MAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,GAAG;AACxD,WAAO,CAAC,EAAE,IAAI,SAAS,MAAM,MAAM,KAAK,EAAE,CAAC;AAAA,EAC7C;AACA,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM,QAAQ,CAAC,OAAO,UAAwB;AACnD,QAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,GAAG;AACxD,aAAO,CAAC,EAAE,IAAI,OAAO,QAAQ,CAAC,IAAI,MAAM,MAAM,KAAK,EAAE,CAAC;AAAA,IACxD;AACA,QAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,YAAM,CAAC,OAAO,MAAM,IAAI;AACxB,UAAI,OAAO,UAAU,YAAY,OAAO,WAAW,UAAU;AAC3D,eAAO,CAAC,EAAE,IAAI,OAAO,MAAM,OAAO,KAAK,EAAE,CAAC;AAAA,MAC5C;AACA,aAAO,MAAM;AAAA,QAAQ,CAAC,QAAQ,gBAC5B,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,SAAS;AAAA,UACnC,GAAG;AAAA,UACH,IAAI,IAAI,GAAG,WAAW,MAAM,IAAI,OAAO,QAAQ,CAAC,IAAI,cAAc,CAAC,KAAK,IAAI;AAAA,QAC9E,EAAE;AAAA,MACJ;AAAA,IACF;AACA,QAAI,CAAC,SAAS,KAAK,EAAG,QAAO,CAAC;AAC9B,UAAM,OACJ,WAAW,MAAM,IAAI,KACrB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,QAAQ,KACzB,WAAW,MAAM,YAAY,KAC7B,WAAW,MAAM,KAAK;AACxB,QAAI,CAAC,KAAM,QAAO,CAAC;AACnB,UAAM,KACJ,WAAW,MAAM,EAAE,KACnB,WAAW,MAAM,KAAK,KACtB,WAAW,MAAM,MAAM,KACvB,WAAW,MAAM,WAAW,KAC5B,OAAO,QAAQ,CAAC;AAClB,UAAM,cAAc,MAAM,YAAY,MAAM,eAAe,MAAM,SAAS,MAAM;AAChF,UAAM,WACJ,OAAO,gBAAgB,YACnB,cACA,OAAO,gBAAgB,WACrB,cAAc,IACd;AACR,WAAO;AAAA,MACL;AAAA,QACE;AAAA,QACA;AAAA,QACA,GAAI,WAAW,MAAM,KAAK,IAAI,EAAE,OAAO,WAAW,MAAM,KAAK,EAAE,IAAI,CAAC;AAAA,QACpE,GAAI,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,IAChD,EAAE,QAAQ,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,EAAE,IAC5D,CAAC;AAAA,QACL,GAAI,aAAa,SAAY,EAAE,SAAS,IAAI,CAAC;AAAA,MAC/C;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEO,SAAS,aAAa,UAAyC;AACpE,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,SAAO,SACJ;AAAA,IAAI,CAAC,KAAK,UACT;AAAA,MACE,IAAI,QAAQ,CAAC,KAAK,IAAI,SAAS,IAAI,EAAE;AAAA,MACrC,IAAI,SAAS,WAAW,IAAI,MAAM,KAAK;AAAA,MACvC,IAAI;AAAA,IACN,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,EACd,EACC,KAAK,MAAM;AAChB;AAEO,SAAS,SAAS,OAAkD;AACzE,SAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK;AAC5E;","names":[]}

package/dist/chunk-XXFF3RRD.js

@@ -1,162 +0,0 @@
-import {
-  benchRoot,
-  runVenvPython
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/webarena-verified.ts
-import { access, readFile, stat } from "fs/promises";
-import { join, resolve } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "webarena-verified.json");
-var DATASET_REL = join("assets", "dataset", "webarena-verified.json");
-var webarenaDir = () => process.env.WEBARENA_VERIFIED_DIR;
-var webarenaOutputDirOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:text|path)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-async function assertFile(path, label) {
-  try {
-    await access(path);
-  } catch (err) {
-    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-async function assertDir(path, label) {
-  try {
-    const s = await stat(path);
-    if (!s.isDirectory()) throw new Error("not a directory");
-  } catch (err) {
-    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-function rowToTask(row) {
-  const meta = {
-    taskId: row.task_id,
-    intentTemplateId: row.intent_template_id,
-    sites: row.sites ?? [],
-    startUrls: row.start_urls ?? [],
-    revision: row.revision,
-    eval: row.eval
-  };
-  return {
-    id: String(row.task_id),
-    prompt: [
-      "Run this WebArena-Verified browser task in the official environment.",
-      `Goal: ${row.intent}`,
-      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(", ")}` : void 0,
-      row.sites?.length ? `Sites: ${row.sites.join(", ")}` : void 0,
-      "",
-      "Return the path to the official WebArena-Verified run output directory for this task.",
-      "The judge expects that directory to contain the task response and network trace files."
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "number") {
-    throw new Error(`webarena-verified task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, opts) {
-  let tasks = rows.map(rowToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`);
-  return tasks;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectRows(rows, opts);
-}
-async function loadOfficialRows(dir, opts) {
-  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), "utf8"));
-  return selectRows(rows, opts);
-}
-async function runOfficialEval(root, taskId, outputDir) {
-  const script = `
-import json, os, subprocess, sys
-from pathlib import Path
-
-root = Path(sys.argv[1])
-task_id = sys.argv[2]
-output_dir = Path(sys.argv[3])
-env = os.environ.copy()
-env["PYTHONPATH"] = str(root / "src") + os.pathsep + env.get("PYTHONPATH", "")
-cmd = [sys.executable, "-m", "webarena_verified", "eval-tasks", "--task-ids", task_id, "--output-dir", str(output_dir)]
-proc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)
-if proc.returncode != 0:
-    raise SystemExit((proc.stderr or proc.stdout or f"exit {proc.returncode}")[:2000])
-result_path = output_dir / task_id / "eval_result.json"
-if not result_path.exists():
-    raise SystemExit(f"official evaluator did not write {result_path}")
-print(json.dumps(json.loads(result_path.read_text())))
-`;
-  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0);
-  return JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-}
-function createWebArenaVerifiedAdapter() {
-  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === "1";
-  return {
-    name: "webarena-verified",
-    output: webarenaOutputDirOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = webarenaDir();
-      if (!dir) {
-        throw new Error(
-          "WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified."
-        );
-      }
-      await assertFile(join(dir, DATASET_REL), "official dataset");
-      await assertFile(join(dir, "src", "webarena_verified", "__main__.py"), "official CLI module");
-      await runVenvPython(
-        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print("ok")',
-        [join(dir, "src")]
-      );
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const dir = webarenaDir();
-      if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks");
-      return loadOfficialRows(dir, opts);
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const dir = webarenaDir();
-      if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts");
-      const outputDir = resolve(artifact.trim());
-      await assertDir(outputDir, "run output directory");
-      const meta = readMeta(task);
-      const report = await runOfficialEval(dir, meta.taskId, outputDir);
-      const score = typeof report.score === "number" ? report.score : 0;
-      const status = typeof report.status === "string" ? report.status : "unknown";
-      return {
-        resolved: score === 1,
-        score,
-        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir })
-      };
-    }
-  };
-}
-
-export {
-  webarenaOutputDirOutput,
-  createWebArenaVerifiedAdapter
-};
-//# sourceMappingURL=chunk-XXFF3RRD.js.map

package/dist/chunk-XXFF3RRD.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/webarena-verified.ts"],"sourcesContent":["/**\n * WebArena-Verified adapter (ServiceNow/webarena-verified).\n *\n * Worker artifact = a WebArena-Verified run output directory, not final chat text.\n * Judge = the official `webarena_verified eval-tasks` evaluator over that output\n * directory. The adapter refuses to score a plain answer so we do not turn a DOM\n * benchmark into a fake text benchmark.\n */\n\nimport { access, readFile, stat } from 'node:fs/promises'\nimport { join, resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'webarena-verified.json')\nconst DATASET_REL = join('assets', 'dataset', 'webarena-verified.json')\n\ninterface WebArenaRow {\n  task_id: number\n  intent: string\n  intent_template_id?: number\n  sites?: string[]\n  start_urls?: string[]\n  eval?: unknown[]\n  revision?: number\n}\n\ninterface WebArenaMeta {\n  taskId: number\n  intentTemplateId?: number\n  sites: string[]\n  startUrls: string[]\n  revision?: number\n  eval?: unknown[]\n}\n\nconst webarenaDir = (): string | undefined => process.env.WEBARENA_VERIFIED_DIR\n\nexport const webarenaOutputDirOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertDir(path: string, label: string): Promise<void> {\n  try {\n    const s = await stat(path)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction rowToTask(row: WebArenaRow): BenchTask {\n  const meta: WebArenaMeta = {\n    taskId: row.task_id,\n    intentTemplateId: row.intent_template_id,\n    sites: row.sites ?? [],\n    startUrls: row.start_urls ?? [],\n    revision: row.revision,\n    eval: row.eval,\n  }\n  return {\n    id: String(row.task_id),\n    prompt: [\n      'Run this WebArena-Verified browser task in the official environment.',\n      `Goal: ${row.intent}`,\n      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(', ')}` : undefined,\n      row.sites?.length ? `Sites: ${row.sites.join(', ')}` : undefined,\n      '',\n      'Return the path to the official WebArena-Verified run output directory for this task.',\n      'The judge expects that directory to contain the task response and network trace files.',\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): WebArenaMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number') {\n    throw new Error(`webarena-verified task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as WebArenaMeta\n}\n\nfunction selectRows(rows: WebArenaRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as WebArenaRow[]\n  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts)\n}\n\nasync function loadOfficialRows(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), 'utf8')) as WebArenaRow[]\n  return selectRows(rows, opts)\n}\n\nasync function runOfficialEval(root: string, taskId: number, outputDir: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, os, subprocess, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\noutput_dir = Path(sys.argv[3])\nenv = os.environ.copy()\nenv[\"PYTHONPATH\"] = str(root / \"src\") + os.pathsep + env.get(\"PYTHONPATH\", \"\")\ncmd = [sys.executable, \"-m\", \"webarena_verified\", \"eval-tasks\", \"--task-ids\", task_id, \"--output-dir\", str(output_dir)]\nproc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)\nif proc.returncode != 0:\n    raise SystemExit((proc.stderr or proc.stdout or f\"exit {proc.returncode}\")[:2000])\nresult_path = output_dir / task_id / \"eval_result.json\"\nif not result_path.exists():\n    raise SystemExit(f\"official evaluator did not write {result_path}\")\nprint(json.dumps(json.loads(result_path.read_text())))\n`\n  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createWebArenaVerifiedAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === '1'\n\n  return {\n    name: 'webarena-verified',\n    output: webarenaOutputDirOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = webarenaDir()\n      if (!dir) {\n        throw new Error(\n          'WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified.',\n        )\n      }\n      await assertFile(join(dir, DATASET_REL), 'official dataset')\n      await assertFile(join(dir, 'src', 'webarena_verified', '__main__.py'), 'official CLI module')\n      await runVenvPython(\n        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print(\"ok\")',\n        [join(dir, 'src')],\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks')\n      return loadOfficialRows(dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts')\n      const outputDir = resolve(artifact.trim())\n      await assertDir(outputDir, 'run output directory')\n      const meta = readMeta(task)\n      const report = await runOfficialEval(dir, meta.taskId, outputDir)\n      const score = typeof report.score === 'number' ? report.score : 0\n      const status = typeof report.status === 'string' ? report.status : 'unknown'\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;AASA,SAAS,QAAQ,UAAU,YAAY;AACvC,SAAS,MAAM,eAAe;AAK9B,IAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,IAAM,cAAc,KAAK,UAAU,WAAW,wBAAwB;AAqBtE,IAAM,cAAc,MAA0B,QAAQ,IAAI;AAEnD,IAAM,0BAAiD;AAAA,EAC5D,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,sCAAsC,CAAC;AACxE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,WAAW,MAAc,OAA8B;AACpE,MAAI;AACF,UAAM,OAAO,IAAI;AAAA,EACnB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,8BAA8B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAChH;AACF;AAEA,eAAe,UAAU,MAAc,OAA8B;AACnE,MAAI;AACF,UAAM,IAAI,MAAM,KAAK,IAAI;AACzB,QAAI,CAAC,EAAE,YAAY,EAAG,OAAM,IAAI,MAAM,iBAAiB;AAAA,EACzD,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,8BAA8B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAChH;AACF;AAEA,SAAS,UAAU,KAA6B;AAC9C,QAAM,OAAqB;AAAA,IACzB,QAAQ,IAAI;AAAA,IACZ,kBAAkB,IAAI;AAAA,IACtB,OAAO,IAAI,SAAS,CAAC;AAAA,IACrB,WAAW,IAAI,cAAc,CAAC;AAAA,IAC9B,UAAU,IAAI;AAAA,IACd,MAAM,IAAI;AAAA,EACZ;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,OAAO;AAAA,IACtB,QAAQ;AAAA,MACN;AAAA,MACA,SAAS,IAAI,MAAM;AAAA,MACnB,IAAI,YAAY,SAAS,wBAAwB,IAAI,WAAW,KAAK,IAAI,CAAC,KAAK;AAAA,MAC/E,IAAI,OAAO,SAAS,UAAU,IAAI,MAAM,KAAK,IAAI,CAAC,KAAK;AAAA,MACvD;AAAA,MACA;AAAA,MACA;AAAA,IACF,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAAU;AACxC,UAAM,IAAI,MAAM,0BAA0B,KAAK,EAAE,wDAAmD;AAAA,EACtG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAqB,MAAgC;AACvE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC,KAAK,UAAU,IAAI,CAAC,EAAE;AACrG,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,mEAA8D,KAAK,MAAM,mBAAmB;AACzG,SAAO,WAAW,MAAM,IAAI;AAC9B;AAEA,eAAe,iBAAiB,KAAa,MAAyC;AACpF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,KAAK,KAAK,WAAW,GAAG,MAAM,CAAC;AACtE,SAAO,WAAW,MAAM,IAAI;AAC9B;AAEA,eAAe,gBAAgB,MAAc,QAAgB,WAAqD;AAChH,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,MAAM,OAAO,MAAM,GAAG,SAAS,GAAG,CAAC;AAC/E,SAAO,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,+BAA+B;AAEhE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,WAAW,KAAK,KAAK,WAAW,GAAG,kBAAkB;AAC3D,YAAM,WAAW,KAAK,KAAK,OAAO,qBAAqB,aAAa,GAAG,qBAAqB;AAC5F,YAAM;AAAA,QACJ;AAAA,QACA,CAAC,KAAK,KAAK,KAAK,CAAC;AAAA,MACnB;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,4EAA4E;AACtG,aAAO,iBAAiB,KAAK,IAAI;AAAA,IACnC;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wEAAwE;AAClG,YAAM,YAAY,QAAQ,SAAS,KAAK,CAAC;AACzC,YAAM,UAAU,WAAW,sBAAsB;AACjD,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK,QAAQ,SAAS;AAChE,YAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;AAChE,YAAM,SAAS,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS;AACnE,aAAO;AAAA,QACL,UAAU,UAAU;AAAA,QACpB;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,UAAU,CAAC;AAAA,MACnE;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-ZFNOM7WR.js

@@ -1,27 +0,0 @@
-import {
-  createTauBenchAdapter
-} from "./chunk-GCHL6XPM.js";
-import {
-  benchRoot
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/tau3-banking.ts
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "tau3-banking.json");
-function createTau3BankingAdapter() {
-  return createTauBenchAdapter({
-    name: "tau3-banking",
-    fixturePath: FIXTURES,
-    fixturesEnv: "TAU3_FIXTURES",
-    dirEnv: "TAU3_BENCH_DIR",
-    domainEnv: "TAU3_DOMAIN",
-    defaultDomain: "banking_knowledge",
-    taskIntro: "Run this tau3 banking task in the official tau3 knowledge benchmark.",
-    installHint: "clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench."
-  });
-}
-
-export {
-  createTau3BankingAdapter
-};
-//# sourceMappingURL=chunk-ZFNOM7WR.js.map

package/dist/chunk-ZFNOM7WR.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/tau3-banking.ts"],"sourcesContent":["/**\n * tau3-banking adapter.\n *\n * The current upstream tau3 release is the `sierra-research/tau2-bench` repo with\n * package namespace `tau2` and a new `banking_knowledge` domain. This adapter\n * reuses the shared tau reward recomputation path and only changes the default\n * domain/env names.\n */\n\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport { createTauBenchAdapter } from './tau-bench-shared'\nimport type { BenchmarkAdapter } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'tau3-banking.json')\n\nexport function createTau3BankingAdapter(): BenchmarkAdapter {\n  return createTauBenchAdapter({\n    name: 'tau3-banking',\n    fixturePath: FIXTURES,\n    fixturesEnv: 'TAU3_FIXTURES',\n    dirEnv: 'TAU3_BENCH_DIR',\n    domainEnv: 'TAU3_DOMAIN',\n    defaultDomain: 'banking_knowledge',\n    taskIntro: 'Run this tau3 banking task in the official tau3 knowledge benchmark.',\n    installHint:\n      'clone https://github.com/sierra-research/tau2-bench, run `uv sync --extra knowledge`, install/import it from bench/.venv, and set TAU3_BENCH_DIR=/path/to/tau2-bench.',\n  })\n}\n"],"mappings":";;;;;;;;AASA,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEzD,SAAS,2BAA6C;AAC3D,SAAO,sBAAsB;AAAA,IAC3B,MAAM;AAAA,IACN,aAAa;AAAA,IACb,aAAa;AAAA,IACb,QAAQ;AAAA,IACR,WAAW;AAAA,IACX,eAAe;AAAA,IACf,WAAW;AAAA,IACX,aACE;AAAA,EACJ,CAAC;AACH;","names":[]}