@tangle-network/agent-bench 0.7.1 → 0.8.0

package/dist/benchmarks/trata-hedge.js.map

@@ -1 +1 @@
-{"version":3,"file":"trata-hedge.js","names":[],"sources":["../../src/benchmarks/trata-hedge.ts"],"sourcesContent":["/**\n * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial\n * analysis tasks across 6 domains (private equity, managed care, industrials,\n * vertical SaaS, REITs, insurance). Each task bundles real earnings-call\n * transcripts, financial statements, press releases, and SEC filings under\n * `environment/data/`; the agent must produce a grounded analysis citing those\n * files.\n *\n * Harbor architecture: the original benchmark runs each task in a Docker\n * container with file-read tools. This adapter skips Harbor and embeds the data\n * files directly in the worker prompt so any router backend can score it. Large\n * files (> 30 KB) are truncated to fit within model context limits. The judge\n * reads the actual files from disk for citation verification.\n *\n * Judge: a 3-stage LLM cascade faithful to grade.py —\n *   Task 1: hallucination check (cited-file context + agent answer)\n *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)\n *   Task 3: synthesis check (ground truth + agent answer)\n * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,\n * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;\n * resolved = score 4 (sparse reward).\n *\n * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.\n * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench\n */\n\nimport { readdirSync, readFileSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench'\nconst MAX_FILE_BYTES = 8_000\n// Sandbox API rejects messages > 100K chars; instruction + suffix adds ~500 chars overhead.\nconst MAX_DATA_CHARS = 85_000\nconst WORKER_SUFFIX = [\n  '',\n  'The data files above are your only source. Cite each claim with the filename that supports it.',\n  'Write your complete analysis to the text block that starts with \"ANALYSIS:\" on a line by itself.',\n  'Take a clear position on the topic. Every factual claim must name the file it comes from.',\n].join('\\n')\n\n// Judge model: Gemini 2.5 Pro via router (matches gemini-3.1-pro-preview tier used by grade.py).\nconst JUDGE_MODEL = 'gemini-2.5-pro'\n\ninterface TrataTaskMeta {\n  taskDir: string\n  taskId: string\n  dataDir: string\n  groundTruth: string\n  gradingTask1: string\n  gradingTask2: string\n  gradingTask3: string\n  /** [(label, n_moves)] in rubric order. */\n  themeMoveCounts: Array<{ label: string; nMoves: number }>\n}\n\n// ── file reading helpers ──────────────────────────────────────────────────────\n\nfunction readText(p: string, maxBytes = MAX_FILE_BYTES): string {\n  try {\n    const buf = readFileSync(p)\n    if (buf.length <= maxBytes) return buf.toString('utf8')\n    return buf.subarray(0, maxBytes).toString('utf8') + `\\n[… truncated at ${maxBytes} bytes …]`\n  } catch {\n    return `[unreadable: ${p}]`\n  }\n}\n\nfunction listFilesRec(dir: string): string[] {\n  const out: string[] = []\n  try {\n    for (const name of readdirSync(dir)) {\n      const full = join(dir, name)\n      try {\n        if (statSync(full).isDirectory()) out.push(...listFilesRec(full))\n        else out.push(full)\n      } catch {\n        // skip unreadable entries\n      }\n    }\n  } catch {\n    // dir absent\n  }\n  return out.sort()\n}\n\nfunction buildDataBlock(dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  if (files.length === 0) return '(no data files found)'\n  // Prioritise: company profiles → earnings calls → primary-ticker financials →\n  // sec filings → press releases → peers. Stop once total chars hit MAX_DATA_CHARS.\n  const priority = (f: string) => {\n    if (f.includes('company_profiles')) return 0\n    if (f.includes('earnings_call')) return 1\n    if (/financials\\/(income|cash_flow|balance)/.test(f) && !/bam_|bx_|kkr_|ares_|cg_|owl_|fsk_|cat_|de_|tdg_|car_/.test(f)) return 2\n    if (f.includes('sec_filings')) return 3\n    if (f.includes('investor_pres')) return 4\n    if (f.includes('press_release')) return 5\n    return 6\n  }\n  const sorted = [...files].sort((a, b) => priority(a) - priority(b) || a.localeCompare(b))\n  const blocks: string[] = []\n  let total = 0\n  for (const f of sorted) {\n    const rel = f.slice(dataDir.length + 1)\n    const content = readText(f)\n    const chunk = `\\n=== FILE: data/${rel} ===\\n${content}`\n    if (total + chunk.length > MAX_DATA_CHARS) {\n      blocks.push(`\\n[… ${sorted.length - blocks.length} more files omitted to stay within context limit …]`)\n      break\n    }\n    blocks.push(chunk)\n    total += chunk.length\n  }\n  return blocks.join('')\n}\n\nfunction buildCitedDataBlock(answerText: string, dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  const cited: string[] = []\n  for (const f of files) {\n    const rel = f.slice(dataDir.length + 1)\n    const name = rel.split('/').pop() ?? ''\n    if (answerText.includes(rel) || answerText.includes(name)) cited.push(f)\n  }\n  if (cited.length === 0) {\n    return '(The agent did not cite any data files, or no cited file could be located.)'\n  }\n  const blocks: string[] = []\n  for (const f of cited) {\n    const rel = f.slice(dataDir.length + 1)\n    blocks.push(`\\n=== FILE: data/${rel} ===\\n${readText(f)}`)\n  }\n  return blocks.join('')\n}\n\n// ── ground-truth parser ───────────────────────────────────────────────────────\n\nfunction parseThemeMoveCounts(groundTruth: string): Array<{ label: string; nMoves: number }> {\n  const lines = groundTruth.split('\\n')\n  const themeStarts = lines.reduce<number[]>((acc, l, i) => {\n    if (/^\\d+\\./.test(l.trim())) acc.push(i)\n    return acc\n  }, [])\n  return themeStarts.map((start, k) => {\n    const end = k + 1 < themeStarts.length ? themeStarts[k + 1] : lines.length\n    const block = lines.slice(start, end).join('\\n')\n    const label = lines[start].replace(/^\\d+\\.\\s*/, '').trim()\n    const nMoves = (block.match(/^\\s*\\[[a-z]\\]\\s+/gm) ?? []).length\n    return { label, nMoves }\n  })\n}\n\nfunction coverageThreshold(nMoves: number): number {\n  return Math.max(1, Math.min(nMoves - 1, 3))\n}\n\n// ── router judge calls ────────────────────────────────────────────────────────\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the Trata hedge-bench judge')\n  const model = process.env.JUDGE_MODEL ?? JUDGE_MODEL\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\nfunction parseJsonFallback(raw: string): unknown {\n  let s = raw.trim()\n  if (s.startsWith('```')) s = s.split('\\n').slice(1).join('\\n')\n  if (s.endsWith('```')) s = s.slice(0, s.lastIndexOf('```'))\n  s = s.replace(/,(\\s*[}\\]])/g, '$1').trim()\n  try {\n    return JSON.parse(s)\n  } catch {\n    try {\n      const m = s.match(/\\{[\\s\\S]*\\}/)\n      if (m) return JSON.parse(m[0])\n    } catch {\n      // fallthrough\n    }\n  }\n  return null\n}\n\nasync function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise<unknown> {\n  for (let i = 0; i < maxAttempts; i++) {\n    const res = await fetch(`${router.baseUrl}/chat/completions`, {\n      method: 'POST',\n      headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n      body: JSON.stringify({\n        model: router.model,\n        temperature: 0,\n        max_tokens: 16384,\n        messages: [{ role: 'user', content: prompt }],\n      }),\n    })\n    if (!res.ok) {\n      if (i < maxAttempts - 1) continue\n      throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n    }\n    const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n    const content = body.choices?.[0]?.message?.content\n    if (typeof content !== 'string') continue\n    const parsed = parseJsonFallback(content)\n    if (parsed !== null) return parsed\n  }\n  return null\n}\n\ninterface Task1Result {\n  hallucinations_detected?: boolean | string\n  unverifiable_claims?: Array<{ claim: string; cited_file?: string; issue?: string; evidence_check?: string }>\n}\n\ninterface Task2ThemeResult {\n  label?: string\n  moves_hit?: string[]\n  moves_missed?: string[]\n  moves_tainted?: string[]\n  move_reasoning?: Record<string, string>\n}\n\ninterface Task2Result {\n  themes?: Task2ThemeResult[]\n}\n\ninterface Task3Result {\n  synthesis_found?: string | null\n}\n\nasync function task1HallucinationCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task1Result> {\n  const citedBlock = buildCitedDataBlock(answer, meta.dataDir)\n  const dataContext =\n    'The block below contains the contents of the data files the agent cited in their answer. ' +\n    'Treat this as the ground truth for factual verification.\\n\\n' +\n    `<cited_data>\\n${citedBlock}\\n</cited_data>`\n  const prompt = meta.gradingTask1\n    .replace('{data_context}', dataContext)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task1Result) ?? { hallucinations_detected: false, unverifiable_claims: [] }\n}\n\nasync function task2PerMoveCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  flaggedClaims: Task1Result['unverifiable_claims'],\n  router: JudgeRouter,\n): Promise<Task2Result> {\n  const claimsBlock =\n    flaggedClaims && flaggedClaims.length > 0\n      ? JSON.stringify(flaggedClaims, null, 2)\n      : '(No claims were flagged as hallucinations.)'\n  const prompt = meta.gradingTask2\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n    .replace('{flagged_claims_block}', claimsBlock)\n    .replace('{num_themes}', String(meta.themeMoveCounts.length))\n  const result = await callJudge(router, prompt)\n  return (result as Task2Result) ?? { themes: [] }\n}\n\nasync function task3SynthesisCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task3Result> {\n  const prompt = meta.gradingTask3\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task3Result) ?? { synthesis_found: null }\n}\n\nasync function runTriataJudge(meta: TrataTaskMeta, answer: string, router: JudgeRouter): Promise<BenchScore> {\n  if (!answer.trim()) {\n    return { resolved: false, score: 0, detail: JSON.stringify({ error: 'empty answer' }) }\n  }\n\n  // Stage 1: hallucination check (serial — feeds into stage 2).\n  const t1 = await task1HallucinationCheck(meta, answer, router)\n  const flaggedClaims = t1.unverifiable_claims ?? []\n\n  // Stages 2 + 3 in parallel.\n  const [t2, t3] = await Promise.all([\n    task2PerMoveCheck(meta, answer, flaggedClaims, router),\n    task3SynthesisCheck(meta, answer, router),\n  ])\n\n  const judgeThemes = t2.themes ?? []\n  const themesHit: string[] = []\n  const themesMissed: string[] = []\n\n  for (let i = 0; i < meta.themeMoveCounts.length; i++) {\n    const { label, nMoves } = meta.themeMoveCounts[i]\n    const judge = judgeThemes[i] ?? {}\n    const movesHit = (judge.moves_hit ?? []).map((m: string) => m.replace(/[\\[\\]]/g, ''))\n    const tainted = new Set((judge.moves_tainted ?? []).map((m: string) => m.replace(/[\\[\\]]/g, '')))\n    const validHit = movesHit.filter((m: string) => !tainted.has(m))\n    const threshold = coverageThreshold(nMoves)\n    if (validHit.length >= threshold) themesHit.push(label)\n    else themesMissed.push(label)\n  }\n\n  const numThemes = meta.themeMoveCounts.length\n  const synth = t3.synthesis_found\n  const hasSynthesis = typeof synth === 'string' && synth.length > 0 && synth !== 'null'\n\n  let rawScore: number\n  if (numThemes === 0) rawScore = 0\n  else if (themesHit.length === numThemes && hasSynthesis) rawScore = 4\n  else if (themesHit.length === numThemes) rawScore = 3\n  else if (themesHit.length >= 2) rawScore = 2\n  else if (themesHit.length >= 1) rawScore = 1\n  else rawScore = 0\n\n  const hallRaw = t1.hallucinations_detected\n  const hallucinations = hallRaw === true || hallRaw === 'true'\n\n  return {\n    resolved: rawScore === 4,\n    score: numThemes > 0 ? rawScore / 4 : 0,\n    detail: JSON.stringify({\n      rawScore,\n      themesHit,\n      themesMissed,\n      hallucinations,\n      unverifiableClaims: flaggedClaims.length,\n      synthesis: synth ?? null,\n      judgeModel: router.model,\n    }),\n  }\n}\n\n// ── task loading ──────────────────────────────────────────────────────────────\n\nfunction benchRoot(): string {\n  return process.env.TRATA_BENCH_ROOT ?? DEFAULT_BENCH_ROOT\n}\n\nfunction loadTask(taskId: string): { task: BenchTask; meta: TrataTaskMeta } {\n  const taskDir = join(benchRoot(), 'environments', taskId)\n  const dataDir = join(taskDir, 'environment', 'data')\n  const testsDir = join(taskDir, 'tests')\n\n  const instruction = readText(join(taskDir, 'instruction.md'), Infinity)\n  const dataBlock = buildDataBlock(dataDir)\n  const groundTruth = readText(join(testsDir, 'ground_truth.txt'), Infinity)\n  const gradingTask1 = readText(join(testsDir, 'grading_prompt_task1.md'), Infinity)\n  const gradingTask2 = readText(join(testsDir, 'grading_prompt_task2.md'), Infinity)\n  const gradingTask3 = readText(join(testsDir, 'grading_prompt_task3.md'), Infinity)\n\n  const themeMoveCounts = parseThemeMoveCounts(groundTruth)\n\n  // Rewrite the instruction so the agent knows data is inline (no /app/data/ FS).\n  const prompt = [\n    instruction\n      .replace(\n        \"You are a financial analyst with access to the data in `/app/data/`.\",\n        'You are a financial analyst. The relevant financial data files are provided inline below.',\n      )\n      .replace(/Write your full analysis to `\\/app\\/answer\\.txt`\\.?/, 'Write your full analysis below.')\n      .replace(/`\\/app\\/data\\/`/g, 'the data files below'),\n    '',\n    '## Data files (inline)',\n    dataBlock,\n    WORKER_SUFFIX,\n  ].join('\\n')\n\n  const meta: TrataTaskMeta = {\n    taskDir,\n    taskId,\n    dataDir,\n    groundTruth,\n    gradingTask1,\n    gradingTask2,\n    gradingTask3,\n    themeMoveCounts,\n  }\n\n  // Store meta in the BenchTask so the judge can access it without re-reading.\n  const task: BenchTask = {\n    id: taskId,\n    prompt,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n\n  return { task, meta }\n}\n\nfunction readMeta(task: BenchTask): TrataTaskMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskDir !== 'string' || typeof md.groundTruth !== 'string') {\n    throw new Error(`Trata task ${task.id} missing judge metadata — was it loaded by this adapter?`)\n  }\n  return md as unknown as TrataTaskMeta\n}\n\n/** List all task ids from the environments directory. */\nfunction listTaskIds(root: string): string[] {\n  try {\n    return readdirSync(join(root, 'environments'))\n      .filter((name) => {\n        try {\n          return statSync(join(root, 'environments', name)).isDirectory()\n        } catch {\n          return false\n        }\n      })\n      .sort()\n  } catch (err) {\n    throw new Error(\n      `Trata benchmark environments not found at ${join(root, 'environments')}: ` +\n        `${err instanceof Error ? err.message : err}.\\n` +\n        `Clone the repo: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n    )\n  }\n}\n\n// ── extract agent analysis from artifact ─────────────────────────────────────\n\nfunction extractAnalysis(artifact: string): string {\n  // Model may write \"ANALYSIS:\" as its very first token (no leading newline) or\n  // after some preamble. Both are valid; fail-closed on absence.\n  if (artifact.startsWith('ANALYSIS:')) return artifact.slice('ANALYSIS:'.length).trim()\n  const marker = artifact.indexOf('\\nANALYSIS:')\n  if (marker !== -1) return artifact.slice(marker + '\\nANALYSIS:'.length).trim()\n  return ''\n}\n\n// ── adapter factory ───────────────────────────────────────────────────────────\n\nexport function createTrataHedgeAdapter(): BenchmarkAdapter {\n  return {\n    name: 'trata-hedge',\n\n    async preflight() {\n      const root = benchRoot()\n      const envDir = join(root, 'environments')\n      try {\n        const stat = statSync(envDir)\n        if (!stat.isDirectory()) throw new Error('not a directory')\n      } catch {\n        throw new Error(\n          `Trata hedge-bench not found at ${envDir}.\\n` +\n            `Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n        )\n      }\n      // Validate judge router connectivity.\n      judgeRouter()\n      const ids = listTaskIds(root)\n      if (ids.length === 0) throw new Error(`No task directories found under ${envDir}`)\n      console.warn(\n        `[trata-hedge] ${ids.length} tasks loaded from ${root}. ` +\n          `Worker model reads embedded data inline (Harbor/Docker not required). ` +\n          `Judge uses ${process.env.JUDGE_MODEL ?? JUDGE_MODEL} via Tangle router.`,\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const root = benchRoot()\n      let ids = listTaskIds(root)\n      if (opts.ids) {\n        const want = new Set(opts.ids)\n        ids = ids.filter((id) => want.has(id))\n      } else if (opts.limit !== undefined) {\n        ids = ids.slice(0, opts.limit)\n      }\n      return ids.map((id) => loadTask(id).task)\n    },\n\n    async goldArtifact(_task: BenchTask): Promise<string | undefined> {\n      // No single gold answer: the score comes from rubric theme coverage, not\n      // string-match against a reference response. Return undefined — the bench\n      // harness will skip judge self-verification for this adapter.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const answer = extractAnalysis(artifact)\n      return runTriataJudge(meta, answer, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAM,qBAAqB;AAC3B,MAAM,iBAAiB;AAEvB,MAAM,iBAAiB;AACvB,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAGX,MAAM,cAAc;AAgBpB,SAAS,SAAS,GAAW,WAAW,gBAAwB;CAC9D,IAAI;EACF,MAAM,MAAM,aAAa,CAAC;EAC1B,IAAI,IAAI,UAAU,UAAU,OAAO,IAAI,SAAS,MAAM;EACtD,OAAO,IAAI,SAAS,GAAG,QAAQ,CAAC,CAAC,SAAS,MAAM,IAAI,qBAAqB,SAAS;CACpF,QAAQ;EACN,OAAO,gBAAgB,EAAE;CAC3B;AACF;AAEA,SAAS,aAAa,KAAuB;CAC3C,MAAM,MAAgB,CAAC;CACvB,IAAI;EACF,KAAK,MAAM,QAAQ,YAAY,GAAG,GAAG;GACnC,MAAM,OAAO,KAAK,KAAK,IAAI;GAC3B,IAAI;IACF,IAAI,SAAS,IAAI,CAAC,CAAC,YAAY,GAAG,IAAI,KAAK,GAAG,aAAa,IAAI,CAAC;SAC3D,IAAI,KAAK,IAAI;GACpB,QAAQ,CAER;EACF;CACF,QAAQ,CAER;CACA,OAAO,IAAI,KAAK;AAClB;AAEA,SAAS,eAAe,SAAyB;CAC/C,MAAM,QAAQ,aAAa,OAAO;CAClC,IAAI,MAAM,WAAW,GAAG,OAAO;CAG/B,MAAM,YAAY,MAAc;EAC9B,IAAI,EAAE,SAAS,kBAAkB,GAAG,OAAO;EAC3C,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,yCAAyC,KAAK,CAAC,KAAK,CAAC,uDAAuD,KAAK,CAAC,GAAG,OAAO;EAChI,IAAI,EAAE,SAAS,aAAa,GAAG,OAAO;EACtC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,OAAO;CACT;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;CACxF,MAAM,SAAmB,CAAC;CAC1B,IAAI,QAAQ;CACZ,KAAK,MAAM,KAAK,QAAQ;EAGtB,MAAM,QAAQ,oBAFF,EAAE,MAAM,QAAQ,SAAS,CAED,EAAE,QADtB,SAAS,CAC2B;EACpD,IAAI,QAAQ,MAAM,SAAS,gBAAgB;GACzC,OAAO,KAAK,QAAQ,OAAO,SAAS,OAAO,OAAO,oDAAoD;GACtG;EACF;EACA,OAAO,KAAK,KAAK;EACjB,SAAS,MAAM;CACjB;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAEA,SAAS,oBAAoB,YAAoB,SAAyB;CACxE,MAAM,QAAQ,aAAa,OAAO;CAClC,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;EACrC,IAAI,WAAW,SAAS,GAAG,KAAK,WAAW,SAAS,IAAI,GAAG,MAAM,KAAK,CAAC;CACzE;CACA,IAAI,MAAM,WAAW,GACnB,OAAO;CAET,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,OAAO,KAAK,oBAAoB,IAAI,QAAQ,SAAS,CAAC,GAAG;CAC3D;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAIA,SAAS,qBAAqB,aAA+D;CAC3F,MAAM,QAAQ,YAAY,MAAM,IAAI;CACpC,MAAM,cAAc,MAAM,QAAkB,KAAK,GAAG,MAAM;EACxD,IAAI,SAAS,KAAK,EAAE,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC;EACvC,OAAO;CACT,GAAG,CAAC,CAAC;CACL,OAAO,YAAY,KAAK,OAAO,MAAM;EACnC,MAAM,MAAM,IAAI,IAAI,YAAY,SAAS,YAAY,IAAI,KAAK,MAAM;EACpE,MAAM,QAAQ,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI;EAG/C,OAAO;GAAE,OAFK,MAAM,MAAM,CAAC,QAAQ,aAAa,EAAE,CAAC,CAAC,KAEvC;GAAG,SADA,MAAM,MAAM,oBAAoB,KAAK,CAAC,EAAA,CAAG;EAClC;CACzB,CAAC;AACH;AAEA,SAAS,kBAAkB,QAAwB;CACjD,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,GAAG,CAAC,CAAC;AAC5C;AAUA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,4DAA4D;CACtF,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;AAEA,SAAS,kBAAkB,KAAsB;CAC/C,IAAI,IAAI,IAAI,KAAK;CACjB,IAAI,EAAE,WAAW,KAAK,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI;CAC7D,IAAI,EAAE,SAAS,KAAK,GAAG,IAAI,EAAE,MAAM,GAAG,EAAE,YAAY,KAAK,CAAC;CAC1D,IAAI,EAAE,QAAQ,gBAAgB,IAAI,CAAC,CAAC,KAAK;CACzC,IAAI;EACF,OAAO,KAAK,MAAM,CAAC;CACrB,QAAQ;EACN,IAAI;GACF,MAAM,IAAI,EAAE,MAAM,aAAa;GAC/B,IAAI,GAAG,OAAO,KAAK,MAAM,EAAE,EAAE;EAC/B,QAAQ,CAER;CACF;CACA,OAAO;AACT;AAEA,eAAe,UAAU,QAAqB,QAAgB,cAAc,GAAqB;CAC/F,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;EACpC,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;GAC5D,QAAQ;GACR,SAAS;IAAE,gBAAgB;IAAoB,eAAe,UAAU,OAAO;GAAM;GACrF,MAAM,KAAK,UAAU;IACnB,OAAO,OAAO;IACd,aAAa;IACb,YAAY;IACZ,UAAU,CAAC;KAAE,MAAM;KAAQ,SAAS;IAAO,CAAC;GAC9C,CAAC;EACH,CAAC;EACD,IAAI,CAAC,IAAI,IAAI;GACX,IAAI,IAAI,cAAc,GAAG;GACzB,MAAM,IAAI,MAAM,oBAAoB,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EACvF;EAEA,MAAM,WAAU,MADI,IAAI,KAAK,EAAA,CACR,UAAU,EAAE,EAAE,SAAS;EAC5C,IAAI,OAAO,YAAY,UAAU;EACjC,MAAM,SAAS,kBAAkB,OAAO;EACxC,IAAI,WAAW,MAAM,OAAO;CAC9B;CACA,OAAO;AACT;AAuBA,eAAe,wBACb,MACA,QACA,QACsB;CAEtB,MAAM,cACJ;;gBAFiB,oBAAoB,QAAQ,KAAK,OAIxB,EAAE;CAK9B,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,WAAW,CAAC,CACtC,QAAQ,kBAAkB,MACe,CAAC,KACX;EAAE,yBAAyB;EAAO,qBAAqB,CAAC;CAAE;AAC9F;AAEA,eAAe,kBACb,MACA,QACA,eACA,QACsB;CACtB,MAAM,cACJ,iBAAiB,cAAc,SAAS,IACpC,KAAK,UAAU,eAAe,MAAM,CAAC,IACrC;CAON,OAAQ,MADa,UAAU,QALhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MAAM,CAAC,CACjC,QAAQ,0BAA0B,WAAW,CAAC,CAC9C,QAAQ,gBAAgB,OAAO,KAAK,gBAAgB,MAAM,CACjB,CAAC,KACX,EAAE,QAAQ,CAAC,EAAE;AACjD;AAEA,eAAe,oBACb,MACA,QACA,QACsB;CAKtB,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MACe,CAAC,KACX,EAAE,iBAAiB,KAAK;AAC5D;AAEA,eAAe,eAAe,MAAqB,QAAgB,QAA0C;CAC3G,IAAI,CAAC,OAAO,KAAK,GACf,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,QAAQ,KAAK,UAAU,EAAE,OAAO,eAAe,CAAC;CAAE;CAIxF,MAAM,KAAK,MAAM,wBAAwB,MAAM,QAAQ,MAAM;CAC7D,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;CAGjD,MAAM,CAAC,IAAI,MAAM,MAAM,QAAQ,IAAI,CACjC,kBAAkB,MAAM,QAAQ,eAAe,MAAM,GACrD,oBAAoB,MAAM,QAAQ,MAAM,CAC1C,CAAC;CAED,MAAM,cAAc,GAAG,UAAU,CAAC;CAClC,MAAM,YAAsB,CAAC;CAC7B,MAAM,eAAyB,CAAC;CAEhC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,gBAAgB,QAAQ,KAAK;EACpD,MAAM,EAAE,OAAO,WAAW,KAAK,gBAAgB;EAC/C,MAAM,QAAQ,YAAY,MAAM,CAAC;EACjC,MAAM,YAAY,MAAM,aAAa,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC;EACpF,MAAM,UAAU,IAAI,KAAK,MAAM,iBAAiB,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC,CAAC;EAChG,MAAM,WAAW,SAAS,QAAQ,MAAc,CAAC,QAAQ,IAAI,CAAC,CAAC;EAC/D,MAAM,YAAY,kBAAkB,MAAM;EAC1C,IAAI,SAAS,UAAU,WAAW,UAAU,KAAK,KAAK;OACjD,aAAa,KAAK,KAAK;CAC9B;CAEA,MAAM,YAAY,KAAK,gBAAgB;CACvC,MAAM,QAAQ,GAAG;CACjB,MAAM,eAAe,OAAO,UAAU,YAAY,MAAM,SAAS,KAAK,UAAU;CAEhF,IAAI;CACJ,IAAI,cAAc,GAAG,WAAW;MAC3B,IAAI,UAAU,WAAW,aAAa,cAAc,WAAW;MAC/D,IAAI,UAAU,WAAW,WAAW,WAAW;MAC/C,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,WAAW;CAEhB,MAAM,UAAU,GAAG;CACnB,MAAM,iBAAiB,YAAY,QAAQ,YAAY;CAEvD,OAAO;EACL,UAAU,aAAa;EACvB,OAAO,YAAY,IAAI,WAAW,IAAI;EACtC,QAAQ,KAAK,UAAU;GACrB;GACA;GACA;GACA;GACA,oBAAoB,cAAc;GAClC,WAAW,SAAS;GACpB,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAIA,SAAS,YAAoB;CAC3B,OAAO,QAAQ,IAAI,oBAAoB;AACzC;AAEA,SAAS,SAAS,QAA0D;CAC1E,MAAM,UAAU,KAAK,UAAU,GAAG,gBAAgB,MAAM;CACxD,MAAM,UAAU,KAAK,SAAS,eAAe,MAAM;CACnD,MAAM,WAAW,KAAK,SAAS,OAAO;CAEtC,MAAM,cAAc,SAAS,KAAK,SAAS,gBAAgB,GAAG,QAAQ;CACtE,MAAM,YAAY,eAAe,OAAO;CACxC,MAAM,cAAc,SAAS,KAAK,UAAU,kBAAkB,GAAG,QAAQ;CACzE,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CAEjF,MAAM,kBAAkB,qBAAqB,WAAW;CAGxD,MAAM,SAAS;EACb,YACG,QACC,wEACA,2FACF,CAAC,CACA,QAAQ,uDAAuD,iCAAiC,CAAC,CACjG,QAAQ,oBAAoB,sBAAsB;EACrD;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CAEX,MAAM,OAAsB;EAC1B;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF;CASA,OAAO;EAAE,MAAA;GALP,IAAI;GACJ;GACA,UAAU;EAGA;EAAG;CAAK;AACtB;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,YAAY,YAAY,OAAO,GAAG,gBAAgB,UACrE,MAAM,IAAI,MAAM,cAAc,KAAK,GAAG,yDAAyD;CAEjG,OAAO;AACT;;AAGA,SAAS,YAAY,MAAwB;CAC3C,IAAI;EACF,OAAO,YAAY,KAAK,MAAM,cAAc,CAAC,CAAC,CAC3C,QAAQ,SAAS;GAChB,IAAI;IACF,OAAO,SAAS,KAAK,MAAM,gBAAgB,IAAI,CAAC,CAAC,CAAC,YAAY;GAChE,QAAQ;IACN,OAAO;GACT;EACF,CAAC,CAAC,CACD,KAAK;CACV,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,6CAA6C,KAAK,MAAM,cAAc,EAAE,IACnE,eAAe,QAAQ,IAAI,UAAU,IAAI,wFAC0C,MAC1F;CACF;AACF;AAIA,SAAS,gBAAgB,UAA0B;CAGjD,IAAI,SAAS,WAAW,WAAW,GAAG,OAAO,SAAS,MAAM,CAAkB,CAAC,CAAC,KAAK;CACrF,MAAM,SAAS,SAAS,QAAQ,aAAa;CAC7C,IAAI,WAAW,IAAI,OAAO,SAAS,MAAM,SAAS,EAAoB,CAAC,CAAC,KAAK;CAC7E,OAAO;AACT;AAIA,SAAgB,0BAA4C;CAC1D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,OAAO,UAAU;GACvB,MAAM,SAAS,KAAK,MAAM,cAAc;GACxC,IAAI;IAEF,IAAI,CADS,SAAS,MACd,CAAC,CAAC,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;GAC5D,QAAQ;IACN,MAAM,IAAI,MACR,kCAAkC,OAAO,+EACsC,MACjF;GACF;GAEA,YAAY;GACZ,MAAM,MAAM,YAAY,IAAI;GAC5B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,mCAAmC,QAAQ;GACjF,QAAQ,KACN,iBAAiB,IAAI,OAAO,qBAAqB,KAAK,qFAEtC,QAAQ,IAAI,eAAe,YAAY,oBACzD;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,IAAI,MAAM,YADG,UACY,CAAC;GAC1B,IAAI,KAAK,KAAK;IACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;IAC7B,MAAM,IAAI,QAAQ,OAAO,KAAK,IAAI,EAAE,CAAC;GACvC,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;GAE/B,OAAO,IAAI,KAAK,OAAO,SAAS,EAAE,CAAC,CAAC,IAAI;EAC1C;EAEA,MAAM,aAAa,OAA+C,CAKlE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAGlE,OAAO,eAFM,SAAS,IAEG,GADV,gBAAgB,QACE,GAAG,YAAY,CAAC;EACnD;CACF;AACF"}
+{"version":3,"file":"trata-hedge.js","names":[],"sources":["../../src/benchmarks/trata-hedge.ts"],"sourcesContent":["/**\n * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial\n * analysis tasks across 6 domains (private equity, managed care, industrials,\n * vertical SaaS, REITs, insurance). Each task bundles real earnings-call\n * transcripts, financial statements, press releases, and SEC filings under\n * `environment/data/`; the agent must produce a grounded analysis citing those\n * files.\n *\n * Harbor architecture: the original benchmark runs each task in a Docker\n * container with file-read tools. This adapter skips Harbor and embeds the data\n * files directly in the worker prompt so any router backend can score it. Large\n * files (> 30 KB) are truncated to fit within model context limits. The judge\n * reads the actual files from disk for citation verification.\n *\n * Judge: a 3-stage LLM cascade faithful to grade.py —\n *   Task 1: hallucination check (cited-file context + agent answer)\n *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)\n *   Task 3: synthesis check (ground truth + agent answer)\n * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,\n * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;\n * resolved = score 4 (sparse reward).\n *\n * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.\n * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench\n */\n\nimport { readdirSync, readFileSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench'\nconst MAX_FILE_BYTES = 8_000\n// Sandbox API rejects messages > 100K chars; instruction + suffix adds ~500 chars overhead.\nconst MAX_DATA_CHARS = 85_000\nconst WORKER_SUFFIX = [\n  '',\n  'The data files above are your only source. Cite each claim with the filename that supports it.',\n  'Write your complete analysis to the text block that starts with \"ANALYSIS:\" on a line by itself.',\n  'Take a clear position on the topic. Every factual claim must name the file it comes from.',\n].join('\\n')\n\n// Judge model: Gemini 2.5 Pro via router (matches gemini-3.1-pro-preview tier used by grade.py).\nconst JUDGE_MODEL = 'gemini-2.5-pro'\n\ninterface TrataTaskMeta {\n  taskDir: string\n  taskId: string\n  dataDir: string\n  groundTruth: string\n  gradingTask1: string\n  gradingTask2: string\n  gradingTask3: string\n  /** [(label, n_moves)] in rubric order. */\n  themeMoveCounts: Array<{ label: string; nMoves: number }>\n}\n\n// ── file reading helpers ──────────────────────────────────────────────────────\n\nfunction readText(p: string, maxBytes = MAX_FILE_BYTES): string {\n  try {\n    const buf = readFileSync(p)\n    if (buf.length <= maxBytes) return buf.toString('utf8')\n    return buf.subarray(0, maxBytes).toString('utf8') + `\\n[… truncated at ${maxBytes} bytes …]`\n  } catch {\n    return `[unreadable: ${p}]`\n  }\n}\n\nfunction listFilesRec(dir: string): string[] {\n  const out: string[] = []\n  try {\n    for (const name of readdirSync(dir)) {\n      const full = join(dir, name)\n      try {\n        if (statSync(full).isDirectory()) out.push(...listFilesRec(full))\n        else out.push(full)\n      } catch {\n        // skip unreadable entries\n      }\n    }\n  } catch {\n    // dir absent\n  }\n  return out.sort()\n}\n\nfunction buildDataBlock(dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  if (files.length === 0) return '(no data files found)'\n  // Prioritise: company profiles → earnings calls → primary-ticker financials →\n  // sec filings → press releases → peers. Stop once total chars hit MAX_DATA_CHARS.\n  const priority = (f: string) => {\n    if (f.includes('company_profiles')) return 0\n    if (f.includes('earnings_call')) return 1\n    if (/financials\\/(income|cash_flow|balance)/.test(f) && !/bam_|bx_|kkr_|ares_|cg_|owl_|fsk_|cat_|de_|tdg_|car_/.test(f)) return 2\n    if (f.includes('sec_filings')) return 3\n    if (f.includes('investor_pres')) return 4\n    if (f.includes('press_release')) return 5\n    return 6\n  }\n  const sorted = [...files].sort((a, b) => priority(a) - priority(b) || a.localeCompare(b))\n  const blocks: string[] = []\n  let total = 0\n  for (const f of sorted) {\n    const rel = f.slice(dataDir.length + 1)\n    const content = readText(f)\n    const chunk = `\\n=== FILE: data/${rel} ===\\n${content}`\n    if (total + chunk.length > MAX_DATA_CHARS) {\n      blocks.push(`\\n[… ${sorted.length - blocks.length} more files omitted to stay within context limit …]`)\n      break\n    }\n    blocks.push(chunk)\n    total += chunk.length\n  }\n  return blocks.join('')\n}\n\nfunction buildCitedDataBlock(answerText: string, dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  const cited: string[] = []\n  for (const f of files) {\n    const rel = f.slice(dataDir.length + 1)\n    const name = rel.split('/').pop() ?? ''\n    if (answerText.includes(rel) || answerText.includes(name)) cited.push(f)\n  }\n  if (cited.length === 0) {\n    return '(The agent did not cite any data files, or no cited file could be located.)'\n  }\n  const blocks: string[] = []\n  for (const f of cited) {\n    const rel = f.slice(dataDir.length + 1)\n    blocks.push(`\\n=== FILE: data/${rel} ===\\n${readText(f)}`)\n  }\n  return blocks.join('')\n}\n\n// ── ground-truth parser ───────────────────────────────────────────────────────\n\nfunction parseThemeMoveCounts(groundTruth: string): Array<{ label: string; nMoves: number }> {\n  const lines = groundTruth.split('\\n')\n  const themeStarts = lines.reduce<number[]>((acc, l, i) => {\n    if (/^\\d+\\./.test(l.trim())) acc.push(i)\n    return acc\n  }, [])\n  return themeStarts.map((start, k) => {\n    const end = k + 1 < themeStarts.length ? themeStarts[k + 1] : lines.length\n    const block = lines.slice(start, end).join('\\n')\n    const label = lines[start].replace(/^\\d+\\.\\s*/, '').trim()\n    const nMoves = (block.match(/^\\s*\\[[a-z]\\]\\s+/gm) ?? []).length\n    return { label, nMoves }\n  })\n}\n\nfunction coverageThreshold(nMoves: number): number {\n  return Math.max(1, Math.min(nMoves - 1, 3))\n}\n\n// ── router judge calls ────────────────────────────────────────────────────────\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the Trata hedge-bench judge')\n  const model = process.env.JUDGE_MODEL ?? JUDGE_MODEL\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\nfunction parseJsonFallback(raw: string): unknown {\n  let s = raw.trim()\n  if (s.startsWith('```')) s = s.split('\\n').slice(1).join('\\n')\n  if (s.endsWith('```')) s = s.slice(0, s.lastIndexOf('```'))\n  s = s.replace(/,(\\s*[}\\]])/g, '$1').trim()\n  try {\n    return JSON.parse(s)\n  } catch {\n    try {\n      const m = s.match(/\\{[\\s\\S]*\\}/)\n      if (m) return JSON.parse(m[0])\n    } catch {\n      // fallthrough\n    }\n  }\n  return null\n}\n\nasync function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise<unknown> {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: router.baseUrl,\n          routerKey: router.key,\n          profile: {\n            name: 'trata-hedge-judge',\n            harness: 'cli-base',\n            model: {\n              provider: 'tangle-router',\n              default: router.model,\n              metadata: {\n                temperature: 0,\n                maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 16384),\n              },\n            },\n          },\n        },\n        prompt,\n      )\n      const content = turn.finalText\n      const parsed = parseJsonFallback(content)\n      if (parsed !== null) return parsed\n    } catch (error) {\n      if (i + 1 === maxAttempts) throw error\n    }\n  }\n  return null\n}\n\ninterface Task1Result {\n  hallucinations_detected?: boolean | string\n  unverifiable_claims?: Array<{ claim: string; cited_file?: string; issue?: string; evidence_check?: string }>\n}\n\ninterface Task2ThemeResult {\n  label?: string\n  moves_hit?: string[]\n  moves_missed?: string[]\n  moves_tainted?: string[]\n  move_reasoning?: Record<string, string>\n}\n\ninterface Task2Result {\n  themes?: Task2ThemeResult[]\n}\n\ninterface Task3Result {\n  synthesis_found?: string | null\n}\n\nasync function task1HallucinationCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task1Result> {\n  const citedBlock = buildCitedDataBlock(answer, meta.dataDir)\n  const dataContext =\n    'The block below contains the contents of the data files the agent cited in their answer. ' +\n    'Treat this as the ground truth for factual verification.\\n\\n' +\n    `<cited_data>\\n${citedBlock}\\n</cited_data>`\n  const prompt = meta.gradingTask1\n    .replace('{data_context}', dataContext)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task1Result) ?? { hallucinations_detected: false, unverifiable_claims: [] }\n}\n\nasync function task2PerMoveCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  flaggedClaims: Task1Result['unverifiable_claims'],\n  router: JudgeRouter,\n): Promise<Task2Result> {\n  const claimsBlock =\n    flaggedClaims && flaggedClaims.length > 0\n      ? JSON.stringify(flaggedClaims, null, 2)\n      : '(No claims were flagged as hallucinations.)'\n  const prompt = meta.gradingTask2\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n    .replace('{flagged_claims_block}', claimsBlock)\n    .replace('{num_themes}', String(meta.themeMoveCounts.length))\n  const result = await callJudge(router, prompt)\n  return (result as Task2Result) ?? { themes: [] }\n}\n\nasync function task3SynthesisCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task3Result> {\n  const prompt = meta.gradingTask3\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task3Result) ?? { synthesis_found: null }\n}\n\nasync function runTriataJudge(meta: TrataTaskMeta, answer: string, router: JudgeRouter): Promise<BenchScore> {\n  if (!answer.trim()) {\n    return { resolved: false, score: 0, detail: JSON.stringify({ error: 'empty answer' }) }\n  }\n\n  // Stage 1: hallucination check (serial — feeds into stage 2).\n  const t1 = await task1HallucinationCheck(meta, answer, router)\n  const flaggedClaims = t1.unverifiable_claims ?? []\n\n  // Stages 2 + 3 in parallel.\n  const [t2, t3] = await Promise.all([\n    task2PerMoveCheck(meta, answer, flaggedClaims, router),\n    task3SynthesisCheck(meta, answer, router),\n  ])\n\n  const judgeThemes = t2.themes ?? []\n  const themesHit: string[] = []\n  const themesMissed: string[] = []\n\n  for (let i = 0; i < meta.themeMoveCounts.length; i++) {\n    const { label, nMoves } = meta.themeMoveCounts[i]\n    const judge = judgeThemes[i] ?? {}\n    const movesHit = (judge.moves_hit ?? []).map((m: string) => m.replace(/[\\[\\]]/g, ''))\n    const tainted = new Set((judge.moves_tainted ?? []).map((m: string) => m.replace(/[\\[\\]]/g, '')))\n    const validHit = movesHit.filter((m: string) => !tainted.has(m))\n    const threshold = coverageThreshold(nMoves)\n    if (validHit.length >= threshold) themesHit.push(label)\n    else themesMissed.push(label)\n  }\n\n  const numThemes = meta.themeMoveCounts.length\n  const synth = t3.synthesis_found\n  const hasSynthesis = typeof synth === 'string' && synth.length > 0 && synth !== 'null'\n\n  let rawScore: number\n  if (numThemes === 0) rawScore = 0\n  else if (themesHit.length === numThemes && hasSynthesis) rawScore = 4\n  else if (themesHit.length === numThemes) rawScore = 3\n  else if (themesHit.length >= 2) rawScore = 2\n  else if (themesHit.length >= 1) rawScore = 1\n  else rawScore = 0\n\n  const hallRaw = t1.hallucinations_detected\n  const hallucinations = hallRaw === true || hallRaw === 'true'\n\n  return {\n    resolved: rawScore === 4,\n    score: numThemes > 0 ? rawScore / 4 : 0,\n    detail: JSON.stringify({\n      rawScore,\n      themesHit,\n      themesMissed,\n      hallucinations,\n      unverifiableClaims: flaggedClaims.length,\n      synthesis: synth ?? null,\n      judgeModel: router.model,\n    }),\n  }\n}\n\n// ── task loading ──────────────────────────────────────────────────────────────\n\nfunction benchRoot(): string {\n  return process.env.TRATA_BENCH_ROOT ?? DEFAULT_BENCH_ROOT\n}\n\nfunction loadTask(taskId: string): { task: BenchTask; meta: TrataTaskMeta } {\n  const taskDir = join(benchRoot(), 'environments', taskId)\n  const dataDir = join(taskDir, 'environment', 'data')\n  const testsDir = join(taskDir, 'tests')\n\n  const instruction = readText(join(taskDir, 'instruction.md'), Infinity)\n  const dataBlock = buildDataBlock(dataDir)\n  const groundTruth = readText(join(testsDir, 'ground_truth.txt'), Infinity)\n  const gradingTask1 = readText(join(testsDir, 'grading_prompt_task1.md'), Infinity)\n  const gradingTask2 = readText(join(testsDir, 'grading_prompt_task2.md'), Infinity)\n  const gradingTask3 = readText(join(testsDir, 'grading_prompt_task3.md'), Infinity)\n\n  const themeMoveCounts = parseThemeMoveCounts(groundTruth)\n\n  // Rewrite the instruction so the agent knows data is inline (no /app/data/ FS).\n  const prompt = [\n    instruction\n      .replace(\n        \"You are a financial analyst with access to the data in `/app/data/`.\",\n        'You are a financial analyst. The relevant financial data files are provided inline below.',\n      )\n      .replace(/Write your full analysis to `\\/app\\/answer\\.txt`\\.?/, 'Write your full analysis below.')\n      .replace(/`\\/app\\/data\\/`/g, 'the data files below'),\n    '',\n    '## Data files (inline)',\n    dataBlock,\n    WORKER_SUFFIX,\n  ].join('\\n')\n\n  const meta: TrataTaskMeta = {\n    taskDir,\n    taskId,\n    dataDir,\n    groundTruth,\n    gradingTask1,\n    gradingTask2,\n    gradingTask3,\n    themeMoveCounts,\n  }\n\n  // Store meta in the BenchTask so the judge can access it without re-reading.\n  const task: BenchTask = {\n    id: taskId,\n    prompt,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n\n  return { task, meta }\n}\n\nfunction readMeta(task: BenchTask): TrataTaskMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskDir !== 'string' || typeof md.groundTruth !== 'string') {\n    throw new Error(`Trata task ${task.id} missing judge metadata — was it loaded by this adapter?`)\n  }\n  return md as unknown as TrataTaskMeta\n}\n\n/** List all task ids from the environments directory. */\nfunction listTaskIds(root: string): string[] {\n  try {\n    return readdirSync(join(root, 'environments'))\n      .filter((name) => {\n        try {\n          return statSync(join(root, 'environments', name)).isDirectory()\n        } catch {\n          return false\n        }\n      })\n      .sort()\n  } catch (err) {\n    throw new Error(\n      `Trata benchmark environments not found at ${join(root, 'environments')}: ` +\n        `${err instanceof Error ? err.message : err}.\\n` +\n        `Clone the repo: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n    )\n  }\n}\n\n// ── extract agent analysis from artifact ─────────────────────────────────────\n\nfunction extractAnalysis(artifact: string): string {\n  // Model may write \"ANALYSIS:\" as its very first token (no leading newline) or\n  // after some preamble. Both are valid; fail-closed on absence.\n  if (artifact.startsWith('ANALYSIS:')) return artifact.slice('ANALYSIS:'.length).trim()\n  const marker = artifact.indexOf('\\nANALYSIS:')\n  if (marker !== -1) return artifact.slice(marker + '\\nANALYSIS:'.length).trim()\n  return ''\n}\n\n// ── adapter factory ───────────────────────────────────────────────────────────\n\nexport function createTrataHedgeAdapter(): BenchmarkAdapter {\n  return {\n    name: 'trata-hedge',\n\n    async preflight() {\n      const root = benchRoot()\n      const envDir = join(root, 'environments')\n      try {\n        const stat = statSync(envDir)\n        if (!stat.isDirectory()) throw new Error('not a directory')\n      } catch {\n        throw new Error(\n          `Trata hedge-bench not found at ${envDir}.\\n` +\n            `Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n        )\n      }\n      // Validate judge router connectivity.\n      judgeRouter()\n      const ids = listTaskIds(root)\n      if (ids.length === 0) throw new Error(`No task directories found under ${envDir}`)\n      console.warn(\n        `[trata-hedge] ${ids.length} tasks loaded from ${root}. ` +\n          `Worker model reads embedded data inline (Harbor/Docker not required). ` +\n          `Judge uses ${process.env.JUDGE_MODEL ?? JUDGE_MODEL} via Tangle router.`,\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const root = benchRoot()\n      let ids = listTaskIds(root)\n      if (opts.ids) {\n        const want = new Set(opts.ids)\n        ids = ids.filter((id) => want.has(id))\n      } else if (opts.limit !== undefined) {\n        ids = ids.slice(0, opts.limit)\n      }\n      return ids.map((id) => loadTask(id).task)\n    },\n\n    async goldArtifact(_task: BenchTask): Promise<string | undefined> {\n      // No single gold answer: the score comes from rubric theme coverage, not\n      // string-match against a reference response. Return undefined — the bench\n      // harness will skip judge self-verification for this adapter.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const answer = extractAnalysis(artifact)\n      return runTriataJudge(meta, answer, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,MAAM,qBAAqB;AAC3B,MAAM,iBAAiB;AAEvB,MAAM,iBAAiB;AACvB,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAGX,MAAM,cAAc;AAgBpB,SAAS,SAAS,GAAW,WAAW,gBAAwB;CAC9D,IAAI;EACF,MAAM,MAAM,aAAa,CAAC;EAC1B,IAAI,IAAI,UAAU,UAAU,OAAO,IAAI,SAAS,MAAM;EACtD,OAAO,IAAI,SAAS,GAAG,QAAQ,CAAC,CAAC,SAAS,MAAM,IAAI,qBAAqB,SAAS;CACpF,QAAQ;EACN,OAAO,gBAAgB,EAAE;CAC3B;AACF;AAEA,SAAS,aAAa,KAAuB;CAC3C,MAAM,MAAgB,CAAC;CACvB,IAAI;EACF,KAAK,MAAM,QAAQ,YAAY,GAAG,GAAG;GACnC,MAAM,OAAO,KAAK,KAAK,IAAI;GAC3B,IAAI;IACF,IAAI,SAAS,IAAI,CAAC,CAAC,YAAY,GAAG,IAAI,KAAK,GAAG,aAAa,IAAI,CAAC;SAC3D,IAAI,KAAK,IAAI;GACpB,QAAQ,CAER;EACF;CACF,QAAQ,CAER;CACA,OAAO,IAAI,KAAK;AAClB;AAEA,SAAS,eAAe,SAAyB;CAC/C,MAAM,QAAQ,aAAa,OAAO;CAClC,IAAI,MAAM,WAAW,GAAG,OAAO;CAG/B,MAAM,YAAY,MAAc;EAC9B,IAAI,EAAE,SAAS,kBAAkB,GAAG,OAAO;EAC3C,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,yCAAyC,KAAK,CAAC,KAAK,CAAC,uDAAuD,KAAK,CAAC,GAAG,OAAO;EAChI,IAAI,EAAE,SAAS,aAAa,GAAG,OAAO;EACtC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,OAAO;CACT;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;CACxF,MAAM,SAAmB,CAAC;CAC1B,IAAI,QAAQ;CACZ,KAAK,MAAM,KAAK,QAAQ;EAGtB,MAAM,QAAQ,oBAFF,EAAE,MAAM,QAAQ,SAAS,CAED,EAAE,QADtB,SAAS,CAC2B;EACpD,IAAI,QAAQ,MAAM,SAAS,gBAAgB;GACzC,OAAO,KAAK,QAAQ,OAAO,SAAS,OAAO,OAAO,oDAAoD;GACtG;EACF;EACA,OAAO,KAAK,KAAK;EACjB,SAAS,MAAM;CACjB;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAEA,SAAS,oBAAoB,YAAoB,SAAyB;CACxE,MAAM,QAAQ,aAAa,OAAO;CAClC,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;EACrC,IAAI,WAAW,SAAS,GAAG,KAAK,WAAW,SAAS,IAAI,GAAG,MAAM,KAAK,CAAC;CACzE;CACA,IAAI,MAAM,WAAW,GACnB,OAAO;CAET,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,OAAO,KAAK,oBAAoB,IAAI,QAAQ,SAAS,CAAC,GAAG;CAC3D;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAIA,SAAS,qBAAqB,aAA+D;CAC3F,MAAM,QAAQ,YAAY,MAAM,IAAI;CACpC,MAAM,cAAc,MAAM,QAAkB,KAAK,GAAG,MAAM;EACxD,IAAI,SAAS,KAAK,EAAE,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC;EACvC,OAAO;CACT,GAAG,CAAC,CAAC;CACL,OAAO,YAAY,KAAK,OAAO,MAAM;EACnC,MAAM,MAAM,IAAI,IAAI,YAAY,SAAS,YAAY,IAAI,KAAK,MAAM;EACpE,MAAM,QAAQ,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI;EAG/C,OAAO;GAAE,OAFK,MAAM,MAAM,CAAC,QAAQ,aAAa,EAAE,CAAC,CAAC,KAEvC;GAAG,SADA,MAAM,MAAM,oBAAoB,KAAK,CAAC,EAAA,CAAG;EAClC;CACzB,CAAC;AACH;AAEA,SAAS,kBAAkB,QAAwB;CACjD,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,GAAG,CAAC,CAAC;AAC5C;AAUA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,4DAA4D;CACtF,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;AAEA,SAAS,kBAAkB,KAAsB;CAC/C,IAAI,IAAI,IAAI,KAAK;CACjB,IAAI,EAAE,WAAW,KAAK,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI;CAC7D,IAAI,EAAE,SAAS,KAAK,GAAG,IAAI,EAAE,MAAM,GAAG,EAAE,YAAY,KAAK,CAAC;CAC1D,IAAI,EAAE,QAAQ,gBAAgB,IAAI,CAAC,CAAC,KAAK;CACzC,IAAI;EACF,OAAO,KAAK,MAAM,CAAC;CACrB,QAAQ;EACN,IAAI;GACF,MAAM,IAAI,EAAE,MAAM,aAAa;GAC/B,IAAI,GAAG,OAAO,KAAK,MAAM,EAAE,EAAE;EAC/B,QAAQ,CAER;CACF;CACA,OAAO;AACT;AAEA,eAAe,UAAU,QAAqB,QAAgB,cAAc,GAAqB;CAC/F,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EAoBF,MAAM,WAAU,MAnBG,mBACjB;GACE,eAAe,OAAO;GACtB,WAAW,OAAO;GAClB,SAAS;IACP,MAAM;IACN,SAAS;IACT,OAAO;KACL,UAAU;KACV,SAAS,OAAO;KAChB,UAAU;MACR,aAAa;MACb,WAAW,OAAO,QAAQ,IAAI,oBAAoB,KAAK;KACzD;IACF;GACF;EACF,GACA,MACF,EAAA,CACqB;EACrB,MAAM,SAAS,kBAAkB,OAAO;EACxC,IAAI,WAAW,MAAM,OAAO;CAC9B,SAAS,OAAO;EACd,IAAI,IAAI,MAAM,aAAa,MAAM;CACnC;CAEF,OAAO;AACT;AAuBA,eAAe,wBACb,MACA,QACA,QACsB;CAEtB,MAAM,cACJ;;gBAFiB,oBAAoB,QAAQ,KAAK,OAIxB,EAAE;CAK9B,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,WAAW,CAAC,CACtC,QAAQ,kBAAkB,MACe,CAAC,KACX;EAAE,yBAAyB;EAAO,qBAAqB,CAAC;CAAE;AAC9F;AAEA,eAAe,kBACb,MACA,QACA,eACA,QACsB;CACtB,MAAM,cACJ,iBAAiB,cAAc,SAAS,IACpC,KAAK,UAAU,eAAe,MAAM,CAAC,IACrC;CAON,OAAQ,MADa,UAAU,QALhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MAAM,CAAC,CACjC,QAAQ,0BAA0B,WAAW,CAAC,CAC9C,QAAQ,gBAAgB,OAAO,KAAK,gBAAgB,MAAM,CACjB,CAAC,KACX,EAAE,QAAQ,CAAC,EAAE;AACjD;AAEA,eAAe,oBACb,MACA,QACA,QACsB;CAKtB,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MACe,CAAC,KACX,EAAE,iBAAiB,KAAK;AAC5D;AAEA,eAAe,eAAe,MAAqB,QAAgB,QAA0C;CAC3G,IAAI,CAAC,OAAO,KAAK,GACf,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,QAAQ,KAAK,UAAU,EAAE,OAAO,eAAe,CAAC;CAAE;CAIxF,MAAM,KAAK,MAAM,wBAAwB,MAAM,QAAQ,MAAM;CAC7D,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;CAGjD,MAAM,CAAC,IAAI,MAAM,MAAM,QAAQ,IAAI,CACjC,kBAAkB,MAAM,QAAQ,eAAe,MAAM,GACrD,oBAAoB,MAAM,QAAQ,MAAM,CAC1C,CAAC;CAED,MAAM,cAAc,GAAG,UAAU,CAAC;CAClC,MAAM,YAAsB,CAAC;CAC7B,MAAM,eAAyB,CAAC;CAEhC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,gBAAgB,QAAQ,KAAK;EACpD,MAAM,EAAE,OAAO,WAAW,KAAK,gBAAgB;EAC/C,MAAM,QAAQ,YAAY,MAAM,CAAC;EACjC,MAAM,YAAY,MAAM,aAAa,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC;EACpF,MAAM,UAAU,IAAI,KAAK,MAAM,iBAAiB,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC,CAAC;EAChG,MAAM,WAAW,SAAS,QAAQ,MAAc,CAAC,QAAQ,IAAI,CAAC,CAAC;EAC/D,MAAM,YAAY,kBAAkB,MAAM;EAC1C,IAAI,SAAS,UAAU,WAAW,UAAU,KAAK,KAAK;OACjD,aAAa,KAAK,KAAK;CAC9B;CAEA,MAAM,YAAY,KAAK,gBAAgB;CACvC,MAAM,QAAQ,GAAG;CACjB,MAAM,eAAe,OAAO,UAAU,YAAY,MAAM,SAAS,KAAK,UAAU;CAEhF,IAAI;CACJ,IAAI,cAAc,GAAG,WAAW;MAC3B,IAAI,UAAU,WAAW,aAAa,cAAc,WAAW;MAC/D,IAAI,UAAU,WAAW,WAAW,WAAW;MAC/C,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,WAAW;CAEhB,MAAM,UAAU,GAAG;CACnB,MAAM,iBAAiB,YAAY,QAAQ,YAAY;CAEvD,OAAO;EACL,UAAU,aAAa;EACvB,OAAO,YAAY,IAAI,WAAW,IAAI;EACtC,QAAQ,KAAK,UAAU;GACrB;GACA;GACA;GACA;GACA,oBAAoB,cAAc;GAClC,WAAW,SAAS;GACpB,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAIA,SAAS,YAAoB;CAC3B,OAAO,QAAQ,IAAI,oBAAoB;AACzC;AAEA,SAAS,SAAS,QAA0D;CAC1E,MAAM,UAAU,KAAK,UAAU,GAAG,gBAAgB,MAAM;CACxD,MAAM,UAAU,KAAK,SAAS,eAAe,MAAM;CACnD,MAAM,WAAW,KAAK,SAAS,OAAO;CAEtC,MAAM,cAAc,SAAS,KAAK,SAAS,gBAAgB,GAAG,QAAQ;CACtE,MAAM,YAAY,eAAe,OAAO;CACxC,MAAM,cAAc,SAAS,KAAK,UAAU,kBAAkB,GAAG,QAAQ;CACzE,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CAEjF,MAAM,kBAAkB,qBAAqB,WAAW;CAGxD,MAAM,SAAS;EACb,YACG,QACC,wEACA,2FACF,CAAC,CACA,QAAQ,uDAAuD,iCAAiC,CAAC,CACjG,QAAQ,oBAAoB,sBAAsB;EACrD;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CAEX,MAAM,OAAsB;EAC1B;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF;CASA,OAAO;EAAE,MAAA;GALP,IAAI;GACJ;GACA,UAAU;EAGA;EAAG;CAAK;AACtB;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,YAAY,YAAY,OAAO,GAAG,gBAAgB,UACrE,MAAM,IAAI,MAAM,cAAc,KAAK,GAAG,yDAAyD;CAEjG,OAAO;AACT;;AAGA,SAAS,YAAY,MAAwB;CAC3C,IAAI;EACF,OAAO,YAAY,KAAK,MAAM,cAAc,CAAC,CAAC,CAC3C,QAAQ,SAAS;GAChB,IAAI;IACF,OAAO,SAAS,KAAK,MAAM,gBAAgB,IAAI,CAAC,CAAC,CAAC,YAAY;GAChE,QAAQ;IACN,OAAO;GACT;EACF,CAAC,CAAC,CACD,KAAK;CACV,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,6CAA6C,KAAK,MAAM,cAAc,EAAE,IACnE,eAAe,QAAQ,IAAI,UAAU,IAAI,wFAC0C,MAC1F;CACF;AACF;AAIA,SAAS,gBAAgB,UAA0B;CAGjD,IAAI,SAAS,WAAW,WAAW,GAAG,OAAO,SAAS,MAAM,CAAkB,CAAC,CAAC,KAAK;CACrF,MAAM,SAAS,SAAS,QAAQ,aAAa;CAC7C,IAAI,WAAW,IAAI,OAAO,SAAS,MAAM,SAAS,EAAoB,CAAC,CAAC,KAAK;CAC7E,OAAO;AACT;AAIA,SAAgB,0BAA4C;CAC1D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,OAAO,UAAU;GACvB,MAAM,SAAS,KAAK,MAAM,cAAc;GACxC,IAAI;IAEF,IAAI,CADS,SAAS,MACd,CAAC,CAAC,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;GAC5D,QAAQ;IACN,MAAM,IAAI,MACR,kCAAkC,OAAO,+EACsC,MACjF;GACF;GAEA,YAAY;GACZ,MAAM,MAAM,YAAY,IAAI;GAC5B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,mCAAmC,QAAQ;GACjF,QAAQ,KACN,iBAAiB,IAAI,OAAO,qBAAqB,KAAK,qFAEtC,QAAQ,IAAI,eAAe,YAAY,oBACzD;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,IAAI,MAAM,YADG,UACY,CAAC;GAC1B,IAAI,KAAK,KAAK;IACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;IAC7B,MAAM,IAAI,QAAQ,OAAO,KAAK,IAAI,EAAE,CAAC;GACvC,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;GAE/B,OAAO,IAAI,KAAK,OAAO,SAAS,EAAE,CAAC,CAAC,IAAI;EAC1C;EAEA,MAAM,aAAa,OAA+C,CAKlE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAGlE,OAAO,eAFM,SAAS,IAEG,GADV,gBAAgB,QACE,GAAG,YAAY,CAAC;EACnD;CACF;AACF"}

package/dist/{cadbench-BrpwOU6A.js → cadbench-BLSyxR1N.js}

@@ -1,9 +1,9 @@
+import { t as runBenchRouterTurn } from "./router-turn-C2wMiDoo.js";
 import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
 import { tmpdir } from "node:os";
 import { promisify } from "node:util";
-import "@tangle-network/agent-runtime/kernel";
 //#region src/worker-blender.ts
 /**
 * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender
@@ -162,24 +162,25 @@ async function judgeCriteria(instruction, script, criteria, renders) {
 		type: "image_url",
 		image_url: { url }
 	});
-	const res = await fetch(`${base}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${key}`
-		},
-		body: JSON.stringify({
-			model,
-			max_tokens: 1500,
-			temperature: 0,
-			messages: [{
-				role: "user",
-				content
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`);
-	const raw = (await res.json()).choices?.[0]?.message?.content ?? "";
+	const raw = (await runBenchRouterTurn({
+		routerBaseUrl: base,
+		routerKey: key,
+		profile: {
+			name: "cadbench-vision-judge",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: model,
+				metadata: {
+					temperature: 0,
+					maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500)
+				}
+			}
+		}
+	}, { messages: [{
+		role: "user",
+		content
+	}] })).finalText;
 	const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw);
 	if (!m) return {
 		passed: criteria.map(() => false),
@@ -282,4 +283,4 @@ function createCadBenchAdapter() {
 //#endregion
 export { createCadBenchAdapter as t };
 
-//# sourceMappingURL=cadbench-BrpwOU6A.js.map
+//# sourceMappingURL=cadbench-BLSyxR1N.js.map

package/dist/cadbench-BLSyxR1N.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cadbench-BLSyxR1N.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n    print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n    for c in o.bound_box:\n        w=o.matrix_world @ mathutils.Vector(c)\n        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n    az=math.radians(40 + v*360.0/nviews)\n    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n *  Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n  const views = Math.max(1, opts.views ?? 4)\n  const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n  const runnerPath = join(dir, 'runner.py')\n  const scriptPath = join(dir, 'model.py')\n  try {\n    await writeFile(runnerPath, RUNNER_PY)\n    await writeFile(scriptPath, script)\n    const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n    const out = `${run.stdout}\\n${run.stderr}`\n    const built = /RENDER_DONE/.test(out)\n    if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n    const renders: string[] = []\n    for (let v = 0; v < views; v++) {\n      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n      if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n    }\n    return { built: renders.length > 0, renders }\n  } finally {\n    await rm(dir, { recursive: true, force: true }).catch(() => {})\n  }\n}\n\nexport interface BlenderLocalConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** N standardized views to render (CADBench uses 4). Default 4. */\n  views?: number\n  /** The bpy authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface BlenderShot {\n  /** The bpy script the agent wrote — the artifact. */\n  artifact: string\n  /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n  renders: string[]\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  /** True if the script executed and produced at least one mesh. */\n  built: boolean\n  detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const views = Math.max(1, cfg.views ?? 4)\n  const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n  const trace: Span[] = []\n  const runId = `cadbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the bpy script; the\n  // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n  let renders: string[] = []\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n  // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n  // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n  // lastErr + the prior script verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: async () => {\n      const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n      await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n      return dir\n    },\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const runnerPath = join(dir, 'runner.py')\n      const scriptPath = join(dir, 'model.py')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'blender-worker',\n            harness: 'cli-base',\n            model: {\n              provider: 'tangle-router',\n              default: cfg.model,\n              metadata: { temperature: 0.3 },\n            },\n            prompt: { systemPrompt: directive },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const script = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, script)\n      const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n      const out = `${run.stdout}\\n${run.stderr}`\n      built = /RENDER_DONE/.test(out)\n      lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n      trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n      if (built) {\n        const collected: string[] = []\n        for (let v = 0; v < views; v++) {\n          const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n          if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n        }\n        renders = collected\n        // first view carries the screen span (run-capsule reveal)\n        trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n        built = collected.length > 0\n      }\n      return { artifact: script }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  const script = res.final.artifact\n  return {\n    artifact: script,\n    renders,\n    trace,\n    usage,\n    ok: script.trim().length > 0,\n    built,\n    detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\nimport { runBenchRouterTurn } from '../router-turn'\n\ninterface CadBenchMeta {\n  name: string\n  type: string\n  criteria: string[]\n}\n\nfunction must(name: string): string {\n  const v = process.env[name]\n  if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n  return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n *  criteria → a JSON array of booleans (true = satisfied). Faithful to the\n *  paper's combined image+script evaluation. Throws on transport failure (never\n *  a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n  instruction: string,\n  script: string,\n  criteria: string[],\n  renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n  const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n  const key = must('TANGLE_API_KEY')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n  const text =\n    `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n    `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n    `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n    `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n  const content: unknown[] = [{ type: 'text', text }]\n  for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: base,\n      routerKey: key,\n      profile: {\n        name: 'cadbench-vision-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: model,\n          metadata: {\n            temperature: 0,\n            maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500),\n          },\n        },\n      },\n    },\n    { messages: [{ role: 'user', content }] },\n  )\n  const raw = turn.finalText\n  const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n  let arr: unknown\n  try {\n    arr = JSON.parse(m[0].toLowerCase())\n  } catch {\n    return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n  }\n  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n  // Pad/truncate to criteria length (a short array scores the missing as fail).\n  const passed = criteria.map((_, i) => bools[i] === true)\n  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n  let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n  async function load(): Promise<typeof cache & object> {\n    if (cache) return cache\n    const path = process.env.CADBENCH_PATH\n    if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n    const text = await readFile(path, 'utf8')\n    cache = text\n      .split('\\n')\n      .filter((l) => l.trim())\n      .map((l) => {\n        const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n        return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n      })\n    return cache\n  }\n\n  return {\n    name: 'cadbench',\n\n    async preflight() {\n      const { execFile } = await import('node:child_process')\n      const { promisify } = await import('node:util')\n      const exec = promisify(execFile)\n      try {\n        await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        throw new Error(\n          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n            `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n        )\n      }\n      await load()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let rows = await load()\n      if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n      // TYPE filter (Simulative|Wild) via env, applied before limit.\n      const t = process.env.CADBENCH_TYPE\n      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n      if (opts.limit != null) rows = rows.slice(0, opts.limit)\n      return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // no reference bpy script ships with the benchmark\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = task.metadata as unknown as CadBenchMeta\n      const criteria = meta.criteria ?? []\n      if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n      if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n      const r = await renderBpy(artifact, { views: 4 })\n      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n      const score = passed.filter(Boolean).length / criteria.length\n      return { resolved: score === 1, score, detail: note }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;ACzFA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CAoBjF,MAAM,OAAM,MAnBO,mBACjB;EACE,eAAe;EACf,WAAW;EACX,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS;IACT,UAAU;KACR,aAAa;KACb,WAAW,OAAO,QAAQ,IAAI,oBAAoB,IAAI;IACxD;GACF;EACF;CACF,GACA,EAAE,UAAU,CAAC;EAAE,MAAM;EAAQ;CAAQ,CAAC,EAAE,CAC1C,EAAA,CACiB;CACjB,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}

package/dist/{cadgenbench-DF7hYHdl.js → cadgenbench-x2OFkf8y.js}

@@ -1,10 +1,10 @@
+import "./router-turn-C2wMiDoo.js";
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
 import "node:fs";
 import { tmpdir } from "node:os";
 import { promisify } from "node:util";
-import "@tangle-network/agent-runtime/kernel";
 promisify(execFile);
 const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? "/tmp/cgb-venv/bin/python";
 const CGB_DIR = process.env.CADGENBENCH_DIR ?? "/tmp/cadgenbench";
@@ -148,4 +148,4 @@ function createCadGenBenchAdapter() {
 //#endregion
 export { createCadGenBenchAdapter as t };
 
-//# sourceMappingURL=cadgenbench-DF7hYHdl.js.map
+//# sourceMappingURL=cadgenbench-x2OFkf8y.js.map

package/dist/cadgenbench-x2OFkf8y.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cadgenbench-x2OFkf8y.js","names":["execFileAsync"],"sources":["../src/worker-build123d.ts","../src/benchmarks/cadgenbench.ts"],"sourcesContent":["/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'build123d-worker',\n            model: { provider: 'tangle-router', default: cfg.model },\n            prompt: { systemPrompt: sys },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AA2BsB,UAAU,QAAQ;AAExC,MAAa,cAAc,QAAQ,IAAI,oBAAoB;AAC3D,MAAa,UAAU,QAAQ,IAAI,mBAAmB;;;;;;;;;;;;;;;;;;ACNtD,MAAM,gBAAgB,UAAU,QAAQ;;;;AAKxC,MAAM,WAAW;;;;;;;;;;;;;;;EAef,KAAK;;;AASP,SAAS,eAAsE;CAC7E,MAAM,IAAI,KAAK,SAAS,yBAAyB;CACjD,OAAO;EACL;GAAE,IAAI;GAAgB,QAAQ;GAAyF,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EAC5J;GAAE,IAAI;GAAW,QAAQ;GAAmC,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EACjG;GAAE,IAAI;GAAa,QAAQ;GAAwD,QAAQ,KAAK,GAAG,gBAAgB;EAAE;CACvH;AACF;AAEA,SAAgB,2BAA6C;CAC3D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,IAAI,MAAM,cAAc,aAAa,CAAC,MAAM,iFAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,CAAC,CAAC,OACtJ,OAAO;IAAE,QAAQ;IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;GAAE,EAC3E;GACA,IAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GACrB,MAAM,IAAI,MACR,sCAAsC,YAAY,KAAK,EAAE,OAAO,MAAM,GAAG,GAAG,EAAE,8DACf,QAAQ,4EAA4E,SACrJ;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GAGtC,IAAI,QAAQ,aAAa;GACzB,MAAM,OAAO,QAAQ,IAAI;GACzB,IAAI,MAAM;IACR,MAAM,EAAE,aAAa,MAAM,OAAO;IAClC,QAAQ,KAAK,MAAM,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG,MAAM,CAAC;GACrE;GACA,IAAI,KAAK,KAAK,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAClE,IAAI,KAAK,SAAS,MAAM,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;GACzD,MAAM,QAAQ,YAA6B;IAAE;IAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,EAAG;GAAE;GACxH,OAAO,MAAM,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAQ,UAAU,KAAK,EAAE,MAAM;GAAwC,EAAE;EACrI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,QAAQ,qBAAqB,KAAK;GAC1C,IAAI,CAAC,SAAS,SAAS,cAAc,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAA8B;GAClH,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,YAAY,CAAC;GACtD,MAAM,OAAO,KAAK,KAAK,gBAAgB;GACvC,MAAM,SAAS,KAAK,KAAK,UAAU;GACnC,IAAI;IACF,MAAM,UAAU,MAAM,QAAQ;IAC9B,MAAM,UAAU,QAAQ,QAAQ;IAEhC,MAAM,IAAI,MAAM,cAAc,YAAY;KAAC;KAAM;KAAa;KAAQ;KAAM;IAAM,GAAG;KAAE,WAAW,KAAK;KAAI,SAAS;IAAQ,CAAC,CAAC,CAAC,OAC5H,OAAO;KAAE,QAAS,EAA0B,UAAU;KAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;IAAE,EAChH;IACA,MAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;IAC5C,IAAI,CAAC,GAAG,OAAO;KAAE,UAAU;KAAO,OAAO;KAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,OAAA,CAAQ,MAAM,GAAG,GAAG;IAAI;IAC1H,MAAM,IAAI,KAAK,MAAM,EAAE,EAAE;IACzB,MAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;IAC9D,OAAO;KAAE,UAAU,SAAS;KAAkB;KAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,MAAM,KAAK;IAAK;GACnJ,UAAU;IACR,MAAM,GAAG,KAAK;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAChE;EACF;CACF;AACF"}

package/dist/index.d.ts

@@ -17,7 +17,6 @@ interface ResolveBenchClientOptions {
   backend: string;
   routerBaseUrl: string;
   routerKey: string;
-  model: string;
   /** When set on the `router` backend, the off-box worker becomes a `router-tools` agentic loop
    *  with a live `web_search` tool backed by this provider (`you`/`exa`/…). */
   searchProvider?: string;

package/dist/index.js

@@ -112,7 +112,7 @@ const makeSearchExecutor = (cfg) => {
 //#region src/resolve-client.ts
 /**
 * The ONE harness → worker-client mapping for the bench layer. Every bench entrypoint
-* (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runLoop` drives from a single
+* (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runAgentRounds` drives from a single
 * selector instead of hand-rolling the branch:
 *
 *   - `router`               → OFF-BOX: a router chat-completion as the leaf executor, presented
@@ -123,22 +123,21 @@ const makeSearchExecutor = (cfg) => {
 *   - `bridge`               → OFF-BOX: a local cli-bridge fronting a harness CLI
 *                              (opencode/kimi-code/…) as the leaf executor. Same resumable
 *                              `bridgeExecutor` the supervisor uses; harness+model ride the
-*                              bridge `model` id (`${harness}/${model}`).
+*                              exact harness/provider/model come from the turn's AgentProfile.
 *   - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend
 *                              TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`;
-*                              this only decides off-box-vs-in-box transport for `runLoop`.
+*                              this only decides off-box-vs-in-box transport for `runAgentRounds`.
 *
 * Centralizing it here means a new entrypoint gets the full off-box/in-box matrix for free and
 * the mapping can't drift between callers.
 */
 function resolveBenchClient(opts) {
-	const { backend, routerBaseUrl, routerKey, model, searchProvider } = opts;
+	const { backend, routerBaseUrl, routerKey, searchProvider } = opts;
 	if (backend === "router") {
 		if (searchProvider) return inlineSandboxClient(createExecutor({
 			backend: "router-tools",
 			routerBaseUrl,
 			routerKey,
-			model,
 			tools: [webSearchTool],
 			executeToolCall: makeSearchExecutor({
 				routerBaseUrl,
@@ -149,8 +148,7 @@ function resolveBenchClient(opts) {
 		return inlineSandboxClient(createExecutor({
 			backend: "router",
 			routerBaseUrl,
-			routerKey,
-			model
+			routerKey
 		}));
 	}
 	if (backend === "bridge") {
@@ -161,7 +159,6 @@ function resolveBenchClient(opts) {
 			bridge: {
 				url: opts.bridgeUrl,
 				bearer: bridgeBearer,
-				model,
 				timeoutMs: opts.timeoutMs
 			}
 		});
@@ -261,7 +258,6 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 		backend: cell.backend ?? "router",
 		routerBaseUrl,
 		routerKey,
-		model: cell.model,
 		...bridgeUrl ? { bridgeUrl } : {},
 		...bridgeBearer ? { bridgeBearer } : {},
 		...sandboxBaseUrl ? { sandboxBaseUrl } : {},
@@ -271,6 +267,11 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 	const harness = cell.harness ?? cell.profile?.metadata?.backendType ?? "opencode";
 	const profile = cell.profile ?? {
 		name: cell.label,
+		harness,
+		model: {
+			provider: "tangle-router",
+			default: cell.model
+		},
 		metadata: { backendType: harness }
 	};
 	const uniq = Math.random().toString(36).slice(2, 8);