@tangle-network/agent-bench 0.7.1 → 0.8.1

package/dist/benchmarks/simpleqa.js.map

@@ -1 +1 @@
-{"version":3,"file":"simpleqa.js","names":[],"sources":["../../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,MAAM,cACJ;AACF,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;;;;;;;AASA,MAAM,iBAAiB,UAAkB,MAAc,cACrD;CACE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,eAA6B;CACpC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6EAA6E;CACvG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;CAChB,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CAAC;IAAE,MAAM;IAAQ,SAAS,cAAc,UAAU,MAAM,SAAS;GAAE,CAAC;EAChF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MAAM,wBAAwB,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAE3F,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UACrB,MAAM,IAAI,MAAM,gDAAgD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAEtG,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,GAAG;CACzG;CACA,IAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBACjF,OAAO,OAAO;CAEhB,MAAM,IAAI,MACR,mEAAmE,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GACxG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;CAC7D,MAAM,OAAqB;EACzB,MAAM,IAAI;EACV,aAAa,IAAI;EACjB,OAAO,IAAI;EACX,YAAY,IAAI;EAChB,UAAU,IAAI;CAChB;CACA,OAAO;EACL,IAAI,YAAY;EAChB,OAAO;EACP,QAAQ,IAAI,UAAU;EACtB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAC/D,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,gEAAgE;CAE3G,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,OAAO,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC3C;CACA,IAAI,KAAK,UAAU,KAAA,GAAW,OAAO,MAAM,MAAM,GAAG,KAAK,KAAK;CAC9D,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,4CAA4C,KAAK,OAAO,iCAAiC;CACtG,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;;;;;;;AAQA,eAAe,SAAS,MAAyC;CAmB/D,MAAM,SAAS,MAAM,GAAG;;;eAfX,KAAK,UAAU,WAAW,EAAE;;;;;;;;;;;;;;CAeX;CAE9B,OAAO,YADM,KAAK,MAAM,MACF,CAAC,CAAC,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,aAAa;GACb,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,2BAA2B,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC1F;IACF,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,WAAW,EAAE;;;YAInC;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,8BAA8B,IAAI,uGAEA,YAAY,6EAEhD;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,OAAO,SAAS,IAAI;EACtB;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAC7C,MAAM,YAAY,eAAe,QAAQ;GAEzC,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,OAAO;KACP,aAAa;KACb,QAAQ;KACR,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;GAGF,MAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;GACrF,MAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;GAC9E,MAAM,WAAW,UAAU;GAC3B,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KACrB;KACA;KACA,MAAM,KAAK;KACX,WAAW;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;EACF;CACF;AACF"}
+{"version":3,"file":"simpleqa.js","names":[],"sources":["../../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'simpleqa-grader',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0 },\n        },\n      },\n    },\n    GRADER_PROMPT(question, gold, predicted),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('SimpleQA grader returned no message content')\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmCA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,MAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,MAAM,cACJ;AACF,MAAM,wBAAwB;;AAG9B,MAAM,kBAAkB;CACtB;CACA;CACA;CACA,iDAAiD,sBAAsB;CACvE;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAClE,WAAW,OAAO,OAAO,IAC3B,CAAC;CACD,OAAO;AACT;;;;;;;AAQA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,KAAK,MAAM,MAAM,GAAA,CAAI,KAAK;EAChC,IAAI,EAAE,SAAS,GAAG,OAAO;CAC3B;CACA,OAAO;AACT;;AAGA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,KAAK,SAAS,SAAS,yBAAyB,GACzD,KAAK,IAAI,EAAE,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAEtC,OAAO,CAAC,GAAG,IAAI;AACjB;;;;;;;;AASA,MAAM,iBAAiB,UAAkB,MAAc,cACrD;CACE;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,aAAa;CACb,gBAAgB;CAChB,qBAAqB;CACrB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAQb,SAAS,eAA6B;CACpC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6EAA6E;CACvG,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;CAiBhB,MAAM,WAAU,MAhBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU,EAAE,aAAa,EAAE;GAC7B;EACF;CACF,GACA,cAAc,UAAU,MAAM,SAAS,CACzC,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,6CAA6C;CAC3E,MAAM,SAAS,QAAQ,MAAM,8BAA8B;CAC3D,MAAM,OAAO,SAAS,OAAO,KAAK,QAAA,EAAU,KAAK,KAAK;CACtD,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,GAAG;CACzB,QAAQ;EACN,MAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,GAAG;CACzG;CACA,IAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBACjF,OAAO,OAAO;CAEhB,MAAM,IAAI,MACR,mEAAmE,KAAK,UAAU,MAAM,CAAC,CAAC,MAAM,GAAG,GAAG,GACxG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;CAC7D,MAAM,OAAqB;EACzB,MAAM,IAAI;EACV,aAAa,IAAI;EACjB,OAAO,IAAI;EACX,YAAY,IAAI;EAChB,UAAU,IAAI;CAChB;CACA,OAAO;EACL,IAAI,YAAY;EAChB,OAAO;EACP,QAAQ,IAAI,UAAU;EACtB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAC/D,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,gEAAgE;CAE3G,OAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;CACvE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,OAAO,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC3C;CACA,IAAI,KAAK,UAAU,KAAA,GAAW,OAAO,MAAM,MAAM,GAAG,KAAK,KAAK;CAC9D,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,4CAA4C,KAAK,OAAO,iCAAiC;CACtG,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;;;;;;;AAQA,eAAe,SAAS,MAAyC;CAmB/D,MAAM,SAAS,MAAM,GAAG;;;eAfX,KAAK,UAAU,WAAW,EAAE;;;;;;;;;;;;;;CAeX;CAE9B,OAAO,YADM,KAAK,MAAM,MACF,CAAC,CAAC,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAGhB,aAAa;GACb,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MACR,2BAA2B,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAC1F;IACF,CAAC;IACD;GACF;GACA,IAAI;IACF,MAAM,GACJ;eACK,KAAK,UAAU,WAAW,EAAE;;;YAInC;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,8BAA8B,IAAI,uGAEA,YAAY,6EAEhD;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,OAAO,SAAS,IAAI;EACtB;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,GAAG,sBAAsB,GAAG,KAAK;EAC1C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,cAAc,iBAAiB,QAAQ;GAC7C,MAAM,YAAY,eAAe,QAAQ;GAEzC,IAAI,YAAY,WAAW,GAEzB,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KACrB,OAAO;KACP,aAAa;KACb,QAAQ;KACR,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;GAGF,MAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;GACrF,MAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;GAC9E,MAAM,WAAW,UAAU;GAC3B,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KACrB;KACA;KACA,MAAM,KAAK;KACX,WAAW;KACX,OAAO,KAAK;KACZ,YAAY,KAAK;KACjB,eAAe,UAAU;IAC3B,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/trata-hedge.js

@@ -1,3 +1,4 @@
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { join } from "node:path";
 import { readFileSync, readdirSync, statSync } from "node:fs";
 //#region src/benchmarks/trata-hedge.ts
@@ -144,31 +145,27 @@ function parseJsonFallback(raw) {
 	return null;
 }
 async function callJudge(router, prompt, maxAttempts = 2) {
-	for (let i = 0; i < maxAttempts; i++) {
-		const res = await fetch(`${router.baseUrl}/chat/completions`, {
-			method: "POST",
-			headers: {
-				"content-type": "application/json",
-				authorization: `Bearer ${router.key}`
-			},
-			body: JSON.stringify({
-				model: router.model,
-				temperature: 0,
-				max_tokens: 16384,
-				messages: [{
-					role: "user",
-					content: prompt
-				}]
-			})
-		});
-		if (!res.ok) {
-			if (i < maxAttempts - 1) continue;
-			throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-		}
-		const content = (await res.json()).choices?.[0]?.message?.content;
-		if (typeof content !== "string") continue;
+	for (let i = 0; i < maxAttempts; i++) try {
+		const content = (await runBenchRouterTurn({
+			routerBaseUrl: router.baseUrl,
+			routerKey: router.key,
+			profile: {
+				name: "trata-hedge-judge",
+				harness: "cli-base",
+				model: {
+					provider: "tangle-router",
+					default: router.model,
+					metadata: {
+						temperature: 0,
+						maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 16384)
+					}
+				}
+			}
+		}, prompt)).finalText;
 		const parsed = parseJsonFallback(content);
 		if (parsed !== null) return parsed;
+	} catch (error) {
+		if (i + 1 === maxAttempts) throw error;
 	}
 	return null;
 }

package/dist/benchmarks/trata-hedge.js.map

@@ -1 +1 @@
-{"version":3,"file":"trata-hedge.js","names":[],"sources":["../../src/benchmarks/trata-hedge.ts"],"sourcesContent":["/**\n * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial\n * analysis tasks across 6 domains (private equity, managed care, industrials,\n * vertical SaaS, REITs, insurance). Each task bundles real earnings-call\n * transcripts, financial statements, press releases, and SEC filings under\n * `environment/data/`; the agent must produce a grounded analysis citing those\n * files.\n *\n * Harbor architecture: the original benchmark runs each task in a Docker\n * container with file-read tools. This adapter skips Harbor and embeds the data\n * files directly in the worker prompt so any router backend can score it. Large\n * files (> 30 KB) are truncated to fit within model context limits. The judge\n * reads the actual files from disk for citation verification.\n *\n * Judge: a 3-stage LLM cascade faithful to grade.py —\n *   Task 1: hallucination check (cited-file context + agent answer)\n *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)\n *   Task 3: synthesis check (ground truth + agent answer)\n * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,\n * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;\n * resolved = score 4 (sparse reward).\n *\n * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.\n * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench\n */\n\nimport { readdirSync, readFileSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench'\nconst MAX_FILE_BYTES = 8_000\n// Sandbox API rejects messages > 100K chars; instruction + suffix adds ~500 chars overhead.\nconst MAX_DATA_CHARS = 85_000\nconst WORKER_SUFFIX = [\n  '',\n  'The data files above are your only source. Cite each claim with the filename that supports it.',\n  'Write your complete analysis to the text block that starts with \"ANALYSIS:\" on a line by itself.',\n  'Take a clear position on the topic. Every factual claim must name the file it comes from.',\n].join('\\n')\n\n// Judge model: Gemini 2.5 Pro via router (matches gemini-3.1-pro-preview tier used by grade.py).\nconst JUDGE_MODEL = 'gemini-2.5-pro'\n\ninterface TrataTaskMeta {\n  taskDir: string\n  taskId: string\n  dataDir: string\n  groundTruth: string\n  gradingTask1: string\n  gradingTask2: string\n  gradingTask3: string\n  /** [(label, n_moves)] in rubric order. */\n  themeMoveCounts: Array<{ label: string; nMoves: number }>\n}\n\n// ── file reading helpers ──────────────────────────────────────────────────────\n\nfunction readText(p: string, maxBytes = MAX_FILE_BYTES): string {\n  try {\n    const buf = readFileSync(p)\n    if (buf.length <= maxBytes) return buf.toString('utf8')\n    return buf.subarray(0, maxBytes).toString('utf8') + `\\n[… truncated at ${maxBytes} bytes …]`\n  } catch {\n    return `[unreadable: ${p}]`\n  }\n}\n\nfunction listFilesRec(dir: string): string[] {\n  const out: string[] = []\n  try {\n    for (const name of readdirSync(dir)) {\n      const full = join(dir, name)\n      try {\n        if (statSync(full).isDirectory()) out.push(...listFilesRec(full))\n        else out.push(full)\n      } catch {\n        // skip unreadable entries\n      }\n    }\n  } catch {\n    // dir absent\n  }\n  return out.sort()\n}\n\nfunction buildDataBlock(dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  if (files.length === 0) return '(no data files found)'\n  // Prioritise: company profiles → earnings calls → primary-ticker financials →\n  // sec filings → press releases → peers. Stop once total chars hit MAX_DATA_CHARS.\n  const priority = (f: string) => {\n    if (f.includes('company_profiles')) return 0\n    if (f.includes('earnings_call')) return 1\n    if (/financials\\/(income|cash_flow|balance)/.test(f) && !/bam_|bx_|kkr_|ares_|cg_|owl_|fsk_|cat_|de_|tdg_|car_/.test(f)) return 2\n    if (f.includes('sec_filings')) return 3\n    if (f.includes('investor_pres')) return 4\n    if (f.includes('press_release')) return 5\n    return 6\n  }\n  const sorted = [...files].sort((a, b) => priority(a) - priority(b) || a.localeCompare(b))\n  const blocks: string[] = []\n  let total = 0\n  for (const f of sorted) {\n    const rel = f.slice(dataDir.length + 1)\n    const content = readText(f)\n    const chunk = `\\n=== FILE: data/${rel} ===\\n${content}`\n    if (total + chunk.length > MAX_DATA_CHARS) {\n      blocks.push(`\\n[… ${sorted.length - blocks.length} more files omitted to stay within context limit …]`)\n      break\n    }\n    blocks.push(chunk)\n    total += chunk.length\n  }\n  return blocks.join('')\n}\n\nfunction buildCitedDataBlock(answerText: string, dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  const cited: string[] = []\n  for (const f of files) {\n    const rel = f.slice(dataDir.length + 1)\n    const name = rel.split('/').pop() ?? ''\n    if (answerText.includes(rel) || answerText.includes(name)) cited.push(f)\n  }\n  if (cited.length === 0) {\n    return '(The agent did not cite any data files, or no cited file could be located.)'\n  }\n  const blocks: string[] = []\n  for (const f of cited) {\n    const rel = f.slice(dataDir.length + 1)\n    blocks.push(`\\n=== FILE: data/${rel} ===\\n${readText(f)}`)\n  }\n  return blocks.join('')\n}\n\n// ── ground-truth parser ───────────────────────────────────────────────────────\n\nfunction parseThemeMoveCounts(groundTruth: string): Array<{ label: string; nMoves: number }> {\n  const lines = groundTruth.split('\\n')\n  const themeStarts = lines.reduce<number[]>((acc, l, i) => {\n    if (/^\\d+\\./.test(l.trim())) acc.push(i)\n    return acc\n  }, [])\n  return themeStarts.map((start, k) => {\n    const end = k + 1 < themeStarts.length ? themeStarts[k + 1] : lines.length\n    const block = lines.slice(start, end).join('\\n')\n    const label = lines[start].replace(/^\\d+\\.\\s*/, '').trim()\n    const nMoves = (block.match(/^\\s*\\[[a-z]\\]\\s+/gm) ?? []).length\n    return { label, nMoves }\n  })\n}\n\nfunction coverageThreshold(nMoves: number): number {\n  return Math.max(1, Math.min(nMoves - 1, 3))\n}\n\n// ── router judge calls ────────────────────────────────────────────────────────\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the Trata hedge-bench judge')\n  const model = process.env.JUDGE_MODEL ?? JUDGE_MODEL\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\nfunction parseJsonFallback(raw: string): unknown {\n  let s = raw.trim()\n  if (s.startsWith('```')) s = s.split('\\n').slice(1).join('\\n')\n  if (s.endsWith('```')) s = s.slice(0, s.lastIndexOf('```'))\n  s = s.replace(/,(\\s*[}\\]])/g, '$1').trim()\n  try {\n    return JSON.parse(s)\n  } catch {\n    try {\n      const m = s.match(/\\{[\\s\\S]*\\}/)\n      if (m) return JSON.parse(m[0])\n    } catch {\n      // fallthrough\n    }\n  }\n  return null\n}\n\nasync function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise<unknown> {\n  for (let i = 0; i < maxAttempts; i++) {\n    const res = await fetch(`${router.baseUrl}/chat/completions`, {\n      method: 'POST',\n      headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n      body: JSON.stringify({\n        model: router.model,\n        temperature: 0,\n        max_tokens: 16384,\n        messages: [{ role: 'user', content: prompt }],\n      }),\n    })\n    if (!res.ok) {\n      if (i < maxAttempts - 1) continue\n      throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n    }\n    const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n    const content = body.choices?.[0]?.message?.content\n    if (typeof content !== 'string') continue\n    const parsed = parseJsonFallback(content)\n    if (parsed !== null) return parsed\n  }\n  return null\n}\n\ninterface Task1Result {\n  hallucinations_detected?: boolean | string\n  unverifiable_claims?: Array<{ claim: string; cited_file?: string; issue?: string; evidence_check?: string }>\n}\n\ninterface Task2ThemeResult {\n  label?: string\n  moves_hit?: string[]\n  moves_missed?: string[]\n  moves_tainted?: string[]\n  move_reasoning?: Record<string, string>\n}\n\ninterface Task2Result {\n  themes?: Task2ThemeResult[]\n}\n\ninterface Task3Result {\n  synthesis_found?: string | null\n}\n\nasync function task1HallucinationCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task1Result> {\n  const citedBlock = buildCitedDataBlock(answer, meta.dataDir)\n  const dataContext =\n    'The block below contains the contents of the data files the agent cited in their answer. ' +\n    'Treat this as the ground truth for factual verification.\\n\\n' +\n    `<cited_data>\\n${citedBlock}\\n</cited_data>`\n  const prompt = meta.gradingTask1\n    .replace('{data_context}', dataContext)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task1Result) ?? { hallucinations_detected: false, unverifiable_claims: [] }\n}\n\nasync function task2PerMoveCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  flaggedClaims: Task1Result['unverifiable_claims'],\n  router: JudgeRouter,\n): Promise<Task2Result> {\n  const claimsBlock =\n    flaggedClaims && flaggedClaims.length > 0\n      ? JSON.stringify(flaggedClaims, null, 2)\n      : '(No claims were flagged as hallucinations.)'\n  const prompt = meta.gradingTask2\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n    .replace('{flagged_claims_block}', claimsBlock)\n    .replace('{num_themes}', String(meta.themeMoveCounts.length))\n  const result = await callJudge(router, prompt)\n  return (result as Task2Result) ?? { themes: [] }\n}\n\nasync function task3SynthesisCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task3Result> {\n  const prompt = meta.gradingTask3\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task3Result) ?? { synthesis_found: null }\n}\n\nasync function runTriataJudge(meta: TrataTaskMeta, answer: string, router: JudgeRouter): Promise<BenchScore> {\n  if (!answer.trim()) {\n    return { resolved: false, score: 0, detail: JSON.stringify({ error: 'empty answer' }) }\n  }\n\n  // Stage 1: hallucination check (serial — feeds into stage 2).\n  const t1 = await task1HallucinationCheck(meta, answer, router)\n  const flaggedClaims = t1.unverifiable_claims ?? []\n\n  // Stages 2 + 3 in parallel.\n  const [t2, t3] = await Promise.all([\n    task2PerMoveCheck(meta, answer, flaggedClaims, router),\n    task3SynthesisCheck(meta, answer, router),\n  ])\n\n  const judgeThemes = t2.themes ?? []\n  const themesHit: string[] = []\n  const themesMissed: string[] = []\n\n  for (let i = 0; i < meta.themeMoveCounts.length; i++) {\n    const { label, nMoves } = meta.themeMoveCounts[i]\n    const judge = judgeThemes[i] ?? {}\n    const movesHit = (judge.moves_hit ?? []).map((m: string) => m.replace(/[\\[\\]]/g, ''))\n    const tainted = new Set((judge.moves_tainted ?? []).map((m: string) => m.replace(/[\\[\\]]/g, '')))\n    const validHit = movesHit.filter((m: string) => !tainted.has(m))\n    const threshold = coverageThreshold(nMoves)\n    if (validHit.length >= threshold) themesHit.push(label)\n    else themesMissed.push(label)\n  }\n\n  const numThemes = meta.themeMoveCounts.length\n  const synth = t3.synthesis_found\n  const hasSynthesis = typeof synth === 'string' && synth.length > 0 && synth !== 'null'\n\n  let rawScore: number\n  if (numThemes === 0) rawScore = 0\n  else if (themesHit.length === numThemes && hasSynthesis) rawScore = 4\n  else if (themesHit.length === numThemes) rawScore = 3\n  else if (themesHit.length >= 2) rawScore = 2\n  else if (themesHit.length >= 1) rawScore = 1\n  else rawScore = 0\n\n  const hallRaw = t1.hallucinations_detected\n  const hallucinations = hallRaw === true || hallRaw === 'true'\n\n  return {\n    resolved: rawScore === 4,\n    score: numThemes > 0 ? rawScore / 4 : 0,\n    detail: JSON.stringify({\n      rawScore,\n      themesHit,\n      themesMissed,\n      hallucinations,\n      unverifiableClaims: flaggedClaims.length,\n      synthesis: synth ?? null,\n      judgeModel: router.model,\n    }),\n  }\n}\n\n// ── task loading ──────────────────────────────────────────────────────────────\n\nfunction benchRoot(): string {\n  return process.env.TRATA_BENCH_ROOT ?? DEFAULT_BENCH_ROOT\n}\n\nfunction loadTask(taskId: string): { task: BenchTask; meta: TrataTaskMeta } {\n  const taskDir = join(benchRoot(), 'environments', taskId)\n  const dataDir = join(taskDir, 'environment', 'data')\n  const testsDir = join(taskDir, 'tests')\n\n  const instruction = readText(join(taskDir, 'instruction.md'), Infinity)\n  const dataBlock = buildDataBlock(dataDir)\n  const groundTruth = readText(join(testsDir, 'ground_truth.txt'), Infinity)\n  const gradingTask1 = readText(join(testsDir, 'grading_prompt_task1.md'), Infinity)\n  const gradingTask2 = readText(join(testsDir, 'grading_prompt_task2.md'), Infinity)\n  const gradingTask3 = readText(join(testsDir, 'grading_prompt_task3.md'), Infinity)\n\n  const themeMoveCounts = parseThemeMoveCounts(groundTruth)\n\n  // Rewrite the instruction so the agent knows data is inline (no /app/data/ FS).\n  const prompt = [\n    instruction\n      .replace(\n        \"You are a financial analyst with access to the data in `/app/data/`.\",\n        'You are a financial analyst. The relevant financial data files are provided inline below.',\n      )\n      .replace(/Write your full analysis to `\\/app\\/answer\\.txt`\\.?/, 'Write your full analysis below.')\n      .replace(/`\\/app\\/data\\/`/g, 'the data files below'),\n    '',\n    '## Data files (inline)',\n    dataBlock,\n    WORKER_SUFFIX,\n  ].join('\\n')\n\n  const meta: TrataTaskMeta = {\n    taskDir,\n    taskId,\n    dataDir,\n    groundTruth,\n    gradingTask1,\n    gradingTask2,\n    gradingTask3,\n    themeMoveCounts,\n  }\n\n  // Store meta in the BenchTask so the judge can access it without re-reading.\n  const task: BenchTask = {\n    id: taskId,\n    prompt,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n\n  return { task, meta }\n}\n\nfunction readMeta(task: BenchTask): TrataTaskMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskDir !== 'string' || typeof md.groundTruth !== 'string') {\n    throw new Error(`Trata task ${task.id} missing judge metadata — was it loaded by this adapter?`)\n  }\n  return md as unknown as TrataTaskMeta\n}\n\n/** List all task ids from the environments directory. */\nfunction listTaskIds(root: string): string[] {\n  try {\n    return readdirSync(join(root, 'environments'))\n      .filter((name) => {\n        try {\n          return statSync(join(root, 'environments', name)).isDirectory()\n        } catch {\n          return false\n        }\n      })\n      .sort()\n  } catch (err) {\n    throw new Error(\n      `Trata benchmark environments not found at ${join(root, 'environments')}: ` +\n        `${err instanceof Error ? err.message : err}.\\n` +\n        `Clone the repo: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n    )\n  }\n}\n\n// ── extract agent analysis from artifact ─────────────────────────────────────\n\nfunction extractAnalysis(artifact: string): string {\n  // Model may write \"ANALYSIS:\" as its very first token (no leading newline) or\n  // after some preamble. Both are valid; fail-closed on absence.\n  if (artifact.startsWith('ANALYSIS:')) return artifact.slice('ANALYSIS:'.length).trim()\n  const marker = artifact.indexOf('\\nANALYSIS:')\n  if (marker !== -1) return artifact.slice(marker + '\\nANALYSIS:'.length).trim()\n  return ''\n}\n\n// ── adapter factory ───────────────────────────────────────────────────────────\n\nexport function createTrataHedgeAdapter(): BenchmarkAdapter {\n  return {\n    name: 'trata-hedge',\n\n    async preflight() {\n      const root = benchRoot()\n      const envDir = join(root, 'environments')\n      try {\n        const stat = statSync(envDir)\n        if (!stat.isDirectory()) throw new Error('not a directory')\n      } catch {\n        throw new Error(\n          `Trata hedge-bench not found at ${envDir}.\\n` +\n            `Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n        )\n      }\n      // Validate judge router connectivity.\n      judgeRouter()\n      const ids = listTaskIds(root)\n      if (ids.length === 0) throw new Error(`No task directories found under ${envDir}`)\n      console.warn(\n        `[trata-hedge] ${ids.length} tasks loaded from ${root}. ` +\n          `Worker model reads embedded data inline (Harbor/Docker not required). ` +\n          `Judge uses ${process.env.JUDGE_MODEL ?? JUDGE_MODEL} via Tangle router.`,\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const root = benchRoot()\n      let ids = listTaskIds(root)\n      if (opts.ids) {\n        const want = new Set(opts.ids)\n        ids = ids.filter((id) => want.has(id))\n      } else if (opts.limit !== undefined) {\n        ids = ids.slice(0, opts.limit)\n      }\n      return ids.map((id) => loadTask(id).task)\n    },\n\n    async goldArtifact(_task: BenchTask): Promise<string | undefined> {\n      // No single gold answer: the score comes from rubric theme coverage, not\n      // string-match against a reference response. Return undefined — the bench\n      // harness will skip judge self-verification for this adapter.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const answer = extractAnalysis(artifact)\n      return runTriataJudge(meta, answer, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,MAAM,qBAAqB;AAC3B,MAAM,iBAAiB;AAEvB,MAAM,iBAAiB;AACvB,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAGX,MAAM,cAAc;AAgBpB,SAAS,SAAS,GAAW,WAAW,gBAAwB;CAC9D,IAAI;EACF,MAAM,MAAM,aAAa,CAAC;EAC1B,IAAI,IAAI,UAAU,UAAU,OAAO,IAAI,SAAS,MAAM;EACtD,OAAO,IAAI,SAAS,GAAG,QAAQ,CAAC,CAAC,SAAS,MAAM,IAAI,qBAAqB,SAAS;CACpF,QAAQ;EACN,OAAO,gBAAgB,EAAE;CAC3B;AACF;AAEA,SAAS,aAAa,KAAuB;CAC3C,MAAM,MAAgB,CAAC;CACvB,IAAI;EACF,KAAK,MAAM,QAAQ,YAAY,GAAG,GAAG;GACnC,MAAM,OAAO,KAAK,KAAK,IAAI;GAC3B,IAAI;IACF,IAAI,SAAS,IAAI,CAAC,CAAC,YAAY,GAAG,IAAI,KAAK,GAAG,aAAa,IAAI,CAAC;SAC3D,IAAI,KAAK,IAAI;GACpB,QAAQ,CAER;EACF;CACF,QAAQ,CAER;CACA,OAAO,IAAI,KAAK;AAClB;AAEA,SAAS,eAAe,SAAyB;CAC/C,MAAM,QAAQ,aAAa,OAAO;CAClC,IAAI,MAAM,WAAW,GAAG,OAAO;CAG/B,MAAM,YAAY,MAAc;EAC9B,IAAI,EAAE,SAAS,kBAAkB,GAAG,OAAO;EAC3C,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,yCAAyC,KAAK,CAAC,KAAK,CAAC,uDAAuD,KAAK,CAAC,GAAG,OAAO;EAChI,IAAI,EAAE,SAAS,aAAa,GAAG,OAAO;EACtC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,OAAO;CACT;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;CACxF,MAAM,SAAmB,CAAC;CAC1B,IAAI,QAAQ;CACZ,KAAK,MAAM,KAAK,QAAQ;EAGtB,MAAM,QAAQ,oBAFF,EAAE,MAAM,QAAQ,SAAS,CAED,EAAE,QADtB,SAAS,CAC2B;EACpD,IAAI,QAAQ,MAAM,SAAS,gBAAgB;GACzC,OAAO,KAAK,QAAQ,OAAO,SAAS,OAAO,OAAO,oDAAoD;GACtG;EACF;EACA,OAAO,KAAK,KAAK;EACjB,SAAS,MAAM;CACjB;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAEA,SAAS,oBAAoB,YAAoB,SAAyB;CACxE,MAAM,QAAQ,aAAa,OAAO;CAClC,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;EACrC,IAAI,WAAW,SAAS,GAAG,KAAK,WAAW,SAAS,IAAI,GAAG,MAAM,KAAK,CAAC;CACzE;CACA,IAAI,MAAM,WAAW,GACnB,OAAO;CAET,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,OAAO,KAAK,oBAAoB,IAAI,QAAQ,SAAS,CAAC,GAAG;CAC3D;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAIA,SAAS,qBAAqB,aAA+D;CAC3F,MAAM,QAAQ,YAAY,MAAM,IAAI;CACpC,MAAM,cAAc,MAAM,QAAkB,KAAK,GAAG,MAAM;EACxD,IAAI,SAAS,KAAK,EAAE,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC;EACvC,OAAO;CACT,GAAG,CAAC,CAAC;CACL,OAAO,YAAY,KAAK,OAAO,MAAM;EACnC,MAAM,MAAM,IAAI,IAAI,YAAY,SAAS,YAAY,IAAI,KAAK,MAAM;EACpE,MAAM,QAAQ,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI;EAG/C,OAAO;GAAE,OAFK,MAAM,MAAM,CAAC,QAAQ,aAAa,EAAE,CAAC,CAAC,KAEvC;GAAG,SADA,MAAM,MAAM,oBAAoB,KAAK,CAAC,EAAA,CAAG;EAClC;CACzB,CAAC;AACH;AAEA,SAAS,kBAAkB,QAAwB;CACjD,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,GAAG,CAAC,CAAC;AAC5C;AAUA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,4DAA4D;CACtF,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;AAEA,SAAS,kBAAkB,KAAsB;CAC/C,IAAI,IAAI,IAAI,KAAK;CACjB,IAAI,EAAE,WAAW,KAAK,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI;CAC7D,IAAI,EAAE,SAAS,KAAK,GAAG,IAAI,EAAE,MAAM,GAAG,EAAE,YAAY,KAAK,CAAC;CAC1D,IAAI,EAAE,QAAQ,gBAAgB,IAAI,CAAC,CAAC,KAAK;CACzC,IAAI;EACF,OAAO,KAAK,MAAM,CAAC;CACrB,QAAQ;EACN,IAAI;GACF,MAAM,IAAI,EAAE,MAAM,aAAa;GAC/B,IAAI,GAAG,OAAO,KAAK,MAAM,EAAE,EAAE;EAC/B,QAAQ,CAER;CACF;CACA,OAAO;AACT;AAEA,eAAe,UAAU,QAAqB,QAAgB,cAAc,GAAqB;CAC/F,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;EACpC,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;GAC5D,QAAQ;GACR,SAAS;IAAE,gBAAgB;IAAoB,eAAe,UAAU,OAAO;GAAM;GACrF,MAAM,KAAK,UAAU;IACnB,OAAO,OAAO;IACd,aAAa;IACb,YAAY;IACZ,UAAU,CAAC;KAAE,MAAM;KAAQ,SAAS;IAAO,CAAC;GAC9C,CAAC;EACH,CAAC;EACD,IAAI,CAAC,IAAI,IAAI;GACX,IAAI,IAAI,cAAc,GAAG;GACzB,MAAM,IAAI,MAAM,oBAAoB,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;EACvF;EAEA,MAAM,WAAU,MADI,IAAI,KAAK,EAAA,CACR,UAAU,EAAE,EAAE,SAAS;EAC5C,IAAI,OAAO,YAAY,UAAU;EACjC,MAAM,SAAS,kBAAkB,OAAO;EACxC,IAAI,WAAW,MAAM,OAAO;CAC9B;CACA,OAAO;AACT;AAuBA,eAAe,wBACb,MACA,QACA,QACsB;CAEtB,MAAM,cACJ;;gBAFiB,oBAAoB,QAAQ,KAAK,OAIxB,EAAE;CAK9B,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,WAAW,CAAC,CACtC,QAAQ,kBAAkB,MACe,CAAC,KACX;EAAE,yBAAyB;EAAO,qBAAqB,CAAC;CAAE;AAC9F;AAEA,eAAe,kBACb,MACA,QACA,eACA,QACsB;CACtB,MAAM,cACJ,iBAAiB,cAAc,SAAS,IACpC,KAAK,UAAU,eAAe,MAAM,CAAC,IACrC;CAON,OAAQ,MADa,UAAU,QALhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MAAM,CAAC,CACjC,QAAQ,0BAA0B,WAAW,CAAC,CAC9C,QAAQ,gBAAgB,OAAO,KAAK,gBAAgB,MAAM,CACjB,CAAC,KACX,EAAE,QAAQ,CAAC,EAAE;AACjD;AAEA,eAAe,oBACb,MACA,QACA,QACsB;CAKtB,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MACe,CAAC,KACX,EAAE,iBAAiB,KAAK;AAC5D;AAEA,eAAe,eAAe,MAAqB,QAAgB,QAA0C;CAC3G,IAAI,CAAC,OAAO,KAAK,GACf,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,QAAQ,KAAK,UAAU,EAAE,OAAO,eAAe,CAAC;CAAE;CAIxF,MAAM,KAAK,MAAM,wBAAwB,MAAM,QAAQ,MAAM;CAC7D,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;CAGjD,MAAM,CAAC,IAAI,MAAM,MAAM,QAAQ,IAAI,CACjC,kBAAkB,MAAM,QAAQ,eAAe,MAAM,GACrD,oBAAoB,MAAM,QAAQ,MAAM,CAC1C,CAAC;CAED,MAAM,cAAc,GAAG,UAAU,CAAC;CAClC,MAAM,YAAsB,CAAC;CAC7B,MAAM,eAAyB,CAAC;CAEhC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,gBAAgB,QAAQ,KAAK;EACpD,MAAM,EAAE,OAAO,WAAW,KAAK,gBAAgB;EAC/C,MAAM,QAAQ,YAAY,MAAM,CAAC;EACjC,MAAM,YAAY,MAAM,aAAa,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC;EACpF,MAAM,UAAU,IAAI,KAAK,MAAM,iBAAiB,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC,CAAC;EAChG,MAAM,WAAW,SAAS,QAAQ,MAAc,CAAC,QAAQ,IAAI,CAAC,CAAC;EAC/D,MAAM,YAAY,kBAAkB,MAAM;EAC1C,IAAI,SAAS,UAAU,WAAW,UAAU,KAAK,KAAK;OACjD,aAAa,KAAK,KAAK;CAC9B;CAEA,MAAM,YAAY,KAAK,gBAAgB;CACvC,MAAM,QAAQ,GAAG;CACjB,MAAM,eAAe,OAAO,UAAU,YAAY,MAAM,SAAS,KAAK,UAAU;CAEhF,IAAI;CACJ,IAAI,cAAc,GAAG,WAAW;MAC3B,IAAI,UAAU,WAAW,aAAa,cAAc,WAAW;MAC/D,IAAI,UAAU,WAAW,WAAW,WAAW;MAC/C,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,WAAW;CAEhB,MAAM,UAAU,GAAG;CACnB,MAAM,iBAAiB,YAAY,QAAQ,YAAY;CAEvD,OAAO;EACL,UAAU,aAAa;EACvB,OAAO,YAAY,IAAI,WAAW,IAAI;EACtC,QAAQ,KAAK,UAAU;GACrB;GACA;GACA;GACA;GACA,oBAAoB,cAAc;GAClC,WAAW,SAAS;GACpB,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAIA,SAAS,YAAoB;CAC3B,OAAO,QAAQ,IAAI,oBAAoB;AACzC;AAEA,SAAS,SAAS,QAA0D;CAC1E,MAAM,UAAU,KAAK,UAAU,GAAG,gBAAgB,MAAM;CACxD,MAAM,UAAU,KAAK,SAAS,eAAe,MAAM;CACnD,MAAM,WAAW,KAAK,SAAS,OAAO;CAEtC,MAAM,cAAc,SAAS,KAAK,SAAS,gBAAgB,GAAG,QAAQ;CACtE,MAAM,YAAY,eAAe,OAAO;CACxC,MAAM,cAAc,SAAS,KAAK,UAAU,kBAAkB,GAAG,QAAQ;CACzE,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CAEjF,MAAM,kBAAkB,qBAAqB,WAAW;CAGxD,MAAM,SAAS;EACb,YACG,QACC,wEACA,2FACF,CAAC,CACA,QAAQ,uDAAuD,iCAAiC,CAAC,CACjG,QAAQ,oBAAoB,sBAAsB;EACrD;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CAEX,MAAM,OAAsB;EAC1B;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF;CASA,OAAO;EAAE,MAAA;GALP,IAAI;GACJ;GACA,UAAU;EAGA;EAAG;CAAK;AACtB;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,YAAY,YAAY,OAAO,GAAG,gBAAgB,UACrE,MAAM,IAAI,MAAM,cAAc,KAAK,GAAG,yDAAyD;CAEjG,OAAO;AACT;;AAGA,SAAS,YAAY,MAAwB;CAC3C,IAAI;EACF,OAAO,YAAY,KAAK,MAAM,cAAc,CAAC,CAAC,CAC3C,QAAQ,SAAS;GAChB,IAAI;IACF,OAAO,SAAS,KAAK,MAAM,gBAAgB,IAAI,CAAC,CAAC,CAAC,YAAY;GAChE,QAAQ;IACN,OAAO;GACT;EACF,CAAC,CAAC,CACD,KAAK;CACV,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,6CAA6C,KAAK,MAAM,cAAc,EAAE,IACnE,eAAe,QAAQ,IAAI,UAAU,IAAI,wFAC0C,MAC1F;CACF;AACF;AAIA,SAAS,gBAAgB,UAA0B;CAGjD,IAAI,SAAS,WAAW,WAAW,GAAG,OAAO,SAAS,MAAM,CAAkB,CAAC,CAAC,KAAK;CACrF,MAAM,SAAS,SAAS,QAAQ,aAAa;CAC7C,IAAI,WAAW,IAAI,OAAO,SAAS,MAAM,SAAS,EAAoB,CAAC,CAAC,KAAK;CAC7E,OAAO;AACT;AAIA,SAAgB,0BAA4C;CAC1D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,OAAO,UAAU;GACvB,MAAM,SAAS,KAAK,MAAM,cAAc;GACxC,IAAI;IAEF,IAAI,CADS,SAAS,MACd,CAAC,CAAC,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;GAC5D,QAAQ;IACN,MAAM,IAAI,MACR,kCAAkC,OAAO,+EACsC,MACjF;GACF;GAEA,YAAY;GACZ,MAAM,MAAM,YAAY,IAAI;GAC5B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,mCAAmC,QAAQ;GACjF,QAAQ,KACN,iBAAiB,IAAI,OAAO,qBAAqB,KAAK,qFAEtC,QAAQ,IAAI,eAAe,YAAY,oBACzD;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,IAAI,MAAM,YADG,UACY,CAAC;GAC1B,IAAI,KAAK,KAAK;IACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;IAC7B,MAAM,IAAI,QAAQ,OAAO,KAAK,IAAI,EAAE,CAAC;GACvC,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;GAE/B,OAAO,IAAI,KAAK,OAAO,SAAS,EAAE,CAAC,CAAC,IAAI;EAC1C;EAEA,MAAM,aAAa,OAA+C,CAKlE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAGlE,OAAO,eAFM,SAAS,IAEG,GADV,gBAAgB,QACE,GAAG,YAAY,CAAC;EACnD;CACF;AACF"}
+{"version":3,"file":"trata-hedge.js","names":[],"sources":["../../src/benchmarks/trata-hedge.ts"],"sourcesContent":["/**\n * Trata Hedge-Bench adapter (Trata-Inc/trata-hedge-bench). 102 financial\n * analysis tasks across 6 domains (private equity, managed care, industrials,\n * vertical SaaS, REITs, insurance). Each task bundles real earnings-call\n * transcripts, financial statements, press releases, and SEC filings under\n * `environment/data/`; the agent must produce a grounded analysis citing those\n * files.\n *\n * Harbor architecture: the original benchmark runs each task in a Docker\n * container with file-read tools. This adapter skips Harbor and embeds the data\n * files directly in the worker prompt so any router backend can score it. Large\n * files (> 30 KB) are truncated to fit within model context limits. The judge\n * reads the actual files from disk for citation verification.\n *\n * Judge: a 3-stage LLM cascade faithful to grade.py —\n *   Task 1: hallucination check (cited-file context + agent answer)\n *   Task 2: per-move coverage check (ground truth + agent answer + flagged claims)\n *   Task 3: synthesis check (ground truth + agent answer)\n * Score 0–4: themes_covered == num_themes && synthesis → 4, all themes → 3,\n * ≥2 themes → 2, ≥1 theme → 1, else → 0. Normalized to 0..1 for BenchScore;\n * resolved = score 4 (sparse reward).\n *\n * Requires TANGLE_API_KEY and TRATA_BENCH_ROOT=/tmp/trata-hedge-bench.\n * Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench /tmp/trata-hedge-bench\n */\n\nimport { readdirSync, readFileSync, statSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { runBenchRouterTurn } from '../router-turn'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench'\nconst MAX_FILE_BYTES = 8_000\n// Sandbox API rejects messages > 100K chars; instruction + suffix adds ~500 chars overhead.\nconst MAX_DATA_CHARS = 85_000\nconst WORKER_SUFFIX = [\n  '',\n  'The data files above are your only source. Cite each claim with the filename that supports it.',\n  'Write your complete analysis to the text block that starts with \"ANALYSIS:\" on a line by itself.',\n  'Take a clear position on the topic. Every factual claim must name the file it comes from.',\n].join('\\n')\n\n// Judge model: Gemini 2.5 Pro via router (matches gemini-3.1-pro-preview tier used by grade.py).\nconst JUDGE_MODEL = 'gemini-2.5-pro'\n\ninterface TrataTaskMeta {\n  taskDir: string\n  taskId: string\n  dataDir: string\n  groundTruth: string\n  gradingTask1: string\n  gradingTask2: string\n  gradingTask3: string\n  /** [(label, n_moves)] in rubric order. */\n  themeMoveCounts: Array<{ label: string; nMoves: number }>\n}\n\n// ── file reading helpers ──────────────────────────────────────────────────────\n\nfunction readText(p: string, maxBytes = MAX_FILE_BYTES): string {\n  try {\n    const buf = readFileSync(p)\n    if (buf.length <= maxBytes) return buf.toString('utf8')\n    return buf.subarray(0, maxBytes).toString('utf8') + `\\n[… truncated at ${maxBytes} bytes …]`\n  } catch {\n    return `[unreadable: ${p}]`\n  }\n}\n\nfunction listFilesRec(dir: string): string[] {\n  const out: string[] = []\n  try {\n    for (const name of readdirSync(dir)) {\n      const full = join(dir, name)\n      try {\n        if (statSync(full).isDirectory()) out.push(...listFilesRec(full))\n        else out.push(full)\n      } catch {\n        // skip unreadable entries\n      }\n    }\n  } catch {\n    // dir absent\n  }\n  return out.sort()\n}\n\nfunction buildDataBlock(dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  if (files.length === 0) return '(no data files found)'\n  // Prioritise: company profiles → earnings calls → primary-ticker financials →\n  // sec filings → press releases → peers. Stop once total chars hit MAX_DATA_CHARS.\n  const priority = (f: string) => {\n    if (f.includes('company_profiles')) return 0\n    if (f.includes('earnings_call')) return 1\n    if (/financials\\/(income|cash_flow|balance)/.test(f) && !/bam_|bx_|kkr_|ares_|cg_|owl_|fsk_|cat_|de_|tdg_|car_/.test(f)) return 2\n    if (f.includes('sec_filings')) return 3\n    if (f.includes('investor_pres')) return 4\n    if (f.includes('press_release')) return 5\n    return 6\n  }\n  const sorted = [...files].sort((a, b) => priority(a) - priority(b) || a.localeCompare(b))\n  const blocks: string[] = []\n  let total = 0\n  for (const f of sorted) {\n    const rel = f.slice(dataDir.length + 1)\n    const content = readText(f)\n    const chunk = `\\n=== FILE: data/${rel} ===\\n${content}`\n    if (total + chunk.length > MAX_DATA_CHARS) {\n      blocks.push(`\\n[… ${sorted.length - blocks.length} more files omitted to stay within context limit …]`)\n      break\n    }\n    blocks.push(chunk)\n    total += chunk.length\n  }\n  return blocks.join('')\n}\n\nfunction buildCitedDataBlock(answerText: string, dataDir: string): string {\n  const files = listFilesRec(dataDir)\n  const cited: string[] = []\n  for (const f of files) {\n    const rel = f.slice(dataDir.length + 1)\n    const name = rel.split('/').pop() ?? ''\n    if (answerText.includes(rel) || answerText.includes(name)) cited.push(f)\n  }\n  if (cited.length === 0) {\n    return '(The agent did not cite any data files, or no cited file could be located.)'\n  }\n  const blocks: string[] = []\n  for (const f of cited) {\n    const rel = f.slice(dataDir.length + 1)\n    blocks.push(`\\n=== FILE: data/${rel} ===\\n${readText(f)}`)\n  }\n  return blocks.join('')\n}\n\n// ── ground-truth parser ───────────────────────────────────────────────────────\n\nfunction parseThemeMoveCounts(groundTruth: string): Array<{ label: string; nMoves: number }> {\n  const lines = groundTruth.split('\\n')\n  const themeStarts = lines.reduce<number[]>((acc, l, i) => {\n    if (/^\\d+\\./.test(l.trim())) acc.push(i)\n    return acc\n  }, [])\n  return themeStarts.map((start, k) => {\n    const end = k + 1 < themeStarts.length ? themeStarts[k + 1] : lines.length\n    const block = lines.slice(start, end).join('\\n')\n    const label = lines[start].replace(/^\\d+\\.\\s*/, '').trim()\n    const nMoves = (block.match(/^\\s*\\[[a-z]\\]\\s+/gm) ?? []).length\n    return { label, nMoves }\n  })\n}\n\nfunction coverageThreshold(nMoves: number): number {\n  return Math.max(1, Math.min(nMoves - 1, 3))\n}\n\n// ── router judge calls ────────────────────────────────────────────────────────\n\ninterface JudgeRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction judgeRouter(): JudgeRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the Trata hedge-bench judge')\n  const model = process.env.JUDGE_MODEL ?? JUDGE_MODEL\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\nfunction parseJsonFallback(raw: string): unknown {\n  let s = raw.trim()\n  if (s.startsWith('```')) s = s.split('\\n').slice(1).join('\\n')\n  if (s.endsWith('```')) s = s.slice(0, s.lastIndexOf('```'))\n  s = s.replace(/,(\\s*[}\\]])/g, '$1').trim()\n  try {\n    return JSON.parse(s)\n  } catch {\n    try {\n      const m = s.match(/\\{[\\s\\S]*\\}/)\n      if (m) return JSON.parse(m[0])\n    } catch {\n      // fallthrough\n    }\n  }\n  return null\n}\n\nasync function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise<unknown> {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: router.baseUrl,\n          routerKey: router.key,\n          profile: {\n            name: 'trata-hedge-judge',\n            harness: 'cli-base',\n            model: {\n              provider: 'tangle-router',\n              default: router.model,\n              metadata: {\n                temperature: 0,\n                maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 16384),\n              },\n            },\n          },\n        },\n        prompt,\n      )\n      const content = turn.finalText\n      const parsed = parseJsonFallback(content)\n      if (parsed !== null) return parsed\n    } catch (error) {\n      if (i + 1 === maxAttempts) throw error\n    }\n  }\n  return null\n}\n\ninterface Task1Result {\n  hallucinations_detected?: boolean | string\n  unverifiable_claims?: Array<{ claim: string; cited_file?: string; issue?: string; evidence_check?: string }>\n}\n\ninterface Task2ThemeResult {\n  label?: string\n  moves_hit?: string[]\n  moves_missed?: string[]\n  moves_tainted?: string[]\n  move_reasoning?: Record<string, string>\n}\n\ninterface Task2Result {\n  themes?: Task2ThemeResult[]\n}\n\ninterface Task3Result {\n  synthesis_found?: string | null\n}\n\nasync function task1HallucinationCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task1Result> {\n  const citedBlock = buildCitedDataBlock(answer, meta.dataDir)\n  const dataContext =\n    'The block below contains the contents of the data files the agent cited in their answer. ' +\n    'Treat this as the ground truth for factual verification.\\n\\n' +\n    `<cited_data>\\n${citedBlock}\\n</cited_data>`\n  const prompt = meta.gradingTask1\n    .replace('{data_context}', dataContext)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task1Result) ?? { hallucinations_detected: false, unverifiable_claims: [] }\n}\n\nasync function task2PerMoveCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  flaggedClaims: Task1Result['unverifiable_claims'],\n  router: JudgeRouter,\n): Promise<Task2Result> {\n  const claimsBlock =\n    flaggedClaims && flaggedClaims.length > 0\n      ? JSON.stringify(flaggedClaims, null, 2)\n      : '(No claims were flagged as hallucinations.)'\n  const prompt = meta.gradingTask2\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n    .replace('{flagged_claims_block}', claimsBlock)\n    .replace('{num_themes}', String(meta.themeMoveCounts.length))\n  const result = await callJudge(router, prompt)\n  return (result as Task2Result) ?? { themes: [] }\n}\n\nasync function task3SynthesisCheck(\n  meta: TrataTaskMeta,\n  answer: string,\n  router: JudgeRouter,\n): Promise<Task3Result> {\n  const prompt = meta.gradingTask3\n    .replace('{ground_truth}', meta.groundTruth)\n    .replace('{agent_answer}', answer)\n  const result = await callJudge(router, prompt)\n  return (result as Task3Result) ?? { synthesis_found: null }\n}\n\nasync function runTriataJudge(meta: TrataTaskMeta, answer: string, router: JudgeRouter): Promise<BenchScore> {\n  if (!answer.trim()) {\n    return { resolved: false, score: 0, detail: JSON.stringify({ error: 'empty answer' }) }\n  }\n\n  // Stage 1: hallucination check (serial — feeds into stage 2).\n  const t1 = await task1HallucinationCheck(meta, answer, router)\n  const flaggedClaims = t1.unverifiable_claims ?? []\n\n  // Stages 2 + 3 in parallel.\n  const [t2, t3] = await Promise.all([\n    task2PerMoveCheck(meta, answer, flaggedClaims, router),\n    task3SynthesisCheck(meta, answer, router),\n  ])\n\n  const judgeThemes = t2.themes ?? []\n  const themesHit: string[] = []\n  const themesMissed: string[] = []\n\n  for (let i = 0; i < meta.themeMoveCounts.length; i++) {\n    const { label, nMoves } = meta.themeMoveCounts[i]\n    const judge = judgeThemes[i] ?? {}\n    const movesHit = (judge.moves_hit ?? []).map((m: string) => m.replace(/[\\[\\]]/g, ''))\n    const tainted = new Set((judge.moves_tainted ?? []).map((m: string) => m.replace(/[\\[\\]]/g, '')))\n    const validHit = movesHit.filter((m: string) => !tainted.has(m))\n    const threshold = coverageThreshold(nMoves)\n    if (validHit.length >= threshold) themesHit.push(label)\n    else themesMissed.push(label)\n  }\n\n  const numThemes = meta.themeMoveCounts.length\n  const synth = t3.synthesis_found\n  const hasSynthesis = typeof synth === 'string' && synth.length > 0 && synth !== 'null'\n\n  let rawScore: number\n  if (numThemes === 0) rawScore = 0\n  else if (themesHit.length === numThemes && hasSynthesis) rawScore = 4\n  else if (themesHit.length === numThemes) rawScore = 3\n  else if (themesHit.length >= 2) rawScore = 2\n  else if (themesHit.length >= 1) rawScore = 1\n  else rawScore = 0\n\n  const hallRaw = t1.hallucinations_detected\n  const hallucinations = hallRaw === true || hallRaw === 'true'\n\n  return {\n    resolved: rawScore === 4,\n    score: numThemes > 0 ? rawScore / 4 : 0,\n    detail: JSON.stringify({\n      rawScore,\n      themesHit,\n      themesMissed,\n      hallucinations,\n      unverifiableClaims: flaggedClaims.length,\n      synthesis: synth ?? null,\n      judgeModel: router.model,\n    }),\n  }\n}\n\n// ── task loading ──────────────────────────────────────────────────────────────\n\nfunction benchRoot(): string {\n  return process.env.TRATA_BENCH_ROOT ?? DEFAULT_BENCH_ROOT\n}\n\nfunction loadTask(taskId: string): { task: BenchTask; meta: TrataTaskMeta } {\n  const taskDir = join(benchRoot(), 'environments', taskId)\n  const dataDir = join(taskDir, 'environment', 'data')\n  const testsDir = join(taskDir, 'tests')\n\n  const instruction = readText(join(taskDir, 'instruction.md'), Infinity)\n  const dataBlock = buildDataBlock(dataDir)\n  const groundTruth = readText(join(testsDir, 'ground_truth.txt'), Infinity)\n  const gradingTask1 = readText(join(testsDir, 'grading_prompt_task1.md'), Infinity)\n  const gradingTask2 = readText(join(testsDir, 'grading_prompt_task2.md'), Infinity)\n  const gradingTask3 = readText(join(testsDir, 'grading_prompt_task3.md'), Infinity)\n\n  const themeMoveCounts = parseThemeMoveCounts(groundTruth)\n\n  // Rewrite the instruction so the agent knows data is inline (no /app/data/ FS).\n  const prompt = [\n    instruction\n      .replace(\n        \"You are a financial analyst with access to the data in `/app/data/`.\",\n        'You are a financial analyst. The relevant financial data files are provided inline below.',\n      )\n      .replace(/Write your full analysis to `\\/app\\/answer\\.txt`\\.?/, 'Write your full analysis below.')\n      .replace(/`\\/app\\/data\\/`/g, 'the data files below'),\n    '',\n    '## Data files (inline)',\n    dataBlock,\n    WORKER_SUFFIX,\n  ].join('\\n')\n\n  const meta: TrataTaskMeta = {\n    taskDir,\n    taskId,\n    dataDir,\n    groundTruth,\n    gradingTask1,\n    gradingTask2,\n    gradingTask3,\n    themeMoveCounts,\n  }\n\n  // Store meta in the BenchTask so the judge can access it without re-reading.\n  const task: BenchTask = {\n    id: taskId,\n    prompt,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n\n  return { task, meta }\n}\n\nfunction readMeta(task: BenchTask): TrataTaskMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskDir !== 'string' || typeof md.groundTruth !== 'string') {\n    throw new Error(`Trata task ${task.id} missing judge metadata — was it loaded by this adapter?`)\n  }\n  return md as unknown as TrataTaskMeta\n}\n\n/** List all task ids from the environments directory. */\nfunction listTaskIds(root: string): string[] {\n  try {\n    return readdirSync(join(root, 'environments'))\n      .filter((name) => {\n        try {\n          return statSync(join(root, 'environments', name)).isDirectory()\n        } catch {\n          return false\n        }\n      })\n      .sort()\n  } catch (err) {\n    throw new Error(\n      `Trata benchmark environments not found at ${join(root, 'environments')}: ` +\n        `${err instanceof Error ? err.message : err}.\\n` +\n        `Clone the repo: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n    )\n  }\n}\n\n// ── extract agent analysis from artifact ─────────────────────────────────────\n\nfunction extractAnalysis(artifact: string): string {\n  // Model may write \"ANALYSIS:\" as its very first token (no leading newline) or\n  // after some preamble. Both are valid; fail-closed on absence.\n  if (artifact.startsWith('ANALYSIS:')) return artifact.slice('ANALYSIS:'.length).trim()\n  const marker = artifact.indexOf('\\nANALYSIS:')\n  if (marker !== -1) return artifact.slice(marker + '\\nANALYSIS:'.length).trim()\n  return ''\n}\n\n// ── adapter factory ───────────────────────────────────────────────────────────\n\nexport function createTrataHedgeAdapter(): BenchmarkAdapter {\n  return {\n    name: 'trata-hedge',\n\n    async preflight() {\n      const root = benchRoot()\n      const envDir = join(root, 'environments')\n      try {\n        const stat = statSync(envDir)\n        if (!stat.isDirectory()) throw new Error('not a directory')\n      } catch {\n        throw new Error(\n          `Trata hedge-bench not found at ${envDir}.\\n` +\n            `Clone: git clone --depth 1 https://github.com/Trata-Inc/trata-hedge-bench ${root}`,\n        )\n      }\n      // Validate judge router connectivity.\n      judgeRouter()\n      const ids = listTaskIds(root)\n      if (ids.length === 0) throw new Error(`No task directories found under ${envDir}`)\n      console.warn(\n        `[trata-hedge] ${ids.length} tasks loaded from ${root}. ` +\n          `Worker model reads embedded data inline (Harbor/Docker not required). ` +\n          `Judge uses ${process.env.JUDGE_MODEL ?? JUDGE_MODEL} via Tangle router.`,\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const root = benchRoot()\n      let ids = listTaskIds(root)\n      if (opts.ids) {\n        const want = new Set(opts.ids)\n        ids = ids.filter((id) => want.has(id))\n      } else if (opts.limit !== undefined) {\n        ids = ids.slice(0, opts.limit)\n      }\n      return ids.map((id) => loadTask(id).task)\n    },\n\n    async goldArtifact(_task: BenchTask): Promise<string | undefined> {\n      // No single gold answer: the score comes from rubric theme coverage, not\n      // string-match against a reference response. Return undefined — the bench\n      // harness will skip judge self-verification for this adapter.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const answer = extractAnalysis(artifact)\n      return runTriataJudge(meta, answer, judgeRouter())\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,MAAM,qBAAqB;AAC3B,MAAM,iBAAiB;AAEvB,MAAM,iBAAiB;AACvB,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAGX,MAAM,cAAc;AAgBpB,SAAS,SAAS,GAAW,WAAW,gBAAwB;CAC9D,IAAI;EACF,MAAM,MAAM,aAAa,CAAC;EAC1B,IAAI,IAAI,UAAU,UAAU,OAAO,IAAI,SAAS,MAAM;EACtD,OAAO,IAAI,SAAS,GAAG,QAAQ,CAAC,CAAC,SAAS,MAAM,IAAI,qBAAqB,SAAS;CACpF,QAAQ;EACN,OAAO,gBAAgB,EAAE;CAC3B;AACF;AAEA,SAAS,aAAa,KAAuB;CAC3C,MAAM,MAAgB,CAAC;CACvB,IAAI;EACF,KAAK,MAAM,QAAQ,YAAY,GAAG,GAAG;GACnC,MAAM,OAAO,KAAK,KAAK,IAAI;GAC3B,IAAI;IACF,IAAI,SAAS,IAAI,CAAC,CAAC,YAAY,GAAG,IAAI,KAAK,GAAG,aAAa,IAAI,CAAC;SAC3D,IAAI,KAAK,IAAI;GACpB,QAAQ,CAER;EACF;CACF,QAAQ,CAER;CACA,OAAO,IAAI,KAAK;AAClB;AAEA,SAAS,eAAe,SAAyB;CAC/C,MAAM,QAAQ,aAAa,OAAO;CAClC,IAAI,MAAM,WAAW,GAAG,OAAO;CAG/B,MAAM,YAAY,MAAc;EAC9B,IAAI,EAAE,SAAS,kBAAkB,GAAG,OAAO;EAC3C,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,yCAAyC,KAAK,CAAC,KAAK,CAAC,uDAAuD,KAAK,CAAC,GAAG,OAAO;EAChI,IAAI,EAAE,SAAS,aAAa,GAAG,OAAO;EACtC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,IAAI,EAAE,SAAS,eAAe,GAAG,OAAO;EACxC,OAAO;CACT;CACA,MAAM,SAAS,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,SAAS,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;CACxF,MAAM,SAAmB,CAAC;CAC1B,IAAI,QAAQ;CACZ,KAAK,MAAM,KAAK,QAAQ;EAGtB,MAAM,QAAQ,oBAFF,EAAE,MAAM,QAAQ,SAAS,CAED,EAAE,QADtB,SAAS,CAC2B;EACpD,IAAI,QAAQ,MAAM,SAAS,gBAAgB;GACzC,OAAO,KAAK,QAAQ,OAAO,SAAS,OAAO,OAAO,oDAAoD;GACtG;EACF;EACA,OAAO,KAAK,KAAK;EACjB,SAAS,MAAM;CACjB;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAEA,SAAS,oBAAoB,YAAoB,SAAyB;CACxE,MAAM,QAAQ,aAAa,OAAO;CAClC,MAAM,QAAkB,CAAC;CACzB,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;EACrC,IAAI,WAAW,SAAS,GAAG,KAAK,WAAW,SAAS,IAAI,GAAG,MAAM,KAAK,CAAC;CACzE;CACA,IAAI,MAAM,WAAW,GACnB,OAAO;CAET,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,MAAM,EAAE,MAAM,QAAQ,SAAS,CAAC;EACtC,OAAO,KAAK,oBAAoB,IAAI,QAAQ,SAAS,CAAC,GAAG;CAC3D;CACA,OAAO,OAAO,KAAK,EAAE;AACvB;AAIA,SAAS,qBAAqB,aAA+D;CAC3F,MAAM,QAAQ,YAAY,MAAM,IAAI;CACpC,MAAM,cAAc,MAAM,QAAkB,KAAK,GAAG,MAAM;EACxD,IAAI,SAAS,KAAK,EAAE,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC;EACvC,OAAO;CACT,GAAG,CAAC,CAAC;CACL,OAAO,YAAY,KAAK,OAAO,MAAM;EACnC,MAAM,MAAM,IAAI,IAAI,YAAY,SAAS,YAAY,IAAI,KAAK,MAAM;EACpE,MAAM,QAAQ,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI;EAG/C,OAAO;GAAE,OAFK,MAAM,MAAM,CAAC,QAAQ,aAAa,EAAE,CAAC,CAAC,KAEvC;GAAG,SADA,MAAM,MAAM,oBAAoB,KAAK,CAAC,EAAA,CAAG;EAClC;CACzB,CAAC;AACH;AAEA,SAAS,kBAAkB,QAAwB;CACjD,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,SAAS,GAAG,CAAC,CAAC;AAC5C;AAUA,SAAS,cAA2B;CAClC,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,4DAA4D;CACtF,MAAM,QAAQ,QAAQ,IAAI,eAAe;CAEzC,OAAO;EAAE,SADO,QAAQ,IAAI,eAAe;EACzB;EAAK;CAAM;AAC/B;AAEA,SAAS,kBAAkB,KAAsB;CAC/C,IAAI,IAAI,IAAI,KAAK;CACjB,IAAI,EAAE,WAAW,KAAK,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI;CAC7D,IAAI,EAAE,SAAS,KAAK,GAAG,IAAI,EAAE,MAAM,GAAG,EAAE,YAAY,KAAK,CAAC;CAC1D,IAAI,EAAE,QAAQ,gBAAgB,IAAI,CAAC,CAAC,KAAK;CACzC,IAAI;EACF,OAAO,KAAK,MAAM,CAAC;CACrB,QAAQ;EACN,IAAI;GACF,MAAM,IAAI,EAAE,MAAM,aAAa;GAC/B,IAAI,GAAG,OAAO,KAAK,MAAM,EAAE,EAAE;EAC/B,QAAQ,CAER;CACF;CACA,OAAO;AACT;AAEA,eAAe,UAAU,QAAqB,QAAgB,cAAc,GAAqB;CAC/F,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EAoBF,MAAM,WAAU,MAnBG,mBACjB;GACE,eAAe,OAAO;GACtB,WAAW,OAAO;GAClB,SAAS;IACP,MAAM;IACN,SAAS;IACT,OAAO;KACL,UAAU;KACV,SAAS,OAAO;KAChB,UAAU;MACR,aAAa;MACb,WAAW,OAAO,QAAQ,IAAI,oBAAoB,KAAK;KACzD;IACF;GACF;EACF,GACA,MACF,EAAA,CACqB;EACrB,MAAM,SAAS,kBAAkB,OAAO;EACxC,IAAI,WAAW,MAAM,OAAO;CAC9B,SAAS,OAAO;EACd,IAAI,IAAI,MAAM,aAAa,MAAM;CACnC;CAEF,OAAO;AACT;AAuBA,eAAe,wBACb,MACA,QACA,QACsB;CAEtB,MAAM,cACJ;;gBAFiB,oBAAoB,QAAQ,KAAK,OAIxB,EAAE;CAK9B,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,WAAW,CAAC,CACtC,QAAQ,kBAAkB,MACe,CAAC,KACX;EAAE,yBAAyB;EAAO,qBAAqB,CAAC;CAAE;AAC9F;AAEA,eAAe,kBACb,MACA,QACA,eACA,QACsB;CACtB,MAAM,cACJ,iBAAiB,cAAc,SAAS,IACpC,KAAK,UAAU,eAAe,MAAM,CAAC,IACrC;CAON,OAAQ,MADa,UAAU,QALhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MAAM,CAAC,CACjC,QAAQ,0BAA0B,WAAW,CAAC,CAC9C,QAAQ,gBAAgB,OAAO,KAAK,gBAAgB,MAAM,CACjB,CAAC,KACX,EAAE,QAAQ,CAAC,EAAE;AACjD;AAEA,eAAe,oBACb,MACA,QACA,QACsB;CAKtB,OAAQ,MADa,UAAU,QAHhB,KAAK,aACjB,QAAQ,kBAAkB,KAAK,WAAW,CAAC,CAC3C,QAAQ,kBAAkB,MACe,CAAC,KACX,EAAE,iBAAiB,KAAK;AAC5D;AAEA,eAAe,eAAe,MAAqB,QAAgB,QAA0C;CAC3G,IAAI,CAAC,OAAO,KAAK,GACf,OAAO;EAAE,UAAU;EAAO,OAAO;EAAG,QAAQ,KAAK,UAAU,EAAE,OAAO,eAAe,CAAC;CAAE;CAIxF,MAAM,KAAK,MAAM,wBAAwB,MAAM,QAAQ,MAAM;CAC7D,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;CAGjD,MAAM,CAAC,IAAI,MAAM,MAAM,QAAQ,IAAI,CACjC,kBAAkB,MAAM,QAAQ,eAAe,MAAM,GACrD,oBAAoB,MAAM,QAAQ,MAAM,CAC1C,CAAC;CAED,MAAM,cAAc,GAAG,UAAU,CAAC;CAClC,MAAM,YAAsB,CAAC;CAC7B,MAAM,eAAyB,CAAC;CAEhC,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,gBAAgB,QAAQ,KAAK;EACpD,MAAM,EAAE,OAAO,WAAW,KAAK,gBAAgB;EAC/C,MAAM,QAAQ,YAAY,MAAM,CAAC;EACjC,MAAM,YAAY,MAAM,aAAa,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC;EACpF,MAAM,UAAU,IAAI,KAAK,MAAM,iBAAiB,CAAC,EAAA,CAAG,KAAK,MAAc,EAAE,QAAQ,WAAW,EAAE,CAAC,CAAC;EAChG,MAAM,WAAW,SAAS,QAAQ,MAAc,CAAC,QAAQ,IAAI,CAAC,CAAC;EAC/D,MAAM,YAAY,kBAAkB,MAAM;EAC1C,IAAI,SAAS,UAAU,WAAW,UAAU,KAAK,KAAK;OACjD,aAAa,KAAK,KAAK;CAC9B;CAEA,MAAM,YAAY,KAAK,gBAAgB;CACvC,MAAM,QAAQ,GAAG;CACjB,MAAM,eAAe,OAAO,UAAU,YAAY,MAAM,SAAS,KAAK,UAAU;CAEhF,IAAI;CACJ,IAAI,cAAc,GAAG,WAAW;MAC3B,IAAI,UAAU,WAAW,aAAa,cAAc,WAAW;MAC/D,IAAI,UAAU,WAAW,WAAW,WAAW;MAC/C,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,IAAI,UAAU,UAAU,GAAG,WAAW;MACtC,WAAW;CAEhB,MAAM,UAAU,GAAG;CACnB,MAAM,iBAAiB,YAAY,QAAQ,YAAY;CAEvD,OAAO;EACL,UAAU,aAAa;EACvB,OAAO,YAAY,IAAI,WAAW,IAAI;EACtC,QAAQ,KAAK,UAAU;GACrB;GACA;GACA;GACA;GACA,oBAAoB,cAAc;GAClC,WAAW,SAAS;GACpB,YAAY,OAAO;EACrB,CAAC;CACH;AACF;AAIA,SAAS,YAAoB;CAC3B,OAAO,QAAQ,IAAI,oBAAoB;AACzC;AAEA,SAAS,SAAS,QAA0D;CAC1E,MAAM,UAAU,KAAK,UAAU,GAAG,gBAAgB,MAAM;CACxD,MAAM,UAAU,KAAK,SAAS,eAAe,MAAM;CACnD,MAAM,WAAW,KAAK,SAAS,OAAO;CAEtC,MAAM,cAAc,SAAS,KAAK,SAAS,gBAAgB,GAAG,QAAQ;CACtE,MAAM,YAAY,eAAe,OAAO;CACxC,MAAM,cAAc,SAAS,KAAK,UAAU,kBAAkB,GAAG,QAAQ;CACzE,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CACjF,MAAM,eAAe,SAAS,KAAK,UAAU,yBAAyB,GAAG,QAAQ;CAEjF,MAAM,kBAAkB,qBAAqB,WAAW;CAGxD,MAAM,SAAS;EACb,YACG,QACC,wEACA,2FACF,CAAC,CACA,QAAQ,uDAAuD,iCAAiC,CAAC,CACjG,QAAQ,oBAAoB,sBAAsB;EACrD;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CAEX,MAAM,OAAsB;EAC1B;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF;CASA,OAAO;EAAE,MAAA;GALP,IAAI;GACJ;GACA,UAAU;EAGA;EAAG;CAAK;AACtB;AAEA,SAAS,SAAS,MAAgC;CAChD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,YAAY,YAAY,OAAO,GAAG,gBAAgB,UACrE,MAAM,IAAI,MAAM,cAAc,KAAK,GAAG,yDAAyD;CAEjG,OAAO;AACT;;AAGA,SAAS,YAAY,MAAwB;CAC3C,IAAI;EACF,OAAO,YAAY,KAAK,MAAM,cAAc,CAAC,CAAC,CAC3C,QAAQ,SAAS;GAChB,IAAI;IACF,OAAO,SAAS,KAAK,MAAM,gBAAgB,IAAI,CAAC,CAAC,CAAC,YAAY;GAChE,QAAQ;IACN,OAAO;GACT;EACF,CAAC,CAAC,CACD,KAAK;CACV,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,6CAA6C,KAAK,MAAM,cAAc,EAAE,IACnE,eAAe,QAAQ,IAAI,UAAU,IAAI,wFAC0C,MAC1F;CACF;AACF;AAIA,SAAS,gBAAgB,UAA0B;CAGjD,IAAI,SAAS,WAAW,WAAW,GAAG,OAAO,SAAS,MAAM,CAAkB,CAAC,CAAC,KAAK;CACrF,MAAM,SAAS,SAAS,QAAQ,aAAa;CAC7C,IAAI,WAAW,IAAI,OAAO,SAAS,MAAM,SAAS,EAAoB,CAAC,CAAC,KAAK;CAC7E,OAAO;AACT;AAIA,SAAgB,0BAA4C;CAC1D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,OAAO,UAAU;GACvB,MAAM,SAAS,KAAK,MAAM,cAAc;GACxC,IAAI;IAEF,IAAI,CADS,SAAS,MACd,CAAC,CAAC,YAAY,GAAG,MAAM,IAAI,MAAM,iBAAiB;GAC5D,QAAQ;IACN,MAAM,IAAI,MACR,kCAAkC,OAAO,+EACsC,MACjF;GACF;GAEA,YAAY;GACZ,MAAM,MAAM,YAAY,IAAI;GAC5B,IAAI,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,mCAAmC,QAAQ;GACjF,QAAQ,KACN,iBAAiB,IAAI,OAAO,qBAAqB,KAAK,qFAEtC,QAAQ,IAAI,eAAe,YAAY,oBACzD;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,IAAI,MAAM,YADG,UACY,CAAC;GAC1B,IAAI,KAAK,KAAK;IACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;IAC7B,MAAM,IAAI,QAAQ,OAAO,KAAK,IAAI,EAAE,CAAC;GACvC,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,MAAM,IAAI,MAAM,GAAG,KAAK,KAAK;GAE/B,OAAO,IAAI,KAAK,OAAO,SAAS,EAAE,CAAC,CAAC,IAAI;EAC1C;EAEA,MAAM,aAAa,OAA+C,CAKlE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAGlE,OAAO,eAFM,SAAS,IAEG,GADV,gBAAgB,QACE,GAAG,YAAY,CAAC;EACnD;CACF;AACF"}

package/dist/{cadbench-BrpwOU6A.js → cadbench-BLSyxR1N.js}

@@ -1,9 +1,9 @@
+import { t as runBenchRouterTurn } from "./router-turn-C2wMiDoo.js";
 import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
 import { tmpdir } from "node:os";
 import { promisify } from "node:util";
-import "@tangle-network/agent-runtime/kernel";
 //#region src/worker-blender.ts
 /**
 * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender
@@ -162,24 +162,25 @@ async function judgeCriteria(instruction, script, criteria, renders) {
 		type: "image_url",
 		image_url: { url }
 	});
-	const res = await fetch(`${base}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${key}`
-		},
-		body: JSON.stringify({
-			model,
-			max_tokens: 1500,
-			temperature: 0,
-			messages: [{
-				role: "user",
-				content
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`);
-	const raw = (await res.json()).choices?.[0]?.message?.content ?? "";
+	const raw = (await runBenchRouterTurn({
+		routerBaseUrl: base,
+		routerKey: key,
+		profile: {
+			name: "cadbench-vision-judge",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: model,
+				metadata: {
+					temperature: 0,
+					maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500)
+				}
+			}
+		}
+	}, { messages: [{
+		role: "user",
+		content
+	}] })).finalText;
 	const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw);
 	if (!m) return {
 		passed: criteria.map(() => false),
@@ -282,4 +283,4 @@ function createCadBenchAdapter() {
 //#endregion
 export { createCadBenchAdapter as t };
 
-//# sourceMappingURL=cadbench-BrpwOU6A.js.map
+//# sourceMappingURL=cadbench-BLSyxR1N.js.map

package/dist/cadbench-BLSyxR1N.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cadbench-BLSyxR1N.js","names":[],"sources":["../src/worker-blender.ts","../src/benchmarks/cadbench.ts"],"sourcesContent":["/**\n * BlenderLLM / CADBench worker. The deliverable for a CADBench task is a Blender\n * `bpy` Python script that builds the described 3D model. We author it via the\n * router, execute it headless in Blender (Cycles CPU, no GPU), auto-frame the\n * produced geometry, and render N standardized views — the images the CADBench\n * criteria judge scores. The authoring directive is the GEPA-optimizable surface.\n *\n * Requires `blender` + `xvfb-run` on PATH (apt blender 4.x). No GPU: Cycles CPU\n * with denoising off (the apt build ships without OpenImageDenoise).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BLENDER_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 180_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\n\n/** Strip markdown fences so we keep just the Python. */\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\n/**\n * The standardized Blender runner (written to a temp file per run). It clears the\n * scene, executes the agent's bpy script, auto-frames the produced meshes, sets\n * up neutral lighting, and renders N azimuth views with Cycles CPU.\n */\nconst RUNNER_PY = `\nimport bpy, sys, math, mathutils, traceback, os\nagent_script, outdir, nviews = sys.argv[-3], sys.argv[-2], int(sys.argv[-1])\nbpy.ops.object.select_all(action='SELECT'); bpy.ops.object.delete()\nok=True\ntry:\n    g={'bpy':bpy,'math':math,'mathutils':mathutils,'__name__':'__main__'}\n    exec(compile(open(agent_script).read(), agent_script, 'exec'), g)\nexcept Exception as e:\n    traceback.print_exc(); print('AGENT_SCRIPT_ERROR:'+repr(e)); ok=False\nmeshes=[o for o in bpy.context.scene.objects if o.type=='MESH']\nif not meshes:\n    print('NO_MESH'); sys.exit(0 if ok else 3)\nmn=[1e18]*3; mx=[-1e18]*3\nfor o in meshes:\n    for c in o.bound_box:\n        w=o.matrix_world @ mathutils.Vector(c)\n        for i in range(3): mn[i]=min(mn[i],w[i]); mx[i]=max(mx[i],w[i])\ncenter=mathutils.Vector(((mn[0]+mx[0])/2,(mn[1]+mx[1])/2,(mn[2]+mx[2])/2))\nsize=max(mx[i]-mn[i] for i in range(3)) or 1.0\n# standardize: drop any agent-added cameras/lights\nfor o in list(bpy.context.scene.objects):\n    if o.type in ('CAMERA','LIGHT'): bpy.data.objects.remove(o, do_unlink=True)\nw=bpy.context.scene.world or bpy.data.worlds.new('W'); bpy.context.scene.world=w\nw.use_nodes=True\ntry: w.node_tree.nodes['Background'].inputs[1].default_value=0.6\nexcept Exception: pass\nbpy.ops.object.light_add(type='SUN'); sun=bpy.context.object; sun.data.energy=4.0; sun.rotation_euler=mathutils.Euler((0.6,0.2,0.4))\nbpy.ops.object.camera_add(); cam=bpy.context.object; bpy.context.scene.camera=cam\nsc=bpy.context.scene\nsc.render.engine='CYCLES'; sc.cycles.samples=20; sc.cycles.device='CPU'; sc.cycles.use_denoising=False\nsc.render.resolution_x=640; sc.render.resolution_y=640; sc.render.film_transparent=False\ndist=size*2.4\nel=math.radians(58)\nfor v in range(nviews):\n    az=math.radians(40 + v*360.0/nviews)\n    cam.location=center+mathutils.Vector((math.cos(az)*math.sin(el), math.sin(az)*math.sin(el), math.cos(el)))*dist\n    d=(center-cam.location); cam.rotation_euler=d.to_track_quat('-Z','Y').to_euler()\n    sc.render.filepath=os.path.join(outdir, 'view_%d.png'%v); bpy.ops.render.render(write_still=True)\nprint('RENDER_DONE')\n`.trim()\n\n/** Execute a bpy script headless + render N standardized views — no authoring.\n *  Used by the CADBench judge to render an artifact before vision-scoring it. */\nexport async function renderBpy(script: string, opts: { views?: number } = {}): Promise<{ built: boolean; renders: string[]; error?: string }> {\n  const views = Math.max(1, opts.views ?? 4)\n  const dir = await mkdtemp(join(tmpdir(), 'blender-judge-'))\n  const runnerPath = join(dir, 'runner.py')\n  const scriptPath = join(dir, 'model.py')\n  try {\n    await writeFile(runnerPath, RUNNER_PY)\n    await writeFile(scriptPath, script)\n    const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n    const out = `${run.stdout}\\n${run.stderr}`\n    const built = /RENDER_DONE/.test(out)\n    if (!built) return { built: false, renders: [], error: (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-400)) }\n    const renders: string[] = []\n    for (let v = 0; v < views; v++) {\n      const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n      if (buf) renders.push(`data:image/png;base64,${buf.toString('base64')}`)\n    }\n    return { built: renders.length > 0, renders }\n  } finally {\n    await rm(dir, { recursive: true, force: true }).catch(() => {})\n  }\n}\n\nexport interface BlenderLocalConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** N standardized views to render (CADBench uses 4). Default 4. */\n  views?: number\n  /** The bpy authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface BlenderShot {\n  /** The bpy script the agent wrote — the artifact. */\n  artifact: string\n  /** Rendered view PNGs as data URIs (the images the criteria judge scores). */\n  renders: string[]\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  /** True if the script executed and produced at least one mesh. */\n  built: boolean\n  detail?: string\n}\n\n/**\n * Author a bpy script for the task via the router, execute + render it headless\n * in Blender, refine on execution errors across rounds. Returns the script, the\n * rendered views, a screenshot-rich trace, and real token usage.\n */\nexport async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig): Promise<BlenderShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const views = Math.max(1, cfg.views ?? 4)\n  const directive = cfg.directive ?? DEFAULT_BLENDER_DIRECTIVE\n  const trace: Span[] = []\n  const runId = `cadbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the bpy script; the\n  // render PNGs + built flag + lastErr persist outside the loop). usage is REAL.\n  let renders: string[] = []\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp dir (with the runner.py written once\n  // in setup) is the Ctx; built (RENDER_DONE + ≥1 collected view) is the early-stop,\n  // modeled as a judge so default-decide stops the loop. The round-2+ steer carries\n  // lastErr + the prior script verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: async () => {\n      const dir = await mkdtemp(join(tmpdir(), 'blender-'))\n      await writeFile(join(dir, 'runner.py'), RUNNER_PY)\n      return dir\n    },\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous bpy script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs under \\`blender --background --python\\` and builds the object as mesh(es). Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const runnerPath = join(dir, 'runner.py')\n      const scriptPath = join(dir, 'model.py')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'blender-worker',\n            harness: 'cli-base',\n            model: {\n              provider: 'tangle-router',\n              default: cfg.model,\n              metadata: { temperature: 0.3 },\n            },\n            prompt: { systemPrompt: directive },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const script = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.py', content: script }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, script)\n      const run = await runLocal('xvfb-run', ['-a', 'blender', '--background', '--python', runnerPath, '--', scriptPath, dir, String(views)], dir)\n      const out = `${run.stdout}\\n${run.stderr}`\n      built = /RENDER_DONE/.test(out)\n      lastErr = built ? '' : (/(AGENT_SCRIPT_ERROR:.*|NO_MESH)/.exec(out)?.[0] ?? out.trim().slice(-800))\n      trace.push({ spanId: `s-blender-${round}`, runId, kind: 'tool', name: `blender r${round}`, toolName: 'shell.exec', args: 'blender --background --python runner.py model.py', result: (built ? 'RENDER_DONE' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n\n      if (built) {\n        const collected: string[] = []\n        for (let v = 0; v < views; v++) {\n          const buf = await readFile(join(dir, `view_${v}.png`)).catch(() => undefined)\n          if (buf) collected.push(`data:image/png;base64,${buf.toString('base64')}`)\n        }\n        renders = collected\n        // first view carries the screen span (run-capsule reveal)\n        trace.push({ spanId: `s-render-${round}`, runId, kind: 'tool', name: 'render', toolName: 'render.screenshot', args: { action: 'rendered model', url: 'view_0.png' }, attributes: collected[0] ? { screenshot: collected[0] } : {}, startedAt: tick(), endedAt: tick(), status: collected.length ? 'ok' : 'error', error: collected.length ? undefined : 'render produced no image' } as Span)\n        built = collected.length > 0\n      }\n      return { artifact: script }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  const script = res.final.artifact\n  return {\n    artifact: script,\n    renders,\n    trace,\n    usage,\n    ok: script.trim().length > 0,\n    built,\n    detail: built ? `built + rendered ${renders.length} views` : `did not build in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).\n * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria\n * eval: render the produced model to standardized views, then a vision judge\n * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the\n * rendered images + the script text. score = fraction of criteria satisfied.\n *\n * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,\n * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL\n * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from\n * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { renderBpy } from '../worker-blender'\nimport { runBenchRouterTurn } from '../router-turn'\n\ninterface CadBenchMeta {\n  name: string\n  type: string\n  criteria: string[]\n}\n\nfunction must(name: string): string {\n  const v = process.env[name]\n  if (!v) throw new Error(`env ${name} is required for the CADBench judge`)\n  return v\n}\n\n/** One batched vision call: rendered views + the bpy script + the numbered\n *  criteria → a JSON array of booleans (true = satisfied). Faithful to the\n *  paper's combined image+script evaluation. Throws on transport failure (never\n *  a silent zero); a parse miss falls back to \"all fail\" with a note. */\nasync function judgeCriteria(\n  instruction: string,\n  script: string,\n  criteria: string[],\n  renders: string[],\n): Promise<{ passed: boolean[]; note: string }> {\n  const base = (process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1').replace(/\\/$/, '')\n  const key = must('TANGLE_API_KEY')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const numbered = criteria.map((c, i) => `${i + 1}. ${c}`).join('\\n')\n  const text =\n    `You are strictly grading a 3D model that was built by a Blender bpy script for this instruction:\\n\"${instruction}\"\\n\\n` +\n    `Below are ${renders.length} rendered views of the produced model, and the script that built it. ` +\n    `For EACH numbered criterion, decide whether it is satisfied (judge geometry/shape/proportion/structure from the IMAGES; judge color/size/material reasonableness from the SCRIPT where the images are ambiguous). ` +\n    `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\\n\\nCRITERIA:\\n${numbered}\\n\\nSCRIPT:\\n\\`\\`\\`python\\n${script.slice(0, 6000)}\\n\\`\\`\\``\n  const content: unknown[] = [{ type: 'text', text }]\n  for (const url of renders) content.push({ type: 'image_url', image_url: { url } })\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: base,\n      routerKey: key,\n      profile: {\n        name: 'cadbench-vision-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: model,\n          metadata: {\n            temperature: 0,\n            maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500),\n          },\n        },\n      },\n    },\n    { messages: [{ role: 'user', content }] },\n  )\n  const raw = turn.finalText\n  const m = /\\[\\s*(?:true|false)[\\s\\S]*?\\]/i.exec(raw)\n  if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` }\n  let arr: unknown\n  try {\n    arr = JSON.parse(m[0].toLowerCase())\n  } catch {\n    return { passed: criteria.map(() => false), note: 'judge verdict not valid JSON' }\n  }\n  const bools = Array.isArray(arr) ? arr.map((x) => x === true) : []\n  // Pad/truncate to criteria length (a short array scores the missing as fail).\n  const passed = criteria.map((_, i) => bools[i] === true)\n  return { passed, note: `${passed.filter(Boolean).length}/${criteria.length} criteria` }\n}\n\nexport function createCadBenchAdapter(): BenchmarkAdapter {\n  let cache: Array<{ id: string; instruction: string; meta: CadBenchMeta }> | null = null\n\n  async function load(): Promise<typeof cache & object> {\n    if (cache) return cache\n    const path = process.env.CADBENCH_PATH\n    if (!path) throw new Error('CADBENCH_PATH must point at the cleaned CADBench JSONL ({id,instruction,type,criteria:[]} per line)')\n    const text = await readFile(path, 'utf8')\n    cache = text\n      .split('\\n')\n      .filter((l) => l.trim())\n      .map((l) => {\n        const r = JSON.parse(l) as { id: string; name?: string; instruction: string; type?: string; criteria: string[] }\n        return { id: r.id, instruction: r.instruction, meta: { name: r.name ?? '', type: r.type ?? '', criteria: r.criteria } }\n      })\n    return cache\n  }\n\n  return {\n    name: 'cadbench',\n\n    async preflight() {\n      const { execFile } = await import('node:child_process')\n      const { promisify } = await import('node:util')\n      const exec = promisify(execFile)\n      try {\n        await exec('xvfb-run', ['-a', 'blender', '--version'], { timeout: 30_000 })\n      } catch (err) {\n        throw new Error(\n          `cadbench preflight failed: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}\\n` +\n            `Fix: install Blender + Xvfb (sudo apt-get install -y blender xvfb). The judge runs \\`xvfb-run -a blender --background --python\\`.`,\n        )\n      }\n      await load()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      let rows = await load()\n      if (opts.ids) rows = rows.filter((r) => opts.ids!.includes(r.id))\n      // TYPE filter (Simulative|Wild) via env, applied before limit.\n      const t = process.env.CADBENCH_TYPE\n      if (t) rows = rows.filter((r) => r.meta.type.toLowerCase() === t.toLowerCase())\n      if (opts.limit != null) rows = rows.slice(0, opts.limit)\n      return rows.map((r): BenchTask => ({ id: r.id, prompt: r.instruction, metadata: r.meta as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // no reference bpy script ships with the benchmark\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = task.metadata as unknown as CadBenchMeta\n      const criteria = meta.criteria ?? []\n      if (!artifact.trim()) return { resolved: false, score: 0, detail: 'empty artifact' }\n      if (criteria.length === 0) return { resolved: false, score: 0, detail: 'task has no criteria' }\n      const r = await renderBpy(artifact, { views: 4 })\n      if (!r.built) return { resolved: false, score: 0, detail: `did not build/render: ${r.error ?? 'no mesh'}` }\n      const { passed, note } = await judgeCriteria(task.prompt, artifact, criteria, r.renders)\n      const score = passed.filter(Boolean).length / criteria.length\n      return { resolved: score === 1, score, detail: note }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAwBA,MAAM,gBAAgB,UAAU,QAAQ;AAExC,eAAe,SAAS,KAAa,MAAgB,KAAa,YAAY,MAAoE;CAChJ,IAAI;EACF,MAAM,EAAE,QAAQ,WAAW,MAAM,cAAc,KAAK,MAAM;GAAE;GAAK,WAAW,KAAK;GAAI,SAAS;EAAU,CAAC;EACzG,OAAO;GAAE,MAAM;GAAG;GAAQ;EAAO;CACnC,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,OAAO;GAAE,MAAM,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;GAAG,QAAQ,EAAE,UAAU;GAAI,QAAQ,EAAE,UAAU,EAAE,WAAW,OAAO,GAAG;EAAE;CAC/H;AACF;;;;;;AAcA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwChB,KAAK;;;AAIP,eAAsB,UAAU,QAAgB,OAA2B,CAAC,GAAmE;CAC7I,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,SAAS,CAAC;CACzC,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,gBAAgB,CAAC;CAC1D,MAAM,aAAa,KAAK,KAAK,WAAW;CACxC,MAAM,aAAa,KAAK,KAAK,UAAU;CACvC,IAAI;EACF,MAAM,UAAU,YAAY,SAAS;EACrC,MAAM,UAAU,YAAY,MAAM;EAClC,MAAM,MAAM,MAAM,SAAS,YAAY;GAAC;GAAM;GAAW;GAAgB;GAAY;GAAY;GAAM;GAAY;GAAK,OAAO,KAAK;EAAC,GAAG,GAAG;EAC3I,MAAM,MAAM,GAAG,IAAI,OAAO,IAAI,IAAI;EAElC,IAAI,CADU,cAAc,KAAK,GACxB,GAAG,OAAO;GAAE,OAAO;GAAO,SAAS,CAAC;GAAG,OAAQ,kCAAkC,KAAK,GAAG,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,MAAM,IAAI;EAAG;EACpI,MAAM,UAAoB,CAAC;EAC3B,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,MAAM,MAAM,SAAS,KAAK,KAAK,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GAC5E,IAAI,KAAK,QAAQ,KAAK,yBAAyB,IAAI,SAAS,QAAQ,GAAG;EACzE;EACA,OAAO;GAAE,OAAO,QAAQ,SAAS;GAAG;EAAQ;CAC9C,UAAU;EACR,MAAM,GAAG,KAAK;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAChE;AACF;;;;;;;;;;;;;;;ACzFA,SAAS,KAAK,MAAsB;CAClC,MAAM,IAAI,QAAQ,IAAI;CACtB,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,KAAK,oCAAoC;CACxE,OAAO;AACT;;;;;AAMA,eAAe,cACb,aACA,QACA,UACA,SAC8C;CAC9C,MAAM,QAAQ,QAAQ,IAAI,eAAe,iCAAA,CAAkC,QAAQ,OAAO,EAAE;CAC5F,MAAM,MAAM,KAAK,gBAAgB;CACjC,MAAM,QAAQ,QAAQ,IAAI,eAAe;CACzC,MAAM,WAAW,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,IAAI;CAMnE,MAAM,UAAqB,CAAC;EAAE,MAAM;EAAQ,MAAA,sGAJ4D,YAAY,iBACrG,QAAQ,OAAO,6TAEW,SAAS,OAAO,2EAA2E,SAAS,6BAA6B,OAAO,MAAM,GAAG,GAAI,EAAE;CAC/I,CAAC;CAClD,KAAK,MAAM,OAAO,SAAS,QAAQ,KAAK;EAAE,MAAM;EAAa,WAAW,EAAE,IAAI;CAAE,CAAC;CAoBjF,MAAM,OAAM,MAnBO,mBACjB;EACE,eAAe;EACf,WAAW;EACX,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS;IACT,UAAU;KACR,aAAa;KACb,WAAW,OAAO,QAAQ,IAAI,oBAAoB,IAAI;IACxD;GACF;EACF;CACF,GACA,EAAE,UAAU,CAAC;EAAE,MAAM;EAAQ;CAAQ,CAAC,EAAE,CAC1C,EAAA,CACiB;CACjB,MAAM,IAAI,iCAAiC,KAAK,GAAG;CACnD,IAAI,CAAC,GAAG,OAAO;EAAE,QAAQ,SAAS,UAAU,KAAK;EAAG,MAAM,wCAAwC,IAAI,MAAM,GAAG,EAAE;CAAI;CACrH,IAAI;CACJ,IAAI;EACF,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC;CACrC,QAAQ;EACN,OAAO;GAAE,QAAQ,SAAS,UAAU,KAAK;GAAG,MAAM;EAA+B;CACnF;CACA,MAAM,QAAQ,MAAM,QAAQ,GAAG,IAAI,IAAI,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC;CAEjE,MAAM,SAAS,SAAS,KAAK,GAAG,MAAM,MAAM,OAAO,IAAI;CACvD,OAAO;EAAE;EAAQ,MAAM,GAAG,OAAO,OAAO,OAAO,CAAC,CAAC,OAAO,GAAG,SAAS,OAAO;CAAW;AACxF;AAEA,SAAgB,wBAA0C;CACxD,IAAI,QAA+E;CAEnF,eAAe,OAAuC;EACpD,IAAI,OAAO,OAAO;EAClB,MAAM,OAAO,QAAQ,IAAI;EACzB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qGAAqG;EAEhI,SAAQ,MADW,SAAS,MAAM,MAAM,EAAA,CAErC,MAAM,IAAI,CAAC,CACX,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,CACvB,KAAK,MAAM;GACV,MAAM,IAAI,KAAK,MAAM,CAAC;GACtB,OAAO;IAAE,IAAI,EAAE;IAAI,aAAa,EAAE;IAAa,MAAM;KAAE,MAAM,EAAE,QAAQ;KAAI,MAAM,EAAE,QAAQ;KAAI,UAAU,EAAE;IAAS;GAAE;EACxH,CAAC;EACH,OAAO;CACT;CAEA,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,EAAE,aAAa,MAAM,OAAO;GAClC,MAAM,EAAE,cAAc,MAAM,OAAO;GACnC,MAAM,OAAO,UAAU,QAAQ;GAC/B,IAAI;IACF,MAAM,KAAK,YAAY;KAAC;KAAM;KAAW;IAAW,GAAG,EAAE,SAAS,IAAO,CAAC;GAC5E,SAAS,KAAK;IACZ,MAAM,IAAI,MACR,+BAA+B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,GAAG,EAAE,oIAEjG;GACF;GACA,MAAM,KAAK;EACb;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,OAAO,MAAM,KAAK;GACtB,IAAI,KAAK,KAAK,OAAO,KAAK,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAEhE,MAAM,IAAI,QAAQ,IAAI;GACtB,IAAI,GAAG,OAAO,KAAK,QAAQ,MAAM,EAAE,KAAK,KAAK,YAAY,MAAM,EAAE,YAAY,CAAC;GAC9E,IAAI,KAAK,SAAS,MAAM,OAAO,KAAK,MAAM,GAAG,KAAK,KAAK;GACvD,OAAO,KAAK,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAa,UAAU,EAAE;GAA2C,EAAE;EACjI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAElE,MAAM,WADO,KAAK,SACI,YAAY,CAAC;GACnC,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAiB;GACnF,IAAI,SAAS,WAAW,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAAuB;GAC9F,MAAM,IAAI,MAAM,UAAU,UAAU,EAAE,OAAO,EAAE,CAAC;GAChD,IAAI,CAAC,EAAE,OAAO,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ,yBAAyB,EAAE,SAAS;GAAY;GAC1G,MAAM,EAAE,QAAQ,SAAS,MAAM,cAAc,KAAK,QAAQ,UAAU,UAAU,EAAE,OAAO;GACvF,MAAM,QAAQ,OAAO,OAAO,OAAO,CAAC,CAAC,SAAS,SAAS;GACvD,OAAO;IAAE,UAAU,UAAU;IAAG;IAAO,QAAQ;GAAK;EACtD;CACF;AACF"}

package/dist/{cadgenbench-DF7hYHdl.js → cadgenbench-x2OFkf8y.js}

@@ -1,10 +1,10 @@
+import "./router-turn-C2wMiDoo.js";
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { execFile } from "node:child_process";
 import "node:fs";
 import { tmpdir } from "node:os";
 import { promisify } from "node:util";
-import "@tangle-network/agent-runtime/kernel";
 promisify(execFile);
 const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? "/tmp/cgb-venv/bin/python";
 const CGB_DIR = process.env.CADGENBENCH_DIR ?? "/tmp/cadgenbench";
@@ -148,4 +148,4 @@ function createCadGenBenchAdapter() {
 //#endregion
 export { createCadGenBenchAdapter as t };
 
-//# sourceMappingURL=cadgenbench-DF7hYHdl.js.map
+//# sourceMappingURL=cadgenbench-x2OFkf8y.js.map

package/dist/cadgenbench-x2OFkf8y.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cadgenbench-x2OFkf8y.js","names":["execFileAsync"],"sources":["../src/worker-build123d.ts","../src/benchmarks/cadgenbench.ts"],"sourcesContent":["/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { runBenchRouterTurn } from './router-turn'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const turn = await runBenchRouterTurn(\n        {\n          routerBaseUrl: cfg.routerBaseUrl,\n          routerKey: cfg.routerKey,\n          profile: {\n            name: 'build123d-worker',\n            model: { provider: 'tangle-router', default: cfg.model },\n            prompt: { systemPrompt: sys },\n          },\n        },\n        user,\n      )\n      const content = turn.finalText\n      if (turn.usage.tokensKnown !== false) {\n        usage.input += turn.usage.input\n        usage.output += turn.usage.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n","/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AA2BsB,UAAU,QAAQ;AAExC,MAAa,cAAc,QAAQ,IAAI,oBAAoB;AAC3D,MAAa,UAAU,QAAQ,IAAI,mBAAmB;;;;;;;;;;;;;;;;;;ACNtD,MAAM,gBAAgB,UAAU,QAAQ;;;;AAKxC,MAAM,WAAW;;;;;;;;;;;;;;;EAef,KAAK;;;AASP,SAAS,eAAsE;CAC7E,MAAM,IAAI,KAAK,SAAS,yBAAyB;CACjD,OAAO;EACL;GAAE,IAAI;GAAgB,QAAQ;GAAyF,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EAC5J;GAAE,IAAI;GAAW,QAAQ;GAAmC,QAAQ,KAAK,GAAG,mBAAmB;EAAE;EACjG;GAAE,IAAI;GAAa,QAAQ;GAAwD,QAAQ,KAAK,GAAG,gBAAgB;EAAE;CACvH;AACF;AAEA,SAAgB,2BAA6C;CAC3D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,MAAM,IAAI,MAAM,cAAc,aAAa,CAAC,MAAM,iFAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,CAAC,CAAC,OACtJ,OAAO;IAAE,QAAQ;IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;GAAE,EAC3E;GACA,IAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GACrB,MAAM,IAAI,MACR,sCAAsC,YAAY,KAAK,EAAE,OAAO,MAAM,GAAG,GAAG,EAAE,8DACf,QAAQ,4EAA4E,SACrJ;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GAGtC,IAAI,QAAQ,aAAa;GACzB,MAAM,OAAO,QAAQ,IAAI;GACzB,IAAI,MAAM;IACR,MAAM,EAAE,aAAa,MAAM,OAAO;IAClC,QAAQ,KAAK,MAAM,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG,MAAM,CAAC;GACrE;GACA,IAAI,KAAK,KAAK,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;GAClE,IAAI,KAAK,SAAS,MAAM,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;GACzD,MAAM,QAAQ,YAA6B;IAAE;IAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,EAAG;GAAE;GACxH,OAAO,MAAM,KAAK,OAAkB;IAAE,IAAI,EAAE;IAAI,QAAQ,EAAE;IAAQ,UAAU,KAAK,EAAE,MAAM;GAAwC,EAAE;EACrI;EAEA,MAAM,eAAe,CAErB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,QAAQ,qBAAqB,KAAK;GAC1C,IAAI,CAAC,SAAS,SAAS,cAAc,GAAG,OAAO;IAAE,UAAU;IAAO,OAAO;IAAG,QAAQ;GAA8B;GAClH,MAAM,MAAM,MAAM,QAAQ,KAAK,OAAO,GAAG,YAAY,CAAC;GACtD,MAAM,OAAO,KAAK,KAAK,gBAAgB;GACvC,MAAM,SAAS,KAAK,KAAK,UAAU;GACnC,IAAI;IACF,MAAM,UAAU,MAAM,QAAQ;IAC9B,MAAM,UAAU,QAAQ,QAAQ;IAEhC,MAAM,IAAI,MAAM,cAAc,YAAY;KAAC;KAAM;KAAa;KAAQ;KAAM;IAAM,GAAG;KAAE,WAAW,KAAK;KAAI,SAAS;IAAQ,CAAC,CAAC,CAAC,OAC5H,OAAO;KAAE,QAAS,EAA0B,UAAU;KAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC;IAAE,EAChH;IACA,MAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;IAC5C,IAAI,CAAC,GAAG,OAAO;KAAE,UAAU;KAAO,OAAO;KAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,OAAA,CAAQ,MAAM,GAAG,GAAG;IAAI;IAC1H,MAAM,IAAI,KAAK,MAAM,EAAE,EAAE;IACzB,MAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;IAC9D,OAAO;KAAE,UAAU,SAAS;KAAkB;KAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,KAAK,EAAE,MAAM,KAAK;IAAK;GACnJ,UAAU;IACR,MAAM,GAAG,KAAK;KAAE,WAAW;KAAM,OAAO;IAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;GAChE;EACF;CACF;AACF"}

package/dist/index.d.ts

@@ -17,7 +17,6 @@ interface ResolveBenchClientOptions {
   backend: string;
   routerBaseUrl: string;
   routerKey: string;
-  model: string;
   /** When set on the `router` backend, the off-box worker becomes a `router-tools` agentic loop
    *  with a live `web_search` tool backed by this provider (`you`/`exa`/…). */
   searchProvider?: string;

package/dist/index.js

@@ -112,7 +112,7 @@ const makeSearchExecutor = (cfg) => {
 //#region src/resolve-client.ts
 /**
 * The ONE harness → worker-client mapping for the bench layer. Every bench entrypoint
-* (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runLoop` drives from a single
+* (rsi.ts, run.ts, …) derives the `SandboxClient` the kernel's `runAgentRounds` drives from a single
 * selector instead of hand-rolling the branch:
 *
 *   - `router`               → OFF-BOX: a router chat-completion as the leaf executor, presented
@@ -123,22 +123,21 @@ const makeSearchExecutor = (cfg) => {
 *   - `bridge`               → OFF-BOX: a local cli-bridge fronting a harness CLI
 *                              (opencode/kimi-code/…) as the leaf executor. Same resumable
 *                              `bridgeExecutor` the supervisor uses; harness+model ride the
-*                              bridge `model` id (`${harness}/${model}`).
+*                              exact harness/provider/model come from the turn's AgentProfile.
 *   - anything else (`sandbox`/a BackendType) → IN-BOX: a real `Sandbox`. The in-box backend
 *                              TYPE (opencode/codex/…) is set separately on the `AgentRunSpec`;
-*                              this only decides off-box-vs-in-box transport for `runLoop`.
+*                              this only decides off-box-vs-in-box transport for `runAgentRounds`.
 *
 * Centralizing it here means a new entrypoint gets the full off-box/in-box matrix for free and
 * the mapping can't drift between callers.
 */
 function resolveBenchClient(opts) {
-	const { backend, routerBaseUrl, routerKey, model, searchProvider } = opts;
+	const { backend, routerBaseUrl, routerKey, searchProvider } = opts;
 	if (backend === "router") {
 		if (searchProvider) return inlineSandboxClient(createExecutor({
 			backend: "router-tools",
 			routerBaseUrl,
 			routerKey,
-			model,
 			tools: [webSearchTool],
 			executeToolCall: makeSearchExecutor({
 				routerBaseUrl,
@@ -149,8 +148,7 @@ function resolveBenchClient(opts) {
 		return inlineSandboxClient(createExecutor({
 			backend: "router",
 			routerBaseUrl,
-			routerKey,
-			model
+			routerKey
 		}));
 	}
 	if (backend === "bridge") {
@@ -161,7 +159,6 @@ function resolveBenchClient(opts) {
 			bridge: {
 				url: opts.bridgeUrl,
 				bearer: bridgeBearer,
-				model,
 				timeoutMs: opts.timeoutMs
 			}
 		});
@@ -261,7 +258,6 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 		backend: cell.backend ?? "router",
 		routerBaseUrl,
 		routerKey,
-		model: cell.model,
 		...bridgeUrl ? { bridgeUrl } : {},
 		...bridgeBearer ? { bridgeBearer } : {},
 		...sandboxBaseUrl ? { sandboxBaseUrl } : {},
@@ -271,6 +267,11 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 	const harness = cell.harness ?? cell.profile?.metadata?.backendType ?? "opencode";
 	const profile = cell.profile ?? {
 		name: cell.label,
+		harness,
+		model: {
+			provider: "tangle-router",
+			default: cell.model
+		},
 		metadata: { backendType: harness }
 	};
 	const uniq = Math.random().toString(36).slice(2, 8);