@tangle-network/agent-bench 0.3.5 → 0.3.6

package/dist/chunk-YCGY7UIZ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillTemplate(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AASA,SAAS,UAAU,YAAY;AAC/B,SAAS,YAAY;AAIrB,IAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,eAAgE;AACvE,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,kEAAkE;AAC5F,SAAO;AAAA,IACL,SAAS,QAAQ,IAAI,eAAe;AAAA,IACpC;AAAA,IACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;AAAA,EAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;AACxE,MAAI;AACF,UAAM,KAAK,IAAI;AAAA,EACjB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,6BAA6B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;AACrD,QAAM,UAAU,IAAI,KAAK;AACzB,MAAI,QAAQ,WAAW,GAAG,EAAG,QAAO,KAAK,MAAM,OAAO;AACtD,SAAO,QACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,EAChC,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;AAC1F,QAAM,UAAU,QACb,OAAO,CAAC,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,EACtE,IAAI,CAAC,QAAQ,IAAI,EAAE;AACtB,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,IAAI;AAAA,MACR,mCAAmC,MAAM,uCAAuC,QAAQ,MAAM,IAAI,QAAQ,MAAM,YAAY,QAAQ,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IAE5J;AAAA,EACF;AACF;AAEA,SAAS,UAAU,KAAwB,cAAkC;AAC3E,QAAM,kBAAkB,IAAI,oBAAoB;AAChD,QAAM,kBAAkB,IAAI,oBAAoB;AAChD,QAAM,OAAwB;AAAA,IAC5B,IAAI,IAAI;AAAA,IACR,UAAU,IAAI,YAAY;AAAA,IAC1B,UAAU,IAAI;AAAA,IACd;AAAA,IACA;AAAA,IACA,WAAW,IAAI,cAAc;AAAA,IAC7B,QAAQ,IAAI,UAAU;AAAA,IACtB,mBAAmB,IAAI;AAAA,IACvB,qBAAqB,IAAI;AAAA,IACzB,SAAS,eAAe,4BAA4B;AAAA,EACtD;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,IAAI;AAAA,IACN,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAkC;AAClD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAAU;AAC1C,UAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,wDAAmD;AAAA,EACrG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;AACpG,MAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,UAAU,KAAK,YAAY,CAAC;AAC1D,MAAI,KAAK,MAAO,SAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK,KAAK;AACxE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,CAAC,EAAE;AACpG,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;AAC5D,UAAQ,KAAK,iEAA4D,QAAQ,MAAM,mBAAmB;AAC1G,SAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;AACtF,QAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;AACxD,wBAAsB,SAAS,IAAI;AACnC,SAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;AACrE,QAAM,WAAW,KAAK;AACtB,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,gCAAgC;AAC/F,SAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,EACtC,WAAW,cAAc,QAAQ,EACjC,WAAW,sBAAsB,KAAK,eAAe,EACrD,WAAW,sBAAsB,KAAK,eAAe,EACrD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,EAClE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;AACzE,QAAM,aAAuB,CAAC;AAC9B,aAAW,KAAK,QAAQ,SAAS,+BAA+B,EAAG,YAAW,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC;AAC9F,aAAW,KAAK,QAAQ,SAAS,eAAe,EAAG,YAAW,KAAK,EAAE,CAAC,CAAC;AACvE,aAAW,KAAK,QAAQ,KAAK,CAAC;AAC9B,aAAW,aAAa,YAAY;AAClC,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,SAAS;AAAA,IAC/B,QAAQ;AACN;AAAA,IACF;AACA,UAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;AACjF,UAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;AAClF,QAAI,CAAC,OAAO,SAAS,CAAC,EAAG;AACzB,QAAI,IAAI,EAAG,OAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,CAAC,EAAE;AAChG,UAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;AAClD,QAAI,QAAQ,EAAG,OAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,CAAC,EAAE;AAChH,WAAO,EAAE,OAAO,KAAK,OAAO;AAAA,EAC9B;AACA,QAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,CAAC,EAAE;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;AAC5F,MAAI,CAAC,KAAK,kBAAmB,OAAM,IAAI,MAAM,yBAAyB,KAAK,EAAE,8BAA8B;AAC3G,QAAM,SAAS,aAAa;AAC5B,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,IAC5D,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,IACrF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,OAAO;AAAA,MACd,aAAa;AAAA,MACb,UAAU;AAAA,QACR,EAAE,MAAM,UAAU,SAAS,KAAK,kBAAkB;AAAA,QAClD,EAAE,MAAM,QAAQ,SAAS,aAAa,MAAM,QAAQ,EAAE;AAAA,MACxD;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAC7G,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,MAAI,OAAO,YAAY,SAAU,OAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAC5I,QAAM,EAAE,OAAO,IAAI,IAAI,gBAAgB,OAAO;AAC9C,SAAO;AAAA,IACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,GAAG;AAAA,IAC5E;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,UAAU,YAAY,OAAO,OAAO,IAAI,CAAC;AAAA,EAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;AAC5C,SAAO,MAAM,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;AACzE,QAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;AACzE,QAAM,WAAW,cAAc,QAAQ;AACvC,QAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;AACnE,SAAO;AAAA,IACL,UAAU,UAAU;AAAA,IACpB;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,SAAS,CAAC;AAAA,EAC3E;AACF;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,8BAA8B;AAE/D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,eAAe,UAAU,cAAc;AAC7C;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,mBAAa;AACb,YAAM,eAAe,MAAM,WAAW;AACtC,YAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;AAAA,IAC5C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,sEAAsE;AACjG,aAAO,kBAAkB,MAAM,IAAI;AAAA,IACrC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,KAAK,mBAAmB,KAAK,mBAAmB;AAAA,IACzD;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI,aAAc,QAAO,aAAa,MAAM,QAAQ;AACpD,UAAI,SAAS,KAAK,EAAE,WAAW,GAAG;AAChC,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,UAAU,KAAK,UAAU,QAAQ,eAAe,CAAC;AAAA,QACnG;AAAA,MACF;AACA,aAAO,iBAAiB,MAAM,QAAQ;AAAA,IACxC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-Z7ML6L77.js

@@ -0,0 +1,162 @@
+import {
+  benchRoot,
+  runVenvPython
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/webarena-verified.ts
+import { access, readFile, stat } from "fs/promises";
+import { join, resolve } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "webarena-verified.json");
+var DATASET_REL = join("assets", "dataset", "webarena-verified.json");
+var webarenaDir = () => process.env.WEBARENA_VERIFIED_DIR;
+var webarenaOutputDirOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:text|path)?\s*\n([\s\S]*?)```/g)];
+    return (fences.at(-1)?.[1] ?? text).trim();
+  }
+};
+async function assertFile(path, label) {
+  try {
+    await access(path);
+  } catch (err) {
+    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+  }
+}
+async function assertDir(path, label) {
+  try {
+    const s = await stat(path);
+    if (!s.isDirectory()) throw new Error("not a directory");
+  } catch (err) {
+    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+  }
+}
+function rowToTask(row) {
+  const meta = {
+    taskId: row.task_id,
+    intentTemplateId: row.intent_template_id,
+    sites: row.sites ?? [],
+    startUrls: row.start_urls ?? [],
+    revision: row.revision,
+    eval: row.eval
+  };
+  return {
+    id: String(row.task_id),
+    prompt: [
+      "Run this WebArena-Verified browser task in the official environment.",
+      `Goal: ${row.intent}`,
+      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(", ")}` : void 0,
+      row.sites?.length ? `Sites: ${row.sites.join(", ")}` : void 0,
+      "",
+      "Return the path to the official WebArena-Verified run output directory for this task.",
+      "The judge expects that directory to contain the task response and network trace files."
+    ].filter(Boolean).join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.taskId !== "number") {
+    throw new Error(`webarena-verified task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectRows(rows, opts) {
+  let tasks = rows.map(rowToTask);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((task) => want.has(task.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`);
+  return tasks;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
+  return selectRows(rows, opts);
+}
+async function loadOfficialRows(dir, opts) {
+  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), "utf8"));
+  return selectRows(rows, opts);
+}
+async function runOfficialEval(root, taskId, outputDir) {
+  const script = `
+import json, os, subprocess, sys
+from pathlib import Path
+
+root = Path(sys.argv[1])
+task_id = sys.argv[2]
+output_dir = Path(sys.argv[3])
+env = os.environ.copy()
+env["PYTHONPATH"] = str(root / "src") + os.pathsep + env.get("PYTHONPATH", "")
+cmd = [sys.executable, "-m", "webarena_verified", "eval-tasks", "--task-ids", task_id, "--output-dir", str(output_dir)]
+proc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)
+if proc.returncode != 0:
+    raise SystemExit((proc.stderr or proc.stdout or f"exit {proc.returncode}")[:2000])
+result_path = output_dir / task_id / "eval_result.json"
+if not result_path.exists():
+    raise SystemExit(f"official evaluator did not write {result_path}")
+print(json.dumps(json.loads(result_path.read_text())))
+`;
+  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0);
+  return JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+}
+function createWebArenaVerifiedAdapter() {
+  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === "1";
+  return {
+    name: "webarena-verified",
+    output: webarenaOutputDirOutput,
+    async preflight() {
+      if (fixturesMode) return;
+      const dir = webarenaDir();
+      if (!dir) {
+        throw new Error(
+          "WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified."
+        );
+      }
+      await assertFile(join(dir, DATASET_REL), "official dataset");
+      await assertFile(join(dir, "src", "webarena_verified", "__main__.py"), "official CLI module");
+      await runVenvPython(
+        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print("ok")',
+        [join(dir, "src")]
+      );
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const dir = webarenaDir();
+      if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks");
+      return loadOfficialRows(dir, opts);
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const dir = webarenaDir();
+      if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts");
+      const outputDir = resolve(artifact.trim());
+      await assertDir(outputDir, "run output directory");
+      const meta = readMeta(task);
+      const report = await runOfficialEval(dir, meta.taskId, outputDir);
+      const score = typeof report.score === "number" ? report.score : 0;
+      const status = typeof report.status === "string" ? report.status : "unknown";
+      return {
+        resolved: score === 1,
+        score,
+        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir })
+      };
+    }
+  };
+}
+
+export {
+  webarenaOutputDirOutput,
+  createWebArenaVerifiedAdapter
+};
+//# sourceMappingURL=chunk-Z7ML6L77.js.map

package/dist/chunk-Z7ML6L77.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/webarena-verified.ts"],"sourcesContent":["/**\n * WebArena-Verified adapter (ServiceNow/webarena-verified).\n *\n * Worker artifact = a WebArena-Verified run output directory, not final chat text.\n * Judge = the official `webarena_verified eval-tasks` evaluator over that output\n * directory. The adapter refuses to score a plain answer so we do not turn a DOM\n * benchmark into a fake text benchmark.\n */\n\nimport { access, readFile, stat } from 'node:fs/promises'\nimport { join, resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'webarena-verified.json')\nconst DATASET_REL = join('assets', 'dataset', 'webarena-verified.json')\n\ninterface WebArenaRow {\n  task_id: number\n  intent: string\n  intent_template_id?: number\n  sites?: string[]\n  start_urls?: string[]\n  eval?: unknown[]\n  revision?: number\n}\n\ninterface WebArenaMeta {\n  taskId: number\n  intentTemplateId?: number\n  sites: string[]\n  startUrls: string[]\n  revision?: number\n  eval?: unknown[]\n}\n\nconst webarenaDir = (): string | undefined => process.env.WEBARENA_VERIFIED_DIR\n\nexport const webarenaOutputDirOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertDir(path: string, label: string): Promise<void> {\n  try {\n    const s = await stat(path)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction rowToTask(row: WebArenaRow): BenchTask {\n  const meta: WebArenaMeta = {\n    taskId: row.task_id,\n    intentTemplateId: row.intent_template_id,\n    sites: row.sites ?? [],\n    startUrls: row.start_urls ?? [],\n    revision: row.revision,\n    eval: row.eval,\n  }\n  return {\n    id: String(row.task_id),\n    prompt: [\n      'Run this WebArena-Verified browser task in the official environment.',\n      `Goal: ${row.intent}`,\n      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(', ')}` : undefined,\n      row.sites?.length ? `Sites: ${row.sites.join(', ')}` : undefined,\n      '',\n      'Return the path to the official WebArena-Verified run output directory for this task.',\n      'The judge expects that directory to contain the task response and network trace files.',\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): WebArenaMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number') {\n    throw new Error(`webarena-verified task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as WebArenaMeta\n}\n\nfunction selectRows(rows: WebArenaRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as WebArenaRow[]\n  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts)\n}\n\nasync function loadOfficialRows(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), 'utf8')) as WebArenaRow[]\n  return selectRows(rows, opts)\n}\n\nasync function runOfficialEval(root: string, taskId: number, outputDir: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, os, subprocess, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\noutput_dir = Path(sys.argv[3])\nenv = os.environ.copy()\nenv[\"PYTHONPATH\"] = str(root / \"src\") + os.pathsep + env.get(\"PYTHONPATH\", \"\")\ncmd = [sys.executable, \"-m\", \"webarena_verified\", \"eval-tasks\", \"--task-ids\", task_id, \"--output-dir\", str(output_dir)]\nproc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)\nif proc.returncode != 0:\n    raise SystemExit((proc.stderr or proc.stdout or f\"exit {proc.returncode}\")[:2000])\nresult_path = output_dir / task_id / \"eval_result.json\"\nif not result_path.exists():\n    raise SystemExit(f\"official evaluator did not write {result_path}\")\nprint(json.dumps(json.loads(result_path.read_text())))\n`\n  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createWebArenaVerifiedAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === '1'\n\n  return {\n    name: 'webarena-verified',\n    output: webarenaOutputDirOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = webarenaDir()\n      if (!dir) {\n        throw new Error(\n          'WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified.',\n        )\n      }\n      await assertFile(join(dir, DATASET_REL), 'official dataset')\n      await assertFile(join(dir, 'src', 'webarena_verified', '__main__.py'), 'official CLI module')\n      await runVenvPython(\n        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print(\"ok\")',\n        [join(dir, 'src')],\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks')\n      return loadOfficialRows(dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts')\n      const outputDir = resolve(artifact.trim())\n      await assertDir(outputDir, 'run output directory')\n      const meta = readMeta(task)\n      const report = await runOfficialEval(dir, meta.taskId, outputDir)\n      const score = typeof report.score === 'number' ? report.score : 0\n      const status = typeof report.status === 'string' ? report.status : 'unknown'\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;AASA,SAAS,QAAQ,UAAU,YAAY;AACvC,SAAS,MAAM,eAAe;AAK9B,IAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,IAAM,cAAc,KAAK,UAAU,WAAW,wBAAwB;AAqBtE,IAAM,cAAc,MAA0B,QAAQ,IAAI;AAEnD,IAAM,0BAAiD;AAAA,EAC5D,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,sCAAsC,CAAC;AACxE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,WAAW,MAAc,OAA8B;AACpE,MAAI;AACF,UAAM,OAAO,IAAI;AAAA,EACnB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,8BAA8B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAChH;AACF;AAEA,eAAe,UAAU,MAAc,OAA8B;AACnE,MAAI;AACF,UAAM,IAAI,MAAM,KAAK,IAAI;AACzB,QAAI,CAAC,EAAE,YAAY,EAAG,OAAM,IAAI,MAAM,iBAAiB;AAAA,EACzD,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,8BAA8B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAChH;AACF;AAEA,SAAS,UAAU,KAA6B;AAC9C,QAAM,OAAqB;AAAA,IACzB,QAAQ,IAAI;AAAA,IACZ,kBAAkB,IAAI;AAAA,IACtB,OAAO,IAAI,SAAS,CAAC;AAAA,IACrB,WAAW,IAAI,cAAc,CAAC;AAAA,IAC9B,UAAU,IAAI;AAAA,IACd,MAAM,IAAI;AAAA,EACZ;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,OAAO;AAAA,IACtB,QAAQ;AAAA,MACN;AAAA,MACA,SAAS,IAAI,MAAM;AAAA,MACnB,IAAI,YAAY,SAAS,wBAAwB,IAAI,WAAW,KAAK,IAAI,CAAC,KAAK;AAAA,MAC/E,IAAI,OAAO,SAAS,UAAU,IAAI,MAAM,KAAK,IAAI,CAAC,KAAK;AAAA,MACvD;AAAA,MACA;AAAA,MACA;AAAA,IACF,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAAU;AACxC,UAAM,IAAI,MAAM,0BAA0B,KAAK,EAAE,wDAAmD;AAAA,EACtG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAqB,MAAgC;AACvE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC,KAAK,UAAU,IAAI,CAAC,EAAE;AACrG,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,mEAA8D,KAAK,MAAM,mBAAmB;AACzG,SAAO,WAAW,MAAM,IAAI;AAC9B;AAEA,eAAe,iBAAiB,KAAa,MAAyC;AACpF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,KAAK,KAAK,WAAW,GAAG,MAAM,CAAC;AACtE,SAAO,WAAW,MAAM,IAAI;AAC9B;AAEA,eAAe,gBAAgB,MAAc,QAAgB,WAAqD;AAChH,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,MAAM,OAAO,MAAM,GAAG,SAAS,GAAG,CAAC;AAC/E,SAAO,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,+BAA+B;AAEhE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,WAAW,KAAK,KAAK,WAAW,GAAG,kBAAkB;AAC3D,YAAM,WAAW,KAAK,KAAK,OAAO,qBAAqB,aAAa,GAAG,qBAAqB;AAC5F,YAAM;AAAA,QACJ;AAAA,QACA,CAAC,KAAK,KAAK,KAAK,CAAC;AAAA,MACnB;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,4EAA4E;AACtG,aAAO,iBAAiB,KAAK,IAAI;AAAA,IACnC;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wEAAwE;AAClG,YAAM,YAAY,QAAQ,SAAS,KAAK,CAAC;AACzC,YAAM,UAAU,WAAW,sBAAsB;AACjD,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK,QAAQ,SAAS;AAChE,YAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;AAChE,YAAM,SAAS,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS;AACnE,aAAO;AAAA,QACL,UAAU,UAAU;AAAA,QACpB;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,UAAU,CAAC;AAAA,MACnE;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-ZEWMTR5M.js

@@ -0,0 +1,136 @@
+import {
+  benchRoot,
+  preflightVenvImports,
+  runVenvScriptStdin
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/programbench.ts
+import { join } from "path";
+import { readFile } from "fs/promises";
+var FIXTURES = join(benchRoot, "fixtures", "programbench.json");
+var TESTS_DATASET = "programbench/ProgramBench-Tests";
+var TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`;
+var PROMPT = [
+  "This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.",
+  "Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.",
+  "You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.",
+  "Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence."
+].join("\n");
+function fixtureToTask(f) {
+  const meta = { instanceId: f.instance_id, imageName: f.image_name };
+  return {
+    id: f.instance_id,
+    prompt: PROMPT,
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.instanceId !== "string") {
+    throw new Error(`programbench task ${task.id} missing metadata.instanceId \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectTasks(fixtures, opts) {
+  let tasks = fixtures.map(fixtureToTask);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((t) => want.has(t.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  return tasks;
+}
+function imageName(instanceId) {
+  return instanceId.split(".")[0].replace("__", "/");
+}
+async function loadFixtures(opts) {
+  const fixtures = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(
+    `[programbench] PROGRAMBENCH_FIXTURES=1 \u2014 loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`
+  );
+  return selectTasks(fixtures, opts);
+}
+async function fetchInstanceIds() {
+  const res = await fetch(TREE_API);
+  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`);
+  const tree = await res.json();
+  const ids = tree.filter((e) => e.type === "directory" && e.path.includes("__")).map((e) => e.path);
+  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`);
+  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }));
+}
+async function runHarness(meta, submission) {
+  const judge = join(benchRoot, "scripts", "programbench_judge.py");
+  let stdout;
+  try {
+    stdout = await runVenvScriptStdin(judge, ["--instance", meta.instanceId], submission, { cwd: benchRoot });
+  } catch (err) {
+    const e = err;
+    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`);
+  }
+  const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`);
+  if (typeof report.passed !== "number" || typeof report.total !== "number") {
+    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`);
+  }
+  const score = report.total > 0 ? report.passed / report.total : 0;
+  return {
+    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),
+    score,
+    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total })
+  };
+}
+var programbenchSubmissionOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const blocks = [...text.matchAll(/```path:([^\n`]+)\n([\s\S]*?)```/g)];
+    if (blocks.length === 0) return "";
+    return blocks.map((m) => `===FILE:${m[1].trim()}===
+${m[2]}`).join("\n");
+  }
+};
+function createProgrambenchAdapter() {
+  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === "1";
+  return {
+    name: "programbench",
+    output: programbenchSubmissionOutput,
+    async preflight() {
+      await preflightVenvImports({
+        modules: ["programbench"],
+        requireDocker: true,
+        fix: `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; (2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; (3) HF access for the hidden test blobs (the driver runs \`programbench blob sync\`). Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`
+      });
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      let fixtures;
+      try {
+        fixtures = await fetchInstanceIds();
+      } catch (err) {
+        console.warn(
+          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`
+        );
+        return loadFixtures(opts);
+      }
+      return selectTasks(fixtures, opts);
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      return runHarness(meta, artifact);
+    }
+  };
+}
+
+export {
+  programbenchSubmissionOutput,
+  createProgrambenchAdapter
+};
+//# sourceMappingURL=chunk-ZEWMTR5M.js.map

package/dist/chunk-ZEWMTR5M.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/programbench.ts"],"sourcesContent":["/**\n * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom\n * reverse-engineering: rebuild a black-box executable's behavior from scratch.\n * The agent is given only the gold `./executable` (run-only) + stripped docs and\n * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`\n * with identical observable behavior. Judge = the official `programbench`\n * harness: it extracts the submission, runs compile.sh in the per-task cleanroom\n * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via\n * `programbench blob sync`). Score = fraction of non-ignored tests passed\n * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests\n * pass. Fully deterministic — no LLM judge.\n *\n * OutputAdapter is stream-only, so the worker emits its codebase as fenced\n * `path:`-prefixed file blocks (including compile.sh); the adapter materializes\n * those into submission.tar.gz. Test execution is delegated to the real\n * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT\n * reimplemented here.\n *\n * Requires for a live run: the bench `.venv` with `programbench` installed +\n * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for\n * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load\n * the committed instance ids (bench/fixtures/programbench.json); judging still\n * needs the harness + Docker and fails loud without them — never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'programbench.json')\n\nconst TESTS_DATASET = 'programbench/ProgramBench-Tests'\nconst TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`\n\ninterface PbFixture {\n  instance_id: string\n  image_name: string\n  note?: string\n}\n\ninterface PbMeta {\n  instanceId: string\n  imageName: string\n}\n\nconst PROMPT = [\n  'This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.',\n  'Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.',\n  'You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.',\n  'Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence.',\n].join('\\n')\n\nfunction fixtureToTask(f: PbFixture): BenchTask {\n  const meta: PbMeta = { instanceId: f.instance_id, imageName: f.image_name }\n  return {\n    id: f.instance_id,\n    prompt: PROMPT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): PbMeta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string') {\n    throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`)\n  }\n  return md as unknown as PbMeta\n}\n\nfunction selectTasks(fixtures: PbFixture[], opts: LoadOptions): BenchTask[] {\n  let tasks = fixtures.map(fixtureToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\n/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */\nfunction imageName(instanceId: string): string {\n  return instanceId.split('.')[0].replace('__', '/')\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixtures = JSON.parse(await readFile(FIXTURES, 'utf8')) as PbFixture[]\n  console.warn(\n    `[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`,\n  )\n  return selectTasks(fixtures, opts)\n}\n\n/** Enumerate real instance ids from the HF Tests dataset tree (one dir per\n *  instance). Throws on a non-OK response (fail loud). */\nasync function fetchInstanceIds(): Promise<PbFixture[]> {\n  const res = await fetch(TREE_API)\n  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`)\n  const tree = (await res.json()) as Array<{ path: string; type: string }>\n  const ids = tree.filter((e) => e.type === 'directory' && e.path.includes('__')).map((e) => e.path)\n  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`)\n  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }))\n}\n\n/**\n * Run the official programbench harness for one instance over the worker's\n * file-manifest submission. The driver builds the run-dir layout (one folder\n * holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,\n * then parses <instance_id>.eval.json applying the tests.json ignore mask via\n * `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.\n */\nasync function runHarness(meta: PbMeta, submission: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'programbench_judge.py')\n  let stdout: string\n  try {\n    // The submission manifest is piped to the driver's stdin via the shared\n    // stdin-aware runner — execFile's `input` option is not honored async and\n    // hangs the driver's sys.stdin.read() forever.\n    stdout = await runVenvScriptStdin(judge, ['--instance', meta.instanceId], submission, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    resolved?: boolean\n    error?: string\n  }\n  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passed / report.total : 0\n  return {\n    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\n/**\n * Parse the worker stream into the submission text the driver materializes: the\n * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to\n * the python driver, which tars it. Empty when the worker emitted no file block\n * (fail-closed → the harness scores a missing compile.sh as 0).\n */\nexport const programbenchSubmissionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const blocks = [...text.matchAll(/```path:([^\\n`]+)\\n([\\s\\S]*?)```/g)]\n    if (blocks.length === 0) return ''\n    // Re-serialize in the same path:<p> envelope the driver splits on.\n    return blocks.map((m) => `===FILE:${m[1].trim()}===\\n${m[2]}`).join('\\n')\n  },\n}\n\nexport function createProgrambenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === '1'\n\n  return {\n    name: 'programbench',\n    output: programbenchSubmissionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['programbench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; ` +\n          `(2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; ` +\n          `(3) HF access for the hidden test blobs (the driver runs \\`programbench blob sync\\`). ` +\n          `Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let fixtures: PbFixture[]\n      try {\n        fixtures = await fetchInstanceIds()\n      } catch (err) {\n        console.warn(\n          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectTasks(fixtures, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the original repo's source (stripped from the task image),\n      // not redistributable as a portable submission string. Judge correctness is\n      // proven by running the real harness on a real solve, not a synthetic gold.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;AAyBA,SAAS,YAAY;AACrB,SAAS,gBAAgB;AAKzB,IAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEhE,IAAM,gBAAgB;AACtB,IAAM,WAAW,uCAAuC,aAAa;AAarE,IAAM,SAAS;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,SAAS,cAAc,GAAyB;AAC9C,QAAM,OAAe,EAAE,YAAY,EAAE,aAAa,WAAW,EAAE,WAAW;AAC1E,SAAO;AAAA,IACL,IAAI,EAAE;AAAA,IACN,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAyB;AACzC,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,eAAe,UAAU;AAC5C,UAAM,IAAI,MAAM,qBAAqB,KAAK,EAAE,mEAA8D;AAAA,EAC5G;AACA,SAAO;AACT;AAEA,SAAS,YAAY,UAAuB,MAAgC;AAC1E,MAAI,QAAQ,SAAS,IAAI,aAAa;AACtC,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAGA,SAAS,UAAU,YAA4B;AAC7C,SAAO,WAAW,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,MAAM,GAAG;AACnD;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,WAAW,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AAC5D,UAAQ;AAAA,IACN,yDAAoD,SAAS,MAAM,gCAAgC,QAAQ;AAAA,EAC7G;AACA,SAAO,YAAY,UAAU,IAAI;AACnC;AAIA,eAAe,mBAAyC;AACtD,QAAM,MAAM,MAAM,MAAM,QAAQ;AAChC,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,0BAA0B,IAAI,MAAM,KAAK,QAAQ,EAAE;AAChF,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,KAAK,SAAS,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AACjG,MAAI,IAAI,WAAW,EAAG,OAAM,IAAI,MAAM,qCAAqC,aAAa,OAAO;AAC/F,SAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,aAAa,IAAI,YAAY,UAAU,EAAE,EAAE,EAAE;AACzE;AASA,eAAe,WAAW,MAAc,YAAyC;AAC/E,QAAM,QAAQ,KAAK,WAAW,WAAW,uBAAuB;AAChE,MAAI;AACJ,MAAI;AAIF,aAAS,MAAM,mBAAmB,OAAO,CAAC,cAAc,KAAK,UAAU,GAAG,YAAY,EAAE,KAAK,UAAU,CAAC;AAAA,EAC1G,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,mCAAmC,KAAK,UAAU,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EACpH;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAMlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,kCAAkC,KAAK,UAAU,KAAK,OAAO,KAAK,EAAE;AACtG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,kDAAkD,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC1F;AACA,QAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;AAChE,SAAO;AAAA,IACL,UAAU,OAAO,aAAa,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;AAAA,IAC3E;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,YAAY,KAAK,YAAY,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACpG;AACF;AAQO,IAAM,+BAAsD;AAAA,EACjE,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,mCAAmC,CAAC;AACrE,QAAI,OAAO,WAAW,EAAG,QAAO;AAEhC,WAAO,OAAO,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,EAAE,KAAK,CAAC;AAAA,EAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI;AAAA,EAC1E;AACF;AAEO,SAAS,4BAA8C;AAC5D,QAAM,eAAe,QAAQ,IAAI,0BAA0B;AAE3D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,cAAc;AAAA,QACxB,eAAe;AAAA,QACf,KACE;AAAA,MAIJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,UAAI;AACJ,UAAI;AACF,mBAAW,MAAM,iBAAiB;AAAA,MACpC,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG,4CAA4C,QAAQ;AAAA,QACxI;AACA,eAAO,aAAa,IAAI;AAAA,MAC1B;AACA,aAAO,YAAY,UAAU,IAAI;AAAA,IACnC;AAAA,IAEA,MAAM,eAAe;AAInB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,WAAW,MAAM,QAAQ;AAAA,IAClC;AAAA,EACF;AACF;","names":[]}