@tangle-network/agent-bench 0.1.0 → 0.3.6

package/dist/chunk-X3BTXCJ4.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/rag-shared.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport type { BenchScore, BenchTask, LoadOptions } from './types'\n\nexport const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\nexport interface RagContext {\n  id: string\n  text: string\n  title?: string\n  source?: string\n  relevant?: boolean\n}\n\nexport interface RagAnswerScore {\n  resolved: boolean\n  score: number\n  finalAnswer: string\n  bestGold: string | null\n  exact: boolean\n  numeric: boolean\n  f1: number\n  threshold: number\n}\n\nexport const ragAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const trimmed = (lines[i] ?? '').trim()\n    if (trimmed.length > 0) return trimmed\n  }\n  return ''\n}\n\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const match of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(match[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\nexport function normalizeAnswer(input: string): string {\n  return input\n    .toLowerCase()\n    .replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n    .replace(/[^\\p{L}\\p{N}\\s.-]+/gu, ' ')\n    .split(/\\s+/)\n    .filter((token) => token.length > 0)\n    .filter((token) => !['a', 'an', 'the'].includes(token))\n    .join(' ')\n    .trim()\n}\n\nfunction tokens(input: string): string[] {\n  const normalized = normalizeAnswer(input)\n  return normalized.length === 0 ? [] : normalized.split(/\\s+/)\n}\n\nexport function tokenF1(candidate: string, gold: string): number {\n  const candidateTokens = tokens(candidate)\n  const goldTokens = tokens(gold)\n  if (candidateTokens.length === 0 || goldTokens.length === 0) {\n    return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const counts = new Map<string, number>()\n  for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1)\n  let common = 0\n  for (const token of candidateTokens) {\n    const left = counts.get(token)\n    if (left !== undefined && left > 0) {\n      common += 1\n      counts.set(token, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / candidateTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\nfunction exactOrContains(candidate: string, gold: string): boolean {\n  const normalizedCandidate = normalizeAnswer(candidate)\n  const normalizedGold = normalizeAnswer(gold)\n  if (normalizedGold.length === 0) return false\n  if (normalizedCandidate === normalizedGold) return true\n  const escaped = normalizedGold.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n  return new RegExp(`(^|\\\\s)${escaped}(\\\\s|$)`).test(normalizedCandidate)\n}\n\nfunction numbers(input: string): number[] {\n  return [...input.replace(/,/g, '').matchAll(/-?\\d+(?:\\.\\d+)?/g)]\n    .map((match) => Number(match[0]))\n    .filter(Number.isFinite)\n}\n\nfunction numericMatch(candidate: string, gold: string, relativeTolerance: number): boolean {\n  const got = numbers(candidate)\n  const want = numbers(gold)\n  if (got.length === 0 || want.length === 0) return false\n  return want.some((expected) =>\n    got.some((actual) => {\n      const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance)\n      return Math.abs(actual - expected) <= tolerance\n    }),\n  )\n}\n\nexport function scoreAnswerArtifact(\n  artifact: string,\n  golds: readonly string[],\n  options: { threshold?: number; numericTolerance?: number } = {},\n): RagAnswerScore {\n  const finalAnswer = parseFinalAnswer(artifact)\n  const threshold = options.threshold ?? 0.72\n  const numericTolerance = options.numericTolerance ?? 0.01\n  let best: RagAnswerScore = {\n    resolved: false,\n    score: 0,\n    finalAnswer,\n    bestGold: null,\n    exact: false,\n    numeric: false,\n    f1: 0,\n    threshold,\n  }\n  if (finalAnswer.length === 0 || golds.length === 0) return best\n\n  for (const gold of golds) {\n    const exact = exactOrContains(finalAnswer, gold)\n    const numeric = numericMatch(finalAnswer, gold, numericTolerance)\n    const f1 = tokenF1(finalAnswer, gold)\n    const resolved = exact || numeric || f1 >= threshold\n    const score = exact || numeric ? 1 : f1\n    if (score > best.score || (resolved && !best.resolved)) {\n      best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold }\n    }\n  }\n  return best\n}\n\nexport function answerScoreToBenchScore(\n  score: RagAnswerScore,\n  detail: Record<string, unknown>,\n): BenchScore {\n  return {\n    resolved: score.resolved,\n    score: score.score,\n    detail: JSON.stringify({\n      ...detail,\n      finalAnswer: score.finalAnswer,\n      bestGold: score.bestGold,\n      exact: score.exact,\n      numeric: score.numeric,\n      f1: score.f1,\n      threshold: score.threshold,\n    }),\n  }\n}\n\nexport async function readJsonRows(path: string): Promise<unknown[]> {\n  const raw = (await readFile(path, 'utf8')).trim()\n  if (raw.length === 0) return []\n  if (raw.startsWith('[')) {\n    const parsed = JSON.parse(raw) as unknown\n    if (Array.isArray(parsed)) return parsed\n    throw new Error(`${path} must contain a JSON array when it starts with [`)\n  }\n  if (raw.startsWith('{')) {\n    try {\n      const parsed = JSON.parse(raw) as unknown\n      if (isObject(parsed)) {\n        for (const key of ['rows', 'data', 'examples', 'items']) {\n          const value = parsed[key]\n          if (Array.isArray(value)) return value\n        }\n        return [parsed]\n      }\n      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`)\n    } catch (err) {\n      if (!raw.includes('\\n')) throw err\n    }\n  }\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nexport function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[] {\n  let selected = tasks\n  if (opts.split) selected = selected.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const ids = new Set(opts.ids)\n    selected = selected.filter((task) => ids.has(task.id))\n  } else if (opts.limit !== undefined) {\n    selected = selected.slice(0, opts.limit)\n  }\n  if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`)\n  return selected\n}\n\nexport function stringFrom(value: unknown): string | undefined {\n  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined\n}\n\nexport function stringArrayFrom(value: unknown): string[] {\n  if (typeof value === 'string' && value.trim().length > 0) return [value.trim()]\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry) => {\n    if (typeof entry === 'string' && entry.trim().length > 0) return [entry.trim()]\n    if (isObject(entry)) {\n      const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text)\n      return text ? [text] : []\n    }\n    return []\n  })\n}\n\nexport function firstString(row: Record<string, unknown>, keys: readonly string[]): string {\n  for (const key of keys) {\n    const value = stringFrom(row[key])\n    if (value) return value\n  }\n  return ''\n}\n\nexport function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[] {\n  const out: string[] = []\n  for (const key of keys) out.push(...stringArrayFrom(row[key]))\n  return [...new Set(out)]\n}\n\nexport function contextsFrom(value: unknown): RagContext[] {\n  if (typeof value === 'string' && value.trim().length > 0) {\n    return [{ id: 'ctx-1', text: value.trim() }]\n  }\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry, index): RagContext[] => {\n    if (typeof entry === 'string' && entry.trim().length > 0) {\n      return [{ id: `ctx-${index + 1}`, text: entry.trim() }]\n    }\n    if (Array.isArray(entry)) {\n      const [first, second] = entry\n      if (typeof first === 'string' && typeof second === 'string') {\n        return [{ id: first, text: second.trim() }]\n      }\n      return entry.flatMap((nested, nestedIndex) =>\n        contextsFrom([nested]).map((ctx) => ({\n          ...ctx,\n          id: ctx.id.startsWith('ctx-') ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id,\n        })),\n      )\n    }\n    if (!isObject(entry)) return []\n    const text =\n      stringFrom(entry.text) ??\n      stringFrom(entry.context) ??\n      stringFrom(entry.content) ??\n      stringFrom(entry.passage) ??\n      stringFrom(entry.document) ??\n      stringFrom(entry.page_content) ??\n      stringFrom(entry.chunk)\n    if (!text) return []\n    const id =\n      stringFrom(entry.id) ??\n      stringFrom(entry.docid) ??\n      stringFrom(entry.doc_id) ??\n      stringFrom(entry.document_id) ??\n      `ctx-${index + 1}`\n    const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance\n    const relevant =\n      typeof relevantRaw === 'boolean'\n        ? relevantRaw\n        : typeof relevantRaw === 'number'\n          ? relevantRaw > 0\n          : undefined\n    return [\n      {\n        id,\n        text,\n        ...(stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {}),\n        ...(stringFrom(entry.source) ?? stringFrom(entry.url)\n          ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) }\n          : {}),\n        ...(relevant !== undefined ? { relevant } : {}),\n      },\n    ]\n  })\n}\n\nexport function contextBlock(contexts: readonly RagContext[]): string {\n  if (contexts.length === 0) return ''\n  return contexts\n    .map((ctx, index) =>\n      [\n        `[${index + 1}] ${ctx.title ?? ctx.id}`,\n        ctx.source ? `Source: ${ctx.source}` : undefined,\n        ctx.text,\n      ]\n        .filter(Boolean)\n        .join('\\n'),\n    )\n    .join('\\n\\n')\n}\n\nexport function isObject(value: unknown): value is Record<string, unknown> {\n  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AAIlB,IAAM,wBAAwB;AAqB9B,IAAM,kBAAyC;AAAA,EACpD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEO,SAAS,iBAAiB,UAA0B;AACzD,QAAM,QAAQ,SAAS,MAAM,OAAO;AACpC,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,OAAO,MAAM,CAAC,KAAK;AACzB,UAAM,MAAM,KAAK,YAAY,EAAE,QAAQ,qBAAqB;AAC5D,QAAI,QAAQ,GAAI,QAAO,KAAK,MAAM,MAAM,sBAAsB,MAAM,EAAE,KAAK;AAAA,EAC7E;AACA,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,WAAW,MAAM,CAAC,KAAK,IAAI,KAAK;AACtC,QAAI,QAAQ,SAAS,EAAG,QAAO;AAAA,EACjC;AACA,SAAO;AACT;AAEO,SAAS,eAAe,UAA4B;AACzD,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,SAAS,SAAS,SAAS,yBAAyB,GAAG;AAChE,SAAK,IAAI,MAAM,CAAC,EAAE,QAAQ,WAAW,EAAE,CAAC;AAAA,EAC1C;AACA,SAAO,CAAC,GAAG,IAAI;AACjB;AAEO,SAAS,gBAAgB,OAAuB;AACrD,SAAO,MACJ,YAAY,EACZ,QAAQ,qBAAqB,IAAI,EACjC,QAAQ,wBAAwB,GAAG,EACnC,MAAM,KAAK,EACX,OAAO,CAAC,UAAU,MAAM,SAAS,CAAC,EAClC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC,EACrD,KAAK,GAAG,EACR,KAAK;AACV;AAEA,SAAS,OAAO,OAAyB;AACvC,QAAM,aAAa,gBAAgB,KAAK;AACxC,SAAO,WAAW,WAAW,IAAI,CAAC,IAAI,WAAW,MAAM,KAAK;AAC9D;AAEO,SAAS,QAAQ,WAAmB,MAAsB;AAC/D,QAAM,kBAAkB,OAAO,SAAS;AACxC,QAAM,aAAa,OAAO,IAAI;AAC9B,MAAI,gBAAgB,WAAW,KAAK,WAAW,WAAW,GAAG;AAC3D,WAAO,gBAAgB,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;AAAA,EACvE;AACA,QAAM,SAAS,oBAAI,IAAoB;AACvC,aAAW,SAAS,WAAY,QAAO,IAAI,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;AAC9E,MAAI,SAAS;AACb,aAAW,SAAS,iBAAiB;AACnC,UAAM,OAAO,OAAO,IAAI,KAAK;AAC7B,QAAI,SAAS,UAAa,OAAO,GAAG;AAClC,gBAAU;AACV,aAAO,IAAI,OAAO,OAAO,CAAC;AAAA,IAC5B;AAAA,EACF;AACA,MAAI,WAAW,EAAG,QAAO;AACzB,QAAM,YAAY,SAAS,gBAAgB;AAC3C,QAAM,SAAS,SAAS,WAAW;AACnC,SAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;AAEA,SAAS,gBAAgB,WAAmB,MAAuB;AACjE,QAAM,sBAAsB,gBAAgB,SAAS;AACrD,QAAM,iBAAiB,gBAAgB,IAAI;AAC3C,MAAI,eAAe,WAAW,EAAG,QAAO;AACxC,MAAI,wBAAwB,eAAgB,QAAO;AACnD,QAAM,UAAU,eAAe,QAAQ,uBAAuB,MAAM;AACpE,SAAO,IAAI,OAAO,UAAU,OAAO,SAAS,EAAE,KAAK,mBAAmB;AACxE;AAEA,SAAS,QAAQ,OAAyB;AACxC,SAAO,CAAC,GAAG,MAAM,QAAQ,MAAM,EAAE,EAAE,SAAS,kBAAkB,CAAC,EAC5D,IAAI,CAAC,UAAU,OAAO,MAAM,CAAC,CAAC,CAAC,EAC/B,OAAO,OAAO,QAAQ;AAC3B;AAEA,SAAS,aAAa,WAAmB,MAAc,mBAAoC;AACzF,QAAM,MAAM,QAAQ,SAAS;AAC7B,QAAM,OAAO,QAAQ,IAAI;AACzB,MAAI,IAAI,WAAW,KAAK,KAAK,WAAW,EAAG,QAAO;AAClD,SAAO,KAAK;AAAA,IAAK,CAAC,aAChB,IAAI,KAAK,CAAC,WAAW;AACnB,YAAM,YAAY,KAAK,IAAI,KAAK,IAAI,QAAQ,IAAI,mBAAmB,iBAAiB;AACpF,aAAO,KAAK,IAAI,SAAS,QAAQ,KAAK;AAAA,IACxC,CAAC;AAAA,EACH;AACF;AAEO,SAAS,oBACd,UACA,OACA,UAA6D,CAAC,GAC9C;AAChB,QAAM,cAAc,iBAAiB,QAAQ;AAC7C,QAAM,YAAY,QAAQ,aAAa;AACvC,QAAM,mBAAmB,QAAQ,oBAAoB;AACrD,MAAI,OAAuB;AAAA,IACzB,UAAU;AAAA,IACV,OAAO;AAAA,IACP;AAAA,IACA,UAAU;AAAA,IACV,OAAO;AAAA,IACP,SAAS;AAAA,IACT,IAAI;AAAA,IACJ;AAAA,EACF;AACA,MAAI,YAAY,WAAW,KAAK,MAAM,WAAW,EAAG,QAAO;AAE3D,aAAW,QAAQ,OAAO;AACxB,UAAM,QAAQ,gBAAgB,aAAa,IAAI;AAC/C,UAAM,UAAU,aAAa,aAAa,MAAM,gBAAgB;AAChE,UAAM,KAAK,QAAQ,aAAa,IAAI;AACpC,UAAM,WAAW,SAAS,WAAW,MAAM;AAC3C,UAAM,QAAQ,SAAS,UAAU,IAAI;AACrC,QAAI,QAAQ,KAAK,SAAU,YAAY,CAAC,KAAK,UAAW;AACtD,aAAO,EAAE,UAAU,OAAO,aAAa,UAAU,MAAM,OAAO,SAAS,IAAI,UAAU;AAAA,IACvF;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,wBACd,OACA,QACY;AACZ,SAAO;AAAA,IACL,UAAU,MAAM;AAAA,IAChB,OAAO,MAAM;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,MACrB,GAAG;AAAA,MACH,aAAa,MAAM;AAAA,MACnB,UAAU,MAAM;AAAA,MAChB,OAAO,MAAM;AAAA,MACb,SAAS,MAAM;AAAA,MACf,IAAI,MAAM;AAAA,MACV,WAAW,MAAM;AAAA,IACnB,CAAC;AAAA,EACH;AACF;AAEA,eAAsB,aAAa,MAAkC;AACnE,QAAM,OAAO,MAAM,SAAS,MAAM,MAAM,GAAG,KAAK;AAChD,MAAI,IAAI,WAAW,EAAG,QAAO,CAAC;AAC9B,MAAI,IAAI,WAAW,GAAG,GAAG;AACvB,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,QAAI,MAAM,QAAQ,MAAM,EAAG,QAAO;AAClC,UAAM,IAAI,MAAM,GAAG,IAAI,kDAAkD;AAAA,EAC3E;AACA,MAAI,IAAI,WAAW,GAAG,GAAG;AACvB,QAAI;AACF,YAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,UAAI,SAAS,MAAM,GAAG;AACpB,mBAAW,OAAO,CAAC,QAAQ,QAAQ,YAAY,OAAO,GAAG;AACvD,gBAAM,QAAQ,OAAO,GAAG;AACxB,cAAI,MAAM,QAAQ,KAAK,EAAG,QAAO;AAAA,QACnC;AACA,eAAO,CAAC,MAAM;AAAA,MAChB;AACA,YAAM,IAAI,MAAM,GAAG,IAAI,iGAAiG;AAAA,IAC1H,SAAS,KAAK;AACZ,UAAI,CAAC,IAAI,SAAS,IAAI,EAAG,OAAM;AAAA,IACjC;AAAA,EACF;AACA,SAAO,IACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,EAChC,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEO,SAAS,YAAY,OAAoB,MAAmB,OAA4B;AAC7F,MAAI,WAAW;AACf,MAAI,KAAK,MAAO,YAAW,SAAS,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK,KAAK;AAC9E,MAAI,KAAK,KAAK;AACZ,UAAM,MAAM,IAAI,IAAI,KAAK,GAAG;AAC5B,eAAW,SAAS,OAAO,CAAC,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;AAAA,EACvD,WAAW,KAAK,UAAU,QAAW;AACnC,eAAW,SAAS,MAAM,GAAG,KAAK,KAAK;AAAA,EACzC;AACA,MAAI,SAAS,WAAW,EAAG,OAAM,IAAI,MAAM,GAAG,KAAK,sBAAsB,KAAK,UAAU,IAAI,CAAC,EAAE;AAC/F,SAAO;AACT;AAEO,SAAS,WAAW,OAAoC;AAC7D,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IAAI,MAAM,KAAK,IAAI;AAC/E;AAEO,SAAS,gBAAgB,OAA0B;AACxD,MAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,EAAG,QAAO,CAAC,MAAM,KAAK,CAAC;AAC9E,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM,QAAQ,CAAC,UAAU;AAC9B,QAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,EAAG,QAAO,CAAC,MAAM,KAAK,CAAC;AAC9E,QAAI,SAAS,KAAK,GAAG;AACnB,YAAM,OAAO,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,MAAM,IAAI;AACzF,aAAO,OAAO,CAAC,IAAI,IAAI,CAAC;AAAA,IAC1B;AACA,WAAO,CAAC;AAAA,EACV,CAAC;AACH;AAEO,SAAS,YAAY,KAA8B,MAAiC;AACzF,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,WAAW,IAAI,GAAG,CAAC;AACjC,QAAI,MAAO,QAAO;AAAA,EACpB;AACA,SAAO;AACT;AAEO,SAAS,WAAW,KAA8B,MAAmC;AAC1F,QAAM,MAAgB,CAAC;AACvB,aAAW,OAAO,KAAM,KAAI,KAAK,GAAG,gBAAgB,IAAI,GAAG,CAAC,CAAC;AAC7D,SAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAEO,SAAS,aAAa,OAA8B;AACzD,MAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,GAAG;AACxD,WAAO,CAAC,EAAE,IAAI,SAAS,MAAM,MAAM,KAAK,EAAE,CAAC;AAAA,EAC7C;AACA,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM,QAAQ,CAAC,OAAO,UAAwB;AACnD,QAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,GAAG;AACxD,aAAO,CAAC,EAAE,IAAI,OAAO,QAAQ,CAAC,IAAI,MAAM,MAAM,KAAK,EAAE,CAAC;AAAA,IACxD;AACA,QAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,YAAM,CAAC,OAAO,MAAM,IAAI;AACxB,UAAI,OAAO,UAAU,YAAY,OAAO,WAAW,UAAU;AAC3D,eAAO,CAAC,EAAE,IAAI,OAAO,MAAM,OAAO,KAAK,EAAE,CAAC;AAAA,MAC5C;AACA,aAAO,MAAM;AAAA,QAAQ,CAAC,QAAQ,gBAC5B,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,SAAS;AAAA,UACnC,GAAG;AAAA,UACH,IAAI,IAAI,GAAG,WAAW,MAAM,IAAI,OAAO,QAAQ,CAAC,IAAI,cAAc,CAAC,KAAK,IAAI;AAAA,QAC9E,EAAE;AAAA,MACJ;AAAA,IACF;AACA,QAAI,CAAC,SAAS,KAAK,EAAG,QAAO,CAAC;AAC9B,UAAM,OACJ,WAAW,MAAM,IAAI,KACrB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,QAAQ,KACzB,WAAW,MAAM,YAAY,KAC7B,WAAW,MAAM,KAAK;AACxB,QAAI,CAAC,KAAM,QAAO,CAAC;AACnB,UAAM,KACJ,WAAW,MAAM,EAAE,KACnB,WAAW,MAAM,KAAK,KACtB,WAAW,MAAM,MAAM,KACvB,WAAW,MAAM,WAAW,KAC5B,OAAO,QAAQ,CAAC;AAClB,UAAM,cAAc,MAAM,YAAY,MAAM,eAAe,MAAM,SAAS,MAAM;AAChF,UAAM,WACJ,OAAO,gBAAgB,YACnB,cACA,OAAO,gBAAgB,WACrB,cAAc,IACd;AACR,WAAO;AAAA,MACL;AAAA,QACE;AAAA,QACA;AAAA,QACA,GAAI,WAAW,MAAM,KAAK,IAAI,EAAE,OAAO,WAAW,MAAM,KAAK,EAAE,IAAI,CAAC;AAAA,QACpE,GAAI,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,IAChD,EAAE,QAAQ,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,EAAE,IAC5D,CAAC;AAAA,QACL,GAAI,aAAa,SAAY,EAAE,SAAS,IAAI,CAAC;AAAA,MAC/C;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEO,SAAS,aAAa,UAAyC;AACpE,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,SAAO,SACJ;AAAA,IAAI,CAAC,KAAK,UACT;AAAA,MACE,IAAI,QAAQ,CAAC,KAAK,IAAI,SAAS,IAAI,EAAE;AAAA,MACrC,IAAI,SAAS,WAAW,IAAI,MAAM,KAAK;AAAA,MACvC,IAAI;AAAA,IACN,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,EACd,EACC,KAAK,MAAM;AAChB;AAEO,SAAS,SAAS,OAAkD;AACzE,SAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK;AAC5E;","names":[]}

package/dist/chunk-X5YKXC6V.js

@@ -0,0 +1,211 @@
+import {
+  benchRoot,
+  runVenvScriptStdin
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/enterpriseops-gym.ts
+import { mkdir, readFile, rm, writeFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "enterpriseops-gym.json");
+var JUDGE = join(benchRoot, "scripts", "enterpriseops_gym_judge.py");
+var judgeCallSeq = 0;
+var DATASET = "ServiceNow-AI/EnterpriseOps-Gym";
+var DEFAULT_MODE = "oracle";
+var DEFAULT_DOMAIN = "itsm";
+var rowsApi = (mode, domain) => `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`;
+var enterpriseOpsTranscriptOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:json)?\s*\n([\s\S]*?)```/g)];
+    return (fences.at(-1)?.[1] ?? text).trim();
+  }
+};
+function asArray(v) {
+  return typeof v === "string" ? JSON.parse(v) : v;
+}
+function workerContract(tools) {
+  return [
+    "",
+    `You have exactly these tools available (call NO others): ${tools.join(", ")}.`,
+    "Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.",
+    "Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:",
+    '{ "calls": [ { "tool": "<tool_name>", "arguments": { ... } } ] }',
+    "Include one entry per tool call in execution order. Nothing after the closing fence."
+  ].join("\n");
+}
+function rowToTask(row, mode) {
+  const servers = asArray(row.gym_servers_config);
+  const verifiers = asArray(row.verifiers);
+  const meta = {
+    taskId: row.task_id,
+    domain: row.domain,
+    mode,
+    selectedTools: row.selected_tools,
+    servers,
+    verifiers
+  };
+  return {
+    id: row.task_id,
+    split: row.domain,
+    prompt: [row.system_prompt, "", row.user_prompt, workerContract(row.selected_tools)].join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.taskId !== "string" || !Array.isArray(md.servers) || !Array.isArray(md.verifiers) || md.verifiers.length === 0) {
+    throw new Error(`enterpriseops-gym task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectRows(rows, mode, opts) {
+  let tasks = rows.map((r) => rowToTask(r, mode));
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((t) => want.has(t.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  return tasks;
+}
+async function loadFixtures(mode, opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(
+    `[enterpriseops-gym] EOPS_FIXTURES=1 \u2014 loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`
+  );
+  const domain = opts.split;
+  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows;
+  return selectRows(scoped, mode, opts);
+}
+async function fetchRows(mode, domain, opts) {
+  const target = opts.ids ? opts.ids.length * 4 : opts.limit ?? 16;
+  const rows = [];
+  const want = opts.ids ? new Set(opts.ids) : null;
+  const page = 100;
+  const base = rowsApi(mode, domain);
+  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {
+    const res = await fetch(`${base}&offset=${offset}&length=${page}`);
+    if (!res.ok) {
+      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`);
+    }
+    const body = await res.json();
+    const got = body.rows ?? [];
+    if (got.length === 0) break;
+    for (const r of got) {
+      if (want && !want.has(r.row.task_id)) continue;
+      rows.push(r.row);
+    }
+    if (got.length < page) break;
+  }
+  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`);
+  return rows;
+}
+async function runJudge(meta, artifact) {
+  const taskJsonPath = join(
+    benchRoot,
+    ".eops-task-cache",
+    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`
+  );
+  await writeTaskCache(taskJsonPath, meta);
+  let stdout;
+  try {
+    stdout = await runVenvScriptStdin(JUDGE, ["judge", "--task-json", taskJsonPath], artifact, { cwd: benchRoot });
+  } catch (err) {
+    const e = err;
+    throw new Error(
+      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}
+Fix: (1) run the ${meta.domain} gym server \u2014 docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; (2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir (the judge seeds each task's database from seed_database_file).`
+    );
+  } finally {
+    await rm(taskJsonPath, { force: true }).catch(() => {
+    });
+  }
+  const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
+  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`);
+  if (typeof report.passes !== "number" || typeof report.total !== "number") {
+    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`);
+  }
+  const score = report.total > 0 ? report.passes / report.total : 0;
+  return {
+    resolved: report.success === true && report.total > 0 && report.passes === report.total,
+    score,
+    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total })
+  };
+}
+async function writeTaskCache(path, meta) {
+  await mkdir(join(path, ".."), { recursive: true });
+  await writeFile(
+    path,
+    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers })
+  );
+}
+async function probeServers(servers, domain) {
+  for (const s of servers) {
+    const url = `${s.mcp_server_url.replace(/\/$/, "")}/api/sql-runner`;
+    try {
+      const res = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" });
+      void res.status;
+    } catch (err) {
+      throw new Error(
+        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}
+Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`
+      );
+    }
+  }
+}
+function createEnterpriseOpsGymAdapter() {
+  const fixturesMode = process.env.EOPS_FIXTURES === "1";
+  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE;
+  return {
+    name: "enterpriseops-gym",
+    output: enterpriseOpsTranscriptOutput,
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8").catch((err) => {
+          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+        });
+        return;
+      }
+      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => []);
+      const servers = sample[0] ? readMeta(sample[0]).servers : [];
+      if (servers.length === 0) {
+        throw new Error(
+          `enterpriseops-gym preflight: no gym_servers_config to probe. Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`
+        );
+      }
+      await probeServers(servers, DEFAULT_DOMAIN);
+    },
+    async loadTasks(opts = {}) {
+      const domain = opts.split ?? DEFAULT_DOMAIN;
+      if (fixturesMode) return loadFixtures(mode, opts);
+      let rows;
+      try {
+        rows = await fetchRows(mode, domain, opts);
+      } catch (err) {
+        console.warn(
+          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`
+        );
+        return loadFixtures(mode, opts);
+      }
+      return selectRows(rows, mode, opts);
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      return runJudge(meta, artifact);
+    }
+  };
+}
+
+export {
+  enterpriseOpsTranscriptOutput,
+  createEnterpriseOpsGymAdapter
+};
+//# sourceMappingURL=chunk-X5YKXC6V.js.map

package/dist/chunk-X5YKXC6V.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/enterpriseops-gym.ts"],"sourcesContent":["/**\n * EnterpriseOps-Gym adapter (ServiceNow-AI/EnterpriseOps-Gym, Apache-2.0) —\n * stateful agentic planning + tool use in enterprise settings. Each record is an\n * enterprise-ops task (Customer Service, HR, ITSM, Calendar, Email, Drive, Teams,\n * Hybrid) handed to the agent with a domain `system_prompt`, a `selected_tools`\n * allow-list, and one or more containerized gym MCP servers (`gym_servers_config`).\n * Worker artifact = the ordered tool-call transcript the agent would issue against\n * those servers, emitted as a single fenced ```json block of\n * `{ \"calls\": [ { \"tool\": ..., \"arguments\": {...}, \"gym_name\"?: ... } ] }`.\n *\n * Judge = the benchmark's OWN deterministic state-checker. The driver replays the\n * transcript against a freshly-seeded gym server (mutating its database), then runs\n * each task's `database_state` verifier — an SQL SELECT executed via the gym\n * server's /api/sql-runner endpoint, compared to `expected_value` under\n * `comparison_type` (equals/greater_than/less_than/contains). GRADED: score =\n * (verifiers passing) / (total verifiers) = the bench's verifier_level_pass_rate;\n * binary `resolved` = ALL verifiers pass = the bench's overall_success_rate. Fully\n * deterministic — no LLM judge.\n *\n * loadTasks enumerates the real suite from the HF rows server (config = tool-set\n * MODE oracle|plus_5_tools|plus_10_tools|plus_15_tools; split = DOMAIN); a committed\n * sample (bench/fixtures/enterpriseops-gym.json) loads offline.\n *\n * Requires for a LIVE judge run: a Docker daemon with the domain gym images\n * (`docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-<domain>:latest`) up\n * on the ports in `gym_servers_config`, seeded from gym_dbs.zip. preflight + judge\n * fail loud with the exact pull/run/unzip step when a server is unreachable — never\n * a fabricated score. No portable gold artifact ships, so goldArtifact is undefined.\n */\n\nimport { mkdir, readFile, rm, writeFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'enterpriseops-gym.json')\nconst JUDGE = join(benchRoot, 'scripts', 'enterpriseops_gym_judge.py')\n\n/** Monotonic discriminator so concurrent judge calls stage distinct task-cache files. */\nlet judgeCallSeq = 0\n\nconst DATASET = 'ServiceNow-AI/EnterpriseOps-Gym'\n/** Tool-set mode = HF config; oracle ships exact tools, plus_N adds N distractors. */\nconst DEFAULT_MODE = 'oracle'\n/** Domain = HF split. */\nconst DEFAULT_DOMAIN = 'itsm'\n\nconst rowsApi = (mode: string, domain: string) =>\n  `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(DATASET)}&config=${encodeURIComponent(mode)}&split=${encodeURIComponent(domain)}`\n\ninterface GymServerConfig {\n  mcp_server_name: string\n  mcp_server_url: string\n  seed_database_file: string\n  context?: Record<string, string>\n  user_info?: Record<string, unknown>\n}\n\ninterface Verifier {\n  verifier_type: string\n  name: string\n  description?: string\n  gym_name: string\n  validation_config: { query: string; expected_value: unknown; comparison_type: string }\n}\n\ninterface EopsRow {\n  task_id: string\n  domain: string\n  system_prompt: string\n  user_prompt: string\n  selected_tools: string[]\n  restricted_tools: string[]\n  mcp_endpoint: string\n  number_of_runs: number\n  reset_database_between_runs: boolean\n  /** HF parquet stores these as JSON strings; fixtures store them as parsed arrays. */\n  gym_servers_config: string | GymServerConfig[]\n  verifiers: string | Verifier[]\n}\n\ninterface EopsMeta {\n  taskId: string\n  domain: string\n  mode: string\n  selectedTools: string[]\n  servers: GymServerConfig[]\n  verifiers: Verifier[]\n}\n\n/** Worker transcript = the last fenced ```json block, else the raw text. */\nexport const enterpriseOpsTranscriptOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:json)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction asArray<T>(v: string | T[]): T[] {\n  return typeof v === 'string' ? (JSON.parse(v) as T[]) : v\n}\n\nfunction workerContract(tools: string[]): string {\n  return [\n    '',\n    `You have exactly these tools available (call NO others): ${tools.join(', ')}.`,\n    'Plan the full sequence of tool calls that brings the enterprise database to the required final state, honoring every policy in the role above.',\n    'Emit your COMPLETE plan as the LAST thing in your reply, in a single fenced ```json block, as an object:',\n    '{ \"calls\": [ { \"tool\": \"<tool_name>\", \"arguments\": { ... } } ] }',\n    'Include one entry per tool call in execution order. Nothing after the closing fence.',\n  ].join('\\n')\n}\n\nfunction rowToTask(row: EopsRow, mode: string): BenchTask {\n  const servers = asArray<GymServerConfig>(row.gym_servers_config)\n  const verifiers = asArray<Verifier>(row.verifiers)\n  const meta: EopsMeta = {\n    taskId: row.task_id,\n    domain: row.domain,\n    mode,\n    selectedTools: row.selected_tools,\n    servers,\n    verifiers,\n  }\n  return {\n    id: row.task_id,\n    split: row.domain,\n    prompt: [row.system_prompt, '', row.user_prompt, workerContract(row.selected_tools)].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): EopsMeta {\n  const md = task.metadata\n  if (\n    !md ||\n    typeof md.taskId !== 'string' ||\n    !Array.isArray(md.servers) ||\n    !Array.isArray(md.verifiers) ||\n    (md.verifiers as unknown[]).length === 0\n  ) {\n    throw new Error(`enterpriseops-gym task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as EopsMeta\n}\n\nfunction selectRows(rows: EopsRow[], mode: string, opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map((r) => rowToTask(r, mode))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(mode: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as EopsRow[]\n  console.warn(\n    `[enterpriseops-gym] EOPS_FIXTURES=1 — loading ${rows.length} committed sample rows from ${FIXTURES} (no HF fetch)`,\n  )\n  const domain = opts.split\n  const scoped = domain ? rows.filter((r) => r.domain === domain) : rows\n  return selectRows(scoped, mode, opts)\n}\n\n/** Pull real rows from the HF rows server (paged) for one mode/domain. Throws loud on a non-OK response. */\nasync function fetchRows(mode: string, domain: string, opts: LoadOptions): Promise<EopsRow[]> {\n  const target = opts.ids ? opts.ids.length * 4 : (opts.limit ?? 16)\n  const rows: EopsRow[] = []\n  const want = opts.ids ? new Set(opts.ids) : null\n  const page = 100\n  const base = rowsApi(mode, domain)\n  for (let offset = 0; offset < 1024 && rows.length < target; offset += page) {\n    const res = await fetch(`${base}&offset=${offset}&length=${page}`)\n    if (!res.ok) {\n      throw new Error(`enterpriseops-gym rows HTTP ${res.status} (offset ${offset}): ${(await res.text()).slice(0, 200)}`)\n    }\n    const body = (await res.json()) as { rows?: Array<{ row: EopsRow }> }\n    const got = body.rows ?? []\n    if (got.length === 0) break\n    for (const r of got) {\n      if (want && !want.has(r.row.task_id)) continue\n      rows.push(r.row)\n    }\n    if (got.length < page) break\n  }\n  if (rows.length === 0) throw new Error(`enterpriseops-gym: no rows matched ${JSON.stringify(opts)} for ${mode}/${domain}`)\n  return rows\n}\n\n/**\n * Run the benchmark's own state-checker for one task over the worker's transcript.\n * The driver replays the tool calls against the live gym server, runs each\n * database_state verifier's SQL via /api/sql-runner, and reports {passes,total}.\n * Score = passes/total (verifier_level_pass_rate); resolved = all pass\n * (overall_success_rate). This is the expensive Docker-backed boundary — delegated\n * to the python driver, not reimplemented. The transcript is piped on stdin via the\n * shared stdin-aware runner (execFile's `input` is not honored async and hangs the\n * reader).\n */\nasync function runJudge(meta: EopsMeta, artifact: string): Promise<BenchScore> {\n  // Stage the full task record (servers + verifiers) so the driver has the live\n  // server URLs/contexts and the SQL it must run. The path is UNIQUE per call: the gate\n  // runs k judges for one task concurrently, so a `${taskId}.json` shared path would race.\n  const taskJsonPath = join(\n    benchRoot,\n    '.eops-task-cache',\n    `${meta.taskId}-${process.pid}-${judgeCallSeq++}.json`,\n  )\n  await writeTaskCache(taskJsonPath, meta)\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(JUDGE, ['judge', '--task-json', taskJsonPath], artifact, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(\n      `enterpriseops-gym judge failed for ${meta.taskId}: ${(e.message || String(err)).slice(0, 1500)}\\n` +\n        `Fix: (1) run the ${meta.domain} gym server — ` +\n        `docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `docker run -d -p <host>:8005 shivakrishnareddyma225/enterpriseops-gym-mcp-${meta.domain}:latest ; ` +\n        `(2) unzip gym_dbs.zip (ServiceNow/EnterpriseOps-Gym) and export EOPS_GYM_DBS_DIR to its dir ` +\n        `(the judge seeds each task's database from seed_database_file).`,\n    )\n  } finally {\n    await rm(taskJsonPath, { force: true }).catch(() => {})\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    success?: boolean\n    passes?: number\n    total?: number\n    error?: string\n  }\n  if (report.error) throw new Error(`enterpriseops-gym judge error for ${meta.taskId}: ${report.error}`)\n  if (typeof report.passes !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`enterpriseops-gym judge returned no {passes,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passes / report.total : 0\n  return {\n    resolved: report.success === true && report.total > 0 && report.passes === report.total,\n    score,\n    detail: JSON.stringify({ taskId: meta.taskId, domain: meta.domain, passes: report.passes, total: report.total }),\n  }\n}\n\nasync function writeTaskCache(path: string, meta: EopsMeta): Promise<void> {\n  await mkdir(join(path, '..'), { recursive: true })\n  await writeFile(\n    path,\n    JSON.stringify({ task_id: meta.taskId, domain: meta.domain, gym_servers_config: meta.servers, verifiers: meta.verifiers }),\n  )\n}\n\n/** Ping the configured gym servers' SQL endpoint; throw loud with the docker fix if any is unreachable. */\nasync function probeServers(servers: GymServerConfig[], domain: string): Promise<void> {\n  for (const s of servers) {\n    const url = `${s.mcp_server_url.replace(/\\/$/, '')}/api/sql-runner`\n    try {\n      // A HEAD/empty POST just proves reachability; a real query runs in judge.\n      const res = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' })\n      // Any HTTP response (even an error status) proves the server is up. Only a\n      // transport failure (refused/ENOTFOUND) means the container is not running.\n      void res.status\n    } catch (err) {\n      throw new Error(\n        `enterpriseops-gym preflight: ${s.mcp_server_name} unreachable at ${url}: ${err instanceof Error ? err.message : err}\\n` +\n          `Fix: docker pull shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `docker run -d -p <port>:<port> shivakrishnareddyma225/enterpriseops-gym-mcp-${domain}:latest ; ` +\n          `seed from gym_dbs.zip. Set EOPS_FIXTURES=1 to load sample tasks offline (judge still needs a live server).`,\n      )\n    }\n  }\n}\n\nexport function createEnterpriseOpsGymAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.EOPS_FIXTURES === '1'\n  const mode = process.env.EOPS_MODE ?? DEFAULT_MODE\n\n  return {\n    name: 'enterpriseops-gym',\n    output: enterpriseOpsTranscriptOutput,\n\n    async preflight() {\n      // Fixtures mode proves only that the sample file is readable; a live judge\n      // still requires running gym servers (and fails loud there if absent).\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`EOPS_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      // Live mode: probe the gym servers for the default domain (the suite's tasks\n      // each carry their own server config; preflight verifies reachability up\n      // front so a batch fails fast with the docker fix rather than mid-run).\n      const sample = await loadFixtures(mode, { split: DEFAULT_DOMAIN, limit: 1 }).catch(() => [])\n      const servers = sample[0] ? readMeta(sample[0]).servers : []\n      if (servers.length === 0) {\n        throw new Error(\n          `enterpriseops-gym preflight: no gym_servers_config to probe. ` +\n            `Set EOPS_FIXTURES=1 to load offline, or ensure the dataset rows carry gym_servers_config.`,\n        )\n      }\n      await probeServers(servers, DEFAULT_DOMAIN)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const domain = opts.split ?? DEFAULT_DOMAIN\n      if (fixturesMode) return loadFixtures(mode, opts)\n      let rows: EopsRow[]\n      try {\n        rows = await fetchRows(mode, domain, opts)\n      } catch (err) {\n        console.warn(\n          `[enterpriseops-gym] live rows fetch failed (${err instanceof Error ? err.message : err}); falling back to committed sample at ${FIXTURES}`,\n        )\n        return loadFixtures(mode, opts)\n      }\n      return selectRows(rows, mode, opts)\n    },\n\n    async goldArtifact() {\n      // The benchmark ships no portable per-task oracle transcript — the reference\n      // is the seeded final DB state the verifiers check, not a tool-call script.\n      // Judge correctness is proven by replaying a real solve against the live gym\n      // server, not by a synthetic gold transcript. Returns undefined (documented,\n      // not faked).\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;AA8BA,SAAS,OAAO,UAAU,IAAI,iBAAiB;AAC/C,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,IAAM,QAAQ,KAAK,WAAW,WAAW,4BAA4B;AAGrE,IAAI,eAAe;AAEnB,IAAM,UAAU;AAEhB,IAAM,eAAe;AAErB,IAAM,iBAAiB;AAEvB,IAAM,UAAU,CAAC,MAAc,WAC7B,uDAAuD,mBAAmB,OAAO,CAAC,WAAW,mBAAmB,IAAI,CAAC,UAAU,mBAAmB,MAAM,CAAC;AA2CpJ,IAAM,gCAAuD;AAAA,EAClE,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,iCAAiC,CAAC;AACnE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,SAAS,QAAW,GAAsB;AACxC,SAAO,OAAO,MAAM,WAAY,KAAK,MAAM,CAAC,IAAY;AAC1D;AAEA,SAAS,eAAe,OAAyB;AAC/C,SAAO;AAAA,IACL;AAAA,IACA,4DAA4D,MAAM,KAAK,IAAI,CAAC;AAAA,IAC5E;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAAc,MAAyB;AACxD,QAAM,UAAU,QAAyB,IAAI,kBAAkB;AAC/D,QAAM,YAAY,QAAkB,IAAI,SAAS;AACjD,QAAM,OAAiB;AAAA,IACrB,QAAQ,IAAI;AAAA,IACZ,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,eAAe,IAAI;AAAA,IACnB;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,QAAQ,CAAC,IAAI,eAAe,IAAI,IAAI,aAAa,eAAe,IAAI,cAAc,CAAC,EAAE,KAAK,IAAI;AAAA,IAC9F,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA2B;AAC3C,QAAM,KAAK,KAAK;AAChB,MACE,CAAC,MACD,OAAO,GAAG,WAAW,YACrB,CAAC,MAAM,QAAQ,GAAG,OAAO,KACzB,CAAC,MAAM,QAAQ,GAAG,SAAS,KAC1B,GAAG,UAAwB,WAAW,GACvC;AACA,UAAM,IAAI,MAAM,0BAA0B,KAAK,EAAE,wDAAmD;AAAA,EACtG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAiB,MAAc,MAAgC;AACjF,MAAI,QAAQ,KAAK,IAAI,CAAC,MAAM,UAAU,GAAG,IAAI,CAAC;AAC9C,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAEA,eAAe,aAAa,MAAc,MAAyC;AACjF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ;AAAA,IACN,sDAAiD,KAAK,MAAM,+BAA+B,QAAQ;AAAA,EACrG;AACA,QAAM,SAAS,KAAK;AACpB,QAAM,SAAS,SAAS,KAAK,OAAO,CAAC,MAAM,EAAE,WAAW,MAAM,IAAI;AAClE,SAAO,WAAW,QAAQ,MAAM,IAAI;AACtC;AAGA,eAAe,UAAU,MAAc,QAAgB,MAAuC;AAC5F,QAAM,SAAS,KAAK,MAAM,KAAK,IAAI,SAAS,IAAK,KAAK,SAAS;AAC/D,QAAM,OAAkB,CAAC;AACzB,QAAM,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,IAAI;AAC5C,QAAM,OAAO;AACb,QAAM,OAAO,QAAQ,MAAM,MAAM;AACjC,WAAS,SAAS,GAAG,SAAS,QAAQ,KAAK,SAAS,QAAQ,UAAU,MAAM;AAC1E,UAAM,MAAM,MAAM,MAAM,GAAG,IAAI,WAAW,MAAM,WAAW,IAAI,EAAE;AACjE,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,YAAY,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,IACrH;AACA,UAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,UAAM,MAAM,KAAK,QAAQ,CAAC;AAC1B,QAAI,IAAI,WAAW,EAAG;AACtB,eAAW,KAAK,KAAK;AACnB,UAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,IAAI,OAAO,EAAG;AACtC,WAAK,KAAK,EAAE,GAAG;AAAA,IACjB;AACA,QAAI,IAAI,SAAS,KAAM;AAAA,EACzB;AACA,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,CAAC,QAAQ,IAAI,IAAI,MAAM,EAAE;AACzH,SAAO;AACT;AAYA,eAAe,SAAS,MAAgB,UAAuC;AAI7E,QAAM,eAAe;AAAA,IACnB;AAAA,IACA;AAAA,IACA,GAAG,KAAK,MAAM,IAAI,QAAQ,GAAG,IAAI,cAAc;AAAA,EACjD;AACA,QAAM,eAAe,cAAc,IAAI;AACvC,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,mBAAmB,OAAO,CAAC,SAAS,eAAe,YAAY,GAAG,UAAU,EAAE,KAAK,UAAU,CAAC;AAAA,EAC/G,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI;AAAA,MACR,sCAAsC,KAAK,MAAM,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC;AAAA,mBACzE,KAAK,MAAM,+EAC6B,KAAK,MAAM,uFACM,KAAK,MAAM;AAAA,IAG5F;AAAA,EACF,UAAE;AACA,UAAM,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACxD;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAMlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,qCAAqC,KAAK,MAAM,KAAK,OAAO,KAAK,EAAE;AACrG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,uDAAuD,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC/F;AACA,QAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;AAChE,SAAO;AAAA,IACL,UAAU,OAAO,YAAY,QAAQ,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;AAAA,IAClF;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,KAAK,QAAQ,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACjH;AACF;AAEA,eAAe,eAAe,MAAc,MAA+B;AACzE,QAAM,MAAM,KAAK,MAAM,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,QAAM;AAAA,IACJ;AAAA,IACA,KAAK,UAAU,EAAE,SAAS,KAAK,QAAQ,QAAQ,KAAK,QAAQ,oBAAoB,KAAK,SAAS,WAAW,KAAK,UAAU,CAAC;AAAA,EAC3H;AACF;AAGA,eAAe,aAAa,SAA4B,QAA+B;AACrF,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,GAAG,EAAE,eAAe,QAAQ,OAAO,EAAE,CAAC;AAClD,QAAI;AAEF,YAAM,MAAM,MAAM,MAAM,KAAK,EAAE,QAAQ,QAAQ,SAAS,EAAE,gBAAgB,mBAAmB,GAAG,MAAM,KAAK,CAAC;AAG5G,WAAK,IAAI;AAAA,IACX,SAAS,KAAK;AACZ,YAAM,IAAI;AAAA,QACR,gCAAgC,EAAE,eAAe,mBAAmB,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,gEACjD,MAAM,yFACQ,MAAM;AAAA,MAEzF;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,kBAAkB;AACnD,QAAM,OAAO,QAAQ,IAAI,aAAa;AAEtC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAGhB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI,MAAM,uBAAuB,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG,EAAE;AAAA,QAC3G,CAAC;AACD;AAAA,MACF;AAIA,YAAM,SAAS,MAAM,aAAa,MAAM,EAAE,OAAO,gBAAgB,OAAO,EAAE,CAAC,EAAE,MAAM,MAAM,CAAC,CAAC;AAC3F,YAAM,UAAU,OAAO,CAAC,IAAI,SAAS,OAAO,CAAC,CAAC,EAAE,UAAU,CAAC;AAC3D,UAAI,QAAQ,WAAW,GAAG;AACxB,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,YAAM,aAAa,SAAS,cAAc;AAAA,IAC5C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,SAAS,KAAK,SAAS;AAC7B,UAAI,aAAc,QAAO,aAAa,MAAM,IAAI;AAChD,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,UAAU,MAAM,QAAQ,IAAI;AAAA,MAC3C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,+CAA+C,eAAe,QAAQ,IAAI,UAAU,GAAG,0CAA0C,QAAQ;AAAA,QAC3I;AACA,eAAO,aAAa,MAAM,IAAI;AAAA,MAChC;AACA,aAAO,WAAW,MAAM,MAAM,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,eAAe;AAMnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,SAAS,MAAM,QAAQ;AAAA,IAChC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-Y6O2OCUO.js

@@ -0,0 +1,130 @@
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringFrom
+} from "./chunk-X3BTXCJ4.js";
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/t2-ragbench.ts
+import { readFile } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "t2-ragbench.json");
+var dataFile = () => process.env.T2_RAGBENCH_DATA_FILE;
+function rowToTask(raw, index) {
+  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`);
+  const query = firstString(raw, ["question", "query", "prompt"]);
+  const goldAnswers = allStrings(raw, [
+    "program_answer",
+    "original_answer",
+    "answer",
+    "answers",
+    "reference",
+    "reference_answer",
+    "gold"
+  ]);
+  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`);
+  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`);
+  const baseContexts = contextsFrom(raw.context).length > 0 ? contextsFrom(raw.context) : contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.chunks).length > 0 ? contextsFrom(raw.chunks) : contextsFrom(raw.passages);
+  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text);
+  const contexts = table ? [...baseContexts, { id: "table", title: "Table", text: table }] : baseContexts;
+  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? "unknown";
+  const documentId = stringFrom(raw.context_id) ?? stringFrom(raw.document_id) ?? stringFrom(raw.doc_id) ?? stringFrom(raw.file_name) ?? "unknown";
+  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`;
+  const meta = {
+    benchmark: "t2-ragbench",
+    query,
+    goldAnswers,
+    contexts,
+    subset,
+    documentId
+  };
+  return {
+    id,
+    split: stringFrom(raw.split) ?? subset,
+    prompt: [
+      "Answer this T2-RAGBench text-and-table financial question.",
+      "Do the required numerical reasoning from the supplied context before giving the final value.",
+      "End with a single final line: `FINAL ANSWER: <answer>`.",
+      "",
+      `Question: ${query}`,
+      `Document: ${documentId}`,
+      `Subset: ${subset}`,
+      contexts.length > 0 ? `
+Context:
+${contextBlock(contexts)}` : void 0
+    ].filter(Boolean).join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.goldAnswers)) {
+    throw new Error(`T2-RAGBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+async function loadRows(path) {
+  const rows = await readJsonRows(path);
+  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`);
+  return rows;
+}
+async function loadFixtures(opts) {
+  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
+  return selectTasks(rows.map(rowToTask), opts, "T2-RAGBench");
+}
+function createT2RagBenchAdapter() {
+  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === "1";
+  return {
+    name: "t2-ragbench",
+    output: ragAnswerOutput,
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, "utf8");
+        return;
+      }
+      const path = dataFile();
+      if (!path) {
+        throw new Error(
+          "T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing."
+        );
+      }
+      await loadRows(path);
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const path = dataFile();
+      if (!path) throw new Error("T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks");
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, "T2-RAGBench");
+    },
+    async goldArtifact(task) {
+      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 });
+      return answerScoreToBenchScore(score, {
+        benchmark: meta.benchmark,
+        subset: meta.subset,
+        documentId: meta.documentId,
+        contextCount: meta.contexts.length
+      });
+    }
+  };
+}
+
+export {
+  createT2RagBenchAdapter
+};
+//# sourceMappingURL=chunk-Y6O2OCUO.js.map

package/dist/chunk-Y6O2OCUO.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/t2-ragbench.ts"],"sourcesContent":["/**\n * T2-RAGBench adapter.\n *\n * T2-RAGBench stresses text+table retrieval and numerical reasoning over\n * financial documents. The judge uses the shared deterministic answer scorer\n * with numeric tolerance enabled by default.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 't2-ragbench.json')\n\ninterface T2RagBenchMeta {\n  benchmark: 't2-ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  subset: string\n  documentId: string\n}\n\nconst dataFile = (): string | undefined => process.env.T2_RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'program_answer',\n    'original_answer',\n    'answer',\n    'answers',\n    'reference',\n    'reference_answer',\n    'gold',\n  ])\n  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`)\n  const baseContexts =\n    contextsFrom(raw.context).length > 0\n      ? contextsFrom(raw.context)\n      : contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.chunks).length > 0\n        ? contextsFrom(raw.chunks)\n        : contextsFrom(raw.passages)\n  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text)\n  const contexts = table\n    ? [...baseContexts, { id: 'table', title: 'Table', text: table }]\n    : baseContexts\n  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? 'unknown'\n  const documentId =\n    stringFrom(raw.context_id) ??\n    stringFrom(raw.document_id) ??\n    stringFrom(raw.doc_id) ??\n    stringFrom(raw.file_name) ??\n    'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`\n  const meta: T2RagBenchMeta = {\n    benchmark: 't2-ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    subset,\n    documentId,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? subset,\n    prompt: [\n      'Answer this T2-RAGBench text-and-table financial question.',\n      'Do the required numerical reasoning from the supplied context before giving the final value.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Document: ${documentId}`,\n      `Subset: ${subset}`,\n      contexts.length > 0 ? `\\nContext:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): T2RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`T2-RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as T2RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'T2-RAGBench')\n}\n\nexport function createT2RagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 't2-ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'T2-RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 })\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        subset: meta.subset,\n        documentId: meta.documentId,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAQA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,kBAAkB;AAW/D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,mBAAmB,KAAK,oBAAoB;AAChF,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,QAAQ,CAAC;AAC9D,QAAM,cAAc,WAAW,KAAK;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,mBAAmB,KAAK,yBAAyB;AAC7E,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,mBAAmB,KAAK,iBAAiB;AACvF,QAAM,eACJ,aAAa,IAAI,OAAO,EAAE,SAAS,IAC/B,aAAa,IAAI,OAAO,IACxB,aAAa,IAAI,QAAQ,EAAE,SAAS,IACpC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,MAAM,EAAE,SAAS,IAChC,aAAa,IAAI,MAAM,IACvB,aAAa,IAAI,QAAQ;AACjC,QAAM,QAAQ,WAAW,IAAI,KAAK,KAAK,WAAW,IAAI,UAAU;AAChE,QAAM,WAAW,QACb,CAAC,GAAG,cAAc,EAAE,IAAI,SAAS,OAAO,SAAS,MAAM,MAAM,CAAC,IAC9D;AACJ,QAAM,SAAS,WAAW,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,KAAK;AACpE,QAAM,aACJ,WAAW,IAAI,UAAU,KACzB,WAAW,IAAI,WAAW,KAC1B,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,SAAS,KACxB;AACF,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,QAAQ,KAAK,eAAe,KAAK;AACxG,QAAM,OAAuB;AAAA,IAC3B,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,aAAa,UAAU;AAAA,MACvB,WAAW,MAAM;AAAA,MACjB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAe,aAAa,QAAQ,CAAC,KAAK;AAAA,IAClE,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAiC;AACjD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,oBAAoB,KAAK,EAAE,wDAAmD;AAAA,EAChG;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,2BAA2B,IAAI,EAAE;AACxE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,uDAAkD,KAAK,MAAM,mBAAmB;AAC7F,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,aAAa;AAC7D;AAEO,SAAS,0BAA4C;AAC1D,QAAM,eAAe,QAAQ,IAAI,yBAAyB;AAE1D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,6DAA6D;AACxF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,aAAa;AAAA,IAC/E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,aAAa,EAAE,kBAAkB,KAAK,CAAC;AACxF,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,QAAQ,KAAK;AAAA,QACb,YAAY,KAAK;AAAA,QACjB,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-YCGY7UIZ.js

@@ -0,0 +1,208 @@
+import {
+  benchRoot
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/finresearchbench.ts
+import { readFile, stat } from "fs/promises";
+import { join } from "path";
+var FIXTURES = join(benchRoot, "fixtures", "finresearchbench.json");
+var dataFile = () => process.env.FINRESEARCHBENCH_DATA_FILE;
+function routerConfig() {
+  const key = process.env.TANGLE_API_KEY;
+  if (!key) throw new Error("TANGLE_API_KEY is required for FinResearchBench live LLM judging");
+  return {
+    baseUrl: process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1",
+    key,
+    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? "deepseek-v4-flash"
+  };
+}
+async function assertReadable(path, label) {
+  try {
+    await stat(path);
+  } catch (err) {
+    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
+  }
+}
+function readRecords(raw) {
+  const trimmed = raw.trim();
+  if (trimmed.startsWith("[")) return JSON.parse(trimmed);
+  return trimmed.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+}
+function assertLiveJudgeFields(records, source) {
+  const missing = records.filter((row) => !row.judge_system_prompt || !row.judge_prompt_template).map((row) => row.id);
+  if (missing.length > 0) {
+    throw new Error(
+      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(", ")}. Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.`
+    );
+  }
+}
+function rowToTask(row, fixturesMode) {
+  const referenceAnswer = row.reference_answer ?? "";
+  const referenceReport = row.reference_report ?? referenceAnswer;
+  const meta = {
+    id: row.id,
+    category: row.category ?? "unknown",
+    question: row.question,
+    referenceAnswer,
+    referenceReport,
+    logicTree: row.logic_tree ?? null,
+    rubric: row.rubric ?? null,
+    judgeSystemPrompt: row.judge_system_prompt,
+    judgePromptTemplate: row.judge_prompt_template,
+    scoring: fixturesMode ? "fixture-exact-reference" : "official-logic-tree-judge"
+  };
+  return {
+    id: row.id,
+    split: row.category,
+    prompt: [
+      "Complete this FinResearchBench financial research task.",
+      "Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.",
+      "",
+      row.question
+    ].join("\n"),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.question !== "string") {
+    throw new Error(`FinResearchBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function selectRows(rows, opts, fixturesMode) {
+  let tasks = rows.map((row) => rowToTask(row, fixturesMode));
+  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split);
+  if (opts.ids) {
+    const want = new Set(opts.ids);
+    tasks = tasks.filter((task) => want.has(task.id));
+  } else if (opts.limit !== void 0) {
+    tasks = tasks.slice(0, opts.limit);
+  }
+  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`);
+  return tasks;
+}
+async function loadFixtures(opts) {
+  const records = readRecords(await readFile(FIXTURES, "utf8"));
+  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 \u2014 loading ${records.length} adapter fixtures`);
+  return selectRows(records, opts, true);
+}
+async function loadOfficialTasks(path, opts) {
+  const records = readRecords(await readFile(path, "utf8"));
+  assertLiveJudgeFields(records, path);
+  return selectRows(records, opts, false);
+}
+function fillTemplate(meta, response) {
+  const template = meta.judgePromptTemplate;
+  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`);
+  return template.replaceAll("{question}", meta.question).replaceAll("{response}", response).replaceAll("{reference_answer}", meta.referenceAnswer).replaceAll("{reference_report}", meta.referenceReport).replaceAll("{logic_tree}", JSON.stringify(meta.logicTree, null, 2)).replaceAll("{rubric}", JSON.stringify(meta.rubric, null, 2));
+}
+function parseJudgeScore(content) {
+  const candidates = [];
+  for (const m of content.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)) candidates.push(m[1].trim());
+  for (const m of content.matchAll(/\{[\s\S]*?\}/g)) candidates.push(m[0]);
+  candidates.push(content.trim());
+  for (const candidate of candidates) {
+    let parsed;
+    try {
+      parsed = JSON.parse(candidate);
+    } catch {
+      continue;
+    }
+    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score;
+    const n = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : NaN;
+    if (!Number.isFinite(n)) continue;
+    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`);
+    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100;
+    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`);
+    return { score, raw: parsed };
+  }
+  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`);
+}
+async function runOfficialJudge(meta, response) {
+  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`);
+  const router = routerConfig();
+  const res = await fetch(`${router.baseUrl}/chat/completions`, {
+    method: "POST",
+    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
+    body: JSON.stringify({
+      model: router.model,
+      temperature: 0,
+      messages: [
+        { role: "system", content: meta.judgeSystemPrompt },
+        { role: "user", content: fillTemplate(meta, response) }
+      ]
+    })
+  });
+  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+  const body = await res.json();
+  const content = body.choices?.[0]?.message?.content;
+  if (typeof content !== "string") throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+  const { score, raw } = parseJudgeScore(content);
+  return {
+    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),
+    score,
+    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw })
+  };
+}
+function normalizeText(value) {
+  return value.toLowerCase().replace(/\s+/g, " ").trim();
+}
+function scoreFixture(meta, artifact) {
+  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport);
+  const response = normalizeText(artifact);
+  const score = answer.length > 0 && response.includes(answer) ? 1 : 0;
+  return {
+    resolved: score === 1,
+    score,
+    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category })
+  };
+}
+function createFinResearchBenchAdapter() {
+  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === "1";
+  return {
+    name: "finresearchbench",
+    async preflight() {
+      if (fixturesMode) {
+        await assertReadable(FIXTURES, "fixture file");
+        return;
+      }
+      const file = dataFile();
+      if (!file) {
+        throw new Error(
+          "FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl."
+        );
+      }
+      routerConfig();
+      await assertReadable(file, "data file");
+      await loadOfficialTasks(file, { limit: 1 });
+    },
+    async loadTasks(opts = {}) {
+      if (fixturesMode) return loadFixtures(opts);
+      const file = dataFile();
+      if (!file) throw new Error("FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows");
+      return loadOfficialTasks(file, opts);
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      return meta.referenceReport || meta.referenceAnswer || void 0;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      if (fixturesMode) return scoreFixture(meta, artifact);
+      if (artifact.trim().length === 0) {
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: "empty answer" })
+        };
+      }
+      return runOfficialJudge(meta, artifact);
+    }
+  };
+}
+
+export {
+  createFinResearchBenchAdapter
+};
+//# sourceMappingURL=chunk-YCGY7UIZ.js.map