@tangle-network/agent-bench 0.1.0 → 0.3.6

package/src/benchmarks/nomiracl.ts

@@ -0,0 +1,180 @@
+/**
+ * NoMIRACL adapter.
+ *
+ * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does
+ * not generate an answer here; it classifies whether the supplied passages
+ * contain enough evidence to answer the query. This directly measures false
+ * positive / false negative behavior for RAG abstention.
+ */
+
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { benchRoot } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+import {
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  selectTasks,
+  stringFrom,
+  type RagContext,
+} from './rag-shared'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'nomiracl.json')
+
+interface NoMiraclMeta {
+  benchmark: 'nomiracl'
+  query: string
+  answerable: boolean
+  language: string
+  subset: string
+  contexts: RagContext[]
+}
+
+const dataFile = (): string | undefined => process.env.NOMIRACL_DATA_FILE
+
+function passagesFrom(value: unknown, relevant: boolean): RagContext[] {
+  return contextsFrom(value).map((context) => ({ ...context, relevant }))
+}
+
+function answerableFrom(raw: Record<string, unknown>, contexts: readonly RagContext[]): boolean {
+  const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant
+  if (typeof direct === 'boolean') return direct
+  if (typeof direct === 'number') return direct > 0
+  if (typeof direct === 'string') {
+    const normalized = direct.toLowerCase()
+    if (['true', 't', 'relevant', 'answerable', 'positive', '1', 'yes'].includes(normalized)) return true
+    if (['false', 'f', 'non-relevant', 'non_relevant', 'unanswerable', 'negative', '0', 'no'].includes(normalized)) {
+      return false
+    }
+  }
+  return contexts.some((context) => context.relevant === true)
+}
+
+function rowToTask(raw: unknown, index: number): BenchTask {
+  if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`)
+  const query = firstString(raw, ['query', 'question'])
+  if (!query) throw new Error(`NoMIRACL row ${index} missing query`)
+  const labelledPassages = [
+    ...passagesFrom(raw.positive_passages, true),
+    ...passagesFrom(raw.negative_passages, false),
+  ]
+  const contexts =
+    labelledPassages.length > 0
+      ? labelledPassages
+      : contextsFrom(raw.passages).length > 0
+      ? contextsFrom(raw.passages)
+      : contextsFrom(raw.contexts).length > 0
+        ? contextsFrom(raw.contexts)
+        : contextsFrom(raw.documents)
+  const answerable = answerableFrom(raw, contexts)
+  const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? 'unknown'
+  const subset = stringFrom(raw.subset) ?? (answerable ? 'relevant' : 'non-relevant')
+  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}`
+  const meta: NoMiraclMeta = {
+    benchmark: 'nomiracl',
+    query,
+    answerable,
+    language,
+    subset,
+    contexts,
+  }
+  return {
+    id,
+    split: stringFrom(raw.split) ?? language,
+    prompt: [
+      'Classify this NoMIRACL RAG relevance case.',
+      'Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.',
+      '',
+      `Query: ${query}`,
+      contexts.length > 0 ? `\nPassages:\n${contextBlock(contexts)}` : undefined,
+    ]
+      .filter(Boolean)
+      .join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): NoMiraclMeta {
+  const md = task.metadata
+  if (!md || typeof md.answerable !== 'boolean') {
+    throw new Error(`NoMIRACL task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as NoMiraclMeta
+}
+
+function parseClassification(artifact: string): boolean | null {
+  const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
+  if (/\bunanswerable\b|\bnot answerable\b|\bno answer\b|\bnegative\b|\bfalse\b/.test(normalized)) {
+    return false
+  }
+  if (/\banswerable\b|\brelevant\b|\bpositive\b|\btrue\b|\byes\b/.test(normalized)) return true
+  return null
+}
+
+async function loadRows(path: string): Promise<unknown[]> {
+  const rows = await readJsonRows(path)
+  if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`)
+  return rows
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]
+  console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 — loading ${rows.length} adapter fixtures`)
+  return selectTasks(rows.map(rowToTask), opts, 'NoMIRACL')
+}
+
+export function createNoMiraclAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.NOMIRACL_FIXTURES === '1'
+
+  return {
+    name: 'nomiracl',
+    output: ragAnswerOutput,
+
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, 'utf8')
+        return
+      }
+      const path = dataFile()
+      if (!path) {
+        throw new Error(
+          'NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing.',
+        )
+      }
+      await loadRows(path)
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const path = dataFile()
+      if (!path) throw new Error('NOMIRACL_DATA_FILE is required to load NoMIRACL tasks')
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'NoMIRACL')
+    },
+
+    async goldArtifact(task: BenchTask) {
+      return readMeta(task).answerable ? 'ANSWERABLE' : 'UNANSWERABLE'
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const got = parseClassification(artifact)
+      const resolved = got === meta.answerable
+      return {
+        resolved,
+        score: resolved ? 1 : 0,
+        detail: JSON.stringify({
+          benchmark: meta.benchmark,
+          language: meta.language,
+          subset: meta.subset,
+          expected: meta.answerable ? 'ANSWERABLE' : 'UNANSWERABLE',
+          got: got === null ? null : got ? 'ANSWERABLE' : 'UNANSWERABLE',
+          contextCount: meta.contexts.length,
+        }),
+      }
+    },
+  }
+}

package/src/benchmarks/open-rag-bench.ts

@@ -0,0 +1,153 @@
+/**
+ * Open RAG Bench adapter.
+ *
+ * This targets Vectara-style Open RAG Bench exports over PDF-derived text,
+ * table, and image contexts. The deterministic judge scores final-answer
+ * agreement and surfaces modality/document metadata for diagnostics.
+ */
+
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { benchRoot } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringFrom,
+  type RagContext,
+} from './rag-shared'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'open-rag-bench.json')
+
+interface OpenRagBenchMeta {
+  benchmark: 'open-rag-bench'
+  query: string
+  goldAnswers: string[]
+  contexts: RagContext[]
+  documentId: string
+  modality: string
+}
+
+const dataFile = (): string | undefined => process.env.OPEN_RAG_BENCH_DATA_FILE
+
+function rowToTask(raw: unknown, index: number): BenchTask {
+  if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`)
+  const query = firstString(raw, ['question', 'query', 'prompt'])
+  const goldAnswers = allStrings(raw, ['answer', 'answers', 'reference', 'reference_answer', 'gold'])
+  if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`)
+  if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`)
+  const contexts =
+    contextsFrom(raw.contexts).length > 0
+      ? contextsFrom(raw.contexts)
+      : contextsFrom(raw.chunks).length > 0
+        ? contextsFrom(raw.chunks)
+        : contextsFrom(raw.pages)
+  const documentId =
+    stringFrom(raw.document_id) ??
+    stringFrom(raw.doc_id) ??
+    stringFrom(raw.pdf_id) ??
+    stringFrom(raw.file_name) ??
+    stringFrom(raw.filename) ??
+    'unknown'
+  const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? 'unknown'
+  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}`
+  const meta: OpenRagBenchMeta = {
+    benchmark: 'open-rag-bench',
+    query,
+    goldAnswers,
+    contexts,
+    documentId,
+    modality,
+  }
+  return {
+    id,
+    split: stringFrom(raw.split) ?? modality,
+    prompt: [
+      'Answer this Open RAG Bench PDF question using the supplied document context.',
+      'Use tables, text, and image-derived notes when present.',
+      'End with a single final line: `FINAL ANSWER: <answer>`.',
+      '',
+      `Question: ${query}`,
+      `Document: ${documentId}`,
+      `Modality: ${modality}`,
+      contexts.length > 0 ? `\nDocument context:\n${contextBlock(contexts)}` : undefined,
+    ]
+      .filter(Boolean)
+      .join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): OpenRagBenchMeta {
+  const md = task.metadata
+  if (!md || !Array.isArray(md.goldAnswers)) {
+    throw new Error(`Open RAG Bench task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as OpenRagBenchMeta
+}
+
+async function loadRows(path: string): Promise<unknown[]> {
+  const rows = await readJsonRows(path)
+  if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`)
+  return rows
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]
+  console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)
+  return selectTasks(rows.map(rowToTask), opts, 'Open RAG Bench')
+}
+
+export function createOpenRagBenchAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === '1'
+
+  return {
+    name: 'open-rag-bench',
+    output: ragAnswerOutput,
+
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, 'utf8')
+        return
+      }
+      const path = dataFile()
+      if (!path) {
+        throw new Error(
+          'OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing.',
+        )
+      }
+      await loadRows(path)
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const path = dataFile()
+      if (!path) throw new Error('OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks')
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'Open RAG Bench')
+    },
+
+    async goldArtifact(task: BenchTask) {
+      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)
+      return answerScoreToBenchScore(score, {
+        benchmark: meta.benchmark,
+        documentId: meta.documentId,
+        modality: meta.modality,
+        contextCount: meta.contexts.length,
+      })
+    },
+  }
+}

package/src/benchmarks/rag-benchmarks.test.mts

@@ -0,0 +1,138 @@
+import assert from 'node:assert/strict'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'node:test'
+import { resolveAdapter } from '../adapters'
+import { createCragAdapter } from './crag'
+import { createNoMiraclAdapter } from './nomiracl'
+import { createOpenRagBenchAdapter } from './open-rag-bench'
+import { createRagBenchAdapter } from './ragbench'
+import { FINAL_ANSWER_SENTINEL, readJsonRows } from './rag-shared'
+import { createT2RagBenchAdapter } from './t2-ragbench'
+
+async function withEnv<T>(patch: Record<string, string | undefined>, fn: () => Promise<T>): Promise<T> {
+  const old: Record<string, string | undefined> = {}
+  for (const [key, value] of Object.entries(patch)) {
+    old[key] = process.env[key]
+    if (value === undefined) delete process.env[key]
+    else process.env[key] = value
+  }
+  try {
+    return await fn()
+  } finally {
+    for (const [key, value] of Object.entries(old)) {
+      if (value === undefined) delete process.env[key]
+      else process.env[key] = value
+    }
+  }
+}
+
+test('RAG benchmark adapters are registered', () => {
+  for (const key of ['ragbench', 'crag', 'nomiracl', 'open-rag-bench', 't2-ragbench']) {
+    assert.equal(resolveAdapter(key).name, key)
+  }
+})
+
+test('RAG JSON reader accepts object-starting JSONL exports', async () => {
+  const dir = await mkdtemp(join(tmpdir(), 'rag-jsonl-'))
+  try {
+    const file = join(dir, 'rows.jsonl')
+    await writeFile(file, '{"id":"a","question":"q1"}\n{"id":"b","question":"q2"}\n')
+    assert.deepEqual(await readJsonRows(file), [
+      { id: 'a', question: 'q1' },
+      { id: 'b', question: 'q2' },
+    ])
+  } finally {
+    await rm(dir, { recursive: true, force: true })
+  }
+})
+
+test('RAGBench fixture mode loads rows and scores final answers', async () => {
+  await withEnv({ RAGBENCH_FIXTURES: '1', RAGBENCH_DATA_FILE: undefined }, async () => {
+    const adapter = createRagBenchAdapter()
+    await adapter.preflight()
+    const [task] = await adapter.loadTasks({ limit: 1 })
+    assert.equal(task.id, 'ragbench-fixture-0')
+    const gold = await adapter.goldArtifact(task)
+    assert.equal((await adapter.judge(task, gold ?? '')).resolved, true)
+    assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 90 days`)).resolved, false)
+  })
+
+  await withEnv({ RAGBENCH_FIXTURES: undefined, RAGBENCH_DATA_FILE: undefined }, async () => {
+    await assert.rejects(() => createRagBenchAdapter().preflight(), /RAGBENCH_DATA_FILE is required/)
+  })
+})
+
+test('CRAG fixture mode exposes domain slices and answer judging', async () => {
+  await withEnv({ CRAG_FIXTURES: '1', CRAG_DATA_FILE: undefined }, async () => {
+    const adapter = createCragAdapter()
+    await adapter.preflight()
+    const [task] = await adapter.loadTasks({ limit: 1 })
+    assert.equal(task.split, 'finance')
+    assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 12.4 billion dollars`)).score, 1)
+    assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 9 billion dollars`)).resolved, false)
+  })
+
+  await withEnv({ CRAG_FIXTURES: undefined, CRAG_DATA_FILE: undefined }, async () => {
+    await assert.rejects(() => createCragAdapter().preflight(), /CRAG_DATA_FILE is required/)
+  })
+})
+
+test('NoMIRACL fixture mode scores answerable and unanswerable classifications', async () => {
+  await withEnv({ NOMIRACL_FIXTURES: '1', NOMIRACL_DATA_FILE: undefined }, async () => {
+    const adapter = createNoMiraclAdapter()
+    await adapter.preflight()
+    const tasks = await adapter.loadTasks({})
+    const relevant = tasks.find((task) => task.id === 'nomiracl-fixture-relevant')
+    const nonRelevant = tasks.find((task) => task.id === 'nomiracl-fixture-non-relevant')
+    assert.ok(relevant)
+    assert.ok(nonRelevant)
+    assert.equal((await adapter.judge(relevant, 'ANSWERABLE')).score, 1)
+    assert.equal((await adapter.judge(nonRelevant, 'UNANSWERABLE')).score, 1)
+    assert.equal((await adapter.judge(nonRelevant, 'ANSWERABLE')).score, 0)
+  })
+
+  await withEnv({ NOMIRACL_FIXTURES: undefined, NOMIRACL_DATA_FILE: undefined }, async () => {
+    await assert.rejects(() => createNoMiraclAdapter().preflight(), /NOMIRACL_DATA_FILE is required/)
+  })
+})
+
+test('Open RAG Bench fixture mode keeps PDF modality metadata and scores answers', async () => {
+  await withEnv(
+    { OPEN_RAG_BENCH_FIXTURES: '1', OPEN_RAG_BENCH_DATA_FILE: undefined },
+    async () => {
+      const adapter = createOpenRagBenchAdapter()
+      await adapter.preflight()
+      const [task] = await adapter.loadTasks({ limit: 1 })
+      assert.equal(task.split, 'table')
+      assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} Hybrid BM25`)).resolved, true)
+      assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} Dense`)).resolved, false)
+    },
+  )
+
+  await withEnv(
+    { OPEN_RAG_BENCH_FIXTURES: undefined, OPEN_RAG_BENCH_DATA_FILE: undefined },
+    async () => {
+      await assert.rejects(
+        () => createOpenRagBenchAdapter().preflight(),
+        /OPEN_RAG_BENCH_DATA_FILE is required/,
+      )
+    },
+  )
+})
+
+test('T2-RAGBench fixture mode scores numeric text-table answers', async () => {
+  await withEnv({ T2_RAGBENCH_FIXTURES: '1', T2_RAGBENCH_DATA_FILE: undefined }, async () => {
+    const adapter = createT2RagBenchAdapter()
+    await adapter.preflight()
+    const [task] = await adapter.loadTasks({ limit: 1 })
+    assert.equal(task.split, 'finqa')
+    assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 12.0 percent`)).resolved, true)
+    assert.equal((await adapter.judge(task, `${FINAL_ANSWER_SENTINEL} 8 percent`)).resolved, false)
+  })
+
+  await withEnv({ T2_RAGBENCH_FIXTURES: undefined, T2_RAGBENCH_DATA_FILE: undefined }, async () => {
+    await assert.rejects(() => createT2RagBenchAdapter().preflight(), /T2_RAGBENCH_DATA_FILE is required/)
+  })
+})