@tangle-network/agent-bench 0.1.0 → 0.3.5

package/src/benchmarks/rag-shared.ts

@@ -0,0 +1,327 @@
+import { readFile } from 'node:fs/promises'
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+import type { BenchScore, BenchTask, LoadOptions } from './types'
+
+export const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'
+
+export interface RagContext {
+  id: string
+  text: string
+  title?: string
+  source?: string
+  relevant?: boolean
+}
+
+export interface RagAnswerScore {
+  resolved: boolean
+  score: number
+  finalAnswer: string
+  bestGold: string | null
+  exact: boolean
+  numeric: boolean
+  f1: number
+  threshold: number
+}
+
+export const ragAnswerOutput: OutputAdapter<string> = {
+  parse(events) {
+    let text = ''
+    for (const ev of events) {
+      const d = (ev as { data?: Record<string, unknown> })?.data
+      const t = d?.finalText ?? d?.text ?? d?.result
+      if (typeof t === 'string' && t.length > 0) text = t
+    }
+    return text.trim()
+  },
+}
+
+export function parseFinalAnswer(artifact: string): string {
+  const lines = artifact.split(/\r?\n/)
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const line = lines[i] ?? ''
+    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)
+    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()
+  }
+  for (let i = lines.length - 1; i >= 0; i -= 1) {
+    const trimmed = (lines[i] ?? '').trim()
+    if (trimmed.length > 0) return trimmed
+  }
+  return ''
+}
+
+export function parseCitations(artifact: string): string[] {
+  const urls = new Set<string>()
+  for (const match of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) {
+    urls.add(match[0].replace(/[.,;]+$/, ''))
+  }
+  return [...urls]
+}
+
+export function normalizeAnswer(input: string): string {
+  return input
+    .toLowerCase()
+    .replace(/(\d),(?=\d{3}\b)/g, '$1')
+    .replace(/[^\p{L}\p{N}\s.-]+/gu, ' ')
+    .split(/\s+/)
+    .filter((token) => token.length > 0)
+    .filter((token) => !['a', 'an', 'the'].includes(token))
+    .join(' ')
+    .trim()
+}
+
+function tokens(input: string): string[] {
+  const normalized = normalizeAnswer(input)
+  return normalized.length === 0 ? [] : normalized.split(/\s+/)
+}
+
+export function tokenF1(candidate: string, gold: string): number {
+  const candidateTokens = tokens(candidate)
+  const goldTokens = tokens(gold)
+  if (candidateTokens.length === 0 || goldTokens.length === 0) {
+    return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0
+  }
+  const counts = new Map<string, number>()
+  for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1)
+  let common = 0
+  for (const token of candidateTokens) {
+    const left = counts.get(token)
+    if (left !== undefined && left > 0) {
+      common += 1
+      counts.set(token, left - 1)
+    }
+  }
+  if (common === 0) return 0
+  const precision = common / candidateTokens.length
+  const recall = common / goldTokens.length
+  return (2 * precision * recall) / (precision + recall)
+}
+
+function exactOrContains(candidate: string, gold: string): boolean {
+  const normalizedCandidate = normalizeAnswer(candidate)
+  const normalizedGold = normalizeAnswer(gold)
+  if (normalizedGold.length === 0) return false
+  if (normalizedCandidate === normalizedGold) return true
+  const escaped = normalizedGold.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  return new RegExp(`(^|\\s)${escaped}(\\s|$)`).test(normalizedCandidate)
+}
+
+function numbers(input: string): number[] {
+  return [...input.replace(/,/g, '').matchAll(/-?\d+(?:\.\d+)?/g)]
+    .map((match) => Number(match[0]))
+    .filter(Number.isFinite)
+}
+
+function numericMatch(candidate: string, gold: string, relativeTolerance: number): boolean {
+  const got = numbers(candidate)
+  const want = numbers(gold)
+  if (got.length === 0 || want.length === 0) return false
+  return want.some((expected) =>
+    got.some((actual) => {
+      const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance)
+      return Math.abs(actual - expected) <= tolerance
+    }),
+  )
+}
+
+export function scoreAnswerArtifact(
+  artifact: string,
+  golds: readonly string[],
+  options: { threshold?: number; numericTolerance?: number } = {},
+): RagAnswerScore {
+  const finalAnswer = parseFinalAnswer(artifact)
+  const threshold = options.threshold ?? 0.72
+  const numericTolerance = options.numericTolerance ?? 0.01
+  let best: RagAnswerScore = {
+    resolved: false,
+    score: 0,
+    finalAnswer,
+    bestGold: null,
+    exact: false,
+    numeric: false,
+    f1: 0,
+    threshold,
+  }
+  if (finalAnswer.length === 0 || golds.length === 0) return best
+
+  for (const gold of golds) {
+    const exact = exactOrContains(finalAnswer, gold)
+    const numeric = numericMatch(finalAnswer, gold, numericTolerance)
+    const f1 = tokenF1(finalAnswer, gold)
+    const resolved = exact || numeric || f1 >= threshold
+    const score = exact || numeric ? 1 : f1
+    if (score > best.score || (resolved && !best.resolved)) {
+      best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold }
+    }
+  }
+  return best
+}
+
+export function answerScoreToBenchScore(
+  score: RagAnswerScore,
+  detail: Record<string, unknown>,
+): BenchScore {
+  return {
+    resolved: score.resolved,
+    score: score.score,
+    detail: JSON.stringify({
+      ...detail,
+      finalAnswer: score.finalAnswer,
+      bestGold: score.bestGold,
+      exact: score.exact,
+      numeric: score.numeric,
+      f1: score.f1,
+      threshold: score.threshold,
+    }),
+  }
+}
+
+export async function readJsonRows(path: string): Promise<unknown[]> {
+  const raw = (await readFile(path, 'utf8')).trim()
+  if (raw.length === 0) return []
+  if (raw.startsWith('[')) {
+    const parsed = JSON.parse(raw) as unknown
+    if (Array.isArray(parsed)) return parsed
+    throw new Error(`${path} must contain a JSON array when it starts with [`)
+  }
+  if (raw.startsWith('{')) {
+    try {
+      const parsed = JSON.parse(raw) as unknown
+      if (isObject(parsed)) {
+        for (const key of ['rows', 'data', 'examples', 'items']) {
+          const value = parsed[key]
+          if (Array.isArray(value)) return value
+        }
+        return [parsed]
+      }
+      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`)
+    } catch (err) {
+      if (!raw.includes('\n')) throw err
+    }
+  }
+  return raw
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .map((line) => JSON.parse(line) as unknown)
+}
+
+export function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[] {
+  let selected = tasks
+  if (opts.split) selected = selected.filter((task) => task.split === opts.split)
+  if (opts.ids) {
+    const ids = new Set(opts.ids)
+    selected = selected.filter((task) => ids.has(task.id))
+  } else if (opts.limit !== undefined) {
+    selected = selected.slice(0, opts.limit)
+  }
+  if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`)
+  return selected
+}
+
+export function stringFrom(value: unknown): string | undefined {
+  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined
+}
+
+export function stringArrayFrom(value: unknown): string[] {
+  if (typeof value === 'string' && value.trim().length > 0) return [value.trim()]
+  if (!Array.isArray(value)) return []
+  return value.flatMap((entry) => {
+    if (typeof entry === 'string' && entry.trim().length > 0) return [entry.trim()]
+    if (isObject(entry)) {
+      const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text)
+      return text ? [text] : []
+    }
+    return []
+  })
+}
+
+export function firstString(row: Record<string, unknown>, keys: readonly string[]): string {
+  for (const key of keys) {
+    const value = stringFrom(row[key])
+    if (value) return value
+  }
+  return ''
+}
+
+export function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[] {
+  const out: string[] = []
+  for (const key of keys) out.push(...stringArrayFrom(row[key]))
+  return [...new Set(out)]
+}
+
+export function contextsFrom(value: unknown): RagContext[] {
+  if (typeof value === 'string' && value.trim().length > 0) {
+    return [{ id: 'ctx-1', text: value.trim() }]
+  }
+  if (!Array.isArray(value)) return []
+  return value.flatMap((entry, index): RagContext[] => {
+    if (typeof entry === 'string' && entry.trim().length > 0) {
+      return [{ id: `ctx-${index + 1}`, text: entry.trim() }]
+    }
+    if (Array.isArray(entry)) {
+      const [first, second] = entry
+      if (typeof first === 'string' && typeof second === 'string') {
+        return [{ id: first, text: second.trim() }]
+      }
+      return entry.flatMap((nested, nestedIndex) =>
+        contextsFrom([nested]).map((ctx) => ({
+          ...ctx,
+          id: ctx.id.startsWith('ctx-') ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id,
+        })),
+      )
+    }
+    if (!isObject(entry)) return []
+    const text =
+      stringFrom(entry.text) ??
+      stringFrom(entry.context) ??
+      stringFrom(entry.content) ??
+      stringFrom(entry.passage) ??
+      stringFrom(entry.document) ??
+      stringFrom(entry.page_content) ??
+      stringFrom(entry.chunk)
+    if (!text) return []
+    const id =
+      stringFrom(entry.id) ??
+      stringFrom(entry.docid) ??
+      stringFrom(entry.doc_id) ??
+      stringFrom(entry.document_id) ??
+      `ctx-${index + 1}`
+    const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance
+    const relevant =
+      typeof relevantRaw === 'boolean'
+        ? relevantRaw
+        : typeof relevantRaw === 'number'
+          ? relevantRaw > 0
+          : undefined
+    return [
+      {
+        id,
+        text,
+        ...(stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {}),
+        ...(stringFrom(entry.source) ?? stringFrom(entry.url)
+          ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) }
+          : {}),
+        ...(relevant !== undefined ? { relevant } : {}),
+      },
+    ]
+  })
+}
+
+export function contextBlock(contexts: readonly RagContext[]): string {
+  if (contexts.length === 0) return ''
+  return contexts
+    .map((ctx, index) =>
+      [
+        `[${index + 1}] ${ctx.title ?? ctx.id}`,
+        ctx.source ? `Source: ${ctx.source}` : undefined,
+        ctx.text,
+      ]
+        .filter(Boolean)
+        .join('\n'),
+    )
+    .join('\n\n')
+}
+
+export function isObject(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
+}

package/src/benchmarks/ragbench.ts

@@ -0,0 +1,171 @@
+/**
+ * RAGBench-compatible adapter.
+ *
+ * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a
+ * compatible table. Rows must carry a query and at least one reference answer.
+ * Contexts, TRACe labels, and source metadata are preserved in task metadata
+ * for diagnostics; the deterministic judge scores the worker's final answer
+ * against the reference answer(s).
+ */
+
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { benchRoot } from './_harness'
+import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
+import {
+  FINAL_ANSWER_SENTINEL,
+  allStrings,
+  answerScoreToBenchScore,
+  contextBlock,
+  contextsFrom,
+  firstString,
+  isObject,
+  ragAnswerOutput,
+  readJsonRows,
+  scoreAnswerArtifact,
+  selectTasks,
+  stringFrom,
+  type RagContext,
+} from './rag-shared'
+
+const FIXTURES = join(benchRoot, 'fixtures', 'ragbench.json')
+
+interface RagBenchMeta {
+  benchmark: 'ragbench'
+  query: string
+  goldAnswers: string[]
+  contexts: RagContext[]
+  dataset?: string
+  traceLabels: Record<string, unknown>
+}
+
+const dataFile = (): string | undefined => process.env.RAGBENCH_DATA_FILE
+
+function rowToTask(raw: unknown, index: number): BenchTask {
+  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`)
+  const query = firstString(raw, ['question', 'query', 'user_input', 'prompt'])
+  const goldAnswers = allStrings(raw, [
+    'response',
+    'responses',
+    'model_response',
+    'generated_response',
+    'reference',
+    'references',
+    'reference_answer',
+    'reference_answers',
+    'answer',
+    'answers',
+    'gold',
+    'gold_answer',
+    'ground_truth',
+    'expected_answer',
+  ])
+  if (!query) throw new Error(`RAGBench row ${index} missing question/query`)
+  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`)
+  const contexts =
+    contextsFrom(raw.contexts).length > 0
+      ? contextsFrom(raw.contexts)
+      : contextsFrom(raw.retrieved_contexts).length > 0
+        ? contextsFrom(raw.retrieved_contexts)
+        : contextsFrom(raw.documents)
+  const traceLabels: Record<string, unknown> = {}
+  for (const key of [
+    'adherence',
+    'completeness',
+    'relevance',
+    'utilization',
+    'all_relevant_sentence_keys',
+    'all_utilized_sentence_keys',
+  ]) {
+    if (raw[key] !== undefined) traceLabels[key] = raw[key]
+  }
+  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset)
+  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`
+  const meta: RagBenchMeta = {
+    benchmark: 'ragbench',
+    query,
+    goldAnswers,
+    contexts,
+    ...(dataset ? { dataset } : {}),
+    traceLabels,
+  }
+  return {
+    id,
+    split: stringFrom(raw.split) ?? dataset ?? 'ragbench',
+    prompt: [
+      'Answer this RAGBench question using the supplied retrieved context.',
+      'End with a single final line: `FINAL ANSWER: <answer>`.',
+      '',
+      `Question: ${query}`,
+      contexts.length > 0 ? `\nRetrieved context:\n${contextBlock(contexts)}` : undefined,
+    ]
+      .filter(Boolean)
+      .join('\n'),
+    metadata: meta as unknown as Record<string, unknown>,
+  }
+}
+
+function readMeta(task: BenchTask): RagBenchMeta {
+  const md = task.metadata
+  if (!md || !Array.isArray(md.goldAnswers)) {
+    throw new Error(`RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)
+  }
+  return md as unknown as RagBenchMeta
+}
+
+async function loadRows(path: string): Promise<unknown[]> {
+  const rows = await readJsonRows(path)
+  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`)
+  return rows
+}
+
+async function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {
+  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]
+  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)
+  return selectTasks(rows.map(rowToTask), opts, 'RAGBench')
+}
+
+export function createRagBenchAdapter(): BenchmarkAdapter {
+  const fixturesMode = process.env.RAGBENCH_FIXTURES === '1'
+
+  return {
+    name: 'ragbench',
+    output: ragAnswerOutput,
+
+    async preflight() {
+      if (fixturesMode) {
+        await readFile(FIXTURES, 'utf8')
+        return
+      }
+      const path = dataFile()
+      if (!path) {
+        throw new Error(
+          'RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing.',
+        )
+      }
+      await loadRows(path)
+    },
+
+    async loadTasks(opts: LoadOptions = {}) {
+      if (fixturesMode) return loadFixtures(opts)
+      const path = dataFile()
+      if (!path) throw new Error('RAGBENCH_DATA_FILE is required to load RAGBench tasks')
+      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'RAGBench')
+    },
+
+    async goldArtifact(task: BenchTask) {
+      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`
+    },
+
+    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const meta = readMeta(task)
+      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)
+      return answerScoreToBenchScore(score, {
+        benchmark: meta.benchmark,
+        dataset: meta.dataset ?? null,
+        traceLabels: meta.traceLabels,
+        contextCount: meta.contexts.length,
+      })
+    },
+  }
+}

package/src/benchmarks/swe-bench.test.mts

@@ -0,0 +1,61 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { createSweBenchAdapter, scoreSweReport, sweEvaluationArgv } from './swe-bench'
+
+const taskId = 'django__django-12345'
+
+test('scoreSweReport preserves official resolved, unresolved, and empty-patch outcomes', () => {
+  assert.equal(
+    scoreSweReport(taskId, { submitted_ids: [taskId], completed_ids: [taskId], resolved_ids: [taskId] }).score,
+    1,
+  )
+  assert.equal(
+    scoreSweReport(taskId, { submitted_ids: [taskId], completed_ids: [taskId], unresolved_ids: [taskId] }).score,
+    0,
+  )
+  assert.equal(scoreSweReport(taskId, { submitted_ids: [taskId], empty_patch_ids: [taskId] }).score, 0)
+})
+
+test('scoreSweReport rejects evaluator failures instead of scoring them as agent failures', () => {
+  assert.throws(() => scoreSweReport(taskId, { submitted_ids: [taskId], error_ids: [taskId] }), /evaluator failed/)
+  assert.throws(() => scoreSweReport(taskId, { submitted_ids: [taskId], incomplete_ids: [taskId] }), /evaluator failed/)
+})
+
+test('scoreSweReport rejects missing, ambiguous, malformed, and mismatched outcomes', () => {
+  assert.throws(() => scoreSweReport(taskId, { submitted_ids: [taskId] }), /no unique outcome/)
+  assert.throws(
+    () => scoreSweReport(taskId, { resolved_ids: [taskId], unresolved_ids: [taskId] }),
+    /no unique outcome/,
+  )
+  assert.throws(() => scoreSweReport(taskId, { resolved_ids: taskId }), /malformed resolved_ids/)
+  assert.throws(() => scoreSweReport(taskId, { resolved_ids: ['other__repo-1'] }), /identity mismatch/)
+  assert.throws(() => scoreSweReport(taskId, { resolved_ids: [taskId] }), /lacks a completed evaluation/)
+  assert.throws(
+    () => scoreSweReport(taskId, { submitted_ids: [taskId, 'other__repo-1'], empty_patch_ids: [taskId] }),
+    /identity mismatch/,
+  )
+})
+
+test('createSweBenchAdapter accepts only positive integer evaluation timeouts', () => {
+  assert.doesNotThrow(() => createSweBenchAdapter({ timeoutMs: 1_200_000 }))
+  assert.throws(() => createSweBenchAdapter({ timeoutMs: 0 }), /positive integer/)
+  assert.throws(() => createSweBenchAdapter({ timeoutMs: 1.5 }), /positive integer/)
+})
+
+test('SWE evaluation command preserves the requested instance image', () => {
+  const argv = sweEvaluationArgv({
+    predictionsPath: '/tmp/preds.json',
+    runId: 'r364',
+    instanceId: taskId,
+    cacheLevel: 'instance',
+    namespace: 'none',
+  })
+  const cacheIndex = argv.indexOf('--cache_level')
+  const namespaceIndex = argv.indexOf('--namespace')
+  assert.equal(argv[cacheIndex + 1], 'instance')
+  assert.equal(argv[namespaceIndex + 1], 'none')
+  assert.throws(
+    () => createSweBenchAdapter({ cacheLevel: 'invalid' as 'instance' }),
+    /invalid cacheLevel/,
+  )
+})