@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/rag-shared.js

@@ -1,39 +1,246 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  normalizeAnswer,
-  parseCitations,
-  parseFinalAnswer,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringArrayFrom,
-  stringFrom,
-  tokenF1
-} from "../chunk-X3BTXCJ4.js";
-export {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  normalizeAnswer,
-  parseCitations,
-  parseFinalAnswer,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringArrayFrom,
-  stringFrom,
-  tokenF1
-};
+import { readFile } from "node:fs/promises";
+//#region src/benchmarks/rag-shared.ts
+const FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
+const ragAnswerOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return text.trim();
+} };
+function parseFinalAnswer(artifact) {
+	const lines = artifact.split(/\r?\n/);
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const line = lines[i] ?? "";
+		const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
+		if (idx !== -1) return line.slice(idx + 13).trim();
+	}
+	for (let i = lines.length - 1; i >= 0; i -= 1) {
+		const trimmed = (lines[i] ?? "").trim();
+		if (trimmed.length > 0) return trimmed;
+	}
+	return "";
+}
+function parseCitations(artifact) {
+	const urls = /* @__PURE__ */ new Set();
+	for (const match of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) urls.add(match[0].replace(/[.,;]+$/, ""));
+	return [...urls];
+}
+function normalizeAnswer(input) {
+	return input.toLowerCase().replace(/(\d),(?=\d{3}\b)/g, "$1").replace(/[^\p{L}\p{N}\s.-]+/gu, " ").split(/\s+/).filter((token) => token.length > 0).filter((token) => ![
+		"a",
+		"an",
+		"the"
+	].includes(token)).join(" ").trim();
+}
+function tokens(input) {
+	const normalized = normalizeAnswer(input);
+	return normalized.length === 0 ? [] : normalized.split(/\s+/);
+}
+function tokenF1(candidate, gold) {
+	const candidateTokens = tokens(candidate);
+	const goldTokens = tokens(gold);
+	if (candidateTokens.length === 0 || goldTokens.length === 0) return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0;
+	const counts = /* @__PURE__ */ new Map();
+	for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1);
+	let common = 0;
+	for (const token of candidateTokens) {
+		const left = counts.get(token);
+		if (left !== void 0 && left > 0) {
+			common += 1;
+			counts.set(token, left - 1);
+		}
+	}
+	if (common === 0) return 0;
+	const precision = common / candidateTokens.length;
+	const recall = common / goldTokens.length;
+	return 2 * precision * recall / (precision + recall);
+}
+function exactOrContains(candidate, gold) {
+	const normalizedCandidate = normalizeAnswer(candidate);
+	const normalizedGold = normalizeAnswer(gold);
+	if (normalizedGold.length === 0) return false;
+	if (normalizedCandidate === normalizedGold) return true;
+	const escaped = normalizedGold.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return new RegExp(`(^|\\s)${escaped}(\\s|$)`).test(normalizedCandidate);
+}
+function numbers(input) {
+	return [...input.replace(/,/g, "").matchAll(/-?\d+(?:\.\d+)?/g)].map((match) => Number(match[0])).filter(Number.isFinite);
+}
+function numericMatch(candidate, gold, relativeTolerance) {
+	const got = numbers(candidate);
+	const want = numbers(gold);
+	if (got.length === 0 || want.length === 0) return false;
+	return want.some((expected) => got.some((actual) => {
+		const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance);
+		return Math.abs(actual - expected) <= tolerance;
+	}));
+}
+function scoreAnswerArtifact(artifact, golds, options = {}) {
+	const finalAnswer = parseFinalAnswer(artifact);
+	const threshold = options.threshold ?? .72;
+	const numericTolerance = options.numericTolerance ?? .01;
+	let best = {
+		resolved: false,
+		score: 0,
+		finalAnswer,
+		bestGold: null,
+		exact: false,
+		numeric: false,
+		f1: 0,
+		threshold
+	};
+	if (finalAnswer.length === 0 || golds.length === 0) return best;
+	for (const gold of golds) {
+		const exact = exactOrContains(finalAnswer, gold);
+		const numeric = numericMatch(finalAnswer, gold, numericTolerance);
+		const f1 = tokenF1(finalAnswer, gold);
+		const resolved = exact || numeric || f1 >= threshold;
+		const score = exact || numeric ? 1 : f1;
+		if (score > best.score || resolved && !best.resolved) best = {
+			resolved,
+			score,
+			finalAnswer,
+			bestGold: gold,
+			exact,
+			numeric,
+			f1,
+			threshold
+		};
+	}
+	return best;
+}
+function answerScoreToBenchScore(score, detail) {
+	return {
+		resolved: score.resolved,
+		score: score.score,
+		detail: JSON.stringify({
+			...detail,
+			finalAnswer: score.finalAnswer,
+			bestGold: score.bestGold,
+			exact: score.exact,
+			numeric: score.numeric,
+			f1: score.f1,
+			threshold: score.threshold
+		})
+	};
+}
+async function readJsonRows(path) {
+	const raw = (await readFile(path, "utf8")).trim();
+	if (raw.length === 0) return [];
+	if (raw.startsWith("[")) {
+		const parsed = JSON.parse(raw);
+		if (Array.isArray(parsed)) return parsed;
+		throw new Error(`${path} must contain a JSON array when it starts with [`);
+	}
+	if (raw.startsWith("{")) try {
+		const parsed = JSON.parse(raw);
+		if (isObject(parsed)) {
+			for (const key of [
+				"rows",
+				"data",
+				"examples",
+				"items"
+			]) {
+				const value = parsed[key];
+				if (Array.isArray(value)) return value;
+			}
+			return [parsed];
+		}
+		throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`);
+	} catch (err) {
+		if (!raw.includes("\n")) throw err;
+	}
+	return raw.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
+}
+function selectTasks(tasks, opts, label) {
+	let selected = tasks;
+	if (opts.split) selected = selected.filter((task) => task.split === opts.split);
+	if (opts.ids) {
+		const ids = new Set(opts.ids);
+		selected = selected.filter((task) => ids.has(task.id));
+	} else if (opts.limit !== void 0) selected = selected.slice(0, opts.limit);
+	if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`);
+	return selected;
+}
+function stringFrom(value) {
+	return typeof value === "string" && value.trim().length > 0 ? value.trim() : void 0;
+}
+function stringArrayFrom(value) {
+	if (typeof value === "string" && value.trim().length > 0) return [value.trim()];
+	if (!Array.isArray(value)) return [];
+	return value.flatMap((entry) => {
+		if (typeof entry === "string" && entry.trim().length > 0) return [entry.trim()];
+		if (isObject(entry)) {
+			const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text);
+			return text ? [text] : [];
+		}
+		return [];
+	});
+}
+function firstString(row, keys) {
+	for (const key of keys) {
+		const value = stringFrom(row[key]);
+		if (value) return value;
+	}
+	return "";
+}
+function allStrings(row, keys) {
+	const out = [];
+	for (const key of keys) out.push(...stringArrayFrom(row[key]));
+	return [...new Set(out)];
+}
+function contextsFrom(value) {
+	if (typeof value === "string" && value.trim().length > 0) return [{
+		id: "ctx-1",
+		text: value.trim()
+	}];
+	if (!Array.isArray(value)) return [];
+	return value.flatMap((entry, index) => {
+		if (typeof entry === "string" && entry.trim().length > 0) return [{
+			id: `ctx-${index + 1}`,
+			text: entry.trim()
+		}];
+		if (Array.isArray(entry)) {
+			const [first, second] = entry;
+			if (typeof first === "string" && typeof second === "string") return [{
+				id: first,
+				text: second.trim()
+			}];
+			return entry.flatMap((nested, nestedIndex) => contextsFrom([nested]).map((ctx) => ({
+				...ctx,
+				id: ctx.id.startsWith("ctx-") ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id
+			})));
+		}
+		if (!isObject(entry)) return [];
+		const text = stringFrom(entry.text) ?? stringFrom(entry.context) ?? stringFrom(entry.content) ?? stringFrom(entry.passage) ?? stringFrom(entry.document) ?? stringFrom(entry.page_content) ?? stringFrom(entry.chunk);
+		if (!text) return [];
+		const id = stringFrom(entry.id) ?? stringFrom(entry.docid) ?? stringFrom(entry.doc_id) ?? stringFrom(entry.document_id) ?? `ctx-${index + 1}`;
+		const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance;
+		const relevant = typeof relevantRaw === "boolean" ? relevantRaw : typeof relevantRaw === "number" ? relevantRaw > 0 : void 0;
+		return [{
+			id,
+			text,
+			...stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {},
+			...stringFrom(entry.source) ?? stringFrom(entry.url) ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) } : {},
+			...relevant !== void 0 ? { relevant } : {}
+		}];
+	});
+}
+function contextBlock(contexts) {
+	if (contexts.length === 0) return "";
+	return contexts.map((ctx, index) => [
+		`[${index + 1}] ${ctx.title ?? ctx.id}`,
+		ctx.source ? `Source: ${ctx.source}` : void 0,
+		ctx.text
+	].filter(Boolean).join("\n")).join("\n\n");
+}
+function isObject(value) {
+	return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+//#endregion
+export { FINAL_ANSWER_SENTINEL, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, normalizeAnswer, parseCitations, parseFinalAnswer, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringArrayFrom, stringFrom, tokenF1 };
+
 //# sourceMappingURL=rag-shared.js.map

package/dist/benchmarks/rag-shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"rag-shared.js","names":[],"sources":["../../src/benchmarks/rag-shared.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport type { BenchScore, BenchTask, LoadOptions } from './types'\n\nexport const FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\nexport interface RagContext {\n  id: string\n  text: string\n  title?: string\n  source?: string\n  relevant?: boolean\n}\n\nexport interface RagAnswerScore {\n  resolved: boolean\n  score: number\n  finalAnswer: string\n  bestGold: string | null\n  exact: boolean\n  numeric: boolean\n  f1: number\n  threshold: number\n}\n\nexport const ragAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const trimmed = (lines[i] ?? '').trim()\n    if (trimmed.length > 0) return trimmed\n  }\n  return ''\n}\n\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const match of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(match[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\nexport function normalizeAnswer(input: string): string {\n  return input\n    .toLowerCase()\n    .replace(/(\\d),(?=\\d{3}\\b)/g, '$1')\n    .replace(/[^\\p{L}\\p{N}\\s.-]+/gu, ' ')\n    .split(/\\s+/)\n    .filter((token) => token.length > 0)\n    .filter((token) => !['a', 'an', 'the'].includes(token))\n    .join(' ')\n    .trim()\n}\n\nfunction tokens(input: string): string[] {\n  const normalized = normalizeAnswer(input)\n  return normalized.length === 0 ? [] : normalized.split(/\\s+/)\n}\n\nexport function tokenF1(candidate: string, gold: string): number {\n  const candidateTokens = tokens(candidate)\n  const goldTokens = tokens(gold)\n  if (candidateTokens.length === 0 || goldTokens.length === 0) {\n    return candidateTokens.length === 0 && goldTokens.length === 0 ? 1 : 0\n  }\n  const counts = new Map<string, number>()\n  for (const token of goldTokens) counts.set(token, (counts.get(token) ?? 0) + 1)\n  let common = 0\n  for (const token of candidateTokens) {\n    const left = counts.get(token)\n    if (left !== undefined && left > 0) {\n      common += 1\n      counts.set(token, left - 1)\n    }\n  }\n  if (common === 0) return 0\n  const precision = common / candidateTokens.length\n  const recall = common / goldTokens.length\n  return (2 * precision * recall) / (precision + recall)\n}\n\nfunction exactOrContains(candidate: string, gold: string): boolean {\n  const normalizedCandidate = normalizeAnswer(candidate)\n  const normalizedGold = normalizeAnswer(gold)\n  if (normalizedGold.length === 0) return false\n  if (normalizedCandidate === normalizedGold) return true\n  const escaped = normalizedGold.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n  return new RegExp(`(^|\\\\s)${escaped}(\\\\s|$)`).test(normalizedCandidate)\n}\n\nfunction numbers(input: string): number[] {\n  return [...input.replace(/,/g, '').matchAll(/-?\\d+(?:\\.\\d+)?/g)]\n    .map((match) => Number(match[0]))\n    .filter(Number.isFinite)\n}\n\nfunction numericMatch(candidate: string, gold: string, relativeTolerance: number): boolean {\n  const got = numbers(candidate)\n  const want = numbers(gold)\n  if (got.length === 0 || want.length === 0) return false\n  return want.some((expected) =>\n    got.some((actual) => {\n      const tolerance = Math.max(Math.abs(expected) * relativeTolerance, relativeTolerance)\n      return Math.abs(actual - expected) <= tolerance\n    }),\n  )\n}\n\nexport function scoreAnswerArtifact(\n  artifact: string,\n  golds: readonly string[],\n  options: { threshold?: number; numericTolerance?: number } = {},\n): RagAnswerScore {\n  const finalAnswer = parseFinalAnswer(artifact)\n  const threshold = options.threshold ?? 0.72\n  const numericTolerance = options.numericTolerance ?? 0.01\n  let best: RagAnswerScore = {\n    resolved: false,\n    score: 0,\n    finalAnswer,\n    bestGold: null,\n    exact: false,\n    numeric: false,\n    f1: 0,\n    threshold,\n  }\n  if (finalAnswer.length === 0 || golds.length === 0) return best\n\n  for (const gold of golds) {\n    const exact = exactOrContains(finalAnswer, gold)\n    const numeric = numericMatch(finalAnswer, gold, numericTolerance)\n    const f1 = tokenF1(finalAnswer, gold)\n    const resolved = exact || numeric || f1 >= threshold\n    const score = exact || numeric ? 1 : f1\n    if (score > best.score || (resolved && !best.resolved)) {\n      best = { resolved, score, finalAnswer, bestGold: gold, exact, numeric, f1, threshold }\n    }\n  }\n  return best\n}\n\nexport function answerScoreToBenchScore(\n  score: RagAnswerScore,\n  detail: Record<string, unknown>,\n): BenchScore {\n  return {\n    resolved: score.resolved,\n    score: score.score,\n    detail: JSON.stringify({\n      ...detail,\n      finalAnswer: score.finalAnswer,\n      bestGold: score.bestGold,\n      exact: score.exact,\n      numeric: score.numeric,\n      f1: score.f1,\n      threshold: score.threshold,\n    }),\n  }\n}\n\nexport async function readJsonRows(path: string): Promise<unknown[]> {\n  const raw = (await readFile(path, 'utf8')).trim()\n  if (raw.length === 0) return []\n  if (raw.startsWith('[')) {\n    const parsed = JSON.parse(raw) as unknown\n    if (Array.isArray(parsed)) return parsed\n    throw new Error(`${path} must contain a JSON array when it starts with [`)\n  }\n  if (raw.startsWith('{')) {\n    try {\n      const parsed = JSON.parse(raw) as unknown\n      if (isObject(parsed)) {\n        for (const key of ['rows', 'data', 'examples', 'items']) {\n          const value = parsed[key]\n          if (Array.isArray(value)) return value\n        }\n        return [parsed]\n      }\n      throw new Error(`${path} must contain a JSON object, JSON array, JSONL rows, or an object with rows/data/examples/items`)\n    } catch (err) {\n      if (!raw.includes('\\n')) throw err\n    }\n  }\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nexport function selectTasks(tasks: BenchTask[], opts: LoadOptions, label: string): BenchTask[] {\n  let selected = tasks\n  if (opts.split) selected = selected.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const ids = new Set(opts.ids)\n    selected = selected.filter((task) => ids.has(task.id))\n  } else if (opts.limit !== undefined) {\n    selected = selected.slice(0, opts.limit)\n  }\n  if (selected.length === 0) throw new Error(`${label}: no tasks matched ${JSON.stringify(opts)}`)\n  return selected\n}\n\nexport function stringFrom(value: unknown): string | undefined {\n  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined\n}\n\nexport function stringArrayFrom(value: unknown): string[] {\n  if (typeof value === 'string' && value.trim().length > 0) return [value.trim()]\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry) => {\n    if (typeof entry === 'string' && entry.trim().length > 0) return [entry.trim()]\n    if (isObject(entry)) {\n      const text = stringFrom(entry.answer) ?? stringFrom(entry.value) ?? stringFrom(entry.text)\n      return text ? [text] : []\n    }\n    return []\n  })\n}\n\nexport function firstString(row: Record<string, unknown>, keys: readonly string[]): string {\n  for (const key of keys) {\n    const value = stringFrom(row[key])\n    if (value) return value\n  }\n  return ''\n}\n\nexport function allStrings(row: Record<string, unknown>, keys: readonly string[]): string[] {\n  const out: string[] = []\n  for (const key of keys) out.push(...stringArrayFrom(row[key]))\n  return [...new Set(out)]\n}\n\nexport function contextsFrom(value: unknown): RagContext[] {\n  if (typeof value === 'string' && value.trim().length > 0) {\n    return [{ id: 'ctx-1', text: value.trim() }]\n  }\n  if (!Array.isArray(value)) return []\n  return value.flatMap((entry, index): RagContext[] => {\n    if (typeof entry === 'string' && entry.trim().length > 0) {\n      return [{ id: `ctx-${index + 1}`, text: entry.trim() }]\n    }\n    if (Array.isArray(entry)) {\n      const [first, second] = entry\n      if (typeof first === 'string' && typeof second === 'string') {\n        return [{ id: first, text: second.trim() }]\n      }\n      return entry.flatMap((nested, nestedIndex) =>\n        contextsFrom([nested]).map((ctx) => ({\n          ...ctx,\n          id: ctx.id.startsWith('ctx-') ? `ctx-${index + 1}-${nestedIndex + 1}` : ctx.id,\n        })),\n      )\n    }\n    if (!isObject(entry)) return []\n    const text =\n      stringFrom(entry.text) ??\n      stringFrom(entry.context) ??\n      stringFrom(entry.content) ??\n      stringFrom(entry.passage) ??\n      stringFrom(entry.document) ??\n      stringFrom(entry.page_content) ??\n      stringFrom(entry.chunk)\n    if (!text) return []\n    const id =\n      stringFrom(entry.id) ??\n      stringFrom(entry.docid) ??\n      stringFrom(entry.doc_id) ??\n      stringFrom(entry.document_id) ??\n      `ctx-${index + 1}`\n    const relevantRaw = entry.relevant ?? entry.is_relevant ?? entry.label ?? entry.relevance\n    const relevant =\n      typeof relevantRaw === 'boolean'\n        ? relevantRaw\n        : typeof relevantRaw === 'number'\n          ? relevantRaw > 0\n          : undefined\n    return [\n      {\n        id,\n        text,\n        ...(stringFrom(entry.title) ? { title: stringFrom(entry.title) } : {}),\n        ...(stringFrom(entry.source) ?? stringFrom(entry.url)\n          ? { source: stringFrom(entry.source) ?? stringFrom(entry.url) }\n          : {}),\n        ...(relevant !== undefined ? { relevant } : {}),\n      },\n    ]\n  })\n}\n\nexport function contextBlock(contexts: readonly RagContext[]): string {\n  if (contexts.length === 0) return ''\n  return contexts\n    .map((ctx, index) =>\n      [\n        `[${index + 1}] ${ctx.title ?? ctx.id}`,\n        ctx.source ? `Source: ${ctx.source}` : undefined,\n        ctx.text,\n      ]\n        .filter(Boolean)\n        .join('\\n'),\n    )\n    .join('\\n\\n')\n}\n\nexport function isObject(value: unknown): value is Record<string, unknown> {\n  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)\n}\n"],"mappings":";;AAIA,MAAa,wBAAwB;AAqBrC,MAAa,kBAAyC,EACpD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO,KAAK,KAAK;AACnB,EACF;AAEA,SAAgB,iBAAiB,UAA0B;CACzD,MAAM,QAAQ,SAAS,MAAM,OAAO;CACpC,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,MAAM,KAAK,YAAY,CAAC,CAAC,QAAQ,qBAAqB;EAC5D,IAAI,QAAQ,IAAI,OAAO,KAAK,MAAM,MAAM,EAA4B,CAAC,CAAC,KAAK;CAC7E;CACA,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;EAC7C,MAAM,WAAW,MAAM,MAAM,GAAA,CAAI,KAAK;EACtC,IAAI,QAAQ,SAAS,GAAG,OAAO;CACjC;CACA,OAAO;AACT;AAEA,SAAgB,eAAe,UAA4B;CACzD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,SAAS,SAAS,SAAS,yBAAyB,GAC7D,KAAK,IAAI,MAAM,EAAE,CAAC,QAAQ,WAAW,EAAE,CAAC;CAE1C,OAAO,CAAC,GAAG,IAAI;AACjB;AAEA,SAAgB,gBAAgB,OAAuB;CACrD,OAAO,MACJ,YAAY,CAAC,CACb,QAAQ,qBAAqB,IAAI,CAAC,CAClC,QAAQ,wBAAwB,GAAG,CAAC,CACpC,MAAM,KAAK,CAAC,CACZ,QAAQ,UAAU,MAAM,SAAS,CAAC,CAAC,CACnC,QAAQ,UAAU,CAAC;EAAC;EAAK;EAAM;CAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CACtD,KAAK,GAAG,CAAC,CACT,KAAK;AACV;AAEA,SAAS,OAAO,OAAyB;CACvC,MAAM,aAAa,gBAAgB,KAAK;CACxC,OAAO,WAAW,WAAW,IAAI,CAAC,IAAI,WAAW,MAAM,KAAK;AAC9D;AAEA,SAAgB,QAAQ,WAAmB,MAAsB;CAC/D,MAAM,kBAAkB,OAAO,SAAS;CACxC,MAAM,aAAa,OAAO,IAAI;CAC9B,IAAI,gBAAgB,WAAW,KAAK,WAAW,WAAW,GACxD,OAAO,gBAAgB,WAAW,KAAK,WAAW,WAAW,IAAI,IAAI;CAEvE,MAAM,yBAAS,IAAI,IAAoB;CACvC,KAAK,MAAM,SAAS,YAAY,OAAO,IAAI,QAAQ,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CAC9E,IAAI,SAAS;CACb,KAAK,MAAM,SAAS,iBAAiB;EACnC,MAAM,OAAO,OAAO,IAAI,KAAK;EAC7B,IAAI,SAAS,KAAA,KAAa,OAAO,GAAG;GAClC,UAAU;GACV,OAAO,IAAI,OAAO,OAAO,CAAC;EAC5B;CACF;CACA,IAAI,WAAW,GAAG,OAAO;CACzB,MAAM,YAAY,SAAS,gBAAgB;CAC3C,MAAM,SAAS,SAAS,WAAW;CACnC,OAAQ,IAAI,YAAY,UAAW,YAAY;AACjD;AAEA,SAAS,gBAAgB,WAAmB,MAAuB;CACjE,MAAM,sBAAsB,gBAAgB,SAAS;CACrD,MAAM,iBAAiB,gBAAgB,IAAI;CAC3C,IAAI,eAAe,WAAW,GAAG,OAAO;CACxC,IAAI,wBAAwB,gBAAgB,OAAO;CACnD,MAAM,UAAU,eAAe,QAAQ,uBAAuB,MAAM;CACpE,OAAO,IAAI,OAAO,UAAU,QAAQ,QAAQ,CAAC,CAAC,KAAK,mBAAmB;AACxE;AAEA,SAAS,QAAQ,OAAyB;CACxC,OAAO,CAAC,GAAG,MAAM,QAAQ,MAAM,EAAE,CAAC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAC7D,KAAK,UAAU,OAAO,MAAM,EAAE,CAAC,CAAC,CAChC,OAAO,OAAO,QAAQ;AAC3B;AAEA,SAAS,aAAa,WAAmB,MAAc,mBAAoC;CACzF,MAAM,MAAM,QAAQ,SAAS;CAC7B,MAAM,OAAO,QAAQ,IAAI;CACzB,IAAI,IAAI,WAAW,KAAK,KAAK,WAAW,GAAG,OAAO;CAClD,OAAO,KAAK,MAAM,aAChB,IAAI,MAAM,WAAW;EACnB,MAAM,YAAY,KAAK,IAAI,KAAK,IAAI,QAAQ,IAAI,mBAAmB,iBAAiB;EACpF,OAAO,KAAK,IAAI,SAAS,QAAQ,KAAK;CACxC,CAAC,CACH;AACF;AAEA,SAAgB,oBACd,UACA,OACA,UAA6D,CAAC,GAC9C;CAChB,MAAM,cAAc,iBAAiB,QAAQ;CAC7C,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,mBAAmB,QAAQ,oBAAoB;CACrD,IAAI,OAAuB;EACzB,UAAU;EACV,OAAO;EACP;EACA,UAAU;EACV,OAAO;EACP,SAAS;EACT,IAAI;EACJ;CACF;CACA,IAAI,YAAY,WAAW,KAAK,MAAM,WAAW,GAAG,OAAO;CAE3D,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,QAAQ,gBAAgB,aAAa,IAAI;EAC/C,MAAM,UAAU,aAAa,aAAa,MAAM,gBAAgB;EAChE,MAAM,KAAK,QAAQ,aAAa,IAAI;EACpC,MAAM,WAAW,SAAS,WAAW,MAAM;EAC3C,MAAM,QAAQ,SAAS,UAAU,IAAI;EACrC,IAAI,QAAQ,KAAK,SAAU,YAAY,CAAC,KAAK,UAC3C,OAAO;GAAE;GAAU;GAAO;GAAa,UAAU;GAAM;GAAO;GAAS;GAAI;EAAU;CAEzF;CACA,OAAO;AACT;AAEA,SAAgB,wBACd,OACA,QACY;CACZ,OAAO;EACL,UAAU,MAAM;EAChB,OAAO,MAAM;EACb,QAAQ,KAAK,UAAU;GACrB,GAAG;GACH,aAAa,MAAM;GACnB,UAAU,MAAM;GAChB,OAAO,MAAM;GACb,SAAS,MAAM;GACf,IAAI,MAAM;GACV,WAAW,MAAM;EACnB,CAAC;CACH;AACF;AAEA,eAAsB,aAAa,MAAkC;CACnE,MAAM,OAAO,MAAM,SAAS,MAAM,MAAM,EAAA,CAAG,KAAK;CAChD,IAAI,IAAI,WAAW,GAAG,OAAO,CAAC;CAC9B,IAAI,IAAI,WAAW,GAAG,GAAG;EACvB,MAAM,SAAS,KAAK,MAAM,GAAG;EAC7B,IAAI,MAAM,QAAQ,MAAM,GAAG,OAAO;EAClC,MAAM,IAAI,MAAM,GAAG,KAAK,iDAAiD;CAC3E;CACA,IAAI,IAAI,WAAW,GAAG,GACpB,IAAI;EACF,MAAM,SAAS,KAAK,MAAM,GAAG;EAC7B,IAAI,SAAS,MAAM,GAAG;GACpB,KAAK,MAAM,OAAO;IAAC;IAAQ;IAAQ;IAAY;GAAO,GAAG;IACvD,MAAM,QAAQ,OAAO;IACrB,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO;GACnC;GACA,OAAO,CAAC,MAAM;EAChB;EACA,MAAM,IAAI,MAAM,GAAG,KAAK,gGAAgG;CAC1H,SAAS,KAAK;EACZ,IAAI,CAAC,IAAI,SAAS,IAAI,GAAG,MAAM;CACjC;CAEF,OAAO,IACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAgB,YAAY,OAAoB,MAAmB,OAA4B;CAC7F,IAAI,WAAW;CACf,IAAI,KAAK,OAAO,WAAW,SAAS,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CAC9E,IAAI,KAAK,KAAK;EACZ,MAAM,MAAM,IAAI,IAAI,KAAK,GAAG;EAC5B,WAAW,SAAS,QAAQ,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;CACvD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,WAAW,SAAS,MAAM,GAAG,KAAK,KAAK;CAEzC,IAAI,SAAS,WAAW,GAAG,MAAM,IAAI,MAAM,GAAG,MAAM,qBAAqB,KAAK,UAAU,IAAI,GAAG;CAC/F,OAAO;AACT;AAEA,SAAgB,WAAW,OAAoC;CAC7D,OAAO,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,IAAI,MAAM,KAAK,IAAI,KAAA;AAC/E;AAEA,SAAgB,gBAAgB,OAA0B;CACxD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC;CAC9E,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,SAAS,UAAU;EAC9B,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC;EAC9E,IAAI,SAAS,KAAK,GAAG;GACnB,MAAM,OAAO,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,MAAM,IAAI;GACzF,OAAO,OAAO,CAAC,IAAI,IAAI,CAAC;EAC1B;EACA,OAAO,CAAC;CACV,CAAC;AACH;AAEA,SAAgB,YAAY,KAA8B,MAAiC;CACzF,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,WAAW,IAAI,IAAI;EACjC,IAAI,OAAO,OAAO;CACpB;CACA,OAAO;AACT;AAEA,SAAgB,WAAW,KAA8B,MAAmC;CAC1F,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,OAAO,MAAM,IAAI,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC;CAC7D,OAAO,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC;AACzB;AAEA,SAAgB,aAAa,OAA8B;CACzD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GACrD,OAAO,CAAC;EAAE,IAAI;EAAS,MAAM,MAAM,KAAK;CAAE,CAAC;CAE7C,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,SAAS,OAAO,UAAwB;EACnD,IAAI,OAAO,UAAU,YAAY,MAAM,KAAK,CAAC,CAAC,SAAS,GACrD,OAAO,CAAC;GAAE,IAAI,OAAO,QAAQ;GAAK,MAAM,MAAM,KAAK;EAAE,CAAC;EAExD,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxB,MAAM,CAAC,OAAO,UAAU;GACxB,IAAI,OAAO,UAAU,YAAY,OAAO,WAAW,UACjD,OAAO,CAAC;IAAE,IAAI;IAAO,MAAM,OAAO,KAAK;GAAE,CAAC;GAE5C,OAAO,MAAM,SAAS,QAAQ,gBAC5B,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,SAAS;IACnC,GAAG;IACH,IAAI,IAAI,GAAG,WAAW,MAAM,IAAI,OAAO,QAAQ,EAAE,GAAG,cAAc,MAAM,IAAI;GAC9E,EAAE,CACJ;EACF;EACA,IAAI,CAAC,SAAS,KAAK,GAAG,OAAO,CAAC;EAC9B,MAAM,OACJ,WAAW,MAAM,IAAI,KACrB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,OAAO,KACxB,WAAW,MAAM,QAAQ,KACzB,WAAW,MAAM,YAAY,KAC7B,WAAW,MAAM,KAAK;EACxB,IAAI,CAAC,MAAM,OAAO,CAAC;EACnB,MAAM,KACJ,WAAW,MAAM,EAAE,KACnB,WAAW,MAAM,KAAK,KACtB,WAAW,MAAM,MAAM,KACvB,WAAW,MAAM,WAAW,KAC5B,OAAO,QAAQ;EACjB,MAAM,cAAc,MAAM,YAAY,MAAM,eAAe,MAAM,SAAS,MAAM;EAChF,MAAM,WACJ,OAAO,gBAAgB,YACnB,cACA,OAAO,gBAAgB,WACrB,cAAc,IACd,KAAA;EACR,OAAO,CACL;GACE;GACA;GACA,GAAI,WAAW,MAAM,KAAK,IAAI,EAAE,OAAO,WAAW,MAAM,KAAK,EAAE,IAAI,CAAC;GACpE,GAAI,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,IAChD,EAAE,QAAQ,WAAW,MAAM,MAAM,KAAK,WAAW,MAAM,GAAG,EAAE,IAC5D,CAAC;GACL,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;EAC/C,CACF;CACF,CAAC;AACH;AAEA,SAAgB,aAAa,UAAyC;CACpE,IAAI,SAAS,WAAW,GAAG,OAAO;CAClC,OAAO,SACJ,KAAK,KAAK,UACT;EACE,IAAI,QAAQ,EAAE,IAAI,IAAI,SAAS,IAAI;EACnC,IAAI,SAAS,WAAW,IAAI,WAAW,KAAA;EACvC,IAAI;CACN,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI,CACd,CAAC,CACA,KAAK,MAAM;AAChB;AAEA,SAAgB,SAAS,OAAkD;CACzE,OAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK;AAC5E"}

package/dist/benchmarks/ragbench.d.ts

@@ -1,16 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * RAGBench-compatible adapter.
- *
- * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a
- * compatible table. Rows must carry a query and at least one reference answer.
- * Contexts, TRACe labels, and source metadata are preserved in task metadata
- * for diagnostics; the deterministic judge scores the worker's final answer
- * against the reference answer(s).
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/ragbench.d.ts
 declare function createRagBenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createRagBenchAdapter };
+//# sourceMappingURL=ragbench.d.ts.map

package/dist/benchmarks/ragbench.js

@@ -1,9 +1,128 @@
-import {
-  createRagBenchAdapter
-} from "../chunk-IFVINJ4B.js";
-import "../chunk-X3BTXCJ4.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createRagBenchAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { FINAL_ANSWER_SENTINEL, allStrings, answerScoreToBenchScore, contextBlock, contextsFrom, firstString, isObject, ragAnswerOutput, readJsonRows, scoreAnswerArtifact, selectTasks, stringFrom } from "./rag-shared.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/ragbench.ts
+/**
+* RAGBench-compatible adapter.
+*
+* Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a
+* compatible table. Rows must carry a query and at least one reference answer.
+* Contexts, TRACe labels, and source metadata are preserved in task metadata
+* for diagnostics; the deterministic judge scores the worker's final answer
+* against the reference answer(s).
+*/
+const FIXTURES = join(benchRoot, "fixtures", "ragbench.json");
+const dataFile = () => process.env.RAGBENCH_DATA_FILE;
+function rowToTask(raw, index) {
+	if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`);
+	const query = firstString(raw, [
+		"question",
+		"query",
+		"user_input",
+		"prompt"
+	]);
+	const goldAnswers = allStrings(raw, [
+		"response",
+		"responses",
+		"model_response",
+		"generated_response",
+		"reference",
+		"references",
+		"reference_answer",
+		"reference_answers",
+		"answer",
+		"answers",
+		"gold",
+		"gold_answer",
+		"ground_truth",
+		"expected_answer"
+	]);
+	if (!query) throw new Error(`RAGBench row ${index} missing question/query`);
+	if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`);
+	const contexts = contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.retrieved_contexts).length > 0 ? contextsFrom(raw.retrieved_contexts) : contextsFrom(raw.documents);
+	const traceLabels = {};
+	for (const key of [
+		"adherence",
+		"completeness",
+		"relevance",
+		"utilization",
+		"all_relevant_sentence_keys",
+		"all_utilized_sentence_keys"
+	]) if (raw[key] !== void 0) traceLabels[key] = raw[key];
+	const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset);
+	const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`;
+	const meta = {
+		benchmark: "ragbench",
+		query,
+		goldAnswers,
+		contexts,
+		...dataset ? { dataset } : {},
+		traceLabels
+	};
+	return {
+		id,
+		split: stringFrom(raw.split) ?? dataset ?? "ragbench",
+		prompt: [
+			"Answer this RAGBench question using the supplied retrieved context.",
+			"End with a single final line: `FINAL ANSWER: <answer>`.",
+			"",
+			`Question: ${query}`,
+			contexts.length > 0 ? `\nRetrieved context:\n${contextBlock(contexts)}` : void 0
+		].filter(Boolean).join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.goldAnswers)) throw new Error(`RAGBench task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+async function loadRows(path) {
+	const rows = await readJsonRows(path);
+	if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`);
+	return rows;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[ragbench] RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`);
+	return selectTasks(rows.map(rowToTask), opts, "RAGBench");
+}
+function createRagBenchAdapter() {
+	const fixturesMode = process.env.RAGBENCH_FIXTURES === "1";
+	return {
+		name: "ragbench",
+		output: ragAnswerOutput,
+		async preflight() {
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8");
+				return;
+			}
+			const path = dataFile();
+			if (!path) throw new Error("RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing.");
+			await loadRows(path);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const path = dataFile();
+			if (!path) throw new Error("RAGBENCH_DATA_FILE is required to load RAGBench tasks");
+			return selectTasks((await loadRows(path)).map(rowToTask), opts, "RAGBench");
+		},
+		async goldArtifact(task) {
+			return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			return answerScoreToBenchScore(scoreAnswerArtifact(artifact, meta.goldAnswers), {
+				benchmark: meta.benchmark,
+				dataset: meta.dataset ?? null,
+				traceLabels: meta.traceLabels,
+				contextCount: meta.contexts.length
+			});
+		}
+	};
+}
+//#endregion
+export { createRagBenchAdapter };
+
 //# sourceMappingURL=ragbench.js.map

package/dist/benchmarks/ragbench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"ragbench.js","names":[],"sources":["../../src/benchmarks/ragbench.ts"],"sourcesContent":["/**\n * RAGBench-compatible adapter.\n *\n * Live mode expects a local JSON/JSONL export from rungalileo/ragbench or a\n * compatible table. Rows must carry a query and at least one reference answer.\n * Contexts, TRACe labels, and source metadata are preserved in task metadata\n * for diagnostics; the deterministic judge scores the worker's final answer\n * against the reference answer(s).\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'ragbench.json')\n\ninterface RagBenchMeta {\n  benchmark: 'ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  dataset?: string\n  traceLabels: Record<string, unknown>\n}\n\nconst dataFile = (): string | undefined => process.env.RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'user_input', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'response',\n    'responses',\n    'model_response',\n    'generated_response',\n    'reference',\n    'references',\n    'reference_answer',\n    'reference_answers',\n    'answer',\n    'answers',\n    'gold',\n    'gold_answer',\n    'ground_truth',\n    'expected_answer',\n  ])\n  if (!query) throw new Error(`RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`RAGBench row ${index} missing reference answer`)\n  const contexts =\n    contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.retrieved_contexts).length > 0\n        ? contextsFrom(raw.retrieved_contexts)\n        : contextsFrom(raw.documents)\n  const traceLabels: Record<string, unknown> = {}\n  for (const key of [\n    'adherence',\n    'completeness',\n    'relevance',\n    'utilization',\n    'all_relevant_sentence_keys',\n    'all_utilized_sentence_keys',\n  ]) {\n    if (raw[key] !== undefined) traceLabels[key] = raw[key]\n  }\n  const dataset = stringFrom(raw.dataset) ?? stringFrom(raw.source_dataset)\n  const id = stringFrom(raw.id) ?? stringFrom(raw.example_id) ?? `ragbench-${index}`\n  const meta: RagBenchMeta = {\n    benchmark: 'ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    ...(dataset ? { dataset } : {}),\n    traceLabels,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? dataset ?? 'ragbench',\n    prompt: [\n      'Answer this RAGBench question using the supplied retrieved context.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      contexts.length > 0 ? `\\nRetrieved context:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[ragbench] RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'RAGBench')\n}\n\nexport function createRagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 'ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'RAGBENCH_DATA_FILE is required. Fix: export rungalileo/ragbench rows to JSONL and set RAGBENCH_DATA_FILE=/path/to/ragbench.jsonl, or set RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('RAGBENCH_DATA_FILE is required to load RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        dataset: meta.dataset ?? null,\n        traceLabels: meta.traceLabels,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA8BA,MAAM,WAAW,KAAK,WAAW,YAAY,eAAe;AAW5D,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;CACzD,IAAI,CAAC,SAAS,GAAG,GAAG,MAAM,IAAI,MAAM,gBAAgB,MAAM,mBAAmB;CAC7E,MAAM,QAAQ,YAAY,KAAK;EAAC;EAAY;EAAS;EAAc;CAAQ,CAAC;CAC5E,MAAM,cAAc,WAAW,KAAK;EAClC;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC;CACD,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,gBAAgB,MAAM,wBAAwB;CAC1E,IAAI,YAAY,WAAW,GAAG,MAAM,IAAI,MAAM,gBAAgB,MAAM,0BAA0B;CAC9F,MAAM,WACJ,aAAa,IAAI,QAAQ,CAAC,CAAC,SAAS,IAChC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,kBAAkB,CAAC,CAAC,SAAS,IAC5C,aAAa,IAAI,kBAAkB,IACnC,aAAa,IAAI,SAAS;CAClC,MAAM,cAAuC,CAAC;CAC9C,KAAK,MAAM,OAAO;EAChB;EACA;EACA;EACA;EACA;EACA;CACF,GACE,IAAI,IAAI,SAAS,KAAA,GAAW,YAAY,OAAO,IAAI;CAErD,MAAM,UAAU,WAAW,IAAI,OAAO,KAAK,WAAW,IAAI,cAAc;CACxE,MAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,UAAU,KAAK,YAAY;CAC3E,MAAM,OAAqB;EACzB,WAAW;EACX;EACA;EACA;EACA,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;EAC7B;CACF;CACA,OAAO;EACL;EACA,OAAO,WAAW,IAAI,KAAK,KAAK,WAAW;EAC3C,QAAQ;GACN;GACA;GACA;GACA,aAAa;GACb,SAAS,SAAS,IAAI,yBAAyB,aAAa,QAAQ,MAAM,KAAA;EAC5E,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GACtC,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,kDAAkD;CAE7F,OAAO;AACT;AAEA,eAAe,SAAS,MAAkC;CACxD,MAAM,OAAO,MAAM,aAAa,IAAI;CACpC,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,wBAAwB,MAAM;CACrE,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,4CAA4C,KAAK,OAAO,kBAAkB;CACvF,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,UAAU;AAC1D;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM;IAC/B;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,oLACF;GAEF,MAAM,SAAS,IAAI;EACrB;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,uDAAuD;GAClF,OAAO,aAAa,MAAM,SAAS,IAAI,EAAA,CAAG,IAAI,SAAS,GAAG,MAAM,UAAU;EAC5E;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,GAAG,sBAAsB,GAAG,SAAS,IAAI,CAAC,CAAC,YAAY,MAAM;EACtE;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAE1B,OAAO,wBADO,oBAAoB,UAAU,KAAK,WACd,GAAG;IACpC,WAAW,KAAK;IAChB,SAAS,KAAK,WAAW;IACzB,aAAa,KAAK;IAClB,cAAc,KAAK,SAAS;GAC9B,CAAC;EACH;CACF;AACF"}

package/dist/benchmarks/simpleqa.d.ts

@@ -1,33 +1,5 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact
- * = a single free-text final answer string (optionally preceded by a CITATIONS:
- * block). Each item ships a short gold `answer` plus metadata (topic,
- * answer_type, source urls).
- *
- * Judge is the official SimpleQA grader — an LLM classifier that maps
- * (question, gold target, predicted answer) to exactly one of:
- *   A = CORRECT       — fully contains the gold, no contradiction
- *   B = INCORRECT     — contradicts / contains a different factual value
- *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given
- * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter
- * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from
- * INCORRECT so the scorecard can separate abstention from error.
- *
- * There is no deterministic tier: SimpleQA's rubric (containment + abstention)
- * is the grader's job by design, so judge() always calls the pinned grader model
- * (temperature 0) and fails loud on unparseable grader output. The final-answer
- * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').
- *
- * Requires for a live run: the bench `.venv` with `datasets`/`requests` not
- * needed — the test set is a single public CSV fetched over HTTP — plus a
- * grader key (TANGLE_API_KEY). For offline/CI verification set
- * SIMPLEQA_FIXTURES=1 to load the committed fixtures
- * (bench/fixtures/simpleqa.json) — no network.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/simpleqa.d.ts
 /**
  * Typed seam for the research worker. The benchmark adapter scores a plain
  * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its
@@ -35,20 +7,20 @@ import '@tangle-network/agent-runtime/loops';
  * optional `CITATIONS:` block) into that string before judging.
  */
 interface ResearchTask {
-    id: string;
-    question: string;
-    /** Gold short answer — the grader's target. */
-    gold: string;
-    /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */
-    goldSources: string[];
-    /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */
-    topic: string;
-    /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */
-    answerType: string;
+  id: string;
+  question: string;
+  /** Gold short answer — the grader's target. */
+  gold: string;
+  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */
+  goldSources: string[];
+  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */
+  topic: string;
+  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */
+  answerType: string;
 }
 interface ResearchAnswer {
-    finalAnswer: string;
-    citations: string[];
+  finalAnswer: string;
+  citations: string[];
 }
 /**
  * Parse the worker artifact into the final answer string.
@@ -60,5 +32,6 @@ declare function parseFinalAnswer(artifact: string): string;
 /** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */
 declare function parseCitations(artifact: string): string[];
 declare function createSimpleQaAdapter(): BenchmarkAdapter;
-
-export { type ResearchAnswer, type ResearchTask, createSimpleQaAdapter, parseCitations, parseFinalAnswer };
+//#endregion
+export { ResearchAnswer, ResearchTask, createSimpleQaAdapter, parseCitations, parseFinalAnswer };
+//# sourceMappingURL=simpleqa.d.ts.map