@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-SHM6MRRF.js

@@ -1,130 +0,0 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/t2-ragbench.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "t2-ragbench.json");
-var dataFile = () => process.env.T2_RAGBENCH_DATA_FILE;
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`);
-  const query = firstString(raw, ["question", "query", "prompt"]);
-  const goldAnswers = allStrings(raw, [
-    "program_answer",
-    "original_answer",
-    "answer",
-    "answers",
-    "reference",
-    "reference_answer",
-    "gold"
-  ]);
-  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`);
-  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`);
-  const baseContexts = contextsFrom(raw.context).length > 0 ? contextsFrom(raw.context) : contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.chunks).length > 0 ? contextsFrom(raw.chunks) : contextsFrom(raw.passages);
-  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text);
-  const contexts = table ? [...baseContexts, { id: "table", title: "Table", text: table }] : baseContexts;
-  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? "unknown";
-  const documentId = stringFrom(raw.context_id) ?? stringFrom(raw.document_id) ?? stringFrom(raw.doc_id) ?? stringFrom(raw.file_name) ?? "unknown";
-  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`;
-  const meta = {
-    benchmark: "t2-ragbench",
-    query,
-    goldAnswers,
-    contexts,
-    subset,
-    documentId
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? subset,
-    prompt: [
-      "Answer this T2-RAGBench text-and-table financial question.",
-      "Do the required numerical reasoning from the supplied context before giving the final value.",
-      "End with a single final line: `FINAL ANSWER: <answer>`.",
-      "",
-      `Question: ${query}`,
-      `Document: ${documentId}`,
-      `Subset: ${subset}`,
-      contexts.length > 0 ? `
-Context:
-${contextBlock(contexts)}` : void 0
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldAnswers)) {
-    throw new Error(`T2-RAGBench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "T2-RAGBench");
-}
-function createT2RagBenchAdapter() {
-  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === "1";
-  return {
-    name: "t2-ragbench",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "T2-RAGBench");
-    },
-    async goldArtifact(task) {
-      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 });
-      return answerScoreToBenchScore(score, {
-        benchmark: meta.benchmark,
-        subset: meta.subset,
-        documentId: meta.documentId,
-        contextCount: meta.contexts.length
-      });
-    }
-  };
-}
-
-export {
-  createT2RagBenchAdapter
-};
-//# sourceMappingURL=chunk-SHM6MRRF.js.map

package/dist/chunk-SHM6MRRF.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/t2-ragbench.ts"],"sourcesContent":["/**\n * T2-RAGBench adapter.\n *\n * T2-RAGBench stresses text+table retrieval and numerical reasoning over\n * financial documents. The judge uses the shared deterministic answer scorer\n * with numeric tolerance enabled by default.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 't2-ragbench.json')\n\ninterface T2RagBenchMeta {\n  benchmark: 't2-ragbench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  subset: string\n  documentId: string\n}\n\nconst dataFile = (): string | undefined => process.env.T2_RAGBENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`T2-RAGBench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'prompt'])\n  const goldAnswers = allStrings(raw, [\n    'program_answer',\n    'original_answer',\n    'answer',\n    'answers',\n    'reference',\n    'reference_answer',\n    'gold',\n  ])\n  if (!query) throw new Error(`T2-RAGBench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`T2-RAGBench row ${index} missing answer`)\n  const baseContexts =\n    contextsFrom(raw.context).length > 0\n      ? contextsFrom(raw.context)\n      : contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.chunks).length > 0\n        ? contextsFrom(raw.chunks)\n        : contextsFrom(raw.passages)\n  const table = stringFrom(raw.table) ?? stringFrom(raw.table_text)\n  const contexts = table\n    ? [...baseContexts, { id: 'table', title: 'Table', text: table }]\n    : baseContexts\n  const subset = stringFrom(raw.subset) ?? stringFrom(raw.dataset) ?? 'unknown'\n  const documentId =\n    stringFrom(raw.context_id) ??\n    stringFrom(raw.document_id) ??\n    stringFrom(raw.doc_id) ??\n    stringFrom(raw.file_name) ??\n    'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.qid) ?? stringFrom(raw.query_id) ?? `t2-ragbench-${index}`\n  const meta: T2RagBenchMeta = {\n    benchmark: 't2-ragbench',\n    query,\n    goldAnswers,\n    contexts,\n    subset,\n    documentId,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? subset,\n    prompt: [\n      'Answer this T2-RAGBench text-and-table financial question.',\n      'Do the required numerical reasoning from the supplied context before giving the final value.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Document: ${documentId}`,\n      `Subset: ${subset}`,\n      contexts.length > 0 ? `\\nContext:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): T2RagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`T2-RAGBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as T2RagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`T2-RAGBench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[t2-ragbench] T2_RAGBENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'T2-RAGBench')\n}\n\nexport function createT2RagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.T2_RAGBENCH_FIXTURES === '1'\n\n  return {\n    name: 't2-ragbench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'T2_RAGBENCH_DATA_FILE is required. Fix: export T2-RAGBench rows to JSONL and set T2_RAGBENCH_DATA_FILE=/path/to/t2-ragbench.jsonl, or set T2_RAGBENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('T2_RAGBENCH_DATA_FILE is required to load T2-RAGBench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'T2-RAGBench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers, { numericTolerance: 0.01 })\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        subset: meta.subset,\n        documentId: meta.documentId,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAQA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,kBAAkB;AAW/D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,mBAAmB,KAAK,oBAAoB;AAChF,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,QAAQ,CAAC;AAC9D,QAAM,cAAc,WAAW,KAAK;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,mBAAmB,KAAK,yBAAyB;AAC7E,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,mBAAmB,KAAK,iBAAiB;AACvF,QAAM,eACJ,aAAa,IAAI,OAAO,EAAE,SAAS,IAC/B,aAAa,IAAI,OAAO,IACxB,aAAa,IAAI,QAAQ,EAAE,SAAS,IACpC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,MAAM,EAAE,SAAS,IAChC,aAAa,IAAI,MAAM,IACvB,aAAa,IAAI,QAAQ;AACjC,QAAM,QAAQ,WAAW,IAAI,KAAK,KAAK,WAAW,IAAI,UAAU;AAChE,QAAM,WAAW,QACb,CAAC,GAAG,cAAc,EAAE,IAAI,SAAS,OAAO,SAAS,MAAM,MAAM,CAAC,IAC9D;AACJ,QAAM,SAAS,WAAW,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,KAAK;AACpE,QAAM,aACJ,WAAW,IAAI,UAAU,KACzB,WAAW,IAAI,WAAW,KAC1B,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,SAAS,KACxB;AACF,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,QAAQ,KAAK,eAAe,KAAK;AACxG,QAAM,OAAuB;AAAA,IAC3B,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,aAAa,UAAU;AAAA,MACvB,WAAW,MAAM;AAAA,MACjB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAe,aAAa,QAAQ,CAAC,KAAK;AAAA,IAClE,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAiC;AACjD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,oBAAoB,KAAK,EAAE,wDAAmD;AAAA,EAChG;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,2BAA2B,IAAI,EAAE;AACxE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,uDAAkD,KAAK,MAAM,mBAAmB;AAC7F,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,aAAa;AAC7D;AAEO,SAAS,0BAA4C;AAC1D,QAAM,eAAe,QAAQ,IAAI,yBAAyB;AAE1D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,6DAA6D;AACxF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,aAAa;AAAA,IAC/E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,aAAa,EAAE,kBAAkB,KAAK,CAAC;AACxF,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,QAAQ,KAAK;AAAA,QACb,YAAY,KAAK;AAAA,QACjB,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-SHYIRB7I.js

@@ -1,120 +0,0 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/open-rag-bench.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "open-rag-bench.json");
-var dataFile = () => process.env.OPEN_RAG_BENCH_DATA_FILE;
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`);
-  const query = firstString(raw, ["question", "query", "prompt"]);
-  const goldAnswers = allStrings(raw, ["answer", "answers", "reference", "reference_answer", "gold"]);
-  if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`);
-  if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`);
-  const contexts = contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.chunks).length > 0 ? contextsFrom(raw.chunks) : contextsFrom(raw.pages);
-  const documentId = stringFrom(raw.document_id) ?? stringFrom(raw.doc_id) ?? stringFrom(raw.pdf_id) ?? stringFrom(raw.file_name) ?? stringFrom(raw.filename) ?? "unknown";
-  const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? "unknown";
-  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}`;
-  const meta = {
-    benchmark: "open-rag-bench",
-    query,
-    goldAnswers,
-    contexts,
-    documentId,
-    modality
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? modality,
-    prompt: [
-      "Answer this Open RAG Bench PDF question using the supplied document context.",
-      "Use tables, text, and image-derived notes when present.",
-      "End with a single final line: `FINAL ANSWER: <answer>`.",
-      "",
-      `Question: ${query}`,
-      `Document: ${documentId}`,
-      `Modality: ${modality}`,
-      contexts.length > 0 ? `
-Document context:
-${contextBlock(contexts)}` : void 0
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldAnswers)) {
-    throw new Error(`Open RAG Bench task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "Open RAG Bench");
-}
-function createOpenRagBenchAdapter() {
-  const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === "1";
-  return {
-    name: "open-rag-bench",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "Open RAG Bench");
-    },
-    async goldArtifact(task) {
-      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const score = scoreAnswerArtifact(artifact, meta.goldAnswers);
-      return answerScoreToBenchScore(score, {
-        benchmark: meta.benchmark,
-        documentId: meta.documentId,
-        modality: meta.modality,
-        contextCount: meta.contexts.length
-      });
-    }
-  };
-}
-
-export {
-  createOpenRagBenchAdapter
-};
-//# sourceMappingURL=chunk-SHYIRB7I.js.map

package/dist/chunk-SHYIRB7I.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/open-rag-bench.ts"],"sourcesContent":["/**\n * Open RAG Bench adapter.\n *\n * This targets Vectara-style Open RAG Bench exports over PDF-derived text,\n * table, and image contexts. The deterministic judge scores final-answer\n * agreement and surfaces modality/document metadata for diagnostics.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'open-rag-bench.json')\n\ninterface OpenRagBenchMeta {\n  benchmark: 'open-rag-bench'\n  query: string\n  goldAnswers: string[]\n  contexts: RagContext[]\n  documentId: string\n  modality: string\n}\n\nconst dataFile = (): string | undefined => process.env.OPEN_RAG_BENCH_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`Open RAG Bench row ${index} must be an object`)\n  const query = firstString(raw, ['question', 'query', 'prompt'])\n  const goldAnswers = allStrings(raw, ['answer', 'answers', 'reference', 'reference_answer', 'gold'])\n  if (!query) throw new Error(`Open RAG Bench row ${index} missing question/query`)\n  if (goldAnswers.length === 0) throw new Error(`Open RAG Bench row ${index} missing answer`)\n  const contexts =\n    contextsFrom(raw.contexts).length > 0\n      ? contextsFrom(raw.contexts)\n      : contextsFrom(raw.chunks).length > 0\n        ? contextsFrom(raw.chunks)\n        : contextsFrom(raw.pages)\n  const documentId =\n    stringFrom(raw.document_id) ??\n    stringFrom(raw.doc_id) ??\n    stringFrom(raw.pdf_id) ??\n    stringFrom(raw.file_name) ??\n    stringFrom(raw.filename) ??\n    'unknown'\n  const modality = stringFrom(raw.modality) ?? stringFrom(raw.answer_modality) ?? 'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `open-rag-bench-${index}`\n  const meta: OpenRagBenchMeta = {\n    benchmark: 'open-rag-bench',\n    query,\n    goldAnswers,\n    contexts,\n    documentId,\n    modality,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? modality,\n    prompt: [\n      'Answer this Open RAG Bench PDF question using the supplied document context.',\n      'Use tables, text, and image-derived notes when present.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Document: ${documentId}`,\n      `Modality: ${modality}`,\n      contexts.length > 0 ? `\\nDocument context:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): OpenRagBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`Open RAG Bench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as OpenRagBenchMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`Open RAG Bench: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[open-rag-bench] OPEN_RAG_BENCH_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'Open RAG Bench')\n}\n\nexport function createOpenRagBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.OPEN_RAG_BENCH_FIXTURES === '1'\n\n  return {\n    name: 'open-rag-bench',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'OPEN_RAG_BENCH_DATA_FILE is required. Fix: export vectara/open-rag-bench rows to JSONL and set OPEN_RAG_BENCH_DATA_FILE=/path/to/open-rag-bench.jsonl, or set OPEN_RAG_BENCH_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('OPEN_RAG_BENCH_DATA_FILE is required to load Open RAG Bench tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'Open RAG Bench')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        documentId: meta.documentId,\n        modality: meta.modality,\n        contextCount: meta.contexts.length,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAQA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAmBrB,IAAM,WAAW,KAAK,WAAW,YAAY,qBAAqB;AAWlE,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,sBAAsB,KAAK,oBAAoB;AACnF,QAAM,QAAQ,YAAY,KAAK,CAAC,YAAY,SAAS,QAAQ,CAAC;AAC9D,QAAM,cAAc,WAAW,KAAK,CAAC,UAAU,WAAW,aAAa,oBAAoB,MAAM,CAAC;AAClG,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,sBAAsB,KAAK,yBAAyB;AAChF,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,sBAAsB,KAAK,iBAAiB;AAC1F,QAAM,WACJ,aAAa,IAAI,QAAQ,EAAE,SAAS,IAChC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,MAAM,EAAE,SAAS,IAChC,aAAa,IAAI,MAAM,IACvB,aAAa,IAAI,KAAK;AAC9B,QAAM,aACJ,WAAW,IAAI,WAAW,KAC1B,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,MAAM,KACrB,WAAW,IAAI,SAAS,KACxB,WAAW,IAAI,QAAQ,KACvB;AACF,QAAM,WAAW,WAAW,IAAI,QAAQ,KAAK,WAAW,IAAI,eAAe,KAAK;AAChF,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,kBAAkB,KAAK;AACpF,QAAM,OAAyB;AAAA,IAC7B,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,aAAa,UAAU;AAAA,MACvB,aAAa,QAAQ;AAAA,MACrB,SAAS,SAAS,IAAI;AAAA;AAAA,EAAwB,aAAa,QAAQ,CAAC,KAAK;AAAA,IAC3E,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAmC;AACnD,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,uBAAuB,KAAK,EAAE,wDAAmD;AAAA,EACnG;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,8BAA8B,IAAI,EAAE;AAC3E,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,6DAAwD,KAAK,MAAM,mBAAmB;AACnG,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,gBAAgB;AAChE;AAEO,SAAS,4BAA8C;AAC5D,QAAM,eAAe,QAAQ,IAAI,4BAA4B;AAE7D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,mEAAmE;AAC9F,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,gBAAgB;AAAA,IAClF;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,WAAW;AAC5D,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,YAAY,KAAK;AAAA,QACjB,UAAU,KAAK;AAAA,QACf,cAAc,KAAK,SAAS;AAAA,MAC9B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-SVR2LKYI.js

@@ -1,116 +0,0 @@
-import {
-  FINAL_ANSWER_SENTINEL,
-  allStrings,
-  answerScoreToBenchScore,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  scoreAnswerArtifact,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/crag.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "crag.json");
-var dataFile = () => process.env.CRAG_DATA_FILE;
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`CRAG row ${index} must be an object`);
-  const query = firstString(raw, ["query", "question", "prompt"]);
-  const goldAnswers = allStrings(raw, ["answer", "answers", "gold", "gold_answer", "expected_answer"]);
-  if (!query) throw new Error(`CRAG row ${index} missing query`);
-  if (goldAnswers.length === 0) throw new Error(`CRAG row ${index} missing gold answer`);
-  const domain = stringFrom(raw.domain) ?? "unknown";
-  const questionType = stringFrom(raw.question_type) ?? stringFrom(raw.questionType) ?? "unknown";
-  const dynamism = stringFrom(raw.static_or_dynamic) ?? stringFrom(raw.dynamism) ?? "unknown";
-  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `crag-${index}`;
-  const meta = {
-    benchmark: "crag",
-    query,
-    goldAnswers,
-    domain,
-    questionType,
-    dynamism
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? domain,
-    prompt: [
-      "Answer this CRAG factual question.",
-      "Return a concise answer and do not guess when the evidence is insufficient.",
-      "End with a single final line: `FINAL ANSWER: <answer>`.",
-      "",
-      `Question: ${query}`,
-      `Domain: ${domain}`,
-      `Question type: ${questionType}`,
-      `Dynamism: ${dynamism}`
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldAnswers)) {
-    throw new Error(`CRAG task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`CRAG: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[crag] CRAG_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "CRAG");
-}
-function createCragAdapter() {
-  const fixturesMode = process.env.CRAG_FIXTURES === "1";
-  return {
-    name: "crag",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "CRAG_DATA_FILE is required. Fix: export facebookresearch/CRAG rows to JSONL and set CRAG_DATA_FILE=/path/to/crag.jsonl, or set CRAG_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("CRAG_DATA_FILE is required to load CRAG tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "CRAG");
-    },
-    async goldArtifact(task) {
-      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ""}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const score = scoreAnswerArtifact(artifact, meta.goldAnswers);
-      return answerScoreToBenchScore(score, {
-        benchmark: meta.benchmark,
-        domain: meta.domain,
-        questionType: meta.questionType,
-        dynamism: meta.dynamism
-      });
-    }
-  };
-}
-
-export {
-  createCragAdapter
-};
-//# sourceMappingURL=chunk-SVR2LKYI.js.map

package/dist/chunk-SVR2LKYI.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/crag.ts"],"sourcesContent":["/**\n * CRAG adapter (Comprehensive RAG Benchmark).\n *\n * Live mode expects an official or compatible CRAG JSON/JSONL export. The\n * adapter preserves CRAG domain/type/dynamism tags in metadata and scores final\n * answers deterministically against the provided gold answer list.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  FINAL_ANSWER_SENTINEL,\n  allStrings,\n  answerScoreToBenchScore,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  scoreAnswerArtifact,\n  selectTasks,\n  stringFrom,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'crag.json')\n\ninterface CragMeta {\n  benchmark: 'crag'\n  query: string\n  goldAnswers: string[]\n  domain: string\n  questionType: string\n  dynamism: string\n}\n\nconst dataFile = (): string | undefined => process.env.CRAG_DATA_FILE\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`CRAG row ${index} must be an object`)\n  const query = firstString(raw, ['query', 'question', 'prompt'])\n  const goldAnswers = allStrings(raw, ['answer', 'answers', 'gold', 'gold_answer', 'expected_answer'])\n  if (!query) throw new Error(`CRAG row ${index} missing query`)\n  if (goldAnswers.length === 0) throw new Error(`CRAG row ${index} missing gold answer`)\n  const domain = stringFrom(raw.domain) ?? 'unknown'\n  const questionType = stringFrom(raw.question_type) ?? stringFrom(raw.questionType) ?? 'unknown'\n  const dynamism = stringFrom(raw.static_or_dynamic) ?? stringFrom(raw.dynamism) ?? 'unknown'\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `crag-${index}`\n  const meta: CragMeta = {\n    benchmark: 'crag',\n    query,\n    goldAnswers,\n    domain,\n    questionType,\n    dynamism,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? domain,\n    prompt: [\n      'Answer this CRAG factual question.',\n      'Return a concise answer and do not guess when the evidence is insufficient.',\n      'End with a single final line: `FINAL ANSWER: <answer>`.',\n      '',\n      `Question: ${query}`,\n      `Domain: ${domain}`,\n      `Question type: ${questionType}`,\n      `Dynamism: ${dynamism}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): CragMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.goldAnswers)) {\n    throw new Error(`CRAG task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as CragMeta\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`CRAG: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[crag] CRAG_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'CRAG')\n}\n\nexport function createCragAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.CRAG_FIXTURES === '1'\n\n  return {\n    name: 'crag',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'CRAG_DATA_FILE is required. Fix: export facebookresearch/CRAG rows to JSONL and set CRAG_DATA_FILE=/path/to/crag.jsonl, or set CRAG_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('CRAG_DATA_FILE is required to load CRAG tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'CRAG')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return `${FINAL_ANSWER_SENTINEL} ${readMeta(task).goldAnswers[0] ?? ''}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const score = scoreAnswerArtifact(artifact, meta.goldAnswers)\n      return answerScoreToBenchScore(score, {\n        benchmark: meta.benchmark,\n        domain: meta.domain,\n        questionType: meta.questionType,\n        dynamism: meta.dynamism,\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAQA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAgBrB,IAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AAWxD,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,YAAY,KAAK,oBAAoB;AACzE,QAAM,QAAQ,YAAY,KAAK,CAAC,SAAS,YAAY,QAAQ,CAAC;AAC9D,QAAM,cAAc,WAAW,KAAK,CAAC,UAAU,WAAW,QAAQ,eAAe,iBAAiB,CAAC;AACnG,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,YAAY,KAAK,gBAAgB;AAC7D,MAAI,YAAY,WAAW,EAAG,OAAM,IAAI,MAAM,YAAY,KAAK,sBAAsB;AACrF,QAAM,SAAS,WAAW,IAAI,MAAM,KAAK;AACzC,QAAM,eAAe,WAAW,IAAI,aAAa,KAAK,WAAW,IAAI,YAAY,KAAK;AACtF,QAAM,WAAW,WAAW,IAAI,iBAAiB,KAAK,WAAW,IAAI,QAAQ,KAAK;AAClF,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,QAAQ,KAAK;AAC1E,QAAM,OAAiB;AAAA,IACrB,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,WAAW,MAAM;AAAA,MACjB,kBAAkB,YAAY;AAAA,MAC9B,aAAa,QAAQ;AAAA,IACvB,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA2B;AAC3C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,WAAW,GAAG;AACzC,UAAM,IAAI,MAAM,aAAa,KAAK,EAAE,wDAAmD;AAAA,EACzF;AACA,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,oBAAoB,IAAI,EAAE;AACjE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,yCAAoC,KAAK,MAAM,mBAAmB;AAC/E,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,MAAM;AACtD;AAEO,SAAS,oBAAsC;AACpD,QAAM,eAAe,QAAQ,IAAI,kBAAkB;AAEnD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,+CAA+C;AAC1E,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,MAAM;AAAA,IACxE;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,GAAG,qBAAqB,IAAI,SAAS,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE;AAAA,IACxE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,oBAAoB,UAAU,KAAK,WAAW;AAC5D,aAAO,wBAAwB,OAAO;AAAA,QACpC,WAAW,KAAK;AAAA,QAChB,QAAQ,KAAK;AAAA,QACb,cAAc,KAAK;AAAA,QACnB,UAAU,KAAK;AAAA,MACjB,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-TBKU5XQI.js

@@ -1,228 +0,0 @@
-// src/benchmarks/mind2web.ts
-import { execFile } from "child_process";
-import { mkdir } from "fs/promises";
-import { tmpdir } from "os";
-import { join } from "path";
-import { fileURLToPath } from "url";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
-var PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, ".venv", "bin", "python");
-var DATASET = "osunlp/Multimodal-Mind2Web";
-var DEFAULT_SPLIT = process.env.M2W_SPLIT ?? "test_domain";
-var SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), "m2w-shots");
-var CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30);
-var WORKER_CONTRACT = [
-  "",
-  "Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:",
-  "ELEMENT: <the [id] number of the chosen element>",
-  "ACTION: <CLICK | TYPE | SELECT>",
-  "VALUE: <text to type or option to select; leave empty for CLICK>"
-].join("\n");
-async function py(script, args = []) {
-  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], { maxBuffer: 1024 * 1024 * 256 });
-  return stdout;
-}
-var LOADER = `
-import json, sys, os
-from datasets import load_dataset
-cfg = json.loads(sys.argv[1])
-split = cfg["split"]; limit = cfg.get("limit"); cap = cfg.get("cap", 30)
-shots = cfg["shotsDir"]; ids = set(cfg["ids"]) if cfg.get("ids") else None
-os.makedirs(shots, exist_ok=True)
-KEEP = ("aria_label","aria-label","role","type","name","placeholder","title","alt","value","text","id","class","href")
-def label(tag, attr):
-    try:
-        a = json.loads(attr) if isinstance(attr, str) else (attr or {})
-    except Exception:
-        a = {}
-    parts = []
-    for k in KEEP:
-        v = a.get(k)
-        if v:
-            sv = str(v).replace("\\n", " ").strip()
-            if k == "class": sv = sv[:40]
-            if k == "href": sv = sv[:50]
-            if sv: parts.append(k + "=" + sv[:60])
-    return "<" + str(tag) + "> " + " ".join(parts[:6])
-def cands(raw):
-    out = []
-    for c in raw or []:
-        try:
-            d = json.loads(c) if isinstance(c, str) else c
-        except Exception:
-            continue
-        bid = str(d.get("backend_node_id", ""))
-        if not bid: continue
-        out.append({"id": bid, "label": label(d.get("tag", "?"), d.get("attributes"))})
-    return out
-ds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)
-emitted = []; skipped = 0; scanned = 0
-for r in ds:
-    scanned += 1
-    if ids is not None and r["action_uid"] not in ids:
-        if scanned > 8000: break
-        continue
-    pos = cands(r.get("pos_candidates"))
-    if not pos:
-        skipped += 1
-        continue
-    neg = cands(r.get("neg_candidates"))
-    seen = set(p["id"] for p in pos); merged = list(pos)
-    for n in neg:
-        if len(merged) >= cap: break
-        if n["id"] in seen: continue
-        seen.add(n["id"]); merged.append(n)
-    merged.sort(key=lambda x: int(x["id"]) if x["id"].isdigit() else 0)
-    try:
-        op = json.loads(r["operation"]) if isinstance(r["operation"], str) else r["operation"]
-    except Exception:
-        op = {}
-    sp = os.path.join(shots, str(r["action_uid"]) + ".jpg")
-    try:
-        r["screenshot"].convert("RGB").save(sp, "JPEG", quality=70)
-    except Exception:
-        sp = ""
-    emitted.append({
-        "id": r["action_uid"], "task": r.get("confirmed_task", ""),
-        "website": r.get("website", ""), "domain": r.get("domain", ""), "subdomain": r.get("subdomain", ""),
-        "op": str(op.get("op", "")).upper(), "value": str(op.get("value", "")),
-        "goldIds": [p["id"] for p in pos], "candidates": merged,
-        "screenshotPath": sp, "targetRepr": r.get("target_action_reprs", ""),
-    })
-    if ids is None and limit is not None and len(emitted) >= limit: break
-    if ids is not None and len(emitted) >= len(ids): break
-sys.stderr.write("[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\n" % (len(emitted), skipped, scanned)); sys.stderr.flush()
-print(json.dumps(emitted)); sys.stdout.flush()
-os._exit(0)
-`;
-function buildPrompt(row) {
-  const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join("\n");
-  return [
-    `Web task: ${row.task}`,
-    "",
-    "You are taking the NEXT single action on the current web page. Choose the one element",
-    'to act on from the candidates below (each line is "[id] <tag> attributes"):',
-    choices,
-    WORKER_CONTRACT
-  ].join("\n");
-}
-function rowToTask(row) {
-  const meta = {
-    task: row.task,
-    website: row.website,
-    domain: row.domain,
-    op: row.op,
-    value: row.value,
-    goldIds: row.goldIds,
-    candidateIds: row.candidates.map((c) => c.id),
-    screenshotPath: row.screenshotPath,
-    targetRepr: row.targetRepr
-  };
-  return {
-    id: `mind2web-${row.id}`,
-    split: DEFAULT_SPLIT,
-    prompt: buildPrompt(row),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldIds)) {
-    throw new Error(`mind2web task ${task.id} missing metadata.goldIds \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function normValue(s) {
-  return s.toLowerCase().replace(/\s+/g, " ").trim();
-}
-function parseAction(artifact) {
-  const elem = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact);
-  const op = /ACTION:\s*(CLICK|TYPE|SELECT)/i.exec(artifact);
-  if (!elem?.[1] || !op?.[1]) return null;
-  const val = /VALUE:\s*(.*)/i.exec(artifact);
-  return { elementId: elem[1], op: op[1].toUpperCase(), value: (val?.[1] ?? "").trim() };
-}
-function createMind2WebAdapter() {
-  const split = DEFAULT_SPLIT;
-  return {
-    name: "mind2web",
-    async preflight() {
-      try {
-        await py(
-          `import os, sys
-from datasets import load_dataset
-import PIL
-ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)
-next(iter(ds))
-print('ok'); sys.stdout.flush(); os._exit(0)`
-        );
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `mind2web preflight failed: ${msg}
-Fix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; (2) network access to Hugging Face for ${DATASET} (split ${split}) ; point M2W_PYTHON at the python if not bench/.venv.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      const cfg = {
-        split: opts.split ?? split,
-        limit: opts.ids ? void 0 : opts.limit ?? 10,
-        cap: CANDIDATE_CAP,
-        shotsDir: SHOTS_DIR,
-        ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, "")) : void 0
-      };
-      await mkdir(SHOTS_DIR, { recursive: true });
-      const stdout = await py(LOADER, [JSON.stringify(cfg)]);
-      const rows = JSON.parse(stdout);
-      return rows.map(rowToTask);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      const id = meta.goldIds[0];
-      if (!id) return void 0;
-      return `ELEMENT: ${id}
-ACTION: ${meta.op}
-VALUE: ${meta.value}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const parsed = parseAction(artifact);
-      if (!parsed) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({ reason: "no parseable ELEMENT/ACTION", goldOp: meta.op, goldIds: meta.goldIds })
-        };
-      }
-      const elementCorrect = meta.goldIds.includes(parsed.elementId);
-      const opMatch = parsed.op === meta.op;
-      const valueMatch = meta.op === "CLICK" ? true : normValue(parsed.value) === normValue(meta.value);
-      const operationCorrect = opMatch && valueMatch;
-      const resolved = elementCorrect && operationCorrect;
-      const score = 0.6 * (elementCorrect ? 1 : 0) + 0.4 * (operationCorrect ? 1 : 0);
-      return {
-        resolved,
-        score,
-        detail: JSON.stringify({
-          elementCorrect,
-          opMatch,
-          valueMatch,
-          predicted: parsed,
-          goldOp: meta.op,
-          goldValue: meta.value,
-          goldIds: meta.goldIds,
-          website: meta.website,
-          domain: meta.domain
-        })
-      };
-    }
-  };
-}
-
-export {
-  parseAction,
-  createMind2WebAdapter
-};
-//# sourceMappingURL=chunk-TBKU5XQI.js.map