@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-KP5KD6EN.js

@@ -1,276 +0,0 @@
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/bfcl.ts
-import { readFile, stat } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "bfcl.json");
-var DEFAULT_CATEGORY = "BFCL_v4_simple_python";
-var bfclDir = () => process.env.BFCL_DIR;
-var bfclCategory = () => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY;
-var bfclOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    return text.trim();
-  }
-};
-async function assertPath(path, label) {
-  try {
-    await stat(path);
-  } catch (err) {
-    throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-function readJsonl(raw) {
-  return raw.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
-}
-function taskFile(dir, category) {
-  return process.env.BFCL_DATA_FILE ?? join(dir, "bfcl_eval", "data", `${category}.json`);
-}
-function answerFile(dir, category) {
-  return process.env.BFCL_ANSWER_FILE ?? join(dir, "bfcl_eval", "data", "possible_answer", `${category}.json`);
-}
-function questionText(question) {
-  if (typeof question === "string") return question;
-  if (!Array.isArray(question)) return JSON.stringify(question, null, 2);
-  const turns = [];
-  for (const item of question.flat(3)) {
-    if (item && typeof item === "object") {
-      const role = typeof item.role === "string" ? item.role : "user";
-      const content = item.content;
-      if (typeof content === "string") turns.push(`${role}: ${content}`);
-    }
-  }
-  return turns.length > 0 ? turns.join("\n") : JSON.stringify(question, null, 2);
-}
-function normalizeScalar(value) {
-  if (typeof value === "number") return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)));
-  if (typeof value === "string") return value.trim().toLowerCase();
-  return JSON.stringify(value);
-}
-function normalizeAllowed(value) {
-  return Array.isArray(value) ? value : [value];
-}
-function groundTruthToCalls(value) {
-  if (!Array.isArray(value)) return [];
-  const out = [];
-  for (const item of value) {
-    if (!item || typeof item !== "object") continue;
-    for (const [name, args] of Object.entries(item)) {
-      if (!args || typeof args !== "object" || Array.isArray(args)) continue;
-      const normalized = {};
-      for (const [argName, allowed] of Object.entries(args)) {
-        normalized[argName] = normalizeAllowed(allowed);
-      }
-      out.push({ name, arguments: normalized });
-    }
-  }
-  return out;
-}
-function rowToTask(row, answer, category) {
-  const expected = groundTruthToCalls(answer.ground_truth);
-  if (expected.length === 0) {
-    throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`);
-  }
-  const meta = {
-    rowId: row.id,
-    category,
-    functions: row.function,
-    expected,
-    scoring: "bfcl-ground-truth-subset"
-  };
-  return {
-    id: row.id,
-    split: category,
-    prompt: [
-      "Solve this BFCL function-calling task.",
-      'Return only JSON: {"function_calls":[{"name":"...","arguments":{...}}]}.',
-      "",
-      questionText(row.question),
-      "",
-      `Available functions: ${JSON.stringify(row.function, null, 2)}`
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.expected)) {
-    throw new Error(`BFCL task ${task.id} missing expected calls \u2014 loadTasks did not populate metadata`);
-  }
-  return md;
-}
-function selectTasks(tasks, opts) {
-  let out = tasks;
-  if (opts.split) out = out.filter((task) => task.split === opts.split);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    out = out.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    out = out.slice(0, opts.limit);
-  }
-  if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`);
-  return out;
-}
-function buildTasks(rows, answers, category, opts) {
-  const byId = new Map(answers.map((answer) => [answer.id, answer]));
-  const tasks = rows.map((row) => {
-    const answer = byId.get(row.id);
-    if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`);
-    return rowToTask(row, answer, category);
-  });
-  return selectTasks(tasks, opts);
-}
-async function loadFixtures(opts) {
-  const fixture = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[bfcl] BFCL_FIXTURES=1 \u2014 loading ${fixture.rows.length} adapter fixtures`);
-  return buildTasks(fixture.rows, fixture.answers, fixture.category, opts);
-}
-async function loadOfficialTasks(dir, opts) {
-  const category = bfclCategory();
-  const dataPath = taskFile(dir, category);
-  const answersPath = answerFile(dir, category);
-  const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, "utf8"), readFile(answersPath, "utf8")]);
-  const rows = readJsonl(rawRows);
-  const answers = readJsonl(rawAnswers);
-  return buildTasks(rows, answers, category, opts);
-}
-function extractJsonBlock(text) {
-  const fences = [...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)];
-  const raw = (fences.at(-1)?.[1] ?? text).trim();
-  try {
-    return JSON.parse(raw);
-  } catch {
-    return void 0;
-  }
-}
-function normalizeArguments(value) {
-  if (typeof value === "string") {
-    try {
-      return normalizeArguments(JSON.parse(value));
-    } catch {
-      return {};
-    }
-  }
-  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
-  return value;
-}
-function callFromObject(value) {
-  if (!value || typeof value !== "object") return void 0;
-  const raw = value;
-  if (raw.function && typeof raw.function === "object") {
-    const fn = raw.function;
-    if (typeof fn.name === "string") return { name: fn.name, arguments: normalizeArguments(fn.arguments) };
-  }
-  const name = typeof raw.name === "string" ? raw.name : typeof raw.function_name === "string" ? raw.function_name : typeof raw.tool_name === "string" ? raw.tool_name : void 0;
-  if (!name) return void 0;
-  return { name, arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters) };
-}
-function extractActualCalls(artifact) {
-  const parsed = extractJsonBlock(artifact);
-  if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call) => Boolean(call));
-  if (parsed && typeof parsed === "object") {
-    const raw = parsed;
-    for (const key of ["function_calls", "tool_calls", "calls"]) {
-      if (Array.isArray(raw[key])) {
-        return raw[key].map(callFromObject).filter((call) => Boolean(call));
-      }
-    }
-    const single = callFromObject(raw);
-    if (single) return [single];
-  }
-  return [];
-}
-function argMatches(expectedValues, actual) {
-  return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual));
-}
-function callMatches(expected, actual) {
-  if (expected.name !== actual.name) return false;
-  for (const [argName, allowed] of Object.entries(expected.arguments)) {
-    if (!(argName in actual.arguments)) {
-      if (allowed.some((value) => value === "" || value === null || value === void 0)) continue;
-      return false;
-    }
-    if (!argMatches(allowed, actual.arguments[argName])) return false;
-  }
-  return true;
-}
-function scoreCalls(task, artifact) {
-  const meta = readMeta(task);
-  const actual = extractActualCalls(artifact);
-  const used = /* @__PURE__ */ new Set();
-  let matched = 0;
-  for (const expected of meta.expected) {
-    const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call));
-    if (index >= 0) {
-      used.add(index);
-      matched += 1;
-    }
-  }
-  const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length;
-  const precision = actual.length === 0 ? 0 : matched / actual.length;
-  const score = recall;
-  return {
-    resolved: recall === 1 && precision === 1,
-    score,
-    detail: JSON.stringify({
-      scoring: meta.scoring,
-      category: meta.category,
-      expected: meta.expected,
-      actual,
-      precision,
-      recall,
-      fullBfclLeaderboardScore: null
-    })
-  };
-}
-function createBfclAdapter() {
-  const fixturesMode = process.env.BFCL_FIXTURES === "1";
-  return {
-    name: "bfcl",
-    output: bfclOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = bfclDir();
-      if (!dir) {
-        throw new Error(
-          "BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python."
-        );
-      }
-      const category = bfclCategory();
-      await assertPath(taskFile(dir, category), "BFCL task JSONL");
-      await assertPath(answerFile(dir, category), "BFCL possible_answer JSONL");
-      await loadOfficialTasks(dir, { limit: 1 });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const dir = bfclDir();
-      if (!dir) throw new Error("BFCL_DIR is required to load official BFCL rows");
-      return loadOfficialTasks(dir, opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return JSON.stringify({
-        function_calls: meta.expected.map((call) => ({
-          name: call.name,
-          arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]]))
-        }))
-      }, null, 2);
-    },
-    async judge(task, artifact) {
-      return scoreCalls(task, artifact);
-    }
-  };
-}
-
-export {
-  bfclOutput,
-  createBfclAdapter
-};
-//# sourceMappingURL=chunk-KP5KD6EN.js.map

package/dist/chunk-KP5KD6EN.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/bfcl.ts"],"sourcesContent":["/**\n * Berkeley Function Calling Leaderboard adapter.\n *\n * Scope: deterministic function-call ground-truth categories from the official\n * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic\n * web-search/memory categories and BFCL's own model-response harness remain\n * upstream responsibilities. The adapter loads official JSONL rows plus their\n * `possible_answer` file and scores structured function-call artifacts against\n * allowed function/argument values.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'bfcl.json')\nconst DEFAULT_CATEGORY = 'BFCL_v4_simple_python'\n\ninterface BfclRow {\n  id: string\n  question: unknown\n  function: unknown\n}\n\ninterface BfclAnswerRow {\n  id: string\n  ground_truth: unknown\n}\n\ninterface BfclAllowedCall {\n  name: string\n  arguments: Record<string, unknown[]>\n}\n\ninterface BfclMeta {\n  rowId: string\n  category: string\n  functions: unknown\n  expected: BfclAllowedCall[]\n  scoring: 'bfcl-ground-truth-subset'\n}\n\ninterface BfclActualCall {\n  name: string\n  arguments: Record<string, unknown>\n}\n\nconst bfclDir = (): string | undefined => process.env.BFCL_DIR\nconst bfclCategory = (): string => process.env.BFCL_CATEGORY ?? DEFAULT_CATEGORY\n\nexport const bfclOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text.trim()\n  },\n}\n\nasync function assertPath(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`BFCL: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readJsonl(raw: string): unknown[] {\n  return raw\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as unknown)\n}\n\nfunction taskFile(dir: string, category: string): string {\n  return process.env.BFCL_DATA_FILE ?? join(dir, 'bfcl_eval', 'data', `${category}.json`)\n}\n\nfunction answerFile(dir: string, category: string): string {\n  return process.env.BFCL_ANSWER_FILE ?? join(dir, 'bfcl_eval', 'data', 'possible_answer', `${category}.json`)\n}\n\nfunction questionText(question: unknown): string {\n  if (typeof question === 'string') return question\n  if (!Array.isArray(question)) return JSON.stringify(question, null, 2)\n  const turns: string[] = []\n  for (const item of question.flat(3)) {\n    if (item && typeof item === 'object') {\n      const role = typeof (item as { role?: unknown }).role === 'string' ? (item as { role: string }).role : 'user'\n      const content = (item as { content?: unknown }).content\n      if (typeof content === 'string') turns.push(`${role}: ${content}`)\n    }\n  }\n  return turns.length > 0 ? turns.join('\\n') : JSON.stringify(question, null, 2)\n}\n\nfunction normalizeScalar(value: unknown): string {\n  if (typeof value === 'number') return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)))\n  if (typeof value === 'string') return value.trim().toLowerCase()\n  return JSON.stringify(value)\n}\n\nfunction normalizeAllowed(value: unknown): unknown[] {\n  return Array.isArray(value) ? value : [value]\n}\n\nfunction groundTruthToCalls(value: unknown): BfclAllowedCall[] {\n  if (!Array.isArray(value)) return []\n  const out: BfclAllowedCall[] = []\n  for (const item of value) {\n    if (!item || typeof item !== 'object') continue\n    for (const [name, args] of Object.entries(item as Record<string, unknown>)) {\n      if (!args || typeof args !== 'object' || Array.isArray(args)) continue\n      const normalized: Record<string, unknown[]> = {}\n      for (const [argName, allowed] of Object.entries(args as Record<string, unknown>)) {\n        normalized[argName] = normalizeAllowed(allowed)\n      }\n      out.push({ name, arguments: normalized })\n    }\n  }\n  return out\n}\n\nfunction rowToTask(row: BfclRow, answer: BfclAnswerRow, category: string): BenchTask {\n  const expected = groundTruthToCalls(answer.ground_truth)\n  if (expected.length === 0) {\n    throw new Error(`BFCL ${row.id}: possible_answer has no deterministic ground_truth calls`)\n  }\n  const meta: BfclMeta = {\n    rowId: row.id,\n    category,\n    functions: row.function,\n    expected,\n    scoring: 'bfcl-ground-truth-subset',\n  }\n  return {\n    id: row.id,\n    split: category,\n    prompt: [\n      'Solve this BFCL function-calling task.',\n      'Return only JSON: {\"function_calls\":[{\"name\":\"...\",\"arguments\":{...}}]}.',\n      '',\n      questionText(row.question),\n      '',\n      `Available functions: ${JSON.stringify(row.function, null, 2)}`,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): BfclMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.expected)) {\n    throw new Error(`BFCL task ${task.id} missing expected calls — loadTasks did not populate metadata`)\n  }\n  return md as unknown as BfclMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  let out = tasks\n  if (opts.split) out = out.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    out = out.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    out = out.slice(0, opts.limit)\n  }\n  if (out.length === 0) throw new Error(`BFCL: no tasks matched ${JSON.stringify(opts)}`)\n  return out\n}\n\nfunction buildTasks(rows: BfclRow[], answers: BfclAnswerRow[], category: string, opts: LoadOptions): BenchTask[] {\n  const byId = new Map(answers.map((answer) => [answer.id, answer]))\n  const tasks = rows.map((row) => {\n    const answer = byId.get(row.id)\n    if (!answer) throw new Error(`BFCL ${category}: missing possible_answer for ${row.id}`)\n    return rowToTask(row, answer, category)\n  })\n  return selectTasks(tasks, opts)\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixture = JSON.parse(await readFile(FIXTURES, 'utf8')) as { rows: BfclRow[]; answers: BfclAnswerRow[]; category: string }\n  console.warn(`[bfcl] BFCL_FIXTURES=1 — loading ${fixture.rows.length} adapter fixtures`)\n  return buildTasks(fixture.rows, fixture.answers, fixture.category, opts)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const category = bfclCategory()\n  const dataPath = taskFile(dir, category)\n  const answersPath = answerFile(dir, category)\n  const [rawRows, rawAnswers] = await Promise.all([readFile(dataPath, 'utf8'), readFile(answersPath, 'utf8')])\n  const rows = readJsonl(rawRows) as BfclRow[]\n  const answers = readJsonl(rawAnswers) as BfclAnswerRow[]\n  return buildTasks(rows, answers, category, opts)\n}\n\nfunction extractJsonBlock(text: string): unknown {\n  const fences = [...text.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)]\n  const raw = (fences.at(-1)?.[1] ?? text).trim()\n  try {\n    return JSON.parse(raw)\n  } catch {\n    return undefined\n  }\n}\n\nfunction normalizeArguments(value: unknown): Record<string, unknown> {\n  if (typeof value === 'string') {\n    try {\n      return normalizeArguments(JSON.parse(value))\n    } catch {\n      return {}\n    }\n  }\n  if (!value || typeof value !== 'object' || Array.isArray(value)) return {}\n  return value as Record<string, unknown>\n}\n\nfunction callFromObject(value: unknown): BfclActualCall | undefined {\n  if (!value || typeof value !== 'object') return undefined\n  const raw = value as Record<string, unknown>\n  if (raw.function && typeof raw.function === 'object') {\n    const fn = raw.function as Record<string, unknown>\n    if (typeof fn.name === 'string') return { name: fn.name, arguments: normalizeArguments(fn.arguments) }\n  }\n  const name =\n    typeof raw.name === 'string' ? raw.name\n      : typeof raw.function_name === 'string' ? raw.function_name\n        : typeof raw.tool_name === 'string' ? raw.tool_name\n          : undefined\n  if (!name) return undefined\n  return { name, arguments: normalizeArguments(raw.arguments ?? raw.args ?? raw.parameters) }\n}\n\nfunction extractActualCalls(artifact: string): BfclActualCall[] {\n  const parsed = extractJsonBlock(artifact)\n  if (Array.isArray(parsed)) return parsed.map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n  if (parsed && typeof parsed === 'object') {\n    const raw = parsed as Record<string, unknown>\n    for (const key of ['function_calls', 'tool_calls', 'calls']) {\n      if (Array.isArray(raw[key])) {\n        return raw[key].map(callFromObject).filter((call): call is BfclActualCall => Boolean(call))\n      }\n    }\n    const single = callFromObject(raw)\n    if (single) return [single]\n  }\n  return []\n}\n\nfunction argMatches(expectedValues: unknown[], actual: unknown): boolean {\n  return expectedValues.some((expected) => normalizeScalar(expected) === normalizeScalar(actual))\n}\n\nfunction callMatches(expected: BfclAllowedCall, actual: BfclActualCall): boolean {\n  if (expected.name !== actual.name) return false\n  for (const [argName, allowed] of Object.entries(expected.arguments)) {\n    if (!(argName in actual.arguments)) {\n      if (allowed.some((value) => value === '' || value === null || value === undefined)) continue\n      return false\n    }\n    if (!argMatches(allowed, actual.arguments[argName])) return false\n  }\n  return true\n}\n\nfunction scoreCalls(task: BenchTask, artifact: string): BenchScore {\n  const meta = readMeta(task)\n  const actual = extractActualCalls(artifact)\n  const used = new Set<number>()\n  let matched = 0\n  for (const expected of meta.expected) {\n    const index = actual.findIndex((call, i) => !used.has(i) && callMatches(expected, call))\n    if (index >= 0) {\n      used.add(index)\n      matched += 1\n    }\n  }\n  const recall = meta.expected.length === 0 ? 0 : matched / meta.expected.length\n  const precision = actual.length === 0 ? 0 : matched / actual.length\n  const score = recall\n  return {\n    resolved: recall === 1 && precision === 1,\n    score,\n    detail: JSON.stringify({\n      scoring: meta.scoring,\n      category: meta.category,\n      expected: meta.expected,\n      actual,\n      precision,\n      recall,\n      fullBfclLeaderboardScore: null,\n    }),\n  }\n}\n\nexport function createBfclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.BFCL_FIXTURES === '1'\n\n  return {\n    name: 'bfcl',\n    output: bfclOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = bfclDir()\n      if (!dir) {\n        throw new Error(\n          'BFCL_DIR is required. Fix: clone https://github.com/ShishirPatil/gorilla, set BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard, and optionally set BFCL_CATEGORY=BFCL_v4_simple_python.',\n        )\n      }\n      const category = bfclCategory()\n      await assertPath(taskFile(dir, category), 'BFCL task JSONL')\n      await assertPath(answerFile(dir, category), 'BFCL possible_answer JSONL')\n      await loadOfficialTasks(dir, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = bfclDir()\n      if (!dir) throw new Error('BFCL_DIR is required to load official BFCL rows')\n      return loadOfficialTasks(dir, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return JSON.stringify({\n        function_calls: meta.expected.map((call) => ({\n          name: call.name,\n          arguments: Object.fromEntries(Object.entries(call.arguments).map(([key, values]) => [key, values[0]])),\n        })),\n      }, null, 2)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      return scoreCalls(task, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;AAWA,SAAS,UAAU,YAAY;AAC/B,SAAS,YAAY;AAKrB,IAAM,WAAW,KAAK,WAAW,YAAY,WAAW;AACxD,IAAM,mBAAmB;AA+BzB,IAAM,UAAU,MAA0B,QAAQ,IAAI;AACtD,IAAM,eAAe,MAAc,QAAQ,IAAI,iBAAiB;AAEzD,IAAM,aAAoC;AAAA,EAC/C,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACF;AAEA,eAAe,WAAW,MAAc,OAA8B;AACpE,MAAI;AACF,UAAM,KAAK,IAAI;AAAA,EACjB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,iBAAiB,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACnG;AACF;AAEA,SAAS,UAAU,KAAwB;AACzC,SAAO,IACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,EAChC,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,CAAY;AAC9C;AAEA,SAAS,SAAS,KAAa,UAA0B;AACvD,SAAO,QAAQ,IAAI,kBAAkB,KAAK,KAAK,aAAa,QAAQ,GAAG,QAAQ,OAAO;AACxF;AAEA,SAAS,WAAW,KAAa,UAA0B;AACzD,SAAO,QAAQ,IAAI,oBAAoB,KAAK,KAAK,aAAa,QAAQ,mBAAmB,GAAG,QAAQ,OAAO;AAC7G;AAEA,SAAS,aAAa,UAA2B;AAC/C,MAAI,OAAO,aAAa,SAAU,QAAO;AACzC,MAAI,CAAC,MAAM,QAAQ,QAAQ,EAAG,QAAO,KAAK,UAAU,UAAU,MAAM,CAAC;AACrE,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,SAAS,KAAK,CAAC,GAAG;AACnC,QAAI,QAAQ,OAAO,SAAS,UAAU;AACpC,YAAM,OAAO,OAAQ,KAA4B,SAAS,WAAY,KAA0B,OAAO;AACvG,YAAM,UAAW,KAA+B;AAChD,UAAI,OAAO,YAAY,SAAU,OAAM,KAAK,GAAG,IAAI,KAAK,OAAO,EAAE;AAAA,IACnE;AAAA,EACF;AACA,SAAO,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,UAAU,UAAU,MAAM,CAAC;AAC/E;AAEA,SAAS,gBAAgB,OAAwB;AAC/C,MAAI,OAAO,UAAU,SAAU,QAAO,OAAO,UAAU,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO,OAAO,MAAM,QAAQ,CAAC,CAAC,CAAC;AAC/G,MAAI,OAAO,UAAU,SAAU,QAAO,MAAM,KAAK,EAAE,YAAY;AAC/D,SAAO,KAAK,UAAU,KAAK;AAC7B;AAEA,SAAS,iBAAiB,OAA2B;AACnD,SAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,CAAC,KAAK;AAC9C;AAEA,SAAS,mBAAmB,OAAmC;AAC7D,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,MAAyB,CAAC;AAChC,aAAW,QAAQ,OAAO;AACxB,QAAI,CAAC,QAAQ,OAAO,SAAS,SAAU;AACvC,eAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,IAA+B,GAAG;AAC1E,UAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,EAAG;AAC9D,YAAM,aAAwC,CAAC;AAC/C,iBAAW,CAAC,SAAS,OAAO,KAAK,OAAO,QAAQ,IAA+B,GAAG;AAChF,mBAAW,OAAO,IAAI,iBAAiB,OAAO;AAAA,MAChD;AACA,UAAI,KAAK,EAAE,MAAM,WAAW,WAAW,CAAC;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,UAAU,KAAc,QAAuB,UAA6B;AACnF,QAAM,WAAW,mBAAmB,OAAO,YAAY;AACvD,MAAI,SAAS,WAAW,GAAG;AACzB,UAAM,IAAI,MAAM,QAAQ,IAAI,EAAE,2DAA2D;AAAA,EAC3F;AACA,QAAM,OAAiB;AAAA,IACrB,OAAO,IAAI;AAAA,IACX;AAAA,IACA,WAAW,IAAI;AAAA,IACf;AAAA,IACA,SAAS;AAAA,EACX;AACA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO;AAAA,IACP,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa,IAAI,QAAQ;AAAA,MACzB;AAAA,MACA,wBAAwB,KAAK,UAAU,IAAI,UAAU,MAAM,CAAC,CAAC;AAAA,IAC/D,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA2B;AAC3C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,QAAQ,GAAG;AACtC,UAAM,IAAI,MAAM,aAAa,KAAK,EAAE,oEAA+D;AAAA,EACrG;AACA,SAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;AACvE,MAAI,MAAM;AACV,MAAI,KAAK,MAAO,OAAM,IAAI,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK,KAAK;AACpE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,UAAM,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAC9C,WAAW,KAAK,UAAU,QAAW;AACnC,UAAM,IAAI,MAAM,GAAG,KAAK,KAAK;AAAA,EAC/B;AACA,MAAI,IAAI,WAAW,EAAG,OAAM,IAAI,MAAM,0BAA0B,KAAK,UAAU,IAAI,CAAC,EAAE;AACtF,SAAO;AACT;AAEA,SAAS,WAAW,MAAiB,SAA0B,UAAkB,MAAgC;AAC/G,QAAM,OAAO,IAAI,IAAI,QAAQ,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,MAAM,CAAC,CAAC;AACjE,QAAM,QAAQ,KAAK,IAAI,CAAC,QAAQ;AAC9B,UAAM,SAAS,KAAK,IAAI,IAAI,EAAE;AAC9B,QAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,QAAQ,QAAQ,iCAAiC,IAAI,EAAE,EAAE;AACtF,WAAO,UAAU,KAAK,QAAQ,QAAQ;AAAA,EACxC,CAAC;AACD,SAAO,YAAY,OAAO,IAAI;AAChC;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AAC3D,UAAQ,KAAK,yCAAoC,QAAQ,KAAK,MAAM,mBAAmB;AACvF,SAAO,WAAW,QAAQ,MAAM,QAAQ,SAAS,QAAQ,UAAU,IAAI;AACzE;AAEA,eAAe,kBAAkB,KAAa,MAAyC;AACrF,QAAM,WAAW,aAAa;AAC9B,QAAM,WAAW,SAAS,KAAK,QAAQ;AACvC,QAAM,cAAc,WAAW,KAAK,QAAQ;AAC5C,QAAM,CAAC,SAAS,UAAU,IAAI,MAAM,QAAQ,IAAI,CAAC,SAAS,UAAU,MAAM,GAAG,SAAS,aAAa,MAAM,CAAC,CAAC;AAC3G,QAAM,OAAO,UAAU,OAAO;AAC9B,QAAM,UAAU,UAAU,UAAU;AACpC,SAAO,WAAW,MAAM,SAAS,UAAU,IAAI;AACjD;AAEA,SAAS,iBAAiB,MAAuB;AAC/C,QAAM,SAAS,CAAC,GAAG,KAAK,SAAS,+BAA+B,CAAC;AACjE,QAAM,OAAO,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAC9C,MAAI;AACF,WAAO,KAAK,MAAM,GAAG;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,mBAAmB,OAAyC;AACnE,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI;AACF,aAAO,mBAAmB,KAAK,MAAM,KAAK,CAAC;AAAA,IAC7C,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AACA,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACzE,SAAO;AACT;AAEA,SAAS,eAAe,OAA4C;AAClE,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,MAAM;AACZ,MAAI,IAAI,YAAY,OAAO,IAAI,aAAa,UAAU;AACpD,UAAM,KAAK,IAAI;AACf,QAAI,OAAO,GAAG,SAAS,SAAU,QAAO,EAAE,MAAM,GAAG,MAAM,WAAW,mBAAmB,GAAG,SAAS,EAAE;AAAA,EACvG;AACA,QAAM,OACJ,OAAO,IAAI,SAAS,WAAW,IAAI,OAC/B,OAAO,IAAI,kBAAkB,WAAW,IAAI,gBAC1C,OAAO,IAAI,cAAc,WAAW,IAAI,YACtC;AACV,MAAI,CAAC,KAAM,QAAO;AAClB,SAAO,EAAE,MAAM,WAAW,mBAAmB,IAAI,aAAa,IAAI,QAAQ,IAAI,UAAU,EAAE;AAC5F;AAEA,SAAS,mBAAmB,UAAoC;AAC9D,QAAM,SAAS,iBAAiB,QAAQ;AACxC,MAAI,MAAM,QAAQ,MAAM,EAAG,QAAO,OAAO,IAAI,cAAc,EAAE,OAAO,CAAC,SAAiC,QAAQ,IAAI,CAAC;AACnH,MAAI,UAAU,OAAO,WAAW,UAAU;AACxC,UAAM,MAAM;AACZ,eAAW,OAAO,CAAC,kBAAkB,cAAc,OAAO,GAAG;AAC3D,UAAI,MAAM,QAAQ,IAAI,GAAG,CAAC,GAAG;AAC3B,eAAO,IAAI,GAAG,EAAE,IAAI,cAAc,EAAE,OAAO,CAAC,SAAiC,QAAQ,IAAI,CAAC;AAAA,MAC5F;AAAA,IACF;AACA,UAAM,SAAS,eAAe,GAAG;AACjC,QAAI,OAAQ,QAAO,CAAC,MAAM;AAAA,EAC5B;AACA,SAAO,CAAC;AACV;AAEA,SAAS,WAAW,gBAA2B,QAA0B;AACvE,SAAO,eAAe,KAAK,CAAC,aAAa,gBAAgB,QAAQ,MAAM,gBAAgB,MAAM,CAAC;AAChG;AAEA,SAAS,YAAY,UAA2B,QAAiC;AAC/E,MAAI,SAAS,SAAS,OAAO,KAAM,QAAO;AAC1C,aAAW,CAAC,SAAS,OAAO,KAAK,OAAO,QAAQ,SAAS,SAAS,GAAG;AACnE,QAAI,EAAE,WAAW,OAAO,YAAY;AAClC,UAAI,QAAQ,KAAK,CAAC,UAAU,UAAU,MAAM,UAAU,QAAQ,UAAU,MAAS,EAAG;AACpF,aAAO;AAAA,IACT;AACA,QAAI,CAAC,WAAW,SAAS,OAAO,UAAU,OAAO,CAAC,EAAG,QAAO;AAAA,EAC9D;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAiB,UAA8B;AACjE,QAAM,OAAO,SAAS,IAAI;AAC1B,QAAM,SAAS,mBAAmB,QAAQ;AAC1C,QAAM,OAAO,oBAAI,IAAY;AAC7B,MAAI,UAAU;AACd,aAAW,YAAY,KAAK,UAAU;AACpC,UAAM,QAAQ,OAAO,UAAU,CAAC,MAAM,MAAM,CAAC,KAAK,IAAI,CAAC,KAAK,YAAY,UAAU,IAAI,CAAC;AACvF,QAAI,SAAS,GAAG;AACd,WAAK,IAAI,KAAK;AACd,iBAAW;AAAA,IACb;AAAA,EACF;AACA,QAAM,SAAS,KAAK,SAAS,WAAW,IAAI,IAAI,UAAU,KAAK,SAAS;AACxE,QAAM,YAAY,OAAO,WAAW,IAAI,IAAI,UAAU,OAAO;AAC7D,QAAM,QAAQ;AACd,SAAO;AAAA,IACL,UAAU,WAAW,KAAK,cAAc;AAAA,IACxC;AAAA,IACA,QAAQ,KAAK,UAAU;AAAA,MACrB,SAAS,KAAK;AAAA,MACd,UAAU,KAAK;AAAA,MACf,UAAU,KAAK;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA,0BAA0B;AAAA,IAC5B,CAAC;AAAA,EACH;AACF;AAEO,SAAS,oBAAsC;AACpD,QAAM,eAAe,QAAQ,IAAI,kBAAkB;AAEnD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,QAAQ;AACpB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,WAAW,aAAa;AAC9B,YAAM,WAAW,SAAS,KAAK,QAAQ,GAAG,iBAAiB;AAC3D,YAAM,WAAW,WAAW,KAAK,QAAQ,GAAG,4BAA4B;AACxE,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,IAC3C;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,QAAQ;AACpB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,iDAAiD;AAC3E,aAAO,kBAAkB,KAAK,IAAI;AAAA,IACpC;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,KAAK,UAAU;AAAA,QACpB,gBAAgB,KAAK,SAAS,IAAI,CAAC,UAAU;AAAA,UAC3C,MAAM,KAAK;AAAA,UACX,WAAW,OAAO,YAAY,OAAO,QAAQ,KAAK,SAAS,EAAE,IAAI,CAAC,CAAC,KAAK,MAAM,MAAM,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAAA,QACvG,EAAE;AAAA,MACJ,GAAG,MAAM,CAAC;AAAA,IACZ;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,aAAO,WAAW,MAAM,QAAQ;AAAA,IAClC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-MQMRLGOG.js

@@ -1,136 +0,0 @@
-import {
-  benchRoot,
-  preflightVenvImports,
-  runVenvScriptStdin
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/programbench.ts
-import { join } from "path";
-import { readFile } from "fs/promises";
-var FIXTURES = join(benchRoot, "fixtures", "programbench.json");
-var TESTS_DATASET = "programbench/ProgramBench-Tests";
-var TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`;
-var PROMPT = [
-  "This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.",
-  "Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.",
-  "You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.",
-  "Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence."
-].join("\n");
-function fixtureToTask(f) {
-  const meta = { instanceId: f.instance_id, imageName: f.image_name };
-  return {
-    id: f.instance_id,
-    prompt: PROMPT,
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.instanceId !== "string") {
-    throw new Error(`programbench task ${task.id} missing metadata.instanceId \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectTasks(fixtures, opts) {
-  let tasks = fixtures.map(fixtureToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((t) => want.has(t.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  return tasks;
-}
-function imageName(instanceId) {
-  return instanceId.split(".")[0].replace("__", "/");
-}
-async function loadFixtures(opts) {
-  const fixtures = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(
-    `[programbench] PROGRAMBENCH_FIXTURES=1 \u2014 loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`
-  );
-  return selectTasks(fixtures, opts);
-}
-async function fetchInstanceIds() {
-  const res = await fetch(TREE_API);
-  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`);
-  const tree = await res.json();
-  const ids = tree.filter((e) => e.type === "directory" && e.path.includes("__")).map((e) => e.path);
-  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`);
-  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }));
-}
-async function runHarness(meta, submission) {
-  const judge = join(benchRoot, "scripts", "programbench_judge.py");
-  let stdout;
-  try {
-    stdout = await runVenvScriptStdin(judge, ["--instance", meta.instanceId], submission, { cwd: benchRoot });
-  } catch (err) {
-    const e = err;
-    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`);
-  }
-  const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`);
-  if (typeof report.passed !== "number" || typeof report.total !== "number") {
-    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`);
-  }
-  const score = report.total > 0 ? report.passed / report.total : 0;
-  return {
-    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),
-    score,
-    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total })
-  };
-}
-var programbenchSubmissionOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const blocks = [...text.matchAll(/```path:([^\n`]+)\n([\s\S]*?)```/g)];
-    if (blocks.length === 0) return "";
-    return blocks.map((m) => `===FILE:${m[1].trim()}===
-${m[2]}`).join("\n");
-  }
-};
-function createProgrambenchAdapter() {
-  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === "1";
-  return {
-    name: "programbench",
-    output: programbenchSubmissionOutput,
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["programbench"],
-        requireDocker: true,
-        fix: `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; (2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; (3) HF access for the hidden test blobs (the driver runs \`programbench blob sync\`). Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`
-      });
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      let fixtures;
-      try {
-        fixtures = await fetchInstanceIds();
-      } catch (err) {
-        console.warn(
-          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`
-        );
-        return loadFixtures(opts);
-      }
-      return selectTasks(fixtures, opts);
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      return runHarness(meta, artifact);
-    }
-  };
-}
-
-export {
-  programbenchSubmissionOutput,
-  createProgrambenchAdapter
-};
-//# sourceMappingURL=chunk-MQMRLGOG.js.map

package/dist/chunk-MQMRLGOG.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/programbench.ts"],"sourcesContent":["/**\n * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom\n * reverse-engineering: rebuild a black-box executable's behavior from scratch.\n * The agent is given only the gold `./executable` (run-only) + stripped docs and\n * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`\n * with identical observable behavior. Judge = the official `programbench`\n * harness: it extracts the submission, runs compile.sh in the per-task cleanroom\n * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via\n * `programbench blob sync`). Score = fraction of non-ignored tests passed\n * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests\n * pass. Fully deterministic — no LLM judge.\n *\n * OutputAdapter is stream-only, so the worker emits its codebase as fenced\n * `path:`-prefixed file blocks (including compile.sh); the adapter materializes\n * those into submission.tar.gz. Test execution is delegated to the real\n * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT\n * reimplemented here.\n *\n * Requires for a live run: the bench `.venv` with `programbench` installed +\n * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for\n * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load\n * the committed instance ids (bench/fixtures/programbench.json); judging still\n * needs the harness + Docker and fails loud without them — never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'programbench.json')\n\nconst TESTS_DATASET = 'programbench/ProgramBench-Tests'\nconst TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`\n\ninterface PbFixture {\n  instance_id: string\n  image_name: string\n  note?: string\n}\n\ninterface PbMeta {\n  instanceId: string\n  imageName: string\n}\n\nconst PROMPT = [\n  'This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.',\n  'Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.',\n  'You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.',\n  'Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence.',\n].join('\\n')\n\nfunction fixtureToTask(f: PbFixture): BenchTask {\n  const meta: PbMeta = { instanceId: f.instance_id, imageName: f.image_name }\n  return {\n    id: f.instance_id,\n    prompt: PROMPT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): PbMeta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string') {\n    throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`)\n  }\n  return md as unknown as PbMeta\n}\n\nfunction selectTasks(fixtures: PbFixture[], opts: LoadOptions): BenchTask[] {\n  let tasks = fixtures.map(fixtureToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\n/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */\nfunction imageName(instanceId: string): string {\n  return instanceId.split('.')[0].replace('__', '/')\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixtures = JSON.parse(await readFile(FIXTURES, 'utf8')) as PbFixture[]\n  console.warn(\n    `[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`,\n  )\n  return selectTasks(fixtures, opts)\n}\n\n/** Enumerate real instance ids from the HF Tests dataset tree (one dir per\n *  instance). Throws on a non-OK response (fail loud). */\nasync function fetchInstanceIds(): Promise<PbFixture[]> {\n  const res = await fetch(TREE_API)\n  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`)\n  const tree = (await res.json()) as Array<{ path: string; type: string }>\n  const ids = tree.filter((e) => e.type === 'directory' && e.path.includes('__')).map((e) => e.path)\n  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`)\n  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }))\n}\n\n/**\n * Run the official programbench harness for one instance over the worker's\n * file-manifest submission. The driver builds the run-dir layout (one folder\n * holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,\n * then parses <instance_id>.eval.json applying the tests.json ignore mask via\n * `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.\n */\nasync function runHarness(meta: PbMeta, submission: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'programbench_judge.py')\n  let stdout: string\n  try {\n    // The submission manifest is piped to the driver's stdin via the shared\n    // stdin-aware runner — execFile's `input` option is not honored async and\n    // hangs the driver's sys.stdin.read() forever.\n    stdout = await runVenvScriptStdin(judge, ['--instance', meta.instanceId], submission, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    resolved?: boolean\n    error?: string\n  }\n  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passed / report.total : 0\n  return {\n    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\n/**\n * Parse the worker stream into the submission text the driver materializes: the\n * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to\n * the python driver, which tars it. Empty when the worker emitted no file block\n * (fail-closed → the harness scores a missing compile.sh as 0).\n */\nexport const programbenchSubmissionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const blocks = [...text.matchAll(/```path:([^\\n`]+)\\n([\\s\\S]*?)```/g)]\n    if (blocks.length === 0) return ''\n    // Re-serialize in the same path:<p> envelope the driver splits on.\n    return blocks.map((m) => `===FILE:${m[1].trim()}===\\n${m[2]}`).join('\\n')\n  },\n}\n\nexport function createProgrambenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === '1'\n\n  return {\n    name: 'programbench',\n    output: programbenchSubmissionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['programbench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; ` +\n          `(2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; ` +\n          `(3) HF access for the hidden test blobs (the driver runs \\`programbench blob sync\\`). ` +\n          `Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let fixtures: PbFixture[]\n      try {\n        fixtures = await fetchInstanceIds()\n      } catch (err) {\n        console.warn(\n          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectTasks(fixtures, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the original repo's source (stripped from the task image),\n      // not redistributable as a portable submission string. Judge correctness is\n      // proven by running the real harness on a real solve, not a synthetic gold.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;AAyBA,SAAS,YAAY;AACrB,SAAS,gBAAgB;AAKzB,IAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEhE,IAAM,gBAAgB;AACtB,IAAM,WAAW,uCAAuC,aAAa;AAarE,IAAM,SAAS;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,SAAS,cAAc,GAAyB;AAC9C,QAAM,OAAe,EAAE,YAAY,EAAE,aAAa,WAAW,EAAE,WAAW;AAC1E,SAAO;AAAA,IACL,IAAI,EAAE;AAAA,IACN,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAyB;AACzC,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,eAAe,UAAU;AAC5C,UAAM,IAAI,MAAM,qBAAqB,KAAK,EAAE,mEAA8D;AAAA,EAC5G;AACA,SAAO;AACT;AAEA,SAAS,YAAY,UAAuB,MAAgC;AAC1E,MAAI,QAAQ,SAAS,IAAI,aAAa;AACtC,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAGA,SAAS,UAAU,YAA4B;AAC7C,SAAO,WAAW,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,MAAM,GAAG;AACnD;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,WAAW,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AAC5D,UAAQ;AAAA,IACN,yDAAoD,SAAS,MAAM,gCAAgC,QAAQ;AAAA,EAC7G;AACA,SAAO,YAAY,UAAU,IAAI;AACnC;AAIA,eAAe,mBAAyC;AACtD,QAAM,MAAM,MAAM,MAAM,QAAQ;AAChC,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,0BAA0B,IAAI,MAAM,KAAK,QAAQ,EAAE;AAChF,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,KAAK,SAAS,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AACjG,MAAI,IAAI,WAAW,EAAG,OAAM,IAAI,MAAM,qCAAqC,aAAa,OAAO;AAC/F,SAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,aAAa,IAAI,YAAY,UAAU,EAAE,EAAE,EAAE;AACzE;AASA,eAAe,WAAW,MAAc,YAAyC;AAC/E,QAAM,QAAQ,KAAK,WAAW,WAAW,uBAAuB;AAChE,MAAI;AACJ,MAAI;AAIF,aAAS,MAAM,mBAAmB,OAAO,CAAC,cAAc,KAAK,UAAU,GAAG,YAAY,EAAE,KAAK,UAAU,CAAC;AAAA,EAC1G,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,mCAAmC,KAAK,UAAU,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EACpH;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAMlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,kCAAkC,KAAK,UAAU,KAAK,OAAO,KAAK,EAAE;AACtG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,kDAAkD,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC1F;AACA,QAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;AAChE,SAAO;AAAA,IACL,UAAU,OAAO,aAAa,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;AAAA,IAC3E;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,YAAY,KAAK,YAAY,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACpG;AACF;AAQO,IAAM,+BAAsD;AAAA,EACjE,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,mCAAmC,CAAC;AACrE,QAAI,OAAO,WAAW,EAAG,QAAO;AAEhC,WAAO,OAAO,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,EAAE,KAAK,CAAC;AAAA,EAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI;AAAA,EAC1E;AACF;AAEO,SAAS,4BAA8C;AAC5D,QAAM,eAAe,QAAQ,IAAI,0BAA0B;AAE3D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,cAAc;AAAA,QACxB,eAAe;AAAA,QACf,KACE;AAAA,MAIJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,UAAI;AACJ,UAAI;AACF,mBAAW,MAAM,iBAAiB;AAAA,MACpC,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG,4CAA4C,QAAQ;AAAA,QACxI;AACA,eAAO,aAAa,IAAI;AAAA,MAC1B;AACA,aAAO,YAAY,UAAU,IAAI;AAAA,IACnC;AAAA,IAEA,MAAM,eAAe;AAInB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,WAAW,MAAM,QAAQ;AAAA,IAClC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-NQG5XDSB.js

@@ -1,147 +0,0 @@
-import {
-  contextBlock,
-  contextsFrom,
-  firstString,
-  isObject,
-  ragAnswerOutput,
-  readJsonRows,
-  selectTasks,
-  stringFrom
-} from "./chunk-X3BTXCJ4.js";
-import {
-  benchRoot
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/nomiracl.ts
-import { readFile } from "fs/promises";
-import { join } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "nomiracl.json");
-var dataFile = () => process.env.NOMIRACL_DATA_FILE;
-function passagesFrom(value, relevant) {
-  return contextsFrom(value).map((context) => ({ ...context, relevant }));
-}
-function answerableFrom(raw, contexts) {
-  const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant;
-  if (typeof direct === "boolean") return direct;
-  if (typeof direct === "number") return direct > 0;
-  if (typeof direct === "string") {
-    const normalized = direct.toLowerCase();
-    if (["true", "t", "relevant", "answerable", "positive", "1", "yes"].includes(normalized)) return true;
-    if (["false", "f", "non-relevant", "non_relevant", "unanswerable", "negative", "0", "no"].includes(normalized)) {
-      return false;
-    }
-  }
-  return contexts.some((context) => context.relevant === true);
-}
-function rowToTask(raw, index) {
-  if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`);
-  const query = firstString(raw, ["query", "question"]);
-  if (!query) throw new Error(`NoMIRACL row ${index} missing query`);
-  const labelledPassages = [
-    ...passagesFrom(raw.positive_passages, true),
-    ...passagesFrom(raw.negative_passages, false)
-  ];
-  const contexts = labelledPassages.length > 0 ? labelledPassages : contextsFrom(raw.passages).length > 0 ? contextsFrom(raw.passages) : contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.documents);
-  const answerable = answerableFrom(raw, contexts);
-  const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? "unknown";
-  const subset = stringFrom(raw.subset) ?? (answerable ? "relevant" : "non-relevant");
-  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}`;
-  const meta = {
-    benchmark: "nomiracl",
-    query,
-    answerable,
-    language,
-    subset,
-    contexts
-  };
-  return {
-    id,
-    split: stringFrom(raw.split) ?? language,
-    prompt: [
-      "Classify this NoMIRACL RAG relevance case.",
-      "Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.",
-      "",
-      `Query: ${query}`,
-      contexts.length > 0 ? `
-Passages:
-${contextBlock(contexts)}` : void 0
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.answerable !== "boolean") {
-    throw new Error(`NoMIRACL task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function parseClassification(artifact) {
-  const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
-  if (/\bunanswerable\b|\bnot answerable\b|\bno answer\b|\bnegative\b|\bfalse\b/.test(normalized)) {
-    return false;
-  }
-  if (/\banswerable\b|\brelevant\b|\bpositive\b|\btrue\b|\byes\b/.test(normalized)) return true;
-  return null;
-}
-async function loadRows(path) {
-  const rows = await readJsonRows(path);
-  if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`);
-  return rows;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectTasks(rows.map(rowToTask), opts, "NoMIRACL");
-}
-function createNoMiraclAdapter() {
-  const fixturesMode = process.env.NOMIRACL_FIXTURES === "1";
-  return {
-    name: "nomiracl",
-    output: ragAnswerOutput,
-    async preflight() {
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8");
-        return;
-      }
-      const path = dataFile();
-      if (!path) {
-        throw new Error(
-          "NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing."
-        );
-      }
-      await loadRows(path);
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const path = dataFile();
-      if (!path) throw new Error("NOMIRACL_DATA_FILE is required to load NoMIRACL tasks");
-      return selectTasks((await loadRows(path)).map(rowToTask), opts, "NoMIRACL");
-    },
-    async goldArtifact(task) {
-      return readMeta(task).answerable ? "ANSWERABLE" : "UNANSWERABLE";
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const got = parseClassification(artifact);
-      const resolved = got === meta.answerable;
-      return {
-        resolved,
-        score: resolved ? 1 : 0,
-        detail: JSON.stringify({
-          benchmark: meta.benchmark,
-          language: meta.language,
-          subset: meta.subset,
-          expected: meta.answerable ? "ANSWERABLE" : "UNANSWERABLE",
-          got: got === null ? null : got ? "ANSWERABLE" : "UNANSWERABLE",
-          contextCount: meta.contexts.length
-        })
-      };
-    }
-  };
-}
-
-export {
-  createNoMiraclAdapter
-};
-//# sourceMappingURL=chunk-NQG5XDSB.js.map

package/dist/chunk-NQG5XDSB.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/nomiracl.ts"],"sourcesContent":["/**\n * NoMIRACL adapter.\n *\n * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does\n * not generate an answer here; it classifies whether the supplied passages\n * contain enough evidence to answer the query. This directly measures false\n * positive / false negative behavior for RAG abstention.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'nomiracl.json')\n\ninterface NoMiraclMeta {\n  benchmark: 'nomiracl'\n  query: string\n  answerable: boolean\n  language: string\n  subset: string\n  contexts: RagContext[]\n}\n\nconst dataFile = (): string | undefined => process.env.NOMIRACL_DATA_FILE\n\nfunction passagesFrom(value: unknown, relevant: boolean): RagContext[] {\n  return contextsFrom(value).map((context) => ({ ...context, relevant }))\n}\n\nfunction answerableFrom(raw: Record<string, unknown>, contexts: readonly RagContext[]): boolean {\n  const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant\n  if (typeof direct === 'boolean') return direct\n  if (typeof direct === 'number') return direct > 0\n  if (typeof direct === 'string') {\n    const normalized = direct.toLowerCase()\n    if (['true', 't', 'relevant', 'answerable', 'positive', '1', 'yes'].includes(normalized)) return true\n    if (['false', 'f', 'non-relevant', 'non_relevant', 'unanswerable', 'negative', '0', 'no'].includes(normalized)) {\n      return false\n    }\n  }\n  return contexts.some((context) => context.relevant === true)\n}\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`)\n  const query = firstString(raw, ['query', 'question'])\n  if (!query) throw new Error(`NoMIRACL row ${index} missing query`)\n  const labelledPassages = [\n    ...passagesFrom(raw.positive_passages, true),\n    ...passagesFrom(raw.negative_passages, false),\n  ]\n  const contexts =\n    labelledPassages.length > 0\n      ? labelledPassages\n      : contextsFrom(raw.passages).length > 0\n      ? contextsFrom(raw.passages)\n      : contextsFrom(raw.contexts).length > 0\n        ? contextsFrom(raw.contexts)\n        : contextsFrom(raw.documents)\n  const answerable = answerableFrom(raw, contexts)\n  const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? 'unknown'\n  const subset = stringFrom(raw.subset) ?? (answerable ? 'relevant' : 'non-relevant')\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}`\n  const meta: NoMiraclMeta = {\n    benchmark: 'nomiracl',\n    query,\n    answerable,\n    language,\n    subset,\n    contexts,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? language,\n    prompt: [\n      'Classify this NoMIRACL RAG relevance case.',\n      'Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.',\n      '',\n      `Query: ${query}`,\n      contexts.length > 0 ? `\\nPassages:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): NoMiraclMeta {\n  const md = task.metadata\n  if (!md || typeof md.answerable !== 'boolean') {\n    throw new Error(`NoMIRACL task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as NoMiraclMeta\n}\n\nfunction parseClassification(artifact: string): boolean | null {\n  const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()\n  if (/\\bunanswerable\\b|\\bnot answerable\\b|\\bno answer\\b|\\bnegative\\b|\\bfalse\\b/.test(normalized)) {\n    return false\n  }\n  if (/\\banswerable\\b|\\brelevant\\b|\\bpositive\\b|\\btrue\\b|\\byes\\b/.test(normalized)) return true\n  return null\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'NoMIRACL')\n}\n\nexport function createNoMiraclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.NOMIRACL_FIXTURES === '1'\n\n  return {\n    name: 'nomiracl',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('NOMIRACL_DATA_FILE is required to load NoMIRACL tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'NoMIRACL')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return readMeta(task).answerable ? 'ANSWERABLE' : 'UNANSWERABLE'\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const got = parseClassification(artifact)\n      const resolved = got === meta.answerable\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          benchmark: meta.benchmark,\n          language: meta.language,\n          subset: meta.subset,\n          expected: meta.answerable ? 'ANSWERABLE' : 'UNANSWERABLE',\n          got: got === null ? null : got ? 'ANSWERABLE' : 'UNANSWERABLE',\n          contextCount: meta.contexts.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AASA,SAAS,gBAAgB;AACzB,SAAS,YAAY;AAerB,IAAM,WAAW,KAAK,WAAW,YAAY,eAAe;AAW5D,IAAM,WAAW,MAA0B,QAAQ,IAAI;AAEvD,SAAS,aAAa,OAAgB,UAAiC;AACrE,SAAO,aAAa,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,GAAG,SAAS,SAAS,EAAE;AACxE;AAEA,SAAS,eAAe,KAA8B,UAA0C;AAC9F,QAAM,SAAS,IAAI,cAAc,IAAI,iBAAiB,IAAI,cAAc,IAAI,SAAS,IAAI;AACzF,MAAI,OAAO,WAAW,UAAW,QAAO;AACxC,MAAI,OAAO,WAAW,SAAU,QAAO,SAAS;AAChD,MAAI,OAAO,WAAW,UAAU;AAC9B,UAAM,aAAa,OAAO,YAAY;AACtC,QAAI,CAAC,QAAQ,KAAK,YAAY,cAAc,YAAY,KAAK,KAAK,EAAE,SAAS,UAAU,EAAG,QAAO;AACjG,QAAI,CAAC,SAAS,KAAK,gBAAgB,gBAAgB,gBAAgB,YAAY,KAAK,IAAI,EAAE,SAAS,UAAU,GAAG;AAC9G,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO,SAAS,KAAK,CAAC,YAAY,QAAQ,aAAa,IAAI;AAC7D;AAEA,SAAS,UAAU,KAAc,OAA0B;AACzD,MAAI,CAAC,SAAS,GAAG,EAAG,OAAM,IAAI,MAAM,gBAAgB,KAAK,oBAAoB;AAC7E,QAAM,QAAQ,YAAY,KAAK,CAAC,SAAS,UAAU,CAAC;AACpD,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,gBAAgB,KAAK,gBAAgB;AACjE,QAAM,mBAAmB;AAAA,IACvB,GAAG,aAAa,IAAI,mBAAmB,IAAI;AAAA,IAC3C,GAAG,aAAa,IAAI,mBAAmB,KAAK;AAAA,EAC9C;AACA,QAAM,WACJ,iBAAiB,SAAS,IACtB,mBACA,aAAa,IAAI,QAAQ,EAAE,SAAS,IACpC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,QAAQ,EAAE,SAAS,IAClC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,SAAS;AAClC,QAAM,aAAa,eAAe,KAAK,QAAQ;AAC/C,QAAM,WAAW,WAAW,IAAI,IAAI,KAAK,WAAW,IAAI,QAAQ,KAAK;AACrE,QAAM,SAAS,WAAW,IAAI,MAAM,MAAM,aAAa,aAAa;AACpE,QAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,YAAY,KAAK;AAC9E,QAAM,OAAqB;AAAA,IACzB,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,OAAO,WAAW,IAAI,KAAK,KAAK;AAAA,IAChC,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,KAAK;AAAA,MACf,SAAS,SAAS,IAAI;AAAA;AAAA,EAAgB,aAAa,QAAQ,CAAC,KAAK;AAAA,IACnE,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,eAAe,WAAW;AAC7C,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,wDAAmD;AAAA,EAC7F;AACA,SAAO;AACT;AAEA,SAAS,oBAAoB,UAAkC;AAC7D,QAAM,aAAa,SAAS,YAAY,EAAE,QAAQ,eAAe,GAAG,EAAE,KAAK;AAC3E,MAAI,2EAA2E,KAAK,UAAU,GAAG;AAC/F,WAAO;AAAA,EACT;AACA,MAAI,4DAA4D,KAAK,UAAU,EAAG,QAAO;AACzF,SAAO;AACT;AAEA,eAAe,SAAS,MAAkC;AACxD,QAAM,OAAO,MAAM,aAAa,IAAI;AACpC,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,wBAAwB,IAAI,EAAE;AACrE,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,mBAAmB;AACvF,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,UAAU;AAC1D;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM;AAC/B;AAAA,MACF;AACA,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,SAAS,IAAI;AAAA,IACrB;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,OAAO,SAAS;AACtB,UAAI,CAAC,KAAM,OAAM,IAAI,MAAM,uDAAuD;AAClF,aAAO,aAAa,MAAM,SAAS,IAAI,GAAG,IAAI,SAAS,GAAG,MAAM,UAAU;AAAA,IAC5E;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,aAAO,SAAS,IAAI,EAAE,aAAa,eAAe;AAAA,IACpD;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAM,oBAAoB,QAAQ;AACxC,YAAM,WAAW,QAAQ,KAAK;AAC9B,aAAO;AAAA,QACL;AAAA,QACA,OAAO,WAAW,IAAI;AAAA,QACtB,QAAQ,KAAK,UAAU;AAAA,UACrB,WAAW,KAAK;AAAA,UAChB,UAAU,KAAK;AAAA,UACf,QAAQ,KAAK;AAAA,UACb,UAAU,KAAK,aAAa,eAAe;AAAA,UAC3C,KAAK,QAAQ,OAAO,OAAO,MAAM,eAAe;AAAA,UAChD,cAAc,KAAK,SAAS;AAAA,QAC9B,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}