@tangle-network/agent-bench 0.4.1 → 0.4.4

package/dist/benchmarks/humaneval.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"humaneval.js","names":[],"sources":["../../src/benchmarks/humaneval.ts"],"sourcesContent":["/**\n * HumanEval adapter — the deployable-checker domain as a `BenchmarkAdapter`, so the\n * gate runner (`runGate`) can A/B the STEERING regime on it: a real rollout\n * through the `Supervisor` that self-corrects across rounds, vs blind\n * random@k. This is the experiment `humaneval-gate.mts` names as \"the next one\" —\n * the gate measures SELECTION over stateless single completions; this measures\n * whether observe→steer (self-correction) beats blind compute at equal k.\n *\n * Worker artifact = the model's reply (a Python function, fenced or raw). The\n * DETERMINISTIC judge runs the candidate against the task's own `test` in an\n * isolated `--network=none` python container — exit 0 = pass. No gold\n * `canonical_solution` is ever shown to the model; `goldArtifact` returns it only\n * to self-verify the judge before spending tokens.\n *\n * The primitives (loader / extractor / Docker checker) live here and are reused by\n * `humaneval-gate.mts` — one home, no duplication.\n */\n\nimport { execFile } from 'node:child_process'\nimport { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { gunzipSync } from 'node:zlib'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst humanevalUrl = 'https://github.com/openai/human-eval/raw/master/data/HumanEval.jsonl.gz'\nconst dockerImage = 'python:3.12-slim'\nconst dockerTimeoutMs = Number(process.env.DOCKER_TIMEOUT_MS ?? 20000)\n\nexport interface HumanEvalTask {\n  taskId: string\n  prompt: string\n  test: string\n  entryPoint: string\n  /** Reference solution body — used ONLY to self-verify the judge, never shown to the worker. */\n  canonicalSolution?: string\n}\n\n/** Pull the 164-task HumanEval JSONL.gz and parse it. Fail loud on a non-OK fetch\n *  or a malformed line — a silently-short task set would poison the gate. `offset`\n *  selects a deeper slice (the later tasks are harder) so the worker has a\n *  correctable middle band rather than a saturated easy prefix. */\nexport async function loadHumanEval(limit: number, offset = 0): Promise<HumanEvalTask[]> {\n  // Prefer a locally-cached .jsonl.gz (HUMANEVAL_GZ) — the GitHub raw URL rate-limits\n  // (429) under repeated runs. Fall back to the network fetch when unset.\n  const localGz = process.env.HUMANEVAL_GZ\n  let gz: Buffer\n  if (localGz) {\n    gz = readFileSync(localGz)\n  } else {\n    const res = await fetch(humanevalUrl)\n    if (!res.ok) throw new Error(`HumanEval fetch HTTP ${res.status}: ${humanevalUrl}`)\n    gz = Buffer.from(await res.arrayBuffer())\n  }\n  const text = gunzipSync(gz).toString('utf8')\n  const tasks: HumanEvalTask[] = []\n  for (const line of text.split('\\n')) {\n    if (line.trim() === '') continue\n    const d = JSON.parse(line) as {\n      task_id?: string\n      prompt?: string\n      test?: string\n      entry_point?: string\n      canonical_solution?: string\n    }\n    if (!d.task_id || !d.prompt || !d.test || !d.entry_point) {\n      throw new Error(`malformed HumanEval record: ${line.slice(0, 120)}`)\n    }\n    tasks.push({\n      taskId: d.task_id,\n      prompt: d.prompt,\n      test: d.test,\n      entryPoint: d.entry_point,\n      ...(d.canonical_solution ? { canonicalSolution: d.canonical_solution } : {}),\n    })\n  }\n  if (tasks.length === 0) throw new Error('HumanEval parsed to 0 tasks')\n  if (offset >= tasks.length) throw new Error(`OFFSET ${offset} >= dataset size ${tasks.length}`)\n  return tasks.slice(offset, offset + limit)\n}\n\nconst solveInstruction =\n  'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.'\n\nexport function basePrompt(task: HumanEvalTask): string {\n  return `${solveInstruction}\\n\\n\\`\\`\\`python\\n${task.prompt}\\`\\`\\``\n}\n\n/** Extract the function source from a model reply: prefer a fenced ```python (or\n *  bare ```) block, else fall back to the raw text. The deployable program adds the\n *  prompt header (imports + signature context), so a candidate that returns only a\n *  body still runs; a candidate that re-defines the function shadows the header. */\nexport function extractCode(reply: string): string {\n  const fenced = reply.match(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i)\n  if (fenced && typeof fenced[1] === 'string') return fenced[1].trim()\n  return reply.trim()\n}\n\n/** The deployable test program: the prompt header (imports + signature/docstring the\n *  model was given), then the candidate (its def shadows the header's stub), then the\n *  task's own check() suite and the call. No gold solution anywhere. */\nfunction buildProgram(task: HumanEvalTask, candidate: string): string {\n  return `${task.prompt}\\n${candidate}\\n\\n${task.test}\\n\\ncheck(${task.entryPoint})\\n`\n}\n\nexport interface CheckResult {\n  /** {0,1} pass-count for this candidate (1 = the check() suite passed). */\n  pass: number\n  /** On failure: the interpreter stderr tail (traceback / failing assertion). The\n   *  execution-grounded feedback a self-repair loop steers on; ignored by selection. */\n  detail?: string\n}\n\nexport interface PythonProgramResult {\n  exitCode: number\n  stdout: string\n  stderr: string\n}\n\n/** Run one candidate's deployable test program in an isolated container:\n *  read-only root, bounded writable `/tmp`, no network, one CPU, and 512 MiB.\n *  Exit 0 → pass. A docker invocation error (binary missing, daemon down, image\n *  unavailable) is NOT a test failure — it throws so the harness fails loud rather\n *  than scoring every candidate 0 from a broken checker. */\nlet dockerRunSeq = 0\n\n/** Run arbitrary Python in the same isolated container used by the HumanEval\n * checker. This is also the execution-tool primitive for experiments that let\n * a model test its own snippets; model-written code must never run on the host. */\nexport function runPythonProgram(\n  program: string,\n  timeoutMs = dockerTimeoutMs,\n): Promise<PythonProgramResult> {\n  const dir = mkdtempSync(join(tmpdir(), 'hev-'))\n  // Rootless Docker maps container root to a subordinate host uid that cannot\n  // traverse mkdtemp's default 0700 directory. The directory contains only the\n  // disposable candidate script and is mounted read-only into the container.\n  chmodSync(dir, 0o755)\n  writeFileSync(join(dir, 'p.py'), program, { mode: 0o644 })\n  // Unique container name so we can force-reap it regardless of the docker client's state.\n  const name = `hev-${process.pid}-${dockerRunSeq++}`\n  const startMarker = `__AGENT_RUNTIME_CANDIDATE_STARTED_${name}__`\n  return new Promise<PythonProgramResult>((resolvePromise, reject) => {\n    let settled = false\n    const cleanup = () => {\n      rmSync(dir, { recursive: true, force: true })\n      // `execFile`'s `timeout` kills the docker CLIENT, not the container — a hung\n      // `python` would otherwise pin a CPU forever. Force-reap by name (fire-and-forget;\n      // the name is unique, so no reuse race).\n      execFile('docker', ['rm', '-f', name], () => {})\n    }\n    const finish = (res: PythonProgramResult) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      resolvePromise(res)\n    }\n    const fail = (e: Error) => {\n      if (settled) return\n      settled = true\n      clearTimeout(backstop)\n      cleanup()\n      reject(e)\n    }\n    // Candidate timeouts are enforced inside the container. If the outer Docker\n    // client or daemon misses this larger deadline, that is infrastructure failure,\n    // not a wrong answer.\n    const outerTimeoutMs = timeoutMs + 3_000\n    const backstop = setTimeout(\n      () => fail(new Error(`docker checker did not return within ${outerTimeoutMs + 3_000}ms`)),\n      outerTimeoutMs + 3_000,\n    )\n    const inContainerSeconds = Math.max(1, Math.ceil(timeoutMs / 1_000))\n    execFile(\n      'docker',\n      [\n        'run',\n        '--rm',\n        '--pull=never',\n        '--name',\n        name,\n        '--network=none',\n        '--cpus=1',\n        '--memory=512m',\n        '--pids-limit=64',\n        '--cap-drop=ALL',\n        '--security-opt=no-new-privileges',\n        '--read-only',\n        '--tmpfs',\n        '/tmp:rw,nosuid,nodev,noexec,size=64m,mode=1777',\n        '--user',\n        '65534:65534',\n        '-v',\n        `${dir}:/w:ro`,\n        '-w',\n        '/w',\n        dockerImage,\n        'sh',\n        '-c',\n        'command -v timeout >/dev/null 2>&1 && command -v python >/dev/null 2>&1 && [ -r /w/p.py ] || exit 126; printf \"%s\\\\n\" \"$1\"; exec timeout -s KILL \"$2\" python /w/p.py',\n        'agent-runtime-checker',\n        startMarker,\n        `${inContainerSeconds}s`,\n      ],\n      { timeout: outerTimeoutMs, killSignal: 'SIGKILL', maxBuffer: 4 * 1024 * 1024 },\n      (err, stdout, stderr) => {\n        const markerLine = `${startMarker}\\n`\n        const candidateStarted = stdout?.startsWith(markerLine) === true\n        const candidateStdout = candidateStarted ? stdout.slice(markerLine.length) : (stdout ?? '')\n        if (err) {\n          const e = err as Error & { killed?: boolean; code?: number | string }\n          if (e.code === 'ENOENT') {\n            fail(new Error('docker binary not found on PATH — cannot run the deployable checker'))\n            return\n          }\n          if (e.killed) {\n            fail(new Error(`docker checker exceeded its ${outerTimeoutMs}ms outer timeout`))\n            return\n          }\n          if (!candidateStarted) {\n            if (\n              /cannot connect to the docker daemon|is the docker daemon running|permission denied while trying to connect/i.test(\n                stderr,\n              )\n            ) {\n              fail(new Error(`docker daemon unreachable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            if (/unable to find image|no such image|pull access denied|manifest unknown/i.test(stderr)) {\n              fail(new Error(`docker image ${dockerImage} unavailable: ${stderr.slice(0, 200)}`))\n              return\n            }\n            fail(\n              new Error(\n                `docker checker did not start the candidate: ${stderr.slice(0, 200) || e.message}`,\n              ),\n            )\n            return\n          }\n          const exitCode = typeof e.code === 'number' ? e.code : 1\n          finish({\n            exitCode,\n            stdout: candidateStdout,\n            stderr: stderr ?? '',\n          })\n          return\n        }\n        if (!candidateStarted) {\n          fail(new Error('docker checker exited without starting the candidate'))\n          return\n        }\n        finish({ exitCode: 0, stdout: candidateStdout, stderr: stderr ?? '' })\n      },\n    )\n  })\n}\n\nexport async function runChecker(task: HumanEvalTask, candidate: string): Promise<CheckResult> {\n  const result = await runPythonProgram(buildProgram(task, candidate))\n  return result.exitCode === 0\n    ? { pass: 1 }\n    : { pass: 0, detail: result.stderr.slice(-600) || 'timed out (no output)' }\n}\n\n/** A HumanEval task carries its checker inputs in metadata so the deterministic\n *  judge can rebuild the deployable program from a `BenchTask` alone. */\ninterface HumanEvalMeta extends Record<string, unknown> {\n  promptHeader: string\n  test: string\n  entryPoint: string\n  canonicalSolution?: string\n}\n\nfunction toBenchTask(t: HumanEvalTask): BenchTask {\n  const metadata: HumanEvalMeta = {\n    promptHeader: t.prompt,\n    test: t.test,\n    entryPoint: t.entryPoint,\n    ...(t.canonicalSolution ? { canonicalSolution: t.canonicalSolution } : {}),\n  }\n  return { id: t.taskId, prompt: basePrompt(t), metadata }\n}\n\nfunction taskFromMeta(task: BenchTask): HumanEvalTask {\n  const m = task.metadata as HumanEvalMeta | undefined\n  if (!m?.promptHeader || !m.test || !m.entryPoint) {\n    throw new Error(`HumanEval judge: task ${task.id} missing checker metadata`)\n  }\n  return { taskId: task.id, prompt: m.promptHeader, test: m.test, entryPoint: m.entryPoint }\n}\n\n/** The HumanEval `BenchmarkAdapter`. OFFSET (env) selects the correctable middle\n *  band; loadTasks honors `limit`/`ids`. The judge is the Docker deployable checker. */\nexport function createHumanEvalAdapter(): BenchmarkAdapter {\n  return {\n    name: 'humaneval',\n    async preflight() {\n      // Prove the daemon and exact local image before any model call. Scoring uses\n      // --pull=never so a registry or cold pull cannot become a candidate outcome.\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['version', '--format', '{{.Server.Version}}'], (err) => {\n          if (err) reject(new Error('HumanEval judge needs a running Docker daemon (python:3.12-slim, --network=none)'))\n          else resolve()\n        })\n      })\n      await new Promise<void>((resolve, reject) => {\n        execFile('docker', ['image', 'inspect', dockerImage], (err) => {\n          if (err) reject(new Error(`HumanEval judge needs the cached Docker image ${dockerImage}`))\n          else resolve()\n        })\n      })\n    },\n    async loadTasks(opts?: LoadOptions) {\n      const offset = Number(process.env.OFFSET ?? 0)\n      // Pull a generous window when filtering by id, else exactly `limit` from offset.\n      const all = await loadHumanEval(opts?.ids ? 164 : (opts?.limit ?? 8), offset)\n      const picked = opts?.ids ? all.filter((t) => opts.ids?.includes(t.taskId)) : all\n      return picked.map(toBenchTask)\n    },\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { pass } = await runChecker(taskFromMeta(task), extractCode(artifact))\n      return { resolved: pass === 1, score: pass, detail: pass === 1 ? 'tests passed' : 'tests failed' }\n    },\n    async goldArtifact(task: BenchTask) {\n      const m = task.metadata as HumanEvalMeta | undefined\n      const sol = m?.canonicalSolution\n      // Return the COMPLETE function (signature header + canonical body), i.e. what a\n      // real worker emits — NOT the body alone. The judge runs `extractCode`, whose\n      // unfenced fallback is `reply.trim()`; trimming a body-only string strips its\n      // leading indent and breaks it, so a body-only gold fails its own judge. A full\n      // def starts at column 0, trims safely, and self-verifies.\n      return sol ? `${m!.promptHeader}${sol}` : undefined\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAyBA,MAAM,eAAe;AACrB,MAAM,cAAc;AACpB,MAAM,kBAAkB,OAAO,QAAQ,IAAI,qBAAqB,GAAK;;;;;AAerE,eAAsB,cAAc,OAAe,SAAS,GAA6B;CAGvF,MAAM,UAAU,QAAQ,IAAI;CAC5B,IAAI;CACJ,IAAI,SACF,KAAK,aAAa,OAAO;MACpB;EACL,MAAM,MAAM,MAAM,MAAM,YAAY;EACpC,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,wBAAwB,IAAI,OAAO,IAAI,cAAc;EAClF,KAAK,OAAO,KAAK,MAAM,IAAI,YAAY,CAAC;CAC1C;CACA,MAAM,OAAO,WAAW,EAAE,CAAC,CAAC,SAAS,MAAM;CAC3C,MAAM,QAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,KAAK,MAAM,IAAI,GAAG;EACnC,IAAI,KAAK,KAAK,MAAM,IAAI;EACxB,MAAM,IAAI,KAAK,MAAM,IAAI;EAOzB,IAAI,CAAC,EAAE,WAAW,CAAC,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,EAAE,aAC3C,MAAM,IAAI,MAAM,+BAA+B,KAAK,MAAM,GAAG,GAAG,GAAG;EAErE,MAAM,KAAK;GACT,QAAQ,EAAE;GACV,QAAQ,EAAE;GACV,MAAM,EAAE;GACR,YAAY,EAAE;GACd,GAAI,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,mBAAmB,IAAI,CAAC;EAC5E,CAAC;CACH;CACA,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,6BAA6B;CACrE,IAAI,UAAU,MAAM,QAAQ,MAAM,IAAI,MAAM,UAAU,OAAO,mBAAmB,MAAM,QAAQ;CAC9F,OAAO,MAAM,MAAM,QAAQ,SAAS,KAAK;AAC3C;AAEA,MAAM,mBACJ;AAEF,SAAgB,WAAW,MAA6B;CACtD,OAAO,GAAG,iBAAiB,oBAAoB,KAAK,OAAO;AAC7D;;;;;AAMA,SAAgB,YAAY,OAAuB;CACjD,MAAM,SAAS,MAAM,MAAM,sCAAsC;CACjE,IAAI,UAAU,OAAO,OAAO,OAAO,UAAU,OAAO,OAAO,EAAE,CAAC,KAAK;CACnE,OAAO,MAAM,KAAK;AACpB;;;;AAKA,SAAS,aAAa,MAAqB,WAA2B;CACpE,OAAO,GAAG,KAAK,OAAO,IAAI,UAAU,MAAM,KAAK,KAAK,YAAY,KAAK,WAAW;AAClF;;;;;;AAqBA,IAAI,eAAe;;;;AAKnB,SAAgB,iBACd,SACA,YAAY,iBACkB;CAC9B,MAAM,MAAM,YAAY,KAAK,OAAO,GAAG,MAAM,CAAC;CAI9C,UAAU,KAAK,GAAK;CACpB,cAAc,KAAK,KAAK,MAAM,GAAG,SAAS,EAAE,MAAM,IAAM,CAAC;CAEzD,MAAM,OAAO,OAAO,QAAQ,IAAI,GAAG;CACnC,MAAM,cAAc,qCAAqC,KAAK;CAC9D,OAAO,IAAI,SAA8B,gBAAgB,WAAW;EAClE,IAAI,UAAU;EACd,MAAM,gBAAgB;GACpB,OAAO,KAAK;IAAE,WAAW;IAAM,OAAO;GAAK,CAAC;GAI5C,SAAS,UAAU;IAAC;IAAM;IAAM;GAAI,SAAS,CAAC,CAAC;EACjD;EACA,MAAM,UAAU,QAA6B;GAC3C,IAAI,SAAS;GACb,UAAU;GACV,aAAa,QAAQ;GACrB,QAAQ;GACR,eAAe,GAAG;EACpB;EACA,MAAM,QAAQ,MAAa;GACzB,IAAI,SAAS;GACb,UAAU;GACV,aAAa,QAAQ;GACrB,QAAQ;GACR,OAAO,CAAC;EACV;EAIA,MAAM,iBAAiB,YAAY;EACnC,MAAM,WAAW,iBACT,qBAAK,IAAI,MAAM,wCAAwC,iBAAiB,IAAM,GAAG,CAAC,GACxF,iBAAiB,GACnB;EACA,MAAM,qBAAqB,KAAK,IAAI,GAAG,KAAK,KAAK,YAAY,GAAK,CAAC;EACnE,SACE,UACA;GACE;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,GAAG,IAAI;GACP;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,GAAG,mBAAmB;EACxB,GACA;GAAE,SAAS;GAAgB,YAAY;GAAW,WAAW,IAAI,OAAO;EAAK,IAC5E,KAAK,QAAQ,WAAW;GACvB,MAAM,aAAa,GAAG,YAAY;GAClC,MAAM,mBAAmB,QAAQ,WAAW,UAAU,MAAM;GAC5D,MAAM,kBAAkB,mBAAmB,OAAO,MAAM,WAAW,MAAM,IAAK,UAAU;GACxF,IAAI,KAAK;IACP,MAAM,IAAI;IACV,IAAI,EAAE,SAAS,UAAU;KACvB,qBAAK,IAAI,MAAM,qEAAqE,CAAC;KACrF;IACF;IACA,IAAI,EAAE,QAAQ;KACZ,qBAAK,IAAI,MAAM,+BAA+B,eAAe,iBAAiB,CAAC;KAC/E;IACF;IACA,IAAI,CAAC,kBAAkB;KACrB,IACE,8GAA8G,KAC5G,MACF,GACA;MACA,qBAAK,IAAI,MAAM,8BAA8B,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC;MACpE;KACF;KACA,IAAI,0EAA0E,KAAK,MAAM,GAAG;MAC1F,qBAAK,IAAI,MAAM,gBAAgB,YAAY,gBAAgB,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC;MAClF;KACF;KACA,qBACE,IAAI,MACF,+CAA+C,OAAO,MAAM,GAAG,GAAG,KAAK,EAAE,SAC3E,CACF;KACA;IACF;IACA,MAAM,WAAW,OAAO,EAAE,SAAS,WAAW,EAAE,OAAO;IACvD,OAAO;KACL;KACA,QAAQ;KACR,QAAQ,UAAU;IACpB,CAAC;IACD;GACF;GACA,IAAI,CAAC,kBAAkB;IACrB,qBAAK,IAAI,MAAM,sDAAsD,CAAC;IACtE;GACF;GACA,OAAO;IAAE,UAAU;IAAG,QAAQ;IAAiB,QAAQ,UAAU;GAAG,CAAC;EACvE,CACF;CACF,CAAC;AACH;AAEA,eAAsB,WAAW,MAAqB,WAAyC;CAC7F,MAAM,SAAS,MAAM,iBAAiB,aAAa,MAAM,SAAS,CAAC;CACnE,OAAO,OAAO,aAAa,IACvB,EAAE,MAAM,EAAE,IACV;EAAE,MAAM;EAAG,QAAQ,OAAO,OAAO,MAAM,IAAI,KAAK;CAAwB;AAC9E;AAWA,SAAS,YAAY,GAA6B;CAChD,MAAM,WAA0B;EAC9B,cAAc,EAAE;EAChB,MAAM,EAAE;EACR,YAAY,EAAE;EACd,GAAI,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,kBAAkB,IAAI,CAAC;CAC1E;CACA,OAAO;EAAE,IAAI,EAAE;EAAQ,QAAQ,WAAW,CAAC;EAAG;CAAS;AACzD;AAEA,SAAS,aAAa,MAAgC;CACpD,MAAM,IAAI,KAAK;CACf,IAAI,CAAC,GAAG,gBAAgB,CAAC,EAAE,QAAQ,CAAC,EAAE,YACpC,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,0BAA0B;CAE7E,OAAO;EAAE,QAAQ,KAAK;EAAI,QAAQ,EAAE;EAAc,MAAM,EAAE;EAAM,YAAY,EAAE;CAAW;AAC3F;;;AAIA,SAAgB,yBAA2C;CACzD,OAAO;EACL,MAAM;EACN,MAAM,YAAY;GAGhB,MAAM,IAAI,SAAe,SAAS,WAAW;IAC3C,SAAS,UAAU;KAAC;KAAW;KAAY;IAAqB,IAAI,QAAQ;KAC1E,IAAI,KAAK,uBAAO,IAAI,MAAM,kFAAkF,CAAC;UACxG,QAAQ;IACf,CAAC;GACH,CAAC;GACD,MAAM,IAAI,SAAe,SAAS,WAAW;IAC3C,SAAS,UAAU;KAAC;KAAS;KAAW;IAAW,IAAI,QAAQ;KAC7D,IAAI,KAAK,uBAAO,IAAI,MAAM,iDAAiD,aAAa,CAAC;UACpF,QAAQ;IACf,CAAC;GACH,CAAC;EACH;EACA,MAAM,UAAU,MAAoB;GAClC,MAAM,SAAS,OAAO,QAAQ,IAAI,UAAU,CAAC;GAE7C,MAAM,MAAM,MAAM,cAAc,MAAM,MAAM,MAAO,MAAM,SAAS,GAAI,MAAM;GAE5E,QADe,MAAM,MAAM,IAAI,QAAQ,MAAM,KAAK,KAAK,SAAS,EAAE,MAAM,CAAC,IAAI,IAAA,CAC/D,IAAI,WAAW;EAC/B;EACA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,EAAE,SAAS,MAAM,WAAW,aAAa,IAAI,GAAG,YAAY,QAAQ,CAAC;GAC3E,OAAO;IAAE,UAAU,SAAS;IAAG,OAAO;IAAM,QAAQ,SAAS,IAAI,iBAAiB;GAAe;EACnG;EACA,MAAM,aAAa,MAAiB;GAClC,MAAM,IAAI,KAAK;GACf,MAAM,MAAM,GAAG;GAMf,OAAO,MAAM,GAAG,EAAG,eAAe,QAAQ,KAAA;EAC5C;CACF;AACF"}

package/dist/benchmarks/mind2web.d.ts

@@ -1,41 +1,14 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).
- * Web-agent ACTION PREDICTION: each task is one step of a real web task — given
- * the natural-language goal and the page's candidate elements, the worker picks
- * the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).
- *
- * Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:
- *   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids
- *   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches
- *   resolved (Step-SR) ⇔ element correct AND operation correct
- *   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right
- *           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)
- * This is the low-noise reward a certifiable directive-lift needs: the number is a
- * programmatic match against human-verified ground truth, not a judge's opinion.
- *
- * The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the
- * judge extracts deterministically; the prompt presents the candidate set as a
- * choice over backend_node_id ordered by id (position uncorrelated with the answer).
- *
- * Each step carries the dataset's OWN page screenshot (written to a temp file by
- * the loader); the worker drops it into a browser.<op> span so run-capsule's screen
- * capsule turns the run into a film — the real page, not a re-rendered DOM.
- *
- * Requires for a live run: a python with `datasets` + `pillow` and network to
- * Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/mind2web.d.ts
 interface ParsedAction {
-    elementId: string;
-    op: string;
-    value: string;
+  elementId: string;
+  op: string;
+  value: string;
 }
 /** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.
  *  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */
 declare function parseAction(artifact: string): ParsedAction | null;
 declare function createMind2WebAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createMind2WebAdapter, parseAction };
+//# sourceMappingURL=mind2web.d.ts.map

package/dist/benchmarks/mind2web.js

@@ -1,9 +1,258 @@
-import {
-  createMind2WebAdapter,
-  parseAction
-} from "../chunk-TBKU5XQI.js";
-export {
-  createMind2WebAdapter,
-  parseAction
-};
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { execFile } from "node:child_process";
+import { tmpdir } from "node:os";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+//#region src/benchmarks/mind2web.ts
+/**
+* Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).
+* Web-agent ACTION PREDICTION: each task is one step of a real web task — given
+* the natural-language goal and the page's candidate elements, the worker picks
+* the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).
+*
+* Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:
+*   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids
+*   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches
+*   resolved (Step-SR) ⇔ element correct AND operation correct
+*   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right
+*           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)
+* This is the low-noise reward a certifiable directive-lift needs: the number is a
+* programmatic match against human-verified ground truth, not a judge's opinion.
+*
+* The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the
+* judge extracts deterministically; the prompt presents the candidate set as a
+* choice over backend_node_id ordered by id (position uncorrelated with the answer).
+*
+* Each step carries the dataset's OWN page screenshot (written to a temp file by
+* the loader); the worker drops it into a browser.<op> span so run-capsule's screen
+* capsule turns the run into a film — the real page, not a re-rendered DOM.
+*
+* Requires for a live run: a python with `datasets` + `pillow` and network to
+* Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).
+*/
+const execFileAsync = promisify(execFile);
+const BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+const PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, ".venv", "bin", "python");
+const DATASET = "osunlp/Multimodal-Mind2Web";
+const DEFAULT_SPLIT = process.env.M2W_SPLIT ?? "test_domain";
+const SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), "m2w-shots");
+/** Candidate-set cap presented to the worker (always keeps all pos candidates). */
+const CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30);
+/** The worker contract appended to every task prompt; the judge keys off these. */
+const WORKER_CONTRACT = [
+	"",
+	"Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:",
+	"ELEMENT: <the [id] number of the chosen element>",
+	"ACTION: <CLICK | TYPE | SELECT>",
+	"VALUE: <text to type or option to select; leave empty for CLICK>"
+].join("\n");
+/** Run the bench python with a script on stdin; return stdout (throws on nonzero). */
+async function py(script, args = []) {
+	const { stdout } = await execFileAsync(PY, [
+		"-c",
+		script,
+		...args
+	], { maxBuffer: 1024 * 1024 * 256 });
+	return stdout;
+}
+/** The loader: stream the split, skip steps with no positive candidate (fail-loud,
+*  never a silent score-0), cap the candidate set, save each screenshot to a file. */
+const LOADER = `
+import json, sys, os
+from datasets import load_dataset
+cfg = json.loads(sys.argv[1])
+split = cfg["split"]; limit = cfg.get("limit"); cap = cfg.get("cap", 30)
+shots = cfg["shotsDir"]; ids = set(cfg["ids"]) if cfg.get("ids") else None
+os.makedirs(shots, exist_ok=True)
+KEEP = ("aria_label","aria-label","role","type","name","placeholder","title","alt","value","text","id","class","href")
+def label(tag, attr):
+    try:
+        a = json.loads(attr) if isinstance(attr, str) else (attr or {})
+    except Exception:
+        a = {}
+    parts = []
+    for k in KEEP:
+        v = a.get(k)
+        if v:
+            sv = str(v).replace("\\n", " ").strip()
+            if k == "class": sv = sv[:40]
+            if k == "href": sv = sv[:50]
+            if sv: parts.append(k + "=" + sv[:60])
+    return "<" + str(tag) + "> " + " ".join(parts[:6])
+def cands(raw):
+    out = []
+    for c in raw or []:
+        try:
+            d = json.loads(c) if isinstance(c, str) else c
+        except Exception:
+            continue
+        bid = str(d.get("backend_node_id", ""))
+        if not bid: continue
+        out.append({"id": bid, "label": label(d.get("tag", "?"), d.get("attributes"))})
+    return out
+ds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)
+emitted = []; skipped = 0; scanned = 0
+for r in ds:
+    scanned += 1
+    if ids is not None and r["action_uid"] not in ids:
+        if scanned > 8000: break
+        continue
+    pos = cands(r.get("pos_candidates"))
+    if not pos:
+        skipped += 1
+        continue
+    neg = cands(r.get("neg_candidates"))
+    seen = set(p["id"] for p in pos); merged = list(pos)
+    for n in neg:
+        if len(merged) >= cap: break
+        if n["id"] in seen: continue
+        seen.add(n["id"]); merged.append(n)
+    merged.sort(key=lambda x: int(x["id"]) if x["id"].isdigit() else 0)
+    try:
+        op = json.loads(r["operation"]) if isinstance(r["operation"], str) else r["operation"]
+    except Exception:
+        op = {}
+    sp = os.path.join(shots, str(r["action_uid"]) + ".jpg")
+    try:
+        r["screenshot"].convert("RGB").save(sp, "JPEG", quality=70)
+    except Exception:
+        sp = ""
+    emitted.append({
+        "id": r["action_uid"], "task": r.get("confirmed_task", ""),
+        "website": r.get("website", ""), "domain": r.get("domain", ""), "subdomain": r.get("subdomain", ""),
+        "op": str(op.get("op", "")).upper(), "value": str(op.get("value", "")),
+        "goldIds": [p["id"] for p in pos], "candidates": merged,
+        "screenshotPath": sp, "targetRepr": r.get("target_action_reprs", ""),
+    })
+    if ids is None and limit is not None and len(emitted) >= limit: break
+    if ids is not None and len(emitted) >= len(ids): break
+sys.stderr.write("[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\n" % (len(emitted), skipped, scanned)); sys.stderr.flush()
+print(json.dumps(emitted)); sys.stdout.flush()
+os._exit(0)
+`;
+function buildPrompt(row) {
+	const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join("\n");
+	return [
+		`Web task: ${row.task}`,
+		"",
+		"You are taking the NEXT single action on the current web page. Choose the one element",
+		"to act on from the candidates below (each line is \"[id] <tag> attributes\"):",
+		choices,
+		WORKER_CONTRACT
+	].join("\n");
+}
+function rowToTask(row) {
+	const meta = {
+		task: row.task,
+		website: row.website,
+		domain: row.domain,
+		op: row.op,
+		value: row.value,
+		goldIds: row.goldIds,
+		candidateIds: row.candidates.map((c) => c.id),
+		screenshotPath: row.screenshotPath,
+		targetRepr: row.targetRepr
+	};
+	return {
+		id: `mind2web-${row.id}`,
+		split: DEFAULT_SPLIT,
+		prompt: buildPrompt(row),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.goldIds)) throw new Error(`mind2web task ${task.id} missing metadata.goldIds — loadTasks did not populate it`);
+	return md;
+}
+/** Normalize a TYPE/SELECT value for comparison: lowercase, collapse whitespace. */
+function normValue(s) {
+	return s.toLowerCase().replace(/\s+/g, " ").trim();
+}
+/** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.
+*  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */
+function parseAction(artifact) {
+	const elem = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact);
+	const op = /ACTION:\s*(CLICK|TYPE|SELECT)/i.exec(artifact);
+	if (!elem?.[1] || !op?.[1]) return null;
+	const val = /VALUE:\s*(.*)/i.exec(artifact);
+	return {
+		elementId: elem[1],
+		op: op[1].toUpperCase(),
+		value: (val?.[1] ?? "").trim()
+	};
+}
+function createMind2WebAdapter() {
+	const split = DEFAULT_SPLIT;
+	return {
+		name: "mind2web",
+		async preflight() {
+			try {
+				await py(`import os, sys
+from datasets import load_dataset
+import PIL
+ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)
+next(iter(ds))
+print('ok'); sys.stdout.flush(); os._exit(0)`);
+			} catch (err) {
+				const msg = err instanceof Error ? err.message : String(err);
+				throw new Error(`mind2web preflight failed: ${msg}\nFix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; (2) network access to Hugging Face for ${DATASET} (split ${split}) ; point M2W_PYTHON at the python if not bench/.venv.`);
+			}
+		},
+		async loadTasks(opts = {}) {
+			const cfg = {
+				split: opts.split ?? split,
+				limit: opts.ids ? void 0 : opts.limit ?? 10,
+				cap: CANDIDATE_CAP,
+				shotsDir: SHOTS_DIR,
+				ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, "")) : void 0
+			};
+			await mkdir(SHOTS_DIR, { recursive: true });
+			const stdout = await py(LOADER, [JSON.stringify(cfg)]);
+			return JSON.parse(stdout).map(rowToTask);
+		},
+		async goldArtifact(task) {
+			const meta = readMeta(task);
+			const id = meta.goldIds[0];
+			if (!id) return void 0;
+			return `ELEMENT: ${id}\nACTION: ${meta.op}\nVALUE: ${meta.value}`;
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const parsed = parseAction(artifact);
+			if (!parsed) return {
+				resolved: false,
+				score: 0,
+				detail: JSON.stringify({
+					reason: "no parseable ELEMENT/ACTION",
+					goldOp: meta.op,
+					goldIds: meta.goldIds
+				})
+			};
+			const elementCorrect = meta.goldIds.includes(parsed.elementId);
+			const opMatch = parsed.op === meta.op;
+			const valueMatch = meta.op === "CLICK" ? true : normValue(parsed.value) === normValue(meta.value);
+			const operationCorrect = opMatch && valueMatch;
+			return {
+				resolved: elementCorrect && operationCorrect,
+				score: .6 * (elementCorrect ? 1 : 0) + .4 * (operationCorrect ? 1 : 0),
+				detail: JSON.stringify({
+					elementCorrect,
+					opMatch,
+					valueMatch,
+					predicted: parsed,
+					goldOp: meta.op,
+					goldValue: meta.value,
+					goldIds: meta.goldIds,
+					website: meta.website,
+					domain: meta.domain
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createMind2WebAdapter, parseAction };
+
 //# sourceMappingURL=mind2web.js.map

package/dist/benchmarks/mind2web.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"mind2web.js","names":[],"sources":["../../src/benchmarks/mind2web.ts"],"sourcesContent":["/**\n * Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).\n * Web-agent ACTION PREDICTION: each task is one step of a real web task — given\n * the natural-language goal and the page's candidate elements, the worker picks\n * the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).\n *\n * Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:\n *   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids\n *   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches\n *   resolved (Step-SR) ⇔ element correct AND operation correct\n *   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right\n *           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)\n * This is the low-noise reward a certifiable directive-lift needs: the number is a\n * programmatic match against human-verified ground truth, not a judge's opinion.\n *\n * The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the\n * judge extracts deterministically; the prompt presents the candidate set as a\n * choice over backend_node_id ordered by id (position uncorrelated with the answer).\n *\n * Each step carries the dataset's OWN page screenshot (written to a temp file by\n * the loader); the worker drops it into a browser.<op> span so run-capsule's screen\n * capsule turns the run into a film — the real page, not a re-rendered DOM.\n *\n * Requires for a live run: a python with `datasets` + `pillow` and network to\n * Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdir } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, '.venv', 'bin', 'python')\n\nconst DATASET = 'osunlp/Multimodal-Mind2Web'\nconst DEFAULT_SPLIT = process.env.M2W_SPLIT ?? 'test_domain'\nconst SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), 'm2w-shots')\n/** Candidate-set cap presented to the worker (always keeps all pos candidates). */\nconst CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30)\n\n/** The worker contract appended to every task prompt; the judge keys off these. */\nconst WORKER_CONTRACT = [\n  '',\n  'Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:',\n  'ELEMENT: <the [id] number of the chosen element>',\n  'ACTION: <CLICK | TYPE | SELECT>',\n  'VALUE: <text to type or option to select; leave empty for CLICK>',\n].join('\\n')\n\ninterface RawCandidate {\n  id: string\n  label: string\n}\ninterface Mind2WebRow {\n  id: string\n  task: string\n  website: string\n  domain: string\n  subdomain: string\n  op: string\n  value: string\n  goldIds: string[]\n  candidates: RawCandidate[]\n  screenshotPath: string\n  targetRepr: string\n}\ninterface Mind2WebMeta {\n  task: string\n  website: string\n  domain: string\n  op: string\n  value: string\n  goldIds: string[]\n  candidateIds: string[]\n  screenshotPath: string\n  targetRepr: string\n}\n\n/** Run the bench python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], { maxBuffer: 1024 * 1024 * 256 })\n  return stdout\n}\n\n/** The loader: stream the split, skip steps with no positive candidate (fail-loud,\n *  never a silent score-0), cap the candidate set, save each screenshot to a file. */\nconst LOADER = `\nimport json, sys, os\nfrom datasets import load_dataset\ncfg = json.loads(sys.argv[1])\nsplit = cfg[\"split\"]; limit = cfg.get(\"limit\"); cap = cfg.get(\"cap\", 30)\nshots = cfg[\"shotsDir\"]; ids = set(cfg[\"ids\"]) if cfg.get(\"ids\") else None\nos.makedirs(shots, exist_ok=True)\nKEEP = (\"aria_label\",\"aria-label\",\"role\",\"type\",\"name\",\"placeholder\",\"title\",\"alt\",\"value\",\"text\",\"id\",\"class\",\"href\")\ndef label(tag, attr):\n    try:\n        a = json.loads(attr) if isinstance(attr, str) else (attr or {})\n    except Exception:\n        a = {}\n    parts = []\n    for k in KEEP:\n        v = a.get(k)\n        if v:\n            sv = str(v).replace(\"\\\\n\", \" \").strip()\n            if k == \"class\": sv = sv[:40]\n            if k == \"href\": sv = sv[:50]\n            if sv: parts.append(k + \"=\" + sv[:60])\n    return \"<\" + str(tag) + \"> \" + \" \".join(parts[:6])\ndef cands(raw):\n    out = []\n    for c in raw or []:\n        try:\n            d = json.loads(c) if isinstance(c, str) else c\n        except Exception:\n            continue\n        bid = str(d.get(\"backend_node_id\", \"\"))\n        if not bid: continue\n        out.append({\"id\": bid, \"label\": label(d.get(\"tag\", \"?\"), d.get(\"attributes\"))})\n    return out\nds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)\nemitted = []; skipped = 0; scanned = 0\nfor r in ds:\n    scanned += 1\n    if ids is not None and r[\"action_uid\"] not in ids:\n        if scanned > 8000: break\n        continue\n    pos = cands(r.get(\"pos_candidates\"))\n    if not pos:\n        skipped += 1\n        continue\n    neg = cands(r.get(\"neg_candidates\"))\n    seen = set(p[\"id\"] for p in pos); merged = list(pos)\n    for n in neg:\n        if len(merged) >= cap: break\n        if n[\"id\"] in seen: continue\n        seen.add(n[\"id\"]); merged.append(n)\n    merged.sort(key=lambda x: int(x[\"id\"]) if x[\"id\"].isdigit() else 0)\n    try:\n        op = json.loads(r[\"operation\"]) if isinstance(r[\"operation\"], str) else r[\"operation\"]\n    except Exception:\n        op = {}\n    sp = os.path.join(shots, str(r[\"action_uid\"]) + \".jpg\")\n    try:\n        r[\"screenshot\"].convert(\"RGB\").save(sp, \"JPEG\", quality=70)\n    except Exception:\n        sp = \"\"\n    emitted.append({\n        \"id\": r[\"action_uid\"], \"task\": r.get(\"confirmed_task\", \"\"),\n        \"website\": r.get(\"website\", \"\"), \"domain\": r.get(\"domain\", \"\"), \"subdomain\": r.get(\"subdomain\", \"\"),\n        \"op\": str(op.get(\"op\", \"\")).upper(), \"value\": str(op.get(\"value\", \"\")),\n        \"goldIds\": [p[\"id\"] for p in pos], \"candidates\": merged,\n        \"screenshotPath\": sp, \"targetRepr\": r.get(\"target_action_reprs\", \"\"),\n    })\n    if ids is None and limit is not None and len(emitted) >= limit: break\n    if ids is not None and len(emitted) >= len(ids): break\nsys.stderr.write(\"[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\\\n\" % (len(emitted), skipped, scanned)); sys.stderr.flush()\nprint(json.dumps(emitted)); sys.stdout.flush()\nos._exit(0)\n`\n\nfunction buildPrompt(row: Mind2WebRow): string {\n  const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join('\\n')\n  return [\n    `Web task: ${row.task}`,\n    '',\n    'You are taking the NEXT single action on the current web page. Choose the one element',\n    'to act on from the candidates below (each line is \"[id] <tag> attributes\"):',\n    choices,\n    WORKER_CONTRACT,\n  ].join('\\n')\n}\n\nfunction rowToTask(row: Mind2WebRow): BenchTask {\n  const meta: Mind2WebMeta = {\n    task: row.task,\n    website: row.website,\n    domain: row.domain,\n    op: row.op,\n    value: row.value,\n    goldIds: row.goldIds,\n    candidateIds: row.candidates.map((c) => c.id),\n    screenshotPath: row.screenshotPath,\n    targetRepr: row.targetRepr,\n  }\n  return {\n    id: `mind2web-${row.id}`,\n    split: DEFAULT_SPLIT,\n    prompt: buildPrompt(row),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): Mind2WebMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray((md as { goldIds?: unknown }).goldIds)) {\n    throw new Error(`mind2web task ${task.id} missing metadata.goldIds — loadTasks did not populate it`)\n  }\n  return md as unknown as Mind2WebMeta\n}\n\n/** Normalize a TYPE/SELECT value for comparison: lowercase, collapse whitespace. */\nfunction normValue(s: string): string {\n  return s.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\ninterface ParsedAction {\n  elementId: string\n  op: string\n  value: string\n}\n\n/** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.\n *  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */\nexport function parseAction(artifact: string): ParsedAction | null {\n  const elem = /ELEMENT:\\s*\\[?(\\d+)\\]?/i.exec(artifact)\n  const op = /ACTION:\\s*(CLICK|TYPE|SELECT)/i.exec(artifact)\n  if (!elem?.[1] || !op?.[1]) return null\n  const val = /VALUE:\\s*(.*)/i.exec(artifact)\n  return { elementId: elem[1], op: op[1].toUpperCase(), value: (val?.[1] ?? '').trim() }\n}\n\nexport function createMind2WebAdapter(): BenchmarkAdapter {\n  const split = DEFAULT_SPLIT\n\n  return {\n    name: 'mind2web',\n\n    async preflight() {\n      try {\n        await py(\n          `import os, sys\nfrom datasets import load_dataset\nimport PIL\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)\nnext(iter(ds))\nprint('ok'); sys.stdout.flush(); os._exit(0)`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `mind2web preflight failed: ${msg}\\n` +\n            `Fix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; ` +\n            `(2) network access to Hugging Face for ${DATASET} (split ${split}) ; ` +\n            `point M2W_PYTHON at the python if not bench/.venv.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const cfg = {\n        split: opts.split ?? split,\n        limit: opts.ids ? undefined : (opts.limit ?? 10),\n        cap: CANDIDATE_CAP,\n        shotsDir: SHOTS_DIR,\n        ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, '')) : undefined,\n      }\n      await mkdir(SHOTS_DIR, { recursive: true })\n      const stdout = await py(LOADER, [JSON.stringify(cfg)])\n      const rows = JSON.parse(stdout) as Mind2WebRow[]\n      return rows.map(rowToTask)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the worker-contract serialization of the ground-truth action, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes. The first accepted positive id is the canonical target.\n      const meta = readMeta(task)\n      const id = meta.goldIds[0]\n      if (!id) return undefined\n      return `ELEMENT: ${id}\\nACTION: ${meta.op}\\nVALUE: ${meta.value}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const parsed = parseAction(artifact)\n      if (!parsed) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ reason: 'no parseable ELEMENT/ACTION', goldOp: meta.op, goldIds: meta.goldIds }),\n        }\n      }\n      const elementCorrect = meta.goldIds.includes(parsed.elementId)\n      const opMatch = parsed.op === meta.op\n      const valueMatch = meta.op === 'CLICK' ? true : normValue(parsed.value) === normValue(meta.value)\n      const operationCorrect = opMatch && valueMatch\n      const resolved = elementCorrect && operationCorrect\n      const score = 0.6 * (elementCorrect ? 1 : 0) + 0.4 * (operationCorrect ? 1 : 0)\n      return {\n        resolved,\n        score,\n        detail: JSON.stringify({\n          elementCorrect,\n          opMatch,\n          valueMatch,\n          predicted: parsed,\n          goldOp: meta.op,\n          goldValue: meta.value,\n          goldIds: meta.goldIds,\n          website: meta.website,\n          domain: meta.domain,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmCA,MAAM,gBAAgB,UAAU,QAAQ;AACxC,MAAM,aAAa,cAAc,IAAI,IAAI,SAAS,OAAO,KAAK,GAAG,CAAC;AAClE,MAAM,KAAK,QAAQ,IAAI,cAAc,KAAK,YAAY,SAAS,OAAO,QAAQ;AAE9E,MAAM,UAAU;AAChB,MAAM,gBAAgB,QAAQ,IAAI,aAAa;AAC/C,MAAM,YAAY,QAAQ,IAAI,aAAa,KAAK,OAAO,GAAG,WAAW;;AAErE,MAAM,gBAAgB,OAAO,QAAQ,IAAI,qBAAqB,EAAE;;AAGhE,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;;AAgCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;CACtE,MAAM,EAAE,WAAW,MAAM,cAAc,IAAI;EAAC;EAAM;EAAQ,GAAG;CAAI,GAAG,EAAE,WAAW,OAAO,OAAO,IAAI,CAAC;CACpG,OAAO;AACT;;;AAIA,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAiCK,KAAK,UAAU,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyC5C,SAAS,YAAY,KAA0B;CAC7C,MAAM,UAAU,IAAI,WAAW,KAAK,MAAM,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,IAAI;CAC7E,OAAO;EACL,aAAa,IAAI;EACjB;EACA;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAA6B;CAC9C,MAAM,OAAqB;EACzB,MAAM,IAAI;EACV,SAAS,IAAI;EACb,QAAQ,IAAI;EACZ,IAAI,IAAI;EACR,OAAO,IAAI;EACX,SAAS,IAAI;EACb,cAAc,IAAI,WAAW,KAAK,MAAM,EAAE,EAAE;EAC5C,gBAAgB,IAAI;EACpB,YAAY,IAAI;CAClB;CACA,OAAO;EACL,IAAI,YAAY,IAAI;EACpB,OAAO;EACP,QAAQ,YAAY,GAAG;EACvB,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAS,GAA6B,OAAO,GAC7D,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,0DAA0D;CAErG,OAAO;AACT;;AAGA,SAAS,UAAU,GAAmB;CACpC,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;AACnD;;;AAUA,SAAgB,YAAY,UAAuC;CACjE,MAAM,OAAO,0BAA0B,KAAK,QAAQ;CACpD,MAAM,KAAK,iCAAiC,KAAK,QAAQ;CACzD,IAAI,CAAC,OAAO,MAAM,CAAC,KAAK,IAAI,OAAO;CACnC,MAAM,MAAM,iBAAiB,KAAK,QAAQ;CAC1C,OAAO;EAAE,WAAW,KAAK;EAAI,IAAI,GAAG,EAAE,CAAC,YAAY;EAAG,QAAQ,MAAM,MAAM,GAAA,CAAI,KAAK;CAAE;AACvF;AAEA,SAAgB,wBAA0C;CACxD,MAAM,QAAQ;CAEd,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,IAAI;IACF,MAAM,GACJ;;;oBAGU,KAAK,UAAU,OAAO,EAAE,UAAU,KAAK,UAAU,KAAK,EAAE;;6CAGpE;GACF,SAAS,KAAK;IACZ,MAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;IAC3D,MAAM,IAAI,MACR,8BAA8B,IAAI,mKAEU,QAAQ,UAAU,MAAM,uDAEtE;GACF;EACF;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,MAAM;IACV,OAAO,KAAK,SAAS;IACrB,OAAO,KAAK,MAAM,KAAA,IAAa,KAAK,SAAS;IAC7C,KAAK;IACL,UAAU;IACV,KAAK,KAAK,MAAM,KAAK,IAAI,KAAK,OAAO,GAAG,QAAQ,cAAc,EAAE,CAAC,IAAI,KAAA;GACvE;GACA,MAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;GAC1C,MAAM,SAAS,MAAM,GAAG,QAAQ,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC;GAErD,OADa,KAAK,MAAM,MACd,CAAC,CAAC,IAAI,SAAS;EAC3B;EAEA,MAAM,aAAa,MAAiB;GAIlC,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,KAAK,KAAK,QAAQ;GACxB,IAAI,CAAC,IAAI,OAAO,KAAA;GAChB,OAAO,YAAY,GAAG,YAAY,KAAK,GAAG,WAAW,KAAK;EAC5D;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,SAAS,YAAY,QAAQ;GACnC,IAAI,CAAC,QACH,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,QAAQ;KAA+B,QAAQ,KAAK;KAAI,SAAS,KAAK;IAAQ,CAAC;GAC1G;GAEF,MAAM,iBAAiB,KAAK,QAAQ,SAAS,OAAO,SAAS;GAC7D,MAAM,UAAU,OAAO,OAAO,KAAK;GACnC,MAAM,aAAa,KAAK,OAAO,UAAU,OAAO,UAAU,OAAO,KAAK,MAAM,UAAU,KAAK,KAAK;GAChG,MAAM,mBAAmB,WAAW;GAGpC,OAAO;IACL,UAHe,kBAAkB;IAIjC,OAHY,MAAO,iBAAiB,IAAI,KAAK,MAAO,mBAAmB,IAAI;IAI3E,QAAQ,KAAK,UAAU;KACrB;KACA;KACA;KACA,WAAW;KACX,QAAQ,KAAK;KACb,WAAW,KAAK;KAChB,SAAS,KAAK;KACd,SAAS,KAAK;KACd,QAAQ,KAAK;IACf,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/nomiracl.d.ts

@@ -1,15 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * NoMIRACL adapter.
- *
- * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does
- * not generate an answer here; it classifies whether the supplied passages
- * contain enough evidence to answer the query. This directly measures false
- * positive / false negative behavior for RAG abstention.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/nomiracl.d.ts
 declare function createNoMiraclAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createNoMiraclAdapter };
+//# sourceMappingURL=nomiracl.d.ts.map

package/dist/benchmarks/nomiracl.js

@@ -1,9 +1,147 @@
-import {
-  createNoMiraclAdapter
-} from "../chunk-7FKBWOQT.js";
-import "../chunk-X3BTXCJ4.js";
-import "../chunk-EIETHPD5.js";
-export {
-  createNoMiraclAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { contextBlock, contextsFrom, firstString, isObject, ragAnswerOutput, readJsonRows, selectTasks, stringFrom } from "./rag-shared.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/nomiracl.ts
+/**
+* NoMIRACL adapter.
+*
+* NoMIRACL tests robustness to irrelevant retrieved passages. The worker does
+* not generate an answer here; it classifies whether the supplied passages
+* contain enough evidence to answer the query. This directly measures false
+* positive / false negative behavior for RAG abstention.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "nomiracl.json");
+const dataFile = () => process.env.NOMIRACL_DATA_FILE;
+function passagesFrom(value, relevant) {
+	return contextsFrom(value).map((context) => ({
+		...context,
+		relevant
+	}));
+}
+function answerableFrom(raw, contexts) {
+	const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant;
+	if (typeof direct === "boolean") return direct;
+	if (typeof direct === "number") return direct > 0;
+	if (typeof direct === "string") {
+		const normalized = direct.toLowerCase();
+		if ([
+			"true",
+			"t",
+			"relevant",
+			"answerable",
+			"positive",
+			"1",
+			"yes"
+		].includes(normalized)) return true;
+		if ([
+			"false",
+			"f",
+			"non-relevant",
+			"non_relevant",
+			"unanswerable",
+			"negative",
+			"0",
+			"no"
+		].includes(normalized)) return false;
+	}
+	return contexts.some((context) => context.relevant === true);
+}
+function rowToTask(raw, index) {
+	if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`);
+	const query = firstString(raw, ["query", "question"]);
+	if (!query) throw new Error(`NoMIRACL row ${index} missing query`);
+	const labelledPassages = [...passagesFrom(raw.positive_passages, true), ...passagesFrom(raw.negative_passages, false)];
+	const contexts = labelledPassages.length > 0 ? labelledPassages : contextsFrom(raw.passages).length > 0 ? contextsFrom(raw.passages) : contextsFrom(raw.contexts).length > 0 ? contextsFrom(raw.contexts) : contextsFrom(raw.documents);
+	const answerable = answerableFrom(raw, contexts);
+	const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? "unknown";
+	const subset = stringFrom(raw.subset) ?? (answerable ? "relevant" : "non-relevant");
+	const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}`;
+	const meta = {
+		benchmark: "nomiracl",
+		query,
+		answerable,
+		language,
+		subset,
+		contexts
+	};
+	return {
+		id,
+		split: stringFrom(raw.split) ?? language,
+		prompt: [
+			"Classify this NoMIRACL RAG relevance case.",
+			"Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.",
+			"",
+			`Query: ${query}`,
+			contexts.length > 0 ? `\nPassages:\n${contextBlock(contexts)}` : void 0
+		].filter(Boolean).join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.answerable !== "boolean") throw new Error(`NoMIRACL task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function parseClassification(artifact) {
+	const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+	if (/\bunanswerable\b|\bnot answerable\b|\bno answer\b|\bnegative\b|\bfalse\b/.test(normalized)) return false;
+	if (/\banswerable\b|\brelevant\b|\bpositive\b|\btrue\b|\byes\b/.test(normalized)) return true;
+	return null;
+}
+async function loadRows(path) {
+	const rows = await readJsonRows(path);
+	if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`);
+	return rows;
+}
+async function loadFixtures(opts) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 — loading ${rows.length} adapter fixtures`);
+	return selectTasks(rows.map(rowToTask), opts, "NoMIRACL");
+}
+function createNoMiraclAdapter() {
+	const fixturesMode = process.env.NOMIRACL_FIXTURES === "1";
+	return {
+		name: "nomiracl",
+		output: ragAnswerOutput,
+		async preflight() {
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8");
+				return;
+			}
+			const path = dataFile();
+			if (!path) throw new Error("NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing.");
+			await loadRows(path);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			const path = dataFile();
+			if (!path) throw new Error("NOMIRACL_DATA_FILE is required to load NoMIRACL tasks");
+			return selectTasks((await loadRows(path)).map(rowToTask), opts, "NoMIRACL");
+		},
+		async goldArtifact(task) {
+			return readMeta(task).answerable ? "ANSWERABLE" : "UNANSWERABLE";
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const got = parseClassification(artifact);
+			const resolved = got === meta.answerable;
+			return {
+				resolved,
+				score: resolved ? 1 : 0,
+				detail: JSON.stringify({
+					benchmark: meta.benchmark,
+					language: meta.language,
+					subset: meta.subset,
+					expected: meta.answerable ? "ANSWERABLE" : "UNANSWERABLE",
+					got: got === null ? null : got ? "ANSWERABLE" : "UNANSWERABLE",
+					contextCount: meta.contexts.length
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { createNoMiraclAdapter };
+
 //# sourceMappingURL=nomiracl.js.map

package/dist/benchmarks/nomiracl.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"nomiracl.js","names":[],"sources":["../../src/benchmarks/nomiracl.ts"],"sourcesContent":["/**\n * NoMIRACL adapter.\n *\n * NoMIRACL tests robustness to irrelevant retrieved passages. The worker does\n * not generate an answer here; it classifies whether the supplied passages\n * contain enough evidence to answer the query. This directly measures false\n * positive / false negative behavior for RAG abstention.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\nimport {\n  contextBlock,\n  contextsFrom,\n  firstString,\n  isObject,\n  ragAnswerOutput,\n  readJsonRows,\n  selectTasks,\n  stringFrom,\n  type RagContext,\n} from './rag-shared'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'nomiracl.json')\n\ninterface NoMiraclMeta {\n  benchmark: 'nomiracl'\n  query: string\n  answerable: boolean\n  language: string\n  subset: string\n  contexts: RagContext[]\n}\n\nconst dataFile = (): string | undefined => process.env.NOMIRACL_DATA_FILE\n\nfunction passagesFrom(value: unknown, relevant: boolean): RagContext[] {\n  return contextsFrom(value).map((context) => ({ ...context, relevant }))\n}\n\nfunction answerableFrom(raw: Record<string, unknown>, contexts: readonly RagContext[]): boolean {\n  const direct = raw.answerable ?? raw.is_answerable ?? raw.has_answer ?? raw.label ?? raw.relevant\n  if (typeof direct === 'boolean') return direct\n  if (typeof direct === 'number') return direct > 0\n  if (typeof direct === 'string') {\n    const normalized = direct.toLowerCase()\n    if (['true', 't', 'relevant', 'answerable', 'positive', '1', 'yes'].includes(normalized)) return true\n    if (['false', 'f', 'non-relevant', 'non_relevant', 'unanswerable', 'negative', '0', 'no'].includes(normalized)) {\n      return false\n    }\n  }\n  return contexts.some((context) => context.relevant === true)\n}\n\nfunction rowToTask(raw: unknown, index: number): BenchTask {\n  if (!isObject(raw)) throw new Error(`NoMIRACL row ${index} must be an object`)\n  const query = firstString(raw, ['query', 'question'])\n  if (!query) throw new Error(`NoMIRACL row ${index} missing query`)\n  const labelledPassages = [\n    ...passagesFrom(raw.positive_passages, true),\n    ...passagesFrom(raw.negative_passages, false),\n  ]\n  const contexts =\n    labelledPassages.length > 0\n      ? labelledPassages\n      : contextsFrom(raw.passages).length > 0\n      ? contextsFrom(raw.passages)\n      : contextsFrom(raw.contexts).length > 0\n        ? contextsFrom(raw.contexts)\n        : contextsFrom(raw.documents)\n  const answerable = answerableFrom(raw, contexts)\n  const language = stringFrom(raw.lang) ?? stringFrom(raw.language) ?? 'unknown'\n  const subset = stringFrom(raw.subset) ?? (answerable ? 'relevant' : 'non-relevant')\n  const id = stringFrom(raw.id) ?? stringFrom(raw.query_id) ?? `nomiracl-${index}`\n  const meta: NoMiraclMeta = {\n    benchmark: 'nomiracl',\n    query,\n    answerable,\n    language,\n    subset,\n    contexts,\n  }\n  return {\n    id,\n    split: stringFrom(raw.split) ?? language,\n    prompt: [\n      'Classify this NoMIRACL RAG relevance case.',\n      'Return exactly one word: ANSWERABLE if at least one supplied passage contains enough evidence to answer the query, otherwise UNANSWERABLE.',\n      '',\n      `Query: ${query}`,\n      contexts.length > 0 ? `\\nPassages:\\n${contextBlock(contexts)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): NoMiraclMeta {\n  const md = task.metadata\n  if (!md || typeof md.answerable !== 'boolean') {\n    throw new Error(`NoMIRACL task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as NoMiraclMeta\n}\n\nfunction parseClassification(artifact: string): boolean | null {\n  const normalized = artifact.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()\n  if (/\\bunanswerable\\b|\\bnot answerable\\b|\\bno answer\\b|\\bnegative\\b|\\bfalse\\b/.test(normalized)) {\n    return false\n  }\n  if (/\\banswerable\\b|\\brelevant\\b|\\bpositive\\b|\\btrue\\b|\\byes\\b/.test(normalized)) return true\n  return null\n}\n\nasync function loadRows(path: string): Promise<unknown[]> {\n  const rows = await readJsonRows(path)\n  if (rows.length === 0) throw new Error(`NoMIRACL: no rows in ${path}`)\n  return rows\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as unknown[]\n  console.warn(`[nomiracl] NOMIRACL_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectTasks(rows.map(rowToTask), opts, 'NoMIRACL')\n}\n\nexport function createNoMiraclAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.NOMIRACL_FIXTURES === '1'\n\n  return {\n    name: 'nomiracl',\n    output: ragAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8')\n        return\n      }\n      const path = dataFile()\n      if (!path) {\n        throw new Error(\n          'NOMIRACL_DATA_FILE is required. Fix: export project-miracl/nomiracl rows to JSONL and set NOMIRACL_DATA_FILE=/path/to/nomiracl.jsonl, or set NOMIRACL_FIXTURES=1 for adapter plumbing.',\n        )\n      }\n      await loadRows(path)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const path = dataFile()\n      if (!path) throw new Error('NOMIRACL_DATA_FILE is required to load NoMIRACL tasks')\n      return selectTasks((await loadRows(path)).map(rowToTask), opts, 'NoMIRACL')\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return readMeta(task).answerable ? 'ANSWERABLE' : 'UNANSWERABLE'\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const got = parseClassification(artifact)\n      const resolved = got === meta.answerable\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          benchmark: meta.benchmark,\n          language: meta.language,\n          subset: meta.subset,\n          expected: meta.answerable ? 'ANSWERABLE' : 'UNANSWERABLE',\n          got: got === null ? null : got ? 'ANSWERABLE' : 'UNANSWERABLE',\n          contextCount: meta.contexts.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAyBA,MAAM,WAAW,KAAK,WAAW,YAAY,eAAe;AAW5D,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,aAAa,OAAgB,UAAiC;CACrE,OAAO,aAAa,KAAK,CAAC,CAAC,KAAK,aAAa;EAAE,GAAG;EAAS;CAAS,EAAE;AACxE;AAEA,SAAS,eAAe,KAA8B,UAA0C;CAC9F,MAAM,SAAS,IAAI,cAAc,IAAI,iBAAiB,IAAI,cAAc,IAAI,SAAS,IAAI;CACzF,IAAI,OAAO,WAAW,WAAW,OAAO;CACxC,IAAI,OAAO,WAAW,UAAU,OAAO,SAAS;CAChD,IAAI,OAAO,WAAW,UAAU;EAC9B,MAAM,aAAa,OAAO,YAAY;EACtC,IAAI;GAAC;GAAQ;GAAK;GAAY;GAAc;GAAY;GAAK;EAAK,CAAC,CAAC,SAAS,UAAU,GAAG,OAAO;EACjG,IAAI;GAAC;GAAS;GAAK;GAAgB;GAAgB;GAAgB;GAAY;GAAK;EAAI,CAAC,CAAC,SAAS,UAAU,GAC3G,OAAO;CAEX;CACA,OAAO,SAAS,MAAM,YAAY,QAAQ,aAAa,IAAI;AAC7D;AAEA,SAAS,UAAU,KAAc,OAA0B;CACzD,IAAI,CAAC,SAAS,GAAG,GAAG,MAAM,IAAI,MAAM,gBAAgB,MAAM,mBAAmB;CAC7E,MAAM,QAAQ,YAAY,KAAK,CAAC,SAAS,UAAU,CAAC;CACpD,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,gBAAgB,MAAM,eAAe;CACjE,MAAM,mBAAmB,CACvB,GAAG,aAAa,IAAI,mBAAmB,IAAI,GAC3C,GAAG,aAAa,IAAI,mBAAmB,KAAK,CAC9C;CACA,MAAM,WACJ,iBAAiB,SAAS,IACtB,mBACA,aAAa,IAAI,QAAQ,CAAC,CAAC,SAAS,IACpC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,QAAQ,CAAC,CAAC,SAAS,IAClC,aAAa,IAAI,QAAQ,IACzB,aAAa,IAAI,SAAS;CAClC,MAAM,aAAa,eAAe,KAAK,QAAQ;CAC/C,MAAM,WAAW,WAAW,IAAI,IAAI,KAAK,WAAW,IAAI,QAAQ,KAAK;CACrE,MAAM,SAAS,WAAW,IAAI,MAAM,MAAM,aAAa,aAAa;CACpE,MAAM,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,IAAI,QAAQ,KAAK,YAAY;CACzE,MAAM,OAAqB;EACzB,WAAW;EACX;EACA;EACA;EACA;EACA;CACF;CACA,OAAO;EACL;EACA,OAAO,WAAW,IAAI,KAAK,KAAK;EAChC,QAAQ;GACN;GACA;GACA;GACA,UAAU;GACV,SAAS,SAAS,IAAI,gBAAgB,aAAa,QAAQ,MAAM,KAAA;EACnE,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,eAAe,WAClC,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,kDAAkD;CAE7F,OAAO;AACT;AAEA,SAAS,oBAAoB,UAAkC;CAC7D,MAAM,aAAa,SAAS,YAAY,CAAC,CAAC,QAAQ,eAAe,GAAG,CAAC,CAAC,KAAK;CAC3E,IAAI,2EAA2E,KAAK,UAAU,GAC5F,OAAO;CAET,IAAI,4DAA4D,KAAK,UAAU,GAAG,OAAO;CACzF,OAAO;AACT;AAEA,eAAe,SAAS,MAAkC;CACxD,MAAM,OAAO,MAAM,aAAa,IAAI;CACpC,IAAI,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM,wBAAwB,MAAM;CACrE,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,4CAA4C,KAAK,OAAO,kBAAkB;CACvF,OAAO,YAAY,KAAK,IAAI,SAAS,GAAG,MAAM,UAAU;AAC1D;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,sBAAsB;CAEvD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM;IAC/B;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,wLACF;GAEF,MAAM,SAAS,IAAI;EACrB;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,uDAAuD;GAClF,OAAO,aAAa,MAAM,SAAS,IAAI,EAAA,CAAG,IAAI,SAAS,GAAG,MAAM,UAAU;EAC5E;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,SAAS,IAAI,CAAC,CAAC,aAAa,eAAe;EACpD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAM,oBAAoB,QAAQ;GACxC,MAAM,WAAW,QAAQ,KAAK;GAC9B,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KACrB,WAAW,KAAK;KAChB,UAAU,KAAK;KACf,QAAQ,KAAK;KACb,UAAU,KAAK,aAAa,eAAe;KAC3C,KAAK,QAAQ,OAAO,OAAO,MAAM,eAAe;KAChD,cAAc,KAAK,SAAS;IAC9B,CAAC;GACH;EACF;CACF;AACF"}

package/dist/benchmarks/open-rag-bench.d.ts

@@ -1,14 +1,6 @@
-import { BenchmarkAdapter } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * Open RAG Bench adapter.
- *
- * This targets Vectara-style Open RAG Bench exports over PDF-derived text,
- * table, and image contexts. The deterministic judge scores final-answer
- * agreement and surfaces modality/document metadata for diagnostics.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+//#region src/benchmarks/open-rag-bench.d.ts
 declare function createOpenRagBenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { createOpenRagBenchAdapter };
+//# sourceMappingURL=open-rag-bench.d.ts.map