@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-QZZEAHWJ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/programbench.ts"],"sourcesContent":["/**\n * ProgramBench adapter (facebookresearch/ProgramBench) — cleanroom\n * reverse-engineering: rebuild a black-box executable's behavior from scratch.\n * The agent is given only the gold `./executable` (run-only) + stripped docs and\n * must produce a `submission.tar.gz` whose `./compile.sh` builds an `./executable`\n * with identical observable behavior. Judge = the official `programbench`\n * harness: it extracts the submission, runs compile.sh in the per-task cleanroom\n * Docker image, then runs the HIDDEN behavioral pytest suites (pulled via\n * `programbench blob sync`). Score = fraction of non-ignored tests passed\n * (GRADED, applying the tests.json ignore mask); resolved = all non-ignored tests\n * pass. Fully deterministic — no LLM judge.\n *\n * OutputAdapter is stream-only, so the worker emits its codebase as fenced\n * `path:`-prefixed file blocks (including compile.sh); the adapter materializes\n * those into submission.tar.gz. Test execution is delegated to the real\n * `programbench eval`/`info` harness — pytest scoring + the ignore mask are NOT\n * reimplemented here.\n *\n * Requires for a live run: the bench `.venv` with `programbench` installed +\n * Docker on linux/amd64 (per-task `<image>:task_cleanroom` images) + HF access for\n * the hidden test blobs. For offline/CI listing set PROGRAMBENCH_FIXTURES=1 to load\n * the committed instance ids (bench/fixtures/programbench.json); judging still\n * needs the harness + Docker and fails loud without them — never a fabricated score.\n */\n\nimport { join } from 'node:path'\nimport { readFile } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'programbench.json')\n\nconst TESTS_DATASET = 'programbench/ProgramBench-Tests'\nconst TREE_API = `https://huggingface.co/api/datasets/${TESTS_DATASET}/tree/main`\n\ninterface PbFixture {\n  instance_id: string\n  image_name: string\n  note?: string\n}\n\ninterface PbMeta {\n  instanceId: string\n  imageName: string\n}\n\nconst PROMPT = [\n  'This is a cleanroom reverse-engineering task. You are given a compiled black-box `./executable` (run-only) and stripped documentation in the workspace.',\n  'Write an ORIGINAL codebase from scratch that, when built, produces an `./executable` with byte-for-byte identical observable behavior (stdout/stderr/exit codes/filesystem effects) to the original. You may run `./executable` to probe its behavior.',\n  'You MUST include a `./compile.sh` at the workspace root that builds your source into `./executable` at the workspace root.',\n  'Emit your COMPLETE submission as the LAST thing in your reply: one fenced block per file, each opening fence line being exactly ```path:<relative/path>``` followed by the file contents, then a closing fence. Include compile.sh and every source file. Nothing after the final closing fence.',\n].join('\\n')\n\nfunction fixtureToTask(f: PbFixture): BenchTask {\n  const meta: PbMeta = { instanceId: f.instance_id, imageName: f.image_name }\n  return {\n    id: f.instance_id,\n    prompt: PROMPT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): PbMeta {\n  const md = task.metadata\n  if (!md || typeof md.instanceId !== 'string') {\n    throw new Error(`programbench task ${task.id} missing metadata.instanceId — loadTasks did not populate it`)\n  }\n  return md as unknown as PbMeta\n}\n\nfunction selectTasks(fixtures: PbFixture[], opts: LoadOptions): BenchTask[] {\n  let tasks = fixtures.map(fixtureToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\n/** `owner__repo.commit` → DockerHub image name `owner/repo` for `<image>:task_cleanroom`. */\nfunction imageName(instanceId: string): string {\n  return instanceId.split('.')[0].replace('__', '/')\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const fixtures = JSON.parse(await readFile(FIXTURES, 'utf8')) as PbFixture[]\n  console.warn(\n    `[programbench] PROGRAMBENCH_FIXTURES=1 — loading ${fixtures.length} committed instance ids from ${FIXTURES} (no HF tree fetch)`,\n  )\n  return selectTasks(fixtures, opts)\n}\n\n/** Enumerate real instance ids from the HF Tests dataset tree (one dir per\n *  instance). Throws on a non-OK response (fail loud). */\nasync function fetchInstanceIds(): Promise<PbFixture[]> {\n  const res = await fetch(TREE_API)\n  if (!res.ok) throw new Error(`programbench tree HTTP ${res.status}: ${TREE_API}`)\n  const tree = (await res.json()) as Array<{ path: string; type: string }>\n  const ids = tree.filter((e) => e.type === 'directory' && e.path.includes('__')).map((e) => e.path)\n  if (ids.length === 0) throw new Error(`programbench: no instance dirs in ${TESTS_DATASET} tree`)\n  return ids.map((id) => ({ instance_id: id, image_name: imageName(id) }))\n}\n\n/**\n * Run the official programbench harness for one instance over the worker's\n * file-manifest submission. The driver builds the run-dir layout (one folder\n * holding submission.tar.gz), runs `programbench blob sync` + `programbench eval`,\n * then parses <instance_id>.eval.json applying the tests.json ignore mask via\n * `programbench info` logic. Emits {passed,total,resolved} as the last stdout line.\n */\nasync function runHarness(meta: PbMeta, submission: string): Promise<BenchScore> {\n  const judge = join(benchRoot, 'scripts', 'programbench_judge.py')\n  let stdout: string\n  try {\n    // The submission manifest is piped to the driver's stdin via the shared\n    // stdin-aware runner — execFile's `input` option is not honored async and\n    // hangs the driver's sys.stdin.read() forever.\n    stdout = await runVenvScriptStdin(judge, ['--instance', meta.instanceId], submission, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`programbench harness failed for ${meta.instanceId}: ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as {\n    passed?: number\n    total?: number\n    resolved?: boolean\n    error?: string\n  }\n  if (report.error) throw new Error(`programbench harness error for ${meta.instanceId}: ${report.error}`)\n  if (typeof report.passed !== 'number' || typeof report.total !== 'number') {\n    throw new Error(`programbench judge returned no {passed,total}: ${stdout.slice(0, 400)}`)\n  }\n  const score = report.total > 0 ? report.passed / report.total : 0\n  return {\n    resolved: report.resolved ?? (report.total > 0 && report.passed === report.total),\n    score,\n    detail: JSON.stringify({ instanceId: meta.instanceId, passed: report.passed, total: report.total }),\n  }\n}\n\n/**\n * Parse the worker stream into the submission text the driver materializes: the\n * concatenation of every ```path:<p>``` fenced block. Passed through verbatim to\n * the python driver, which tars it. Empty when the worker emitted no file block\n * (fail-closed → the harness scores a missing compile.sh as 0).\n */\nexport const programbenchSubmissionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const blocks = [...text.matchAll(/```path:([^\\n`]+)\\n([\\s\\S]*?)```/g)]\n    if (blocks.length === 0) return ''\n    // Re-serialize in the same path:<p> envelope the driver splits on.\n    return blocks.map((m) => `===FILE:${m[1].trim()}===\\n${m[2]}`).join('\\n')\n  },\n}\n\nexport function createProgrambenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.PROGRAMBENCH_FIXTURES === '1'\n\n  return {\n    name: 'programbench',\n    output: programbenchSubmissionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['programbench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install programbench ; ` +\n          `(2) Docker on linux/amd64 (per-task <image>:task_cleanroom images; will NOT run on ARM) ; ` +\n          `(3) HF access for the hidden test blobs (the driver runs \\`programbench blob sync\\`). ` +\n          `Set PROGRAMBENCH_FIXTURES=1 to list the committed instance ids offline.`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      let fixtures: PbFixture[]\n      try {\n        fixtures = await fetchInstanceIds()\n      } catch (err) {\n        console.warn(\n          `[programbench] live tree fetch failed (${err instanceof Error ? err.message : err}); falling back to committed fixtures at ${FIXTURES}`,\n        )\n        return loadFixtures(opts)\n      }\n      return selectTasks(fixtures, opts)\n    },\n\n    async goldArtifact() {\n      // The oracle is the original repo's source (stripped from the task image),\n      // not redistributable as a portable submission string. Judge correctness is\n      // proven by running the real harness on a real solve, not a synthetic gold.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      return runHarness(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;AAyBA,SAAS,YAAY;AACrB,SAAS,gBAAgB;AAKzB,IAAM,WAAW,KAAK,WAAW,YAAY,mBAAmB;AAEhE,IAAM,gBAAgB;AACtB,IAAM,WAAW,uCAAuC,aAAa;AAarE,IAAM,SAAS;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,SAAS,cAAc,GAAyB;AAC9C,QAAM,OAAe,EAAE,YAAY,EAAE,aAAa,WAAW,EAAE,WAAW;AAC1E,SAAO;AAAA,IACL,IAAI,EAAE;AAAA,IACN,QAAQ;AAAA,IACR,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAAyB;AACzC,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,eAAe,UAAU;AAC5C,UAAM,IAAI,MAAM,qBAAqB,KAAK,EAAE,mEAA8D;AAAA,EAC5G;AACA,SAAO;AACT;AAEA,SAAS,YAAY,UAAuB,MAAgC;AAC1E,MAAI,QAAQ,SAAS,IAAI,aAAa;AACtC,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC5C,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,SAAO;AACT;AAGA,SAAS,UAAU,YAA4B;AAC7C,SAAO,WAAW,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,MAAM,GAAG;AACnD;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,WAAW,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AAC5D,UAAQ;AAAA,IACN,yDAAoD,SAAS,MAAM,gCAAgC,QAAQ;AAAA,EAC7G;AACA,SAAO,YAAY,UAAU,IAAI;AACnC;AAIA,eAAe,mBAAyC;AACtD,QAAM,MAAM,MAAM,MAAM,QAAQ;AAChC,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,0BAA0B,IAAI,MAAM,KAAK,QAAQ,EAAE;AAChF,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,KAAK,SAAS,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AACjG,MAAI,IAAI,WAAW,EAAG,OAAM,IAAI,MAAM,qCAAqC,aAAa,OAAO;AAC/F,SAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,aAAa,IAAI,YAAY,UAAU,EAAE,EAAE,EAAE;AACzE;AASA,eAAe,WAAW,MAAc,YAAyC;AAC/E,QAAM,QAAQ,KAAK,WAAW,WAAW,uBAAuB;AAChE,MAAI;AACJ,MAAI;AAIF,aAAS,MAAM,mBAAmB,OAAO,CAAC,cAAc,KAAK,UAAU,GAAG,YAAY,EAAE,KAAK,UAAU,CAAC;AAAA,EAC1G,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,mCAAmC,KAAK,UAAU,MAAM,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EACpH;AACA,QAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAMlE,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,kCAAkC,KAAK,UAAU,KAAK,OAAO,KAAK,EAAE;AACtG,MAAI,OAAO,OAAO,WAAW,YAAY,OAAO,OAAO,UAAU,UAAU;AACzE,UAAM,IAAI,MAAM,kDAAkD,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC1F;AACA,QAAM,QAAQ,OAAO,QAAQ,IAAI,OAAO,SAAS,OAAO,QAAQ;AAChE,SAAO;AAAA,IACL,UAAU,OAAO,aAAa,OAAO,QAAQ,KAAK,OAAO,WAAW,OAAO;AAAA,IAC3E;AAAA,IACA,QAAQ,KAAK,UAAU,EAAE,YAAY,KAAK,YAAY,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC;AAAA,EACpG;AACF;AAQO,IAAM,+BAAsD;AAAA,EACjE,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,mCAAmC,CAAC;AACrE,QAAI,OAAO,WAAW,EAAG,QAAO;AAEhC,WAAO,OAAO,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,EAAE,KAAK,CAAC;AAAA,EAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,IAAI;AAAA,EAC1E;AACF;AAEO,SAAS,4BAA8C;AAC5D,QAAM,eAAe,QAAQ,IAAI,0BAA0B;AAE3D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,cAAc;AAAA,QACxB,eAAe;AAAA,QACf,KACE;AAAA,MAIJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,UAAI;AACJ,UAAI;AACF,mBAAW,MAAM,iBAAiB;AAAA,MACpC,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG,4CAA4C,QAAQ;AAAA,QACxI;AACA,eAAO,aAAa,IAAI;AAAA,MAC1B;AACA,aAAO,YAAY,UAAU,IAAI;AAAA,IACnC;AAAA,IAEA,MAAM,eAAe;AAInB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,WAAW,MAAM,QAAQ;AAAA,IAClC;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-SEVJPLZC.js

@@ -1,260 +0,0 @@
-// src/benchmarks/simpleqa.ts
-import { execFile } from "child_process";
-import { readFile } from "fs/promises";
-import { join } from "path";
-import { fileURLToPath } from "url";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
-var PY = join(BENCH_ROOT, ".venv", "bin", "python");
-var FIXTURES = join(BENCH_ROOT, "fixtures", "simpleqa.json");
-var DATASET_URL = "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv";
-var FINAL_ANSWER_SENTINEL = "FINAL ANSWER:";
-var WORKER_CONTRACT = [
-  "",
-  "Research the question using live web sources and answer it with a short, specific factual value.",
-  "Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.",
-  `End your response with a single final line: \`${FINAL_ANSWER_SENTINEL} <answer>\``,
-  "The answer after the sentinel must be the bare value only (no explanation on that line).",
-  "If you do not know the answer, state that you do not know rather than guessing."
-].join("\n");
-async function py(script, args = []) {
-  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], {
-    maxBuffer: 1024 * 1024 * 256
-  });
-  return stdout;
-}
-function parseFinalAnswer(artifact) {
-  const lines = artifact.split(/\r?\n/);
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const line = lines[i] ?? "";
-    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL);
-    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim();
-  }
-  for (let i = lines.length - 1; i >= 0; i -= 1) {
-    const t = (lines[i] ?? "").trim();
-    if (t.length > 0) return t;
-  }
-  return "";
-}
-function parseCitations(artifact) {
-  const urls = /* @__PURE__ */ new Set();
-  for (const m of artifact.matchAll(/https?:\/\/[^\s)<>"']+/g)) {
-    urls.add(m[0].replace(/[.,;]+$/, ""));
-  }
-  return [...urls];
-}
-var GRADER_PROMPT = (question, gold, predicted) => [
-  "Your job is to grade a predicted answer to a factual question against the single gold target answer.",
-  "Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.",
-  "First, follow these grading rules carefully.",
-  "",
-  "CORRECT:",
-  "  The predicted answer fully contains the gold target without contradicting it.",
-  "  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.",
-  '  Hedging that still commits to the gold value (e.g. "I think it is X" where X is the gold) is CORRECT.',
-  "  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.",
-  "  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.",
-  "",
-  "INCORRECT:",
-  "  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.",
-  "  A prediction that states a different factual value than the gold is INCORRECT.",
-  "",
-  "NOT_ATTEMPTED:",
-  "  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,",
-  "  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.",
-  "",
-  "Also note the following rules:",
-  "  Do not punish a prediction for omitting information that would not change whether the gold target is contained.",
-  "  Grade ONLY whether the gold value is present and uncontradicted \u2014 not the overall quality of the response.",
-  "",
-  `Question: ${question}`,
-  `Gold target: ${gold}`,
-  `Predicted answer: ${predicted}`,
-  "",
-  "Respond with ONLY a fenced JSON block and nothing else:",
-  "```json",
-  '{"grade": "CORRECT" | "INCORRECT" | "NOT_ATTEMPTED"}',
-  "```"
-].join("\n");
-function graderRouter() {
-  const key = process.env.TANGLE_API_KEY;
-  if (!key) throw new Error("TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)");
-  const model = process.env.JUDGE_MODEL ?? "deepseek-v4-flash";
-  const baseUrl = process.env.ROUTER_BASE ?? "https://router.tangle.tools/v1";
-  return { baseUrl, key, model };
-}
-async function gradeAnswer(question, gold, predicted, router) {
-  const res = await fetch(`${router.baseUrl}/chat/completions`, {
-    method: "POST",
-    headers: { "content-type": "application/json", authorization: `Bearer ${router.key}` },
-    body: JSON.stringify({
-      model: router.model,
-      temperature: 0,
-      messages: [{ role: "user", content: GRADER_PROMPT(question, gold, predicted) }]
-    })
-  });
-  if (!res.ok) {
-    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-  }
-  const body = await res.json();
-  const content = body.choices?.[0]?.message?.content;
-  if (typeof content !== "string") {
-    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
-  }
-  const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/);
-  const raw = (fenced ? fenced[1] : content)?.trim() ?? "";
-  let parsed;
-  try {
-    parsed = JSON.parse(raw);
-  } catch {
-    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`);
-  }
-  if (parsed.grade === "CORRECT" || parsed.grade === "INCORRECT" || parsed.grade === "NOT_ATTEMPTED") {
-    return parsed.grade;
-  }
-  throw new Error(
-    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`
-  );
-}
-function rowToTask(row, index) {
-  const meta = {
-    gold: row.answer,
-    goldSources: row.urls,
-    topic: row.topic,
-    answerType: row.answer_type,
-    question: row.problem
-  };
-  return {
-    id: `simpleqa-${index}`,
-    split: "test",
-    prompt: row.problem + WORKER_CONTRACT,
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.gold !== "string" || typeof md.question !== "string") {
-    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectTasks(tasks, opts) {
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    return tasks.filter((t) => want.has(t.id));
-  }
-  if (opts.limit !== void 0) return tasks.slice(0, opts.limit);
-  return tasks;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 \u2014 loading ${rows.length} committed fixtures (no network)`);
-  return selectTasks(rows.map(rowToTask), opts);
-}
-async function loadLive(opts) {
-  const script = `
-import csv, ast, json, io, sys
-from urllib.request import urlopen
-with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
-    text = resp.read().decode('utf-8')
-rows = list(csv.DictReader(io.StringIO(text)))
-out = []
-for r in rows:
-    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}
-    out.append({
-        'problem': r.get('problem', ''),
-        'answer': r.get('answer', ''),
-        'topic': str(meta.get('topic', '')),
-        'answer_type': str(meta.get('answer_type', '')),
-        'urls': list(meta.get('urls', [])),
-    })
-print(json.dumps(out))
-`;
-  const stdout = await py(script);
-  const rows = JSON.parse(stdout);
-  return selectTasks(rows.map(rowToTask), opts);
-}
-function createSimpleQaAdapter() {
-  const fixturesMode = process.env.SIMPLEQA_FIXTURES === "1";
-  return {
-    name: "simpleqa",
-    async preflight() {
-      graderRouter();
-      if (fixturesMode) {
-        await readFile(FIXTURES, "utf8").catch((err) => {
-          throw new Error(
-            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`
-          );
-        });
-        return;
-      }
-      try {
-        await py(
-          `from urllib.request import urlopen
-with urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:
-    head = resp.read(64).decode('utf-8', 'replace')
-assert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]
-print('ok')`
-        );
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `simpleqa preflight failed: ${msg}
-Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; (2) ensure network access to ${DATASET_URL} ; or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      return loadLive(opts);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const finalAnswer = parseFinalAnswer(artifact);
-      const citations = parseCitations(artifact);
-      if (finalAnswer.length === 0) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({
-            grade: "NOT_ATTEMPTED",
-            gradeLetter: "C",
-            reason: "no parseable answer",
-            gold: meta.gold,
-            topic: meta.topic,
-            answerType: meta.answerType,
-            citationCount: citations.length
-          })
-        };
-      }
-      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter());
-      const gradeLetter = grade === "CORRECT" ? "A" : grade === "INCORRECT" ? "B" : "C";
-      const resolved = grade === "CORRECT";
-      return {
-        resolved,
-        score: resolved ? 1 : 0,
-        detail: JSON.stringify({
-          grade,
-          gradeLetter,
-          gold: meta.gold,
-          predicted: finalAnswer,
-          topic: meta.topic,
-          answerType: meta.answerType,
-          citationCount: citations.length
-        })
-      };
-    }
-  };
-}
-
-export {
-  parseFinalAnswer,
-  parseCitations,
-  createSimpleQaAdapter
-};
-//# sourceMappingURL=chunk-SEVJPLZC.js.map

package/dist/chunk-SEVJPLZC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/simpleqa.ts"],"sourcesContent":["/**\n * SimpleQA adapter (OpenAI simple-evals; short-form factual QA). Worker artifact\n * = a single free-text final answer string (optionally preceded by a CITATIONS:\n * block). Each item ships a short gold `answer` plus metadata (topic,\n * answer_type, source urls).\n *\n * Judge is the official SimpleQA grader — an LLM classifier that maps\n * (question, gold target, predicted answer) to exactly one of:\n *   A = CORRECT       — fully contains the gold, no contradiction\n *   B = INCORRECT     — contradicts / contains a different factual value\n *   C = NOT_ATTEMPTED — hedged, non-committal, or no value given\n * resolved = (grade == CORRECT). score = 1 for CORRECT else 0. The grade letter\n * (with its label) rides in `detail`; NOT_ATTEMPTED is surfaced distinctly from\n * INCORRECT so the scorecard can separate abstention from error.\n *\n * There is no deterministic tier: SimpleQA's rubric (containment + abstention)\n * is the grader's job by design, so judge() always calls the pinned grader model\n * (temperature 0) and fails loud on unparseable grader output. The final-answer\n * extraction IS deterministic (FINAL ANSWER: sentinel, fail-closed to '').\n *\n * Requires for a live run: the bench `.venv` with `datasets`/`requests` not\n * needed — the test set is a single public CSV fetched over HTTP — plus a\n * grader key (TANGLE_API_KEY). For offline/CI verification set\n * SIMPLEQA_FIXTURES=1 to load the committed fixtures\n * (bench/fixtures/simpleqa.json) — no network.\n */\n\nimport { execFile } from 'node:child_process'\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = join(BENCH_ROOT, '.venv', 'bin', 'python')\nconst FIXTURES = join(BENCH_ROOT, 'fixtures', 'simpleqa.json')\n\nconst DATASET_URL =\n  'https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv'\nconst FINAL_ANSWER_SENTINEL = 'FINAL ANSWER:'\n\n/** Worker contract appended to every task prompt. Answer extraction keys off the sentinel. */\nconst WORKER_CONTRACT = [\n  '',\n  'Research the question using live web sources and answer it with a short, specific factual value.',\n  'Cite the sources you used as full URLs, one per line, in a `CITATIONS:` block.',\n  `End your response with a single final line: \\`${FINAL_ANSWER_SENTINEL} <answer>\\``,\n  'The answer after the sentinel must be the bare value only (no explanation on that line).',\n  'If you do not know the answer, state that you do not know rather than guessing.',\n].join('\\n')\n\n/**\n * Typed seam for the research worker. The benchmark adapter scores a plain\n * `string` artifact (the BenchmarkAdapter contract); the loop worker decodes its\n * agent runs into a {@link ResearchAnswer} and serializes `finalAnswer` (+ an\n * optional `CITATIONS:` block) into that string before judging.\n */\nexport interface ResearchTask {\n  id: string\n  question: string\n  /** Gold short answer — the grader's target. */\n  gold: string\n  /** Source URLs cited by the dataset for this item — SOFT provenance signal, never a hard gate. */\n  goldSources: string[]\n  /** SimpleQA topic, e.g. 'Politics' — slices the scorecard by domain. */\n  topic: string\n  /** SimpleQA answer_type, e.g. 'Person' | 'Date' | 'Number' | 'Place'. */\n  answerType: string\n}\n\nexport interface ResearchAnswer {\n  finalAnswer: string\n  citations: string[]\n}\n\n/** Normalized fixture/loader row — the single shape both the CSV loader and fixtures emit. */\ninterface SimpleQaRow {\n  problem: string\n  answer: string\n  topic: string\n  answer_type: string\n  urls: string[]\n}\n\ninterface SimpleQaMeta {\n  gold: string\n  goldSources: string[]\n  topic: string\n  answerType: string\n  question: string\n}\n\n/** The three official SimpleQA grades. */\ntype Grade = 'CORRECT' | 'INCORRECT' | 'NOT_ATTEMPTED'\n\n/** Run the bench venv python with a script; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], {\n    maxBuffer: 1024 * 1024 * 256,\n  })\n  return stdout\n}\n\n/**\n * Parse the worker artifact into the final answer string.\n * Order: the `FINAL ANSWER:` sentinel first (deterministic-extraction discipline);\n * fall back to the last non-empty line. Returns '' when nothing is parseable\n * (fail-closed — never guess); judge() grades '' as NOT_ATTEMPTED.\n */\nexport function parseFinalAnswer(artifact: string): string {\n  const lines = artifact.split(/\\r?\\n/)\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const line = lines[i] ?? ''\n    const idx = line.toUpperCase().indexOf(FINAL_ANSWER_SENTINEL)\n    if (idx !== -1) return line.slice(idx + FINAL_ANSWER_SENTINEL.length).trim()\n  }\n  for (let i = lines.length - 1; i >= 0; i -= 1) {\n    const t = (lines[i] ?? '').trim()\n    if (t.length > 0) return t\n  }\n  return ''\n}\n\n/** Extract every bare URL in the artifact (CITATIONS: block + inline). SOFT provenance signal. */\nexport function parseCitations(artifact: string): string[] {\n  const urls = new Set<string>()\n  for (const m of artifact.matchAll(/https?:\\/\\/[^\\s)<>\"']+/g)) {\n    urls.add(m[0].replace(/[.,;]+$/, ''))\n  }\n  return [...urls]\n}\n\n/**\n * The official SimpleQA grader prompt (faithful paraphrase of OpenAI's\n * simple-evals GRADER_TEMPLATE). Classifies the predicted answer against the\n * gold target into A=CORRECT / B=INCORRECT / C=NOT_ATTEMPTED. The predicted\n * answer must fully contain the gold value with no contradicting statement;\n * hedged or non-committal answers are NOT_ATTEMPTED.\n */\nconst GRADER_PROMPT = (question: string, gold: string, predicted: string): string =>\n  [\n    'Your job is to grade a predicted answer to a factual question against the single gold target answer.',\n    'Grade the prediction into exactly one of three categories: CORRECT, INCORRECT, or NOT_ATTEMPTED.',\n    'First, follow these grading rules carefully.',\n    '',\n    'CORRECT:',\n    '  The predicted answer fully contains the gold target without contradicting it.',\n    '  Predictions with more detail than the gold are still CORRECT, as long as no part contradicts the gold.',\n    '  Hedging that still commits to the gold value (e.g. \"I think it is X\" where X is the gold) is CORRECT.',\n    '  Minor formatting, capitalization, ordering, or obvious-typo differences do not matter.',\n    '  For numeric/date targets, the prediction is CORRECT only if it matches the gold to the precision the gold specifies.',\n    '',\n    'INCORRECT:',\n    '  The predicted answer contradicts the gold target in any way, even if the contradiction is hedged.',\n    '  A prediction that states a different factual value than the gold is INCORRECT.',\n    '',\n    'NOT_ATTEMPTED:',\n    '  The gold target is neither fully given nor contradicted: the prediction declines to answer, says it does not know,',\n    '  asks for clarification, or gives a non-committal, vague, or empty response that commits to no specific value.',\n    '',\n    'Also note the following rules:',\n    '  Do not punish a prediction for omitting information that would not change whether the gold target is contained.',\n    '  Grade ONLY whether the gold value is present and uncontradicted — not the overall quality of the response.',\n    '',\n    `Question: ${question}`,\n    `Gold target: ${gold}`,\n    `Predicted answer: ${predicted}`,\n    '',\n    'Respond with ONLY a fenced JSON block and nothing else:',\n    '```json',\n    '{\"grade\": \"CORRECT\" | \"INCORRECT\" | \"NOT_ATTEMPTED\"}',\n    '```',\n  ].join('\\n')\n\ninterface GraderRouter {\n  baseUrl: string\n  key: string\n  model: string\n}\n\nfunction graderRouter(): GraderRouter {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for the SimpleQA grader (set the Tangle API key)')\n  const model = process.env.JUDGE_MODEL ?? 'deepseek-v4-flash'\n  const baseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'\n  return { baseUrl, key, model }\n}\n\n/** Call the official grader. Pinned model, temperature 0; fail loud on unparseable output. */\nasync function gradeAnswer(\n  question: string,\n  gold: string,\n  predicted: string,\n  router: GraderRouter,\n): Promise<Grade> {\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }],\n    }),\n  })\n  if (!res.ok) {\n    throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  }\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') {\n    throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  }\n  const fenced = content.match(/```(?:json)?\\s*([\\s\\S]*?)```/)\n  const raw = (fenced ? fenced[1] : content)?.trim() ?? ''\n  let parsed: { grade?: unknown }\n  try {\n    parsed = JSON.parse(raw) as { grade?: unknown }\n  } catch {\n    throw new Error(`SimpleQA grader produced unparseable output (no JSON grade): ${content.slice(0, 300)}`)\n  }\n  if (parsed.grade === 'CORRECT' || parsed.grade === 'INCORRECT' || parsed.grade === 'NOT_ATTEMPTED') {\n    return parsed.grade\n  }\n  throw new Error(\n    `SimpleQA grader grade not in {CORRECT,INCORRECT,NOT_ATTEMPTED}: ${JSON.stringify(parsed).slice(0, 200)}`,\n  )\n}\n\nfunction rowToTask(row: SimpleQaRow, index: number): BenchTask {\n  const meta: SimpleQaMeta = {\n    gold: row.answer,\n    goldSources: row.urls,\n    topic: row.topic,\n    answerType: row.answer_type,\n    question: row.problem,\n  }\n  return {\n    id: `simpleqa-${index}`,\n    split: 'test',\n    prompt: row.problem + WORKER_CONTRACT,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): SimpleQaMeta {\n  const md = task.metadata\n  if (!md || typeof md.gold !== 'string' || typeof md.question !== 'string') {\n    throw new Error(`SimpleQA task ${task.id} missing metadata.gold/question — loadTasks did not populate it`)\n  }\n  return md as unknown as SimpleQaMeta\n}\n\nfunction selectTasks(tasks: BenchTask[], opts: LoadOptions): BenchTask[] {\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    return tasks.filter((t) => want.has(t.id))\n  }\n  if (opts.limit !== undefined) return tasks.slice(0, opts.limit)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as SimpleQaRow[]\n  console.warn(`[simpleqa] SIMPLEQA_FIXTURES=1 — loading ${rows.length} committed fixtures (no network)`)\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\n/**\n * Load the live test set: fetch the public CSV via the bench venv python, parse\n * with `csv` (handles quoted/embedded-comma fields) and `ast.literal_eval` for\n * the python-repr `metadata` dict. Emits the same normalized SimpleQaRow shape\n * the fixtures use, so loadTasks/judge share one parse path.\n */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  const script = `\nimport csv, ast, json, io, sys\nfrom urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    text = resp.read().decode('utf-8')\nrows = list(csv.DictReader(io.StringIO(text)))\nout = []\nfor r in rows:\n    meta = ast.literal_eval(r['metadata']) if r.get('metadata') else {}\n    out.append({\n        'problem': r.get('problem', ''),\n        'answer': r.get('answer', ''),\n        'topic': str(meta.get('topic', '')),\n        'answer_type': str(meta.get('answer_type', '')),\n        'urls': list(meta.get('urls', [])),\n    })\nprint(json.dumps(out))\n`\n  const stdout = await py(script)\n  const rows = JSON.parse(stdout) as SimpleQaRow[]\n  return selectTasks(rows.map(rowToTask), opts)\n}\n\nexport function createSimpleQaAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.SIMPLEQA_FIXTURES === '1'\n\n  return {\n    name: 'simpleqa',\n\n    async preflight() {\n      // The grader router must be configured in both modes — SimpleQA's score is\n      // defined by the grader, so a run without it is meaningless.\n      graderRouter()\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(\n            `SIMPLEQA_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`,\n          )\n        })\n        return\n      }\n      try {\n        await py(\n          `from urllib.request import urlopen\nwith urlopen(${JSON.stringify(DATASET_URL)}, timeout=120) as resp:\n    head = resp.read(64).decode('utf-8', 'replace')\nassert head.startswith('metadata,problem,answer'), 'unexpected CSV header: ' + head[:40]\nprint('ok')`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `simpleqa preflight failed: ${msg}\\n` +\n            `Fix: (1) ensure the bench venv exists (python3 -m venv bench/.venv) ; ` +\n            `(2) ensure network access to ${DATASET_URL} ; ` +\n            `or set SIMPLEQA_FIXTURES=1 to run against the committed fixtures offline.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      return loadLive(opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold artifact = the worker-contract serialization of the gold answer, so\n      // verify-judge proves gold→CORRECT through the SAME parse path the real\n      // artifact takes.\n      const meta = readMeta(task)\n      return `${FINAL_ANSWER_SENTINEL} ${meta.gold}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const finalAnswer = parseFinalAnswer(artifact)\n      const citations = parseCitations(artifact)\n\n      if (finalAnswer.length === 0) {\n        // No parseable value committed — NOT_ATTEMPTED by construction, no grader call.\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({\n            grade: 'NOT_ATTEMPTED',\n            gradeLetter: 'C',\n            reason: 'no parseable answer',\n            gold: meta.gold,\n            topic: meta.topic,\n            answerType: meta.answerType,\n            citationCount: citations.length,\n          }),\n        }\n      }\n\n      const grade = await gradeAnswer(meta.question, meta.gold, finalAnswer, graderRouter())\n      const gradeLetter = grade === 'CORRECT' ? 'A' : grade === 'INCORRECT' ? 'B' : 'C'\n      const resolved = grade === 'CORRECT'\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({\n          grade,\n          gradeLetter,\n          gold: meta.gold,\n          predicted: finalAnswer,\n          topic: meta.topic,\n          answerType: meta.answerType,\n          citationCount: citations.length,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";AA2BA,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AACzB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AACxC,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,KAAK,KAAK,YAAY,SAAS,OAAO,QAAQ;AACpD,IAAM,WAAW,KAAK,YAAY,YAAY,eAAe;AAE7D,IAAM,cACJ;AACF,IAAM,wBAAwB;AAG9B,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA,iDAAiD,qBAAqB;AAAA,EACtE;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AA+CX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;AACtE,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,IAAI,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG;AAAA,IAClE,WAAW,OAAO,OAAO;AAAA,EAC3B,CAAC;AACD,SAAO;AACT;AAQO,SAAS,iBAAiB,UAA0B;AACzD,QAAM,QAAQ,SAAS,MAAM,OAAO;AACpC,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,OAAO,MAAM,CAAC,KAAK;AACzB,UAAM,MAAM,KAAK,YAAY,EAAE,QAAQ,qBAAqB;AAC5D,QAAI,QAAQ,GAAI,QAAO,KAAK,MAAM,MAAM,sBAAsB,MAAM,EAAE,KAAK;AAAA,EAC7E;AACA,WAAS,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK,GAAG;AAC7C,UAAM,KAAK,MAAM,CAAC,KAAK,IAAI,KAAK;AAChC,QAAI,EAAE,SAAS,EAAG,QAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAGO,SAAS,eAAe,UAA4B;AACzD,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,SAAS,SAAS,yBAAyB,GAAG;AAC5D,SAAK,IAAI,EAAE,CAAC,EAAE,QAAQ,WAAW,EAAE,CAAC;AAAA,EACtC;AACA,SAAO,CAAC,GAAG,IAAI;AACjB;AASA,IAAM,gBAAgB,CAAC,UAAkB,MAAc,cACrD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa,QAAQ;AAAA,EACrB,gBAAgB,IAAI;AAAA,EACpB,qBAAqB,SAAS;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAQb,SAAS,eAA6B;AACpC,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,6EAA6E;AACvG,QAAM,QAAQ,QAAQ,IAAI,eAAe;AACzC,QAAM,UAAU,QAAQ,IAAI,eAAe;AAC3C,SAAO,EAAE,SAAS,KAAK,MAAM;AAC/B;AAGA,eAAe,YACb,UACA,MACA,WACA,QACgB;AAChB,QAAM,MAAM,MAAM,MAAM,GAAG,OAAO,OAAO,qBAAqB;AAAA,IAC5D,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,oBAAoB,eAAe,UAAU,OAAO,GAAG,GAAG;AAAA,IACrF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,OAAO;AAAA,MACd,aAAa;AAAA,MACb,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,cAAc,UAAU,MAAM,SAAS,EAAE,CAAC;AAAA,IAChF,CAAC;AAAA,EACH,CAAC;AACD,MAAI,CAAC,IAAI,IAAI;AACX,UAAM,IAAI,MAAM,wBAAwB,IAAI,MAAM,MAAM,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EAC3F;AACA,QAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,QAAM,UAAU,KAAK,UAAU,CAAC,GAAG,SAAS;AAC5C,MAAI,OAAO,YAAY,UAAU;AAC/B,UAAM,IAAI,MAAM,gDAAgD,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACtG;AACA,QAAM,SAAS,QAAQ,MAAM,8BAA8B;AAC3D,QAAM,OAAO,SAAS,OAAO,CAAC,IAAI,UAAU,KAAK,KAAK;AACtD,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;AAAA,EACzB,QAAQ;AACN,UAAM,IAAI,MAAM,gEAAgE,QAAQ,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACzG;AACA,MAAI,OAAO,UAAU,aAAa,OAAO,UAAU,eAAe,OAAO,UAAU,iBAAiB;AAClG,WAAO,OAAO;AAAA,EAChB;AACA,QAAM,IAAI;AAAA,IACR,mEAAmE,KAAK,UAAU,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC;AAAA,EACzG;AACF;AAEA,SAAS,UAAU,KAAkB,OAA0B;AAC7D,QAAM,OAAqB;AAAA,IACzB,MAAM,IAAI;AAAA,IACV,aAAa,IAAI;AAAA,IACjB,OAAO,IAAI;AAAA,IACX,YAAY,IAAI;AAAA,IAChB,UAAU,IAAI;AAAA,EAChB;AACA,SAAO;AAAA,IACL,IAAI,YAAY,KAAK;AAAA,IACrB,OAAO;AAAA,IACP,QAAQ,IAAI,UAAU;AAAA,IACtB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,SAAS,YAAY,OAAO,GAAG,aAAa,UAAU;AACzE,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,sEAAiE;AAAA,EAC3G;AACA,SAAO;AACT;AAEA,SAAS,YAAY,OAAoB,MAAgC;AACvE,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,WAAO,MAAM,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;AAAA,EAC3C;AACA,MAAI,KAAK,UAAU,OAAW,QAAO,MAAM,MAAM,GAAG,KAAK,KAAK;AAC9D,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,iDAA4C,KAAK,MAAM,kCAAkC;AACtG,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;AAQA,eAAe,SAAS,MAAyC;AAC/D,QAAM,SAAS;AAAA;AAAA;AAAA,eAGF,KAAK,UAAU,WAAW,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAexC,QAAM,SAAS,MAAM,GAAG,MAAM;AAC9B,QAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,SAAO,YAAY,KAAK,IAAI,SAAS,GAAG,IAAI;AAC9C;AAEO,SAAS,wBAA0C;AACxD,QAAM,eAAe,QAAQ,IAAI,sBAAsB;AAEvD,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAGhB,mBAAa;AACb,UAAI,cAAc;AAChB,cAAM,SAAS,UAAU,MAAM,EAAE,MAAM,CAAC,QAAQ;AAC9C,gBAAM,IAAI;AAAA,YACR,2BAA2B,QAAQ,gBAAgB,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAC7F;AAAA,QACF,CAAC;AACD;AAAA,MACF;AACA,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,eACK,KAAK,UAAU,WAAW,CAAC;AAAA;AAAA;AAAA;AAAA,QAIlC;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,8BAA8B,GAAG;AAAA,qGAEC,WAAW;AAAA,QAE/C;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,aAAO,SAAS,IAAI;AAAA,IACtB;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,aAAO,GAAG,qBAAqB,IAAI,KAAK,IAAI;AAAA,IAC9C;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,cAAc,iBAAiB,QAAQ;AAC7C,YAAM,YAAY,eAAe,QAAQ;AAEzC,UAAI,YAAY,WAAW,GAAG;AAE5B,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU;AAAA,YACrB,OAAO;AAAA,YACP,aAAa;AAAA,YACb,QAAQ;AAAA,YACR,MAAM,KAAK;AAAA,YACX,OAAO,KAAK;AAAA,YACZ,YAAY,KAAK;AAAA,YACjB,eAAe,UAAU;AAAA,UAC3B,CAAC;AAAA,QACH;AAAA,MACF;AAEA,YAAM,QAAQ,MAAM,YAAY,KAAK,UAAU,KAAK,MAAM,aAAa,aAAa,CAAC;AACrF,YAAM,cAAc,UAAU,YAAY,MAAM,UAAU,cAAc,MAAM;AAC9E,YAAM,WAAW,UAAU;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,OAAO,WAAW,IAAI;AAAA,QACtB,QAAQ,KAAK,UAAU;AAAA,UACrB;AAAA,UACA;AAAA,UACA,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,OAAO,KAAK;AAAA,UACZ,YAAY,KAAK;AAAA,UACjB,eAAe,UAAU;AAAA,QAC3B,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-TBKU5XQI.js

@@ -1,228 +0,0 @@
-// src/benchmarks/mind2web.ts
-import { execFile } from "child_process";
-import { mkdir } from "fs/promises";
-import { tmpdir } from "os";
-import { join } from "path";
-import { fileURLToPath } from "url";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
-var PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, ".venv", "bin", "python");
-var DATASET = "osunlp/Multimodal-Mind2Web";
-var DEFAULT_SPLIT = process.env.M2W_SPLIT ?? "test_domain";
-var SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), "m2w-shots");
-var CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30);
-var WORKER_CONTRACT = [
-  "",
-  "Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:",
-  "ELEMENT: <the [id] number of the chosen element>",
-  "ACTION: <CLICK | TYPE | SELECT>",
-  "VALUE: <text to type or option to select; leave empty for CLICK>"
-].join("\n");
-async function py(script, args = []) {
-  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], { maxBuffer: 1024 * 1024 * 256 });
-  return stdout;
-}
-var LOADER = `
-import json, sys, os
-from datasets import load_dataset
-cfg = json.loads(sys.argv[1])
-split = cfg["split"]; limit = cfg.get("limit"); cap = cfg.get("cap", 30)
-shots = cfg["shotsDir"]; ids = set(cfg["ids"]) if cfg.get("ids") else None
-os.makedirs(shots, exist_ok=True)
-KEEP = ("aria_label","aria-label","role","type","name","placeholder","title","alt","value","text","id","class","href")
-def label(tag, attr):
-    try:
-        a = json.loads(attr) if isinstance(attr, str) else (attr or {})
-    except Exception:
-        a = {}
-    parts = []
-    for k in KEEP:
-        v = a.get(k)
-        if v:
-            sv = str(v).replace("\\n", " ").strip()
-            if k == "class": sv = sv[:40]
-            if k == "href": sv = sv[:50]
-            if sv: parts.append(k + "=" + sv[:60])
-    return "<" + str(tag) + "> " + " ".join(parts[:6])
-def cands(raw):
-    out = []
-    for c in raw or []:
-        try:
-            d = json.loads(c) if isinstance(c, str) else c
-        except Exception:
-            continue
-        bid = str(d.get("backend_node_id", ""))
-        if not bid: continue
-        out.append({"id": bid, "label": label(d.get("tag", "?"), d.get("attributes"))})
-    return out
-ds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)
-emitted = []; skipped = 0; scanned = 0
-for r in ds:
-    scanned += 1
-    if ids is not None and r["action_uid"] not in ids:
-        if scanned > 8000: break
-        continue
-    pos = cands(r.get("pos_candidates"))
-    if not pos:
-        skipped += 1
-        continue
-    neg = cands(r.get("neg_candidates"))
-    seen = set(p["id"] for p in pos); merged = list(pos)
-    for n in neg:
-        if len(merged) >= cap: break
-        if n["id"] in seen: continue
-        seen.add(n["id"]); merged.append(n)
-    merged.sort(key=lambda x: int(x["id"]) if x["id"].isdigit() else 0)
-    try:
-        op = json.loads(r["operation"]) if isinstance(r["operation"], str) else r["operation"]
-    except Exception:
-        op = {}
-    sp = os.path.join(shots, str(r["action_uid"]) + ".jpg")
-    try:
-        r["screenshot"].convert("RGB").save(sp, "JPEG", quality=70)
-    except Exception:
-        sp = ""
-    emitted.append({
-        "id": r["action_uid"], "task": r.get("confirmed_task", ""),
-        "website": r.get("website", ""), "domain": r.get("domain", ""), "subdomain": r.get("subdomain", ""),
-        "op": str(op.get("op", "")).upper(), "value": str(op.get("value", "")),
-        "goldIds": [p["id"] for p in pos], "candidates": merged,
-        "screenshotPath": sp, "targetRepr": r.get("target_action_reprs", ""),
-    })
-    if ids is None and limit is not None and len(emitted) >= limit: break
-    if ids is not None and len(emitted) >= len(ids): break
-sys.stderr.write("[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\n" % (len(emitted), skipped, scanned)); sys.stderr.flush()
-print(json.dumps(emitted)); sys.stdout.flush()
-os._exit(0)
-`;
-function buildPrompt(row) {
-  const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join("\n");
-  return [
-    `Web task: ${row.task}`,
-    "",
-    "You are taking the NEXT single action on the current web page. Choose the one element",
-    'to act on from the candidates below (each line is "[id] <tag> attributes"):',
-    choices,
-    WORKER_CONTRACT
-  ].join("\n");
-}
-function rowToTask(row) {
-  const meta = {
-    task: row.task,
-    website: row.website,
-    domain: row.domain,
-    op: row.op,
-    value: row.value,
-    goldIds: row.goldIds,
-    candidateIds: row.candidates.map((c) => c.id),
-    screenshotPath: row.screenshotPath,
-    targetRepr: row.targetRepr
-  };
-  return {
-    id: `mind2web-${row.id}`,
-    split: DEFAULT_SPLIT,
-    prompt: buildPrompt(row),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || !Array.isArray(md.goldIds)) {
-    throw new Error(`mind2web task ${task.id} missing metadata.goldIds \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function normValue(s) {
-  return s.toLowerCase().replace(/\s+/g, " ").trim();
-}
-function parseAction(artifact) {
-  const elem = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact);
-  const op = /ACTION:\s*(CLICK|TYPE|SELECT)/i.exec(artifact);
-  if (!elem?.[1] || !op?.[1]) return null;
-  const val = /VALUE:\s*(.*)/i.exec(artifact);
-  return { elementId: elem[1], op: op[1].toUpperCase(), value: (val?.[1] ?? "").trim() };
-}
-function createMind2WebAdapter() {
-  const split = DEFAULT_SPLIT;
-  return {
-    name: "mind2web",
-    async preflight() {
-      try {
-        await py(
-          `import os, sys
-from datasets import load_dataset
-import PIL
-ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)
-next(iter(ds))
-print('ok'); sys.stdout.flush(); os._exit(0)`
-        );
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `mind2web preflight failed: ${msg}
-Fix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; (2) network access to Hugging Face for ${DATASET} (split ${split}) ; point M2W_PYTHON at the python if not bench/.venv.`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      const cfg = {
-        split: opts.split ?? split,
-        limit: opts.ids ? void 0 : opts.limit ?? 10,
-        cap: CANDIDATE_CAP,
-        shotsDir: SHOTS_DIR,
-        ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, "")) : void 0
-      };
-      await mkdir(SHOTS_DIR, { recursive: true });
-      const stdout = await py(LOADER, [JSON.stringify(cfg)]);
-      const rows = JSON.parse(stdout);
-      return rows.map(rowToTask);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      const id = meta.goldIds[0];
-      if (!id) return void 0;
-      return `ELEMENT: ${id}
-ACTION: ${meta.op}
-VALUE: ${meta.value}`;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const parsed = parseAction(artifact);
-      if (!parsed) {
-        return {
-          resolved: false,
-          score: 0,
-          detail: JSON.stringify({ reason: "no parseable ELEMENT/ACTION", goldOp: meta.op, goldIds: meta.goldIds })
-        };
-      }
-      const elementCorrect = meta.goldIds.includes(parsed.elementId);
-      const opMatch = parsed.op === meta.op;
-      const valueMatch = meta.op === "CLICK" ? true : normValue(parsed.value) === normValue(meta.value);
-      const operationCorrect = opMatch && valueMatch;
-      const resolved = elementCorrect && operationCorrect;
-      const score = 0.6 * (elementCorrect ? 1 : 0) + 0.4 * (operationCorrect ? 1 : 0);
-      return {
-        resolved,
-        score,
-        detail: JSON.stringify({
-          elementCorrect,
-          opMatch,
-          valueMatch,
-          predicted: parsed,
-          goldOp: meta.op,
-          goldValue: meta.value,
-          goldIds: meta.goldIds,
-          website: meta.website,
-          domain: meta.domain
-        })
-      };
-    }
-  };
-}
-
-export {
-  parseAction,
-  createMind2WebAdapter
-};
-//# sourceMappingURL=chunk-TBKU5XQI.js.map

package/dist/chunk-TBKU5XQI.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/mind2web.ts"],"sourcesContent":["/**\n * Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).\n * Web-agent ACTION PREDICTION: each task is one step of a real web task — given\n * the natural-language goal and the page's candidate elements, the worker picks\n * the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).\n *\n * Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:\n *   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids\n *   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches\n *   resolved (Step-SR) ⇔ element correct AND operation correct\n *   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right\n *           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)\n * This is the low-noise reward a certifiable directive-lift needs: the number is a\n * programmatic match against human-verified ground truth, not a judge's opinion.\n *\n * The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the\n * judge extracts deterministically; the prompt presents the candidate set as a\n * choice over backend_node_id ordered by id (position uncorrelated with the answer).\n *\n * Each step carries the dataset's OWN page screenshot (written to a temp file by\n * the loader); the worker drops it into a browser.<op> span so run-capsule's screen\n * capsule turns the run into a film — the real page, not a re-rendered DOM.\n *\n * Requires for a live run: a python with `datasets` + `pillow` and network to\n * Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdir } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, '.venv', 'bin', 'python')\n\nconst DATASET = 'osunlp/Multimodal-Mind2Web'\nconst DEFAULT_SPLIT = process.env.M2W_SPLIT ?? 'test_domain'\nconst SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), 'm2w-shots')\n/** Candidate-set cap presented to the worker (always keeps all pos candidates). */\nconst CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30)\n\n/** The worker contract appended to every task prompt; the judge keys off these. */\nconst WORKER_CONTRACT = [\n  '',\n  'Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:',\n  'ELEMENT: <the [id] number of the chosen element>',\n  'ACTION: <CLICK | TYPE | SELECT>',\n  'VALUE: <text to type or option to select; leave empty for CLICK>',\n].join('\\n')\n\ninterface RawCandidate {\n  id: string\n  label: string\n}\ninterface Mind2WebRow {\n  id: string\n  task: string\n  website: string\n  domain: string\n  subdomain: string\n  op: string\n  value: string\n  goldIds: string[]\n  candidates: RawCandidate[]\n  screenshotPath: string\n  targetRepr: string\n}\ninterface Mind2WebMeta {\n  task: string\n  website: string\n  domain: string\n  op: string\n  value: string\n  goldIds: string[]\n  candidateIds: string[]\n  screenshotPath: string\n  targetRepr: string\n}\n\n/** Run the bench python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], { maxBuffer: 1024 * 1024 * 256 })\n  return stdout\n}\n\n/** The loader: stream the split, skip steps with no positive candidate (fail-loud,\n *  never a silent score-0), cap the candidate set, save each screenshot to a file. */\nconst LOADER = `\nimport json, sys, os\nfrom datasets import load_dataset\ncfg = json.loads(sys.argv[1])\nsplit = cfg[\"split\"]; limit = cfg.get(\"limit\"); cap = cfg.get(\"cap\", 30)\nshots = cfg[\"shotsDir\"]; ids = set(cfg[\"ids\"]) if cfg.get(\"ids\") else None\nos.makedirs(shots, exist_ok=True)\nKEEP = (\"aria_label\",\"aria-label\",\"role\",\"type\",\"name\",\"placeholder\",\"title\",\"alt\",\"value\",\"text\",\"id\",\"class\",\"href\")\ndef label(tag, attr):\n    try:\n        a = json.loads(attr) if isinstance(attr, str) else (attr or {})\n    except Exception:\n        a = {}\n    parts = []\n    for k in KEEP:\n        v = a.get(k)\n        if v:\n            sv = str(v).replace(\"\\\\n\", \" \").strip()\n            if k == \"class\": sv = sv[:40]\n            if k == \"href\": sv = sv[:50]\n            if sv: parts.append(k + \"=\" + sv[:60])\n    return \"<\" + str(tag) + \"> \" + \" \".join(parts[:6])\ndef cands(raw):\n    out = []\n    for c in raw or []:\n        try:\n            d = json.loads(c) if isinstance(c, str) else c\n        except Exception:\n            continue\n        bid = str(d.get(\"backend_node_id\", \"\"))\n        if not bid: continue\n        out.append({\"id\": bid, \"label\": label(d.get(\"tag\", \"?\"), d.get(\"attributes\"))})\n    return out\nds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)\nemitted = []; skipped = 0; scanned = 0\nfor r in ds:\n    scanned += 1\n    if ids is not None and r[\"action_uid\"] not in ids:\n        if scanned > 8000: break\n        continue\n    pos = cands(r.get(\"pos_candidates\"))\n    if not pos:\n        skipped += 1\n        continue\n    neg = cands(r.get(\"neg_candidates\"))\n    seen = set(p[\"id\"] for p in pos); merged = list(pos)\n    for n in neg:\n        if len(merged) >= cap: break\n        if n[\"id\"] in seen: continue\n        seen.add(n[\"id\"]); merged.append(n)\n    merged.sort(key=lambda x: int(x[\"id\"]) if x[\"id\"].isdigit() else 0)\n    try:\n        op = json.loads(r[\"operation\"]) if isinstance(r[\"operation\"], str) else r[\"operation\"]\n    except Exception:\n        op = {}\n    sp = os.path.join(shots, str(r[\"action_uid\"]) + \".jpg\")\n    try:\n        r[\"screenshot\"].convert(\"RGB\").save(sp, \"JPEG\", quality=70)\n    except Exception:\n        sp = \"\"\n    emitted.append({\n        \"id\": r[\"action_uid\"], \"task\": r.get(\"confirmed_task\", \"\"),\n        \"website\": r.get(\"website\", \"\"), \"domain\": r.get(\"domain\", \"\"), \"subdomain\": r.get(\"subdomain\", \"\"),\n        \"op\": str(op.get(\"op\", \"\")).upper(), \"value\": str(op.get(\"value\", \"\")),\n        \"goldIds\": [p[\"id\"] for p in pos], \"candidates\": merged,\n        \"screenshotPath\": sp, \"targetRepr\": r.get(\"target_action_reprs\", \"\"),\n    })\n    if ids is None and limit is not None and len(emitted) >= limit: break\n    if ids is not None and len(emitted) >= len(ids): break\nsys.stderr.write(\"[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\\\n\" % (len(emitted), skipped, scanned)); sys.stderr.flush()\nprint(json.dumps(emitted)); sys.stdout.flush()\nos._exit(0)\n`\n\nfunction buildPrompt(row: Mind2WebRow): string {\n  const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join('\\n')\n  return [\n    `Web task: ${row.task}`,\n    '',\n    'You are taking the NEXT single action on the current web page. Choose the one element',\n    'to act on from the candidates below (each line is \"[id] <tag> attributes\"):',\n    choices,\n    WORKER_CONTRACT,\n  ].join('\\n')\n}\n\nfunction rowToTask(row: Mind2WebRow): BenchTask {\n  const meta: Mind2WebMeta = {\n    task: row.task,\n    website: row.website,\n    domain: row.domain,\n    op: row.op,\n    value: row.value,\n    goldIds: row.goldIds,\n    candidateIds: row.candidates.map((c) => c.id),\n    screenshotPath: row.screenshotPath,\n    targetRepr: row.targetRepr,\n  }\n  return {\n    id: `mind2web-${row.id}`,\n    split: DEFAULT_SPLIT,\n    prompt: buildPrompt(row),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): Mind2WebMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray((md as { goldIds?: unknown }).goldIds)) {\n    throw new Error(`mind2web task ${task.id} missing metadata.goldIds — loadTasks did not populate it`)\n  }\n  return md as unknown as Mind2WebMeta\n}\n\n/** Normalize a TYPE/SELECT value for comparison: lowercase, collapse whitespace. */\nfunction normValue(s: string): string {\n  return s.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\ninterface ParsedAction {\n  elementId: string\n  op: string\n  value: string\n}\n\n/** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.\n *  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */\nexport function parseAction(artifact: string): ParsedAction | null {\n  const elem = /ELEMENT:\\s*\\[?(\\d+)\\]?/i.exec(artifact)\n  const op = /ACTION:\\s*(CLICK|TYPE|SELECT)/i.exec(artifact)\n  if (!elem?.[1] || !op?.[1]) return null\n  const val = /VALUE:\\s*(.*)/i.exec(artifact)\n  return { elementId: elem[1], op: op[1].toUpperCase(), value: (val?.[1] ?? '').trim() }\n}\n\nexport function createMind2WebAdapter(): BenchmarkAdapter {\n  const split = DEFAULT_SPLIT\n\n  return {\n    name: 'mind2web',\n\n    async preflight() {\n      try {\n        await py(\n          `import os, sys\nfrom datasets import load_dataset\nimport PIL\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)\nnext(iter(ds))\nprint('ok'); sys.stdout.flush(); os._exit(0)`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `mind2web preflight failed: ${msg}\\n` +\n            `Fix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; ` +\n            `(2) network access to Hugging Face for ${DATASET} (split ${split}) ; ` +\n            `point M2W_PYTHON at the python if not bench/.venv.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const cfg = {\n        split: opts.split ?? split,\n        limit: opts.ids ? undefined : (opts.limit ?? 10),\n        cap: CANDIDATE_CAP,\n        shotsDir: SHOTS_DIR,\n        ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, '')) : undefined,\n      }\n      await mkdir(SHOTS_DIR, { recursive: true })\n      const stdout = await py(LOADER, [JSON.stringify(cfg)])\n      const rows = JSON.parse(stdout) as Mind2WebRow[]\n      return rows.map(rowToTask)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the worker-contract serialization of the ground-truth action, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes. The first accepted positive id is the canonical target.\n      const meta = readMeta(task)\n      const id = meta.goldIds[0]\n      if (!id) return undefined\n      return `ELEMENT: ${id}\\nACTION: ${meta.op}\\nVALUE: ${meta.value}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const parsed = parseAction(artifact)\n      if (!parsed) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ reason: 'no parseable ELEMENT/ACTION', goldOp: meta.op, goldIds: meta.goldIds }),\n        }\n      }\n      const elementCorrect = meta.goldIds.includes(parsed.elementId)\n      const opMatch = parsed.op === meta.op\n      const valueMatch = meta.op === 'CLICK' ? true : normValue(parsed.value) === normValue(meta.value)\n      const operationCorrect = opMatch && valueMatch\n      const resolved = elementCorrect && operationCorrect\n      const score = 0.6 * (elementCorrect ? 1 : 0) + 0.4 * (operationCorrect ? 1 : 0)\n      return {\n        resolved,\n        score,\n        detail: JSON.stringify({\n          elementCorrect,\n          opMatch,\n          valueMatch,\n          predicted: parsed,\n          goldOp: meta.op,\n          goldValue: meta.value,\n          goldIds: meta.goldIds,\n          website: meta.website,\n          domain: meta.domain,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";AA2BA,SAAS,gBAAgB;AACzB,SAAS,aAAa;AACtB,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AACxC,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,KAAK,QAAQ,IAAI,cAAc,KAAK,YAAY,SAAS,OAAO,QAAQ;AAE9E,IAAM,UAAU;AAChB,IAAM,gBAAgB,QAAQ,IAAI,aAAa;AAC/C,IAAM,YAAY,QAAQ,IAAI,aAAa,KAAK,OAAO,GAAG,WAAW;AAErE,IAAM,gBAAgB,OAAO,QAAQ,IAAI,qBAAqB,EAAE;AAGhE,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAgCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;AACtE,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,IAAI,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAE,WAAW,OAAO,OAAO,IAAI,CAAC;AACpG,SAAO;AACT;AAIA,IAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oBAiCK,KAAK,UAAU,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAyC3C,SAAS,YAAY,KAA0B;AAC7C,QAAM,UAAU,IAAI,WAAW,IAAI,CAAC,MAAM,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,KAAK,IAAI;AAC7E,SAAO;AAAA,IACL,aAAa,IAAI,IAAI;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAA6B;AAC9C,QAAM,OAAqB;AAAA,IACzB,MAAM,IAAI;AAAA,IACV,SAAS,IAAI;AAAA,IACb,QAAQ,IAAI;AAAA,IACZ,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,SAAS,IAAI;AAAA,IACb,cAAc,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,EAAE;AAAA,IAC5C,gBAAgB,IAAI;AAAA,IACpB,YAAY,IAAI;AAAA,EAClB;AACA,SAAO;AAAA,IACL,IAAI,YAAY,IAAI,EAAE;AAAA,IACtB,OAAO;AAAA,IACP,QAAQ,YAAY,GAAG;AAAA,IACvB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAS,GAA6B,OAAO,GAAG;AAChE,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,gEAA2D;AAAA,EACrG;AACA,SAAO;AACT;AAGA,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACnD;AAUO,SAAS,YAAY,UAAuC;AACjE,QAAM,OAAO,0BAA0B,KAAK,QAAQ;AACpD,QAAM,KAAK,iCAAiC,KAAK,QAAQ;AACzD,MAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAG,QAAO;AACnC,QAAM,MAAM,iBAAiB,KAAK,QAAQ;AAC1C,SAAO,EAAE,WAAW,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,YAAY,GAAG,QAAQ,MAAM,CAAC,KAAK,IAAI,KAAK,EAAE;AACvF;AAEO,SAAS,wBAA0C;AACxD,QAAM,QAAQ;AAEd,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI;AACF,cAAM;AAAA,UACJ;AAAA;AAAA;AAAA,oBAGU,KAAK,UAAU,OAAO,CAAC,WAAW,KAAK,UAAU,KAAK,CAAC;AAAA;AAAA;AAAA,QAGnE;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,8BAA8B,GAAG;AAAA,iKAEW,OAAO,WAAW,KAAK;AAAA,QAErE;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,MAAM;AAAA,QACV,OAAO,KAAK,SAAS;AAAA,QACrB,OAAO,KAAK,MAAM,SAAa,KAAK,SAAS;AAAA,QAC7C,KAAK;AAAA,QACL,UAAU;AAAA,QACV,KAAK,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,GAAG,QAAQ,cAAc,EAAE,CAAC,IAAI;AAAA,MACvE;AACA,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,SAAS,MAAM,GAAG,QAAQ,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC;AACrD,YAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,aAAO,KAAK,IAAI,SAAS;AAAA,IAC3B;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,KAAK,KAAK,QAAQ,CAAC;AACzB,UAAI,CAAC,GAAI,QAAO;AAChB,aAAO,YAAY,EAAE;AAAA,UAAa,KAAK,EAAE;AAAA,SAAY,KAAK,KAAK;AAAA,IACjE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,SAAS,YAAY,QAAQ;AACnC,UAAI,CAAC,QAAQ;AACX,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU,EAAE,QAAQ,+BAA+B,QAAQ,KAAK,IAAI,SAAS,KAAK,QAAQ,CAAC;AAAA,QAC1G;AAAA,MACF;AACA,YAAM,iBAAiB,KAAK,QAAQ,SAAS,OAAO,SAAS;AAC7D,YAAM,UAAU,OAAO,OAAO,KAAK;AACnC,YAAM,aAAa,KAAK,OAAO,UAAU,OAAO,UAAU,OAAO,KAAK,MAAM,UAAU,KAAK,KAAK;AAChG,YAAM,mBAAmB,WAAW;AACpC,YAAM,WAAW,kBAAkB;AACnC,YAAM,QAAQ,OAAO,iBAAiB,IAAI,KAAK,OAAO,mBAAmB,IAAI;AAC7E,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,QAAQ,KAAK,UAAU;AAAA,UACrB;AAAA,UACA;AAAA,UACA;AAAA,UACA,WAAW;AAAA,UACX,QAAQ,KAAK;AAAA,UACb,WAAW,KAAK;AAAA,UAChB,SAAS,KAAK;AAAA,UACd,SAAS,KAAK;AAAA,UACd,QAAQ,KAAK;AAAA,QACf,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}