@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/chunk-C7T7WEK2.js

@@ -1,103 +0,0 @@
-// src/benchmarks/cadgenbench.ts
-import { execFile as execFile2 } from "child_process";
-import { mkdtemp as mkdtemp2, rm as rm2, writeFile as writeFile2 } from "fs/promises";
-import { tmpdir as tmpdir2 } from "os";
-import { join as join2 } from "path";
-import { promisify as promisify2 } from "util";
-
-// src/worker-build123d.ts
-import { execFile } from "child_process";
-import { existsSync, readFileSync } from "fs";
-import { mkdtemp, readFile, rm, writeFile } from "fs/promises";
-import { tmpdir } from "os";
-import { join } from "path";
-import { promisify } from "util";
-import { routerChatWithUsage } from "@tangle-network/agent-runtime/loops";
-var execFileAsync = promisify(execFile);
-var CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? "/tmp/cgb-venv/bin/python";
-var CGB_DIR = process.env.CADGENBENCH_DIR ?? "/tmp/cadgenbench";
-
-// src/benchmarks/cadgenbench.ts
-var execFileAsync2 = promisify2(execFile2);
-var SCORE_PY = `
-import sys, json, tempfile, shutil
-from pathlib import Path
-from cadgenbench.eval.evaluate import evaluate_result
-cand, gt = Path(sys.argv[1]), Path(sys.argv[2])
-with tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:
-    rd, gd = Path(rd), Path(gd)
-    (rd / 'result.json').write_text('{}')
-    shutil.copy(gt, gd / 'ground_truth.step')
-    try:
-        evaluate_result(rd, gd, candidate_step=cand)
-        d = json.loads((rd / 'result.json').read_text())
-        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))
-    except Exception as e:
-        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))
-`.trim();
-function fixtureTasks() {
-  const g = join2(CGB_DIR, "tests/fixtures/geometry");
-  return [
-    { id: "box-10x20x30", prompt: "A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).", gtStep: join2(g, "box_10_20_30.step") },
-    { id: "cube-10", prompt: "A cube, 10 units on every side.", gtStep: join2(g, "box_10_10_10.step") },
-    { id: "sphere-10", prompt: "A sphere of radius 10 units, centered at the origin.", gtStep: join2(g, "sphere_10.step") }
-  ];
-}
-function createCadGenBenchAdapter() {
-  return {
-    name: "cadgenbench",
-    async preflight() {
-      const r = await execFileAsync2(CGB_VENV_PY, ["-c", 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print("ok")'], { timeout: 6e4 }).catch(
-        (e) => ({ stdout: "", stderr: e instanceof Error ? e.message : String(e) })
-      );
-      if (!/ok/.test(r.stdout)) {
-        throw new Error(
-          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}
-Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`
-        );
-      }
-    },
-    async loadTasks(opts = {}) {
-      let tasks = fixtureTasks();
-      const hard = process.env.CGB_HARD_DIR;
-      if (hard) {
-        const { readFile: readFile2 } = await import("fs/promises");
-        tasks = JSON.parse(await readFile2(join2(hard, "tasks.json"), "utf8"));
-      }
-      if (opts.ids) tasks = tasks.filter((t) => opts.ids.includes(t.id));
-      if (opts.limit != null) tasks = tasks.slice(0, opts.limit);
-      const meta = (gtStep) => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) });
-      return tasks.map((t) => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) }));
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const { gtStep, resolveThreshold } = task.metadata;
-      if (!artifact.includes("ISO-10303-21")) return { resolved: false, score: 0, detail: "artifact is not a STEP file" };
-      const dir = await mkdtemp2(join2(tmpdir2(), "cgb-judge-"));
-      const cand = join2(dir, "candidate.step");
-      const scorer = join2(dir, "score.py");
-      try {
-        await writeFile2(cand, artifact);
-        await writeFile2(scorer, SCORE_PY);
-        const r = await execFileAsync2("xvfb-run", ["-a", CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 18e4 }).catch(
-          (e) => ({ stdout: e.stdout ?? "", stderr: e instanceof Error ? e.message : String(e) })
-        );
-        const m = /CGB_SCORE (\{.*\})/.exec(r.stdout);
-        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` };
-        const v = JSON.parse(m[1]);
-        const score = typeof v.cad_score === "number" ? v.cad_score : 0;
-        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ""}` };
-      } finally {
-        await rm2(dir, { recursive: true, force: true }).catch(() => {
-        });
-      }
-    }
-  };
-}
-
-export {
-  createCadGenBenchAdapter
-};
-//# sourceMappingURL=chunk-C7T7WEK2.js.map

package/dist/chunk-C7T7WEK2.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/cadgenbench.ts","../src/worker-build123d.ts"],"sourcesContent":["/**\n * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part\n * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN\n * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to\n * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.\n * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.\n *\n * The official task set (private GT, server-side graded) isn't released yet, so\n * tasks here are seeded from the repo's dimension-named geometry fixtures (real\n * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap\n * loadTasks to read the published fixtures' description.yaml + ground_truth.step.\n *\n * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +\n * xvfb (the scorer's alignment renders need a display).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdtemp, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types'\nimport { CGB_DIR, CGB_VENV_PY } from '../worker-build123d'\n\nconst execFileAsync = promisify(execFile)\n\n/** Self-contained scorer wrapper (written to a temp file, run in the venv).\n *  Scores a candidate STEP against a ground-truth STEP via the benchmark's own\n *  evaluate_result, printing the cad_score line. */\nconst SCORE_PY = `\nimport sys, json, tempfile, shutil\nfrom pathlib import Path\nfrom cadgenbench.eval.evaluate import evaluate_result\ncand, gt = Path(sys.argv[1]), Path(sys.argv[2])\nwith tempfile.TemporaryDirectory() as rd, tempfile.TemporaryDirectory() as gd:\n    rd, gd = Path(rd), Path(gd)\n    (rd / 'result.json').write_text('{}')\n    shutil.copy(gt, gd / 'ground_truth.step')\n    try:\n        evaluate_result(rd, gd, candidate_step=cand)\n        d = json.loads((rd / 'result.json').read_text())\n        print('CGB_SCORE ' + json.dumps({'cad_score': d.get('cad_score', 0.0), 'status': d.get('status', 'unknown')}))\n    except Exception as e:\n        print('CGB_SCORE ' + json.dumps({'cad_score': 0.0, 'status': 'error', 'error': str(e)[:200]}))\n`.trim()\n\ninterface CgbMeta {\n  gtStep: string\n  resolveThreshold: number\n}\n\n/** Fixture-seeded tasks (real GT STEPs from the repo, dim-named so the spec is\n *  exact). Replaced by the published dataset when CADGENBENCH_DATA_DIR is set. */\nfunction fixtureTasks(): Array<{ id: string; prompt: string; gtStep: string }> {\n  const g = join(CGB_DIR, 'tests/fixtures/geometry')\n  return [\n    { id: 'box-10x20x30', prompt: 'A rectangular solid box, 10 units wide (X), 20 units deep (Y), and 30 units tall (Z).', gtStep: join(g, 'box_10_20_30.step') },\n    { id: 'cube-10', prompt: 'A cube, 10 units on every side.', gtStep: join(g, 'box_10_10_10.step') },\n    { id: 'sphere-10', prompt: 'A sphere of radius 10 units, centered at the origin.', gtStep: join(g, 'sphere_10.step') },\n  ]\n}\n\nexport function createCadGenBenchAdapter(): BenchmarkAdapter {\n  return {\n    name: 'cadgenbench',\n\n    async preflight() {\n      const r = await execFileAsync(CGB_VENV_PY, ['-c', 'import cadgenbench.eval.evaluate, build123d, trimesh, manifold3d; print(\"ok\")'], { timeout: 60_000 }).catch(\n        (e) => ({ stdout: '', stderr: e instanceof Error ? e.message : String(e) }),\n      )\n      if (!/ok/.test(r.stdout)) {\n        throw new Error(\n          `cadgenbench preflight failed (venv=${CGB_VENV_PY}): ${r.stderr.slice(0, 200)}\\n` +\n            `Fix: git clone https://github.com/huggingface/cadgenbench ${CGB_DIR}; python3 -m venv $CADGENBENCH_VENV; $CADGENBENCH_VENV/bin/pip install -e ${CGB_DIR}`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      // CGB_HARD_DIR (a dir with tasks.json = [{id,prompt,gtStep}]) overrides the\n      // trivial fixture primitives with hard multi-feature parts (real headroom).\n      let tasks = fixtureTasks()\n      const hard = process.env.CGB_HARD_DIR\n      if (hard) {\n        const { readFile } = await import('node:fs/promises')\n        tasks = JSON.parse(await readFile(join(hard, 'tasks.json'), 'utf8')) as Array<{ id: string; prompt: string; gtStep: string }>\n      }\n      if (opts.ids) tasks = tasks.filter((t) => opts.ids!.includes(t.id))\n      if (opts.limit != null) tasks = tasks.slice(0, opts.limit)\n      const meta = (gtStep: string): CgbMeta => ({ gtStep, resolveThreshold: Number(process.env.CGB_RESOLVE_THRESHOLD ?? 0.9) })\n      return tasks.map((t): BenchTask => ({ id: t.id, prompt: t.prompt, metadata: meta(t.gtStep) as unknown as Record<string, unknown> }))\n    },\n\n    async goldArtifact() {\n      return undefined // GT is a STEP file scored by the kernel, not a returnable artifact\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const { gtStep, resolveThreshold } = task.metadata as unknown as CgbMeta\n      if (!artifact.includes('ISO-10303-21')) return { resolved: false, score: 0, detail: 'artifact is not a STEP file' }\n      const dir = await mkdtemp(join(tmpdir(), 'cgb-judge-'))\n      const cand = join(dir, 'candidate.step')\n      const scorer = join(dir, 'score.py')\n      try {\n        await writeFile(cand, artifact)\n        await writeFile(scorer, SCORE_PY)\n        // xvfb: the scorer's alignment step renders; needs a display.\n        const r = await execFileAsync('xvfb-run', ['-a', CGB_VENV_PY, scorer, cand, gtStep], { maxBuffer: 1 << 26, timeout: 180_000 }).catch(\n          (e) => ({ stdout: (e as { stdout?: string }).stdout ?? '', stderr: e instanceof Error ? e.message : String(e) }),\n        )\n        const m = /CGB_SCORE (\\{.*\\})/.exec(r.stdout)\n        if (!m) return { resolved: false, score: 0, detail: `scorer produced no verdict: ${(r.stderr || r.stdout).slice(0, 160)}` }\n        const v = JSON.parse(m[1]) as { cad_score: number; status: string; error?: string }\n        const score = typeof v.cad_score === 'number' ? v.cad_score : 0\n        return { resolved: score >= resolveThreshold, score, detail: `cad_score=${score.toFixed(3)} status=${v.status}${v.error ? ` (${v.error})` : ''}` }\n      } finally {\n        await rm(dir, { recursive: true, force: true }).catch(() => {})\n      }\n    },\n  }\n}\n","/**\n * CADGenBench worker. The deliverable is a STEP B-rep solid (output.step). We\n * author a build123d (Python on the OpenCascade kernel) script via the router,\n * execute it in the CADGenBench venv, and read back the produced output.step —\n * exactly the reference baseline's contract. The artifact returned IS the STEP\n * text, which the CADGenBench geometric scorer grades against the ground truth.\n *\n * The build123d authoring directive is the GEPA-optimizable surface; the\n * build123d API cheat sheet (shipped in the cadgenbench package) is appended as\n * fixed reference context. Requires the CADGenBench venv (CADGENBENCH_VENV) +\n * its clone (CADGENBENCH_DIR for the cheat sheet).\n */\n\nimport { execFile } from 'node:child_process'\nimport { existsSync, readFileSync } from 'node:fs'\nimport { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { promisify } from 'node:util'\nimport type { Span } from '@tangle-network/agent-eval'\nimport type { BenchTask } from './benchmarks/types'\nimport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\nimport { runRefineLoop } from './refine-loop'\nimport { routerChatWithUsage } from '@tangle-network/agent-runtime/loops'\n\nexport { DEFAULT_BUILD123D_DIRECTIVE } from './directives'\n\nconst execFileAsync = promisify(execFile)\n\nexport const CGB_VENV_PY = process.env.CADGENBENCH_VENV ?? '/tmp/cgb-venv/bin/python'\nexport const CGB_DIR = process.env.CADGENBENCH_DIR ?? '/tmp/cadgenbench'\n\nasync function runLocal(cmd: string, args: string[], cwd: string, timeoutMs = 120_000): Promise<{ code: number; stdout: string; stderr: string }> {\n  try {\n    const { stdout, stderr } = await execFileAsync(cmd, args, { cwd, maxBuffer: 1 << 26, timeout: timeoutMs })\n    return { code: 0, stdout, stderr }\n  } catch (err) {\n    const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }\n    return { code: typeof e.code === 'number' ? e.code : 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? String(err) }\n  }\n}\n\nfunction extractPy(text: string): string {\n  const fence = /```(?:python|py)?\\s*\\n([\\s\\S]*?)```/i.exec(text)\n  return (fence ? fence[1] : text).trim()\n}\n\nlet _cheat: string | null = null\nfunction cheatSheet(): string {\n  if (_cheat != null) return _cheat\n  const p = join(CGB_DIR, 'src/cadgenbench/baseline/build123d_cheat_sheet.md')\n  _cheat = existsSync(p) ? readFileSync(p, 'utf8').slice(0, 12000) : ''\n  return _cheat\n}\n\nexport interface Build123dConfig {\n  routerBaseUrl: string\n  routerKey: string\n  model: string\n  rounds?: number\n  /** The build123d authoring directive — the GEPA-optimizable surface. */\n  directive?: string\n}\n\nexport interface Build123dShot {\n  /** The produced STEP text (the artifact the CADGenBench scorer grades). */\n  artifact: string\n  /** The Python source the agent wrote. */\n  source: string\n  trace: Span[]\n  usage: { input: number; output: number }\n  ok: boolean\n  built: boolean\n  detail?: string\n}\n\n/** Author a build123d script via the router, execute it in the CADGenBench venv,\n *  read back output.step. Refine on execution error / missing STEP. */\nexport async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig): Promise<Build123dShot> {\n  const rounds = Math.max(1, cfg.rounds ?? 2)\n  const directive = cfg.directive ?? DEFAULT_BUILD123D_DIRECTIVE\n  const sys = `${directive}\\n\\nbuild123d API reference:\\n${cheatSheet()}`\n  const trace: Span[] = []\n  const runId = `cadgenbench-${task.id}`\n  let ts = Date.now()\n  const tick = () => (ts += 1)\n  const usage = { input: 0, output: 0 }\n  // Carried across rounds in closures (the round Artifact is the Python source; the\n  // STEP text + built flag + lastErr persist outside the loop). usage is REAL.\n  let step = ''\n  let built = false\n  let lastErr = ''\n\n  trace.push({ spanId: 's-brief', runId, kind: 'llm', name: 'brief', model: cfg.model, messages: [{ role: 'user', content: task.prompt }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n  // Migrated onto runRefineLoop: the mkdtemp scratch dir is the Ctx; built (STEP\n  // produced) is the early-stop, modeled as a judge so default-decide stops the\n  // loop. The round-2+ steer carries lastErr + the prior source verbatim.\n  const res = await runRefineLoop<string, string>({\n    rounds,\n    setup: () => mkdtemp(join(tmpdir(), 'b123d-')),\n    prompt: (round, history) =>\n      round === 1\n        ? task.prompt\n        : `Your previous build123d script failed:\\n${lastErr}\\n\\nPrevious script:\\n${history[history.length - 1]?.artifact ?? ''}\\n\\nFix it so it runs in python and writes a valid output.step. Brief:\\n${task.prompt}`,\n    runShot: async (user, round, dir) => {\n      const scriptPath = join(dir, 'build.py')\n      const stepPath = join(dir, 'output.step')\n      const { content, usage: u } = await routerChatWithUsage(cfg, [\n        { role: 'system', content: sys },\n        { role: 'user', content: user },\n      ])\n      if (u) {\n        usage.input += u.input\n        usage.output += u.output\n      }\n      const source = extractPy(content)\n      trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n      trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'build.py', content: source }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span)\n\n      await writeFile(scriptPath, source)\n      const run = await runLocal(CGB_VENV_PY, [scriptPath], dir)\n      const got = existsSync(stepPath) ? await readFile(stepPath, 'utf8').catch(() => '') : ''\n      built = got.includes('ISO-10303-21') && got.length > 200\n      lastErr = built ? '' : `${run.stdout}\\n${run.stderr}`.trim().slice(-800) || 'no output.step written'\n      if (built) step = got\n      trace.push({ spanId: `s-exec-${round}`, runId, kind: 'tool', name: `build123d r${round}`, toolName: 'shell.exec', args: 'python build.py', result: (built ? 'wrote output.step' : lastErr).slice(0, 1500), startedAt: tick(), endedAt: tick(), status: built ? 'ok' : 'error', error: built ? undefined : `exit ${run.code}` } as Span)\n      return { artifact: source }\n    },\n    judge: async () => ({ valid: built }),\n    teardown: (dir) => rm(dir, { recursive: true, force: true }).then(() => {}, () => {}),\n  })\n\n  return {\n    artifact: step,\n    source: res.final.artifact,\n    trace,\n    usage,\n    ok: res.final.artifact.trim().length > 0,\n    built,\n    detail: built ? 'exported output.step' : `did not produce a STEP in ${rounds} rounds${lastErr ? `; last: ${lastErr.slice(0, 140)}` : ''}`,\n  }\n}\n"],"mappings":";AAgBA,SAAS,YAAAA,iBAAgB;AACzB,SAAS,WAAAC,UAAS,MAAAC,KAAI,aAAAC,kBAAiB;AACvC,SAAS,UAAAC,eAAc;AACvB,SAAS,QAAAC,aAAY;AACrB,SAAS,aAAAC,kBAAiB;;;ACP1B,SAAS,gBAAgB;AACzB,SAAS,YAAY,oBAAoB;AACzC,SAAS,SAAS,UAAU,IAAI,iBAAiB;AACjD,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAK1B,SAAS,2BAA2B;AAIpC,IAAM,gBAAgB,UAAU,QAAQ;AAEjC,IAAM,cAAc,QAAQ,IAAI,oBAAoB;AACpD,IAAM,UAAU,QAAQ,IAAI,mBAAmB;;;ADNtD,IAAMC,iBAAgBC,WAAUC,SAAQ;AAKxC,IAAM,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAef,KAAK;AASP,SAAS,eAAsE;AAC7E,QAAM,IAAIC,MAAK,SAAS,yBAAyB;AACjD,SAAO;AAAA,IACL,EAAE,IAAI,gBAAgB,QAAQ,yFAAyF,QAAQA,MAAK,GAAG,mBAAmB,EAAE;AAAA,IAC5J,EAAE,IAAI,WAAW,QAAQ,mCAAmC,QAAQA,MAAK,GAAG,mBAAmB,EAAE;AAAA,IACjG,EAAE,IAAI,aAAa,QAAQ,wDAAwD,QAAQA,MAAK,GAAG,gBAAgB,EAAE;AAAA,EACvH;AACF;AAEO,SAAS,2BAA6C;AAC3D,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,YAAM,IAAI,MAAMH,eAAc,aAAa,CAAC,MAAM,+EAA+E,GAAG,EAAE,SAAS,IAAO,CAAC,EAAE;AAAA,QACvJ,CAAC,OAAO,EAAE,QAAQ,IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,EAAE;AAAA,MAC3E;AACA,UAAI,CAAC,KAAK,KAAK,EAAE,MAAM,GAAG;AACxB,cAAM,IAAI;AAAA,UACR,sCAAsC,WAAW,MAAM,EAAE,OAAO,MAAM,GAAG,GAAG,CAAC;AAAA,4DACd,OAAO,6EAA6E,OAAO;AAAA,QAC5J;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AAGtC,UAAI,QAAQ,aAAa;AACzB,YAAM,OAAO,QAAQ,IAAI;AACzB,UAAI,MAAM;AACR,cAAM,EAAE,UAAAI,UAAS,IAAI,MAAM,OAAO,aAAkB;AACpD,gBAAQ,KAAK,MAAM,MAAMA,UAASD,MAAK,MAAM,YAAY,GAAG,MAAM,CAAC;AAAA,MACrE;AACA,UAAI,KAAK,IAAK,SAAQ,MAAM,OAAO,CAAC,MAAM,KAAK,IAAK,SAAS,EAAE,EAAE,CAAC;AAClE,UAAI,KAAK,SAAS,KAAM,SAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AACzD,YAAM,OAAO,CAAC,YAA6B,EAAE,QAAQ,kBAAkB,OAAO,QAAQ,IAAI,yBAAyB,GAAG,EAAE;AACxH,aAAO,MAAM,IAAI,CAAC,OAAkB,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,QAAQ,UAAU,KAAK,EAAE,MAAM,EAAwC,EAAE;AAAA,IACrI;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,EAAE,QAAQ,iBAAiB,IAAI,KAAK;AAC1C,UAAI,CAAC,SAAS,SAAS,cAAc,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,8BAA8B;AAClH,YAAM,MAAM,MAAME,SAAQF,MAAKG,QAAO,GAAG,YAAY,CAAC;AACtD,YAAM,OAAOH,MAAK,KAAK,gBAAgB;AACvC,YAAM,SAASA,MAAK,KAAK,UAAU;AACnC,UAAI;AACF,cAAMI,WAAU,MAAM,QAAQ;AAC9B,cAAMA,WAAU,QAAQ,QAAQ;AAEhC,cAAM,IAAI,MAAMP,eAAc,YAAY,CAAC,MAAM,aAAa,QAAQ,MAAM,MAAM,GAAG,EAAE,WAAW,KAAK,IAAI,SAAS,KAAQ,CAAC,EAAE;AAAA,UAC7H,CAAC,OAAO,EAAE,QAAS,EAA0B,UAAU,IAAI,QAAQ,aAAa,QAAQ,EAAE,UAAU,OAAO,CAAC,EAAE;AAAA,QAChH;AACA,cAAM,IAAI,qBAAqB,KAAK,EAAE,MAAM;AAC5C,YAAI,CAAC,EAAG,QAAO,EAAE,UAAU,OAAO,OAAO,GAAG,QAAQ,gCAAgC,EAAE,UAAU,EAAE,QAAQ,MAAM,GAAG,GAAG,CAAC,GAAG;AAC1H,cAAM,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC;AACzB,cAAM,QAAQ,OAAO,EAAE,cAAc,WAAW,EAAE,YAAY;AAC9D,eAAO,EAAE,UAAU,SAAS,kBAAkB,OAAO,QAAQ,aAAa,MAAM,QAAQ,CAAC,CAAC,WAAW,EAAE,MAAM,GAAG,EAAE,QAAQ,KAAK,EAAE,KAAK,MAAM,EAAE,GAAG;AAAA,MACnJ,UAAE;AACA,cAAMQ,IAAG,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AACF;","names":["execFile","mkdtemp","rm","writeFile","tmpdir","join","promisify","execFileAsync","promisify","execFile","join","readFile","mkdtemp","tmpdir","writeFile","rm"]}

package/dist/chunk-HWST3SED.js

@@ -1,162 +0,0 @@
-import {
-  benchRoot,
-  runVenvPython
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/webarena-verified.ts
-import { access, readFile, stat } from "fs/promises";
-import { join, resolve } from "path";
-var FIXTURES = join(benchRoot, "fixtures", "webarena-verified.json");
-var DATASET_REL = join("assets", "dataset", "webarena-verified.json");
-var webarenaDir = () => process.env.WEBARENA_VERIFIED_DIR;
-var webarenaOutputDirOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:text|path)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-async function assertFile(path, label) {
-  try {
-    await access(path);
-  } catch (err) {
-    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-async function assertDir(path, label) {
-  try {
-    const s = await stat(path);
-    if (!s.isDirectory()) throw new Error("not a directory");
-  } catch (err) {
-    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-function rowToTask(row) {
-  const meta = {
-    taskId: row.task_id,
-    intentTemplateId: row.intent_template_id,
-    sites: row.sites ?? [],
-    startUrls: row.start_urls ?? [],
-    revision: row.revision,
-    eval: row.eval
-  };
-  return {
-    id: String(row.task_id),
-    prompt: [
-      "Run this WebArena-Verified browser task in the official environment.",
-      `Goal: ${row.intent}`,
-      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(", ")}` : void 0,
-      row.sites?.length ? `Sites: ${row.sites.join(", ")}` : void 0,
-      "",
-      "Return the path to the official WebArena-Verified run output directory for this task.",
-      "The judge expects that directory to contain the task response and network trace files."
-    ].filter(Boolean).join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "number") {
-    throw new Error(`webarena-verified task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, opts) {
-  let tasks = rows.map(rowToTask);
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`);
-  return tasks;
-}
-async function loadFixtures(opts) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures`);
-  return selectRows(rows, opts);
-}
-async function loadOfficialRows(dir, opts) {
-  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), "utf8"));
-  return selectRows(rows, opts);
-}
-async function runOfficialEval(root, taskId, outputDir) {
-  const script = `
-import json, os, subprocess, sys
-from pathlib import Path
-
-root = Path(sys.argv[1])
-task_id = sys.argv[2]
-output_dir = Path(sys.argv[3])
-env = os.environ.copy()
-env["PYTHONPATH"] = str(root / "src") + os.pathsep + env.get("PYTHONPATH", "")
-cmd = [sys.executable, "-m", "webarena_verified", "eval-tasks", "--task-ids", task_id, "--output-dir", str(output_dir)]
-proc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)
-if proc.returncode != 0:
-    raise SystemExit((proc.stderr or proc.stdout or f"exit {proc.returncode}")[:2000])
-result_path = output_dir / task_id / "eval_result.json"
-if not result_path.exists():
-    raise SystemExit(f"official evaluator did not write {result_path}")
-print(json.dumps(json.loads(result_path.read_text())))
-`;
-  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0);
-  return JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-}
-function createWebArenaVerifiedAdapter() {
-  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === "1";
-  return {
-    name: "webarena-verified",
-    output: webarenaOutputDirOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = webarenaDir();
-      if (!dir) {
-        throw new Error(
-          "WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified."
-        );
-      }
-      await assertFile(join(dir, DATASET_REL), "official dataset");
-      await assertFile(join(dir, "src", "webarena_verified", "__main__.py"), "official CLI module");
-      await runVenvPython(
-        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print("ok")',
-        [join(dir, "src")]
-      );
-    },
-    async loadTasks(opts = {}) {
-      if (fixturesMode) return loadFixtures(opts);
-      const dir = webarenaDir();
-      if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks");
-      return loadOfficialRows(dir, opts);
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const dir = webarenaDir();
-      if (!dir) throw new Error("WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts");
-      const outputDir = resolve(artifact.trim());
-      await assertDir(outputDir, "run output directory");
-      const meta = readMeta(task);
-      const report = await runOfficialEval(dir, meta.taskId, outputDir);
-      const score = typeof report.score === "number" ? report.score : 0;
-      const status = typeof report.status === "string" ? report.status : "unknown";
-      return {
-        resolved: score === 1,
-        score,
-        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir })
-      };
-    }
-  };
-}
-
-export {
-  webarenaOutputDirOutput,
-  createWebArenaVerifiedAdapter
-};
-//# sourceMappingURL=chunk-HWST3SED.js.map

package/dist/chunk-HWST3SED.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/webarena-verified.ts"],"sourcesContent":["/**\n * WebArena-Verified adapter (ServiceNow/webarena-verified).\n *\n * Worker artifact = a WebArena-Verified run output directory, not final chat text.\n * Judge = the official `webarena_verified eval-tasks` evaluator over that output\n * directory. The adapter refuses to score a plain answer so we do not turn a DOM\n * benchmark into a fake text benchmark.\n */\n\nimport { access, readFile, stat } from 'node:fs/promises'\nimport { join, resolve } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'webarena-verified.json')\nconst DATASET_REL = join('assets', 'dataset', 'webarena-verified.json')\n\ninterface WebArenaRow {\n  task_id: number\n  intent: string\n  intent_template_id?: number\n  sites?: string[]\n  start_urls?: string[]\n  eval?: unknown[]\n  revision?: number\n}\n\ninterface WebArenaMeta {\n  taskId: number\n  intentTemplateId?: number\n  sites: string[]\n  startUrls: string[]\n  revision?: number\n  eval?: unknown[]\n}\n\nconst webarenaDir = (): string | undefined => process.env.WEBARENA_VERIFIED_DIR\n\nexport const webarenaOutputDirOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|path)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertDir(path: string, label: string): Promise<void> {\n  try {\n    const s = await stat(path)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`WebArena-Verified: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction rowToTask(row: WebArenaRow): BenchTask {\n  const meta: WebArenaMeta = {\n    taskId: row.task_id,\n    intentTemplateId: row.intent_template_id,\n    sites: row.sites ?? [],\n    startUrls: row.start_urls ?? [],\n    revision: row.revision,\n    eval: row.eval,\n  }\n  return {\n    id: String(row.task_id),\n    prompt: [\n      'Run this WebArena-Verified browser task in the official environment.',\n      `Goal: ${row.intent}`,\n      row.start_urls?.length ? `Start URL templates: ${row.start_urls.join(', ')}` : undefined,\n      row.sites?.length ? `Sites: ${row.sites.join(', ')}` : undefined,\n      '',\n      'Return the path to the official WebArena-Verified run output directory for this task.',\n      'The judge expects that directory to contain the task response and network trace files.',\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): WebArenaMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number') {\n    throw new Error(`webarena-verified task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as WebArenaMeta\n}\n\nfunction selectRows(rows: WebArenaRow[], opts: LoadOptions): BenchTask[] {\n  let tasks = rows.map(rowToTask)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`WebArena-Verified: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as WebArenaRow[]\n  console.warn(`[webarena-verified] WEBARENA_VERIFIED_FIXTURES=1 — loading ${rows.length} adapter fixtures`)\n  return selectRows(rows, opts)\n}\n\nasync function loadOfficialRows(dir: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(join(dir, DATASET_REL), 'utf8')) as WebArenaRow[]\n  return selectRows(rows, opts)\n}\n\nasync function runOfficialEval(root: string, taskId: number, outputDir: string): Promise<Record<string, unknown>> {\n  const script = `\nimport json, os, subprocess, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\ntask_id = sys.argv[2]\noutput_dir = Path(sys.argv[3])\nenv = os.environ.copy()\nenv[\"PYTHONPATH\"] = str(root / \"src\") + os.pathsep + env.get(\"PYTHONPATH\", \"\")\ncmd = [sys.executable, \"-m\", \"webarena_verified\", \"eval-tasks\", \"--task-ids\", task_id, \"--output-dir\", str(output_dir)]\nproc = subprocess.run(cmd, cwd=root, env=env, text=True, capture_output=True)\nif proc.returncode != 0:\n    raise SystemExit((proc.stderr or proc.stdout or f\"exit {proc.returncode}\")[:2000])\nresult_path = output_dir / task_id / \"eval_result.json\"\nif not result_path.exists():\n    raise SystemExit(f\"official evaluator did not write {result_path}\")\nprint(json.dumps(json.loads(result_path.read_text())))\n`\n  const stdout = await runVenvPython(script, [root, String(taskId), outputDir], 0)\n  return JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as Record<string, unknown>\n}\n\nexport function createWebArenaVerifiedAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.WEBARENA_VERIFIED_FIXTURES === '1'\n\n  return {\n    name: 'webarena-verified',\n    output: webarenaOutputDirOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = webarenaDir()\n      if (!dir) {\n        throw new Error(\n          'WEBARENA_VERIFIED_DIR is required. Fix: clone https://github.com/ServiceNow/webarena-verified, install its deps in bench/.venv, and set WEBARENA_VERIFIED_DIR=/path/to/webarena-verified.',\n        )\n      }\n      await assertFile(join(dir, DATASET_REL), 'official dataset')\n      await assertFile(join(dir, 'src', 'webarena_verified', '__main__.py'), 'official CLI module')\n      await runVenvPython(\n        'import sys; sys.path.insert(0, sys.argv[1]); import webarena_verified; print(\"ok\")',\n        [join(dir, 'src')],\n      )\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to load official WebArena-Verified tasks')\n      return loadOfficialRows(dir, opts)\n    },\n\n    async goldArtifact() {\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const dir = webarenaDir()\n      if (!dir) throw new Error('WEBARENA_VERIFIED_DIR is required to judge WebArena-Verified artifacts')\n      const outputDir = resolve(artifact.trim())\n      await assertDir(outputDir, 'run output directory')\n      const meta = readMeta(task)\n      const report = await runOfficialEval(dir, meta.taskId, outputDir)\n      const score = typeof report.score === 'number' ? report.score : 0\n      const status = typeof report.status === 'string' ? report.status : 'unknown'\n      return {\n        resolved: score === 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, status, outputDir }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;AASA,SAAS,QAAQ,UAAU,YAAY;AACvC,SAAS,MAAM,eAAe;AAK9B,IAAM,WAAW,KAAK,WAAW,YAAY,wBAAwB;AACrE,IAAM,cAAc,KAAK,UAAU,WAAW,wBAAwB;AAqBtE,IAAM,cAAc,MAA0B,QAAQ,IAAI;AAEnD,IAAM,0BAAiD;AAAA,EAC5D,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,sCAAsC,CAAC;AACxE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,eAAe,WAAW,MAAc,OAA8B;AACpE,MAAI;AACF,UAAM,OAAO,IAAI;AAAA,EACnB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,8BAA8B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAChH;AACF;AAEA,eAAe,UAAU,MAAc,OAA8B;AACnE,MAAI;AACF,UAAM,IAAI,MAAM,KAAK,IAAI;AACzB,QAAI,CAAC,EAAE,YAAY,EAAG,OAAM,IAAI,MAAM,iBAAiB;AAAA,EACzD,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,8BAA8B,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EAChH;AACF;AAEA,SAAS,UAAU,KAA6B;AAC9C,QAAM,OAAqB;AAAA,IACzB,QAAQ,IAAI;AAAA,IACZ,kBAAkB,IAAI;AAAA,IACtB,OAAO,IAAI,SAAS,CAAC;AAAA,IACrB,WAAW,IAAI,cAAc,CAAC;AAAA,IAC9B,UAAU,IAAI;AAAA,IACd,MAAM,IAAI;AAAA,EACZ;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,OAAO;AAAA,IACtB,QAAQ;AAAA,MACN;AAAA,MACA,SAAS,IAAI,MAAM;AAAA,MACnB,IAAI,YAAY,SAAS,wBAAwB,IAAI,WAAW,KAAK,IAAI,CAAC,KAAK;AAAA,MAC/E,IAAI,OAAO,SAAS,UAAU,IAAI,MAAM,KAAK,IAAI,CAAC,KAAK;AAAA,MACvD;AAAA,MACA;AAAA,MACA;AAAA,IACF,EACG,OAAO,OAAO,EACd,KAAK,IAAI;AAAA,IACZ,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAAU;AACxC,UAAM,IAAI,MAAM,0BAA0B,KAAK,EAAE,wDAAmD;AAAA,EACtG;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAqB,MAAgC;AACvE,MAAI,QAAQ,KAAK,IAAI,SAAS;AAC9B,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC,KAAK,UAAU,IAAI,CAAC,EAAE;AACrG,SAAO;AACT;AAEA,eAAe,aAAa,MAAyC;AACnE,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,mEAA8D,KAAK,MAAM,mBAAmB;AACzG,SAAO,WAAW,MAAM,IAAI;AAC9B;AAEA,eAAe,iBAAiB,KAAa,MAAyC;AACpF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,KAAK,KAAK,WAAW,GAAG,MAAM,CAAC;AACtE,SAAO,WAAW,MAAM,IAAI;AAC9B;AAEA,eAAe,gBAAgB,MAAc,QAAgB,WAAqD;AAChH,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,MAAM,OAAO,MAAM,GAAG,SAAS,GAAG,CAAC;AAC/E,SAAO,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAC5D;AAEO,SAAS,gCAAkD;AAChE,QAAM,eAAe,QAAQ,IAAI,+BAA+B;AAEhE,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,WAAW,KAAK,KAAK,WAAW,GAAG,kBAAkB;AAC3D,YAAM,WAAW,KAAK,KAAK,OAAO,qBAAqB,aAAa,GAAG,qBAAqB;AAC5F,YAAM;AAAA,QACJ;AAAA,QACA,CAAC,KAAK,KAAK,KAAK,CAAC;AAAA,MACnB;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,UAAI,aAAc,QAAO,aAAa,IAAI;AAC1C,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,4EAA4E;AACtG,aAAO,iBAAiB,KAAK,IAAI;AAAA,IACnC;AAAA,IAEA,MAAM,eAAe;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,MAAM,YAAY;AACxB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wEAAwE;AAClG,YAAM,YAAY,QAAQ,SAAS,KAAK,CAAC;AACzC,YAAM,UAAU,WAAW,sBAAsB;AACjD,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK,QAAQ,SAAS;AAChE,YAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;AAChE,YAAM,SAAS,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS;AACnE,aAAO;AAAA,QACL,UAAU,UAAU;AAAA,QACpB;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,QAAQ,UAAU,CAAC;AAAA,MACnE;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/chunk-IA2FBTWC.js

@@ -1,318 +0,0 @@
-import {
-  benchRoot,
-  preflightVenvImports,
-  runVenvScriptStdin,
-  venvPython
-} from "./chunk-WSKWVEQB.js";
-
-// src/benchmarks/appworld.ts
-import { spawn } from "child_process";
-import { join } from "path";
-import { createInterface } from "readline";
-import { routerToolLoop } from "@tangle-network/agent-runtime/loops";
-var DRIVER = join(benchRoot, "scripts", "appworld_driver.py");
-var DEFAULT_SPLIT = "test_normal";
-var appworldSolutionOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-var WORKER_CONTRACT = [
-  "",
-  "Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
-  "Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.",
-  "Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence."
-].join("\n");
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "string") {
-    throw new Error(`appworld task ${task.id} missing metadata.taskId \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function driver(args, input = "") {
-  let stdout;
-  try {
-    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot });
-  } catch (err) {
-    const e = err;
-    throw new Error(`appworld driver failed (${args.join(" ")}): ${(e.message || String(err)).slice(0, 1500)}`);
-  }
-  const last = stdout.trim().split("\n").at(-1) ?? "{}";
-  const parsed = JSON.parse(last);
-  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`);
-  return parsed;
-}
-function createAppWorldAdapter() {
-  return {
-    name: "appworld",
-    output: appworldSolutionOutput,
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["appworld"],
-        requireDocker: false,
-        fix: "Fix: bench/.venv/bin/pip install appworld ; bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data (unpacks the engine + downloads the simulated-app data/tasks). Set APPWORLD_ROOT to the data root if not the default."
-      });
-    },
-    async loadTasks(opts = {}) {
-      const split = opts.split ?? DEFAULT_SPLIT;
-      const out = await driver([
-        "load",
-        "--split",
-        split,
-        ...opts.limit !== void 0 ? ["--limit", String(opts.limit)] : [],
-        ...opts.ids ? ["--ids", opts.ids.join(",")] : []
-      ]);
-      const tasks = out.tasks ?? [];
-      if (tasks.length === 0) {
-        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`);
-      }
-      return tasks.map(
-        (t) => ({
-          id: t.task_id,
-          split,
-          prompt: t.instruction + WORKER_CONTRACT,
-          metadata: { taskId: t.task_id, split }
-        })
-      );
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const out = await driver(["evaluate", "--task-id", meta.taskId, "--split", meta.split], artifact);
-      const passes = out.passes ?? 0;
-      const fails = out.fails ?? 0;
-      const total = out.num_tests ?? passes + fails;
-      const score = total > 0 ? passes / total : 0;
-      const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
-      return {
-        resolved: out.success === true,
-        score,
-        detail: JSON.stringify({
-          taskId: meta.taskId,
-          success: out.success,
-          passes,
-          fails,
-          total,
-          ...failures.length ? { failures } : {}
-        })
-      };
-    }
-  };
-}
-var REACT_HEADER = /^@appworld-react (\S+) (\S+)\n?/;
-var SESSION_SYSTEM = [
-  "You are completing a task in AppWorld, a simulated multi-app environment.",
-  "Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).",
-  "Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
-  "Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).",
-  "Work incrementally: small snippets, read each output, correct course.",
-  "When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again."
-].join("\n");
-var EXECUTE_TOOL = {
-  type: "function",
-  function: {
-    name: "execute_python",
-    description: "Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).",
-    parameters: {
-      type: "object",
-      properties: { code: { type: "string", description: "Python code calling apis.<app>.<fn>(...)" } },
-      required: ["code"]
-    }
-  }
-};
-async function withWorldSession(taskId, split, fn) {
-  const child = spawn(venvPython, [DRIVER, "session", "--task-id", taskId, "--split", split], {
-    cwd: benchRoot
-  });
-  const rl = createInterface({ input: child.stdout });
-  const pending = [];
-  const backlog = [];
-  rl.on("line", (l) => {
-    const next = pending.shift();
-    if (next) next(l);
-    else backlog.push(l);
-  });
-  let stderr = "";
-  child.stderr.on("data", (c) => {
-    stderr += c.toString("utf8");
-  });
-  const nextLine = (timeoutMs) => new Promise((resolve, reject) => {
-    const fromBacklog = backlog.shift();
-    if (fromBacklog !== void 0) return resolve(fromBacklog);
-    const t = setTimeout(
-      () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),
-      timeoutMs
-    );
-    const onExit = (code) => {
-      clearTimeout(t);
-      reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`));
-    };
-    pending.push((l) => {
-      clearTimeout(t);
-      child.removeListener("exit", onExit);
-      resolve(l);
-    });
-    child.once("exit", onExit);
-  });
-  try {
-    const ready = JSON.parse(await nextLine(12e4));
-    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? "no ready line"}`);
-    const call = async (cmd) => {
-      child.stdin.write(`${JSON.stringify(cmd)}
-`);
-      const res = JSON.parse(await nextLine(18e4));
-      if (typeof res.error === "string") throw new Error(`appworld session op failed: ${res.error}`);
-      return res;
-    };
-    return await fn(call, ready.instruction ?? "");
-  } finally {
-    child.stdin.end();
-    child.kill("SIGTERM");
-  }
-}
-function appworldToolLoopClient(cfg) {
-  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40);
-  let seq = 0;
-  return {
-    async create() {
-      const id = `appworld-toolloop-${seq++}`;
-      return {
-        id,
-        async *streamPrompt(prompt) {
-          const m = prompt.match(REACT_HEADER);
-          if (!m) {
-            throw new Error(
-              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header \u2014 got: ${prompt.slice(0, 120)}`
-            );
-          }
-          const [, taskId, split] = m;
-          const directive = prompt.replace(REACT_HEADER, "").trim();
-          const out = await withWorldSession(taskId, split, async (call, instruction) => {
-            const system = directive ? `${SESSION_SYSTEM}
-
-${directive}` : SESSION_SYSTEM;
-            const loop = await routerToolLoop(
-              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },
-              system,
-              `Task: ${instruction}`,
-              [EXECUTE_TOOL],
-              async (name, args) => {
-                if (name !== "execute_python") return `error: unknown tool ${name}`;
-                const res = await call({ op: "execute", code: String(args.code ?? "") });
-                const done = res.task_completed === true;
-                return `${String(res.output ?? "")}${done ? "\n\n[TASK MARKED COMPLETE \u2014 reply with a final summary and do not call the tool again]" : ""}`;
-              },
-              { maxTurns }
-            );
-            const verdict = await call({ op: "evaluate" });
-            const transcript = loop.toolTrace.slice(-3).map((t) => `CODE:
-${t.args.slice(0, 600)}
-OUTPUT:
-${t.result.slice(0, 600)}`).join("\n---\n").slice(0, 1600);
-            return {
-              ...verdict,
-              turns: loop.turns,
-              input_tokens: loop.usage.input,
-              output_tokens: loop.usage.output,
-              transcript
-            };
-          });
-          if (out.input_tokens || out.output_tokens) {
-            yield {
-              type: "llm_call",
-              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model }
-            };
-          }
-          yield { type: "result", data: { finalText: JSON.stringify(out) } };
-        },
-        async delete() {
-        }
-      };
-    }
-  };
-}
-var reactEpisodeOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    return text;
-  }
-};
-function createAppWorldReactAdapter() {
-  const base = createAppWorldAdapter();
-  return {
-    name: "appworld-react",
-    output: reactEpisodeOutput,
-    preflight: () => base.preflight(),
-    async loadTasks(opts = {}) {
-      const tasks = await base.loadTasks(opts);
-      return tasks.map((t) => {
-        const meta = readMeta(t);
-        return {
-          ...t,
-          // Header carries task identity to the leaf; the body (empty at round 0)
-          // is the directive slot the arms append their steer into.
-          prompt: `@appworld-react ${meta.taskId} ${meta.split}
-`
-        };
-      });
-    },
-    goldArtifact: () => Promise.resolve(void 0),
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      let out;
-      try {
-        out = JSON.parse(artifact);
-      } catch {
-        throw new Error(
-          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`
-        );
-      }
-      if (typeof out.success !== "boolean" || typeof out.num_tests !== "number") {
-        throw new Error(
-          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`
-        );
-      }
-      const passes = out.passes ?? 0;
-      const total = out.num_tests;
-      const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
-      return {
-        resolved: out.success === true,
-        score: total > 0 ? passes / total : 0,
-        detail: JSON.stringify({
-          taskId: meta.taskId,
-          success: out.success,
-          passes,
-          fails: out.fails ?? 0,
-          total,
-          turns: out.turns,
-          ...failures.length ? { failures } : {},
-          ...out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}
-        })
-      };
-    },
-    leafClient: (c) => appworldToolLoopClient(c)
-  };
-}
-
-export {
-  appworldSolutionOutput,
-  createAppWorldAdapter,
-  appworldToolLoopClient,
-  createAppWorldReactAdapter
-};
-//# sourceMappingURL=chunk-IA2FBTWC.js.map