@tangle-network/agent-bench 0.1.0 → 0.3.6

package/dist/chunk-SYDW647C.js

@@ -0,0 +1,318 @@
+import {
+  benchRoot,
+  preflightVenvImports,
+  runVenvScriptStdin,
+  venvPython
+} from "./chunk-LRRD7NAG.js";
+
+// src/benchmarks/appworld.ts
+import { spawn } from "child_process";
+import { join } from "path";
+import { createInterface } from "readline";
+import { routerToolLoop } from "@tangle-network/agent-runtime/loops";
+var DRIVER = join(benchRoot, "scripts", "appworld_driver.py");
+var DEFAULT_SPLIT = "test_normal";
+var appworldSolutionOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText ?? d?.text ?? d?.result;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    const fences = [...text.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/g)];
+    return (fences.at(-1)?.[1] ?? text).trim();
+  }
+};
+var WORKER_CONTRACT = [
+  "",
+  "Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
+  "Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.",
+  "Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence."
+].join("\n");
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || typeof md.taskId !== "string") {
+    throw new Error(`appworld task ${task.id} missing metadata.taskId \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+async function driver(args, input = "") {
+  let stdout;
+  try {
+    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot });
+  } catch (err) {
+    const e = err;
+    throw new Error(`appworld driver failed (${args.join(" ")}): ${(e.message || String(err)).slice(0, 1500)}`);
+  }
+  const last = stdout.trim().split("\n").at(-1) ?? "{}";
+  const parsed = JSON.parse(last);
+  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`);
+  return parsed;
+}
+function createAppWorldAdapter() {
+  return {
+    name: "appworld",
+    output: appworldSolutionOutput,
+    async preflight() {
+      await preflightVenvImports({
+        modules: ["appworld"],
+        requireDocker: false,
+        fix: "Fix: bench/.venv/bin/pip install appworld ; bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data (unpacks the engine + downloads the simulated-app data/tasks). Set APPWORLD_ROOT to the data root if not the default."
+      });
+    },
+    async loadTasks(opts = {}) {
+      const split = opts.split ?? DEFAULT_SPLIT;
+      const out = await driver([
+        "load",
+        "--split",
+        split,
+        ...opts.limit !== void 0 ? ["--limit", String(opts.limit)] : [],
+        ...opts.ids ? ["--ids", opts.ids.join(",")] : []
+      ]);
+      const tasks = out.tasks ?? [];
+      if (tasks.length === 0) {
+        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`);
+      }
+      return tasks.map(
+        (t) => ({
+          id: t.task_id,
+          split,
+          prompt: t.instruction + WORKER_CONTRACT,
+          metadata: { taskId: t.task_id, split }
+        })
+      );
+    },
+    async goldArtifact() {
+      return void 0;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const out = await driver(["evaluate", "--task-id", meta.taskId, "--split", meta.split], artifact);
+      const passes = out.passes ?? 0;
+      const fails = out.fails ?? 0;
+      const total = out.num_tests ?? passes + fails;
+      const score = total > 0 ? passes / total : 0;
+      const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
+      return {
+        resolved: out.success === true,
+        score,
+        detail: JSON.stringify({
+          taskId: meta.taskId,
+          success: out.success,
+          passes,
+          fails,
+          total,
+          ...failures.length ? { failures } : {}
+        })
+      };
+    }
+  };
+}
+var REACT_HEADER = /^@appworld-react (\S+) (\S+)\n?/;
+var SESSION_SYSTEM = [
+  "You are completing a task in AppWorld, a simulated multi-app environment.",
+  "Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).",
+  "Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
+  "Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).",
+  "Work incrementally: small snippets, read each output, correct course.",
+  "When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again."
+].join("\n");
+var EXECUTE_TOOL = {
+  type: "function",
+  function: {
+    name: "execute_python",
+    description: "Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).",
+    parameters: {
+      type: "object",
+      properties: { code: { type: "string", description: "Python code calling apis.<app>.<fn>(...)" } },
+      required: ["code"]
+    }
+  }
+};
+async function withWorldSession(taskId, split, fn) {
+  const child = spawn(venvPython, [DRIVER, "session", "--task-id", taskId, "--split", split], {
+    cwd: benchRoot
+  });
+  const rl = createInterface({ input: child.stdout });
+  const pending = [];
+  const backlog = [];
+  rl.on("line", (l) => {
+    const next = pending.shift();
+    if (next) next(l);
+    else backlog.push(l);
+  });
+  let stderr = "";
+  child.stderr.on("data", (c) => {
+    stderr += c.toString("utf8");
+  });
+  const nextLine = (timeoutMs) => new Promise((resolve, reject) => {
+    const fromBacklog = backlog.shift();
+    if (fromBacklog !== void 0) return resolve(fromBacklog);
+    const t = setTimeout(
+      () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),
+      timeoutMs
+    );
+    const onExit = (code) => {
+      clearTimeout(t);
+      reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`));
+    };
+    pending.push((l) => {
+      clearTimeout(t);
+      child.removeListener("exit", onExit);
+      resolve(l);
+    });
+    child.once("exit", onExit);
+  });
+  try {
+    const ready = JSON.parse(await nextLine(12e4));
+    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? "no ready line"}`);
+    const call = async (cmd) => {
+      child.stdin.write(`${JSON.stringify(cmd)}
+`);
+      const res = JSON.parse(await nextLine(18e4));
+      if (typeof res.error === "string") throw new Error(`appworld session op failed: ${res.error}`);
+      return res;
+    };
+    return await fn(call, ready.instruction ?? "");
+  } finally {
+    child.stdin.end();
+    child.kill("SIGTERM");
+  }
+}
+function appworldToolLoopClient(cfg) {
+  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40);
+  let seq = 0;
+  return {
+    async create() {
+      const id = `appworld-toolloop-${seq++}`;
+      return {
+        id,
+        async *streamPrompt(prompt) {
+          const m = prompt.match(REACT_HEADER);
+          if (!m) {
+            throw new Error(
+              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header \u2014 got: ${prompt.slice(0, 120)}`
+            );
+          }
+          const [, taskId, split] = m;
+          const directive = prompt.replace(REACT_HEADER, "").trim();
+          const out = await withWorldSession(taskId, split, async (call, instruction) => {
+            const system = directive ? `${SESSION_SYSTEM}
+
+${directive}` : SESSION_SYSTEM;
+            const loop = await routerToolLoop(
+              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },
+              system,
+              `Task: ${instruction}`,
+              [EXECUTE_TOOL],
+              async (name, args) => {
+                if (name !== "execute_python") return `error: unknown tool ${name}`;
+                const res = await call({ op: "execute", code: String(args.code ?? "") });
+                const done = res.task_completed === true;
+                return `${String(res.output ?? "")}${done ? "\n\n[TASK MARKED COMPLETE \u2014 reply with a final summary and do not call the tool again]" : ""}`;
+              },
+              { maxTurns }
+            );
+            const verdict = await call({ op: "evaluate" });
+            const transcript = loop.toolTrace.slice(-3).map((t) => `CODE:
+${t.args.slice(0, 600)}
+OUTPUT:
+${t.result.slice(0, 600)}`).join("\n---\n").slice(0, 1600);
+            return {
+              ...verdict,
+              turns: loop.turns,
+              input_tokens: loop.usage.input,
+              output_tokens: loop.usage.output,
+              transcript
+            };
+          });
+          if (out.input_tokens || out.output_tokens) {
+            yield {
+              type: "llm_call",
+              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model }
+            };
+          }
+          yield { type: "result", data: { finalText: JSON.stringify(out) } };
+        },
+        async delete() {
+        }
+      };
+    }
+  };
+}
+var reactEpisodeOutput = {
+  parse(events) {
+    let text = "";
+    for (const ev of events) {
+      const d = ev?.data;
+      const t = d?.finalText;
+      if (typeof t === "string" && t.length > 0) text = t;
+    }
+    return text;
+  }
+};
+function createAppWorldReactAdapter() {
+  const base = createAppWorldAdapter();
+  return {
+    name: "appworld-react",
+    output: reactEpisodeOutput,
+    preflight: () => base.preflight(),
+    async loadTasks(opts = {}) {
+      const tasks = await base.loadTasks(opts);
+      return tasks.map((t) => {
+        const meta = readMeta(t);
+        return {
+          ...t,
+          // Header carries task identity to the leaf; the body (empty at round 0)
+          // is the directive slot the arms append their steer into.
+          prompt: `@appworld-react ${meta.taskId} ${meta.split}
+`
+        };
+      });
+    },
+    goldArtifact: () => Promise.resolve(void 0),
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      let out;
+      try {
+        out = JSON.parse(artifact);
+      } catch {
+        throw new Error(
+          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`
+        );
+      }
+      if (typeof out.success !== "boolean" || typeof out.num_tests !== "number") {
+        throw new Error(
+          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`
+        );
+      }
+      const passes = out.passes ?? 0;
+      const total = out.num_tests;
+      const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
+      return {
+        resolved: out.success === true,
+        score: total > 0 ? passes / total : 0,
+        detail: JSON.stringify({
+          taskId: meta.taskId,
+          success: out.success,
+          passes,
+          fails: out.fails ?? 0,
+          total,
+          turns: out.turns,
+          ...failures.length ? { failures } : {},
+          ...out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}
+        })
+      };
+    },
+    leafClient: (c) => appworldToolLoopClient(c)
+  };
+}
+
+export {
+  appworldSolutionOutput,
+  createAppWorldAdapter,
+  appworldToolLoopClient,
+  createAppWorldReactAdapter
+};
+//# sourceMappingURL=chunk-SYDW647C.js.map

package/dist/chunk-SYDW647C.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\ninterface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  transcript?: string\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))\n      }\n      pending.push((l) => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        resolve(l)\n      })\n      child.once('exit', onExit)\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    child.stdin.end()\n    child.kill('SIGTERM')\n  }\n}\n\n/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string) {\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const loop = await routerToolLoop(\n              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },\n              system,\n              `Task: ${instruction}`,\n              [EXECUTE_TOOL],\n              async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                return `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n              },\n              { maxTurns },\n            )\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = loop.toolTrace\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            return {\n              ...verdict,\n              turns: loop.turns,\n              input_tokens: loop.usage.input,\n              output_tokens: loop.usage.output,\n              transcript,\n            } satisfies ReactResult\n          })\n          // Real usage from the episode — flat llm_call so the kernel meters it.\n          if (out.input_tokens || out.output_tokens) {\n            yield {\n              type: 'llm_call',\n              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },\n            }\n          }\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;AAqBA,SAAS,aAAa;AACtB,SAAS,YAAY;AACrB,SAAS,uBAAuB;AAChC,SAA6B,sBAAqC;AAIlE,IAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;AAG9D,IAAM,gBAAgB;AAQf,IAAM,yBAAgD;AAAA,EAC3D,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,sCAAsC,CAAC;AACxE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAAU;AACxC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,+DAA0D;AAAA,EACpG;AACA,SAAO;AACT;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;AAClE,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;AAAA,EAC3E,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EAC5G;AACA,QAAM,OAAO,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK;AACjD,QAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,0BAA0B,OAAO,KAAK,EAAE;AAC1E,SAAO;AACT;AAEO,SAAS,wBAA0C;AACxD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,UAAU;AAAA,QACpB,eAAe;AAAA,QACf,KACE;AAAA,MAIJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;AAC5D,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,MAAO,MAAM,OAAO;AAAA,QACxB;AAAA,QACA;AAAA,QAAW;AAAA,QACX,GAAI,KAAK,UAAU,SAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;AAAA,QAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;AAAA,MAClD,CAAC;AACD,YAAM,QAAQ,IAAI,SAAS,CAAC;AAC5B,UAAI,MAAM,WAAW,GAAG;AACtB,cAAM,IAAI,MAAM,kDAAkD,KAAK,IAAI,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,MACnG;AACA,aAAO,MAAM;AAAA,QACX,CAAC,OAAkB;AAAA,UACjB,IAAI,EAAE;AAAA,UACN;AAAA,UACA,QAAQ,EAAE,cAAc;AAAA,UACxB,UAAU,EAAE,QAAQ,EAAE,SAAS,MAAM;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,eAAe;AAOnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAO,MAAM,OAAO,CAAC,YAAY,aAAa,KAAK,QAAQ,WAAW,KAAK,KAAK,GAAG,QAAQ;AAOjG,YAAM,SAAS,IAAI,UAAU;AAC7B,YAAM,QAAQ,IAAI,SAAS;AAI3B,YAAM,QAAQ,IAAI,aAAa,SAAS;AACxC,YAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;AAG3C,YAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;AACzE,aAAO;AAAA,QACL,UAAU,IAAI,YAAY;AAAA,QAC1B;AAAA,QACA,QAAQ,KAAK,UAAU;AAAA,UACrB,QAAQ,KAAK;AAAA,UACb,SAAS,IAAI;AAAA,UACb;AAAA,UACA;AAAA,UACA;AAAA,UACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;AAmCA,IAAM,eAAe;AAErB,IAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,IAAM,eAAyB;AAAA,EAC7B,MAAM;AAAA,EACN,UAAU;AAAA,IACR,MAAM;AAAA,IACN,aACE;AAAA,IACF,YAAY;AAAA,MACV,MAAM;AAAA,MACN,YAAY,EAAE,MAAM,EAAE,MAAM,UAAU,aAAa,2CAA2C,EAAE;AAAA,MAChG,UAAU,CAAC,MAAM;AAAA,IACnB;AAAA,EACF;AACF;AAGA,eAAe,iBACb,QACA,OACA,IACY;AACZ,QAAM,QAAQ,MAAM,YAAY,CAAC,QAAQ,WAAW,aAAa,QAAQ,WAAW,KAAK,GAAG;AAAA,IAC1F,KAAK;AAAA,EACP,CAAC;AACD,QAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;AAClD,QAAM,UAAyC,CAAC;AAChD,QAAM,UAAoB,CAAC;AAC3B,KAAG,GAAG,QAAQ,CAAC,MAAM;AACnB,UAAM,OAAO,QAAQ,MAAM;AAC3B,QAAI,KAAM,MAAK,CAAC;AAAA,QACX,SAAQ,KAAK,CAAC;AAAA,EACrB,CAAC;AACD,MAAI,SAAS;AACb,QAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,cAAU,EAAE,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,QAAM,WAAW,CAAC,cAChB,IAAI,QAAQ,CAAC,SAAS,WAAW;AAC/B,UAAM,cAAc,QAAQ,MAAM;AAClC,QAAI,gBAAgB,OAAW,QAAO,QAAQ,WAAW;AACzD,UAAM,IAAI;AAAA,MACR,MAAM,OAAO,IAAI,MAAM,oCAAoC,SAAS,eAAe,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,MACxG;AAAA,IACF;AAGA,UAAM,SAAS,CAAC,SAA8B;AAC5C,mBAAa,CAAC;AACd,aAAO,IAAI,MAAM,4BAA4B,IAAI,cAAc,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,IACtF;AACA,YAAQ,KAAK,CAAC,MAAM;AAClB,mBAAa,CAAC;AACd,YAAM,eAAe,QAAQ,MAAM;AACnC,cAAQ,CAAC;AAAA,IACX,CAAC;AACD,UAAM,KAAK,QAAQ,MAAM;AAAA,EAC3B,CAAC;AACH,MAAI;AACF,UAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;AAChD,QAAI,CAAC,MAAM,MAAO,OAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,eAAe,EAAE;AACvG,UAAM,OAAO,OAAO,QAAmE;AACrF,YAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,CAAC;AAAA,CAAI;AAC5C,YAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;AAC9C,UAAI,OAAO,IAAI,UAAU,SAAU,OAAM,IAAI,MAAM,+BAA+B,IAAI,KAAK,EAAE;AAC7F,aAAO;AAAA,IACT;AACA,WAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;AAAA,EAC/C,UAAE;AACA,UAAM,MAAM,IAAI;AAChB,UAAM,KAAK,SAAS;AAAA,EACtB;AACF;AAGO,SAAS,uBAAuB,KAK3B;AACV,QAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;AACzE,MAAI,MAAM;AACV,SAAO;AAAA,IACL,MAAM,SAAS;AACb,YAAM,KAAK,qBAAqB,KAAK;AACrC,aAAO;AAAA,QACL;AAAA,QACA,OAAO,aAAa,QAAgB;AAClC,gBAAM,IAAI,OAAO,MAAM,YAAY;AACnC,cAAI,CAAC,GAAG;AACN,kBAAM,IAAI;AAAA,cACR,6FAAwF,OAAO,MAAM,GAAG,GAAG,CAAC;AAAA,YAC9G;AAAA,UACF;AACA,gBAAM,CAAC,EAAE,QAAQ,KAAK,IAAI;AAC1B,gBAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,EAAE,KAAK;AACxD,gBAAM,MAAM,MAAM,iBAAiB,QAAkB,OAAiB,OAAO,MAAM,gBAAgB;AACjG,kBAAM,SAAS,YAAY,GAAG,cAAc;AAAA;AAAA,EAAO,SAAS,KAAK;AACjE,kBAAM,OAAO,MAAM;AAAA,cACjB,EAAE,eAAe,IAAI,eAAe,WAAW,IAAI,WAAW,OAAO,IAAI,MAAM;AAAA,cAC/E;AAAA,cACA,SAAS,WAAW;AAAA,cACpB,CAAC,YAAY;AAAA,cACb,OAAO,MAAM,SAAS;AACpB,oBAAI,SAAS,iBAAkB,QAAO,uBAAuB,IAAI;AACjE,sBAAM,MAAM,MAAM,KAAK,EAAE,IAAI,WAAW,MAAM,OAAO,KAAK,QAAQ,EAAE,EAAE,CAAC;AACvE,sBAAM,OAAO,IAAI,mBAAmB;AACpC,uBAAO,GAAG,OAAO,IAAI,UAAU,EAAE,CAAC,GAAG,OAAO,gGAA2F,EAAE;AAAA,cAC3I;AAAA,cACA,EAAE,SAAS;AAAA,YACb;AACA,kBAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;AAC9C,kBAAM,aAAa,KAAK,UACrB,MAAM,EAAE,EACR,IAAI,CAAC,MAAM;AAAA,EAAU,EAAE,KAAK,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA,EAAc,EAAE,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,EAC/E,KAAK,SAAS,EACd,MAAM,GAAG,IAAI;AAChB,mBAAO;AAAA,cACL,GAAG;AAAA,cACH,OAAO,KAAK;AAAA,cACZ,cAAc,KAAK,MAAM;AAAA,cACzB,eAAe,KAAK,MAAM;AAAA,cAC1B;AAAA,YACF;AAAA,UACF,CAAC;AAED,cAAI,IAAI,gBAAgB,IAAI,eAAe;AACzC,kBAAM;AAAA,cACJ,MAAM;AAAA,cACN,MAAM,EAAE,UAAU,IAAI,gBAAgB,GAAG,WAAW,IAAI,iBAAiB,GAAG,OAAO,IAAI,MAAM;AAAA,YAC/F;AAAA,UACF;AACA,gBAAM,EAAE,MAAM,UAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE,EAAE;AAAA,QACnE;AAAA,QACA,MAAM,SAAS;AAAA,QAAC;AAAA,MAClB;AAAA,IACF;AAAA,EACF;AACF;AAGA,IAAM,qBAA4C;AAAA,EAChD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG;AACb,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AACF;AAEO,SAAS,6BAA+C;AAC7D,QAAM,OAAO,sBAAsB;AACnC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,WAAW,MAAM,KAAK,UAAU;AAAA,IAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;AAC5D,YAAM,QAAQ,MAAM,KAAK,UAAU,IAAI;AACvC,aAAO,MAAM,IAAI,CAAC,MAAM;AACtB,cAAM,OAAO,SAAS,CAAC;AACvB,eAAO;AAAA,UACL,GAAG;AAAA;AAAA;AAAA,UAGH,QAAQ,mBAAmB,KAAK,MAAM,IAAI,KAAK,KAAK;AAAA;AAAA,QACtD;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,cAAc,MAAM,QAAQ,QAAQ,MAAS;AAAA,IAE7C,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI;AACJ,UAAI;AACF,cAAM,KAAK,MAAM,QAAQ;AAAA,MAC3B,QAAQ;AACN,cAAM,IAAI;AAAA,UACR,6EAA6E,KAAK,MAAM,MAAM,SAAS,MAAM,GAAG,GAAG,CAAC;AAAA,QACtH;AAAA,MACF;AACA,UAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAAU;AACzE,cAAM,IAAI;AAAA,UACR,sEAAsE,KAAK,MAAM,MAAM,SAAS,MAAM,GAAG,GAAG,CAAC;AAAA,QAC/G;AAAA,MACF;AACA,YAAM,SAAS,IAAI,UAAU;AAC7B,YAAM,QAAQ,IAAI;AAClB,YAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;AACzE,aAAO;AAAA,QACL,UAAU,IAAI,YAAY;AAAA,QAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;AAAA,QACpC,QAAQ,KAAK,UAAU;AAAA,UACrB,QAAQ,KAAK;AAAA,UACb,SAAS,IAAI;AAAA,UACb;AAAA,UACA,OAAO,IAAI,SAAS;AAAA,UACpB;AAAA,UACA,OAAO,IAAI;AAAA,UACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;AAAA,UACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;AAAA,QACzE,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IAEA,YAAY,CAAC,MAAM,uBAAuB,CAAC;AAAA,EAC7C;AACF;","names":[]}

package/dist/chunk-TBKU5XQI.js

@@ -0,0 +1,228 @@
+// src/benchmarks/mind2web.ts
+import { execFile } from "child_process";
+import { mkdir } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { fileURLToPath } from "url";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+var BENCH_ROOT = fileURLToPath(new URL("../..", import.meta.url));
+var PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, ".venv", "bin", "python");
+var DATASET = "osunlp/Multimodal-Mind2Web";
+var DEFAULT_SPLIT = process.env.M2W_SPLIT ?? "test_domain";
+var SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), "m2w-shots");
+var CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30);
+var WORKER_CONTRACT = [
+  "",
+  "Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:",
+  "ELEMENT: <the [id] number of the chosen element>",
+  "ACTION: <CLICK | TYPE | SELECT>",
+  "VALUE: <text to type or option to select; leave empty for CLICK>"
+].join("\n");
+async function py(script, args = []) {
+  const { stdout } = await execFileAsync(PY, ["-c", script, ...args], { maxBuffer: 1024 * 1024 * 256 });
+  return stdout;
+}
+var LOADER = `
+import json, sys, os
+from datasets import load_dataset
+cfg = json.loads(sys.argv[1])
+split = cfg["split"]; limit = cfg.get("limit"); cap = cfg.get("cap", 30)
+shots = cfg["shotsDir"]; ids = set(cfg["ids"]) if cfg.get("ids") else None
+os.makedirs(shots, exist_ok=True)
+KEEP = ("aria_label","aria-label","role","type","name","placeholder","title","alt","value","text","id","class","href")
+def label(tag, attr):
+    try:
+        a = json.loads(attr) if isinstance(attr, str) else (attr or {})
+    except Exception:
+        a = {}
+    parts = []
+    for k in KEEP:
+        v = a.get(k)
+        if v:
+            sv = str(v).replace("\\n", " ").strip()
+            if k == "class": sv = sv[:40]
+            if k == "href": sv = sv[:50]
+            if sv: parts.append(k + "=" + sv[:60])
+    return "<" + str(tag) + "> " + " ".join(parts[:6])
+def cands(raw):
+    out = []
+    for c in raw or []:
+        try:
+            d = json.loads(c) if isinstance(c, str) else c
+        except Exception:
+            continue
+        bid = str(d.get("backend_node_id", ""))
+        if not bid: continue
+        out.append({"id": bid, "label": label(d.get("tag", "?"), d.get("attributes"))})
+    return out
+ds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)
+emitted = []; skipped = 0; scanned = 0
+for r in ds:
+    scanned += 1
+    if ids is not None and r["action_uid"] not in ids:
+        if scanned > 8000: break
+        continue
+    pos = cands(r.get("pos_candidates"))
+    if not pos:
+        skipped += 1
+        continue
+    neg = cands(r.get("neg_candidates"))
+    seen = set(p["id"] for p in pos); merged = list(pos)
+    for n in neg:
+        if len(merged) >= cap: break
+        if n["id"] in seen: continue
+        seen.add(n["id"]); merged.append(n)
+    merged.sort(key=lambda x: int(x["id"]) if x["id"].isdigit() else 0)
+    try:
+        op = json.loads(r["operation"]) if isinstance(r["operation"], str) else r["operation"]
+    except Exception:
+        op = {}
+    sp = os.path.join(shots, str(r["action_uid"]) + ".jpg")
+    try:
+        r["screenshot"].convert("RGB").save(sp, "JPEG", quality=70)
+    except Exception:
+        sp = ""
+    emitted.append({
+        "id": r["action_uid"], "task": r.get("confirmed_task", ""),
+        "website": r.get("website", ""), "domain": r.get("domain", ""), "subdomain": r.get("subdomain", ""),
+        "op": str(op.get("op", "")).upper(), "value": str(op.get("value", "")),
+        "goldIds": [p["id"] for p in pos], "candidates": merged,
+        "screenshotPath": sp, "targetRepr": r.get("target_action_reprs", ""),
+    })
+    if ids is None and limit is not None and len(emitted) >= limit: break
+    if ids is not None and len(emitted) >= len(ids): break
+sys.stderr.write("[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\n" % (len(emitted), skipped, scanned)); sys.stderr.flush()
+print(json.dumps(emitted)); sys.stdout.flush()
+os._exit(0)
+`;
+function buildPrompt(row) {
+  const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join("\n");
+  return [
+    `Web task: ${row.task}`,
+    "",
+    "You are taking the NEXT single action on the current web page. Choose the one element",
+    'to act on from the candidates below (each line is "[id] <tag> attributes"):',
+    choices,
+    WORKER_CONTRACT
+  ].join("\n");
+}
+function rowToTask(row) {
+  const meta = {
+    task: row.task,
+    website: row.website,
+    domain: row.domain,
+    op: row.op,
+    value: row.value,
+    goldIds: row.goldIds,
+    candidateIds: row.candidates.map((c) => c.id),
+    screenshotPath: row.screenshotPath,
+    targetRepr: row.targetRepr
+  };
+  return {
+    id: `mind2web-${row.id}`,
+    split: DEFAULT_SPLIT,
+    prompt: buildPrompt(row),
+    metadata: meta
+  };
+}
+function readMeta(task) {
+  const md = task.metadata;
+  if (!md || !Array.isArray(md.goldIds)) {
+    throw new Error(`mind2web task ${task.id} missing metadata.goldIds \u2014 loadTasks did not populate it`);
+  }
+  return md;
+}
+function normValue(s) {
+  return s.toLowerCase().replace(/\s+/g, " ").trim();
+}
+function parseAction(artifact) {
+  const elem = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact);
+  const op = /ACTION:\s*(CLICK|TYPE|SELECT)/i.exec(artifact);
+  if (!elem?.[1] || !op?.[1]) return null;
+  const val = /VALUE:\s*(.*)/i.exec(artifact);
+  return { elementId: elem[1], op: op[1].toUpperCase(), value: (val?.[1] ?? "").trim() };
+}
+function createMind2WebAdapter() {
+  const split = DEFAULT_SPLIT;
+  return {
+    name: "mind2web",
+    async preflight() {
+      try {
+        await py(
+          `import os, sys
+from datasets import load_dataset
+import PIL
+ds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)
+next(iter(ds))
+print('ok'); sys.stdout.flush(); os._exit(0)`
+        );
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `mind2web preflight failed: ${msg}
+Fix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; (2) network access to Hugging Face for ${DATASET} (split ${split}) ; point M2W_PYTHON at the python if not bench/.venv.`
+        );
+      }
+    },
+    async loadTasks(opts = {}) {
+      const cfg = {
+        split: opts.split ?? split,
+        limit: opts.ids ? void 0 : opts.limit ?? 10,
+        cap: CANDIDATE_CAP,
+        shotsDir: SHOTS_DIR,
+        ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, "")) : void 0
+      };
+      await mkdir(SHOTS_DIR, { recursive: true });
+      const stdout = await py(LOADER, [JSON.stringify(cfg)]);
+      const rows = JSON.parse(stdout);
+      return rows.map(rowToTask);
+    },
+    async goldArtifact(task) {
+      const meta = readMeta(task);
+      const id = meta.goldIds[0];
+      if (!id) return void 0;
+      return `ELEMENT: ${id}
+ACTION: ${meta.op}
+VALUE: ${meta.value}`;
+    },
+    async judge(task, artifact) {
+      const meta = readMeta(task);
+      const parsed = parseAction(artifact);
+      if (!parsed) {
+        return {
+          resolved: false,
+          score: 0,
+          detail: JSON.stringify({ reason: "no parseable ELEMENT/ACTION", goldOp: meta.op, goldIds: meta.goldIds })
+        };
+      }
+      const elementCorrect = meta.goldIds.includes(parsed.elementId);
+      const opMatch = parsed.op === meta.op;
+      const valueMatch = meta.op === "CLICK" ? true : normValue(parsed.value) === normValue(meta.value);
+      const operationCorrect = opMatch && valueMatch;
+      const resolved = elementCorrect && operationCorrect;
+      const score = 0.6 * (elementCorrect ? 1 : 0) + 0.4 * (operationCorrect ? 1 : 0);
+      return {
+        resolved,
+        score,
+        detail: JSON.stringify({
+          elementCorrect,
+          opMatch,
+          valueMatch,
+          predicted: parsed,
+          goldOp: meta.op,
+          goldValue: meta.value,
+          goldIds: meta.goldIds,
+          website: meta.website,
+          domain: meta.domain
+        })
+      };
+    }
+  };
+}
+
+export {
+  parseAction,
+  createMind2WebAdapter
+};
+//# sourceMappingURL=chunk-TBKU5XQI.js.map

package/dist/chunk-TBKU5XQI.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/benchmarks/mind2web.ts"],"sourcesContent":["/**\n * Mind2Web adapter (osunlp/Multimodal-Mind2Web, split 'test_domain' by default).\n * Web-agent ACTION PREDICTION: each task is one step of a real web task — given\n * the natural-language goal and the page's candidate elements, the worker picks\n * the SINGLE element to act on and the action (CLICK / TYPE / SELECT [+ value]).\n *\n * Judge is the published Mind2Web step metric, FULLY DETERMINISTIC — no LLM:\n *   element correct  ⇔ predicted backend_node_id ∈ pos_candidates ids\n *   operation correct ⇔ op-type matches AND (TYPE/SELECT) the value matches\n *   resolved (Step-SR) ⇔ element correct AND operation correct\n *   score = 0.6·element + 0.4·operation  (a gradient for the optimizer; the right\n *           element is most of the credit, mirroring Element-Acc ≫ Op as the lever)\n * This is the low-noise reward a certifiable directive-lift needs: the number is a\n * programmatic match against human-verified ground truth, not a judge's opinion.\n *\n * The worker artifact is three sentinel lines (ELEMENT / ACTION / VALUE) so the\n * judge extracts deterministically; the prompt presents the candidate set as a\n * choice over backend_node_id ordered by id (position uncorrelated with the answer).\n *\n * Each step carries the dataset's OWN page screenshot (written to a temp file by\n * the loader); the worker drops it into a browser.<op> span so run-capsule's screen\n * capsule turns the run into a film — the real page, not a re-rendered DOM.\n *\n * Requires for a live run: a python with `datasets` + `pillow` and network to\n * Hugging Face. Point M2W_PYTHON at it (defaults to bench/.venv/bin/python).\n */\n\nimport { execFile } from 'node:child_process'\nimport { mkdir } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join } from 'node:path'\nimport { fileURLToPath } from 'node:url'\nimport { promisify } from 'node:util'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst execFileAsync = promisify(execFile)\nconst BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url))\nconst PY = process.env.M2W_PYTHON ?? join(BENCH_ROOT, '.venv', 'bin', 'python')\n\nconst DATASET = 'osunlp/Multimodal-Mind2Web'\nconst DEFAULT_SPLIT = process.env.M2W_SPLIT ?? 'test_domain'\nconst SHOTS_DIR = process.env.M2W_SHOTS ?? join(tmpdir(), 'm2w-shots')\n/** Candidate-set cap presented to the worker (always keeps all pos candidates). */\nconst CANDIDATE_CAP = Number(process.env.M2W_CANDIDATE_CAP ?? 30)\n\n/** The worker contract appended to every task prompt; the judge keys off these. */\nconst WORKER_CONTRACT = [\n  '',\n  'Pick the SINGLE next element to act on, then end your response with EXACTLY these three lines:',\n  'ELEMENT: <the [id] number of the chosen element>',\n  'ACTION: <CLICK | TYPE | SELECT>',\n  'VALUE: <text to type or option to select; leave empty for CLICK>',\n].join('\\n')\n\ninterface RawCandidate {\n  id: string\n  label: string\n}\ninterface Mind2WebRow {\n  id: string\n  task: string\n  website: string\n  domain: string\n  subdomain: string\n  op: string\n  value: string\n  goldIds: string[]\n  candidates: RawCandidate[]\n  screenshotPath: string\n  targetRepr: string\n}\ninterface Mind2WebMeta {\n  task: string\n  website: string\n  domain: string\n  op: string\n  value: string\n  goldIds: string[]\n  candidateIds: string[]\n  screenshotPath: string\n  targetRepr: string\n}\n\n/** Run the bench python with a script on stdin; return stdout (throws on nonzero). */\nasync function py(script: string, args: string[] = []): Promise<string> {\n  const { stdout } = await execFileAsync(PY, ['-c', script, ...args], { maxBuffer: 1024 * 1024 * 256 })\n  return stdout\n}\n\n/** The loader: stream the split, skip steps with no positive candidate (fail-loud,\n *  never a silent score-0), cap the candidate set, save each screenshot to a file. */\nconst LOADER = `\nimport json, sys, os\nfrom datasets import load_dataset\ncfg = json.loads(sys.argv[1])\nsplit = cfg[\"split\"]; limit = cfg.get(\"limit\"); cap = cfg.get(\"cap\", 30)\nshots = cfg[\"shotsDir\"]; ids = set(cfg[\"ids\"]) if cfg.get(\"ids\") else None\nos.makedirs(shots, exist_ok=True)\nKEEP = (\"aria_label\",\"aria-label\",\"role\",\"type\",\"name\",\"placeholder\",\"title\",\"alt\",\"value\",\"text\",\"id\",\"class\",\"href\")\ndef label(tag, attr):\n    try:\n        a = json.loads(attr) if isinstance(attr, str) else (attr or {})\n    except Exception:\n        a = {}\n    parts = []\n    for k in KEEP:\n        v = a.get(k)\n        if v:\n            sv = str(v).replace(\"\\\\n\", \" \").strip()\n            if k == \"class\": sv = sv[:40]\n            if k == \"href\": sv = sv[:50]\n            if sv: parts.append(k + \"=\" + sv[:60])\n    return \"<\" + str(tag) + \"> \" + \" \".join(parts[:6])\ndef cands(raw):\n    out = []\n    for c in raw or []:\n        try:\n            d = json.loads(c) if isinstance(c, str) else c\n        except Exception:\n            continue\n        bid = str(d.get(\"backend_node_id\", \"\"))\n        if not bid: continue\n        out.append({\"id\": bid, \"label\": label(d.get(\"tag\", \"?\"), d.get(\"attributes\"))})\n    return out\nds = load_dataset(${JSON.stringify(DATASET)}, split=split, streaming=True)\nemitted = []; skipped = 0; scanned = 0\nfor r in ds:\n    scanned += 1\n    if ids is not None and r[\"action_uid\"] not in ids:\n        if scanned > 8000: break\n        continue\n    pos = cands(r.get(\"pos_candidates\"))\n    if not pos:\n        skipped += 1\n        continue\n    neg = cands(r.get(\"neg_candidates\"))\n    seen = set(p[\"id\"] for p in pos); merged = list(pos)\n    for n in neg:\n        if len(merged) >= cap: break\n        if n[\"id\"] in seen: continue\n        seen.add(n[\"id\"]); merged.append(n)\n    merged.sort(key=lambda x: int(x[\"id\"]) if x[\"id\"].isdigit() else 0)\n    try:\n        op = json.loads(r[\"operation\"]) if isinstance(r[\"operation\"], str) else r[\"operation\"]\n    except Exception:\n        op = {}\n    sp = os.path.join(shots, str(r[\"action_uid\"]) + \".jpg\")\n    try:\n        r[\"screenshot\"].convert(\"RGB\").save(sp, \"JPEG\", quality=70)\n    except Exception:\n        sp = \"\"\n    emitted.append({\n        \"id\": r[\"action_uid\"], \"task\": r.get(\"confirmed_task\", \"\"),\n        \"website\": r.get(\"website\", \"\"), \"domain\": r.get(\"domain\", \"\"), \"subdomain\": r.get(\"subdomain\", \"\"),\n        \"op\": str(op.get(\"op\", \"\")).upper(), \"value\": str(op.get(\"value\", \"\")),\n        \"goldIds\": [p[\"id\"] for p in pos], \"candidates\": merged,\n        \"screenshotPath\": sp, \"targetRepr\": r.get(\"target_action_reprs\", \"\"),\n    })\n    if ids is None and limit is not None and len(emitted) >= limit: break\n    if ids is not None and len(emitted) >= len(ids): break\nsys.stderr.write(\"[mind2web] emitted=%d skipped_empty_pos=%d scanned=%d\\\\n\" % (len(emitted), skipped, scanned)); sys.stderr.flush()\nprint(json.dumps(emitted)); sys.stdout.flush()\nos._exit(0)\n`\n\nfunction buildPrompt(row: Mind2WebRow): string {\n  const choices = row.candidates.map((c) => `  [${c.id}] ${c.label}`).join('\\n')\n  return [\n    `Web task: ${row.task}`,\n    '',\n    'You are taking the NEXT single action on the current web page. Choose the one element',\n    'to act on from the candidates below (each line is \"[id] <tag> attributes\"):',\n    choices,\n    WORKER_CONTRACT,\n  ].join('\\n')\n}\n\nfunction rowToTask(row: Mind2WebRow): BenchTask {\n  const meta: Mind2WebMeta = {\n    task: row.task,\n    website: row.website,\n    domain: row.domain,\n    op: row.op,\n    value: row.value,\n    goldIds: row.goldIds,\n    candidateIds: row.candidates.map((c) => c.id),\n    screenshotPath: row.screenshotPath,\n    targetRepr: row.targetRepr,\n  }\n  return {\n    id: `mind2web-${row.id}`,\n    split: DEFAULT_SPLIT,\n    prompt: buildPrompt(row),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): Mind2WebMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray((md as { goldIds?: unknown }).goldIds)) {\n    throw new Error(`mind2web task ${task.id} missing metadata.goldIds — loadTasks did not populate it`)\n  }\n  return md as unknown as Mind2WebMeta\n}\n\n/** Normalize a TYPE/SELECT value for comparison: lowercase, collapse whitespace. */\nfunction normValue(s: string): string {\n  return s.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\ninterface ParsedAction {\n  elementId: string\n  op: string\n  value: string\n}\n\n/** Parse the worker artifact's ELEMENT / ACTION / VALUE sentinel lines.\n *  Fail-closed: a missing ELEMENT or ACTION returns null (judge → resolved=false). */\nexport function parseAction(artifact: string): ParsedAction | null {\n  const elem = /ELEMENT:\\s*\\[?(\\d+)\\]?/i.exec(artifact)\n  const op = /ACTION:\\s*(CLICK|TYPE|SELECT)/i.exec(artifact)\n  if (!elem?.[1] || !op?.[1]) return null\n  const val = /VALUE:\\s*(.*)/i.exec(artifact)\n  return { elementId: elem[1], op: op[1].toUpperCase(), value: (val?.[1] ?? '').trim() }\n}\n\nexport function createMind2WebAdapter(): BenchmarkAdapter {\n  const split = DEFAULT_SPLIT\n\n  return {\n    name: 'mind2web',\n\n    async preflight() {\n      try {\n        await py(\n          `import os, sys\nfrom datasets import load_dataset\nimport PIL\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)}, streaming=True)\nnext(iter(ds))\nprint('ok'); sys.stdout.flush(); os._exit(0)`,\n        )\n      } catch (err) {\n        const msg = err instanceof Error ? err.message : String(err)\n        throw new Error(\n          `mind2web preflight failed: ${msg}\\n` +\n            `Fix: (1) a python with datasets + pillow (python3 -m venv bench/.venv && bench/.venv/bin/pip install datasets pillow) ; ` +\n            `(2) network access to Hugging Face for ${DATASET} (split ${split}) ; ` +\n            `point M2W_PYTHON at the python if not bench/.venv.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const cfg = {\n        split: opts.split ?? split,\n        limit: opts.ids ? undefined : (opts.limit ?? 10),\n        cap: CANDIDATE_CAP,\n        shotsDir: SHOTS_DIR,\n        ids: opts.ids ? opts.ids.map((id) => id.replace(/^mind2web-/, '')) : undefined,\n      }\n      await mkdir(SHOTS_DIR, { recursive: true })\n      const stdout = await py(LOADER, [JSON.stringify(cfg)])\n      const rows = JSON.parse(stdout) as Mind2WebRow[]\n      return rows.map(rowToTask)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the worker-contract serialization of the ground-truth action, so\n      // verify-judge proves gold→resolved through the SAME parse path the real\n      // artifact takes. The first accepted positive id is the canonical target.\n      const meta = readMeta(task)\n      const id = meta.goldIds[0]\n      if (!id) return undefined\n      return `ELEMENT: ${id}\\nACTION: ${meta.op}\\nVALUE: ${meta.value}`\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const parsed = parseAction(artifact)\n      if (!parsed) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ reason: 'no parseable ELEMENT/ACTION', goldOp: meta.op, goldIds: meta.goldIds }),\n        }\n      }\n      const elementCorrect = meta.goldIds.includes(parsed.elementId)\n      const opMatch = parsed.op === meta.op\n      const valueMatch = meta.op === 'CLICK' ? true : normValue(parsed.value) === normValue(meta.value)\n      const operationCorrect = opMatch && valueMatch\n      const resolved = elementCorrect && operationCorrect\n      const score = 0.6 * (elementCorrect ? 1 : 0) + 0.4 * (operationCorrect ? 1 : 0)\n      return {\n        resolved,\n        score,\n        detail: JSON.stringify({\n          elementCorrect,\n          opMatch,\n          valueMatch,\n          predicted: parsed,\n          goldOp: meta.op,\n          goldValue: meta.value,\n          goldIds: meta.goldIds,\n          website: meta.website,\n          domain: meta.domain,\n        }),\n      }\n    },\n  }\n}\n"],"mappings":";AA2BA,SAAS,gBAAgB;AACzB,SAAS,aAAa;AACtB,SAAS,cAAc;AACvB,SAAS,YAAY;AACrB,SAAS,qBAAqB;AAC9B,SAAS,iBAAiB;AAG1B,IAAM,gBAAgB,UAAU,QAAQ;AACxC,IAAM,aAAa,cAAc,IAAI,IAAI,SAAS,YAAY,GAAG,CAAC;AAClE,IAAM,KAAK,QAAQ,IAAI,cAAc,KAAK,YAAY,SAAS,OAAO,QAAQ;AAE9E,IAAM,UAAU;AAChB,IAAM,gBAAgB,QAAQ,IAAI,aAAa;AAC/C,IAAM,YAAY,QAAQ,IAAI,aAAa,KAAK,OAAO,GAAG,WAAW;AAErE,IAAM,gBAAgB,OAAO,QAAQ,IAAI,qBAAqB,EAAE;AAGhE,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAgCX,eAAe,GAAG,QAAgB,OAAiB,CAAC,GAAoB;AACtE,QAAM,EAAE,OAAO,IAAI,MAAM,cAAc,IAAI,CAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAE,WAAW,OAAO,OAAO,IAAI,CAAC;AACpG,SAAO;AACT;AAIA,IAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oBAiCK,KAAK,UAAU,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAyC3C,SAAS,YAAY,KAA0B;AAC7C,QAAM,UAAU,IAAI,WAAW,IAAI,CAAC,MAAM,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,KAAK,IAAI;AAC7E,SAAO;AAAA,IACL,aAAa,IAAI,IAAI;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,UAAU,KAA6B;AAC9C,QAAM,OAAqB;AAAA,IACzB,MAAM,IAAI;AAAA,IACV,SAAS,IAAI;AAAA,IACb,QAAQ,IAAI;AAAA,IACZ,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,SAAS,IAAI;AAAA,IACb,cAAc,IAAI,WAAW,IAAI,CAAC,MAAM,EAAE,EAAE;AAAA,IAC5C,gBAAgB,IAAI;AAAA,IACpB,YAAY,IAAI;AAAA,EAClB;AACA,SAAO;AAAA,IACL,IAAI,YAAY,IAAI,EAAE;AAAA,IACtB,OAAO;AAAA,IACP,QAAQ,YAAY,GAAG;AAAA,IACvB,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,CAAC,MAAM,QAAS,GAA6B,OAAO,GAAG;AAChE,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,gEAA2D;AAAA,EACrG;AACA,SAAO;AACT;AAGA,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,YAAY,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACnD;AAUO,SAAS,YAAY,UAAuC;AACjE,QAAM,OAAO,0BAA0B,KAAK,QAAQ;AACpD,QAAM,KAAK,iCAAiC,KAAK,QAAQ;AACzD,MAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAG,QAAO;AACnC,QAAM,MAAM,iBAAiB,KAAK,QAAQ;AAC1C,SAAO,EAAE,WAAW,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,YAAY,GAAG,QAAQ,MAAM,CAAC,KAAK,IAAI,KAAK,EAAE;AACvF;AAEO,SAAS,wBAA0C;AACxD,QAAM,QAAQ;AAEd,SAAO;AAAA,IACL,MAAM;AAAA,IAEN,MAAM,YAAY;AAChB,UAAI;AACF,cAAM;AAAA,UACJ;AAAA;AAAA;AAAA,oBAGU,KAAK,UAAU,OAAO,CAAC,WAAW,KAAK,UAAU,KAAK,CAAC;AAAA;AAAA;AAAA,QAGnE;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,cAAM,IAAI;AAAA,UACR,8BAA8B,GAAG;AAAA,iKAEW,OAAO,WAAW,KAAK;AAAA,QAErE;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,MAAM;AAAA,QACV,OAAO,KAAK,SAAS;AAAA,QACrB,OAAO,KAAK,MAAM,SAAa,KAAK,SAAS;AAAA,QAC7C,KAAK;AAAA,QACL,UAAU;AAAA,QACV,KAAK,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,GAAG,QAAQ,cAAc,EAAE,CAAC,IAAI;AAAA,MACvE;AACA,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,SAAS,MAAM,GAAG,QAAQ,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC;AACrD,YAAM,OAAO,KAAK,MAAM,MAAM;AAC9B,aAAO,KAAK,IAAI,SAAS;AAAA,IAC3B;AAAA,IAEA,MAAM,aAAa,MAAiB;AAIlC,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,KAAK,KAAK,QAAQ,CAAC;AACzB,UAAI,CAAC,GAAI,QAAO;AAChB,aAAO,YAAY,EAAE;AAAA,UAAa,KAAK,EAAE;AAAA,SAAY,KAAK,KAAK;AAAA,IACjE;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,SAAS,YAAY,QAAQ;AACnC,UAAI,CAAC,QAAQ;AACX,eAAO;AAAA,UACL,UAAU;AAAA,UACV,OAAO;AAAA,UACP,QAAQ,KAAK,UAAU,EAAE,QAAQ,+BAA+B,QAAQ,KAAK,IAAI,SAAS,KAAK,QAAQ,CAAC;AAAA,QAC1G;AAAA,MACF;AACA,YAAM,iBAAiB,KAAK,QAAQ,SAAS,OAAO,SAAS;AAC7D,YAAM,UAAU,OAAO,OAAO,KAAK;AACnC,YAAM,aAAa,KAAK,OAAO,UAAU,OAAO,UAAU,OAAO,KAAK,MAAM,UAAU,KAAK,KAAK;AAChG,YAAM,mBAAmB,WAAW;AACpC,YAAM,WAAW,kBAAkB;AACnC,YAAM,QAAQ,OAAO,iBAAiB,IAAI,KAAK,OAAO,mBAAmB,IAAI;AAC7E,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA,QAAQ,KAAK,UAAU;AAAA,UACrB;AAAA,UACA;AAAA,UACA;AAAA,UACA,WAAW;AAAA,UACX,QAAQ,KAAK;AAAA,UACb,WAAW,KAAK;AAAA,UAChB,SAAS,KAAK;AAAA,UACd,SAAS,KAAK;AAAA,UACd,QAAQ,KAAK;AAAA,QACf,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;","names":[]}