@tangle-network/agent-bench 0.4.1 → 0.4.3

package/dist/chunk-CXDUTWQE.js

@@ -1,318 +0,0 @@
-import {
-  benchRoot,
-  preflightVenvImports,
-  runVenvScriptStdin,
-  venvPython
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/appworld.ts
-import { spawn } from "child_process";
-import { join } from "path";
-import { createInterface } from "readline";
-import { routerToolLoop } from "@tangle-network/agent-runtime/loops";
-var DRIVER = join(benchRoot, "scripts", "appworld_driver.py");
-var DEFAULT_SPLIT = "test_normal";
-var appworldSolutionOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-var WORKER_CONTRACT = [
-  "",
-  "Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
-  "Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.",
-  "Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence."
-].join("\n");
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "string") {
-    throw new Error(`appworld task ${task.id} missing metadata.taskId \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-async function driver(args, input = "") {
-  let stdout;
-  try {
-    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot });
-  } catch (err) {
-    const e = err;
-    throw new Error(`appworld driver failed (${args.join(" ")}): ${(e.message || String(err)).slice(0, 1500)}`);
-  }
-  const last = stdout.trim().split("\n").at(-1) ?? "{}";
-  const parsed = JSON.parse(last);
-  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`);
-  return parsed;
-}
-function createAppWorldAdapter() {
-  return {
-    name: "appworld",
-    output: appworldSolutionOutput,
-    async preflight() {
-      await preflightVenvImports({
-        modules: ["appworld"],
-        requireDocker: false,
-        fix: "Fix: bench/.venv/bin/pip install appworld ; bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data (unpacks the engine + downloads the simulated-app data/tasks). Set APPWORLD_ROOT to the data root if not the default."
-      });
-    },
-    async loadTasks(opts = {}) {
-      const split = opts.split ?? DEFAULT_SPLIT;
-      const out = await driver([
-        "load",
-        "--split",
-        split,
-        ...opts.limit !== void 0 ? ["--limit", String(opts.limit)] : [],
-        ...opts.ids ? ["--ids", opts.ids.join(",")] : []
-      ]);
-      const tasks = out.tasks ?? [];
-      if (tasks.length === 0) {
-        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`);
-      }
-      return tasks.map(
-        (t) => ({
-          id: t.task_id,
-          split,
-          prompt: t.instruction + WORKER_CONTRACT,
-          metadata: { taskId: t.task_id, split }
-        })
-      );
-    },
-    async goldArtifact() {
-      return void 0;
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const out = await driver(["evaluate", "--task-id", meta.taskId, "--split", meta.split], artifact);
-      const passes = out.passes ?? 0;
-      const fails = out.fails ?? 0;
-      const total = out.num_tests ?? passes + fails;
-      const score = total > 0 ? passes / total : 0;
-      const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
-      return {
-        resolved: out.success === true,
-        score,
-        detail: JSON.stringify({
-          taskId: meta.taskId,
-          success: out.success,
-          passes,
-          fails,
-          total,
-          ...failures.length ? { failures } : {}
-        })
-      };
-    }
-  };
-}
-var REACT_HEADER = /^@appworld-react (\S+) (\S+)\n?/;
-var SESSION_SYSTEM = [
-  "You are completing a task in AppWorld, a simulated multi-app environment.",
-  "Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).",
-  "Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
-  "Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).",
-  "Work incrementally: small snippets, read each output, correct course.",
-  "When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again."
-].join("\n");
-var EXECUTE_TOOL = {
-  type: "function",
-  function: {
-    name: "execute_python",
-    description: "Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).",
-    parameters: {
-      type: "object",
-      properties: { code: { type: "string", description: "Python code calling apis.<app>.<fn>(...)" } },
-      required: ["code"]
-    }
-  }
-};
-async function withWorldSession(taskId, split, fn) {
-  const child = spawn(venvPython, [DRIVER, "session", "--task-id", taskId, "--split", split], {
-    cwd: benchRoot
-  });
-  const rl = createInterface({ input: child.stdout });
-  const pending = [];
-  const backlog = [];
-  rl.on("line", (l) => {
-    const next = pending.shift();
-    if (next) next(l);
-    else backlog.push(l);
-  });
-  let stderr = "";
-  child.stderr.on("data", (c) => {
-    stderr += c.toString("utf8");
-  });
-  const nextLine = (timeoutMs) => new Promise((resolve, reject) => {
-    const fromBacklog = backlog.shift();
-    if (fromBacklog !== void 0) return resolve(fromBacklog);
-    const t = setTimeout(
-      () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),
-      timeoutMs
-    );
-    const onExit = (code) => {
-      clearTimeout(t);
-      reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`));
-    };
-    pending.push((l) => {
-      clearTimeout(t);
-      child.removeListener("exit", onExit);
-      resolve(l);
-    });
-    child.once("exit", onExit);
-  });
-  try {
-    const ready = JSON.parse(await nextLine(12e4));
-    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? "no ready line"}`);
-    const call = async (cmd) => {
-      child.stdin.write(`${JSON.stringify(cmd)}
-`);
-      const res = JSON.parse(await nextLine(18e4));
-      if (typeof res.error === "string") throw new Error(`appworld session op failed: ${res.error}`);
-      return res;
-    };
-    return await fn(call, ready.instruction ?? "");
-  } finally {
-    child.stdin.end();
-    child.kill("SIGTERM");
-  }
-}
-function appworldToolLoopClient(cfg) {
-  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40);
-  let seq = 0;
-  return {
-    async create() {
-      const id = `appworld-toolloop-${seq++}`;
-      return {
-        id,
-        async *streamPrompt(prompt) {
-          const m = prompt.match(REACT_HEADER);
-          if (!m) {
-            throw new Error(
-              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header \u2014 got: ${prompt.slice(0, 120)}`
-            );
-          }
-          const [, taskId, split] = m;
-          const directive = prompt.replace(REACT_HEADER, "").trim();
-          const out = await withWorldSession(taskId, split, async (call, instruction) => {
-            const system = directive ? `${SESSION_SYSTEM}
-
-${directive}` : SESSION_SYSTEM;
-            const loop = await routerToolLoop(
-              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },
-              system,
-              `Task: ${instruction}`,
-              [EXECUTE_TOOL],
-              async (name, args) => {
-                if (name !== "execute_python") return `error: unknown tool ${name}`;
-                const res = await call({ op: "execute", code: String(args.code ?? "") });
-                const done = res.task_completed === true;
-                return `${String(res.output ?? "")}${done ? "\n\n[TASK MARKED COMPLETE \u2014 reply with a final summary and do not call the tool again]" : ""}`;
-              },
-              { maxTurns }
-            );
-            const verdict = await call({ op: "evaluate" });
-            const transcript = loop.toolTrace.slice(-3).map((t) => `CODE:
-${t.args.slice(0, 600)}
-OUTPUT:
-${t.result.slice(0, 600)}`).join("\n---\n").slice(0, 1600);
-            return {
-              ...verdict,
-              turns: loop.turns,
-              input_tokens: loop.usage.input,
-              output_tokens: loop.usage.output,
-              transcript
-            };
-          });
-          if (out.input_tokens || out.output_tokens) {
-            yield {
-              type: "llm_call",
-              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model }
-            };
-          }
-          yield { type: "result", data: { finalText: JSON.stringify(out) } };
-        },
-        async delete() {
-        }
-      };
-    }
-  };
-}
-var reactEpisodeOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    return text;
-  }
-};
-function createAppWorldReactAdapter() {
-  const base = createAppWorldAdapter();
-  return {
-    name: "appworld-react",
-    output: reactEpisodeOutput,
-    preflight: () => base.preflight(),
-    async loadTasks(opts = {}) {
-      const tasks = await base.loadTasks(opts);
-      return tasks.map((t) => {
-        const meta = readMeta(t);
-        return {
-          ...t,
-          // Header carries task identity to the leaf; the body (empty at round 0)
-          // is the directive slot the arms append their steer into.
-          prompt: `@appworld-react ${meta.taskId} ${meta.split}
-`
-        };
-      });
-    },
-    goldArtifact: () => Promise.resolve(void 0),
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      let out;
-      try {
-        out = JSON.parse(artifact);
-      } catch {
-        throw new Error(
-          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`
-        );
-      }
-      if (typeof out.success !== "boolean" || typeof out.num_tests !== "number") {
-        throw new Error(
-          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`
-        );
-      }
-      const passes = out.passes ?? 0;
-      const total = out.num_tests;
-      const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
-      return {
-        resolved: out.success === true,
-        score: total > 0 ? passes / total : 0,
-        detail: JSON.stringify({
-          taskId: meta.taskId,
-          success: out.success,
-          passes,
-          fails: out.fails ?? 0,
-          total,
-          turns: out.turns,
-          ...failures.length ? { failures } : {},
-          ...out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}
-        })
-      };
-    },
-    leafClient: (c) => appworldToolLoopClient(c)
-  };
-}
-
-export {
-  appworldSolutionOutput,
-  createAppWorldAdapter,
-  appworldToolLoopClient,
-  createAppWorldReactAdapter
-};
-//# sourceMappingURL=chunk-CXDUTWQE.js.map

package/dist/chunk-CXDUTWQE.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\ninterface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  transcript?: string\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))\n      }\n      pending.push((l) => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        resolve(l)\n      })\n      child.once('exit', onExit)\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    child.stdin.end()\n    child.kill('SIGTERM')\n  }\n}\n\n/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string) {\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const loop = await routerToolLoop(\n              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },\n              system,\n              `Task: ${instruction}`,\n              [EXECUTE_TOOL],\n              async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                return `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n              },\n              { maxTurns },\n            )\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = loop.toolTrace\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            return {\n              ...verdict,\n              turns: loop.turns,\n              input_tokens: loop.usage.input,\n              output_tokens: loop.usage.output,\n              transcript,\n            } satisfies ReactResult\n          })\n          // Real usage from the episode — flat llm_call so the kernel meters it.\n          if (out.input_tokens || out.output_tokens) {\n            yield {\n              type: 'llm_call',\n              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },\n            }\n          }\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;AAqBA,SAAS,aAAa;AACtB,SAAS,YAAY;AACrB,SAAS,uBAAuB;AAChC,SAA6B,sBAAqC;AAIlE,IAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;AAG9D,IAAM,gBAAgB;AAQf,IAAM,yBAAgD;AAAA,EAC3D,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,sCAAsC,CAAC;AACxE,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,IAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;AAC/C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAAU;AACxC,UAAM,IAAI,MAAM,iBAAiB,KAAK,EAAE,+DAA0D;AAAA,EACpG;AACA,SAAO;AACT;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;AAClE,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;AAAA,EAC3E,SAAS,KAAK;AACZ,UAAM,IAAI;AACV,UAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,WAAW,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,EAAE;AAAA,EAC5G;AACA,QAAM,OAAO,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK;AACjD,QAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,MAAI,OAAO,MAAO,OAAM,IAAI,MAAM,0BAA0B,OAAO,KAAK,EAAE;AAC1E,SAAO;AACT;AAEO,SAAS,wBAA0C;AACxD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,YAAM,qBAAqB;AAAA,QACzB,SAAS,CAAC,UAAU;AAAA,QACpB,eAAe;AAAA,QACf,KACE;AAAA,MAIJ,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;AAC5D,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,MAAO,MAAM,OAAO;AAAA,QACxB;AAAA,QACA;AAAA,QAAW;AAAA,QACX,GAAI,KAAK,UAAU,SAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;AAAA,QAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;AAAA,MAClD,CAAC;AACD,YAAM,QAAQ,IAAI,SAAS,CAAC;AAC5B,UAAI,MAAM,WAAW,GAAG;AACtB,cAAM,IAAI,MAAM,kDAAkD,KAAK,IAAI,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,MACnG;AACA,aAAO,MAAM;AAAA,QACX,CAAC,OAAkB;AAAA,UACjB,IAAI,EAAE;AAAA,UACN;AAAA,UACA,QAAQ,EAAE,cAAc;AAAA,UACxB,UAAU,EAAE,QAAQ,EAAE,SAAS,MAAM;AAAA,QACvC;AAAA,MACF;AAAA,IACF;AAAA,IAEA,MAAM,eAAe;AAOnB,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAO,MAAM,OAAO,CAAC,YAAY,aAAa,KAAK,QAAQ,WAAW,KAAK,KAAK,GAAG,QAAQ;AAOjG,YAAM,SAAS,IAAI,UAAU;AAC7B,YAAM,QAAQ,IAAI,SAAS;AAI3B,YAAM,QAAQ,IAAI,aAAa,SAAS;AACxC,YAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;AAG3C,YAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;AACzE,aAAO;AAAA,QACL,UAAU,IAAI,YAAY;AAAA,QAC1B;AAAA,QACA,QAAQ,KAAK,UAAU;AAAA,UACrB,QAAQ,KAAK;AAAA,UACb,SAAS,IAAI;AAAA,UACb;AAAA,UACA;AAAA,UACA;AAAA,UACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;AAAA,QACxC,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;AAmCA,IAAM,eAAe;AAErB,IAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAEX,IAAM,eAAyB;AAAA,EAC7B,MAAM;AAAA,EACN,UAAU;AAAA,IACR,MAAM;AAAA,IACN,aACE;AAAA,IACF,YAAY;AAAA,MACV,MAAM;AAAA,MACN,YAAY,EAAE,MAAM,EAAE,MAAM,UAAU,aAAa,2CAA2C,EAAE;AAAA,MAChG,UAAU,CAAC,MAAM;AAAA,IACnB;AAAA,EACF;AACF;AAGA,eAAe,iBACb,QACA,OACA,IACY;AACZ,QAAM,QAAQ,MAAM,YAAY,CAAC,QAAQ,WAAW,aAAa,QAAQ,WAAW,KAAK,GAAG;AAAA,IAC1F,KAAK;AAAA,EACP,CAAC;AACD,QAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;AAClD,QAAM,UAAyC,CAAC;AAChD,QAAM,UAAoB,CAAC;AAC3B,KAAG,GAAG,QAAQ,CAAC,MAAM;AACnB,UAAM,OAAO,QAAQ,MAAM;AAC3B,QAAI,KAAM,MAAK,CAAC;AAAA,QACX,SAAQ,KAAK,CAAC;AAAA,EACrB,CAAC;AACD,MAAI,SAAS;AACb,QAAM,OAAO,GAAG,QAAQ,CAAC,MAAc;AACrC,cAAU,EAAE,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,QAAM,WAAW,CAAC,cAChB,IAAI,QAAQ,CAAC,SAAS,WAAW;AAC/B,UAAM,cAAc,QAAQ,MAAM;AAClC,QAAI,gBAAgB,OAAW,QAAO,QAAQ,WAAW;AACzD,UAAM,IAAI;AAAA,MACR,MAAM,OAAO,IAAI,MAAM,oCAAoC,SAAS,eAAe,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,MACxG;AAAA,IACF;AAGA,UAAM,SAAS,CAAC,SAA8B;AAC5C,mBAAa,CAAC;AACd,aAAO,IAAI,MAAM,4BAA4B,IAAI,cAAc,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC;AAAA,IACtF;AACA,YAAQ,KAAK,CAAC,MAAM;AAClB,mBAAa,CAAC;AACd,YAAM,eAAe,QAAQ,MAAM;AACnC,cAAQ,CAAC;AAAA,IACX,CAAC;AACD,UAAM,KAAK,QAAQ,MAAM;AAAA,EAC3B,CAAC;AACH,MAAI;AACF,UAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;AAChD,QAAI,CAAC,MAAM,MAAO,OAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,eAAe,EAAE;AACvG,UAAM,OAAO,OAAO,QAAmE;AACrF,YAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,CAAC;AAAA,CAAI;AAC5C,YAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;AAC9C,UAAI,OAAO,IAAI,UAAU,SAAU,OAAM,IAAI,MAAM,+BAA+B,IAAI,KAAK,EAAE;AAC7F,aAAO;AAAA,IACT;AACA,WAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;AAAA,EAC/C,UAAE;AACA,UAAM,MAAM,IAAI;AAChB,UAAM,KAAK,SAAS;AAAA,EACtB;AACF;AAGO,SAAS,uBAAuB,KAK3B;AACV,QAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;AACzE,MAAI,MAAM;AACV,SAAO;AAAA,IACL,MAAM,SAAS;AACb,YAAM,KAAK,qBAAqB,KAAK;AACrC,aAAO;AAAA,QACL;AAAA,QACA,OAAO,aAAa,QAAgB;AAClC,gBAAM,IAAI,OAAO,MAAM,YAAY;AACnC,cAAI,CAAC,GAAG;AACN,kBAAM,IAAI;AAAA,cACR,6FAAwF,OAAO,MAAM,GAAG,GAAG,CAAC;AAAA,YAC9G;AAAA,UACF;AACA,gBAAM,CAAC,EAAE,QAAQ,KAAK,IAAI;AAC1B,gBAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,EAAE,KAAK;AACxD,gBAAM,MAAM,MAAM,iBAAiB,QAAkB,OAAiB,OAAO,MAAM,gBAAgB;AACjG,kBAAM,SAAS,YAAY,GAAG,cAAc;AAAA;AAAA,EAAO,SAAS,KAAK;AACjE,kBAAM,OAAO,MAAM;AAAA,cACjB,EAAE,eAAe,IAAI,eAAe,WAAW,IAAI,WAAW,OAAO,IAAI,MAAM;AAAA,cAC/E;AAAA,cACA,SAAS,WAAW;AAAA,cACpB,CAAC,YAAY;AAAA,cACb,OAAO,MAAM,SAAS;AACpB,oBAAI,SAAS,iBAAkB,QAAO,uBAAuB,IAAI;AACjE,sBAAM,MAAM,MAAM,KAAK,EAAE,IAAI,WAAW,MAAM,OAAO,KAAK,QAAQ,EAAE,EAAE,CAAC;AACvE,sBAAM,OAAO,IAAI,mBAAmB;AACpC,uBAAO,GAAG,OAAO,IAAI,UAAU,EAAE,CAAC,GAAG,OAAO,gGAA2F,EAAE;AAAA,cAC3I;AAAA,cACA,EAAE,SAAS;AAAA,YACb;AACA,kBAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;AAC9C,kBAAM,aAAa,KAAK,UACrB,MAAM,EAAE,EACR,IAAI,CAAC,MAAM;AAAA,EAAU,EAAE,KAAK,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA,EAAc,EAAE,OAAO,MAAM,GAAG,GAAG,CAAC,EAAE,EAC/E,KAAK,SAAS,EACd,MAAM,GAAG,IAAI;AAChB,mBAAO;AAAA,cACL,GAAG;AAAA,cACH,OAAO,KAAK;AAAA,cACZ,cAAc,KAAK,MAAM;AAAA,cACzB,eAAe,KAAK,MAAM;AAAA,cAC1B;AAAA,YACF;AAAA,UACF,CAAC;AAED,cAAI,IAAI,gBAAgB,IAAI,eAAe;AACzC,kBAAM;AAAA,cACJ,MAAM;AAAA,cACN,MAAM,EAAE,UAAU,IAAI,gBAAgB,GAAG,WAAW,IAAI,iBAAiB,GAAG,OAAO,IAAI,MAAM;AAAA,YAC/F;AAAA,UACF;AACA,gBAAM,EAAE,MAAM,UAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE,EAAE;AAAA,QACnE;AAAA,QACA,MAAM,SAAS;AAAA,QAAC;AAAA,MAClB;AAAA,IACF;AAAA,EACF;AACF;AAGA,IAAM,qBAA4C;AAAA,EAChD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG;AACb,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AACF;AAEO,SAAS,6BAA+C;AAC7D,QAAM,OAAO,sBAAsB;AACnC,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,WAAW,MAAM,KAAK,UAAU;AAAA,IAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;AAC5D,YAAM,QAAQ,MAAM,KAAK,UAAU,IAAI;AACvC,aAAO,MAAM,IAAI,CAAC,MAAM;AACtB,cAAM,OAAO,SAAS,CAAC;AACvB,eAAO;AAAA,UACL,GAAG;AAAA;AAAA;AAAA,UAGH,QAAQ,mBAAmB,KAAK,MAAM,IAAI,KAAK,KAAK;AAAA;AAAA,QACtD;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,cAAc,MAAM,QAAQ,QAAQ,MAAS;AAAA,IAE7C,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,UAAI;AACJ,UAAI;AACF,cAAM,KAAK,MAAM,QAAQ;AAAA,MAC3B,QAAQ;AACN,cAAM,IAAI;AAAA,UACR,6EAA6E,KAAK,MAAM,MAAM,SAAS,MAAM,GAAG,GAAG,CAAC;AAAA,QACtH;AAAA,MACF;AACA,UAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAAU;AACzE,cAAM,IAAI;AAAA,UACR,sEAAsE,KAAK,MAAM,MAAM,SAAS,MAAM,GAAG,GAAG,CAAC;AAAA,QAC/G;AAAA,MACF;AACA,YAAM,SAAS,IAAI,UAAU;AAC7B,YAAM,QAAQ,IAAI;AAClB,YAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;AACzE,aAAO;AAAA,QACL,UAAU,IAAI,YAAY;AAAA,QAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;AAAA,QACpC,QAAQ,KAAK,UAAU;AAAA,UACrB,QAAQ,KAAK;AAAA,UACb,SAAS,IAAI;AAAA,UACb;AAAA,UACA,OAAO,IAAI,SAAS;AAAA,UACpB;AAAA,UACA,OAAO,IAAI;AAAA,UACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;AAAA,UACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;AAAA,QACzE,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IAEA,YAAY,CAAC,MAAM,uBAAuB,CAAC;AAAA,EAC7C;AACF;","names":[]}

package/dist/chunk-DWALFME7.js

@@ -1,182 +0,0 @@
-import {
-  benchRoot,
-  runVenvPython,
-  runVenvScriptStdin
-} from "./chunk-EIETHPD5.js";
-
-// src/benchmarks/dabstep.ts
-import { join } from "path";
-import { access, readFile, stat } from "fs/promises";
-var FIXTURES = join(benchRoot, "fixtures", "dabstep.json");
-var DEFAULT_SPLIT = "easy";
-var dabstepDir = () => process.env.DABSTEP_DIR;
-var gradeFile = (dir) => join(dir, "grade.py");
-var resourceRoot = (dir) => join(dir, "files");
-async function assertFile(path, label) {
-  try {
-    await access(path);
-  } catch (err) {
-    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`);
-  }
-}
-async function assertOfficialFiles(dir, split) {
-  await assertFile(join(dir, "dataset.csv"), "released dataset.csv");
-  await assertFile(join(dir, "splits", `${split}.txt`), `${split} split file`);
-  await assertFile(gradeFile(dir), "official grade.py");
-  const files = resourceRoot(dir);
-  try {
-    const s = await stat(files);
-    if (!s.isDirectory()) throw new Error("not a directory");
-  } catch (err) {
-    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`);
-  }
-}
-var dabstepAnswerOutput = {
-  parse(events) {
-    let text = "";
-    for (const ev of events) {
-      const d = ev?.data;
-      const t = d?.finalText ?? d?.text ?? d?.result;
-      if (typeof t === "string" && t.length > 0) text = t;
-    }
-    const fences = [...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)];
-    return (fences.at(-1)?.[1] ?? text).trim();
-  }
-};
-function rowToTask(row, split, dir) {
-  const meta = {
-    taskId: row.task_id,
-    split,
-    golds: row.all_golds_by_task,
-    ...dir ? { resourceRoot: resourceRoot(dir) } : {}
-  };
-  return {
-    id: String(row.task_id),
-    split,
-    prompt: [
-      "Solve this DABStep data-analysis task using the mounted payment files.",
-      "Use code or shell commands as needed, then return only the final answer.",
-      "",
-      row.instructions
-    ].join("\n"),
-    metadata: meta
-  };
-}
-function readMeta(task) {
-  const md = task.metadata;
-  if (!md || typeof md.taskId !== "number" || !Array.isArray(md.golds)) {
-    throw new Error(`dabstep task ${task.id} missing metadata \u2014 loadTasks did not populate it`);
-  }
-  return md;
-}
-function selectRows(rows, opts, split, dir) {
-  let tasks = rows.map((row) => rowToTask(row, split, dir));
-  if (opts.ids) {
-    const want = new Set(opts.ids);
-    tasks = tasks.filter((task) => want.has(task.id));
-  } else if (opts.limit !== void 0) {
-    tasks = tasks.slice(0, opts.limit);
-  }
-  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`);
-  return tasks;
-}
-async function loadFixtures(opts, split) {
-  const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
-  console.warn(`[dabstep] DABSTEP_FIXTURES=1 \u2014 loading ${rows.length} adapter fixtures from ${FIXTURES}`);
-  return selectRows(rows, opts, split);
-}
-async function loadOfficialTasks(dir, opts, split) {
-  const script = `
-import ast, csv, json, sys
-from pathlib import Path
-
-root = Path(sys.argv[1])
-split = sys.argv[2]
-limit = None if sys.argv[3] == "" else int(sys.argv[3])
-ids = set(json.loads(sys.argv[4]))
-dataset = root / "dataset.csv"
-split_file = root / "splits" / f"{split}.txt"
-if not dataset.exists():
-    raise SystemExit(f"missing official DABStep dataset.csv at {dataset}")
-if not split_file.exists():
-    raise SystemExit(f"missing official DABStep split file at {split_file}")
-split_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}
-out = []
-with dataset.open(newline="") as f:
-    for row in csv.DictReader(f):
-        task_id = int(row["task_id"])
-        if task_id not in split_ids:
-            continue
-        if ids and str(task_id) not in ids:
-            continue
-        out.append({
-            "task_id": task_id,
-            "instructions": f"{row['question']}\\n{row['guidelines']}",
-            "all_golds_by_task": ast.literal_eval(str(row["all_golds_by_task"])),
-        })
-        if limit is not None and len(out) >= limit:
-            break
-if not out:
-    raise SystemExit(f"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}")
-print(json.dumps(out))
-`;
-  const stdout = await runVenvPython(script, [dir, split, opts.limit === void 0 ? "" : String(opts.limit), JSON.stringify(opts.ids ?? [])]);
-  return selectRows(JSON.parse(stdout), opts, split, dir);
-}
-function createDabstepAdapter() {
-  const fixturesMode = process.env.DABSTEP_FIXTURES === "1";
-  return {
-    name: "dabstep",
-    output: dabstepAnswerOutput,
-    async preflight() {
-      if (fixturesMode) return;
-      const dir = dabstepDir();
-      if (!dir) {
-        throw new Error(
-          "DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep."
-        );
-      }
-      await assertOfficialFiles(dir, DEFAULT_SPLIT);
-      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT);
-    },
-    async loadTasks(opts = {}) {
-      const split = opts.split ?? DEFAULT_SPLIT;
-      if (fixturesMode) return loadFixtures(opts, split);
-      const dir = dabstepDir();
-      if (!dir) throw new Error("DABSTEP_DIR is required to load official DABStep tasks");
-      return loadOfficialTasks(dir, opts, split);
-    },
-    async goldArtifact(task) {
-      const meta = readMeta(task);
-      const first = meta.golds[0];
-      if (!first) return void 0;
-      const value = first.value;
-      return value === void 0 ? void 0 : String(value);
-    },
-    async judge(task, artifact) {
-      const meta = readMeta(task);
-      const dir = dabstepDir();
-      if (!dir) throw new Error("DABSTEP_DIR is required to judge DABStep tasks with the official grade.py");
-      const stdout = await runVenvScriptStdin(
-        join(benchRoot, "scripts", "dabstep_judge.py"),
-        ["--grade-file", gradeFile(dir)],
-        JSON.stringify({ prediction: artifact, golds: meta.golds }),
-        { cwd: benchRoot }
-      );
-      const report = JSON.parse(stdout.trim().split("\n").at(-1) ?? "{}");
-      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`);
-      const score = typeof report.score === "number" ? report.score : 0;
-      return {
-        resolved: report.correct === true,
-        score,
-        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct })
-      };
-    }
-  };
-}
-
-export {
-  dabstepAnswerOutput,
-  createDabstepAdapter
-};
-//# sourceMappingURL=chunk-DWALFME7.js.map

package/dist/chunk-DWALFME7.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/benchmarks/dabstep.ts"],"sourcesContent":["/**\n * DABStep adapter (EnvCommons/DABStep) — data-analysis questions over synthetic\n * payment files. Worker artifact = final answer text. Judge = the official\n * DABStep `grade.py` normalization/matching function. No LLM judge.\n *\n * Live tasks require `DABSTEP_DIR` pointing at an official DABStep checkout that\n * includes `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`. The adapter\n * exposes `metadata.resourceRoot` so runners can mount the benchmark files into\n * AgentProfile.resources.files; it does not paste the dataset into prompt text.\n */\n\nimport { join } from 'node:path'\nimport { access, readFile, stat } from 'node:fs/promises'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, runVenvPython, runVenvScriptStdin } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'dabstep.json')\nconst DEFAULT_SPLIT = 'easy'\n\ninterface DabstepFixtureRow {\n  task_id: number\n  instructions: string\n  all_golds_by_task: Array<Record<string, unknown>>\n}\n\ninterface DabstepMeta {\n  taskId: number\n  split: string\n  golds: Array<Record<string, unknown>>\n  resourceRoot?: string\n}\n\nconst dabstepDir = (): string | undefined => process.env.DABSTEP_DIR\nconst gradeFile = (dir: string): string => join(dir, 'grade.py')\nconst resourceRoot = (dir: string): string => join(dir, 'files')\n\nasync function assertFile(path: string, label: string): Promise<void> {\n  try {\n    await access(path)\n  } catch (err) {\n    throw new Error(`DABStep: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nasync function assertOfficialFiles(dir: string, split: string): Promise<void> {\n  await assertFile(join(dir, 'dataset.csv'), 'released dataset.csv')\n  await assertFile(join(dir, 'splits', `${split}.txt`), `${split} split file`)\n  await assertFile(gradeFile(dir), 'official grade.py')\n  const files = resourceRoot(dir)\n  try {\n    const s = await stat(files)\n    if (!s.isDirectory()) throw new Error('not a directory')\n  } catch (err) {\n    throw new Error(`DABStep: missing benchmark files directory at ${files} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nexport const dabstepAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction rowToTask(row: DabstepFixtureRow, split: string, dir?: string): BenchTask {\n  const meta: DabstepMeta = {\n    taskId: row.task_id,\n    split,\n    golds: row.all_golds_by_task,\n    ...(dir ? { resourceRoot: resourceRoot(dir) } : {}),\n  }\n  return {\n    id: String(row.task_id),\n    split,\n    prompt: [\n      'Solve this DABStep data-analysis task using the mounted payment files.',\n      'Use code or shell commands as needed, then return only the final answer.',\n      '',\n      row.instructions,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): DabstepMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'number' || !Array.isArray(md.golds)) {\n    throw new Error(`dabstep task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as DabstepMeta\n}\n\nfunction selectRows(rows: DabstepFixtureRow[], opts: LoadOptions, split: string, dir?: string): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, split, dir))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`DABStep: no tasks matched ${JSON.stringify(opts)} for split=${split}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as DabstepFixtureRow[]\n  console.warn(`[dabstep] DABSTEP_FIXTURES=1 — loading ${rows.length} adapter fixtures from ${FIXTURES}`)\n  return selectRows(rows, opts, split)\n}\n\nasync function loadOfficialTasks(dir: string, opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const script = `\nimport ast, csv, json, sys\nfrom pathlib import Path\n\nroot = Path(sys.argv[1])\nsplit = sys.argv[2]\nlimit = None if sys.argv[3] == \"\" else int(sys.argv[3])\nids = set(json.loads(sys.argv[4]))\ndataset = root / \"dataset.csv\"\nsplit_file = root / \"splits\" / f\"{split}.txt\"\nif not dataset.exists():\n    raise SystemExit(f\"missing official DABStep dataset.csv at {dataset}\")\nif not split_file.exists():\n    raise SystemExit(f\"missing official DABStep split file at {split_file}\")\nsplit_ids = {int(line.strip()) for line in split_file.read_text().splitlines() if line.strip()}\nout = []\nwith dataset.open(newline=\"\") as f:\n    for row in csv.DictReader(f):\n        task_id = int(row[\"task_id\"])\n        if task_id not in split_ids:\n            continue\n        if ids and str(task_id) not in ids:\n            continue\n        out.append({\n            \"task_id\": task_id,\n            \"instructions\": f\"{row['question']}\\\\n{row['guidelines']}\",\n            \"all_golds_by_task\": ast.literal_eval(str(row[\"all_golds_by_task\"])),\n        })\n        if limit is not None and len(out) >= limit:\n            break\nif not out:\n    raise SystemExit(f\"no DABStep rows matched split={split} ids={sorted(ids)} limit={limit}\")\nprint(json.dumps(out))\n`\n  const stdout = await runVenvPython(script, [dir, split, opts.limit === undefined ? '' : String(opts.limit), JSON.stringify(opts.ids ?? [])])\n  return selectRows(JSON.parse(stdout) as DabstepFixtureRow[], opts, split, dir)\n}\n\nexport function createDabstepAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.DABSTEP_FIXTURES === '1'\n\n  return {\n    name: 'dabstep',\n    output: dabstepAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = dabstepDir()\n      if (!dir) {\n        throw new Error(\n          'DABSTEP_DIR is required. Fix: clone https://github.com/EnvCommons/DABStep, add the released dataset.csv under that checkout, then set DABSTEP_DIR=/path/to/DABStep.',\n        )\n      }\n      await assertOfficialFiles(dir, DEFAULT_SPLIT)\n      await loadOfficialTasks(dir, { limit: 1 }, DEFAULT_SPLIT)\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to load official DABStep tasks')\n      return loadOfficialTasks(dir, opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      const first = meta.golds[0]\n      if (!first) return undefined\n      const value = first.value\n      return value === undefined ? undefined : String(value)\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const dir = dabstepDir()\n      if (!dir) throw new Error('DABSTEP_DIR is required to judge DABStep tasks with the official grade.py')\n      const stdout = await runVenvScriptStdin(\n        join(benchRoot, 'scripts', 'dabstep_judge.py'),\n        ['--grade-file', gradeFile(dir)],\n        JSON.stringify({ prediction: artifact, golds: meta.golds }),\n        { cwd: benchRoot },\n      )\n      const report = JSON.parse(stdout.trim().split('\\n').at(-1) ?? '{}') as { correct?: boolean; score?: number; error?: string }\n      if (report.error) throw new Error(`DABStep judge error for ${task.id}: ${report.error}`)\n      const score = typeof report.score === 'number' ? report.score : 0\n      return {\n        resolved: report.correct === true,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, split: meta.split, correct: report.correct }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;AAWA,SAAS,YAAY;AACrB,SAAS,QAAQ,UAAU,YAAY;AAKvC,IAAM,WAAW,KAAK,WAAW,YAAY,cAAc;AAC3D,IAAM,gBAAgB;AAetB,IAAM,aAAa,MAA0B,QAAQ,IAAI;AACzD,IAAM,YAAY,CAAC,QAAwB,KAAK,KAAK,UAAU;AAC/D,IAAM,eAAe,CAAC,QAAwB,KAAK,KAAK,OAAO;AAE/D,eAAe,WAAW,MAAc,OAA8B;AACpE,MAAI;AACF,UAAM,OAAO,IAAI;AAAA,EACnB,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,oBAAoB,KAAK,OAAO,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACtG;AACF;AAEA,eAAe,oBAAoB,KAAa,OAA8B;AAC5E,QAAM,WAAW,KAAK,KAAK,aAAa,GAAG,sBAAsB;AACjE,QAAM,WAAW,KAAK,KAAK,UAAU,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,aAAa;AAC3E,QAAM,WAAW,UAAU,GAAG,GAAG,mBAAmB;AACpD,QAAM,QAAQ,aAAa,GAAG;AAC9B,MAAI;AACF,UAAM,IAAI,MAAM,KAAK,KAAK;AAC1B,QAAI,CAAC,EAAE,YAAY,EAAG,OAAM,IAAI,MAAM,iBAAiB;AAAA,EACzD,SAAS,KAAK;AACZ,UAAM,IAAI,MAAM,iDAAiD,KAAK,KAAK,eAAe,QAAQ,IAAI,UAAU,GAAG,GAAG;AAAA,EACxH;AACF;AAEO,IAAM,sBAA6C;AAAA,EACxD,MAAM,QAAQ;AACZ,QAAI,OAAO;AACX,eAAW,MAAM,QAAQ;AACvB,YAAM,IAAK,IAA2C;AACtD,YAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;AACxC,UAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,IACpD;AACA,UAAM,SAAS,CAAC,GAAG,KAAK,SAAS,wCAAwC,CAAC;AAC1E,YAAQ,OAAO,GAAG,EAAE,IAAI,CAAC,KAAK,MAAM,KAAK;AAAA,EAC3C;AACF;AAEA,SAAS,UAAU,KAAwB,OAAe,KAAyB;AACjF,QAAM,OAAoB;AAAA,IACxB,QAAQ,IAAI;AAAA,IACZ;AAAA,IACA,OAAO,IAAI;AAAA,IACX,GAAI,MAAM,EAAE,cAAc,aAAa,GAAG,EAAE,IAAI,CAAC;AAAA,EACnD;AACA,SAAO;AAAA,IACL,IAAI,OAAO,IAAI,OAAO;AAAA,IACtB;AAAA,IACA,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,IAAI;AAAA,IACN,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ;AACF;AAEA,SAAS,SAAS,MAA8B;AAC9C,QAAM,KAAK,KAAK;AAChB,MAAI,CAAC,MAAM,OAAO,GAAG,WAAW,YAAY,CAAC,MAAM,QAAQ,GAAG,KAAK,GAAG;AACpE,UAAM,IAAI,MAAM,gBAAgB,KAAK,EAAE,wDAAmD;AAAA,EAC5F;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,OAAe,KAA2B;AAC1G,MAAI,QAAQ,KAAK,IAAI,CAAC,QAAQ,UAAU,KAAK,OAAO,GAAG,CAAC;AACxD,MAAI,KAAK,KAAK;AACZ,UAAM,OAAO,IAAI,IAAI,KAAK,GAAG;AAC7B,YAAQ,MAAM,OAAO,CAAC,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EAClD,WAAW,KAAK,UAAU,QAAW;AACnC,YAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,6BAA6B,KAAK,UAAU,IAAI,CAAC,cAAc,KAAK,EAAE;AAC9G,SAAO;AACT;AAEA,eAAe,aAAa,MAAmB,OAAqC;AAClF,QAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;AACxD,UAAQ,KAAK,+CAA0C,KAAK,MAAM,0BAA0B,QAAQ,EAAE;AACtG,SAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,eAAe,kBAAkB,KAAa,MAAmB,OAAqC;AACpG,QAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkCf,QAAM,SAAS,MAAM,cAAc,QAAQ,CAAC,KAAK,OAAO,KAAK,UAAU,SAAY,KAAK,OAAO,KAAK,KAAK,GAAG,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;AAC3I,SAAO,WAAW,KAAK,MAAM,MAAM,GAA0B,MAAM,OAAO,GAAG;AAC/E;AAEO,SAAS,uBAAyC;AACvD,QAAM,eAAe,QAAQ,IAAI,qBAAqB;AAEtD,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ;AAAA,IAER,MAAM,YAAY;AAChB,UAAI,aAAc;AAClB,YAAM,MAAM,WAAW;AACvB,UAAI,CAAC,KAAK;AACR,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,oBAAoB,KAAK,aAAa;AAC5C,YAAM,kBAAkB,KAAK,EAAE,OAAO,EAAE,GAAG,aAAa;AAAA,IAC1D;AAAA,IAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;AACtC,YAAM,QAAQ,KAAK,SAAS;AAC5B,UAAI,aAAc,QAAO,aAAa,MAAM,KAAK;AACjD,YAAM,MAAM,WAAW;AACvB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wDAAwD;AAClF,aAAO,kBAAkB,KAAK,MAAM,KAAK;AAAA,IAC3C;AAAA,IAEA,MAAM,aAAa,MAAiB;AAClC,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,CAAC,MAAO,QAAO;AACnB,YAAM,QAAQ,MAAM;AACpB,aAAO,UAAU,SAAY,SAAY,OAAO,KAAK;AAAA,IACvD;AAAA,IAEA,MAAM,MAAM,MAAiB,UAAuC;AAClE,YAAM,OAAO,SAAS,IAAI;AAC1B,YAAM,MAAM,WAAW;AACvB,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,2EAA2E;AACrG,YAAM,SAAS,MAAM;AAAA,QACnB,KAAK,WAAW,WAAW,kBAAkB;AAAA,QAC7C,CAAC,gBAAgB,UAAU,GAAG,CAAC;AAAA,QAC/B,KAAK,UAAU,EAAE,YAAY,UAAU,OAAO,KAAK,MAAM,CAAC;AAAA,QAC1D,EAAE,KAAK,UAAU;AAAA,MACnB;AACA,YAAM,SAAS,KAAK,MAAM,OAAO,KAAK,EAAE,MAAM,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI;AAClE,UAAI,OAAO,MAAO,OAAM,IAAI,MAAM,2BAA2B,KAAK,EAAE,KAAK,OAAO,KAAK,EAAE;AACvF,YAAM,QAAQ,OAAO,OAAO,UAAU,WAAW,OAAO,QAAQ;AAChE,aAAO;AAAA,QACL,UAAU,OAAO,YAAY;AAAA,QAC7B;AAAA,QACA,QAAQ,KAAK,UAAU,EAAE,QAAQ,KAAK,QAAQ,OAAO,KAAK,OAAO,SAAS,OAAO,QAAQ,CAAC;AAAA,MAC5F;AAAA,IACF;AAAA,EACF;AACF;","names":[]}