@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/appworld.js

@@ -1,14 +1,319 @@
-import {
-  appworldSolutionOutput,
-  appworldToolLoopClient,
-  createAppWorldAdapter,
-  createAppWorldReactAdapter
-} from "../chunk-IA2FBTWC.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  appworldSolutionOutput,
-  appworldToolLoopClient,
-  createAppWorldAdapter,
-  createAppWorldReactAdapter
+import { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from "./_harness.js";
+import { join } from "node:path";
+import { spawn } from "node:child_process";
+import { createInterface } from "node:readline";
+import { routerToolLoop } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/appworld.ts
+/**
+* AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's
+* Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`
+* surface AppWorld exposes inside `world.execute(...)`), ending in
+* `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic
+* evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,
+* then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`
+* (per-requirement total) and the `passes`/`failures` lists. Score =
+* passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.
+*
+* loadTasks enumerates the real task suite via `load_task_ids(split)`
+* (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.
+* The OutputAdapter is stream-only, so the worker emits its solution as a fenced
+* ```python block which the driver executes.
+*
+* Requires for a live run: the bench `.venv` with `appworld` installed + the
+* unpacked engine + downloaded data (`appworld install` ; `appworld download
+* data`). preflight + loadTasks + judge all fail loud with the exact step when the
+* engine/data is absent — never a fabricated score.
+*/
+const DRIVER = join(benchRoot, "scripts", "appworld_driver.py");
+/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */
+const DEFAULT_SPLIT = "test_normal";
+/** Worker solution code = the last fenced ```python block, else the raw text. */
+const appworldSolutionOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return ([...text.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+} };
+const WORKER_CONTRACT = [
+	"",
+	"Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
+	"Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.",
+	"Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence."
+].join("\n");
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.taskId !== "string") throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`);
+	return md;
+}
+/**
+* Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.
+* The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —
+* execFile's `input` option is not honored async and hangs the driver's
+* sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.
+*/
+async function driver(args, input = "") {
+	let stdout;
+	try {
+		stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot });
+	} catch (err) {
+		const e = err;
+		throw new Error(`appworld driver failed (${args.join(" ")}): ${(e.message || String(err)).slice(0, 1500)}`);
+	}
+	const last = stdout.trim().split("\n").at(-1) ?? "{}";
+	const parsed = JSON.parse(last);
+	if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`);
+	return parsed;
+}
+function createAppWorldAdapter() {
+	return {
+		name: "appworld",
+		output: appworldSolutionOutput,
+		async preflight() {
+			await preflightVenvImports({
+				modules: ["appworld"],
+				requireDocker: false,
+				fix: "Fix: bench/.venv/bin/pip install appworld ; bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data (unpacks the engine + downloads the simulated-app data/tasks). Set APPWORLD_ROOT to the data root if not the default."
+			});
+		},
+		async loadTasks(opts = {}) {
+			const split = opts.split ?? DEFAULT_SPLIT;
+			const tasks = (await driver([
+				"load",
+				"--split",
+				split,
+				...opts.limit !== void 0 ? ["--limit", String(opts.limit)] : [],
+				...opts.ids ? ["--ids", opts.ids.join(",")] : []
+			])).tasks ?? [];
+			if (tasks.length === 0) throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`);
+			return tasks.map((t) => ({
+				id: t.task_id,
+				split,
+				prompt: t.instruction + WORKER_CONTRACT,
+				metadata: {
+					taskId: t.task_id,
+					split
+				}
+			}));
+		},
+		async goldArtifact() {},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const out = await driver([
+				"evaluate",
+				"--task-id",
+				meta.taskId,
+				"--split",
+				meta.split
+			], artifact);
+			const passes = out.passes ?? 0;
+			const fails = out.fails ?? 0;
+			const total = out.num_tests ?? passes + fails;
+			const score = total > 0 ? passes / total : 0;
+			const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
+			return {
+				resolved: out.success === true,
+				score,
+				detail: JSON.stringify({
+					taskId: meta.taskId,
+					success: out.success,
+					passes,
+					fails,
+					total,
+					...failures.length ? { failures } : {}
+				})
+			};
+		}
+	};
+}
+const REACT_HEADER = /^@appworld-react (\S+) (\S+)\n?/;
+const SESSION_SYSTEM = [
+	"You are completing a task in AppWorld, a simulated multi-app environment.",
+	"Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).",
+	"Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.",
+	"Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).",
+	"Work incrementally: small snippets, read each output, correct course.",
+	"When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again."
+].join("\n");
+const EXECUTE_TOOL = {
+	type: "function",
+	function: {
+		name: "execute_python",
+		description: "Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).",
+		parameters: {
+			type: "object",
+			properties: { code: {
+				type: "string",
+				description: "Python code calling apis.<app>.<fn>(...)"
+			} },
+			required: ["code"]
+		}
+	}
 };
+/** One persistent world session: line-JSONL request/response over the driver. */
+async function withWorldSession(taskId, split, fn) {
+	const child = spawn(venvPython, [
+		DRIVER,
+		"session",
+		"--task-id",
+		taskId,
+		"--split",
+		split
+	], { cwd: benchRoot });
+	const rl = createInterface({ input: child.stdout });
+	const pending = [];
+	const backlog = [];
+	rl.on("line", (l) => {
+		const next = pending.shift();
+		if (next) next(l);
+		else backlog.push(l);
+	});
+	let stderr = "";
+	child.stderr.on("data", (c) => {
+		stderr += c.toString("utf8");
+	});
+	const nextLine = (timeoutMs) => new Promise((resolve, reject) => {
+		const fromBacklog = backlog.shift();
+		if (fromBacklog !== void 0) return resolve(fromBacklog);
+		const t = setTimeout(() => reject(/* @__PURE__ */ new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)), timeoutMs);
+		const onExit = (code) => {
+			clearTimeout(t);
+			reject(/* @__PURE__ */ new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`));
+		};
+		pending.push((l) => {
+			clearTimeout(t);
+			child.removeListener("exit", onExit);
+			resolve(l);
+		});
+		child.once("exit", onExit);
+	});
+	try {
+		const ready = JSON.parse(await nextLine(12e4));
+		if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? "no ready line"}`);
+		const call = async (cmd) => {
+			child.stdin.write(`${JSON.stringify(cmd)}\n`);
+			const res = JSON.parse(await nextLine(18e4));
+			if (typeof res.error === "string") throw new Error(`appworld session op failed: ${res.error}`);
+			return res;
+		};
+		return await fn(call, ready.instruction ?? "");
+	} finally {
+		child.stdin.end();
+		child.kill("SIGTERM");
+	}
+}
+/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */
+function appworldToolLoopClient(cfg) {
+	const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40);
+	let seq = 0;
+	return { async create() {
+		return {
+			id: `appworld-toolloop-${seq++}`,
+			async *streamPrompt(prompt) {
+				const m = prompt.match(REACT_HEADER);
+				if (!m) throw new Error(`appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`);
+				const [, taskId, split] = m;
+				const directive = prompt.replace(REACT_HEADER, "").trim();
+				const out = await withWorldSession(taskId, split, async (call, instruction) => {
+					const system = directive ? `${SESSION_SYSTEM}\n\n${directive}` : SESSION_SYSTEM;
+					const loop = await routerToolLoop({
+						routerBaseUrl: cfg.routerBaseUrl,
+						routerKey: cfg.routerKey,
+						model: cfg.model
+					}, system, `Task: ${instruction}`, [EXECUTE_TOOL], async (name, args) => {
+						if (name !== "execute_python") return `error: unknown tool ${name}`;
+						const res = await call({
+							op: "execute",
+							code: String(args.code ?? "")
+						});
+						const done = res.task_completed === true;
+						return `${String(res.output ?? "")}${done ? "\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]" : ""}`;
+					}, { maxTurns });
+					const verdict = await call({ op: "evaluate" });
+					const transcript = loop.toolTrace.slice(-3).map((t) => `CODE:\n${t.args.slice(0, 600)}\nOUTPUT:\n${t.result.slice(0, 600)}`).join("\n---\n").slice(0, 1600);
+					return {
+						...verdict,
+						turns: loop.turns,
+						input_tokens: loop.usage.input,
+						output_tokens: loop.usage.output,
+						transcript
+					};
+				});
+				if (out.input_tokens || out.output_tokens) yield {
+					type: "llm_call",
+					data: {
+						tokensIn: out.input_tokens ?? 0,
+						tokensOut: out.output_tokens ?? 0,
+						model: cfg.model
+					}
+				};
+				yield {
+					type: "result",
+					data: { finalText: JSON.stringify(out) }
+				};
+			},
+			async delete() {}
+		};
+	} };
+}
+/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */
+const reactEpisodeOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const t = (ev?.data)?.finalText;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return text;
+} };
+function createAppWorldReactAdapter() {
+	const base = createAppWorldAdapter();
+	return {
+		name: "appworld-react",
+		output: reactEpisodeOutput,
+		preflight: () => base.preflight(),
+		async loadTasks(opts = {}) {
+			return (await base.loadTasks(opts)).map((t) => {
+				const meta = readMeta(t);
+				return {
+					...t,
+					prompt: `@appworld-react ${meta.taskId} ${meta.split}\n`
+				};
+			});
+		},
+		goldArtifact: () => Promise.resolve(void 0),
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			let out;
+			try {
+				out = JSON.parse(artifact);
+			} catch {
+				throw new Error(`appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`);
+			}
+			if (typeof out.success !== "boolean" || typeof out.num_tests !== "number") throw new Error(`appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`);
+			const passes = out.passes ?? 0;
+			const total = out.num_tests;
+			const failures = Array.isArray(out.failure_names) ? out.failure_names : [];
+			return {
+				resolved: out.success === true,
+				score: total > 0 ? passes / total : 0,
+				detail: JSON.stringify({
+					taskId: meta.taskId,
+					success: out.success,
+					passes,
+					fails: out.fails ?? 0,
+					total,
+					turns: out.turns,
+					...failures.length ? { failures } : {},
+					...out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}
+				})
+			};
+		},
+		leafClient: (c) => appworldToolLoopClient(c)
+	};
+}
+//#endregion
+export { appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };
+
 //# sourceMappingURL=appworld.js.map

package/dist/benchmarks/appworld.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"appworld.js","names":[],"sources":["../../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\ninterface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  transcript?: string\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))\n      }\n      pending.push((l) => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        resolve(l)\n      })\n      child.once('exit', onExit)\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    child.stdin.end()\n    child.kill('SIGTERM')\n  }\n}\n\n/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string) {\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const loop = await routerToolLoop(\n              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },\n              system,\n              `Task: ${instruction}`,\n              [EXECUTE_TOOL],\n              async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                return `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n              },\n              { maxTurns },\n            )\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = loop.toolTrace\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            return {\n              ...verdict,\n              turns: loop.turns,\n              input_tokens: loop.usage.input,\n              output_tokens: loop.usage.output,\n              transcript,\n            } satisfies ReactResult\n          })\n          // Real usage from the episode — flat llm_call so the kernel meters it.\n          if (out.input_tokens || out.output_tokens) {\n            yield {\n              type: 'llm_call',\n              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },\n            }\n          }\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;;AAG9D,MAAM,gBAAgB;;AAQtB,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,yDAAyD;CAEpG,OAAO;AACT;;;;;;;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;CAClE,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;CAC3E,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC5G;CACA,MAAM,OAAO,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;CACjD,MAAM,SAAS,KAAK,MAAM,IAAI;CAC9B,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,0BAA0B,OAAO,OAAO;CAC1E,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAC5D,MAAM,QAAQ,KAAK,SAAS;GAO5B,MAAM,SAAQ,MANK,OAAO;IACxB;IACA;IAAW;IACX,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;IAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;GAClD,CAAC,EAAA,CACiB,SAAS,CAAC;GAC5B,IAAI,MAAM,WAAW,GACnB,MAAM,IAAI,MAAM,kDAAkD,MAAM,GAAG,KAAK,UAAU,IAAI,GAAG;GAEnG,OAAO,MAAM,KACV,OAAkB;IACjB,IAAI,EAAE;IACN;IACA,QAAQ,EAAE,cAAc;IACxB,UAAU;KAAE,QAAQ,EAAE;KAAS;IAAM;GACvC,EACF;EACF;EAEA,MAAM,eAAe,CAQrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAO,MAAM,OAAO;IAAC;IAAY;IAAa,KAAK;IAAQ;IAAW,KAAK;GAAK,GAAG,QAAQ;GAOjG,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI,SAAS;GAI3B,MAAM,QAAQ,IAAI,aAAa,SAAS;GACxC,MAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;GAG3C,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA;KACA;KACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;IACxC,CAAC;GACH;EACF;CACF;AACF;AAmCA,MAAM,eAAe;AAErB,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,MAAM,eAAyB;CAC7B,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY,EAAE,MAAM;IAAE,MAAM;IAAU,aAAa;GAA2C,EAAE;GAChG,UAAU,CAAC,MAAM;EACnB;CACF;AACF;;AAGA,eAAe,iBACb,QACA,OACA,IACY;CACZ,MAAM,QAAQ,MAAM,YAAY;EAAC;EAAQ;EAAW;EAAa;EAAQ;EAAW;CAAK,GAAG,EAC1F,KAAK,UACP,CAAC;CACD,MAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;CAClD,MAAM,UAAyC,CAAC;CAChD,MAAM,UAAoB,CAAC;CAC3B,GAAG,GAAG,SAAS,MAAM;EACnB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK,CAAC;OACX,QAAQ,KAAK,CAAC;CACrB,CAAC;CACD,IAAI,SAAS;CACb,MAAM,OAAO,GAAG,SAAS,MAAc;EACrC,UAAU,EAAE,SAAS,MAAM;CAC7B,CAAC;CACD,MAAM,YAAY,cAChB,IAAI,SAAS,SAAS,WAAW;EAC/B,MAAM,cAAc,QAAQ,MAAM;EAClC,IAAI,gBAAgB,KAAA,GAAW,OAAO,QAAQ,WAAW;EACzD,MAAM,IAAI,iBACF,uBAAO,IAAI,MAAM,oCAAoC,UAAU,cAAc,OAAO,MAAM,IAAI,GAAG,CAAC,GACxG,SACF;EAGA,MAAM,UAAU,SAA8B;GAC5C,aAAa,CAAC;GACd,uBAAO,IAAI,MAAM,4BAA4B,KAAK,aAAa,OAAO,MAAM,IAAI,GAAG,CAAC;EACtF;EACA,QAAQ,MAAM,MAAM;GAClB,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,QAAQ,CAAC;EACX,CAAC;EACD,MAAM,KAAK,QAAQ,MAAM;CAC3B,CAAC;CACH,IAAI;EACF,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;EAChD,IAAI,CAAC,MAAM,OAAO,MAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,iBAAiB;EACvG,MAAM,OAAO,OAAO,QAAmE;GACrF,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG;GAC5C,MAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;GAC9C,IAAI,OAAO,IAAI,UAAU,UAAU,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO;GAC7F,OAAO;EACT;EACA,OAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;CAC/C,UAAU;EACR,MAAM,MAAM,IAAI;EAChB,MAAM,KAAK,SAAS;CACtB;AACF;;AAGA,SAAgB,uBAAuB,KAK3B;CACV,MAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;CACzE,IAAI,MAAM;CACV,OAAO,EACL,MAAM,SAAS;EAEb,OAAO;GACL,IAAA,qBAF8B;GAG9B,OAAO,aAAa,QAAgB;IAClC,MAAM,IAAI,OAAO,MAAM,YAAY;IACnC,IAAI,CAAC,GACH,MAAM,IAAI,MACR,wFAAwF,OAAO,MAAM,GAAG,GAAG,GAC7G;IAEF,MAAM,GAAG,QAAQ,SAAS;IAC1B,MAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,CAAC,CAAC,KAAK;IACxD,MAAM,MAAM,MAAM,iBAAiB,QAAkB,OAAiB,OAAO,MAAM,gBAAgB;KACjG,MAAM,SAAS,YAAY,GAAG,eAAe,MAAM,cAAc;KACjE,MAAM,OAAO,MAAM,eACjB;MAAE,eAAe,IAAI;MAAe,WAAW,IAAI;MAAW,OAAO,IAAI;KAAM,GAC/E,QACA,SAAS,eACT,CAAC,YAAY,GACb,OAAO,MAAM,SAAS;MACpB,IAAI,SAAS,kBAAkB,OAAO,uBAAuB;MAC7D,MAAM,MAAM,MAAM,KAAK;OAAE,IAAI;OAAW,MAAM,OAAO,KAAK,QAAQ,EAAE;MAAE,CAAC;MACvE,MAAM,OAAO,IAAI,mBAAmB;MACpC,OAAO,GAAG,OAAO,IAAI,UAAU,EAAE,IAAI,OAAO,2FAA2F;KACzI,GACA,EAAE,SAAS,CACb;KACA,MAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;KAC9C,MAAM,aAAa,KAAK,UACrB,MAAM,EAAE,CAAC,CACT,KAAK,MAAM,UAAU,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,aAAa,EAAE,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC,CAChF,KAAK,SAAS,CAAC,CACf,MAAM,GAAG,IAAI;KAChB,OAAO;MACL,GAAG;MACH,OAAO,KAAK;MACZ,cAAc,KAAK,MAAM;MACzB,eAAe,KAAK,MAAM;MAC1B;KACF;IACF,CAAC;IAED,IAAI,IAAI,gBAAgB,IAAI,eAC1B,MAAM;KACJ,MAAM;KACN,MAAM;MAAE,UAAU,IAAI,gBAAgB;MAAG,WAAW,IAAI,iBAAiB;MAAG,OAAO,IAAI;KAAM;IAC/F;IAEF,MAAM;KAAE,MAAM;KAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE;IAAE;GACnE;GACA,MAAM,SAAS,CAAC;EAClB;CACF,EACF;AACF;;AAGA,MAAM,qBAA4C,EAChD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EAEvB,MAAM,KADK,IAA2C,KAAA,EACzC;EACb,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO;AACT,EACF;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,OAAO,sBAAsB;CACnC,OAAO;EACL,MAAM;EACN,QAAQ;EACR,iBAAiB,KAAK,UAAU;EAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,QAAO,MADa,KAAK,UAAU,IAAI,EAAA,CAC1B,KAAK,MAAM;IACtB,MAAM,OAAO,SAAS,CAAC;IACvB,OAAO;KACL,GAAG;KAGH,QAAQ,mBAAmB,KAAK,OAAO,GAAG,KAAK,MAAM;IACvD;GACF,CAAC;EACH;EAEA,oBAAoB,QAAQ,QAAQ,KAAA,CAAS;EAE7C,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,QAAQ;GAC3B,QAAQ;IACN,MAAM,IAAI,MACR,6EAA6E,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GACrH;GACF;GACA,IAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAC/D,MAAM,IAAI,MACR,sEAAsE,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GAC9G;GAEF,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI;GAClB,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;IACpC,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA,OAAO,IAAI,SAAS;KACpB;KACA,OAAO,IAAI;KACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;KACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;IACzE,CAAC;GACH;EACF;EAEA,aAAa,MAAM,uBAAuB,CAAC;CAC7C;AACF"}

package/dist/benchmarks/bfcl.d.ts

@@ -1,18 +1,8 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * Berkeley Function Calling Leaderboard adapter.
- *
- * Scope: deterministic function-call ground-truth categories from the official
- * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic
- * web-search/memory categories and BFCL's own model-response harness remain
- * upstream responsibilities. The adapter loads official JSONL rows plus their
- * `possible_answer` file and scores structured function-call artifacts against
- * allowed function/argument values.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/bfcl.d.ts
 declare const bfclOutput: OutputAdapter<string>;
 declare function createBfclAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { bfclOutput, createBfclAdapter };
+//# sourceMappingURL=bfcl.d.ts.map