@tangle-network/agent-bench 0.7.1 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.8.0
+
+- Consume Runtime 0.131.0, Eval 0.144.6, Interface 0.46.1, Knowledge 7.1.2, and Sandbox 0.19.4 as one compatible dependency set.
+
+## 0.7.2
+
+- Consume Runtime 0.129.0, Eval 0.144.4, Interface 0.43.1, Knowledge 7.0.11, and Sandbox 0.19.1 so benchmark model calls use the exact-profile execution boundary and the released optimizer callback contract without loading duplicate agent contracts.
+
 ## 0.7.1
 
 - Consume Runtime 0.126.0 with Eval 0.143.0 and Knowledge 7.0.8, so campaign cost remains observed, estimated, or explicitly uncaptured across the complete benchmark dependency tree.

package/HARNESS.md

@@ -34,7 +34,7 @@ probe (`bench/src/humaneval-gate.mts`) uses exactly that degenerate shape — it
 directly and does **not** route through `AgentProfile` / the sandbox / the keystone — so its numbers
 are the **no-self-correction lower bound** on the selector, distinct from the rollout-based keystone
 gate above. Bridge it to the product by running the same arms with real rollouts (an `AgentProfile`
-through `runLoop`), dialing `maxTurns`.
+through `runAgentRounds`), dialing `maxTurns`.
 
 Two things to keep straight: today's judges grade a single
 *correctness* scalar (the multi-objective vector is the open contract, architecture.md §6), and every

package/dist/adapters.js

@@ -2,9 +2,9 @@ import { createAecBenchAdapter } from "./benchmarks/aec-bench.js";
 import { createAgentBenchAdapter } from "./benchmarks/agentbench.js";
 import { createAppWorldAdapter, createAppWorldReactAdapter } from "./benchmarks/appworld.js";
 import { createBfclAdapter } from "./benchmarks/bfcl.js";
-import { t as createCadBenchAdapter } from "./cadbench-BrpwOU6A.js";
+import { t as createCadBenchAdapter } from "./cadbench-BLSyxR1N.js";
 import { createCadDesignAdapter } from "./benchmarks/cad-design.js";
-import { t as createCadGenBenchAdapter } from "./cadgenbench-DF7hYHdl.js";
+import { t as createCadGenBenchAdapter } from "./cadgenbench-x2OFkf8y.js";
 import { createCommit0Adapter } from "./benchmarks/commit0.js";
 import { createCragAdapter } from "./benchmarks/crag.js";
 import { createDabstepAdapter } from "./benchmarks/dabstep.js";

package/dist/benchmarks/appworld.d.ts

@@ -4,14 +4,73 @@ import { OutputAdapter } from "@tangle-network/agent-runtime/kernel";
 /** Worker solution code = the last fenced ```python block, else the raw text. */
 declare const appworldSolutionOutput: OutputAdapter<string>;
 declare function createAppWorldAdapter(): BenchmarkAdapter;
-/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */
+/**
+ * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is
+ * Runtime's profile-bound `router-tools` executor with one tool —
+ * `execute_python` — bound to a persistent AppWorld world session. The driver's
+ * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,
+ * evaluate → verdict); every inference turn, the metering, and the typed
+ * toolTrace the analyst steers on belong to the runtime, so runtime
+ * improvements are what this benchmark measures.
+ *
+ * The one-shot codegen adapter above plays a strictly harder game (no execution
+ * feedback — the first wrong API call kills the whole program at judge time),
+ * which flatlines the score against ANY steering; this mode is what the
+ * benchmark's published baselines use, where behavior can move sub-tests.
+ *
+ * Protocol: the round task string is `@appworld-react <taskId> <split>` on
+ * line 1; everything after line 1 is the steer (an analyst correction, a push
+ * directive) appended to the system prompt — so the existing arms steer this
+ * worker without modification. The artifact is the episode evaluation JSON
+ * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.
+ */
+interface ReactResult {
+  success?: boolean;
+  passes?: number;
+  fails?: number;
+  num_tests?: number;
+  failure_names?: string[];
+  turns?: number;
+  input_tokens?: number;
+  output_tokens?: number;
+  cost_usd?: number;
+  transcript?: string;
+}
+interface ReactRuntimeUsage {
+  input: number;
+  output: number;
+  costUsd?: number;
+  tokensKnown?: boolean;
+  usdKnown?: boolean;
+}
+/** Preserve a completed scientific/task result even when one accounting dimension is incomplete.
+ * Unknown usage fields stay absent; later comparison/reporting can refuse a cost claim without
+ * discarding the episode's task evidence. */
+declare function appworldReactResultWithUsage(verdict: ReactResult, usage: ReactRuntimeUsage, turns: number | undefined, transcript: string): ReactResult;
+/** Emit only usage the Runtime actually knows. Catalog estimates never become observed dollars. */
+declare function appworldReactUsageEvent(result: ReactResult, model: string): {
+  type: 'llm_call';
+  data: Record<string, unknown>;
+} | undefined;
+/** One persistent world session: line-JSONL request/response over the driver. */
+declare function withWorldSession<T>(taskId: string, split: string, signal: AbortSignal, fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>): Promise<T>;
+type AppWorldWorldSession = typeof withWorldSession;
+type AppWorldComplete = (body: Record<string, unknown>, request?: {
+  readonly headers: Readonly<Record<string, string>>;
+  readonly signal?: AbortSignal;
+}) => Promise<unknown>;
+/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */
 declare function appworldToolLoopClient(cfg: {
   model: string;
   routerBaseUrl: string;
   routerKey: string;
   maxTurns?: number;
+  /** Offline-test seam; production always uses the Python AppWorld session above. */
+  runWorldSession?: AppWorldWorldSession;
+  /** Offline-test seam; production uses Runtime's Router HTTP transport. */
+  complete?: AppWorldComplete;
 }): unknown;
 declare function createAppWorldReactAdapter(): BenchmarkAdapter;
 //#endregion
-export { appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };
+export { ReactResult, appworldReactResultWithUsage, appworldReactUsageEvent, appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };
 //# sourceMappingURL=appworld.d.ts.map

package/dist/benchmarks/appworld.js

@@ -2,7 +2,7 @@ import { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from
 import { join } from "node:path";
 import { spawn } from "node:child_process";
 import { createInterface } from "node:readline";
-import { routerToolLoop } from "@tangle-network/agent-runtime/kernel";
+import { collectAgentTurn, createExecutor, streamAgentTurn } from "@tangle-network/agent-runtime/kernel";
 //#region src/benchmarks/appworld.ts
 /**
 * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's
@@ -128,6 +128,38 @@ function createAppWorldAdapter() {
 		}
 	};
 }
+/** Preserve a completed scientific/task result even when one accounting dimension is incomplete.
+* Unknown usage fields stay absent; later comparison/reporting can refuse a cost claim without
+* discarding the episode's task evidence. */
+function appworldReactResultWithUsage(verdict, usage, turns, transcript) {
+	return {
+		...verdict,
+		...turns !== void 0 ? { turns } : {},
+		...usage.tokensKnown === false ? {} : {
+			input_tokens: usage.input,
+			output_tokens: usage.output
+		},
+		...usage.usdKnown === false || usage.costUsd === void 0 ? {} : { cost_usd: usage.costUsd },
+		transcript
+	};
+}
+/** Emit only usage the Runtime actually knows. Catalog estimates never become observed dollars. */
+function appworldReactUsageEvent(result, model) {
+	const hasTokens = typeof result.input_tokens === "number" && typeof result.output_tokens === "number";
+	const hasCost = typeof result.cost_usd === "number";
+	if (!hasTokens && !hasCost) return void 0;
+	return {
+		type: "llm_call",
+		data: {
+			model,
+			...hasTokens ? {
+				tokensIn: result.input_tokens,
+				tokensOut: result.output_tokens
+			} : {},
+			...hasCost ? { costUsd: result.cost_usd } : {}
+		}
+	};
+}
 const REACT_HEADER = /^@appworld-react (\S+) (\S+)\n?/;
 const SESSION_SYSTEM = [
 	"You are completing a task in AppWorld, a simulated multi-app environment.",
@@ -153,7 +185,8 @@ const EXECUTE_TOOL = {
 	}
 };
 /** One persistent world session: line-JSONL request/response over the driver. */
-async function withWorldSession(taskId, split, fn) {
+async function withWorldSession(taskId, split, signal, fn) {
+	signal.throwIfAborted();
 	const child = spawn(venvPython, [
 		DRIVER,
 		"session",
@@ -162,6 +195,10 @@ async function withWorldSession(taskId, split, fn) {
 		"--split",
 		split
 	], { cwd: benchRoot });
+	const stopChild = () => {
+		if (!child.killed) child.kill("SIGTERM");
+	};
+	signal.addEventListener("abort", stopChild, { once: true });
 	const rl = createInterface({ input: child.stdout });
 	const pending = [];
 	const backlog = [];
@@ -175,19 +212,34 @@ async function withWorldSession(taskId, split, fn) {
 		stderr += c.toString("utf8");
 	});
 	const nextLine = (timeoutMs) => new Promise((resolve, reject) => {
+		if (signal.aborted) {
+			reject(signal.reason);
+			return;
+		}
 		const fromBacklog = backlog.shift();
 		if (fromBacklog !== void 0) return resolve(fromBacklog);
 		const t = setTimeout(() => reject(/* @__PURE__ */ new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)), timeoutMs);
 		const onExit = (code) => {
 			clearTimeout(t);
-			reject(/* @__PURE__ */ new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`));
+			signal.removeEventListener("abort", onAbort);
+			reject(signal.aborted ? signal.reason : /* @__PURE__ */ new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`));
+		};
+		const onAbort = () => {
+			clearTimeout(t);
+			child.removeListener("exit", onExit);
+			const index = pending.indexOf(onLine);
+			if (index >= 0) pending.splice(index, 1);
+			reject(signal.reason);
 		};
-		pending.push((l) => {
+		const onLine = (line) => {
 			clearTimeout(t);
 			child.removeListener("exit", onExit);
-			resolve(l);
-		});
+			signal.removeEventListener("abort", onAbort);
+			resolve(line);
+		};
+		pending.push(onLine);
 		child.once("exit", onExit);
+		signal.addEventListener("abort", onAbort, { once: true });
 	});
 	try {
 		const ready = JSON.parse(await nextLine(12e4));
@@ -200,55 +252,74 @@ async function withWorldSession(taskId, split, fn) {
 		};
 		return await fn(call, ready.instruction ?? "");
 	} finally {
+		signal.removeEventListener("abort", stopChild);
 		child.stdin.end();
-		child.kill("SIGTERM");
+		stopChild();
 	}
 }
-/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */
+/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */
 function appworldToolLoopClient(cfg) {
 	const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40);
+	const runWorldSession = cfg.runWorldSession ?? withWorldSession;
 	let seq = 0;
 	return { async create() {
 		return {
 			id: `appworld-toolloop-${seq++}`,
-			async *streamPrompt(prompt) {
+			async *streamPrompt(prompt, promptOpts) {
+				const signal = promptOpts?.signal ?? new AbortController().signal;
+				signal.throwIfAborted();
 				const m = prompt.match(REACT_HEADER);
 				if (!m) throw new Error(`appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`);
 				const [, taskId, split] = m;
 				const directive = prompt.replace(REACT_HEADER, "").trim();
-				const out = await withWorldSession(taskId, split, async (call, instruction) => {
+				const out = await runWorldSession(taskId, split, signal, async (call, instruction) => {
 					const system = directive ? `${SESSION_SYSTEM}\n\n${directive}` : SESSION_SYSTEM;
-					const loop = await routerToolLoop({
-						routerBaseUrl: cfg.routerBaseUrl,
-						routerKey: cfg.routerKey,
-						model: cfg.model
-					}, system, `Task: ${instruction}`, [EXECUTE_TOOL], async (name, args) => {
-						if (name !== "execute_python") return `error: unknown tool ${name}`;
-						const res = await call({
-							op: "execute",
-							code: String(args.code ?? "")
-						});
-						const done = res.task_completed === true;
-						return `${String(res.output ?? "")}${done ? "\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]" : ""}`;
-					}, { maxTurns });
-					const verdict = await call({ op: "evaluate" });
-					const transcript = loop.toolTrace.slice(-3).map((t) => `CODE:\n${t.args.slice(0, 600)}\nOUTPUT:\n${t.result.slice(0, 600)}`).join("\n---\n").slice(0, 1600);
-					return {
-						...verdict,
-						turns: loop.turns,
-						input_tokens: loop.usage.input,
-						output_tokens: loop.usage.output,
-						transcript
+					const transcriptSteps = [];
+					const profile = {
+						name: "appworld-react-worker",
+						harness: "cli-base",
+						model: {
+							provider: "tangle-router",
+							default: cfg.model,
+							metadata: { maxTurns }
+						},
+						prompt: { systemPrompt: system },
+						tools: { execute_python: true }
 					};
+					const loop = await collectAgentTurn(streamAgentTurn({
+						kind: "executor",
+						factory: createExecutor({
+							backend: "router-tools",
+							routerBaseUrl: cfg.routerBaseUrl,
+							routerKey: cfg.routerKey,
+							...cfg.complete ? { complete: cfg.complete } : {},
+							tools: [EXECUTE_TOOL],
+							executeToolCall: async (name, args) => {
+								if (name !== "execute_python") return `error: unknown tool ${name}`;
+								const res = await call({
+									op: "execute",
+									code: String(args.code ?? "")
+								});
+								const done = res.task_completed === true;
+								const result = `${String(res.output ?? "")}${done ? "\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]" : ""}`;
+								transcriptSteps.push({
+									args: JSON.stringify(args),
+									result
+								});
+								return result;
+							}
+						}),
+						profile
+					}, `Task: ${instruction}`, { signal }));
+					if (loop.status !== "completed") throw new Error(loop.error?.message ?? `AppWorld turn ended with ${loop.status}`);
+					const verdict = await call({ op: "evaluate" });
+					const transcript = transcriptSteps.slice(-3).map((t) => `CODE:\n${t.args.slice(0, 600)}\nOUTPUT:\n${t.result.slice(0, 600)}`).join("\n---\n").slice(0, 1600);
+					const finalEvent = loop.events.at(-1);
+					const resultMetadata = finalEvent?.type === "final" && finalEvent.metadata?.result ? finalEvent.metadata.result : void 0;
+					return appworldReactResultWithUsage(verdict, loop.usage, resultMetadata?.spent?.iterations, transcript);
 				});
-				if (out.input_tokens || out.output_tokens) yield {
-					type: "llm_call",
-					data: {
-						tokensIn: out.input_tokens ?? 0,
-						tokensOut: out.output_tokens ?? 0,
-						model: cfg.model
-					}
-				};
+				const usageEvent = appworldReactUsageEvent(out, cfg.model);
+				if (usageEvent) yield usageEvent;
 				yield {
 					type: "result",
 					data: { finalText: JSON.stringify(out) }
@@ -314,6 +385,6 @@ function createAppWorldReactAdapter() {
 	};
 }
 //#endregion
-export { appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };
+export { appworldReactResultWithUsage, appworldReactUsageEvent, appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };
 
 //# sourceMappingURL=appworld.js.map

package/dist/benchmarks/appworld.js.map

@@ -1 +1 @@
-{"version":3,"file":"appworld.js","names":[],"sources":["../../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * `routerToolLoop` (the runtime's off-box agentic tool loop) with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\ninterface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  transcript?: string\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        reject(new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`))\n      }\n      pending.push((l) => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        resolve(l)\n      })\n      child.once('exit', onExit)\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    child.stdin.end()\n    child.kill('SIGTERM')\n  }\n}\n\n/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string) {\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const loop = await routerToolLoop(\n              { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model },\n              system,\n              `Task: ${instruction}`,\n              [EXECUTE_TOOL],\n              async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                return `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n              },\n              { maxTurns },\n            )\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = loop.toolTrace\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            return {\n              ...verdict,\n              turns: loop.turns,\n              input_tokens: loop.usage.input,\n              output_tokens: loop.usage.output,\n              transcript,\n            } satisfies ReactResult\n          })\n          // Real usage from the episode — flat llm_call so the kernel meters it.\n          if (out.input_tokens || out.output_tokens) {\n            yield {\n              type: 'llm_call',\n              data: { tokensIn: out.input_tokens ?? 0, tokensOut: out.output_tokens ?? 0, model: cfg.model },\n            }\n          }\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;;AAG9D,MAAM,gBAAgB;;AAQtB,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,yDAAyD;CAEpG,OAAO;AACT;;;;;;;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;CAClE,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;CAC3E,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC5G;CACA,MAAM,OAAO,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;CACjD,MAAM,SAAS,KAAK,MAAM,IAAI;CAC9B,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,0BAA0B,OAAO,OAAO;CAC1E,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAC5D,MAAM,QAAQ,KAAK,SAAS;GAO5B,MAAM,SAAQ,MANK,OAAO;IACxB;IACA;IAAW;IACX,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;IAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;GAClD,CAAC,EAAA,CACiB,SAAS,CAAC;GAC5B,IAAI,MAAM,WAAW,GACnB,MAAM,IAAI,MAAM,kDAAkD,MAAM,GAAG,KAAK,UAAU,IAAI,GAAG;GAEnG,OAAO,MAAM,KACV,OAAkB;IACjB,IAAI,EAAE;IACN;IACA,QAAQ,EAAE,cAAc;IACxB,UAAU;KAAE,QAAQ,EAAE;KAAS;IAAM;GACvC,EACF;EACF;EAEA,MAAM,eAAe,CAQrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAO,MAAM,OAAO;IAAC;IAAY;IAAa,KAAK;IAAQ;IAAW,KAAK;GAAK,GAAG,QAAQ;GAOjG,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI,SAAS;GAI3B,MAAM,QAAQ,IAAI,aAAa,SAAS;GACxC,MAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;GAG3C,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA;KACA;KACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;IACxC,CAAC;GACH;EACF;CACF;AACF;AAmCA,MAAM,eAAe;AAErB,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,MAAM,eAAyB;CAC7B,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY,EAAE,MAAM;IAAE,MAAM;IAAU,aAAa;GAA2C,EAAE;GAChG,UAAU,CAAC,MAAM;EACnB;CACF;AACF;;AAGA,eAAe,iBACb,QACA,OACA,IACY;CACZ,MAAM,QAAQ,MAAM,YAAY;EAAC;EAAQ;EAAW;EAAa;EAAQ;EAAW;CAAK,GAAG,EAC1F,KAAK,UACP,CAAC;CACD,MAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;CAClD,MAAM,UAAyC,CAAC;CAChD,MAAM,UAAoB,CAAC;CAC3B,GAAG,GAAG,SAAS,MAAM;EACnB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK,CAAC;OACX,QAAQ,KAAK,CAAC;CACrB,CAAC;CACD,IAAI,SAAS;CACb,MAAM,OAAO,GAAG,SAAS,MAAc;EACrC,UAAU,EAAE,SAAS,MAAM;CAC7B,CAAC;CACD,MAAM,YAAY,cAChB,IAAI,SAAS,SAAS,WAAW;EAC/B,MAAM,cAAc,QAAQ,MAAM;EAClC,IAAI,gBAAgB,KAAA,GAAW,OAAO,QAAQ,WAAW;EACzD,MAAM,IAAI,iBACF,uBAAO,IAAI,MAAM,oCAAoC,UAAU,cAAc,OAAO,MAAM,IAAI,GAAG,CAAC,GACxG,SACF;EAGA,MAAM,UAAU,SAA8B;GAC5C,aAAa,CAAC;GACd,uBAAO,IAAI,MAAM,4BAA4B,KAAK,aAAa,OAAO,MAAM,IAAI,GAAG,CAAC;EACtF;EACA,QAAQ,MAAM,MAAM;GAClB,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,QAAQ,CAAC;EACX,CAAC;EACD,MAAM,KAAK,QAAQ,MAAM;CAC3B,CAAC;CACH,IAAI;EACF,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;EAChD,IAAI,CAAC,MAAM,OAAO,MAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,iBAAiB;EACvG,MAAM,OAAO,OAAO,QAAmE;GACrF,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG;GAC5C,MAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;GAC9C,IAAI,OAAO,IAAI,UAAU,UAAU,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO;GAC7F,OAAO;EACT;EACA,OAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;CAC/C,UAAU;EACR,MAAM,MAAM,IAAI;EAChB,MAAM,KAAK,SAAS;CACtB;AACF;;AAGA,SAAgB,uBAAuB,KAK3B;CACV,MAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;CACzE,IAAI,MAAM;CACV,OAAO,EACL,MAAM,SAAS;EAEb,OAAO;GACL,IAAA,qBAF8B;GAG9B,OAAO,aAAa,QAAgB;IAClC,MAAM,IAAI,OAAO,MAAM,YAAY;IACnC,IAAI,CAAC,GACH,MAAM,IAAI,MACR,wFAAwF,OAAO,MAAM,GAAG,GAAG,GAC7G;IAEF,MAAM,GAAG,QAAQ,SAAS;IAC1B,MAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,CAAC,CAAC,KAAK;IACxD,MAAM,MAAM,MAAM,iBAAiB,QAAkB,OAAiB,OAAO,MAAM,gBAAgB;KACjG,MAAM,SAAS,YAAY,GAAG,eAAe,MAAM,cAAc;KACjE,MAAM,OAAO,MAAM,eACjB;MAAE,eAAe,IAAI;MAAe,WAAW,IAAI;MAAW,OAAO,IAAI;KAAM,GAC/E,QACA,SAAS,eACT,CAAC,YAAY,GACb,OAAO,MAAM,SAAS;MACpB,IAAI,SAAS,kBAAkB,OAAO,uBAAuB;MAC7D,MAAM,MAAM,MAAM,KAAK;OAAE,IAAI;OAAW,MAAM,OAAO,KAAK,QAAQ,EAAE;MAAE,CAAC;MACvE,MAAM,OAAO,IAAI,mBAAmB;MACpC,OAAO,GAAG,OAAO,IAAI,UAAU,EAAE,IAAI,OAAO,2FAA2F;KACzI,GACA,EAAE,SAAS,CACb;KACA,MAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;KAC9C,MAAM,aAAa,KAAK,UACrB,MAAM,EAAE,CAAC,CACT,KAAK,MAAM,UAAU,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,aAAa,EAAE,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC,CAChF,KAAK,SAAS,CAAC,CACf,MAAM,GAAG,IAAI;KAChB,OAAO;MACL,GAAG;MACH,OAAO,KAAK;MACZ,cAAc,KAAK,MAAM;MACzB,eAAe,KAAK,MAAM;MAC1B;KACF;IACF,CAAC;IAED,IAAI,IAAI,gBAAgB,IAAI,eAC1B,MAAM;KACJ,MAAM;KACN,MAAM;MAAE,UAAU,IAAI,gBAAgB;MAAG,WAAW,IAAI,iBAAiB;MAAG,OAAO,IAAI;KAAM;IAC/F;IAEF,MAAM;KAAE,MAAM;KAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE;IAAE;GACnE;GACA,MAAM,SAAS,CAAC;EAClB;CACF,EACF;AACF;;AAGA,MAAM,qBAA4C,EAChD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EAEvB,MAAM,KADK,IAA2C,KAAA,EACzC;EACb,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO;AACT,EACF;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,OAAO,sBAAsB;CACnC,OAAO;EACL,MAAM;EACN,QAAQ;EACR,iBAAiB,KAAK,UAAU;EAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,QAAO,MADa,KAAK,UAAU,IAAI,EAAA,CAC1B,KAAK,MAAM;IACtB,MAAM,OAAO,SAAS,CAAC;IACvB,OAAO;KACL,GAAG;KAGH,QAAQ,mBAAmB,KAAK,OAAO,GAAG,KAAK,MAAM;IACvD;GACF,CAAC;EACH;EAEA,oBAAoB,QAAQ,QAAQ,KAAA,CAAS;EAE7C,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,QAAQ;GAC3B,QAAQ;IACN,MAAM,IAAI,MACR,6EAA6E,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GACrH;GACF;GACA,IAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAC/D,MAAM,IAAI,MACR,sEAAsE,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GAC9G;GAEF,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI;GAClB,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;IACpC,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA,OAAO,IAAI,SAAS;KACpB;KACA,OAAO,IAAI;KACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;KACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;IACzE,CAAC;GACH;EACF;EAEA,aAAa,MAAM,uBAAuB,CAAC;CAC7C;AACF"}
+{"version":3,"file":"appworld.js","names":[],"sources":["../../src/benchmarks/appworld.ts"],"sourcesContent":["/**\n * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's\n * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`\n * surface AppWorld exposes inside `world.execute(...)`), ending in\n * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic\n * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,\n * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`\n * (per-requirement total) and the `passes`/`failures` lists. Score =\n * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.\n *\n * loadTasks enumerates the real task suite via `load_task_ids(split)`\n * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.\n * The OutputAdapter is stream-only, so the worker emits its solution as a fenced\n * ```python block which the driver executes.\n *\n * Requires for a live run: the bench `.venv` with `appworld` installed + the\n * unpacked engine + downloaded data (`appworld install` ; `appworld download\n * data`). preflight + loadTasks + judge all fail loud with the exact step when the\n * engine/data is absent — never a fabricated score.\n */\n\nimport { spawn } from 'node:child_process'\nimport { join } from 'node:path'\nimport { createInterface } from 'node:readline'\nimport {\n  collectAgentTurn,\n  createExecutor,\n  type OutputAdapter,\n  streamAgentTurn,\n  type ToolSpec,\n} from '@tangle-network/agent-runtime/kernel'\nimport { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst DRIVER = join(benchRoot, 'scripts', 'appworld_driver.py')\n\n/** AppWorld splits; only the test splits ship evaluation-only (no setup/solution). */\nconst DEFAULT_SPLIT = 'test_normal'\n\ninterface AppWorldMeta {\n  taskId: string\n  split: string\n}\n\n/** Worker solution code = the last fenced ```python block, else the raw text. */\nexport const appworldSolutionOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:python|py)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nconst WORKER_CONTRACT = [\n  '',\n  'Solve this by writing Python that calls the available app APIs (the `apis.<app>.<function>(...)` surface). You may inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials, perform every step the task requires, and FINISH with `apis.supervisor.complete_task()`.',\n  'Emit your COMPLETE solution as the LAST thing in your reply, in a single fenced ```python block. Nothing after the closing fence.',\n].join('\\n')\n\nfunction readMeta(task: BenchTask): AppWorldMeta {\n  const md = task.metadata\n  if (!md || typeof md.taskId !== 'string') {\n    throw new Error(`appworld task ${task.id} missing metadata.taskId — loadTasks did not populate it`)\n  }\n  return md as unknown as AppWorldMeta\n}\n\n/**\n * Run the appworld engine driver with a subcommand; JSON on the LAST stdout line.\n * The solution code (evaluate) is piped to stdin via the shared stdin-aware runner —\n * execFile's `input` option is not honored async and hangs the driver's\n * sys.stdin.read() forever. `load` ignores stdin, so an empty pipe is harmless.\n */\nasync function driver(args: string[], input = ''): Promise<unknown> {\n  let stdout: string\n  try {\n    stdout = await runVenvScriptStdin(DRIVER, args, input, { cwd: benchRoot })\n  } catch (err) {\n    const e = err as { message?: string }\n    throw new Error(`appworld driver failed (${args.join(' ')}): ${(e.message || String(err)).slice(0, 1500)}`)\n  }\n  const last = stdout.trim().split('\\n').at(-1) ?? '{}'\n  const parsed = JSON.parse(last) as { error?: string }\n  if (parsed.error) throw new Error(`appworld driver error: ${parsed.error}`)\n  return parsed\n}\n\nexport function createAppWorldAdapter(): BenchmarkAdapter {\n  return {\n    name: 'appworld',\n    output: appworldSolutionOutput,\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['appworld'],\n        requireDocker: false,\n        fix:\n          'Fix: bench/.venv/bin/pip install appworld ; ' +\n          'bench/.venv/bin/appworld install ; bench/.venv/bin/appworld download data ' +\n          '(unpacks the engine + downloads the simulated-app data/tasks). ' +\n          'Set APPWORLD_ROOT to the data root if not the default.',\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const split = opts.split ?? DEFAULT_SPLIT\n      const out = (await driver([\n        'load',\n        '--split', split,\n        ...(opts.limit !== undefined ? ['--limit', String(opts.limit)] : []),\n        ...(opts.ids ? ['--ids', opts.ids.join(',')] : []),\n      ])) as { tasks?: Array<{ task_id: string; instruction: string }> }\n      const tasks = out.tasks ?? []\n      if (tasks.length === 0) {\n        throw new Error(`appworld loadTasks returned no tasks for split=${split} ${JSON.stringify(opts)}`)\n      }\n      return tasks.map(\n        (t): BenchTask => ({\n          id: t.task_id,\n          split,\n          prompt: t.instruction + WORKER_CONTRACT,\n          metadata: { taskId: t.task_id, split } as unknown as Record<string, unknown>,\n        }),\n      )\n    },\n\n    async goldArtifact() {\n      // Reference solution code ships only for train/dev, and only inside the\n      // engine's decrypted `.bundle` (it is not a portable string this adapter can\n      // emit across splits). The test splits are evaluation-only. So verify-judge\n      // here requires a real solve on a train/dev task through the live engine\n      // rather than a synthetic gold — returning a fabricated artifact would be a\n      // fake. Returns undefined.\n      return undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const out = (await driver(['evaluate', '--task-id', meta.taskId, '--split', meta.split], artifact)) as {\n        success?: boolean\n        passes?: number\n        fails?: number\n        num_tests?: number\n        failure_names?: string[]\n      }\n      const passes = out.passes ?? 0\n      const fails = out.fails ?? 0\n      // num_tests is the evaluator's authoritative per-requirement count; prefer it\n      // over passes+fails (which can disagree if a requirement neither passed nor\n      // failed). Never default the total to a phantom denominator.\n      const total = out.num_tests ?? passes + fails\n      const score = total > 0 ? passes / total : 0\n      // failure_names = WHICH sub-tests failed — the evidence a trace analyst\n      // steers on. Carried in `detail` so it reaches the verdict's `notes`.\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails,\n          total,\n          ...(failures.length ? { failures } : {}),\n        }),\n      }\n    },\n  }\n}\n\n/**\n * AppWorld in its NATIVE protocol, run by OUR runtime: the worker is\n * Runtime's profile-bound `router-tools` executor with one tool —\n * `execute_python` — bound to a persistent AppWorld world session. The driver's\n * `session` subcommand is a dumb world shim (stdin JSONL: execute → output,\n * evaluate → verdict); every inference turn, the metering, and the typed\n * toolTrace the analyst steers on belong to the runtime, so runtime\n * improvements are what this benchmark measures.\n *\n * The one-shot codegen adapter above plays a strictly harder game (no execution\n * feedback — the first wrong API call kills the whole program at judge time),\n * which flatlines the score against ANY steering; this mode is what the\n * benchmark's published baselines use, where behavior can move sub-tests.\n *\n * Protocol: the round task string is `@appworld-react <taskId> <split>` on\n * line 1; everything after line 1 is the steer (an analyst correction, a push\n * directive) appended to the system prompt — so the existing arms steer this\n * worker without modification. The artifact is the episode evaluation JSON\n * (AppWorld's evaluator ran in-world); judge() parses it, never re-executes.\n */\n\nexport interface ReactResult {\n  success?: boolean\n  passes?: number\n  fails?: number\n  num_tests?: number\n  failure_names?: string[]\n  turns?: number\n  input_tokens?: number\n  output_tokens?: number\n  cost_usd?: number\n  transcript?: string\n}\n\ninterface ReactRuntimeUsage {\n  input: number\n  output: number\n  costUsd?: number\n  tokensKnown?: boolean\n  usdKnown?: boolean\n}\n\n/** Preserve a completed scientific/task result even when one accounting dimension is incomplete.\n * Unknown usage fields stay absent; later comparison/reporting can refuse a cost claim without\n * discarding the episode's task evidence. */\nexport function appworldReactResultWithUsage(\n  verdict: ReactResult,\n  usage: ReactRuntimeUsage,\n  turns: number | undefined,\n  transcript: string,\n): ReactResult {\n  return {\n    ...verdict,\n    ...(turns !== undefined ? { turns } : {}),\n    ...(usage.tokensKnown === false\n      ? {}\n      : { input_tokens: usage.input, output_tokens: usage.output }),\n    ...(usage.usdKnown === false || usage.costUsd === undefined\n      ? {}\n      : { cost_usd: usage.costUsd }),\n    transcript,\n  }\n}\n\n/** Emit only usage the Runtime actually knows. Catalog estimates never become observed dollars. */\nexport function appworldReactUsageEvent(\n  result: ReactResult,\n  model: string,\n): { type: 'llm_call'; data: Record<string, unknown> } | undefined {\n  const hasTokens =\n    typeof result.input_tokens === 'number' && typeof result.output_tokens === 'number'\n  const hasCost = typeof result.cost_usd === 'number'\n  if (!hasTokens && !hasCost) return undefined\n  return {\n    type: 'llm_call',\n    data: {\n      model,\n      ...(hasTokens\n        ? { tokensIn: result.input_tokens, tokensOut: result.output_tokens }\n        : {}),\n      ...(hasCost ? { costUsd: result.cost_usd } : {}),\n    },\n  }\n}\n\nconst REACT_HEADER = /^@appworld-react (\\S+) (\\S+)\\n?/\n\nconst SESSION_SYSTEM = [\n  'You are completing a task in AppWorld, a simulated multi-app environment.',\n  'Use the execute_python tool to run Python that calls the app APIs (the `apis.<app>.<function>(...)` surface).',\n  'Inspect API docs with `apis.api_docs.show_api_descriptions(app_name=...)` and `apis.api_docs.show_api_doc(app_name=..., api_name=...)`.',\n  'Authenticate where needed via the supervisor-provided credentials (`apis.supervisor.show_account_passwords()`).',\n  'Work incrementally: small snippets, read each output, correct course.',\n  'When every step of the task is done, run `apis.supervisor.complete_task()` and then reply WITHOUT calling the tool again.',\n].join('\\n')\n\nconst EXECUTE_TOOL: ToolSpec = {\n  type: 'function',\n  function: {\n    name: 'execute_python',\n    description:\n      'Execute a Python snippet in the persistent AppWorld world. State persists across calls. Returns the execution output (API results or errors).',\n    parameters: {\n      type: 'object',\n      properties: { code: { type: 'string', description: 'Python code calling apis.<app>.<fn>(...)' } },\n      required: ['code'],\n    },\n  },\n}\n\n/** One persistent world session: line-JSONL request/response over the driver. */\nasync function withWorldSession<T>(\n  taskId: string,\n  split: string,\n  signal: AbortSignal,\n  fn: (call: (cmd: Record<string, unknown>) => Promise<Record<string, unknown>>, instruction: string) => Promise<T>,\n): Promise<T> {\n  signal.throwIfAborted()\n  const child = spawn(venvPython, [DRIVER, 'session', '--task-id', taskId, '--split', split], {\n    cwd: benchRoot,\n  })\n  const stopChild = (): void => {\n    if (!child.killed) child.kill('SIGTERM')\n  }\n  signal.addEventListener('abort', stopChild, { once: true })\n  const rl = createInterface({ input: child.stdout })\n  const pending: Array<(line: string) => void> = []\n  const backlog: string[] = []\n  rl.on('line', (l) => {\n    const next = pending.shift()\n    if (next) next(l)\n    else backlog.push(l)\n  })\n  let stderr = ''\n  child.stderr.on('data', (c: Buffer) => {\n    stderr += c.toString('utf8')\n  })\n  const nextLine = (timeoutMs: number): Promise<string> =>\n    new Promise((resolve, reject) => {\n      if (signal.aborted) {\n        reject(signal.reason)\n        return\n      }\n      const fromBacklog = backlog.shift()\n      if (fromBacklog !== undefined) return resolve(fromBacklog)\n      const t = setTimeout(\n        () => reject(new Error(`appworld session: no response in ${timeoutMs}ms; stderr: ${stderr.slice(-400)}`)),\n        timeoutMs,\n      )\n      // One exit listener per await leaks (25-turn episodes blow the listener\n      // cap) — remove it on the resolve path.\n      const onExit = (code: number | null): void => {\n        clearTimeout(t)\n        signal.removeEventListener('abort', onAbort)\n        reject(\n          signal.aborted\n            ? signal.reason\n            : new Error(`appworld session exited (${code}); stderr: ${stderr.slice(-400)}`),\n        )\n      }\n      const onAbort = (): void => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        const index = pending.indexOf(onLine)\n        if (index >= 0) pending.splice(index, 1)\n        reject(signal.reason)\n      }\n      const onLine = (line: string): void => {\n        clearTimeout(t)\n        child.removeListener('exit', onExit)\n        signal.removeEventListener('abort', onAbort)\n        resolve(line)\n      }\n      pending.push(onLine)\n      child.once('exit', onExit)\n      signal.addEventListener('abort', onAbort, { once: true })\n    })\n  try {\n    const ready = JSON.parse(await nextLine(120_000)) as { ready?: boolean; instruction?: string; error?: string }\n    if (!ready.ready) throw new Error(`appworld session failed to start: ${ready.error ?? 'no ready line'}`)\n    const call = async (cmd: Record<string, unknown>): Promise<Record<string, unknown>> => {\n      child.stdin.write(`${JSON.stringify(cmd)}\\n`)\n      const res = JSON.parse(await nextLine(180_000)) as Record<string, unknown>\n      if (typeof res.error === 'string') throw new Error(`appworld session op failed: ${res.error}`)\n      return res\n    }\n    return await fn(call, ready.instruction ?? '')\n  } finally {\n    signal.removeEventListener('abort', stopChild)\n    child.stdin.end()\n    stopChild()\n  }\n}\n\ntype AppWorldWorldSession = typeof withWorldSession\ntype AppWorldComplete = (\n  body: Record<string, unknown>,\n  request?: {\n    readonly headers: Readonly<Record<string, string>>\n    readonly signal?: AbortSignal\n  },\n) => Promise<unknown>\n\n/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */\nexport function appworldToolLoopClient(cfg: {\n  model: string\n  routerBaseUrl: string\n  routerKey: string\n  maxTurns?: number\n  /** Offline-test seam; production always uses the Python AppWorld session above. */\n  runWorldSession?: AppWorldWorldSession\n  /** Offline-test seam; production uses Runtime's Router HTTP transport. */\n  complete?: AppWorldComplete\n}): unknown {\n  const maxTurns = cfg.maxTurns ?? Number(process.env.REACT_MAX_TURNS ?? 40)\n  const runWorldSession = cfg.runWorldSession ?? withWorldSession\n  let seq = 0\n  return {\n    async create() {\n      const id = `appworld-toolloop-${seq++}`\n      return {\n        id,\n        async *streamPrompt(prompt: string, promptOpts?: { signal?: AbortSignal }) {\n          const signal = promptOpts?.signal ?? new AbortController().signal\n          signal.throwIfAborted()\n          const m = prompt.match(REACT_HEADER)\n          if (!m) {\n            throw new Error(\n              `appworld-react leaf: prompt missing '@appworld-react <taskId> <split>' header — got: ${prompt.slice(0, 120)}`,\n            )\n          }\n          const [, taskId, split] = m\n          const directive = prompt.replace(REACT_HEADER, '').trim()\n          const out = await runWorldSession(taskId as string, split as string, signal, async (call, instruction) => {\n            const system = directive ? `${SESSION_SYSTEM}\\n\\n${directive}` : SESSION_SYSTEM\n            const transcriptSteps: Array<{ args: string; result: string }> = []\n            const profile = {\n              name: 'appworld-react-worker',\n              harness: 'cli-base' as const,\n              model: {\n                provider: 'tangle-router',\n                default: cfg.model,\n                metadata: { maxTurns },\n              },\n              prompt: { systemPrompt: system },\n              tools: { execute_python: true },\n            }\n            const factory = createExecutor({\n              backend: 'router-tools',\n              routerBaseUrl: cfg.routerBaseUrl,\n              routerKey: cfg.routerKey,\n              ...(cfg.complete ? { complete: cfg.complete } : {}),\n              tools: [EXECUTE_TOOL],\n              executeToolCall: async (name, args) => {\n                if (name !== 'execute_python') return `error: unknown tool ${name}`\n                const res = await call({ op: 'execute', code: String(args.code ?? '') })\n                const done = res.task_completed === true\n                const result = `${String(res.output ?? '')}${done ? '\\n\\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}`\n                transcriptSteps.push({ args: JSON.stringify(args), result })\n                return result\n              },\n            })\n            const loop = await collectAgentTurn(\n              streamAgentTurn(\n                { kind: 'executor', factory, profile },\n                `Task: ${instruction}`,\n                { signal },\n              ),\n            )\n            if (loop.status !== 'completed') {\n              throw new Error(loop.error?.message ?? `AppWorld turn ended with ${loop.status}`)\n            }\n            const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult\n            const transcript = transcriptSteps\n              .slice(-3)\n              .map((t) => `CODE:\\n${t.args.slice(0, 600)}\\nOUTPUT:\\n${t.result.slice(0, 600)}`)\n              .join('\\n---\\n')\n              .slice(0, 1600)\n            const finalEvent = loop.events.at(-1)\n            const resultMetadata =\n              finalEvent?.type === 'final' && finalEvent.metadata?.result\n                ? (finalEvent.metadata.result as { spent?: { iterations?: number } })\n                : undefined\n            return appworldReactResultWithUsage(\n              verdict,\n              loop.usage,\n              resultMetadata?.spent?.iterations,\n              transcript,\n            )\n          })\n          const usageEvent = appworldReactUsageEvent(out, cfg.model)\n          if (usageEvent) yield usageEvent\n          yield { type: 'result', data: { finalText: JSON.stringify(out) } }\n        },\n        async delete() {},\n      }\n    },\n  }\n}\n\n/** Artifact = the episode's evaluation JSON, verbatim (no fence extraction). */\nconst reactEpisodeOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    return text\n  },\n}\n\nexport function createAppWorldReactAdapter(): BenchmarkAdapter {\n  const base = createAppWorldAdapter()\n  return {\n    name: 'appworld-react',\n    output: reactEpisodeOutput,\n    preflight: () => base.preflight(),\n\n    async loadTasks(opts: LoadOptions = {}): Promise<BenchTask[]> {\n      const tasks = await base.loadTasks(opts)\n      return tasks.map((t) => {\n        const meta = readMeta(t)\n        return {\n          ...t,\n          // Header carries task identity to the leaf; the body (empty at round 0)\n          // is the directive slot the arms append their steer into.\n          prompt: `@appworld-react ${meta.taskId} ${meta.split}\\n`,\n        }\n      })\n    },\n\n    goldArtifact: () => Promise.resolve(undefined),\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      let out: ReactResult\n      try {\n        out = JSON.parse(artifact) as ReactResult\n      } catch {\n        throw new Error(\n          `appworld-react judge: artifact is not the episode's evaluation JSON (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      if (typeof out.success !== 'boolean' || typeof out.num_tests !== 'number') {\n        throw new Error(\n          `appworld-react judge: episode JSON missing success/num_tests (task ${meta.taskId}): ${artifact.slice(0, 200)}`,\n        )\n      }\n      const passes = out.passes ?? 0\n      const total = out.num_tests\n      const failures = Array.isArray(out.failure_names) ? out.failure_names : []\n      return {\n        resolved: out.success === true,\n        score: total > 0 ? passes / total : 0,\n        detail: JSON.stringify({\n          taskId: meta.taskId,\n          success: out.success,\n          passes,\n          fails: out.fails ?? 0,\n          total,\n          turns: out.turns,\n          ...(failures.length ? { failures } : {}),\n          ...(out.transcript ? { transcriptTail: out.transcript.slice(-800) } : {}),\n        }),\n      }\n    },\n\n    leafClient: (c) => appworldToolLoopClient(c),\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,SAAS,KAAK,WAAW,WAAW,oBAAoB;;AAG9D,MAAM,gBAAgB;;AAQtB,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,sCAAsC,CAC1D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,SAAS,SAAS,MAA+B;CAC/C,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,WAAW,UAC9B,MAAM,IAAI,MAAM,iBAAiB,KAAK,GAAG,yDAAyD;CAEpG,OAAO;AACT;;;;;;;AAQA,eAAe,OAAO,MAAgB,QAAQ,IAAsB;CAClE,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,mBAAmB,QAAQ,MAAM,OAAO,EAAE,KAAK,UAAU,CAAC;CAC3E,SAAS,KAAK;EACZ,MAAM,IAAI;EACV,MAAM,IAAI,MAAM,2BAA2B,KAAK,KAAK,GAAG,EAAE,MAAM,EAAE,WAAW,OAAO,GAAG,EAAA,CAAG,MAAM,GAAG,IAAI,GAAG;CAC5G;CACA,MAAM,OAAO,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK;CACjD,MAAM,SAAS,KAAK,MAAM,IAAI;CAC9B,IAAI,OAAO,OAAO,MAAM,IAAI,MAAM,0BAA0B,OAAO,OAAO;CAC1E,OAAO;AACT;AAEA,SAAgB,wBAA0C;CACxD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAIJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAC5D,MAAM,QAAQ,KAAK,SAAS;GAO5B,MAAM,SAAQ,MANK,OAAO;IACxB;IACA;IAAW;IACX,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,WAAW,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC;IAClE,GAAI,KAAK,MAAM,CAAC,SAAS,KAAK,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC;GAClD,CAAC,EAAA,CACiB,SAAS,CAAC;GAC5B,IAAI,MAAM,WAAW,GACnB,MAAM,IAAI,MAAM,kDAAkD,MAAM,GAAG,KAAK,UAAU,IAAI,GAAG;GAEnG,OAAO,MAAM,KACV,OAAkB;IACjB,IAAI,EAAE;IACN;IACA,QAAQ,EAAE,cAAc;IACxB,UAAU;KAAE,QAAQ,EAAE;KAAS;IAAM;GACvC,EACF;EACF;EAEA,MAAM,eAAe,CAQrB;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAO,MAAM,OAAO;IAAC;IAAY;IAAa,KAAK;IAAQ;IAAW,KAAK;GAAK,GAAG,QAAQ;GAOjG,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI,SAAS;GAI3B,MAAM,QAAQ,IAAI,aAAa,SAAS;GACxC,MAAM,QAAQ,QAAQ,IAAI,SAAS,QAAQ;GAG3C,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B;IACA,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA;KACA;KACA,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;IACxC,CAAC;GACH;EACF;CACF;AACF;;;;AA+CA,SAAgB,6BACd,SACA,OACA,OACA,YACa;CACb,OAAO;EACL,GAAG;EACH,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;EACvC,GAAI,MAAM,gBAAgB,QACtB,CAAC,IACD;GAAE,cAAc,MAAM;GAAO,eAAe,MAAM;EAAO;EAC7D,GAAI,MAAM,aAAa,SAAS,MAAM,YAAY,KAAA,IAC9C,CAAC,IACD,EAAE,UAAU,MAAM,QAAQ;EAC9B;CACF;AACF;;AAGA,SAAgB,wBACd,QACA,OACiE;CACjE,MAAM,YACJ,OAAO,OAAO,iBAAiB,YAAY,OAAO,OAAO,kBAAkB;CAC7E,MAAM,UAAU,OAAO,OAAO,aAAa;CAC3C,IAAI,CAAC,aAAa,CAAC,SAAS,OAAO,KAAA;CACnC,OAAO;EACL,MAAM;EACN,MAAM;GACJ;GACA,GAAI,YACA;IAAE,UAAU,OAAO;IAAc,WAAW,OAAO;GAAc,IACjE,CAAC;GACL,GAAI,UAAU,EAAE,SAAS,OAAO,SAAS,IAAI,CAAC;EAChD;CACF;AACF;AAEA,MAAM,eAAe;AAErB,MAAM,iBAAiB;CACrB;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,IAAI;AAEX,MAAM,eAAyB;CAC7B,MAAM;CACN,UAAU;EACR,MAAM;EACN,aACE;EACF,YAAY;GACV,MAAM;GACN,YAAY,EAAE,MAAM;IAAE,MAAM;IAAU,aAAa;GAA2C,EAAE;GAChG,UAAU,CAAC,MAAM;EACnB;CACF;AACF;;AAGA,eAAe,iBACb,QACA,OACA,QACA,IACY;CACZ,OAAO,eAAe;CACtB,MAAM,QAAQ,MAAM,YAAY;EAAC;EAAQ;EAAW;EAAa;EAAQ;EAAW;CAAK,GAAG,EAC1F,KAAK,UACP,CAAC;CACD,MAAM,kBAAwB;EAC5B,IAAI,CAAC,MAAM,QAAQ,MAAM,KAAK,SAAS;CACzC;CACA,OAAO,iBAAiB,SAAS,WAAW,EAAE,MAAM,KAAK,CAAC;CAC1D,MAAM,KAAK,gBAAgB,EAAE,OAAO,MAAM,OAAO,CAAC;CAClD,MAAM,UAAyC,CAAC;CAChD,MAAM,UAAoB,CAAC;CAC3B,GAAG,GAAG,SAAS,MAAM;EACnB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK,CAAC;OACX,QAAQ,KAAK,CAAC;CACrB,CAAC;CACD,IAAI,SAAS;CACb,MAAM,OAAO,GAAG,SAAS,MAAc;EACrC,UAAU,EAAE,SAAS,MAAM;CAC7B,CAAC;CACD,MAAM,YAAY,cAChB,IAAI,SAAS,SAAS,WAAW;EAC/B,IAAI,OAAO,SAAS;GAClB,OAAO,OAAO,MAAM;GACpB;EACF;EACA,MAAM,cAAc,QAAQ,MAAM;EAClC,IAAI,gBAAgB,KAAA,GAAW,OAAO,QAAQ,WAAW;EACzD,MAAM,IAAI,iBACF,uBAAO,IAAI,MAAM,oCAAoC,UAAU,cAAc,OAAO,MAAM,IAAI,GAAG,CAAC,GACxG,SACF;EAGA,MAAM,UAAU,SAA8B;GAC5C,aAAa,CAAC;GACd,OAAO,oBAAoB,SAAS,OAAO;GAC3C,OACE,OAAO,UACH,OAAO,yBACP,IAAI,MAAM,4BAA4B,KAAK,aAAa,OAAO,MAAM,IAAI,GAAG,CAClF;EACF;EACA,MAAM,gBAAsB;GAC1B,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,MAAM,QAAQ,QAAQ,QAAQ,MAAM;GACpC,IAAI,SAAS,GAAG,QAAQ,OAAO,OAAO,CAAC;GACvC,OAAO,OAAO,MAAM;EACtB;EACA,MAAM,UAAU,SAAuB;GACrC,aAAa,CAAC;GACd,MAAM,eAAe,QAAQ,MAAM;GACnC,OAAO,oBAAoB,SAAS,OAAO;GAC3C,QAAQ,IAAI;EACd;EACA,QAAQ,KAAK,MAAM;EACnB,MAAM,KAAK,QAAQ,MAAM;EACzB,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;CAC1D,CAAC;CACH,IAAI;EACF,MAAM,QAAQ,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;EAChD,IAAI,CAAC,MAAM,OAAO,MAAM,IAAI,MAAM,qCAAqC,MAAM,SAAS,iBAAiB;EACvG,MAAM,OAAO,OAAO,QAAmE;GACrF,MAAM,MAAM,MAAM,GAAG,KAAK,UAAU,GAAG,EAAE,GAAG;GAC5C,MAAM,MAAM,KAAK,MAAM,MAAM,SAAS,IAAO,CAAC;GAC9C,IAAI,OAAO,IAAI,UAAU,UAAU,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO;GAC7F,OAAO;EACT;EACA,OAAO,MAAM,GAAG,MAAM,MAAM,eAAe,EAAE;CAC/C,UAAU;EACR,OAAO,oBAAoB,SAAS,SAAS;EAC7C,MAAM,MAAM,IAAI;EAChB,UAAU;CACZ;AACF;;AAYA,SAAgB,uBAAuB,KAS3B;CACV,MAAM,WAAW,IAAI,YAAY,OAAO,QAAQ,IAAI,mBAAmB,EAAE;CACzE,MAAM,kBAAkB,IAAI,mBAAmB;CAC/C,IAAI,MAAM;CACV,OAAO,EACL,MAAM,SAAS;EAEb,OAAO;GACL,IAAA,qBAF8B;GAG9B,OAAO,aAAa,QAAgB,YAAuC;IACzE,MAAM,SAAS,YAAY,UAAU,IAAI,gBAAgB,CAAC,CAAC;IAC3D,OAAO,eAAe;IACtB,MAAM,IAAI,OAAO,MAAM,YAAY;IACnC,IAAI,CAAC,GACH,MAAM,IAAI,MACR,wFAAwF,OAAO,MAAM,GAAG,GAAG,GAC7G;IAEF,MAAM,GAAG,QAAQ,SAAS;IAC1B,MAAM,YAAY,OAAO,QAAQ,cAAc,EAAE,CAAC,CAAC,KAAK;IACxD,MAAM,MAAM,MAAM,gBAAgB,QAAkB,OAAiB,QAAQ,OAAO,MAAM,gBAAgB;KACxG,MAAM,SAAS,YAAY,GAAG,eAAe,MAAM,cAAc;KACjE,MAAM,kBAA2D,CAAC;KAClE,MAAM,UAAU;MACd,MAAM;MACN,SAAS;MACT,OAAO;OACL,UAAU;OACV,SAAS,IAAI;OACb,UAAU,EAAE,SAAS;MACvB;MACA,QAAQ,EAAE,cAAc,OAAO;MAC/B,OAAO,EAAE,gBAAgB,KAAK;KAChC;KAgBA,MAAM,OAAO,MAAM,iBACjB,gBACE;MAAE,MAAM;MAAY,SAjBR,eAAe;OAC7B,SAAS;OACT,eAAe,IAAI;OACnB,WAAW,IAAI;OACf,GAAI,IAAI,WAAW,EAAE,UAAU,IAAI,SAAS,IAAI,CAAC;OACjD,OAAO,CAAC,YAAY;OACpB,iBAAiB,OAAO,MAAM,SAAS;QACrC,IAAI,SAAS,kBAAkB,OAAO,uBAAuB;QAC7D,MAAM,MAAM,MAAM,KAAK;SAAE,IAAI;SAAW,MAAM,OAAO,KAAK,QAAQ,EAAE;QAAE,CAAC;QACvE,MAAM,OAAO,IAAI,mBAAmB;QACpC,MAAM,SAAS,GAAG,OAAO,IAAI,UAAU,EAAE,IAAI,OAAO,2FAA2F;QAC/I,gBAAgB,KAAK;SAAE,MAAM,KAAK,UAAU,IAAI;SAAG;QAAO,CAAC;QAC3D,OAAO;OACT;MACF,CAG8B;MAAG;KAAQ,GACrC,SAAS,eACT,EAAE,OAAO,CACX,CACF;KACA,IAAI,KAAK,WAAW,aAClB,MAAM,IAAI,MAAM,KAAK,OAAO,WAAW,4BAA4B,KAAK,QAAQ;KAElF,MAAM,UAAW,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC;KAC9C,MAAM,aAAa,gBAChB,MAAM,EAAE,CAAC,CACT,KAAK,MAAM,UAAU,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,aAAa,EAAE,OAAO,MAAM,GAAG,GAAG,GAAG,CAAC,CAChF,KAAK,SAAS,CAAC,CACf,MAAM,GAAG,IAAI;KAChB,MAAM,aAAa,KAAK,OAAO,GAAG,EAAE;KACpC,MAAM,iBACJ,YAAY,SAAS,WAAW,WAAW,UAAU,SAChD,WAAW,SAAS,SACrB,KAAA;KACN,OAAO,6BACL,SACA,KAAK,OACL,gBAAgB,OAAO,YACvB,UACF;IACF,CAAC;IACD,MAAM,aAAa,wBAAwB,KAAK,IAAI,KAAK;IACzD,IAAI,YAAY,MAAM;IACtB,MAAM;KAAE,MAAM;KAAU,MAAM,EAAE,WAAW,KAAK,UAAU,GAAG,EAAE;IAAE;GACnE;GACA,MAAM,SAAS,CAAC;EAClB;CACF,EACF;AACF;;AAGA,MAAM,qBAA4C,EAChD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EAEvB,MAAM,KADK,IAA2C,KAAA,EACzC;EACb,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CACA,OAAO;AACT,EACF;AAEA,SAAgB,6BAA+C;CAC7D,MAAM,OAAO,sBAAsB;CACnC,OAAO;EACL,MAAM;EACN,QAAQ;EACR,iBAAiB,KAAK,UAAU;EAEhC,MAAM,UAAU,OAAoB,CAAC,GAAyB;GAE5D,QAAO,MADa,KAAK,UAAU,IAAI,EAAA,CAC1B,KAAK,MAAM;IACtB,MAAM,OAAO,SAAS,CAAC;IACvB,OAAO;KACL,GAAG;KAGH,QAAQ,mBAAmB,KAAK,OAAO,GAAG,KAAK,MAAM;IACvD;GACF,CAAC;EACH;EAEA,oBAAoB,QAAQ,QAAQ,KAAA,CAAS;EAE7C,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI;GACJ,IAAI;IACF,MAAM,KAAK,MAAM,QAAQ;GAC3B,QAAQ;IACN,MAAM,IAAI,MACR,6EAA6E,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GACrH;GACF;GACA,IAAI,OAAO,IAAI,YAAY,aAAa,OAAO,IAAI,cAAc,UAC/D,MAAM,IAAI,MACR,sEAAsE,KAAK,OAAO,KAAK,SAAS,MAAM,GAAG,GAAG,GAC9G;GAEF,MAAM,SAAS,IAAI,UAAU;GAC7B,MAAM,QAAQ,IAAI;GAClB,MAAM,WAAW,MAAM,QAAQ,IAAI,aAAa,IAAI,IAAI,gBAAgB,CAAC;GACzE,OAAO;IACL,UAAU,IAAI,YAAY;IAC1B,OAAO,QAAQ,IAAI,SAAS,QAAQ;IACpC,QAAQ,KAAK,UAAU;KACrB,QAAQ,KAAK;KACb,SAAS,IAAI;KACb;KACA,OAAO,IAAI,SAAS;KACpB;KACA,OAAO,IAAI;KACX,GAAI,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;KACtC,GAAI,IAAI,aAAa,EAAE,gBAAgB,IAAI,WAAW,MAAM,IAAI,EAAE,IAAI,CAAC;IACzE,CAAC;GACH;EACF;EAEA,aAAa,MAAM,uBAAuB,CAAC;CAC7C;AACF"}

package/dist/benchmarks/cadbench.js

@@ -1,2 +1,2 @@
-import { t as createCadBenchAdapter } from "../cadbench-BrpwOU6A.js";
+import { t as createCadBenchAdapter } from "../cadbench-BLSyxR1N.js";
 export { createCadBenchAdapter };

package/dist/benchmarks/cadgenbench.js

@@ -1,2 +1,2 @@
-import { t as createCadGenBenchAdapter } from "../cadgenbench-DF7hYHdl.js";
+import { t as createCadGenBenchAdapter } from "../cadgenbench-x2OFkf8y.js";
 export { createCadGenBenchAdapter };

package/dist/benchmarks/finresearchbench.js

@@ -1,4 +1,5 @@
 import { benchRoot } from "./_harness.js";
+import { t as runBenchRouterTurn } from "../router-turn-C2wMiDoo.js";
 import { readFile, stat } from "node:fs/promises";
 import { join } from "node:path";
 //#region src/benchmarks/finresearchbench.ts
@@ -122,28 +123,21 @@ function parseJudgeScore(content) {
 async function runOfficialJudge(meta, response) {
 	if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`);
 	const router = routerConfig();
-	const res = await fetch(`${router.baseUrl}/chat/completions`, {
-		method: "POST",
-		headers: {
-			"content-type": "application/json",
-			authorization: `Bearer ${router.key}`
-		},
-		body: JSON.stringify({
-			model: router.model,
-			temperature: 0,
-			messages: [{
-				role: "system",
-				content: meta.judgeSystemPrompt
-			}, {
-				role: "user",
-				content: fillTemplate(meta, response)
-			}]
-		})
-	});
-	if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
-	const body = await res.json();
-	const content = body.choices?.[0]?.message?.content;
-	if (typeof content !== "string") throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`);
+	const content = (await runBenchRouterTurn({
+		routerBaseUrl: router.baseUrl,
+		routerKey: router.key,
+		profile: {
+			name: "finresearchbench-judge",
+			harness: "cli-base",
+			model: {
+				provider: "tangle-router",
+				default: router.model,
+				metadata: { temperature: 0 }
+			},
+			prompt: { systemPrompt: meta.judgeSystemPrompt }
+		}
+	}, fillTemplate(meta, response))).finalText;
+	if (!content) throw new Error("FinResearchBench judge returned no message content");
 	const { score, raw } = parseJudgeScore(content);
 	return {
 		resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? .8),

package/dist/benchmarks/finresearchbench.js.map

@@ -1 +1 @@
-{"version":3,"file":"finresearchbench.js","names":[],"sources":["../../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const res = await fetch(`${router.baseUrl}/chat/completions`, {\n    method: 'POST',\n    headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` },\n    body: JSON.stringify({\n      model: router.model,\n      temperature: 0,\n      messages: [\n        { role: 'system', content: meta.judgeSystemPrompt },\n        { role: 'user', content: fillTemplate(meta, response) },\n      ],\n    }),\n  })\n  if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)\n  const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }\n  const content = body.choices?.[0]?.message?.content\n  if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`)\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAcA,MAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,eAAgE;CACvE,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,kEAAkE;CAC5F,OAAO;EACL,SAAS,QAAQ,IAAI,eAAe;EACpC;EACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;CAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;CACxE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,6BAA6B,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;CACrD,MAAM,UAAU,IAAI,KAAK;CACzB,IAAI,QAAQ,WAAW,GAAG,GAAG,OAAO,KAAK,MAAM,OAAO;CACtD,OAAO,QACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;CAC1F,MAAM,UAAU,QACb,QAAQ,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,CAAC,CACvE,KAAK,QAAQ,IAAI,EAAE;CACtB,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,mCAAmC,OAAO,sCAAsC,QAAQ,OAAO,GAAG,QAAQ,OAAO,WAAW,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,kHAE7J;AAEJ;AAEA,SAAS,UAAU,KAAwB,cAAkC;CAC3E,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,OAAwB;EAC5B,IAAI,IAAI;EACR,UAAU,IAAI,YAAY;EAC1B,UAAU,IAAI;EACd;EACA;EACA,WAAW,IAAI,cAAc;EAC7B,QAAQ,IAAI,UAAU;EACtB,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,SAAS,eAAe,4BAA4B;CACtD;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAChC,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,kDAAkD;CAErG,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;CACpG,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,YAAY,CAAC;CAC1D,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACxE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,GAAG;CACpG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KAAK,4DAA4D,QAAQ,OAAO,kBAAkB;CAC1G,OAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;CACtF,MAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;CACxD,sBAAsB,SAAS,IAAI;CACnC,OAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;CACrE,MAAM,WAAW,KAAK;CACtB,IAAI,CAAC,UAAU,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,+BAA+B;CAC/F,OAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,CAAC,CACvC,WAAW,cAAc,QAAQ,CAAC,CAClC,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,CAAC,CACnE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;CACzE,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,eAAe,GAAG,WAAW,KAAK,EAAE,EAAE;CACvE,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,aAAa,YAAY;EAClC,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,SAAS;EAC/B,QAAQ;GACN;EACF;EACA,MAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;EACjF,MAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;EAClF,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG;EACzB,IAAI,IAAI,GAAG,MAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,GAAG;EAChG,MAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;EAClD,IAAI,QAAQ,GAAG,MAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,GAAG;EAChH,OAAO;GAAE;GAAO,KAAK;EAAO;CAC9B;CACA,MAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,GAAG;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;CAC5F,IAAI,CAAC,KAAK,mBAAmB,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,6BAA6B;CAC3G,MAAM,SAAS,aAAa;CAC5B,MAAM,MAAM,MAAM,MAAM,GAAG,OAAO,QAAQ,oBAAoB;EAC5D,QAAQ;EACR,SAAS;GAAE,gBAAgB;GAAoB,eAAe,UAAU,OAAO;EAAM;EACrF,MAAM,KAAK,UAAU;GACnB,OAAO,OAAO;GACd,aAAa;GACb,UAAU,CACR;IAAE,MAAM;IAAU,SAAS,KAAK;GAAkB,GAClD;IAAE,MAAM;IAAQ,SAAS,aAAa,MAAM,QAAQ;GAAE,CACxD;EACF,CAAC;CACH,CAAC;CACD,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,+BAA+B,IAAI,OAAO,KAAK,MAAM,IAAI,KAAK,EAAA,CAAG,MAAM,GAAG,GAAG,GAAG;CAC7G,MAAM,OAAQ,MAAM,IAAI,KAAK;CAC7B,MAAM,UAAU,KAAK,UAAU,EAAE,EAAE,SAAS;CAC5C,IAAI,OAAO,YAAY,UAAU,MAAM,IAAI,MAAM,uDAAuD,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,GAAG;CAC5I,MAAM,EAAE,OAAO,QAAQ,gBAAgB,OAAO;CAC9C,OAAO;EACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,EAAG;EAC5E;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;GAAU,YAAY,OAAO;GAAO;EAAI,CAAC;CAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;CAC5C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;CACzE,MAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;CACzE,MAAM,WAAW,cAAc,QAAQ;CACvC,MAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;CACnE,OAAO;EACL,UAAU,UAAU;EACpB;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;EAAS,CAAC;CAC3E;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,8BAA8B;CAE/D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,eAAe,UAAU,cAAc;IAC7C;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,4NACF;GAEF,aAAa;GACb,MAAM,eAAe,MAAM,WAAW;GACtC,MAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,sEAAsE;GACjG,OAAO,kBAAkB,MAAM,IAAI;EACrC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,mBAAmB,KAAK,mBAAmB,KAAA;EACzD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,cAAc,OAAO,aAAa,MAAM,QAAQ;GACpD,IAAI,SAAS,KAAK,CAAC,CAAC,WAAW,GAC7B,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,SAAS,KAAK;KAAS,UAAU,KAAK;KAAU,QAAQ;IAAe,CAAC;GACnG;GAEF,OAAO,iBAAiB,MAAM,QAAQ;EACxC;CACF;AACF"}
+{"version":3,"file":"finresearchbench.js","names":[],"sources":["../../src/benchmarks/finresearchbench.ts"],"sourcesContent":["/**\n * FinResearchBench-compatible adapter.\n *\n * The paper defines a logic-tree Agent-as-a-Judge benchmark for financial\n * research reports, but there is no stable public scorer package wired here.\n * Live mode therefore requires a local data export whose rows carry the official\n * judge prompt/template/logic tree. The adapter refuses to invent a judge.\n */\n\nimport { readFile, stat } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { runBenchRouterTurn } from '../router-turn'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'finresearchbench.json')\n\ninterface FinResearchRecord {\n  id: string\n  question: string\n  category?: string\n  reference_answer?: string\n  reference_report?: string\n  logic_tree?: unknown\n  rubric?: unknown\n  judge_system_prompt?: string\n  judge_prompt_template?: string\n}\n\ninterface FinResearchMeta {\n  id: string\n  category: string\n  question: string\n  referenceAnswer: string\n  referenceReport: string\n  logicTree: unknown\n  rubric: unknown\n  judgeSystemPrompt?: string\n  judgePromptTemplate?: string\n  scoring: 'official-logic-tree-judge' | 'fixture-exact-reference'\n}\n\nconst dataFile = (): string | undefined => process.env.FINRESEARCHBENCH_DATA_FILE\n\nfunction routerConfig(): { baseUrl: string; key: string; model: string } {\n  const key = process.env.TANGLE_API_KEY\n  if (!key) throw new Error('TANGLE_API_KEY is required for FinResearchBench live LLM judging')\n  return {\n    baseUrl: process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1',\n    key,\n    model: process.env.FINRESEARCHBENCH_JUDGE_MODEL ?? process.env.JUDGE_MODEL ?? 'deepseek-v4-flash',\n  }\n}\n\nasync function assertReadable(path: string, label: string): Promise<void> {\n  try {\n    await stat(path)\n  } catch (err) {\n    throw new Error(`FinResearchBench: missing ${label} at ${path} (${err instanceof Error ? err.message : err})`)\n  }\n}\n\nfunction readRecords(raw: string): FinResearchRecord[] {\n  const trimmed = raw.trim()\n  if (trimmed.startsWith('[')) return JSON.parse(trimmed) as FinResearchRecord[]\n  return trimmed\n    .split(/\\r?\\n/)\n    .map((line) => line.trim())\n    .filter((line) => line.length > 0)\n    .map((line) => JSON.parse(line) as FinResearchRecord)\n}\n\nfunction assertLiveJudgeFields(records: readonly FinResearchRecord[], source: string): void {\n  const missing = records\n    .filter((row) => !row.judge_system_prompt || !row.judge_prompt_template)\n    .map((row) => row.id)\n  if (missing.length > 0) {\n    throw new Error(\n      `FinResearchBench live rows from ${source} missing official judge prompts for ${missing.length}/${records.length} row(s): ${missing.slice(0, 5).join(', ')}. ` +\n        'Use a benchmark export with judge_system_prompt and judge_prompt_template, or do not score this benchmark live.',\n    )\n  }\n}\n\nfunction rowToTask(row: FinResearchRecord, fixturesMode: boolean): BenchTask {\n  const referenceAnswer = row.reference_answer ?? ''\n  const referenceReport = row.reference_report ?? referenceAnswer\n  const meta: FinResearchMeta = {\n    id: row.id,\n    category: row.category ?? 'unknown',\n    question: row.question,\n    referenceAnswer,\n    referenceReport,\n    logicTree: row.logic_tree ?? null,\n    rubric: row.rubric ?? null,\n    judgeSystemPrompt: row.judge_system_prompt,\n    judgePromptTemplate: row.judge_prompt_template,\n    scoring: fixturesMode ? 'fixture-exact-reference' : 'official-logic-tree-judge',\n  }\n  return {\n    id: row.id,\n    split: row.category,\n    prompt: [\n      'Complete this FinResearchBench financial research task.',\n      'Produce a decision-grade research answer with explicit reasoning, evidence, and final conclusion.',\n      '',\n      row.question,\n    ].join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): FinResearchMeta {\n  const md = task.metadata\n  if (!md || typeof md.question !== 'string') {\n    throw new Error(`FinResearchBench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as FinResearchMeta\n}\n\nfunction selectRows(rows: FinResearchRecord[], opts: LoadOptions, fixturesMode: boolean): BenchTask[] {\n  let tasks = rows.map((row) => rowToTask(row, fixturesMode))\n  if (opts.split) tasks = tasks.filter((task) => task.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`FinResearchBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(FIXTURES, 'utf8'))\n  console.warn(`[finresearchbench] FINRESEARCHBENCH_FIXTURES=1 — loading ${records.length} adapter fixtures`)\n  return selectRows(records, opts, true)\n}\n\nasync function loadOfficialTasks(path: string, opts: LoadOptions): Promise<BenchTask[]> {\n  const records = readRecords(await readFile(path, 'utf8'))\n  assertLiveJudgeFields(records, path)\n  return selectRows(records, opts, false)\n}\n\nfunction fillTemplate(meta: FinResearchMeta, response: string): string {\n  const template = meta.judgePromptTemplate\n  if (!template) throw new Error(`FinResearchBench task ${meta.id} missing judge_prompt_template`)\n  return template\n    .replaceAll('{question}', meta.question)\n    .replaceAll('{response}', response)\n    .replaceAll('{reference_answer}', meta.referenceAnswer)\n    .replaceAll('{reference_report}', meta.referenceReport)\n    .replaceAll('{logic_tree}', JSON.stringify(meta.logicTree, null, 2))\n    .replaceAll('{rubric}', JSON.stringify(meta.rubric, null, 2))\n}\n\nfunction parseJudgeScore(content: string): { score: number; raw: unknown } {\n  const candidates: string[] = []\n  for (const m of content.matchAll(/```(?:json)?\\s*([\\s\\S]*?)```/g)) candidates.push(m[1].trim())\n  for (const m of content.matchAll(/\\{[\\s\\S]*?\\}/g)) candidates.push(m[0])\n  candidates.push(content.trim())\n  for (const candidate of candidates) {\n    let parsed: Record<string, unknown>\n    try {\n      parsed = JSON.parse(candidate) as Record<string, unknown>\n    } catch {\n      continue\n    }\n    const raw = parsed.score ?? parsed.overall_score ?? parsed.total_score ?? parsed.answer_score\n    const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN\n    if (!Number.isFinite(n)) continue\n    if (n < 0) throw new Error(`FinResearchBench judge score is negative: ${JSON.stringify(parsed)}`)\n    const score = n <= 1 ? n : n <= 10 ? n / 10 : n / 100\n    if (score > 1) throw new Error(`FinResearchBench judge score outside supported range: ${JSON.stringify(parsed)}`)\n    return { score, raw: parsed }\n  }\n  throw new Error(`FinResearchBench judge produced no parseable JSON score: ${content.slice(0, 400)}`)\n}\n\nasync function runOfficialJudge(meta: FinResearchMeta, response: string): Promise<BenchScore> {\n  if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`)\n  const router = routerConfig()\n  const turn = await runBenchRouterTurn(\n    {\n      routerBaseUrl: router.baseUrl,\n      routerKey: router.key,\n      profile: {\n        name: 'finresearchbench-judge',\n        harness: 'cli-base',\n        model: {\n          provider: 'tangle-router',\n          default: router.model,\n          metadata: { temperature: 0 },\n        },\n        prompt: { systemPrompt: meta.judgeSystemPrompt },\n      },\n    },\n    fillTemplate(meta, response),\n  )\n  const content = turn.finalText\n  if (!content) throw new Error('FinResearchBench judge returned no message content')\n  const { score, raw } = parseJudgeScore(content)\n  return {\n    resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8),\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, judgeModel: router.model, raw }),\n  }\n}\n\nfunction normalizeText(value: string): string {\n  return value.toLowerCase().replace(/\\s+/g, ' ').trim()\n}\n\nfunction scoreFixture(meta: FinResearchMeta, artifact: string): BenchScore {\n  const answer = normalizeText(meta.referenceAnswer || meta.referenceReport)\n  const response = normalizeText(artifact)\n  const score = answer.length > 0 && response.includes(answer) ? 1 : 0\n  return {\n    resolved: score === 1,\n    score,\n    detail: JSON.stringify({ scoring: meta.scoring, category: meta.category }),\n  }\n}\n\nexport function createFinResearchBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.FINRESEARCHBENCH_FIXTURES === '1'\n\n  return {\n    name: 'finresearchbench',\n\n    async preflight() {\n      if (fixturesMode) {\n        await assertReadable(FIXTURES, 'fixture file')\n        return\n      }\n      const file = dataFile()\n      if (!file) {\n        throw new Error(\n          'FINRESEARCHBENCH_DATA_FILE is required. Fix: export the official FinResearchBench rows as JSON/JSONL with judge_system_prompt and judge_prompt_template fields, then set FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl.',\n        )\n      }\n      routerConfig()\n      await assertReadable(file, 'data file')\n      await loadOfficialTasks(file, { limit: 1 })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      const file = dataFile()\n      if (!file) throw new Error('FINRESEARCHBENCH_DATA_FILE is required to load FinResearchBench rows')\n      return loadOfficialTasks(file, opts)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const meta = readMeta(task)\n      return meta.referenceReport || meta.referenceAnswer || undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      if (fixturesMode) return scoreFixture(meta, artifact)\n      if (artifact.trim().length === 0) {\n        return {\n          resolved: false,\n          score: 0,\n          detail: JSON.stringify({ scoring: meta.scoring, category: meta.category, reason: 'empty answer' }),\n        }\n      }\n      return runOfficialJudge(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,uBAAuB;AA2BpE,MAAM,iBAAqC,QAAQ,IAAI;AAEvD,SAAS,eAAgE;CACvE,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,kEAAkE;CAC5F,OAAO;EACL,SAAS,QAAQ,IAAI,eAAe;EACpC;EACA,OAAO,QAAQ,IAAI,gCAAgC,QAAQ,IAAI,eAAe;CAChF;AACF;AAEA,eAAe,eAAe,MAAc,OAA8B;CACxE,IAAI;EACF,MAAM,KAAK,IAAI;CACjB,SAAS,KAAK;EACZ,MAAM,IAAI,MAAM,6BAA6B,MAAM,MAAM,KAAK,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,EAAE;CAC/G;AACF;AAEA,SAAS,YAAY,KAAkC;CACrD,MAAM,UAAU,IAAI,KAAK;CACzB,IAAI,QAAQ,WAAW,GAAG,GAAG,OAAO,KAAK,MAAM,OAAO;CACtD,OAAO,QACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,QAAQ,SAAS,KAAK,SAAS,CAAC,CAAC,CACjC,KAAK,SAAS,KAAK,MAAM,IAAI,CAAsB;AACxD;AAEA,SAAS,sBAAsB,SAAuC,QAAsB;CAC1F,MAAM,UAAU,QACb,QAAQ,QAAQ,CAAC,IAAI,uBAAuB,CAAC,IAAI,qBAAqB,CAAC,CACvE,KAAK,QAAQ,IAAI,EAAE;CACtB,IAAI,QAAQ,SAAS,GACnB,MAAM,IAAI,MACR,mCAAmC,OAAO,sCAAsC,QAAQ,OAAO,GAAG,QAAQ,OAAO,WAAW,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,kHAE7J;AAEJ;AAEA,SAAS,UAAU,KAAwB,cAAkC;CAC3E,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,kBAAkB,IAAI,oBAAoB;CAChD,MAAM,OAAwB;EAC5B,IAAI,IAAI;EACR,UAAU,IAAI,YAAY;EAC1B,UAAU,IAAI;EACd;EACA;EACA,WAAW,IAAI,cAAc;EAC7B,QAAQ,IAAI,UAAU;EACtB,mBAAmB,IAAI;EACvB,qBAAqB,IAAI;EACzB,SAAS,eAAe,4BAA4B;CACtD;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EACX,QAAQ;GACN;GACA;GACA;GACA,IAAI;EACN,CAAC,CAAC,KAAK,IAAI;EACX,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,OAAO,GAAG,aAAa,UAChC,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,kDAAkD;CAErG,OAAO;AACT;AAEA,SAAS,WAAW,MAA2B,MAAmB,cAAoC;CACpG,IAAI,QAAQ,KAAK,KAAK,QAAQ,UAAU,KAAK,YAAY,CAAC;CAC1D,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,SAAS,KAAK,UAAU,KAAK,KAAK;CACxE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,sCAAsC,KAAK,UAAU,IAAI,GAAG;CACpG,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,YAAY,MAAM,SAAS,UAAU,MAAM,CAAC;CAC5D,QAAQ,KAAK,4DAA4D,QAAQ,OAAO,kBAAkB;CAC1G,OAAO,WAAW,SAAS,MAAM,IAAI;AACvC;AAEA,eAAe,kBAAkB,MAAc,MAAyC;CACtF,MAAM,UAAU,YAAY,MAAM,SAAS,MAAM,MAAM,CAAC;CACxD,sBAAsB,SAAS,IAAI;CACnC,OAAO,WAAW,SAAS,MAAM,KAAK;AACxC;AAEA,SAAS,aAAa,MAAuB,UAA0B;CACrE,MAAM,WAAW,KAAK;CACtB,IAAI,CAAC,UAAU,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,+BAA+B;CAC/F,OAAO,SACJ,WAAW,cAAc,KAAK,QAAQ,CAAC,CACvC,WAAW,cAAc,QAAQ,CAAC,CAClC,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,sBAAsB,KAAK,eAAe,CAAC,CACtD,WAAW,gBAAgB,KAAK,UAAU,KAAK,WAAW,MAAM,CAAC,CAAC,CAAC,CACnE,WAAW,YAAY,KAAK,UAAU,KAAK,QAAQ,MAAM,CAAC,CAAC;AAChE;AAEA,SAAS,gBAAgB,SAAkD;CACzE,MAAM,aAAuB,CAAC;CAC9B,KAAK,MAAM,KAAK,QAAQ,SAAS,+BAA+B,GAAG,WAAW,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC;CAC9F,KAAK,MAAM,KAAK,QAAQ,SAAS,eAAe,GAAG,WAAW,KAAK,EAAE,EAAE;CACvE,WAAW,KAAK,QAAQ,KAAK,CAAC;CAC9B,KAAK,MAAM,aAAa,YAAY;EAClC,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,SAAS;EAC/B,QAAQ;GACN;EACF;EACA,MAAM,MAAM,OAAO,SAAS,OAAO,iBAAiB,OAAO,eAAe,OAAO;EACjF,MAAM,IAAI,OAAO,QAAQ,WAAW,MAAM,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI;EAClF,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG;EACzB,IAAI,IAAI,GAAG,MAAM,IAAI,MAAM,6CAA6C,KAAK,UAAU,MAAM,GAAG;EAChG,MAAM,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI,KAAK,IAAI;EAClD,IAAI,QAAQ,GAAG,MAAM,IAAI,MAAM,yDAAyD,KAAK,UAAU,MAAM,GAAG;EAChH,OAAO;GAAE;GAAO,KAAK;EAAO;CAC9B;CACA,MAAM,IAAI,MAAM,4DAA4D,QAAQ,MAAM,GAAG,GAAG,GAAG;AACrG;AAEA,eAAe,iBAAiB,MAAuB,UAAuC;CAC5F,IAAI,CAAC,KAAK,mBAAmB,MAAM,IAAI,MAAM,yBAAyB,KAAK,GAAG,6BAA6B;CAC3G,MAAM,SAAS,aAAa;CAkB5B,MAAM,WAAU,MAjBG,mBACjB;EACE,eAAe,OAAO;EACtB,WAAW,OAAO;EAClB,SAAS;GACP,MAAM;GACN,SAAS;GACT,OAAO;IACL,UAAU;IACV,SAAS,OAAO;IAChB,UAAU,EAAE,aAAa,EAAE;GAC7B;GACA,QAAQ,EAAE,cAAc,KAAK,kBAAkB;EACjD;CACF,GACA,aAAa,MAAM,QAAQ,CAC7B,EAAA,CACqB;CACrB,IAAI,CAAC,SAAS,MAAM,IAAI,MAAM,oDAAoD;CAClF,MAAM,EAAE,OAAO,QAAQ,gBAAgB,OAAO;CAC9C,OAAO;EACL,UAAU,SAAS,OAAO,QAAQ,IAAI,mCAAmC,EAAG;EAC5E;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;GAAU,YAAY,OAAO;GAAO;EAAI,CAAC;CAC1G;AACF;AAEA,SAAS,cAAc,OAAuB;CAC5C,OAAO,MAAM,YAAY,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;AACvD;AAEA,SAAS,aAAa,MAAuB,UAA8B;CACzE,MAAM,SAAS,cAAc,KAAK,mBAAmB,KAAK,eAAe;CACzE,MAAM,WAAW,cAAc,QAAQ;CACvC,MAAM,QAAQ,OAAO,SAAS,KAAK,SAAS,SAAS,MAAM,IAAI,IAAI;CACnE,OAAO;EACL,UAAU,UAAU;EACpB;EACA,QAAQ,KAAK,UAAU;GAAE,SAAS,KAAK;GAAS,UAAU,KAAK;EAAS,CAAC;CAC3E;AACF;AAEA,SAAgB,gCAAkD;CAChE,MAAM,eAAe,QAAQ,IAAI,8BAA8B;CAE/D,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAChB,IAAI,cAAc;IAChB,MAAM,eAAe,UAAU,cAAc;IAC7C;GACF;GACA,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MACH,MAAM,IAAI,MACR,4NACF;GAEF,aAAa;GACb,MAAM,eAAe,MAAM,WAAW;GACtC,MAAM,kBAAkB,MAAM,EAAE,OAAO,EAAE,CAAC;EAC5C;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,MAAM,OAAO,SAAS;GACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,sEAAsE;GACjG,OAAO,kBAAkB,MAAM,IAAI;EACrC;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,SAAS,IAAI;GAC1B,OAAO,KAAK,mBAAmB,KAAK,mBAAmB,KAAA;EACzD;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,IAAI,cAAc,OAAO,aAAa,MAAM,QAAQ;GACpD,IAAI,SAAS,KAAK,CAAC,CAAC,WAAW,GAC7B,OAAO;IACL,UAAU;IACV,OAAO;IACP,QAAQ,KAAK,UAAU;KAAE,SAAS,KAAK;KAAS,UAAU,KAAK;KAAU,QAAQ;IAAe,CAAC;GACnG;GAEF,OAAO,iBAAiB,MAAM,QAAQ;EACxC;CACF;AACF"}