@tangle-network/agent-bench 0.4.0 → 0.4.3

package/dist/benchmarks/aec-bench.js

@@ -1,8 +1,243 @@
-import {
-  createAecBenchAdapter
-} from "../chunk-K3BQGZCT.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  createAecBenchAdapter
-};
+import { benchRoot, preflightVenvImports, readJsonReport, runStagedJudge, stageFile, venvPython } from "./_harness.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/aec-bench.ts
+/**
+* AEC-Bench adapter (TheodoreGalanos/aec-bench, MIT) — closed-form
+* Architecture/Engineering/Construction calculation tasks. Worker artifact = a
+* markdown solution ending in a fenced ```json block with the required numeric
+* fields. Judge = the task's OWN `tests/verify.py`, run with python3: it
+* recomputes ground truth from the embedded engineering formulas, extracts the
+* last JSON block from the artifact, scores each field by math.isclose within a
+* per-field rel_tol, and writes {"reward": mean} + per-field details.json.
+* GRADED / partial-credit, FULLY DETERMINISTIC — no LLM judge.
+*
+* Distinct from the multimodal nomic-ai/aec-bench; this is the deterministic
+* calculation platform. The runnable-instance verify.py only needs python3 (no
+* Docker, no Harbor) for the pure-calc disciplines, so the local gate runs at
+* conc<=2 without a container backend.
+*
+* Requires for a live run: network to raw.githubusercontent.com /
+* api.github.com (the in-repo tasks tree) + a python3 interpreter (the bench
+* venv) to run verify.py. For offline/CI, set AEC_FIXTURES=1 to load the
+* committed fixtures (bench/fixtures/aec-bench.json) — never a silent fallback.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "aec-bench.json");
+const REPO = "TheodoreGalanos/aec-bench";
+const RAW = `https://raw.githubusercontent.com/${REPO}/main`;
+const TREE = `https://api.github.com/repos/${REPO}/git/trees/main?recursive=1`;
+/** Matches every runnable-instance task id at ANY depth under tasks/. */
+const verifyPathPattern = /^tasks\/(.+)\/tests\/verify\.py$/;
+/** Default cap on tasks enumerated before a `limit` is applied. */
+const DEFAULT_LIMIT = 10;
+function recordToTask(rec) {
+	const meta = {
+		taskId: rec.id,
+		discipline: rec.discipline,
+		taskToml: rec.task_toml,
+		verifyPy: rec.verify_py,
+		goldenPassMd: rec.golden_pass_md
+	};
+	return {
+		id: rec.id,
+		split: rec.discipline,
+		prompt: rec.instruction,
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || typeof md.verifyPy !== "string" || typeof md.taskId !== "string") throw new Error(`aec-bench task ${task.id} missing verifier metadata — loadTasks did not populate it`);
+	return md;
+}
+async function fetchText(url) {
+	const res = await fetch(url);
+	if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`);
+	return res.text();
+}
+/** Like fetchText but returns null on a 404 (absent optional file); throws on any other non-OK. */
+async function fetchTextOrNull(url) {
+	const res = await fetch(url);
+	if (res.status === 404) return null;
+	if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`);
+	return res.text();
+}
+/**
+* One recursive git-tree call enumerates EVERY runnable-instance id at any depth:
+* a task is runnable iff it ships tests/verify.py. The captured group is the id
+* `tasks/<id>/tests/verify.py` → `<id>` (e.g. 'electrical/pf-droop', or a deeper
+* '<discipline>/<family>/<task>'). Throws loud on a non-OK tree response.
+*/
+async function listAllInstances() {
+	const res = await fetch(TREE);
+	if (!res.ok) throw new Error(`aec-bench tree ${res.status}: ${TREE}`);
+	const { tree } = await res.json();
+	const ids = [];
+	for (const entry of tree) {
+		const m = verifyPathPattern.exec(entry.path);
+		if (m?.[1]) ids.push(m[1]);
+	}
+	return ids;
+}
+/**
+* Fetch one task's instruction.md + task.toml + tests/verify.py + golden_pass.md.
+* Returns null when the dir is a SEED (no runnable verify.py) so enumeration can
+* skip it without faking a task. Gold is OPTIONAL — a task that ships a non-md
+* ground truth (e.g. tests/ground_truth.json) yields golden_pass_md=null; the
+* deterministic judge needs only verify.py.
+*/
+async function fetchInstance(id) {
+	const base = `${RAW}/tasks/${id}`;
+	const verify = await fetch(`${base}/tests/verify.py`);
+	if (verify.status === 404) return null;
+	if (!verify.ok) throw new Error(`aec-bench fetch ${verify.status}: ${id}/tests/verify.py`);
+	const [instruction, task_toml, golden_pass_md] = await Promise.all([
+		fetchText(`${base}/instruction.md`),
+		fetchText(`${base}/task.toml`),
+		fetchTextOrNull(`${base}/tests/fixtures/golden_pass.md`)
+	]);
+	return {
+		id,
+		discipline: id.split("/")[0] ?? "",
+		instruction,
+		task_toml,
+		verify_py: await verify.text(),
+		golden_pass_md
+	};
+}
+function selectFixtures(records, opts) {
+	let tasks = records.map(recordToTask);
+	if (opts.split) tasks = tasks.filter((t) => t.split === opts.split);
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((t) => want.has(t.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	return tasks;
+}
+async function loadFixtures(opts) {
+	const records = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[aec-bench] AEC_FIXTURES=1 — loading ${records.length} committed fixtures from ${FIXTURES} (no GitHub fetch)`);
+	return selectFixtures(records, opts);
+}
+/** Enumerate live tasks: explicit ids (each required — throws on a bad id), or a
+*  capped slice of the recursive tree (optionally filtered to one split). Per-task
+*  resilient: a single fetch failure warns + SKIPS that task, never aborting the
+*  batch. Skips seed dirs (no verify.py) — never fabricates. */
+async function loadLive(opts) {
+	if (opts.ids) {
+		const records = [];
+		for (const id of opts.ids) {
+			const rec = await fetchInstance(id);
+			if (!rec) throw new Error(`aec-bench: ${id} has no tests/verify.py (seed-only or wrong id)`);
+			records.push(rec);
+		}
+		return records.map(recordToTask);
+	}
+	const limit = opts.limit ?? DEFAULT_LIMIT;
+	let ids = await listAllInstances();
+	if (opts.split) ids = ids.filter((id) => id.startsWith(`${opts.split}/`));
+	const records = [];
+	for (const id of ids) {
+		if (records.length >= limit) break;
+		try {
+			const rec = await fetchInstance(id);
+			if (rec) records.push(rec);
+		} catch (err) {
+			console.warn(`[aec-bench] skipping ${id}: ${err instanceof Error ? err.message : String(err)}`);
+		}
+	}
+	if (records.length === 0) throw new Error(`aec-bench loadTasks found no runnable instances for ${JSON.stringify(opts)} (no tests/verify.py matched the requested split). Set AEC_FIXTURES=1 to run offline.`);
+	return records.map(recordToTask);
+}
+/**
+* Run the task's own verify.py with python3 over the artifact via the shared
+* staged-judge spine (mkdtemp → stage → spawn → parseReport → cleanup). verify.py
+* writes {"reward": mean} to --output and per-field details.json as a sibling. We
+* read both: reward → graded score; details → sub-scores for the trace-analyst.
+* Fail loud if the verifier never wrote a numeric reward (a crashed verifier is
+* NOT a silent 0 — verify.py's own except-trap writes reward 0.0, so an absent /
+* non-numeric reward.json is a real bug, surfaced by readJsonReport).
+*/
+async function runVerifier(meta, artifact) {
+	return runStagedJudge({
+		tmpPrefix: "aecbench-",
+		timeoutMs: 12e4,
+		async stage(dir) {
+			await Promise.all([stageFile(join(dir, "output.md"), artifact), stageFile(join(dir, "verify.py"), meta.verifyPy)]);
+		},
+		bin: venvPython,
+		argv: (dir) => [
+			join(dir, "verify.py"),
+			"--input",
+			join(dir, "output.md"),
+			"--output",
+			join(dir, "reward.json")
+		],
+		async parseReport(dir) {
+			const report = await readJsonReport(join(dir, "reward.json"));
+			const reward = report.reward;
+			if (typeof reward !== "number" || !Number.isFinite(reward)) throw new Error(`aec-bench verify.py wrote no numeric reward for ${meta.taskId}: ${JSON.stringify(report)}`);
+			const details = await readFile(join(dir, "details.json"), "utf8").then((s) => JSON.parse(s), () => ({}));
+			const score = Math.max(0, Math.min(1, reward));
+			return {
+				resolved: score >= 1,
+				score,
+				detail: JSON.stringify({
+					taskId: meta.taskId,
+					discipline: meta.discipline,
+					reward,
+					fields: details
+				})
+			};
+		}
+	});
+}
+function createAecBenchAdapter() {
+	const fixturesMode = process.env.AEC_FIXTURES === "1";
+	return {
+		name: "aec-bench",
+		async preflight() {
+			await preflightVenvImports({
+				modules: [
+					"math",
+					"json",
+					"re"
+				],
+				requireDocker: false,
+				fix: "Fix: python3 -m venv bench/.venv (verify.py only needs the stdlib — no pip install)."
+			});
+			if (fixturesMode) {
+				await readFile(FIXTURES, "utf8").catch((err) => {
+					throw new Error(`AEC_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`);
+				});
+				return;
+			}
+			const res = await fetch(`${RAW}/README.md`, { method: "HEAD" }).catch((err) => {
+				throw new Error(`aec-bench preflight failed reaching ${RAW}: ${err instanceof Error ? err.message : err}\nFix: ensure network access to raw.githubusercontent.com, or set AEC_FIXTURES=1 to run offline.`);
+			});
+			if (!res.ok) throw new Error(`aec-bench preflight: ${REPO} README HEAD ${res.status}. Set AEC_FIXTURES=1 to run against committed fixtures.`);
+		},
+		async loadTasks(opts = {}) {
+			if (fixturesMode) return loadFixtures(opts);
+			try {
+				return await loadLive(opts);
+			} catch (err) {
+				if (err instanceof Error && /fetch \d|ENOTFOUND|getaddrinfo|network/i.test(err.message)) {
+					console.warn(`[aec-bench] live fetch failed (${err.message.slice(0, 160)}); falling back to committed fixtures at ${FIXTURES}`);
+					return loadFixtures(opts);
+				}
+				throw err;
+			}
+		},
+		async goldArtifact(task) {
+			return readMeta(task).goldenPassMd ?? void 0;
+		},
+		async judge(task, artifact) {
+			return runVerifier(readMeta(task), artifact);
+		}
+	};
+}
+//#endregion
+export { createAecBenchAdapter };
+
 //# sourceMappingURL=aec-bench.js.map

package/dist/benchmarks/aec-bench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"aec-bench.js","names":[],"sources":["../../src/benchmarks/aec-bench.ts"],"sourcesContent":["/**\n * AEC-Bench adapter (TheodoreGalanos/aec-bench, MIT) — closed-form\n * Architecture/Engineering/Construction calculation tasks. Worker artifact = a\n * markdown solution ending in a fenced ```json block with the required numeric\n * fields. Judge = the task's OWN `tests/verify.py`, run with python3: it\n * recomputes ground truth from the embedded engineering formulas, extracts the\n * last JSON block from the artifact, scores each field by math.isclose within a\n * per-field rel_tol, and writes {\"reward\": mean} + per-field details.json.\n * GRADED / partial-credit, FULLY DETERMINISTIC — no LLM judge.\n *\n * Distinct from the multimodal nomic-ai/aec-bench; this is the deterministic\n * calculation platform. The runnable-instance verify.py only needs python3 (no\n * Docker, no Harbor) for the pure-calc disciplines, so the local gate runs at\n * conc<=2 without a container backend.\n *\n * Requires for a live run: network to raw.githubusercontent.com /\n * api.github.com (the in-repo tasks tree) + a python3 interpreter (the bench\n * venv) to run verify.py. For offline/CI, set AEC_FIXTURES=1 to load the\n * committed fixtures (bench/fixtures/aec-bench.json) — never a silent fallback.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport { benchRoot, preflightVenvImports, readJsonReport, runStagedJudge, stageFile, venvPython } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'aec-bench.json')\n\nconst REPO = 'TheodoreGalanos/aec-bench'\nconst RAW = `https://raw.githubusercontent.com/${REPO}/main`\nconst TREE = `https://api.github.com/repos/${REPO}/git/trees/main?recursive=1`\n\n/** Matches every runnable-instance task id at ANY depth under tasks/. */\nconst verifyPathPattern = /^tasks\\/(.+)\\/tests\\/verify\\.py$/\n\n/** Default cap on tasks enumerated before a `limit` is applied. */\nconst DEFAULT_LIMIT = 10\n\ninterface AecRecord {\n  /** Canonical task id `<discipline>/<task>`, e.g. 'electrical/catenary-sag'. */\n  id: string\n  discipline: string\n  /** instruction.md — the self-contained prompt (table + required outputs + JSON schema). */\n  instruction: string\n  /** task.toml — metadata/difficulty/timeouts (carried for trace context). */\n  task_toml: string\n  /** tests/verify.py — the deterministic verifier (recomputes GT, scores fields). */\n  verify_py: string\n  /** tests/fixtures/golden_pass.md — the oracle artifact that scores reward 1.0,\n   *  when the task ships one. Null when the task only ships a non-md ground truth\n   *  (e.g. tests/ground_truth.json); the judge never needs it, only goldArtifact does. */\n  golden_pass_md: string | null\n}\n\ninterface AecMeta {\n  taskId: string\n  discipline: string\n  taskToml: string\n  verifyPy: string\n  goldenPassMd: string | null\n}\n\nfunction recordToTask(rec: AecRecord): BenchTask {\n  const meta: AecMeta = {\n    taskId: rec.id,\n    discipline: rec.discipline,\n    taskToml: rec.task_toml,\n    verifyPy: rec.verify_py,\n    goldenPassMd: rec.golden_pass_md,\n  }\n  return {\n    id: rec.id,\n    split: rec.discipline,\n    // instruction.md is fully self-contained and already specifies the exact JSON\n    // output schema + the \"write to /workspace/output.md\" contract the verifier\n    // keys off — we pass it through verbatim so the verify.py extractor matches.\n    prompt: rec.instruction,\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): AecMeta {\n  const md = task.metadata\n  // Gold is optional (goldenPassMd may be null) — only the verifier + id are required.\n  if (!md || typeof md.verifyPy !== 'string' || typeof md.taskId !== 'string') {\n    throw new Error(`aec-bench task ${task.id} missing verifier metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as AecMeta\n}\n\nasync function fetchText(url: string): Promise<string> {\n  const res = await fetch(url)\n  if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`)\n  return res.text()\n}\n\n/** Like fetchText but returns null on a 404 (absent optional file); throws on any other non-OK. */\nasync function fetchTextOrNull(url: string): Promise<string | null> {\n  const res = await fetch(url)\n  if (res.status === 404) return null\n  if (!res.ok) throw new Error(`aec-bench fetch ${res.status}: ${url}`)\n  return res.text()\n}\n\ninterface GitTree {\n  tree: Array<{ path: string; type: string }>\n}\n\n/**\n * One recursive git-tree call enumerates EVERY runnable-instance id at any depth:\n * a task is runnable iff it ships tests/verify.py. The captured group is the id\n * `tasks/<id>/tests/verify.py` → `<id>` (e.g. 'electrical/pf-droop', or a deeper\n * '<discipline>/<family>/<task>'). Throws loud on a non-OK tree response.\n */\nasync function listAllInstances(): Promise<string[]> {\n  const res = await fetch(TREE)\n  if (!res.ok) throw new Error(`aec-bench tree ${res.status}: ${TREE}`)\n  const { tree } = (await res.json()) as GitTree\n  const ids: string[] = []\n  for (const entry of tree) {\n    const m = verifyPathPattern.exec(entry.path)\n    if (m?.[1]) ids.push(m[1])\n  }\n  return ids\n}\n\n/**\n * Fetch one task's instruction.md + task.toml + tests/verify.py + golden_pass.md.\n * Returns null when the dir is a SEED (no runnable verify.py) so enumeration can\n * skip it without faking a task. Gold is OPTIONAL — a task that ships a non-md\n * ground truth (e.g. tests/ground_truth.json) yields golden_pass_md=null; the\n * deterministic judge needs only verify.py.\n */\nasync function fetchInstance(id: string): Promise<AecRecord | null> {\n  const base = `${RAW}/tasks/${id}`\n  const verify = await fetch(`${base}/tests/verify.py`)\n  if (verify.status === 404) return null\n  if (!verify.ok) throw new Error(`aec-bench fetch ${verify.status}: ${id}/tests/verify.py`)\n  const [instruction, task_toml, golden_pass_md] = await Promise.all([\n    fetchText(`${base}/instruction.md`),\n    fetchText(`${base}/task.toml`),\n    fetchTextOrNull(`${base}/tests/fixtures/golden_pass.md`),\n  ])\n  return {\n    id,\n    discipline: id.split('/')[0] ?? '',\n    instruction,\n    task_toml,\n    verify_py: await verify.text(),\n    golden_pass_md,\n  }\n}\n\nfunction selectFixtures(records: AecRecord[], opts: LoadOptions): BenchTask[] {\n  let tasks = records.map(recordToTask)\n  if (opts.split) tasks = tasks.filter((t) => t.split === opts.split)\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((t) => want.has(t.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  return tasks\n}\n\nasync function loadFixtures(opts: LoadOptions): Promise<BenchTask[]> {\n  const records = JSON.parse(await readFile(FIXTURES, 'utf8')) as AecRecord[]\n  console.warn(\n    `[aec-bench] AEC_FIXTURES=1 — loading ${records.length} committed fixtures from ${FIXTURES} (no GitHub fetch)`,\n  )\n  return selectFixtures(records, opts)\n}\n\n/** Enumerate live tasks: explicit ids (each required — throws on a bad id), or a\n *  capped slice of the recursive tree (optionally filtered to one split). Per-task\n *  resilient: a single fetch failure warns + SKIPS that task, never aborting the\n *  batch. Skips seed dirs (no verify.py) — never fabricates. */\nasync function loadLive(opts: LoadOptions): Promise<BenchTask[]> {\n  if (opts.ids) {\n    const records: AecRecord[] = []\n    for (const id of opts.ids) {\n      const rec = await fetchInstance(id)\n      if (!rec) throw new Error(`aec-bench: ${id} has no tests/verify.py (seed-only or wrong id)`)\n      records.push(rec)\n    }\n    return records.map(recordToTask)\n  }\n  const limit = opts.limit ?? DEFAULT_LIMIT\n  let ids = await listAllInstances()\n  if (opts.split) ids = ids.filter((id) => id.startsWith(`${opts.split}/`))\n  const records: AecRecord[] = []\n  for (const id of ids) {\n    if (records.length >= limit) break\n    try {\n      const rec = await fetchInstance(id)\n      if (rec) records.push(rec)\n    } catch (err) {\n      // One bad task must NEVER abort the batch — warn and skip it.\n      console.warn(`[aec-bench] skipping ${id}: ${err instanceof Error ? err.message : String(err)}`)\n    }\n  }\n  if (records.length === 0) {\n    throw new Error(\n      `aec-bench loadTasks found no runnable instances for ${JSON.stringify(opts)} ` +\n        `(no tests/verify.py matched the requested split). Set AEC_FIXTURES=1 to run offline.`,\n    )\n  }\n  return records.map(recordToTask)\n}\n\n/**\n * Run the task's own verify.py with python3 over the artifact via the shared\n * staged-judge spine (mkdtemp → stage → spawn → parseReport → cleanup). verify.py\n * writes {\"reward\": mean} to --output and per-field details.json as a sibling. We\n * read both: reward → graded score; details → sub-scores for the trace-analyst.\n * Fail loud if the verifier never wrote a numeric reward (a crashed verifier is\n * NOT a silent 0 — verify.py's own except-trap writes reward 0.0, so an absent /\n * non-numeric reward.json is a real bug, surfaced by readJsonReport).\n */\nasync function runVerifier(meta: AecMeta, artifact: string): Promise<BenchScore> {\n  return runStagedJudge({\n    tmpPrefix: 'aecbench-',\n    timeoutMs: 120_000,\n    async stage(dir) {\n      await Promise.all([\n        stageFile(join(dir, 'output.md'), artifact),\n        stageFile(join(dir, 'verify.py'), meta.verifyPy),\n      ])\n    },\n    bin: venvPython,\n    argv: (dir) => [join(dir, 'verify.py'), '--input', join(dir, 'output.md'), '--output', join(dir, 'reward.json')],\n    async parseReport(dir) {\n      const report = await readJsonReport<{ reward?: number }>(join(dir, 'reward.json'))\n      const reward = report.reward\n      if (typeof reward !== 'number' || !Number.isFinite(reward)) {\n        throw new Error(`aec-bench verify.py wrote no numeric reward for ${meta.taskId}: ${JSON.stringify(report)}`)\n      }\n      const details = await readFile(join(dir, 'details.json'), 'utf8').then(\n        (s) => JSON.parse(s) as Record<string, number>,\n        () => ({}),\n      )\n      const score = Math.max(0, Math.min(1, reward))\n      return {\n        // resolved = full credit (all fields within tolerance), matching aec-bench's perfect_rate.\n        resolved: score >= 1,\n        score,\n        detail: JSON.stringify({ taskId: meta.taskId, discipline: meta.discipline, reward, fields: details }),\n      }\n    },\n  })\n}\n\nexport function createAecBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.AEC_FIXTURES === '1'\n\n  return {\n    name: 'aec-bench',\n\n    async preflight() {\n      // The verifier is python3 over the stdlib (math/json/re) — no pip install.\n      // Reuse the shared import-probe so the bench venv interpreter is proven to\n      // exist + run before any judge spawns verify.py.\n      await preflightVenvImports({\n        modules: ['math', 'json', 're'],\n        requireDocker: false,\n        fix: 'Fix: python3 -m venv bench/.venv (verify.py only needs the stdlib — no pip install).',\n      })\n      if (fixturesMode) {\n        await readFile(FIXTURES, 'utf8').catch((err) => {\n          throw new Error(`AEC_FIXTURES=1 but ${FIXTURES} unreadable: ${err instanceof Error ? err.message : err}`)\n        })\n        return\n      }\n      const res = await fetch(`${RAW}/README.md`, { method: 'HEAD' }).catch((err) => {\n        throw new Error(\n          `aec-bench preflight failed reaching ${RAW}: ${err instanceof Error ? err.message : err}\\n` +\n            `Fix: ensure network access to raw.githubusercontent.com, or set AEC_FIXTURES=1 to run offline.`,\n        )\n      })\n      if (!res.ok) {\n        throw new Error(\n          `aec-bench preflight: ${REPO} README HEAD ${res.status}. Set AEC_FIXTURES=1 to run against committed fixtures.`,\n        )\n      }\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      if (fixturesMode) return loadFixtures(opts)\n      try {\n        return await loadLive(opts)\n      } catch (err) {\n        // A reachability failure falls back to fixtures with an explicit warn; a\n        // wrong-id / seed-only error (the loader's own throw) propagates.\n        if (err instanceof Error && /fetch \\d|ENOTFOUND|getaddrinfo|network/i.test(err.message)) {\n          console.warn(\n            `[aec-bench] live fetch failed (${err.message.slice(0, 160)}); falling back to committed fixtures at ${FIXTURES}`,\n          )\n          return loadFixtures(opts)\n        }\n        throw err\n      }\n    },\n\n    async goldArtifact(task: BenchTask) {\n      // Gold = the task's own golden_pass.md (scores reward 1.0 through the SAME\n      // verify.py the real artifact takes), proving the judge end-to-end. Tasks\n      // without a golden_pass.md (non-md ground truth) have no oracle artifact.\n      const meta = readMeta(task)\n      return meta.goldenPassMd ?? undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      // verify.py fail-closes an empty/unparseable artifact to reward 0.0 itself,\n      // so we pass it straight through (no pre-judging here).\n      return runVerifier(meta, artifact)\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA0BA,MAAM,WAAW,KAAK,WAAW,YAAY,gBAAgB;AAE7D,MAAM,OAAO;AACb,MAAM,MAAM,qCAAqC,KAAK;AACtD,MAAM,OAAO,gCAAgC,KAAK;;AAGlD,MAAM,oBAAoB;;AAG1B,MAAM,gBAAgB;AA0BtB,SAAS,aAAa,KAA2B;CAC/C,MAAM,OAAgB;EACpB,QAAQ,IAAI;EACZ,YAAY,IAAI;EAChB,UAAU,IAAI;EACd,UAAU,IAAI;EACd,cAAc,IAAI;CACpB;CACA,OAAO;EACL,IAAI,IAAI;EACR,OAAO,IAAI;EAIX,QAAQ,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAA0B;CAC1C,MAAM,KAAK,KAAK;CAEhB,IAAI,CAAC,MAAM,OAAO,GAAG,aAAa,YAAY,OAAO,GAAG,WAAW,UACjE,MAAM,IAAI,MAAM,kBAAkB,KAAK,GAAG,2DAA2D;CAEvG,OAAO;AACT;AAEA,eAAe,UAAU,KAA8B;CACrD,MAAM,MAAM,MAAM,MAAM,GAAG;CAC3B,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,mBAAmB,IAAI,OAAO,IAAI,KAAK;CACpE,OAAO,IAAI,KAAK;AAClB;;AAGA,eAAe,gBAAgB,KAAqC;CAClE,MAAM,MAAM,MAAM,MAAM,GAAG;CAC3B,IAAI,IAAI,WAAW,KAAK,OAAO;CAC/B,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,mBAAmB,IAAI,OAAO,IAAI,KAAK;CACpE,OAAO,IAAI,KAAK;AAClB;;;;;;;AAYA,eAAe,mBAAsC;CACnD,MAAM,MAAM,MAAM,MAAM,IAAI;CAC5B,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,kBAAkB,IAAI,OAAO,IAAI,MAAM;CACpE,MAAM,EAAE,SAAU,MAAM,IAAI,KAAK;CACjC,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,SAAS,MAAM;EACxB,MAAM,IAAI,kBAAkB,KAAK,MAAM,IAAI;EAC3C,IAAI,IAAI,IAAI,IAAI,KAAK,EAAE,EAAE;CAC3B;CACA,OAAO;AACT;;;;;;;;AASA,eAAe,cAAc,IAAuC;CAClE,MAAM,OAAO,GAAG,IAAI,SAAS;CAC7B,MAAM,SAAS,MAAM,MAAM,GAAG,KAAK,iBAAiB;CACpD,IAAI,OAAO,WAAW,KAAK,OAAO;CAClC,IAAI,CAAC,OAAO,IAAI,MAAM,IAAI,MAAM,mBAAmB,OAAO,OAAO,IAAI,GAAG,iBAAiB;CACzF,MAAM,CAAC,aAAa,WAAW,kBAAkB,MAAM,QAAQ,IAAI;EACjE,UAAU,GAAG,KAAK,gBAAgB;EAClC,UAAU,GAAG,KAAK,WAAW;EAC7B,gBAAgB,GAAG,KAAK,+BAA+B;CACzD,CAAC;CACD,OAAO;EACL;EACA,YAAY,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM;EAChC;EACA;EACA,WAAW,MAAM,OAAO,KAAK;EAC7B;CACF;AACF;AAEA,SAAS,eAAe,SAAsB,MAAgC;CAC5E,IAAI,QAAQ,QAAQ,IAAI,YAAY;CACpC,IAAI,KAAK,OAAO,QAAQ,MAAM,QAAQ,MAAM,EAAE,UAAU,KAAK,KAAK;CAClE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,MAAM,KAAK,IAAI,EAAE,EAAE,CAAC;CAC5C,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,OAAO;AACT;AAEA,eAAe,aAAa,MAAyC;CACnE,MAAM,UAAU,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CAC3D,QAAQ,KACN,wCAAwC,QAAQ,OAAO,2BAA2B,SAAS,mBAC7F;CACA,OAAO,eAAe,SAAS,IAAI;AACrC;;;;;AAMA,eAAe,SAAS,MAAyC;CAC/D,IAAI,KAAK,KAAK;EACZ,MAAM,UAAuB,CAAC;EAC9B,KAAK,MAAM,MAAM,KAAK,KAAK;GACzB,MAAM,MAAM,MAAM,cAAc,EAAE;GAClC,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,cAAc,GAAG,gDAAgD;GAC3F,QAAQ,KAAK,GAAG;EAClB;EACA,OAAO,QAAQ,IAAI,YAAY;CACjC;CACA,MAAM,QAAQ,KAAK,SAAS;CAC5B,IAAI,MAAM,MAAM,iBAAiB;CACjC,IAAI,KAAK,OAAO,MAAM,IAAI,QAAQ,OAAO,GAAG,WAAW,GAAG,KAAK,MAAM,EAAE,CAAC;CACxE,MAAM,UAAuB,CAAC;CAC9B,KAAK,MAAM,MAAM,KAAK;EACpB,IAAI,QAAQ,UAAU,OAAO;EAC7B,IAAI;GACF,MAAM,MAAM,MAAM,cAAc,EAAE;GAClC,IAAI,KAAK,QAAQ,KAAK,GAAG;EAC3B,SAAS,KAAK;GAEZ,QAAQ,KAAK,wBAAwB,GAAG,IAAI,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GAAG;EAChG;CACF;CACA,IAAI,QAAQ,WAAW,GACrB,MAAM,IAAI,MACR,uDAAuD,KAAK,UAAU,IAAI,EAAE,sFAE9E;CAEF,OAAO,QAAQ,IAAI,YAAY;AACjC;;;;;;;;;;AAWA,eAAe,YAAY,MAAe,UAAuC;CAC/E,OAAO,eAAe;EACpB,WAAW;EACX,WAAW;EACX,MAAM,MAAM,KAAK;GACf,MAAM,QAAQ,IAAI,CAChB,UAAU,KAAK,KAAK,WAAW,GAAG,QAAQ,GAC1C,UAAU,KAAK,KAAK,WAAW,GAAG,KAAK,QAAQ,CACjD,CAAC;EACH;EACA,KAAK;EACL,OAAO,QAAQ;GAAC,KAAK,KAAK,WAAW;GAAG;GAAW,KAAK,KAAK,WAAW;GAAG;GAAY,KAAK,KAAK,aAAa;EAAC;EAC/G,MAAM,YAAY,KAAK;GACrB,MAAM,SAAS,MAAM,eAAoC,KAAK,KAAK,aAAa,CAAC;GACjF,MAAM,SAAS,OAAO;GACtB,IAAI,OAAO,WAAW,YAAY,CAAC,OAAO,SAAS,MAAM,GACvD,MAAM,IAAI,MAAM,mDAAmD,KAAK,OAAO,IAAI,KAAK,UAAU,MAAM,GAAG;GAE7G,MAAM,UAAU,MAAM,SAAS,KAAK,KAAK,cAAc,GAAG,MAAM,CAAC,CAAC,MAC/D,MAAM,KAAK,MAAM,CAAC,UACZ,CAAC,EACV;GACA,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;GAC7C,OAAO;IAEL,UAAU,SAAS;IACnB;IACA,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,YAAY,KAAK;KAAY;KAAQ,QAAQ;IAAQ,CAAC;GACtG;EACF;CACF,CAAC;AACH;AAEA,SAAgB,wBAA0C;CACxD,MAAM,eAAe,QAAQ,IAAI,iBAAiB;CAElD,OAAO;EACL,MAAM;EAEN,MAAM,YAAY;GAIhB,MAAM,qBAAqB;IACzB,SAAS;KAAC;KAAQ;KAAQ;IAAI;IAC9B,eAAe;IACf,KAAK;GACP,CAAC;GACD,IAAI,cAAc;IAChB,MAAM,SAAS,UAAU,MAAM,CAAC,CAAC,OAAO,QAAQ;KAC9C,MAAM,IAAI,MAAM,sBAAsB,SAAS,eAAe,eAAe,QAAQ,IAAI,UAAU,KAAK;IAC1G,CAAC;IACD;GACF;GACA,MAAM,MAAM,MAAM,MAAM,GAAG,IAAI,aAAa,EAAE,QAAQ,OAAO,CAAC,CAAC,CAAC,OAAO,QAAQ;IAC7E,MAAM,IAAI,MACR,uCAAuC,IAAI,IAAI,eAAe,QAAQ,IAAI,UAAU,IAAI,iGAE1F;GACF,CAAC;GACD,IAAI,CAAC,IAAI,IACP,MAAM,IAAI,MACR,wBAAwB,KAAK,eAAe,IAAI,OAAO,wDACzD;EAEJ;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,IAAI,cAAc,OAAO,aAAa,IAAI;GAC1C,IAAI;IACF,OAAO,MAAM,SAAS,IAAI;GAC5B,SAAS,KAAK;IAGZ,IAAI,eAAe,SAAS,0CAA0C,KAAK,IAAI,OAAO,GAAG;KACvF,QAAQ,KACN,kCAAkC,IAAI,QAAQ,MAAM,GAAG,GAAG,EAAE,2CAA2C,UACzG;KACA,OAAO,aAAa,IAAI;IAC1B;IACA,MAAM;GACR;EACF;EAEA,MAAM,aAAa,MAAiB;GAKlC,OADa,SAAS,IACZ,CAAC,CAAC,gBAAgB,KAAA;EAC9B;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAIlE,OAAO,YAHM,SAAS,IAGA,GAAG,QAAQ;EACnC;CACF;AACF"}

package/dist/benchmarks/agentbench.d.ts

@@ -1,16 +1,8 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * AgentBench deterministic subset adapter.
- *
- * This targets AgentBench DBBench rows only: question + table + published label.
- * It does not wrap AgentBench's controller protocol or the non-deterministic game
- * environments. Worker artifact = final answer text. Judge = exact match against
- * the official DBBench label list after light whitespace/case normalization.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/agentbench.d.ts
 declare const agentbenchAnswerOutput: OutputAdapter<string>;
 declare function createAgentBenchAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { agentbenchAnswerOutput, createAgentBenchAdapter };
+//# sourceMappingURL=agentbench.d.ts.map

package/dist/benchmarks/agentbench.js

@@ -1,10 +1,115 @@
-import {
-  agentbenchAnswerOutput,
-  createAgentBenchAdapter
-} from "../chunk-PB64GYIG.js";
-import "../chunk-WSKWVEQB.js";
-export {
-  agentbenchAnswerOutput,
-  createAgentBenchAdapter
-};
+import { benchRoot } from "./_harness.js";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+//#region src/benchmarks/agentbench.ts
+/**
+* AgentBench deterministic subset adapter.
+*
+* This targets AgentBench DBBench rows only: question + table + published label.
+* It does not wrap AgentBench's controller protocol or the non-deterministic game
+* environments. Worker artifact = final answer text. Judge = exact match against
+* the official DBBench label list after light whitespace/case normalization.
+*/
+const FIXTURES = join(benchRoot, "fixtures", "agentbench-dbbench.json");
+const DEFAULT_SPLIT = "dev";
+const agentbenchDir = () => process.env.AGENTBENCH_DIR;
+const agentbenchAnswerOutput = { parse(events) {
+	let text = "";
+	for (const ev of events) {
+		const d = ev?.data;
+		const t = d?.finalText ?? d?.text ?? d?.result;
+		if (typeof t === "string" && t.length > 0) text = t;
+	}
+	return ([...text.matchAll(/```(?:text|answer)?\s*\n([\s\S]*?)```/g)].at(-1)?.[1] ?? text).trim();
+} };
+function normalizeAnswer(value) {
+	return value.toLowerCase().replace(/\s+/g, " ").replace(/^["'`]+|["'`]+$/g, "").trim();
+}
+function rowToTask(row, index, split) {
+	const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ""}`).join(", ");
+	const sampleRows = row.table?.table_info?.rows?.slice(0, 40);
+	const meta = {
+		labels: row.label,
+		split,
+		subset: "dbbench",
+		table: row.table
+	};
+	return {
+		id: `dbbench-${split}-${index}`,
+		split,
+		prompt: [
+			"Answer this AgentBench DBBench question using the table below.",
+			"Return only the answer value.",
+			"",
+			`Question: ${row.description}`,
+			row.table?.table_name ? `Table: ${row.table.table_name}` : void 0,
+			columns ? `Columns: ${columns}` : void 0,
+			sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : void 0
+		].filter(Boolean).join("\n"),
+		metadata: meta
+	};
+}
+function readMeta(task) {
+	const md = task.metadata;
+	if (!md || !Array.isArray(md.labels)) throw new Error(`agentbench task ${task.id} missing metadata — loadTasks did not populate it`);
+	return md;
+}
+function selectRows(rows, opts, split) {
+	let tasks = rows.map((row, index) => rowToTask(row, index, split));
+	if (opts.ids) {
+		const want = new Set(opts.ids);
+		tasks = tasks.filter((task) => want.has(task.id));
+	} else if (opts.limit !== void 0) tasks = tasks.slice(0, opts.limit);
+	if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`);
+	return tasks;
+}
+async function loadJsonl(path) {
+	return (await readFile(path, "utf8")).split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
+}
+async function loadFixtures(opts, split) {
+	const rows = JSON.parse(await readFile(FIXTURES, "utf8"));
+	console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 — loading ${rows.length} DBBench adapter fixtures`);
+	return selectRows(rows, opts, split);
+}
+function createAgentBenchAdapter() {
+	const fixturesMode = process.env.AGENTBENCH_FIXTURES === "1";
+	return {
+		name: "agentbench",
+		output: agentbenchAnswerOutput,
+		async preflight() {
+			if (fixturesMode) return;
+			const dir = agentbenchDir();
+			if (!dir) throw new Error("AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.");
+			await loadJsonl(join(dir, "data", "dbbench", `${DEFAULT_SPLIT}.jsonl`));
+		},
+		async loadTasks(opts = {}) {
+			const split = opts.split ?? DEFAULT_SPLIT;
+			if (fixturesMode) return loadFixtures(opts, split);
+			const dir = agentbenchDir();
+			if (!dir) throw new Error("AGENTBENCH_DIR is required to load AgentBench DBBench tasks");
+			return selectRows(await loadJsonl(join(dir, "data", "dbbench", `${split}.jsonl`)), opts, split);
+		},
+		async goldArtifact(task) {
+			return readMeta(task).labels[0];
+		},
+		async judge(task, artifact) {
+			const meta = readMeta(task);
+			const got = normalizeAnswer(artifact);
+			const resolved = meta.labels.map(normalizeAnswer).includes(got);
+			return {
+				resolved,
+				score: resolved ? 1 : 0,
+				detail: JSON.stringify({
+					subset: meta.subset,
+					split: meta.split,
+					expected: meta.labels,
+					got: artifact
+				})
+			};
+		}
+	};
+}
+//#endregion
+export { agentbenchAnswerOutput, createAgentBenchAdapter };
+
 //# sourceMappingURL=agentbench.js.map

package/dist/benchmarks/agentbench.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"agentbench.js","names":[],"sources":["../../src/benchmarks/agentbench.ts"],"sourcesContent":["/**\n * AgentBench deterministic subset adapter.\n *\n * This targets AgentBench DBBench rows only: question + table + published label.\n * It does not wrap AgentBench's controller protocol or the non-deterministic game\n * environments. Worker artifact = final answer text. Judge = exact match against\n * the official DBBench label list after light whitespace/case normalization.\n */\n\nimport { readFile } from 'node:fs/promises'\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/loops'\nimport { benchRoot } from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\nconst FIXTURES = join(benchRoot, 'fixtures', 'agentbench-dbbench.json')\nconst DEFAULT_SPLIT = 'dev'\n\ninterface AgentBenchDbRow {\n  description: string\n  label: string[]\n  table?: {\n    table_name?: string\n    table_info?: {\n      columns?: Array<{ name: string; type?: string }>\n      rows?: unknown[][]\n    }\n  }\n}\n\ninterface AgentBenchMeta {\n  labels: string[]\n  split: string\n  subset: 'dbbench'\n  table?: AgentBenchDbRow['table']\n}\n\nconst agentbenchDir = (): string | undefined => process.env.AGENTBENCH_DIR\n\nexport const agentbenchAnswerOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    const fences = [...text.matchAll(/```(?:text|answer)?\\s*\\n([\\s\\S]*?)```/g)]\n    return (fences.at(-1)?.[1] ?? text).trim()\n  },\n}\n\nfunction normalizeAnswer(value: string): string {\n  return value\n    .toLowerCase()\n    .replace(/\\s+/g, ' ')\n    .replace(/^[\"'`]+|[\"'`]+$/g, '')\n    .trim()\n}\n\nfunction rowToTask(row: AgentBenchDbRow, index: number, split: string): BenchTask {\n  const columns = row.table?.table_info?.columns?.map((c) => `${c.name}${c.type ? ` (${c.type})` : ''}`).join(', ')\n  const sampleRows = row.table?.table_info?.rows?.slice(0, 40)\n  const meta: AgentBenchMeta = {\n    labels: row.label,\n    split,\n    subset: 'dbbench',\n    table: row.table,\n  }\n  return {\n    id: `dbbench-${split}-${index}`,\n    split,\n    prompt: [\n      'Answer this AgentBench DBBench question using the table below.',\n      'Return only the answer value.',\n      '',\n      `Question: ${row.description}`,\n      row.table?.table_name ? `Table: ${row.table.table_name}` : undefined,\n      columns ? `Columns: ${columns}` : undefined,\n      sampleRows ? `Rows JSON: ${JSON.stringify(sampleRows)}` : undefined,\n    ]\n      .filter(Boolean)\n      .join('\\n'),\n    metadata: meta as unknown as Record<string, unknown>,\n  }\n}\n\nfunction readMeta(task: BenchTask): AgentBenchMeta {\n  const md = task.metadata\n  if (!md || !Array.isArray(md.labels)) {\n    throw new Error(`agentbench task ${task.id} missing metadata — loadTasks did not populate it`)\n  }\n  return md as unknown as AgentBenchMeta\n}\n\nfunction selectRows(rows: AgentBenchDbRow[], opts: LoadOptions, split: string): BenchTask[] {\n  let tasks = rows.map((row, index) => rowToTask(row, index, split))\n  if (opts.ids) {\n    const want = new Set(opts.ids)\n    tasks = tasks.filter((task) => want.has(task.id))\n  } else if (opts.limit !== undefined) {\n    tasks = tasks.slice(0, opts.limit)\n  }\n  if (tasks.length === 0) throw new Error(`AgentBench DBBench: no tasks matched ${JSON.stringify(opts)}`)\n  return tasks\n}\n\nasync function loadJsonl(path: string): Promise<AgentBenchDbRow[]> {\n  const raw = await readFile(path, 'utf8')\n  return raw\n    .split('\\n')\n    .map((line) => line.trim())\n    .filter(Boolean)\n    .map((line) => JSON.parse(line) as AgentBenchDbRow)\n}\n\nasync function loadFixtures(opts: LoadOptions, split: string): Promise<BenchTask[]> {\n  const rows = JSON.parse(await readFile(FIXTURES, 'utf8')) as AgentBenchDbRow[]\n  console.warn(`[agentbench] AGENTBENCH_FIXTURES=1 — loading ${rows.length} DBBench adapter fixtures`)\n  return selectRows(rows, opts, split)\n}\n\nexport function createAgentBenchAdapter(): BenchmarkAdapter {\n  const fixturesMode = process.env.AGENTBENCH_FIXTURES === '1'\n\n  return {\n    name: 'agentbench',\n    output: agentbenchAnswerOutput,\n\n    async preflight() {\n      if (fixturesMode) return\n      const dir = agentbenchDir()\n      if (!dir) {\n        throw new Error('AGENTBENCH_DIR is required. Fix: clone https://github.com/THUDM/AgentBench and set AGENTBENCH_DIR=/path/to/AgentBench.')\n      }\n      await loadJsonl(join(dir, 'data', 'dbbench', `${DEFAULT_SPLIT}.jsonl`))\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const split = opts.split ?? DEFAULT_SPLIT\n      if (fixturesMode) return loadFixtures(opts, split)\n      const dir = agentbenchDir()\n      if (!dir) throw new Error('AGENTBENCH_DIR is required to load AgentBench DBBench tasks')\n      return selectRows(await loadJsonl(join(dir, 'data', 'dbbench', `${split}.jsonl`)), opts, split)\n    },\n\n    async goldArtifact(task: BenchTask) {\n      return readMeta(task).labels[0]\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const meta = readMeta(task)\n      const got = normalizeAnswer(artifact)\n      const expected = meta.labels.map(normalizeAnswer)\n      const resolved = expected.includes(got)\n      return {\n        resolved,\n        score: resolved ? 1 : 0,\n        detail: JSON.stringify({ subset: meta.subset, split: meta.split, expected: meta.labels, got: artifact }),\n      }\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAeA,MAAM,WAAW,KAAK,WAAW,YAAY,yBAAyB;AACtE,MAAM,gBAAgB;AAqBtB,MAAM,sBAA0C,QAAQ,IAAI;AAE5D,MAAa,yBAAgD,EAC3D,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAEA,QAAQ,CADQ,GAAG,KAAK,SAAS,wCAAwC,CAC5D,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,KAAA,CAAM,KAAK;AAC3C,EACF;AAEA,SAAS,gBAAgB,OAAuB;CAC9C,OAAO,MACJ,YAAY,CAAC,CACb,QAAQ,QAAQ,GAAG,CAAC,CACpB,QAAQ,oBAAoB,EAAE,CAAC,CAC/B,KAAK;AACV;AAEA,SAAS,UAAU,KAAsB,OAAe,OAA0B;CAChF,MAAM,UAAU,IAAI,OAAO,YAAY,SAAS,KAAK,MAAM,GAAG,EAAE,OAAO,EAAE,OAAO,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI;CAChH,MAAM,aAAa,IAAI,OAAO,YAAY,MAAM,MAAM,GAAG,EAAE;CAC3D,MAAM,OAAuB;EAC3B,QAAQ,IAAI;EACZ;EACA,QAAQ;EACR,OAAO,IAAI;CACb;CACA,OAAO;EACL,IAAI,WAAW,MAAM,GAAG;EACxB;EACA,QAAQ;GACN;GACA;GACA;GACA,aAAa,IAAI;GACjB,IAAI,OAAO,aAAa,UAAU,IAAI,MAAM,eAAe,KAAA;GAC3D,UAAU,YAAY,YAAY,KAAA;GAClC,aAAa,cAAc,KAAK,UAAU,UAAU,MAAM,KAAA;EAC5D,CAAC,CACE,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;EACZ,UAAU;CACZ;AACF;AAEA,SAAS,SAAS,MAAiC;CACjD,MAAM,KAAK,KAAK;CAChB,IAAI,CAAC,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,GACjC,MAAM,IAAI,MAAM,mBAAmB,KAAK,GAAG,kDAAkD;CAE/F,OAAO;AACT;AAEA,SAAS,WAAW,MAAyB,MAAmB,OAA4B;CAC1F,IAAI,QAAQ,KAAK,KAAK,KAAK,UAAU,UAAU,KAAK,OAAO,KAAK,CAAC;CACjE,IAAI,KAAK,KAAK;EACZ,MAAM,OAAO,IAAI,IAAI,KAAK,GAAG;EAC7B,QAAQ,MAAM,QAAQ,SAAS,KAAK,IAAI,KAAK,EAAE,CAAC;CAClD,OAAO,IAAI,KAAK,UAAU,KAAA,GACxB,QAAQ,MAAM,MAAM,GAAG,KAAK,KAAK;CAEnC,IAAI,MAAM,WAAW,GAAG,MAAM,IAAI,MAAM,wCAAwC,KAAK,UAAU,IAAI,GAAG;CACtG,OAAO;AACT;AAEA,eAAe,UAAU,MAA0C;CAEjE,QAAO,MADW,SAAS,MAAM,MAAM,EAAA,CAEpC,MAAM,IAAI,CAAC,CACX,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,KAAK,SAAS,KAAK,MAAM,IAAI,CAAoB;AACtD;AAEA,eAAe,aAAa,MAAmB,OAAqC;CAClF,MAAM,OAAO,KAAK,MAAM,MAAM,SAAS,UAAU,MAAM,CAAC;CACxD,QAAQ,KAAK,gDAAgD,KAAK,OAAO,0BAA0B;CACnG,OAAO,WAAW,MAAM,MAAM,KAAK;AACrC;AAEA,SAAgB,0BAA4C;CAC1D,MAAM,eAAe,QAAQ,IAAI,wBAAwB;CAEzD,OAAO;EACL,MAAM;EACN,QAAQ;EAER,MAAM,YAAY;GAChB,IAAI,cAAc;GAClB,MAAM,MAAM,cAAc;GAC1B,IAAI,CAAC,KACH,MAAM,IAAI,MAAM,wHAAwH;GAE1I,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,cAAc,OAAO,CAAC;EACxE;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,IAAI,cAAc,OAAO,aAAa,MAAM,KAAK;GACjD,MAAM,MAAM,cAAc;GAC1B,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,6DAA6D;GACvF,OAAO,WAAW,MAAM,UAAU,KAAK,KAAK,QAAQ,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,MAAM,KAAK;EAChG;EAEA,MAAM,aAAa,MAAiB;GAClC,OAAO,SAAS,IAAI,CAAC,CAAC,OAAO;EAC/B;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,OAAO,SAAS,IAAI;GAC1B,MAAM,MAAM,gBAAgB,QAAQ;GAEpC,MAAM,WADW,KAAK,OAAO,IAAI,eACT,CAAC,CAAC,SAAS,GAAG;GACtC,OAAO;IACL;IACA,OAAO,WAAW,IAAI;IACtB,QAAQ,KAAK,UAAU;KAAE,QAAQ,KAAK;KAAQ,OAAO,KAAK;KAAO,UAAU,KAAK;KAAQ,KAAK;IAAS,CAAC;GACzG;EACF;CACF;AACF"}

package/dist/benchmarks/appworld.d.ts

@@ -1,37 +1,17 @@
-import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
-import { BenchmarkAdapter } from './types.js';
-
-/**
- * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's
- * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`
- * surface AppWorld exposes inside `world.execute(...)`), ending in
- * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic
- * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,
- * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`
- * (per-requirement total) and the `passes`/`failures` lists. Score =
- * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.
- *
- * loadTasks enumerates the real task suite via `load_task_ids(split)`
- * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.
- * The OutputAdapter is stream-only, so the worker emits its solution as a fenced
- * ```python block which the driver executes.
- *
- * Requires for a live run: the bench `.venv` with `appworld` installed + the
- * unpacked engine + downloaded data (`appworld install` ; `appworld download
- * data`). preflight + loadTasks + judge all fail loud with the exact step when the
- * engine/data is absent — never a fabricated score.
- */
-
+import { BenchmarkAdapter } from "./types.js";
+import { OutputAdapter } from "@tangle-network/agent-runtime/loops";
+//#region src/benchmarks/appworld.d.ts
 /** Worker solution code = the last fenced ```python block, else the raw text. */
 declare const appworldSolutionOutput: OutputAdapter<string>;
 declare function createAppWorldAdapter(): BenchmarkAdapter;
 /** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */
 declare function appworldToolLoopClient(cfg: {
-    model: string;
-    routerBaseUrl: string;
-    routerKey: string;
-    maxTurns?: number;
+  model: string;
+  routerBaseUrl: string;
+  routerKey: string;
+  maxTurns?: number;
 }): unknown;
 declare function createAppWorldReactAdapter(): BenchmarkAdapter;
-
+//#endregion
 export { appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };
+//# sourceMappingURL=appworld.d.ts.map