@tangle-network/agent-bench 0.8.0 → 0.8.1

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.8.1
+
+- Consume Eval 0.144.8 and Knowledge 7.1.3 so Bench uses the same published dependency family as Runtime 0.131.6.
+
 ## 0.8.0
 
 - Consume Runtime 0.131.0, Eval 0.144.6, Interface 0.46.1, Knowledge 7.1.2, and Sandbox 0.19.4 as one compatible dependency set.

package/HARNESS.md

@@ -259,7 +259,7 @@ SWE-bench callers that need a hard evaluation deadline construct `createSweBench
 Callers that must inspect the exact per-task image after scoring pass `cacheLevel: 'instance'`; the default remains `'env'`.
 Callers retain the complete official evaluator tree plus raw process output with `captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({ destination })`; the returned score (or `StagedJudgeError`) carries a receipt with every file hash and a whole-tree hash.
 - **Real, runnable with ZERO extra deps:** finsearchcomp (GitHub dataset + fixtures + LLM judge — the gate bench), hotpotqa + simpleqa + frames (HF/web QA + F1/LLM judge; `*_FIXTURES=1` offline), **ragbench**, **crag**, **nomiracl**, **open-rag-bench**, **t2-ragbench** (SOTA RAG/knowledge benchmarks with committed fixtures and deterministic answer/relevance judges; live mode reads explicit `*_DATA_FILE` JSON/JSONL exports), **aec-bench** (real GitHub task tree + fixtures; judge = the task's own `tests/verify.py` over python3 stdlib — **deterministic, graded per-field partial credit, no Docker, no LLM** → the candidate non-oracle correctable-middle-band bench for the open gate).
-- **Real code, needs an external harness/tools to run (fail loud with the exact install/Docker fix; never a fabricated score):** swe-bench + terminal-bench (`bench/.venv` + Docker), **commit0** (ISOLATED `bench/.venv-commit0` via `python3 -m venv bench/.venv-commit0 && bench/.venv-commit0/bin/pip install commit0 datasets` — its deps conflict with the shared `.venv`; override dir with `COMMIT0_VENV` — plus Docker; judge = official pytest harness, graded (passed+xfail)/total; the rollout prompt stages in-box (clones `commit-0/<repo>` @ `base_commit`, emits `git diff`); `COMMIT0_FIXTURES=1` for offline listing), **programbench** (`pip install programbench` + Docker on linux/amd64 + HF blobs; judge = official cleanroom eval, graded passed/total; `PROGRAMBENCH_FIXTURES=1` offline), **appworld** (`pip install appworld` + `appworld install` + `appworld download data`; judge = AppWorld's own `world.evaluate()`, graded passes/num_tests — NO committed fixture: task data exists only after `download data`, so loadTasks fails loud rather than fabricate a task), **dabstep** (`DABSTEP_DIR=/path/to/EnvCommons/DABStep` with the released `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`; judge delegates to official `grade.py`; `DABSTEP_FIXTURES=1` only tests adapter plumbing and does not fabricate benchmark scores), **webarena-verified** (`WEBARENA_VERIFIED_DIR=/path/to/webarena-verified`; judge delegates to official `eval-tasks` over a run output directory), **tau2-bench** (`TAU2_BENCH_DIR=/path/to/tau2-bench`; judge recomputes tau2 trajectory rewards), **tau3-banking** (`TAU3_BENCH_DIR=/path/to/tau2-bench`; default domain `banking_knowledge`; judge recomputes tau trajectory rewards through the upstream tau3 package), **agentbench** DBBench subset (`AGENTBENCH_DIR=/path/to/AgentBench`; exact-match deterministic label judge), **bfcl** deterministic function-call subset (`BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard`; loads official BFCL JSONL + `possible_answer`; score = structured call/argument match, not the full BFCL leaderboard evaluator), **toollm** API-selection subset (`TOOLBENCH_DIR=/path/to/ToolBench`; score = recall of ToolBench `relevant APIs` labels, resolved only when the worker emits the requested structured JSON call list; official ToolEval pass rate remains LLM-judged/stochastic), **finresearchbench** (`FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl`; rows must carry official `judge_system_prompt` + `judge_prompt_template`; no self-authored live judge), mind2web, cad-design + cadbench + cadgenbench (openscad/blender/build123d).
+- **Real code, needs an external harness/tools to run (fail loud with the exact install/Docker fix; never a fabricated score):** swe-bench + terminal-bench (`bench/.venv` + Docker), **commit0** (ISOLATED `bench/.venv-commit0` via `python3 -m venv bench/.venv-commit0 && bench/.venv-commit0/bin/pip install commit0 datasets` — its deps conflict with the shared `.venv`; override dir with `COMMIT0_VENV` — plus Docker; judge = official pytest harness, graded (passed+xfail)/total; the rollout prompt stages in-box (clones `commit-0/<repo>` @ `base_commit`, emits `git diff`); `COMMIT0_FIXTURES=1` for offline listing), **programbench** (`pip install programbench` + Docker on linux/amd64 + HF blobs; judge = official cleanroom eval, graded passed/total; `PROGRAMBENCH_FIXTURES=1` offline), **appworld** (`pip install appworld` + `appworld install` + `appworld download data`; judge = AppWorld's own `world.evaluate()`, graded passes/num_tests — NO committed fixture: task data exists only after `download data`, so loadTasks fails loud rather than fabricate a task), **dabstep** (`DABSTEP_DIR=/path/to/EnvCommons/DABStep` with the released `dataset.csv`, `splits/*.txt`, `files/*`, and `grade.py`; judge delegates to official `grade.py`; `DABSTEP_FIXTURES=1` only tests adapter plumbing and does not fabricate benchmark scores), **webarena-verified** (`WEBARENA_VERIFIED_DIR=/path/to/webarena-verified`; judge delegates to official `eval-tasks` over a run output directory), **tau2-bench** (`TAU2_BENCH_DIR=/path/to/tau2-bench`; judge recomputes tau2 trajectory rewards), **tau3-banking** (`TAU3_BENCH_DIR=/path/to/tau2-bench`; default domain `banking_knowledge`; judge recomputes tau trajectory rewards through the upstream tau3 package), **agentbench** DBBench subset (`AGENTBENCH_DIR=/path/to/AgentBench`; exact-match deterministic label judge), **bfcl** deterministic function-call subset (`BFCL_DIR=/path/to/gorilla/berkeley-function-call-leaderboard`; loads official BFCL JSONL + `possible_answer`; score = structured call/argument match, not the full BFCL leaderboard evaluator), **toollm** API-selection subset (`TOOLBENCH_DIR=/path/to/ToolBench`; score = recall of ToolBench `relevant APIs` labels, resolved only when the worker emits the requested structured JSON call list; official ToolEval pass rate remains LLM-judged/stochastic), **finresearchbench** (`FINRESEARCHBENCH_DATA_FILE=/path/to/export.jsonl`; rows must carry official `judge_system_prompt` + `judge_prompt_template`; no self-authored live judge), mind2web, cad-design + cadbench + cadgenbench (openscad/blender/build123d), **mcad** (openscad + xvfb-run; 10 dimensioned mechanical parts adopted from the MIT text-to-cad benchmark set; judge = OpenSCAD compile then pure-TS mesh assertions — bbox, closed-form volume bands, body count, and ray-parity point-in-solid probes at the spec's pinned hole coordinates; all 10 tasks carry a verified gold, and the must-reject direction is tested: one missing bore of four fails on its named probe), **mcad-cq** (the SAME 10 parts, tasks and spec assertions, but the worker writes a Python **CadQuery** script that must export BOTH `part.step` and an ASCII `part.stl` — which closes the format deviation `mcad-tasks.ts` documents, since upstream asks for STEP and OpenSCAD cannot emit it; the geometry engine is imported from `mcad-bench.ts`, not copied, and STEP delivery is one EXTRA scored check `stepEmitted` = `part.step` starts with `ISO-10303-21`; 120 s deadline per script, all 10 tasks carry a verified CadQuery gold at score 1.0, and four must-rejects are tested — one bore deleted, no STL, a binary STL, and the STEP export dropped. Needs an ISOLATED `bench/.venv-cadquery`: `cd bench && uv venv --python 3.12 .venv-cadquery && uv pip install --python .venv-cadquery/bin/python 'cadquery==2.4.0' 'numpy<2'` — both pins are load-bearing, cadquery-ocp 7.7.2 has no cp313 wheel and cadquery 2.4.0's pinned nptyping 2.0.1 reads the numpy-1.x-only `np.bool8`; override with the absolute path `MCAD_CQ_PYTHON`. The judge RUNS the worker's Python on the host, same trust level as `mcad` running arbitrary OpenSCAD).
 - **goldArtifact:** aec-bench returns the task's real `golden_pass.md` (verify-judge works fully offline). commit0 / programbench / appworld return `undefined` — the oracle is a git ref / stripped source / engine-bundled solution, not a portable string; judge correctness is proven by a real solve through the harness, not a synthetic gold (documented + fail-loud, not a fake).
 - **Absent (not built):** swe-gym, swe-bench-multimodal, and the rest of the survey set.
 Every unbuilt/scaffold adapter fails LOUD (throws with the integration step) rather than faking a score — no silent zeros in any corpus. Offline fixture tests: `benchmarks/{aec-bench,commit0,programbench,appworld,rag-benchmarks}.test.mts` (`tsx --test`).

package/dist/adapters.js

@@ -14,6 +14,8 @@ import { createFinsearchcompAdapter } from "./benchmarks/finsearchcomp.js";
 import { createFramesAdapter } from "./benchmarks/frames.js";
 import { createHotpotqaAdapter } from "./benchmarks/hotpotqa.js";
 import { createHumanEvalAdapter } from "./benchmarks/humaneval.js";
+import { createMcadBenchAdapter } from "./benchmarks/mcad-bench.js";
+import { createMcadCqAdapter } from "./benchmarks/mcad-cq-bench.js";
 import { createMind2WebAdapter } from "./benchmarks/mind2web.js";
 import { createNoMiraclAdapter } from "./benchmarks/nomiracl.js";
 import { createOpenRagBenchAdapter } from "./benchmarks/open-rag-bench.js";
@@ -52,6 +54,8 @@ const ADAPTERS = {
 	"appworld-react": createAppWorldReactAdapter,
 	"enterpriseops-gym": createEnterpriseOpsGymAdapter,
 	"cad-design": createCadDesignAdapter,
+	mcad: createMcadBenchAdapter,
+	"mcad-cq": createMcadCqAdapter,
 	cadbench: createCadBenchAdapter,
 	cadgenbench: createCadGenBenchAdapter,
 	frames: createFramesAdapter,

package/dist/adapters.js.map

@@ -1 +1 @@
-{"version":3,"file":"adapters.js","names":[],"sources":["../src/adapters.ts"],"sourcesContent":["/**\n * The benchmark registry — the single source of truth for every wired benchmark.\n * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`\n * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it\n * here rather than each keeping their own copy.\n */\n\nimport { createAecBenchAdapter } from './benchmarks/aec-bench'\nimport { createAgentBenchAdapter } from './benchmarks/agentbench'\nimport { createAppWorldAdapter, createAppWorldReactAdapter } from './benchmarks/appworld'\nimport { createBfclAdapter } from './benchmarks/bfcl'\nimport { createCadBenchAdapter } from './benchmarks/cadbench'\nimport { createCadDesignAdapter } from './benchmarks/cad-design'\nimport { createCadGenBenchAdapter } from './benchmarks/cadgenbench'\nimport { createCommit0Adapter } from './benchmarks/commit0'\nimport { createCragAdapter } from './benchmarks/crag'\nimport { createDabstepAdapter } from './benchmarks/dabstep'\nimport { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'\nimport { createFinResearchBenchAdapter } from './benchmarks/finresearchbench'\nimport { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'\nimport { createFramesAdapter } from './benchmarks/frames'\nimport { createHotpotqaAdapter } from './benchmarks/hotpotqa'\nimport { createHumanEvalAdapter } from './benchmarks/humaneval'\nimport { createMind2WebAdapter } from './benchmarks/mind2web'\nimport { createNoMiraclAdapter } from './benchmarks/nomiracl'\nimport { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'\nimport { createProgrambenchAdapter } from './benchmarks/programbench'\nimport { createRagBenchAdapter } from './benchmarks/ragbench'\nimport { createSimpleQaAdapter } from './benchmarks/simpleqa'\nimport { createSweBenchAdapter } from './benchmarks/swe-bench'\nimport { createT2RagBenchAdapter } from './benchmarks/t2-ragbench'\nimport { createTau2BenchAdapter } from './benchmarks/tau2-bench'\nimport { createTau3BankingAdapter } from './benchmarks/tau3-banking'\nimport { createTerminalBenchAdapter } from './benchmarks/terminal-bench'\nimport { createToolLlmAdapter } from './benchmarks/toollm'\nimport { createTrataHedgeAdapter } from './benchmarks/trata-hedge'\nimport { createWebArenaVerifiedAdapter } from './benchmarks/webarena-verified'\nimport type { BenchmarkAdapter } from './benchmarks/types'\n\nexport const ADAPTERS: Record<string, () => BenchmarkAdapter> = {\n  'swe-bench': createSweBenchAdapter,\n  'terminal-bench': createTerminalBenchAdapter,\n  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).\n  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge\n  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.\n  'aec-bench': createAecBenchAdapter,\n  commit0: createCommit0Adapter,\n  dabstep: createDabstepAdapter,\n  programbench: createProgrambenchAdapter,\n  'webarena-verified': createWebArenaVerifiedAdapter,\n  'tau2-bench': createTau2BenchAdapter,\n  'tau3-banking': createTau3BankingAdapter,\n  agentbench: createAgentBenchAdapter,\n  bfcl: createBfclAdapter,\n  toollm: createToolLlmAdapter,\n  appworld: createAppWorldAdapter,\n  // AppWorld's native interactive protocol — the worker is the in-engine ReAct\n  // episode (execution feedback every turn), the mode published baselines use.\n  'appworld-react': createAppWorldReactAdapter,\n  'enterpriseops-gym': createEnterpriseOpsGymAdapter,\n  'cad-design': createCadDesignAdapter,\n  cadbench: createCadBenchAdapter,\n  cadgenbench: createCadGenBenchAdapter,\n  frames: createFramesAdapter,\n  ragbench: createRagBenchAdapter,\n  crag: createCragAdapter,\n  nomiracl: createNoMiraclAdapter,\n  'open-rag-bench': createOpenRagBenchAdapter,\n  't2-ragbench': createT2RagBenchAdapter,\n  finresearchbench: createFinResearchBenchAdapter,\n  finsearchcomp: createFinsearchcompAdapter,\n  simpleqa: createSimpleQaAdapter,\n  hotpotqa: createHotpotqaAdapter,\n  // Deployable-checker code domain: worker = router completion, judge = Docker test\n  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).\n  humaneval: createHumanEvalAdapter,\n  mind2web: createMind2WebAdapter,\n  'trata-hedge': createTrataHedgeAdapter,\n}\n\n/** Resolve a benchmark key to its adapter, failing loud with the known keys. */\nexport function resolveAdapter(key: string): BenchmarkAdapter {\n  const make = ADAPTERS[key]\n  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(', ')})`)\n  return make()\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,MAAa,WAAmD;CAC9D,aAAa;CACb,kBAAkB;CAIlB,aAAa;CACb,SAAS;CACT,SAAS;CACT,cAAc;CACd,qBAAqB;CACrB,cAAc;CACd,gBAAgB;CAChB,YAAY;CACZ,MAAM;CACN,QAAQ;CACR,UAAU;CAGV,kBAAkB;CAClB,qBAAqB;CACrB,cAAc;CACd,UAAU;CACV,aAAa;CACb,QAAQ;CACR,UAAU;CACV,MAAM;CACN,UAAU;CACV,kBAAkB;CAClB,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,UAAU;CACV,UAAU;CAGV,WAAW;CACX,UAAU;CACV,eAAe;AACjB;;AAGA,SAAgB,eAAe,KAA+B;CAC5D,MAAM,OAAO,SAAS;CACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB,KAAK,UAAU,GAAG,EAAE,UAAU,OAAO,KAAK,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE;CACjH,OAAO,KAAK;AACd"}
+{"version":3,"file":"adapters.js","names":[],"sources":["../src/adapters.ts"],"sourcesContent":["/**\n * The benchmark registry — the single source of truth for every wired benchmark.\n * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`\n * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it\n * here rather than each keeping their own copy.\n */\n\nimport { createAecBenchAdapter } from './benchmarks/aec-bench'\nimport { createAgentBenchAdapter } from './benchmarks/agentbench'\nimport { createAppWorldAdapter, createAppWorldReactAdapter } from './benchmarks/appworld'\nimport { createBfclAdapter } from './benchmarks/bfcl'\nimport { createCadBenchAdapter } from './benchmarks/cadbench'\nimport { createCadDesignAdapter } from './benchmarks/cad-design'\nimport { createCadGenBenchAdapter } from './benchmarks/cadgenbench'\nimport { createCommit0Adapter } from './benchmarks/commit0'\nimport { createCragAdapter } from './benchmarks/crag'\nimport { createDabstepAdapter } from './benchmarks/dabstep'\nimport { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'\nimport { createFinResearchBenchAdapter } from './benchmarks/finresearchbench'\nimport { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'\nimport { createFramesAdapter } from './benchmarks/frames'\nimport { createHotpotqaAdapter } from './benchmarks/hotpotqa'\nimport { createHumanEvalAdapter } from './benchmarks/humaneval'\nimport { createMcadBenchAdapter } from './benchmarks/mcad-bench'\nimport { createMcadCqAdapter } from './benchmarks/mcad-cq-bench'\nimport { createMind2WebAdapter } from './benchmarks/mind2web'\nimport { createNoMiraclAdapter } from './benchmarks/nomiracl'\nimport { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'\nimport { createProgrambenchAdapter } from './benchmarks/programbench'\nimport { createRagBenchAdapter } from './benchmarks/ragbench'\nimport { createSimpleQaAdapter } from './benchmarks/simpleqa'\nimport { createSweBenchAdapter } from './benchmarks/swe-bench'\nimport { createT2RagBenchAdapter } from './benchmarks/t2-ragbench'\nimport { createTau2BenchAdapter } from './benchmarks/tau2-bench'\nimport { createTau3BankingAdapter } from './benchmarks/tau3-banking'\nimport { createTerminalBenchAdapter } from './benchmarks/terminal-bench'\nimport { createToolLlmAdapter } from './benchmarks/toollm'\nimport { createTrataHedgeAdapter } from './benchmarks/trata-hedge'\nimport { createWebArenaVerifiedAdapter } from './benchmarks/webarena-verified'\nimport type { BenchmarkAdapter } from './benchmarks/types'\n\nexport const ADAPTERS: Record<string, () => BenchmarkAdapter> = {\n  'swe-bench': createSweBenchAdapter,\n  'terminal-bench': createTerminalBenchAdapter,\n  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).\n  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge\n  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.\n  'aec-bench': createAecBenchAdapter,\n  commit0: createCommit0Adapter,\n  dabstep: createDabstepAdapter,\n  programbench: createProgrambenchAdapter,\n  'webarena-verified': createWebArenaVerifiedAdapter,\n  'tau2-bench': createTau2BenchAdapter,\n  'tau3-banking': createTau3BankingAdapter,\n  agentbench: createAgentBenchAdapter,\n  bfcl: createBfclAdapter,\n  toollm: createToolLlmAdapter,\n  appworld: createAppWorldAdapter,\n  // AppWorld's native interactive protocol — the worker is the in-engine ReAct\n  // episode (execution feedback every turn), the mode published baselines use.\n  'appworld-react': createAppWorldReactAdapter,\n  'enterpriseops-gym': createEnterpriseOpsGymAdapter,\n  'cad-design': createCadDesignAdapter,\n  // Dimensioned mechanical parts: same OpenSCAD+xvfb judge deps as cad-design, but\n  // the spec is metric (bbox / volume / body count / point-in-solid hole probes)\n  // rather than qualitative, and every task carries a verified gold.\n  mcad: createMcadBenchAdapter,\n  // The same 10 parts and the same spec assertions, but the worker writes a Python\n  // CadQuery script that must export STEP as well as the ASCII STL — which closes\n  // the format deviation `mcad-tasks.ts` documents (upstream asks for STEP;\n  // OpenSCAD cannot emit it). STEP delivery is one extra SCORED check.\n  'mcad-cq': createMcadCqAdapter,\n  cadbench: createCadBenchAdapter,\n  cadgenbench: createCadGenBenchAdapter,\n  frames: createFramesAdapter,\n  ragbench: createRagBenchAdapter,\n  crag: createCragAdapter,\n  nomiracl: createNoMiraclAdapter,\n  'open-rag-bench': createOpenRagBenchAdapter,\n  't2-ragbench': createT2RagBenchAdapter,\n  finresearchbench: createFinResearchBenchAdapter,\n  finsearchcomp: createFinsearchcompAdapter,\n  simpleqa: createSimpleQaAdapter,\n  hotpotqa: createHotpotqaAdapter,\n  // Deployable-checker code domain: worker = router completion, judge = Docker test\n  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).\n  humaneval: createHumanEvalAdapter,\n  mind2web: createMind2WebAdapter,\n  'trata-hedge': createTrataHedgeAdapter,\n}\n\n/** Resolve a benchmark key to its adapter, failing loud with the known keys. */\nexport function resolveAdapter(key: string): BenchmarkAdapter {\n  const make = ADAPTERS[key]\n  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(', ')})`)\n  return make()\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,MAAa,WAAmD;CAC9D,aAAa;CACb,kBAAkB;CAIlB,aAAa;CACb,SAAS;CACT,SAAS;CACT,cAAc;CACd,qBAAqB;CACrB,cAAc;CACd,gBAAgB;CAChB,YAAY;CACZ,MAAM;CACN,QAAQ;CACR,UAAU;CAGV,kBAAkB;CAClB,qBAAqB;CACrB,cAAc;CAId,MAAM;CAKN,WAAW;CACX,UAAU;CACV,aAAa;CACb,QAAQ;CACR,UAAU;CACV,MAAM;CACN,UAAU;CACV,kBAAkB;CAClB,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,UAAU;CACV,UAAU;CAGV,WAAW;CACX,UAAU;CACV,eAAe;AACjB;;AAGA,SAAgB,eAAe,KAA+B;CAC5D,MAAM,OAAO,SAAS;CACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB,KAAK,UAAU,GAAG,EAAE,UAAU,OAAO,KAAK,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE;CACjH,OAAO,KAAK;AACd"}

package/dist/benchmarks/mcad-bench.d.ts

@@ -0,0 +1,106 @@
+import { BenchmarkAdapter } from "./types.js";
+import { McadSpec } from "./mcad-tasks.js";
+//#region src/benchmarks/mcad-bench.d.ts
+interface Vec3 {
+  x: number;
+  y: number;
+  z: number;
+}
+/** One triangle as three vertices, in the STL's stated winding order. */
+type Tri = readonly [Vec3, Vec3, Vec3];
+interface McadGeometry {
+  triangles: number;
+  /** Edges NOT shared by exactly two faces. 0 iff the surface is closed + 2-manifold. */
+  openEdges: number;
+  watertight: boolean;
+  /** Zero-area faces — bad geometry even when the topology closes. */
+  degenerateFaces: number;
+  /** Enclosed volume (absolute) via the signed-tetrahedron sum. */
+  volume: number;
+  /** Connected components of the triangle-adjacency graph = disconnected bodies. */
+  solids: number;
+  bbox: {
+    min: Vec3;
+    max: Vec3;
+    size: Vec3;
+  };
+}
+/** Parse an ASCII STL into triangles by reading `vertex x y z` lines in groups of three. */
+declare function parseAsciiStl(stl: string): Tri[];
+/**
+ * Measure the closed-solid properties of a triangle soup. Pure: no I/O, no deps.
+ *
+ * `solids` is the number of connected components of the graph whose nodes are
+ * triangles and whose edges join two triangles that share a quantised mesh edge.
+ * On a watertight mesh that count IS the number of disconnected bodies (an
+ * enclosed internal cavity is its own shell and counts, which is the behaviour
+ * the specs want — a "hollow" body that is really two nested shells is not one
+ * fused solid).
+ */
+declare function measureMesh(tris: Tri[]): McadGeometry;
+/**
+ * Point-in-solid membership by ray parity, made robust three ways.
+ *
+ * ROBUSTNESS ARGUMENT. A parity test is exact except on a measure-zero set: rays
+ * that graze a triangle edge or vertex (the crossing is counted twice or zero
+ * times), rays coplanar with a face, and origins lying on the surface. Those cases
+ * are not merely rare here — they are SYSTEMATIC, because OpenSCAD emits
+ * axis-aligned meshes whose vertices land on the same round millimetre lattice the
+ * spec's probe coordinates come from, so an axis-aligned ray from a probe point
+ * hits shared edges constantly. Three defences, in order:
+ *   1. DIRECTIONS. The three fixed directions are irrational combinations, so a ray
+ *      from a lattice point cannot stay in an axis-aligned face plane and cannot
+ *      run along a lattice edge.
+ *   2. DETECTION + DETERMINISTIC RE-JITTER. Grazing is DETECTED (a barycentric
+ *      coordinate within BARY_EPS of 0, |det| below DET_EPS with the origin in the
+ *      plane, or |t| within T_EPS) rather than hoped away. A detected ray is re-cast
+ *      with a small direction perturbation drawn from a mulberry32 PRNG seeded by
+ *      the probe coordinates and ray index, so the whole judge stays deterministic:
+ *      the same mesh and the same point always take the same sequence of re-casts.
+ *   3. MAJORITY VOTE. The verdict is the majority of three independent directions,
+ *      so even an undetected miscount on one ray cannot flip the answer.
+ * A point sitting exactly ON the surface has no correct answer; every direction
+ * degenerates there and the vote falls back to whatever the jittered casts say.
+ * The specs therefore place probes with >=1 mm clearance from any surface.
+ */
+declare function pointInSolid(tris: Tri[], point: Vec3): boolean;
+interface McadCheck {
+  name: string;
+  ok: boolean;
+  measured: string;
+  expected: string;
+}
+interface McadScoring {
+  checks: McadCheck[];
+  failed: McadCheck[];
+  score: number;
+  resolved: boolean;
+}
+/** Score a measured mesh against the task's spec — one named check per assertion. */
+declare function scoreAgainstSpec(geo: McadGeometry, tris: Tri[], spec: McadSpec): McadScoring;
+/** Task metadata carried onto every `BenchTask`, so the judge needs no lookup table. */
+interface McadTaskMeta extends Record<string, unknown> {
+  spec: McadSpec;
+  source: string;
+  calibrated: boolean;
+}
+/**
+ * Strip a markdown fence if the model wrapped its answer in one. The prompt asks
+ * for bare source; a fenced answer is a formatting slip, not a geometry failure,
+ * and the geometric gate downstream is unchanged either way.
+ */
+declare function stripCodeFence(artifact: string): string;
+/** Compile OpenSCAD source and measure the resulting mesh. Judge-time only. */
+declare function compileAndMeasure(src: string): Promise<{
+  ok: true;
+  geo: McadGeometry;
+  tris: Tri[];
+  stlPath: string;
+} | {
+  ok: false;
+  detail: string;
+}>;
+declare function createMcadBenchAdapter(): BenchmarkAdapter;
+//#endregion
+export { McadCheck, McadGeometry, McadScoring, McadTaskMeta, Tri, Vec3, compileAndMeasure, createMcadBenchAdapter, measureMesh, parseAsciiStl, pointInSolid, scoreAgainstSpec, stripCodeFence };
+//# sourceMappingURL=mcad-bench.d.ts.map