@tangle-network/agent-bench 0.3.5 → 0.3.7

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.3.7
+
+- Declare the public executable search path for Pier candidate entrypoints so the exact process contract can replay them against `@tangle-network/agent-runtime@0.102.0`.
+- Align the portable agent contract with `@tangle-network/agent-interface@0.32.0`.
+- Add the trace-driven SWE supervisor improvement loop with measured candidate worktrees and official Docker judging.
+- Settle cancelled worker trees, isolate each cell's state, serialize judges with a kernel lock, and reject unfinished processes before patch extraction or scoring.
+
+## 0.3.6
+
+- Ship compiled ESM and declarations so Node can import the installed package without a TypeScript runtime.
+- Check root, registry, and benchmark subpath imports with plain Node in addition to TypeScript 5, TypeScript 6, and Python package checks.
+- Build explicitly before verification and publication so disabled lifecycle scripts cannot produce a package without compiled output.
+
 ## 0.3.5
 
 - Build the workspace runtime before source type checks so the published-package verification command works from a clean checkout; the packed consumer still installs `@tangle-network/agent-runtime@0.97.0` from npm.

package/README.md

@@ -2,7 +2,19 @@
 
 Published as `@tangle-network/agent-bench`, with independent CI and release checks for its TypeScript and Python surfaces.
 
-**Read [`bench/HARNESS.md`](./HARNESS.md) FIRST.** It is the one maintained map: the commands, the `rollout → corpus → selector → CI → gate` data flow, the canonical-suite table, the wired/needs-creds/scaffolded matrix, and the gate one-liners — kept verified against source.
+**Read [`bench/HARNESS.md`](./HARNESS.md) FIRST.** It is the one maintained map: the commands, the `rollout → corpus → selector → CI → gate` data flow, the canonical-suite table, the wired/needs-creds/scaffolded matrix, and the gate one-liners, kept verified against source.
+
+## Use
+
+```bash
+pnpm add -D @tangle-network/agent-bench
+```
+
+```ts
+import { resolveAdapter } from '@tangle-network/agent-bench'
+
+const crag = resolveAdapter('crag')
+```
 
 ## SWE-bench judge setup (the one block not in HARNESS.md)
 

package/dist/adapters.d.ts

@@ -0,0 +1,15 @@
+import { BenchmarkAdapter } from './benchmarks/types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * The benchmark registry — the single source of truth for every wired benchmark.
+ * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`
+ * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it
+ * here rather than each keeping their own copy.
+ */
+
+declare const ADAPTERS: Record<string, () => BenchmarkAdapter>;
+/** Resolve a benchmark key to its adapter, failing loud with the known keys. */
+declare function resolveAdapter(key: string): BenchmarkAdapter;
+
+export { ADAPTERS, resolveAdapter };

package/dist/adapters.js

@@ -0,0 +1,43 @@
+import {
+  ADAPTERS,
+  resolveAdapter
+} from "./chunk-PWQVGAJB.js";
+import "./chunk-2PVVP7GN.js";
+import "./chunk-VQRS7VUC.js";
+import "./chunk-Z7ML6L77.js";
+import "./chunk-R67DFVLO.js";
+import "./chunk-SEVJPLZC.js";
+import "./chunk-CKUVRZ2T.js";
+import "./chunk-Y6O2OCUO.js";
+import "./chunk-HBSWHQNJ.js";
+import "./chunk-UAIOHCUK.js";
+import "./chunk-R36V2VP7.js";
+import "./chunk-JRWWGMK7.js";
+import "./chunk-INNOYXCP.js";
+import "./chunk-PA2ZKHJC.js";
+import "./chunk-5H5XV76F.js";
+import "./chunk-TBKU5XQI.js";
+import "./chunk-TSWPNOYM.js";
+import "./chunk-KDIKRJGB.js";
+import "./chunk-ZEWMTR5M.js";
+import "./chunk-PUIRNYI7.js";
+import "./chunk-C7T7WEK2.js";
+import "./chunk-JTHWEDEW.js";
+import "./chunk-2XU6OGEN.js";
+import "./chunk-HHXFIHXC.js";
+import "./chunk-X3BTXCJ4.js";
+import "./chunk-J3KDJNX2.js";
+import "./chunk-X5YKXC6V.js";
+import "./chunk-YCGY7UIZ.js";
+import "./chunk-UPAMRDX4.js";
+import "./chunk-ODT47UAY.js";
+import "./chunk-7WSD27QQ.js";
+import "./chunk-SYDW647C.js";
+import "./chunk-IFAV6KEM.js";
+import "./chunk-LRRD7NAG.js";
+import "./chunk-53UPUNBZ.js";
+export {
+  ADAPTERS,
+  resolveAdapter
+};
+//# sourceMappingURL=adapters.js.map

package/dist/adapters.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/_harness.d.ts

@@ -0,0 +1,125 @@
+import { JudgeArtifactReceipt, BenchScore } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * Shared code-bench harness. The "stage the artifact → run the benchmark's own
+ * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)
+ * → read its JSON report → { resolved, score }" spine that swe-bench,
+ * terminal-bench, commit0, programbench and aec-bench all need. Factored out so
+ * the Docker/venv/report-reading logic lives in ONE place instead of being
+ * copy-pasted per adapter.
+ *
+ * It owns NO benchmark policy: each adapter passes the argv for its evaluator
+ * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The
+ * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config
+ * and fail-loud diagnostics.
+ *
+ * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`
+ * (the adapter passes the import line + the exact fix). A staged run that exits
+ * nonzero throws with the captured stderr — never a fabricated score.
+ */
+
+/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */
+declare const benchRoot: string;
+/** Resolve the shared interpreter without requiring an installed package to contain a venv. */
+declare function resolveBenchPython(env?: Readonly<{
+    AGENT_BENCH_PYTHON?: string;
+}>, root?: string): string;
+/** The shared interpreter every Python-backed evaluator runs through. */
+declare const venvPython: string;
+/** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip
+ *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)
+ *  get their own venv and pass its python explicitly — keeping the shared one clean. */
+declare const venvPythonAt: (venvDir: string) => string;
+/** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */
+declare const bigBuffer: number;
+/** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */
+declare function venvBin(name: string): string;
+/**
+ * Run the bench venv python with an inline script (`-c`); return stdout. Throws
+ * (with stderr) on a nonzero exit — the loaders rely on this to fail loud rather
+ * than parse a partial dump.
+ */
+declare function runVenvPython(script: string, args?: string[], timeoutMs?: number, python?: string): Promise<string>;
+/**
+ * Preflight a python-backed harness: import the module(s) and (optionally) ping
+ * Docker, all inside the bench venv. On failure THROWS the captured error joined
+ * to the adapter's `fix` guidance — the contract every code-bench preflight wants.
+ */
+declare function preflightVenvImports(opts: {
+    /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */
+    modules: string[];
+    /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */
+    requireDocker?: boolean;
+    /** Actionable remediation appended to the thrown message. */
+    fix: string;
+    /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */
+    python?: string;
+}): Promise<void>;
+/**
+ * Run a bench-local python driver script (in the bench venv) while piping
+ * `input` to its stdin, returning stdout. The driver's verdict JSON is its LAST
+ * stdout line; callers parse that and inspect an `error` field (fail loud).
+ *
+ * Uses spawn + an explicit `stdin.end(input)` rather than promisify(execFile)'s
+ * `input` option, because that option is NOT honored by async execFile — stdin
+ * is left open and a driver that does `sys.stdin.read()` blocks forever. The
+ * artifact-piping judges (commit0, appworld) MUST go through this.
+ */
+declare function runVenvScriptStdin(scriptPath: string, args: string[], input: string, opts?: {
+    cwd?: string;
+    timeoutMs?: number;
+    python?: string;
+}): Promise<string>;
+interface StagedRunSpec {
+    /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */
+    tmpPrefix: string;
+    /**
+     * Write the artifact + any harness inputs into the temp dir. Returns nothing;
+     * `argv`/`cwd` consume `dir` to point the evaluator at what was written.
+     */
+    stage(dir: string): Promise<void>;
+    /** The external evaluator to spawn. `bin` defaults to the bench venv python. */
+    bin?: string;
+    /** argv for the evaluator, computed from the temp `dir`. */
+    argv(dir: string): string[];
+    /** Working directory for the evaluator. Defaults to the temp `dir`. */
+    cwd?(dir: string): string;
+    /** Hard timeout for the evaluator (ms); 0 = none. */
+    timeoutMs?: number;
+    /**
+     * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws
+     * if the expected report is absent/malformed (fail loud — no default score).
+     */
+    parseReport(dir: string): Promise<BenchScore>;
+    /**
+     * Copy the complete evaluator directory plus raw process stdout/stderr to this
+     * caller-owned directory before cleanup. The destination must not exist.
+     */
+    capture?: StagedRunCaptureSpec;
+    /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */
+    keepTmp?: boolean;
+}
+interface StagedRunCaptureSpec {
+    /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */
+    destination: string;
+}
+/** A staged run failed after any requested evidence was durably retained. */
+declare class StagedJudgeError extends Error {
+    readonly judgeArtifacts?: JudgeArtifactReceipt;
+    constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions);
+}
+/**
+ * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →
+ * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is
+ * always removed in `finally` unless `keepTmp`.
+ */
+declare function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore>;
+/** Read + JSON.parse a report file from a staged run; throws with the path on failure. */
+declare function readJsonReport<T>(path: string): Promise<T>;
+/** Write a UTF-8 file into a staged dir (artifact / preds.json / attempt.sh / …). */
+declare function stageFile(path: string, content: string): Promise<void>;
+/** Sanitize an instance id into a filesystem/run-id-safe token. */
+declare function safeRunId(prefix: string, id: string): string;
+
+export { StagedJudgeError, type StagedRunCaptureSpec, type StagedRunSpec, benchRoot, bigBuffer, preflightVenvImports, readJsonReport, resolveBenchPython, runStagedJudge, runVenvPython, runVenvScriptStdin, safeRunId, stageFile, venvBin, venvPython, venvPythonAt };

package/dist/benchmarks/_harness.js

@@ -0,0 +1,33 @@
+import {
+  StagedJudgeError,
+  benchRoot,
+  bigBuffer,
+  preflightVenvImports,
+  readJsonReport,
+  resolveBenchPython,
+  runStagedJudge,
+  runVenvPython,
+  runVenvScriptStdin,
+  safeRunId,
+  stageFile,
+  venvBin,
+  venvPython,
+  venvPythonAt
+} from "../chunk-LRRD7NAG.js";
+export {
+  StagedJudgeError,
+  benchRoot,
+  bigBuffer,
+  preflightVenvImports,
+  readJsonReport,
+  resolveBenchPython,
+  runStagedJudge,
+  runVenvPython,
+  runVenvScriptStdin,
+  safeRunId,
+  stageFile,
+  venvBin,
+  venvPython,
+  venvPythonAt
+};
+//# sourceMappingURL=_harness.js.map

package/dist/benchmarks/_harness.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/aec-bench.d.ts

@@ -0,0 +1,27 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * AEC-Bench adapter (TheodoreGalanos/aec-bench, MIT) — closed-form
+ * Architecture/Engineering/Construction calculation tasks. Worker artifact = a
+ * markdown solution ending in a fenced ```json block with the required numeric
+ * fields. Judge = the task's OWN `tests/verify.py`, run with python3: it
+ * recomputes ground truth from the embedded engineering formulas, extracts the
+ * last JSON block from the artifact, scores each field by math.isclose within a
+ * per-field rel_tol, and writes {"reward": mean} + per-field details.json.
+ * GRADED / partial-credit, FULLY DETERMINISTIC — no LLM judge.
+ *
+ * Distinct from the multimodal nomic-ai/aec-bench; this is the deterministic
+ * calculation platform. The runnable-instance verify.py only needs python3 (no
+ * Docker, no Harbor) for the pure-calc disciplines, so the local gate runs at
+ * conc<=2 without a container backend.
+ *
+ * Requires for a live run: network to raw.githubusercontent.com /
+ * api.github.com (the in-repo tasks tree) + a python3 interpreter (the bench
+ * venv) to run verify.py. For offline/CI, set AEC_FIXTURES=1 to load the
+ * committed fixtures (bench/fixtures/aec-bench.json) — never a silent fallback.
+ */
+
+declare function createAecBenchAdapter(): BenchmarkAdapter;
+
+export { createAecBenchAdapter };

package/dist/benchmarks/aec-bench.js

@@ -0,0 +1,8 @@
+import {
+  createAecBenchAdapter
+} from "../chunk-ODT47UAY.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createAecBenchAdapter
+};
+//# sourceMappingURL=aec-bench.js.map

package/dist/benchmarks/aec-bench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/agentbench.d.ts

@@ -0,0 +1,16 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * AgentBench deterministic subset adapter.
+ *
+ * This targets AgentBench DBBench rows only: question + table + published label.
+ * It does not wrap AgentBench's controller protocol or the non-deterministic game
+ * environments. Worker artifact = final answer text. Judge = exact match against
+ * the official DBBench label list after light whitespace/case normalization.
+ */
+
+declare const agentbenchAnswerOutput: OutputAdapter<string>;
+declare function createAgentBenchAdapter(): BenchmarkAdapter;
+
+export { agentbenchAnswerOutput, createAgentBenchAdapter };

package/dist/benchmarks/agentbench.js

@@ -0,0 +1,10 @@
+import {
+  agentbenchAnswerOutput,
+  createAgentBenchAdapter
+} from "../chunk-7WSD27QQ.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  agentbenchAnswerOutput,
+  createAgentBenchAdapter
+};
+//# sourceMappingURL=agentbench.js.map

package/dist/benchmarks/agentbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/appworld.d.ts

@@ -0,0 +1,37 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * AppWorld adapter (StonyBrookNLP/appworld). Worker artifact = the agent's
+ * Python solution that calls the simulated apps' APIs (the same `apis.<app>.<fn>`
+ * surface AppWorld exposes inside `world.execute(...)`), ending in
+ * `apis.supervisor.complete_task()`. Judge = AppWorld's OWN programmatic
+ * evaluator: a driver runs the solution in a fresh `AppWorld(task_id=...)` world,
+ * then `world.evaluate().to_dict()` reports `success` (binary TGC), `num_tests`
+ * (per-requirement total) and the `passes`/`failures` lists. Score =
+ * passes / num_tests — GRADED; resolved = success. Fully deterministic — no LLM judge.
+ *
+ * loadTasks enumerates the real task suite via `load_task_ids(split)`
+ * (train|dev|test_normal|test_challenge); the prompt = `world.task.instruction`.
+ * The OutputAdapter is stream-only, so the worker emits its solution as a fenced
+ * ```python block which the driver executes.
+ *
+ * Requires for a live run: the bench `.venv` with `appworld` installed + the
+ * unpacked engine + downloaded data (`appworld install` ; `appworld download
+ * data`). preflight + loadTasks + judge all fail loud with the exact step when the
+ * engine/data is absent — never a fabricated score.
+ */
+
+/** Worker solution code = the last fenced ```python block, else the raw text. */
+declare const appworldSolutionOutput: OutputAdapter<string>;
+declare function createAppWorldAdapter(): BenchmarkAdapter;
+/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */
+declare function appworldToolLoopClient(cfg: {
+    model: string;
+    routerBaseUrl: string;
+    routerKey: string;
+    maxTurns?: number;
+}): unknown;
+declare function createAppWorldReactAdapter(): BenchmarkAdapter;
+
+export { appworldSolutionOutput, appworldToolLoopClient, createAppWorldAdapter, createAppWorldReactAdapter };

package/dist/benchmarks/appworld.js

@@ -0,0 +1,14 @@
+import {
+  appworldSolutionOutput,
+  appworldToolLoopClient,
+  createAppWorldAdapter,
+  createAppWorldReactAdapter
+} from "../chunk-SYDW647C.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  appworldSolutionOutput,
+  appworldToolLoopClient,
+  createAppWorldAdapter,
+  createAppWorldReactAdapter
+};
+//# sourceMappingURL=appworld.js.map

package/dist/benchmarks/appworld.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/bfcl.d.ts

@@ -0,0 +1,18 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * Berkeley Function Calling Leaderboard adapter.
+ *
+ * Scope: deterministic function-call ground-truth categories from the official
+ * BFCL data files. This is NOT the full live BFCL leaderboard evaluator: agentic
+ * web-search/memory categories and BFCL's own model-response harness remain
+ * upstream responsibilities. The adapter loads official JSONL rows plus their
+ * `possible_answer` file and scores structured function-call artifacts against
+ * allowed function/argument values.
+ */
+
+declare const bfclOutput: OutputAdapter<string>;
+declare function createBfclAdapter(): BenchmarkAdapter;
+
+export { bfclOutput, createBfclAdapter };

package/dist/benchmarks/bfcl.js

@@ -0,0 +1,10 @@
+import {
+  bfclOutput,
+  createBfclAdapter
+} from "../chunk-IFAV6KEM.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  bfclOutput,
+  createBfclAdapter
+};
+//# sourceMappingURL=bfcl.js.map

package/dist/benchmarks/bfcl.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/cad-design.d.ts

@@ -0,0 +1,45 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CAD-Design adapter. Worker artifact = OpenSCAD source (`.scad`). Judge = the
+ * REAL OpenSCAD engine: compile + export STL, then measure the produced solid
+ * geometry against the task's spec. Fully deterministic — the authoritative CAD
+ * kernel is the gate, not an LLM and not the agent's own claim.
+ *
+ * This is verifiable-reward CAD: an agent is far better at *writing* exact
+ * parametric code than at clicking a GUI, and the kernel either produces the
+ * specified geometry or it doesn't. Every judged artifact also renders a PNG,
+ * so a refine loop's attempts become a watchable "model getting better" reel
+ * (run-capsule consumes the renders).
+ *
+ * Spec checks are geometric and ungameable:
+ *   - compiles      : `openscad -o out.stl` exits 0 (hard gate)
+ *   - volumes       : disconnected-solid count within [min,max]
+ *   - bbox          : overall X/Y/Z extent within bounds (the thing is the right size)
+ *   - detail        : triangle count ≥ floor (not a degenerate single cube)
+ *   - pitchedRoof   : the top band's XY footprint NARROWS vs the base — a flat
+ *                     box can't fake this; a real gabled/hipped roof tapers
+ *   - hollow        : interior cavity present (walls, not a solid block) — the
+ *                     bbox volume materially exceeds the printed solid volume
+ *
+ * Requires only `openscad` + `xvfb-run` on PATH (no venv, no Docker, no network).
+ */
+
+/** Spec assertions a CAD task can require. All are deterministic + geometric. */
+interface CadSpec {
+    volumes?: [number, number];
+    bbox?: {
+        x?: [number, number];
+        y?: [number, number];
+        z?: [number, number];
+    };
+    minTriangles?: number;
+    /** Top-band XY footprint must be < this fraction of the base footprint. */
+    pitchedRoof?: number;
+    /** Printed solid volume must be < this fraction of the bbox volume (hollow). */
+    hollowBelow?: number;
+}
+declare function createCadDesignAdapter(): BenchmarkAdapter;
+
+export { type CadSpec, createCadDesignAdapter };

package/dist/benchmarks/cad-design.js

@@ -0,0 +1,7 @@
+import {
+  createCadDesignAdapter
+} from "../chunk-53UPUNBZ.js";
+export {
+  createCadDesignAdapter
+};
+//# sourceMappingURL=cad-design.js.map

package/dist/benchmarks/cad-design.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/cadbench.d.ts

@@ -0,0 +1,19 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CADBench / BlenderLLM adapter (FreedomIntelligence/CADBench, arXiv:2412.14203).
+ * Task = NL instruction → a Blender `bpy` script. Score = the paper's criteria
+ * eval: render the produced model to standardized views, then a vision judge
+ * (GPT-4o-class) marks each per-task criterion bullet pass/fail against the
+ * rendered images + the script text. score = fraction of criteria satisfied.
+ *
+ * Data: the published dataset's `criteria` flattened to a bullet list (700 tasks,
+ * 500 Simulative + 200 Wild). Point CADBENCH_PATH at the cleaned JSONL
+ * ({id,name,instruction,type,criteria:string[]} per line). Judge creds from
+ * TANGLE_API_KEY / ROUTER_BASE / JUDGE_MODEL (default gpt-4o).
+ */
+
+declare function createCadBenchAdapter(): BenchmarkAdapter;
+
+export { createCadBenchAdapter };

package/dist/benchmarks/cadbench.js

@@ -0,0 +1,8 @@
+import {
+  createCadBenchAdapter
+} from "../chunk-PUIRNYI7.js";
+import "../chunk-JTHWEDEW.js";
+export {
+  createCadBenchAdapter
+};
+//# sourceMappingURL=cadbench.js.map

package/dist/benchmarks/cadbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/cadgenbench.d.ts

@@ -0,0 +1,22 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CADGenBench adapter (huggingface/cadgenbench, Apache-2.0). Task = a part
+ * description → a STEP B-rep solid (output.step). Score = the benchmark's OWN
+ * deterministic geometric metric (cad_score): validity gate → PCA/ICP align to
+ * the ground truth → point-cloud F1 + volume IoU + edge F1 + topology match.
+ * NOT an LLM judge, NOT self-defined checks — the published CAD kernel decides.
+ *
+ * The official task set (private GT, server-side graded) isn't released yet, so
+ * tasks here are seeded from the repo's dimension-named geometry fixtures (real
+ * GT STEPs scored by the real scorer). When CADGENBENCH_DATA_DIR is set, swap
+ * loadTasks to read the published fixtures' description.yaml + ground_truth.step.
+ *
+ * Requires the CADGenBench venv (CADGENBENCH_VENV) + clone (CADGENBENCH_DIR) +
+ * xvfb (the scorer's alignment renders need a display).
+ */
+
+declare function createCadGenBenchAdapter(): BenchmarkAdapter;
+
+export { createCadGenBenchAdapter };

package/dist/benchmarks/cadgenbench.js

@@ -0,0 +1,8 @@
+import {
+  createCadGenBenchAdapter
+} from "../chunk-C7T7WEK2.js";
+import "../chunk-JTHWEDEW.js";
+export {
+  createCadGenBenchAdapter
+};
+//# sourceMappingURL=cadgenbench.js.map

package/dist/benchmarks/cadgenbench.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/commit0.d.ts

@@ -0,0 +1,31 @@
+import { OutputAdapter } from '@tangle-network/agent-runtime/loops';
+import { BenchmarkAdapter } from './types.js';
+
+/**
+ * Commit0 adapter (wentingzhao/commit0_combined) — implement-a-library-from-scratch.
+ * Each record is a Python repo stubbed at `base_commit` (public function bodies
+ * emptied to `pass`) + the full test suite + a natural-language spec URL. Worker
+ * artifact = a unified diff that fills in `src_dir`. Judge = the official
+ * `commit0` test harness: it stages the starter repo, applies the worker's diff,
+ * builds the library's deps and runs `pytest`, writing a per-repo pytest-json
+ * `report.json`. Score = (passed + xfail) / total — GRADED / partial-credit (the
+ * macro-averaged unit-test pass-rate the leaderboard reports). Fully
+ * deterministic — no LLM judge.
+ *
+ * The OutputAdapter is stream-only (the SWE wrinkle), so the worker emits its
+ * implementation as a fenced ```diff against the stubbed repo — same deliverable
+ * shape as swe-bench. The expensive per-repo clone+build+test is delegated to the
+ * real `commit0` harness on a local Docker backend (NOT reimplemented here).
+ *
+ * Requires for a live run: an ISOLATED `.venv-commit0` with `commit0` installed
+ * (its deps conflict with the shared bench `.venv`; override with COMMIT0_VENV) +
+ * a Docker daemon (`--backend local`). For offline/CI dataset listing set
+ * COMMIT0_FIXTURES=1 to load the committed lite rows (bench/fixtures/commit0.json)
+ * — judging still needs the harness + Docker and fails loud, never a fabricated score.
+ */
+
+/** Reuse the SWE patch extractor shape: last fenced diff/patch, else raw text. */
+declare const commit0DiffOutput: OutputAdapter<string>;
+declare function createCommit0Adapter(): BenchmarkAdapter;
+
+export { commit0DiffOutput, createCommit0Adapter };

package/dist/benchmarks/commit0.js

@@ -0,0 +1,10 @@
+import {
+  commit0DiffOutput,
+  createCommit0Adapter
+} from "../chunk-2XU6OGEN.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  commit0DiffOutput,
+  createCommit0Adapter
+};
+//# sourceMappingURL=commit0.js.map

package/dist/benchmarks/commit0.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/crag.d.ts

@@ -0,0 +1,14 @@
+import { BenchmarkAdapter } from './types.js';
+import '@tangle-network/agent-runtime/loops';
+
+/**
+ * CRAG adapter (Comprehensive RAG Benchmark).
+ *
+ * Live mode expects an official or compatible CRAG JSON/JSONL export. The
+ * adapter preserves CRAG domain/type/dynamism tags in metadata and scores final
+ * answers deterministically against the provided gold answer list.
+ */
+
+declare function createCragAdapter(): BenchmarkAdapter;
+
+export { createCragAdapter };

package/dist/benchmarks/crag.js

@@ -0,0 +1,9 @@
+import {
+  createCragAdapter
+} from "../chunk-HHXFIHXC.js";
+import "../chunk-X3BTXCJ4.js";
+import "../chunk-LRRD7NAG.js";
+export {
+  createCragAdapter
+};
+//# sourceMappingURL=crag.js.map

package/dist/benchmarks/crag.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}