@tangle-network/agent-bench 0.4.0 → 0.4.3

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 0.4.3
+
+- Consume Runtime 0.107.1 with Eval 0.133.0, Interface 0.36.0, and Knowledge 6.1.2.
+
+## 0.4.2
+
+- Consume `@tangle-network/agent-runtime` 0.106.x with Eval 0.131.0, Interface 0.35.0, and Knowledge 6.1.0.
+- Build the package before checking its exports so verification works from a clean checkout.
+
+## 0.4.1
+
+- Read Runtime 0.105 candidate plans from their signed run cell, benchmark records, and profile activation.
+- Allow only the fixed public executable path signed by the Runtime plan.
+- Restore the real Pier failure/success proof against the current receipt layout.
+
 ## 0.4.0
 
 - Add the resumable SWE improvement loop backed by the official GEPA engine.

package/dist/adapters.d.ts

@@ -1,15 +1,8 @@
-import { BenchmarkAdapter } from './benchmarks/types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * The benchmark registry — the single source of truth for every wired benchmark.
- * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`
- * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it
- * here rather than each keeping their own copy.
- */
-
+import { BenchmarkAdapter } from "./benchmarks/types.js";
+//#region src/adapters.d.ts
 declare const ADAPTERS: Record<string, () => BenchmarkAdapter>;
 /** Resolve a benchmark key to its adapter, failing loud with the known keys. */
 declare function resolveAdapter(key: string): BenchmarkAdapter;
-
+//#endregion
 export { ADAPTERS, resolveAdapter };
+//# sourceMappingURL=adapters.d.ts.map

package/dist/adapters.js

@@ -1,43 +1,80 @@
-import {
-  ADAPTERS,
-  resolveAdapter
-} from "./chunk-V7AEBY6U.js";
-import "./chunk-XKEFIFIC.js";
-import "./chunk-VQRS7VUC.js";
-import "./chunk-HWST3SED.js";
-import "./chunk-IFVINJ4B.js";
-import "./chunk-SEVJPLZC.js";
-import "./chunk-3U5TXJZS.js";
-import "./chunk-SHM6MRRF.js";
-import "./chunk-RCYQEFNX.js";
-import "./chunk-SFLA7OH3.js";
-import "./chunk-IZ5M6OAC.js";
-import "./chunk-XYA4XSNU.js";
-import "./chunk-INNOYXCP.js";
-import "./chunk-PA2ZKHJC.js";
-import "./chunk-5H5XV76F.js";
-import "./chunk-TBKU5XQI.js";
-import "./chunk-NQG5XDSB.js";
-import "./chunk-SHYIRB7I.js";
-import "./chunk-MQMRLGOG.js";
-import "./chunk-PUIRNYI7.js";
-import "./chunk-C7T7WEK2.js";
-import "./chunk-JTHWEDEW.js";
-import "./chunk-Z4TZ76N7.js";
-import "./chunk-SVR2LKYI.js";
-import "./chunk-X3BTXCJ4.js";
-import "./chunk-RH5F53JT.js";
-import "./chunk-YSMEKBTD.js";
-import "./chunk-7GRVHU22.js";
-import "./chunk-UPAMRDX4.js";
-import "./chunk-K3BQGZCT.js";
-import "./chunk-PB64GYIG.js";
-import "./chunk-IA2FBTWC.js";
-import "./chunk-KP5KD6EN.js";
-import "./chunk-WSKWVEQB.js";
-import "./chunk-53UPUNBZ.js";
-export {
-  ADAPTERS,
-  resolveAdapter
+import { createAecBenchAdapter } from "./benchmarks/aec-bench.js";
+import { createAgentBenchAdapter } from "./benchmarks/agentbench.js";
+import { createAppWorldAdapter, createAppWorldReactAdapter } from "./benchmarks/appworld.js";
+import { createBfclAdapter } from "./benchmarks/bfcl.js";
+import { t as createCadBenchAdapter } from "./cadbench-DpQWZHp4.js";
+import { createCadDesignAdapter } from "./benchmarks/cad-design.js";
+import { t as createCadGenBenchAdapter } from "./cadgenbench-DRhczfsG.js";
+import { createCommit0Adapter } from "./benchmarks/commit0.js";
+import { createCragAdapter } from "./benchmarks/crag.js";
+import { createDabstepAdapter } from "./benchmarks/dabstep.js";
+import { createEnterpriseOpsGymAdapter } from "./benchmarks/enterpriseops-gym.js";
+import { createFinResearchBenchAdapter } from "./benchmarks/finresearchbench.js";
+import { createFinsearchcompAdapter } from "./benchmarks/finsearchcomp.js";
+import { createFramesAdapter } from "./benchmarks/frames.js";
+import { createHotpotqaAdapter } from "./benchmarks/hotpotqa.js";
+import { createHumanEvalAdapter } from "./benchmarks/humaneval.js";
+import { createMind2WebAdapter } from "./benchmarks/mind2web.js";
+import { createNoMiraclAdapter } from "./benchmarks/nomiracl.js";
+import { createOpenRagBenchAdapter } from "./benchmarks/open-rag-bench.js";
+import { createProgrambenchAdapter } from "./benchmarks/programbench.js";
+import { createRagBenchAdapter } from "./benchmarks/ragbench.js";
+import { createSimpleQaAdapter } from "./benchmarks/simpleqa.js";
+import { createSweBenchAdapter } from "./benchmarks/swe-bench.js";
+import { createT2RagBenchAdapter } from "./benchmarks/t2-ragbench.js";
+import { createTau2BenchAdapter } from "./benchmarks/tau2-bench.js";
+import { createTau3BankingAdapter } from "./benchmarks/tau3-banking.js";
+import { createTerminalBenchAdapter } from "./benchmarks/terminal-bench.js";
+import { createToolLlmAdapter } from "./benchmarks/toollm.js";
+import { createTrataHedgeAdapter } from "./benchmarks/trata-hedge.js";
+import { createWebArenaVerifiedAdapter } from "./benchmarks/webarena-verified.js";
+//#region src/adapters.ts
+/**
+* The benchmark registry — the single source of truth for every wired benchmark.
+* One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`
+* (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it
+* here rather than each keeping their own copy.
+*/
+const ADAPTERS = {
+	"swe-bench": createSweBenchAdapter,
+	"terminal-bench": createTerminalBenchAdapter,
+	"aec-bench": createAecBenchAdapter,
+	commit0: createCommit0Adapter,
+	dabstep: createDabstepAdapter,
+	programbench: createProgrambenchAdapter,
+	"webarena-verified": createWebArenaVerifiedAdapter,
+	"tau2-bench": createTau2BenchAdapter,
+	"tau3-banking": createTau3BankingAdapter,
+	agentbench: createAgentBenchAdapter,
+	bfcl: createBfclAdapter,
+	toollm: createToolLlmAdapter,
+	appworld: createAppWorldAdapter,
+	"appworld-react": createAppWorldReactAdapter,
+	"enterpriseops-gym": createEnterpriseOpsGymAdapter,
+	"cad-design": createCadDesignAdapter,
+	cadbench: createCadBenchAdapter,
+	cadgenbench: createCadGenBenchAdapter,
+	frames: createFramesAdapter,
+	ragbench: createRagBenchAdapter,
+	crag: createCragAdapter,
+	nomiracl: createNoMiraclAdapter,
+	"open-rag-bench": createOpenRagBenchAdapter,
+	"t2-ragbench": createT2RagBenchAdapter,
+	finresearchbench: createFinResearchBenchAdapter,
+	finsearchcomp: createFinsearchcompAdapter,
+	simpleqa: createSimpleQaAdapter,
+	hotpotqa: createHotpotqaAdapter,
+	humaneval: createHumanEvalAdapter,
+	mind2web: createMind2WebAdapter,
+	"trata-hedge": createTrataHedgeAdapter
 };
+/** Resolve a benchmark key to its adapter, failing loud with the known keys. */
+function resolveAdapter(key) {
+	const make = ADAPTERS[key];
+	if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(", ")})`);
+	return make();
+}
+//#endregion
+export { ADAPTERS, resolveAdapter };
+
 //# sourceMappingURL=adapters.js.map

package/dist/adapters.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
+{"version":3,"file":"adapters.js","names":[],"sources":["../src/adapters.ts"],"sourcesContent":["/**\n * The benchmark registry — the single source of truth for every wired benchmark.\n * One key per benchmark; the value is its `BenchmarkAdapter` factory. `runBenchmarks`\n * (the unifier) maps over this; `run.ts`, `rsi.ts`, and `corpus-replay.mts` all read it\n * here rather than each keeping their own copy.\n */\n\nimport { createAecBenchAdapter } from './benchmarks/aec-bench'\nimport { createAgentBenchAdapter } from './benchmarks/agentbench'\nimport { createAppWorldAdapter, createAppWorldReactAdapter } from './benchmarks/appworld'\nimport { createBfclAdapter } from './benchmarks/bfcl'\nimport { createCadBenchAdapter } from './benchmarks/cadbench'\nimport { createCadDesignAdapter } from './benchmarks/cad-design'\nimport { createCadGenBenchAdapter } from './benchmarks/cadgenbench'\nimport { createCommit0Adapter } from './benchmarks/commit0'\nimport { createCragAdapter } from './benchmarks/crag'\nimport { createDabstepAdapter } from './benchmarks/dabstep'\nimport { createEnterpriseOpsGymAdapter } from './benchmarks/enterpriseops-gym'\nimport { createFinResearchBenchAdapter } from './benchmarks/finresearchbench'\nimport { createFinsearchcompAdapter } from './benchmarks/finsearchcomp'\nimport { createFramesAdapter } from './benchmarks/frames'\nimport { createHotpotqaAdapter } from './benchmarks/hotpotqa'\nimport { createHumanEvalAdapter } from './benchmarks/humaneval'\nimport { createMind2WebAdapter } from './benchmarks/mind2web'\nimport { createNoMiraclAdapter } from './benchmarks/nomiracl'\nimport { createOpenRagBenchAdapter } from './benchmarks/open-rag-bench'\nimport { createProgrambenchAdapter } from './benchmarks/programbench'\nimport { createRagBenchAdapter } from './benchmarks/ragbench'\nimport { createSimpleQaAdapter } from './benchmarks/simpleqa'\nimport { createSweBenchAdapter } from './benchmarks/swe-bench'\nimport { createT2RagBenchAdapter } from './benchmarks/t2-ragbench'\nimport { createTau2BenchAdapter } from './benchmarks/tau2-bench'\nimport { createTau3BankingAdapter } from './benchmarks/tau3-banking'\nimport { createTerminalBenchAdapter } from './benchmarks/terminal-bench'\nimport { createToolLlmAdapter } from './benchmarks/toollm'\nimport { createTrataHedgeAdapter } from './benchmarks/trata-hedge'\nimport { createWebArenaVerifiedAdapter } from './benchmarks/webarena-verified'\nimport type { BenchmarkAdapter } from './benchmarks/types'\n\nexport const ADAPTERS: Record<string, () => BenchmarkAdapter> = {\n  'swe-bench': createSweBenchAdapter,\n  'terminal-bench': createTerminalBenchAdapter,\n  // Code-benches sharing ./benchmarks/_harness (stage → external evaluator → report).\n  // loadTasks fetches the REAL dataset (committed fixtures fallback offline); judge\n  // delegates to the benchmark's own harness and fails loud when it/Docker is absent.\n  'aec-bench': createAecBenchAdapter,\n  commit0: createCommit0Adapter,\n  dabstep: createDabstepAdapter,\n  programbench: createProgrambenchAdapter,\n  'webarena-verified': createWebArenaVerifiedAdapter,\n  'tau2-bench': createTau2BenchAdapter,\n  'tau3-banking': createTau3BankingAdapter,\n  agentbench: createAgentBenchAdapter,\n  bfcl: createBfclAdapter,\n  toollm: createToolLlmAdapter,\n  appworld: createAppWorldAdapter,\n  // AppWorld's native interactive protocol — the worker is the in-engine ReAct\n  // episode (execution feedback every turn), the mode published baselines use.\n  'appworld-react': createAppWorldReactAdapter,\n  'enterpriseops-gym': createEnterpriseOpsGymAdapter,\n  'cad-design': createCadDesignAdapter,\n  cadbench: createCadBenchAdapter,\n  cadgenbench: createCadGenBenchAdapter,\n  frames: createFramesAdapter,\n  ragbench: createRagBenchAdapter,\n  crag: createCragAdapter,\n  nomiracl: createNoMiraclAdapter,\n  'open-rag-bench': createOpenRagBenchAdapter,\n  't2-ragbench': createT2RagBenchAdapter,\n  finresearchbench: createFinResearchBenchAdapter,\n  finsearchcomp: createFinsearchcompAdapter,\n  simpleqa: createSimpleQaAdapter,\n  hotpotqa: createHotpotqaAdapter,\n  // Deployable-checker code domain: worker = router completion, judge = Docker test\n  // run (--network=none). The steering A/B counterpart to humaneval-gate.mts (selection).\n  humaneval: createHumanEvalAdapter,\n  mind2web: createMind2WebAdapter,\n  'trata-hedge': createTrataHedgeAdapter,\n}\n\n/** Resolve a benchmark key to its adapter, failing loud with the known keys. */\nexport function resolveAdapter(key: string): BenchmarkAdapter {\n  const make = ADAPTERS[key]\n  if (!make) throw new Error(`unknown benchmark ${JSON.stringify(key)} (have: ${Object.keys(ADAPTERS).join(', ')})`)\n  return make()\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,MAAa,WAAmD;CAC9D,aAAa;CACb,kBAAkB;CAIlB,aAAa;CACb,SAAS;CACT,SAAS;CACT,cAAc;CACd,qBAAqB;CACrB,cAAc;CACd,gBAAgB;CAChB,YAAY;CACZ,MAAM;CACN,QAAQ;CACR,UAAU;CAGV,kBAAkB;CAClB,qBAAqB;CACrB,cAAc;CACd,UAAU;CACV,aAAa;CACb,QAAQ;CACR,UAAU;CACV,MAAM;CACN,UAAU;CACV,kBAAkB;CAClB,eAAe;CACf,kBAAkB;CAClB,eAAe;CACf,UAAU;CACV,UAAU;CAGV,WAAW;CACX,UAAU;CACV,eAAe;AACjB;;AAGA,SAAgB,eAAe,KAA+B;CAC5D,MAAM,OAAO,SAAS;CACtB,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,qBAAqB,KAAK,UAAU,GAAG,EAAE,UAAU,OAAO,KAAK,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE;CACjH,OAAO,KAAK;AACd"}

package/dist/benchmarks/_harness.d.ts

@@ -1,29 +1,10 @@
-import { JudgeArtifactReceipt, BenchScore } from './types.js';
-import '@tangle-network/agent-runtime/loops';
-
-/**
- * Shared code-bench harness. The "stage the artifact → run the benchmark's own
- * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)
- * → read its JSON report → { resolved, score }" spine that swe-bench,
- * terminal-bench, commit0, programbench and aec-bench all need. Factored out so
- * the Docker/venv/report-reading logic lives in ONE place instead of being
- * copy-pasted per adapter.
- *
- * It owns NO benchmark policy: each adapter passes the argv for its evaluator
- * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The
- * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config
- * and fail-loud diagnostics.
- *
- * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`
- * (the adapter passes the import line + the exact fix). A staged run that exits
- * nonzero throws with the captured stderr — never a fabricated score.
- */
-
+import { BenchScore, JudgeArtifactReceipt } from "./types.js";
+//#region src/benchmarks/_harness.d.ts
 /** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */
 declare const benchRoot: string;
 /** Resolve the shared interpreter without requiring an installed package to contain a venv. */
 declare function resolveBenchPython(env?: Readonly<{
-    AGENT_BENCH_PYTHON?: string;
+  AGENT_BENCH_PYTHON?: string;
 }>, root?: string): string;
 /** The shared interpreter every Python-backed evaluator runs through. */
 declare const venvPython: string;
@@ -31,6 +12,9 @@ declare const venvPython: string;
  *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)
  *  get their own venv and pass its python explicitly — keeping the shared one clean. */
 declare const venvPythonAt: (venvDir: string) => string;
+/** Resolve an executable in an isolated venv. Relative venv paths are package-owned;
+ * absolute paths allow installed consumers to keep large environments elsewhere. */
+declare function venvBinAt(venvDir: string, name: string): string;
 /** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */
 declare const bigBuffer: number;
 /** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */
@@ -47,14 +31,14 @@ declare function runVenvPython(script: string, args?: string[], timeoutMs?: numb
  * to the adapter's `fix` guidance — the contract every code-bench preflight wants.
  */
 declare function preflightVenvImports(opts: {
-    /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */
-    modules: string[];
-    /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */
-    requireDocker?: boolean;
-    /** Actionable remediation appended to the thrown message. */
-    fix: string;
-    /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */
-    python?: string;
+  /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */
+  modules: string[];
+  /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */
+  requireDocker?: boolean;
+  /** Actionable remediation appended to the thrown message. */
+  fix: string;
+  /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */
+  python?: string;
 }): Promise<void>;
 /**
  * Run a bench-local python driver script (in the bench venv) while piping
@@ -67,47 +51,47 @@ declare function preflightVenvImports(opts: {
  * artifact-piping judges (commit0, appworld) MUST go through this.
  */
 declare function runVenvScriptStdin(scriptPath: string, args: string[], input: string, opts?: {
-    cwd?: string;
-    timeoutMs?: number;
-    python?: string;
+  cwd?: string;
+  timeoutMs?: number;
+  python?: string;
 }): Promise<string>;
 interface StagedRunSpec {
-    /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */
-    tmpPrefix: string;
-    /**
-     * Write the artifact + any harness inputs into the temp dir. Returns nothing;
-     * `argv`/`cwd` consume `dir` to point the evaluator at what was written.
-     */
-    stage(dir: string): Promise<void>;
-    /** The external evaluator to spawn. `bin` defaults to the bench venv python. */
-    bin?: string;
-    /** argv for the evaluator, computed from the temp `dir`. */
-    argv(dir: string): string[];
-    /** Working directory for the evaluator. Defaults to the temp `dir`. */
-    cwd?(dir: string): string;
-    /** Hard timeout for the evaluator (ms); 0 = none. */
-    timeoutMs?: number;
-    /**
-     * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws
-     * if the expected report is absent/malformed (fail loud — no default score).
-     */
-    parseReport(dir: string): Promise<BenchScore>;
-    /**
-     * Copy the complete evaluator directory plus raw process stdout/stderr to this
-     * caller-owned directory before cleanup. The destination must not exist.
-     */
-    capture?: StagedRunCaptureSpec;
-    /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */
-    keepTmp?: boolean;
+  /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */
+  tmpPrefix: string;
+  /**
+   * Write the artifact + any harness inputs into the temp dir. Returns nothing;
+   * `argv`/`cwd` consume `dir` to point the evaluator at what was written.
+   */
+  stage(dir: string): Promise<void>;
+  /** The external evaluator to spawn. `bin` defaults to the bench venv python. */
+  bin?: string;
+  /** argv for the evaluator, computed from the temp `dir`. */
+  argv(dir: string): string[];
+  /** Working directory for the evaluator. Defaults to the temp `dir`. */
+  cwd?(dir: string): string;
+  /** Hard timeout for the evaluator (ms); 0 = none. */
+  timeoutMs?: number;
+  /**
+   * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws
+   * if the expected report is absent/malformed (fail loud — no default score).
+   */
+  parseReport(dir: string): Promise<BenchScore>;
+  /**
+   * Copy the complete evaluator directory plus raw process stdout/stderr to this
+   * caller-owned directory before cleanup. The destination must not exist.
+   */
+  capture?: StagedRunCaptureSpec;
+  /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */
+  keepTmp?: boolean;
 }
 interface StagedRunCaptureSpec {
-    /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */
-    destination: string;
+  /** Destination for `evaluator/`, `process/`, and the hashed `receipt.json`. */
+  destination: string;
 }
 /** A staged run failed after any requested evidence was durably retained. */
 declare class StagedJudgeError extends Error {
-    readonly judgeArtifacts?: JudgeArtifactReceipt;
-    constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions);
+  readonly judgeArtifacts?: JudgeArtifactReceipt;
+  constructor(message: string, judgeArtifacts?: JudgeArtifactReceipt, options?: ErrorOptions);
 }
 /**
  * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →
@@ -121,5 +105,6 @@ declare function readJsonReport<T>(path: string): Promise<T>;
 declare function stageFile(path: string, content: string): Promise<void>;
 /** Sanitize an instance id into a filesystem/run-id-safe token. */
 declare function safeRunId(prefix: string, id: string): string;
-
-export { StagedJudgeError, type StagedRunCaptureSpec, type StagedRunSpec, benchRoot, bigBuffer, preflightVenvImports, readJsonReport, resolveBenchPython, runStagedJudge, runVenvPython, runVenvScriptStdin, safeRunId, stageFile, venvBin, venvPython, venvPythonAt };
+//#endregion
+export { StagedJudgeError, StagedRunCaptureSpec, StagedRunSpec, benchRoot, bigBuffer, preflightVenvImports, readJsonReport, resolveBenchPython, runStagedJudge, runVenvPython, runVenvScriptStdin, safeRunId, stageFile, venvBin, venvBinAt, venvPython, venvPythonAt };
+//# sourceMappingURL=_harness.d.ts.map