@tangle-network/agent-bench 0.3.7 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/HARNESS.md +43 -0
- package/README.md +7 -0
- package/dist/adapters.js +23 -23
- package/dist/benchmarks/_harness.d.ts +1 -1
- package/dist/benchmarks/_harness.js +1 -1
- package/dist/benchmarks/aec-bench.js +2 -2
- package/dist/benchmarks/agentbench.js +2 -2
- package/dist/benchmarks/appworld.js +2 -2
- package/dist/benchmarks/bfcl.js +2 -2
- package/dist/benchmarks/commit0.js +2 -2
- package/dist/benchmarks/crag.js +2 -2
- package/dist/benchmarks/dabstep.js +2 -2
- package/dist/benchmarks/enterpriseops-gym.js +2 -2
- package/dist/benchmarks/finresearchbench.js +2 -2
- package/dist/benchmarks/nomiracl.js +2 -2
- package/dist/benchmarks/open-rag-bench.js +2 -2
- package/dist/benchmarks/programbench.js +2 -2
- package/dist/benchmarks/ragbench.js +2 -2
- package/dist/benchmarks/swe-bench.js +2 -2
- package/dist/benchmarks/t2-ragbench.js +2 -2
- package/dist/benchmarks/tau-bench-shared.js +2 -2
- package/dist/benchmarks/tau2-bench.js +3 -3
- package/dist/benchmarks/tau3-banking.js +3 -3
- package/dist/benchmarks/terminal-bench.js +2 -2
- package/dist/benchmarks/toollm.js +2 -2
- package/dist/benchmarks/webarena-verified.js +2 -2
- package/dist/{chunk-CKUVRZ2T.js → chunk-3U5TXJZS.js} +2 -2
- package/dist/{chunk-YCGY7UIZ.js → chunk-7GRVHU22.js} +2 -2
- package/dist/{chunk-Z7ML6L77.js → chunk-HWST3SED.js} +2 -2
- package/dist/{chunk-SYDW647C.js → chunk-IA2FBTWC.js} +2 -2
- package/dist/{chunk-R67DFVLO.js → chunk-IFVINJ4B.js} +2 -2
- package/dist/{chunk-R36V2VP7.js → chunk-IZ5M6OAC.js} +2 -2
- package/dist/{chunk-ODT47UAY.js → chunk-K3BQGZCT.js} +2 -2
- package/dist/{chunk-IFAV6KEM.js → chunk-KP5KD6EN.js} +2 -2
- package/dist/{chunk-ZEWMTR5M.js → chunk-MQMRLGOG.js} +2 -2
- package/dist/{chunk-TSWPNOYM.js → chunk-NQG5XDSB.js} +2 -2
- package/dist/{chunk-7WSD27QQ.js → chunk-PB64GYIG.js} +2 -2
- package/dist/{chunk-HBSWHQNJ.js → chunk-RCYQEFNX.js} +3 -3
- package/dist/{chunk-J3KDJNX2.js → chunk-RH5F53JT.js} +2 -2
- package/dist/{chunk-UAIOHCUK.js → chunk-SFLA7OH3.js} +3 -3
- package/dist/{chunk-Y6O2OCUO.js → chunk-SHM6MRRF.js} +2 -2
- package/dist/{chunk-KDIKRJGB.js → chunk-SHYIRB7I.js} +2 -2
- package/dist/{chunk-HHXFIHXC.js → chunk-SVR2LKYI.js} +2 -2
- package/dist/{chunk-PWQVGAJB.js → chunk-V7AEBY6U.js} +21 -21
- package/dist/{chunk-LRRD7NAG.js → chunk-WSKWVEQB.js} +18 -2
- package/dist/chunk-WSKWVEQB.js.map +1 -0
- package/dist/{chunk-2PVVP7GN.js → chunk-XKEFIFIC.js} +2 -2
- package/dist/{chunk-JRWWGMK7.js → chunk-XYA4XSNU.js} +2 -2
- package/dist/{chunk-X5YKXC6V.js → chunk-YSMEKBTD.js} +2 -2
- package/dist/{chunk-2XU6OGEN.js → chunk-Z4TZ76N7.js} +2 -2
- package/dist/index.js +23 -23
- package/package.json +5 -5
- package/scripts/trata-hedge/README.md +6 -5
- package/scripts/verify-packed-consumer.mjs +1 -1
- package/src/benchmarks/_harness.ts +20 -2
- package/src/benchmarks/humaneval.test.mts +2 -2
- package/src/gate.ts +1 -1
- package/src/hev-eval.mts +5 -2
- package/src/hev-improve.mts +118 -73
- package/src/official-optimizer-config.mts +89 -0
- package/src/official-optimizer-config.test.mts +88 -0
- package/src/profiles.ts +2 -2
- package/src/quant-arena/README.md +144 -0
- package/src/quant-arena/backtest.test.mts +135 -0
- package/src/quant-arena/backtest.ts +218 -0
- package/src/quant-arena/data.test.mts +44 -0
- package/src/quant-arena/data.ts +141 -0
- package/src/quant-arena/driver.test.mts +253 -0
- package/src/quant-arena/driver.ts +219 -0
- package/src/quant-arena/fixtures/data/PROVENANCE.md +26 -0
- package/src/quant-arena/fixtures/data/holdout/IDX.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S01.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S02.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S03.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S04.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S05.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S06.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S07.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S08.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S09.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S10.csv +523 -0
- package/src/quant-arena/fixtures/data/insample/IDX.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S01.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S02.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S03.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S04.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S05.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S06.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S07.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S08.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S09.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S10.csv +2087 -0
- package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +16 -0
- package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +5 -0
- package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +171 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +119 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +119 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +105 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +102 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +4 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +2 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +84 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +117 -0
- package/src/quant-arena/holdout-certify.mts +206 -0
- package/src/quant-arena/holdout-certify.test.mts +82 -0
- package/src/quant-arena/leak-audit.test.mts +79 -0
- package/src/quant-arena/leak-audit.ts +95 -0
- package/src/quant-arena/make-fixtures.mts +161 -0
- package/src/quant-arena/multiplicity.test.mts +68 -0
- package/src/quant-arena/multiplicity.ts +87 -0
- package/src/quant-arena/nautilus-certify.ts +31 -0
- package/src/quant-arena/oms.ts +90 -0
- package/src/quant-arena/profiles/quant-researcher.profile.json +7 -0
- package/src/quant-arena/python/pyproject.toml +8 -0
- package/src/quant-arena/python/uv.lock +1297 -0
- package/src/quant-arena/python/vbt-worker.py +192 -0
- package/src/quant-arena/quant-loop.mts +813 -0
- package/src/quant-arena/quant-loop.test.mts +75 -0
- package/src/quant-arena/strategies/buy-hold-index/strategy.ts +11 -0
- package/src/quant-arena/strategies/equal-weight/strategy.ts +20 -0
- package/src/quant-arena/strategies/sma-crossover/strategy.ts +42 -0
- package/src/quant-arena/types.ts +133 -0
- package/src/quant-arena/vbt-client.ts +321 -0
- package/src/quant-arena/vbt-parity.test.mts +183 -0
- package/src/quant-arena/windows.test.mts +45 -0
- package/src/quant-arena/windows.ts +54 -0
- package/src/rollout-ledger/backfill-swe-arena.mts +606 -0
- package/src/rollout-ledger/backfill-swe-arena.test.mts +342 -0
- package/src/rollout-ledger/settle-capture.mts +442 -0
- package/src/rollout-ledger/settle-capture.test.mts +270 -0
- package/src/smoke-structural-rollout.mts +15 -9
- package/src/swe-arena/activation.mts +225 -0
- package/src/swe-arena/activation.test.mts +300 -0
- package/src/swe-arena/arms.ts +30 -14
- package/src/swe-arena/briefing.mts +217 -0
- package/src/swe-arena/briefing.test.mts +178 -0
- package/src/swe-arena/calibrate.ts +102 -1
- package/src/swe-arena/cell-evidence.mts +41 -9
- package/src/swe-arena/factory-command-container.ts +284 -0
- package/src/swe-arena/factory-judge-child.mts +228 -0
- package/src/swe-arena/factory.test.mts +643 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +51 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +29 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +64 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +48 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +29 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +48 -0
- package/src/swe-arena/fixtures/factory/loops-28/calibration.md +47 -0
- package/src/swe-arena/fixtures/factory/loops-28/manifest.json +30 -0
- package/src/swe-arena/fixtures/factory/loops-28/spec.md +50 -0
- package/src/swe-arena/fixtures/run-report/README.md +43 -0
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +173 -0
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +100 -0
- package/src/swe-arena/fixtures/run-report/gen3-rollup.json +551 -0
- package/src/swe-arena/fixtures/run-report/gen3-rollup.md +64 -0
- package/src/swe-arena/fixtures.ts +135 -2
- package/src/swe-arena/gepa-seat.mts +877 -0
- package/src/swe-arena/gepa-seat.test.mts +1059 -0
- package/src/swe-arena/implementation-ref.test.mts +64 -0
- package/src/swe-arena/implementation-ref.ts +62 -0
- package/src/swe-arena/ledger-orphans.mts +77 -0
- package/src/swe-arena/ledger-orphans.test.mts +147 -0
- package/src/swe-arena/outer-loop.mts +710 -33
- package/src/swe-arena/premeasured-from-cells.mts +281 -0
- package/src/swe-arena/premeasured-from-cells.test.mts +180 -0
- package/src/swe-arena/proposer-fanout.mts +327 -31
- package/src/swe-arena/proposer-fanout.test.mts +246 -0
- package/src/swe-arena/proposer-provenance.mts +172 -0
- package/src/swe-arena/proposer-provenance.test.mts +106 -0
- package/src/swe-arena/run-experiment.mts +367 -1
- package/src/swe-arena/run-report.mts +75 -0
- package/src/swe-arena/run-supervisor.test.mts +5 -3
- package/src/swe-arena/score-split.mts +140 -0
- package/src/swe-arena/score-split.test.mts +123 -0
- package/src/swe-arena/scratch-worktree.test.mts +55 -0
- package/src/swe-arena/scratch-worktree.ts +34 -0
- package/src/swe-arena/types.ts +52 -0
- package/src/swe-code-improve.mts +24 -25
- package/src/swe-improve.mts +129 -96
- package/src/swe-local-proof.mts +6 -1
- package/src/swe-stream.mts +4 -2
- package/src/swe-structural.mts +245 -837
- package/src/tb-container-executor.test.mts +30 -6
- package/src/tb-supervisor-sidecar.mts +2 -1
- package/src/trata-gepa.mts +182 -245
- package/dist/chunk-LRRD7NAG.js.map +0 -1
- package/src/live-improve-campaign-mbpp.mts +0 -641
- package/src/live-improve-campaign.mts +0 -500
- /package/dist/{chunk-CKUVRZ2T.js.map → chunk-3U5TXJZS.js.map} +0 -0
- /package/dist/{chunk-YCGY7UIZ.js.map → chunk-7GRVHU22.js.map} +0 -0
- /package/dist/{chunk-Z7ML6L77.js.map → chunk-HWST3SED.js.map} +0 -0
- /package/dist/{chunk-SYDW647C.js.map → chunk-IA2FBTWC.js.map} +0 -0
- /package/dist/{chunk-R67DFVLO.js.map → chunk-IFVINJ4B.js.map} +0 -0
- /package/dist/{chunk-R36V2VP7.js.map → chunk-IZ5M6OAC.js.map} +0 -0
- /package/dist/{chunk-ODT47UAY.js.map → chunk-K3BQGZCT.js.map} +0 -0
- /package/dist/{chunk-IFAV6KEM.js.map → chunk-KP5KD6EN.js.map} +0 -0
- /package/dist/{chunk-ZEWMTR5M.js.map → chunk-MQMRLGOG.js.map} +0 -0
- /package/dist/{chunk-TSWPNOYM.js.map → chunk-NQG5XDSB.js.map} +0 -0
- /package/dist/{chunk-7WSD27QQ.js.map → chunk-PB64GYIG.js.map} +0 -0
- /package/dist/{chunk-HBSWHQNJ.js.map → chunk-RCYQEFNX.js.map} +0 -0
- /package/dist/{chunk-J3KDJNX2.js.map → chunk-RH5F53JT.js.map} +0 -0
- /package/dist/{chunk-UAIOHCUK.js.map → chunk-SFLA7OH3.js.map} +0 -0
- /package/dist/{chunk-Y6O2OCUO.js.map → chunk-SHM6MRRF.js.map} +0 -0
- /package/dist/{chunk-KDIKRJGB.js.map → chunk-SHYIRB7I.js.map} +0 -0
- /package/dist/{chunk-HHXFIHXC.js.map → chunk-SVR2LKYI.js.map} +0 -0
- /package/dist/{chunk-PWQVGAJB.js.map → chunk-V7AEBY6U.js.map} +0 -0
- /package/dist/{chunk-2PVVP7GN.js.map → chunk-XKEFIFIC.js.map} +0 -0
- /package/dist/{chunk-JRWWGMK7.js.map → chunk-XYA4XSNU.js.map} +0 -0
- /package/dist/{chunk-X5YKXC6V.js.map → chunk-YSMEKBTD.js.map} +0 -0
- /package/dist/{chunk-2XU6OGEN.js.map → chunk-Z4TZ76N7.js.map} +0 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.4.0
|
|
4
|
+
|
|
5
|
+
- Add the resumable SWE improvement loop backed by the official GEPA engine.
|
|
6
|
+
- Isolate every concurrent candidate in its own Git worktree and authorize the exact candidate before execution.
|
|
7
|
+
- Include the complete Runtime, Bench, and installed SWE judge implementations in resume identity.
|
|
8
|
+
- Consume `@tangle-network/agent-runtime` 0.105.x, `@tangle-network/agent-eval` 0.126.6, and `@tangle-network/agent-knowledge` 5.0.1.
|
|
9
|
+
|
|
10
|
+
## 0.3.8
|
|
11
|
+
|
|
12
|
+
- Resolve package-owned fixtures and scripts from both source and compiled installs, and prove ToolLLM fixture loading from the packed package.
|
|
13
|
+
|
|
3
14
|
## 0.3.7
|
|
4
15
|
|
|
5
16
|
- Declare the public executable search path for Pier candidate entrypoints so the exact process contract can replay them against `@tangle-network/agent-runtime@0.102.0`.
|
package/HARNESS.md
CHANGED
|
@@ -71,6 +71,48 @@ this codebase. `agent-lab/runs/RUNS.md` is the index mapping artifacts → verdi
|
|
|
71
71
|
findings gist. **Set `OUT=runs/<date>/<name>.json` (in agent-lab) on every run — never `/tmp`**
|
|
72
72
|
(a reboot erases ramdisk; ~20 runs nearly died there once).
|
|
73
73
|
|
|
74
|
+
## Run observability — never hand-grep a journal again
|
|
75
|
+
|
|
76
|
+
Every supervisor cell writes `run-report.json` + `run-report.md` next to its artifacts, and
|
|
77
|
+
appends a headline block to the run log, automatically: `outer-loop.mts` and both
|
|
78
|
+
`run-experiment.mts` paths call `writeRunReportSafe` after each cell and `reportRound` at
|
|
79
|
+
the gate. Source: `swe-arena/run-report.mts` (pure `computeRunReport` + an I/O collector).
|
|
80
|
+
|
|
81
|
+
What it answers, all deterministic, all from artifacts already on disk (`journal.jsonl`,
|
|
82
|
+
`workers/*.ndjson`, `workers/*.inbox.ndjson`, `state.json`, `result.json`, `judge.json`,
|
|
83
|
+
the ledger, `driver.log`, the delivered patch, the opencode session store):
|
|
84
|
+
|
|
85
|
+
- **orchestration** — workers spawned/settled/cancelled, **steer count with a per-worker
|
|
86
|
+
breakdown**, spawn waves + sizes, max concurrency, respawns, repeated labels, delegation
|
|
87
|
+
depth, time-to-first-spawn, supervisor wall, idle wall (zero live workers), worker
|
|
88
|
+
utilization (Σ worker wall ÷ supervisor wall).
|
|
89
|
+
- **decision quality** — settled by status/verdict, accepted vs rejected vs empty-pass,
|
|
90
|
+
evidence→respawn sequences vs blind respawns, review actions, evidence bytes returned.
|
|
91
|
+
- **economics** — tokens + USD by role (brain from journal `metered`; workers from journal
|
|
92
|
+
`settled` spend plus the opencode session join), per-worker rows, cost per accepted patch,
|
|
93
|
+
worker wall distribution.
|
|
94
|
+
- **outcome** — supervisor status/verdict/delivered, judge verdict + score (from `judge.json`
|
|
95
|
+
or the matching ledger row, with provenance), verify gate, patch file/line/test-touch stats.
|
|
96
|
+
|
|
97
|
+
Every metric is UNAVAILABLE-aware: a missing artifact yields `{"unavailable": "<reason>"}`
|
|
98
|
+
and renders as `unavailable — <reason>`, never `0`. A supervisor that steered nobody and a
|
|
99
|
+
supervisor whose worker logs were lost are visually different, because they have driven
|
|
100
|
+
opposite conclusions about the same architecture.
|
|
101
|
+
|
|
102
|
+
Standalone / backfill (use `--report-dir` when the run directory must stay READ-ONLY):
|
|
103
|
+
```
|
|
104
|
+
tsx src/swe-arena/run-report.mts <cellDir> [--patch p.diff] [--ledger ledger.jsonl] [--log run.log] [--report-dir out/]
|
|
105
|
+
tsx src/swe-arena/run-report.mts --round <outDir> [--report-dir out/]
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Relationship to the published `traces` CLI (`npx @tangle-network/traces analyze`): traces
|
|
109
|
+
covers the HARNESS-SESSION layer — per-session model calls, token/cost tables, latency
|
|
110
|
+
distributions, stuck-loop detection, tool-error rates, skill/subagent adoption — reading the
|
|
111
|
+
opencode/claude-code/codex session stores. It has no supervision-tree reader: no journal
|
|
112
|
+
parsing, no steer/wave/concurrency/utilization/delegation-depth metric, no accepted-vs-rejected
|
|
113
|
+
worker accounting. The two are complementary; run-report prints the traces command for the
|
|
114
|
+
same cell rather than re-deriving session-level facts.
|
|
115
|
+
|
|
74
116
|
## Data flow (the whole experiment in one line)
|
|
75
117
|
`rollout (worker → answer) → adapter.judge (valid?) → CORPUS RunRecord (k attempts, output+valid each) → corpus-replay --selector (pick WITHOUT the judge) → corpus-report CI → gate verdict`
|
|
76
118
|
The expensive part (rollouts) produces a **reusable corpus**; selection + stats are free
|
|
@@ -135,6 +177,7 @@ the gate + measurement tools:
|
|
|
135
177
|
commit0-env-run.mts the HARD domain through `runBenchmark` (the optimization suite)
|
|
136
178
|
terminal-compare.ts Terminal-Bench compare (own main)
|
|
137
179
|
pnpm verify:pier zero-model failure/pass Pier controls through a separate verifier
|
|
180
|
+
swe-arena/run-report.mts deterministic per-cell run report (steers, waves, idle, cost by role) + round rollup
|
|
138
181
|
unit tests (the only fully-green, cred-free runnable surface besides offline replay):
|
|
139
182
|
node --test --import tsx src/{selector,refine-loop}.test.mts
|
|
140
183
|
tsx src/gate.test.mts # offline plumbing test (no creds)
|
package/README.md
CHANGED
|
@@ -26,6 +26,13 @@ pnpm install # tsx + link parent
|
|
|
26
26
|
|
|
27
27
|
The judge needs only Docker; workers need a model key (Tangle router `TANGLE_API_KEY`, or a direct provider).
|
|
28
28
|
|
|
29
|
+
Live optimizer scripts require explicit token prices so cost records cannot be guessed.
|
|
30
|
+
Use the `REFLECT_` prefix for the general benchmark scripts and `GEPA_OPTIMIZER_` for the SWE arena seat.
|
|
31
|
+
For either prefix, set `INPUT_USD_PER_MILLION`, `CACHED_INPUT_USD_PER_MILLION`, `CACHE_WRITE_USD_PER_MILLION`, and `OUTPUT_USD_PER_MILLION`.
|
|
32
|
+
The SWE arena seat reads its model from `GEPA_OPTIMIZER_MODEL`, its URL from `GEPA_OPTIMIZER_BASE_URL`, and its key from `GEPA_OPTIMIZER_API_KEY`.
|
|
33
|
+
It falls back to the configured driver model, `ROUTER_BASE`, and `TANGLE_API_KEY`.
|
|
34
|
+
Missing prices fail before an optimizer model call.
|
|
35
|
+
|
|
29
36
|
Retain every official per-test log and report before the temporary evaluator directory is removed:
|
|
30
37
|
|
|
31
38
|
```ts
|
package/dist/adapters.js
CHANGED
|
@@ -1,40 +1,40 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ADAPTERS,
|
|
3
3
|
resolveAdapter
|
|
4
|
-
} from "./chunk-
|
|
5
|
-
import "./chunk-
|
|
4
|
+
} from "./chunk-V7AEBY6U.js";
|
|
5
|
+
import "./chunk-XKEFIFIC.js";
|
|
6
6
|
import "./chunk-VQRS7VUC.js";
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
7
|
+
import "./chunk-HWST3SED.js";
|
|
8
|
+
import "./chunk-IFVINJ4B.js";
|
|
9
9
|
import "./chunk-SEVJPLZC.js";
|
|
10
|
-
import "./chunk-
|
|
11
|
-
import "./chunk-
|
|
12
|
-
import "./chunk-
|
|
13
|
-
import "./chunk-
|
|
14
|
-
import "./chunk-
|
|
15
|
-
import "./chunk-
|
|
10
|
+
import "./chunk-3U5TXJZS.js";
|
|
11
|
+
import "./chunk-SHM6MRRF.js";
|
|
12
|
+
import "./chunk-RCYQEFNX.js";
|
|
13
|
+
import "./chunk-SFLA7OH3.js";
|
|
14
|
+
import "./chunk-IZ5M6OAC.js";
|
|
15
|
+
import "./chunk-XYA4XSNU.js";
|
|
16
16
|
import "./chunk-INNOYXCP.js";
|
|
17
17
|
import "./chunk-PA2ZKHJC.js";
|
|
18
18
|
import "./chunk-5H5XV76F.js";
|
|
19
19
|
import "./chunk-TBKU5XQI.js";
|
|
20
|
-
import "./chunk-
|
|
21
|
-
import "./chunk-
|
|
22
|
-
import "./chunk-
|
|
20
|
+
import "./chunk-NQG5XDSB.js";
|
|
21
|
+
import "./chunk-SHYIRB7I.js";
|
|
22
|
+
import "./chunk-MQMRLGOG.js";
|
|
23
23
|
import "./chunk-PUIRNYI7.js";
|
|
24
24
|
import "./chunk-C7T7WEK2.js";
|
|
25
25
|
import "./chunk-JTHWEDEW.js";
|
|
26
|
-
import "./chunk-
|
|
27
|
-
import "./chunk-
|
|
26
|
+
import "./chunk-Z4TZ76N7.js";
|
|
27
|
+
import "./chunk-SVR2LKYI.js";
|
|
28
28
|
import "./chunk-X3BTXCJ4.js";
|
|
29
|
-
import "./chunk-
|
|
30
|
-
import "./chunk-
|
|
31
|
-
import "./chunk-
|
|
29
|
+
import "./chunk-RH5F53JT.js";
|
|
30
|
+
import "./chunk-YSMEKBTD.js";
|
|
31
|
+
import "./chunk-7GRVHU22.js";
|
|
32
32
|
import "./chunk-UPAMRDX4.js";
|
|
33
|
-
import "./chunk-
|
|
34
|
-
import "./chunk-
|
|
35
|
-
import "./chunk-
|
|
36
|
-
import "./chunk-
|
|
37
|
-
import "./chunk-
|
|
33
|
+
import "./chunk-K3BQGZCT.js";
|
|
34
|
+
import "./chunk-PB64GYIG.js";
|
|
35
|
+
import "./chunk-IA2FBTWC.js";
|
|
36
|
+
import "./chunk-KP5KD6EN.js";
|
|
37
|
+
import "./chunk-WSKWVEQB.js";
|
|
38
38
|
import "./chunk-53UPUNBZ.js";
|
|
39
39
|
export {
|
|
40
40
|
ADAPTERS,
|
|
@@ -19,7 +19,7 @@ import '@tangle-network/agent-runtime/loops';
|
|
|
19
19
|
* nonzero throws with the captured stderr — never a fabricated score.
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
/**
|
|
22
|
+
/** Package root for agent-bench, so `.venv`, scripts, and fixtures resolve. */
|
|
23
23
|
declare const benchRoot: string;
|
|
24
24
|
/** Resolve the shared interpreter without requiring an installed package to contain a venv. */
|
|
25
25
|
declare function resolveBenchPython(env?: Readonly<{
|
|
@@ -3,8 +3,8 @@ import {
|
|
|
3
3
|
appworldToolLoopClient,
|
|
4
4
|
createAppWorldAdapter,
|
|
5
5
|
createAppWorldReactAdapter
|
|
6
|
-
} from "../chunk-
|
|
7
|
-
import "../chunk-
|
|
6
|
+
} from "../chunk-IA2FBTWC.js";
|
|
7
|
+
import "../chunk-WSKWVEQB.js";
|
|
8
8
|
export {
|
|
9
9
|
appworldSolutionOutput,
|
|
10
10
|
appworldToolLoopClient,
|
package/dist/benchmarks/bfcl.js
CHANGED
package/dist/benchmarks/crag.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createEnterpriseOpsGymAdapter,
|
|
3
3
|
enterpriseOpsTranscriptOutput
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-YSMEKBTD.js";
|
|
5
|
+
import "../chunk-WSKWVEQB.js";
|
|
6
6
|
export {
|
|
7
7
|
createEnterpriseOpsGymAdapter,
|
|
8
8
|
enterpriseOpsTranscriptOutput
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createProgrambenchAdapter,
|
|
3
3
|
programbenchSubmissionOutput
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-MQMRLGOG.js";
|
|
5
|
+
import "../chunk-WSKWVEQB.js";
|
|
6
6
|
export {
|
|
7
7
|
createProgrambenchAdapter,
|
|
8
8
|
programbenchSubmissionOutput
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createTau2BenchAdapter,
|
|
3
3
|
tau2ResultsOutput
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
6
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-RCYQEFNX.js";
|
|
5
|
+
import "../chunk-IZ5M6OAC.js";
|
|
6
|
+
import "../chunk-WSKWVEQB.js";
|
|
7
7
|
export {
|
|
8
8
|
createTau2BenchAdapter,
|
|
9
9
|
tau2ResultsOutput
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createTau3BankingAdapter
|
|
3
|
-
} from "../chunk-
|
|
4
|
-
import "../chunk-
|
|
5
|
-
import "../chunk-
|
|
3
|
+
} from "../chunk-SFLA7OH3.js";
|
|
4
|
+
import "../chunk-IZ5M6OAC.js";
|
|
5
|
+
import "../chunk-WSKWVEQB.js";
|
|
6
6
|
export {
|
|
7
7
|
createTau3BankingAdapter
|
|
8
8
|
};
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createWebArenaVerifiedAdapter,
|
|
3
3
|
webarenaOutputDirOutput
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-HWST3SED.js";
|
|
5
|
+
import "../chunk-WSKWVEQB.js";
|
|
6
6
|
export {
|
|
7
7
|
createWebArenaVerifiedAdapter,
|
|
8
8
|
webarenaOutputDirOutput
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
runVenvPython,
|
|
6
6
|
safeRunId,
|
|
7
7
|
stageFile
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-WSKWVEQB.js";
|
|
9
9
|
|
|
10
10
|
// src/benchmarks/swe-bench.ts
|
|
11
11
|
import { join } from "path";
|
|
@@ -248,4 +248,4 @@ export {
|
|
|
248
248
|
sweEvaluationArgv,
|
|
249
249
|
createSweBenchAdapter
|
|
250
250
|
};
|
|
251
|
-
//# sourceMappingURL=chunk-
|
|
251
|
+
//# sourceMappingURL=chunk-3U5TXJZS.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
benchRoot
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-WSKWVEQB.js";
|
|
4
4
|
|
|
5
5
|
// src/benchmarks/finresearchbench.ts
|
|
6
6
|
import { readFile, stat } from "fs/promises";
|
|
@@ -205,4 +205,4 @@ function createFinResearchBenchAdapter() {
|
|
|
205
205
|
export {
|
|
206
206
|
createFinResearchBenchAdapter
|
|
207
207
|
};
|
|
208
|
-
//# sourceMappingURL=chunk-
|
|
208
|
+
//# sourceMappingURL=chunk-7GRVHU22.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
benchRoot,
|
|
3
3
|
runVenvPython
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-WSKWVEQB.js";
|
|
5
5
|
|
|
6
6
|
// src/benchmarks/webarena-verified.ts
|
|
7
7
|
import { access, readFile, stat } from "fs/promises";
|
|
@@ -159,4 +159,4 @@ export {
|
|
|
159
159
|
webarenaOutputDirOutput,
|
|
160
160
|
createWebArenaVerifiedAdapter
|
|
161
161
|
};
|
|
162
|
-
//# sourceMappingURL=chunk-
|
|
162
|
+
//# sourceMappingURL=chunk-HWST3SED.js.map
|
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
preflightVenvImports,
|
|
4
4
|
runVenvScriptStdin,
|
|
5
5
|
venvPython
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-WSKWVEQB.js";
|
|
7
7
|
|
|
8
8
|
// src/benchmarks/appworld.ts
|
|
9
9
|
import { spawn } from "child_process";
|
|
@@ -315,4 +315,4 @@ export {
|
|
|
315
315
|
appworldToolLoopClient,
|
|
316
316
|
createAppWorldReactAdapter
|
|
317
317
|
};
|
|
318
|
-
//# sourceMappingURL=chunk-
|
|
318
|
+
//# sourceMappingURL=chunk-IA2FBTWC.js.map
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
} from "./chunk-X3BTXCJ4.js";
|
|
15
15
|
import {
|
|
16
16
|
benchRoot
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-WSKWVEQB.js";
|
|
18
18
|
|
|
19
19
|
// src/benchmarks/ragbench.ts
|
|
20
20
|
import { readFile } from "fs/promises";
|
|
@@ -139,4 +139,4 @@ function createRagBenchAdapter() {
|
|
|
139
139
|
export {
|
|
140
140
|
createRagBenchAdapter
|
|
141
141
|
};
|
|
142
|
-
//# sourceMappingURL=chunk-
|
|
142
|
+
//# sourceMappingURL=chunk-IFVINJ4B.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
runVenvPython
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-WSKWVEQB.js";
|
|
4
4
|
|
|
5
5
|
// src/benchmarks/tau-bench-shared.ts
|
|
6
6
|
import { readFile, stat } from "fs/promises";
|
|
@@ -166,4 +166,4 @@ export {
|
|
|
166
166
|
tauResultsOutput,
|
|
167
167
|
createTauBenchAdapter
|
|
168
168
|
};
|
|
169
|
-
//# sourceMappingURL=chunk-
|
|
169
|
+
//# sourceMappingURL=chunk-IZ5M6OAC.js.map
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
runStagedJudge,
|
|
6
6
|
stageFile,
|
|
7
7
|
venvPython
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-WSKWVEQB.js";
|
|
9
9
|
|
|
10
10
|
// src/benchmarks/aec-bench.ts
|
|
11
11
|
import { readFile } from "fs/promises";
|
|
@@ -218,4 +218,4 @@ Fix: ensure network access to raw.githubusercontent.com, or set AEC_FIXTURES=1 t
|
|
|
218
218
|
export {
|
|
219
219
|
createAecBenchAdapter
|
|
220
220
|
};
|
|
221
|
-
//# sourceMappingURL=chunk-
|
|
221
|
+
//# sourceMappingURL=chunk-K3BQGZCT.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
benchRoot
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-WSKWVEQB.js";
|
|
4
4
|
|
|
5
5
|
// src/benchmarks/bfcl.ts
|
|
6
6
|
import { readFile, stat } from "fs/promises";
|
|
@@ -273,4 +273,4 @@ export {
|
|
|
273
273
|
bfclOutput,
|
|
274
274
|
createBfclAdapter
|
|
275
275
|
};
|
|
276
|
-
//# sourceMappingURL=chunk-
|
|
276
|
+
//# sourceMappingURL=chunk-KP5KD6EN.js.map
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
benchRoot,
|
|
3
3
|
preflightVenvImports,
|
|
4
4
|
runVenvScriptStdin
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-WSKWVEQB.js";
|
|
6
6
|
|
|
7
7
|
// src/benchmarks/programbench.ts
|
|
8
8
|
import { join } from "path";
|
|
@@ -133,4 +133,4 @@ export {
|
|
|
133
133
|
programbenchSubmissionOutput,
|
|
134
134
|
createProgrambenchAdapter
|
|
135
135
|
};
|
|
136
|
-
//# sourceMappingURL=chunk-
|
|
136
|
+
//# sourceMappingURL=chunk-MQMRLGOG.js.map
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
} from "./chunk-X3BTXCJ4.js";
|
|
11
11
|
import {
|
|
12
12
|
benchRoot
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-WSKWVEQB.js";
|
|
14
14
|
|
|
15
15
|
// src/benchmarks/nomiracl.ts
|
|
16
16
|
import { readFile } from "fs/promises";
|
|
@@ -144,4 +144,4 @@ function createNoMiraclAdapter() {
|
|
|
144
144
|
export {
|
|
145
145
|
createNoMiraclAdapter
|
|
146
146
|
};
|
|
147
|
-
//# sourceMappingURL=chunk-
|
|
147
|
+
//# sourceMappingURL=chunk-NQG5XDSB.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
benchRoot
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-WSKWVEQB.js";
|
|
4
4
|
|
|
5
5
|
// src/benchmarks/agentbench.ts
|
|
6
6
|
import { readFile } from "fs/promises";
|
|
@@ -115,4 +115,4 @@ export {
|
|
|
115
115
|
agentbenchAnswerOutput,
|
|
116
116
|
createAgentBenchAdapter
|
|
117
117
|
};
|
|
118
|
-
//# sourceMappingURL=chunk-
|
|
118
|
+
//# sourceMappingURL=chunk-PB64GYIG.js.map
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createTauBenchAdapter,
|
|
3
3
|
tauResultsOutput
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-IZ5M6OAC.js";
|
|
5
5
|
import {
|
|
6
6
|
benchRoot
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-WSKWVEQB.js";
|
|
8
8
|
|
|
9
9
|
// src/benchmarks/tau2-bench.ts
|
|
10
10
|
import { join } from "path";
|
|
@@ -27,4 +27,4 @@ export {
|
|
|
27
27
|
tau2ResultsOutput,
|
|
28
28
|
createTau2BenchAdapter
|
|
29
29
|
};
|
|
30
|
-
//# sourceMappingURL=chunk-
|
|
30
|
+
//# sourceMappingURL=chunk-RCYQEFNX.js.map
|
|
@@ -2,7 +2,7 @@ import {
|
|
|
2
2
|
benchRoot,
|
|
3
3
|
runVenvPython,
|
|
4
4
|
runVenvScriptStdin
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-WSKWVEQB.js";
|
|
6
6
|
|
|
7
7
|
// src/benchmarks/dabstep.ts
|
|
8
8
|
import { join } from "path";
|
|
@@ -179,4 +179,4 @@ export {
|
|
|
179
179
|
dabstepAnswerOutput,
|
|
180
180
|
createDabstepAdapter
|
|
181
181
|
};
|
|
182
|
-
//# sourceMappingURL=chunk-
|
|
182
|
+
//# sourceMappingURL=chunk-RH5F53JT.js.map
|