@arcforge/platform 2.0.166
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/package.json +69 -0
- package/src/bin/supervisor.ts +102 -0
- package/src/bin/test-api.ts +30 -0
- package/src/bin/test-preload.ts +308 -0
- package/src/bin/update-helper.ts +32 -0
- package/src/build/bench/CLAUDE.md +106 -0
- package/src/build/bench/SPEC.md +209 -0
- package/src/build/bench/agents.ts +105 -0
- package/src/build/bench/axes.ts +84 -0
- package/src/build/bench/bench.ts +96 -0
- package/src/build/bench/discover.ts +147 -0
- package/src/build/bench/index.ts +7 -0
- package/src/build/bench/log.ts +47 -0
- package/src/build/bench/matrix.ts +23 -0
- package/src/build/bench/preload.ts +440 -0
- package/src/build/bench/prepare.ts +61 -0
- package/src/build/bench/projection.ts +112 -0
- package/src/build/bench/resolve.ts +54 -0
- package/src/build/bench/runner.ts +309 -0
- package/src/build/bench/schema/extract.ts +148 -0
- package/src/build/bench/schema/index.ts +2 -0
- package/src/build/bench/schema/schema.ts +95 -0
- package/src/build/bench/schema/worker.ts +30 -0
- package/src/build/bench/typegen.ts +119 -0
- package/src/build/bench/workspace/files.ts +139 -0
- package/src/build/bench/workspace/handle.ts +133 -0
- package/src/build/bench/workspace/index.ts +11 -0
- package/src/build/bench/workspace/workspace.ts +14 -0
- package/src/build/blueprint/blueprint.ts +398 -0
- package/src/build/blueprint/cognet/abi.ts +207 -0
- package/src/build/blueprint/cognet/bundle-worker.ts +93 -0
- package/src/build/blueprint/cognet/bundle.ts +664 -0
- package/src/build/blueprint/cognet/cognet.ts +249 -0
- package/src/build/blueprint/cognet/index.ts +10 -0
- package/src/build/blueprint/cognet/manifest.ts +38 -0
- package/src/build/blueprint/cognet/resolve.ts +113 -0
- package/src/build/blueprint/collisions.ts +51 -0
- package/src/build/blueprint/index.ts +29 -0
- package/src/build/blueprint/modules/entries.ts +117 -0
- package/src/build/blueprint/modules/index.ts +300 -0
- package/src/build/blueprint/modules/meta.ts +174 -0
- package/src/build/blueprint/scan/boot.ts +34 -0
- package/src/build/blueprint/scan/cognetImports.ts +56 -0
- package/src/build/blueprint/scan/components.ts +45 -0
- package/src/build/blueprint/scan/config.ts +234 -0
- package/src/build/blueprint/scan/configImports.ts +232 -0
- package/src/build/blueprint/scan/declare-worker.ts +61 -0
- package/src/build/blueprint/scan/declare.ts +393 -0
- package/src/build/blueprint/scan/h3-globals.ts +137 -0
- package/src/build/blueprint/scan/knowledge.ts +101 -0
- package/src/build/blueprint/scan/middleware.ts +110 -0
- package/src/build/blueprint/scan/moduleImports.ts +79 -0
- package/src/build/blueprint/scan/plugins.ts +93 -0
- package/src/build/blueprint/scan/promptPackages.ts +45 -0
- package/src/build/blueprint/scan/prompts.ts +164 -0
- package/src/build/blueprint/scan/published.ts +131 -0
- package/src/build/blueprint/scan/resolve.ts +398 -0
- package/src/build/blueprint/scan/routes.ts +109 -0
- package/src/build/blueprint/scan/scripts.ts +119 -0
- package/src/build/blueprint/scan/tool-bundle-worker.ts +69 -0
- package/src/build/blueprint/scan/tools.ts +610 -0
- package/src/build/blueprint/types.ts +28 -0
- package/src/build/extensions/config.ts +261 -0
- package/src/build/extensions/edit.ts +684 -0
- package/src/build/extensions/extensions.ts +731 -0
- package/src/build/extensions/index.ts +10 -0
- package/src/build/extensions/load.ts +350 -0
- package/src/build/extensions/registrations.ts +189 -0
- package/src/build/extensions/store.ts +200 -0
- package/src/build/frame/frame.ts +145 -0
- package/src/build/frame/index.ts +5 -0
- package/src/build/frame/migrate.ts +246 -0
- package/src/build/project/bundle/agent.ts +382 -0
- package/src/build/project/bundle/artifacts.ts +88 -0
- package/src/build/project/bundle/assets.ts +385 -0
- package/src/build/project/bundle/bundle.ts +160 -0
- package/src/build/project/bundle/imports.ts +102 -0
- package/src/build/project/bundle/index.ts +16 -0
- package/src/build/project/bundle/module.ts +130 -0
- package/src/build/project/bundle/source.ts +136 -0
- package/src/build/project/bundle/stage.ts +97 -0
- package/src/build/project/bundle/types.ts +111 -0
- package/src/build/project/cognet.ts +156 -0
- package/src/build/project/create/agent.ts +52 -0
- package/src/build/project/create/bench.ts +81 -0
- package/src/build/project/create/cognet.ts +102 -0
- package/src/build/project/create/extension.ts +63 -0
- package/src/build/project/create/module.ts +43 -0
- package/src/build/project/create/profile.ts +102 -0
- package/src/build/project/create/prompt.ts +68 -0
- package/src/build/project/create/templates.ts +88 -0
- package/src/build/project/deploy.ts +96 -0
- package/src/build/project/index.ts +40 -0
- package/src/build/project/installer.ts +616 -0
- package/src/build/project/kinds.ts +374 -0
- package/src/build/project/links.ts +33 -0
- package/src/build/project/manifest/bunfig.ts +139 -0
- package/src/build/project/manifest/config.ts +246 -0
- package/src/build/project/manifest/env.ts +184 -0
- package/src/build/project/manifest/index.ts +9 -0
- package/src/build/project/manifest/manifest.ts +49 -0
- package/src/build/project/manifest/model.ts +124 -0
- package/src/build/project/manifest/package.ts +386 -0
- package/src/build/project/manifest/text.ts +19 -0
- package/src/build/project/models/fetch.ts +123 -0
- package/src/build/project/models/index.ts +8 -0
- package/src/build/project/models/models.ts +111 -0
- package/src/build/project/models/specifier.ts +106 -0
- package/src/build/project/models/store.ts +202 -0
- package/src/build/project/modules/hash.ts +31 -0
- package/src/build/project/modules/index.ts +5 -0
- package/src/build/project/modules/lock.ts +78 -0
- package/src/build/project/modules/modules.ts +155 -0
- package/src/build/project/prepare.ts +709 -0
- package/src/build/project/project.ts +253 -0
- package/src/build/project/projects.ts +164 -0
- package/src/build/project/propagation.ts +120 -0
- package/src/build/project/publish/verify.ts +312 -0
- package/src/build/project/publish.ts +155 -0
- package/src/build/project/reconcile.ts +93 -0
- package/src/build/project/specifier.ts +42 -0
- package/src/build/project/tree.ts +615 -0
- package/src/build/project/treecache.ts +356 -0
- package/src/build/project/typegen/axon-dts.ts +362 -0
- package/src/build/project/typegen/cognet-dts.ts +91 -0
- package/src/build/project/typegen/components.ts +45 -0
- package/src/build/project/typegen/env.ts +59 -0
- package/src/build/project/typegen/generators.ts +108 -0
- package/src/build/project/typegen/hooks.ts +71 -0
- package/src/build/project/typegen/index.ts +137 -0
- package/src/build/project/typegen/prompt-dts.ts +40 -0
- package/src/build/project/typegen/prompts.ts +36 -0
- package/src/build/project/typegen/registry-dts.ts +109 -0
- package/src/build/project/typegen/scripts.ts +37 -0
- package/src/build/project/typegen/tools.ts +31 -0
- package/src/build/project/typegen/tsconfig.ts +243 -0
- package/src/build/project/typegen/tui-dts.ts +120 -0
- package/src/build/project/typegen/write.ts +40 -0
- package/src/build/project/verify.ts +268 -0
- package/src/build/project/watcher.ts +266 -0
- package/src/build/runtime/agent.ts +550 -0
- package/src/build/runtime/escalations.ts +159 -0
- package/src/build/runtime/index.ts +15 -0
- package/src/build/runtime/instances.ts +604 -0
- package/src/build/runtime/profile.ts +91 -0
- package/src/build/runtime/progress.ts +143 -0
- package/src/build/runtime/requests.ts +167 -0
- package/src/build/runtime/resolve.ts +159 -0
- package/src/build/runtime/runtime.ts +233 -0
- package/src/build/runtime/sessions/index.ts +5 -0
- package/src/build/runtime/sessions/record.ts +145 -0
- package/src/build/runtime/sessions/sessions.ts +240 -0
- package/src/build/runtime/zeno.ts +120 -0
- package/src/index.ts +11 -0
- package/src/link/agent-main.ts +395 -0
- package/src/link/index.ts +70 -0
- package/src/link/services.ts +120 -0
- package/src/platform.ts +293 -0
- package/src/procs/index.ts +15 -0
- package/src/procs/tree.ts +177 -0
- package/src/services/cloud.ts +419 -0
- package/src/services/control/client.ts +126 -0
- package/src/services/control/dispatch.ts +156 -0
- package/src/services/control/index.ts +17 -0
- package/src/services/control/protocol.ts +179 -0
- package/src/services/control/server.ts +241 -0
- package/src/services/deployments.ts +81 -0
- package/src/services/files.ts +329 -0
- package/src/services/index.ts +5 -0
- package/src/services/mic/capture.ts +248 -0
- package/src/services/mic/config.ts +109 -0
- package/src/services/mic/fft.ts +93 -0
- package/src/services/mic/index.ts +139 -0
- package/src/services/mic/visualizer.ts +46 -0
- package/src/services/ollama/catalog.ts +122 -0
- package/src/services/ollama/http.ts +124 -0
- package/src/services/ollama/index.ts +13 -0
- package/src/services/ollama/models.ts +153 -0
- package/src/services/ollama/ollama.ts +89 -0
- package/src/services/ollama/registry.ts +112 -0
- package/src/services/ollama/types.ts +72 -0
- package/src/services/prompts.ts +76 -0
- package/src/services/registry.ts +220 -0
- package/src/services/resources/hardware.ts +125 -0
- package/src/services/resources/index.ts +6 -0
- package/src/services/resources/reservations.ts +162 -0
- package/src/services/resources/resources.ts +156 -0
- package/src/services/running/index.ts +1 -0
- package/src/services/running/running.ts +200 -0
- package/src/services/store/disk.ts +97 -0
- package/src/services/store/index.ts +15 -0
- package/src/services/store/store.ts +729 -0
- package/src/services/store/types.ts +303 -0
- package/src/services/test/child.ts +150 -0
- package/src/services/test/events.ts +102 -0
- package/src/services/test/files.ts +26 -0
- package/src/services/test/frames.ts +39 -0
- package/src/services/test/index.ts +7 -0
- package/src/services/test/runner.ts +122 -0
- package/src/services/test/types.ts +26 -0
- package/src/services/update/contract.ts +77 -0
- package/src/services/update/index.ts +12 -0
- package/src/services/update/installer.ts +139 -0
- package/src/services/update/updates.ts +137 -0
- package/src/utils/fs.ts +148 -0
- package/src/utils/tsast.ts +102 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# bench
|
|
2
|
+
|
|
3
|
+
## What This Is
|
|
4
|
+
|
|
5
|
+
Benchmarks as a project kind, alongside agents, modules, cognets and prompts. A
|
|
6
|
+
bench runs one scenario against many configurations of an agent and records
|
|
7
|
+
what happened, so a claim like "this cognet is better at refactoring" has
|
|
8
|
+
evidence under it.
|
|
9
|
+
|
|
10
|
+
Consumed by `axon bench init|prepare|run|result` and `axon publish` when the
|
|
11
|
+
nearest project is a bench. Published to the registry as a definition, so
|
|
12
|
+
someone else can clone it and run it against their own agent.
|
|
13
|
+
|
|
14
|
+
**This module owns benchmarking, not project management.** Finding, opening,
|
|
15
|
+
scaffolding, preparing and publishing a bench are `Project()`'s, exactly as for
|
|
16
|
+
every other kind — a bench's row in `build/project/kinds.ts` is what makes it a
|
|
17
|
+
kind. `Bench()` hangs off `project.bench` and holds only the verbs no other
|
|
18
|
+
kind has. There used to be a parallel `Benches()` manager duplicating find /
|
|
19
|
+
open / create / publish; it is gone.
|
|
20
|
+
|
|
21
|
+
## The Design
|
|
22
|
+
|
|
23
|
+
**One variable at a time, or a deliberate grid.** `matrix` is a plain object:
|
|
24
|
+
each key is a binding point, each value the variations to try there. One key
|
|
25
|
+
is the controlled experiment — the shape to reach for, because it is what
|
|
26
|
+
makes a result attributable. Several keys multiply, and the Cartesian growth
|
|
27
|
+
is the author's to manage rather than the runtime's to hide.
|
|
28
|
+
|
|
29
|
+
Keys are either named axes (`agent`, `cognet`, `model`, `env`) or dotted paths
|
|
30
|
+
into the blueprint (`config.engine.temperature`). The named set stays small
|
|
31
|
+
because the paths are the escape hatch. An earlier design had eight "factor
|
|
32
|
+
kinds" of which three were ever wired — the others (`dataset`, `judge`,
|
|
33
|
+
`toolset`) were not binding points at all, and modelling them as such is what
|
|
34
|
+
made the taxonomy grow.
|
|
35
|
+
|
|
36
|
+
**`expect` and `observe` are not the same tool.** A test encodes what you
|
|
37
|
+
already know should happen; a benchmark exists because you do not. So:
|
|
38
|
+
|
|
39
|
+
- `expect` — did the SCENARIO hold? Workspace copied, agent booted. A failure
|
|
40
|
+
means the trial measures nothing and must stop rather than score zero.
|
|
41
|
+
- `observe` — how did the SUBJECT do? Recorded whatever the value is, never a
|
|
42
|
+
failure. A model that scores badly is the finding.
|
|
43
|
+
|
|
44
|
+
This is why `observe` cannot collapse into `expect`: one halts on surprise,
|
|
45
|
+
the other records it, and a benchmark whose weak conditions all "failed" would
|
|
46
|
+
report itself as broken.
|
|
47
|
+
|
|
48
|
+
**Evidence, then projection.** `run()` executes and persists events; the
|
|
49
|
+
result is derived by REPLAY over that log. `BenchRunResult` deliberately holds
|
|
50
|
+
no aggregates — they are reproducible projections over the immutable record.
|
|
51
|
+
Physics (duration, tokens, cost, engine calls) are derived from the kernel log
|
|
52
|
+
with the author declaring nothing.
|
|
53
|
+
|
|
54
|
+
**Directories mean one thing each.**
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
bench.config.ts matrix, measurements, setup
|
|
58
|
+
workspace/ the agent's world — copied fresh per iteration
|
|
59
|
+
fixtures/ author-side inputs; never visible to the agent
|
|
60
|
+
tests/ the scenarios, and where measurements are recorded
|
|
61
|
+
.bench/ generated declarations, run logs, materialized workspaces
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The `workspace/` ↔ `fixtures/` split is an invariant, not a convention: if
|
|
65
|
+
fixtures leaked into the agent's view, a run would depend on the bench's own
|
|
66
|
+
layout and stop being reproducible.
|
|
67
|
+
|
|
68
|
+
## Key Interfaces
|
|
69
|
+
|
|
70
|
+
```typescript
|
|
71
|
+
const project = await platform.projects.openAs("bench", cwd)
|
|
72
|
+
|
|
73
|
+
await project.prepare() // shared frame + the bench half below
|
|
74
|
+
await project.publish() // ship the definition (never the results)
|
|
75
|
+
|
|
76
|
+
await project.bench.coordinates() // the expanded matrix, without running
|
|
77
|
+
await project.bench.run() // execute, persist, project → BenchRunResult
|
|
78
|
+
await project.bench.result(runId) // rebuild a past result by replay
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`project.bench` is null on every other kind, so reaching for a benchmark verb
|
|
82
|
+
on an agent is a type error rather than a runtime surprise.
|
|
83
|
+
|
|
84
|
+
Authoring surface, narrowed by generated declarations in `.bench/bench.d.ts`:
|
|
85
|
+
|
|
86
|
+
```typescript
|
|
87
|
+
bench.axis("model") // the variation this cell got
|
|
88
|
+
bench.observe("resolved", true) // a measurement
|
|
89
|
+
bench.attach("diff", patch) // content-hashed evidence
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Known Debt
|
|
93
|
+
|
|
94
|
+
- **The workspace assertion API does not exist.** `registry/benches/benchmark0`
|
|
95
|
+
contains `tests/fix-bug.bench.ts` written against the intended shape
|
|
96
|
+
(`workspace.tests.pass()`, `workspace.changed()`) as the target to satisfy.
|
|
97
|
+
Today `bench.workspace` is a bare path, so authors hand-roll `Bun.$`.
|
|
98
|
+
- **`toolCalls` is hardcoded to 0** in `preload.ts` — for coding benchmarks
|
|
99
|
+
this is one of the most diagnostic numbers there is.
|
|
100
|
+
- **`aggregate.ts` and `contribution.ts` are types with no consumers.** The
|
|
101
|
+
collective-submission layer (cohorts, tiers, shared aggregates) is designed
|
|
102
|
+
and unwired. Worth building against a real result set rather than in the
|
|
103
|
+
abstract — that is how they got ahead of the code in the first place.
|
|
104
|
+
- **Cases are unresolved.** `BenchTrialRecord.case` exists and nothing fills
|
|
105
|
+
it. A dataset is the population you measure over, not an axis, so it must
|
|
106
|
+
not enter the Cartesian product — but nothing expresses that yet.
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# Bench API — the locked shape
|
|
2
|
+
|
|
3
|
+
Status: design agreed, not yet implemented. This is the target; `CLAUDE.md`
|
|
4
|
+
describes what exists today.
|
|
5
|
+
|
|
6
|
+
## The whole surface
|
|
7
|
+
|
|
8
|
+
```ts
|
|
9
|
+
type Schema = {
|
|
10
|
+
/** Has the agent resolved the bug? */
|
|
11
|
+
resolved: boolean
|
|
12
|
+
|
|
13
|
+
/** Files edited beyond the target. @objective minimize */
|
|
14
|
+
collateral: number
|
|
15
|
+
|
|
16
|
+
/** Strategy the agent took. */
|
|
17
|
+
approach: "refactor" | "patch" | "rewrite"
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export default defineBench<Schema>({
|
|
21
|
+
description: "Can the agent fix a real bug without collateral damage?",
|
|
22
|
+
|
|
23
|
+
workspace: { source: "./workspace", retain: "failed" },
|
|
24
|
+
|
|
25
|
+
matrix: {
|
|
26
|
+
model: [
|
|
27
|
+
Mock({ fix: run("...correct patch...") }),
|
|
28
|
+
Mock({ fix: "I can't do that" }),
|
|
29
|
+
],
|
|
30
|
+
},
|
|
31
|
+
|
|
32
|
+
trials: 3,
|
|
33
|
+
|
|
34
|
+
async setup() {
|
|
35
|
+
// after the workspace copy, before the subject boots
|
|
36
|
+
},
|
|
37
|
+
})
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
// tests/fix-bug.bench.ts
|
|
42
|
+
it("resolves the failing test", async ({ workspace }) => {
|
|
43
|
+
expect(workspace).toHaveFile("src/parser.ts")
|
|
44
|
+
|
|
45
|
+
const { axon } = await Axon()
|
|
46
|
+
await axon.request("Fix the failing test in src/parser.ts")
|
|
47
|
+
|
|
48
|
+
observe("resolved", await workspace.tests.pass())
|
|
49
|
+
observe("collateral", (await workspace.changed()).length - 1)
|
|
50
|
+
})
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Four concepts: **what varies** (matrix), **the world** (workspace), **the
|
|
54
|
+
scenario** (tests), **what is measured** (Schema). Nothing else is required.
|
|
55
|
+
|
|
56
|
+
## Why each piece is shaped this way
|
|
57
|
+
|
|
58
|
+
### `expect` vs `observe`
|
|
59
|
+
|
|
60
|
+
A test encodes what you already know should happen. A benchmark exists
|
|
61
|
+
because you do not.
|
|
62
|
+
|
|
63
|
+
- **`expect`** — did the SCENARIO hold? A failure means this trial measures
|
|
64
|
+
nothing; it is excluded, not scored zero.
|
|
65
|
+
- **`observe`** — how did the SUBJECT do? Recorded whatever the value is.
|
|
66
|
+
|
|
67
|
+
They cannot merge. `expect` throws on first failure, so a benchmark whose
|
|
68
|
+
weak conditions all failed would report itself broken — and every measurement
|
|
69
|
+
after the first failure would be lost, precisely on the runs that need them.
|
|
70
|
+
|
|
71
|
+
### `matrix`
|
|
72
|
+
|
|
73
|
+
Every key is a binding point in the assembled blueprint; every value list is
|
|
74
|
+
an axis. One key is the controlled experiment. Several multiply, and the
|
|
75
|
+
growth is the author's to manage rather than the runtime's to hide.
|
|
76
|
+
|
|
77
|
+
Named axes (`agent`, `cognet`, `model`, `env`) cover the common cases; dotted
|
|
78
|
+
paths (`config.engine.temperature`) reach anything else. The named set stays
|
|
79
|
+
small BECAUSE the paths exist — an earlier design had eight "factor kinds" of
|
|
80
|
+
which three were ever wired.
|
|
81
|
+
|
|
82
|
+
Variation ids are derived from values, not positions: `model[0]` would mean
|
|
83
|
+
something different after a reorder, and cell ids built from it would stop
|
|
84
|
+
matching earlier runs of the same benchmark. Collisions (`Mock()` twice)
|
|
85
|
+
suffix rather than reject — two mocks with different reply maps are a
|
|
86
|
+
meaningful pair the id cannot otherwise distinguish.
|
|
87
|
+
|
|
88
|
+
### `Schema` as a TypeScript type
|
|
89
|
+
|
|
90
|
+
The type is the authoring surface; JSON is not. A doc comment is where a
|
|
91
|
+
description naturally lives, and a union type is how anyone would write a
|
|
92
|
+
category.
|
|
93
|
+
|
|
94
|
+
But a `type` is erased at compile time, and the manifest needs the schema as a
|
|
95
|
+
VALUE:
|
|
96
|
+
|
|
97
|
+
- `BenchCohortKey` hashes it to decide whether two people's results are
|
|
98
|
+
comparable
|
|
99
|
+
- `BenchCoverage` reports "expected 4, observed 3" — undetectable if you only
|
|
100
|
+
know about measurements that fired
|
|
101
|
+
|
|
102
|
+
So `prepare` extracts the type into `.bench/schema.json` via
|
|
103
|
+
`ts.createProgram()`, and the runtime reads that. The author writes a type;
|
|
104
|
+
the system keeps a value. This is the same machinery `declareTools()` already
|
|
105
|
+
uses (worker-isolated, content-hash cached — see `scan/tools.ts`).
|
|
106
|
+
|
|
107
|
+
Derived from the type:
|
|
108
|
+
|
|
109
|
+
| from | to |
|
|
110
|
+
|---|---|
|
|
111
|
+
| property name | measurement id |
|
|
112
|
+
| property type | `value.kind` (`boolean`, `number`, union → `category`) |
|
|
113
|
+
| JSDoc text | `description` |
|
|
114
|
+
| `@objective minimize` | direction, where it is not obvious |
|
|
115
|
+
| `@unit ms` | display suffix |
|
|
116
|
+
|
|
117
|
+
`aggregate` defaults by kind — boolean → `rate`, number → `mean`, category →
|
|
118
|
+
`count`. `weighting` / `grain` / `missing` remain available and out of sight.
|
|
119
|
+
|
|
120
|
+
**`description` is required to publish, optional to run.** A shared benchmark
|
|
121
|
+
whose measurements have no stated meaning cannot be interpreted by whoever
|
|
122
|
+
reads the leaderboard; a bench you are iterating on locally should not nag.
|
|
123
|
+
|
|
124
|
+
### Visualisation is derived, never declared
|
|
125
|
+
|
|
126
|
+
No `chart`, `display`, or `format` field. The moment one exists you need
|
|
127
|
+
`scatter`, `heatmap`, `stacked`, `logScale`, `colorBy` — and every bench
|
|
128
|
+
author becomes a dashboard designer. That is exactly how `measurements`
|
|
129
|
+
reached ten fields the first time.
|
|
130
|
+
|
|
131
|
+
Chart type follows from data shape, not author intent:
|
|
132
|
+
|
|
133
|
+
| data | chart |
|
|
134
|
+
|---|---|
|
|
135
|
+
| boolean × cells | bar of rates |
|
|
136
|
+
| number × cells | box plot (trials give the distribution) |
|
|
137
|
+
| category × cells | stacked bar |
|
|
138
|
+
| number × number | scatter |
|
|
139
|
+
| number, step grain | line |
|
|
140
|
+
| two matrix axes | grid / heatmap |
|
|
141
|
+
|
|
142
|
+
`objective` supplies the only intent a chart needs: which direction is good.
|
|
143
|
+
|
|
144
|
+
Presentation is a projection over the semantic schema, exactly as aggregates
|
|
145
|
+
are a projection over observations. Add a presentation field only when a real
|
|
146
|
+
chart demonstrably cannot be derived — earned by a chart that exists, not
|
|
147
|
+
anticipated.
|
|
148
|
+
|
|
149
|
+
## Directory contract
|
|
150
|
+
|
|
151
|
+
```
|
|
152
|
+
bench.config.ts matrix, measurements schema, setup
|
|
153
|
+
workspace/ the agent's world — copied fresh per iteration
|
|
154
|
+
fixtures/ author-side inputs; NEVER visible to the agent
|
|
155
|
+
tests/ scenarios, and where measurements are recorded
|
|
156
|
+
.bench/ generated declarations, extracted schema, run logs
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The `workspace/` ↔ `fixtures/` split is an invariant. If fixtures leaked into
|
|
160
|
+
the agent's view, a run would depend on the bench's own layout and stop being
|
|
161
|
+
reproducible.
|
|
162
|
+
|
|
163
|
+
## Testing without live APIs
|
|
164
|
+
|
|
165
|
+
`Mock()` varies behaviour, not just labels — pattern maps, ordered sequences,
|
|
166
|
+
and `run()` steps that execute in the capsule:
|
|
167
|
+
|
|
168
|
+
```ts
|
|
169
|
+
matrix: {
|
|
170
|
+
model: [
|
|
171
|
+
Mock({ fix: run("...correct patch...") }), // succeeds
|
|
172
|
+
Mock({ fix: "I can't do that" }), // fails
|
|
173
|
+
Mock({ fix: [run("...wrong..."), run("...right...")] }), // succeeds on retry
|
|
174
|
+
],
|
|
175
|
+
}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
That exercises matrix expansion, subject binding, measurement collection and
|
|
179
|
+
replay end to end, with a real pass/fail spread and no API cost.
|
|
180
|
+
|
|
181
|
+
## Build order
|
|
182
|
+
|
|
183
|
+
1. **Object-keyed measurements** — `{ resolved: { description } }`. Small
|
|
184
|
+
change to working code, cuts nine fields to one or two, and carries
|
|
185
|
+
identical information to a `Schema` type so the migration is mechanical.
|
|
186
|
+
2. **Workspace assertions** — `workspace.tests.pass()`, `.changed()`,
|
|
187
|
+
`toHaveFile()`. The blocker for any real bench;
|
|
188
|
+
`benchmark0/tests/fix-bug.bench.ts` is written against this and marked
|
|
189
|
+
not-yet-runnable.
|
|
190
|
+
3. **`toolCalls`** — hardcoded to `0` in `preload.ts`, and one of the most
|
|
191
|
+
diagnostic numbers for a coding benchmark.
|
|
192
|
+
4. **Schema extraction** — after two or three real benches exist, so the
|
|
193
|
+
extractor is designed against real schemas rather than imagined ones.
|
|
194
|
+
5. **Aggregation** — `aggregate.ts` is types with no consumers. Build against
|
|
195
|
+
a real result set; designing it in the abstract is how it got ahead of the
|
|
196
|
+
code.
|
|
197
|
+
|
|
198
|
+
## Still open
|
|
199
|
+
|
|
200
|
+
**Cases.** `BenchTrialRecord.case` exists and nothing fills it. A dataset
|
|
201
|
+
(SWE-bench's 2,294 instances) is the population you measure OVER, not an axis
|
|
202
|
+
— it must not enter the Cartesian product. Nothing expresses that yet, and it
|
|
203
|
+
is the one design question the current shape does not answer.
|
|
204
|
+
|
|
205
|
+
**Multi-agent.** `BenchPhysics` already separates `subject` from `assessor`,
|
|
206
|
+
so the types anticipate a judge; the runtime boots one agent. An LLM judge
|
|
207
|
+
must be held CONSTANT across the matrix, or the measuring instrument varies
|
|
208
|
+
with the thing measured. Blocked on whether `Axon()` becomes a manager of
|
|
209
|
+
several instances — a runtime question, not a bench one.
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { existsSync } from "node:fs"
|
|
2
|
+
import { isAbsolute, join, resolve } from "node:path"
|
|
3
|
+
import { err } from "@arcforge/err"
|
|
4
|
+
import type { BenchAxis } from "@arcforge/types"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Resolve the agents a matrix names, so every cell has something bootable.
|
|
8
|
+
*
|
|
9
|
+
* An `agent` variation is either a path you wrote or a registry ref you trust:
|
|
10
|
+
*
|
|
11
|
+
* agent: "./fixtures/subject" — authored here, committed
|
|
12
|
+
* agent: "@axon/coding-base" — resolved from the registry, fetched
|
|
13
|
+
*
|
|
14
|
+
* Registry agents land under `.bench/agents/`, not `fixtures/`. The split is
|
|
15
|
+
* the same one that makes `workspace/` ↔ `fixtures/` clean: `fixtures/` is
|
|
16
|
+
* source you author and commit, `.bench/` is generated and ignored. Vendoring
|
|
17
|
+
* a copy of a shared base into source would put it back under your control —
|
|
18
|
+
* which is precisely what naming a registry version is meant to avoid, since
|
|
19
|
+
* two benchmarks claiming to measure against "a standard coding agent" should
|
|
20
|
+
* be measuring against the same one.
|
|
21
|
+
*
|
|
22
|
+
* The version lands in the directory name because it lands in the manifest: an
|
|
23
|
+
* `agent` pin that says `@axon/coding-base@1.2.0` is a real identity, where a
|
|
24
|
+
* hash of the string "./fixtures/subject" says nothing anyone else can use.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
export type ResolvedAgent = {
|
|
28
|
+
/** As written in the matrix. */
|
|
29
|
+
ref: string
|
|
30
|
+
/** Absolute path to a prepared agent project. */
|
|
31
|
+
root: string
|
|
32
|
+
/** Manifest identity: a registry `name@version`, or the ref for a local path. */
|
|
33
|
+
pin: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export type AgentsOpts = {
|
|
37
|
+
root: string
|
|
38
|
+
/** Fetch, extract and prepare a registry artifact. */
|
|
39
|
+
clone(ref: string, cwd: string, options: { dir?: string }): Promise<{ name: string; version: string; root: string }>
|
|
40
|
+
/** Install deps and generate types for a local project. */
|
|
41
|
+
prepare(root: string): Promise<void>
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** A scoped registry name, as opposed to a filesystem path. */
|
|
45
|
+
export function isRegistryRef(ref: string): boolean {
|
|
46
|
+
return ref.startsWith("@") && !ref.startsWith("./") && !ref.startsWith("../")
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function Agents(opts: AgentsOpts) {
|
|
50
|
+
const cacheDir = join(opts.root, ".bench", "agents")
|
|
51
|
+
|
|
52
|
+
async function resolveOne(ref: string): Promise<ResolvedAgent> {
|
|
53
|
+
if (!isRegistryRef(ref)) {
|
|
54
|
+
const root = isAbsolute(ref) ? ref : resolve(opts.root, ref)
|
|
55
|
+
if (!existsSync(join(root, "axon.config.ts"))) {
|
|
56
|
+
throw err("BENCH_AGENT_UNRESOLVED", {
|
|
57
|
+
detail: `${ref} is not an agent project — no axon.config.ts at ${root}`,
|
|
58
|
+
context: { ref, root },
|
|
59
|
+
})
|
|
60
|
+
}
|
|
61
|
+
// A local agent still needs node_modules and generated types before
|
|
62
|
+
// it can boot, and a bench author should not have to know that.
|
|
63
|
+
await opts.prepare(root)
|
|
64
|
+
return { ref, root, pin: ref }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const cloned = await opts.clone(ref, cacheDir, { dir: slug(ref) })
|
|
68
|
+
return { ref, root: cloned.root, pin: `${cloned.name}@${cloned.version}` }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return {
|
|
72
|
+
/**
|
|
73
|
+
* Every agent the matrix names, resolved and prepared.
|
|
74
|
+
*
|
|
75
|
+
* Returns a map keyed by the ref as written, so the runtime can look up
|
|
76
|
+
* what a cell selected without re-deciding local-versus-registry.
|
|
77
|
+
*/
|
|
78
|
+
async resolve(axes: BenchAxis[]): Promise<Map<string, ResolvedAgent>> {
|
|
79
|
+
const axis = axes.find(item => item.key === "agent")
|
|
80
|
+
if (!axis) return new Map()
|
|
81
|
+
|
|
82
|
+
const resolved = new Map<string, ResolvedAgent>()
|
|
83
|
+
for (const variation of axis.values) {
|
|
84
|
+
const ref = variation.value
|
|
85
|
+
if (typeof ref !== "string") {
|
|
86
|
+
throw err("BENCH_AGENT_UNRESOLVED", {
|
|
87
|
+
detail: `agent variations are paths or registry refs, got ${JSON.stringify(ref)}`,
|
|
88
|
+
context: { value: ref },
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
// Sequential on purpose: two variations of the same registry
|
|
92
|
+
// agent would otherwise race to write the same directory.
|
|
93
|
+
if (!resolved.has(ref)) resolved.set(ref, await resolveOne(ref))
|
|
94
|
+
}
|
|
95
|
+
return resolved
|
|
96
|
+
},
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** `@axon/coding-base@1.2.0` → `axon-coding-base@1.2.0`, safe as a directory name. */
|
|
101
|
+
function slug(ref: string): string {
|
|
102
|
+
return ref.replace(/^@/, "").replace(/\//g, "-")
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export type AgentsT = ReturnType<typeof Agents>
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { BenchAxis, BenchAxisKey, BenchAxisVariation, BenchMatrix } from "@arcforge/types"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Turn the authored `matrix` object into ordered axes.
|
|
5
|
+
*
|
|
6
|
+
* The authored form is deliberately terse — a key and a list of values, with
|
|
7
|
+
* nothing to declare twice because the key IS the binding point. Everything
|
|
8
|
+
* the manifest needs beyond that (stable ids, labels) is derived here, once,
|
|
9
|
+
* at the seam.
|
|
10
|
+
*
|
|
11
|
+
* Ids come from the value itself rather than its position: `model[0]` would
|
|
12
|
+
* mean a different thing the moment someone reorders the list, and cell ids
|
|
13
|
+
* built from it would silently stop matching earlier runs of the same
|
|
14
|
+
* benchmark. A label derived from the value stays stable under reordering.
|
|
15
|
+
*/
|
|
16
|
+
export function toAxes(matrix: BenchMatrix | undefined): BenchAxis[] {
|
|
17
|
+
if (!matrix) return []
|
|
18
|
+
|
|
19
|
+
return Object.entries(matrix)
|
|
20
|
+
.filter(([, values]) => values !== undefined)
|
|
21
|
+
.map(([key, values]) => ({
|
|
22
|
+
key: key as BenchAxisKey,
|
|
23
|
+
// A bare value is a held constant — `agent: "@axon/coding-base"`
|
|
24
|
+
// says the same thing as a one-element array and reads like what it
|
|
25
|
+
// means. It is still an axis: one cell, and the value still pinned
|
|
26
|
+
// into the manifest, because a result is only comparable if what did
|
|
27
|
+
// NOT vary is recorded too.
|
|
28
|
+
values: withDistinctIds(Array.isArray(values) ? values : [values]),
|
|
29
|
+
}))
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Derive ids, disambiguating any that collide.
|
|
34
|
+
*
|
|
35
|
+
* Two variations can legitimately share a derived label — `Mock()` twice with
|
|
36
|
+
* different reply maps is the obvious case, since the engine name is the same
|
|
37
|
+
* and the behaviour lives in a closure the id cannot see. Suffixing keeps the
|
|
38
|
+
* readable part while staying unique, rather than rejecting a config that is
|
|
39
|
+
* perfectly meaningful.
|
|
40
|
+
*/
|
|
41
|
+
function withDistinctIds(values: unknown[]): BenchAxisVariation[] {
|
|
42
|
+
const variations = values.map(toVariation)
|
|
43
|
+
const counts = new Map<string, number>()
|
|
44
|
+
for (const variation of variations) counts.set(variation.id, (counts.get(variation.id) ?? 0) + 1)
|
|
45
|
+
|
|
46
|
+
const seen = new Map<string, number>()
|
|
47
|
+
return variations.map(variation => {
|
|
48
|
+
if ((counts.get(variation.id) ?? 0) < 2) return variation
|
|
49
|
+
const nth = (seen.get(variation.id) ?? 0) + 1
|
|
50
|
+
seen.set(variation.id, nth)
|
|
51
|
+
return { ...variation, id: `${variation.id}-${nth}`, label: `${variation.label} #${nth}` }
|
|
52
|
+
})
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Derive a stable, readable id for one variation.
|
|
57
|
+
*
|
|
58
|
+
* An engine ref carries its model name, a path carries its directory — both
|
|
59
|
+
* are what an author would call the variation anyway, so a run's cell ids
|
|
60
|
+
* read as `model=claude-sonnet-4.6` rather than `model=0`.
|
|
61
|
+
*/
|
|
62
|
+
function toVariation(value: unknown, index: number): BenchAxisVariation {
|
|
63
|
+
return { id: variationId(value, index), label: variationLabel(value), value }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function variationId(value: unknown, index: number): string {
|
|
67
|
+
const label = variationLabel(value)
|
|
68
|
+
if (!label) return String(index)
|
|
69
|
+
// Cell ids end up in file paths and manifest keys, so keep them boring.
|
|
70
|
+
return label.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64) || String(index)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function variationLabel(value: unknown): string {
|
|
74
|
+
if (typeof value === "string") return value
|
|
75
|
+
if (value && typeof value === "object") {
|
|
76
|
+
const record = value as Record<string, unknown>
|
|
77
|
+
// EngineRef and friends: the model name is the identity an author means.
|
|
78
|
+
for (const field of ["model", "id", "name", "ref"]) {
|
|
79
|
+
const candidate = record[field]
|
|
80
|
+
if (typeof candidate === "string" && candidate.length > 0) return candidate
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return typeof value === "number" || typeof value === "boolean" ? String(value) : ""
|
|
84
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { AxonCloudClient } from "@arcforge/cloud"
|
|
2
|
+
import type { BenchConfig, BenchRunResult } from "@arcforge/types"
|
|
3
|
+
import { Discover } from "./discover"
|
|
4
|
+
import { BenchLog } from "./log"
|
|
5
|
+
import { Matrix } from "./matrix"
|
|
6
|
+
import { Projection } from "./projection"
|
|
7
|
+
import { Resolver } from "./resolve"
|
|
8
|
+
import { Runner } from "./runner"
|
|
9
|
+
import type { TestRunnerT } from "../../services/test"
|
|
10
|
+
import { BenchTypegen } from "./typegen"
|
|
11
|
+
import { PrepareBench } from "./prepare"
|
|
12
|
+
import { Agents, type AgentsOpts } from "./agents"
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The collaborators a bench needs that no other project kind does. Threaded
|
|
16
|
+
* from Platform through Projects, so Project() can build a Bench without
|
|
17
|
+
* knowing what a test runner is.
|
|
18
|
+
*/
|
|
19
|
+
export type BenchExtras = {
|
|
20
|
+
tests: TestRunnerT
|
|
21
|
+
clone: AgentsOpts["clone"]
|
|
22
|
+
prepare: AgentsOpts["prepare"]
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export type BenchOpts = BenchExtras & {
|
|
26
|
+
root: string
|
|
27
|
+
cloud: AxonCloudClient
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The benchmark verbs of a bench project — run, result, coordinates, config.
|
|
32
|
+
*
|
|
33
|
+
* Composition only; behavior lives in the leaves. This is NOT a project
|
|
34
|
+
* manager: finding, opening, scaffolding, preparing and publishing a bench are
|
|
35
|
+
* Project()'s, exactly as for every other kind. What lives here is only what is
|
|
36
|
+
* genuinely about benchmarking.
|
|
37
|
+
*/
|
|
38
|
+
export function Bench(opts: BenchOpts) {
|
|
39
|
+
const discover = Discover({ root: opts.root })
|
|
40
|
+
const matrix = Matrix()
|
|
41
|
+
const log = BenchLog({ root: opts.root })
|
|
42
|
+
const resolver = Resolver()
|
|
43
|
+
const runner = Runner({ root: opts.root, log, resolver, tests: opts.tests })
|
|
44
|
+
const projection = Projection()
|
|
45
|
+
const typegen = BenchTypegen({ root: opts.root })
|
|
46
|
+
const agents = Agents({ root: opts.root, clone: opts.clone, prepare: opts.prepare })
|
|
47
|
+
const prepare = PrepareBench({ root: opts.root, discover, typegen, agents })
|
|
48
|
+
|
|
49
|
+
return {
|
|
50
|
+
root: opts.root,
|
|
51
|
+
|
|
52
|
+
config(): Promise<BenchConfig> {
|
|
53
|
+
return discover.load()
|
|
54
|
+
},
|
|
55
|
+
|
|
56
|
+
async coordinates() {
|
|
57
|
+
const config = await discover.load()
|
|
58
|
+
return matrix.build(config.axes)
|
|
59
|
+
},
|
|
60
|
+
|
|
61
|
+
/** Execute Bun, persist events, then derive the portable result by replay. */
|
|
62
|
+
async run(): Promise<BenchRunResult> {
|
|
63
|
+
const config = await discover.load()
|
|
64
|
+
const coordinates = matrix.build(config.axes)
|
|
65
|
+
// Resolve agents before running: a registry ref must be fetched and
|
|
66
|
+
// prepared, and its version is what the manifest pins.
|
|
67
|
+
const resolvedAgents = await agents.resolve(config.axes)
|
|
68
|
+
const agentPins = new Map([...resolvedAgents].map(([ref, agent]) => [ref, agent.pin]))
|
|
69
|
+
const agentRoots = Object.fromEntries([...resolvedAgents].map(([ref, agent]) => [ref, agent.root]))
|
|
70
|
+
const runId = await runner.run(config, coordinates, agentPins, agentRoots)
|
|
71
|
+
const result = projection.build(config, await log.read(runId))
|
|
72
|
+
await log.writeManifest(runId, result.manifest)
|
|
73
|
+
await log.writeResult(runId, result)
|
|
74
|
+
return result
|
|
75
|
+
},
|
|
76
|
+
|
|
77
|
+
/** Rebuild a result without executing anything. */
|
|
78
|
+
async result(runId: string): Promise<BenchRunResult> {
|
|
79
|
+
const config = await discover.load()
|
|
80
|
+
return projection.build(config, await log.read(runId))
|
|
81
|
+
},
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The bench half of `axon prepare` — validate the matrix's local
|
|
85
|
+
* refs, resolve the agents it names, materialize the workspace
|
|
86
|
+
* template, and generate declarations from the config's axes.
|
|
87
|
+
*
|
|
88
|
+
* Project.prepare() calls this after writing the shared frame, so
|
|
89
|
+
* `axon prepare` in a bench does the whole job in one verb. It stays
|
|
90
|
+
* exposed because run() needs its resolved result.
|
|
91
|
+
*/
|
|
92
|
+
prepare,
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export type BenchT = ReturnType<typeof Bench>
|