@arcforge/platform 2.0.166

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/README.md +31 -0
  2. package/package.json +69 -0
  3. package/src/bin/supervisor.ts +102 -0
  4. package/src/bin/test-api.ts +30 -0
  5. package/src/bin/test-preload.ts +308 -0
  6. package/src/bin/update-helper.ts +32 -0
  7. package/src/build/bench/CLAUDE.md +106 -0
  8. package/src/build/bench/SPEC.md +209 -0
  9. package/src/build/bench/agents.ts +105 -0
  10. package/src/build/bench/axes.ts +84 -0
  11. package/src/build/bench/bench.ts +96 -0
  12. package/src/build/bench/discover.ts +147 -0
  13. package/src/build/bench/index.ts +7 -0
  14. package/src/build/bench/log.ts +47 -0
  15. package/src/build/bench/matrix.ts +23 -0
  16. package/src/build/bench/preload.ts +440 -0
  17. package/src/build/bench/prepare.ts +61 -0
  18. package/src/build/bench/projection.ts +112 -0
  19. package/src/build/bench/resolve.ts +54 -0
  20. package/src/build/bench/runner.ts +309 -0
  21. package/src/build/bench/schema/extract.ts +148 -0
  22. package/src/build/bench/schema/index.ts +2 -0
  23. package/src/build/bench/schema/schema.ts +95 -0
  24. package/src/build/bench/schema/worker.ts +30 -0
  25. package/src/build/bench/typegen.ts +119 -0
  26. package/src/build/bench/workspace/files.ts +139 -0
  27. package/src/build/bench/workspace/handle.ts +133 -0
  28. package/src/build/bench/workspace/index.ts +11 -0
  29. package/src/build/bench/workspace/workspace.ts +14 -0
  30. package/src/build/blueprint/blueprint.ts +398 -0
  31. package/src/build/blueprint/cognet/abi.ts +207 -0
  32. package/src/build/blueprint/cognet/bundle-worker.ts +93 -0
  33. package/src/build/blueprint/cognet/bundle.ts +664 -0
  34. package/src/build/blueprint/cognet/cognet.ts +249 -0
  35. package/src/build/blueprint/cognet/index.ts +10 -0
  36. package/src/build/blueprint/cognet/manifest.ts +38 -0
  37. package/src/build/blueprint/cognet/resolve.ts +113 -0
  38. package/src/build/blueprint/collisions.ts +51 -0
  39. package/src/build/blueprint/index.ts +29 -0
  40. package/src/build/blueprint/modules/entries.ts +117 -0
  41. package/src/build/blueprint/modules/index.ts +300 -0
  42. package/src/build/blueprint/modules/meta.ts +174 -0
  43. package/src/build/blueprint/scan/boot.ts +34 -0
  44. package/src/build/blueprint/scan/cognetImports.ts +56 -0
  45. package/src/build/blueprint/scan/components.ts +45 -0
  46. package/src/build/blueprint/scan/config.ts +234 -0
  47. package/src/build/blueprint/scan/configImports.ts +232 -0
  48. package/src/build/blueprint/scan/declare-worker.ts +61 -0
  49. package/src/build/blueprint/scan/declare.ts +393 -0
  50. package/src/build/blueprint/scan/h3-globals.ts +137 -0
  51. package/src/build/blueprint/scan/knowledge.ts +101 -0
  52. package/src/build/blueprint/scan/middleware.ts +110 -0
  53. package/src/build/blueprint/scan/moduleImports.ts +79 -0
  54. package/src/build/blueprint/scan/plugins.ts +93 -0
  55. package/src/build/blueprint/scan/promptPackages.ts +45 -0
  56. package/src/build/blueprint/scan/prompts.ts +164 -0
  57. package/src/build/blueprint/scan/published.ts +131 -0
  58. package/src/build/blueprint/scan/resolve.ts +398 -0
  59. package/src/build/blueprint/scan/routes.ts +109 -0
  60. package/src/build/blueprint/scan/scripts.ts +119 -0
  61. package/src/build/blueprint/scan/tool-bundle-worker.ts +69 -0
  62. package/src/build/blueprint/scan/tools.ts +610 -0
  63. package/src/build/blueprint/types.ts +28 -0
  64. package/src/build/extensions/config.ts +261 -0
  65. package/src/build/extensions/edit.ts +684 -0
  66. package/src/build/extensions/extensions.ts +731 -0
  67. package/src/build/extensions/index.ts +10 -0
  68. package/src/build/extensions/load.ts +350 -0
  69. package/src/build/extensions/registrations.ts +189 -0
  70. package/src/build/extensions/store.ts +200 -0
  71. package/src/build/frame/frame.ts +145 -0
  72. package/src/build/frame/index.ts +5 -0
  73. package/src/build/frame/migrate.ts +246 -0
  74. package/src/build/project/bundle/agent.ts +382 -0
  75. package/src/build/project/bundle/artifacts.ts +88 -0
  76. package/src/build/project/bundle/assets.ts +385 -0
  77. package/src/build/project/bundle/bundle.ts +160 -0
  78. package/src/build/project/bundle/imports.ts +102 -0
  79. package/src/build/project/bundle/index.ts +16 -0
  80. package/src/build/project/bundle/module.ts +130 -0
  81. package/src/build/project/bundle/source.ts +136 -0
  82. package/src/build/project/bundle/stage.ts +97 -0
  83. package/src/build/project/bundle/types.ts +111 -0
  84. package/src/build/project/cognet.ts +156 -0
  85. package/src/build/project/create/agent.ts +52 -0
  86. package/src/build/project/create/bench.ts +81 -0
  87. package/src/build/project/create/cognet.ts +102 -0
  88. package/src/build/project/create/extension.ts +63 -0
  89. package/src/build/project/create/module.ts +43 -0
  90. package/src/build/project/create/profile.ts +102 -0
  91. package/src/build/project/create/prompt.ts +68 -0
  92. package/src/build/project/create/templates.ts +88 -0
  93. package/src/build/project/deploy.ts +96 -0
  94. package/src/build/project/index.ts +40 -0
  95. package/src/build/project/installer.ts +616 -0
  96. package/src/build/project/kinds.ts +374 -0
  97. package/src/build/project/links.ts +33 -0
  98. package/src/build/project/manifest/bunfig.ts +139 -0
  99. package/src/build/project/manifest/config.ts +246 -0
  100. package/src/build/project/manifest/env.ts +184 -0
  101. package/src/build/project/manifest/index.ts +9 -0
  102. package/src/build/project/manifest/manifest.ts +49 -0
  103. package/src/build/project/manifest/model.ts +124 -0
  104. package/src/build/project/manifest/package.ts +386 -0
  105. package/src/build/project/manifest/text.ts +19 -0
  106. package/src/build/project/models/fetch.ts +123 -0
  107. package/src/build/project/models/index.ts +8 -0
  108. package/src/build/project/models/models.ts +111 -0
  109. package/src/build/project/models/specifier.ts +106 -0
  110. package/src/build/project/models/store.ts +202 -0
  111. package/src/build/project/modules/hash.ts +31 -0
  112. package/src/build/project/modules/index.ts +5 -0
  113. package/src/build/project/modules/lock.ts +78 -0
  114. package/src/build/project/modules/modules.ts +155 -0
  115. package/src/build/project/prepare.ts +709 -0
  116. package/src/build/project/project.ts +253 -0
  117. package/src/build/project/projects.ts +164 -0
  118. package/src/build/project/propagation.ts +120 -0
  119. package/src/build/project/publish/verify.ts +312 -0
  120. package/src/build/project/publish.ts +155 -0
  121. package/src/build/project/reconcile.ts +93 -0
  122. package/src/build/project/specifier.ts +42 -0
  123. package/src/build/project/tree.ts +615 -0
  124. package/src/build/project/treecache.ts +356 -0
  125. package/src/build/project/typegen/axon-dts.ts +362 -0
  126. package/src/build/project/typegen/cognet-dts.ts +91 -0
  127. package/src/build/project/typegen/components.ts +45 -0
  128. package/src/build/project/typegen/env.ts +59 -0
  129. package/src/build/project/typegen/generators.ts +108 -0
  130. package/src/build/project/typegen/hooks.ts +71 -0
  131. package/src/build/project/typegen/index.ts +137 -0
  132. package/src/build/project/typegen/prompt-dts.ts +40 -0
  133. package/src/build/project/typegen/prompts.ts +36 -0
  134. package/src/build/project/typegen/registry-dts.ts +109 -0
  135. package/src/build/project/typegen/scripts.ts +37 -0
  136. package/src/build/project/typegen/tools.ts +31 -0
  137. package/src/build/project/typegen/tsconfig.ts +243 -0
  138. package/src/build/project/typegen/tui-dts.ts +120 -0
  139. package/src/build/project/typegen/write.ts +40 -0
  140. package/src/build/project/verify.ts +268 -0
  141. package/src/build/project/watcher.ts +266 -0
  142. package/src/build/runtime/agent.ts +550 -0
  143. package/src/build/runtime/escalations.ts +159 -0
  144. package/src/build/runtime/index.ts +15 -0
  145. package/src/build/runtime/instances.ts +604 -0
  146. package/src/build/runtime/profile.ts +91 -0
  147. package/src/build/runtime/progress.ts +143 -0
  148. package/src/build/runtime/requests.ts +167 -0
  149. package/src/build/runtime/resolve.ts +159 -0
  150. package/src/build/runtime/runtime.ts +233 -0
  151. package/src/build/runtime/sessions/index.ts +5 -0
  152. package/src/build/runtime/sessions/record.ts +145 -0
  153. package/src/build/runtime/sessions/sessions.ts +240 -0
  154. package/src/build/runtime/zeno.ts +120 -0
  155. package/src/index.ts +11 -0
  156. package/src/link/agent-main.ts +395 -0
  157. package/src/link/index.ts +70 -0
  158. package/src/link/services.ts +120 -0
  159. package/src/platform.ts +293 -0
  160. package/src/procs/index.ts +15 -0
  161. package/src/procs/tree.ts +177 -0
  162. package/src/services/cloud.ts +419 -0
  163. package/src/services/control/client.ts +126 -0
  164. package/src/services/control/dispatch.ts +156 -0
  165. package/src/services/control/index.ts +17 -0
  166. package/src/services/control/protocol.ts +179 -0
  167. package/src/services/control/server.ts +241 -0
  168. package/src/services/deployments.ts +81 -0
  169. package/src/services/files.ts +329 -0
  170. package/src/services/index.ts +5 -0
  171. package/src/services/mic/capture.ts +248 -0
  172. package/src/services/mic/config.ts +109 -0
  173. package/src/services/mic/fft.ts +93 -0
  174. package/src/services/mic/index.ts +139 -0
  175. package/src/services/mic/visualizer.ts +46 -0
  176. package/src/services/ollama/catalog.ts +122 -0
  177. package/src/services/ollama/http.ts +124 -0
  178. package/src/services/ollama/index.ts +13 -0
  179. package/src/services/ollama/models.ts +153 -0
  180. package/src/services/ollama/ollama.ts +89 -0
  181. package/src/services/ollama/registry.ts +112 -0
  182. package/src/services/ollama/types.ts +72 -0
  183. package/src/services/prompts.ts +76 -0
  184. package/src/services/registry.ts +220 -0
  185. package/src/services/resources/hardware.ts +125 -0
  186. package/src/services/resources/index.ts +6 -0
  187. package/src/services/resources/reservations.ts +162 -0
  188. package/src/services/resources/resources.ts +156 -0
  189. package/src/services/running/index.ts +1 -0
  190. package/src/services/running/running.ts +200 -0
  191. package/src/services/store/disk.ts +97 -0
  192. package/src/services/store/index.ts +15 -0
  193. package/src/services/store/store.ts +729 -0
  194. package/src/services/store/types.ts +303 -0
  195. package/src/services/test/child.ts +150 -0
  196. package/src/services/test/events.ts +102 -0
  197. package/src/services/test/files.ts +26 -0
  198. package/src/services/test/frames.ts +39 -0
  199. package/src/services/test/index.ts +7 -0
  200. package/src/services/test/runner.ts +122 -0
  201. package/src/services/test/types.ts +26 -0
  202. package/src/services/update/contract.ts +77 -0
  203. package/src/services/update/index.ts +12 -0
  204. package/src/services/update/installer.ts +139 -0
  205. package/src/services/update/updates.ts +137 -0
  206. package/src/utils/fs.ts +148 -0
  207. package/src/utils/tsast.ts +102 -0
@@ -0,0 +1,106 @@
1
+ # bench
2
+
3
+ ## What This Is
4
+
5
+ Benchmarks as a project kind, alongside agents, modules, cognets and prompts. A
6
+ bench runs one scenario against many configurations of an agent and records
7
+ what happened, so a claim like "this cognet is better at refactoring" has
8
+ evidence under it.
9
+
10
+ Consumed by `axon bench init|prepare|run|result` and `axon publish` when the
11
+ nearest project is a bench. Published to the registry as a definition, so
12
+ someone else can clone it and run it against their own agent.
13
+
14
+ **This module owns benchmarking, not project management.** Finding, opening,
15
+ scaffolding, preparing and publishing a bench are `Project()`'s, exactly as for
16
+ every other kind — a bench's row in `build/project/kinds.ts` is what makes it a
17
+ kind. `Bench()` hangs off `project.bench` and holds only the verbs no other
18
+ kind has. There used to be a parallel `Benches()` manager duplicating find /
19
+ open / create / publish; it is gone.
20
+
21
+ ## The Design
22
+
23
+ **One variable at a time, or a deliberate grid.** `matrix` is a plain object:
24
+ each key is a binding point, each value the variations to try there. One key
25
+ is the controlled experiment — the shape to reach for, because it is what
26
+ makes a result attributable. Several keys multiply, and the Cartesian growth
27
+ is the author's to manage rather than the runtime's to hide.
28
+
29
+ Keys are either named axes (`agent`, `cognet`, `model`, `env`) or dotted paths
30
+ into the blueprint (`config.engine.temperature`). The named set stays small
31
+ because the paths are the escape hatch. An earlier design had eight "factor
32
+ kinds" of which three were ever wired — the others (`dataset`, `judge`,
33
+ `toolset`) were not binding points at all, and modelling them as such is what
34
+ made the taxonomy grow.
35
+
36
+ **`expect` and `observe` are not the same tool.** A test encodes what you
37
+ already know should happen; a benchmark exists because you do not. So:
38
+
39
+ - `expect` — did the SCENARIO hold? Workspace copied, agent booted. A failure
40
+ means the trial measures nothing and must stop rather than score zero.
41
+ - `observe` — how did the SUBJECT do? Recorded whatever the value is, never a
42
+ failure. A model that scores badly is the finding.
43
+
44
+ This is why `observe` cannot collapse into `expect`: one halts on surprise,
45
+ the other records it, and a benchmark whose weak conditions all "failed" would
46
+ report itself as broken.
47
+
48
+ **Evidence, then projection.** `run()` executes and persists events; the
49
+ result is derived by REPLAY over that log. `BenchRunResult` deliberately holds
50
+ no aggregates — they are reproducible projections over the immutable record.
51
+ Physics (duration, tokens, cost, engine calls) are derived from the kernel log
52
+ with the author declaring nothing.
53
+
54
+ **Directories mean one thing each.**
55
+
56
+ ```
57
+ bench.config.ts matrix, measurements, setup
58
+ workspace/ the agent's world — copied fresh per iteration
59
+ fixtures/ author-side inputs; never visible to the agent
60
+ tests/ the scenarios, and where measurements are recorded
61
+ .bench/ generated declarations, run logs, materialized workspaces
62
+ ```
63
+
64
+ The `workspace/` ↔ `fixtures/` split is an invariant, not a convention: if
65
+ fixtures leaked into the agent's view, a run would depend on the bench's own
66
+ layout and stop being reproducible.
67
+
68
+ ## Key Interfaces
69
+
70
+ ```typescript
71
+ const project = await platform.projects.openAs("bench", cwd)
72
+
73
+ await project.prepare() // shared frame + the bench half below
74
+ await project.publish() // ship the definition (never the results)
75
+
76
+ await project.bench.coordinates() // the expanded matrix, without running
77
+ await project.bench.run() // execute, persist, project → BenchRunResult
78
+ await project.bench.result(runId) // rebuild a past result by replay
79
+ ```
80
+
81
+ `project.bench` is null on every other kind, so reaching for a benchmark verb
82
+ on an agent is a type error rather than a runtime surprise.
83
+
84
+ Authoring surface, narrowed by generated declarations in `.bench/bench.d.ts`:
85
+
86
+ ```typescript
87
+ bench.axis("model") // the variation this cell got
88
+ bench.observe("resolved", true) // a measurement
89
+ bench.attach("diff", patch) // content-hashed evidence
90
+ ```
91
+
92
+ ## Known Debt
93
+
94
+ - **The workspace assertion API does not exist.** `registry/benches/benchmark0`
95
+ contains `tests/fix-bug.bench.ts` written against the intended shape
96
+ (`workspace.tests.pass()`, `workspace.changed()`) as the target to satisfy.
97
+ Today `bench.workspace` is a bare path, so authors hand-roll `Bun.$`.
98
+ - **`toolCalls` is hardcoded to 0** in `preload.ts` — for coding benchmarks
99
+ this is one of the most diagnostic numbers there is.
100
+ - **`aggregate.ts` and `contribution.ts` are types with no consumers.** The
101
+ collective-submission layer (cohorts, tiers, shared aggregates) is designed
102
+ and unwired. Worth building against a real result set rather than in the
103
+ abstract — that is how they got ahead of the code in the first place.
104
+ - **Cases are unresolved.** `BenchTrialRecord.case` exists and nothing fills
105
+ it. A dataset is the population you measure over, not an axis, so it must
106
+ not enter the Cartesian product — but nothing expresses that yet.
@@ -0,0 +1,209 @@
1
+ # Bench API — the locked shape
2
+
3
+ Status: design agreed, not yet implemented. This is the target; `CLAUDE.md`
4
+ describes what exists today.
5
+
6
+ ## The whole surface
7
+
8
+ ```ts
9
+ type Schema = {
10
+ /** Has the agent resolved the bug? */
11
+ resolved: boolean
12
+
13
+ /** Files edited beyond the target. @objective minimize */
14
+ collateral: number
15
+
16
+ /** Strategy the agent took. */
17
+ approach: "refactor" | "patch" | "rewrite"
18
+ }
19
+
20
+ export default defineBench<Schema>({
21
+ description: "Can the agent fix a real bug without collateral damage?",
22
+
23
+ workspace: { source: "./workspace", retain: "failed" },
24
+
25
+ matrix: {
26
+ model: [
27
+ Mock({ fix: run("...correct patch...") }),
28
+ Mock({ fix: "I can't do that" }),
29
+ ],
30
+ },
31
+
32
+ trials: 3,
33
+
34
+ async setup() {
35
+ // after the workspace copy, before the subject boots
36
+ },
37
+ })
38
+ ```
39
+
40
+ ```ts
41
+ // tests/fix-bug.bench.ts
42
+ it("resolves the failing test", async ({ workspace }) => {
43
+ expect(workspace).toHaveFile("src/parser.ts")
44
+
45
+ const { axon } = await Axon()
46
+ await axon.request("Fix the failing test in src/parser.ts")
47
+
48
+ observe("resolved", await workspace.tests.pass())
49
+ observe("collateral", (await workspace.changed()).length - 1)
50
+ })
51
+ ```
52
+
53
+ Four concepts: **what varies** (matrix), **the world** (workspace), **the
54
+ scenario** (tests), **what is measured** (Schema). Nothing else is required.
55
+
56
+ ## Why each piece is shaped this way
57
+
58
+ ### `expect` vs `observe`
59
+
60
+ A test encodes what you already know should happen. A benchmark exists
61
+ because you do not.
62
+
63
+ - **`expect`** — did the SCENARIO hold? A failure means this trial measures
64
+ nothing; it is excluded, not scored zero.
65
+ - **`observe`** — how did the SUBJECT do? Recorded whatever the value is.
66
+
67
+ They cannot merge. `expect` throws on first failure, so a benchmark whose
68
+ weak conditions all failed would report itself broken — and every measurement
69
+ after the first failure would be lost, precisely on the runs that need them.
70
+
71
+ ### `matrix`
72
+
73
+ Every key is a binding point in the assembled blueprint; every value list is
74
+ an axis. One key is the controlled experiment. Several multiply, and the
75
+ growth is the author's to manage rather than the runtime's to hide.
76
+
77
+ Named axes (`agent`, `cognet`, `model`, `env`) cover the common cases; dotted
78
+ paths (`config.engine.temperature`) reach anything else. The named set stays
79
+ small BECAUSE the paths exist — an earlier design had eight "factor kinds" of
80
+ which three were ever wired.
81
+
82
+ Variation ids are derived from values, not positions: `model[0]` would mean
83
+ something different after a reorder, and cell ids built from it would stop
84
+ matching earlier runs of the same benchmark. Collisions (`Mock()` twice)
85
+ suffix rather than reject — two mocks with different reply maps are a
86
+ meaningful pair the id cannot otherwise distinguish.
87
+
88
+ ### `Schema` as a TypeScript type
89
+
90
+ The type is the authoring surface; JSON is not. A doc comment is where a
91
+ description naturally lives, and a union type is how anyone would write a
92
+ category.
93
+
94
+ But a `type` is erased at compile time, and the manifest needs the schema as a
95
+ VALUE:
96
+
97
+ - `BenchCohortKey` hashes it to decide whether two people's results are
98
+ comparable
99
+ - `BenchCoverage` reports "expected 4, observed 3" — undetectable if you only
100
+ know about measurements that fired
101
+
102
+ So `prepare` extracts the type into `.bench/schema.json` via
103
+ `ts.createProgram()`, and the runtime reads that. The author writes a type;
104
+ the system keeps a value. This is the same machinery `declareTools()` already
105
+ uses (worker-isolated, content-hash cached — see `scan/tools.ts`).
106
+
107
+ Derived from the type:
108
+
109
+ | from | to |
110
+ |---|---|
111
+ | property name | measurement id |
112
+ | property type | `value.kind` (`boolean`, `number`, union → `category`) |
113
+ | JSDoc text | `description` |
114
+ | `@objective minimize` | direction, where it is not obvious |
115
+ | `@unit ms` | display suffix |
116
+
117
+ `aggregate` defaults by kind — boolean → `rate`, number → `mean`, category →
118
+ `count`. `weighting` / `grain` / `missing` remain available and out of sight.
119
+
120
+ **`description` is required to publish, optional to run.** A shared benchmark
121
+ whose measurements have no stated meaning cannot be interpreted by whoever
122
+ reads the leaderboard; a bench you are iterating on locally should not nag.
123
+
124
+ ### Visualisation is derived, never declared
125
+
126
+ No `chart`, `display`, or `format` field. The moment one exists you need
127
+ `scatter`, `heatmap`, `stacked`, `logScale`, `colorBy` — and every bench
128
+ author becomes a dashboard designer. That is exactly how `measurements`
129
+ reached ten fields the first time.
130
+
131
+ Chart type follows from data shape, not author intent:
132
+
133
+ | data | chart |
134
+ |---|---|
135
+ | boolean × cells | bar of rates |
136
+ | number × cells | box plot (trials give the distribution) |
137
+ | category × cells | stacked bar |
138
+ | number × number | scatter |
139
+ | number, step grain | line |
140
+ | two matrix axes | grid / heatmap |
141
+
142
+ `objective` supplies the only intent a chart needs: which direction is good.
143
+
144
+ Presentation is a projection over the semantic schema, exactly as aggregates
145
+ are a projection over observations. Add a presentation field only when a real
146
+ chart demonstrably cannot be derived — earned by a chart that exists, not
147
+ anticipated.
148
+
149
+ ## Directory contract
150
+
151
+ ```
152
+ bench.config.ts matrix, measurements schema, setup
153
+ workspace/ the agent's world — copied fresh per iteration
154
+ fixtures/ author-side inputs; NEVER visible to the agent
155
+ tests/ scenarios, and where measurements are recorded
156
+ .bench/ generated declarations, extracted schema, run logs
157
+ ```
158
+
159
+ The `workspace/` ↔ `fixtures/` split is an invariant. If fixtures leaked into
160
+ the agent's view, a run would depend on the bench's own layout and stop being
161
+ reproducible.
162
+
163
+ ## Testing without live APIs
164
+
165
+ `Mock()` varies behaviour, not just labels — pattern maps, ordered sequences,
166
+ and `run()` steps that execute in the capsule:
167
+
168
+ ```ts
169
+ matrix: {
170
+ model: [
171
+ Mock({ fix: run("...correct patch...") }), // succeeds
172
+ Mock({ fix: "I can't do that" }), // fails
173
+ Mock({ fix: [run("...wrong..."), run("...right...")] }), // succeeds on retry
174
+ ],
175
+ }
176
+ ```
177
+
178
+ That exercises matrix expansion, subject binding, measurement collection and
179
+ replay end to end, with a real pass/fail spread and no API cost.
180
+
181
+ ## Build order
182
+
183
+ 1. **Object-keyed measurements** — `{ resolved: { description } }`. Small
184
+ change to working code, cuts nine fields to one or two, and carries
185
+ identical information to a `Schema` type so the migration is mechanical.
186
+ 2. **Workspace assertions** — `workspace.tests.pass()`, `.changed()`,
187
+ `toHaveFile()`. The blocker for any real bench;
188
+ `benchmark0/tests/fix-bug.bench.ts` is written against this and marked
189
+ not-yet-runnable.
190
+ 3. **`toolCalls`** — hardcoded to `0` in `preload.ts`, and one of the most
191
+ diagnostic numbers for a coding benchmark.
192
+ 4. **Schema extraction** — after two or three real benches exist, so the
193
+ extractor is designed against real schemas rather than imagined ones.
194
+ 5. **Aggregation** — `aggregate.ts` is types with no consumers. Build against
195
+ a real result set; designing it in the abstract is how it got ahead of the
196
+ code.
197
+
198
+ ## Still open
199
+
200
+ **Cases.** `BenchTrialRecord.case` exists and nothing fills it. A dataset
201
+ (SWE-bench's 2,294 instances) is the population you measure OVER, not an axis
202
+ — it must not enter the Cartesian product. Nothing expresses that yet, and it
203
+ is the one design question the current shape does not answer.
204
+
205
+ **Multi-agent.** `BenchPhysics` already separates `subject` from `assessor`,
206
+ so the types anticipate a judge; the runtime boots one agent. An LLM judge
207
+ must be held CONSTANT across the matrix, or the measuring instrument varies
208
+ with the thing measured. Blocked on whether `Axon()` becomes a manager of
209
+ several instances — a runtime question, not a bench one.
@@ -0,0 +1,105 @@
1
+ import { existsSync } from "node:fs"
2
+ import { isAbsolute, join, resolve } from "node:path"
3
+ import { err } from "@arcforge/err"
4
+ import type { BenchAxis } from "@arcforge/types"
5
+
6
+ /**
7
+ * Resolve the agents a matrix names, so every cell has something bootable.
8
+ *
9
+ * An `agent` variation is either a path you wrote or a registry ref you trust:
10
+ *
11
+ * agent: "./fixtures/subject" — authored here, committed
12
+ * agent: "@axon/coding-base" — resolved from the registry, fetched
13
+ *
14
+ * Registry agents land under `.bench/agents/`, not `fixtures/`. The split is
15
+ * the same one that makes `workspace/` ↔ `fixtures/` clean: `fixtures/` is
16
+ * source you author and commit, `.bench/` is generated and ignored. Vendoring
17
+ * a copy of a shared base into source would put it back under your control —
18
+ * which is precisely what naming a registry version is meant to avoid, since
19
+ * two benchmarks claiming to measure against "a standard coding agent" should
20
+ * be measuring against the same one.
21
+ *
22
+ * The version lands in the directory name because it lands in the manifest: an
23
+ * `agent` pin that says `@axon/coding-base@1.2.0` is a real identity, where a
24
+ * hash of the string "./fixtures/subject" says nothing anyone else can use.
25
+ */
26
+
27
+ export type ResolvedAgent = {
28
+ /** As written in the matrix. */
29
+ ref: string
30
+ /** Absolute path to a prepared agent project. */
31
+ root: string
32
+ /** Manifest identity: a registry `name@version`, or the ref for a local path. */
33
+ pin: string
34
+ }
35
+
36
+ export type AgentsOpts = {
37
+ root: string
38
+ /** Fetch, extract and prepare a registry artifact. */
39
+ clone(ref: string, cwd: string, options: { dir?: string }): Promise<{ name: string; version: string; root: string }>
40
+ /** Install deps and generate types for a local project. */
41
+ prepare(root: string): Promise<void>
42
+ }
43
+
44
+ /** A scoped registry name, as opposed to a filesystem path. */
45
+ export function isRegistryRef(ref: string): boolean {
46
+ return ref.startsWith("@") && !ref.startsWith("./") && !ref.startsWith("../")
47
+ }
48
+
49
+ export function Agents(opts: AgentsOpts) {
50
+ const cacheDir = join(opts.root, ".bench", "agents")
51
+
52
+ async function resolveOne(ref: string): Promise<ResolvedAgent> {
53
+ if (!isRegistryRef(ref)) {
54
+ const root = isAbsolute(ref) ? ref : resolve(opts.root, ref)
55
+ if (!existsSync(join(root, "axon.config.ts"))) {
56
+ throw err("BENCH_AGENT_UNRESOLVED", {
57
+ detail: `${ref} is not an agent project — no axon.config.ts at ${root}`,
58
+ context: { ref, root },
59
+ })
60
+ }
61
+ // A local agent still needs node_modules and generated types before
62
+ // it can boot, and a bench author should not have to know that.
63
+ await opts.prepare(root)
64
+ return { ref, root, pin: ref }
65
+ }
66
+
67
+ const cloned = await opts.clone(ref, cacheDir, { dir: slug(ref) })
68
+ return { ref, root: cloned.root, pin: `${cloned.name}@${cloned.version}` }
69
+ }
70
+
71
+ return {
72
+ /**
73
+ * Every agent the matrix names, resolved and prepared.
74
+ *
75
+ * Returns a map keyed by the ref as written, so the runtime can look up
76
+ * what a cell selected without re-deciding local-versus-registry.
77
+ */
78
+ async resolve(axes: BenchAxis[]): Promise<Map<string, ResolvedAgent>> {
79
+ const axis = axes.find(item => item.key === "agent")
80
+ if (!axis) return new Map()
81
+
82
+ const resolved = new Map<string, ResolvedAgent>()
83
+ for (const variation of axis.values) {
84
+ const ref = variation.value
85
+ if (typeof ref !== "string") {
86
+ throw err("BENCH_AGENT_UNRESOLVED", {
87
+ detail: `agent variations are paths or registry refs, got ${JSON.stringify(ref)}`,
88
+ context: { value: ref },
89
+ })
90
+ }
91
+ // Sequential on purpose: two variations of the same registry
92
+ // agent would otherwise race to write the same directory.
93
+ if (!resolved.has(ref)) resolved.set(ref, await resolveOne(ref))
94
+ }
95
+ return resolved
96
+ },
97
+ }
98
+ }
99
+
100
+ /** `@axon/coding-base@1.2.0` → `axon-coding-base@1.2.0`, safe as a directory name. */
101
+ function slug(ref: string): string {
102
+ return ref.replace(/^@/, "").replace(/\//g, "-")
103
+ }
104
+
105
+ export type AgentsT = ReturnType<typeof Agents>
@@ -0,0 +1,84 @@
1
+ import type { BenchAxis, BenchAxisKey, BenchAxisVariation, BenchMatrix } from "@arcforge/types"
2
+
3
+ /**
4
+ * Turn the authored `matrix` object into ordered axes.
5
+ *
6
+ * The authored form is deliberately terse — a key and a list of values, with
7
+ * nothing to declare twice because the key IS the binding point. Everything
8
+ * the manifest needs beyond that (stable ids, labels) is derived here, once,
9
+ * at the seam.
10
+ *
11
+ * Ids come from the value itself rather than its position: `model[0]` would
12
+ * mean a different thing the moment someone reorders the list, and cell ids
13
+ * built from it would silently stop matching earlier runs of the same
14
+ * benchmark. A label derived from the value stays stable under reordering.
15
+ */
16
+ export function toAxes(matrix: BenchMatrix | undefined): BenchAxis[] {
17
+ if (!matrix) return []
18
+
19
+ return Object.entries(matrix)
20
+ .filter(([, values]) => values !== undefined)
21
+ .map(([key, values]) => ({
22
+ key: key as BenchAxisKey,
23
+ // A bare value is a held constant — `agent: "@axon/coding-base"`
24
+ // says the same thing as a one-element array and reads like what it
25
+ // means. It is still an axis: one cell, and the value still pinned
26
+ // into the manifest, because a result is only comparable if what did
27
+ // NOT vary is recorded too.
28
+ values: withDistinctIds(Array.isArray(values) ? values : [values]),
29
+ }))
30
+ }
31
+
32
+ /**
33
+ * Derive ids, disambiguating any that collide.
34
+ *
35
+ * Two variations can legitimately share a derived label — `Mock()` twice with
36
+ * different reply maps is the obvious case, since the engine name is the same
37
+ * and the behaviour lives in a closure the id cannot see. Suffixing keeps the
38
+ * readable part while staying unique, rather than rejecting a config that is
39
+ * perfectly meaningful.
40
+ */
41
+ function withDistinctIds(values: unknown[]): BenchAxisVariation[] {
42
+ const variations = values.map(toVariation)
43
+ const counts = new Map<string, number>()
44
+ for (const variation of variations) counts.set(variation.id, (counts.get(variation.id) ?? 0) + 1)
45
+
46
+ const seen = new Map<string, number>()
47
+ return variations.map(variation => {
48
+ if ((counts.get(variation.id) ?? 0) < 2) return variation
49
+ const nth = (seen.get(variation.id) ?? 0) + 1
50
+ seen.set(variation.id, nth)
51
+ return { ...variation, id: `${variation.id}-${nth}`, label: `${variation.label} #${nth}` }
52
+ })
53
+ }
54
+
55
+ /**
56
+ * Derive a stable, readable id for one variation.
57
+ *
58
+ * An engine ref carries its model name, a path carries its directory — both
59
+ * are what an author would call the variation anyway, so a run's cell ids
60
+ * read as `model=claude-sonnet-4.6` rather than `model=0`.
61
+ */
62
+ function toVariation(value: unknown, index: number): BenchAxisVariation {
63
+ return { id: variationId(value, index), label: variationLabel(value), value }
64
+ }
65
+
66
+ function variationId(value: unknown, index: number): string {
67
+ const label = variationLabel(value)
68
+ if (!label) return String(index)
69
+ // Cell ids end up in file paths and manifest keys, so keep them boring.
70
+ return label.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64) || String(index)
71
+ }
72
+
73
+ function variationLabel(value: unknown): string {
74
+ if (typeof value === "string") return value
75
+ if (value && typeof value === "object") {
76
+ const record = value as Record<string, unknown>
77
+ // EngineRef and friends: the model name is the identity an author means.
78
+ for (const field of ["model", "id", "name", "ref"]) {
79
+ const candidate = record[field]
80
+ if (typeof candidate === "string" && candidate.length > 0) return candidate
81
+ }
82
+ }
83
+ return typeof value === "number" || typeof value === "boolean" ? String(value) : ""
84
+ }
@@ -0,0 +1,96 @@
1
+ import type { AxonCloudClient } from "@arcforge/cloud"
2
+ import type { BenchConfig, BenchRunResult } from "@arcforge/types"
3
+ import { Discover } from "./discover"
4
+ import { BenchLog } from "./log"
5
+ import { Matrix } from "./matrix"
6
+ import { Projection } from "./projection"
7
+ import { Resolver } from "./resolve"
8
+ import { Runner } from "./runner"
9
+ import type { TestRunnerT } from "../../services/test"
10
+ import { BenchTypegen } from "./typegen"
11
+ import { PrepareBench } from "./prepare"
12
+ import { Agents, type AgentsOpts } from "./agents"
13
+
14
+ /**
15
+ * The collaborators a bench needs that no other project kind does. Threaded
16
+ * from Platform through Projects, so Project() can build a Bench without
17
+ * knowing what a test runner is.
18
+ */
19
+ export type BenchExtras = {
20
+ tests: TestRunnerT
21
+ clone: AgentsOpts["clone"]
22
+ prepare: AgentsOpts["prepare"]
23
+ }
24
+
25
+ export type BenchOpts = BenchExtras & {
26
+ root: string
27
+ cloud: AxonCloudClient
28
+ }
29
+
30
+ /**
31
+ * The benchmark verbs of a bench project — run, result, coordinates, config.
32
+ *
33
+ * Composition only; behavior lives in the leaves. This is NOT a project
34
+ * manager: finding, opening, scaffolding, preparing and publishing a bench are
35
+ * Project()'s, exactly as for every other kind. What lives here is only what is
36
+ * genuinely about benchmarking.
37
+ */
38
+ export function Bench(opts: BenchOpts) {
39
+ const discover = Discover({ root: opts.root })
40
+ const matrix = Matrix()
41
+ const log = BenchLog({ root: opts.root })
42
+ const resolver = Resolver()
43
+ const runner = Runner({ root: opts.root, log, resolver, tests: opts.tests })
44
+ const projection = Projection()
45
+ const typegen = BenchTypegen({ root: opts.root })
46
+ const agents = Agents({ root: opts.root, clone: opts.clone, prepare: opts.prepare })
47
+ const prepare = PrepareBench({ root: opts.root, discover, typegen, agents })
48
+
49
+ return {
50
+ root: opts.root,
51
+
52
+ config(): Promise<BenchConfig> {
53
+ return discover.load()
54
+ },
55
+
56
+ async coordinates() {
57
+ const config = await discover.load()
58
+ return matrix.build(config.axes)
59
+ },
60
+
61
+ /** Execute Bun, persist events, then derive the portable result by replay. */
62
+ async run(): Promise<BenchRunResult> {
63
+ const config = await discover.load()
64
+ const coordinates = matrix.build(config.axes)
65
+ // Resolve agents before running: a registry ref must be fetched and
66
+ // prepared, and its version is what the manifest pins.
67
+ const resolvedAgents = await agents.resolve(config.axes)
68
+ const agentPins = new Map([...resolvedAgents].map(([ref, agent]) => [ref, agent.pin]))
69
+ const agentRoots = Object.fromEntries([...resolvedAgents].map(([ref, agent]) => [ref, agent.root]))
70
+ const runId = await runner.run(config, coordinates, agentPins, agentRoots)
71
+ const result = projection.build(config, await log.read(runId))
72
+ await log.writeManifest(runId, result.manifest)
73
+ await log.writeResult(runId, result)
74
+ return result
75
+ },
76
+
77
+ /** Rebuild a result without executing anything. */
78
+ async result(runId: string): Promise<BenchRunResult> {
79
+ const config = await discover.load()
80
+ return projection.build(config, await log.read(runId))
81
+ },
82
+
83
+ /**
84
+ * The bench half of `axon prepare` — validate the matrix's local
85
+ * refs, resolve the agents it names, materialize the workspace
86
+ * template, and generate declarations from the config's axes.
87
+ *
88
+ * Project.prepare() calls this after writing the shared frame, so
89
+ * `axon prepare` in a bench does the whole job in one verb. It stays
90
+ * exposed because run() needs its resolved result.
91
+ */
92
+ prepare,
93
+ }
94
+ }
95
+
96
+ export type BenchT = ReturnType<typeof Bench>