@arcforge/platform 2.0.166

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/README.md +31 -0
  2. package/package.json +69 -0
  3. package/src/bin/supervisor.ts +102 -0
  4. package/src/bin/test-api.ts +30 -0
  5. package/src/bin/test-preload.ts +308 -0
  6. package/src/bin/update-helper.ts +32 -0
  7. package/src/build/bench/CLAUDE.md +106 -0
  8. package/src/build/bench/SPEC.md +209 -0
  9. package/src/build/bench/agents.ts +105 -0
  10. package/src/build/bench/axes.ts +84 -0
  11. package/src/build/bench/bench.ts +96 -0
  12. package/src/build/bench/discover.ts +147 -0
  13. package/src/build/bench/index.ts +7 -0
  14. package/src/build/bench/log.ts +47 -0
  15. package/src/build/bench/matrix.ts +23 -0
  16. package/src/build/bench/preload.ts +440 -0
  17. package/src/build/bench/prepare.ts +61 -0
  18. package/src/build/bench/projection.ts +112 -0
  19. package/src/build/bench/resolve.ts +54 -0
  20. package/src/build/bench/runner.ts +309 -0
  21. package/src/build/bench/schema/extract.ts +148 -0
  22. package/src/build/bench/schema/index.ts +2 -0
  23. package/src/build/bench/schema/schema.ts +95 -0
  24. package/src/build/bench/schema/worker.ts +30 -0
  25. package/src/build/bench/typegen.ts +119 -0
  26. package/src/build/bench/workspace/files.ts +139 -0
  27. package/src/build/bench/workspace/handle.ts +133 -0
  28. package/src/build/bench/workspace/index.ts +11 -0
  29. package/src/build/bench/workspace/workspace.ts +14 -0
  30. package/src/build/blueprint/blueprint.ts +398 -0
  31. package/src/build/blueprint/cognet/abi.ts +207 -0
  32. package/src/build/blueprint/cognet/bundle-worker.ts +93 -0
  33. package/src/build/blueprint/cognet/bundle.ts +664 -0
  34. package/src/build/blueprint/cognet/cognet.ts +249 -0
  35. package/src/build/blueprint/cognet/index.ts +10 -0
  36. package/src/build/blueprint/cognet/manifest.ts +38 -0
  37. package/src/build/blueprint/cognet/resolve.ts +113 -0
  38. package/src/build/blueprint/collisions.ts +51 -0
  39. package/src/build/blueprint/index.ts +29 -0
  40. package/src/build/blueprint/modules/entries.ts +117 -0
  41. package/src/build/blueprint/modules/index.ts +300 -0
  42. package/src/build/blueprint/modules/meta.ts +174 -0
  43. package/src/build/blueprint/scan/boot.ts +34 -0
  44. package/src/build/blueprint/scan/cognetImports.ts +56 -0
  45. package/src/build/blueprint/scan/components.ts +45 -0
  46. package/src/build/blueprint/scan/config.ts +234 -0
  47. package/src/build/blueprint/scan/configImports.ts +232 -0
  48. package/src/build/blueprint/scan/declare-worker.ts +61 -0
  49. package/src/build/blueprint/scan/declare.ts +393 -0
  50. package/src/build/blueprint/scan/h3-globals.ts +137 -0
  51. package/src/build/blueprint/scan/knowledge.ts +101 -0
  52. package/src/build/blueprint/scan/middleware.ts +110 -0
  53. package/src/build/blueprint/scan/moduleImports.ts +79 -0
  54. package/src/build/blueprint/scan/plugins.ts +93 -0
  55. package/src/build/blueprint/scan/promptPackages.ts +45 -0
  56. package/src/build/blueprint/scan/prompts.ts +164 -0
  57. package/src/build/blueprint/scan/published.ts +131 -0
  58. package/src/build/blueprint/scan/resolve.ts +398 -0
  59. package/src/build/blueprint/scan/routes.ts +109 -0
  60. package/src/build/blueprint/scan/scripts.ts +119 -0
  61. package/src/build/blueprint/scan/tool-bundle-worker.ts +69 -0
  62. package/src/build/blueprint/scan/tools.ts +610 -0
  63. package/src/build/blueprint/types.ts +28 -0
  64. package/src/build/extensions/config.ts +261 -0
  65. package/src/build/extensions/edit.ts +684 -0
  66. package/src/build/extensions/extensions.ts +731 -0
  67. package/src/build/extensions/index.ts +10 -0
  68. package/src/build/extensions/load.ts +350 -0
  69. package/src/build/extensions/registrations.ts +189 -0
  70. package/src/build/extensions/store.ts +200 -0
  71. package/src/build/frame/frame.ts +145 -0
  72. package/src/build/frame/index.ts +5 -0
  73. package/src/build/frame/migrate.ts +246 -0
  74. package/src/build/project/bundle/agent.ts +382 -0
  75. package/src/build/project/bundle/artifacts.ts +88 -0
  76. package/src/build/project/bundle/assets.ts +385 -0
  77. package/src/build/project/bundle/bundle.ts +160 -0
  78. package/src/build/project/bundle/imports.ts +102 -0
  79. package/src/build/project/bundle/index.ts +16 -0
  80. package/src/build/project/bundle/module.ts +130 -0
  81. package/src/build/project/bundle/source.ts +136 -0
  82. package/src/build/project/bundle/stage.ts +97 -0
  83. package/src/build/project/bundle/types.ts +111 -0
  84. package/src/build/project/cognet.ts +156 -0
  85. package/src/build/project/create/agent.ts +52 -0
  86. package/src/build/project/create/bench.ts +81 -0
  87. package/src/build/project/create/cognet.ts +102 -0
  88. package/src/build/project/create/extension.ts +63 -0
  89. package/src/build/project/create/module.ts +43 -0
  90. package/src/build/project/create/profile.ts +102 -0
  91. package/src/build/project/create/prompt.ts +68 -0
  92. package/src/build/project/create/templates.ts +88 -0
  93. package/src/build/project/deploy.ts +96 -0
  94. package/src/build/project/index.ts +40 -0
  95. package/src/build/project/installer.ts +616 -0
  96. package/src/build/project/kinds.ts +374 -0
  97. package/src/build/project/links.ts +33 -0
  98. package/src/build/project/manifest/bunfig.ts +139 -0
  99. package/src/build/project/manifest/config.ts +246 -0
  100. package/src/build/project/manifest/env.ts +184 -0
  101. package/src/build/project/manifest/index.ts +9 -0
  102. package/src/build/project/manifest/manifest.ts +49 -0
  103. package/src/build/project/manifest/model.ts +124 -0
  104. package/src/build/project/manifest/package.ts +386 -0
  105. package/src/build/project/manifest/text.ts +19 -0
  106. package/src/build/project/models/fetch.ts +123 -0
  107. package/src/build/project/models/index.ts +8 -0
  108. package/src/build/project/models/models.ts +111 -0
  109. package/src/build/project/models/specifier.ts +106 -0
  110. package/src/build/project/models/store.ts +202 -0
  111. package/src/build/project/modules/hash.ts +31 -0
  112. package/src/build/project/modules/index.ts +5 -0
  113. package/src/build/project/modules/lock.ts +78 -0
  114. package/src/build/project/modules/modules.ts +155 -0
  115. package/src/build/project/prepare.ts +709 -0
  116. package/src/build/project/project.ts +253 -0
  117. package/src/build/project/projects.ts +164 -0
  118. package/src/build/project/propagation.ts +120 -0
  119. package/src/build/project/publish/verify.ts +312 -0
  120. package/src/build/project/publish.ts +155 -0
  121. package/src/build/project/reconcile.ts +93 -0
  122. package/src/build/project/specifier.ts +42 -0
  123. package/src/build/project/tree.ts +615 -0
  124. package/src/build/project/treecache.ts +356 -0
  125. package/src/build/project/typegen/axon-dts.ts +362 -0
  126. package/src/build/project/typegen/cognet-dts.ts +91 -0
  127. package/src/build/project/typegen/components.ts +45 -0
  128. package/src/build/project/typegen/env.ts +59 -0
  129. package/src/build/project/typegen/generators.ts +108 -0
  130. package/src/build/project/typegen/hooks.ts +71 -0
  131. package/src/build/project/typegen/index.ts +137 -0
  132. package/src/build/project/typegen/prompt-dts.ts +40 -0
  133. package/src/build/project/typegen/prompts.ts +36 -0
  134. package/src/build/project/typegen/registry-dts.ts +109 -0
  135. package/src/build/project/typegen/scripts.ts +37 -0
  136. package/src/build/project/typegen/tools.ts +31 -0
  137. package/src/build/project/typegen/tsconfig.ts +243 -0
  138. package/src/build/project/typegen/tui-dts.ts +120 -0
  139. package/src/build/project/typegen/write.ts +40 -0
  140. package/src/build/project/verify.ts +268 -0
  141. package/src/build/project/watcher.ts +266 -0
  142. package/src/build/runtime/agent.ts +550 -0
  143. package/src/build/runtime/escalations.ts +159 -0
  144. package/src/build/runtime/index.ts +15 -0
  145. package/src/build/runtime/instances.ts +604 -0
  146. package/src/build/runtime/profile.ts +91 -0
  147. package/src/build/runtime/progress.ts +143 -0
  148. package/src/build/runtime/requests.ts +167 -0
  149. package/src/build/runtime/resolve.ts +159 -0
  150. package/src/build/runtime/runtime.ts +233 -0
  151. package/src/build/runtime/sessions/index.ts +5 -0
  152. package/src/build/runtime/sessions/record.ts +145 -0
  153. package/src/build/runtime/sessions/sessions.ts +240 -0
  154. package/src/build/runtime/zeno.ts +120 -0
  155. package/src/index.ts +11 -0
  156. package/src/link/agent-main.ts +395 -0
  157. package/src/link/index.ts +70 -0
  158. package/src/link/services.ts +120 -0
  159. package/src/platform.ts +293 -0
  160. package/src/procs/index.ts +15 -0
  161. package/src/procs/tree.ts +177 -0
  162. package/src/services/cloud.ts +419 -0
  163. package/src/services/control/client.ts +126 -0
  164. package/src/services/control/dispatch.ts +156 -0
  165. package/src/services/control/index.ts +17 -0
  166. package/src/services/control/protocol.ts +179 -0
  167. package/src/services/control/server.ts +241 -0
  168. package/src/services/deployments.ts +81 -0
  169. package/src/services/files.ts +329 -0
  170. package/src/services/index.ts +5 -0
  171. package/src/services/mic/capture.ts +248 -0
  172. package/src/services/mic/config.ts +109 -0
  173. package/src/services/mic/fft.ts +93 -0
  174. package/src/services/mic/index.ts +139 -0
  175. package/src/services/mic/visualizer.ts +46 -0
  176. package/src/services/ollama/catalog.ts +122 -0
  177. package/src/services/ollama/http.ts +124 -0
  178. package/src/services/ollama/index.ts +13 -0
  179. package/src/services/ollama/models.ts +153 -0
  180. package/src/services/ollama/ollama.ts +89 -0
  181. package/src/services/ollama/registry.ts +112 -0
  182. package/src/services/ollama/types.ts +72 -0
  183. package/src/services/prompts.ts +76 -0
  184. package/src/services/registry.ts +220 -0
  185. package/src/services/resources/hardware.ts +125 -0
  186. package/src/services/resources/index.ts +6 -0
  187. package/src/services/resources/reservations.ts +162 -0
  188. package/src/services/resources/resources.ts +156 -0
  189. package/src/services/running/index.ts +1 -0
  190. package/src/services/running/running.ts +200 -0
  191. package/src/services/store/disk.ts +97 -0
  192. package/src/services/store/index.ts +15 -0
  193. package/src/services/store/store.ts +729 -0
  194. package/src/services/store/types.ts +303 -0
  195. package/src/services/test/child.ts +150 -0
  196. package/src/services/test/events.ts +102 -0
  197. package/src/services/test/files.ts +26 -0
  198. package/src/services/test/frames.ts +39 -0
  199. package/src/services/test/index.ts +7 -0
  200. package/src/services/test/runner.ts +122 -0
  201. package/src/services/test/types.ts +26 -0
  202. package/src/services/update/contract.ts +77 -0
  203. package/src/services/update/index.ts +12 -0
  204. package/src/services/update/installer.ts +139 -0
  205. package/src/services/update/updates.ts +137 -0
  206. package/src/utils/fs.ts +148 -0
  207. package/src/utils/tsast.ts +102 -0
@@ -0,0 +1,61 @@
1
+ import { resolveTestFiles } from "../../services/test"
2
+ import { isAbsolute, resolve } from "node:path"
3
+ import { err } from "@arcforge/err"
4
+ import { fsx } from "../../utils/fs"
5
+ import type { DiscoverT } from "./discover"
6
+ import type { BenchTypegenT } from "./typegen"
7
+ import type { BenchAxis } from "@arcforge/types"
8
+ import type { AgentsT } from "./agents"
9
+ import { Workspace } from "./workspace"
10
+
11
+ export type BenchPrepareResult = {
12
+ typegen: Awaited<ReturnType<BenchTypegenT["write"]>>
13
+ tests: string[]
14
+ /** Agents the matrix names, with the identity each resolved to. */
15
+ agents: Array<{ ref: string; pin: string }>
16
+ warnings: string[]
17
+ workspace: Awaited<ReturnType<ReturnType<typeof Workspace>["prepare"]>>["template"]
18
+ }
19
+
20
+ /**
21
+ * Variations that name something on disk, so prepare can fail early rather
22
+ * than mid-run. `agent` and `cognet` variations are bare paths or registry
23
+ * refs; everything else is a value the runtime merges, not a location.
24
+ */
25
+ function localRefs(axis: BenchAxis, value: unknown): string[] {
26
+ // `agent` is handled by Agents(), which resolves registry refs as well as
27
+ // paths. This check is for the axes that can only ever be local.
28
+ if (axis.key !== "cognet") return []
29
+ return typeof value === "string" ? [value] : []
30
+ }
31
+
32
+ export function PrepareBench(opts: { root: string; discover: DiscoverT; typegen: BenchTypegenT; agents: AgentsT }) {
33
+ return async function prepare(): Promise<BenchPrepareResult> {
34
+ const config = await opts.discover.load()
35
+ const tests = await resolveTestFiles(config.tests, opts.root)
36
+ if (tests.length === 0) throw err("BENCH_TESTS_NOT_FOUND", { detail: config.tests.join(", "), context: { declared: config.tests } })
37
+ for (const axis of config.axes) {
38
+ for (const variation of axis.values) {
39
+ for (const ref of localRefs(axis, variation.value)) {
40
+ if (!ref.startsWith(".") && !isAbsolute(ref)) continue
41
+ const path = isAbsolute(ref) ? ref : resolve(opts.root, ref)
42
+ if (!fsx.exists(path)) throw err("BENCH_LOCAL_REF_NOT_FOUND", { detail: `${axis.key}/${variation.id} -> ${ref}`, context: { axis: axis.key, valueId: variation.id, ref } })
43
+ }
44
+ }
45
+ }
46
+ // Fetch registry agents and prepare every agent the matrix names —
47
+ // local or cloned. A freshly cloned agent has no node_modules and no
48
+ // generated types, so without this every cell fails at boot with an
49
+ // error that says nothing about the cause.
50
+ const agents = await opts.agents.resolve(config.axes)
51
+
52
+ const workspace = await Workspace({ root: opts.root, definition: config.workspace }).prepare()
53
+ return {
54
+ typegen: await opts.typegen.write(config),
55
+ tests,
56
+ workspace: workspace.template,
57
+ agents: [...agents.values()].map(agent => ({ ref: agent.ref, pin: agent.pin })),
58
+ warnings: [],
59
+ }
60
+ }
61
+ }
@@ -0,0 +1,112 @@
1
+ import type {
2
+ BenchArtifactRef,
3
+ BenchCaseRef,
4
+ BenchConfig,
5
+ BenchCoverage,
6
+ BenchEvent,
7
+ BenchFault,
8
+ BenchObservation,
9
+ BenchPhysics,
10
+ BenchRunResult,
11
+ BenchSessionRef,
12
+ BenchTestRef,
13
+ BenchTrialRecord,
14
+ } from "@arcforge/types"
15
+ import { err } from "@arcforge/err"
16
+
17
+ const emptyPhysics = (): BenchPhysics => ({
18
+ subject: { durationMs: 0, tokens: { input: 0, output: 0 }, engineCalls: 0, toolCalls: 0, errors: 0 },
19
+ harness: { durationMs: 0 },
20
+ })
21
+
22
+ function key(context: BenchEvent["context"]): string {
23
+ return `${context.cellId ?? ""}:${context.trial ?? 0}:${context.testId ?? ""}:${context.attempt ?? 0}`
24
+ }
25
+
26
+ /** Pure fold from the immutable JSONL source of truth into portable run data. */
27
+ export function Projection() {
28
+ return {
29
+ build(_config: BenchConfig, events: BenchEvent[]): BenchRunResult {
30
+ const start = events.find(event => event.type === "bench:run:start")
31
+ if (!start || start.type !== "bench:run:start") throw err("BENCH_LOG_INVALID", { detail: "missing bench:run:start" })
32
+ const manifest = structuredClone(start.data.manifest)
33
+ const observations: BenchObservation[] = []
34
+ const artifacts: BenchArtifactRef[] = []
35
+ const sessions: BenchSessionRef[] = []
36
+ const faults: BenchFault[] = []
37
+ const tests = new Map<string, BenchTestRef>()
38
+ const physics = new Map<string, BenchPhysics>()
39
+ const observationIds = new Map<string, string[]>()
40
+ const artifactIds = new Map<string, string[]>()
41
+ const trials: BenchTrialRecord[] = []
42
+ let coverage: BenchCoverage = {
43
+ expectedTrials: 0, executedTrials: 0, completedTrials: 0,
44
+ expectedMeasurements: 0, observedMeasurements: 0,
45
+ missing: {}, measurements: [],
46
+ }
47
+
48
+ for (const event of events) {
49
+ const eventKey = key(event.context)
50
+ if (event.type === "bench:case:declare") tests.set(event.data.test.id, event.data.test)
51
+ if (event.type === "bench:observation") {
52
+ // `seq` is the event's identity — see AxonEvent. Unique
53
+ // within one bench log, which is the scope these ids are
54
+ // correlated in.
55
+ const id = String(event.time.seq)
56
+ observations.push({
57
+ ...event.data,
58
+ id,
59
+ time: new Date(event.time.ms).toISOString(),
60
+ context: {
61
+ runId: event.context.benchRunId,
62
+ cellId: event.context.cellId!,
63
+ testId: event.context.testId,
64
+ caseId: event.context.caseId,
65
+ trial: event.context.trial,
66
+ attempt: event.context.attempt,
67
+ },
68
+ })
69
+ observationIds.set(eventKey, [...(observationIds.get(eventKey) ?? []), id])
70
+ }
71
+ if (event.type === "bench:artifact") {
72
+ artifacts.push(event.data.artifact)
73
+ artifactIds.set(eventKey, [...(artifactIds.get(eventKey) ?? []), event.data.artifact.id])
74
+ }
75
+ if (event.type === "bench:session:attach") sessions.push(event.data.session)
76
+ if (event.type === "bench:process:fault") faults.push(event.data.fault)
77
+ if (event.type === "bench:trial:complete") {
78
+ physics.set(`${event.context.cellId}:${event.context.trial}`, event.data.physics)
79
+ }
80
+ if (event.type === "bench:case:complete") {
81
+ const test = tests.get(event.context.testId ?? "")
82
+ const cell = manifest.cells.find(item => item.id === event.context.cellId)
83
+ if (!test || !cell) throw err("BENCH_LOG_INVALID", { detail: "case completion has no declaration or cell" })
84
+ trials.push({
85
+ id: String(event.time.seq),
86
+ test,
87
+ cell,
88
+ trial: event.context.trial ?? 0,
89
+ attempt: event.context.attempt ?? 0,
90
+ status: event.data.status,
91
+ ...(event.data.error ? { error: event.data.error } : {}),
92
+ physics: physics.get(`${cell.id}:${event.context.trial}`) ?? emptyPhysics(),
93
+ sessions: [],
94
+ observationIds: observationIds.get(eventKey) ?? [],
95
+ artifactIds: artifactIds.get(eventKey) ?? [],
96
+ })
97
+ }
98
+ if (event.type === "bench:run:complete") {
99
+ coverage = event.data.coverage
100
+ manifest.completedAt = new Date(event.time.ms).toISOString()
101
+ }
102
+ }
103
+
104
+ manifest.tests = [...tests.values()]
105
+ manifest.cases = manifest.tests.map(test => ({ id: test.id, testId: test.id, label: test.name } satisfies BenchCaseRef))
106
+ for (const trial of trials) trial.physics = physics.get(`${trial.cell.id}:${trial.trial}`) ?? trial.physics
107
+ return { manifest, trials, observations, artifacts, sessions, faults, coverage }
108
+ },
109
+ }
110
+ }
111
+
112
+ export type ProjectionT = ReturnType<typeof Projection>
@@ -0,0 +1,54 @@
1
+ import type { BenchAxis, BenchCell, BenchCoordinate, BenchHash, BenchValue } from "@arcforge/types"
2
+ import { err } from "@arcforge/err"
3
+
4
+ function canonical(value: BenchValue): string {
5
+ if (value === null || typeof value !== "object") return JSON.stringify(value)
6
+ if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`
7
+ return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key]!)}`).join(",")}}`
8
+ }
9
+
10
+ function pin(value: unknown): BenchHash {
11
+ const hash = new Bun.CryptoHasher("sha256").update(canonical(value as BenchValue)).digest("hex")
12
+ return `sha256:${hash}`
13
+ }
14
+
15
+ function findAxis(axes: BenchAxis[], key: string): BenchAxis {
16
+ const axis = axes.find(item => item.key === key)
17
+ if (!axis) throw err("BENCH_AXIS_NOT_FOUND", { context: { axis: key } })
18
+ return axis
19
+ }
20
+
21
+ export function Resolver() {
22
+ return {
23
+ /**
24
+ * @param agentPins resolved identities keyed by the ref as written, so
25
+ * an `agent` axis pins `@axon/coding-base@1.2.0` rather than a hash of
26
+ * the string "./fixtures/subject" — which says nothing anyone else
27
+ * could compare against.
28
+ */
29
+ async resolve(axes: BenchAxis[], coordinate: BenchCoordinate, agentPins?: Map<string, string>): Promise<BenchCell> {
30
+ const resolved = coordinate.axes.map(selection => {
31
+ const axis = findAxis(axes, selection.key)
32
+ const selected = axis.values.find(item => item.id === selection.valueId)
33
+ if (!selected) {
34
+ throw err("BENCH_AXIS_VALUE_NOT_FOUND", {
35
+ detail: `${selection.key}/${selection.valueId}`,
36
+ context: { axis: selection.key, valueId: selection.valueId },
37
+ })
38
+ }
39
+ const resolvedPin = selection.key === "agent" && typeof selected.value === "string"
40
+ ? agentPins?.get(selected.value)
41
+ : undefined
42
+
43
+ return {
44
+ ...selection,
45
+ label: selected.label || selected.id,
46
+ pin: resolvedPin ?? pin(selected.value),
47
+ }
48
+ })
49
+ return { id: coordinate.id, axes: resolved }
50
+ },
51
+ }
52
+ }
53
+
54
+ export type ResolverT = ReturnType<typeof Resolver>
@@ -0,0 +1,309 @@
1
+ import { join } from "node:path"
2
+ import type {
3
+ AxonTestEvent,
4
+ BenchAxis,
5
+ BenchCoordinate,
6
+ BenchCoverage,
7
+ BenchEventMap,
8
+ BenchHash,
9
+ BenchNormalizedConfig,
10
+ BenchRunManifest,
11
+ BenchMeasurementState,
12
+ BenchResourceUsage,
13
+ BenchTestRef,
14
+ } from "@arcforge/types"
15
+ import { BENCH_PROTOCOL } from "@arcforge/types"
16
+ import { err } from "@arcforge/err"
17
+ import type { BenchLogT } from "./log"
18
+ import type { ResolverT } from "./resolve"
19
+ import type { TestRunnerT } from "../../services/test"
20
+ import { Workspace } from "./workspace"
21
+
22
+ export type RunnerOpts = { root: string; log: BenchLogT; resolver: ResolverT; tests: TestRunnerT }
23
+
24
+ /**
25
+ * Parse a budget string into minor units of USD.
26
+ *
27
+ * Accepts "$0.50", "0.50", "50c". Returns null for undefined so an unset
28
+ * budget is unmistakably "no ceiling" rather than zero — a budget of zero
29
+ * would halt the run before the first trial, which is never what anyone means.
30
+ */
31
+ function budgetUsd(value: string | undefined): number | null {
32
+ if (!value) return null
33
+ const trimmed = value.trim()
34
+ const cents = /^(\d+(?:\.\d+)?)\s*c$/i.exec(trimmed)
35
+ if (cents) return Number(cents[1]) / 100
36
+ const dollars = /^\$?\s*(\d+(?:\.\d+)?)$/.exec(trimmed)
37
+ if (dollars) return Number(dollars[1])
38
+ throw err("BENCH_CONFIG_INVALID", { detail: `budget ${JSON.stringify(value)} is not an amount like "$0.50"` })
39
+ }
40
+
41
+ function hash(value: unknown): BenchHash {
42
+ return `sha256:${new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex")}`
43
+ }
44
+
45
+ function testRef(event: Extract<AxonTestEvent, { type: "test:case:declare" }>): BenchTestRef {
46
+ return {
47
+ id: event.context.testId!,
48
+ file: event.context.file!,
49
+ suite: event.data.suite,
50
+ name: event.data.name,
51
+ }
52
+ }
53
+
54
+ /** The concrete value selected on each axis, keyed by axis — what the subject is built from. */
55
+ function selectedValues(axes: BenchAxis[], coordinate: BenchCoordinate): Record<string, unknown> {
56
+ return Object.fromEntries(coordinate.axes.map(selection => {
57
+ const axis = axes.find(item => item.key === selection.key)
58
+ const value = axis?.values.find(item => item.id === selection.valueId)
59
+ if (!value) {
60
+ throw err("BENCH_AXIS_VALUE_NOT_FOUND", {
61
+ detail: `${selection.key}/${selection.valueId}`,
62
+ context: { axis: selection.key, valueId: selection.valueId },
63
+ })
64
+ }
65
+ return [selection.key, value.value]
66
+ }))
67
+ }
68
+
69
+ function isBenchMessage(value: unknown): value is { channel: "axon:bench"; frame: { type: keyof BenchEventMap; context: Record<string, unknown>; data: unknown } } {
70
+ if (!value || typeof value !== "object") return false
71
+ const message = value as any
72
+ return message.channel === "axon:bench"
73
+ && [
74
+ "bench:observation", "bench:artifact",
75
+ "bench:session:attach", "bench:session:detach", "bench:session:usage",
76
+ "bench:workspace:materialized", "bench:workspace:captured",
77
+ "bench:workspace:retained", "bench:workspace:cleaned",
78
+ ].includes(message.frame?.type)
79
+ && typeof message.frame.context === "object"
80
+ }
81
+
82
+ export function Runner(opts: RunnerOpts) {
83
+ return {
84
+ async run(
85
+ config: BenchNormalizedConfig,
86
+ coordinates: BenchCoordinate[],
87
+ agentPins?: Map<string, string>,
88
+ agentRoots?: Record<string, string>,
89
+ ): Promise<string> {
90
+ const runId = Bun.randomUUIDv7()
91
+ const started = performance.now()
92
+ const preparedWorkspace = await Workspace({ root: opts.root, definition: config.workspace }).prepare()
93
+ const manifest: BenchRunManifest = {
94
+ runId,
95
+ bench: { ref: config.identity.name, name: config.identity.name, version: config.identity.version, hash: hash(config) },
96
+ protocols: BENCH_PROTOCOL,
97
+ schema: {
98
+ hash: hash({ measurements: config.measurements, dimensions: config.dimensions, artifacts: config.artifacts }),
99
+ measurements: config.measurements,
100
+ dimensions: config.dimensions,
101
+ artifacts: config.artifacts,
102
+ },
103
+ axes: config.axes.map(axis => ({ key: axis.key, label: axis.label ?? axis.key, values: axis.values.map(value => value.id) })),
104
+ cells: [],
105
+ trials: config.trials,
106
+ tests: [],
107
+ cases: [],
108
+ caseSetPin: hash(config.tests),
109
+ harness: { name: "axon", version: "0.1.0", runtime: process.version, bun: Bun.version },
110
+ workspace: preparedWorkspace.template,
111
+ startedAt: new Date().toISOString(),
112
+ }
113
+ const cells = []
114
+ for (const coordinate of coordinates) cells.push(await opts.resolver.resolve(config.axes, coordinate, agentPins))
115
+ manifest.cells = cells
116
+ await opts.log.emit({ benchRunId: runId }, "bench:run:start", { manifest })
117
+ await opts.log.writeManifest(runId, manifest)
118
+ await opts.log.emit({ benchRunId: runId }, "bench:workspace:prepared", { template: preparedWorkspace.template })
119
+
120
+ const perTrialBudget = budgetUsd(config.budget?.perTrial)
121
+ const totalBudget = budgetUsd(config.budget?.total)
122
+ let spentUsd = 0
123
+ let budgetExhausted = false
124
+
125
+ let expectedTrials = 0
126
+ let executedTrials = 0
127
+ let completedTrials = 0
128
+ let completedCases = 0
129
+ const required = config.measurements.filter(item => item.required).length
130
+ const requiredIds = new Set(config.measurements.filter(item => item.required).map(item => item.id))
131
+ const measured = new Set<string>()
132
+ const measurementStates: BenchMeasurementState[] = []
133
+ const preload = new URL("./preload.ts", import.meta.url).pathname
134
+
135
+ try {
136
+ for (let cellIndex = 0; cellIndex < coordinates.length && !budgetExhausted; cellIndex++) {
137
+ const coordinate = coordinates[cellIndex]!
138
+ const cell = cells[cellIndex]!
139
+ const cellStarted = performance.now()
140
+ await opts.log.emit({ benchRunId: runId, cellId: cell.id }, "bench:cell:start", { cell })
141
+ // Each level closes its OWN bracket on the way out. The
142
+ // run-level catch below cannot do it for them: by the time
143
+ // it runs, which cell and trial were in flight is lost.
144
+ try {
145
+ for (let trial = 0; trial < config.trials && !budgetExhausted; trial++) {
146
+ expectedTrials++
147
+ const trialStarted = performance.now()
148
+ const base = { benchRunId: runId, cellId: cell.id, trial }
149
+ await opts.log.emit(base, "bench:trial:start", {})
150
+ try {
151
+ let cases = 0
152
+ let completed = 0
153
+ const subject: BenchResourceUsage = { durationMs: 0, tokens: { input: 0, output: 0 }, engineCalls: 0, toolCalls: 0, errors: 0 }
154
+ let subjectCost = 0
155
+ let subjectCostKnown = true
156
+ const context = {
157
+ runId,
158
+ cellId: cell.id,
159
+ trial,
160
+ axes: cell.axes,
161
+ axisValues: selectedValues(config.axes, coordinate),
162
+ measurements: config.measurements,
163
+ dimensions: config.dimensions,
164
+ artifacts: config.artifacts,
165
+ artifactDir: join(opts.log.runDir(runId), "artifacts"),
166
+ benchRoot: opts.root,
167
+ // setup() is a function and the context crosses as
168
+ // JSON — the preload re-imports the config to reach it.
169
+ configPath: join(opts.root, "bench.config.ts"),
170
+ agentRoots,
171
+ workspace: {
172
+ definition: config.workspace,
173
+ template: preparedWorkspace.template,
174
+ sourcePath: preparedWorkspace.sourcePath,
175
+ root: join(opts.root, ".bench", "workspace"),
176
+ },
177
+ }
178
+ await opts.tests.run({
179
+ cwd: opts.root,
180
+ files: config.tests,
181
+ preloads: [preload],
182
+ env: { AXON_BENCH_CONTEXT: JSON.stringify(context) },
183
+ async onMessage(message) {
184
+ if (!isBenchMessage(message)) return
185
+ const frame = message.frame
186
+ const eventContext = { ...base, ...(frame.context as object) }
187
+ const emitted = await opts.log.emit(eventContext, frame.type, frame.data as never)
188
+ if (frame.type === "bench:session:usage") {
189
+ const usage = (frame.data as { usage: BenchResourceUsage }).usage
190
+ subject.durationMs += usage.durationMs
191
+ subject.tokens.input += usage.tokens.input
192
+ subject.tokens.output += usage.tokens.output
193
+ subject.engineCalls += usage.engineCalls
194
+ subject.toolCalls += usage.toolCalls
195
+ subject.errors += usage.errors
196
+ if (usage.costUsd === undefined) subjectCostKnown = false
197
+ else subjectCost += usage.costUsd
198
+ }
199
+ if (frame.type === "bench:observation") {
200
+ const measurementId = (frame.data as { measurementId?: string }).measurementId
201
+ if (measurementId && requiredIds.has(measurementId)) {
202
+ const slot = `${cell.id}:${trial}:${frame.context.testId ?? ""}:${frame.context.attempt ?? 0}:${measurementId}`
203
+ if (!measured.has(slot)) {
204
+ measured.add(slot)
205
+ measurementStates.push({ kind: "observed", observationId: String(emitted.time.seq) })
206
+ }
207
+ }
208
+ }
209
+ },
210
+ async onEvent(event) {
211
+ const eventContext = { ...base, testId: event.context.testId, attempt: event.context.attempt }
212
+ if (event.type === "test:case:declare") {
213
+ cases++
214
+ await opts.log.emit(eventContext, "bench:case:declare", { test: testRef(event) })
215
+ } else if (event.type === "test:case:pass" || event.type === "test:case:fail") {
216
+ completed++
217
+ completedCases++
218
+ for (const measurementId of requiredIds) {
219
+ const slot = `${cell.id}:${trial}:${event.context.testId ?? ""}:${event.context.attempt ?? 0}:${measurementId}`
220
+ if (!measured.has(slot)) measurementStates.push({ kind: "missing", reason: "not_emitted" })
221
+ }
222
+ await opts.log.emit(eventContext, "bench:case:complete", {
223
+ status: event.type === "test:case:pass" ? "passed" : "failed",
224
+ durationMs: event.data.durationMs,
225
+ ...(event.type === "test:case:fail" ? { error: event.data.error } : {}),
226
+ })
227
+ } else if (event.type === "test:case:skip" || event.type === "test:case:todo") {
228
+ completed++
229
+ completedCases++
230
+ for (const _measurementId of requiredIds) measurementStates.push({ kind: "missing", reason: "filtered" })
231
+ await opts.log.emit(eventContext, "bench:case:complete", {
232
+ status: event.type === "test:case:skip" ? "skipped" : "todo",
233
+ durationMs: 0,
234
+ })
235
+ }
236
+ },
237
+ })
238
+ if (cases === 0) throw err("BENCH_NO_CASES")
239
+ executedTrials++
240
+ if (completed === cases) completedTrials++
241
+
242
+ // Budget is checked AFTER the trial, not during: a trial
243
+ // is the smallest unit whose cost is knowable, and killing
244
+ // one mid-flight would leave a half-run that measures
245
+ // nothing while still having been paid for. Exceeding the
246
+ // ceiling is a fault, so the trial is excluded from scoring
247
+ // rather than counted as a failure of the agent.
248
+ if (subjectCostKnown) {
249
+ spentUsd += subjectCost
250
+ if (perTrialBudget !== null && subjectCost > perTrialBudget) {
251
+ await opts.log.emit(base, "bench:process:fault", {
252
+ fault: { code: "budget", message: `trial spent $${subjectCost.toFixed(4)}, over the $${perTrialBudget} per-trial budget`, context: { runId, cellId: cell.id, trial } },
253
+ })
254
+ }
255
+ if (totalBudget !== null && spentUsd > totalBudget) {
256
+ await opts.log.emit(base, "bench:process:fault", {
257
+ fault: { code: "budget", message: `run spent $${spentUsd.toFixed(4)}, over the $${totalBudget} total budget — stopping`, context: { runId, cellId: cell.id, trial } },
258
+ })
259
+ budgetExhausted = true
260
+ }
261
+ }
262
+
263
+ await opts.log.emit(base, "bench:trial:complete", {
264
+ durationMs: performance.now() - trialStarted,
265
+ physics: {
266
+ subject: { ...subject, ...(subjectCostKnown ? { costUsd: subjectCost } : {}) },
267
+ harness: { durationMs: performance.now() - trialStarted },
268
+ },
269
+ })
270
+ } catch (error) {
271
+ await opts.log.emit(base, "bench:trial:failed", {
272
+ durationMs: performance.now() - trialStarted,
273
+ error: err(error),
274
+ })
275
+ throw error
276
+ }
277
+ }
278
+ await opts.log.emit({ benchRunId: runId, cellId: cell.id }, "bench:cell:complete", { durationMs: performance.now() - cellStarted })
279
+ } catch (error) {
280
+ await opts.log.emit({ benchRunId: runId, cellId: cell.id }, "bench:cell:failed", {
281
+ durationMs: performance.now() - cellStarted,
282
+ error: err(error),
283
+ })
284
+ throw error
285
+ }
286
+ }
287
+ const coverage: BenchCoverage = {
288
+ expectedTrials,
289
+ executedTrials,
290
+ completedTrials,
291
+ expectedMeasurements: completedCases * required,
292
+ observedMeasurements: measured.size,
293
+ missing: completedCases * required > measured.size
294
+ ? { not_emitted: measurementStates.filter(item => item.kind === "missing" && item.reason === "not_emitted").length,
295
+ filtered: measurementStates.filter(item => item.kind === "missing" && item.reason === "filtered").length }
296
+ : {},
297
+ measurements: measurementStates,
298
+ }
299
+ await opts.log.emit({ benchRunId: runId }, "bench:run:complete", { durationMs: performance.now() - started, coverage })
300
+ return runId
301
+ } catch (error) {
302
+ await opts.log.emit({ benchRunId: runId }, "bench:run:failed", { error: err(error) })
303
+ throw error
304
+ }
305
+ },
306
+ }
307
+ }
308
+
309
+ export type RunnerT = ReturnType<typeof Runner>