@arcforge/platform 2.0.166

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/README.md +31 -0
  2. package/package.json +69 -0
  3. package/src/bin/supervisor.ts +102 -0
  4. package/src/bin/test-api.ts +30 -0
  5. package/src/bin/test-preload.ts +308 -0
  6. package/src/bin/update-helper.ts +32 -0
  7. package/src/build/bench/CLAUDE.md +106 -0
  8. package/src/build/bench/SPEC.md +209 -0
  9. package/src/build/bench/agents.ts +105 -0
  10. package/src/build/bench/axes.ts +84 -0
  11. package/src/build/bench/bench.ts +96 -0
  12. package/src/build/bench/discover.ts +147 -0
  13. package/src/build/bench/index.ts +7 -0
  14. package/src/build/bench/log.ts +47 -0
  15. package/src/build/bench/matrix.ts +23 -0
  16. package/src/build/bench/preload.ts +440 -0
  17. package/src/build/bench/prepare.ts +61 -0
  18. package/src/build/bench/projection.ts +112 -0
  19. package/src/build/bench/resolve.ts +54 -0
  20. package/src/build/bench/runner.ts +309 -0
  21. package/src/build/bench/schema/extract.ts +148 -0
  22. package/src/build/bench/schema/index.ts +2 -0
  23. package/src/build/bench/schema/schema.ts +95 -0
  24. package/src/build/bench/schema/worker.ts +30 -0
  25. package/src/build/bench/typegen.ts +119 -0
  26. package/src/build/bench/workspace/files.ts +139 -0
  27. package/src/build/bench/workspace/handle.ts +133 -0
  28. package/src/build/bench/workspace/index.ts +11 -0
  29. package/src/build/bench/workspace/workspace.ts +14 -0
  30. package/src/build/blueprint/blueprint.ts +398 -0
  31. package/src/build/blueprint/cognet/abi.ts +207 -0
  32. package/src/build/blueprint/cognet/bundle-worker.ts +93 -0
  33. package/src/build/blueprint/cognet/bundle.ts +664 -0
  34. package/src/build/blueprint/cognet/cognet.ts +249 -0
  35. package/src/build/blueprint/cognet/index.ts +10 -0
  36. package/src/build/blueprint/cognet/manifest.ts +38 -0
  37. package/src/build/blueprint/cognet/resolve.ts +113 -0
  38. package/src/build/blueprint/collisions.ts +51 -0
  39. package/src/build/blueprint/index.ts +29 -0
  40. package/src/build/blueprint/modules/entries.ts +117 -0
  41. package/src/build/blueprint/modules/index.ts +300 -0
  42. package/src/build/blueprint/modules/meta.ts +174 -0
  43. package/src/build/blueprint/scan/boot.ts +34 -0
  44. package/src/build/blueprint/scan/cognetImports.ts +56 -0
  45. package/src/build/blueprint/scan/components.ts +45 -0
  46. package/src/build/blueprint/scan/config.ts +234 -0
  47. package/src/build/blueprint/scan/configImports.ts +232 -0
  48. package/src/build/blueprint/scan/declare-worker.ts +61 -0
  49. package/src/build/blueprint/scan/declare.ts +393 -0
  50. package/src/build/blueprint/scan/h3-globals.ts +137 -0
  51. package/src/build/blueprint/scan/knowledge.ts +101 -0
  52. package/src/build/blueprint/scan/middleware.ts +110 -0
  53. package/src/build/blueprint/scan/moduleImports.ts +79 -0
  54. package/src/build/blueprint/scan/plugins.ts +93 -0
  55. package/src/build/blueprint/scan/promptPackages.ts +45 -0
  56. package/src/build/blueprint/scan/prompts.ts +164 -0
  57. package/src/build/blueprint/scan/published.ts +131 -0
  58. package/src/build/blueprint/scan/resolve.ts +398 -0
  59. package/src/build/blueprint/scan/routes.ts +109 -0
  60. package/src/build/blueprint/scan/scripts.ts +119 -0
  61. package/src/build/blueprint/scan/tool-bundle-worker.ts +69 -0
  62. package/src/build/blueprint/scan/tools.ts +610 -0
  63. package/src/build/blueprint/types.ts +28 -0
  64. package/src/build/extensions/config.ts +261 -0
  65. package/src/build/extensions/edit.ts +684 -0
  66. package/src/build/extensions/extensions.ts +731 -0
  67. package/src/build/extensions/index.ts +10 -0
  68. package/src/build/extensions/load.ts +350 -0
  69. package/src/build/extensions/registrations.ts +189 -0
  70. package/src/build/extensions/store.ts +200 -0
  71. package/src/build/frame/frame.ts +145 -0
  72. package/src/build/frame/index.ts +5 -0
  73. package/src/build/frame/migrate.ts +246 -0
  74. package/src/build/project/bundle/agent.ts +382 -0
  75. package/src/build/project/bundle/artifacts.ts +88 -0
  76. package/src/build/project/bundle/assets.ts +385 -0
  77. package/src/build/project/bundle/bundle.ts +160 -0
  78. package/src/build/project/bundle/imports.ts +102 -0
  79. package/src/build/project/bundle/index.ts +16 -0
  80. package/src/build/project/bundle/module.ts +130 -0
  81. package/src/build/project/bundle/source.ts +136 -0
  82. package/src/build/project/bundle/stage.ts +97 -0
  83. package/src/build/project/bundle/types.ts +111 -0
  84. package/src/build/project/cognet.ts +156 -0
  85. package/src/build/project/create/agent.ts +52 -0
  86. package/src/build/project/create/bench.ts +81 -0
  87. package/src/build/project/create/cognet.ts +102 -0
  88. package/src/build/project/create/extension.ts +63 -0
  89. package/src/build/project/create/module.ts +43 -0
  90. package/src/build/project/create/profile.ts +102 -0
  91. package/src/build/project/create/prompt.ts +68 -0
  92. package/src/build/project/create/templates.ts +88 -0
  93. package/src/build/project/deploy.ts +96 -0
  94. package/src/build/project/index.ts +40 -0
  95. package/src/build/project/installer.ts +616 -0
  96. package/src/build/project/kinds.ts +374 -0
  97. package/src/build/project/links.ts +33 -0
  98. package/src/build/project/manifest/bunfig.ts +139 -0
  99. package/src/build/project/manifest/config.ts +246 -0
  100. package/src/build/project/manifest/env.ts +184 -0
  101. package/src/build/project/manifest/index.ts +9 -0
  102. package/src/build/project/manifest/manifest.ts +49 -0
  103. package/src/build/project/manifest/model.ts +124 -0
  104. package/src/build/project/manifest/package.ts +386 -0
  105. package/src/build/project/manifest/text.ts +19 -0
  106. package/src/build/project/models/fetch.ts +123 -0
  107. package/src/build/project/models/index.ts +8 -0
  108. package/src/build/project/models/models.ts +111 -0
  109. package/src/build/project/models/specifier.ts +106 -0
  110. package/src/build/project/models/store.ts +202 -0
  111. package/src/build/project/modules/hash.ts +31 -0
  112. package/src/build/project/modules/index.ts +5 -0
  113. package/src/build/project/modules/lock.ts +78 -0
  114. package/src/build/project/modules/modules.ts +155 -0
  115. package/src/build/project/prepare.ts +709 -0
  116. package/src/build/project/project.ts +253 -0
  117. package/src/build/project/projects.ts +164 -0
  118. package/src/build/project/propagation.ts +120 -0
  119. package/src/build/project/publish/verify.ts +312 -0
  120. package/src/build/project/publish.ts +155 -0
  121. package/src/build/project/reconcile.ts +93 -0
  122. package/src/build/project/specifier.ts +42 -0
  123. package/src/build/project/tree.ts +615 -0
  124. package/src/build/project/treecache.ts +356 -0
  125. package/src/build/project/typegen/axon-dts.ts +362 -0
  126. package/src/build/project/typegen/cognet-dts.ts +91 -0
  127. package/src/build/project/typegen/components.ts +45 -0
  128. package/src/build/project/typegen/env.ts +59 -0
  129. package/src/build/project/typegen/generators.ts +108 -0
  130. package/src/build/project/typegen/hooks.ts +71 -0
  131. package/src/build/project/typegen/index.ts +137 -0
  132. package/src/build/project/typegen/prompt-dts.ts +40 -0
  133. package/src/build/project/typegen/prompts.ts +36 -0
  134. package/src/build/project/typegen/registry-dts.ts +109 -0
  135. package/src/build/project/typegen/scripts.ts +37 -0
  136. package/src/build/project/typegen/tools.ts +31 -0
  137. package/src/build/project/typegen/tsconfig.ts +243 -0
  138. package/src/build/project/typegen/tui-dts.ts +120 -0
  139. package/src/build/project/typegen/write.ts +40 -0
  140. package/src/build/project/verify.ts +268 -0
  141. package/src/build/project/watcher.ts +266 -0
  142. package/src/build/runtime/agent.ts +550 -0
  143. package/src/build/runtime/escalations.ts +159 -0
  144. package/src/build/runtime/index.ts +15 -0
  145. package/src/build/runtime/instances.ts +604 -0
  146. package/src/build/runtime/profile.ts +91 -0
  147. package/src/build/runtime/progress.ts +143 -0
  148. package/src/build/runtime/requests.ts +167 -0
  149. package/src/build/runtime/resolve.ts +159 -0
  150. package/src/build/runtime/runtime.ts +233 -0
  151. package/src/build/runtime/sessions/index.ts +5 -0
  152. package/src/build/runtime/sessions/record.ts +145 -0
  153. package/src/build/runtime/sessions/sessions.ts +240 -0
  154. package/src/build/runtime/zeno.ts +120 -0
  155. package/src/index.ts +11 -0
  156. package/src/link/agent-main.ts +395 -0
  157. package/src/link/index.ts +70 -0
  158. package/src/link/services.ts +120 -0
  159. package/src/platform.ts +293 -0
  160. package/src/procs/index.ts +15 -0
  161. package/src/procs/tree.ts +177 -0
  162. package/src/services/cloud.ts +419 -0
  163. package/src/services/control/client.ts +126 -0
  164. package/src/services/control/dispatch.ts +156 -0
  165. package/src/services/control/index.ts +17 -0
  166. package/src/services/control/protocol.ts +179 -0
  167. package/src/services/control/server.ts +241 -0
  168. package/src/services/deployments.ts +81 -0
  169. package/src/services/files.ts +329 -0
  170. package/src/services/index.ts +5 -0
  171. package/src/services/mic/capture.ts +248 -0
  172. package/src/services/mic/config.ts +109 -0
  173. package/src/services/mic/fft.ts +93 -0
  174. package/src/services/mic/index.ts +139 -0
  175. package/src/services/mic/visualizer.ts +46 -0
  176. package/src/services/ollama/catalog.ts +122 -0
  177. package/src/services/ollama/http.ts +124 -0
  178. package/src/services/ollama/index.ts +13 -0
  179. package/src/services/ollama/models.ts +153 -0
  180. package/src/services/ollama/ollama.ts +89 -0
  181. package/src/services/ollama/registry.ts +112 -0
  182. package/src/services/ollama/types.ts +72 -0
  183. package/src/services/prompts.ts +76 -0
  184. package/src/services/registry.ts +220 -0
  185. package/src/services/resources/hardware.ts +125 -0
  186. package/src/services/resources/index.ts +6 -0
  187. package/src/services/resources/reservations.ts +162 -0
  188. package/src/services/resources/resources.ts +156 -0
  189. package/src/services/running/index.ts +1 -0
  190. package/src/services/running/running.ts +200 -0
  191. package/src/services/store/disk.ts +97 -0
  192. package/src/services/store/index.ts +15 -0
  193. package/src/services/store/store.ts +729 -0
  194. package/src/services/store/types.ts +303 -0
  195. package/src/services/test/child.ts +150 -0
  196. package/src/services/test/events.ts +102 -0
  197. package/src/services/test/files.ts +26 -0
  198. package/src/services/test/frames.ts +39 -0
  199. package/src/services/test/index.ts +7 -0
  200. package/src/services/test/runner.ts +122 -0
  201. package/src/services/test/types.ts +26 -0
  202. package/src/services/update/contract.ts +77 -0
  203. package/src/services/update/index.ts +12 -0
  204. package/src/services/update/installer.ts +139 -0
  205. package/src/services/update/updates.ts +137 -0
  206. package/src/utils/fs.ts +148 -0
  207. package/src/utils/tsast.ts +102 -0
@@ -0,0 +1,147 @@
1
+ import { resolve } from "node:path"
2
+ import { pathToFileURL } from "node:url"
3
+ import type { BenchConfig, BenchDefinition, BenchMeasurementDefinition, BenchNormalizedConfig, BenchResolvedWorkspaceDefinition } from "@arcforge/types"
4
+ import { toAxes } from "./axes"
5
+ import { BenchSchema } from "./schema"
6
+ import { defineBench } from "@arcforge/types"
7
+ import { err } from "@arcforge/err"
8
+ import { fsx } from "../../utils/fs"
9
+
10
+ export type DiscoverOpts = { root: string }
11
+
12
+ function fail(message: string): never {
13
+ throw err("BENCH_CONFIG_INVALID", { detail: message })
14
+ }
15
+
16
+ /**
17
+ * Duplicate check keyed by something other than `id`.
18
+ *
19
+ * Matrix axes are keyed by their binding point ("model", "config.engine.
20
+ * temperature"), which is deliberately not an identifier — dotted paths are
21
+ * the escape hatch that keeps the named axis set small.
22
+ */
23
+ function uniqueBy<T>(items: T[], key: (item: T) => string, kind: string): void {
24
+ const seen = new Set<string>()
25
+ for (const item of items) {
26
+ const value = key(item)
27
+ if (!value) fail(`${kind} key is empty`)
28
+ if (seen.has(value)) fail(`duplicate ${kind} ${JSON.stringify(value)}`)
29
+ seen.add(value)
30
+ }
31
+ }
32
+
33
+ function unique(items: Array<{ id: string }>, kind: string): void {
34
+ const ids = new Set<string>()
35
+ for (const item of items) {
36
+ if (!item.id || !/^[a-zA-Z][a-zA-Z0-9._-]*$/.test(item.id)) fail(`${kind} id ${JSON.stringify(item.id)} is invalid`)
37
+ if (ids.has(item.id)) fail(`duplicate ${kind} id ${JSON.stringify(item.id)}`)
38
+ ids.add(item.id)
39
+ }
40
+ }
41
+
42
+ export function normalizeBenchConfig(config: BenchConfig, opts: { identity: { name: string; version: string; description?: string }; hasWorkspace: boolean; measurements?: BenchMeasurementDefinition[] }): BenchNormalizedConfig {
43
+ // `source` accepts a bare path because that is what an author means
44
+ // nine times out of ten; the object form stays for `{ kind: "empty" }`.
45
+ const declared = config.workspace?.source
46
+ const source = typeof declared === "string"
47
+ ? { kind: "directory" as const, path: declared }
48
+ : declared
49
+ ?? (opts.hasWorkspace ? { kind: "directory" as const, path: "./workspace" } : { kind: "empty" as const })
50
+ const workspace: BenchResolvedWorkspaceDefinition = {
51
+ source,
52
+ retain: config.workspace?.retain ?? "failed",
53
+ capture: {
54
+ changes: config.workspace?.capture?.changes ?? true,
55
+ display: config.workspace?.capture?.display ?? "hidden",
56
+ ignore: config.workspace?.capture?.ignore ?? [".git", "node_modules", ".axon"],
57
+ maxBytes: config.workspace?.capture?.maxBytes ?? 10 * 1024 * 1024,
58
+ },
59
+ }
60
+ const normalized: BenchNormalizedConfig = {
61
+ ...config,
62
+ identity: opts.identity,
63
+ axes: toAxes(config.matrix),
64
+ trials: config.trials ?? 1,
65
+ tests: config.tests ?? ["tests/**/*.bench.ts"],
66
+ measurements: opts.measurements ?? config.measurements ?? [],
67
+ dimensions: config.dimensions ?? [],
68
+ artifacts: config.artifacts ?? [],
69
+ workspace,
70
+ }
71
+ if (!Number.isInteger(normalized.trials) || normalized.trials < 1) fail("trials must be a positive integer")
72
+ if (normalized.tests.length === 0) fail("at least one test pattern is required")
73
+ uniqueBy(normalized.axes, axis => axis.key, "matrix axis")
74
+ unique(normalized.measurements, "measurement")
75
+ unique(normalized.dimensions, "dimension")
76
+ unique(normalized.artifacts, "artifact")
77
+ for (const axis of normalized.axes) {
78
+ if (axis.values.length === 0) fail(`matrix axis ${JSON.stringify(axis.key)} has no values`)
79
+ // uniqueBy, not unique: variation ids are DERIVED from the value
80
+ // ("../agents/a" -> "..-agents-a", 0.7 -> "0.7"), so the identifier
81
+ // rule that governs hand-authored ids does not apply. They only have
82
+ // to be distinct within their axis.
83
+ uniqueBy(axis.values, variation => variation.id, `value on axis ${JSON.stringify(axis.key)}`)
84
+ }
85
+ for (const measurement of normalized.measurements) {
86
+ if (!measurement.label?.trim() || !measurement.description?.trim()) fail(`measurement ${JSON.stringify(measurement.id)} needs label and description`)
87
+ if (measurement.aggregate === "auc" && (measurement.value.kind !== "number" || measurement.grain !== "step")) {
88
+ fail(`measurement ${JSON.stringify(measurement.id)} uses auc but is not a step-grained number`)
89
+ }
90
+ if (measurement.objective && measurement.value.kind !== "number" && typeof measurement.objective === "object") {
91
+ fail(`measurement ${JSON.stringify(measurement.id)} has a numeric objective but is not numeric`)
92
+ }
93
+ }
94
+ return normalized
95
+ }
96
+
97
+ function isDefinition(value: unknown): value is BenchDefinition {
98
+ return !!value && typeof value === "object" && (value as { _kind?: unknown })._kind === "bench" && !!(value as { config?: unknown }).config
99
+ }
100
+
101
+ export function Discover(opts: DiscoverOpts) {
102
+ const root = resolve(opts.root)
103
+ return {
104
+ async load(): Promise<BenchNormalizedConfig> {
105
+ const path = resolve(root, "bench.config.ts")
106
+ if (!fsx.exists(path)) throw err("BENCH_NOT_FOUND", { context: { path: root } })
107
+ const globals = globalThis as typeof globalThis & { defineBench?: typeof defineBench }
108
+ const previous = globals.defineBench
109
+ globals.defineBench = defineBench
110
+ let module: { default?: unknown }
111
+ try {
112
+ module = await import(`${pathToFileURL(path).href}?axon=${Date.now()}`) as { default?: unknown }
113
+ } finally {
114
+ if (previous) globals.defineBench = previous
115
+ else delete globals.defineBench
116
+ }
117
+ if (!isDefinition(module.default)) fail("default export must be defineBench({...})")
118
+ const pkg = await fsx.readJson<{ name?: string; version?: string; description?: string }>(resolve(root, "package.json"))
119
+ if (!pkg) throw err("BENCH_PACKAGE_NOT_FOUND", { context: { root } })
120
+ if (!pkg.name?.trim()) throw err("BENCH_PACKAGE_NAME_REQUIRED")
121
+ if (!pkg.version?.trim()) throw err("BENCH_PACKAGE_VERSION_REQUIRED")
122
+ // Measurements come from the `defineBench<Schema>` type argument,
123
+ // not from the config object — the type is the authoring surface,
124
+ // and this is where it becomes a value the manifest can carry.
125
+ return normalizeBenchConfig(module.default.config, {
126
+ // Identity is package.json's, not the config's — name, version
127
+ // and description are what the registry publishes, and stating
128
+ // them twice is how the two come to disagree.
129
+ identity: { name: pkg.name, version: pkg.version, description: pkg.description },
130
+ hasWorkspace: fsx.exists(resolve(root, "workspace")),
131
+ measurements: await BenchSchema({ root }).load(),
132
+ })
133
+ },
134
+
135
+ async list(globPattern = "**/bench.config.ts"): Promise<Array<{ path: string; config: BenchNormalizedConfig }>> {
136
+ const found: Array<{ path: string; config: BenchNormalizedConfig }> = []
137
+ const glob = new Bun.Glob(globPattern)
138
+ for await (const path of glob.scan({ cwd: root, onlyFiles: true, dot: false })) {
139
+ const benchRoot = resolve(root, path, "..")
140
+ found.push({ path: benchRoot, config: await Discover({ root: benchRoot }).load() })
141
+ }
142
+ return found.sort((a, b) => a.path.localeCompare(b.path))
143
+ },
144
+ }
145
+ }
146
+
147
+ export type DiscoverT = ReturnType<typeof Discover>
@@ -0,0 +1,7 @@
1
+ // bench — native Bun experiments over agents, models, cognets, and values.
2
+ //
3
+ // A bench is a project kind: finding, opening, scaffolding, preparing and
4
+ // publishing one are Project()'s, like every other kind. What lives here is
5
+ // the benchmarking itself — the verbs Project hangs off `project.bench`.
6
+
7
+ export { Bench, type BenchT, type BenchExtras } from "./bench"
@@ -0,0 +1,47 @@
1
+ import { appendFile, mkdir } from "node:fs/promises"
2
+ import { join } from "node:path"
3
+ import type { BenchEvent, BenchEventContext, BenchEventMap } from "@arcforge/types"
4
+ import { err } from "@arcforge/err"
5
+
6
+ export type BenchLogOpts = { root: string }
7
+
8
+ export function BenchLog(opts: BenchLogOpts) {
9
+ const sequences = new Map<string, number>()
10
+ const runDir = (runId: string) => join(opts.root, ".bench", "runs", runId)
11
+ return {
12
+ runDir,
13
+ async emit<K extends keyof BenchEventMap>(context: BenchEventContext, type: K, data: BenchEventMap[K]): Promise<BenchEvent> {
14
+ const seq = sequences.get(context.benchRunId) ?? 0
15
+ sequences.set(context.benchRunId, seq + 1)
16
+ const event = {
17
+ id: Bun.randomUUIDv7(),
18
+ type,
19
+ time: { ms: Date.now(), seq },
20
+ context,
21
+ data,
22
+ } as BenchEvent
23
+ const dir = runDir(context.benchRunId)
24
+ await mkdir(dir, { recursive: true })
25
+ await appendFile(join(dir, "events.jsonl"), JSON.stringify(event) + "\n")
26
+ return event
27
+ },
28
+ async read(runId: string): Promise<BenchEvent[]> {
29
+ const file = Bun.file(join(runDir(runId), "events.jsonl"))
30
+ if (!await file.exists()) throw err("BENCH_RUN_NOT_FOUND", { context: { runId } })
31
+ const text = await file.text()
32
+ return text.split("\n").filter(Boolean).map(line => JSON.parse(line) as BenchEvent)
33
+ },
34
+ async writeResult(runId: string, result: unknown): Promise<void> {
35
+ const dir = runDir(runId)
36
+ await mkdir(dir, { recursive: true })
37
+ await Bun.write(join(dir, "result.json"), JSON.stringify(result, null, 2) + "\n")
38
+ },
39
+ async writeManifest(runId: string, manifest: unknown): Promise<void> {
40
+ const dir = runDir(runId)
41
+ await mkdir(dir, { recursive: true })
42
+ await Bun.write(join(dir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n")
43
+ },
44
+ }
45
+ }
46
+
47
+ export type BenchLogT = ReturnType<typeof BenchLog>
@@ -0,0 +1,23 @@
1
+ import type { BenchAxis, BenchCoordinate } from "@arcforge/types"
2
+
3
+ function coordinateId(axes: BenchCoordinate["axes"]): string {
4
+ return axes.length === 0 ? "default" : axes.map(item => `${item.key}=${item.valueId}`).join(";")
5
+ }
6
+
7
+ /** Pure Cartesian expansion. Single values remain present as pinned controls. */
8
+ export function Matrix() {
9
+ return {
10
+ build(axes: BenchAxis[]): BenchCoordinate[] {
11
+ let coordinates: BenchCoordinate["axes"][] = [[]]
12
+ for (const axis of axes) {
13
+ coordinates = coordinates.flatMap(existing => axis.values.map(value => [
14
+ ...existing,
15
+ { key: axis.key, valueId: value.id },
16
+ ]))
17
+ }
18
+ return coordinates.map(selected => ({ id: coordinateId(selected), axes: selected }))
19
+ },
20
+ }
21
+ }
22
+
23
+ export type MatrixT = ReturnType<typeof Matrix>
@@ -0,0 +1,440 @@
1
+ import { mkdir } from "node:fs/promises"
2
+ import { join } from "node:path"
3
+ import { isAbsolute, resolve } from "node:path"
4
+ import { Axon as createAxon } from "@arcforge/core"
5
+ import { err } from "@arcforge/err"
6
+ import { Blueprint } from "../blueprint/blueprint"
7
+ import {
8
+ materializeWorkspace,
9
+ WorkspaceHandle,
10
+ removeWorkspace,
11
+ snapshotWorkspace,
12
+ workspaceChanges,
13
+ type WorkspaceSnapshot,
14
+ } from "./workspace"
15
+ import type {
16
+ ProviderEntry,
17
+ BenchArtifactRef,
18
+ BenchDimensionDefinition,
19
+ BenchExecutionContext,
20
+ BenchHandle,
21
+ BenchMeasurementDefinition,
22
+ BenchObservationPayload,
23
+ BenchAxisKey,
24
+ BenchResolvedAxis,
25
+ BenchResolvedWorkspaceDefinition,
26
+ BenchTestStatus,
27
+ BenchWorkspaceInstance,
28
+ BenchWorkspaceTemplate,
29
+ } from "@arcforge/types"
30
+
31
+ type TestContext = { testId?: string; attempt?: number; resources?: Record<string, unknown> }
32
+ type TestOutcome = { status: "passed" } | { status: "failed"; error: unknown }
33
+ type ActiveWorkspace = { instance: BenchWorkspaceInstance; before: WorkspaceSnapshot }
34
+ type RuntimeContext = {
35
+ runId: string
36
+ cellId: string
37
+ trial: number
38
+ axes: BenchResolvedAxis[]
39
+ axisValues: Record<string, unknown>
40
+ measurements: BenchMeasurementDefinition[]
41
+ dimensions: BenchDimensionDefinition[]
42
+ artifacts: Array<{ id: string; role?: BenchArtifactRef["role"]; mediaTypes?: string[]; schema?: string }>
43
+ artifactDir: string
44
+ benchRoot: string
45
+ workspace: {
46
+ definition: BenchResolvedWorkspaceDefinition
47
+ template: BenchWorkspaceTemplate
48
+ sourcePath?: string
49
+ root: string
50
+ }
51
+ /** Absolute path to bench.config.ts — setup() is a function, so it cannot cross the IPC boundary. */
52
+ configPath?: string
53
+ /** Prepared agent roots, keyed by the ref as written in the matrix. */
54
+ agentRoots?: Record<string, string>
55
+ }
56
+
57
+ const encoded = process.env.AXON_BENCH_CONTEXT
58
+ if (!encoded) throw err("BENCH_CONTEXT_MISSING")
59
+ const runtime = JSON.parse(encoded) as RuntimeContext
60
+
61
+ function active(): TestContext {
62
+ const bridge = (globalThis as any).__axon_test_context__ as { current?: () => TestContext | undefined } | undefined
63
+ const current = bridge?.current?.()
64
+ if (!current?.testId) throw err("BENCH_NO_ACTIVE_CASE")
65
+ return current
66
+ }
67
+
68
+ function execution(current = active()): BenchExecutionContext {
69
+ return { runId: runtime.runId, cellId: runtime.cellId, testId: current.testId, trial: runtime.trial, attempt: current.attempt ?? 0 }
70
+ }
71
+
72
+ function workspaceResource(current = active()): ActiveWorkspace {
73
+ const workspace = current.resources?.workspace as ActiveWorkspace | undefined
74
+ if (!workspace) throw err("BENCH_WORKSPACE_UNAVAILABLE")
75
+ return workspace
76
+ }
77
+
78
+ function pathSegment(value: string | number): string {
79
+ return new Bun.CryptoHasher("sha256").update(String(value)).digest("hex").slice(0, 16)
80
+ }
81
+
82
+ function send(type: string, data: unknown, current = active()): void {
83
+ const ipc = process as typeof process & { send?: (message: unknown) => void }
84
+ ipc.send?.({ channel: "axon:bench", frame: { type, context: execution(current), data } })
85
+ }
86
+
87
+ /** The variation selected on one axis for this cell, or undefined if the axis is not in the matrix. */
88
+ function selected(key: BenchAxisKey): unknown {
89
+ return runtime.axisValues[key]
90
+ }
91
+
92
+ function resourceUsage(kernelLog: Array<{ type: string; data: any }>, durationMs: number) {
93
+ const completed = kernelLog.filter(event => event.type === "kernel:engine:complete")
94
+ const priced = completed.every(event => event.data.meta.cost?.total !== undefined)
95
+ return {
96
+ durationMs,
97
+ tokens: {
98
+ input: completed.reduce((sum, event) => sum + (event.data.meta.tokens?.in ?? 0), 0),
99
+ output: completed.reduce((sum, event) => sum + (event.data.meta.tokens?.out ?? 0), 0),
100
+ },
101
+ ...(priced ? { costUsd: completed.reduce((sum, event) => sum + event.data.meta.cost.total, 0) } : {}),
102
+ engineCalls: completed.length,
103
+ // Every reach for the world goes through kernel.run() — the capsule is
104
+ // the only door. Counting completions rather than starts means a run
105
+ // the kernel refused is an error, not a tool call.
106
+ toolCalls: kernelLog.filter(event => event.type === "kernel:run:complete").length,
107
+ errors: kernelLog.filter(event => event.type === "kernel:engine:failed" || event.type === "kernel:run:failed").length,
108
+ }
109
+ }
110
+
111
+ /**
112
+ * Invoke the config's setup() for this iteration.
113
+ *
114
+ * Loaded from disk here rather than passed in: the runtime context crosses to
115
+ * this process as JSON, and a function does not survive that. Importing the
116
+ * config in the preload is the same module the runner already validated, so
117
+ * there is no second source of truth — only a second reader.
118
+ */
119
+ let preparingWorkspace: string | null = null
120
+ let setupHook: ((workspace: string) => Promise<void> | void) | null | undefined
121
+ async function runSetup(workspace: string): Promise<void> {
122
+ if (setupHook === undefined) {
123
+ if (!runtime.configPath) {
124
+ setupHook = null
125
+ } else {
126
+ // bench.config.ts calls defineBench at module scope, and in this
127
+ // process that global does not exist — importing without it throws
128
+ // a ReferenceError that surfaces as an unexplained trial failure.
129
+ // Install it for the duration of the import only.
130
+ const globals = globalThis as typeof globalThis & { defineBench?: unknown }
131
+ const previous = globals.defineBench
132
+ globals.defineBench = (config: unknown) => ({ _kind: "bench", config })
133
+ try {
134
+ const loaded = (await import(runtime.configPath)) as { default?: { config?: { setup?: () => Promise<void> | void } } }
135
+ const declared = loaded.default?.config?.setup
136
+ setupHook = declared ? async () => { await declared() } : null
137
+ } finally {
138
+ if (previous) globals.defineBench = previous
139
+ else delete globals.defineBench
140
+ }
141
+ }
142
+ }
143
+ if (!setupHook) return
144
+
145
+ preparingWorkspace = workspace
146
+ try {
147
+ await setupHook(workspace)
148
+ } finally {
149
+ preparingWorkspace = null
150
+ }
151
+ }
152
+
153
+ async function bootSubject() {
154
+ const current = active()
155
+ // Each named axis is a binding point in the assembled blueprint. `agent`
156
+ // decides WHICH blueprint is loaded, so it resolves first; `cognet` edits
157
+ // the agent's source before load; the rest merge onto the loaded config.
158
+ // A variation is a path or a registry ref. Registry refs were fetched into
159
+ // .bench/agents/ during prepare, so resolution here is a lookup rather than
160
+ // a second fetch — the run must not reach the network.
161
+ const agent = selected("agent") as string | undefined
162
+ const agentRoot = agent
163
+ ? (runtime.agentRoots?.[agent] ?? (isAbsolute(agent) ? agent : resolve(runtime.benchRoot, agent)))
164
+ : runtime.benchRoot
165
+ if (!await Bun.file(join(agentRoot, "axon.config.ts")).exists()) {
166
+ throw err("BENCH_AGENT_UNRESOLVED", { detail: `${agent ?? agentRoot} is not a prepared local agent`, context: { agentRoot, ref: agent } })
167
+ }
168
+ const blueprintHandle = Blueprint({ root: agentRoot })
169
+ // A bench's `cognet` axis names a path on disk, not an installed package:
170
+ // the point is running an agent against a brain it does not declare.
171
+ const cognet = selected("cognet") as string | undefined
172
+ if (cognet) {
173
+ await blueprintHandle.cognet.compile({
174
+ kind: "source",
175
+ dir: isAbsolute(cognet) ? cognet : resolve(runtime.benchRoot, cognet),
176
+ })
177
+ }
178
+ const loaded = await blueprintHandle.load({ compile: !cognet })
179
+ /**
180
+ * The `model` axis, in either form the inference model allows.
181
+ *
182
+ * A STRING is a cortex pin (`"codex:gpt-5.6-terra"`) and lands on
183
+ * `config.model`. A PROVIDER ENTRY is a source (`Ollama({ url })`) and
184
+ * lands on `config.providers`, REPLACING what the agent declared for the
185
+ * duration of this cell — a bench varying inference has to replace rather
186
+ * than append, or every cell resolves against the first provider that can
187
+ * answer and the axis does nothing.
188
+ *
189
+ * `profileProviders` is cleared alongside for the same reason: profile
190
+ * entries rank ahead of an agent's, so leaving them would let the runner's
191
+ * own machine decide what every cell actually ran on.
192
+ *
193
+ * This previously wrote `config.engine`, a field nothing reads — so every
194
+ * cell of a model matrix silently ran on the SAME inference while the
195
+ * manifest recorded them as different.
196
+ */
197
+ const model = selected("model") as string | ProviderEntry | undefined
198
+ const env = selected("env") as Record<string, string> | undefined
199
+ const inference = model === undefined
200
+ ? {}
201
+ : typeof model === "string"
202
+ ? { config: { ...loaded.blueprint.config, model } }
203
+ : { config: { ...loaded.blueprint.config, providers: [model] }, profileProviders: [] }
204
+ const blueprint = {
205
+ ...loaded.blueprint,
206
+ ...inference,
207
+ ...(env ? { env: { ...loaded.blueprint.env, ...env } } : {}),
208
+ paths: {
209
+ root: workspaceResource(current).instance.path,
210
+ data: join(workspaceResource(current).instance.path, ".axon", "data"),
211
+ },
212
+ }
213
+ const started = performance.now()
214
+ const instance = await createAxon({ blueprint })
215
+ const session = {
216
+ agentId: instance.blueprint.agent.name,
217
+ sessionId: instance.blueprint.session.id,
218
+ role: "subject" as const,
219
+ }
220
+ send("bench:session:attach", { session }, current)
221
+ let stopped = false
222
+ const stop = async () => {
223
+ if (stopped) return
224
+ stopped = true
225
+ await instance.shutdown()
226
+ send("bench:session:usage", { session, usage: resourceUsage(instance.session.kernelLog as any[], performance.now() - started) }, current)
227
+ send("bench:session:detach", { session }, current)
228
+ }
229
+ return Object.assign(instance, { stop })
230
+ }
231
+
232
+ function observation(measurementId: string, raw: number | boolean | string, options?: Parameters<BenchHandle["observe"]>[2]): BenchObservationPayload {
233
+ const definition = runtime.measurements.find(item => item.id === measurementId)
234
+ if (!definition) throw err("BENCH_MEASUREMENT_UNKNOWN", { context: { measurementId } })
235
+ if (definition.value.kind === "number" && typeof raw !== "number") throw err("BENCH_MEASUREMENT_TYPE", { detail: `${measurementId} expects number`, context: { measurementId } })
236
+ if (definition.value.kind === "boolean" && typeof raw !== "boolean") throw err("BENCH_MEASUREMENT_TYPE", { detail: `${measurementId} expects boolean`, context: { measurementId } })
237
+ if ((definition.value.kind === "category" || definition.value.kind === "text") && typeof raw !== "string") throw err("BENCH_MEASUREMENT_TYPE", { detail: `${measurementId} expects string`, context: { measurementId } })
238
+ if (definition.value.kind === "number" && definition.value.domain) {
239
+ if (definition.value.domain.min !== undefined && (raw as number) < definition.value.domain.min) throw err("BENCH_MEASUREMENT_DOMAIN", { detail: `${measurementId} is below minimum`, context: { measurementId } })
240
+ if (definition.value.domain.max !== undefined && (raw as number) > definition.value.domain.max) throw err("BENCH_MEASUREMENT_DOMAIN", { detail: `${measurementId} is above maximum`, context: { measurementId } })
241
+ }
242
+ if (definition.value.kind === "category" && definition.value.values && !definition.value.open && !definition.value.values.includes(raw as string)) {
243
+ throw err("BENCH_MEASUREMENT_DOMAIN", { detail: `${measurementId} has unknown category ${JSON.stringify(raw)}`, context: { measurementId } })
244
+ }
245
+ for (const [id, value] of Object.entries(options?.dimensions ?? {})) {
246
+ const dimension = runtime.dimensions.find(item => item.id === id)
247
+ if (!dimension) throw err("BENCH_DIMENSION_UNKNOWN", { context: { dimensionId: id } })
248
+ if (dimension.value.kind === "category" && !dimension.value.open && !dimension.value.values.includes(String(value))) {
249
+ throw err("BENCH_DIMENSION_DOMAIN", { detail: `${id} has unknown category ${JSON.stringify(value)}`, context: { dimensionId: id } })
250
+ }
251
+ }
252
+ const value = definition.value.kind === "number"
253
+ ? { kind: "number" as const, value: raw as number }
254
+ : definition.value.kind === "boolean"
255
+ ? { kind: "boolean" as const, value: raw as boolean }
256
+ : definition.value.kind === "category"
257
+ ? { kind: "category" as const, value: raw as string }
258
+ : { kind: "text" as const, value: raw as string }
259
+ return { measurementId, value, dimensions: options?.dimensions, index: options?.at, producer: { kind: "benchmark", sourceHash: "runtime" } }
260
+ }
261
+
262
+ const bench: BenchHandle = {
263
+ get workspace() {
264
+ // During setup() the workspace exists on disk but its resource is not
265
+ // registered yet (the baseline snapshot has not been taken, because
266
+ // setup is what it must be taken AFTER). Fall back to a handle over the
267
+ // path being prepared so a setup hook can address the world it is
268
+ // setting up — with an empty baseline, since nothing has changed yet.
269
+ if (preparingWorkspace) {
270
+ return WorkspaceHandle({
271
+ path: preparingWorkspace,
272
+ before: new Map(),
273
+ ignore: runtime.workspace.definition.capture.ignore,
274
+ })
275
+ }
276
+ const resource = workspaceResource()
277
+ return WorkspaceHandle({
278
+ path: resource.instance.path,
279
+ before: resource.before,
280
+ ignore: runtime.workspace.definition.capture.ignore,
281
+ })
282
+ },
283
+ // BenchHandle declares axis<T>(key): T so generated per-bench declarations
284
+ // can narrow the return. The runtime map is untyped by nature, so the cast
285
+ // to the caller's T happens here, once, at that seam — after the key has
286
+ // been validated against the manifest.
287
+ axis<T = unknown>(key: string): T {
288
+ if (!(key in runtime.axisValues)) throw err("BENCH_AXIS_UNKNOWN", { context: { axis: key } })
289
+ return runtime.axisValues[key] as T
290
+ },
291
+ observe(id, value, options) {
292
+ send("bench:observation", observation(id, value, options))
293
+ },
294
+ async attach(id, content, options = {}) {
295
+ const definition = runtime.artifacts.find(item => item.id === id)
296
+ if (!definition) throw err("BENCH_ARTIFACT_UNKNOWN", { context: { artifactId: id } })
297
+ const mediaType = options.mediaType ?? definition.mediaTypes?.[0] ?? "application/json"
298
+ if (definition.mediaTypes?.length && !definition.mediaTypes.includes(mediaType)) {
299
+ throw err("BENCH_ARTIFACT_MEDIA_TYPE", { detail: `${id} does not allow ${mediaType}`, context: { artifactId: id, mediaType } })
300
+ }
301
+ const bytes = content instanceof Uint8Array
302
+ ? content
303
+ : typeof content === "string"
304
+ ? new TextEncoder().encode(content)
305
+ : new TextEncoder().encode(JSON.stringify(content))
306
+ const digest = new Bun.CryptoHasher("sha256").update(bytes).digest("hex")
307
+ const artifactId = Bun.randomUUIDv7()
308
+ await mkdir(runtime.artifactDir, { recursive: true })
309
+ const path = join(runtime.artifactDir, digest)
310
+ await Bun.write(path, bytes)
311
+ const artifact: BenchArtifactRef = {
312
+ id: artifactId,
313
+ definitionId: id,
314
+ role: options.role ?? definition.role ?? "evidence",
315
+ mediaType,
316
+ schema: options.schema ?? definition.schema,
317
+ hash: `sha256:${digest}`,
318
+ bytes: bytes.byteLength,
319
+ ref: `artifacts/${digest}`,
320
+ context: execution(),
321
+ createdAt: new Date().toISOString(),
322
+ }
323
+ send("bench:artifact", { artifact })
324
+ },
325
+ }
326
+
327
+ async function persistWorkspaceArtifact(current: TestContext, result: unknown): Promise<BenchArtifactRef> {
328
+ const content = new TextEncoder().encode(JSON.stringify(result))
329
+ const digest = new Bun.CryptoHasher("sha256").update(content).digest("hex")
330
+ await mkdir(runtime.artifactDir, { recursive: true })
331
+ await Bun.write(join(runtime.artifactDir, digest), content)
332
+ return {
333
+ id: Bun.randomUUIDv7(),
334
+ definitionId: "axon.workspace.changes",
335
+ role: "output",
336
+ owner: "framework",
337
+ display: runtime.workspace.definition.capture.display,
338
+ mediaType: "application/vnd.axon.workspace-changes+json",
339
+ hash: `sha256:${digest}`,
340
+ bytes: content.byteLength,
341
+ ref: `artifacts/${digest}`,
342
+ context: execution(current),
343
+ createdAt: new Date().toISOString(),
344
+ }
345
+ }
346
+
347
+ const lifecycle = (globalThis as any).__axon_test_context__ as {
348
+ beforeCase(handler: (context: TestContext) => void | Promise<void>): () => void
349
+ afterCase(handler: (context: TestContext, outcome: TestOutcome) => void | Promise<void>): () => void
350
+ }
351
+
352
+ lifecycle.beforeCase(async current => {
353
+ const context = execution(current)
354
+ const destination = join(
355
+ runtime.workspace.root,
356
+ pathSegment(runtime.runId),
357
+ pathSegment(runtime.cellId),
358
+ String(runtime.trial),
359
+ pathSegment(current.testId!),
360
+ String(current.attempt ?? 0),
361
+ )
362
+ try {
363
+ await materializeWorkspace(runtime.workspace.sourcePath, destination, runtime.workspace.definition.capture.ignore)
364
+
365
+ // setup() runs BEFORE the baseline snapshot, not after. Installing
366
+ // dependencies or seeding data is preparing the world, not the agent
367
+ // acting on it — snapshotting first would attribute every file setup
368
+ // created to the subject and make workspace.changed() meaningless.
369
+ await runSetup(destination)
370
+
371
+ const before = await snapshotWorkspace(destination, {
372
+ ignore: runtime.workspace.definition.capture.ignore,
373
+ persistDir: runtime.artifactDir,
374
+ maxBytes: runtime.workspace.definition.capture.maxBytes,
375
+ })
376
+ const instance: BenchWorkspaceInstance = {
377
+ id: Bun.randomUUIDv7(),
378
+ path: destination,
379
+ context,
380
+ template: runtime.workspace.template,
381
+ }
382
+ current.resources ??= {}
383
+ current.resources.workspace = { instance, before: before.files } satisfies ActiveWorkspace
384
+ } catch (error) {
385
+ await removeWorkspace(destination)
386
+ throw error
387
+ }
388
+ const instance = workspaceResource(current).instance
389
+ send("bench:workspace:materialized", {
390
+ workspaceId: instance.id,
391
+ templateHash: instance.template.hash,
392
+ }, current)
393
+ })
394
+
395
+ lifecycle.afterCase(async (current, outcome) => {
396
+ const activeWorkspace = current.resources?.workspace as ActiveWorkspace | undefined
397
+ if (!activeWorkspace) return
398
+ const definition = runtime.workspace.definition
399
+ const after = await snapshotWorkspace(activeWorkspace.instance.path, {
400
+ ignore: definition.capture.ignore,
401
+ persistDir: runtime.artifactDir,
402
+ maxBytes: definition.capture.maxBytes,
403
+ })
404
+ const changes = definition.capture.changes ? workspaceChanges(activeWorkspace.before, after.files) : []
405
+ const status: BenchTestStatus = outcome.status === "passed" ? "passed" : "failed"
406
+ const retain = definition.retain === "always" || (definition.retain === "failed" && status === "failed")
407
+ const result = {
408
+ workspaceId: activeWorkspace.instance.id,
409
+ templateHash: activeWorkspace.instance.template.hash,
410
+ retained: retain,
411
+ outcome: status,
412
+ changes,
413
+ summary: {
414
+ added: changes.filter(change => change.kind === "added").length,
415
+ modified: changes.filter(change => change.kind === "modified").length,
416
+ deleted: changes.filter(change => change.kind === "deleted").length,
417
+ bytesChanged: changes.reduce((sum, change) => sum + Math.max(change.before?.bytes ?? 0, change.after?.bytes ?? 0), 0),
418
+ },
419
+ }
420
+ send("bench:workspace:captured", { result }, current)
421
+ send("bench:artifact", { artifact: await persistWorkspaceArtifact(current, result) }, current)
422
+ if (retain) {
423
+ send("bench:workspace:retained", {
424
+ workspaceId: activeWorkspace.instance.id,
425
+ reason: definition.retain === "always" ? "always" : "failed",
426
+ }, current)
427
+ } else {
428
+ await removeWorkspace(activeWorkspace.instance.path)
429
+ send("bench:workspace:cleaned", { workspaceId: activeWorkspace.instance.id }, current)
430
+ }
431
+ })
432
+
433
+ ;(globalThis as any).bench = bench
434
+ ;(globalThis as any).Axon = bootSubject
435
+
436
+ // `observe` is bare rather than only bench.observe: recording a measurement is
437
+ // the single thing every scenario does, and it reads alongside expect/it as a
438
+ // peer verb rather than a method on a harness object. bench.* keeps the rarer
439
+ // surface (axis, attach, workspace).
440
+ ;(globalThis as any).observe = bench.observe.bind(bench)