@arcforge/platform 2.0.166

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/README.md +31 -0
  2. package/package.json +69 -0
  3. package/src/bin/supervisor.ts +102 -0
  4. package/src/bin/test-api.ts +30 -0
  5. package/src/bin/test-preload.ts +308 -0
  6. package/src/bin/update-helper.ts +32 -0
  7. package/src/build/bench/CLAUDE.md +106 -0
  8. package/src/build/bench/SPEC.md +209 -0
  9. package/src/build/bench/agents.ts +105 -0
  10. package/src/build/bench/axes.ts +84 -0
  11. package/src/build/bench/bench.ts +96 -0
  12. package/src/build/bench/discover.ts +147 -0
  13. package/src/build/bench/index.ts +7 -0
  14. package/src/build/bench/log.ts +47 -0
  15. package/src/build/bench/matrix.ts +23 -0
  16. package/src/build/bench/preload.ts +440 -0
  17. package/src/build/bench/prepare.ts +61 -0
  18. package/src/build/bench/projection.ts +112 -0
  19. package/src/build/bench/resolve.ts +54 -0
  20. package/src/build/bench/runner.ts +309 -0
  21. package/src/build/bench/schema/extract.ts +148 -0
  22. package/src/build/bench/schema/index.ts +2 -0
  23. package/src/build/bench/schema/schema.ts +95 -0
  24. package/src/build/bench/schema/worker.ts +30 -0
  25. package/src/build/bench/typegen.ts +119 -0
  26. package/src/build/bench/workspace/files.ts +139 -0
  27. package/src/build/bench/workspace/handle.ts +133 -0
  28. package/src/build/bench/workspace/index.ts +11 -0
  29. package/src/build/bench/workspace/workspace.ts +14 -0
  30. package/src/build/blueprint/blueprint.ts +398 -0
  31. package/src/build/blueprint/cognet/abi.ts +207 -0
  32. package/src/build/blueprint/cognet/bundle-worker.ts +93 -0
  33. package/src/build/blueprint/cognet/bundle.ts +664 -0
  34. package/src/build/blueprint/cognet/cognet.ts +249 -0
  35. package/src/build/blueprint/cognet/index.ts +10 -0
  36. package/src/build/blueprint/cognet/manifest.ts +38 -0
  37. package/src/build/blueprint/cognet/resolve.ts +113 -0
  38. package/src/build/blueprint/collisions.ts +51 -0
  39. package/src/build/blueprint/index.ts +29 -0
  40. package/src/build/blueprint/modules/entries.ts +117 -0
  41. package/src/build/blueprint/modules/index.ts +300 -0
  42. package/src/build/blueprint/modules/meta.ts +174 -0
  43. package/src/build/blueprint/scan/boot.ts +34 -0
  44. package/src/build/blueprint/scan/cognetImports.ts +56 -0
  45. package/src/build/blueprint/scan/components.ts +45 -0
  46. package/src/build/blueprint/scan/config.ts +234 -0
  47. package/src/build/blueprint/scan/configImports.ts +232 -0
  48. package/src/build/blueprint/scan/declare-worker.ts +61 -0
  49. package/src/build/blueprint/scan/declare.ts +393 -0
  50. package/src/build/blueprint/scan/h3-globals.ts +137 -0
  51. package/src/build/blueprint/scan/knowledge.ts +101 -0
  52. package/src/build/blueprint/scan/middleware.ts +110 -0
  53. package/src/build/blueprint/scan/moduleImports.ts +79 -0
  54. package/src/build/blueprint/scan/plugins.ts +93 -0
  55. package/src/build/blueprint/scan/promptPackages.ts +45 -0
  56. package/src/build/blueprint/scan/prompts.ts +164 -0
  57. package/src/build/blueprint/scan/published.ts +131 -0
  58. package/src/build/blueprint/scan/resolve.ts +398 -0
  59. package/src/build/blueprint/scan/routes.ts +109 -0
  60. package/src/build/blueprint/scan/scripts.ts +119 -0
  61. package/src/build/blueprint/scan/tool-bundle-worker.ts +69 -0
  62. package/src/build/blueprint/scan/tools.ts +610 -0
  63. package/src/build/blueprint/types.ts +28 -0
  64. package/src/build/extensions/config.ts +261 -0
  65. package/src/build/extensions/edit.ts +684 -0
  66. package/src/build/extensions/extensions.ts +731 -0
  67. package/src/build/extensions/index.ts +10 -0
  68. package/src/build/extensions/load.ts +350 -0
  69. package/src/build/extensions/registrations.ts +189 -0
  70. package/src/build/extensions/store.ts +200 -0
  71. package/src/build/frame/frame.ts +145 -0
  72. package/src/build/frame/index.ts +5 -0
  73. package/src/build/frame/migrate.ts +246 -0
  74. package/src/build/project/bundle/agent.ts +382 -0
  75. package/src/build/project/bundle/artifacts.ts +88 -0
  76. package/src/build/project/bundle/assets.ts +385 -0
  77. package/src/build/project/bundle/bundle.ts +160 -0
  78. package/src/build/project/bundle/imports.ts +102 -0
  79. package/src/build/project/bundle/index.ts +16 -0
  80. package/src/build/project/bundle/module.ts +130 -0
  81. package/src/build/project/bundle/source.ts +136 -0
  82. package/src/build/project/bundle/stage.ts +97 -0
  83. package/src/build/project/bundle/types.ts +111 -0
  84. package/src/build/project/cognet.ts +156 -0
  85. package/src/build/project/create/agent.ts +52 -0
  86. package/src/build/project/create/bench.ts +81 -0
  87. package/src/build/project/create/cognet.ts +102 -0
  88. package/src/build/project/create/extension.ts +63 -0
  89. package/src/build/project/create/module.ts +43 -0
  90. package/src/build/project/create/profile.ts +102 -0
  91. package/src/build/project/create/prompt.ts +68 -0
  92. package/src/build/project/create/templates.ts +88 -0
  93. package/src/build/project/deploy.ts +96 -0
  94. package/src/build/project/index.ts +40 -0
  95. package/src/build/project/installer.ts +616 -0
  96. package/src/build/project/kinds.ts +374 -0
  97. package/src/build/project/links.ts +33 -0
  98. package/src/build/project/manifest/bunfig.ts +139 -0
  99. package/src/build/project/manifest/config.ts +246 -0
  100. package/src/build/project/manifest/env.ts +184 -0
  101. package/src/build/project/manifest/index.ts +9 -0
  102. package/src/build/project/manifest/manifest.ts +49 -0
  103. package/src/build/project/manifest/model.ts +124 -0
  104. package/src/build/project/manifest/package.ts +386 -0
  105. package/src/build/project/manifest/text.ts +19 -0
  106. package/src/build/project/models/fetch.ts +123 -0
  107. package/src/build/project/models/index.ts +8 -0
  108. package/src/build/project/models/models.ts +111 -0
  109. package/src/build/project/models/specifier.ts +106 -0
  110. package/src/build/project/models/store.ts +202 -0
  111. package/src/build/project/modules/hash.ts +31 -0
  112. package/src/build/project/modules/index.ts +5 -0
  113. package/src/build/project/modules/lock.ts +78 -0
  114. package/src/build/project/modules/modules.ts +155 -0
  115. package/src/build/project/prepare.ts +709 -0
  116. package/src/build/project/project.ts +253 -0
  117. package/src/build/project/projects.ts +164 -0
  118. package/src/build/project/propagation.ts +120 -0
  119. package/src/build/project/publish/verify.ts +312 -0
  120. package/src/build/project/publish.ts +155 -0
  121. package/src/build/project/reconcile.ts +93 -0
  122. package/src/build/project/specifier.ts +42 -0
  123. package/src/build/project/tree.ts +615 -0
  124. package/src/build/project/treecache.ts +356 -0
  125. package/src/build/project/typegen/axon-dts.ts +362 -0
  126. package/src/build/project/typegen/cognet-dts.ts +91 -0
  127. package/src/build/project/typegen/components.ts +45 -0
  128. package/src/build/project/typegen/env.ts +59 -0
  129. package/src/build/project/typegen/generators.ts +108 -0
  130. package/src/build/project/typegen/hooks.ts +71 -0
  131. package/src/build/project/typegen/index.ts +137 -0
  132. package/src/build/project/typegen/prompt-dts.ts +40 -0
  133. package/src/build/project/typegen/prompts.ts +36 -0
  134. package/src/build/project/typegen/registry-dts.ts +109 -0
  135. package/src/build/project/typegen/scripts.ts +37 -0
  136. package/src/build/project/typegen/tools.ts +31 -0
  137. package/src/build/project/typegen/tsconfig.ts +243 -0
  138. package/src/build/project/typegen/tui-dts.ts +120 -0
  139. package/src/build/project/typegen/write.ts +40 -0
  140. package/src/build/project/verify.ts +268 -0
  141. package/src/build/project/watcher.ts +266 -0
  142. package/src/build/runtime/agent.ts +550 -0
  143. package/src/build/runtime/escalations.ts +159 -0
  144. package/src/build/runtime/index.ts +15 -0
  145. package/src/build/runtime/instances.ts +604 -0
  146. package/src/build/runtime/profile.ts +91 -0
  147. package/src/build/runtime/progress.ts +143 -0
  148. package/src/build/runtime/requests.ts +167 -0
  149. package/src/build/runtime/resolve.ts +159 -0
  150. package/src/build/runtime/runtime.ts +233 -0
  151. package/src/build/runtime/sessions/index.ts +5 -0
  152. package/src/build/runtime/sessions/record.ts +145 -0
  153. package/src/build/runtime/sessions/sessions.ts +240 -0
  154. package/src/build/runtime/zeno.ts +120 -0
  155. package/src/index.ts +11 -0
  156. package/src/link/agent-main.ts +395 -0
  157. package/src/link/index.ts +70 -0
  158. package/src/link/services.ts +120 -0
  159. package/src/platform.ts +293 -0
  160. package/src/procs/index.ts +15 -0
  161. package/src/procs/tree.ts +177 -0
  162. package/src/services/cloud.ts +419 -0
  163. package/src/services/control/client.ts +126 -0
  164. package/src/services/control/dispatch.ts +156 -0
  165. package/src/services/control/index.ts +17 -0
  166. package/src/services/control/protocol.ts +179 -0
  167. package/src/services/control/server.ts +241 -0
  168. package/src/services/deployments.ts +81 -0
  169. package/src/services/files.ts +329 -0
  170. package/src/services/index.ts +5 -0
  171. package/src/services/mic/capture.ts +248 -0
  172. package/src/services/mic/config.ts +109 -0
  173. package/src/services/mic/fft.ts +93 -0
  174. package/src/services/mic/index.ts +139 -0
  175. package/src/services/mic/visualizer.ts +46 -0
  176. package/src/services/ollama/catalog.ts +122 -0
  177. package/src/services/ollama/http.ts +124 -0
  178. package/src/services/ollama/index.ts +13 -0
  179. package/src/services/ollama/models.ts +153 -0
  180. package/src/services/ollama/ollama.ts +89 -0
  181. package/src/services/ollama/registry.ts +112 -0
  182. package/src/services/ollama/types.ts +72 -0
  183. package/src/services/prompts.ts +76 -0
  184. package/src/services/registry.ts +220 -0
  185. package/src/services/resources/hardware.ts +125 -0
  186. package/src/services/resources/index.ts +6 -0
  187. package/src/services/resources/reservations.ts +162 -0
  188. package/src/services/resources/resources.ts +156 -0
  189. package/src/services/running/index.ts +1 -0
  190. package/src/services/running/running.ts +200 -0
  191. package/src/services/store/disk.ts +97 -0
  192. package/src/services/store/index.ts +15 -0
  193. package/src/services/store/store.ts +729 -0
  194. package/src/services/store/types.ts +303 -0
  195. package/src/services/test/child.ts +150 -0
  196. package/src/services/test/events.ts +102 -0
  197. package/src/services/test/files.ts +26 -0
  198. package/src/services/test/frames.ts +39 -0
  199. package/src/services/test/index.ts +7 -0
  200. package/src/services/test/runner.ts +122 -0
  201. package/src/services/test/types.ts +26 -0
  202. package/src/services/update/contract.ts +77 -0
  203. package/src/services/update/index.ts +12 -0
  204. package/src/services/update/installer.ts +139 -0
  205. package/src/services/update/updates.ts +137 -0
  206. package/src/utils/fs.ts +148 -0
  207. package/src/utils/tsast.ts +102 -0
@@ -0,0 +1,148 @@
1
+ import ts from "typescript"
2
+ import { err } from "@arcforge/err"
3
+ import type { BenchMeasurementDefinition, BenchMeasurementReducer, BenchMeasurementValueDefinition } from "@arcforge/types"
4
+
5
+ /**
6
+ * Read the measurement schema out of `defineBench<Schema>(...)`.
7
+ *
8
+ * The schema is authored as a TypeScript type because that is the right tool
9
+ * for describing types — a doc comment is where a description naturally lives,
10
+ * and a union is how anyone would write a fixed set of categories. Modelling
11
+ * the same thing in JSON means building a worse TypeScript inside a config file.
12
+ *
13
+ * But a type is erased at compile time, and two things need the schema as a
14
+ * VALUE at runtime: coverage cannot report "expected four, observed three"
15
+ * unless it knows the fourth exists, and results only enter a shared aggregate
16
+ * when their schemas hash equal. So the type is extracted here, once, at
17
+ * prepare — the author writes a type, the system keeps a value.
18
+ */
19
+
20
+ const COMPILER_OPTIONS: ts.CompilerOptions = {
21
+ allowJs: false,
22
+ noEmit: true,
23
+ target: ts.ScriptTarget.ESNext,
24
+ module: ts.ModuleKind.ESNext,
25
+ moduleResolution: ts.ModuleResolutionKind.Bundler,
26
+ skipLibCheck: true,
27
+ strict: true,
28
+ }
29
+
30
+ /** Aggregation follows from the kind unless the author says otherwise. */
31
+ const DEFAULT_AGGREGATE: Record<BenchMeasurementValueDefinition["kind"], BenchMeasurementReducer> = {
32
+ boolean: "rate",
33
+ number: "mean",
34
+ category: "count",
35
+ text: "last",
36
+ }
37
+
38
+ export function extractBenchSchema(configPath: string): BenchMeasurementDefinition[] {
39
+ const program = ts.createProgram([configPath], COMPILER_OPTIONS)
40
+ const source = program.getSourceFile(configPath)
41
+ if (!source) throw err("BENCH_SCHEMA_UNREADABLE", { detail: configPath, context: { configPath } })
42
+
43
+ const checker = program.getTypeChecker()
44
+ const call = findDefineBenchCall(source)
45
+
46
+ // No type argument is legal: a benchmark that records nothing still runs.
47
+ // An EMPTY schema and a FAILED extraction must never look the same, which
48
+ // is why this returns [] here and throws everywhere else.
49
+ if (!call?.typeArguments?.length) return []
50
+
51
+ const type = checker.getTypeFromTypeNode(call.typeArguments[0]!)
52
+ const properties = checker.getPropertiesOfType(type)
53
+ if (properties.length === 0) {
54
+ throw err("BENCH_SCHEMA_EMPTY", {
55
+ detail: "defineBench<Schema> was given a type with no properties — declare at least one measurement or drop the type argument",
56
+ context: { configPath },
57
+ })
58
+ }
59
+
60
+ return properties.map(property => toMeasurement(property, checker, configPath))
61
+ }
62
+
63
+ function findDefineBenchCall(source: ts.SourceFile): ts.CallExpression | null {
64
+ let found: ts.CallExpression | null = null
65
+ const visit = (node: ts.Node): void => {
66
+ if (found) return
67
+ if (ts.isCallExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === "defineBench") {
68
+ found = node
69
+ return
70
+ }
71
+ ts.forEachChild(node, visit)
72
+ }
73
+ ts.forEachChild(source, visit)
74
+ return found
75
+ }
76
+
77
+ function toMeasurement(
78
+ property: ts.Symbol,
79
+ checker: ts.TypeChecker,
80
+ configPath: string,
81
+ ): BenchMeasurementDefinition {
82
+ const id = property.getName()
83
+ const declaration = property.valueDeclaration ?? property.declarations?.[0]
84
+ const type = declaration
85
+ ? checker.getTypeOfSymbolAtLocation(property, declaration)
86
+ : checker.getDeclaredTypeOfSymbol(property)
87
+
88
+ const value = toValueDefinition(type, checker, id, configPath)
89
+ const description = ts.displayPartsToString(property.getDocumentationComment(checker)).trim()
90
+ const tags = Object.fromEntries(
91
+ property.getJsDocTags(checker).map(tag => [tag.name, ts.displayPartsToString(tag.text).trim()]),
92
+ )
93
+
94
+ return {
95
+ id,
96
+ // The property name is the label unless the author overrides it: an id
97
+ // is already a human-chosen word, so restating it is pure ceremony.
98
+ label: tags.label || sentenceCase(id),
99
+ description,
100
+ value: tags.unit && value.kind === "number" ? { ...value, unit: tags.unit } : value,
101
+ aggregate: (tags.aggregate as BenchMeasurementReducer | undefined) ?? DEFAULT_AGGREGATE[value.kind],
102
+ ...(tags.objective ? { objective: parseObjective(tags.objective, id, configPath) } : {}),
103
+ ...(tags.required === "" || tags.required === "true" ? { required: true } : {}),
104
+ }
105
+ }
106
+
107
+ function toValueDefinition(
108
+ type: ts.Type,
109
+ checker: ts.TypeChecker,
110
+ id: string,
111
+ configPath: string,
112
+ ): BenchMeasurementValueDefinition {
113
+ if (type.flags & ts.TypeFlags.BooleanLike) return { kind: "boolean" }
114
+ if (type.flags & ts.TypeFlags.NumberLike) return { kind: "number" }
115
+
116
+ // A union of string literals is a category with a closed value set — the
117
+ // natural way to write "one of these", and the reason the schema is a type.
118
+ if (type.isUnion()) {
119
+ const literals = type.types.filter(member => member.isStringLiteral())
120
+ if (literals.length === type.types.length && literals.length > 0) {
121
+ return { kind: "category", values: literals.map(member => (member as ts.StringLiteralType).value) }
122
+ }
123
+ if (type.types.every(member => member.flags & ts.TypeFlags.BooleanLike)) return { kind: "boolean" }
124
+ }
125
+
126
+ if (type.flags & ts.TypeFlags.StringLike) return { kind: "text" }
127
+
128
+ throw err("BENCH_SCHEMA_UNSUPPORTED_TYPE", {
129
+ detail: `measurement ${JSON.stringify(id)} is ${checker.typeToString(type)} — measurements are boolean, number, a union of string literals, or string`,
130
+ context: { configPath, measurementId: id, type: checker.typeToString(type) },
131
+ })
132
+ }
133
+
134
+ function parseObjective(raw: string, id: string, configPath: string) {
135
+ const value = raw.trim()
136
+ if (value === "maximize" || value === "minimize") return value
137
+ const target = /^target\s+(-?\d+(?:\.\d+)?)$/.exec(value)
138
+ if (target) return { target: Number(target[1]) }
139
+ throw err("BENCH_SCHEMA_UNSUPPORTED_TYPE", {
140
+ detail: `measurement ${JSON.stringify(id)} has @objective ${JSON.stringify(value)} — expected "maximize", "minimize", or "target <n>"`,
141
+ context: { configPath, measurementId: id },
142
+ })
143
+ }
144
+
145
+ function sentenceCase(id: string): string {
146
+ const spaced = id.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2")
147
+ return spaced.charAt(0).toUpperCase() + spaced.slice(1)
148
+ }
@@ -0,0 +1,2 @@
1
+ export { extractBenchSchema } from "./extract"
2
+ export { BenchSchema, type BenchSchemaT } from "./schema"
@@ -0,0 +1,95 @@
1
+ import { createHash } from "node:crypto"
2
+ import { mkdir } from "node:fs/promises"
3
+ import { join } from "node:path"
4
+ import { err } from "@arcforge/err"
5
+ import type { BenchMeasurementDefinition } from "@arcforge/types"
6
+ import { Frame } from "../../frame"
7
+
8
+ /**
9
+ * The measurement schema, extracted from bench.config.ts and kept on disk.
10
+ *
11
+ * Extraction is real ts.createProgram() work — expensive and blocking — so it
12
+ * runs in a worker and is gated on a content hash of the config, exactly like
13
+ * declareTools(). The common case (prepare with nothing changed) never spawns
14
+ * the worker at all.
15
+ */
16
+ export function BenchSchema(opts: { root: string }) {
17
+ const configPath = join(opts.root, "bench.config.ts")
18
+ // A hash-keyed record of what the last extraction found, skipped entirely
19
+ // when the config is unchanged — a cache in the strict sense, so it lives
20
+ // with the others rather than loose in the frame root.
21
+ const frame = Frame({ root: opts.root, kind: "bench" })
22
+ const outputPath = frame.file("cache", "schema.json")
23
+
24
+ type Cached = { hash: string; measurements: BenchMeasurementDefinition[] }
25
+
26
+ async function read(): Promise<Cached | null> {
27
+ const file = Bun.file(outputPath)
28
+ if (!(await file.exists())) return null
29
+ try {
30
+ return (await file.json()) as Cached
31
+ } catch {
32
+ // A truncated or hand-edited schema.json is not a reason to fail —
33
+ // it is regenerated from the config, which is the source of truth.
34
+ return null
35
+ }
36
+ }
37
+
38
+ async function extract(): Promise<BenchMeasurementDefinition[]> {
39
+ const worker = new URL("./worker.ts", import.meta.url).pathname
40
+ const proc = Bun.spawn(["bun", "run", worker, configPath], { stdout: "pipe", stderr: "pipe" })
41
+ const [stdout, stderr, code] = await Promise.all([
42
+ new Response(proc.stdout).text(),
43
+ new Response(proc.stderr).text(),
44
+ proc.exited,
45
+ ])
46
+
47
+ // A worker that could not even report is a different failure from a
48
+ // worker that reported a bad schema, and both must be loud. An empty
49
+ // schema silently makes every measurement undeclared, so coverage
50
+ // stops meaning anything — never fall back to one.
51
+ if (code !== 0) {
52
+ throw err("BENCH_SCHEMA_EXTRACTION_FAILED", {
53
+ detail: stderr.trim() || `schema worker exited ${code}`,
54
+ context: { configPath },
55
+ })
56
+ }
57
+
58
+ let parsed: { ok: boolean; measurements?: BenchMeasurementDefinition[]; message?: string }
59
+ try {
60
+ parsed = JSON.parse(stdout.trim().split("\n").at(-1) ?? "") as typeof parsed
61
+ } catch {
62
+ throw err("BENCH_SCHEMA_EXTRACTION_FAILED", {
63
+ detail: `schema worker printed no result: ${stdout.slice(0, 400)}`,
64
+ context: { configPath },
65
+ })
66
+ }
67
+
68
+ if (!parsed.ok || !parsed.measurements) {
69
+ throw err("BENCH_SCHEMA_EXTRACTION_FAILED", {
70
+ detail: parsed.message ?? "schema worker reported failure without a message",
71
+ context: { configPath },
72
+ })
73
+ }
74
+
75
+ return parsed.measurements
76
+ }
77
+
78
+ return {
79
+ /** Extract if the config changed, otherwise reuse what is on disk. */
80
+ async load(): Promise<BenchMeasurementDefinition[]> {
81
+ const source = await Bun.file(configPath).text()
82
+ const hash = createHash("sha256").update(source).digest("hex")
83
+
84
+ const cached = await read()
85
+ if (cached?.hash === hash) return cached.measurements
86
+
87
+ const measurements = await extract()
88
+ frame.ensure("cache")
89
+ await Bun.write(outputPath, JSON.stringify({ hash, measurements }, null, 2) + "\n")
90
+ return measurements
91
+ },
92
+ }
93
+ }
94
+
95
+ export type BenchSchemaT = ReturnType<typeof BenchSchema>
@@ -0,0 +1,30 @@
1
+ #!/usr/bin/env bun
2
+ /**
3
+ * schema-worker — the ONLY thing this process does: extract one bench's
4
+ * measurement schema and report it on stdout as JSON, then exit.
5
+ *
6
+ * ts.createProgram() is native, single-threaded compiler work with no
7
+ * cooperative yield points — inline it freezes VTerm's render/input loop for
8
+ * the whole call. Spawned as a subprocess instead, same reasoning and shape as
9
+ * declare-worker.ts and bundle-worker.ts.
10
+ *
11
+ * Contract: argv[2] is the absolute path to bench.config.ts. Prints exactly one
12
+ * JSON line — { ok: true, measurements } or { ok: false, message } — and exits
13
+ * 0 either way, so a non-zero code always means the worker itself broke rather
14
+ * than the extraction failing.
15
+ */
16
+ import { extractBenchSchema } from "./extract"
17
+
18
+ const configPath = process.argv[2]
19
+ if (!configPath) {
20
+ console.log(JSON.stringify({ ok: false, message: "schema-worker: no config path given" }))
21
+ process.exit(0)
22
+ }
23
+
24
+ try {
25
+ console.log(JSON.stringify({ ok: true, measurements: extractBenchSchema(configPath) }))
26
+ } catch (error) {
27
+ const message = error instanceof Error ? (error.stack ?? error.message) : String(error)
28
+ console.log(JSON.stringify({ ok: false, message }))
29
+ }
30
+ process.exit(0)
@@ -0,0 +1,119 @@
1
+ import { writeFile } from "node:fs/promises"
2
+ import { join } from "node:path"
3
+ import type { BenchAxis, BenchNormalizedConfig } from "@arcforge/types"
4
+ import { fsx } from "../../utils/fs"
5
+ import { Frame } from "../frame"
6
+
7
+ export type BenchTypegenResult = {
8
+ axes: number
9
+ measurements: number
10
+ dimensions: number
11
+ artifacts: number
12
+ }
13
+
14
+ function valueType(kind: BenchNormalizedConfig["measurements"][number]["value"]): string {
15
+ if (kind.kind === "number") return "number"
16
+ if (kind.kind === "boolean") return "boolean"
17
+ if (kind.kind === "text") return "string"
18
+ if (kind.values?.length) return kind.values.map(value => JSON.stringify(value)).join(" | ")
19
+ return "string"
20
+ }
21
+
22
+ function dimensionType(value: BenchNormalizedConfig["dimensions"][number]["value"]): string {
23
+ if (value.kind === "number") return "number"
24
+ if (value.kind === "boolean") return "boolean"
25
+ if (value.kind === "category" && !value.open) return value.values.map(item => JSON.stringify(item)).join(" | ") || "string"
26
+ return "string"
27
+ }
28
+
29
+ /**
30
+ * The union of variations declared on one axis, so `bench.axis("model")`
31
+ * narrows to exactly what the matrix can produce rather than `unknown`.
32
+ */
33
+ function axisRuntimeValue(axis: BenchAxis): string {
34
+ const values = axis.values.map(item => JSON.stringify(item.value)).join(" | ")
35
+ return values || "unknown"
36
+ }
37
+
38
+ export function benchDts(config: BenchNormalizedConfig): string {
39
+ const measurements = config.measurements.map(item => ` ${JSON.stringify(item.id)}: ${valueType(item.value)}`).join("\n")
40
+ const dimensions = config.dimensions.map(item => ` ${JSON.stringify(item.id)}?: ${dimensionType(item.value)}`).join("\n")
41
+ const axes = config.axes.map(axis => ` ${JSON.stringify(axis.key)}: ${axisRuntimeValue(axis)}`).join("\n")
42
+ const artifactIds = config.artifacts.length ? config.artifacts.map(item => JSON.stringify(item.id)).join(" | ") : "never"
43
+
44
+ return `// .bench/bench.d.ts — generated by axon bench prepare — do not edit
45
+ import type { BenchObserveOptions } from "@arcforge/types"
46
+
47
+ type BenchMeasurements = {
48
+ ${measurements}
49
+ }
50
+
51
+ type BenchDimensions = {
52
+ ${dimensions}
53
+ }
54
+
55
+ type BenchAxes = {
56
+ ${axes}
57
+ }
58
+
59
+ type GeneratedBenchOptions = Omit<BenchObserveOptions, "dimensions"> & {
60
+ dimensions?: BenchDimensions
61
+ }
62
+
63
+ interface GeneratedBenchHandle {
64
+ readonly workspace: import("@arcforge/types").BenchWorkspaceHandle
65
+ axis<K extends keyof BenchAxes>(key: K): BenchAxes[K]
66
+ observe<K extends keyof BenchMeasurements>(id: K, value: BenchMeasurements[K], options?: GeneratedBenchOptions): void
67
+ attach(id: ${artifactIds}, content: unknown, options?: { mediaType?: string; schema?: string; role?: "evidence" | "output" | "replay" | "report" }): Promise<void>
68
+ }
69
+
70
+ declare module "bun:test" {
71
+ interface TestContext {
72
+ /** The agent's world for this trial. */
73
+ workspace: import("@arcforge/types").BenchWorkspaceHandle
74
+ }
75
+ }
76
+
77
+ declare global {
78
+ const bench: GeneratedBenchHandle
79
+ const observe: GeneratedBenchHandle["observe"]
80
+ const Axon: () => Promise<Awaited<ReturnType<typeof import("@arcforge/core").Axon>> & { stop(): Promise<void> }>
81
+ const defineBench: typeof import("@arcforge/types").defineBench
82
+ }
83
+
84
+ export {}
85
+ `
86
+ }
87
+
88
+ export function BenchTypegen(opts: { root: string }) {
89
+ return {
90
+ async write(config: BenchNormalizedConfig): Promise<BenchTypegenResult> {
91
+ const out = Frame({ root: opts.root, kind: "bench" }).ensure("types")
92
+ const declarations = benchDts(config)
93
+
94
+ // Only the declarations derived from THIS config. The .bench
95
+ // tsconfig and the root pointer into it belong to the shared frame
96
+ // writer (project/typegen/tsconfig.ts), which runs first — writing
97
+ // them here too meant two authors for one file, and the includes
98
+ // silently drifted apart.
99
+ await writeFile(join(out, "bench.d.ts"), declarations)
100
+
101
+ // A bench nested inside another project must also be visible to
102
+ // that project's scope, or `bench.observe` is untyped in its tests.
103
+ // Into the host's `types/` area, which is what its tsconfig globs.
104
+ const hostFrames = (["agent", "module", "cognet"] as const)
105
+ .map(kind => Frame({ root: opts.root, kind: kind }))
106
+ .filter(frame => fsx.exists(frame.dir))
107
+ await Promise.all(hostFrames.map(frame =>
108
+ writeFile(join(frame.ensure("types"), "bench.d.ts"), declarations)))
109
+ return {
110
+ axes: config.axes.length,
111
+ measurements: config.measurements.length,
112
+ dimensions: config.dimensions.length,
113
+ artifacts: config.artifacts.length,
114
+ }
115
+ },
116
+ }
117
+ }
118
+
119
+ export type BenchTypegenT = ReturnType<typeof BenchTypegen>
@@ -0,0 +1,139 @@
1
+ import { cp, lstat, mkdir, readdir, readFile, readlink, rm } from "node:fs/promises"
2
+ import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"
3
+ import type {
4
+ BenchHash,
5
+ BenchResolvedWorkspaceDefinition,
6
+ BenchWorkspaceChange,
7
+ BenchWorkspaceFileState,
8
+ BenchWorkspaceTemplate,
9
+ } from "@arcforge/types"
10
+ import { err } from "@arcforge/err"
11
+
12
+ export type WorkspaceSnapshot = Map<string, BenchWorkspaceFileState>
13
+
14
+ function hash(bytes: Uint8Array | string): BenchHash {
15
+ return `sha256:${new Bun.CryptoHasher("sha256").update(bytes).digest("hex")}`
16
+ }
17
+
18
+ function ignored(path: string, patterns: string[]): boolean {
19
+ return patterns.some(pattern => path === pattern || path.startsWith(`${pattern}/`) || new Bun.Glob(pattern).match(path))
20
+ }
21
+
22
+ export async function snapshotWorkspace(
23
+ root: string,
24
+ opts: { ignore: string[]; persistDir?: string; maxBytes?: number },
25
+ ): Promise<{ files: WorkspaceSnapshot; bytes: number }> {
26
+ const files: WorkspaceSnapshot = new Map()
27
+ let bytes = 0
28
+
29
+ async function walk(dir: string, prefix: string): Promise<void> {
30
+ for (const entry of await readdir(dir, { withFileTypes: true })) {
31
+ const rel = prefix ? `${prefix}/${entry.name}` : entry.name
32
+ if (ignored(rel, opts.ignore)) continue
33
+ const absolute = join(dir, entry.name)
34
+ if (entry.isSymbolicLink()) {
35
+ const target = await readlink(absolute)
36
+ const resolved = resolve(dirname(absolute), target)
37
+ const boundary = resolve(root)
38
+ if (resolved !== boundary && !resolved.startsWith(`${boundary}${sep}`)) {
39
+ throw err("BENCH_WORKSPACE_ESCAPE", { detail: `${rel} resolves outside the workspace source`, context: { path: rel } })
40
+ }
41
+ throw err("BENCH_WORKSPACE_SYMLINK_UNSUPPORTED", { context: { path: rel } })
42
+ }
43
+ if (entry.isDirectory()) {
44
+ await walk(absolute, rel)
45
+ continue
46
+ }
47
+ if (!entry.isFile()) continue
48
+ const data = new Uint8Array(await readFile(absolute))
49
+ const info = await lstat(absolute)
50
+ const digest = hash(data)
51
+ const state: BenchWorkspaceFileState = { hash: digest, bytes: data.byteLength, mode: info.mode & 0o777 }
52
+ if (opts.persistDir && data.byteLength <= (opts.maxBytes ?? Number.POSITIVE_INFINITY)) {
53
+ await mkdir(opts.persistDir, { recursive: true })
54
+ const name = digest.slice("sha256:".length)
55
+ await Bun.write(join(opts.persistDir, name), data)
56
+ state.ref = `artifacts/${name}`
57
+ }
58
+ files.set(rel, state)
59
+ bytes += data.byteLength
60
+ }
61
+ }
62
+
63
+ if (await Bun.file(root).exists() || await lstat(root).then(() => true, () => false)) await walk(root, "")
64
+ return { files, bytes }
65
+ }
66
+
67
+ export function snapshotHash(snapshot: WorkspaceSnapshot): BenchHash {
68
+ const body = [...snapshot.entries()]
69
+ .sort(([a], [b]) => a.localeCompare(b))
70
+ .map(([path, state]) => `${path}\0${state.hash}\0${state.mode}`)
71
+ .join("\n")
72
+ return hash(body)
73
+ }
74
+
75
+ export async function prepareWorkspaceTemplate(
76
+ benchRoot: string,
77
+ definition: BenchResolvedWorkspaceDefinition,
78
+ ): Promise<{ template: BenchWorkspaceTemplate; sourcePath?: string }> {
79
+ if (definition.source.kind === "empty") {
80
+ return { template: { kind: "empty", hash: hash(""), files: 0, bytes: 0 } }
81
+ }
82
+ const sourcePath = isAbsolute(definition.source.path)
83
+ ? resolve(definition.source.path)
84
+ : resolve(benchRoot, definition.source.path)
85
+ const boundary = resolve(benchRoot)
86
+ if (sourcePath !== boundary && !sourcePath.startsWith(`${boundary}${sep}`)) {
87
+ throw err("BENCH_WORKSPACE_OUTSIDE_ROOT", { detail: definition.source.path, context: { path: definition.source.path } })
88
+ }
89
+ const info = await lstat(sourcePath).catch(() => null)
90
+ if (!info?.isDirectory()) throw err("BENCH_WORKSPACE_SOURCE_NOT_FOUND", { detail: definition.source.path, context: { path: definition.source.path } })
91
+ const snapshot = await snapshotWorkspace(sourcePath, { ignore: definition.capture.ignore })
92
+ return {
93
+ sourcePath,
94
+ template: {
95
+ kind: "directory",
96
+ hash: snapshotHash(snapshot.files),
97
+ files: snapshot.files.size,
98
+ bytes: snapshot.bytes,
99
+ },
100
+ }
101
+ }
102
+
103
+ export async function materializeWorkspace(sourcePath: string | undefined, destination: string, ignore: string[] = []): Promise<void> {
104
+ await rm(destination, { recursive: true, force: true })
105
+ if (!sourcePath) {
106
+ await mkdir(destination, { recursive: true })
107
+ return
108
+ }
109
+ await mkdir(dirname(destination), { recursive: true })
110
+ await cp(sourcePath, destination, {
111
+ recursive: true,
112
+ force: false,
113
+ errorOnExist: true,
114
+ preserveTimestamps: true,
115
+ filter(source) {
116
+ const rel = relative(sourcePath, source).split(sep).join("/")
117
+ return rel === "" || !ignored(rel, ignore)
118
+ },
119
+ })
120
+ }
121
+
122
+ export function workspaceChanges(before: WorkspaceSnapshot, after: WorkspaceSnapshot): BenchWorkspaceChange[] {
123
+ const paths = new Set([...before.keys(), ...after.keys()])
124
+ const changes: BenchWorkspaceChange[] = []
125
+ for (const path of [...paths].sort()) {
126
+ const prior = before.get(path)
127
+ const next = after.get(path)
128
+ if (!prior && next) changes.push({ path, kind: "added", after: next })
129
+ else if (prior && !next) changes.push({ path, kind: "deleted", before: prior })
130
+ else if (prior && next && (prior.hash !== next.hash || prior.mode !== next.mode)) {
131
+ changes.push({ path, kind: "modified", before: prior, after: next })
132
+ }
133
+ }
134
+ return changes
135
+ }
136
+
137
+ export async function removeWorkspace(path: string): Promise<void> {
138
+ await rm(path, { recursive: true, force: true })
139
+ }
@@ -0,0 +1,133 @@
1
+ import { join } from "node:path"
2
+ import type { BenchWorkspaceChange } from "@arcforge/types"
3
+ import { snapshotWorkspace, workspaceChanges, type WorkspaceSnapshot } from "./files"
4
+
5
+ /**
6
+ * What a scenario can ask about the world after the agent has acted.
7
+ *
8
+ * Every coding benchmark needs the same three questions — did the suite pass,
9
+ * what changed, what does the patch look like — and two of them are
10
+ * unanswerable from inside the test. `changed()` and `diff()` need the world
11
+ * as it was BEFORE the agent booted, and by the time a test runs that state
12
+ * only exists in the harness's baseline snapshot.
13
+ *
14
+ * `toString()` returns the path, so the handle can still be passed anywhere a
15
+ * directory is expected without callers reaching for a property.
16
+ */
17
+ export type WorkspaceHandle = {
18
+ readonly path: string
19
+ toString(): string
20
+
21
+ /** Read a file relative to the workspace root. */
22
+ read(relativePath: string): Promise<string>
23
+ exists(relativePath: string): Promise<boolean>
24
+
25
+ /** Paths the agent added, modified, or removed, relative to the root. */
26
+ changed(): Promise<string[]>
27
+ /** The same set with detail: kind, sizes, hashes. */
28
+ changes(): Promise<BenchWorkspaceChange[]>
29
+ /** Unified diff of everything that changed. */
30
+ diff(): Promise<string>
31
+
32
+ tests: {
33
+ /** Run the workspace's own suite. True when it exits clean. */
34
+ pass(opts?: { timeoutMs?: number }): Promise<boolean>
35
+ /** The suite's combined output — worth attaching when a trial scores badly. */
36
+ output(opts?: { timeoutMs?: number }): Promise<{ ok: boolean; text: string }>
37
+ }
38
+ }
39
+
40
+ export type WorkspaceHandleOpts = {
41
+ path: string
42
+ before: WorkspaceSnapshot
43
+ ignore: string[]
44
+ /** Command that runs the workspace's suite. Defaults to `bun test`. */
45
+ testCommand?: string[]
46
+ }
47
+
48
+ /** How long the workspace's own suite may run before it is treated as failing. */
49
+ const DEFAULT_TEST_TIMEOUT_MS = 120_000
50
+
51
+ export function WorkspaceHandle(opts: WorkspaceHandleOpts): WorkspaceHandle {
52
+ async function after(): Promise<WorkspaceSnapshot> {
53
+ return (await snapshotWorkspace(opts.path, { ignore: opts.ignore })).files
54
+ }
55
+
56
+ async function changes(): Promise<BenchWorkspaceChange[]> {
57
+ return workspaceChanges(opts.before, await after())
58
+ }
59
+
60
+ async function runTests(timeoutMs: number): Promise<{ ok: boolean; text: string }> {
61
+ const command = opts.testCommand ?? ["bun", "test"]
62
+ const proc = Bun.spawn(command, {
63
+ cwd: opts.path,
64
+ stdout: "pipe",
65
+ stderr: "pipe",
66
+ // The suite belongs to the task, not to this process. Without its
67
+ // own group a runaway test tree survives the timeout below and
68
+ // holds the whole benchmark open.
69
+ detached: true,
70
+ })
71
+
72
+ const timeout = Bun.sleep(timeoutMs).then(() => "timeout" as const)
73
+ const finished = Promise.all([
74
+ new Response(proc.stdout).text(),
75
+ new Response(proc.stderr).text(),
76
+ proc.exited,
77
+ ])
78
+
79
+ const outcome = await Promise.race([finished, timeout])
80
+ if (outcome === "timeout") {
81
+ try {
82
+ process.kill(-proc.pid, "SIGKILL")
83
+ } catch {
84
+ proc.kill()
85
+ }
86
+ return { ok: false, text: `workspace test suite exceeded ${timeoutMs}ms and was killed` }
87
+ }
88
+
89
+ const [stdout, stderr, code] = outcome
90
+ return { ok: code === 0, text: `${stdout}${stderr}` }
91
+ }
92
+
93
+ return {
94
+ get path() {
95
+ return opts.path
96
+ },
97
+ toString() {
98
+ return opts.path
99
+ },
100
+
101
+ async read(relativePath) {
102
+ return Bun.file(join(opts.path, relativePath)).text()
103
+ },
104
+ async exists(relativePath) {
105
+ return Bun.file(join(opts.path, relativePath)).exists()
106
+ },
107
+
108
+ async changed() {
109
+ return (await changes()).map(change => change.path)
110
+ },
111
+ changes,
112
+
113
+ async diff() {
114
+ // `git diff --no-index` against the baseline is not available — the
115
+ // baseline is a hash snapshot, not a tree on disk. Rendering from
116
+ // the change records keeps the output honest about what is known:
117
+ // which paths moved and how, without inventing line-level detail
118
+ // the snapshot never captured.
119
+ const entries = await changes()
120
+ if (entries.length === 0) return ""
121
+ return entries.map(change => `${change.kind}\t${change.path}`).join("\n")
122
+ },
123
+
124
+ tests: {
125
+ async pass(options = {}) {
126
+ return (await runTests(options.timeoutMs ?? DEFAULT_TEST_TIMEOUT_MS)).ok
127
+ },
128
+ async output(options = {}) {
129
+ return runTests(options.timeoutMs ?? DEFAULT_TEST_TIMEOUT_MS)
130
+ },
131
+ },
132
+ }
133
+ }
@@ -0,0 +1,11 @@
1
+ export { WorkspaceHandle, type WorkspaceHandle as WorkspaceHandleT } from "./handle"
2
+ export { Workspace, type WorkspaceT } from "./workspace"
3
+ export {
4
+ materializeWorkspace,
5
+ prepareWorkspaceTemplate,
6
+ removeWorkspace,
7
+ snapshotHash,
8
+ snapshotWorkspace,
9
+ workspaceChanges,
10
+ type WorkspaceSnapshot,
11
+ } from "./files"