@arcforge/platform 2.0.166
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/package.json +69 -0
- package/src/bin/supervisor.ts +102 -0
- package/src/bin/test-api.ts +30 -0
- package/src/bin/test-preload.ts +308 -0
- package/src/bin/update-helper.ts +32 -0
- package/src/build/bench/CLAUDE.md +106 -0
- package/src/build/bench/SPEC.md +209 -0
- package/src/build/bench/agents.ts +105 -0
- package/src/build/bench/axes.ts +84 -0
- package/src/build/bench/bench.ts +96 -0
- package/src/build/bench/discover.ts +147 -0
- package/src/build/bench/index.ts +7 -0
- package/src/build/bench/log.ts +47 -0
- package/src/build/bench/matrix.ts +23 -0
- package/src/build/bench/preload.ts +440 -0
- package/src/build/bench/prepare.ts +61 -0
- package/src/build/bench/projection.ts +112 -0
- package/src/build/bench/resolve.ts +54 -0
- package/src/build/bench/runner.ts +309 -0
- package/src/build/bench/schema/extract.ts +148 -0
- package/src/build/bench/schema/index.ts +2 -0
- package/src/build/bench/schema/schema.ts +95 -0
- package/src/build/bench/schema/worker.ts +30 -0
- package/src/build/bench/typegen.ts +119 -0
- package/src/build/bench/workspace/files.ts +139 -0
- package/src/build/bench/workspace/handle.ts +133 -0
- package/src/build/bench/workspace/index.ts +11 -0
- package/src/build/bench/workspace/workspace.ts +14 -0
- package/src/build/blueprint/blueprint.ts +398 -0
- package/src/build/blueprint/cognet/abi.ts +207 -0
- package/src/build/blueprint/cognet/bundle-worker.ts +93 -0
- package/src/build/blueprint/cognet/bundle.ts +664 -0
- package/src/build/blueprint/cognet/cognet.ts +249 -0
- package/src/build/blueprint/cognet/index.ts +10 -0
- package/src/build/blueprint/cognet/manifest.ts +38 -0
- package/src/build/blueprint/cognet/resolve.ts +113 -0
- package/src/build/blueprint/collisions.ts +51 -0
- package/src/build/blueprint/index.ts +29 -0
- package/src/build/blueprint/modules/entries.ts +117 -0
- package/src/build/blueprint/modules/index.ts +300 -0
- package/src/build/blueprint/modules/meta.ts +174 -0
- package/src/build/blueprint/scan/boot.ts +34 -0
- package/src/build/blueprint/scan/cognetImports.ts +56 -0
- package/src/build/blueprint/scan/components.ts +45 -0
- package/src/build/blueprint/scan/config.ts +234 -0
- package/src/build/blueprint/scan/configImports.ts +232 -0
- package/src/build/blueprint/scan/declare-worker.ts +61 -0
- package/src/build/blueprint/scan/declare.ts +393 -0
- package/src/build/blueprint/scan/h3-globals.ts +137 -0
- package/src/build/blueprint/scan/knowledge.ts +101 -0
- package/src/build/blueprint/scan/middleware.ts +110 -0
- package/src/build/blueprint/scan/moduleImports.ts +79 -0
- package/src/build/blueprint/scan/plugins.ts +93 -0
- package/src/build/blueprint/scan/promptPackages.ts +45 -0
- package/src/build/blueprint/scan/prompts.ts +164 -0
- package/src/build/blueprint/scan/published.ts +131 -0
- package/src/build/blueprint/scan/resolve.ts +398 -0
- package/src/build/blueprint/scan/routes.ts +109 -0
- package/src/build/blueprint/scan/scripts.ts +119 -0
- package/src/build/blueprint/scan/tool-bundle-worker.ts +69 -0
- package/src/build/blueprint/scan/tools.ts +610 -0
- package/src/build/blueprint/types.ts +28 -0
- package/src/build/extensions/config.ts +261 -0
- package/src/build/extensions/edit.ts +684 -0
- package/src/build/extensions/extensions.ts +731 -0
- package/src/build/extensions/index.ts +10 -0
- package/src/build/extensions/load.ts +350 -0
- package/src/build/extensions/registrations.ts +189 -0
- package/src/build/extensions/store.ts +200 -0
- package/src/build/frame/frame.ts +145 -0
- package/src/build/frame/index.ts +5 -0
- package/src/build/frame/migrate.ts +246 -0
- package/src/build/project/bundle/agent.ts +382 -0
- package/src/build/project/bundle/artifacts.ts +88 -0
- package/src/build/project/bundle/assets.ts +385 -0
- package/src/build/project/bundle/bundle.ts +160 -0
- package/src/build/project/bundle/imports.ts +102 -0
- package/src/build/project/bundle/index.ts +16 -0
- package/src/build/project/bundle/module.ts +130 -0
- package/src/build/project/bundle/source.ts +136 -0
- package/src/build/project/bundle/stage.ts +97 -0
- package/src/build/project/bundle/types.ts +111 -0
- package/src/build/project/cognet.ts +156 -0
- package/src/build/project/create/agent.ts +52 -0
- package/src/build/project/create/bench.ts +81 -0
- package/src/build/project/create/cognet.ts +102 -0
- package/src/build/project/create/extension.ts +63 -0
- package/src/build/project/create/module.ts +43 -0
- package/src/build/project/create/profile.ts +102 -0
- package/src/build/project/create/prompt.ts +68 -0
- package/src/build/project/create/templates.ts +88 -0
- package/src/build/project/deploy.ts +96 -0
- package/src/build/project/index.ts +40 -0
- package/src/build/project/installer.ts +616 -0
- package/src/build/project/kinds.ts +374 -0
- package/src/build/project/links.ts +33 -0
- package/src/build/project/manifest/bunfig.ts +139 -0
- package/src/build/project/manifest/config.ts +246 -0
- package/src/build/project/manifest/env.ts +184 -0
- package/src/build/project/manifest/index.ts +9 -0
- package/src/build/project/manifest/manifest.ts +49 -0
- package/src/build/project/manifest/model.ts +124 -0
- package/src/build/project/manifest/package.ts +386 -0
- package/src/build/project/manifest/text.ts +19 -0
- package/src/build/project/models/fetch.ts +123 -0
- package/src/build/project/models/index.ts +8 -0
- package/src/build/project/models/models.ts +111 -0
- package/src/build/project/models/specifier.ts +106 -0
- package/src/build/project/models/store.ts +202 -0
- package/src/build/project/modules/hash.ts +31 -0
- package/src/build/project/modules/index.ts +5 -0
- package/src/build/project/modules/lock.ts +78 -0
- package/src/build/project/modules/modules.ts +155 -0
- package/src/build/project/prepare.ts +709 -0
- package/src/build/project/project.ts +253 -0
- package/src/build/project/projects.ts +164 -0
- package/src/build/project/propagation.ts +120 -0
- package/src/build/project/publish/verify.ts +312 -0
- package/src/build/project/publish.ts +155 -0
- package/src/build/project/reconcile.ts +93 -0
- package/src/build/project/specifier.ts +42 -0
- package/src/build/project/tree.ts +615 -0
- package/src/build/project/treecache.ts +356 -0
- package/src/build/project/typegen/axon-dts.ts +362 -0
- package/src/build/project/typegen/cognet-dts.ts +91 -0
- package/src/build/project/typegen/components.ts +45 -0
- package/src/build/project/typegen/env.ts +59 -0
- package/src/build/project/typegen/generators.ts +108 -0
- package/src/build/project/typegen/hooks.ts +71 -0
- package/src/build/project/typegen/index.ts +137 -0
- package/src/build/project/typegen/prompt-dts.ts +40 -0
- package/src/build/project/typegen/prompts.ts +36 -0
- package/src/build/project/typegen/registry-dts.ts +109 -0
- package/src/build/project/typegen/scripts.ts +37 -0
- package/src/build/project/typegen/tools.ts +31 -0
- package/src/build/project/typegen/tsconfig.ts +243 -0
- package/src/build/project/typegen/tui-dts.ts +120 -0
- package/src/build/project/typegen/write.ts +40 -0
- package/src/build/project/verify.ts +268 -0
- package/src/build/project/watcher.ts +266 -0
- package/src/build/runtime/agent.ts +550 -0
- package/src/build/runtime/escalations.ts +159 -0
- package/src/build/runtime/index.ts +15 -0
- package/src/build/runtime/instances.ts +604 -0
- package/src/build/runtime/profile.ts +91 -0
- package/src/build/runtime/progress.ts +143 -0
- package/src/build/runtime/requests.ts +167 -0
- package/src/build/runtime/resolve.ts +159 -0
- package/src/build/runtime/runtime.ts +233 -0
- package/src/build/runtime/sessions/index.ts +5 -0
- package/src/build/runtime/sessions/record.ts +145 -0
- package/src/build/runtime/sessions/sessions.ts +240 -0
- package/src/build/runtime/zeno.ts +120 -0
- package/src/index.ts +11 -0
- package/src/link/agent-main.ts +395 -0
- package/src/link/index.ts +70 -0
- package/src/link/services.ts +120 -0
- package/src/platform.ts +293 -0
- package/src/procs/index.ts +15 -0
- package/src/procs/tree.ts +177 -0
- package/src/services/cloud.ts +419 -0
- package/src/services/control/client.ts +126 -0
- package/src/services/control/dispatch.ts +156 -0
- package/src/services/control/index.ts +17 -0
- package/src/services/control/protocol.ts +179 -0
- package/src/services/control/server.ts +241 -0
- package/src/services/deployments.ts +81 -0
- package/src/services/files.ts +329 -0
- package/src/services/index.ts +5 -0
- package/src/services/mic/capture.ts +248 -0
- package/src/services/mic/config.ts +109 -0
- package/src/services/mic/fft.ts +93 -0
- package/src/services/mic/index.ts +139 -0
- package/src/services/mic/visualizer.ts +46 -0
- package/src/services/ollama/catalog.ts +122 -0
- package/src/services/ollama/http.ts +124 -0
- package/src/services/ollama/index.ts +13 -0
- package/src/services/ollama/models.ts +153 -0
- package/src/services/ollama/ollama.ts +89 -0
- package/src/services/ollama/registry.ts +112 -0
- package/src/services/ollama/types.ts +72 -0
- package/src/services/prompts.ts +76 -0
- package/src/services/registry.ts +220 -0
- package/src/services/resources/hardware.ts +125 -0
- package/src/services/resources/index.ts +6 -0
- package/src/services/resources/reservations.ts +162 -0
- package/src/services/resources/resources.ts +156 -0
- package/src/services/running/index.ts +1 -0
- package/src/services/running/running.ts +200 -0
- package/src/services/store/disk.ts +97 -0
- package/src/services/store/index.ts +15 -0
- package/src/services/store/store.ts +729 -0
- package/src/services/store/types.ts +303 -0
- package/src/services/test/child.ts +150 -0
- package/src/services/test/events.ts +102 -0
- package/src/services/test/files.ts +26 -0
- package/src/services/test/frames.ts +39 -0
- package/src/services/test/index.ts +7 -0
- package/src/services/test/runner.ts +122 -0
- package/src/services/test/types.ts +26 -0
- package/src/services/update/contract.ts +77 -0
- package/src/services/update/index.ts +12 -0
- package/src/services/update/installer.ts +139 -0
- package/src/services/update/updates.ts +137 -0
- package/src/utils/fs.ts +148 -0
- package/src/utils/tsast.ts +102 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { resolveTestFiles } from "../../services/test"
|
|
2
|
+
import { isAbsolute, resolve } from "node:path"
|
|
3
|
+
import { err } from "@arcforge/err"
|
|
4
|
+
import { fsx } from "../../utils/fs"
|
|
5
|
+
import type { DiscoverT } from "./discover"
|
|
6
|
+
import type { BenchTypegenT } from "./typegen"
|
|
7
|
+
import type { BenchAxis } from "@arcforge/types"
|
|
8
|
+
import type { AgentsT } from "./agents"
|
|
9
|
+
import { Workspace } from "./workspace"
|
|
10
|
+
|
|
11
|
+
export type BenchPrepareResult = {
|
|
12
|
+
typegen: Awaited<ReturnType<BenchTypegenT["write"]>>
|
|
13
|
+
tests: string[]
|
|
14
|
+
/** Agents the matrix names, with the identity each resolved to. */
|
|
15
|
+
agents: Array<{ ref: string; pin: string }>
|
|
16
|
+
warnings: string[]
|
|
17
|
+
workspace: Awaited<ReturnType<ReturnType<typeof Workspace>["prepare"]>>["template"]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Variations that name something on disk, so prepare can fail early rather
|
|
22
|
+
* than mid-run. `agent` and `cognet` variations are bare paths or registry
|
|
23
|
+
* refs; everything else is a value the runtime merges, not a location.
|
|
24
|
+
*/
|
|
25
|
+
function localRefs(axis: BenchAxis, value: unknown): string[] {
|
|
26
|
+
// `agent` is handled by Agents(), which resolves registry refs as well as
|
|
27
|
+
// paths. This check is for the axes that can only ever be local.
|
|
28
|
+
if (axis.key !== "cognet") return []
|
|
29
|
+
return typeof value === "string" ? [value] : []
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function PrepareBench(opts: { root: string; discover: DiscoverT; typegen: BenchTypegenT; agents: AgentsT }) {
|
|
33
|
+
return async function prepare(): Promise<BenchPrepareResult> {
|
|
34
|
+
const config = await opts.discover.load()
|
|
35
|
+
const tests = await resolveTestFiles(config.tests, opts.root)
|
|
36
|
+
if (tests.length === 0) throw err("BENCH_TESTS_NOT_FOUND", { detail: config.tests.join(", "), context: { declared: config.tests } })
|
|
37
|
+
for (const axis of config.axes) {
|
|
38
|
+
for (const variation of axis.values) {
|
|
39
|
+
for (const ref of localRefs(axis, variation.value)) {
|
|
40
|
+
if (!ref.startsWith(".") && !isAbsolute(ref)) continue
|
|
41
|
+
const path = isAbsolute(ref) ? ref : resolve(opts.root, ref)
|
|
42
|
+
if (!fsx.exists(path)) throw err("BENCH_LOCAL_REF_NOT_FOUND", { detail: `${axis.key}/${variation.id} -> ${ref}`, context: { axis: axis.key, valueId: variation.id, ref } })
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// Fetch registry agents and prepare every agent the matrix names —
|
|
47
|
+
// local or cloned. A freshly cloned agent has no node_modules and no
|
|
48
|
+
// generated types, so without this every cell fails at boot with an
|
|
49
|
+
// error that says nothing about the cause.
|
|
50
|
+
const agents = await opts.agents.resolve(config.axes)
|
|
51
|
+
|
|
52
|
+
const workspace = await Workspace({ root: opts.root, definition: config.workspace }).prepare()
|
|
53
|
+
return {
|
|
54
|
+
typegen: await opts.typegen.write(config),
|
|
55
|
+
tests,
|
|
56
|
+
workspace: workspace.template,
|
|
57
|
+
agents: [...agents.values()].map(agent => ({ ref: agent.ref, pin: agent.pin })),
|
|
58
|
+
warnings: [],
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
BenchArtifactRef,
|
|
3
|
+
BenchCaseRef,
|
|
4
|
+
BenchConfig,
|
|
5
|
+
BenchCoverage,
|
|
6
|
+
BenchEvent,
|
|
7
|
+
BenchFault,
|
|
8
|
+
BenchObservation,
|
|
9
|
+
BenchPhysics,
|
|
10
|
+
BenchRunResult,
|
|
11
|
+
BenchSessionRef,
|
|
12
|
+
BenchTestRef,
|
|
13
|
+
BenchTrialRecord,
|
|
14
|
+
} from "@arcforge/types"
|
|
15
|
+
import { err } from "@arcforge/err"
|
|
16
|
+
|
|
17
|
+
const emptyPhysics = (): BenchPhysics => ({
|
|
18
|
+
subject: { durationMs: 0, tokens: { input: 0, output: 0 }, engineCalls: 0, toolCalls: 0, errors: 0 },
|
|
19
|
+
harness: { durationMs: 0 },
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
function key(context: BenchEvent["context"]): string {
|
|
23
|
+
return `${context.cellId ?? ""}:${context.trial ?? 0}:${context.testId ?? ""}:${context.attempt ?? 0}`
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Pure fold from the immutable JSONL source of truth into portable run data. */
|
|
27
|
+
export function Projection() {
|
|
28
|
+
return {
|
|
29
|
+
build(_config: BenchConfig, events: BenchEvent[]): BenchRunResult {
|
|
30
|
+
const start = events.find(event => event.type === "bench:run:start")
|
|
31
|
+
if (!start || start.type !== "bench:run:start") throw err("BENCH_LOG_INVALID", { detail: "missing bench:run:start" })
|
|
32
|
+
const manifest = structuredClone(start.data.manifest)
|
|
33
|
+
const observations: BenchObservation[] = []
|
|
34
|
+
const artifacts: BenchArtifactRef[] = []
|
|
35
|
+
const sessions: BenchSessionRef[] = []
|
|
36
|
+
const faults: BenchFault[] = []
|
|
37
|
+
const tests = new Map<string, BenchTestRef>()
|
|
38
|
+
const physics = new Map<string, BenchPhysics>()
|
|
39
|
+
const observationIds = new Map<string, string[]>()
|
|
40
|
+
const artifactIds = new Map<string, string[]>()
|
|
41
|
+
const trials: BenchTrialRecord[] = []
|
|
42
|
+
let coverage: BenchCoverage = {
|
|
43
|
+
expectedTrials: 0, executedTrials: 0, completedTrials: 0,
|
|
44
|
+
expectedMeasurements: 0, observedMeasurements: 0,
|
|
45
|
+
missing: {}, measurements: [],
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
for (const event of events) {
|
|
49
|
+
const eventKey = key(event.context)
|
|
50
|
+
if (event.type === "bench:case:declare") tests.set(event.data.test.id, event.data.test)
|
|
51
|
+
if (event.type === "bench:observation") {
|
|
52
|
+
// `seq` is the event's identity — see AxonEvent. Unique
|
|
53
|
+
// within one bench log, which is the scope these ids are
|
|
54
|
+
// correlated in.
|
|
55
|
+
const id = String(event.time.seq)
|
|
56
|
+
observations.push({
|
|
57
|
+
...event.data,
|
|
58
|
+
id,
|
|
59
|
+
time: new Date(event.time.ms).toISOString(),
|
|
60
|
+
context: {
|
|
61
|
+
runId: event.context.benchRunId,
|
|
62
|
+
cellId: event.context.cellId!,
|
|
63
|
+
testId: event.context.testId,
|
|
64
|
+
caseId: event.context.caseId,
|
|
65
|
+
trial: event.context.trial,
|
|
66
|
+
attempt: event.context.attempt,
|
|
67
|
+
},
|
|
68
|
+
})
|
|
69
|
+
observationIds.set(eventKey, [...(observationIds.get(eventKey) ?? []), id])
|
|
70
|
+
}
|
|
71
|
+
if (event.type === "bench:artifact") {
|
|
72
|
+
artifacts.push(event.data.artifact)
|
|
73
|
+
artifactIds.set(eventKey, [...(artifactIds.get(eventKey) ?? []), event.data.artifact.id])
|
|
74
|
+
}
|
|
75
|
+
if (event.type === "bench:session:attach") sessions.push(event.data.session)
|
|
76
|
+
if (event.type === "bench:process:fault") faults.push(event.data.fault)
|
|
77
|
+
if (event.type === "bench:trial:complete") {
|
|
78
|
+
physics.set(`${event.context.cellId}:${event.context.trial}`, event.data.physics)
|
|
79
|
+
}
|
|
80
|
+
if (event.type === "bench:case:complete") {
|
|
81
|
+
const test = tests.get(event.context.testId ?? "")
|
|
82
|
+
const cell = manifest.cells.find(item => item.id === event.context.cellId)
|
|
83
|
+
if (!test || !cell) throw err("BENCH_LOG_INVALID", { detail: "case completion has no declaration or cell" })
|
|
84
|
+
trials.push({
|
|
85
|
+
id: String(event.time.seq),
|
|
86
|
+
test,
|
|
87
|
+
cell,
|
|
88
|
+
trial: event.context.trial ?? 0,
|
|
89
|
+
attempt: event.context.attempt ?? 0,
|
|
90
|
+
status: event.data.status,
|
|
91
|
+
...(event.data.error ? { error: event.data.error } : {}),
|
|
92
|
+
physics: physics.get(`${cell.id}:${event.context.trial}`) ?? emptyPhysics(),
|
|
93
|
+
sessions: [],
|
|
94
|
+
observationIds: observationIds.get(eventKey) ?? [],
|
|
95
|
+
artifactIds: artifactIds.get(eventKey) ?? [],
|
|
96
|
+
})
|
|
97
|
+
}
|
|
98
|
+
if (event.type === "bench:run:complete") {
|
|
99
|
+
coverage = event.data.coverage
|
|
100
|
+
manifest.completedAt = new Date(event.time.ms).toISOString()
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
manifest.tests = [...tests.values()]
|
|
105
|
+
manifest.cases = manifest.tests.map(test => ({ id: test.id, testId: test.id, label: test.name } satisfies BenchCaseRef))
|
|
106
|
+
for (const trial of trials) trial.physics = physics.get(`${trial.cell.id}:${trial.trial}`) ?? trial.physics
|
|
107
|
+
return { manifest, trials, observations, artifacts, sessions, faults, coverage }
|
|
108
|
+
},
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export type ProjectionT = ReturnType<typeof Projection>
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { BenchAxis, BenchCell, BenchCoordinate, BenchHash, BenchValue } from "@arcforge/types"
|
|
2
|
+
import { err } from "@arcforge/err"
|
|
3
|
+
|
|
4
|
+
function canonical(value: BenchValue): string {
|
|
5
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value)
|
|
6
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`
|
|
7
|
+
return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key]!)}`).join(",")}}`
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function pin(value: unknown): BenchHash {
|
|
11
|
+
const hash = new Bun.CryptoHasher("sha256").update(canonical(value as BenchValue)).digest("hex")
|
|
12
|
+
return `sha256:${hash}`
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function findAxis(axes: BenchAxis[], key: string): BenchAxis {
|
|
16
|
+
const axis = axes.find(item => item.key === key)
|
|
17
|
+
if (!axis) throw err("BENCH_AXIS_NOT_FOUND", { context: { axis: key } })
|
|
18
|
+
return axis
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function Resolver() {
|
|
22
|
+
return {
|
|
23
|
+
/**
|
|
24
|
+
* @param agentPins resolved identities keyed by the ref as written, so
|
|
25
|
+
* an `agent` axis pins `@axon/coding-base@1.2.0` rather than a hash of
|
|
26
|
+
* the string "./fixtures/subject" — which says nothing anyone else
|
|
27
|
+
* could compare against.
|
|
28
|
+
*/
|
|
29
|
+
async resolve(axes: BenchAxis[], coordinate: BenchCoordinate, agentPins?: Map<string, string>): Promise<BenchCell> {
|
|
30
|
+
const resolved = coordinate.axes.map(selection => {
|
|
31
|
+
const axis = findAxis(axes, selection.key)
|
|
32
|
+
const selected = axis.values.find(item => item.id === selection.valueId)
|
|
33
|
+
if (!selected) {
|
|
34
|
+
throw err("BENCH_AXIS_VALUE_NOT_FOUND", {
|
|
35
|
+
detail: `${selection.key}/${selection.valueId}`,
|
|
36
|
+
context: { axis: selection.key, valueId: selection.valueId },
|
|
37
|
+
})
|
|
38
|
+
}
|
|
39
|
+
const resolvedPin = selection.key === "agent" && typeof selected.value === "string"
|
|
40
|
+
? agentPins?.get(selected.value)
|
|
41
|
+
: undefined
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
...selection,
|
|
45
|
+
label: selected.label || selected.id,
|
|
46
|
+
pin: resolvedPin ?? pin(selected.value),
|
|
47
|
+
}
|
|
48
|
+
})
|
|
49
|
+
return { id: coordinate.id, axes: resolved }
|
|
50
|
+
},
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export type ResolverT = ReturnType<typeof Resolver>
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { join } from "node:path"
|
|
2
|
+
import type {
|
|
3
|
+
AxonTestEvent,
|
|
4
|
+
BenchAxis,
|
|
5
|
+
BenchCoordinate,
|
|
6
|
+
BenchCoverage,
|
|
7
|
+
BenchEventMap,
|
|
8
|
+
BenchHash,
|
|
9
|
+
BenchNormalizedConfig,
|
|
10
|
+
BenchRunManifest,
|
|
11
|
+
BenchMeasurementState,
|
|
12
|
+
BenchResourceUsage,
|
|
13
|
+
BenchTestRef,
|
|
14
|
+
} from "@arcforge/types"
|
|
15
|
+
import { BENCH_PROTOCOL } from "@arcforge/types"
|
|
16
|
+
import { err } from "@arcforge/err"
|
|
17
|
+
import type { BenchLogT } from "./log"
|
|
18
|
+
import type { ResolverT } from "./resolve"
|
|
19
|
+
import type { TestRunnerT } from "../../services/test"
|
|
20
|
+
import { Workspace } from "./workspace"
|
|
21
|
+
|
|
22
|
+
export type RunnerOpts = { root: string; log: BenchLogT; resolver: ResolverT; tests: TestRunnerT }
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Parse a budget string into minor units of USD.
|
|
26
|
+
*
|
|
27
|
+
* Accepts "$0.50", "0.50", "50c". Returns null for undefined so an unset
|
|
28
|
+
* budget is unmistakably "no ceiling" rather than zero — a budget of zero
|
|
29
|
+
* would halt the run before the first trial, which is never what anyone means.
|
|
30
|
+
*/
|
|
31
|
+
function budgetUsd(value: string | undefined): number | null {
|
|
32
|
+
if (!value) return null
|
|
33
|
+
const trimmed = value.trim()
|
|
34
|
+
const cents = /^(\d+(?:\.\d+)?)\s*c$/i.exec(trimmed)
|
|
35
|
+
if (cents) return Number(cents[1]) / 100
|
|
36
|
+
const dollars = /^\$?\s*(\d+(?:\.\d+)?)$/.exec(trimmed)
|
|
37
|
+
if (dollars) return Number(dollars[1])
|
|
38
|
+
throw err("BENCH_CONFIG_INVALID", { detail: `budget ${JSON.stringify(value)} is not an amount like "$0.50"` })
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function hash(value: unknown): BenchHash {
|
|
42
|
+
return `sha256:${new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex")}`
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function testRef(event: Extract<AxonTestEvent, { type: "test:case:declare" }>): BenchTestRef {
|
|
46
|
+
return {
|
|
47
|
+
id: event.context.testId!,
|
|
48
|
+
file: event.context.file!,
|
|
49
|
+
suite: event.data.suite,
|
|
50
|
+
name: event.data.name,
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The concrete value selected on each axis, keyed by axis — what the subject is built from. */
|
|
55
|
+
function selectedValues(axes: BenchAxis[], coordinate: BenchCoordinate): Record<string, unknown> {
|
|
56
|
+
return Object.fromEntries(coordinate.axes.map(selection => {
|
|
57
|
+
const axis = axes.find(item => item.key === selection.key)
|
|
58
|
+
const value = axis?.values.find(item => item.id === selection.valueId)
|
|
59
|
+
if (!value) {
|
|
60
|
+
throw err("BENCH_AXIS_VALUE_NOT_FOUND", {
|
|
61
|
+
detail: `${selection.key}/${selection.valueId}`,
|
|
62
|
+
context: { axis: selection.key, valueId: selection.valueId },
|
|
63
|
+
})
|
|
64
|
+
}
|
|
65
|
+
return [selection.key, value.value]
|
|
66
|
+
}))
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function isBenchMessage(value: unknown): value is { channel: "axon:bench"; frame: { type: keyof BenchEventMap; context: Record<string, unknown>; data: unknown } } {
|
|
70
|
+
if (!value || typeof value !== "object") return false
|
|
71
|
+
const message = value as any
|
|
72
|
+
return message.channel === "axon:bench"
|
|
73
|
+
&& [
|
|
74
|
+
"bench:observation", "bench:artifact",
|
|
75
|
+
"bench:session:attach", "bench:session:detach", "bench:session:usage",
|
|
76
|
+
"bench:workspace:materialized", "bench:workspace:captured",
|
|
77
|
+
"bench:workspace:retained", "bench:workspace:cleaned",
|
|
78
|
+
].includes(message.frame?.type)
|
|
79
|
+
&& typeof message.frame.context === "object"
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function Runner(opts: RunnerOpts) {
|
|
83
|
+
return {
|
|
84
|
+
async run(
|
|
85
|
+
config: BenchNormalizedConfig,
|
|
86
|
+
coordinates: BenchCoordinate[],
|
|
87
|
+
agentPins?: Map<string, string>,
|
|
88
|
+
agentRoots?: Record<string, string>,
|
|
89
|
+
): Promise<string> {
|
|
90
|
+
const runId = Bun.randomUUIDv7()
|
|
91
|
+
const started = performance.now()
|
|
92
|
+
const preparedWorkspace = await Workspace({ root: opts.root, definition: config.workspace }).prepare()
|
|
93
|
+
const manifest: BenchRunManifest = {
|
|
94
|
+
runId,
|
|
95
|
+
bench: { ref: config.identity.name, name: config.identity.name, version: config.identity.version, hash: hash(config) },
|
|
96
|
+
protocols: BENCH_PROTOCOL,
|
|
97
|
+
schema: {
|
|
98
|
+
hash: hash({ measurements: config.measurements, dimensions: config.dimensions, artifacts: config.artifacts }),
|
|
99
|
+
measurements: config.measurements,
|
|
100
|
+
dimensions: config.dimensions,
|
|
101
|
+
artifacts: config.artifacts,
|
|
102
|
+
},
|
|
103
|
+
axes: config.axes.map(axis => ({ key: axis.key, label: axis.label ?? axis.key, values: axis.values.map(value => value.id) })),
|
|
104
|
+
cells: [],
|
|
105
|
+
trials: config.trials,
|
|
106
|
+
tests: [],
|
|
107
|
+
cases: [],
|
|
108
|
+
caseSetPin: hash(config.tests),
|
|
109
|
+
harness: { name: "axon", version: "0.1.0", runtime: process.version, bun: Bun.version },
|
|
110
|
+
workspace: preparedWorkspace.template,
|
|
111
|
+
startedAt: new Date().toISOString(),
|
|
112
|
+
}
|
|
113
|
+
const cells = []
|
|
114
|
+
for (const coordinate of coordinates) cells.push(await opts.resolver.resolve(config.axes, coordinate, agentPins))
|
|
115
|
+
manifest.cells = cells
|
|
116
|
+
await opts.log.emit({ benchRunId: runId }, "bench:run:start", { manifest })
|
|
117
|
+
await opts.log.writeManifest(runId, manifest)
|
|
118
|
+
await opts.log.emit({ benchRunId: runId }, "bench:workspace:prepared", { template: preparedWorkspace.template })
|
|
119
|
+
|
|
120
|
+
const perTrialBudget = budgetUsd(config.budget?.perTrial)
|
|
121
|
+
const totalBudget = budgetUsd(config.budget?.total)
|
|
122
|
+
let spentUsd = 0
|
|
123
|
+
let budgetExhausted = false
|
|
124
|
+
|
|
125
|
+
let expectedTrials = 0
|
|
126
|
+
let executedTrials = 0
|
|
127
|
+
let completedTrials = 0
|
|
128
|
+
let completedCases = 0
|
|
129
|
+
const required = config.measurements.filter(item => item.required).length
|
|
130
|
+
const requiredIds = new Set(config.measurements.filter(item => item.required).map(item => item.id))
|
|
131
|
+
const measured = new Set<string>()
|
|
132
|
+
const measurementStates: BenchMeasurementState[] = []
|
|
133
|
+
const preload = new URL("./preload.ts", import.meta.url).pathname
|
|
134
|
+
|
|
135
|
+
try {
|
|
136
|
+
for (let cellIndex = 0; cellIndex < coordinates.length && !budgetExhausted; cellIndex++) {
|
|
137
|
+
const coordinate = coordinates[cellIndex]!
|
|
138
|
+
const cell = cells[cellIndex]!
|
|
139
|
+
const cellStarted = performance.now()
|
|
140
|
+
await opts.log.emit({ benchRunId: runId, cellId: cell.id }, "bench:cell:start", { cell })
|
|
141
|
+
// Each level closes its OWN bracket on the way out. The
|
|
142
|
+
// run-level catch below cannot do it for them: by the time
|
|
143
|
+
// it runs, which cell and trial were in flight is lost.
|
|
144
|
+
try {
|
|
145
|
+
for (let trial = 0; trial < config.trials && !budgetExhausted; trial++) {
|
|
146
|
+
expectedTrials++
|
|
147
|
+
const trialStarted = performance.now()
|
|
148
|
+
const base = { benchRunId: runId, cellId: cell.id, trial }
|
|
149
|
+
await opts.log.emit(base, "bench:trial:start", {})
|
|
150
|
+
try {
|
|
151
|
+
let cases = 0
|
|
152
|
+
let completed = 0
|
|
153
|
+
const subject: BenchResourceUsage = { durationMs: 0, tokens: { input: 0, output: 0 }, engineCalls: 0, toolCalls: 0, errors: 0 }
|
|
154
|
+
let subjectCost = 0
|
|
155
|
+
let subjectCostKnown = true
|
|
156
|
+
const context = {
|
|
157
|
+
runId,
|
|
158
|
+
cellId: cell.id,
|
|
159
|
+
trial,
|
|
160
|
+
axes: cell.axes,
|
|
161
|
+
axisValues: selectedValues(config.axes, coordinate),
|
|
162
|
+
measurements: config.measurements,
|
|
163
|
+
dimensions: config.dimensions,
|
|
164
|
+
artifacts: config.artifacts,
|
|
165
|
+
artifactDir: join(opts.log.runDir(runId), "artifacts"),
|
|
166
|
+
benchRoot: opts.root,
|
|
167
|
+
// setup() is a function and the context crosses as
|
|
168
|
+
// JSON — the preload re-imports the config to reach it.
|
|
169
|
+
configPath: join(opts.root, "bench.config.ts"),
|
|
170
|
+
agentRoots,
|
|
171
|
+
workspace: {
|
|
172
|
+
definition: config.workspace,
|
|
173
|
+
template: preparedWorkspace.template,
|
|
174
|
+
sourcePath: preparedWorkspace.sourcePath,
|
|
175
|
+
root: join(opts.root, ".bench", "workspace"),
|
|
176
|
+
},
|
|
177
|
+
}
|
|
178
|
+
await opts.tests.run({
|
|
179
|
+
cwd: opts.root,
|
|
180
|
+
files: config.tests,
|
|
181
|
+
preloads: [preload],
|
|
182
|
+
env: { AXON_BENCH_CONTEXT: JSON.stringify(context) },
|
|
183
|
+
async onMessage(message) {
|
|
184
|
+
if (!isBenchMessage(message)) return
|
|
185
|
+
const frame = message.frame
|
|
186
|
+
const eventContext = { ...base, ...(frame.context as object) }
|
|
187
|
+
const emitted = await opts.log.emit(eventContext, frame.type, frame.data as never)
|
|
188
|
+
if (frame.type === "bench:session:usage") {
|
|
189
|
+
const usage = (frame.data as { usage: BenchResourceUsage }).usage
|
|
190
|
+
subject.durationMs += usage.durationMs
|
|
191
|
+
subject.tokens.input += usage.tokens.input
|
|
192
|
+
subject.tokens.output += usage.tokens.output
|
|
193
|
+
subject.engineCalls += usage.engineCalls
|
|
194
|
+
subject.toolCalls += usage.toolCalls
|
|
195
|
+
subject.errors += usage.errors
|
|
196
|
+
if (usage.costUsd === undefined) subjectCostKnown = false
|
|
197
|
+
else subjectCost += usage.costUsd
|
|
198
|
+
}
|
|
199
|
+
if (frame.type === "bench:observation") {
|
|
200
|
+
const measurementId = (frame.data as { measurementId?: string }).measurementId
|
|
201
|
+
if (measurementId && requiredIds.has(measurementId)) {
|
|
202
|
+
const slot = `${cell.id}:${trial}:${frame.context.testId ?? ""}:${frame.context.attempt ?? 0}:${measurementId}`
|
|
203
|
+
if (!measured.has(slot)) {
|
|
204
|
+
measured.add(slot)
|
|
205
|
+
measurementStates.push({ kind: "observed", observationId: String(emitted.time.seq) })
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
},
|
|
210
|
+
async onEvent(event) {
|
|
211
|
+
const eventContext = { ...base, testId: event.context.testId, attempt: event.context.attempt }
|
|
212
|
+
if (event.type === "test:case:declare") {
|
|
213
|
+
cases++
|
|
214
|
+
await opts.log.emit(eventContext, "bench:case:declare", { test: testRef(event) })
|
|
215
|
+
} else if (event.type === "test:case:pass" || event.type === "test:case:fail") {
|
|
216
|
+
completed++
|
|
217
|
+
completedCases++
|
|
218
|
+
for (const measurementId of requiredIds) {
|
|
219
|
+
const slot = `${cell.id}:${trial}:${event.context.testId ?? ""}:${event.context.attempt ?? 0}:${measurementId}`
|
|
220
|
+
if (!measured.has(slot)) measurementStates.push({ kind: "missing", reason: "not_emitted" })
|
|
221
|
+
}
|
|
222
|
+
await opts.log.emit(eventContext, "bench:case:complete", {
|
|
223
|
+
status: event.type === "test:case:pass" ? "passed" : "failed",
|
|
224
|
+
durationMs: event.data.durationMs,
|
|
225
|
+
...(event.type === "test:case:fail" ? { error: event.data.error } : {}),
|
|
226
|
+
})
|
|
227
|
+
} else if (event.type === "test:case:skip" || event.type === "test:case:todo") {
|
|
228
|
+
completed++
|
|
229
|
+
completedCases++
|
|
230
|
+
for (const _measurementId of requiredIds) measurementStates.push({ kind: "missing", reason: "filtered" })
|
|
231
|
+
await opts.log.emit(eventContext, "bench:case:complete", {
|
|
232
|
+
status: event.type === "test:case:skip" ? "skipped" : "todo",
|
|
233
|
+
durationMs: 0,
|
|
234
|
+
})
|
|
235
|
+
}
|
|
236
|
+
},
|
|
237
|
+
})
|
|
238
|
+
if (cases === 0) throw err("BENCH_NO_CASES")
|
|
239
|
+
executedTrials++
|
|
240
|
+
if (completed === cases) completedTrials++
|
|
241
|
+
|
|
242
|
+
// Budget is checked AFTER the trial, not during: a trial
|
|
243
|
+
// is the smallest unit whose cost is knowable, and killing
|
|
244
|
+
// one mid-flight would leave a half-run that measures
|
|
245
|
+
// nothing while still having been paid for. Exceeding the
|
|
246
|
+
// ceiling is a fault, so the trial is excluded from scoring
|
|
247
|
+
// rather than counted as a failure of the agent.
|
|
248
|
+
if (subjectCostKnown) {
|
|
249
|
+
spentUsd += subjectCost
|
|
250
|
+
if (perTrialBudget !== null && subjectCost > perTrialBudget) {
|
|
251
|
+
await opts.log.emit(base, "bench:process:fault", {
|
|
252
|
+
fault: { code: "budget", message: `trial spent $${subjectCost.toFixed(4)}, over the $${perTrialBudget} per-trial budget`, context: { runId, cellId: cell.id, trial } },
|
|
253
|
+
})
|
|
254
|
+
}
|
|
255
|
+
if (totalBudget !== null && spentUsd > totalBudget) {
|
|
256
|
+
await opts.log.emit(base, "bench:process:fault", {
|
|
257
|
+
fault: { code: "budget", message: `run spent $${spentUsd.toFixed(4)}, over the $${totalBudget} total budget — stopping`, context: { runId, cellId: cell.id, trial } },
|
|
258
|
+
})
|
|
259
|
+
budgetExhausted = true
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
await opts.log.emit(base, "bench:trial:complete", {
|
|
264
|
+
durationMs: performance.now() - trialStarted,
|
|
265
|
+
physics: {
|
|
266
|
+
subject: { ...subject, ...(subjectCostKnown ? { costUsd: subjectCost } : {}) },
|
|
267
|
+
harness: { durationMs: performance.now() - trialStarted },
|
|
268
|
+
},
|
|
269
|
+
})
|
|
270
|
+
} catch (error) {
|
|
271
|
+
await opts.log.emit(base, "bench:trial:failed", {
|
|
272
|
+
durationMs: performance.now() - trialStarted,
|
|
273
|
+
error: err(error),
|
|
274
|
+
})
|
|
275
|
+
throw error
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
await opts.log.emit({ benchRunId: runId, cellId: cell.id }, "bench:cell:complete", { durationMs: performance.now() - cellStarted })
|
|
279
|
+
} catch (error) {
|
|
280
|
+
await opts.log.emit({ benchRunId: runId, cellId: cell.id }, "bench:cell:failed", {
|
|
281
|
+
durationMs: performance.now() - cellStarted,
|
|
282
|
+
error: err(error),
|
|
283
|
+
})
|
|
284
|
+
throw error
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
const coverage: BenchCoverage = {
|
|
288
|
+
expectedTrials,
|
|
289
|
+
executedTrials,
|
|
290
|
+
completedTrials,
|
|
291
|
+
expectedMeasurements: completedCases * required,
|
|
292
|
+
observedMeasurements: measured.size,
|
|
293
|
+
missing: completedCases * required > measured.size
|
|
294
|
+
? { not_emitted: measurementStates.filter(item => item.kind === "missing" && item.reason === "not_emitted").length,
|
|
295
|
+
filtered: measurementStates.filter(item => item.kind === "missing" && item.reason === "filtered").length }
|
|
296
|
+
: {},
|
|
297
|
+
measurements: measurementStates,
|
|
298
|
+
}
|
|
299
|
+
await opts.log.emit({ benchRunId: runId }, "bench:run:complete", { durationMs: performance.now() - started, coverage })
|
|
300
|
+
return runId
|
|
301
|
+
} catch (error) {
|
|
302
|
+
await opts.log.emit({ benchRunId: runId }, "bench:run:failed", { error: err(error) })
|
|
303
|
+
throw error
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
export type RunnerT = ReturnType<typeof Runner>
|