@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/bench/README.md +94 -0
- package/bench/bfcl/checker.ts +313 -0
- package/bench/bfcl/data.ts +93 -0
- package/bench/bfcl/observe.ts +93 -0
- package/bench/bfcl/run.ts +213 -0
- package/bench/bfcl/tools.ts +72 -0
- package/bench/bridge/bridge.ts +392 -0
- package/bench/bridge/openai.ts +150 -0
- package/bench/bridge/python/qvac_bench/__init__.py +3 -0
- package/bench/bridge/python/qvac_bench/airline.py +42 -0
- package/bench/bridge/python/qvac_bench/approval.py +26 -0
- package/bench/bridge/python/qvac_bench/bfcl_multi_turn.py +152 -0
- package/bench/bridge/python/qvac_bench/client.py +88 -0
- package/bench/bridge/python/qvac_bench/harbor_agent.py +85 -0
- package/bench/bridge/python/qvac_bench/session.py +107 -0
- package/bench/bridge/python/qvac_bench/speed.py +30 -0
- package/bench/bridge/python/qvac_bench/tau2_agent.py +106 -0
- package/bench/bridge/python/qvac_bench/tools.py +11 -0
- package/bench/bridge/python/run_bfcl_multi_turn.py +95 -0
- package/bench/bridge/python/run_tau2.py +150 -0
- package/bench/bridge/python/smoke.py +47 -0
- package/bench/bridge/python/test_airline.py +28 -0
- package/bench/bridge/python/test_approval.py +57 -0
- package/bench/bridge/python/test_bfcl_multi_turn.py +185 -0
- package/bench/bridge/python/test_harbor_agent.py +107 -0
- package/bench/bridge/serve.ts +52 -0
- package/bench/bridge/transcript.ts +52 -0
- package/bench/bun-tsconfig.json +15 -0
- package/bench/cpu-main.mjs +216 -0
- package/bench/cpu.ts +457 -0
- package/bench/evidence/2026-09-28-review15.md +225 -0
- package/bench/evidence/redact-bun1.3.14.json +92 -0
- package/bench/evidence/redact-node24.11.json +92 -0
- package/bench/graph-metrics.ts +43 -0
- package/bench/harness.ts +73 -0
- package/bench/issue129-matrix.mjs +722 -0
- package/bench/redact-adversarial.ts +256 -0
- package/bench/repeat-cpu.mjs +281 -0
- package/bench/report.ts +1075 -0
- package/bench/skill-eval/harness.ts +30 -0
- package/bench/skill-eval/laws.ts +37 -0
- package/bench/skill-eval/mermaid/client.ts +132 -0
- package/bench/skill-eval/mermaid/validate.mjs +115 -0
- package/bench/skill-eval/pack.ts +61 -0
- package/bench/skill-eval/packs/diagrams/cases.ts +281 -0
- package/bench/skill-eval/packs/diagrams/index.ts +50 -0
- package/bench/skill-eval/packs/diagrams/score.ts +74 -0
- package/bench/skill-eval/packs/index.ts +5 -0
- package/bench/skill-eval/packs/weather/cases.ts +279 -0
- package/bench/skill-eval/packs/weather/fixtures/wttr.json +566 -0
- package/bench/skill-eval/packs/weather/fixtures.ts +84 -0
- package/bench/skill-eval/packs/weather/index.ts +55 -0
- package/bench/skill-eval/packs/weather/score.ts +92 -0
- package/bench/skill-eval/report.ts +194 -0
- package/bench/skill-eval/run.ts +316 -0
- package/bench/speed.ts +80 -0
- package/package.json +4 -13
package/bench/cpu.ts
ADDED
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
// CPU and run-log benchmark for Node, Bun, and Bare.
|
|
2
|
+
import { CompletionEngine } from '../lib/capabilities/completion-engine/index.ts'
|
|
3
|
+
import type { CompletionSdk, Message, ToolDefinition } from '../lib/contract/index.ts'
|
|
4
|
+
import b4a from 'b4a'
|
|
5
|
+
import process from 'process'
|
|
6
|
+
import { RunLog, type EngineBoot, type EngineStep } from '../lib/run-log/index.ts'
|
|
7
|
+
import { driveCompletion } from '../lib/capabilities/completion-engine/driver.ts'
|
|
8
|
+
import { HookPipeline } from '../lib/hooks/index.ts'
|
|
9
|
+
import { defaultAgentPorts } from '../lib/capabilities/agent-ports.ts'
|
|
10
|
+
import { outboundGuard } from '../lib/tools/outbound-guard.ts'
|
|
11
|
+
import { installAbortControllerPolyfill } from '../lib/helpers/abort-polyfill.ts'
|
|
12
|
+
|
|
13
|
+
installAbortControllerPolyfill()
|
|
14
|
+
|
|
15
|
+
type RedactionMode = 'nohost' | 'plain' | 'bench' | 'production'
|
|
16
|
+
type Spacing = 'immediate' | 'timeout'
|
|
17
|
+
type Workload = 'tools' | 'stream'
|
|
18
|
+
interface CliOptions {
|
|
19
|
+
runs: number
|
|
20
|
+
mode: string
|
|
21
|
+
spacing: string
|
|
22
|
+
workload: string
|
|
23
|
+
json: boolean
|
|
24
|
+
}
|
|
25
|
+
interface WorkloadInput {
|
|
26
|
+
sdk: CompletionSdk
|
|
27
|
+
args: Parameters<typeof CompletionEngine.run>[1]
|
|
28
|
+
host: ReturnType<typeof countingHost>
|
|
29
|
+
}
|
|
30
|
+
interface CountOptions {
|
|
31
|
+
count?: boolean
|
|
32
|
+
}
|
|
33
|
+
interface ModeResultsOptions {
|
|
34
|
+
mode: RedactionMode
|
|
35
|
+
runs: number
|
|
36
|
+
spacings: Spacing[]
|
|
37
|
+
workloads: Workload[]
|
|
38
|
+
}
|
|
39
|
+
type BenchResult =
|
|
40
|
+
| {
|
|
41
|
+
workload: 'tools'
|
|
42
|
+
mode: RedactionMode
|
|
43
|
+
runs: number
|
|
44
|
+
cpuMs: number
|
|
45
|
+
cpuSamplesMs: number[]
|
|
46
|
+
logBytes: number
|
|
47
|
+
records: number
|
|
48
|
+
}
|
|
49
|
+
| {
|
|
50
|
+
workload: 'stream'
|
|
51
|
+
mode: RedactionMode
|
|
52
|
+
spacing: Spacing
|
|
53
|
+
runs: number
|
|
54
|
+
cpuMs: number
|
|
55
|
+
cpuSamplesMs: number[]
|
|
56
|
+
usPerToken: number
|
|
57
|
+
contentEvents: number
|
|
58
|
+
logBytes: number
|
|
59
|
+
records: number
|
|
60
|
+
}
|
|
61
|
+
const TOKENS = 2000
|
|
62
|
+
// Keep every token observable by starting outside round.ts's burst window.
|
|
63
|
+
const FIRST_TOKEN_MS = 110
|
|
64
|
+
const USAGE = [
|
|
65
|
+
'Usage: cpu.ts [runs] [--mode=nohost|plain|bench|production|all]',
|
|
66
|
+
' [--spacing=immediate|timeout|all] [--workload=tools|stream|all] [--json]',
|
|
67
|
+
'Node: node --experimental-strip-types bench/cpu.ts',
|
|
68
|
+
'Bun: bun --tsconfig-override=bench/bun-tsconfig.json bench/cpu.ts',
|
|
69
|
+
'Bare: bare bench/cpu.ts'
|
|
70
|
+
].join('\n')
|
|
71
|
+
|
|
72
|
+
function parseOptions(args: string[]): CliOptions {
|
|
73
|
+
let runs = 3
|
|
74
|
+
let mode = 'all'
|
|
75
|
+
let spacing = 'all'
|
|
76
|
+
let workload = 'all'
|
|
77
|
+
let json = false
|
|
78
|
+
let positionalRuns = false
|
|
79
|
+
|
|
80
|
+
for (const arg of args) {
|
|
81
|
+
if (arg === '--json') {
|
|
82
|
+
json = true
|
|
83
|
+
} else if (arg === '--help' || arg === '-h') {
|
|
84
|
+
console.log(USAGE)
|
|
85
|
+
process.exit(0)
|
|
86
|
+
} else if (arg.startsWith('--mode=')) {
|
|
87
|
+
mode = arg.slice('--mode='.length)
|
|
88
|
+
} else if (arg.startsWith('--spacing=')) {
|
|
89
|
+
spacing = arg.slice('--spacing='.length)
|
|
90
|
+
} else if (arg.startsWith('--workload=')) {
|
|
91
|
+
workload = arg.slice('--workload='.length)
|
|
92
|
+
} else if (!arg.startsWith('-') && !positionalRuns) {
|
|
93
|
+
runs = Number(arg)
|
|
94
|
+
positionalRuns = true
|
|
95
|
+
} else {
|
|
96
|
+
throw new Error(`Unknown option: ${arg}`)
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (!Number.isInteger(runs) || runs < 1) {
|
|
101
|
+
throw new Error('runs must be a positive integer')
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return { runs, mode, spacing, workload, json }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function selectValues<Value extends string>(
|
|
108
|
+
value: string,
|
|
109
|
+
allowed: readonly Value[],
|
|
110
|
+
label: string
|
|
111
|
+
): Value[] {
|
|
112
|
+
if (value === 'all') {
|
|
113
|
+
return [...allowed]
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const selected = allowed.find((option) => option === value)
|
|
117
|
+
if (selected === undefined) {
|
|
118
|
+
throw new Error(`Unknown ${label}: ${value}`)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return [selected]
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function options() {
|
|
125
|
+
const parsed = parseOptions(process.argv.slice(2))
|
|
126
|
+
return {
|
|
127
|
+
runs: parsed.runs,
|
|
128
|
+
modes: selectValues(parsed.mode, ['nohost', 'plain', 'bench', 'production'], 'mode'),
|
|
129
|
+
spacings: selectValues(parsed.spacing, ['immediate', 'timeout'], 'spacing'),
|
|
130
|
+
workloads: selectValues(parsed.workload, ['tools', 'stream'], 'workload'),
|
|
131
|
+
json: parsed.json
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async function collect<Event extends { type?: string }>(
|
|
136
|
+
run: AsyncIterable<Event>
|
|
137
|
+
): Promise<Event[]> {
|
|
138
|
+
const events: Event[] = []
|
|
139
|
+
for await (const event of run) {
|
|
140
|
+
events.push(event)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
return events
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function cpuMs(start: NodeJS.CpuUsage): number {
|
|
147
|
+
const d = process.cpuUsage(start)
|
|
148
|
+
return (d.user + d.system) / 1000
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function median(values: number[]): number {
|
|
152
|
+
const sorted = [...values].sort((a, b) => a - b)
|
|
153
|
+
const value = sorted[Math.floor(sorted.length / 2)]
|
|
154
|
+
if (value === undefined) {
|
|
155
|
+
throw new Error('Cannot find the median of an empty sample')
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return value
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// 101 messages, ~200 KB: a review-sized conversation.
|
|
162
|
+
function history(): Message[] {
|
|
163
|
+
return Array.from({ length: 101 }, (_, i) => ({
|
|
164
|
+
role: i % 2 ? 'assistant' : 'user',
|
|
165
|
+
content: `${i} ${'x'.repeat(2000)}`
|
|
166
|
+
}))
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function readTool(name: string): ToolDefinition {
|
|
170
|
+
return {
|
|
171
|
+
name,
|
|
172
|
+
description: 'read something',
|
|
173
|
+
toolset: null,
|
|
174
|
+
parameters: { type: 'object', properties: { path: { type: 'string' } } },
|
|
175
|
+
effect: () => 'read',
|
|
176
|
+
autoApprove: true,
|
|
177
|
+
execute: () => Promise.resolve({ content: `${name} ${'r'.repeat(2048)}` })
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// 3 tool rounds x 2 read calls/round, then a plain answer round.
|
|
182
|
+
function toolWorkloadSdk(tools: ToolDefinition[]): CompletionSdk {
|
|
183
|
+
let round = 0
|
|
184
|
+
return {
|
|
185
|
+
completion: () => {
|
|
186
|
+
const r = round++
|
|
187
|
+
if (r < 3) {
|
|
188
|
+
return {
|
|
189
|
+
tokenStream: (async function* () {})(),
|
|
190
|
+
final: Promise.resolve({
|
|
191
|
+
contentText: '',
|
|
192
|
+
toolCalls: tools.map((t, i) => ({
|
|
193
|
+
id: `call-${r}-${i}`,
|
|
194
|
+
name: t.name,
|
|
195
|
+
arguments: { path: `f${r}${i}` }
|
|
196
|
+
})),
|
|
197
|
+
raw: { fullText: '' }
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
tokenStream: (async function* () {
|
|
204
|
+
yield 'done'
|
|
205
|
+
})(),
|
|
206
|
+
final: Promise.resolve({ contentText: 'done', toolCalls: [], raw: { fullText: 'done' } })
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function countingHost(count: boolean, mode: RedactionMode) {
|
|
213
|
+
const core = memoryCore()
|
|
214
|
+
const records: unknown[] = []
|
|
215
|
+
const stats = { bytes: 0, records: 0 }
|
|
216
|
+
let boot: EngineBoot = RunLog.createEngineBoot()
|
|
217
|
+
return {
|
|
218
|
+
core,
|
|
219
|
+
stats,
|
|
220
|
+
get boot() {
|
|
221
|
+
return boot
|
|
222
|
+
},
|
|
223
|
+
async onStep(step: EngineStep) {
|
|
224
|
+
const next = RunLog.applyEngineStep(boot, step)
|
|
225
|
+
if (mode === 'production') {
|
|
226
|
+
for (const record of step.records) {
|
|
227
|
+
await core.append(record)
|
|
228
|
+
}
|
|
229
|
+
} else if (mode === 'bench') {
|
|
230
|
+
for (const record of step.records) {
|
|
231
|
+
records.push(JSON.parse(JSON.stringify(record)))
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (count) {
|
|
236
|
+
for (const record of step.records) {
|
|
237
|
+
stats.bytes += b4a.byteLength(JSON.stringify(record))
|
|
238
|
+
stats.records++
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
boot = next
|
|
243
|
+
},
|
|
244
|
+
digestKey: new Uint8Array(32).fill(7)
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function memoryCore() {
|
|
249
|
+
const records: unknown[] = []
|
|
250
|
+
return {
|
|
251
|
+
get length() {
|
|
252
|
+
return records.length
|
|
253
|
+
},
|
|
254
|
+
ready: () => Promise.resolve(),
|
|
255
|
+
get(index: number) {
|
|
256
|
+
const value = records[index]
|
|
257
|
+
return Promise.resolve(value === undefined ? null : JSON.parse(JSON.stringify(value)))
|
|
258
|
+
},
|
|
259
|
+
append(value: unknown) {
|
|
260
|
+
records.push(JSON.parse(JSON.stringify(value)))
|
|
261
|
+
return Promise.resolve()
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function runWithMode(mode: RedactionMode, { sdk, args, host }: WorkloadInput) {
|
|
267
|
+
// No onStep: the run log journal still runs, so only the host append is left out.
|
|
268
|
+
if (mode === 'nohost') {
|
|
269
|
+
return collect(CompletionEngine.run(sdk, args))
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const completionOptions = {
|
|
273
|
+
boot: host.boot,
|
|
274
|
+
onStep: host.onStep,
|
|
275
|
+
requestDigestKey: host.digestKey
|
|
276
|
+
}
|
|
277
|
+
if (mode === 'production') {
|
|
278
|
+
return collect(CompletionEngine.run(sdk, args, completionOptions))
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const redactor = mode === 'bench' ? RunLog.createRedactor(args.tools ?? []) : undefined
|
|
282
|
+
const driverOptions: Parameters<typeof driveCompletion>[2] = {
|
|
283
|
+
...completionOptions,
|
|
284
|
+
pipeline: new HookPipeline(HookPipeline.defaults()),
|
|
285
|
+
outbound: outboundGuard,
|
|
286
|
+
agentPorts: defaultAgentPorts
|
|
287
|
+
}
|
|
288
|
+
if (redactor) {
|
|
289
|
+
Object.assign(driverOptions, {
|
|
290
|
+
redact: redactor,
|
|
291
|
+
redactShared: redactor.redactShared,
|
|
292
|
+
redactOperationResult: redactor.redactOperationResult,
|
|
293
|
+
redactStep: redactor.redactStep
|
|
294
|
+
})
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return collect(driveCompletion(args, () => Promise.resolve({ sdk }), driverOptions))
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async function runToolWorkload(mode: RedactionMode, options: CountOptions = {}) {
|
|
301
|
+
const { count = false } = options
|
|
302
|
+
const tools = [readTool('read_a'), readTool('read_b')]
|
|
303
|
+
const host = countingHost(count, mode)
|
|
304
|
+
const { stats } = host
|
|
305
|
+
const args = { runId: 'bench-tools', modelId: 'm', messages: history(), tools }
|
|
306
|
+
const start = process.cpuUsage()
|
|
307
|
+
await runWithMode(mode, { sdk: toolWorkloadSdk(tools), args, host })
|
|
308
|
+
return { ms: cpuMs(start), ...stats }
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function streamingSdk(spacing: Spacing): CompletionSdk {
|
|
312
|
+
return {
|
|
313
|
+
completion: () => ({
|
|
314
|
+
tokenStream: (async function* () {
|
|
315
|
+
await new Promise((resolve) => setTimeout(resolve, FIRST_TOKEN_MS))
|
|
316
|
+
for (let i = 0; i < TOKENS; i++) {
|
|
317
|
+
await new Promise((resolve) =>
|
|
318
|
+
spacing === 'immediate' ? setImmediate(resolve) : setTimeout(resolve, 0)
|
|
319
|
+
)
|
|
320
|
+
yield 't'
|
|
321
|
+
}
|
|
322
|
+
})(),
|
|
323
|
+
final: Promise.resolve({
|
|
324
|
+
contentText: 't'.repeat(TOKENS),
|
|
325
|
+
toolCalls: [],
|
|
326
|
+
raw: { fullText: '' }
|
|
327
|
+
})
|
|
328
|
+
})
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
async function runStreaming(mode: RedactionMode, spacing: Spacing, options: CountOptions = {}) {
|
|
333
|
+
const { count = false } = options
|
|
334
|
+
const host = countingHost(count, mode)
|
|
335
|
+
const { stats } = host
|
|
336
|
+
const args = { runId: `bench-stream-${Math.random()}`, modelId: 'm', messages: history() }
|
|
337
|
+
const start = process.cpuUsage()
|
|
338
|
+
const events = await runWithMode(mode, { sdk: streamingSdk(spacing), args, host })
|
|
339
|
+
const ms = cpuMs(start)
|
|
340
|
+
return {
|
|
341
|
+
ms,
|
|
342
|
+
contentEvents: events.filter((event) => event.type === 'content').length,
|
|
343
|
+
...stats
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
async function measureTools(mode: RedactionMode, runs: number): Promise<BenchResult> {
|
|
348
|
+
const log = await runToolWorkload(mode, { count: true })
|
|
349
|
+
const samples = []
|
|
350
|
+
for (let i = 0; i < runs; i++) {
|
|
351
|
+
samples.push(await runToolWorkload(mode))
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
return {
|
|
355
|
+
workload: 'tools',
|
|
356
|
+
mode,
|
|
357
|
+
runs,
|
|
358
|
+
cpuMs: median(samples.map((sample) => sample.ms)),
|
|
359
|
+
cpuSamplesMs: samples.map((sample) => sample.ms),
|
|
360
|
+
logBytes: log.bytes,
|
|
361
|
+
records: log.records
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
async function measureStream(
|
|
366
|
+
mode: RedactionMode,
|
|
367
|
+
spacing: Spacing,
|
|
368
|
+
runs: number
|
|
369
|
+
): Promise<BenchResult> {
|
|
370
|
+
const log = await runStreaming(mode, spacing, { count: true })
|
|
371
|
+
const samples = []
|
|
372
|
+
for (let i = 0; i < runs; i++) {
|
|
373
|
+
samples.push(await runStreaming(mode, spacing))
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const cpu = median(samples.map((sample) => sample.ms))
|
|
377
|
+
|
|
378
|
+
return {
|
|
379
|
+
workload: 'stream',
|
|
380
|
+
mode,
|
|
381
|
+
spacing,
|
|
382
|
+
runs,
|
|
383
|
+
cpuMs: cpu,
|
|
384
|
+
cpuSamplesMs: samples.map((sample) => sample.ms),
|
|
385
|
+
usPerToken: (cpu * 1000) / TOKENS,
|
|
386
|
+
contentEvents: log.contentEvents,
|
|
387
|
+
logBytes: log.bytes,
|
|
388
|
+
records: log.records
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
async function resultsForMode({
|
|
393
|
+
mode,
|
|
394
|
+
runs,
|
|
395
|
+
spacings,
|
|
396
|
+
workloads
|
|
397
|
+
}: ModeResultsOptions): Promise<BenchResult[]> {
|
|
398
|
+
const results: BenchResult[] = []
|
|
399
|
+
if (workloads.includes('tools')) {
|
|
400
|
+
results.push(await measureTools(mode, runs))
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
if (workloads.includes('stream')) {
|
|
404
|
+
for (const spacing of spacings) {
|
|
405
|
+
results.push(await measureStream(mode, spacing, runs))
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
return results
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function runtimeInfo() {
|
|
413
|
+
const bun = Reflect.get(globalThis, 'Bun')
|
|
414
|
+
if (bun && typeof bun === 'object' && 'version' in bun && typeof bun.version === 'string') {
|
|
415
|
+
return { runtime: 'bun', version: bun.version }
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
return {
|
|
419
|
+
runtime: process.release?.name ?? (process.version.startsWith('v1.') ? 'bare' : 'unknown'),
|
|
420
|
+
version: process.version
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
async function main() {
|
|
425
|
+
const { runs, modes, spacings, workloads, json } = options()
|
|
426
|
+
const results: BenchResult[] = []
|
|
427
|
+
for (const mode of modes) {
|
|
428
|
+
results.push(...(await resultsForMode({ mode, runs, spacings, workloads })))
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const report = { ...runtimeInfo(), runs, results }
|
|
432
|
+
if (json) {
|
|
433
|
+
console.log(JSON.stringify(report))
|
|
434
|
+
return
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
console.log(`${report.runtime} ${report.version}; median of ${runs} runs`)
|
|
438
|
+
for (const result of results) {
|
|
439
|
+
const logMiB = (result.logBytes / (1024 * 1024)).toFixed(3)
|
|
440
|
+
if (result.workload === 'tools') {
|
|
441
|
+
console.log(
|
|
442
|
+
`tools mode=${result.mode} cpu=${result.cpuMs.toFixed(2)}ms log=${logMiB}MiB records=${result.records}`
|
|
443
|
+
)
|
|
444
|
+
} else {
|
|
445
|
+
const usPerToken =
|
|
446
|
+
'usPerToken' in result && typeof result.usPerToken === 'number' ? result.usPerToken : 0
|
|
447
|
+
console.log(
|
|
448
|
+
`stream mode=${result.mode} spacing=${result.spacing} cpu=${result.cpuMs.toFixed(2)}ms usPerToken=${usPerToken.toFixed(2)} contentEvents=${result.contentEvents} log=${logMiB}MiB records=${result.records}`
|
|
449
|
+
)
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
main().catch((error) => {
|
|
455
|
+
console.error(error)
|
|
456
|
+
process.exitCode = 1
|
|
457
|
+
})
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# Review 15 performance re-measure — 2026-09-28
|
|
2
|
+
|
|
3
|
+
No tuning. Both runtimes exceed the 1.5× streaming and 2× tool-workload targets in this measurement.
|
|
4
|
+
|
|
5
|
+
## Revisions and method
|
|
6
|
+
|
|
7
|
+
- Branch: `refactor/add-state-machine`, HEAD `a029f2ec07de03c9bdce306c0d22494917399d4e` plus the uncommitted Lot 1 ownership fix in `lib/run-log-redact.ts`. Lots 2–3 change documentation and tests.
|
|
8
|
+
- Main comparison: merge-base `7d4a1ec282ed29b2480b551e6a4670d448441e63`, obtained with `git merge-base HEAD origin/main`.
|
|
9
|
+
- Machine: Darwin arm64. Runtimes: Node v24.11.0 and Bun 1.3.14.
|
|
10
|
+
- Method: [issue #129](https://github.com/tetherto/qvac-harness/issues/129), “Measurement method.” Fake SDK, 101 messages (roughly 200 KB), 2,000 token events, 110 ms initial delay followed by `setImmediate` spacing; 3 rounds × 2 auto-approved read tools followed by an answer.
|
|
11
|
+
- Three fresh processes per runtime/revision, three timed runs per workload per process. Each workload first runs once to count log bytes and warm the process. CPU time is `process.cpuUsage()`, not elapsed time. Results below are medians of the three process medians; all nine CPU samples are retained below.
|
|
12
|
+
- Process pairs ran sequentially: branch/main, main/branch, branch/main; Node pair then Bun pair in each pass. No benchmark processes overlapped. The shared machine was not reserved, so other agent activity and Bun’s wide stream samples remain sources of variance.
|
|
13
|
+
- Branch used the existing `bench/cpu.ts` in production mode with host boot folding and JSON-copy storage. Every stream measurement reported 2,000 content events.
|
|
14
|
+
- Baseline deviation: the merge-base predates both `bench/cpu.ts` and the run-log API. Its temporary copy used identical workloads, warm-up, timing, and reporting, with only the entry-point/storage adapter changed to `completion(sdk, args)`. Main accepts two arguments and emits no journal records; its log bytes/records are zero. The exact adapter diff is below.
|
|
15
|
+
- Benchmark SHA-256: `305ff7270f08b058e706efd6de75a57a109817a7f03106b608bc9ec1b2c15ba1`.
|
|
16
|
+
- Measured redactor SHA-256: `007bf7c3146e7e2132062c50f62f709e6495e13ae03ebe3017f49621a0f298b0`.
|
|
17
|
+
|
|
18
|
+
## Results
|
|
19
|
+
|
|
20
|
+
| Runtime | Workload | Branch | Main | Branch/main | Limit | Meets limit |
|
|
21
|
+
| ------- | -------- | ---------------: | ---------------: | ----------: | ----: | ----------- |
|
|
22
|
+
| Node | stream | 14.3875 µs/token | 7.6265 µs/token | 1.887× | 1.5× | no |
|
|
23
|
+
| Node | tools | 27.7580 ms | 1.3510 ms | 20.546× | 2× | no |
|
|
24
|
+
| Bun | stream | 24.7500 µs/token | 15.8645 µs/token | 1.560× | 1.5× | no |
|
|
25
|
+
| Bun | tools | 36.8500 ms | 2.0140 ms | 18.297× | 2× | no |
|
|
26
|
+
|
|
27
|
+
| Runtime | Revision | Tool log bytes / records | Stream log bytes / records (processes 1, 2, 3) |
|
|
28
|
+
| ------- | -------- | -------------------------------- | ---------------------------------------------- |
|
|
29
|
+
| Node | branch | 324611, 324611, 324611 / 57 each | 248235, 248245, 248235 / 9 each |
|
|
30
|
+
| Node | main | 0, 0, 0 / 0 each | 0, 0, 0 / 0 each |
|
|
31
|
+
| Bun | branch | 324611, 324611, 324611 / 57 each | 248235, 248235, 248240 / 9 each |
|
|
32
|
+
| Bun | main | 0, 0, 0 / 0 each | 0, 0, 0 / 0 each |
|
|
33
|
+
|
|
34
|
+
Stream log sizes vary slightly because the benchmark generates a random run ID. Tool log sizes were identical across these processes. These measurements do not verify the golden-bytes fixture; it was not regenerated.
|
|
35
|
+
|
|
36
|
+
## Exact benchmark commands
|
|
37
|
+
|
|
38
|
+
Run each command three times per revision, in the interleaved order above:
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
QVAC_NO_SKILL_BIN_PROBE=1 /tmp/node-v24.11.0-darwin-arm64/bin/node --experimental-strip-types bench/cpu.ts 3 --mode=production --spacing=immediate --workload=all --json
|
|
42
|
+
QVAC_NO_SKILL_BIN_PROBE=1 /Users/matheus/.bun/bin/bun --tsconfig-override=bench/bun-tsconfig.json bench/cpu.ts 3 --mode=production --spacing=immediate --workload=all --json
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The baseline was created with `git worktree add --detach /tmp/qvac-review15-giokq5_1/main 7d4a1ec282ed29b2480b551e6a4670d448441e63`. Its `node_modules` symlink reused the branch’s installed dependency resolutions, and `bun-tsconfig.json` was copied unchanged. The temporary worktree was removed after measurement.
|
|
46
|
+
|
|
47
|
+
## Raw CPU samples
|
|
48
|
+
|
|
49
|
+
| Runtime | Revision | Process | Tool ms (3 runs) | Stream CPU ms (3 runs; divide by 2 for µs/token) |
|
|
50
|
+
| ------- | -------- | ------: | ---------------------- | ------------------------------------------------ |
|
|
51
|
+
| node | branch | 1 | 30.668, 27.576, 29.124 | 25.374, 32.936, 32.531 |
|
|
52
|
+
| node | main | 1 | 1.743, 1.351, 1.123 | 10.491, 15.769, 15.307 |
|
|
53
|
+
| bun | branch | 1 | 58.115, 35.710, 36.574 | 29.654, 49.500, 107.315 |
|
|
54
|
+
| bun | main | 1 | 3.041, 2.292, 1.223 | 36.103, 13.894, 31.729 |
|
|
55
|
+
| node | main | 2 | 1.584, 1.494, 1.095 | 15.253, 12.610, 17.444 |
|
|
56
|
+
| node | branch | 2 | 32.229, 24.765, 27.758 | 28.775, 25.674, 31.390 |
|
|
57
|
+
| bun | main | 2 | 3.081, 2.014, 1.081 | 35.047, 16.004, 20.184 |
|
|
58
|
+
| bun | branch | 2 | 54.465, 34.368, 38.625 | 43.572, 48.618, 69.273 |
|
|
59
|
+
| node | branch | 3 | 29.307, 24.338, 27.701 | 26.108, 26.253, 27.608 |
|
|
60
|
+
| node | main | 3 | 1.519, 1.345, 1.032 | 11.542, 16.398, 13.744 |
|
|
61
|
+
| bun | branch | 3 | 54.344, 34.818, 36.850 | 39.626, 73.367, 72.450 |
|
|
62
|
+
| bun | main | 3 | 2.887, 1.857, 1.112 | 39.319, 41.242, 71.949 |
|
|
63
|
+
|
|
64
|
+
## Baseline adapter diff
|
|
65
|
+
|
|
66
|
+
Apply to a temporary copy of the branch benchmark; it does not alter the workload bodies or measurement loops.
|
|
67
|
+
|
|
68
|
+
```diff
|
|
69
|
+
--- branch/bench/cpu.ts
|
|
70
|
+
+++ main/bench/cpu.ts
|
|
71
|
+
@@ -6,17 +6,6 @@
|
|
72
|
+
type ToolDefinition
|
|
73
|
+
} from '../../lib/capabilities/completion.ts'
|
|
74
|
+
import process from 'process'
|
|
75
|
+
-import {
|
|
76
|
+
- applyEngineStep,
|
|
77
|
+
- createEngineBoot,
|
|
78
|
+
- type EngineBoot,
|
|
79
|
+
- type EngineStep
|
|
80
|
+
-} from '../../lib/run-log-protocol.ts'
|
|
81
|
+
-import { driveCompletion } from '../../lib/capabilities/completion-engine/driver.ts'
|
|
82
|
+
-import { createRunLogRedactor } from '../../lib/run-log-redact.ts'
|
|
83
|
+
-import { installAbortControllerPolyfill } from '../../lib/capabilities/completion-engine/abort-polyfill.ts'
|
|
84
|
+
-
|
|
85
|
+
-installAbortControllerPolyfill()
|
|
86
|
+
|
|
87
|
+
type RedactionMode = 'plain' | 'bench' | 'production'
|
|
88
|
+
type Spacing = 'immediate' | 'timeout'
|
|
89
|
+
@@ -216,40 +205,8 @@
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
-function countingHost(count: boolean, mode: RedactionMode) {
|
|
94
|
+
- const core = memoryCore()
|
|
95
|
+
- const records: unknown[] = []
|
|
96
|
+
- const stats = { bytes: 0, records: 0 }
|
|
97
|
+
- let boot: EngineBoot = createEngineBoot()
|
|
98
|
+
- return {
|
|
99
|
+
- core,
|
|
100
|
+
- stats,
|
|
101
|
+
- get boot() {
|
|
102
|
+
- return boot
|
|
103
|
+
- },
|
|
104
|
+
- async onStep(step: EngineStep) {
|
|
105
|
+
- const next = applyEngineStep(boot, step)
|
|
106
|
+
- if (mode === 'production') {
|
|
107
|
+
- for (const record of step.records) {
|
|
108
|
+
- await core.append(record)
|
|
109
|
+
- }
|
|
110
|
+
- } else if (mode === 'bench') {
|
|
111
|
+
- for (const record of step.records) {
|
|
112
|
+
- records.push(JSON.parse(JSON.stringify(record)))
|
|
113
|
+
- }
|
|
114
|
+
- }
|
|
115
|
+
-
|
|
116
|
+
- if (count) {
|
|
117
|
+
- for (const record of step.records) {
|
|
118
|
+
- stats.bytes += textEncoder.encode(JSON.stringify(record)).byteLength
|
|
119
|
+
- stats.records++
|
|
120
|
+
- }
|
|
121
|
+
- }
|
|
122
|
+
-
|
|
123
|
+
- boot = next
|
|
124
|
+
- },
|
|
125
|
+
- digestKey: new Uint8Array(32).fill(7)
|
|
126
|
+
- }
|
|
127
|
+
+function countingHost(_count: boolean, _mode: RedactionMode) {
|
|
128
|
+
+ return { core: memoryCore(), stats: { bytes: 0, records: 0 } }
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function memoryCore() {
|
|
132
|
+
@@ -270,28 +227,8 @@
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
-function runWithMode(mode: RedactionMode, { sdk, args, host }: WorkloadInput) {
|
|
137
|
+
- const completionOptions = {
|
|
138
|
+
- boot: host.boot,
|
|
139
|
+
- onStep: host.onStep,
|
|
140
|
+
- requestDigestKey: host.digestKey
|
|
141
|
+
- }
|
|
142
|
+
- if (mode === 'production') {
|
|
143
|
+
- return collect(completion(sdk, args, completionOptions))
|
|
144
|
+
- }
|
|
145
|
+
-
|
|
146
|
+
- const redactor = mode === 'bench' ? createRunLogRedactor(args.tools ?? []) : undefined
|
|
147
|
+
- const driverOptions: Parameters<typeof driveCompletion>[2] = { ...completionOptions }
|
|
148
|
+
- if (redactor) {
|
|
149
|
+
- Object.assign(driverOptions, {
|
|
150
|
+
- redact: redactor,
|
|
151
|
+
- redactShared: redactor.redactShared,
|
|
152
|
+
- redactOperationResult: redactor.redactOperationResult,
|
|
153
|
+
- redactStep: redactor.redactStep
|
|
154
|
+
- })
|
|
155
|
+
- }
|
|
156
|
+
-
|
|
157
|
+
- return collect(driveCompletion(args, () => Promise.resolve({ sdk }), driverOptions))
|
|
158
|
+
+function runWithMode(_mode: RedactionMode, { sdk, args }: WorkloadInput) {
|
|
159
|
+
+ return collect(completion(sdk, args))
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async function runToolWorkload(mode: RedactionMode, options: CountOptions = {}) {
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Tool-workload profile and re-measure — 2026-09-28, same day
|
|
166
|
+
|
|
167
|
+
Same revisions as above (HEAD `a029f2e`, merge-base `7d4a1ec`), after the changes listed under “What changed”. Machine load average was 6–7 throughout; differences under ~15% are noise.
|
|
168
|
+
|
|
169
|
+
### Profile (Node, `--cpu-prof --cpu-prof-interval=100`, 30 timed tool runs)
|
|
170
|
+
|
|
171
|
+
Top self-time frames before any change, module loading excluded (664 ms of workload over 31 runs):
|
|
172
|
+
|
|
173
|
+
| Self ms | Share | Frame |
|
|
174
|
+
| ------: | ----: | -------------------------------------------------------------- |
|
|
175
|
+
| 108.2 | 16.3% | `brandCapturedRecord` lib/run-log/protocol.ts:305 |
|
|
176
|
+
| 45.0 | 6.8% | garbage collector |
|
|
177
|
+
| 26.1 | 3.9% | `append` bench/cpu.ts:266 (host JSON copy) |
|
|
178
|
+
| 26.0 | 3.9% | `compactHistory` map callback lib/run-log/history-codec.ts:417 |
|
|
179
|
+
| 25.0 | 3.8% | `define` lib/run-log-redact.ts:167 |
|
|
180
|
+
|
|
181
|
+
Self time by file, per run: protocol.ts 4.9 ms, run-log-redact.ts 3.0 ms, history-codec.ts 2.5 ms, xstate 1.8 ms, GC 1.5 ms, bench host 0.9 ms, stable-stringify.ts 0.8 ms; nothing else above 0.6 ms.
|
|
182
|
+
|
|
183
|
+
Deterministic section timing (hrtime around each call, 10 timed runs) per run: compactSnapshot 2.25 ms, redactShared 2.19 ms, event encode 1.40 ms, append redactRecord+capture 1.33 ms, host append 1.12 ms, applyEngineStep 0.77 ms, snapshot diff 0.45 ms, getPersistedSnapshot 0.33 ms. These sum to about 10 ms of a 26–29 ms run; the rest is spread across the XState actor loop, the driver, operation records and GC.
|
|
184
|
+
|
|
185
|
+
Branding counters, per run: about 5,000 freshly branded nodes at 700 ns each (3.5 ms), from redactRecord 2,494, redactShared 1,227, event encode 580, delta patch 515, compactSnapshot 183. The uncommitted always-clone in walkRunRecord/redactRecord accounted for the redactRecord share: it re-cloned already-branded patches and snapshots and branded them again.
|
|
186
|
+
|
|
187
|
+
### What changed
|
|
188
|
+
|
|
189
|
+
- `lib/run-log-redact.ts`: captured values reaching `walkRunRecord`, `redactRecord` and `copyPatch` are reused by reference again (zero-copy). `cloneStructure` still copies captured input, and `walkHistoryEntry` still clones captured messages, so nothing reachable from live `context.history` is branded (N84). Fresh brand visits per run drop from about 5,000 to 3,460.
|
|
190
|
+
- `test/run-log-redact.ts`: the assertions that required copying captured input now assert the pass-through, and the identity tests (live nodes never shared, 40-round restored history) are unchanged.
|
|
191
|
+
- Append-path copy count over the tool workload (`captureRunRecord` misses, 11 runs × 57 records): **0**.
|
|
192
|
+
- A brand-walk micro-optimisation (array index loop) was tried and reverted: unmeasurable at this noise level.
|
|
193
|
+
- Log bytes unchanged: tools 324611 / 57 records; `test/run-log-golden-bytes.ts` and `test/completion-recovery-matrix.ts` pass.
|
|
194
|
+
|
|
195
|
+
### Results (same commands and interleaving as above)
|
|
196
|
+
|
|
197
|
+
| Runtime | Workload | Branch | Main | Branch/main | Limit | Meets limit |
|
|
198
|
+
| ------- | -------- | ---------------: | ---------------: | ----------: | ----: | ----------- |
|
|
199
|
+
| Node | stream | 13.4825 µs/token | 7.7775 µs/token | 1.734× | 1.5× | no |
|
|
200
|
+
| Node | tools | 29.1920 ms | 1.4280 ms | 20.443× | 2× | no |
|
|
201
|
+
| Bun | stream | 18.1330 µs/token | 16.0295 µs/token | 1.131× | 1.5× | yes |
|
|
202
|
+
| Bun | tools | 38.6190 ms | 2.1580 ms | 17.896× | 2× | no |
|
|
203
|
+
|
|
204
|
+
The tool workload did not move: the cut removed about 1.5 ms of branding, which is inside run-to-run noise on this machine. Redactor SHA-256: `d64b6103f081969ef9d56a6bbe5f2d6f3f9fe35a7dcad22734593cbe46d79183`. Benchmark SHA-256: `305ff7270f08b058e706efd6de75a57a109817a7f03106b608bc9ec1b2c15ba1` (unchanged).
|
|
205
|
+
|
|
206
|
+
### Raw CPU samples
|
|
207
|
+
|
|
208
|
+
| Runtime | Revision | Process | Tool ms (3 runs) | Stream CPU ms (3 runs; divide by 2 for µs/token) |
|
|
209
|
+
| ------- | -------- | ------: | ---------------------- | ------------------------------------------------ |
|
|
210
|
+
| node | branch | 1 | 33.290, 28.572, 29.864 | 41.942, 28.681, 30.363 |
|
|
211
|
+
| node | main | 1 | 1.507, 1.556, 1.072 | 11.554, 14.193, 14.610 |
|
|
212
|
+
| bun | branch | 1 | 58.033, 34.929, 38.619 | 26.353, 36.266, 96.164 |
|
|
213
|
+
| bun | main | 1 | 3.180, 2.158, 1.034 | 25.701, 33.694, 38.158 |
|
|
214
|
+
| node | main | 2 | 2.230, 1.427, 1.153 | 12.278, 22.986, 15.555 |
|
|
215
|
+
| node | branch | 2 | 32.094, 27.156, 29.012 | 26.243, 27.721, 26.965 |
|
|
216
|
+
| bun | main | 2 | 3.378, 2.315, 1.252 | 25.693, 12.538, 17.701 |
|
|
217
|
+
| bun | branch | 2 | 53.866, 32.177, 37.547 | 38.825, 79.629, 102.442 |
|
|
218
|
+
| node | branch | 3 | 29.694, 29.192, 25.870 | 26.662, 26.331, 27.544 |
|
|
219
|
+
| node | main | 3 | 1.479, 1.428, 1.076 | 12.326, 23.221, 15.700 |
|
|
220
|
+
| bun | branch | 3 | 54.822, 32.487, 39.635 | 28.558, 32.516, 60.522 |
|
|
221
|
+
| bun | main | 3 | 3.243, 2.143, 1.210 | 32.059, 31.021, 49.043 |
|
|
222
|
+
|
|
223
|
+
### Where the 20× comes from
|
|
224
|
+
|
|
225
|
+
No frame is above 16% of the workload. Each run writes 57 records (1 metadata of 219 KB, 1 checkpoint, 31 deltas, 24 operations) and each of the 31 deltas pays a persisted snapshot, a redaction walk, history compaction, a structural diff, a record redaction, a capture and the host's JSON copy: roughly 0.3–0.5 ms per delta on Node. Main writes nothing. Reaching 2× (about 2.9 ms) needs fewer journal writes per round or a much cheaper per-write path, not another frame-level cut; both change the record stream that `test/run-log-golden-bytes.ts` pins.
|