@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +12 -12
  2. package/bench/README.md +94 -0
  3. package/bench/bfcl/checker.ts +313 -0
  4. package/bench/bfcl/data.ts +93 -0
  5. package/bench/bfcl/observe.ts +93 -0
  6. package/bench/bfcl/run.ts +213 -0
  7. package/bench/bfcl/tools.ts +72 -0
  8. package/bench/bridge/bridge.ts +392 -0
  9. package/bench/bridge/openai.ts +150 -0
  10. package/bench/bridge/python/qvac_bench/__init__.py +3 -0
  11. package/bench/bridge/python/qvac_bench/airline.py +42 -0
  12. package/bench/bridge/python/qvac_bench/approval.py +26 -0
  13. package/bench/bridge/python/qvac_bench/bfcl_multi_turn.py +152 -0
  14. package/bench/bridge/python/qvac_bench/client.py +88 -0
  15. package/bench/bridge/python/qvac_bench/harbor_agent.py +85 -0
  16. package/bench/bridge/python/qvac_bench/session.py +107 -0
  17. package/bench/bridge/python/qvac_bench/speed.py +30 -0
  18. package/bench/bridge/python/qvac_bench/tau2_agent.py +106 -0
  19. package/bench/bridge/python/qvac_bench/tools.py +11 -0
  20. package/bench/bridge/python/run_bfcl_multi_turn.py +95 -0
  21. package/bench/bridge/python/run_tau2.py +150 -0
  22. package/bench/bridge/python/smoke.py +47 -0
  23. package/bench/bridge/python/test_airline.py +28 -0
  24. package/bench/bridge/python/test_approval.py +57 -0
  25. package/bench/bridge/python/test_bfcl_multi_turn.py +185 -0
  26. package/bench/bridge/python/test_harbor_agent.py +107 -0
  27. package/bench/bridge/serve.ts +52 -0
  28. package/bench/bridge/transcript.ts +52 -0
  29. package/bench/bun-tsconfig.json +15 -0
  30. package/bench/cpu-main.mjs +216 -0
  31. package/bench/cpu.ts +457 -0
  32. package/bench/evidence/2026-09-28-review15.md +225 -0
  33. package/bench/evidence/redact-bun1.3.14.json +92 -0
  34. package/bench/evidence/redact-node24.11.json +92 -0
  35. package/bench/graph-metrics.ts +43 -0
  36. package/bench/harness.ts +73 -0
  37. package/bench/issue129-matrix.mjs +722 -0
  38. package/bench/redact-adversarial.ts +256 -0
  39. package/bench/repeat-cpu.mjs +281 -0
  40. package/bench/report.ts +1075 -0
  41. package/bench/skill-eval/harness.ts +30 -0
  42. package/bench/skill-eval/laws.ts +37 -0
  43. package/bench/skill-eval/mermaid/client.ts +132 -0
  44. package/bench/skill-eval/mermaid/validate.mjs +115 -0
  45. package/bench/skill-eval/pack.ts +61 -0
  46. package/bench/skill-eval/packs/diagrams/cases.ts +281 -0
  47. package/bench/skill-eval/packs/diagrams/index.ts +50 -0
  48. package/bench/skill-eval/packs/diagrams/score.ts +74 -0
  49. package/bench/skill-eval/packs/index.ts +5 -0
  50. package/bench/skill-eval/packs/weather/cases.ts +279 -0
  51. package/bench/skill-eval/packs/weather/fixtures/wttr.json +566 -0
  52. package/bench/skill-eval/packs/weather/fixtures.ts +84 -0
  53. package/bench/skill-eval/packs/weather/index.ts +55 -0
  54. package/bench/skill-eval/packs/weather/score.ts +92 -0
  55. package/bench/skill-eval/report.ts +194 -0
  56. package/bench/skill-eval/run.ts +316 -0
  57. package/bench/speed.ts +80 -0
  58. package/package.json +4 -13
package/bench/cpu.ts ADDED
@@ -0,0 +1,457 @@
1
+ // CPU and run-log benchmark for Node, Bun, and Bare.
2
+ import { CompletionEngine } from '../lib/capabilities/completion-engine/index.ts'
3
+ import type { CompletionSdk, Message, ToolDefinition } from '../lib/contract/index.ts'
4
+ import b4a from 'b4a'
5
+ import process from 'process'
6
+ import { RunLog, type EngineBoot, type EngineStep } from '../lib/run-log/index.ts'
7
+ import { driveCompletion } from '../lib/capabilities/completion-engine/driver.ts'
8
+ import { HookPipeline } from '../lib/hooks/index.ts'
9
+ import { defaultAgentPorts } from '../lib/capabilities/agent-ports.ts'
10
+ import { outboundGuard } from '../lib/tools/outbound-guard.ts'
11
+ import { installAbortControllerPolyfill } from '../lib/helpers/abort-polyfill.ts'
12
+
13
+ installAbortControllerPolyfill()
14
+
15
+ type RedactionMode = 'nohost' | 'plain' | 'bench' | 'production'
16
+ type Spacing = 'immediate' | 'timeout'
17
+ type Workload = 'tools' | 'stream'
18
+ interface CliOptions {
19
+ runs: number
20
+ mode: string
21
+ spacing: string
22
+ workload: string
23
+ json: boolean
24
+ }
25
+ interface WorkloadInput {
26
+ sdk: CompletionSdk
27
+ args: Parameters<typeof CompletionEngine.run>[1]
28
+ host: ReturnType<typeof countingHost>
29
+ }
30
+ interface CountOptions {
31
+ count?: boolean
32
+ }
33
+ interface ModeResultsOptions {
34
+ mode: RedactionMode
35
+ runs: number
36
+ spacings: Spacing[]
37
+ workloads: Workload[]
38
+ }
39
+ type BenchResult =
40
+ | {
41
+ workload: 'tools'
42
+ mode: RedactionMode
43
+ runs: number
44
+ cpuMs: number
45
+ cpuSamplesMs: number[]
46
+ logBytes: number
47
+ records: number
48
+ }
49
+ | {
50
+ workload: 'stream'
51
+ mode: RedactionMode
52
+ spacing: Spacing
53
+ runs: number
54
+ cpuMs: number
55
+ cpuSamplesMs: number[]
56
+ usPerToken: number
57
+ contentEvents: number
58
+ logBytes: number
59
+ records: number
60
+ }
61
+ const TOKENS = 2000
62
+ // Keep every token observable by starting outside round.ts's burst window.
63
+ const FIRST_TOKEN_MS = 110
64
+ const USAGE = [
65
+ 'Usage: cpu.ts [runs] [--mode=nohost|plain|bench|production|all]',
66
+ ' [--spacing=immediate|timeout|all] [--workload=tools|stream|all] [--json]',
67
+ 'Node: node --experimental-strip-types bench/cpu.ts',
68
+ 'Bun: bun --tsconfig-override=bench/bun-tsconfig.json bench/cpu.ts',
69
+ 'Bare: bare bench/cpu.ts'
70
+ ].join('\n')
71
+
72
+ function parseOptions(args: string[]): CliOptions {
73
+ let runs = 3
74
+ let mode = 'all'
75
+ let spacing = 'all'
76
+ let workload = 'all'
77
+ let json = false
78
+ let positionalRuns = false
79
+
80
+ for (const arg of args) {
81
+ if (arg === '--json') {
82
+ json = true
83
+ } else if (arg === '--help' || arg === '-h') {
84
+ console.log(USAGE)
85
+ process.exit(0)
86
+ } else if (arg.startsWith('--mode=')) {
87
+ mode = arg.slice('--mode='.length)
88
+ } else if (arg.startsWith('--spacing=')) {
89
+ spacing = arg.slice('--spacing='.length)
90
+ } else if (arg.startsWith('--workload=')) {
91
+ workload = arg.slice('--workload='.length)
92
+ } else if (!arg.startsWith('-') && !positionalRuns) {
93
+ runs = Number(arg)
94
+ positionalRuns = true
95
+ } else {
96
+ throw new Error(`Unknown option: ${arg}`)
97
+ }
98
+ }
99
+
100
+ if (!Number.isInteger(runs) || runs < 1) {
101
+ throw new Error('runs must be a positive integer')
102
+ }
103
+
104
+ return { runs, mode, spacing, workload, json }
105
+ }
106
+
107
+ function selectValues<Value extends string>(
108
+ value: string,
109
+ allowed: readonly Value[],
110
+ label: string
111
+ ): Value[] {
112
+ if (value === 'all') {
113
+ return [...allowed]
114
+ }
115
+
116
+ const selected = allowed.find((option) => option === value)
117
+ if (selected === undefined) {
118
+ throw new Error(`Unknown ${label}: ${value}`)
119
+ }
120
+
121
+ return [selected]
122
+ }
123
+
124
+ function options() {
125
+ const parsed = parseOptions(process.argv.slice(2))
126
+ return {
127
+ runs: parsed.runs,
128
+ modes: selectValues(parsed.mode, ['nohost', 'plain', 'bench', 'production'], 'mode'),
129
+ spacings: selectValues(parsed.spacing, ['immediate', 'timeout'], 'spacing'),
130
+ workloads: selectValues(parsed.workload, ['tools', 'stream'], 'workload'),
131
+ json: parsed.json
132
+ }
133
+ }
134
+
135
+ async function collect<Event extends { type?: string }>(
136
+ run: AsyncIterable<Event>
137
+ ): Promise<Event[]> {
138
+ const events: Event[] = []
139
+ for await (const event of run) {
140
+ events.push(event)
141
+ }
142
+
143
+ return events
144
+ }
145
+
146
+ function cpuMs(start: NodeJS.CpuUsage): number {
147
+ const d = process.cpuUsage(start)
148
+ return (d.user + d.system) / 1000
149
+ }
150
+
151
+ function median(values: number[]): number {
152
+ const sorted = [...values].sort((a, b) => a - b)
153
+ const value = sorted[Math.floor(sorted.length / 2)]
154
+ if (value === undefined) {
155
+ throw new Error('Cannot find the median of an empty sample')
156
+ }
157
+
158
+ return value
159
+ }
160
+
161
+ // 101 messages, ~200 KB: a review-sized conversation.
162
+ function history(): Message[] {
163
+ return Array.from({ length: 101 }, (_, i) => ({
164
+ role: i % 2 ? 'assistant' : 'user',
165
+ content: `${i} ${'x'.repeat(2000)}`
166
+ }))
167
+ }
168
+
169
+ function readTool(name: string): ToolDefinition {
170
+ return {
171
+ name,
172
+ description: 'read something',
173
+ toolset: null,
174
+ parameters: { type: 'object', properties: { path: { type: 'string' } } },
175
+ effect: () => 'read',
176
+ autoApprove: true,
177
+ execute: () => Promise.resolve({ content: `${name} ${'r'.repeat(2048)}` })
178
+ }
179
+ }
180
+
181
+ // 3 tool rounds x 2 read calls/round, then a plain answer round.
182
+ function toolWorkloadSdk(tools: ToolDefinition[]): CompletionSdk {
183
+ let round = 0
184
+ return {
185
+ completion: () => {
186
+ const r = round++
187
+ if (r < 3) {
188
+ return {
189
+ tokenStream: (async function* () {})(),
190
+ final: Promise.resolve({
191
+ contentText: '',
192
+ toolCalls: tools.map((t, i) => ({
193
+ id: `call-${r}-${i}`,
194
+ name: t.name,
195
+ arguments: { path: `f${r}${i}` }
196
+ })),
197
+ raw: { fullText: '' }
198
+ })
199
+ }
200
+ }
201
+
202
+ return {
203
+ tokenStream: (async function* () {
204
+ yield 'done'
205
+ })(),
206
+ final: Promise.resolve({ contentText: 'done', toolCalls: [], raw: { fullText: 'done' } })
207
+ }
208
+ }
209
+ }
210
+ }
211
+
212
+ function countingHost(count: boolean, mode: RedactionMode) {
213
+ const core = memoryCore()
214
+ const records: unknown[] = []
215
+ const stats = { bytes: 0, records: 0 }
216
+ let boot: EngineBoot = RunLog.createEngineBoot()
217
+ return {
218
+ core,
219
+ stats,
220
+ get boot() {
221
+ return boot
222
+ },
223
+ async onStep(step: EngineStep) {
224
+ const next = RunLog.applyEngineStep(boot, step)
225
+ if (mode === 'production') {
226
+ for (const record of step.records) {
227
+ await core.append(record)
228
+ }
229
+ } else if (mode === 'bench') {
230
+ for (const record of step.records) {
231
+ records.push(JSON.parse(JSON.stringify(record)))
232
+ }
233
+ }
234
+
235
+ if (count) {
236
+ for (const record of step.records) {
237
+ stats.bytes += b4a.byteLength(JSON.stringify(record))
238
+ stats.records++
239
+ }
240
+ }
241
+
242
+ boot = next
243
+ },
244
+ digestKey: new Uint8Array(32).fill(7)
245
+ }
246
+ }
247
+
248
+ function memoryCore() {
249
+ const records: unknown[] = []
250
+ return {
251
+ get length() {
252
+ return records.length
253
+ },
254
+ ready: () => Promise.resolve(),
255
+ get(index: number) {
256
+ const value = records[index]
257
+ return Promise.resolve(value === undefined ? null : JSON.parse(JSON.stringify(value)))
258
+ },
259
+ append(value: unknown) {
260
+ records.push(JSON.parse(JSON.stringify(value)))
261
+ return Promise.resolve()
262
+ }
263
+ }
264
+ }
265
+
266
+ function runWithMode(mode: RedactionMode, { sdk, args, host }: WorkloadInput) {
267
+ // No onStep: the run log journal still runs, so only the host append is left out.
268
+ if (mode === 'nohost') {
269
+ return collect(CompletionEngine.run(sdk, args))
270
+ }
271
+
272
+ const completionOptions = {
273
+ boot: host.boot,
274
+ onStep: host.onStep,
275
+ requestDigestKey: host.digestKey
276
+ }
277
+ if (mode === 'production') {
278
+ return collect(CompletionEngine.run(sdk, args, completionOptions))
279
+ }
280
+
281
+ const redactor = mode === 'bench' ? RunLog.createRedactor(args.tools ?? []) : undefined
282
+ const driverOptions: Parameters<typeof driveCompletion>[2] = {
283
+ ...completionOptions,
284
+ pipeline: new HookPipeline(HookPipeline.defaults()),
285
+ outbound: outboundGuard,
286
+ agentPorts: defaultAgentPorts
287
+ }
288
+ if (redactor) {
289
+ Object.assign(driverOptions, {
290
+ redact: redactor,
291
+ redactShared: redactor.redactShared,
292
+ redactOperationResult: redactor.redactOperationResult,
293
+ redactStep: redactor.redactStep
294
+ })
295
+ }
296
+
297
+ return collect(driveCompletion(args, () => Promise.resolve({ sdk }), driverOptions))
298
+ }
299
+
300
+ async function runToolWorkload(mode: RedactionMode, options: CountOptions = {}) {
301
+ const { count = false } = options
302
+ const tools = [readTool('read_a'), readTool('read_b')]
303
+ const host = countingHost(count, mode)
304
+ const { stats } = host
305
+ const args = { runId: 'bench-tools', modelId: 'm', messages: history(), tools }
306
+ const start = process.cpuUsage()
307
+ await runWithMode(mode, { sdk: toolWorkloadSdk(tools), args, host })
308
+ return { ms: cpuMs(start), ...stats }
309
+ }
310
+
311
+ function streamingSdk(spacing: Spacing): CompletionSdk {
312
+ return {
313
+ completion: () => ({
314
+ tokenStream: (async function* () {
315
+ await new Promise((resolve) => setTimeout(resolve, FIRST_TOKEN_MS))
316
+ for (let i = 0; i < TOKENS; i++) {
317
+ await new Promise((resolve) =>
318
+ spacing === 'immediate' ? setImmediate(resolve) : setTimeout(resolve, 0)
319
+ )
320
+ yield 't'
321
+ }
322
+ })(),
323
+ final: Promise.resolve({
324
+ contentText: 't'.repeat(TOKENS),
325
+ toolCalls: [],
326
+ raw: { fullText: '' }
327
+ })
328
+ })
329
+ }
330
+ }
331
+
332
+ async function runStreaming(mode: RedactionMode, spacing: Spacing, options: CountOptions = {}) {
333
+ const { count = false } = options
334
+ const host = countingHost(count, mode)
335
+ const { stats } = host
336
+ const args = { runId: `bench-stream-${Math.random()}`, modelId: 'm', messages: history() }
337
+ const start = process.cpuUsage()
338
+ const events = await runWithMode(mode, { sdk: streamingSdk(spacing), args, host })
339
+ const ms = cpuMs(start)
340
+ return {
341
+ ms,
342
+ contentEvents: events.filter((event) => event.type === 'content').length,
343
+ ...stats
344
+ }
345
+ }
346
+
347
+ async function measureTools(mode: RedactionMode, runs: number): Promise<BenchResult> {
348
+ const log = await runToolWorkload(mode, { count: true })
349
+ const samples = []
350
+ for (let i = 0; i < runs; i++) {
351
+ samples.push(await runToolWorkload(mode))
352
+ }
353
+
354
+ return {
355
+ workload: 'tools',
356
+ mode,
357
+ runs,
358
+ cpuMs: median(samples.map((sample) => sample.ms)),
359
+ cpuSamplesMs: samples.map((sample) => sample.ms),
360
+ logBytes: log.bytes,
361
+ records: log.records
362
+ }
363
+ }
364
+
365
+ async function measureStream(
366
+ mode: RedactionMode,
367
+ spacing: Spacing,
368
+ runs: number
369
+ ): Promise<BenchResult> {
370
+ const log = await runStreaming(mode, spacing, { count: true })
371
+ const samples = []
372
+ for (let i = 0; i < runs; i++) {
373
+ samples.push(await runStreaming(mode, spacing))
374
+ }
375
+
376
+ const cpu = median(samples.map((sample) => sample.ms))
377
+
378
+ return {
379
+ workload: 'stream',
380
+ mode,
381
+ spacing,
382
+ runs,
383
+ cpuMs: cpu,
384
+ cpuSamplesMs: samples.map((sample) => sample.ms),
385
+ usPerToken: (cpu * 1000) / TOKENS,
386
+ contentEvents: log.contentEvents,
387
+ logBytes: log.bytes,
388
+ records: log.records
389
+ }
390
+ }
391
+
392
+ async function resultsForMode({
393
+ mode,
394
+ runs,
395
+ spacings,
396
+ workloads
397
+ }: ModeResultsOptions): Promise<BenchResult[]> {
398
+ const results: BenchResult[] = []
399
+ if (workloads.includes('tools')) {
400
+ results.push(await measureTools(mode, runs))
401
+ }
402
+
403
+ if (workloads.includes('stream')) {
404
+ for (const spacing of spacings) {
405
+ results.push(await measureStream(mode, spacing, runs))
406
+ }
407
+ }
408
+
409
+ return results
410
+ }
411
+
412
+ function runtimeInfo() {
413
+ const bun = Reflect.get(globalThis, 'Bun')
414
+ if (bun && typeof bun === 'object' && 'version' in bun && typeof bun.version === 'string') {
415
+ return { runtime: 'bun', version: bun.version }
416
+ }
417
+
418
+ return {
419
+ runtime: process.release?.name ?? (process.version.startsWith('v1.') ? 'bare' : 'unknown'),
420
+ version: process.version
421
+ }
422
+ }
423
+
424
+ async function main() {
425
+ const { runs, modes, spacings, workloads, json } = options()
426
+ const results: BenchResult[] = []
427
+ for (const mode of modes) {
428
+ results.push(...(await resultsForMode({ mode, runs, spacings, workloads })))
429
+ }
430
+
431
+ const report = { ...runtimeInfo(), runs, results }
432
+ if (json) {
433
+ console.log(JSON.stringify(report))
434
+ return
435
+ }
436
+
437
+ console.log(`${report.runtime} ${report.version}; median of ${runs} runs`)
438
+ for (const result of results) {
439
+ const logMiB = (result.logBytes / (1024 * 1024)).toFixed(3)
440
+ if (result.workload === 'tools') {
441
+ console.log(
442
+ `tools mode=${result.mode} cpu=${result.cpuMs.toFixed(2)}ms log=${logMiB}MiB records=${result.records}`
443
+ )
444
+ } else {
445
+ const usPerToken =
446
+ 'usPerToken' in result && typeof result.usPerToken === 'number' ? result.usPerToken : 0
447
+ console.log(
448
+ `stream mode=${result.mode} spacing=${result.spacing} cpu=${result.cpuMs.toFixed(2)}ms usPerToken=${usPerToken.toFixed(2)} contentEvents=${result.contentEvents} log=${logMiB}MiB records=${result.records}`
449
+ )
450
+ }
451
+ }
452
+ }
453
+
454
+ main().catch((error) => {
455
+ console.error(error)
456
+ process.exitCode = 1
457
+ })
@@ -0,0 +1,225 @@
1
+ # Review 15 performance re-measure — 2026-09-28
2
+
3
+ No tuning. Both runtimes exceed the 1.5× streaming and 2× tool-workload targets in this measurement.
4
+
5
+ ## Revisions and method
6
+
7
+ - Branch: `refactor/add-state-machine`, HEAD `a029f2ec07de03c9bdce306c0d22494917399d4e` plus the uncommitted Lot 1 ownership fix in `lib/run-log-redact.ts`. Lots 2–3 change documentation and tests.
8
+ - Main comparison: merge-base `7d4a1ec282ed29b2480b551e6a4670d448441e63`, obtained with `git merge-base HEAD origin/main`.
9
+ - Machine: Darwin arm64. Runtimes: Node v24.11.0 and Bun 1.3.14.
10
+ - Method: [issue #129](https://github.com/tetherto/qvac-harness/issues/129), “Measurement method.” Fake SDK, 101 messages (roughly 200 KB), 2,000 token events, 110 ms initial delay followed by `setImmediate` spacing; 3 rounds × 2 auto-approved read tools followed by an answer.
11
+ - Three fresh processes per runtime/revision, three timed runs per workload per process. Each workload first runs once to count log bytes and warm the process. CPU time is `process.cpuUsage()`, not elapsed time. Results below are medians of the three process medians; all nine CPU samples are retained below.
12
+ - Process pairs ran sequentially: branch/main, main/branch, branch/main; Node pair then Bun pair in each pass. No benchmark processes overlapped. The shared machine was not reserved, so other agent activity and Bun’s wide stream samples remain sources of variance.
13
+ - Branch used the existing `bench/cpu.ts` in production mode with host boot folding and JSON-copy storage. Every stream measurement reported 2,000 content events.
14
+ - Baseline deviation: the merge-base predates both `bench/cpu.ts` and the run-log API. Its temporary copy used identical workloads, warm-up, timing, and reporting, with only the entry-point/storage adapter changed to `completion(sdk, args)`. Main accepts two arguments and emits no journal records; its log bytes/records are zero. The exact adapter diff is below.
15
+ - Benchmark SHA-256: `305ff7270f08b058e706efd6de75a57a109817a7f03106b608bc9ec1b2c15ba1`.
16
+ - Measured redactor SHA-256: `007bf7c3146e7e2132062c50f62f709e6495e13ae03ebe3017f49621a0f298b0`.
17
+
18
+ ## Results
19
+
20
+ | Runtime | Workload | Branch | Main | Branch/main | Limit | Meets limit |
21
+ | ------- | -------- | ---------------: | ---------------: | ----------: | ----: | ----------- |
22
+ | Node | stream | 14.3875 µs/token | 7.6265 µs/token | 1.887× | 1.5× | no |
23
+ | Node | tools | 27.7580 ms | 1.3510 ms | 20.546× | 2× | no |
24
+ | Bun | stream | 24.7500 µs/token | 15.8645 µs/token | 1.560× | 1.5× | no |
25
+ | Bun | tools | 36.8500 ms | 2.0140 ms | 18.297× | 2× | no |
26
+
27
+ | Runtime | Revision | Tool log bytes / records | Stream log bytes / records (processes 1, 2, 3) |
28
+ | ------- | -------- | -------------------------------- | ---------------------------------------------- |
29
+ | Node | branch | 324611, 324611, 324611 / 57 each | 248235, 248245, 248235 / 9 each |
30
+ | Node | main | 0, 0, 0 / 0 each | 0, 0, 0 / 0 each |
31
+ | Bun | branch | 324611, 324611, 324611 / 57 each | 248235, 248235, 248240 / 9 each |
32
+ | Bun | main | 0, 0, 0 / 0 each | 0, 0, 0 / 0 each |
33
+
34
+ Stream log sizes vary slightly because the benchmark generates a random run ID. Tool log sizes were identical across these processes. These measurements do not verify the golden-bytes fixture; it was not regenerated.
35
+
36
+ ## Exact benchmark commands
37
+
38
+ Run each command three times per revision, in the interleaved order above:
39
+
40
+ ```sh
41
+ QVAC_NO_SKILL_BIN_PROBE=1 /tmp/node-v24.11.0-darwin-arm64/bin/node --experimental-strip-types bench/cpu.ts 3 --mode=production --spacing=immediate --workload=all --json
42
+ QVAC_NO_SKILL_BIN_PROBE=1 /Users/matheus/.bun/bin/bun --tsconfig-override=bench/bun-tsconfig.json bench/cpu.ts 3 --mode=production --spacing=immediate --workload=all --json
43
+ ```
44
+
45
+ The baseline was created with `git worktree add --detach /tmp/qvac-review15-giokq5_1/main 7d4a1ec282ed29b2480b551e6a4670d448441e63`. Its `node_modules` symlink reused the branch’s installed dependency resolutions, and `bun-tsconfig.json` was copied unchanged. The temporary worktree was removed after measurement.
46
+
47
+ ## Raw CPU samples
48
+
49
+ | Runtime | Revision | Process | Tool ms (3 runs) | Stream CPU ms (3 runs; divide by 2 for µs/token) |
50
+ | ------- | -------- | ------: | ---------------------- | ------------------------------------------------ |
51
+ | node | branch | 1 | 30.668, 27.576, 29.124 | 25.374, 32.936, 32.531 |
52
+ | node | main | 1 | 1.743, 1.351, 1.123 | 10.491, 15.769, 15.307 |
53
+ | bun | branch | 1 | 58.115, 35.710, 36.574 | 29.654, 49.500, 107.315 |
54
+ | bun | main | 1 | 3.041, 2.292, 1.223 | 36.103, 13.894, 31.729 |
55
+ | node | main | 2 | 1.584, 1.494, 1.095 | 15.253, 12.610, 17.444 |
56
+ | node | branch | 2 | 32.229, 24.765, 27.758 | 28.775, 25.674, 31.390 |
57
+ | bun | main | 2 | 3.081, 2.014, 1.081 | 35.047, 16.004, 20.184 |
58
+ | bun | branch | 2 | 54.465, 34.368, 38.625 | 43.572, 48.618, 69.273 |
59
+ | node | branch | 3 | 29.307, 24.338, 27.701 | 26.108, 26.253, 27.608 |
60
+ | node | main | 3 | 1.519, 1.345, 1.032 | 11.542, 16.398, 13.744 |
61
+ | bun | branch | 3 | 54.344, 34.818, 36.850 | 39.626, 73.367, 72.450 |
62
+ | bun | main | 3 | 2.887, 1.857, 1.112 | 39.319, 41.242, 71.949 |
63
+
64
+ ## Baseline adapter diff
65
+
66
+ Apply to a temporary copy of the branch benchmark; it does not alter the workload bodies or measurement loops.
67
+
68
+ ```diff
69
+ --- branch/bench/cpu.ts
70
+ +++ main/bench/cpu.ts
71
+ @@ -6,17 +6,6 @@
72
+ type ToolDefinition
73
+ } from '../../lib/capabilities/completion.ts'
74
+ import process from 'process'
75
+ -import {
76
+ - applyEngineStep,
77
+ - createEngineBoot,
78
+ - type EngineBoot,
79
+ - type EngineStep
80
+ -} from '../../lib/run-log-protocol.ts'
81
+ -import { driveCompletion } from '../../lib/capabilities/completion-engine/driver.ts'
82
+ -import { createRunLogRedactor } from '../../lib/run-log-redact.ts'
83
+ -import { installAbortControllerPolyfill } from '../../lib/capabilities/completion-engine/abort-polyfill.ts'
84
+ -
85
+ -installAbortControllerPolyfill()
86
+
87
+ type RedactionMode = 'plain' | 'bench' | 'production'
88
+ type Spacing = 'immediate' | 'timeout'
89
+ @@ -216,40 +205,8 @@
90
+ }
91
+ }
92
+
93
+ -function countingHost(count: boolean, mode: RedactionMode) {
94
+ - const core = memoryCore()
95
+ - const records: unknown[] = []
96
+ - const stats = { bytes: 0, records: 0 }
97
+ - let boot: EngineBoot = createEngineBoot()
98
+ - return {
99
+ - core,
100
+ - stats,
101
+ - get boot() {
102
+ - return boot
103
+ - },
104
+ - async onStep(step: EngineStep) {
105
+ - const next = applyEngineStep(boot, step)
106
+ - if (mode === 'production') {
107
+ - for (const record of step.records) {
108
+ - await core.append(record)
109
+ - }
110
+ - } else if (mode === 'bench') {
111
+ - for (const record of step.records) {
112
+ - records.push(JSON.parse(JSON.stringify(record)))
113
+ - }
114
+ - }
115
+ -
116
+ - if (count) {
117
+ - for (const record of step.records) {
118
+ - stats.bytes += textEncoder.encode(JSON.stringify(record)).byteLength
119
+ - stats.records++
120
+ - }
121
+ - }
122
+ -
123
+ - boot = next
124
+ - },
125
+ - digestKey: new Uint8Array(32).fill(7)
126
+ - }
127
+ +function countingHost(_count: boolean, _mode: RedactionMode) {
128
+ + return { core: memoryCore(), stats: { bytes: 0, records: 0 } }
129
+ }
130
+
131
+ function memoryCore() {
132
+ @@ -270,28 +227,8 @@
133
+ }
134
+ }
135
+
136
+ -function runWithMode(mode: RedactionMode, { sdk, args, host }: WorkloadInput) {
137
+ - const completionOptions = {
138
+ - boot: host.boot,
139
+ - onStep: host.onStep,
140
+ - requestDigestKey: host.digestKey
141
+ - }
142
+ - if (mode === 'production') {
143
+ - return collect(completion(sdk, args, completionOptions))
144
+ - }
145
+ -
146
+ - const redactor = mode === 'bench' ? createRunLogRedactor(args.tools ?? []) : undefined
147
+ - const driverOptions: Parameters<typeof driveCompletion>[2] = { ...completionOptions }
148
+ - if (redactor) {
149
+ - Object.assign(driverOptions, {
150
+ - redact: redactor,
151
+ - redactShared: redactor.redactShared,
152
+ - redactOperationResult: redactor.redactOperationResult,
153
+ - redactStep: redactor.redactStep
154
+ - })
155
+ - }
156
+ -
157
+ - return collect(driveCompletion(args, () => Promise.resolve({ sdk }), driverOptions))
158
+ +function runWithMode(_mode: RedactionMode, { sdk, args }: WorkloadInput) {
159
+ + return collect(completion(sdk, args))
160
+ }
161
+
162
+ async function runToolWorkload(mode: RedactionMode, options: CountOptions = {}) {
163
+ ```
164
+
165
+ ## Tool-workload profile and re-measure — 2026-09-28, same day
166
+
167
+ Same revisions as above (HEAD `a029f2e`, merge-base `7d4a1ec`), after the changes listed under “What changed”. Machine load average was 6–7 throughout; differences under ~15% are noise.
168
+
169
+ ### Profile (Node, `--cpu-prof --cpu-prof-interval=100`, 30 timed tool runs)
170
+
171
+ Top self-time frames before any change, module loading excluded (664 ms of workload over 31 runs):
172
+
173
+ | Self ms | Share | Frame |
174
+ | ------: | ----: | -------------------------------------------------------------- |
175
+ | 108.2 | 16.3% | `brandCapturedRecord` lib/run-log/protocol.ts:305 |
176
+ | 45.0 | 6.8% | garbage collector |
177
+ | 26.1 | 3.9% | `append` bench/cpu.ts:266 (host JSON copy) |
178
+ | 26.0 | 3.9% | `compactHistory` map callback lib/run-log/history-codec.ts:417 |
179
+ | 25.0 | 3.8% | `define` lib/run-log-redact.ts:167 |
180
+
181
+ Self time by file, per run: protocol.ts 4.9 ms, run-log-redact.ts 3.0 ms, history-codec.ts 2.5 ms, xstate 1.8 ms, GC 1.5 ms, bench host 0.9 ms, stable-stringify.ts 0.8 ms; nothing else above 0.6 ms.
182
+
183
+ Deterministic section timing (hrtime around each call, 10 timed runs) per run: compactSnapshot 2.25 ms, redactShared 2.19 ms, event encode 1.40 ms, append redactRecord+capture 1.33 ms, host append 1.12 ms, applyEngineStep 0.77 ms, snapshot diff 0.45 ms, getPersistedSnapshot 0.33 ms. These sum to about 10 ms of a 26–29 ms run; the rest is spread across the XState actor loop, the driver, operation records and GC.
184
+
185
+ Branding counters, per run: about 5,000 freshly branded nodes at 700 ns each (3.5 ms), from redactRecord 2,494, redactShared 1,227, event encode 580, delta patch 515, compactSnapshot 183. The uncommitted always-clone in walkRunRecord/redactRecord accounted for the redactRecord share: it re-cloned already-branded patches and snapshots and branded them again.
186
+
187
+ ### What changed
188
+
189
+ - `lib/run-log-redact.ts`: captured values reaching `walkRunRecord`, `redactRecord` and `copyPatch` are reused by reference again (zero-copy). `cloneStructure` still copies captured input, and `walkHistoryEntry` still clones captured messages, so nothing reachable from live `context.history` is branded (N84). Fresh brand visits per run drop from about 5,000 to 3,460.
190
+ - `test/run-log-redact.ts`: the assertions that required copying captured input now assert the pass-through, and the identity tests (live nodes never shared, 40-round restored history) are unchanged.
191
+ - Append-path copy count over the tool workload (`captureRunRecord` misses, 11 runs × 57 records): **0**.
192
+ - A brand-walk micro-optimisation (array index loop) was tried and reverted: unmeasurable at this noise level.
193
+ - Log bytes unchanged: tools 324611 / 57 records; `test/run-log-golden-bytes.ts` and `test/completion-recovery-matrix.ts` pass.
194
+
195
+ ### Results (same commands and interleaving as above)
196
+
197
+ | Runtime | Workload | Branch | Main | Branch/main | Limit | Meets limit |
198
+ | ------- | -------- | ---------------: | ---------------: | ----------: | ----: | ----------- |
199
+ | Node | stream | 13.4825 µs/token | 7.7775 µs/token | 1.734× | 1.5× | no |
200
+ | Node | tools | 29.1920 ms | 1.4280 ms | 20.443× | 2× | no |
201
+ | Bun | stream | 18.1330 µs/token | 16.0295 µs/token | 1.131× | 1.5× | yes |
202
+ | Bun | tools | 38.6190 ms | 2.1580 ms | 17.896× | 2× | no |
203
+
204
+ The tool workload did not move: the cut removed about 1.5 ms of branding, which is inside run-to-run noise on this machine. Redactor SHA-256: `d64b6103f081969ef9d56a6bbe5f2d6f3f9fe35a7dcad22734593cbe46d79183`. Benchmark SHA-256: `305ff7270f08b058e706efd6de75a57a109817a7f03106b608bc9ec1b2c15ba1` (unchanged).
205
+
206
+ ### Raw CPU samples
207
+
208
+ | Runtime | Revision | Process | Tool ms (3 runs) | Stream CPU ms (3 runs; divide by 2 for µs/token) |
209
+ | ------- | -------- | ------: | ---------------------- | ------------------------------------------------ |
210
+ | node | branch | 1 | 33.290, 28.572, 29.864 | 41.942, 28.681, 30.363 |
211
+ | node | main | 1 | 1.507, 1.556, 1.072 | 11.554, 14.193, 14.610 |
212
+ | bun | branch | 1 | 58.033, 34.929, 38.619 | 26.353, 36.266, 96.164 |
213
+ | bun | main | 1 | 3.180, 2.158, 1.034 | 25.701, 33.694, 38.158 |
214
+ | node | main | 2 | 2.230, 1.427, 1.153 | 12.278, 22.986, 15.555 |
215
+ | node | branch | 2 | 32.094, 27.156, 29.012 | 26.243, 27.721, 26.965 |
216
+ | bun | main | 2 | 3.378, 2.315, 1.252 | 25.693, 12.538, 17.701 |
217
+ | bun | branch | 2 | 53.866, 32.177, 37.547 | 38.825, 79.629, 102.442 |
218
+ | node | branch | 3 | 29.694, 29.192, 25.870 | 26.662, 26.331, 27.544 |
219
+ | node | main | 3 | 1.479, 1.428, 1.076 | 12.326, 23.221, 15.700 |
220
+ | bun | branch | 3 | 54.822, 32.487, 39.635 | 28.558, 32.516, 60.522 |
221
+ | bun | main | 3 | 3.243, 2.143, 1.210 | 32.059, 31.021, 49.043 |
222
+
223
+ ### Where the 20× comes from
224
+
225
+ No frame is above 16% of the workload. Each run writes 57 records (1 metadata of 219 KB, 1 checkpoint, 31 deltas, 24 operations) and each of the 31 deltas pays a persisted snapshot, a redaction walk, history compaction, a structural diff, a record redaction, a capture and the host's JSON copy: roughly 0.3–0.5 ms per delta on Node. Main writes nothing. Reaching 2× (about 2.9 ms) needs fewer journal writes per round or a much cheaper per-write path, not another frame-level cut; both change the record stream that `test/run-log-golden-bytes.ts` pins.