@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/issue129-matrix.mjs

@@ -0,0 +1,722 @@
+// Reproduce the 33-cell Issue 129 matrix; --dry-run and --prepare-only never measure.
+import { execFileSync, spawnSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  realpathSync,
+  renameSync,
+  symlinkSync,
+  writeFileSync
+} from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
+const baselineRevisions = [
+  '7d4a1ec282ed29b2480b551e6a4670d448441e63',
+  '8cbf419e15e6b13aee118f601b3282749cdf7e01'
+]
+const nodeDefault = '/tmp/node-v24.11.0-darwin-arm64/bin/node'
+const omittedDirectories = new Set([
+  '.git',
+  '.turbo',
+  '.cache',
+  'node_modules',
+  'coverage',
+  'dist',
+  'dist-lib'
+])
+const options = parseOptions(process.argv.slice(2))
+const schedule = makeSchedule(options.seed)
+const nodeModules = realpathSync(
+  options.nodeModules ?? process.env.QVAC_MATRIX_NODE_MODULES ?? path.join(root, 'node_modules')
+)
+const nodeBinary = path.resolve(options.node ?? process.env.QVAC_NODE24_BIN ?? nodeDefault)
+const bunBinary = options.bun ?? process.env.QVAC_BUN_BIN ?? 'bun'
+const bareRuntimeBinary = path.resolve(
+  options.bareRuntime ??
+    process.env.QVAC_BARE_RUNTIME_BIN ??
+    path.join(nodeModules, 'bare-runtime/bin/bare')
+)
+
+if (options.dryRun) {
+  console.log(JSON.stringify({ seed: options.seed, jobCount: schedule.length, schedule }))
+  process.exit(0)
+}
+
+const scratch = mkdtempSync(path.join(os.tmpdir(), 'qvac-issue129-matrix-'))
+const candidateRoot = path.join(scratch, 'candidate')
+const baselineRoots = new Map()
+prepareCandidate(candidateRoot)
+for (const revision of baselineRevisions) {
+  const baselineRoot = path.join(scratch, `baseline-${revision.slice(0, 7)}`)
+  prepareBaseline(revision, baselineRoot)
+  baselineRoots.set(revision, baselineRoot)
+}
+
+const runtimes = {
+  node: { executable: nodeBinary, version: versionOf(nodeBinary, ['--version']) },
+  bun: { executable: bunBinary, version: versionOf(bunBinary, ['--version']) },
+  bare: {
+    executable: bareRuntimeBinary,
+    version: versionOf(bareRuntimeBinary, ['--version'])
+  }
+}
+const sourceManifests = {
+  candidate: sourceManifest(candidateRoot),
+  ...Object.fromEntries(
+    [...baselineRoots].map(([revision, baselineRoot]) => [revision, sourceManifest(baselineRoot)])
+  )
+}
+
+if (options.prepareOnly) {
+  console.log(
+    JSON.stringify({
+      scratch,
+      runtimes,
+      dependencyGraph: {
+        nodeModules,
+        packageManifestSha256: hashFile(path.resolve(nodeModules, 'package.json')),
+        lockfileSha256: hashFile(path.resolve(nodeModules, '../bun.lock')),
+        installedPackages: installedPackageVersions(nodeModules, [
+          'xstate',
+          '@qvac/sdk',
+          '@qvac/inference',
+          'b4a',
+          'bare-runtime',
+          'typescript'
+        ])
+      },
+      sourceManifests
+    })
+  )
+  process.exit(0)
+}
+
+const outputPath = path.resolve(options.output)
+const partialPath = `${outputPath}.partial.json`
+mkdirSync(path.dirname(outputPath), { recursive: true })
+
+const report = {
+  name: 'Issue 129 reproducible benchmark matrix',
+  status: 'running',
+  createdAt: new Date().toISOString(),
+  method: {
+    unit: 'process CPU milliseconds',
+    workload:
+      'tools: 101 messages (~200 KB), 3 rounds x 2 read tools x 2 KB results then answer; stream: 2,000 one-character tokens, 110 ms first-token gap, immediate setImmediate or timeout(0) spacing',
+    warmup: 'one untimed selected workload pass per process before three timed samples',
+    processReplicates: 3,
+    timedSamplesPerProcess: 3,
+    aggregate: 'pooled median of nine raw cpuSamplesMs values; process medians retained separately',
+    interleaving: `deterministic Fisher-Yates shuffle with seed ${options.seed}`
+  },
+  host: { platform: process.platform, architecture: process.arch, controller: process.version },
+  runtimes,
+  dependencyGraph: {
+    nodeModules,
+    packageManifestSha256: hashFile(path.resolve(nodeModules, 'package.json')),
+    lockfileSha256: hashFile(path.resolve(nodeModules, '../bun.lock')),
+    installedPackages: installedPackageVersions(nodeModules, [
+      'xstate',
+      '@qvac/sdk',
+      '@qvac/inference',
+      'b4a',
+      'bare-runtime',
+      'typescript'
+    ]),
+    note: 'All snapshots and runtimes use the same selected frozen dependency graph, symlinked into each snapshot. Baseline source is historical, but baseline dependencies are not reinstalled from each historical lockfile.'
+  },
+  snapshots: {
+    directory: scratch,
+    candidate: sourceManifests.candidate,
+    baselines: Object.fromEntries(
+      baselineRevisions.map((revision) => [revision, sourceManifests[revision]])
+    ),
+    overlay: [
+      'bench/cpu-main.mjs',
+      'bench/repeat-cpu.mjs',
+      'bench/bun-tsconfig.json',
+      "candidate-only generated Bare entrypoint: import 'bare-encoding/global'; import('./cpu.ts')"
+    ]
+  },
+  schedule,
+  execution: [],
+  cells: []
+}
+
+writePartial()
+for (let index = 0; index < schedule.length; index++) {
+  const job = schedule[index]
+  const result = runJob(job, index)
+  report.execution.push(result)
+  report.cells = aggregateCells(report.execution)
+  writePartial()
+}
+
+if (
+  report.cells.length !== 33 ||
+  report.cells.some((cell) => cell.processCount !== 3 || cell.rawSampleCount !== 9)
+) {
+  throw new Error(
+    'The matrix is incomplete: expected 33 cells with three processes and nine raw samples each'
+  )
+}
+
+report.status = 'complete'
+report.completedAt = new Date().toISOString()
+writePartial()
+renameSync(partialPath, outputPath)
+console.log(
+  JSON.stringify({ status: report.status, output: outputPath, jobCount: schedule.length })
+)
+
+function writePartial() {
+  writeFileSync(partialPath, `${JSON.stringify(report, null, 2)}\n`)
+}
+
+function prepareCandidate(destination) {
+  mkdirSync(destination, { recursive: true })
+  cpSync(root, destination, {
+    recursive: true,
+    filter(source) {
+      const relative = path.relative(root, source)
+      if (relative === '') {
+        return true
+      }
+
+      const parts = relative.split(path.sep)
+
+      return (
+        !parts.some((part) => omittedDirectories.has(part) || part.startsWith('.')) &&
+        !/\.(?:pem|key)$/i.test(relative)
+      )
+    }
+  })
+  writeFileSync(
+    path.join(destination, 'bench/cpu-bare-runner.mjs'),
+    "import 'bare-encoding/global'\nawait import('./cpu.ts')\n"
+  )
+  linkNodeModules(destination, nodeModules)
+}
+
+function prepareBaseline(revision, destination) {
+  mkdirSync(destination, { recursive: true })
+  const archive = execFileSync('git', ['archive', '--format=tar', revision], {
+    cwd: root,
+    maxBuffer: 512 * 1024 * 1024
+  })
+  const extracted = spawnSync('tar', ['-xf', '-', '-C', destination], {
+    input: archive,
+    encoding: 'utf8'
+  })
+  if (extracted.error) {
+    throw extracted.error
+  }
+
+  if (extracted.status !== 0) {
+    throw new Error(`Could not extract ${revision}: ${extracted.stderr}`)
+  }
+
+  for (const relative of [
+    'bench/cpu-main.mjs',
+    'bench/repeat-cpu.mjs',
+    'bench/bun-tsconfig.json'
+  ]) {
+    const target = path.join(destination, relative)
+    mkdirSync(path.dirname(target), { recursive: true })
+    cpSync(path.join(root, relative), target)
+  }
+
+  linkNodeModules(destination, nodeModules)
+}
+
+function linkNodeModules(destination, graphNodeModules) {
+  const target = path.join(destination, 'node_modules')
+  if (!existsSync(target)) {
+    symlinkSync(graphNodeModules, target, 'dir')
+  }
+}
+
+function sourceManifest(directory) {
+  const files = []
+  listFiles(directory, '', files)
+  files.sort((left, right) => left.localeCompare(right))
+  const hash = createHash('sha256')
+  let byteCount = 0
+  const entries = []
+  for (const relative of files) {
+    const content = readFileSync(path.join(directory, relative))
+    const sha256 = createHash('sha256').update(content).digest('hex')
+    hash.update(relative).update('\0').update(sha256).update('\n')
+    byteCount += content.byteLength
+    if (relative.startsWith('bench/') || relative === 'package.json' || relative === 'bun.lock') {
+      entries.push({ path: relative, sha256 })
+    }
+  }
+
+  return {
+    revision: directory === candidateRoot ? currentRevision() : null,
+    treeSha256: hash.digest('hex'),
+    fileCount: files.length,
+    byteCount,
+    benchmarkFiles: entries
+  }
+}
+
+function listFiles(directory, relative, output) {
+  for (const entry of readdirSync(path.join(directory, relative), { withFileTypes: true })) {
+    if (entry.name.startsWith('.')) {
+      continue
+    }
+
+    const childRelative = path.join(relative, entry.name)
+    if (entry.isDirectory()) {
+      if (!omittedDirectories.has(entry.name)) {
+        listFiles(directory, childRelative, output)
+      }
+    } else if (entry.isFile()) {
+      output.push(childRelative.split(path.sep).join('/'))
+    }
+  }
+}
+
+function currentRevision() {
+  return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim()
+}
+
+function hashFile(filename) {
+  return createHash('sha256').update(readFileSync(filename)).digest('hex')
+}
+
+function makeSchedule(seed) {
+  const cells = [...baselineCells(), ...candidateCells(), ...bareCells()]
+
+  if (cells.length !== 33) {
+    throw new Error(`Expected 33 cells, got ${cells.length}`)
+  }
+
+  return shuffleJobs(
+    cells.flatMap((cell) => [1, 2, 3].map((replicate) => ({ ...cell, replicate }))),
+    seed
+  )
+}
+
+function baselineCells() {
+  const cells = []
+  for (const revision of baselineRevisions) {
+    for (const runtime of ['node', 'bun']) {
+      cells.push({
+        kind: 'baseline',
+        revision,
+        runtime,
+        mode: 'main',
+        workload: 'tools',
+        spacing: null
+      })
+      for (const spacing of ['immediate', 'timeout']) {
+        cells.push({
+          kind: 'baseline',
+          revision,
+          runtime,
+          mode: 'main',
+          workload: 'stream',
+          spacing
+        })
+      }
+    }
+  }
+
+  return cells
+}
+
+function candidateCells() {
+  const cells = []
+  for (const runtime of ['node', 'bun']) {
+    for (const mode of ['plain', 'bench', 'production']) {
+      cells.push({
+        kind: 'candidate',
+        revision: 'candidate',
+        runtime,
+        mode,
+        workload: 'tools',
+        spacing: null
+      })
+      for (const spacing of ['immediate', 'timeout']) {
+        cells.push({
+          kind: 'candidate',
+          revision: 'candidate',
+          runtime,
+          mode,
+          workload: 'stream',
+          spacing
+        })
+      }
+    }
+  }
+
+  return cells
+}
+
+function bareCells() {
+  const cells = []
+  for (const workload of ['tools', 'stream']) {
+    const spacings = workload === 'tools' ? [null] : ['immediate', 'timeout']
+    for (const spacing of spacings) {
+      cells.push({
+        kind: 'candidate',
+        revision: 'candidate',
+        runtime: 'bare',
+        mode: 'production',
+        workload,
+        spacing
+      })
+    }
+  }
+
+  return cells
+}
+
+function shuffleJobs(jobs, seed) {
+  let state = seedHash(seed)
+  for (let index = jobs.length - 1; index > 0; index--) {
+    state ^= state << 13
+    state ^= state >>> 17
+    state ^= state << 5
+    const swapIndex = (state >>> 0) % (index + 1)
+    ;[jobs[index], jobs[swapIndex]] = [jobs[swapIndex], jobs[index]]
+  }
+
+  return jobs
+}
+
+function seedHash(value) {
+  let hash = 2166136261
+  for (const character of value) {
+    hash = Math.imul(hash ^ character.charCodeAt(0), 16777619)
+  }
+
+  return hash >>> 0 || 1
+}
+
+function runJob(job, scheduleIndex) {
+  const directory = job.kind === 'candidate' ? candidateRoot : baselineRoots.get(job.revision)
+  const invocation = createInvocation(job, directory)
+  const startedAt = new Date().toISOString()
+  const started = process.hrtime.bigint()
+  const child = spawnSync(invocation.executable, invocation.args, {
+    cwd: invocation.cwd,
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      PATH: prependExecutableDirectory(job.runtime, process.env.PATH ?? ''),
+      QVAC_NO_SKILL_BIN_PROBE: '1'
+    },
+    maxBuffer: 32 * 1024 * 1024
+  })
+  const wallMs = Number(process.hrtime.bigint() - started) / 1e6
+  if (child.error) {
+    throw child.error
+  }
+
+  if (child.status !== 0) {
+    throw new Error(
+      `${invocation.executable} exited ${child.status} for ${cellKey(job)}:\n${child.stderr || child.stdout}`
+    )
+  }
+
+  const parsed = parseLastJson(child.stdout)
+  const result =
+    job.runtime === 'bare' ? readBareResult(parsed, job) : readRepeatResult(parsed, job)
+  assertRuntimeVersion(parsed, job)
+
+  return reportJob({ job, invocation, scheduleIndex, startedAt, wallMs, parsed, result })
+}
+
+function createInvocation(job, directory) {
+  const script = path.join(directory, 'bench/repeat-cpu.mjs')
+  if (job.runtime === 'bare') {
+    return {
+      executable: bareRuntimeBinary,
+      args: [
+        'bench/cpu-bare-runner.mjs',
+        '3',
+        `--mode=${job.mode}`,
+        `--spacing=${job.spacing ?? 'immediate'}`,
+        `--workload=${job.workload}`,
+        '--json'
+      ],
+      cwd: directory
+    }
+  }
+
+  const args = [
+    `--runtime=${job.runtime}`,
+    `--driver=${job.kind === 'baseline' ? 'main' : 'engine'}`,
+    '--processes=1',
+    '--runs=3',
+    `--spacing=${job.spacing ?? 'immediate'}`,
+    `--workload=${job.workload}`,
+    `--revision=${job.revision}`
+  ]
+  if (job.kind !== 'baseline') {
+    args.push(`--mode=${job.mode}`)
+  }
+
+  return {
+    executable: nodeBinary,
+    args: ['--experimental-strip-types', script, ...args],
+    cwd: directory
+  }
+}
+
+function assertRuntimeVersion(report, job) {
+  if (report.version !== runtimes[job.runtime].version) {
+    throw new Error(`Runtime version changed during ${cellKey(job)}: ${report.version}`)
+  }
+}
+
+function reportJob({ job, invocation, scheduleIndex, startedAt, wallMs, parsed, result }) {
+  return {
+    order: scheduleIndex + 1,
+    startedAt,
+    wallMs,
+    key: cellKey(job),
+    replicate: job.replicate,
+    command: { executable: invocation.executable, args: invocation.args, cwd: invocation.cwd },
+    runtime: parsed.runtime,
+    version: parsed.version,
+    rawCpuSamplesMs: result.cpuSamplesMs,
+    processMedianMs: result.cpuMs,
+    logBytes: result.logBytes ?? null,
+    records: result.records ?? null,
+    contentEvents: result.contentEvents
+  }
+}
+
+function readRepeatResult(report, job) {
+  if (
+    report.runtime !== job.runtime ||
+    !Array.isArray(report.samples) ||
+    report.samples.length !== 1
+  ) {
+    throw new Error(`Unexpected repeat-cpu report for ${cellKey(job)}`)
+  }
+
+  const process = report.samples[0]
+  const expectedMode = job.kind === 'baseline' ? 'main' : job.mode
+  const result = process?.results?.[0]
+  validateResult(result, job, expectedMode)
+  if (!Array.isArray(result.cpuSamplesMs) || result.cpuSamplesMs.length !== 3) {
+    throw new Error(`Expected three raw samples in ${cellKey(job)}`)
+  }
+
+  validateSamples(result, cellKey(job))
+
+  return result
+}
+
+function readBareResult(report, job) {
+  if (report.runtime !== 'bare' || !Array.isArray(report.results) || report.results.length !== 1) {
+    throw new Error(`Unexpected Bare report for ${cellKey(job)}`)
+  }
+
+  const result = report.results[0]
+  validateResult(result, job, job.mode)
+  if (!Array.isArray(result.cpuSamplesMs) || result.cpuSamplesMs.length !== 3) {
+    throw new Error(`Expected three raw Bare samples in ${cellKey(job)}`)
+  }
+
+  validateSamples(result, cellKey(job))
+
+  return result
+}
+
+function validateSamples(result, key) {
+  if (
+    !Array.isArray(result.cpuSamplesMs) ||
+    !result.cpuSamplesMs.every((sample) => Number.isFinite(sample) && sample >= 0) ||
+    result.cpuMs !== median(result.cpuSamplesMs)
+  ) {
+    throw new Error(`Invalid per-process raw CPU samples in ${key}`)
+  }
+}
+
+function validateResult(result, job, mode) {
+  const carriesLogMetrics = job.kind !== 'baseline'
+  if (
+    !result ||
+    result.workload !== job.workload ||
+    result.mode !== mode ||
+    result.runs !== 3 ||
+    !Number.isFinite(result.cpuMs) ||
+    (carriesLogMetrics &&
+      (!Number.isFinite(result.logBytes) || !Number.isFinite(result.records))) ||
+    (!carriesLogMetrics && (result.logBytes !== undefined || result.records !== undefined))
+  ) {
+    throw new Error(`Incomplete benchmark result for ${cellKey(job)}`)
+  }
+
+  if (
+    job.workload === 'stream' &&
+    (result.contentEvents !== 2000 || result.spacing !== job.spacing)
+  ) {
+    throw new Error(`Stream workload did not preserve events/spacing for ${cellKey(job)}`)
+  }
+}
+
+function parseLastJson(stdout) {
+  for (const line of stdout.trim().split(/\r?\n/).reverse()) {
+    try {
+      const value = JSON.parse(line)
+      if (value && typeof value === 'object') {
+        return value
+      }
+    } catch {}
+  }
+
+  throw new Error('Benchmark process did not emit a JSON report')
+}
+
+function aggregateCells(execution) {
+  const cells = new Map()
+  for (const sample of execution) {
+    let cell = cells.get(sample.key)
+    if (!cell) {
+      cell = { key: sample.key, processes: [] }
+      cells.set(sample.key, cell)
+    }
+
+    cell.processes.push(sample)
+  }
+
+  return [...cells.values()].map((cell) => {
+    if (cell.processes.length > 3) {
+      throw new Error(`Too many process samples for ${cell.key}`)
+    }
+
+    const raw = cell.processes.flatMap((process) => process.rawCpuSamplesMs)
+
+    return {
+      key: cell.key,
+      processCount: cell.processes.length,
+      rawSampleCount: raw.length,
+      rawCpuSamplesMs: raw,
+      cpuMs: raw.length === 9 ? median(raw) : null,
+      processMedianMs: cell.processes.map((process) => process.processMedianMs),
+      logBytes: medianOrNull(cell.processes.map((process) => process.logBytes)),
+      records: medianOrNull(cell.processes.map((process) => process.records)),
+      contentEvents: cell.processes[0]?.contentEvents
+    }
+  })
+}
+
+function cellKey(job) {
+  return [job.kind, job.revision, job.runtime, job.mode, job.workload, job.spacing ?? ''].join(':')
+}
+
+function median(values) {
+  const sorted = [...values].sort((left, right) => left - right)
+  return sorted[Math.floor(sorted.length / 2)]
+}
+
+function medianOrNull(values) {
+  const finite = values.filter(Number.isFinite)
+  return finite.length ? median(finite) : null
+}
+
+function installedPackageVersions(graphNodeModules, names) {
+  return Object.fromEntries(
+    names.map((name) => {
+      const manifestPath = path.join(graphNodeModules, name, 'package.json')
+      const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'))
+      return [name, manifest.version]
+    })
+  )
+}
+
+function prependExecutableDirectory(runtime, existingPath) {
+  if (runtime === 'node') {
+    return existingPath
+  }
+
+  const executable = runtime === 'bun' ? bunBinary : bareRuntimeBinary
+  if (!executable.includes(path.sep)) {
+    return existingPath
+  }
+
+  return `${path.dirname(path.resolve(executable))}${path.delimiter}${existingPath}`
+}
+
+function versionOf(executable, args) {
+  const result = spawnSync(executable, args, { encoding: 'utf8', maxBuffer: 1024 * 1024 })
+  if (result.error) {
+    throw result.error
+  }
+
+  if (result.status !== 0) {
+    throw new Error(`Could not read runtime version from ${executable}`)
+  }
+
+  return result.stdout.trim().split(/\r?\n/)[0]
+}
+
+function parseOptions(args) {
+  const result = {
+    seed: 'issue129-final-2026-09',
+    node: undefined,
+    bun: undefined,
+    nodeModules: undefined,
+    bareRuntime: undefined,
+    output: path.join(root, 'output/benchmarks/issue-129/final-matrix.json'),
+    dryRun: false,
+    prepareOnly: false
+  }
+  for (const arg of args) {
+    const [key, value] = arg.split('=', 2)
+    if (arg === '--dry-run') {
+      result.dryRun = true
+    } else if (arg === '--prepare-only') {
+      result.prepareOnly = true
+    } else if (key === '--seed' && value) {
+      result.seed = value
+    } else if (key === '--node' && value) {
+      result.node = value
+    } else if (key === '--bun' && value) {
+      result.bun = value
+    } else if (key === '--node-modules' && value) {
+      result.nodeModules = path.resolve(value)
+    } else if (key === '--bare-runtime' && value) {
+      result.bareRuntime = value
+    } else if (key === '--output' && value) {
+      result.output = value
+    } else {
+      throw new Error(`Unknown option: ${arg}`)
+    }
+  }
+
+  if (!existsSync(optionsNodePath(result))) {
+    throw new Error(`Node 24 binary not found: ${optionsNodePath(result)}`)
+  }
+
+  const graphNodeModules = path.resolve(
+    result.nodeModules ?? process.env.QVAC_MATRIX_NODE_MODULES ?? path.join(root, 'node_modules')
+  )
+  if (!existsSync(graphNodeModules)) {
+    throw new Error(`Validated dependency graph not found: ${graphNodeModules}`)
+  }
+
+  result.nodeModules = realpathSync(graphNodeModules)
+
+  return result
+}
+
+function optionsNodePath(result) {
+  return path.resolve(result.node ?? process.env.QVAC_NODE24_BIN ?? nodeDefault)
+}