@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/report.ts

@@ -0,0 +1,1075 @@
+// every benchmark on one page: bare bench/report.ts --bfcl l=<dir> --tau2 <dir> --out <f>
+import fs from 'fs'
+import path from 'path'
+import process from 'process'
+import { BFCL_COMMIT, CATEGORIES, SHAPES, type Shape } from './bfcl/data.ts'
+import { addSpeed, NO_SPEED, speedOf, type Speed } from './speed.ts'
+
+interface BfclRow {
+  readonly column: string
+  readonly model: string
+  readonly category: string
+  readonly id: string
+  readonly shape?: Shape
+  readonly valid: boolean
+  readonly error: string | null
+  readonly wholeValid: boolean
+  readonly speed?: Speed | null
+}
+
+interface MultiTurnRow {
+  readonly column: string
+  readonly model: string
+  readonly category: string
+  readonly id: string
+  readonly valid: boolean
+  // BFCL's own error type, such as multi_turn:instance_state_mismatch
+  readonly error_type: string | null
+  readonly error: string | null
+  readonly speed?: Speed | null
+}
+
+interface Tau2Row {
+  readonly run: string
+  readonly domain: string
+  readonly model: string
+  readonly task: string
+  readonly writes: boolean
+  readonly reward: number
+  readonly termination: string
+  readonly seconds: number
+  readonly replies?: number
+  readonly failed_turns?: number
+  readonly customer_never_ended?: boolean
+  readonly speed?: Speed
+}
+
+interface TerminalTrial {
+  readonly task: string
+  readonly reward: number
+  readonly model: string | null
+  // absent when the trial left no log
+  readonly limit: number | 'default' | null
+  readonly calls: number
+  readonly speed: Speed | null
+  // the one-minute load average at the start and end of the trial
+  readonly load: number[]
+}
+
+interface BaselineTau2 {
+  readonly run: string
+  readonly model: string
+  readonly tasks: string
+  readonly pass: number
+  readonly writes: number
+  readonly refusals: number
+  readonly capped: string
+  readonly lost: string
+  readonly minutes: string
+}
+
+const TAU2_SAMPLE = 20
+const MULTI_TURN_CATEGORIES = [
+  'multi_turn_base',
+  'multi_turn_miss_func',
+  'multi_turn_miss_param',
+  'multi_turn_long_context'
+]
+
+// figures copied from the #151 / PR #35 page, whose run dirs were not retained
+const BASELINE_COLUMNS = [
+  '#35 · Qwen3.5-2B',
+  '#35 · Qwen3.5-4B',
+  '#35 · Qwen3.5-9B',
+  'prompt asks for independent calls together · Qwen3.5-2B'
+]
+// strict then across rounds, per baseline column
+const BASELINE_BFCL: Record<string, readonly (number | null)[]> = {
+  simple_python: [84, 84, 90, 90, 88, 88, 84, 84],
+  multiple: [84, 84, 86, 86, 90, 90, 82, 82],
+  parallel: [0, 76, 0, 84, 0, 86, 0, 76],
+  parallel_multiple: [0, 72, 0, 88, 0, 90, 0, 84],
+  irrelevance: [74, 74, 86, 86, 84, 84, null, null],
+  live_simple: [66, 66, 68, 68, 80, 80, 58, 58],
+  live_multiple: [50, 50, 72, 72, 74, 74, null, null],
+  live_parallel: [0, 63, 0, 63, 0, 81, 0, 56],
+  live_parallel_multiple: [0, 58, 0, 63, 0, 50, 0, 50],
+  live_irrelevance: [76, 76, 84, 84, 82, 82, null, null],
+  live_relevance: [88, 88, 75, 75, 88, 88, null, null]
+}
+const BEFORE_FIXES = '#35, before fixes (withdrawn)'
+const BASELINE_TAU2: readonly BaselineTau2[] = [
+  {
+    run: BEFORE_FIXES,
+    model: 'Qwen3.5-2B',
+    tasks: '20/20',
+    pass: 20,
+    writes: 15,
+    refusals: 29,
+    capped: '9',
+    lost: '47/195 (24%)',
+    minutes: '6.2'
+  },
+  {
+    run: BEFORE_FIXES,
+    model: 'Qwen3.5-4B',
+    tasks: '20/20',
+    pass: 30,
+    writes: 15,
+    refusals: 57,
+    capped: '4',
+    lost: '37/170 (22%)',
+    minutes: '8.1'
+  },
+  {
+    run: BEFORE_FIXES,
+    model: 'Qwen3.5-9B',
+    tasks: '19/20',
+    pass: 11,
+    writes: 0,
+    refusals: 33,
+    capped: '17',
+    lost: '333/385 (86%)',
+    minutes: '9.5'
+  },
+  {
+    run: '#151, all fixes',
+    model: 'Qwen3.5-2B',
+    tasks: '20/20',
+    pass: 45,
+    writes: 38,
+    refusals: 57,
+    capped: '2',
+    lost: '8/135 (6%)',
+    minutes: '6.2'
+  },
+  {
+    run: '#151, all fixes',
+    model: 'Qwen3.5-9B',
+    tasks: '20/20',
+    pass: 60,
+    writes: 46,
+    refusals: 86,
+    capped: '1 (1 waiting on the customer)',
+    lost: '2/119 (2%)',
+    minutes: '11.5'
+  }
+]
+
+const argv = process.argv.slice(2)
+const all = (name: string) => argv.flatMap((arg, index) => (arg === name ? [argv[index + 1]] : []))
+const out = argv[argv.indexOf('--out') + 1]
+
+const bfcl: BfclRow[] = all('--bfcl').flatMap(columnRows)
+const columns = [...new Set(bfcl.map((row) => row.column))]
+// the first BFCL run is the branch under test; later ones are experiments beside it
+const primary = (all('--bfcl')[0] ?? '').split('=')[0]
+const categories = CATEGORIES.filter((category) => bfcl.some((row) => row.category === category))
+
+const multiTurn: MultiTurnRow[] = all('--bfcl-multi-turn').flatMap(columnRows)
+const multiTurnColumns = [...new Set(multiTurn.map((row) => row.column))]
+
+// a tau2 run is label=dir, oldest first; a file is <domain>-<model>-<stamp>.jsonl, one row per task
+const tau2Specs = all('--tau2').map(labelled)
+const tau2: Tau2Row[] = tau2Specs.flatMap(({ label, dir }) =>
+  fs
+    .readdirSync(dir)
+    .filter((file) => file.endsWith('.jsonl'))
+    .flatMap((file) => {
+      const name = file.replace(/\.jsonl$/, '')
+      const domain = name.slice(0, name.indexOf('-'))
+      const model = name.slice(name.indexOf('-') + 1, name.lastIndexOf('-'))
+      return lines(path.join(dir, file)).map((row) => ({ ...row, run: label, domain, model }))
+    })
+)
+const tau2Models = [...new Set(tau2.map((row) => row.model))].sort(bySize)
+// the latest run is the one the headline cards report
+const currentRun = tau2Specs.at(-1)?.label ?? ''
+
+// a Terminal-Bench run is label=<harbor job dir>; a trial dir holds result.json and agent/qvac.jsonl
+const terminal = all('--terminal').map((spec) => {
+  const [label, dir] = spec.split('=', 2)
+  const trials = fs
+    .readdirSync(dir, { withFileTypes: true })
+    .filter(
+      (entry) => entry.isDirectory() && fs.existsSync(path.join(dir, entry.name, 'result.json'))
+    )
+    .map((entry): TerminalTrial => {
+      const trial = path.join(dir, entry.name)
+      const result = JSON.parse(fs.readFileSync(path.join(trial, 'result.json'), 'utf8'))
+      const logFile = path.join(trial, 'agent', 'qvac.jsonl')
+      const log = fs.existsSync(logFile) ? lines(logFile) : []
+      const start = log.find((row) => row.type === 'start')
+      return {
+        task: result.task_name,
+        reward: result.verifier_result?.rewards?.reward ?? 0,
+        model: start?.model ?? null,
+        limit: start ? (start.max_tool_rounds ?? 'default') : null,
+        calls: log.filter((row) => row.type === 'call').length,
+        speed: speedOf(log.find((row) => row.type === 'done')?.metrics ?? null),
+        load: log.filter((row) => row.load).map((row) => row.load[0])
+      }
+    })
+
+  return { label, trials }
+})
+
+interface RunSpec {
+  readonly label: string
+  readonly dir: string
+}
+
+// a run is label=dir, or a bare dir
+function labelled(spec: string): RunSpec {
+  const [label, dir] = spec.includes('=') ? spec.split('=', 2) : ['', spec]
+  return { label, dir }
+}
+
+// each model of a run's runs.jsonl becomes a column
+function columnRows(spec: string) {
+  const { label, dir } = labelled(spec)
+  return lines(path.join(dir, 'runs.jsonl')).map((row) => ({
+    ...row,
+    column: label ? `${label} · ${short(row.model)}` : short(row.model)
+  }))
+}
+
+function page(): string {
+  const models = [
+    ...new Set([...bfcl, ...multiTurn].map((row) => row.model).concat(tau2Models))
+  ].sort(bySize)
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Harness Benchmarks</title>
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600;700&family=Inconsolata:wght@400;600&display=swap" rel="stylesheet">
+<style>${BRAND_STYLE}${STYLE}</style>
+</head>
+<body>
+<main>
+<header class="topbar"><a class="brand" href="#">${WORDMARK}<span>Harness</span></a><button class="toggle" type="button" id="theme">Theme</button></header>
+<div class="hero">
+<p class="eyebrow">Known benchmarks</p>
+<h1>@qvac/harness on published benchmarks</h1>
+<p class="lede">Built on PR #35 (the completion state machine) and PR #151 (the benchmark fixes); the harness-alignment work (waves 1 to 8, #155 onwards) lands on <code>dev</code>, measured here at 3ca35a56. Local Qwen3.5 models, Q4_K_M, on one Mac. Every number comes from a published benchmark's own tasks and scoring; nothing here is a task we wrote.</p>
+<div class="versions">${models.map((model) => `<span class="chip"><i class="dot ours"></i><b>${escape(short(model))}</b></span>`).join('')}<span class="chip">BFCL v4 · gorilla ${BFCL_COMMIT.slice(0, 7)}</span><span class="chip">τ²-bench airline</span></div>
+</div>
+${overview()}
+${bfclSection()}
+${tau2Section()}
+${speedSection()}
+${terminalSection()}
+<section><h2><span class="num">05</span>Findings</h2><div class="notes">${FINDINGS}</div></section>
+</main>
+<div id="tip" role="tooltip" hidden></div>
+<script>${SCRIPT}${THEME_SCRIPT}</script>
+</body>
+</html>
+`
+}
+
+function overview(): string {
+  const underTest = (model: string) => (primary ? `${primary} · ${short(model)}` : short(model))
+  const models = [...new Set([...bfcl.map((row) => row.model), ...tau2Models])].sort(bySize)
+  const cards = models.map((model) => {
+    const strict = bfclMean(bfcl.filter((row) => row.column === underTest(model)))
+    const own = tau2.filter((row) => row.model === model && row.run === currentRun)
+    const tau = own.length ? `${pct(mean(own.map((row) => row.reward)))}` : '–'
+    return `<div class="card stat"><h3><i class="dot ours"></i>${escape(short(model))}</h3><div class="pair"><div><div class="figure">${pct(strict)}</div><div class="figure-label">BFCL strict</div></div><div><div class="figure">${tau}</div><div class="figure-label">τ² airline pass^1${own.length && own.length < TAU2_SAMPLE ? ` · ${own.length}/${TAU2_SAMPLE} so far` : ''}</div></div></div></div>`
+  })
+
+  return `<div class="cards">${cards.join('')}</div>`
+}
+
+function bfclSection(): string {
+  if (!bfcl.length && !multiTurn.length) {
+    return ''
+  }
+
+  return `<section>
+<h2><span class="num">01</span>BFCL v4 <span class="tag">function calling</span></h2>
+<p class="section-lede">The Berkeley Function Calling Leaderboard's eleven single-turn categories, up to 50 cases each, data and checker from gorilla commit <code>${BFCL_COMMIT.slice(0, 7)}</code>. Each case's functions are the run's only tools. <em>Strict</em> is BFCL's rule, every call in the first response. <em>Across rounds</em> also credits parallel calls the harness made one per round. Hover a strict cell for its failures. The #35 columns are figures from the #151 / PR #35 page; their run dirs were not retained, so they carry no failure detail.</p>
+<div class="table-wrap"><table class="data">
+<thead><tr><th scope="col">Category</th>${[...columns, ...BASELINE_COLUMNS].map((column) => `<th scope="col" colspan="2">${escape(column)}</th>`).join('')}</tr>
+<tr><th></th>${[...columns, ...BASELINE_COLUMNS].map(() => '<th class="sub">strict</th><th class="sub">across rounds</th>').join('')}</tr></thead>
+<tbody>${categories.map(bfclRow).join('')}</tbody>
+</table></div>
+${shapeTable()}
+${multiTurnTable()}
+</section>`
+}
+
+function multiTurnTable(): string {
+  if (!multiTurn.length) {
+    return ''
+  }
+
+  const rows = MULTI_TURN_CATEGORIES.map((category) => {
+    const cells = multiTurnColumns.map((column) => {
+      const own = multiTurn.filter((row) => row.column === column && row.category === category)
+      const tip = own
+        .filter((row) => !row.valid)
+        .slice(0, 6)
+        .map((row) => `${row.id}: ${row.error_type}`)
+        .join('\n')
+
+      return `<td>${own.length}</td>${cell(
+        own.map((row) => row.valid),
+        tip
+      )}`
+    })
+    return `<tr><th scope="row">${category}</th>${cells.join('')}</tr>`
+  })
+
+  return `<p class="section-lede" style="margin-top:20px">Multi-turn: BFCL v4's four multi-turn categories, each user turn one harness run on gorilla's own backends, scored by BFCL's <code>multi_turn_checker</code> applied to the harness's own behaviour (its system prompt, nudges, result cap, and retries that count toward the round limit), not to BFCL's bare prompt.</p>
+<div class="table-wrap"><table class="data">
+<thead><tr><th scope="col">Category</th>${multiTurnColumns.map((column) => `<th scope="col" colspan="2">${escape(column)}</th>`).join('')}</tr>
+<tr><th></th>${multiTurnColumns.map(() => '<th class="sub">entries</th><th class="sub">pass</th>').join('')}</tr></thead>
+<tbody>${rows.join('')}</tbody>
+</table></div>
+${multiTurnFailures()}`
+}
+
+function multiTurnFailures(): string {
+  const types = [...new Set(multiTurn.flatMap((row) => (row.error_type ? [row.error_type] : [])))]
+  const classes = types.map((type) => {
+    const cells = multiTurnColumns.map(
+      (column) =>
+        `<td>${multiTurn.filter((row) => row.column === column && row.error_type === type).length}</td>`
+    )
+    return `<tr><th scope="row">${escape(type.replace('multi_turn:', ''))}</th>${cells.join('')}</tr>`
+  })
+
+  return `<p class="section-lede" style="margin-top:20px">Multi-turn failures by BFCL's own error type.</p>
+<div class="table-wrap"><table class="data">
+<thead><tr><th scope="col">Failure</th>${multiTurnColumns.map((column) => `<th scope="col">${escape(column)}</th>`).join('')}</tr></thead>
+<tbody>${classes.join('')}</tbody>
+</table></div>`
+}
+
+// runs from before the shape tag carry none, so the baseline is compared by its text
+function shapeTable(): string {
+  const tagged = bfcl.filter((row) => row.shape)
+  const shaped = columns.filter((column) => tagged.some((row) => row.column === column))
+  if (!shaped.length) {
+    return ''
+  }
+
+  const rows = SHAPES.map((shape) => {
+    const cells = shaped.map((column) => {
+      const own = tagged.filter((row) => row.column === column && row.shape === shape)
+      return `<td>${own.length}</td>${cell(
+        own.map((row) => row.valid),
+        ''
+      )}${cell(
+        own.map((row) => row.wholeValid),
+        ''
+      )}`
+    })
+    return `<tr><th scope="row">${shape}</th>${cells.join('')}</tr>`
+  })
+
+  return `<p class="section-lede" style="margin-top:20px">By schema shape (#157). <em>Nested</em> cases have a parameter with fields or items, <em>default-only</em> ones a default and no nesting: the two the harness spells out in the tool description because the SDK drops them. <em>Untouched</em> reach the model as written. The #151 baseline was not tagged; it measured nested cases at 57% strict against 92% for flat ones. Strict here includes the parallel categories, which score 0% under the one-call limit and weigh most on untouched cases; without them 9B passes 77% nested, 80% default-only and 88% untouched.</p>
+<div class="table-wrap"><table class="data">
+<thead><tr><th scope="col">Shape</th>${shaped.map((column) => `<th scope="col" colspan="3">${escape(column)}</th>`).join('')}</tr>
+<tr><th></th>${shaped.map(() => '<th class="sub">cases</th><th class="sub">strict</th><th class="sub">across rounds</th>').join('')}</tr></thead>
+<tbody>${rows.join('')}</tbody>
+</table></div>`
+}
+
+function bfclRow(category: string): string {
+  const cells = columns.map((column) => {
+    const own = bfcl.filter((row) => row.column === column && row.category === category)
+    const tip = own
+      .filter((row) => !row.valid)
+      .slice(0, 6)
+      .map((row) => `${row.id}: ${row.error ?? ''}`)
+      .join('\n')
+
+    return (
+      cell(
+        own.map((row) => row.valid),
+        tip
+      ) +
+      cell(
+        own.map((row) => row.wholeValid),
+        ''
+      )
+    )
+  })
+
+  const baseline = (BASELINE_BFCL[category] ?? []).map((percent) => figure(percent, ''))
+
+  return `<tr><th scope="row">${escape(category)}</th>${cells.join('')}${baseline.join('')}</tr>`
+}
+
+function tau2Section(): string {
+  const pairs = tau2Specs.flatMap(({ label }) =>
+    tau2Models
+      .filter((model) => tau2.some((row) => row.run === label && row.model === model))
+      .map((model) => ({ label, model }))
+  )
+  const rows = pairs.map(({ label, model }) => {
+    const own = tau2.filter((row) => row.model === model && row.run === label)
+    const writes = own.filter((row) => row.writes)
+    const refusals = own.filter((row) => !row.writes)
+    const capped = own.filter((row) => row.termination.endsWith('MAX_STEPS')).length
+    const unanswered = own.filter((row) => row.customer_never_ended).length
+    const lost = failedShare(own)
+    const tip = own
+      .map(
+        (row) =>
+          `task ${row.task} ${row.writes ? 'write' : 'read'}: ${row.reward} in ${Math.round(row.seconds)}s, ${row.termination.split('.').pop()}${row.customer_never_ended ? ', the customer never ended it' : ''}`
+      )
+      .join('\n')
+
+    return `<tr${label === currentRun ? ' class="current"' : ''}><th scope="row">${escape(label)}</th><th scope="row">${escape(short(model))}</th><td>${own.length}/${TAU2_SAMPLE}</td>${cell(
+      own.map((row) => row.reward === 1),
+      tip
+    )}${cell(
+      writes.map((row) => row.reward === 1),
+      ''
+    )}${cell(
+      refusals.map((row) => row.reward === 1),
+      ''
+    )}<td>${capped}${unanswered ? ` (${unanswered} waiting on the customer)` : ''}</td><td>${lost}</td><td>${own.length ? (mean(own.map((row) => row.seconds)) / 60).toFixed(1) : '–'} min</td></tr>`
+  })
+
+  const baseline = BASELINE_TAU2.map(
+    (row) =>
+      `<tr><th scope="row">${escape(row.run)}</th><th scope="row">${escape(row.model)}</th><td>${row.tasks}</td>${figure(row.pass, '')}${figure(row.writes, '')}${figure(row.refusals, '')}<td>${escape(row.capped)}</td><td>${row.lost}</td><td>${row.minutes} min</td></tr>`
+  )
+
+  return `<section>
+<h2><span class="num">02</span>τ²-bench <span class="tag">multi-turn agent with a policy</span></h2>
+<p class="section-lede">Sierra's τ²-bench airline domain: the harness is the customer-service agent, τ²'s orchestrator runs the tools against its own database, and τ²'s reward (database state and what was communicated) scores each task. ${TAU2_SAMPLE} tasks spread evenly over the 50, 13 of them needing database changes. The simulated customer is local Qwen3.5-9B, not the leaderboard's GPT-4.1, so scores are τ²'s own but not leaderboard-comparable. A round is cut off at three minutes. <em>Replies lost to errors</em> counts turns where the harness returned no answer; in the first run almost all came from one SDK grammar error (see findings). Those turns count against the agent, as the app would show the user an error. <em>Waiting on the customer</em> counts tasks that hit the step limit because the simulated customer never sent τ²'s end token: it answered a correct hand-off with \"I will continue to hold\" instead of <code>###TRANSFER###</code>, or wrote a bare STOP instead of <code>###STOP###</code>. τ² scores those 0; the loss is the local customer model's, not the agent's.</p>
+${
+  rows.length
+    ? `<div class="table-wrap"><table class="data">
+<thead><tr><th scope="col">Run</th><th scope="col">Agent model</th><th scope="col">Tasks</th><th scope="col">pass^1</th><th scope="col">needs a change</th><th scope="col">policy says refuse</th><th scope="col">hit 60 steps</th><th scope="col">replies lost to errors</th><th scope="col">avg per task</th></tr></thead>
+<tbody>${[...baseline, ...rows].join('')}</tbody>
+</table></div>`
+    : '<p class="pending">Running; no task finished yet.</p>'
+}
+${TAU2_CHANGES}
+</section>`
+}
+
+// every failure traced to a cause, and the issue that acts on it
+const FINDINGS = `
+<h2>Re-run on dev</h2>
+<p>
+  <a href="https://github.com/tetherto/qvac-harness/issues/184">#184</a> re-runs both benchmarks on
+  <code>dev</code> at 3ca35a56, after waves 1 to 8, with the #151 settings. BFCL across rounds,
+  all 456 cases against the #35 figures: 2B 72.1% to 74.8%, 4B 80.3% to 83.8%, 9B 82.5% to 82.9%.
+  Simple, multiple and parallel calls gained (9B <code>simple_python</code> 88% to 98%, 9B
+  <code>live_parallel_multiple</code> 50% to 75% across rounds). The irrelevance categories
+  regressed, most on 9B <code>live_irrelevance</code>, 82% to 64%: models call an offered tool
+  where none applies (<a href="https://github.com/tetherto/qvac-harness/issues/252">#252</a>, being
+  bisected). τ²-bench regressed against #151: pass^1 45% to 30% on 2B and 60% to 25% on 9B, with
+  1 of 13 tasks that need a database change passed by each model (38% and 46% at #151). Replies
+  lost to errors stayed low, 5 of 130 on 2B and 9 of 179 on 9B, all empty replies, none to the
+  grammar fault. Not yet traced; the runs overlapped heavy load on the Mac (see speed).
+</p>
+<h2>Every failure, traced</h2>
+<p>
+  Every failure in both benchmarks was traced to a root cause: 619 BFCL strict failures, each τ²
+  transcript, and the original BFCL <code>ast_checker.py</code> re-run on our rows, which flips 0
+  verdicts. They split into bugs that code can fix, limits in the SDK, model weaknesses, and gaps
+  that need a design change.
+</p>
+<table class="findings">
+  <thead>
+    <tr>
+      <th>Failure</th>
+      <th>Kind</th>
+      <th>Evidence</th>
+      <th>Status</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>
+        <strong>τ² runs leaked into later tasks.</strong> A task ended on an unanswered call left
+        its harness run alive, so later tasks got replies one step behind and earlier writes hit
+        later tasks' databases.
+      </td>
+      <td>Code bug (our τ² adapter)</td>
+      <td>Decided 15 of 16 2B and 8 of 14 4B τ² failures</td>
+      <td>
+        Fixed in PR #151; the first τ² run is withdrawn (see the before and after table above).
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>Wrong clock.</strong> The harness stamps today (2026) on the user's turn; τ²'s
+        policy is set on 2024-05-15, so the agent refuses "past" flights. BFCL dates drift to 2026
+        too.
+      </td>
+      <td>Harness gap</td>
+      <td>τ² refusals; 11 BFCL rows</td>
+      <td>
+        Fixed: <code>CompletionContext.now</code>. BFCL re-run on <code>dev</code> done
+        (<a href="https://github.com/tetherto/qvac-harness/issues/184">#184</a>).
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>llama.cpp tool-grammar fault</strong> ("Unexpected empty grammar stack after
+        accepting piece") ends long tool-calling turns.
+      </td>
+      <td>SDK bug</td>
+      <td>
+        86% of 9B's τ² replies before #151; about a quarter of 2B's and 4B's. None lost
+        to it on <code>dev</code> at 3ca35a56: 5 of 130 2B and 9 of 179 9B replies lost, all empty
+        replies
+      </td>
+      <td>
+        Worked around: the harness retries the round without the grammar. Draft PR #146 finds the
+        cause, past calls written back in a non-Qwen format the model copies. Structured history
+        merged (<a href="https://github.com/tetherto/qvac-harness/issues/169">#169</a>, on PR #146).
+        SDK ask in <a href="https://github.com/tetherto/qvac-harness/issues/185">#185</a>.
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>One tool call per response.</strong> The SDK returns exactly one parsed call, so
+        independent calls each cost a full round.
+      </td>
+      <td>SDK limit</td>
+      <td>
+        410 BFCL rows; parallel 0% strict vs 72–90% across rounds; a prompt rewording changed
+        nothing
+      </td>
+      <td>
+        The cap comes from llama.cpp's tool grammar (<a
+          href="https://github.com/tetherto/qvac-harness/issues/161"
+          >#161</a
+        >); SDK ask in <a href="https://github.com/tetherto/qvac-harness/issues/185">#185</a>.
+        Concurrent completions on one model:
+        <a href="https://github.com/tetherto/qvac-harness/issues/192">#192</a>.
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>Nested tool schemas stripped.</strong> The SDK's <code>toolSchema</code> drops
+        nested <code>properties</code>, <code>items</code> enums, <code>required</code> and
+        <code>default</code>, so models invent field names.
+      </td>
+      <td>SDK bug</td>
+      <td>32 strict and 7 parallel BFCL rows; nested-schema cases pass 57% vs 92%</td>
+      <td>
+        Harness stopgap merged: nested fields are described in the tool description (<a
+          href="https://github.com/tetherto/qvac-harness/issues/157"
+          >#157</a
+        >); dev re-run: 9B <code>simple_python</code> 88% to 98%. SDK ask in <a href="https://github.com/tetherto/qvac-harness/issues/185">#185</a>.
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>A tool's own <code>intent</code> argument stripped</strong> along with the
+        harness's.
+      </td>
+      <td>Code bug (harness)</td>
+      <td>6 BFCL rows; hits any MCP or skill tool with an intent field</td>
+      <td>Fixed in PR #151.</td>
+    </tr>
+    <tr>
+      <td>
+        <strong>Run tools clashed with same-named built-ins</strong>; the refused request also broke
+        the next run's cache write.
+      </td>
+      <td>Code bug (harness)</td>
+      <td>6 BFCL rows</td>
+      <td>Fixed (run tool replaces the built-in). The cache cascade needs its own look.</td>
+    </tr>
+    <tr>
+      <td><strong>An SDK error mid-stream leaves an unhandled promise rejection.</strong></td>
+      <td>Code bug (harness, pre-existing)</td>
+      <td>Reproduced on #35 without this PR</td>
+      <td>
+        Does not reproduce on <code>dev</code>; regression tests merged (<a
+          href="https://github.com/tetherto/qvac-harness/issues/160"
+          >#160</a
+        >).
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>Text-only history between turns.</strong> Earlier tool results reach later turns
+        only as a short text log, so facts get lost and results get invented.
+      </td>
+      <td>Adapter, and a design question for the app</td>
+      <td>Invented flight in one τ² task; a flight number cut off in another</td>
+      <td>
+        Log raised from 600 to 4000 characters. Structured tool turns merged
+        (<a href="https://github.com/tetherto/qvac-harness/issues/169">#169</a>).
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>Calls a tool when none applies</strong>, or
+        <strong>asks for clarification instead of calling</strong>.
+      </td>
+      <td>Model, plus a prompt trade-off</td>
+      <td>51 and 22 BFCL rows</td>
+      <td>Worse on <code>dev</code>: live_irrelevance 82% to 64% (9B), 84% to 78% (4B) and 76% to
+        72% (2B); irrelevance 74% to 70% (2B). Bisecting in
+        <a href="https://github.com/tetherto/qvac-harness/issues/252">#252</a>.</td>
+    </tr>
+    <tr>
+      <td><strong>Wrong values, wrong function, invented parameters</strong></td>
+      <td>Model weakness</td>
+      <td>About 50 BFCL rows, 13 of the 20 genuine value errors on 2B</td>
+      <td>
+        Argument conversion to the schema's types merged (<a
+          href="https://github.com/tetherto/qvac-harness/issues/156"
+          >#156</a
+        >); refusing cut-off calls merged (<a
+          href="https://github.com/tetherto/qvac-harness/issues/155"
+          >#155</a
+        >). The rest: model routing, constrained decoding or fine-tuning.
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>Policy slips in τ²:</strong> quoting a price for one passenger instead of three,
+        upgrading without quoting the cost, cancelling against policy, retrying with a wrong
+        argument name.
+      </td>
+      <td>Model, plus a design gap</td>
+      <td>The clean 4B τ² tasks</td>
+      <td>
+        Design: quote and confirm before any write, and a policy check before cancellations or
+        compensation (<a href="https://github.com/tetherto/qvac-harness/issues/183">#183</a>).
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <strong>BFCL answers we'd dispute</strong> (percentages expected as decimals, "City, ST"
+        against the schema's own example)
+      </td>
+      <td>Benchmark key</td>
+      <td>26 rows</td>
+      <td>Kept as scored, so the numbers stay comparable with the leaderboard.</td>
+    </tr>
+  </tbody>
+</table>
+<h2>Skill coverage gaps (inventory, not a benchmark)</h2>
+<ul>
+  <li>
+    <strong>No browser tool.</strong> Nothing can open, click, fill or screenshot a page.
+    <code>web_fetch</code> reads a URL and <code>http_request</code> calls an allow-listed API;
+    neither drives a browser. The same goes for timers, notifications, the clipboard and the screen.
+  </li>
+  <li>
+    <strong>Mobile gets 5 of 20 skills</strong>: diagrams, github, notion, spotify and weather.
+    Word, Excel, PowerPoint and PDF run Python in Pyodide and are desktop-only; image and music
+    generation are desktop-only too.
+  </li>
+  <li>
+    <strong>9 skills need a connected account</strong> (Asana, GitHub, Gmail, Google
+    Calendar/Docs/Drive/Sheets, Notion, Spotify). Apple Notes and Reminders are macOS-only and need
+    a host binary.
+  </li>
+  <li>
+    <strong>No skill is gated by model size.</strong> The catalog supports
+    <code>minCapabilityProfile</code>, but no bundled skill sets it, so the 2B model is offered the
+    document skills that are flaky on small models (<a
+      href="https://github.com/tetherto/qvac-harness/issues/182"
+      >#182</a
+    >). None of the benchmarks here covers documents.
+  </li>
+  <li>
+    <strong>Other harness findings:</strong> exec's approval label hides a Python run;
+    <code>rag_search</code> was never registered and is removed (<a
+      href="https://github.com/tetherto/qvac-harness/issues/162"
+      >#162</a
+    >; retrieval returns as a plugin in
+    <a href="https://github.com/tetherto/qvac-harness/issues/197">#197</a>); the round limit is an
+    input since <a href="https://github.com/tetherto/qvac-harness/issues/159">#159</a>; BFCL and
+    τ² on <code>dev</code> ran on the default limit with no regression; an agent with
+    <code>skillsScope</code> unset gets no skills.
+  </li>
+</ul>
+`
+
+// what separates the runs in the table; each line is a commit on PR #151
+const TAU2_CHANGES = `<div class="cards" style="margin-top:14px"><div class="card"><h3>What changed between the runs</h3>
+<p>The first run is withdrawn: tracing every failure showed most were ours, not the model's. Each fix below is a commit on PR #151 with its own test.</p>
+<ul class="marks plus">
+<li><strong>Runs leaked across tasks.</strong> A task that ended on an unanswered call left its harness run alive, so later tasks got replies one step behind and writes hit the wrong database. The bridge now matches frames by run and cancels an abandoned run.</li>
+<li><strong>The wrong date.</strong> The harness stamped today on every turn while τ²'s policy is set in May 2024, so the agent refused "past" flights. A run can now set its clock (<code>CompletionContext.now</code>).</li>
+<li><strong>The llama.cpp tool-grammar fault</strong> ("Unexpected empty grammar stack") ended 86% of 9B's replies. The harness now retries such a round without the grammar.</li>
+<li><strong>A tool's own <code>intent</code> argument</strong> was stripped with the harness's, and a run's tools clashed with same-named built-ins.</li>
+<li><strong>The runaway deadline</strong> cut off whole turns of many short rounds; it now applies to each round.</li>
+<li><strong>The simulated customer</strong> dropped the hashes from τ²'s end tokens, so finished tasks ran to the step limit; a bare <code>STOP</code> is now read as <code>###STOP###</code>.</li>
+</ul></div></div>`
+
+function terminalSection(): string {
+  const rows = terminal.map(({ label, trials }) => {
+    const models = [
+      ...new Set(trials.flatMap((trial) => (trial.model ? [short(trial.model)] : [])))
+    ]
+    const limits = [
+      ...new Set(trials.flatMap((trial) => (trial.limit === null ? [] : [trial.limit])))
+    ]
+    const speed = trials.reduce(
+      (sum, trial) => (trial.speed ? addSpeed(sum, trial.speed) : sum),
+      NO_SPEED
+    )
+    const decode = speed.decodeMs ? (speed.generatedTokens / speed.decodeMs) * 1000 : null
+    const prefill = speed.prefillMs ? (speed.promptTokens / speed.prefillMs) * 1000 : null
+    const load = trials.flatMap((trial) => trial.load)
+    const tip = trials
+      .map((trial) => `${trial.task}: ${trial.reward}, ${trial.calls} calls`)
+      .join('\n')
+
+    return `<tr><th scope="row">${escape(label)}</th><th scope="row">${escape(models.join(', ') || '–')}</th><td>${trials.length}</td>${cell(
+      trials.map((trial) => trial.reward === 1),
+      tip
+    )}<td>${escape(limits.join(', '))}</td><td>${trials.length ? (trials.reduce((sum, trial) => sum + trial.calls, 0) / trials.length).toFixed(1) : '–'}</td><td>${prefill === null ? '–' : Math.round(prefill)}</td><td>${decode === null ? '–' : decode.toFixed(1)}</td><td>${load.length ? `${Math.min(...load).toFixed(1)} to ${Math.max(...load).toFixed(1)}` : '–'}</td></tr>`
+  })
+
+  return `<section>
+<h2><span class="num">04</span>Terminal-Bench 2.0 <span class="tag">agentic terminal tasks</span></h2>
+<p class="section-lede">Harbor runs each task in its own container and its verifier scores it; the harness is the agent, with one bash tool that runs in the container (<code>bench/bridge/python/qvac_bench/harbor_agent.py</code>). <em>Pass</em> is the share of tasks the verifier rewarded 1. <em>Round limit</em> is the run's tool-round limit. <em>Load</em> is the one-minute load average at the start and end of each task. Tracked in <a href="https://github.com/tetherto/qvac-harness/issues/249">#249</a>.</p>
+${
+  rows.length
+    ? `<div class="table-wrap"><table class="data">
+<thead><tr><th scope="col">Run</th><th scope="col">Model</th><th scope="col">Tasks</th><th scope="col">Pass</th><th scope="col">Round limit</th><th scope="col">Calls per task</th><th scope="col">Prefill tok/s</th><th scope="col">Decode tok/s</th><th scope="col">Load</th></tr></thead>
+<tbody>${rows.join('')}</tbody>
+</table></div>`
+    : '<p class="pending">Not run yet: it needs a container runtime and the <code>harbor</code> CLI. See <code>bench/README.md</code>.</p>'
+}
+</section>`
+}
+
+function speedSection(): string {
+  const sources = [
+    ...bfcl.map((row) => ({ bench: 'BFCL', model: row.model, speed: row.speed })),
+    ...multiTurn.map((row) => ({ bench: 'BFCL multi-turn', model: row.model, speed: row.speed })),
+    ...tau2
+      .filter((row) => row.run === currentRun)
+      .map((row) => ({ bench: 'τ² airline', model: row.model, speed: row.speed }))
+  ]
+  const groups = new Map<string, { bench: string; model: string; speed: Speed }>()
+  for (const { bench, model, speed } of sources) {
+    if (!speed) continue
+    const key = `${bench}\0${model}`
+    const group = groups.get(key) ?? { bench, model, speed: NO_SPEED }
+    groups.set(key, { ...group, speed: addSpeed(group.speed, speed) })
+  }
+
+  const rows = [...groups.values()]
+    .sort((left, right) => bySize(left.model, right.model))
+    .map(({ bench, model, speed }) => {
+      const prefill = speed.prefillMs ? (speed.promptTokens / speed.prefillMs) * 1000 : null
+      const decode = speed.decodeMs ? (speed.generatedTokens / speed.decodeMs) * 1000 : null
+      const ttft = median(speed.ttftMs)
+      const share = (ms: number) => (speed.wallMs ? pct(ms / speed.wallMs) : '–')
+      return `<tr><th scope="row">${escape(bench)}</th><th scope="row">${escape(short(model))}</th><td>${speed.runs}</td><td>${(speed.rounds / Math.max(1, speed.runs)).toFixed(1)}</td><td>${prefill === null ? '–' : Math.round(prefill)}</td><td>${decode === null ? '–' : decode.toFixed(1)}</td><td>${ttft === null ? '–' : `${Math.round(ttft)} ms`}</td><td>${share(speed.modelMs)}</td><td>${share(speed.toolMs)}</td></tr>`
+    })
+
+  return `<section>
+<h2><span class="num">03</span>Speed <span class="tag">per model</span></h2>
+<p class="section-lede">From each run's <code>metrics</code> event. Prefill and decode rates are weighted by tokens: all the tokens a model prefilled or generated, over the time it took. <em>First token</em> is the median wait for each run's first round. <em>Model</em> and <em>tools</em> are shares of wall time. τ² runs carry no cache key, so every τ² round prefills cold. The <code>dev</code> runs at 3ca35a56 overlapped other work on the Mac: the one-minute load average went from 4 to 330 on 14 cores. Read their rates as lower bounds, not against #151.</p>
+${
+  rows.length
+    ? `<div class="table-wrap"><table class="data">
+<thead><tr><th scope="col">Benchmark</th><th scope="col">Model</th><th scope="col">Runs</th><th scope="col">Rounds per run</th><th scope="col">Prefill tok/s</th><th scope="col">Decode tok/s</th><th scope="col">First token</th><th scope="col">Model</th><th scope="col">Tools</th></tr></thead>
+<tbody>${rows.join('')}</tbody>
+</table></div>`
+    : '<p class="pending">No run here recorded speed yet; runs from before issue #195 carry none.</p>'
+}
+</section>`
+}
+
+function median(values: readonly number[]): number | null {
+  if (!values.length) {
+    return null
+  }
+
+  const sorted = [...values].sort((left, right) => left - right)
+  const middle = Math.floor(sorted.length / 2)
+
+  return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2
+}
+
+function failedShare(rows: readonly Tau2Row[]): string {
+  const replies = rows.reduce((sum, row) => sum + (row.replies ?? 0), 0)
+  const failed = rows.reduce((sum, row) => sum + (row.failed_turns ?? 0), 0)
+
+  return replies ? `${failed}/${replies} (${pct(failed / replies)})` : '–'
+}
+
+function cell(results: readonly boolean[], tip: string): string {
+  return figure(results.length ? Math.round(share(results) * 100) : null, tip)
+}
+
+function figure(percent: number | null, tip: string): string {
+  const data = tip ? ` data-tip="${escape(tip)}"` : ''
+
+  return `<td${data}><div class="bar"><span style="width:${percent ?? 0}%"></span></div><span class="value">${percent === null ? '–' : `${percent}%`}</span></td>`
+}
+
+// a column that ran a subset is averaged over what it ran, not over categories it never saw
+function bfclMean(rows: readonly BfclRow[]): number {
+  const perCategory = categories
+    .map((category) => rows.filter((row) => row.category === category).map((row) => row.valid))
+    .filter((results) => results.length)
+    .map(share)
+
+  return mean(perCategory)
+}
+
+function share(results: readonly boolean[]): number {
+  return results.length ? results.filter(Boolean).length / results.length : 0
+}
+
+function mean(values: readonly number[]): number {
+  return values.reduce((sum, value) => sum + value, 0) / Math.max(1, values.length)
+}
+
+function pct(value: number): string {
+  return `${Math.round(value * 100)}%`
+}
+
+function bySize(left: string, right: string): number {
+  return (
+    parseFloat(left.replace(/^\D+/, '').replace(/^[\d.]+-/, '')) -
+    parseFloat(right.replace(/^\D+/, '').replace(/^[\d.]+-/, ''))
+  )
+}
+
+function short(model: string): string {
+  return model.replace(/-Q\d.*$/, '')
+}
+
+function lines(file: string) {
+  return fs
+    .readFileSync(file, 'utf8')
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => JSON.parse(line))
+}
+
+function escape(text: string): string {
+  return text
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+}
+
+// the QVAC look: tokens from qvac-workbench ui/src/tokens.ts
+const WORDMARK = `<svg aria-label="QVAC" role="img" height="18" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 110 13" fill="none"><path d="M98.1597 0.199383C98.0421 0.0727478 97.8793 0 97.7057 0H80.0164C79.8454 0 79.6799 0.0727478 79.5624 0.199383L76.6921 3.30059C76.5853 3.41645 76.5266 3.57003 76.5266 3.7263V8.15314C76.5266 8.31211 76.5853 8.46299 76.6921 8.57885L79.5624 11.6801C79.6799 11.8067 79.8428 11.8794 80.0164 11.8794H97.7057C97.8766 11.8794 98.0421 11.8067 98.1597 11.6801L101.03 8.57885C101.137 8.46299 101.195 8.30941 101.195 8.15314V7.38524C101.195 7.04037 100.918 6.76015 100.576 6.76015H82.2218C82.051 6.76015 81.8854 6.68741 81.768 6.56077C81.6611 6.44491 81.6024 6.29133 81.6024 6.13506V5.74438C81.6024 5.58541 81.6611 5.43453 81.768 5.31867C81.8854 5.19203 82.0483 5.11929 82.2218 5.11929H100.576C100.918 5.11929 101.195 4.83907 101.195 4.4942V3.7263C101.195 3.56733 101.137 3.41645 101.03 3.30059L98.1597 0.199383Z" fill="currentColor" /><path d="M26.708 6.78433H25.1486V3.75048C25.1486 3.59151 25.09 3.44063 24.9831 3.32477L22.1127 0.220867C21.9953 0.0942321 21.8324 0.0214844 21.6589 0.0214844H3.96954C3.79865 0.0214844 3.63311 0.0942321 3.51563 0.220867L0.645281 3.32208C0.538478 3.43794 0.479736 3.59151 0.479736 3.74779V8.17732C0.479736 8.33629 0.538478 8.48717 0.645281 8.60303L3.51563 11.7042C3.63311 11.8308 3.79599 11.9036 3.96954 11.9036H30.9188C31.0896 11.9009 31.2551 11.8281 31.3727 11.7015C31.3727 11.7015 27.0524 6.78164 26.7106 6.78164L26.708 6.78433ZM20.0756 6.15655C20.0756 6.50142 19.7978 6.78164 19.4561 6.78164H6.17236C6.00148 6.78164 5.83593 6.70889 5.71845 6.58226C5.61164 6.4664 5.55291 6.31282 5.55291 6.15655V5.76856C5.55291 5.60959 5.61164 5.45871 5.72112 5.34285C5.8386 5.21621 6.00148 5.14347 6.17503 5.14347H19.4454C19.6109 5.14347 19.7685 5.20813 19.8833 5.32668L19.8966 5.34015C20.0141 5.45871 20.0781 5.61767 20.0781 5.78203V6.15655H20.0756Z" fill="currentColor" /><path d="M51.2249 3.05271L48.5842 0.199383C48.4666 0.0727478 48.3037 0 48.1302 0H40.7181C40.3763 0 40.0986 0.280214 40.0986 0.625092V6.13506C40.0986 6.3075 40.0265 6.47455 39.901 6.5931C39.7862 6.70088 39.634 6.76285 39.4791 6.76285H39.092C38.9344 6.76285 38.7823 6.70357 38.6701 6.5931C38.5446 6.47455 38.4725 6.31019 38.4725 6.13506V0.625092C38.4725 0.280214 38.1948 0 37.853 0H30.4409C30.27 0 30.1045 0.0727478 29.9869 0.199383L27.3462 3.05271C27.1247 3.29251 27.1247 3.66703 27.3462 3.90682L34.5368 11.6828C34.6543 11.8094 34.8171 11.8821 34.9907 11.8821H43.5804C43.7513 11.8821 43.9168 11.8094 44.0343 11.6828L51.2249 3.90682C51.4464 3.66703 51.4464 3.29251 51.2249 3.05271Z" fill="currentColor" /><path d="M67.3815 0.199383C67.264 0.0727478 67.1012 0 66.9275 0H58.3379C58.167 0 58.0015 0.0727478 57.8839 0.199383L50.6934 7.97531C50.4718 8.21511 50.4718 8.58962 50.6934 8.82942L53.3341 11.6828C53.4516 11.8094 53.6145 11.8821 53.7881 11.8821H61.2002C61.542 11.8821 61.8197 11.6019 61.8197 11.257V5.74707C61.8197 5.57463 61.8918 5.40758 62.0173 5.28903C62.132 5.18126 62.2842 5.11929 62.4391 5.11929H62.8263C62.9838 5.11929 63.136 5.17856 63.2482 5.28903C63.3736 5.40758 63.4458 5.57194 63.4458 5.74707V11.257C63.4458 11.6019 63.7235 11.8821 64.0652 11.8821H71.4774C71.6482 11.8821 71.8138 11.8094 71.9313 11.6828L74.572 8.82942C74.7936 8.58962 74.7936 8.21511 74.572 7.97531L67.3815 0.199383Z" fill="currentColor" /><path d="M109 8.67795L107.954 6.86464C107.911 6.7919 107.831 6.74609 107.746 6.74609H104.082C103.997 6.74609 103.916 6.7919 103.874 6.86464L102.827 8.67795C102.802 8.72106 102.795 8.76956 102.8 8.81536C102.8 8.81536 102.805 8.83422 102.805 8.8423C102.813 8.88811 102.829 8.93391 102.867 8.96894L105.743 11.8007C105.79 11.8465 105.852 11.8708 105.913 11.8681C105.975 11.8681 106.036 11.8465 106.085 11.8007L108.96 8.96894C108.995 8.93391 109.011 8.8908 109.019 8.845C109.019 8.83422 109.024 8.82614 109.024 8.81536C109.027 8.76956 109.024 8.72106 108.997 8.67795H109Z" fill="currentColor" /><path d="M98.1597 0.199383C98.0421 0.0727478 97.8793 0 97.7057 0H80.0164C79.8454 0 79.6799 0.0727478 79.5624 0.199383L76.6921 3.30059C76.5853 3.41645 76.5266 3.57003 76.5266 3.7263V8.15314C76.5266 8.31211 76.5853 8.46299 76.6921 8.57885L79.5624 11.6801C79.6799 11.8067 79.8428 11.8794 80.0164 11.8794H97.7057C97.8766 11.8794 98.0421 11.8067 98.1597 11.6801L101.03 8.57885C101.137 8.46299 101.195 8.30941 101.195 8.15314V7.38525C101.195 7.04037 100.918 6.76015 100.576 6.76015H82.2218C82.051 6.76015 81.8854 6.68741 81.768 6.56077C81.6611 6.44491 81.6024 6.29133 81.6024 6.13506V5.74438C81.6024 5.58541 81.6611 5.43453 81.768 5.31867C81.8854 5.19203 82.0483 5.11929 82.2218 5.11929H100.576C100.918 5.11929 101.195 4.83907 101.195 4.4942V3.7263C101.195 3.56734 101.137 3.41645 101.03 3.30059L98.1597 0.199383Z" fill="currentColor" /><path d="M26.708 6.78433H25.1486V3.75048C25.1486 3.59151 25.09 3.44063 24.9831 3.32477L22.1128 0.220867C21.9953 0.0942322 21.8324 0.0214844 21.6589 0.0214844H3.96954C3.79865 0.0214844 3.63311 0.0942322 3.51563 0.220867L0.645281 3.32208C0.538478 3.43794 0.479736 3.59151 0.479736 3.74779V8.17732C0.479736 8.33629 0.538478 8.48717 0.645281 8.60303L3.51563 11.7042C3.63311 11.8308 3.79599 11.9036 3.96954 11.9036H30.9188C31.0896 11.9009 31.2551 11.8281 31.3727 11.7015C31.3727 11.7015 27.0524 6.78164 26.7106 6.78164L26.708 6.78433ZM20.0756 6.15655C20.0756 6.50142 19.7978 6.78164 19.4561 6.78164H6.17236C6.00148 6.78164 5.83593 6.70889 5.71845 6.58226C5.61164 6.4664 5.55291 6.31282 5.55291 6.15655V5.76856C5.55291 5.60959 5.61164 5.45871 5.72112 5.34285C5.8386 5.21621 6.00148 5.14347 6.17503 5.14347H19.4454C19.6109 5.14347 19.7685 5.20813 19.8833 5.32668L19.8966 5.34015C20.0141 5.45871 20.0781 5.61767 20.0781 5.78203V6.15655H20.0756Z" fill="currentColor" /><path d="M51.2249 3.05271L48.5842 0.199383C48.4666 0.0727478 48.3038 0 48.1302 0H40.7181C40.3763 0 40.0986 0.280214 40.0986 0.625092V6.13506C40.0986 6.3075 40.0265 6.47455 39.901 6.5931C39.7862 6.70088 39.634 6.76285 39.4791 6.76285H39.092C38.9344 6.76285 38.7823 6.70357 38.6701 6.5931C38.5446 6.47455 38.4725 6.3102 38.4725 6.13506V0.625092C38.4725 0.280214 38.1948 0 37.853 0H30.4409C30.27 0 30.1045 0.0727478 29.9869 0.199383L27.3462 3.05271C27.1247 3.29251 27.1247 3.66703 27.3462 3.90682L34.5368 11.6828C34.6543 11.8094 34.8171 11.8821 34.9907 11.8821H43.5804C43.7513 11.8821 43.9168 11.8094 44.0343 11.6828L51.2249 3.90682C51.4464 3.66703 51.4464 3.29251 51.2249 3.05271Z" fill="currentColor" /><path d="M67.3815 0.199383C67.264 0.0727478 67.1012 0 66.9275 0H58.3379C58.167 0 58.0015 0.0727478 57.8839 0.199383L50.6934 7.97531C50.4718 8.21511 50.4718 8.58963 50.6934 8.82942L53.3341 11.6828C53.4516 11.8094 53.6145 11.8821 53.7881 11.8821H61.2002C61.542 11.8821 61.8197 11.6019 61.8197 11.257V5.74707C61.8197 5.57463 61.8918 5.40758 62.0173 5.28903C62.132 5.18126 62.2842 5.11929 62.4391 5.11929H62.8263C62.9839 5.11929 63.136 5.17856 63.2482 5.28903C63.3736 5.40758 63.4458 5.57194 63.4458 5.74707V11.257C63.4458 11.6019 63.7235 11.8821 64.0652 11.8821H71.4774C71.6482 11.8821 71.8138 11.8094 71.9313 11.6828L74.572 8.82942C74.7936 8.58963 74.7936 8.21511 74.572 7.97531L67.3815 0.199383Z" fill="currentColor" /><path d="M109 8.67795L107.954 6.86464C107.911 6.7919 107.831 6.74609 107.746 6.74609H104.082C103.997 6.74609 103.916 6.7919 103.874 6.86464L102.827 8.67795C102.802 8.72106 102.795 8.76956 102.8 8.81536C102.8 8.81536 102.805 8.83422 102.805 8.8423C102.813 8.88811 102.829 8.93391 102.867 8.96894L105.743 11.8007C105.79 11.8465 105.852 11.8708 105.913 11.8681C105.975 11.8681 106.036 11.8465 106.085 11.8007L108.96 8.96894C108.995 8.93391 109.011 8.8908 109.019 8.845C109.019 8.83422 109.024 8.82614 109.024 8.81536C109.027 8.76956 109.024 8.72106 108.997 8.67795H109Z" fill="currentColor" /></svg>`
+
+const BRAND_STYLE = `:root {
+        color-scheme: light;
+        --canvas: #eef0f3; --bg: #f8f9fa; --surface: #ffffff; --raised: #eef0f3;
+        --fg: #1c1e21; --fg-muted: #646a73; --border: #e9ecef;
+        --primary: #08735f; --primary-soft: #e3f4ef; --on-primary: #ffffff;
+        --ours-bg: #c4eae1; --ours-fg: #0a6e62;
+        --pi-bg: #ddd3fa; --pi-fg: #5737b5;
+        --dsh-bg: #ccdff8; --dsh-fg: #1e56a3;
+        --good-bg: #e4f6ea; --good-fg: #1a7a43;
+        --bad-bg: #fde8e8; --bad-fg: #c53030;
+        --warn-bg: #fef3d7; --warn-fg: #8a5a0f;
+        --shadow-soft: 0 1px 2px rgb(16 24 40 / 0.04), 0 1px 3px rgb(16 24 40 / 0.06);
+        --shadow-card: 0 2px 6px -2px rgb(16 24 40 / 0.06), 0 8px 24px -6px rgb(16 24 40 / 0.10);
+      }
+      @media (prefers-color-scheme: dark) {
+        :root:not([data-theme="light"]) {
+          color-scheme: dark;
+          --canvas: #161718; --bg: #161718; --surface: #252728; --raised: #1e1f20;
+          --fg: #ffffff; --fg-muted: #9a9b9c; --border: #2e3031;
+          --primary: #16e3c1; --primary-soft: #095b4d; --on-primary: #0f1010;
+          --ours-bg: #173833; --ours-fg: #5ed3bf;
+          --pi-bg: #2b2442; --pi-fg: #b6a3f6;
+          --dsh-bg: #1b2b42; --dsh-fg: #8db9f5;
+          --good-bg: #18482d; --good-fg: #5cc98a;
+          --bad-bg: #602222; --bad-fg: #f58b8b;
+          --warn-bg: #65511a; --warn-fg: #fdca40;
+          --shadow-soft: inset 0 1px 0 rgb(255 255 255 / 0.03), 0 1px 2px rgb(0 0 0 / 0.30);
+          --shadow-card: inset 0 1px 0 rgb(255 255 255 / 0.04), 0 2px 8px -2px rgb(0 0 0 / 0.35), 0 12px 32px -8px rgb(0 0 0 / 0.45);
+        }
+      }
+      :root[data-theme="dark"] {
+        color-scheme: dark;
+        --canvas: #161718; --bg: #161718; --surface: #252728; --raised: #1e1f20;
+        --fg: #ffffff; --fg-muted: #9a9b9c; --border: #2e3031;
+        --primary: #16e3c1; --primary-soft: #095b4d; --on-primary: #0f1010;
+        --ours-bg: #173833; --ours-fg: #5ed3bf;
+        --pi-bg: #2b2442; --pi-fg: #b6a3f6;
+        --dsh-bg: #1b2b42; --dsh-fg: #8db9f5;
+        --good-bg: #18482d; --good-fg: #5cc98a;
+        --bad-bg: #602222; --bad-fg: #f58b8b;
+        --warn-bg: #65511a; --warn-fg: #fdca40;
+        --shadow-soft: inset 0 1px 0 rgb(255 255 255 / 0.03), 0 1px 2px rgb(0 0 0 / 0.30);
+        --shadow-card: inset 0 1px 0 rgb(255 255 255 / 0.04), 0 2px 8px -2px rgb(0 0 0 / 0.35), 0 12px 32px -8px rgb(0 0 0 / 0.45);
+      }
+      * { box-sizing: border-box; }
+      html { background: var(--canvas); }
+      body { margin: 0; background: var(--canvas); color: var(--fg); font: 15px/1.6 "Geist", "Inter", ui-sans-serif, system-ui, -apple-system, sans-serif; -webkit-font-smoothing: antialiased; }
+      code { font: 600 0.92em/1 "Inconsolata", ui-monospace, monospace; background: var(--raised); padding: 2px 5px; border-radius: 4px; }
+      a { color: var(--primary); text-underline-offset: 3px; }
+      main { max-width: 1200px; margin: 0 auto; padding: 0 16px 80px; }
+
+      .topbar { display: flex; align-items: center; justify-content: space-between; gap: 16px; padding: 20px 0; }
+      .brand { display: flex; align-items: center; gap: 12px; color: var(--primary); text-decoration: none; }
+      .brand svg { display: block; }
+      .brand span { color: var(--fg-muted); font: 500 13px/1 "Inconsolata", ui-monospace, monospace; letter-spacing: 0.06em; text-transform: uppercase; padding-left: 12px; border-left: 1px solid var(--border); }
+      .toggle { font: inherit; font-size: 13px; color: var(--fg-muted); background: var(--surface); border: 0; border-radius: 999px; padding: 6px 12px; box-shadow: var(--shadow-soft); cursor: pointer; }
+      .toggle:hover { color: var(--fg); }
+
+      .hero { background: var(--surface); border-radius: 16px; box-shadow: var(--shadow-card); padding: 40px 40px 32px; margin-bottom: 40px; position: relative; overflow: hidden; }
+      .hero::after { content: ""; position: absolute; inset: 0 0 auto auto; width: 320px; height: 320px; background: radial-gradient(circle at 70% 30%, var(--primary-soft), transparent 65%); pointer-events: none; }
+      .eyebrow { font: 600 12px/1 "Inconsolata", ui-monospace, monospace; letter-spacing: 0.1em; text-transform: uppercase; color: var(--primary); margin: 0 0 12px; }
+      h1 { font-size: clamp(28px, 4vw, 40px); line-height: 1.15; letter-spacing: -0.02em; margin: 0 0 14px; max-width: 760px; position: relative; z-index: 1; }
+      .lede { color: var(--fg-muted); max-width: 820px; margin: 0; position: relative; z-index: 1; }
+      .versions { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 22px; position: relative; z-index: 1; }
+      .chip { display: inline-flex; align-items: center; gap: 6px; font-size: 13px; border-radius: 999px; padding: 4px 12px; background: var(--raised); color: var(--fg-muted); }
+      .chip b { font-weight: 600; }
+      .dot { width: 8px; height: 8px; border-radius: 50%; display: inline-block; flex: none; }
+      .ours .dot, .dot.ours { background: var(--ours-fg); }
+      .pi .dot, .dot.pi { background: var(--pi-fg); }
+      .dsh .dot, .dot.dsh { background: var(--dsh-fg); }
+
+      section { margin-top: 56px; }
+      h2 { font-size: 22px; letter-spacing: -0.01em; margin: 0 0 6px; }
+      .section-lede { color: var(--fg-muted); margin: 0 0 20px; max-width: 820px; }
+      .num { font: 600 13px/1 "Inconsolata", ui-monospace, monospace; color: var(--primary); margin-right: 10px; }
+
+      .stack { display: grid; gap: 8px; }
+      .layer { display: grid; grid-template-columns: 220px 1fr; gap: 16px; align-items: center; background: var(--surface); border-radius: 12px; box-shadow: var(--shadow-soft); padding: 14px 18px; }
+      .layer .tag { font: 600 13px/1.2 "Inconsolata", ui-monospace, monospace; }
+      .layer.now { box-shadow: var(--shadow-card); outline: 2px solid var(--primary); outline-offset: -2px; }
+      .layer .note { color: var(--fg-muted); font-size: 14px; }
+      .layer strong { color: var(--fg); }
+
+      .cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 14px; }
+      .card { background: var(--surface); border-radius: 12px; box-shadow: var(--shadow-card); padding: 22px; display: flex; flex-direction: column; gap: 10px; }
+      .card h3 { margin: 0; font-size: 17px; display: flex; align-items: center; gap: 8px; }
+      .card p { margin: 0; color: var(--fg-muted); font-size: 14px; }
+      .facts { display: flex; flex-wrap: wrap; gap: 6px; margin-top: auto; padding-top: 6px; }
+      .fact { font-size: 12px; padding: 3px 9px; border-radius: 6px; }
+      .ours .fact { background: var(--ours-bg); color: var(--ours-fg); }
+      .pi .fact { background: var(--pi-bg); color: var(--pi-fg); }
+      .dsh .fact { background: var(--dsh-bg); color: var(--dsh-fg); }
+
+      .table-wrap { background: var(--surface); border-radius: 12px; box-shadow: var(--shadow-card); overflow-x: auto; }
+      table { border-collapse: collapse; width: 100%; min-width: 900px; }
+      th, td { text-align: left; vertical-align: top; padding: 12px 14px; font-size: 14px; border-bottom: 1px solid var(--border); }
+      thead th { position: sticky; top: 0; background: var(--surface); font-size: 13px; z-index: 1; }
+      thead th span { display: inline-flex; align-items: center; gap: 8px; }
+      tbody th { font-weight: 600; width: 170px; white-space: nowrap; }
+      tr.group th { font: 600 12px/1 "Inconsolata", ui-monospace, monospace; letter-spacing: 0.08em; text-transform: uppercase; color: var(--primary); background: var(--raised); padding: 10px 14px; border-bottom: 0; }
+      td.lead.ours { background: color-mix(in srgb, var(--ours-bg) 55%, transparent); }
+      td.lead.pi { background: color-mix(in srgb, var(--pi-bg) 55%, transparent); }
+      td.lead.dsh { background: color-mix(in srgb, var(--dsh-bg) 55%, transparent); }
+      .badge { display: inline-flex; align-items: center; gap: 4px; font: 600 11px/1 "Inconsolata", ui-monospace, monospace; letter-spacing: 0.04em; text-transform: uppercase; border-radius: 4px; padding: 3px 6px; margin-bottom: 6px; }
+      td.lead.ours .badge { background: var(--ours-fg); color: var(--surface); }
+      td.lead.pi .badge { background: var(--pi-fg); color: var(--surface); }
+      td.lead.dsh .badge { background: var(--dsh-fg); color: var(--surface); }
+      .badge::before { content: "▲"; font-size: 8px; }
+      td .badge { display: flex; width: fit-content; }
+
+      .pc { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 14px; }
+      .pc .card { gap: 14px; }
+      .list-title { font: 600 12px/1 "Inconsolata", ui-monospace, monospace; letter-spacing: 0.08em; text-transform: uppercase; color: var(--fg-muted); margin: 4px 0 0; }
+      ul.marks { list-style: none; margin: 0; padding: 0; display: grid; gap: 8px; font-size: 14px; }
+      ul.marks li { position: relative; padding-left: 30px; }
+      ul.marks li::before { position: absolute; left: 0; top: 1px; width: 20px; height: 20px; border-radius: 6px; display: grid; place-items: center; font: 700 13px/1 "Geist", sans-serif; }
+      ul.plus li::before { content: "+"; background: var(--good-bg); color: var(--good-fg); }
+      ul.minus li::before { content: "−"; background: var(--bad-bg); color: var(--bad-fg); }
+
+      ol.plan { list-style: none; margin: 0; padding: 0; display: grid; gap: 10px; counter-reset: step; }
+      ol.plan > li { counter-increment: step; display: grid; grid-template-columns: 44px 1fr; gap: 16px; background: var(--surface); border-radius: 12px; box-shadow: var(--shadow-soft); padding: 18px 20px; }
+      ol.plan > li::before { content: counter(step, decimal-leading-zero); font: 600 15px/1 "Inconsolata", ui-monospace, monospace; color: var(--on-primary); background: var(--primary); border-radius: 10px; width: 44px; height: 44px; display: grid; place-items: center; }
+      .step-title { font-weight: 600; font-size: 16px; margin: 0 0 4px; }
+      .step-body { margin: 0; color: var(--fg); font-size: 14px; }
+      .why { display: block; margin-top: 8px; color: var(--fg-muted); font-size: 13px; padding-left: 12px; border-left: 2px solid var(--primary-soft); }
+      .areas { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 10px; }
+      .area { font: 600 11px/1 "Inconsolata", ui-monospace, monospace; letter-spacing: 0.04em; text-transform: uppercase; color: var(--fg-muted); background: var(--raised); border-radius: 4px; padding: 4px 7px; }
+
+      .avoid { background: var(--surface); border-radius: 12px; box-shadow: var(--shadow-card); padding: 22px; display: grid; gap: 12px; }
+      .avoid ul.marks li::before { content: "×"; background: var(--warn-bg); color: var(--warn-fg); }
+
+      td.nw { white-space: nowrap; min-width: 0; }
+      footer { margin-top: 56px; padding-top: 20px; border-top: 1px solid var(--border); color: var(--fg-muted); font-size: 12px; }
+      footer code { font-size: 11px; }
+
+      @media (max-width: 720px) {
+        .hero { padding: 28px 20px 24px; }
+        .layer { grid-template-columns: 1fr; gap: 4px; }
+        ol.plan > li { grid-template-columns: 1fr; }
+        .brand span { display: none; }
+      }
+    `
+
+const THEME_SCRIPT = `
+      const root = document.documentElement
+      const button = document.getElementById('theme')
+      const dark = () => root.dataset.theme ? root.dataset.theme === 'dark' : matchMedia('(prefers-color-scheme: dark)').matches
+      const label = () => { button.textContent = dark() ? 'Light mode' : 'Dark mode' }
+      try { const saved = localStorage.getItem('qvac-theme'); if (saved) root.dataset.theme = saved } catch {}
+      label()
+      button.addEventListener('click', () => {
+        root.dataset.theme = dark() ? 'light' : 'dark'
+        try { localStorage.setItem('qvac-theme', root.dataset.theme) } catch {}
+        label()
+      })
+    `
+
+const STYLE = `
+section { margin-top: 48px; }
+.tag { font: 500 12px/1 "Inconsolata", ui-monospace, monospace; letter-spacing: 0.06em; text-transform: uppercase; color: var(--fg-muted); margin-left: 8px; }
+.cards { margin-top: 0; }
+.stat .pair { display: flex; gap: 28px; }
+.figure { font-size: 34px; font-weight: 600; letter-spacing: -0.02em; font-variant-numeric: tabular-nums; }
+.figure-label { color: var(--fg-muted); font-size: 12px; }
+.pending { color: var(--fg-muted); background: var(--surface); border-radius: 12px; box-shadow: var(--shadow-soft); padding: 16px 20px; }
+table.data { min-width: 640px; font-variant-numeric: tabular-nums; }
+table.data th, table.data td { white-space: nowrap; }
+table.data thead th { text-align: center; }
+table.data th.sub { color: var(--fg-muted); font-weight: 400; font-size: 11px; }
+table.data th[scope=row] { text-align: left; }
+td[data-tip] { cursor: default; }
+td[data-tip]:hover { background: var(--raised); }
+.bar { display: inline-block; width: 64px; height: 6px; background: var(--raised); border-radius: 3px; vertical-align: middle; overflow: hidden; }
+.bar span { display: block; height: 100%; background: var(--primary); border-radius: 3px; }
+.value { margin-left: 8px; }
+.notes { background: var(--surface); border-radius: 12px; box-shadow: var(--shadow-card); padding: 4px 24px 20px; }
+.notes h2 { font-size: 17px; margin: 22px 0 8px; }
+.notes p, .notes li { color: var(--fg); font-size: 14px; }
+.notes ul { padding-left: 20px; }
+tr.current th, tr.current td { background: color-mix(in srgb, var(--primary-soft) 40%, transparent); }
+.findings { margin: 8px 0 16px; }
+.findings th, .findings td { white-space: normal; text-align: left; vertical-align: top; font-size: 13px; }
+.findings td:first-child { min-width: 300px; }
+#tip { position: fixed; z-index: 10; max-width: 480px; white-space: pre-wrap; background: var(--surface); color: var(--fg); border-radius: 8px; padding: 8px 10px; font-size: 12px; box-shadow: var(--shadow-card); pointer-events: none; }
+`
+
+const SCRIPT = `
+const tip = document.getElementById('tip')
+document.addEventListener('pointerover', (event) => {
+  const cell = event.target.closest('td[data-tip]')
+  if (!cell) { tip.hidden = true; return }
+  tip.textContent = cell.dataset.tip
+  tip.hidden = false
+})
+document.addEventListener('pointermove', (event) => {
+  if (tip.hidden) return
+  tip.style.left = Math.min(event.clientX + 14, innerWidth - tip.offsetWidth - 8) + 'px'
+  tip.style.top = event.clientY + 14 + 'px'
+})
+`
+
+fs.mkdirSync(path.dirname(out), { recursive: true })
+fs.writeFileSync(out, page())
+console.log(`wrote ${out}`)