benchforge 0.1.9 → 0.2.4

package/LICENSE

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2026 Benchforge Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -1,10 +1,39 @@
 # Benchforge
 
-A TypeScript benchmarking library with CLI support for running performance tests.
-
-## Browser Profiling
-
-See [Browser Heap Profiling](README-browser.md) for profiling code running in a browser.
+Benchforge helps you make faster JavaScript programs with integrated tools for
+benchmarking and performance analysis in Node.js and Chrome, including features
+designed specifically for analyzing garbage-collected programs.
+
+Garbage collection is intermittent and infrequent, which makes it harder to
+identify true performance issues. Typical perf tools isolate microbenchmarks
+from GC, but that hides a key part of real-world performance. Intermittent
+events also lead to statistically skewed measurement distributions. Perf tools
+that assume normal distributions and noise-free test runs can easily create
+misleading false-positive performance reports. Benchforge captures a truer
+picture of garbage-collected programs:
+
+- **GC-aware statistics** -- bootstrap confidence intervals account for GC
+  variance instead of hiding it.
+- **Heap allocation profiling** -- see which functions allocate the most,
+  including short-lived objects already collected.
+- **GC collection reports** -- allocation rates, scavenge/full GC counts,
+  promotion %, and pause times per iteration.
+- **Visualization** -- distribution plots, icicle charts for allocators, source
+  annotations with allocation and call count metrics.
+- **Archive** -- save traces and source code together to share with your team.
+
+## Timing Distributions
+<img width="326" height="363" alt="stats with distribution curves" src="https://github.com/user-attachments/assets/532702bd-faa1-4cb3-8b33-ad5409631427" />
+
+## Heap Allocation
+Explore memory _allocation_ per function:
+<img width="4444" height="2706" alt="allocation view" src="https://github.com/user-attachments/assets/6d4e2dee-bb72-41ce-a71d-d036bebedb3d" />
+
+## Benchmark Iteration Time Series
+<img width="387" height="306" alt="time series" src="https://github.com/user-attachments/assets/f5676b64-7906-422b-aef3-4eedc325c422" />
+
+## Source Code Annotated with Performance Info
+<img width="1946" height="460" alt="src annotations" src="https://github.com/user-attachments/assets/102cc574-ecf3-4f5f-8143-d20ee7008a72" />
 
 ## Installation
 
@@ -14,9 +43,10 @@ npm install benchforge
 pnpm add benchforge
 ```
 
-## Quick Start
+## Quick Start: Node
 
-The simplest way to benchmark a function: export it as the default export and pass the file to `benchforge`.
+The simplest benchmark: export a default function and pass the file to
+`benchforge`.
 
 ```typescript
 // my-bench.ts
@@ -29,283 +59,92 @@ export default function (): string {
 benchforge my-bench.ts --gc-stats
 ```
 
-### BenchSuite Export
-
-For multiple benchmarks with groups, setup data, and baseline comparison, export a `BenchSuite`:
+For suites with multiple benchmarks, groups, and baseline comparison, see
+[Node.md](Node.md).
 
-```typescript
-// sorting.ts
-import type { BenchGroup, BenchSuite } from 'benchforge';
-
-const sortingGroup: BenchGroup<number[]> = {
-  name: "Array Sorting (1000 numbers)",
-  setup: () => Array.from({ length: 1000 }, () => Math.random()),
-  baseline: { name: "native sort", fn: (arr) => [...arr].sort((a, b) => a - b) },
-  benchmarks: [
-    { name: "quicksort", fn: quickSort },
-    { name: "insertion sort", fn: insertionSort },
-  ],
-};
+## Quick Start: Browser
 
-const suite: BenchSuite = {
-  name: "Performance Tests",
-  groups: [sortingGroup],
-};
+`benchforge --url <page>` opens Chromium and runs your program.
 
-export default suite;
-```
+You can time any page without modification, and compare against a baseline.
 
 ```bash
-benchforge sorting.ts --gc-stats
+benchforge --url http://localhost:5173 --baseline-url http://localhost:5174 \
+  --gc-stats --batches 20 --iterations 10 --headless
 ```
 
-See `examples/simple-cli.ts` for a complete runnable example.
-
-### Worker Mode with Module Imports
+If you export your test function as `window.__bench`, benchforge can run
+multiple iterations in the same tab, which helps reveal the accumulated effect
+of heap allocation over time. Tests also run faster.
 
-For worker mode, benchmarks can reference module exports instead of inline functions. This is essential for proper isolation since functions can't be serialized across process boundaries.
-
-```typescript
-const group: BenchGroup = {
-  name: "Parser Benchmark",
-  setup: () => loadTestData(),
-  benchmarks: [{
-    name: "parse",
-    fn: () => {},  // placeholder - not used in worker mode
-    modulePath: new URL("./benchmarks.ts", import.meta.url).href,
-    exportName: "parse",
-    setupExportName: "setup",  // optional: called once, result passed to exportName fn
-  }],
+```html
+<!-- bench function mode -->
+<script>
+window.__bench = () => {
+  const arr = Array.from({ length: 10000 }, () => Math.random());
+  arr.sort((a, b) => a - b);
 };
+</script>
 ```
 
-When `setupExportName` is provided, the worker:
-1. Imports the module
-2. Calls `setup(params)` once (where params comes from `BenchGroup.setup()`)
-3. Passes the setup result to each benchmark iteration
-
-This eliminates manual caching boilerplate in worker modules.
-
-## CLI Options
-
-### Basic Options
-- `--time <seconds>` - Benchmark duration per test (default: 0.642s)
-- `--iterations <count>` - Exact number of iterations (overrides --time)
-- `--filter <pattern>` - Run only benchmarks matching regex/substring
-- `--worker` / `--no-worker` - Run in isolated worker process (default: true)
-- `--profile` - Run once for profiling (single iteration, no warmup)
-- `--warmup <count>` - Warmup iterations before measurement (default: 0)
-- `--help` - Show all available options
-
-### Memory Profiling
-- `--gc-stats` - Collect GC allocation/collection stats via --trace-gc-nvp
-- `--heap-sample` - Heap sampling allocation attribution (includes garbage)
-- `--heap-interval <bytes>` - Sampling interval in bytes (default: 32768)
-- `--heap-depth <frames>` - Stack depth to capture (default: 64)
-- `--heap-rows <n>` - Number of top allocation sites to show (default: 20)
-
-### Output Options
-- `--html` - Generate HTML report, start server, and open in browser
-- `--export-html <file>` - Export HTML report to file
-- `--json <file>` - Export benchmark data to JSON
-- `--perfetto <file>` - Export Perfetto trace file
-
-## CLI Usage
-
-### Filter benchmarks by name
-
-```bash
-benchforge my-bench.ts --filter "concat"
-benchforge my-bench.ts --filter "^parse" --time 2
-```
-
-### Profiling with external debuggers
-
-Use `--profile` to run benchmarks once for attaching external profilers:
-
-```bash
-# Use with Chrome DevTools profiler
-node --inspect-brk $(which benchforge) my-bench.ts --profile
-
-# Use with other profiling tools
-node --prof $(which benchforge) my-bench.ts --profile
-```
-
-The `--profile` flag executes exactly one iteration with no warmup, making it ideal for debugging and performance profiling.
-
-### Key Concepts
-
-**Setup Functions**: Run once per group and provide shared data to all benchmarks in that group. The data returned by setup is automatically passed as the first parameter to benchmark functions that expect it.
-
-**Baseline Comparison**: When a baseline is specified, all benchmarks in the group show percentage differences (Δ%) compared to baseline. 
-
-## Output
-
-Results are displayed in a formatted table:
-
-```
-╔═════════════════╤═══════════════════════════════════════════╤═════════╗
-║                 │                   time                    │         ║
-║ name            │ mean  Δ% CI                    p50   p99  │ runs    ║
-╟─────────────────┼───────────────────────────────────────────┼─────────╢
-║ quicksort       │ 0.17  +5.5% [+4.7%, +6.2%]     0.15  0.63 │ 1,134   ║
-║ insertion sort  │ 0.24  +25.9% [+25.3%, +27.4%]  0.18  0.36 │ 807     ║
-║ --> native sort │ 0.16                           0.15  0.41 │ 1,210   ║
-╚═════════════════╧═══════════════════════════════════════════╧═════════╝
-```
-
-- **Δ% CI**: Percentage difference from baseline with bootstrap confidence interval
-
-### HTML
-
-The HTML report displays:
-- Histogram + KDE: Bar chart showing the distribution
-- Time Series: Sample values over iterations
-- Allocation Series: Per-sample heap allocation (requires `--heap-sample`)
-
-```bash
-# Generate HTML report, start server, and open in browser
-benchforge my-bench.ts --html
-# Press Ctrl+C to exit when done viewing
-```
-
-### Perfetto Trace Export
-
-Export benchmark data as a Perfetto-compatible trace file for detailed analysis:
-
-```bash
-# Export trace file
-benchforge my-bench.ts --perfetto trace.json
-
-# With V8 GC events (automatically merged after exit)
-node --expose-gc --trace-events-enabled --trace-event-categories=v8,v8.gc \
-  benchforge my-bench.ts --perfetto trace.json
-```
-
-View the trace at https://ui.perfetto.dev by dragging the JSON file.
+See [Browser.md](Browser.md) for setup patterns, completion signals, and the CDP
+flow.
 
-The trace includes:
-- **Heap counter**: Continuous heap usage as a line graph
-- **Sample markers**: Each benchmark iteration with timing
-- **Pause markers**: V8 optimization pause points
-- **V8 GC events**: Automatically merged after process exit (when run with `--trace-events-enabled`)
+## CLI Overview
 
-### GC Statistics
+Core flags for common workflows. Run `benchforge --help` for the full list.
 
-Collect detailed garbage collection statistics via V8's `--trace-gc-nvp`:
+| Flag | What it does |
+|------|-------------|
+| `--gc-stats` | GC allocation/collection stats |
+| `--alloc` | Heap allocation sampling attribution |
+| `--profile` | V8 CPU time sampling profiler |
+| `--call-counts` | Per-function execution counts |
+| `--stats <list>` | Timing columns to display (default: mean,p50,p99) |
+| `--view` | Open interactive viewer in browser |
+| `--archive [file]` | Archive profiles + sources to `.benchforge` file |
+| `--duration <sec>` | Duration per batch (default: 0.642s) |
+| `--iterations <n>` | Exact iterations (overrides --duration) |
+| `--batches <n>` | Interleaved batches for baseline comparison |
+| `--filter <pattern>` | Run only benchmarks matching regex/substring |
+| `--url <url>` | Benchmark a browser page |
+| `--baseline-url <url>` | A/B comparison in browser |
+| `--equiv-margin <pct>` | Equivalence margin (default: 2%) |
 
-```bash
-# Collect GC allocation/collection stats (requires worker mode)
-benchforge my-bench.ts --gc-stats
-```
+See [Profiling.md](Profiling.md) for detailed profiling options and V8 flags.
 
-Adds these columns to the output table:
-- **alloc/iter**: Bytes allocated per iteration
-- **scav**: Number of scavenge (minor) GCs
-- **full**: Number of full (mark-compact) GCs
-- **promo%**: Percentage of allocations promoted to old generation
-- **pause/iter**: GC pause time per iteration
+## Key Concepts
 
-### Heap Sampling
+### Batching
 
-For allocation profiling including garbage (short-lived objects), use `--heap-sample` mode which uses Node's built-in inspector API:
+When comparing against a baseline, use `--batches` to interleave runs and reduce
+ordering bias. Batch 0 is dropped by default (OS cache warmup). For reliable
+comparisons, use 40+ batches.
 
 ```bash
-# Basic heap sampling
-benchforge my-bench.ts --heap-sample --iterations 100
-
-# Smaller interval = more samples = better coverage of rare allocations
-benchforge my-bench.ts --heap-sample --heap-interval 4096 --iterations 100
-
-# Verbose output with clickable file:// paths
-benchforge my-bench.ts --heap-sample --heap-verbose
-
-# Control call stack display depth
-benchforge my-bench.ts --heap-sample --heap-stack 5
-```
-
-**CLI Options:**
-- `--heap-sample` - Enable heap sampling allocation attribution
-- `--heap-interval <bytes>` - Sampling interval in bytes (default: 32768)
-- `--heap-depth <frames>` - Maximum stack depth to capture (default: 64)
-- `--heap-rows <n>` - Number of top allocation sites to show (default: 20)
-- `--heap-stack <n>` - Call stack depth to display (default: 3)
-- `--heap-verbose` - Show full file:// paths with line numbers (cmd-clickable)
-
-**Output (default compact):**
-```
-─── Heap profile: bevy_env_map ───
-Heap allocation sites (top 20, garbage included):
-  13.62 MB  recursiveResolve <- flattenTreeImport <- bindAndTransform
-  12.36 MB  nextToken <- parseBlockStatements <- parseCompoundStatement
-   5.15 MB  coverWithText <- finishElem <- parseVarOrLet
-
-Total (all):       56.98 MB
-Total (user-code): 28.45 MB
-Samples: 1,842
-```
-
-**How V8 Heap Sampling Works:**
-
-V8's sampling profiler uses Poisson-distributed sampling. When an allocation occurs, V8 probabilistically decides whether to record it based on the sampling interval. Key points:
-
-1. **selfSize is scaled**: V8 doesn't report raw sampled bytes. It scales sample counts to estimate total allocations (`selfSize = size × count × scaleFactor`). This means changing `--heap-interval` affects sample count and overhead, but the estimated total converges to the same value.
-
-2. **Smaller intervals = better coverage**: With a smaller interval (e.g., 1024 vs 32768), you get more samples and discover more unique allocation sites, especially rare ones. The total estimate stays similar, but you see more of the distribution.
-
-3. **User-code only**: The report filters out Node.js internals (`node:`, `internal/`). "Total (user-code)" shows filtered allocations; "Total (all)" shows everything.
-
-4. **Measurement window**: Sampling covers benchmark module import + execution. Worker startup and framework init aren't captured (but do appear in `--gc-stats`).
-
-5. **Sites are stack-unique**: The same function appears multiple times with different callers. For example, `nextToken` may show up in several entries with different call stacks, each representing a distinct allocation pattern.
-
-**Limitations:**
-- **Function-level attribution only**: V8 reports the function where allocation occurred, not the specific line. The line:column shown is where the function is *defined*.
-- **Statistical sampling**: Results vary between runs. More iterations = more stable results.
-- **~50% filtered**: Node.js internals account for roughly half of allocations. Use "Total (all)" to see the full picture.
-
-**When to use which:**
-| Tool | Use When |
-|------|----------|
-| `--gc-stats` | Need total allocation/collection bytes, GC pause times |
-| `--heap-sample` | Need to identify which functions allocate the most |
-| Both | Cross-reference attribution with totals |
-
-## Requirements
-
-- Node.js 22.6+ (for native TypeScript support)
-- Use `--expose-gc --allow-natives-syntax` flags for garbage collection monitoring and V8 native functions
-
-## Adaptive Mode (Experimental)
-
-Adaptive mode (`--adaptive`) automatically adjusts iteration count until measurements stabilize. The algorithm is still being tuned — use `--help` for available options.
-
-## Interpreting Results
-
-### Baseline Comparison (Δ% CI)
-```
-0.17  +5.5% [+4.7%, +6.2%]
-```
-The benchmark is 5.5% slower than baseline, with a bootstrap confidence interval of [+4.7%, +6.2%].
-
-### Percentiles
-```
-p50: 0.15ms, p99: 0.27ms
+benchforge sorting.ts --batches 40 --duration 2
 ```
-50% of runs completed in ≤0.15ms and 99% in ≤0.27ms. Use percentiles when you care about consistency and tail latencies.
 
-## Understanding GC Time Measurements
+See [Statistics.md](Statistics.md) for the full explanation of batched
+execution, block bootstrap, and Tukey trimming.
 
-### GC Duration in Node.js Performance Hooks
+### Baseline Comparison
 
-The `duration` field in GC PerformanceEntry records **stop-the-world pause time** - the time when JavaScript execution is actually blocked. This does NOT include:
+When a group has a `baseline`, all benchmarks show Δ% with a bootstrap
+confidence interval. The result is classified as faster, slower, equivalent, or
+inconclusive based on the equivalence margin.
 
-1. **Concurrent GC work** done in parallel threads (concurrent marking, sweeping)
-2. **Performance degradation** from CPU contention and cache effects
-3. **Total GC overhead** including preparation and cleanup
+See [Statistics.md](Statistics.md#equivalence-margin) for how the four verdicts
+work and how to calibrate the margin.
 
-### Key Findings
+## Further Reading
 
-1. **Multiple GC Events**: A single `gc()` call can trigger multiple GC events that are recorded separately
-2. **Incremental GC**: V8 breaks up GC work into smaller increments to reduce pause times
-3. **Duration < Impact**: The recorded duration is often much less than the actual performance impact
+- [Node.md](Node.md) -- Worker mode, module imports, custom metric sections,
+  external debugger attachment
+- [Browser.md](Browser.md) -- Bench function and page-load modes, completion
+  signals, CDP flow
+- [Profiling.md](Profiling.md) -- Allocation sampling, GC stats, V8 flags,
+  Perfetto export
+- [Statistics.md](Statistics.md) -- Column selection (`--stats`), bootstrap
+  methods, batching, equivalence testing
+- [README-tachometer.md](README-tachometer.md) -- Coming from tachometer

package/bin/benchforge

@@ -1,3 +1,2 @@
 #!/usr/bin/env -S node --experimental-strip-types
-import { runDefaultBench } from "../src/index.ts";
-await runDefaultBench();
+import "../src/bin/benchforge.ts";

package/dist/AnalyzeArchive-8NCJhmhS.mjs

@@ -0,0 +1,145 @@
+import { b as splitByOffsets, g as percentile, p as median, t as average, w as tukeyFences } from "./StatisticalUtils-BD92crgM.mjs";
+import { a as formatSignedPercent, c as timeMs, u as colors } from "./Formatters-BWj3d4sv.mjs";
+import { resolve } from "node:path";
+import { readFile } from "node:fs/promises";
+//#region src/cli/AnalyzeArchive.ts
+/** Diagnostic analysis of a .benchforge archive's per-batch statistics. */
+const { bold, dim, red, green, yellow } = colors;
+const blockFenceMultiplier = 3;
+/** Read an archive and print per-batch diagnostic analysis.
+* (for benchforge debugging/development purposes, not a general user tool)
+*/
+async function analyzeArchive(filePath) {
+	const content = await readFile(resolve(filePath), "utf-8");
+	const { report } = JSON.parse(content);
+	if (!report?.groups?.length) {
+		console.error("No report data found in archive.");
+		return;
+	}
+	const batchCount = report.metadata?.cliArgs?.batches;
+	for (const group of report.groups) analyzeGroup(group, batchCount);
+}
+/** Print analysis for all benchmarks in a group. */
+function analyzeGroup(group, batchCount) {
+	console.log(bold(`\n=== ${group.name} ===\n`));
+	const baseline = group.baseline;
+	for (const bench of group.benchmarks) analyzeBenchmark(bench, baseline, batchCount);
+}
+/** Print per-batch analysis for one benchmark entry. */
+function analyzeBenchmark(bench, baseline, batchCount) {
+	const bOffsets = bench.batchOffsets ?? inferOffsets(bench.samples, batchCount);
+	const baseOffsets = baseline?.batchOffsets ?? inferOffsets(baseline?.samples, batchCount);
+	if (!bOffsets?.length) {
+		console.log(dim("  No batch data (single batch run)"));
+		return;
+	}
+	const batches = splitByOffsets(bench.samples, bOffsets);
+	const baseBatches = baseOffsets && baseline ? splitByOffsets(baseline.samples, baseOffsets) : void 0;
+	printBatchHeader(bench, baseline, batches.length);
+	printBatchTable(batches, baseBatches);
+	if (baseBatches && baseBatches.length === batches.length) {
+		printOrderEffect(batches, baseBatches);
+		printPairedDeltas(batches, baseBatches);
+		printTrimmedBlocks(batches, baseBatches, bench.name);
+	}
+	console.log();
+}
+/** Infer equal-sized batch offsets when batchOffsets isn't in the archive. */
+function inferOffsets(samples, batchCount) {
+	if (!samples?.length || !batchCount || batchCount <= 1) return void 0;
+	const size = Math.floor(samples.length / batchCount);
+	return Array.from({ length: batchCount }, (_, i) => i * size);
+}
+/** Print benchmark name with batch/run summary. */
+function printBatchHeader(bench, baseline, nBatches) {
+	const baseRuns = baseline?.samples?.length;
+	const dur = bench.totalTime ? (bench.totalTime / nBatches).toFixed(1) + "s" : "?";
+	const info = dim(` (${nBatches} batches, ${baseRuns ? `${bench.samples.length}+${baseRuns} runs` : `${bench.samples.length} runs`}, ~${dur}/batch)`);
+	console.log(bold(`  ${bench.name}`) + info);
+}
+/** Print per-batch median table for current and baseline. */
+function printBatchTable(benches, baselines) {
+	const header = baselines ? `  ${"batch".padEnd(7)} ${"n".padStart(4)}  ${"current".padStart(10)}  ${"baseline".padStart(10)}  ${"delta".padStart(8)}` : `  ${"batch".padEnd(7)} ${"n".padStart(4)}  ${"median".padStart(10)}`;
+	console.log(dim(header));
+	for (let i = 0; i < benches.length; i++) {
+		const n = String(benches[i].length).padStart(4);
+		const med = (timeMs(median(benches[i])) ?? "").padStart(10);
+		const idx = String(i).padEnd(7);
+		if (!baselines?.[i]) {
+			console.log(`  ${idx} ${n}  ${med}`);
+			continue;
+		}
+		const baseMed = (timeMs(median(baselines[i])) ?? "").padStart(10);
+		const delta = formatDelta(medianDelta(benches[i], baselines[i])).padStart(8);
+		const order = i % 2 === 0 ? dim(" B>C") : dim(" C>B");
+		console.log(`  ${idx} ${n}  ${med}  ${baseMed}  ${delta}${order}`);
+	}
+}
+/** Analyze order effect: does running second make a difference? */
+function printOrderEffect(benches, baselines) {
+	const deltas = benches.map((b, i) => medianDelta(b, baselines[i]));
+	const baseFirstDeltas = deltas.filter((_, i) => i % 2 === 0);
+	const currFirstDeltas = deltas.filter((_, i) => i % 2 === 1);
+	const baseFirstAvg = baseFirstDeltas.length ? average(baseFirstDeltas) : 0;
+	const currFirstAvg = currFirstDeltas.length ? average(currFirstDeltas) : 0;
+	console.log();
+	console.log(bold("  Order effect:"));
+	console.log(`    baseline first (B>C): avg delta ${formatDelta(baseFirstAvg)}` + dim(` (${baseFirstDeltas.length} batches)`));
+	console.log(`    current first  (C>B): avg delta ${formatDelta(currFirstAvg)}` + dim(` (${currFirstDeltas.length} batches)`));
+	const diff = Math.abs(baseFirstAvg - currFirstAvg);
+	if (diff > 2) console.log(yellow(`    ==> ${diff.toFixed(1)}% order effect detected`));
+	else console.log(dim(`    order effect: ${diff.toFixed(1)}% (small)`));
+}
+/** Print paired batch deltas and their consistency. */
+function printPairedDeltas(benches, baselines) {
+	const deltas = benches.map((b, i) => medianDelta(b, baselines[i]));
+	const positive = deltas.filter((d) => d > 0).length;
+	const negative = deltas.filter((d) => d < 0).length;
+	const avgDelta = average(deltas);
+	const med = median(deltas);
+	const spread = percentile(deltas, .75) - percentile(deltas, .25);
+	console.log();
+	console.log(bold("  Paired deltas:"));
+	console.log(`    mean: ${formatDelta(avgDelta)}  median: ${formatDelta(med)}  IQR: ${spread.toFixed(1)}%`);
+	console.log(`    direction: ${positive} slower, ${negative} faster` + dim(` (${deltas.length} batches)`));
+	if (positive > 0 && negative > 0) console.log(green("    ==> batches disagree on direction"));
+	else console.log(red("    ==> all batches agree on direction (systematic bias?)"));
+}
+/** Show which blocks would be Tukey-trimmed per side. */
+function printTrimmedBlocks(benches, baselines, name) {
+	console.log();
+	console.log(bold("  Trimmed blocks:"));
+	const baseMeans = baselines.map((b) => average(b));
+	const benchMeans = benches.map((b) => average(b));
+	printSideTrim("baseline", baseMeans);
+	printSideTrim(name, benchMeans);
+}
+/** Color a percent delta: red if >1%, green if <-1%. */
+function formatDelta(pct) {
+	const str = formatSignedPercent(pct);
+	if (pct > 1) return red(str);
+	if (pct < -1) return green(str);
+	return str;
+}
+/** Percent delta between two medians. */
+function medianDelta(samples, baseSamples) {
+	const med = median(samples);
+	const baseMed = median(baseSamples);
+	return (med - baseMed) / baseMed * 100;
+}
+/** Print trimming info for one side using 3x IQR fences. */
+function printSideTrim(label, means) {
+	const [, hi] = tukeyFences(means, blockFenceMultiplier);
+	const indices = means.map((v, i) => v > hi ? i : -1).filter((i) => i >= 0);
+	if (indices.length === 0) {
+		console.log(dim(`    ${label}: 0 trimmed`));
+		return;
+	}
+	const vals = indices.map((i) => timeMs(means[i]) ?? "?").join(", ");
+	const fence = `hi: ${timeMs(hi)}`;
+	console.log(`    ${label}: ${yellow(`${indices.length} trimmed`)} (${vals})` + dim(`  fence: ${fence}`));
+}
+//#endregion
+export { analyzeArchive };
+
+//# sourceMappingURL=AnalyzeArchive-8NCJhmhS.mjs.map

package/dist/AnalyzeArchive-8NCJhmhS.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"AnalyzeArchive-8NCJhmhS.mjs","names":[],"sources":["../src/cli/AnalyzeArchive.ts"],"sourcesContent":["/** Diagnostic analysis of a .benchforge archive's per-batch statistics. */\nimport { readFile } from \"node:fs/promises\";\nimport { resolve } from \"node:path\";\nimport colors from \"../report/Colors.ts\";\nimport { formatSignedPercent, timeMs } from \"../report/Formatters.ts\";\nimport {\n  average,\n  median,\n  percentile,\n  splitByOffsets,\n  tukeyFences,\n} from \"../stats/StatisticalUtils.ts\";\nimport type { BenchmarkEntry, BenchmarkGroup } from \"../viewer/ReportData.ts\";\n\nconst { bold, dim, red, green, yellow } = colors;\n\nconst blockFenceMultiplier = 3;\n\n/** Read an archive and print per-batch diagnostic analysis.\n * (for benchforge debugging/development purposes, not a general user tool)\n */\nexport async function analyzeArchive(filePath: string): Promise<void> {\n  const absPath = resolve(filePath);\n  const content = await readFile(absPath, \"utf-8\");\n  const { report } = JSON.parse(content);\n  if (!report?.groups?.length) {\n    console.error(\"No report data found in archive.\");\n    return;\n  }\n  const batchCount = report.metadata?.cliArgs?.batches as number | undefined;\n  for (const group of report.groups) {\n    analyzeGroup(group, batchCount);\n  }\n}\n\n/** Print analysis for all benchmarks in a group. */\nfunction analyzeGroup(group: BenchmarkGroup, batchCount?: number): void {\n  console.log(bold(`\\n=== ${group.name} ===\\n`));\n\n  const baseline = group.baseline;\n  for (const bench of group.benchmarks) {\n    analyzeBenchmark(bench, baseline, batchCount);\n  }\n}\n\n/** Print per-batch analysis for one benchmark entry. */\nfunction analyzeBenchmark(\n  bench: BenchmarkEntry,\n  baseline: BenchmarkEntry | undefined,\n  batchCount?: number,\n): void {\n  const bOffsets =\n    bench.batchOffsets ?? inferOffsets(bench.samples, batchCount);\n  const baseOffsets =\n    baseline?.batchOffsets ?? inferOffsets(baseline?.samples, batchCount);\n  if (!bOffsets?.length) {\n    console.log(dim(\"  No batch data (single batch run)\"));\n    return;\n  }\n\n  const batches = splitByOffsets(bench.samples, bOffsets);\n  const baseBatches =\n    baseOffsets && baseline\n      ? splitByOffsets(baseline.samples, baseOffsets)\n      : undefined;\n\n  printBatchHeader(bench, baseline, batches.length);\n  printBatchTable(batches, baseBatches);\n\n  if (baseBatches && baseBatches.length === batches.length) {\n    printOrderEffect(batches, baseBatches);\n    printPairedDeltas(batches, baseBatches);\n    printTrimmedBlocks(batches, baseBatches, bench.name);\n  }\n  console.log();\n}\n\n/** Infer equal-sized batch offsets when batchOffsets isn't in the archive. */\nfunction inferOffsets(\n  samples: number[] | undefined,\n  batchCount?: number,\n): number[] | undefined {\n  if (!samples?.length || !batchCount || batchCount <= 1) return undefined;\n  const size = Math.floor(samples.length / batchCount);\n  return Array.from({ length: batchCount }, (_, i) => i * size);\n}\n\n/** Print benchmark name with batch/run summary. */\nfunction printBatchHeader(\n  bench: BenchmarkEntry,\n  baseline: BenchmarkEntry | undefined,\n  nBatches: number,\n): void {\n  const baseRuns = baseline?.samples?.length;\n  const dur = bench.totalTime\n    ? (bench.totalTime / nBatches).toFixed(1) + \"s\"\n    : \"?\";\n  const runs = baseRuns\n    ? `${bench.samples.length}+${baseRuns} runs`\n    : `${bench.samples.length} runs`;\n  const info = dim(` (${nBatches} batches, ${runs}, ~${dur}/batch)`);\n  console.log(bold(`  ${bench.name}`) + info);\n}\n\n/** Print per-batch median table for current and baseline. */\nfunction printBatchTable(\n  benches: number[][],\n  baselines: number[][] | undefined,\n): void {\n  const header = baselines\n    ? `  ${\"batch\".padEnd(7)} ${\"n\".padStart(4)}  ${\"current\".padStart(10)}  ${\"baseline\".padStart(10)}  ${\"delta\".padStart(8)}`\n    : `  ${\"batch\".padEnd(7)} ${\"n\".padStart(4)}  ${\"median\".padStart(10)}`;\n  console.log(dim(header));\n\n  for (let i = 0; i < benches.length; i++) {\n    const n = String(benches[i].length).padStart(4);\n    const med = (timeMs(median(benches[i])) ?? \"\").padStart(10);\n    const idx = String(i).padEnd(7);\n    if (!baselines?.[i]) {\n      console.log(`  ${idx} ${n}  ${med}`);\n      continue;\n    }\n    const baseMed = (timeMs(median(baselines[i])) ?? \"\").padStart(10);\n    const delta = formatDelta(medianDelta(benches[i], baselines[i])).padStart(\n      8,\n    );\n    const order = i % 2 === 0 ? dim(\" B>C\") : dim(\" C>B\");\n    console.log(`  ${idx} ${n}  ${med}  ${baseMed}  ${delta}${order}`);\n  }\n}\n\n/** Analyze order effect: does running second make a difference? */\nfunction printOrderEffect(benches: number[][], baselines: number[][]): void {\n  // Even batches: baseline runs first (B>C), odd: current runs first (C>B)\n  const deltas = benches.map((b, i) => medianDelta(b, baselines[i]));\n  const baseFirstDeltas = deltas.filter((_, i) => i % 2 === 0);\n  const currFirstDeltas = deltas.filter((_, i) => i % 2 === 1);\n  const baseFirstAvg = baseFirstDeltas.length ? average(baseFirstDeltas) : 0;\n  const currFirstAvg = currFirstDeltas.length ? average(currFirstDeltas) : 0;\n\n  console.log();\n  console.log(bold(\"  Order effect:\"));\n  console.log(\n    `    baseline first (B>C): avg delta ${formatDelta(baseFirstAvg)}` +\n      dim(` (${baseFirstDeltas.length} batches)`),\n  );\n  console.log(\n    `    current first  (C>B): avg delta ${formatDelta(currFirstAvg)}` +\n      dim(` (${currFirstDeltas.length} batches)`),\n  );\n\n  const diff = Math.abs(baseFirstAvg - currFirstAvg);\n  if (diff > 2) {\n    console.log(yellow(`    ==> ${diff.toFixed(1)}% order effect detected`));\n  } else {\n    console.log(dim(`    order effect: ${diff.toFixed(1)}% (small)`));\n  }\n}\n\n/** Print paired batch deltas and their consistency. */\nfunction printPairedDeltas(benches: number[][], baselines: number[][]): void {\n  const deltas = benches.map((b, i) => medianDelta(b, baselines[i]));\n\n  const positive = deltas.filter(d => d > 0).length;\n  const negative = deltas.filter(d => d < 0).length;\n  const avgDelta = average(deltas);\n  const med = median(deltas);\n  const spread = percentile(deltas, 0.75) - percentile(deltas, 0.25);\n\n  console.log();\n  console.log(bold(\"  Paired deltas:\"));\n  console.log(\n    `    mean: ${formatDelta(avgDelta)}  median: ${formatDelta(med)}  IQR: ${spread.toFixed(1)}%`,\n  );\n  console.log(\n    `    direction: ${positive} slower, ${negative} faster` +\n      dim(` (${deltas.length} batches)`),\n  );\n\n  if (positive > 0 && negative > 0) {\n    console.log(green(\"    ==> batches disagree on direction\"));\n  } else {\n    console.log(\n      red(\"    ==> all batches agree on direction (systematic bias?)\"),\n    );\n  }\n}\n\n/** Show which blocks would be Tukey-trimmed per side. */\nfunction printTrimmedBlocks(\n  benches: number[][],\n  baselines: number[][],\n  name: string,\n): void {\n  console.log();\n  console.log(bold(\"  Trimmed blocks:\"));\n  const baseMeans = baselines.map(b => average(b));\n  const benchMeans = benches.map(b => average(b));\n  printSideTrim(\"baseline\", baseMeans);\n  printSideTrim(name, benchMeans);\n}\n\n/** Color a percent delta: red if >1%, green if <-1%. */\nfunction formatDelta(pct: number): string {\n  const str = formatSignedPercent(pct);\n  if (pct > 1) return red(str);\n  if (pct < -1) return green(str);\n  return str;\n}\n\n/** Percent delta between two medians. */\nfunction medianDelta(samples: number[], baseSamples: number[]): number {\n  const med = median(samples);\n  const baseMed = median(baseSamples);\n  return ((med - baseMed) / baseMed) * 100;\n}\n\n/** Print trimming info for one side using 3x IQR fences. */\nfunction printSideTrim(label: string, means: number[]): void {\n  const [, hi] = tukeyFences(means, blockFenceMultiplier);\n  const indices = means.map((v, i) => (v > hi ? i : -1)).filter(i => i >= 0);\n  if (indices.length === 0) {\n    console.log(dim(`    ${label}: 0 trimmed`));\n    return;\n  }\n  const vals = indices.map(i => timeMs(means[i]) ?? \"?\").join(\", \");\n  const fence = `hi: ${timeMs(hi)}`;\n  console.log(\n    `    ${label}: ${yellow(`${indices.length} trimmed`)} (${vals})` +\n      dim(`  fence: ${fence}`),\n  );\n}\n"],"mappings":";;;;;;AAcA,MAAM,EAAE,MAAM,KAAK,KAAK,OAAO,WAAW;AAE1C,MAAM,uBAAuB;;;;AAK7B,eAAsB,eAAe,UAAiC;CAEpE,MAAM,UAAU,MAAM,SADN,QAAQ,SAAS,EACO,QAAQ;CAChD,MAAM,EAAE,WAAW,KAAK,MAAM,QAAQ;AACtC,KAAI,CAAC,QAAQ,QAAQ,QAAQ;AAC3B,UAAQ,MAAM,mCAAmC;AACjD;;CAEF,MAAM,aAAa,OAAO,UAAU,SAAS;AAC7C,MAAK,MAAM,SAAS,OAAO,OACzB,cAAa,OAAO,WAAW;;;AAKnC,SAAS,aAAa,OAAuB,YAA2B;AACtE,SAAQ,IAAI,KAAK,SAAS,MAAM,KAAK,QAAQ,CAAC;CAE9C,MAAM,WAAW,MAAM;AACvB,MAAK,MAAM,SAAS,MAAM,WACxB,kBAAiB,OAAO,UAAU,WAAW;;;AAKjD,SAAS,iBACP,OACA,UACA,YACM;CACN,MAAM,WACJ,MAAM,gBAAgB,aAAa,MAAM,SAAS,WAAW;CAC/D,MAAM,cACJ,UAAU,gBAAgB,aAAa,UAAU,SAAS,WAAW;AACvE,KAAI,CAAC,UAAU,QAAQ;AACrB,UAAQ,IAAI,IAAI,qCAAqC,CAAC;AACtD;;CAGF,MAAM,UAAU,eAAe,MAAM,SAAS,SAAS;CACvD,MAAM,cACJ,eAAe,WACX,eAAe,SAAS,SAAS,YAAY,GAC7C,KAAA;AAEN,kBAAiB,OAAO,UAAU,QAAQ,OAAO;AACjD,iBAAgB,SAAS,YAAY;AAErC,KAAI,eAAe,YAAY,WAAW,QAAQ,QAAQ;AACxD,mBAAiB,SAAS,YAAY;AACtC,oBAAkB,SAAS,YAAY;AACvC,qBAAmB,SAAS,aAAa,MAAM,KAAK;;AAEtD,SAAQ,KAAK;;;AAIf,SAAS,aACP,SACA,YACsB;AACtB,KAAI,CAAC,SAAS,UAAU,CAAC,cAAc,cAAc,EAAG,QAAO,KAAA;CAC/D,MAAM,OAAO,KAAK,MAAM,QAAQ,SAAS,WAAW;AACpD,QAAO,MAAM,KAAK,EAAE,QAAQ,YAAY,GAAG,GAAG,MAAM,IAAI,KAAK;;;AAI/D,SAAS,iBACP,OACA,UACA,UACM;CACN,MAAM,WAAW,UAAU,SAAS;CACpC,MAAM,MAAM,MAAM,aACb,MAAM,YAAY,UAAU,QAAQ,EAAE,GAAG,MAC1C;CAIJ,MAAM,OAAO,IAAI,KAAK,SAAS,YAHlB,WACT,GAAG,MAAM,QAAQ,OAAO,GAAG,SAAS,SACpC,GAAG,MAAM,QAAQ,OAAO,OACoB,KAAK,IAAI,SAAS;AAClE,SAAQ,IAAI,KAAK,KAAK,MAAM,OAAO,GAAG,KAAK;;;AAI7C,SAAS,gBACP,SACA,WACM;CACN,MAAM,SAAS,YACX,KAAK,QAAQ,OAAO,EAAE,CAAC,GAAG,IAAI,SAAS,EAAE,CAAC,IAAI,UAAU,SAAS,GAAG,CAAC,IAAI,WAAW,SAAS,GAAG,CAAC,IAAI,QAAQ,SAAS,EAAE,KACxH,KAAK,QAAQ,OAAO,EAAE,CAAC,GAAG,IAAI,SAAS,EAAE,CAAC,IAAI,SAAS,SAAS,GAAG;AACvE,SAAQ,IAAI,IAAI,OAAO,CAAC;AAExB,MAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,IAAI,OAAO,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE;EAC/C,MAAM,OAAO,OAAO,OAAO,QAAQ,GAAG,CAAC,IAAI,IAAI,SAAS,GAAG;EAC3D,MAAM,MAAM,OAAO,EAAE,CAAC,OAAO,EAAE;AAC/B,MAAI,CAAC,YAAY,IAAI;AACnB,WAAQ,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI,MAAM;AACpC;;EAEF,MAAM,WAAW,OAAO,OAAO,UAAU,GAAG,CAAC,IAAI,IAAI,SAAS,GAAG;EACjE,MAAM,QAAQ,YAAY,YAAY,QAAQ,IAAI,UAAU,GAAG,CAAC,CAAC,SAC/D,EACD;EACD,MAAM,QAAQ,IAAI,MAAM,IAAI,IAAI,OAAO,GAAG,IAAI,OAAO;AACrD,UAAQ,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI,IAAI,IAAI,QAAQ,IAAI,QAAQ,QAAQ;;;;AAKtE,SAAS,iBAAiB,SAAqB,WAA6B;CAE1E,MAAM,SAAS,QAAQ,KAAK,GAAG,MAAM,YAAY,GAAG,UAAU,GAAG,CAAC;CAClE,MAAM,kBAAkB,OAAO,QAAQ,GAAG,MAAM,IAAI,MAAM,EAAE;CAC5D,MAAM,kBAAkB,OAAO,QAAQ,GAAG,MAAM,IAAI,MAAM,EAAE;CAC5D,MAAM,eAAe,gBAAgB,SAAS,QAAQ,gBAAgB,GAAG;CACzE,MAAM,eAAe,gBAAgB,SAAS,QAAQ,gBAAgB,GAAG;AAEzE,SAAQ,KAAK;AACb,SAAQ,IAAI,KAAK,kBAAkB,CAAC;AACpC,SAAQ,IACN,uCAAuC,YAAY,aAAa,KAC9D,IAAI,KAAK,gBAAgB,OAAO,WAAW,CAC9C;AACD,SAAQ,IACN,uCAAuC,YAAY,aAAa,KAC9D,IAAI,KAAK,gBAAgB,OAAO,WAAW,CAC9C;CAED,MAAM,OAAO,KAAK,IAAI,eAAe,aAAa;AAClD,KAAI,OAAO,EACT,SAAQ,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC,yBAAyB,CAAC;KAExE,SAAQ,IAAI,IAAI,qBAAqB,KAAK,QAAQ,EAAE,CAAC,WAAW,CAAC;;;AAKrE,SAAS,kBAAkB,SAAqB,WAA6B;CAC3E,MAAM,SAAS,QAAQ,KAAK,GAAG,MAAM,YAAY,GAAG,UAAU,GAAG,CAAC;CAElE,MAAM,WAAW,OAAO,QAAO,MAAK,IAAI,EAAE,CAAC;CAC3C,MAAM,WAAW,OAAO,QAAO,MAAK,IAAI,EAAE,CAAC;CAC3C,MAAM,WAAW,QAAQ,OAAO;CAChC,MAAM,MAAM,OAAO,OAAO;CAC1B,MAAM,SAAS,WAAW,QAAQ,IAAK,GAAG,WAAW,QAAQ,IAAK;AAElE,SAAQ,KAAK;AACb,SAAQ,IAAI,KAAK,mBAAmB,CAAC;AACrC,SAAQ,IACN,aAAa,YAAY,SAAS,CAAC,YAAY,YAAY,IAAI,CAAC,SAAS,OAAO,QAAQ,EAAE,CAAC,GAC5F;AACD,SAAQ,IACN,kBAAkB,SAAS,WAAW,SAAS,WAC7C,IAAI,KAAK,OAAO,OAAO,WAAW,CACrC;AAED,KAAI,WAAW,KAAK,WAAW,EAC7B,SAAQ,IAAI,MAAM,wCAAwC,CAAC;KAE3D,SAAQ,IACN,IAAI,4DAA4D,CACjE;;;AAKL,SAAS,mBACP,SACA,WACA,MACM;AACN,SAAQ,KAAK;AACb,SAAQ,IAAI,KAAK,oBAAoB,CAAC;CACtC,MAAM,YAAY,UAAU,KAAI,MAAK,QAAQ,EAAE,CAAC;CAChD,MAAM,aAAa,QAAQ,KAAI,MAAK,QAAQ,EAAE,CAAC;AAC/C,eAAc,YAAY,UAAU;AACpC,eAAc,MAAM,WAAW;;;AAIjC,SAAS,YAAY,KAAqB;CACxC,MAAM,MAAM,oBAAoB,IAAI;AACpC,KAAI,MAAM,EAAG,QAAO,IAAI,IAAI;AAC5B,KAAI,MAAM,GAAI,QAAO,MAAM,IAAI;AAC/B,QAAO;;;AAIT,SAAS,YAAY,SAAmB,aAA+B;CACrE,MAAM,MAAM,OAAO,QAAQ;CAC3B,MAAM,UAAU,OAAO,YAAY;AACnC,SAAS,MAAM,WAAW,UAAW;;;AAIvC,SAAS,cAAc,OAAe,OAAuB;CAC3D,MAAM,GAAG,MAAM,YAAY,OAAO,qBAAqB;CACvD,MAAM,UAAU,MAAM,KAAK,GAAG,MAAO,IAAI,KAAK,IAAI,GAAI,CAAC,QAAO,MAAK,KAAK,EAAE;AAC1E,KAAI,QAAQ,WAAW,GAAG;AACxB,UAAQ,IAAI,IAAI,OAAO,MAAM,aAAa,CAAC;AAC3C;;CAEF,MAAM,OAAO,QAAQ,KAAI,MAAK,OAAO,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,KAAK;CACjE,MAAM,QAAQ,OAAO,OAAO,GAAG;AAC/B,SAAQ,IACN,OAAO,MAAM,IAAI,OAAO,GAAG,QAAQ,OAAO,UAAU,CAAC,IAAI,KAAK,KAC5D,IAAI,YAAY,QAAQ,CAC3B"}