benchforge 0.1.9 → 0.2.4

package/src/runners/BenchRunner.ts

@@ -1,15 +1,7 @@
-import type { BenchmarkSpec } from "../Benchmark.ts";
-import type { MeasuredResults } from "../MeasuredResults.ts";
+import type { BenchmarkSpec } from "./BenchmarkSpec.ts";
+import type { MeasuredResults } from "./MeasuredResults.ts";
 
-/** Execute benchmark with optional parameters */
-export function executeBenchmark<T>(
-  benchmark: BenchmarkSpec<T>,
-  params?: T,
-): void {
-  (benchmark.fn as (params?: T) => void)(params);
-}
-
-/** Interface for benchmark execution libraries */
+/** Benchmark execution strategy that collects timing samples. */
 export interface BenchRunner {
   runBench<T = unknown>(
     benchmark: BenchmarkSpec<T>,
@@ -18,6 +10,7 @@ export interface BenchRunner {
   ): Promise<MeasuredResults[]>;
 }
 
+/** Configuration for benchmark execution: timing limits, warmup, profiling, and V8 options. */
 export interface RunnerOptions {
   /** Minimum time to run each benchmark (milliseconds) */
   minTime?: number;
@@ -36,13 +29,11 @@ export interface RunnerOptions {
   /** Minimum samples required - mitata only */
   minSamples?: number;
   /** Force GC after each iteration (requires --expose-gc) */
-  collect?: boolean;
-  /** Enable CPU performance counters (requires root access) */
-  cpuCounters?: boolean;
+  gcForce?: boolean;
   /** Trace V8 optimization tiers (requires --allow-natives-syntax) */
   traceOpt?: boolean;
-  /** Skip post-warmup settle time (default: false) */
-  noSettle?: boolean;
+  /** Post-warmup settle time in ms for V8 background compilation (0 to skip) */
+  pauseWarmup?: number;
   /** Iterations before first pause (then pauseInterval applies) */
   pauseFirst?: number;
   /** Iterations between pauses for V8 optimization (0 to disable) */
@@ -51,10 +42,24 @@ export interface RunnerOptions {
   pauseDuration?: number;
   /** Collect GC stats via --trace-gc-nvp (requires worker mode) */
   gcStats?: boolean;
-  /** Heap sampling allocation attribution */
-  heapSample?: boolean;
-  /** Heap sampling interval in bytes */
-  heapInterval?: number;
-  /** Heap sampling stack depth */
-  heapDepth?: number;
+  /** Allocation sampling attribution */
+  alloc?: boolean;
+  /** Allocation sampling interval in bytes */
+  allocInterval?: number;
+  /** Allocation sampling stack depth */
+  allocDepth?: number;
+  /** V8 CPU time sampling */
+  profile?: boolean;
+  /** CPU sampling interval in microseconds (default 1000) */
+  profileInterval?: number;
+  /** Collect per-function execution counts via V8 precise coverage */
+  callCounts?: boolean;
+}
+
+/** Invoke the benchmark function, forwarding setup params. */
+export function executeBenchmark<T>(
+  benchmark: BenchmarkSpec<T>,
+  params?: T,
+): void {
+  (benchmark.fn as (params?: T) => void)(params);
 }

package/src/{Benchmark.ts → runners/BenchmarkSpec.ts}

@@ -1,4 +1,4 @@
-/** Single benchmark function specification */
+/** Benchmark function with optional module path for worker-mode serialization. */
 export interface BenchmarkSpec<T = unknown> {
   name: string;
   fn: BenchmarkFunction<T>;
@@ -10,23 +10,22 @@ export interface BenchmarkSpec<T = unknown> {
   setupExportName?: string;
 }
 
+/** Benchmark function, optionally receiving setup parameters from the group. */
 export type BenchmarkFunction<T = unknown> =
   | ((params: T) => void)
   | (() => void);
 
-/** Group of benchmarks with shared setup */
+/** Group of benchmarks with shared setup and optional baseline. */
 export interface BenchGroup<T = unknown> {
   name: string;
-  /** Prepare parameters for all benchmarks in this group */
   setup?: () => T | Promise<T>;
   benchmarks: BenchmarkSpec<T>[];
-  /** Baseline benchmark for comparison */
   baseline?: BenchmarkSpec<T>;
-  /** Metadata for reporting (e.g., lines of code) */
+  /** Metadata for reporting (e.g. lines of code). */
   metadata?: Record<string, any>;
 }
 
-/** Collection of benchmark groups */
+/** Named collection of benchmark groups. */
 export interface BenchSuite {
   name: string;
   groups: BenchGroup<any>[];

package/src/runners/CreateRunner.ts

@@ -1,11 +1,9 @@
-import { BasicRunner } from "./BasicRunner.ts";
 import type { BenchRunner } from "./BenchRunner.ts";
+import { TimingRunner } from "./TimingRunner.ts";
 
-export type KnownRunner = "basic";
+export type KnownRunner = "timing";
 
-/** @return benchmark runner */
-export async function createRunner(
-  _runnerName: KnownRunner,
-): Promise<BenchRunner> {
-  return new BasicRunner();
+/** Create a benchmark runner by name. */
+export async function createRunner(_name: KnownRunner): Promise<BenchRunner> {
+  return new TimingRunner();
 }

package/src/runners/GcStats.ts

@@ -1,14 +1,20 @@
-/** GC statistics aggregated from V8 trace events.
- *  Node (--trace-gc-nvp) provides all fields.
- *  Browser (CDP Tracing) provides counts, collected, and pause only. */
+/**
+ * Aggregated GC statistics from V8 trace events.
+ * Node (--trace-gc-nvp) provides all fields; browser (CDP) provides counts, collected, and pause only.
+ */
 export interface GcStats {
   scavenges: number;
   markCompacts: number;
-  totalCollected: number; // bytes freed
-  gcPauseTime: number; // total pause time (ms)
-  totalAllocated?: number; // bytes allocated (Node only)
-  totalPromoted?: number; // bytes promoted to old gen (Node only)
-  totalSurvived?: number; // bytes survived in young gen (Node only)
+  /** Bytes freed by GC. */
+  totalCollected: number;
+  /** Total GC pause time in milliseconds. */
+  gcPauseTime: number;
+  /** Bytes allocated (Node only). */
+  totalAllocated?: number;
+  /** Bytes promoted to old generation (Node only). */
+  totalPromoted?: number;
+  /** Bytes survived in young generation (Node only). */
+  totalSurvived?: number;
 }
 
 /** Single GC event. Node provides all fields; browser provides type, pauseMs, collected. */
@@ -16,79 +22,58 @@ export interface GcEvent {
   type: "scavenge" | "mark-compact" | "minor-ms" | "unknown";
   pauseMs: number;
   collected: number;
-  allocated?: number; // Node only
-  promoted?: number; // Node only
-  survived?: number; // Node only
+  /** Node only. */
+  allocated?: number;
+  /** Node only. */
+  promoted?: number;
+  /** Node only. */
+  survived?: number;
 }
 
-/** Parse a single --trace-gc-nvp stderr line */
+/** Parse a single --trace-gc-nvp stderr line into a GcEvent. */
 export function parseGcLine(line: string): GcEvent | undefined {
-  // V8 format: [pid:addr:gen] N ms: pause=X gc=s ... allocated=N promoted=N ...
   if (!line.includes("pause=")) return undefined;
 
   const fields = parseNvpFields(line);
   if (!fields.gc) return undefined;
 
-  const int = (k: string) => Number.parseInt(fields[k] || "0", 10);
+  const intField = (name: string) => Number.parseInt(fields[name] || "0", 10);
   const type = parseGcType(fields.gc);
   const pauseMs = Number.parseFloat(fields.pause || "0");
-  const allocated = int("allocated");
-  const promoted = int("promoted");
-  // V8 uses "new_space_survived" not "survived"
-  const survived = int("new_space_survived") || int("survived");
-  // Calculate collected from start/end object size if available
-  const startSize = int("start_object_size");
-  const endSize = int("end_object_size");
-  const collected = startSize > endSize ? startSize - endSize : 0;
-
   if (Number.isNaN(pauseMs)) return undefined;
 
-  return { type, pauseMs, allocated, collected, promoted, survived };
-}
-
-/** Parse name=value pairs from trace-gc-nvp line */
-function parseNvpFields(line: string): Record<string, string> {
-  const fields: Record<string, string> = {};
-  // Format: "key=value, key=value, ..." or "key=value key=value"
-  const matches = line.matchAll(/(\w+)=([^\s,]+)/g);
-  for (const [, key, value] of matches) {
-    fields[key] = value;
-  }
-  return fields;
-}
+  const allocated = intField("allocated");
+  const promoted = intField("promoted");
+  // V8 uses "new_space_survived" not "survived"
+  const survived = intField("new_space_survived") || intField("survived");
+  const start = intField("start_object_size");
+  const end = intField("end_object_size");
+  const collected = start > end ? start - end : 0;
 
-/** Map V8 gc type codes to our types */
-function parseGcType(gcField: string): GcEvent["type"] {
-  // V8 uses: s=scavenge, mc=mark-compact, mmc=minor-mc (young gen mark-compact)
-  if (gcField === "s" || gcField === "scavenge") return "scavenge";
-  if (gcField === "mc" || gcField === "ms" || gcField === "mark-compact")
-    return "mark-compact";
-  if (gcField === "mmc" || gcField === "minor-mc" || gcField === "minor-ms")
-    return "minor-ms";
-  return "unknown";
+  return { type, pauseMs, allocated, collected, promoted, survived };
 }
 
-/** Aggregate GC events into summary stats */
+/** Aggregate a list of GC events into summary statistics. */
 export function aggregateGcStats(events: GcEvent[]): GcStats {
   let scavenges = 0;
   let markCompacts = 0;
   let gcPauseTime = 0;
   let totalCollected = 0;
-  let hasNodeFields = false;
+  let hasNode = false;
   let totalAllocated = 0;
   let totalPromoted = 0;
   let totalSurvived = 0;
 
-  for (const e of events) {
-    if (e.type === "scavenge" || e.type === "minor-ms") scavenges++;
-    else if (e.type === "mark-compact") markCompacts++;
-    gcPauseTime += e.pauseMs;
-    totalCollected += e.collected;
-    if (e.allocated != null) {
-      hasNodeFields = true;
-      totalAllocated += e.allocated;
-      totalPromoted += e.promoted ?? 0;
-      totalSurvived += e.survived ?? 0;
+  for (const event of events) {
+    if (event.type === "scavenge" || event.type === "minor-ms") scavenges++;
+    else if (event.type === "mark-compact") markCompacts++;
+    gcPauseTime += event.pauseMs;
+    totalCollected += event.collected;
+    if (event.allocated != null) {
+      hasNode = true;
+      totalAllocated += event.allocated;
+      totalPromoted += event.promoted ?? 0;
+      totalSurvived += event.survived ?? 0;
     }
   }
 
@@ -97,11 +82,23 @@ export function aggregateGcStats(events: GcEvent[]): GcStats {
     markCompacts,
     totalCollected,
     gcPauseTime,
-    ...(hasNodeFields && { totalAllocated, totalPromoted, totalSurvived }),
+    ...(hasNode && { totalAllocated, totalPromoted, totalSurvived }),
   };
 }
 
-/** @return GcStats with all counters zeroed */
-export function emptyGcStats(): GcStats {
-  return { scavenges: 0, markCompacts: 0, totalCollected: 0, gcPauseTime: 0 };
+/** Parse name=value pairs from a trace-gc-nvp line. */
+function parseNvpFields(line: string): Record<string, string> {
+  const pairs = [...line.matchAll(/(\w+)=([^\s,]+)/g)];
+  return Object.fromEntries(pairs.map(([, key, value]) => [key, value]));
+}
+
+/** Map V8 gc type codes to normalized event types. */
+function parseGcType(gcField: string): GcEvent["type"] {
+  // V8 uses: s=scavenge, mc=mark-compact, mmc=minor-mc (young gen mark-compact)
+  if (gcField === "s" || gcField === "scavenge") return "scavenge";
+  if (gcField === "mc" || gcField === "ms" || gcField === "mark-compact")
+    return "mark-compact";
+  if (gcField === "mmc" || gcField === "minor-mc" || gcField === "minor-ms")
+    return "minor-ms";
+  return "unknown";
 }

package/src/{MeasuredResults.ts → runners/MeasuredResults.ts}

@@ -1,13 +1,9 @@
-import type { HeapProfile } from "./heap-sample/HeapSampler.ts";
-import type { NodeGCTime } from "./NodeGC.ts";
-import type { GcStats } from "./runners/GcStats.ts";
-
-/** CPU performance counter stats */
-export interface CpuCounts {
-  instructions?: number;
-  cycles?: number;
-  branchMisses?: number;
-}
+import type { NavTiming } from "../profiling/browser/BrowserProfiler.ts";
+import type { CoverageData } from "../profiling/node/CoverageTypes.ts";
+import type { HeapProfile } from "../profiling/node/HeapSampler.ts";
+import type { TimeProfile } from "../profiling/node/TimeSampler.ts";
+import type { GcStats } from "../runners/GcStats.ts";
+import type { NodeGCTime } from "../runners/NodeGC.ts";
 
 /** Benchmark results: times in milliseconds, sizes in kilobytes */
 export interface MeasuredResults {
@@ -16,7 +12,7 @@ export interface MeasuredResults {
   /** Raw execution time samples for custom statistics */
   samples: number[];
 
-  /** Warmup iteration timings (ms) - captured before gc/settle */
+  /** Warmup iteration timings (ms) - captured before gc/pause-warmup */
   warmupSamples?: number[];
 
   /** Raw allocation samples per iteration (KB) */
@@ -25,10 +21,7 @@ export interface MeasuredResults {
   /** Heap size per sample (bytes) - used for charts */
   heapSamples?: number[];
 
-  /** Wall-clock timestamps per sample (μs since process start) - for Perfetto export */
-  timestamps?: number[];
-
-  /** Execution time in milliseconds (measurement overhead excluded by mitata) */
+  /** Execution time in milliseconds */
   time: {
     min: number;
     max: number;
@@ -51,36 +44,22 @@ export interface MeasuredResults {
     max: number;
   };
 
-  /**
-   * Time for explicit gc() call after test execution (milliseconds).
-   * Does not include GC time during test execution.
-   * Only reported by mitata runner.
-   */
+  /** Time for explicit gc() call after test execution (ms), excludes in-run GC. */
   gcTime?: {
     avg: number;
     min: number;
     max: number;
   };
 
-  /** CPU counter stats from @mitata/counters (requires root access) */
-  cpu?: CpuCounts;
-
-  /** L1 cache miss rate */
-  cpuCacheMiss?: number;
-
-  /** CPU stall rate (macOS only) */
-  cpuStall?: number;
-
-  /**
-   * Stop-the-world GC time blocking main thread (milliseconds).
-   * Measured via Node's performance hooks when nodeObserveGC is true.
-   * Excludes parallel thread collection time and indirect slowdowns.
-   */
+  /** Stop-the-world GC pause time (ms) via Node perf hooks (nodeObserveGC). */
   nodeGcTime?: NodeGCTime;
 
   /** Total time spent collecting samples (seconds) */
   totalTime?: number;
 
+  /** Monotonic start time (μs, hrtime-based) for Perfetto trace alignment. */
+  startTime?: number;
+
   /** Convergence information for adaptive mode */
   convergence?: {
     converged: boolean;
@@ -97,18 +76,28 @@ export interface MeasuredResults {
   /** Points where pauses occurred for V8 optimization */
   pausePoints?: PausePoint[];
 
+  /** Batch boundaries for block bootstrap (indices into samples where each batch starts) */
+  batchOffsets?: number[];
+
   /** GC stats from V8's --trace-gc-nvp (requires --gc-stats and worker mode) */
   gcStats?: GcStats;
 
   /** Heap sampling allocation profile (requires --heap-sample and worker mode) */
   heapProfile?: HeapProfile;
+
+  /** V8 CPU time sampling profile (requires --profile and worker mode) */
+  timeProfile?: TimeProfile;
+
+  /** Per-function execution counts (requires --call-counts) */
+  coverage?: CoverageData;
+
+  /** Navigation timings from page-load mode (one per iteration) */
+  navTimings?: NavTiming[];
 }
 
-/** A pause point during sample collection for V8 optimization */
+/** Pause inserted during sample collection for V8 optimization settling. */
 export interface PausePoint {
-  /** Sample index where pause occurred (after this iteration) */
   sampleIndex: number;
-  /** Pause duration in milliseconds */
   durationMs: number;
 }
 
@@ -125,3 +114,20 @@ export interface OptStatusInfo {
   /** Number of samples with deopt flag set */
   deoptCount: number;
 }
+
+/**
+ * V8 GetOptimizationStatus() return codes ==> human-readable tier names.
+ *   Bit 0 (1): is_function
+ *   Bit 4 (16): is_optimized (TurboFan)
+ *   Bit 5 (32): is_optimized (Maglev)
+ *   Bit 7 (128): is_baseline (Sparkplug)
+ *   Bit 3 (8): maybe_deoptimized
+ */
+export const optStatusNames: Record<number, string> = {
+  1: "interpreted",
+  129: "sparkplug", // 1 + 128
+  17: "turbofan", // 1 + 16
+  33: "maglev", // 1 + 32
+  49: "turbofan+maglev", // 1 + 16 + 32
+  32769: "optimized", // common optimized status
+};

package/src/runners/MergeBatches.ts

@@ -0,0 +1,123 @@
+import type { GcStats } from "./GcStats.ts";
+import type { MeasuredResults } from "./MeasuredResults.ts";
+import { computeStats } from "./SampleStats.ts";
+
+/** Progress update emitted after each batch run. */
+export interface BatchProgress {
+  batch: number;
+  batches: number;
+  label: "baseline" | "current";
+  elapsed: number;
+}
+
+type SamplesFn = (r: MeasuredResults) => number[] | undefined;
+
+/** Merge multiple batch results, concatenating samples and tracking batch boundaries. */
+export function mergeBatchResults(results: MeasuredResults[]): MeasuredResults {
+  if (results.length === 0) {
+    throw new Error("Cannot merge empty results array");
+  }
+  if (results.length === 1) return { ...results[0], batchOffsets: [0] };
+
+  const allSamples = results.flatMap(r => r.samples);
+  const time = computeStats(allSamples);
+
+  const batchOffsets: number[] = [];
+  const offsetPauses: MeasuredResults["pausePoints"] = [];
+  let offset = 0;
+  for (const r of results) {
+    batchOffsets.push(offset);
+    for (const p of r.pausePoints ?? []) {
+      const sampleIndex = p.sampleIndex + offset;
+      offsetPauses.push({ sampleIndex, durationMs: p.durationMs });
+    }
+    offset += r.samples.length;
+  }
+
+  // last batch as base ==> new MeasuredResults fields get "take last"
+  // semantics by default instead of silently disappearing
+  return {
+    ...results[results.length - 1],
+    name: results[0].name,
+    samples: allSamples,
+    warmupSamples: concatOptional(results, r => r.warmupSamples),
+    allocationSamples: concatOptional(results, r => r.allocationSamples),
+    heapSamples: concatOptional(results, r => r.heapSamples),
+    optSamples: concatOptional(results, r => r.optSamples),
+    time,
+    startTime: results[0].startTime,
+    totalTime: results.reduce((sum, r) => sum + (r.totalTime || 0), 0),
+    pausePoints: offsetPauses.length ? offsetPauses : undefined,
+    batchOffsets,
+    gcStats: mergeGcStats(results),
+  };
+}
+
+/** Sum GcStats across batches, or undefined if none collected. */
+export function mergeGcStats(
+  results: { gcStats?: GcStats }[],
+): GcStats | undefined {
+  const stats = results.map(r => r.gcStats).filter(Boolean) as GcStats[];
+  if (!stats.length) return undefined;
+  const sum = (fn: (s: GcStats) => number | undefined) =>
+    stats.reduce((acc, s) => acc + (fn(s) ?? 0), 0);
+  return {
+    scavenges: sum(s => s.scavenges),
+    markCompacts: sum(s => s.markCompacts),
+    totalCollected: sum(s => s.totalCollected),
+    gcPauseTime: sum(s => s.gcPauseTime),
+    totalAllocated: sum(s => s.totalAllocated) || undefined,
+    totalPromoted: sum(s => s.totalPromoted) || undefined,
+    totalSurvived: sum(s => s.totalSurvived) || undefined,
+  };
+}
+
+/** Run N benchmarks + optional baseline in batched alternation, merge results. */
+export async function runBatched(
+  runners: (() => Promise<MeasuredResults>)[],
+  baseline: (() => Promise<MeasuredResults>) | undefined,
+  batches: number,
+  warmupBatch = false,
+  onProgress?: (p: BatchProgress) => void,
+): Promise<{ results: MeasuredResults[]; baseline?: MeasuredResults }> {
+  const runnerBatches: MeasuredResults[][] = runners.map(() => []);
+  const baselineBatches: MeasuredResults[] = [];
+  const start = performance.now();
+
+  const report = (batch: number, label: BatchProgress["label"]) =>
+    onProgress?.({ batch, batches, label, elapsed: performance.now() - start });
+
+  for (let i = 0; i < batches; i++) {
+    const reverse = i % 2 === 1;
+    // baseline runs before benchmarks on even batches, after on odd (alternation)
+    if (!reverse && baseline) {
+      baselineBatches.push(await baseline());
+      report(i, "baseline");
+    }
+    for (let j = 0; j < runners.length; j++) {
+      runnerBatches[j].push(await runners[j]());
+      report(i, "current");
+    }
+    if (reverse && baseline) {
+      baselineBatches.push(await baseline());
+      report(i, "baseline");
+    }
+  }
+
+  if (!warmupBatch && batches > 1) {
+    for (const b of runnerBatches) b.shift();
+    baselineBatches.shift();
+  }
+
+  const results = runnerBatches.map(b => mergeBatchResults(b));
+  const mergedBaseline = baselineBatches.length
+    ? mergeBatchResults(baselineBatches)
+    : undefined;
+  return { results, baseline: mergedBaseline };
+}
+
+/** Concat optional number arrays across batches. */
+function concatOptional(results: MeasuredResults[], fn: SamplesFn) {
+  const all = results.flatMap(r => fn(r) || []);
+  return all.length ? all : undefined;
+}

package/src/{NodeGC.ts → runners/NodeGC.ts}

@@ -31,8 +31,7 @@ export function analyzeGCEntries(
   let collects = 0;
   const events: GcEvent[] = [];
 
-  gcRecords.forEach(record => {
-    const { duration, startTime } = record;
+  for (const { duration, startTime } of gcRecords) {
     if (startTime < start) {
       before += duration;
     } else if (startTime > end) {
@@ -42,7 +41,7 @@ export function analyzeGCEntries(
       collects++;
       events.push({ offset: startTime - start, duration });
     }
-  });
+  }
   const total = inRun + before + after;
   return { inRun, before, after, total, collects, events };
 }