benchforge 0.1.0

package/src/HtmlDataPrep.ts

@@ -0,0 +1,148 @@
+import type {
+  ReportColumnGroup,
+  ReportGroup,
+  ResultsMapper,
+} from "./BenchmarkReport.ts";
+import type { GitVersion } from "./GitUtils.ts";
+import type {
+  BenchmarkData,
+  DifferenceCI,
+  FormattedStat,
+  GroupData,
+  ReportData,
+} from "./html/index.ts";
+import { bootstrapDifferenceCI } from "./StatisticalUtils.ts";
+
+export interface PrepareHtmlOptions {
+  cliArgs?: Record<string, unknown>;
+  sections?: ResultsMapper[];
+  currentVersion?: GitVersion;
+  baselineVersion?: GitVersion;
+}
+
+/** Find higherIsBetter from first comparable column in sections */
+function findHigherIsBetter(sections?: ResultsMapper[]): boolean {
+  const cols = sections?.flatMap(s => s.columns().flatMap(g => g.columns));
+  return cols?.find(c => c.comparable)?.higherIsBetter ?? false;
+}
+
+/** Flip CI percent for metrics where higher is better (e.g., lines/sec) */
+function flipCI(ci: DifferenceCI): DifferenceCI {
+  return {
+    percent: -ci.percent,
+    ci: [-ci.ci[1], -ci.ci[0]],
+    direction: ci.direction,
+    histogram: ci.histogram?.map(bin => ({ x: -bin.x, count: bin.count })),
+  };
+}
+
+/** Prepare ReportData from benchmark results for HTML rendering */
+export function prepareHtmlData(
+  groups: ReportGroup[],
+  options: PrepareHtmlOptions,
+): ReportData {
+  const { cliArgs, sections, currentVersion, baselineVersion } = options;
+  const higherIsBetter = findHigherIsBetter(sections);
+  return {
+    groups: groups.map(group =>
+      prepareGroupData(group, sections, higherIsBetter),
+    ),
+    metadata: {
+      timestamp: new Date().toISOString(),
+      bencherVersion: process.env.npm_package_version || "unknown",
+      cliArgs,
+      gcTrackingEnabled: cliArgs?.["gc-stats"] === true,
+      currentVersion,
+      baselineVersion,
+    },
+  };
+}
+
+/** @return group data with bootstrap CI comparisons against baseline */
+function prepareGroupData(
+  group: ReportGroup,
+  sections?: ResultsMapper[],
+  higherIsBetter?: boolean,
+): GroupData {
+  const baselineSamples = group.baseline?.measuredResults.samples;
+  return {
+    name: group.name,
+    baseline: group.baseline
+      ? prepareBenchmarkData(group.baseline, sections)
+      : undefined,
+    benchmarks: group.reports.map(report => {
+      const samples = report.measuredResults.samples;
+      const rawCI =
+        baselineSamples && samples
+          ? bootstrapDifferenceCI(baselineSamples, samples)
+          : undefined;
+      const comparisonCI = rawCI && higherIsBetter ? flipCI(rawCI) : rawCI;
+      return { ...prepareBenchmarkData(report, sections), comparisonCI };
+    }),
+  };
+}
+
+/** @return benchmark data with samples, stats, and formatted section values */
+function prepareBenchmarkData(
+  report: {
+    name: string;
+    measuredResults: any;
+    metadata?: Record<string, unknown>;
+  },
+  sections?: ResultsMapper[],
+): Omit<BenchmarkData, "comparisonCI"> {
+  const { measuredResults } = report;
+  return {
+    name: report.name,
+    samples: measuredResults.samples,
+    warmupSamples: measuredResults.warmupSamples,
+    allocationSamples: measuredResults.allocationSamples,
+    heapSamples: measuredResults.heapSamples,
+    gcEvents: measuredResults.nodeGcTime?.events,
+    optSamples: measuredResults.optSamples,
+    pausePoints: measuredResults.pausePoints,
+    stats: measuredResults.time,
+    heapSize: measuredResults.heapSize,
+    sectionStats: sections ? extractSectionStats(report, sections) : undefined,
+  };
+}
+
+/** @return formatted stats from all sections for tooltip display */
+function extractSectionStats(
+  report: { measuredResults: any; metadata?: Record<string, unknown> },
+  sections: ResultsMapper[],
+): FormattedStat[] {
+  return sections.flatMap(section => {
+    const vals = section.extract(report.measuredResults, report.metadata);
+    return section.columns().flatMap(g => formatGroupStats(vals, g));
+  });
+}
+
+/** @return formatted stats for one column group, skipping undefined values */
+function formatGroupStats(
+  values: Record<string, unknown>,
+  group: ReportColumnGroup<Record<string, unknown>>,
+): FormattedStat[] {
+  return group.columns
+    .map(c => formatColumnStat(values, c, group.groupTitle))
+    .filter((s): s is FormattedStat => s !== undefined);
+}
+
+type ColumnLike = {
+  key: string;
+  title: string;
+  formatter?: (v: unknown) => string | null;
+};
+
+/** @return formatted stat for a single column, or undefined if empty/placeholder */
+function formatColumnStat(
+  values: Record<string, unknown>,
+  col: ColumnLike,
+  groupTitle?: string,
+): FormattedStat | undefined {
+  const raw = values[col.key];
+  if (raw === undefined) return undefined;
+  const formatted = col.formatter ? col.formatter(raw) : String(raw);
+  if (!formatted || formatted === "—" || formatted === "") return undefined;
+  return { label: col.title, value: formatted, groupTitle };
+}

package/src/MeasuredResults.ts

@@ -0,0 +1,127 @@
+import type { HeapProfile } from "./heap-sample/HeapSampler.ts";
+import type { NodeGCTime } from "./NodeGC.ts";
+import type { GcStats } from "./runners/GcStats.ts";
+
+/** CPU performance counter stats */
+export interface CpuCounts {
+  instructions?: number;
+  cycles?: number;
+  branchMisses?: number;
+}
+
+/** Benchmark results: times in milliseconds, sizes in kilobytes */
+export interface MeasuredResults {
+  name: string;
+
+  /** Raw execution time samples for custom statistics */
+  samples: number[];
+
+  /** Warmup iteration timings (ms) - captured before gc/settle */
+  warmupSamples?: number[];
+
+  /** Raw allocation samples per iteration (KB) */
+  allocationSamples?: number[];
+
+  /** Heap size per sample (bytes) - used for charts */
+  heapSamples?: number[];
+
+  /** Wall-clock timestamps per sample (μs since process start) - for Perfetto export */
+  timestamps?: number[];
+
+  /** Execution time in milliseconds (measurement overhead excluded by mitata) */
+  time: {
+    min: number;
+    max: number;
+    avg: number;
+    p25?: number;
+    p50: number;
+    p75: number;
+    p95?: number;
+    p99: number;
+    p999: number;
+    cv?: number;
+    mad?: number;
+    outlierRate?: number;
+  };
+
+  /** Heap size increase during test run (kilobytes) */
+  heapSize?: {
+    avg: number;
+    min: number;
+    max: number;
+  };
+
+  /**
+   * Time for explicit gc() call after test execution (milliseconds).
+   * Does not include GC time during test execution.
+   * Only reported by mitata runner.
+   */
+  gcTime?: {
+    avg: number;
+    min: number;
+    max: number;
+  };
+
+  /** CPU counter stats from @mitata/counters (requires root access) */
+  cpu?: CpuCounts;
+
+  /** L1 cache miss rate */
+  cpuCacheMiss?: number;
+
+  /** CPU stall rate (macOS only) */
+  cpuStall?: number;
+
+  /**
+   * Stop-the-world GC time blocking main thread (milliseconds).
+   * Measured via Node's performance hooks when nodeObserveGC is true.
+   * Excludes parallel thread collection time and indirect slowdowns.
+   */
+  nodeGcTime?: NodeGCTime;
+
+  /** Total time spent collecting samples (seconds) */
+  totalTime?: number;
+
+  /** Convergence information for adaptive mode */
+  convergence?: {
+    converged: boolean;
+    confidence: number;
+    reason: string;
+  };
+
+  /** V8 optimization tier tracking (requires --allow-natives-syntax) */
+  optStatus?: OptStatusInfo;
+
+  /** Per-sample V8 optimization status codes (for chart visualization) */
+  optSamples?: number[];
+
+  /** Points where pauses occurred for V8 optimization */
+  pausePoints?: PausePoint[];
+
+  /** GC stats from V8's --trace-gc-nvp (requires --gc-stats and worker mode) */
+  gcStats?: GcStats;
+
+  /** Heap sampling allocation profile (requires --heap-sample and worker mode) */
+  heapProfile?: HeapProfile;
+}
+
+/** A pause point during sample collection for V8 optimization */
+export interface PausePoint {
+  /** Sample index where pause occurred (after this iteration) */
+  sampleIndex: number;
+  /** Pause duration in milliseconds */
+  durationMs: number;
+}
+
+/** V8 optimization tier distribution */
+export interface OptTierInfo {
+  count: number;
+  medianMs: number;
+}
+
+/** V8 optimization status summary */
+export interface OptStatusInfo {
+  /** Samples by tier name (e.g., "turbofan", "sparkplug") */
+  byTier: Record<string, OptTierInfo>;
+  /** Number of samples with deopt flag set */
+  deoptCount: number;
+}

package/src/NodeGC.ts

@@ -0,0 +1,48 @@
+import type { PerformanceEntry } from "node:perf_hooks";
+
+/** Individual GC event for visualization */
+export interface GcEvent {
+  /** Offset from collection start (ms) - can be negative for warmup GCs */
+  offset: number;
+  /** Duration of GC pause (ms) */
+  duration: number;
+}
+
+/** GC time measured by Node's performance hooks */
+export interface NodeGCTime {
+  inRun: number;
+  before: number;
+  after: number;
+  total: number;
+  collects: number;
+  /** Individual GC events during sample collection (for visualization) */
+  events: GcEvent[];
+}
+
+/** Correlate GC events with benchmark timing */
+export function analyzeGCEntries(
+  gcRecords: PerformanceEntry[],
+  benchTime: [number, number],
+): NodeGCTime {
+  const [start, end] = benchTime;
+  let inRun = 0;
+  let before = 0;
+  let after = 0;
+  let collects = 0;
+  const events: GcEvent[] = [];
+
+  gcRecords.forEach(record => {
+    const { duration, startTime } = record;
+    if (startTime < start) {
+      before += duration;
+    } else if (startTime > end) {
+      after += duration;
+    } else {
+      inRun += duration;
+      collects++;
+      events.push({ offset: startTime - start, duration });
+    }
+  });
+  const total = inRun + before + after;
+  return { inRun, before, after, total, collects, events };
+}

package/src/PermutationTest.ts

@@ -0,0 +1,115 @@
+/**
+ * Permutation-based hypothesis testing for benchmark comparisons.
+ *
+ * Currently unused - the main reporting pipeline uses bootstrapDifferenceCI()
+ * from StatisticalUtils.ts instead, which provides confidence intervals on
+ * the difference rather than p-values.
+ *
+ * Kept for potential future use cases where p-values are needed.
+ */
+
+import { average, percentile } from "./StatisticalUtils.ts";
+
+const significanceThreshold = 0.05;
+const strongSignificance = 0.001;
+const goodSignificance = 0.01;
+const defaultBootstrapSamples = 10000;
+
+/** Statistical comparison between baseline and current benchmark samples */
+export interface ComparisonResult {
+  baselineMedian: number;
+  currentMedian: number;
+  baselineMean: number;
+  currentMean: number;
+
+  medianChange: {
+    absolute: number;
+    percent: number;
+    pValue: number;
+    significant: boolean;
+    significance: "strong" | "good" | "weak" | "none";
+  };
+
+  meanChange: {
+    absolute: number;
+    percent: number;
+    pValue: number;
+    significant: boolean;
+    significance: "strong" | "good" | "weak" | "none";
+  };
+}
+
+/** @return statistical comparison between baseline and current samples */
+export function compareWithBaseline(
+  baseline: number[],
+  current: number[],
+): ComparisonResult {
+  const baselineMedian = percentile(baseline, 0.5);
+  const currentMedian = percentile(current, 0.5);
+  const baselineMean = average(baseline);
+  const currentMean = average(current);
+
+  const median = (s: number[]) => percentile(s, 0.5);
+  const medianPValue = bootstrapDifferenceTest(baseline, current, median);
+  const meanPValue = bootstrapDifferenceTest(baseline, current, average);
+
+  return {
+    baselineMedian,
+    currentMedian,
+    baselineMean,
+    currentMean,
+    medianChange: changeStats(currentMedian, baselineMedian, medianPValue),
+    meanChange: changeStats(currentMean, baselineMean, meanPValue),
+  };
+}
+
+/** @return change statistics for a current vs baseline comparison */
+function changeStats(current: number, base: number, pValue: number) {
+  return {
+    absolute: current - base,
+    percent: ((current - base) / base) * 100,
+    pValue,
+    significant: pValue < significanceThreshold,
+    significance: getSignificance(pValue),
+  };
+}
+
+/** @return significance level based on p-value thresholds */
+function getSignificance(pValue: number): "strong" | "good" | "weak" | "none" {
+  if (pValue < strongSignificance) return "strong";
+  if (pValue < goodSignificance) return "good";
+  if (pValue < significanceThreshold) return "weak";
+  return "none";
+}
+
+/** @return p-value from permutation test for difference in statistics */
+function bootstrapDifferenceTest(
+  sample1: number[],
+  sample2: number[],
+  statistic: (samples: number[]) => number,
+): number {
+  const observedDiff = statistic(sample2) - statistic(sample1);
+  const combined = [...sample1, ...sample2];
+  const n1 = sample1.length;
+
+  let moreExtreme = 0;
+  for (let i = 0; i < defaultBootstrapSamples; i++) {
+    const { resample1, resample2 } = shuffleAndSplit(combined, n1);
+    const diff = statistic(resample2) - statistic(resample1);
+    if (Math.abs(diff) >= Math.abs(observedDiff)) moreExtreme++;
+  }
+  return moreExtreme / defaultBootstrapSamples;
+}
+
+/** @return randomly shuffled samples split at n1 (Fisher-Yates shuffle) */
+function shuffleAndSplit(combined: number[], n1: number) {
+  const shuffled = [...combined];
+  for (let i = shuffled.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+  }
+  return {
+    resample1: shuffled.slice(0, n1),
+    resample2: shuffled.slice(n1),
+  };
+}

package/src/StandardSections.ts

@@ -0,0 +1,268 @@
+import type { ReportColumnGroup, ResultsMapper } from "./BenchmarkReport.ts";
+import type { MeasuredResults } from "./MeasuredResults.ts";
+import { formatConvergence } from "./table-util/ConvergenceFormatters.ts";
+import {
+  formatBytes,
+  integer,
+  percent,
+  percentPrecision,
+  timeMs,
+} from "./table-util/Formatters.ts";
+
+export interface TimeStats {
+  mean?: number;
+  p50?: number;
+  p99?: number;
+  convergence?: number;
+}
+
+/** Section: mean, p50, p99 timing with convergence */
+export const timeSection: ResultsMapper<TimeStats> = {
+  extract: (results: MeasuredResults) => ({
+    mean: results.time?.avg,
+    p50: results.time?.p50,
+    p99: results.time?.p99,
+    convergence: results.convergence?.confidence,
+  }),
+  columns: (): ReportColumnGroup<TimeStats>[] => [
+    {
+      groupTitle: "time",
+      columns: [
+        { key: "mean", title: "mean", formatter: timeMs, comparable: true },
+        { key: "p50", title: "p50", formatter: timeMs, comparable: true },
+        { key: "p99", title: "p99", formatter: timeMs, comparable: true },
+      ],
+    },
+    {
+      columns: [
+        { key: "convergence", title: "conv%", formatter: formatConvergence },
+      ],
+    },
+  ],
+};
+
+export interface GcSectionStats {
+  gc?: number; // GC time as fraction of total bench time
+}
+
+/** Section: GC time as fraction of total benchmark time (Node performance hooks) */
+export const gcSection: ResultsMapper<GcSectionStats> = {
+  extract: (results: MeasuredResults) => {
+    const { nodeGcTime, time, samples } = results;
+    if (!nodeGcTime || !time?.avg) return { gc: undefined };
+    const totalBenchTime = time.avg * samples.length;
+    if (totalBenchTime <= 0) return { gc: undefined };
+    const gcTime = nodeGcTime.inRun / totalBenchTime;
+    // GC time can't exceed total time
+    return { gc: gcTime <= 1 ? gcTime : undefined };
+  },
+  columns: (): ReportColumnGroup<GcSectionStats>[] => [
+    {
+      groupTitle: "gc",
+      columns: [
+        { key: "gc", title: "mean", formatter: percent, comparable: true },
+      ],
+    },
+  ],
+};
+
+export interface GcStatsInfo {
+  allocPerIter?: number;
+  collected?: number;
+  scavenges?: number;
+  fullGCs?: number;
+  promoPercent?: number;
+  pausePerIter?: number;
+}
+
+/** Section: detailed GC stats from --trace-gc-nvp (allocation, promotion, pauses) */
+export const gcStatsSection: ResultsMapper<GcStatsInfo> = {
+  extract: (results: MeasuredResults) => {
+    const { gcStats, samples } = results;
+    if (!gcStats) return {};
+    const iterations = samples.length || 1;
+    const { totalAllocated, totalPromoted } = gcStats;
+    const hasAlloc = totalAllocated && totalAllocated > 0;
+    const promoPercent = hasAlloc
+      ? (totalPromoted ?? 0) / totalAllocated
+      : undefined;
+    return {
+      allocPerIter:
+        totalAllocated != null ? totalAllocated / iterations : undefined,
+      collected: gcStats.totalCollected || undefined,
+      scavenges: gcStats.scavenges,
+      fullGCs: gcStats.markCompacts,
+      promoPercent,
+      pausePerIter: gcStats.gcPauseTime / iterations,
+    };
+  },
+  columns: (): ReportColumnGroup<GcStatsInfo>[] => [
+    {
+      groupTitle: "gc",
+      columns: [
+        { key: "allocPerIter", title: "alloc/iter", formatter: formatBytes },
+        { key: "collected", title: "collected", formatter: formatBytes },
+        { key: "scavenges", title: "scav", formatter: integer },
+        { key: "fullGCs", title: "full", formatter: integer },
+        { key: "promoPercent", title: "promo%", formatter: percent },
+        { key: "pausePerIter", title: "pause/iter", formatter: timeMs },
+      ],
+    },
+  ],
+};
+
+/** Browser GC section: only fields available from CDP tracing */
+export const browserGcStatsSection: ResultsMapper<GcStatsInfo> = {
+  extract: gcStatsSection.extract,
+  columns: (): ReportColumnGroup<GcStatsInfo>[] => [
+    {
+      groupTitle: "gc",
+      columns: [
+        { key: "collected", title: "collected", formatter: formatBytes },
+        { key: "scavenges", title: "scav", formatter: integer },
+        { key: "fullGCs", title: "full", formatter: integer },
+        { key: "pausePerIter", title: "pause", formatter: timeMs },
+      ],
+    },
+  ],
+};
+
+export interface CpuStats {
+  cpuCacheMiss?: number;
+  cpuStall?: number;
+}
+
+/** Section: CPU L1 cache miss rate and stall rate (requires @mitata/counters) */
+export const cpuSection: ResultsMapper<CpuStats> = {
+  extract: (results: MeasuredResults) => ({
+    cpuCacheMiss: results.cpuCacheMiss,
+    cpuStall: results.cpuStall,
+  }),
+  columns: (): ReportColumnGroup<CpuStats>[] => [
+    {
+      groupTitle: "cpu",
+      columns: [
+        { key: "cpuCacheMiss", title: "L1 miss", formatter: percent },
+        { key: "cpuStall", title: "stalls", formatter: percentPrecision(2) },
+      ],
+    },
+  ],
+};
+
+export interface RunStats {
+  runs?: number;
+}
+
+/** Section: number of sample iterations */
+export const runsSection: ResultsMapper<RunStats> = {
+  extract: (results: MeasuredResults) => ({
+    runs: results.samples.length,
+  }),
+  columns: (): ReportColumnGroup<RunStats>[] => [
+    { columns: [{ key: "runs", title: "runs", formatter: integer }] },
+  ],
+};
+
+/** Section: total sampling duration in seconds (brackets if >= 30s) */
+export const totalTimeSection: ResultsMapper<{ totalTime?: number }> = {
+  extract: (results: MeasuredResults) => ({
+    totalTime: results.totalTime,
+  }),
+  columns: (): ReportColumnGroup<{ totalTime?: number }>[] => [
+    {
+      columns: [
+        {
+          key: "totalTime",
+          title: "time",
+          formatter: v => {
+            if (typeof v !== "number") return "";
+            return v >= 30 ? `[${v.toFixed(1)}s]` : `${v.toFixed(1)}s`;
+          },
+        },
+      ],
+    },
+  ],
+};
+
+export interface AdaptiveStats {
+  median?: number;
+  mean?: number;
+  p99?: number;
+  convergence?: number;
+}
+
+/** Section: median, mean, p99, and convergence for adaptive mode */
+export const adaptiveSection: ResultsMapper<AdaptiveStats> = {
+  extract: (results: MeasuredResults) => ({
+    median: results.time?.p50,
+    mean: results.time?.avg,
+    p99: results.time?.p99,
+    convergence: results.convergence?.confidence,
+  }),
+  columns: (): ReportColumnGroup<AdaptiveStats>[] => [
+    {
+      groupTitle: "time",
+      columns: [
+        { key: "median", title: "median", formatter: timeMs, comparable: true },
+        { key: "mean", title: "mean", formatter: timeMs, comparable: true },
+        { key: "p99", title: "p99", formatter: timeMs },
+      ],
+    },
+    {
+      columns: [
+        { key: "convergence", title: "conv%", formatter: formatConvergence },
+      ],
+    },
+  ],
+};
+
+/** Build generic sections based on CLI flags */
+export function buildGenericSections(args: {
+  "gc-stats"?: boolean;
+  "heap-sample"?: boolean;
+}): ResultsMapper[] {
+  const sections: ResultsMapper[] = [];
+  if (args["gc-stats"]) sections.push(gcStatsSection);
+  sections.push(runsSection);
+  return sections;
+}
+
+export interface OptStats {
+  tiers?: string; // tier distribution summary
+  deopt?: number; // deopt count
+}
+
+/** Section: V8 optimization tier distribution and deopt count */
+export const optSection: ResultsMapper<OptStats> = {
+  extract: (results: MeasuredResults) => {
+    const opt = results.optStatus;
+    if (!opt) return {};
+
+    const total = Object.values(opt.byTier).reduce((s, t) => s + t.count, 0);
+    const tierParts = Object.entries(opt.byTier)
+      .sort((a, b) => b[1].count - a[1].count)
+      .map(([name, t]) => `${name}:${((t.count / total) * 100).toFixed(0)}%`);
+
+    return {
+      tiers: tierParts.join(" "),
+      deopt: opt.deoptCount > 0 ? opt.deoptCount : undefined,
+    };
+  },
+  columns: (): ReportColumnGroup<OptStats>[] => [
+    {
+      groupTitle: "v8 opt",
+      columns: [
+        {
+          key: "tiers",
+          title: "tiers",
+          formatter: v => (typeof v === "string" ? v : ""),
+        },
+        {
+          key: "deopt",
+          title: "deopt",
+          formatter: v => (typeof v === "number" ? String(v) : ""),
+        },
+      ],
+    },
+  ],
+};