benchforge 0.1.8 → 0.1.11

package/src/BenchMatrix.ts

@@ -74,13 +74,6 @@ export interface MatrixResults {
   variants: VariantResult[];
 }
 
-/** @return true if variant is a StatefulVariant (has setup + run) */
-export function isStatefulVariant<T, S>(
-  v: Variant<T, S>,
-): v is StatefulVariant<T, S> {
-  return typeof v === "object" && "setup" in v && "run" in v;
-}
-
 /** Options for runMatrix */
 export interface RunMatrixOptions {
   iterations?: number;
@@ -103,6 +96,22 @@ export interface RunMatrixOptions {
   heapDepth?: number;
 }
 
+/** Context for running matrix benchmarks in worker mode */
+interface DirMatrixContext<T> {
+  matrix: BenchMatrix<T>;
+  casesModule?: import("./matrix/CaseLoader.ts").CasesModule<T>;
+  baselineIds: string[];
+  caseIds: string[];
+  runnerOpts: RunnerOptions;
+}
+
+/** @return true if variant is a StatefulVariant (has setup + run) */
+export function isStatefulVariant<T, S>(
+  v: Variant<T, S>,
+): v is StatefulVariant<T, S> {
+  return typeof v === "object" && "setup" in v && "run" in v;
+}
+
 /** Run a BenchMatrix with inline variants or variantDir */
 export async function runMatrix<T>(
   matrix: BenchMatrix<T>,
@@ -127,36 +136,24 @@ function validateBaseline<T>(matrix: BenchMatrix<T>): void {
   if (matrix.baselineDir && matrix.baselineVariant) throw new Error(msg);
 }
 
-function buildRunnerOptions(options: RunMatrixOptions): RunnerOptions {
-  return {
-    maxIterations: options.iterations,
-    maxTime: options.maxTime ?? 1000,
-    warmup: options.warmup ?? 0,
-    collect: options.collect,
-    cpuCounters: options.cpuCounters,
-    traceOpt: options.traceOpt,
-    noSettle: options.noSettle,
-    pauseFirst: options.pauseFirst,
-    pauseInterval: options.pauseInterval,
-    pauseDuration: options.pauseDuration,
-    gcStats: options.gcStats,
-    heapSample: options.heapSample,
-    heapInterval: options.heapInterval,
-    heapDepth: options.heapDepth,
-  };
-}
-
-/** Load cases module and resolve filtered case IDs */
-async function resolveCases<T>(
+/** Run matrix with variantDir (worker mode for memory isolation) */
+async function runMatrixWithDir<T>(
   matrix: BenchMatrix<T>,
   options: RunMatrixOptions,
-) {
-  const casesModule = matrix.casesModule
-    ? await loadCasesModule<T>(matrix.casesModule)
-    : undefined;
-  const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
-  const caseIds = options.filteredCases ?? allCaseIds;
-  return { casesModule, caseIds };
+): Promise<MatrixResults> {
+  const allVariantIds = await discoverVariants(matrix.variantDir!);
+  if (allVariantIds.length === 0) {
+    throw new Error(`No variants found in ${matrix.variantDir}`);
+  }
+  const variantIds = options.filteredVariants ?? allVariantIds;
+
+  const ctx = await createDirContext(matrix, options);
+  const variants = await runDirVariants(variantIds, ctx);
+
+  if (matrix.baselineVariant) {
+    applyBaselineVariant(variants, matrix.baselineVariant);
+  }
+  return { name: matrix.name, variants };
 }
 
 /** Run matrix with inline variants (non-worker mode) */
@@ -205,35 +202,6 @@ async function runMatrixInline<T>(
   return { name: matrix.name, variants };
 }
 
-/** Context for running matrix benchmarks in worker mode */
-interface DirMatrixContext<T> {
-  matrix: BenchMatrix<T>;
-  casesModule?: import("./matrix/CaseLoader.ts").CasesModule<T>;
-  baselineIds: string[];
-  caseIds: string[];
-  runnerOpts: RunnerOptions;
-}
-
-/** Run matrix with variantDir (worker mode for memory isolation) */
-async function runMatrixWithDir<T>(
-  matrix: BenchMatrix<T>,
-  options: RunMatrixOptions,
-): Promise<MatrixResults> {
-  const allVariantIds = await discoverVariants(matrix.variantDir!);
-  if (allVariantIds.length === 0) {
-    throw new Error(`No variants found in ${matrix.variantDir}`);
-  }
-  const variantIds = options.filteredVariants ?? allVariantIds;
-
-  const ctx = await createDirContext(matrix, options);
-  const variants = await runDirVariants(variantIds, ctx);
-
-  if (matrix.baselineVariant) {
-    applyBaselineVariant(variants, matrix.baselineVariant);
-  }
-  return { name: matrix.name, variants };
-}
-
 /** Create context for directory-based matrix execution */
 async function createDirContext<T>(
   matrix: BenchMatrix<T>,
@@ -260,6 +228,89 @@ async function runDirVariants<T>(
   return variants;
 }
 
+/** Apply baselineVariant comparison - one variant is the reference for all others */
+function applyBaselineVariant(
+  variants: VariantResult[],
+  baselineVariantId: string,
+): void {
+  const baselineVariant = variants.find(v => v.id === baselineVariantId);
+  if (!baselineVariant) return;
+
+  const baselineByCase = new Map<string, MeasuredResults>();
+  for (const c of baselineVariant.cases) {
+    baselineByCase.set(c.caseId, c.measured);
+  }
+
+  for (const variant of variants) {
+    if (variant.id === baselineVariantId) continue;
+    for (const caseResult of variant.cases) {
+      const baseline = baselineByCase.get(caseResult.caseId);
+      if (baseline) {
+        caseResult.baseline = baseline;
+        caseResult.deltaPercent = computeDeltaPercent(
+          baseline,
+          caseResult.measured,
+        );
+      }
+    }
+  }
+}
+
+/** Load cases module and resolve filtered case IDs */
+async function resolveCases<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+) {
+  const casesModule = matrix.casesModule
+    ? await loadCasesModule<T>(matrix.casesModule)
+    : undefined;
+  const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
+  const caseIds = options.filteredCases ?? allCaseIds;
+  return { casesModule, caseIds };
+}
+
+function buildRunnerOptions(options: RunMatrixOptions): RunnerOptions {
+  return {
+    maxIterations: options.iterations,
+    maxTime: options.maxTime ?? 1000,
+    warmup: options.warmup ?? 0,
+    collect: options.collect,
+    cpuCounters: options.cpuCounters,
+    traceOpt: options.traceOpt,
+    noSettle: options.noSettle,
+    pauseFirst: options.pauseFirst,
+    pauseInterval: options.pauseInterval,
+    pauseDuration: options.pauseDuration,
+    gcStats: options.gcStats,
+    heapSample: options.heapSample,
+    heapInterval: options.heapInterval,
+    heapDepth: options.heapDepth,
+  };
+}
+
+/** Run a single variant with case data */
+async function runVariant<T>(
+  variant: AnyVariant<T>,
+  caseData: T,
+  name: string,
+  runner: BasicRunner,
+  options: RunnerOptions,
+): Promise<MeasuredResults> {
+  if (isStatefulVariant(variant)) {
+    const state = await variant.setup(caseData);
+    const [result] = await runner.runBench(
+      { name, fn: () => variant.run(state) },
+      options,
+    );
+    return result;
+  }
+  const [result] = await runner.runBench(
+    { name, fn: () => variant(caseData) },
+    options,
+  );
+  return result;
+}
+
 /** Run all cases for a single variant */
 async function runDirVariantCases<T>(
   variantId: string,
@@ -296,6 +347,16 @@ async function runDirVariantCases<T>(
   return cases;
 }
 
+/** Compute delta percentage: (current - baseline) / baseline * 100 */
+function computeDeltaPercent(
+  baseline: MeasuredResults,
+  current: MeasuredResults,
+): number {
+  const baseAvg = average(baseline.samples);
+  if (baseAvg === 0) return 0;
+  return ((average(current.samples) - baseAvg) / baseAvg) * 100;
+}
+
 /** Run baseline variant if it exists in baselineDir */
 async function runBaselineIfExists<T>(
   variantId: string,
@@ -317,64 +378,3 @@ async function runBaselineIfExists<T>(
   });
   return measured;
 }
-
-/** Compute delta percentage: (current - baseline) / baseline * 100 */
-function computeDeltaPercent(
-  baseline: MeasuredResults,
-  current: MeasuredResults,
-): number {
-  const baseAvg = average(baseline.samples);
-  if (baseAvg === 0) return 0;
-  return ((average(current.samples) - baseAvg) / baseAvg) * 100;
-}
-
-/** Apply baselineVariant comparison - one variant is the reference for all others */
-function applyBaselineVariant(
-  variants: VariantResult[],
-  baselineVariantId: string,
-): void {
-  const baselineVariant = variants.find(v => v.id === baselineVariantId);
-  if (!baselineVariant) return;
-
-  const baselineByCase = new Map<string, MeasuredResults>();
-  for (const c of baselineVariant.cases) {
-    baselineByCase.set(c.caseId, c.measured);
-  }
-
-  for (const variant of variants) {
-    if (variant.id === baselineVariantId) continue;
-    for (const caseResult of variant.cases) {
-      const baseline = baselineByCase.get(caseResult.caseId);
-      if (baseline) {
-        caseResult.baseline = baseline;
-        caseResult.deltaPercent = computeDeltaPercent(
-          baseline,
-          caseResult.measured,
-        );
-      }
-    }
-  }
-}
-
-/** Run a single variant with case data */
-async function runVariant<T>(
-  variant: AnyVariant<T>,
-  caseData: T,
-  name: string,
-  runner: BasicRunner,
-  options: RunnerOptions,
-): Promise<MeasuredResults> {
-  if (isStatefulVariant(variant)) {
-    const state = await variant.setup(caseData);
-    const [result] = await runner.runBench(
-      { name, fn: () => variant.run(state) },
-      options,
-    );
-    return result;
-  }
-  const [result] = await runner.runBench(
-    { name, fn: () => variant(caseData) },
-    options,
-  );
-  return result;
-}

package/src/BenchmarkReport.ts

@@ -58,6 +58,11 @@ interface ReportRowBase {
 type ReportRowData<S extends ReadonlyArray<ResultsMapper<any>>> =
   ReportRowBase & UnionToIntersection<SectionStats<S[number]>>;
 
+/** All reports in a group, including the baseline if present */
+export function groupReports(group: ReportGroup): BenchmarkReport[] {
+  return group.baseline ? [...group.reports, group.baseline] : group.reports;
+}
+
 /** @return formatted table report with optional baseline comparisons */
 export function reportResults<S extends ReadonlyArray<ResultsMapper<any>>>(
   groups: ReportGroup[],
@@ -68,6 +73,41 @@ export function reportResults<S extends ReadonlyArray<ResultsMapper<any>>>(
   return buildTable(createColumnGroups(sections, hasBaseline), results);
 }
 
+/** @return rows with stats from sections */
+export function valuesForReports<S extends ReadonlyArray<ResultsMapper<any>>>(
+  reports: BenchmarkReport[],
+  sections: S,
+): ReportRowData<S>[] {
+  return reports.map(report => ({
+    name: truncate(report.name),
+    ...extractReportValues(report, sections),
+  })) as ReportRowData<S>[];
+}
+
+/** @return groups with single CI column after first comparable field */
+export function injectDiffColumns<T>(
+  reportGroups: ReportColumnGroup<T>[],
+): ColumnGroup<T>[] {
+  let ciAdded = false;
+
+  return reportGroups.map(group => ({
+    groupTitle: group.groupTitle,
+    columns: group.columns.flatMap(col => {
+      if (col.comparable && !ciAdded) {
+        ciAdded = true;
+        const fmt = col.higherIsBetter
+          ? formatDiffWithCIHigherIsBetter
+          : formatDiffWithCI;
+        return [
+          col,
+          { title: "Δ% CI", key: "diffCI" as keyof T, formatter: fmt },
+        ];
+      }
+      return [col];
+    }),
+  }));
+}
+
 /** @return values for report group */
 function resultGroupValues<S extends ReadonlyArray<ResultsMapper<any>>>(
   group: ReportGroup,
@@ -95,29 +135,6 @@ function resultGroupValues<S extends ReadonlyArray<ResultsMapper<any>>>(
   return { results, baseline: baselineRow };
 }
 
-/** @return rows with stats from sections */
-export function valuesForReports<S extends ReadonlyArray<ResultsMapper<any>>>(
-  reports: BenchmarkReport[],
-  sections: S,
-): ReportRowData<S>[] {
-  return reports.map(report => ({
-    name: truncate(report.name),
-    ...extractReportValues(report, sections),
-  })) as ReportRowData<S>[];
-}
-
-/** @return merged statistics from all sections */
-function extractReportValues(
-  report: BenchmarkReport,
-  sections: ReadonlyArray<ResultsMapper<any>>,
-): UnknownRecord {
-  const { measuredResults, metadata } = report;
-  const entries = sections.flatMap(s =>
-    Object.entries(s.extract(measuredResults, metadata)),
-  );
-  return Object.fromEntries(entries);
-}
-
 /** @return column groups with diff columns if baseline exists */
 function createColumnGroups<S extends ReadonlyArray<ResultsMapper<any>>>(
   sections: S,
@@ -131,26 +148,14 @@ function createColumnGroups<S extends ReadonlyArray<ResultsMapper<any>>>(
   return [nameColumn, ...(hasBaseline ? injectDiffColumns(groups) : groups)];
 }
 
-/** @return groups with single CI column after first comparable field */
-export function injectDiffColumns<T>(
-  reportGroups: ReportColumnGroup<T>[],
-): ColumnGroup<T>[] {
-  let ciAdded = false;
-
-  return reportGroups.map(group => ({
-    groupTitle: group.groupTitle,
-    columns: group.columns.flatMap(col => {
-      if (col.comparable && !ciAdded) {
-        ciAdded = true;
-        const fmt = col.higherIsBetter
-          ? formatDiffWithCIHigherIsBetter
-          : formatDiffWithCI;
-        return [
-          col,
-          { title: "Δ% CI", key: "diffCI" as keyof T, formatter: fmt },
-        ];
-      }
-      return [col];
-    }),
-  }));
+/** @return merged statistics from all sections */
+function extractReportValues(
+  report: BenchmarkReport,
+  sections: ReadonlyArray<ResultsMapper<any>>,
+): UnknownRecord {
+  const { measuredResults, metadata } = report;
+  const entries = sections.flatMap(s =>
+    Object.entries(s.extract(measuredResults, metadata)),
+  );
+  return Object.fromEntries(entries);
 }

package/src/HtmlDataPrep.ts

@@ -20,21 +20,11 @@ export interface PrepareHtmlOptions {
   baselineVersion?: GitVersion;
 }
 
-/** Find higherIsBetter from first comparable column in sections */
-function findHigherIsBetter(sections?: ResultsMapper[]): boolean {
-  const cols = sections?.flatMap(s => s.columns().flatMap(g => g.columns));
-  return cols?.find(c => c.comparable)?.higherIsBetter ?? false;
-}
-
-/** Flip CI percent for metrics where higher is better (e.g., lines/sec) */
-function flipCI(ci: DifferenceCI): DifferenceCI {
-  return {
-    percent: -ci.percent,
-    ci: [-ci.ci[1], -ci.ci[0]],
-    direction: ci.direction,
-    histogram: ci.histogram?.map(bin => ({ x: -bin.x, count: bin.count })),
-  };
-}
+type ColumnLike = {
+  key: string;
+  title: string;
+  formatter?: (v: unknown) => string | null;
+};
 
 /** Prepare ReportData from benchmark results for HTML rendering */
 export function prepareHtmlData(
@@ -58,6 +48,12 @@ export function prepareHtmlData(
   };
 }
 
+/** Find higherIsBetter from first comparable column in sections */
+function findHigherIsBetter(sections?: ResultsMapper[]): boolean {
+  const cols = sections?.flatMap(s => s.columns().flatMap(g => g.columns));
+  return cols?.find(c => c.comparable)?.higherIsBetter ?? false;
+}
+
 /** @return group data with bootstrap CI comparisons against baseline */
 function prepareGroupData(
   group: ReportGroup,
@@ -107,6 +103,16 @@ function prepareBenchmarkData(
   };
 }
 
+/** Flip CI percent for metrics where higher is better (e.g., lines/sec) */
+function flipCI(ci: DifferenceCI): DifferenceCI {
+  return {
+    percent: -ci.percent,
+    ci: [-ci.ci[1], -ci.ci[0]],
+    direction: ci.direction,
+    histogram: ci.histogram?.map(bin => ({ x: -bin.x, count: bin.count })),
+  };
+}
+
 /** @return formatted stats from all sections for tooltip display */
 function extractSectionStats(
   report: { measuredResults: any; metadata?: Record<string, unknown> },
@@ -128,12 +134,6 @@ function formatGroupStats(
     .filter((s): s is FormattedStat => s !== undefined);
 }
 
-type ColumnLike = {
-  key: string;
-  title: string;
-  formatter?: (v: unknown) => string | null;
-};
-
 /** @return formatted stat for a single column, or undefined if empty/placeholder */
 function formatColumnStat(
   values: Record<string, unknown>,

package/src/PermutationTest.ts

@@ -10,11 +10,6 @@
 
 import { average, percentile } from "./StatisticalUtils.ts";
 
-const significanceThreshold = 0.05;
-const strongSignificance = 0.001;
-const goodSignificance = 0.01;
-const defaultBootstrapSamples = 10000;
-
 /** Statistical comparison between baseline and current benchmark samples */
 export interface ComparisonResult {
   baselineMedian: number;
@@ -39,6 +34,11 @@ export interface ComparisonResult {
   };
 }
 
+const significanceThreshold = 0.05;
+const strongSignificance = 0.001;
+const goodSignificance = 0.01;
+const defaultBootstrapSamples = 10000;
+
 /** @return statistical comparison between baseline and current samples */
 export function compareWithBaseline(
   baseline: number[],
@@ -63,25 +63,6 @@ export function compareWithBaseline(
   };
 }
 
-/** @return change statistics for a current vs baseline comparison */
-function changeStats(current: number, base: number, pValue: number) {
-  return {
-    absolute: current - base,
-    percent: ((current - base) / base) * 100,
-    pValue,
-    significant: pValue < significanceThreshold,
-    significance: getSignificance(pValue),
-  };
-}
-
-/** @return significance level based on p-value thresholds */
-function getSignificance(pValue: number): "strong" | "good" | "weak" | "none" {
-  if (pValue < strongSignificance) return "strong";
-  if (pValue < goodSignificance) return "good";
-  if (pValue < significanceThreshold) return "weak";
-  return "none";
-}
-
 /** @return p-value from permutation test for difference in statistics */
 function bootstrapDifferenceTest(
   sample1: number[],
@@ -101,6 +82,17 @@ function bootstrapDifferenceTest(
   return moreExtreme / defaultBootstrapSamples;
 }
 
+/** @return change statistics for a current vs baseline comparison */
+function changeStats(current: number, base: number, pValue: number) {
+  return {
+    absolute: current - base,
+    percent: ((current - base) / base) * 100,
+    pValue,
+    significant: pValue < significanceThreshold,
+    significance: getSignificance(pValue),
+  };
+}
+
 /** @return randomly shuffled samples split at n1 (Fisher-Yates shuffle) */
 function shuffleAndSplit(combined: number[], n1: number) {
   const shuffled = [...combined];
@@ -113,3 +105,11 @@ function shuffleAndSplit(combined: number[], n1: number) {
     resample2: shuffled.slice(n1),
   };
 }
+
+/** @return significance level based on p-value thresholds */
+function getSignificance(pValue: number): "strong" | "good" | "weak" | "none" {
+  if (pValue < strongSignificance) return "strong";
+  if (pValue < goodSignificance) return "good";
+  if (pValue < significanceThreshold) return "weak";
+  return "none";
+}

package/src/StandardSections.ts

@@ -15,6 +15,40 @@ export interface TimeStats {
   p99?: number;
 }
 
+export interface GcSectionStats {
+  gc?: number; // GC time as fraction of total bench time
+}
+
+export interface GcStatsInfo {
+  allocPerIter?: number;
+  collected?: number;
+  scavenges?: number;
+  fullGCs?: number;
+  promoPercent?: number;
+  pausePerIter?: number;
+}
+
+export interface CpuStats {
+  cpuCacheMiss?: number;
+  cpuStall?: number;
+}
+
+export interface RunStats {
+  runs?: number;
+}
+
+export interface AdaptiveStats {
+  median?: number;
+  mean?: number;
+  p99?: number;
+  convergence?: number;
+}
+
+export interface OptStats {
+  tiers?: string; // tier distribution summary
+  deopt?: number; // deopt count
+}
+
 /** Section: mean, p50, p99 timing */
 export const timeSection: ResultsMapper<TimeStats> = {
   extract: (results: MeasuredResults) => ({
@@ -34,10 +68,6 @@ export const timeSection: ResultsMapper<TimeStats> = {
   ],
 };
 
-export interface GcSectionStats {
-  gc?: number; // GC time as fraction of total bench time
-}
-
 /** Section: GC time as fraction of total benchmark time (Node performance hooks) */
 export const gcSection: ResultsMapper<GcSectionStats> = {
   extract: (results: MeasuredResults) => {
@@ -59,15 +89,6 @@ export const gcSection: ResultsMapper<GcSectionStats> = {
   ],
 };
 
-export interface GcStatsInfo {
-  allocPerIter?: number;
-  collected?: number;
-  scavenges?: number;
-  fullGCs?: number;
-  promoPercent?: number;
-  pausePerIter?: number;
-}
-
 /** Section: detailed GC stats from --trace-gc-nvp (allocation, promotion, pauses) */
 export const gcStatsSection: ResultsMapper<GcStatsInfo> = {
   extract: (results: MeasuredResults) => {
@@ -120,11 +141,6 @@ export const browserGcStatsSection: ResultsMapper<GcStatsInfo> = {
   ],
 };
 
-export interface CpuStats {
-  cpuCacheMiss?: number;
-  cpuStall?: number;
-}
-
 /** Section: CPU L1 cache miss rate and stall rate (requires @mitata/counters) */
 export const cpuSection: ResultsMapper<CpuStats> = {
   extract: (results: MeasuredResults) => ({
@@ -142,10 +158,6 @@ export const cpuSection: ResultsMapper<CpuStats> = {
   ],
 };
 
-export interface RunStats {
-  runs?: number;
-}
-
 /** Section: number of sample iterations */
 export const runsSection: ResultsMapper<RunStats> = {
   extract: (results: MeasuredResults) => ({
@@ -177,13 +189,6 @@ export const totalTimeSection: ResultsMapper<{ totalTime?: number }> = {
   ],
 };
 
-export interface AdaptiveStats {
-  median?: number;
-  mean?: number;
-  p99?: number;
-  convergence?: number;
-}
-
 /** Section: median, mean, p99, and convergence for adaptive mode */
 export const adaptiveSection: ResultsMapper<AdaptiveStats> = {
   extract: (results: MeasuredResults) => ({
@@ -209,22 +214,6 @@ export const adaptiveSection: ResultsMapper<AdaptiveStats> = {
   ],
 };
 
-/** Build generic sections based on CLI flags */
-export function buildGenericSections(args: {
-  "gc-stats"?: boolean;
-  "heap-sample"?: boolean;
-}): ResultsMapper[] {
-  const sections: ResultsMapper[] = [];
-  if (args["gc-stats"]) sections.push(gcStatsSection);
-  sections.push(runsSection);
-  return sections;
-}
-
-export interface OptStats {
-  tiers?: string; // tier distribution summary
-  deopt?: number; // deopt count
-}
-
 /** Section: V8 optimization tier distribution and deopt count */
 export const optSection: ResultsMapper<OptStats> = {
   extract: (results: MeasuredResults) => {
@@ -259,3 +248,14 @@ export const optSection: ResultsMapper<OptStats> = {
     },
   ],
 };
+
+/** Build generic sections based on CLI flags */
+export function buildGenericSections(args: {
+  "gc-stats"?: boolean;
+  "heap-sample"?: boolean;
+}): ResultsMapper[] {
+  const sections: ResultsMapper[] = [];
+  if (args["gc-stats"]) sections.push(gcStatsSection);
+  sections.push(runsSection);
+  return sections;
+}