benchforge 0.1.0

package/src/StatisticalUtils.ts

@@ -0,0 +1,176 @@
+const outlierMultiplier = 1.5; // Tukey's fence multiplier
+const bootstrapSamples = 10000;
+const confidence = 0.95;
+
+/** Options for bootstrap resampling methods */
+type BootstrapOptions = {
+  resamples?: number;
+  confidence?: number;
+};
+
+/** Bootstrap estimate with confidence interval and raw resample data */
+export interface BootstrapResult {
+  estimate: number;
+  ci: [number, number];
+  samples: number[];
+}
+
+/** @return relative standard deviation (coefficient of variation) */
+export function coefficientOfVariation(samples: number[]): number {
+  const mean = average(samples);
+  if (mean === 0) return 0;
+  const stdDev = standardDeviation(samples);
+  return stdDev / mean;
+}
+
+/** @return median absolute deviation for robust variability measure */
+export function medianAbsoluteDeviation(samples: number[]): number {
+  const median = percentile(samples, 0.5);
+  const deviations = samples.map(x => Math.abs(x - median));
+  return percentile(deviations, 0.5);
+}
+
+/** @return outliers detected via Tukey's interquartile range method */
+export function findOutliers(samples: number[]): {
+  rate: number;
+  indices: number[];
+} {
+  const q1 = percentile(samples, 0.25);
+  const q3 = percentile(samples, 0.75);
+  const iqr = q3 - q1;
+  const lowerBound = q1 - outlierMultiplier * iqr;
+  const upperBound = q3 + outlierMultiplier * iqr;
+
+  const indices = samples
+    .map((v, i) => (v < lowerBound || v > upperBound ? i : -1))
+    .filter(i => i >= 0);
+  return { rate: indices.length / samples.length, indices };
+}
+
+/** @return bootstrap confidence interval for median */
+export function bootstrapMedian(
+  samples: number[],
+  options: BootstrapOptions = {},
+): BootstrapResult {
+  const { resamples = bootstrapSamples, confidence: conf = confidence } =
+    options;
+  const medians = generateMedians(samples, resamples);
+  const ci = computeInterval(medians, conf);
+
+  return {
+    estimate: percentile(samples, 0.5),
+    ci,
+    samples: medians,
+  };
+}
+
+/** @return mean of values */
+export function average(values: number[]): number {
+  const sum = values.reduce((a, b) => a + b, 0);
+  return sum / values.length;
+}
+
+/** @return standard deviation with Bessel's correction */
+export function standardDeviation(samples: number[]): number {
+  if (samples.length <= 1) return 0;
+  const mean = average(samples);
+  const variance =
+    samples.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (samples.length - 1);
+  return Math.sqrt(variance);
+}
+
+/** @return value at percentile p (0-1) */
+export function percentile(values: number[], p: number): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.ceil(sorted.length * p) - 1;
+  return sorted[Math.max(0, index)];
+}
+
+/** @return medians from bootstrap resamples */
+function generateMedians(samples: number[], resamples: number): number[] {
+  return Array.from({ length: resamples }, () =>
+    percentile(createResample(samples), 0.5),
+  );
+}
+
+/** @return bootstrap resample with replacement */
+export function createResample(samples: number[]): number[] {
+  const n = samples.length;
+  const rand = () => samples[Math.floor(Math.random() * n)];
+  return Array.from({ length: n }, rand);
+}
+
+/** @return confidence interval [lower, upper] */
+function computeInterval(
+  medians: number[],
+  confidence: number,
+): [number, number] {
+  const alpha = (1 - confidence) / 2;
+  const lower = percentile(medians, alpha);
+  const upper = percentile(medians, 1 - alpha);
+  return [lower, upper];
+}
+
+export type CIDirection = "faster" | "slower" | "uncertain";
+
+/** Binned histogram for efficient transfer to browser */
+export interface HistogramBin {
+  x: number; // bin center
+  count: number;
+}
+
+/** Bootstrap confidence interval for percentage difference between two samples */
+export interface DifferenceCI {
+  percent: number;
+  ci: [number, number];
+  direction: CIDirection;
+  /** Histogram of bootstrap distribution for visualization */
+  histogram?: HistogramBin[];
+}
+
+/** Bin values into histogram for compact visualization */
+function binValues(values: number[], binCount = 30): HistogramBin[] {
+  const sorted = [...values].sort((a, b) => a - b);
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  if (min === max) return [{ x: min, count: values.length }];
+
+  const step = (max - min) / binCount;
+  const counts = new Array(binCount).fill(0);
+  for (const v of values) {
+    const bin = Math.min(Math.floor((v - min) / step), binCount - 1);
+    counts[bin]++;
+  }
+  return counts.map((count, i) => ({ x: min + (i + 0.5) * step, count }));
+}
+
+/** @return bootstrap CI for percentage difference between baseline and current medians */
+export function bootstrapDifferenceCI(
+  baseline: number[],
+  current: number[],
+  options: BootstrapOptions = {},
+): DifferenceCI {
+  const { resamples = bootstrapSamples, confidence: conf = confidence } =
+    options;
+
+  const baselineMedian = percentile(baseline, 0.5);
+  const currentMedian = percentile(current, 0.5);
+  const observedPercent =
+    ((currentMedian - baselineMedian) / baselineMedian) * 100;
+
+  const diffs: number[] = [];
+  for (let i = 0; i < resamples; i++) {
+    const resB = createResample(baseline);
+    const resC = createResample(current);
+    const medB = percentile(resB, 0.5);
+    const medC = percentile(resC, 0.5);
+    diffs.push(((medC - medB) / medB) * 100);
+  }
+
+  const ci = computeInterval(diffs, conf);
+  const excludesZero = ci[0] > 0 || ci[1] < 0;
+  let direction: CIDirection = "uncertain";
+  if (excludesZero) direction = observedPercent < 0 ? "faster" : "slower";
+  const histogram = binValues(diffs);
+  return { percent: observedPercent, ci, direction, histogram };
+}

package/src/TypeUtil.ts

@@ -0,0 +1,8 @@
+/** Convert union to intersection - https://mighdoll.dev/blog/modern-typescript-intersection/ */
+export type UnionToIntersection<U> = (
+  U extends any
+    ? (k: U) => void
+    : never
+) extends (k: infer I extends U) => void
+  ? I
+  : never;

package/src/bin/benchforge.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { runDefaultBench } from "../index.ts";
+
+await runDefaultBench();

package/src/browser/BrowserGcStats.ts

@@ -0,0 +1,44 @@
+import {
+  aggregateGcStats,
+  type GcEvent,
+  type GcStats,
+} from "../runners/GcStats.ts";
+
+/** CDP trace event from Tracing.dataCollected */
+export interface TraceEvent {
+  cat: string;
+  name: string;
+  ph: string;
+  dur?: number; // microseconds
+  args?: Record<string, any>;
+}
+
+/** Parse CDP trace events (MinorGC/MajorGC) into GcEvent[] */
+export function parseGcTraceEvents(traceEvents: TraceEvent[]): GcEvent[] {
+  return traceEvents.flatMap(e => {
+    if (e.ph !== "X") return [];
+    const type = gcType(e.name);
+    if (!type) return [];
+    const durUs = e.dur ?? 0;
+    const heapBefore: number = e.args?.usedHeapSizeBefore ?? 0;
+    const heapAfter: number = e.args?.usedHeapSizeAfter ?? 0;
+    return [
+      {
+        type,
+        pauseMs: durUs / 1000,
+        collected: Math.max(0, heapBefore - heapAfter),
+      },
+    ];
+  });
+}
+
+function gcType(name: string): GcEvent["type"] | undefined {
+  if (name === "MinorGC") return "scavenge";
+  if (name === "MajorGC") return "mark-compact";
+  return undefined;
+}
+
+/** Parse CDP trace events and aggregate into GcStats */
+export function browserGcStats(traceEvents: TraceEvent[]): GcStats {
+  return aggregateGcStats(parseGcTraceEvents(traceEvents));
+}

package/src/browser/BrowserHeapSampler.ts

@@ -0,0 +1,248 @@
+import { type CDPSession, chromium, type Page } from "playwright";
+import type {
+  HeapProfile,
+  HeapSampleOptions,
+} from "../heap-sample/HeapSampler.ts";
+import type { GcStats } from "../runners/GcStats.ts";
+import { browserGcStats, type TraceEvent } from "./BrowserGcStats.ts";
+
+export interface BrowserProfileParams {
+  url: string;
+  heapSample?: boolean;
+  heapOptions?: HeapSampleOptions;
+  gcStats?: boolean;
+  headless?: boolean;
+  timeout?: number; // seconds
+  maxTime?: number; // ms, bench function iteration time limit
+  maxIterations?: number; // exact iteration count (bench function mode)
+}
+
+export interface BrowserProfileResult {
+  heapProfile?: HeapProfile;
+  gcStats?: GcStats;
+  /** Wall-clock ms (lap mode: first start to done, bench function: total loop) */
+  wallTimeMs?: number;
+  /** Per-iteration timing samples (ms) from bench function or lap mode */
+  samples?: number[];
+}
+
+interface LapModeHandle {
+  promise: Promise<BrowserProfileResult>;
+  cancel: () => void;
+}
+
+/** Run browser benchmark, auto-detecting page API mode.
+ *  Bench function (window.__bench): CLI controls iteration and timing.
+ *  Lap mode (__start/__lap/__done): page controls the measured region. */
+export async function profileBrowser(
+  params: BrowserProfileParams,
+): Promise<BrowserProfileResult> {
+  const { url, headless = true, timeout = 60 } = params;
+  const { gcStats: collectGc } = params;
+  const { samplingInterval = 32768 } = params.heapOptions ?? {};
+
+  const browser = await chromium.launch({ headless });
+  try {
+    const page = await browser.newPage();
+    page.setDefaultTimeout(timeout * 1000);
+    const cdp = await page.context().newCDPSession(page);
+
+    const pageErrors: string[] = [];
+    page.on("pageerror", err => pageErrors.push(err.message));
+
+    const traceEvents = collectGc ? await startGcTracing(cdp) : [];
+    const lapMode = await setupLapMode(
+      page,
+      cdp,
+      params,
+      samplingInterval,
+      timeout,
+      pageErrors,
+    );
+
+    await page.goto(url, { waitUntil: "load" });
+    const hasBench = await page.evaluate(
+      () => typeof (globalThis as any).__bench === "function",
+    );
+
+    let result: BrowserProfileResult;
+    if (hasBench) {
+      lapMode.cancel();
+      lapMode.promise.catch(() => {}); // suppress unused rejection
+      result = await runBenchLoop(page, cdp, params, samplingInterval);
+    } else {
+      result = await lapMode.promise;
+      lapMode.cancel();
+    }
+
+    if (collectGc) {
+      result = { ...result, gcStats: await collectTracing(cdp, traceEvents) };
+    }
+    return result;
+  } finally {
+    await browser.close();
+  }
+}
+
+/** Inject __start/__lap as in-page functions, expose __done for results collection.
+ *  __start/__lap are pure in-page (zero CDP overhead). First __start() triggers
+ *  instrument start. __done() stops instruments and collects timing data. */
+async function setupLapMode(
+  page: Page,
+  cdp: CDPSession,
+  params: BrowserProfileParams,
+  samplingInterval: number,
+  timeout: number,
+  pageErrors: string[],
+): Promise<LapModeHandle> {
+  const { heapSample } = params;
+  const { promise, resolve, reject } =
+    Promise.withResolvers<BrowserProfileResult>();
+  let instrumentsStarted = false;
+
+  await page.exposeFunction("__benchInstrumentStart", async () => {
+    if (instrumentsStarted) return;
+    instrumentsStarted = true;
+    if (heapSample) {
+      await cdp.send(
+        "HeapProfiler.startSampling",
+        heapSamplingParams(samplingInterval),
+      );
+    }
+  });
+
+  await page.exposeFunction(
+    "__benchCollect",
+    async (samples: number[], wallTimeMs: number) => {
+      let heapProfile: HeapProfile | undefined;
+      if (heapSample && instrumentsStarted) {
+        const result = await cdp.send("HeapProfiler.stopSampling");
+        heapProfile = result.profile as unknown as HeapProfile;
+      }
+      resolve({ samples, heapProfile, wallTimeMs });
+    },
+  );
+
+  await page.addInitScript(injectLapFunctions);
+
+  const timer = setTimeout(() => {
+    const lines = [`Timed out after ${timeout}s`];
+    if (pageErrors.length) {
+      lines.push("Page JS errors:", ...pageErrors.map(e => `  ${e}`));
+    } else {
+      lines.push("Page did not call __done() or define window.__bench");
+    }
+    reject(new Error(lines.join("\n")));
+  }, timeout * 1000);
+
+  return { promise, cancel: () => clearTimeout(timer) };
+}
+
+/** In-page timing functions injected via addInitScript (zero CDP overhead).
+ *  __start/__lap collect timestamps, __done delegates to exposed __benchCollect. */
+function injectLapFunctions(): void {
+  const g = globalThis as any;
+  g.__benchSamples = [];
+  g.__benchLastTime = 0;
+  g.__benchFirstStart = 0;
+
+  g.__start = () => {
+    const now = performance.now();
+    g.__benchLastTime = now;
+    if (!g.__benchFirstStart) {
+      g.__benchFirstStart = now;
+      return g.__benchInstrumentStart();
+    }
+  };
+
+  g.__lap = () => {
+    const now = performance.now();
+    g.__benchSamples.push(now - g.__benchLastTime);
+    g.__benchLastTime = now;
+  };
+
+  g.__done = () => {
+    const wall = g.__benchFirstStart
+      ? performance.now() - g.__benchFirstStart
+      : 0;
+    return g.__benchCollect(g.__benchSamples.slice(), wall);
+  };
+}
+
+function heapSamplingParams(samplingInterval: number) {
+  return {
+    samplingInterval,
+    includeObjectsCollectedByMajorGC: true,
+    includeObjectsCollectedByMinorGC: true,
+  };
+}
+
+/** Start CDP GC tracing, returns the event collector array. */
+async function startGcTracing(cdp: CDPSession): Promise<TraceEvent[]> {
+  const events: TraceEvent[] = [];
+  cdp.on("Tracing.dataCollected", ({ value }) => {
+    for (const e of value) events.push(e as unknown as TraceEvent);
+  });
+  await cdp.send("Tracing.start", {
+    traceConfig: { includedCategories: ["v8", "v8.gc"] },
+  });
+  return events;
+}
+
+/** Bench function mode: run window.__bench in a timed iteration loop. */
+async function runBenchLoop(
+  page: Page,
+  cdp: CDPSession,
+  params: BrowserProfileParams,
+  samplingInterval: number,
+): Promise<BrowserProfileResult> {
+  const { heapSample } = params;
+  const maxTime = params.maxTime ?? 642;
+  const maxIter = params.maxIterations ?? Number.MAX_SAFE_INTEGER;
+
+  if (heapSample) {
+    await cdp.send(
+      "HeapProfiler.startSampling",
+      heapSamplingParams(samplingInterval),
+    );
+  }
+
+  const { samples, totalMs } = await page.evaluate(
+    async ({ maxTime, maxIter }) => {
+      const bench = (globalThis as any).__bench;
+      const samples: number[] = [];
+      const startAll = performance.now();
+      const deadline = startAll + maxTime;
+      for (let i = 0; i < maxIter && performance.now() < deadline; i++) {
+        const t0 = performance.now();
+        await bench();
+        samples.push(performance.now() - t0);
+      }
+      return { samples, totalMs: performance.now() - startAll };
+    },
+    { maxTime, maxIter },
+  );
+
+  let heapProfile: HeapProfile | undefined;
+  if (heapSample) {
+    const result = await cdp.send("HeapProfiler.stopSampling");
+    heapProfile = result.profile as unknown as HeapProfile;
+  }
+
+  return { samples, heapProfile, wallTimeMs: totalMs };
+}
+
+/** Stop CDP tracing and parse GC events into GcStats. */
+async function collectTracing(
+  cdp: CDPSession,
+  traceEvents: TraceEvent[],
+): Promise<GcStats> {
+  const complete = new Promise<void>(resolve =>
+    cdp.once("Tracing.tracingComplete", () => resolve()),
+  );
+  await cdp.send("Tracing.end");
+  await complete;
+  return browserGcStats(traceEvents);
+}
+
+export { profileBrowser as profileBrowserHeap };

package/src/cli/CliArgs.ts

@@ -0,0 +1,64 @@
+import type { Argv } from "yargs";
+import yargs from "yargs";
+
+export const defaultTime = 0.642;
+export const defaultAdaptiveMaxTime = 20;
+export const defaultPauseInterval = 0;
+export const defaultPauseDuration = 100;
+
+export type Configure<T> = (yargs: Argv) => Argv<T>;
+
+/** @return CLI args type from builder function */
+export type DefaultCliArgs =
+  ReturnType<typeof defaultCliArgs> extends Argv<infer T> ? T : never;
+
+// biome-ignore format: compact option definitions
+const cliOptions = {
+  time:           { type: "number",  default: defaultTime, requiresArg: true, describe: "test duration in seconds" },
+  cpu:            { type: "boolean", default: false, describe: "CPU counter measurements (requires root)" },
+  collect:        { type: "boolean", default: false, describe: "force GC after each iteration" },
+  "gc-stats":     { type: "boolean", default: false, describe: "collect GC statistics (Node: --trace-gc-nvp, browser: CDP tracing)" },
+  profile:        { type: "boolean", default: false, describe: "run once for profiling" },
+  filter:         { type: "string",  requiresArg: true, describe: "filter benchmarks by regex or substring" },
+  all:            { type: "boolean", default: false, describe: "run all cases (ignore defaultCases)" },
+  worker:         { type: "boolean", default: true, describe: "run in worker process for isolation (default: true)" },
+  adaptive:       { type: "boolean", default: false, describe: "use adaptive sampling mode" },
+  "min-time":     { type: "number",  default: 1, describe: "minimum time in seconds before adaptive convergence can stop" },
+  convergence:    { type: "number",  default: 95, describe: "confidence threshold (0-100)" },
+  warmup:         { type: "number",  default: 0, describe: "warmup iterations before measurement" },
+  html:           { type: "boolean", default: false, describe: "generate HTML report and open in browser" },
+  "export-html":  { type: "string",  requiresArg: true, describe: "export HTML report to specified file" },
+  json:           { type: "string",  requiresArg: true, describe: "export benchmark data to JSON file" },
+  perfetto:       { type: "string",  requiresArg: true, describe: "export Perfetto trace file (view at ui.perfetto.dev)" },
+  "trace-opt":    { type: "boolean", default: false, describe: "trace V8 optimization tiers (requires --allow-natives-syntax)" },
+  "skip-settle":  { type: "boolean", default: false, describe: "skip post-warmup settle time (see V8 optimization cold start)" },
+  "pause-first":  { type: "number",  describe: "iterations before first pause (then pause-interval applies)" },
+  "pause-interval": { type: "number", default: defaultPauseInterval, describe: "iterations between pauses for V8 optimization (0 to disable)" },
+  "pause-duration": { type: "number", default: defaultPauseDuration, describe: "pause duration in ms for V8 optimization" },
+  batches:          { type: "number",  default: 1, describe: "divide time into N batches, alternating baseline/current order" },
+  iterations:       { type: "number",  requiresArg: true, describe: "exact number of iterations (overrides --time)" },
+  "heap-sample":    { type: "boolean", default: false, describe: "heap sampling allocation attribution (includes garbage)" },
+  "heap-interval":  { type: "number",  default: 32768, describe: "heap sampling interval in bytes" },
+  "heap-depth":     { type: "number",  default: 64, describe: "heap sampling stack depth" },
+  "heap-rows":      { type: "number",  default: 20, describe: "top allocation sites to show" },
+  "heap-stack":     { type: "number",  default: 3, describe: "call stack depth to display" },
+  "heap-verbose":   { type: "boolean", default: false, describe: "verbose output with file:// paths and line numbers" },
+  "heap-user-only": { type: "boolean", default: false, describe: "filter to user code only (hide node internals)" },
+  url:              { type: "string",  requiresArg: true, describe: "page URL for browser profiling (enables browser mode)" },
+  headless:         { type: "boolean", default: true, describe: "run browser in headless mode" },
+  timeout:          { type: "number",  default: 60, describe: "browser page timeout in seconds" },
+} as const;
+
+/** @return yargs with standard benchmark options */
+export function defaultCliArgs(yargsInstance: Argv) {
+  return yargsInstance.options(cliOptions).help().strict();
+}
+
+/** @return parsed command line arguments */
+export function parseCliArgs<T = DefaultCliArgs>(
+  args: string[],
+  configure: Configure<T> = defaultCliArgs as Configure<T>,
+): T {
+  const yargsInstance = configure(yargs(args));
+  return yargsInstance.parseSync() as T;
+}

package/src/cli/FilterBenchmarks.ts

@@ -0,0 +1,68 @@
+import type { BenchGroup, BenchSuite } from "../Benchmark.ts";
+
+/** Filter benchmarks by name pattern */
+export function filterBenchmarks(
+  suite: BenchSuite,
+  filter?: string,
+  removeEmpty = true,
+): BenchSuite {
+  if (!filter) return suite;
+  const regex = createFilterRegex(filter);
+  const groups = suite.groups
+    .map(group => ({
+      ...group,
+      benchmarks: group.benchmarks.filter(bench =>
+        regex.test(stripCaseSuffix(bench.name)),
+      ),
+      baseline:
+        group.baseline && regex.test(stripCaseSuffix(group.baseline.name))
+          ? group.baseline
+          : undefined,
+    }))
+    .filter(group => !removeEmpty || group.benchmarks.length > 0);
+  validateFilteredSuite(groups, filter);
+  return { name: suite.name, groups };
+}
+
+/** Create regex from filter (literal unless regex-like) */
+function createFilterRegex(filter: string): RegExp {
+  const looksLikeRegex =
+    (filter.startsWith("/") && filter.endsWith("/")) ||
+    filter.includes("*") ||
+    filter.includes("?") ||
+    filter.includes("[") ||
+    filter.includes("|") ||
+    filter.startsWith("^") ||
+    filter.endsWith("$");
+
+  if (looksLikeRegex) {
+    const pattern =
+      filter.startsWith("/") && filter.endsWith("/")
+        ? filter.slice(1, -1)
+        : filter;
+    try {
+      return new RegExp(pattern, "i");
+    } catch {
+      return new RegExp(escapeRegex(filter), "i");
+    }
+  }
+
+  return new RegExp("^" + escapeRegex(filter), "i");
+}
+
+/** Strip case suffix like " [large]" from benchmark name for filtering */
+function stripCaseSuffix(name: string): string {
+  return name.replace(/ \[.*?\]$/, "");
+}
+
+/** Escape regex special characters */
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** Ensure at least one benchmark matches filter */
+function validateFilteredSuite(groups: BenchGroup[], filter?: string): void {
+  if (groups.every(g => g.benchmarks.length === 0)) {
+    throw new Error(`No benchmarks match filter: "${filter}"`);
+  }
+}