benchforge 0.1.9 → 0.2.4

package/src/runners/BasicRunner.ts

@@ -1,364 +0,0 @@
-import { getHeapStatistics } from "node:v8";
-import type { BenchmarkSpec } from "../Benchmark.ts";
-import type {
-  MeasuredResults,
-  OptStatusInfo,
-  PausePoint,
-} from "../MeasuredResults.ts";
-import type { BenchRunner, RunnerOptions } from "./BenchRunner.ts";
-import { executeBenchmark } from "./BenchRunner.ts";
-
-/**
- * Wait time after gc() for V8 to stabilize (ms).
- *
- * V8 has 4 compilation tiers: Ignition (interpreter) -> Sparkplug (baseline) ->
- * Maglev (mid-tier optimizer) -> TurboFan (full optimizer). Tiering thresholds:
- *   - Ignition -> Sparkplug: 8 invocations
- *   - Sparkplug -> Maglev: 500 invocations
- *   - Maglev -> TurboFan: 6000 invocations
- *
- * Optimization compilation happens on background threads and requires idle time
- * on the main thread to complete. Without sufficient warmup + settle time,
- * benchmarks exhibit bimodal timing: slow Sparkplug samples (~30% slower) mixed
- * with fast optimized samples.
- *
- * The warmup iterations trigger the optimization decision, then gcSettleTime
- * provides idle time for background compilation to finish before measurement.
- *
- * @see https://v8.dev/blog/sparkplug
- * @see https://v8.dev/blog/maglev
- * @see https://v8.dev/blog/background-compilation
- */
-const gcSettleTime = 1000;
-
-type CollectParams<T = unknown> = {
-  benchmark: BenchmarkSpec<T>;
-  maxTime: number;
-  maxIterations: number;
-  warmup: number;
-  params?: T;
-  skipWarmup?: boolean;
-  traceOpt?: boolean;
-  noSettle?: boolean;
-  pauseFirst?: number;
-  pauseInterval?: number;
-  pauseDuration?: number;
-};
-
-type CollectResult = {
-  samples: number[];
-  warmupSamples: number[]; // timing of warmup iterations
-  heapGrowth: number; // amortized KB per sample
-  heapSamples?: number[]; // heap size per sample (bytes)
-  timestamps?: number[]; // wall-clock μs per sample for Perfetto
-  optStatus?: OptStatusInfo;
-  optSamples?: number[]; // per-sample V8 opt status codes
-  pausePoints: PausePoint[]; // where pauses occurred
-};
-
-export type SampleTimeStats = {
-  min: number;
-  max: number;
-  avg: number;
-  p50: number;
-  p75: number;
-  p99: number;
-  p999: number;
-};
-
-/** @return runner with time and iteration limits */
-export class BasicRunner implements BenchRunner {
-  async runBench<T = unknown>(
-    benchmark: BenchmarkSpec<T>,
-    options: RunnerOptions,
-    params?: T,
-  ): Promise<MeasuredResults[]> {
-    const opts = { ...defaultCollectOptions, ...(options as any) };
-    const collected = await collectSamples({ benchmark, params, ...opts });
-    return [buildMeasuredResults(benchmark.name, collected)];
-  }
-}
-
-const defaultCollectOptions = {
-  maxTime: 5000,
-  maxIterations: 1000000,
-  warmup: 0,
-  traceOpt: false,
-  noSettle: false,
-};
-
-function buildMeasuredResults(name: string, c: CollectResult): MeasuredResults {
-  const time = computeStats(c.samples);
-  return {
-    name,
-    samples: c.samples,
-    warmupSamples: c.warmupSamples,
-    heapSamples: c.heapSamples,
-    timestamps: c.timestamps,
-    time,
-    heapSize: { avg: c.heapGrowth, min: c.heapGrowth, max: c.heapGrowth },
-    optStatus: c.optStatus,
-    optSamples: c.optSamples,
-    pausePoints: c.pausePoints,
-  };
-}
-
-/** @return timing samples and amortized allocation from benchmark execution */
-async function collectSamples<T>(p: CollectParams<T>): Promise<CollectResult> {
-  if (!p.maxIterations && !p.maxTime) {
-    throw new Error(`At least one of maxIterations or maxTime must be set`);
-  }
-  const warmupSamples = p.skipWarmup ? [] : await runWarmup(p);
-  const heapBefore = process.memoryUsage().heapUsed;
-  const { samples, heapSamples, timestamps, optStatuses, pausePoints } =
-    await runSampleLoop(p);
-  const heapGrowth =
-    Math.max(0, process.memoryUsage().heapUsed - heapBefore) /
-    1024 /
-    samples.length;
-  if (samples.length === 0) {
-    throw new Error(`No samples collected for benchmark: ${p.benchmark.name}`);
-  }
-  const optStatus = p.traceOpt
-    ? analyzeOptStatus(samples, optStatuses)
-    : undefined;
-  const optSamples =
-    p.traceOpt && optStatuses.length > 0 ? optStatuses : undefined;
-  return {
-    samples,
-    warmupSamples,
-    heapGrowth,
-    heapSamples,
-    timestamps,
-    optStatus,
-    optSamples,
-    pausePoints,
-  };
-}
-
-/** Run warmup iterations with gc + settle time for V8 optimization */
-async function runWarmup<T>(p: CollectParams<T>): Promise<number[]> {
-  const gc = gcFunction();
-  const samples = new Array<number>(p.warmup);
-  for (let i = 0; i < p.warmup; i++) {
-    const start = performance.now();
-    executeBenchmark(p.benchmark, p.params);
-    samples[i] = performance.now() - start;
-  }
-  gc();
-  if (!p.noSettle) {
-    await new Promise(r => setTimeout(r, gcSettleTime));
-    gc();
-  }
-  return samples;
-}
-
-type SampleLoopResult = {
-  samples: number[];
-  heapSamples?: number[];
-  timestamps?: number[];
-  optStatuses: number[];
-  pausePoints: PausePoint[];
-};
-
-/** Estimate sample count for pre-allocation */
-function estimateSampleCount(maxTime: number, maxIterations: number): number {
-  return maxIterations || Math.ceil(maxTime / 0.1); // assume 0.1ms per iteration minimum
-}
-
-type SampleArrays = {
-  samples: number[];
-  timestamps: number[];
-  heapSamples: number[];
-  optStatuses: number[];
-  pausePoints: PausePoint[];
-};
-
-/** Pre-allocate arrays to reduce GC pressure during measurement */
-function createSampleArrays(
-  n: number,
-  trackHeap: boolean,
-  trackOpt: boolean,
-): SampleArrays {
-  const arr = (track: boolean) => (track ? new Array<number>(n) : []);
-  return {
-    samples: new Array<number>(n),
-    timestamps: new Array<number>(n),
-    heapSamples: arr(trackHeap),
-    optStatuses: arr(trackOpt),
-    pausePoints: [],
-  };
-}
-
-/** Trim arrays to actual sample count */
-function trimArrays(
-  a: SampleArrays,
-  count: number,
-  trackHeap: boolean,
-  trackOpt: boolean,
-): void {
-  a.samples.length = a.timestamps.length = count;
-  if (trackHeap) a.heapSamples.length = count;
-  if (trackOpt) a.optStatuses.length = count;
-}
-
-/** Collect timing samples with periodic pauses for V8 optimization */
-async function runSampleLoop<T>(
-  p: CollectParams<T>,
-): Promise<SampleLoopResult> {
-  const {
-    maxTime,
-    maxIterations,
-    pauseFirst,
-    pauseInterval = 0,
-    pauseDuration = 100,
-  } = p;
-  const trackHeap = true; // Always track heap for charts
-  const getOptStatus = p.traceOpt ? createOptStatusGetter() : undefined;
-  const estimated = estimateSampleCount(maxTime, maxIterations);
-  const a = createSampleArrays(estimated, trackHeap, !!getOptStatus);
-
-  let count = 0;
-  let elapsed = 0;
-  let totalPauseTime = 0;
-  const loopStart = performance.now();
-
-  while (
-    (!maxIterations || count < maxIterations) &&
-    (!maxTime || elapsed < maxTime)
-  ) {
-    const start = performance.now();
-    executeBenchmark(p.benchmark, p.params);
-    const end = performance.now();
-    a.samples[count] = end - start;
-    a.timestamps[count] = Number(process.hrtime.bigint() / 1000n);
-    if (trackHeap) a.heapSamples[count] = getHeapStatistics().used_heap_size;
-    if (getOptStatus) a.optStatuses[count] = getOptStatus(p.benchmark.fn);
-    count++;
-
-    if (shouldPause(count, pauseFirst, pauseInterval)) {
-      a.pausePoints.push({ sampleIndex: count - 1, durationMs: pauseDuration });
-      const pauseStart = performance.now();
-      await new Promise(r => setTimeout(r, pauseDuration));
-      totalPauseTime += performance.now() - pauseStart;
-    }
-    elapsed = performance.now() - loopStart - totalPauseTime;
-  }
-
-  trimArrays(a, count, trackHeap, !!getOptStatus);
-  return {
-    samples: a.samples,
-    heapSamples: trackHeap ? a.heapSamples : undefined,
-    timestamps: a.timestamps,
-    optStatuses: a.optStatuses,
-    pausePoints: a.pausePoints,
-  };
-}
-
-/** Check if we should pause at this iteration for V8 optimization */
-function shouldPause(
-  iter: number,
-  first: number | undefined,
-  interval: number,
-): boolean {
-  if (first !== undefined && iter === first) return true;
-  if (interval <= 0) return false;
-  if (first === undefined) return iter % interval === 0;
-  return (iter - first) % interval === 0;
-}
-
-/** @return percentiles and basic statistics */
-export function computeStats(samples: number[]): SampleTimeStats {
-  const sorted = [...samples].sort((a, b) => a - b);
-  const avg = samples.reduce((sum, s) => sum + s, 0) / samples.length;
-  return {
-    min: sorted[0],
-    max: sorted[sorted.length - 1],
-    avg,
-    p50: percentile(sorted, 0.5),
-    p75: percentile(sorted, 0.75),
-    p99: percentile(sorted, 0.99),
-    p999: percentile(sorted, 0.999),
-  };
-}
-
-/** @return percentile value with linear interpolation */
-function percentile(sortedArray: number[], p: number): number {
-  const index = (sortedArray.length - 1) * p;
-  const lower = Math.floor(index);
-  const upper = Math.ceil(index);
-  const weight = index % 1;
-
-  if (upper >= sortedArray.length) return sortedArray[sortedArray.length - 1];
-
-  return sortedArray[lower] * (1 - weight) + sortedArray[upper] * weight;
-}
-
-/** @return runtime gc() function, or no-op if unavailable */
-function gcFunction(): () => void {
-  const gc = globalThis.gc || (globalThis as any).__gc;
-  if (gc) return gc;
-  console.warn("gc() not available, run node/bun with --expose-gc");
-  return () => {};
-}
-
-/** @return function to get V8 optimization status (requires --allow-natives-syntax) */
-function createOptStatusGetter(): ((fn: unknown) => number) | undefined {
-  try {
-    // %GetOptimizationStatus returns a bitmask
-    const getter = new Function("f", "return %GetOptimizationStatus(f)");
-    getter(() => {});
-    return getter as (fn: unknown) => number;
-  } catch {
-    return undefined;
-  }
-}
-
-/**
- * V8 optimization status bit meanings:
- *   Bit 0 (1): is_function
- *   Bit 4 (16): is_optimized (TurboFan)
- *   Bit 5 (32): is_optimized (Maglev)
- *   Bit 7 (128): is_baseline (Sparkplug)
- *   Bit 3 (8): maybe_deoptimized
- */
-const statusNames: Record<number, string> = {
-  1: "interpreted",
-  129: "sparkplug", // 1 + 128
-  17: "turbofan", // 1 + 16
-  33: "maglev", // 1 + 32
-  49: "turbofan+maglev", // 1 + 16 + 32
-  32769: "optimized", // common optimized status
-};
-
-/** @return analysis of V8 optimization status per sample */
-function analyzeOptStatus(
-  samples: number[],
-  statuses: number[],
-): OptStatusInfo | undefined {
-  if (statuses.length === 0 || statuses[0] === undefined) return undefined;
-
-  const byStatusCode = new Map<number, number[]>();
-  let deoptCount = 0;
-
-  for (let i = 0; i < samples.length; i++) {
-    const status = statuses[i];
-    if (status === undefined) continue;
-
-    // Check deopt flag (bit 3)
-    if (status & 8) deoptCount++;
-
-    if (!byStatusCode.has(status)) byStatusCode.set(status, []);
-    byStatusCode.get(status)!.push(samples[i]);
-  }
-
-  const byTier: Record<string, { count: number; medianMs: number }> = {};
-  for (const [status, times] of byStatusCode) {
-    const name = statusNames[status] || `status=${status}`;
-    const sorted = [...times].sort((a, b) => a - b);
-    const median = sorted[Math.floor(sorted.length / 2)];
-    byTier[name] = { count: times.length, medianMs: median };
-  }
-
-  return { byTier, deoptCount };
-}

package/src/table-util/ConvergenceFormatters.ts

@@ -1,19 +0,0 @@
-import pico from "picocolors";
-
-const isTest = process.env.NODE_ENV === "test" || process.env.VITEST === "true";
-const { red } = isTest ? { red: (str: string) => str } : pico;
-
-const lowConfidence = 80;
-
-/** @return convergence percentage with color for low values */
-export function formatConvergence(v: unknown): string {
-  if (typeof v !== "number") return "—";
-  const pct = `${Math.round(v)}%`;
-  return v < lowConfidence ? red(pct) : pct;
-}
-
-/** @return coefficient of variation as ±percentage */
-export function formatCV(v: unknown): string {
-  if (typeof v !== "number") return "";
-  return `±${(v * 100).toFixed(1)}%`;
-}

package/src/table-util/Formatters.ts

@@ -1,152 +0,0 @@
-import pico from "picocolors";
-import type { CIDirection, DifferenceCI } from "../StatisticalUtils.ts";
-
-const isTest = process.env.NODE_ENV === "test" || process.env.VITEST === "true";
-const { red, green } = isTest
-  ? { red: (str: string) => str, green: (str: string) => str }
-  : pico;
-
-/** Format floats with custom precision */
-export function floatPrecision(precision: number) {
-  return (x: unknown): string | null => {
-    if (typeof x !== "number") return null;
-    return x.toFixed(precision).replace(/\.?0+$/, "");
-  };
-}
-
-/** Format percentages with custom precision */
-export function percentPrecision(precision: number) {
-  return (x: unknown): string | null => {
-    if (typeof x !== "number") return null;
-    return percent(x, precision);
-  };
-}
-
-/** Format duration in milliseconds with appropriate units */
-export function duration(ms: unknown): string | null {
-  if (typeof ms !== "number") return null;
-  if (ms < 0.001) return `${(ms * 1000000).toFixed(0)}ns`;
-  if (ms < 1) return `${(ms * 1000).toFixed(1)}μs`;
-  if (ms < 1000) return `${ms.toFixed(2)}ms`;
-  return `${(ms / 1000).toFixed(2)}s`;
-}
-
-/** Format time in milliseconds, showing very small values with units */
-export function timeMs(ms: unknown): string | null {
-  if (typeof ms !== "number") return null;
-  if (ms < 0.001) return `${(ms * 1000000).toFixed(0)}ns`;
-  if (ms < 0.01) return `${(ms * 1000).toFixed(1)}μs`;
-  if (ms >= 10) return ms.toFixed(0);
-  return ms.toFixed(2);
-}
-
-/** Format as rate (value per unit) */
-export function rate(unit: string): (value: unknown) => string | null {
-  return (value: unknown) => {
-    if (typeof value !== "number") return null;
-    return `${integer(value)}/${unit}`;
-  };
-}
-
-/** Format integer with thousand separators */
-export function integer(x: unknown): string | null {
-  if (typeof x !== "number") return null;
-  return new Intl.NumberFormat("en-US").format(Math.round(x));
-}
-
-/** Format fraction as percentage (0.473 → 47.3%) */
-export function percent(fraction: unknown, precision = 1): string | null {
-  if (typeof fraction !== "number") return null;
-  return `${Math.abs(fraction * 100).toFixed(precision)}%`;
-}
-
-/** Format percentage difference between two values */
-export function diffPercent(main: unknown, base: unknown): string {
-  if (typeof main !== "number" || typeof base !== "number") return " ";
-  const diff = main - base;
-  return coloredPercent(diff, base);
-}
-
-/** Format percentage difference for benchmarks (lower is better) */
-export function diffPercentBenchmark(main: unknown, base: unknown): string {
-  if (typeof main !== "number" || typeof base !== "number") return " ";
-  const diff = main - base;
-  return coloredPercent(diff, base, false); // negative is good for benchmarks
-}
-
-/** Format fraction as colored +/- percentage */
-function coloredPercent(
-  numerator: number,
-  denominator: number,
-  positiveIsGreen = true,
-): string {
-  const fraction = numerator / denominator;
-  if (Number.isNaN(fraction) || !Number.isFinite(fraction)) {
-    return " ";
-  }
-  const positive = fraction >= 0;
-  const sign = positive ? "+" : "-";
-  const percentStr = `${sign}${percent(fraction)}`;
-  const isGood = positive === positiveIsGreen;
-  return isGood ? green(percentStr) : red(percentStr);
-}
-
-/** Format memory size in KB with appropriate units */
-export function memoryKB(kb: unknown): string | null {
-  if (typeof kb !== "number") return null;
-  if (kb < 1024) return `${kb.toFixed(0)}KB`;
-  return `${(kb / 1024).toFixed(1)}MB`;
-}
-
-/** Format bytes with appropriate units (B, KB, MB, GB) */
-export function formatBytes(bytes: unknown): string | null {
-  if (typeof bytes !== "number") return null;
-  if (bytes < 1024) return `${bytes.toFixed(0)}B`;
-  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
-  if (bytes < 1024 * 1024 * 1024)
-    return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
-  return `${(bytes / 1024 / 1024 / 1024).toFixed(1)}GB`;
-}
-
-/** Format percentage difference with confidence interval */
-export function formatDiffWithCI(value: unknown): string | null {
-  if (!isDifferenceCI(value)) return null;
-  const { percent, ci, direction } = value;
-  return colorByDirection(diffCIText(percent, ci), direction);
-}
-
-/** Format percentage difference with CI for throughput metrics (higher is better) */
-export function formatDiffWithCIHigherIsBetter(value: unknown): string | null {
-  if (!isDifferenceCI(value)) return null;
-  const { percent, ci, direction } = value;
-  // Flip percent sign for "higher is better" metrics (direction stays same)
-  return colorByDirection(diffCIText(-percent, [-ci[1], -ci[0]]), direction);
-}
-
-/** @return formatted "pct [lo, hi]" text for a diff with CI */
-function diffCIText(pct: number, ci: [number, number]): string {
-  return `${formatBound(pct)} [${formatBound(ci[0])}, ${formatBound(ci[1])}]`;
-}
-
-/** @return text colored green for faster, red for slower */
-function colorByDirection(text: string, direction: CIDirection): string {
-  if (direction === "faster") return green(text);
-  if (direction === "slower") return red(text);
-  return text;
-}
-
-/** @return signed percentage string (e.g. "+1.2%", "-3.4%") */
-function formatBound(v: number): string {
-  const sign = v >= 0 ? "+" : "";
-  return `${sign}${v.toFixed(1)}%`;
-}
-
-/** @return true if value is a DifferenceCI object */
-function isDifferenceCI(x: unknown): x is DifferenceCI {
-  return typeof x === "object" && x !== null && "ci" in x && "direction" in x;
-}
-
-/** @return truncated string with ellipsis if over maxLen */
-export function truncate(str: string, maxLen = 30): string {
-  return str.length > maxLen ? str.slice(0, maxLen - 3) + "..." : str;
-}

package/src/table-util/README.md

@@ -1,70 +0,0 @@
-# TableReport 
-
-Utilities for creating formatted text-based tables.
-Under the hood, TableReport uses the npm [table](https://www.npmjs.com/package/table) library.
-
-### Features
-
-*   **Column Grouping:** Group related columns under a common header.
-*   **Difference Columns:** Automatically generate columns that show the percentage difference between a value and a baseline value.
-*   **Custom Formatting:** Declaratively provide custom formatters for columns.
-*   **Simplified Configuration:** A higher-level API for the `table` library.
-
-## `TableReport.ts`
-
-The `buildTable` function in `TableReport.ts` is the main entry point for creating a table. 
-It takes a configuration object for columns and an array of data records for rows.
-
-### Example
-
-Here's a simplified example of how to use `buildTable`:
-
-```typescript
-import { buildTable, ColumnGroup } from './TableReport';
-import { integer, floatPrecision } from './Formatters';
-
-interface MyData {
-  name: string;
-  value: number;
-  score: number;
-}
-
-const data: MyData[] = [
-  { name: 'test A', value: 123, score: 45.6 },
-  { name: 'test B', value: 456, score: 78.9 },
-];
-
-const baselineData: MyData[] = [
-  { name: 'test A', value: 100, score: 50.0 },
-  { name: 'test B', value: 500, score: 75.0 },
-];
-
-const tableConfig: ColumnGroup<MyData>[] = [
-  {
-    columns: [{ key: 'name', title: 'Name' }],
-  },
-  {
-    groupTitle: 'Metrics',
-    columns: [
-      { key: 'value', title: 'Value', formatter: integer },
-      { key: 'value_diff', title: 'Δ%', diffKey: 'value' },
-      { key: 'score', title: 'Score', formatter: floatPrecision(1) },
-      { key: 'score_diff', title: 'Δ%', diffKey: 'score' },
-    ],
-  },
-];
-
-const table = buildTable(tableConfig, data, baselineData);
-console.log(table);
-```
-
-For a more complex example, see `BenchmarkReport.ts`, specifically the `mostlyFullRow` and `tableConfig` variables.
-
-## `Formatters.ts`
-
-This file contains various utility functions for formatting numbers and strings, such as:
-
-*   `float`, `integer`: Format numbers to a specific precision.
-*   `percent`: Format a number as a percentage.
-*   `diffPercent`, `diffPercentNegative`: Format the percentage difference between two numbers, with color-coding for positive/negative changes.
-*   `bytes`, `duration`, `rate`: Format numbers with appropriate units.