benchforge 0.1.11 → 0.2.4

package/src/runners/AdaptiveWrapper.ts

@@ -1,19 +1,22 @@
-import type { BenchmarkSpec } from "../Benchmark.ts";
-import type { MeasuredResults } from "../MeasuredResults.ts";
-import {
-  coefficientOfVariation,
-  medianAbsoluteDeviation,
-  percentile,
-} from "../StatisticalUtils.ts";
+import { median } from "../stats/StatisticalUtils.ts";
+import type { BenchmarkSpec } from "./BenchmarkSpec.ts";
 import type { BenchRunner, RunnerOptions } from "./BenchRunner.ts";
+import type { MeasuredResults } from "./MeasuredResults.ts";
 import { msToNs } from "./RunnerUtils.ts";
+import { computeStats, outlierImpactRatio } from "./SampleStats.ts";
 
+/** Options for adaptive sampling: collects until statistical convergence or timeout. */
 export interface AdaptiveOptions extends RunnerOptions {
+  /** Enable adaptive sampling (default: true when using adaptive runner) */
   adaptive?: boolean;
+  /** Minimum measurement time in ms before convergence can stop sampling (default: 1000) */
   minTime?: number;
+  /** Maximum measurement time in ms, hard stop (default: 10000) */
   maxTime?: number;
+  /** Target confidence percentage to stop early (default: 95) */
   targetConfidence?: number;
-  convergence?: number; // Confidence threshold (0-100)
+  /** Confidence threshold 0-100 (alias for targetConfidence) */
+  convergence?: number;
 }
 
 type Metrics = {
@@ -39,101 +42,85 @@ const initialBatch = 100;
 const continueBatch = 100;
 const continueIterations = 10;
 
-/** @return adaptive sampling runner wrapper */
+/** Wrap a runner with adaptive sampling (convergence detection or timeout). */
 export function createAdaptiveWrapper(
   baseRunner: BenchRunner,
   options: AdaptiveOptions,
 ): BenchRunner {
   return {
     async runBench<T = unknown>(
-      benchmark: BenchmarkSpec<T>,
-      runnerOptions: RunnerOptions,
+      bench: BenchmarkSpec<T>,
+      opts: RunnerOptions,
       params?: T,
     ): Promise<MeasuredResults[]> {
-      return runAdaptiveBench(
-        baseRunner,
-        benchmark,
-        runnerOptions,
-        options,
-        params,
-      );
+      return runAdaptiveBench(baseRunner, bench, opts, options, params);
     },
   };
 }
 
-/** @return convergence based on window stability */
+/** Check convergence by comparing sliding windows of samples for stability. */
 export function checkConvergence(samples: number[]): ConvergenceResult {
   const windowSize = getWindowSize(samples);
   const minSamples = windowSize * 2;
-
   if (samples.length < minSamples) {
-    return buildProgressResult(samples.length, minSamples);
+    const confidence = (samples.length / minSamples) * 100;
+    const reason = `Collecting samples: ${samples.length}/${minSamples}`;
+    return { converged: false, confidence, reason };
   }
-
-  const metrics = getStability(samples, windowSize);
-  return buildConvergence(metrics);
+  return buildConvergence(getStability(samples, windowSize));
 }
 
-/** @return results using adaptive sampling strategy */
+/** Run benchmark with adaptive sampling until convergence or timeout. */
 async function runAdaptiveBench<T>(
-  baseRunner: BenchRunner,
-  benchmark: BenchmarkSpec<T>,
-  runnerOptions: RunnerOptions,
-  options: AdaptiveOptions,
+  runner: BenchRunner,
+  bench: BenchmarkSpec<T>,
+  opts: RunnerOptions,
+  adaptive: AdaptiveOptions,
   params?: T,
 ): Promise<MeasuredResults[]> {
-  const {
-    minTime: min = options.minTime ?? minTime,
-    maxTime: max = options.maxTime ?? maxTime,
-    targetConfidence: target = options.convergence ?? targetConfidence,
-  } = runnerOptions as AdaptiveOptions;
+  const overrides = opts as AdaptiveOptions;
+  const min = overrides.minTime ?? adaptive.minTime ?? minTime;
+  const max = overrides.maxTime ?? adaptive.maxTime ?? maxTime;
+  const target =
+    overrides.convergence ?? adaptive.convergence ?? targetConfidence;
   const allSamples: number[] = [];
 
-  // Collect initial batch (includes warmup + settle)
-  const warmup = await collectInitial(
-    baseRunner,
-    benchmark,
-    runnerOptions,
+  const { warmup, startTime: hrtimeStart } = await collectInitial(
+    runner,
+    bench,
+    opts,
     params,
     allSamples,
   );
-
-  // Start timing AFTER warmup - warmup time doesn't count against maxTime
+  // Start timing after warmup so warmup time doesn't count against maxTime
   const startTime = performance.now();
-
   const limits = {
     minTime: min,
     maxTime: max,
     targetConfidence: target,
     startTime,
   };
-  await collectAdaptive(
-    baseRunner,
-    benchmark,
-    runnerOptions,
-    params,
-    allSamples,
-    limits,
-  );
+  await collectAdaptive(runner, bench, opts, params, allSamples, limits);
 
-  const convergence = checkConvergence(allSamples.map(s => s * msToNs));
+  const samplesNs = allSamples.map(s => s * msToNs);
+  const convergence = checkConvergence(samplesNs);
   return buildResults(
     allSamples,
     startTime,
     convergence,
-    benchmark.name,
+    bench.name,
     warmup,
+    hrtimeStart,
   );
 }
 
-/** @return window size scaled to execution time */
+/** Scale window size inversely with execution time -- fast ops need more samples. */
 function getWindowSize(samples: number[]): number {
-  if (samples.length < 20) return windowSize; // Default for initial samples
+  if (samples.length < 20) return windowSize;
 
   const recentMs = samples.slice(-20).map(s => s / msToNs);
-  const recentMedian = percentile(recentMs, 0.5);
+  const recentMedian = median(recentMs);
 
-  // Inverse scaling with execution time
   if (recentMedian < 0.01) return 200; // <10μs
   if (recentMedian < 0.1) return 100; // <100μs
   if (recentMedian < 1) return 50; // <1ms
@@ -141,91 +128,67 @@ function getWindowSize(samples: number[]): number {
   return 20; // >10ms
 }
 
-/** @return progress when samples insufficient */
-function buildProgressResult(
-  currentSamples: number,
-  minSamples: number,
-): ConvergenceResult {
-  return {
-    converged: false,
-    confidence: (currentSamples / minSamples) * 100,
-    reason: `Collecting samples: ${currentSamples}/${minSamples}`,
-  };
-}
-
-/** @return stability metrics between windows */
-function getStability(samples: number[], windowSize: number): Metrics {
-  const recent = samples.slice(-windowSize);
-  const previous = samples.slice(-windowSize * 2, -windowSize);
-
-  const recentMs = recent.map(s => s / msToNs);
-  const previousMs = previous.map(s => s / msToNs);
-
-  const medianRecent = percentile(recentMs, 0.5);
-  const medianPrevious = percentile(previousMs, 0.5);
-  const medianDrift = Math.abs(medianRecent - medianPrevious) / medianPrevious;
-
-  const impactRecent = getOutlierImpact(recentMs);
-  const impactPrevious = getOutlierImpact(previousMs);
-  const impactDrift = Math.abs(impactRecent.ratio - impactPrevious.ratio);
-
-  return {
-    medianDrift,
-    impactDrift,
-    medianStable: medianDrift < stability,
-    impactStable: impactDrift < stability,
-  };
-}
-
-/** @return convergence from stability metrics */
+/** Convert stability metrics to a convergence result with confidence score. */
 function buildConvergence(metrics: Metrics): ConvergenceResult {
   const { medianDrift, impactDrift, medianStable, impactStable } = metrics;
-
-  if (medianStable && impactStable) {
+  if (medianStable && impactStable)
     return {
       converged: true,
       confidence: 100,
       reason: "Stable performance pattern",
     };
-  }
-
-  const confidence = Math.min(
-    100,
-    (1 - medianDrift / stability) * 50 + (1 - impactDrift / stability) * 50,
-  );
-
+  const raw =
+    (1 - medianDrift / stability) * 50 + (1 - impactDrift / stability) * 50;
+  const confidence = Math.max(0, Math.min(100, raw));
   const reason =
     medianDrift > impactDrift
       ? `Median drifting: ${(medianDrift * 100).toFixed(1)}%`
       : `Outlier impact changing: ${(impactDrift * 100).toFixed(1)}%`;
+  return { converged: false, confidence, reason };
+}
+
+/** Compare median and outlier-impact drift between recent and previous windows. */
+function getStability(samples: number[], windowSize: number): Metrics {
+  const toMs = (s: number) => s / msToNs;
+  const recentMs = samples.slice(-windowSize).map(toMs);
+  const previousMs = samples.slice(-windowSize * 2, -windowSize).map(toMs);
 
-  return { converged: false, confidence: Math.max(0, confidence), reason };
+  const medianRecent = median(recentMs);
+  const medianPrevious = median(previousMs);
+  const medianDrift = Math.abs(medianRecent - medianPrevious) / medianPrevious;
+
+  const impactRecent = outlierImpactRatio(recentMs);
+  const impactPrevious = outlierImpactRatio(previousMs);
+  const impactDrift = Math.abs(impactRecent - impactPrevious);
+
+  const medianStable = medianDrift < stability;
+  const impactStable = impactDrift < stability;
+  return { medianDrift, impactDrift, medianStable, impactStable };
 }
 
-/** @return warmupSamples from initial batch */
+/** Collect the initial batch (warmup + settle), returning warmup samples. */
 async function collectInitial<T>(
-  baseRunner: BenchRunner,
-  benchmark: BenchmarkSpec<T>,
-  runnerOptions: RunnerOptions,
+  runner: BenchRunner,
+  bench: BenchmarkSpec<T>,
+  opts: RunnerOptions,
   params: T | undefined,
   allSamples: number[],
-): Promise<number[] | undefined> {
-  // Don't pass adaptive flag to base runner to avoid double wrapping
-  const opts = {
-    ...(runnerOptions as any),
+): Promise<{ warmup?: number[]; startTime?: number }> {
+  const batchOpts = {
+    ...(opts as any),
     maxTime: initialBatch,
     maxIterations: undefined,
   };
-  const results = await baseRunner.runBench(benchmark, opts, params);
+  const results = await runner.runBench(bench, batchOpts, params);
   appendSamples(results[0], allSamples);
-  return results[0].warmupSamples;
+  return { warmup: results[0].warmupSamples, startTime: results[0].startTime };
 }
 
-/** @return samples until convergence or timeout */
+/** Collect batches until convergence or timeout, with progress logging. */
 async function collectAdaptive<T>(
-  baseRunner: BenchRunner,
-  benchmark: BenchmarkSpec<T>,
-  runnerOptions: RunnerOptions,
+  runner: BenchRunner,
+  bench: BenchmarkSpec<T>,
+  opts: RunnerOptions,
   params: T | undefined,
   allSamples: number[],
   limits: {
@@ -242,150 +205,67 @@ async function collectAdaptive<T>(
     const convergence = checkConvergence(samplesNs);
     const elapsed = performance.now() - startTime;
 
-    if (elapsed - lastLog > 1000) {
-      const elapsedSec = (elapsed / 1000).toFixed(1);
-      const conf = convergence.confidence.toFixed(0);
-      process.stderr.write(
-        `\r◊ ${benchmark.name}: ${conf}% confident (${elapsedSec}s)   `,
-      );
-      lastLog = elapsed;
-    }
-
-    if (shouldStop(convergence, targetConfidence, elapsed, minTime)) {
-      break;
-    }
+    lastLog = logProgress(bench.name, convergence, elapsed, lastLog);
+    if (shouldStop(convergence, targetConfidence, elapsed, minTime)) break;
 
-    // Skip warmup for continuation batches (warmup done in initial batch)
-    const opts = {
-      ...(runnerOptions as any),
+    const batch = {
+      ...(opts as any),
       maxTime: continueBatch,
       maxIterations: continueIterations,
       skipWarmup: true,
     };
-    const batchResults = await baseRunner.runBench(benchmark, opts, params);
-    appendSamples(batchResults[0], allSamples);
+    const results = await runner.runBench(bench, batch, params);
+    appendSamples(results[0], allSamples);
   }
   process.stderr.write("\r" + " ".repeat(60) + "\r");
 }
 
-/** @return measured results with convergence metrics */
+/** Build final MeasuredResults from collected samples and convergence state. */
 function buildResults(
-  samplesMs: number[],
-  startTime: number,
+  samples: number[],
+  elapsedStart: number,
   convergence: ConvergenceResult,
   name: string,
   warmupSamples?: number[],
+  startTime?: number,
 ): MeasuredResults[] {
-  const totalTime = (performance.now() - startTime) / 1000;
-  const samplesNs = samplesMs.map(s => s * msToNs);
-  const timeStats = computeTimeStats(samplesNs);
-
+  const totalTime = (performance.now() - elapsedStart) / 1000;
+  const time = computeStats(samples);
   return [
-    {
-      name,
-      samples: samplesMs,
-      warmupSamples,
-      time: timeStats,
-      totalTime,
-      convergence,
-    },
+    { name, samples, warmupSamples, time, totalTime, startTime, convergence },
   ];
 }
 
-/** @return outlier impact as proportion of total time */
-function getOutlierImpact(samples: number[]): { ratio: number; count: number } {
-  if (samples.length === 0) return { ratio: 0, count: 0 };
-
-  const median = percentile(samples, 0.5);
-  const q75 = percentile(samples, 0.75);
-  const threshold = median + 1.5 * (q75 - median);
-
-  let excessTime = 0;
-  let count = 0;
-
-  for (const sample of samples) {
-    if (sample > threshold) {
-      excessTime += sample - median;
-      count++;
-    }
-  }
-
-  const totalTime = samples.reduce((a, b) => a + b, 0);
-  return {
-    ratio: totalTime > 0 ? excessTime / totalTime : 0,
-    count,
-  };
-}
-
-/** Append samples one-by-one to avoid stack overflow from spread on large arrays */
+/** Append samples one-by-one to avoid stack overflow from spread on large arrays. */
 function appendSamples(result: MeasuredResults, samples: number[]): void {
   if (!result.samples?.length) return;
   for (const sample of result.samples) samples.push(sample);
 }
 
-/** @return true if convergence reached or timeout */
-function shouldStop(
+/** Log adaptive sampling progress at ~1s intervals. */
+function logProgress(
+  name: string,
   convergence: ConvergenceResult,
-  targetConfidence: number,
-  elapsedTime: number,
-  minTime: number,
-): boolean {
-  if (convergence.converged && convergence.confidence >= targetConfidence) {
-    return true;
-  }
-  // After minTime, accept whichever is higher: targetConfidence or fallbackThreshold
-  const threshold = Math.max(targetConfidence, fallbackThreshold);
-  return elapsedTime >= minTime && convergence.confidence >= threshold;
+  elapsed: number,
+  lastLog: number,
+): number {
+  if (elapsed - lastLog <= 1000) return lastLog;
+  const sec = (elapsed / 1000).toFixed(1);
+  const conf = convergence.confidence.toFixed(0);
+  process.stderr.write(`\r◊ ${name}: ${conf}% confident (${sec}s)   `);
+  return elapsed;
 }
 
-/** @return time percentiles and statistics in ms */
-function computeTimeStats(samplesNs: number[]) {
-  const samplesMs = samplesNs.map(s => s / msToNs);
-  const { min, max, sum } = getMinMaxSum(samplesNs);
-  const percentiles = getPercentiles(samplesNs);
-  const robust = getRobustMetrics(samplesMs);
-
-  return {
-    min: min / msToNs,
-    max: max / msToNs,
-    avg: sum / samplesNs.length / msToNs,
-    ...percentiles,
-    ...robust,
-  };
-}
-
-/** @return min, max, sum of samples */
-function getMinMaxSum(samples: number[]) {
-  const min = samples.reduce(
-    (a, b) => Math.min(a, b),
-    Number.POSITIVE_INFINITY,
-  );
-  const max = samples.reduce(
-    (a, b) => Math.max(a, b),
-    Number.NEGATIVE_INFINITY,
+/** @return true if convergence target met, or minTime elapsed with fallback confidence. */
+function shouldStop(
+  convergence: ConvergenceResult,
+  target: number,
+  elapsed: number,
+  minElapsed: number,
+): boolean {
+  if (convergence.converged && convergence.confidence >= target) return true;
+  return (
+    elapsed >= minElapsed &&
+    convergence.confidence >= Math.max(target, fallbackThreshold)
   );
-  const sum = samples.reduce((a, b) => a + b, 0);
-  return { min, max, sum };
-}
-
-/** @return percentiles in ms */
-function getPercentiles(samples: number[]) {
-  return {
-    p25: percentile(samples, 0.25) / msToNs,
-    p50: percentile(samples, 0.5) / msToNs,
-    p75: percentile(samples, 0.75) / msToNs,
-    p95: percentile(samples, 0.95) / msToNs,
-    p99: percentile(samples, 0.99) / msToNs,
-    p999: percentile(samples, 0.999) / msToNs,
-  };
-}
-
-/** @return robust variability metrics */
-function getRobustMetrics(samplesMs: number[]) {
-  const impact = getOutlierImpact(samplesMs);
-  return {
-    cv: coefficientOfVariation(samplesMs),
-    mad: medianAbsoluteDeviation(samplesMs),
-    outlierRate: impact.ratio,
-  };
 }

package/src/runners/BenchRunner.ts

@@ -1,7 +1,7 @@
-import type { BenchmarkSpec } from "../Benchmark.ts";
-import type { MeasuredResults } from "../MeasuredResults.ts";
+import type { BenchmarkSpec } from "./BenchmarkSpec.ts";
+import type { MeasuredResults } from "./MeasuredResults.ts";
 
-/** Interface for benchmark execution libraries */
+/** Benchmark execution strategy that collects timing samples. */
 export interface BenchRunner {
   runBench<T = unknown>(
     benchmark: BenchmarkSpec<T>,
@@ -10,6 +10,7 @@ export interface BenchRunner {
   ): Promise<MeasuredResults[]>;
 }
 
+/** Configuration for benchmark execution: timing limits, warmup, profiling, and V8 options. */
 export interface RunnerOptions {
   /** Minimum time to run each benchmark (milliseconds) */
   minTime?: number;
@@ -28,13 +29,11 @@ export interface RunnerOptions {
   /** Minimum samples required - mitata only */
   minSamples?: number;
   /** Force GC after each iteration (requires --expose-gc) */
-  collect?: boolean;
-  /** Enable CPU performance counters (requires root access) */
-  cpuCounters?: boolean;
+  gcForce?: boolean;
   /** Trace V8 optimization tiers (requires --allow-natives-syntax) */
   traceOpt?: boolean;
-  /** Skip post-warmup settle time (default: false) */
-  noSettle?: boolean;
+  /** Post-warmup settle time in ms for V8 background compilation (0 to skip) */
+  pauseWarmup?: number;
   /** Iterations before first pause (then pauseInterval applies) */
   pauseFirst?: number;
   /** Iterations between pauses for V8 optimization (0 to disable) */
@@ -43,15 +42,21 @@ export interface RunnerOptions {
   pauseDuration?: number;
   /** Collect GC stats via --trace-gc-nvp (requires worker mode) */
   gcStats?: boolean;
-  /** Heap sampling allocation attribution */
-  heapSample?: boolean;
-  /** Heap sampling interval in bytes */
-  heapInterval?: number;
-  /** Heap sampling stack depth */
-  heapDepth?: number;
+  /** Allocation sampling attribution */
+  alloc?: boolean;
+  /** Allocation sampling interval in bytes */
+  allocInterval?: number;
+  /** Allocation sampling stack depth */
+  allocDepth?: number;
+  /** V8 CPU time sampling */
+  profile?: boolean;
+  /** CPU sampling interval in microseconds (default 1000) */
+  profileInterval?: number;
+  /** Collect per-function execution counts via V8 precise coverage */
+  callCounts?: boolean;
 }
 
-/** Execute benchmark with optional parameters */
+/** Invoke the benchmark function, forwarding setup params. */
 export function executeBenchmark<T>(
   benchmark: BenchmarkSpec<T>,
   params?: T,

package/src/{Benchmark.ts → runners/BenchmarkSpec.ts}

@@ -1,4 +1,4 @@
-/** Single benchmark function specification */
+/** Benchmark function with optional module path for worker-mode serialization. */
 export interface BenchmarkSpec<T = unknown> {
   name: string;
   fn: BenchmarkFunction<T>;
@@ -10,23 +10,22 @@ export interface BenchmarkSpec<T = unknown> {
   setupExportName?: string;
 }
 
+/** Benchmark function, optionally receiving setup parameters from the group. */
 export type BenchmarkFunction<T = unknown> =
   | ((params: T) => void)
   | (() => void);
 
-/** Group of benchmarks with shared setup */
+/** Group of benchmarks with shared setup and optional baseline. */
 export interface BenchGroup<T = unknown> {
   name: string;
-  /** Prepare parameters for all benchmarks in this group */
   setup?: () => T | Promise<T>;
   benchmarks: BenchmarkSpec<T>[];
-  /** Baseline benchmark for comparison */
   baseline?: BenchmarkSpec<T>;
-  /** Metadata for reporting (e.g., lines of code) */
+  /** Metadata for reporting (e.g. lines of code). */
   metadata?: Record<string, any>;
 }
 
-/** Collection of benchmark groups */
+/** Named collection of benchmark groups. */
 export interface BenchSuite {
   name: string;
   groups: BenchGroup<any>[];

package/src/runners/CreateRunner.ts

@@ -1,11 +1,9 @@
-import { BasicRunner } from "./BasicRunner.ts";
 import type { BenchRunner } from "./BenchRunner.ts";
+import { TimingRunner } from "./TimingRunner.ts";
 
-export type KnownRunner = "basic";
+export type KnownRunner = "timing";
 
-/** @return benchmark runner */
-export async function createRunner(
-  _runnerName: KnownRunner,
-): Promise<BenchRunner> {
-  return new BasicRunner();
+/** Create a benchmark runner by name. */
+export async function createRunner(_name: KnownRunner): Promise<BenchRunner> {
+  return new TimingRunner();
 }