benchforge 0.1.11 → 0.2.4

package/src/StandardSections.ts

@@ -1,261 +0,0 @@
-import type { ReportColumnGroup, ResultsMapper } from "./BenchmarkReport.ts";
-import type { MeasuredResults } from "./MeasuredResults.ts";
-import { formatConvergence } from "./table-util/ConvergenceFormatters.ts";
-import {
-  formatBytes,
-  integer,
-  percent,
-  percentPrecision,
-  timeMs,
-} from "./table-util/Formatters.ts";
-
-export interface TimeStats {
-  mean?: number;
-  p50?: number;
-  p99?: number;
-}
-
-export interface GcSectionStats {
-  gc?: number; // GC time as fraction of total bench time
-}
-
-export interface GcStatsInfo {
-  allocPerIter?: number;
-  collected?: number;
-  scavenges?: number;
-  fullGCs?: number;
-  promoPercent?: number;
-  pausePerIter?: number;
-}
-
-export interface CpuStats {
-  cpuCacheMiss?: number;
-  cpuStall?: number;
-}
-
-export interface RunStats {
-  runs?: number;
-}
-
-export interface AdaptiveStats {
-  median?: number;
-  mean?: number;
-  p99?: number;
-  convergence?: number;
-}
-
-export interface OptStats {
-  tiers?: string; // tier distribution summary
-  deopt?: number; // deopt count
-}
-
-/** Section: mean, p50, p99 timing */
-export const timeSection: ResultsMapper<TimeStats> = {
-  extract: (results: MeasuredResults) => ({
-    mean: results.time?.avg,
-    p50: results.time?.p50,
-    p99: results.time?.p99,
-  }),
-  columns: (): ReportColumnGroup<TimeStats>[] => [
-    {
-      groupTitle: "time",
-      columns: [
-        { key: "mean", title: "mean", formatter: timeMs, comparable: true },
-        { key: "p50", title: "p50", formatter: timeMs, comparable: true },
-        { key: "p99", title: "p99", formatter: timeMs, comparable: true },
-      ],
-    },
-  ],
-};
-
-/** Section: GC time as fraction of total benchmark time (Node performance hooks) */
-export const gcSection: ResultsMapper<GcSectionStats> = {
-  extract: (results: MeasuredResults) => {
-    const { nodeGcTime, time, samples } = results;
-    if (!nodeGcTime || !time?.avg) return { gc: undefined };
-    const totalBenchTime = time.avg * samples.length;
-    if (totalBenchTime <= 0) return { gc: undefined };
-    const gcTime = nodeGcTime.inRun / totalBenchTime;
-    // GC time can't exceed total time
-    return { gc: gcTime <= 1 ? gcTime : undefined };
-  },
-  columns: (): ReportColumnGroup<GcSectionStats>[] => [
-    {
-      groupTitle: "gc",
-      columns: [
-        { key: "gc", title: "mean", formatter: percent, comparable: true },
-      ],
-    },
-  ],
-};
-
-/** Section: detailed GC stats from --trace-gc-nvp (allocation, promotion, pauses) */
-export const gcStatsSection: ResultsMapper<GcStatsInfo> = {
-  extract: (results: MeasuredResults) => {
-    const { gcStats, samples } = results;
-    if (!gcStats) return {};
-    const iterations = samples.length || 1;
-    const { totalAllocated, totalPromoted } = gcStats;
-    const hasAlloc = totalAllocated && totalAllocated > 0;
-    const promoPercent = hasAlloc
-      ? (totalPromoted ?? 0) / totalAllocated
-      : undefined;
-    return {
-      allocPerIter:
-        totalAllocated != null ? totalAllocated / iterations : undefined,
-      collected: gcStats.totalCollected || undefined,
-      scavenges: gcStats.scavenges,
-      fullGCs: gcStats.markCompacts,
-      promoPercent,
-      pausePerIter: gcStats.gcPauseTime / iterations,
-    };
-  },
-  columns: (): ReportColumnGroup<GcStatsInfo>[] => [
-    {
-      groupTitle: "gc",
-      columns: [
-        { key: "allocPerIter", title: "alloc/iter", formatter: formatBytes },
-        { key: "collected", title: "collected", formatter: formatBytes },
-        { key: "scavenges", title: "scav", formatter: integer },
-        { key: "fullGCs", title: "full", formatter: integer },
-        { key: "promoPercent", title: "promo%", formatter: percent },
-        { key: "pausePerIter", title: "pause/iter", formatter: timeMs },
-      ],
-    },
-  ],
-};
-
-/** Browser GC section: only fields available from CDP tracing */
-export const browserGcStatsSection: ResultsMapper<GcStatsInfo> = {
-  extract: gcStatsSection.extract,
-  columns: (): ReportColumnGroup<GcStatsInfo>[] => [
-    {
-      groupTitle: "gc",
-      columns: [
-        { key: "collected", title: "collected", formatter: formatBytes },
-        { key: "scavenges", title: "scav", formatter: integer },
-        { key: "fullGCs", title: "full", formatter: integer },
-        { key: "pausePerIter", title: "pause", formatter: timeMs },
-      ],
-    },
-  ],
-};
-
-/** Section: CPU L1 cache miss rate and stall rate (requires @mitata/counters) */
-export const cpuSection: ResultsMapper<CpuStats> = {
-  extract: (results: MeasuredResults) => ({
-    cpuCacheMiss: results.cpuCacheMiss,
-    cpuStall: results.cpuStall,
-  }),
-  columns: (): ReportColumnGroup<CpuStats>[] => [
-    {
-      groupTitle: "cpu",
-      columns: [
-        { key: "cpuCacheMiss", title: "L1 miss", formatter: percent },
-        { key: "cpuStall", title: "stalls", formatter: percentPrecision(2) },
-      ],
-    },
-  ],
-};
-
-/** Section: number of sample iterations */
-export const runsSection: ResultsMapper<RunStats> = {
-  extract: (results: MeasuredResults) => ({
-    runs: results.samples.length,
-  }),
-  columns: (): ReportColumnGroup<RunStats>[] => [
-    { columns: [{ key: "runs", title: "runs", formatter: integer }] },
-  ],
-};
-
-/** Section: total sampling duration in seconds (brackets if >= 30s) */
-export const totalTimeSection: ResultsMapper<{ totalTime?: number }> = {
-  extract: (results: MeasuredResults) => ({
-    totalTime: results.totalTime,
-  }),
-  columns: (): ReportColumnGroup<{ totalTime?: number }>[] => [
-    {
-      columns: [
-        {
-          key: "totalTime",
-          title: "time",
-          formatter: v => {
-            if (typeof v !== "number") return "";
-            return v >= 30 ? `[${v.toFixed(1)}s]` : `${v.toFixed(1)}s`;
-          },
-        },
-      ],
-    },
-  ],
-};
-
-/** Section: median, mean, p99, and convergence for adaptive mode */
-export const adaptiveSection: ResultsMapper<AdaptiveStats> = {
-  extract: (results: MeasuredResults) => ({
-    median: results.time?.p50,
-    mean: results.time?.avg,
-    p99: results.time?.p99,
-    convergence: results.convergence?.confidence,
-  }),
-  columns: (): ReportColumnGroup<AdaptiveStats>[] => [
-    {
-      groupTitle: "time",
-      columns: [
-        { key: "median", title: "median", formatter: timeMs, comparable: true },
-        { key: "mean", title: "mean", formatter: timeMs, comparable: true },
-        { key: "p99", title: "p99", formatter: timeMs },
-      ],
-    },
-    {
-      columns: [
-        { key: "convergence", title: "conv%", formatter: formatConvergence },
-      ],
-    },
-  ],
-};
-
-/** Section: V8 optimization tier distribution and deopt count */
-export const optSection: ResultsMapper<OptStats> = {
-  extract: (results: MeasuredResults) => {
-    const opt = results.optStatus;
-    if (!opt) return {};
-
-    const total = Object.values(opt.byTier).reduce((s, t) => s + t.count, 0);
-    const tierParts = Object.entries(opt.byTier)
-      .sort((a, b) => b[1].count - a[1].count)
-      .map(([name, t]) => `${name}:${((t.count / total) * 100).toFixed(0)}%`);
-
-    return {
-      tiers: tierParts.join(" "),
-      deopt: opt.deoptCount > 0 ? opt.deoptCount : undefined,
-    };
-  },
-  columns: (): ReportColumnGroup<OptStats>[] => [
-    {
-      groupTitle: "v8 opt",
-      columns: [
-        {
-          key: "tiers",
-          title: "tiers",
-          formatter: v => (typeof v === "string" ? v : ""),
-        },
-        {
-          key: "deopt",
-          title: "deopt",
-          formatter: v => (typeof v === "number" ? String(v) : ""),
-        },
-      ],
-    },
-  ],
-};
-
-/** Build generic sections based on CLI flags */
-export function buildGenericSections(args: {
-  "gc-stats"?: boolean;
-  "heap-sample"?: boolean;
-}): ResultsMapper[] {
-  const sections: ResultsMapper[] = [];
-  if (args["gc-stats"]) sections.push(gcStatsSection);
-  sections.push(runsSection);
-  return sections;
-}

package/src/StatisticalUtils.ts

@@ -1,175 +0,0 @@
-/** Bootstrap estimate with confidence interval and raw resample data */
-export interface BootstrapResult {
-  estimate: number;
-  ci: [number, number];
-  samples: number[];
-}
-
-export type CIDirection = "faster" | "slower" | "uncertain";
-
-/** Binned histogram for efficient transfer to browser */
-export interface HistogramBin {
-  x: number; // bin center
-  count: number;
-}
-
-/** Bootstrap confidence interval for percentage difference between two samples */
-export interface DifferenceCI {
-  percent: number;
-  ci: [number, number];
-  direction: CIDirection;
-  /** Histogram of bootstrap distribution for visualization */
-  histogram?: HistogramBin[];
-}
-
-/** Options for bootstrap resampling methods */
-type BootstrapOptions = {
-  resamples?: number;
-  confidence?: number;
-};
-const confidence = 0.95;
-const outlierMultiplier = 1.5; // Tukey's fence multiplier
-const bootstrapSamples = 10000;
-
-/** @return relative standard deviation (coefficient of variation) */
-export function coefficientOfVariation(samples: number[]): number {
-  const mean = average(samples);
-  if (mean === 0) return 0;
-  const stdDev = standardDeviation(samples);
-  return stdDev / mean;
-}
-
-/** @return median absolute deviation for robust variability measure */
-export function medianAbsoluteDeviation(samples: number[]): number {
-  const median = percentile(samples, 0.5);
-  const deviations = samples.map(x => Math.abs(x - median));
-  return percentile(deviations, 0.5);
-}
-
-/** @return outliers detected via Tukey's interquartile range method */
-export function findOutliers(samples: number[]): {
-  rate: number;
-  indices: number[];
-} {
-  const q1 = percentile(samples, 0.25);
-  const q3 = percentile(samples, 0.75);
-  const iqr = q3 - q1;
-  const lowerBound = q1 - outlierMultiplier * iqr;
-  const upperBound = q3 + outlierMultiplier * iqr;
-
-  const indices = samples
-    .map((v, i) => (v < lowerBound || v > upperBound ? i : -1))
-    .filter(i => i >= 0);
-  return { rate: indices.length / samples.length, indices };
-}
-
-/** @return bootstrap confidence interval for median */
-export function bootstrapMedian(
-  samples: number[],
-  options: BootstrapOptions = {},
-): BootstrapResult {
-  const { resamples = bootstrapSamples, confidence: conf = confidence } =
-    options;
-  const medians = generateMedians(samples, resamples);
-  const ci = computeInterval(medians, conf);
-
-  return {
-    estimate: percentile(samples, 0.5),
-    ci,
-    samples: medians,
-  };
-}
-
-/** @return mean of values */
-export function average(values: number[]): number {
-  const sum = values.reduce((a, b) => a + b, 0);
-  return sum / values.length;
-}
-
-/** @return standard deviation with Bessel's correction */
-export function standardDeviation(samples: number[]): number {
-  if (samples.length <= 1) return 0;
-  const mean = average(samples);
-  const variance =
-    samples.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (samples.length - 1);
-  return Math.sqrt(variance);
-}
-
-/** @return value at percentile p (0-1) */
-export function percentile(values: number[], p: number): number {
-  const sorted = [...values].sort((a, b) => a - b);
-  const index = Math.ceil(sorted.length * p) - 1;
-  return sorted[Math.max(0, index)];
-}
-
-/** @return bootstrap resample with replacement */
-export function createResample(samples: number[]): number[] {
-  const n = samples.length;
-  const rand = () => samples[Math.floor(Math.random() * n)];
-  return Array.from({ length: n }, rand);
-}
-
-/** @return bootstrap CI for percentage difference between baseline and current medians */
-export function bootstrapDifferenceCI(
-  baseline: number[],
-  current: number[],
-  options: BootstrapOptions = {},
-): DifferenceCI {
-  const { resamples = bootstrapSamples, confidence: conf = confidence } =
-    options;
-
-  const baselineMedian = percentile(baseline, 0.5);
-  const currentMedian = percentile(current, 0.5);
-  const observedPercent =
-    ((currentMedian - baselineMedian) / baselineMedian) * 100;
-
-  const diffs: number[] = [];
-  for (let i = 0; i < resamples; i++) {
-    const resB = createResample(baseline);
-    const resC = createResample(current);
-    const medB = percentile(resB, 0.5);
-    const medC = percentile(resC, 0.5);
-    diffs.push(((medC - medB) / medB) * 100);
-  }
-
-  const ci = computeInterval(diffs, conf);
-  const excludesZero = ci[0] > 0 || ci[1] < 0;
-  let direction: CIDirection = "uncertain";
-  if (excludesZero) direction = observedPercent < 0 ? "faster" : "slower";
-  const histogram = binValues(diffs);
-  return { percent: observedPercent, ci, direction, histogram };
-}
-
-/** @return medians from bootstrap resamples */
-function generateMedians(samples: number[], resamples: number): number[] {
-  return Array.from({ length: resamples }, () =>
-    percentile(createResample(samples), 0.5),
-  );
-}
-
-/** @return confidence interval [lower, upper] */
-function computeInterval(
-  medians: number[],
-  confidence: number,
-): [number, number] {
-  const alpha = (1 - confidence) / 2;
-  const lower = percentile(medians, alpha);
-  const upper = percentile(medians, 1 - alpha);
-  return [lower, upper];
-}
-
-/** Bin values into histogram for compact visualization */
-function binValues(values: number[], binCount = 30): HistogramBin[] {
-  const sorted = [...values].sort((a, b) => a - b);
-  const min = sorted[0];
-  const max = sorted[sorted.length - 1];
-  if (min === max) return [{ x: min, count: values.length }];
-
-  const step = (max - min) / binCount;
-  const counts = new Array(binCount).fill(0);
-  for (const v of values) {
-    const bin = Math.min(Math.floor((v - min) / step), binCount - 1);
-    counts[bin]++;
-  }
-  return counts.map((count, i) => ({ x: min + (i + 0.5) * step, count }));
-}

package/src/TypeUtil.ts

@@ -1,8 +0,0 @@
-/** Convert union to intersection - https://mighdoll.dev/blog/modern-typescript-intersection/ */
-export type UnionToIntersection<U> = (
-  U extends any
-    ? (k: U) => void
-    : never
-) extends (k: infer I extends U) => void
-  ? I
-  : never;

package/src/browser/BrowserGcStats.ts

@@ -1,44 +0,0 @@
-import {
-  aggregateGcStats,
-  type GcEvent,
-  type GcStats,
-} from "../runners/GcStats.ts";
-
-/** CDP trace event from Tracing.dataCollected */
-export interface TraceEvent {
-  cat: string;
-  name: string;
-  ph: string;
-  dur?: number; // microseconds
-  args?: Record<string, any>;
-}
-
-/** Parse CDP trace events (MinorGC/MajorGC) into GcEvent[] */
-export function parseGcTraceEvents(traceEvents: TraceEvent[]): GcEvent[] {
-  return traceEvents.flatMap(e => {
-    if (e.ph !== "X") return [];
-    const type = gcType(e.name);
-    if (!type) return [];
-    const durUs = e.dur ?? 0;
-    const heapBefore: number = e.args?.usedHeapSizeBefore ?? 0;
-    const heapAfter: number = e.args?.usedHeapSizeAfter ?? 0;
-    return [
-      {
-        type,
-        pauseMs: durUs / 1000,
-        collected: Math.max(0, heapBefore - heapAfter),
-      },
-    ];
-  });
-}
-
-/** Parse CDP trace events and aggregate into GcStats */
-export function browserGcStats(traceEvents: TraceEvent[]): GcStats {
-  return aggregateGcStats(parseGcTraceEvents(traceEvents));
-}
-
-function gcType(name: string): GcEvent["type"] | undefined {
-  if (name === "MinorGC") return "scavenge";
-  if (name === "MajorGC") return "mark-compact";
-  return undefined;
-}