benchforge 0.1.11 → 0.2.4

package/src/report/BenchmarkReport.ts

@@ -0,0 +1,146 @@
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { diffCIs } from "../stats/BootstrapDifference.ts";
+import {
+  computeStat,
+  type DifferenceCI,
+  flipCI,
+  type StatKind,
+  swapDirection,
+} from "../stats/StatisticalUtils.ts";
+
+import type { AnyColumn } from "./text/TableReport.ts";
+
+/** Options that affect baseline comparison statistics */
+export interface ComparisonOptions {
+  /** Equivalence margin in percent (0 to disable) */
+  equivMargin?: number;
+  /** Disable Tukey trimming of outlier batches */
+  noBatchTrim?: boolean;
+}
+
+/** Benchmark results with optional baseline for comparison */
+export interface ReportGroup {
+  name: string;
+  reports: BenchmarkReport[];
+  baseline?: BenchmarkReport;
+}
+
+/** Results from a single benchmark run */
+export interface BenchmarkReport {
+  name: string;
+  measuredResults: MeasuredResults;
+  metadata?: UnknownRecord;
+}
+
+/** A titled group of related columns (one per report section) */
+export interface ReportSection {
+  title: string;
+  columns: ReportColumn[];
+}
+
+/** A table column with optional comparison behavior */
+export type ReportColumn = AnyColumn<Record<string, unknown>> & {
+  /** Add diff column after this column when baseline exists */
+  comparable?: boolean;
+  /** Set true for throughput metrics where higher values are better (e.g., lines/sec) */
+  higherIsBetter?: boolean;
+  /** Stat descriptor: framework computes value from samples via computeStat */
+  statKind?: StatKind;
+  /** Accessor for non-sample data (e.g., run count, metadata fields) */
+  value?: (results: MeasuredResults, metadata?: UnknownRecord) => unknown;
+  /** Convert a timing-domain value to display domain (e.g., ms to lines/sec) */
+  toDisplay?: (
+    timingValue: number,
+    metadata?: Record<string, unknown>,
+  ) => number;
+};
+
+export type UnknownRecord = Record<string, unknown>;
+
+/** Compute column values for a section from results + metadata.
+ *  statKind columns: computeStat(samples, kind), then toDisplay.
+ *  value columns: call the accessor directly. */
+export function computeColumnValues(
+  section: ReportSection,
+  results: MeasuredResults,
+  metadata?: UnknownRecord,
+): UnknownRecord {
+  return Object.fromEntries(
+    section.columns.map(col => {
+      const key = col.key ?? col.title;
+      if (col.value) return [key, col.value(results, metadata)];
+      if (col.statKind) {
+        const raw = computeStat(results.samples, col.statKind);
+        return [key, col.toDisplay ? col.toDisplay(raw, metadata) : raw];
+      }
+      return [key, undefined];
+    }),
+  );
+}
+
+/** Run each section's computeColumnValues and merge into one record */
+export function extractSectionValues(
+  measuredResults: MeasuredResults,
+  sections: ReadonlyArray<ReportSection>,
+  metadata?: UnknownRecord,
+): UnknownRecord {
+  const entries = sections.flatMap(s =>
+    Object.entries(computeColumnValues(s, measuredResults, metadata)),
+  );
+  return Object.fromEntries(entries);
+}
+
+/** All reports in a group, including the baseline if present */
+export function groupReports(group: ReportGroup): BenchmarkReport[] {
+  return group.baseline ? [...group.reports, group.baseline] : group.reports;
+}
+
+/** True if any result in the groups has the specified field with a defined value */
+export function hasField(
+  groups: ReportGroup[],
+  field: keyof MeasuredResults,
+): boolean {
+  return groups.some(group =>
+    groupReports(group).some(
+      ({ measuredResults }) => measuredResults[field] !== undefined,
+    ),
+  );
+}
+
+/** @return true if the first comparable column in sections has higherIsBetter set */
+export function isHigherIsBetter(sections: ReportSection[]): boolean {
+  return (
+    sections.flatMap(s => s.columns).find(c => c.comparable)?.higherIsBetter ??
+    false
+  );
+}
+
+/** @return the first comparable column with a statKind across all sections */
+export function findPrimaryColumn(
+  sections?: ReportSection[],
+): ReportColumn | undefined {
+  if (!sections) return undefined;
+  return sections.flatMap(s => s.columns).find(c => c.comparable && c.statKind);
+}
+
+/** Bootstrap difference CI for a column, using batch structure when available */
+export function computeDiffCI(
+  baseline: MeasuredResults | undefined,
+  current: MeasuredResults,
+  statKind: StatKind,
+  comparison?: ComparisonOptions,
+  higherIsBetter?: boolean,
+): DifferenceCI | undefined {
+  if (!baseline?.samples?.length || !current.samples?.length) return undefined;
+  const { equivMargin, noBatchTrim } = comparison ?? {};
+  const rawCIs = diffCIs(
+    baseline.samples,
+    baseline.batchOffsets,
+    current.samples,
+    current.batchOffsets,
+    [statKind],
+    { equivMargin, noBatchTrim },
+  );
+  if (!rawCIs[0]) return undefined;
+  return higherIsBetter ? swapDirection(flipCI(rawCIs[0])) : rawCIs[0];
+}

package/src/report/Colors.ts

@@ -0,0 +1,9 @@
+import pico from "picocolors";
+import type { Colors } from "picocolors/types";
+
+const isTest = process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+
+/** Picocolors instance that disables color in test environments */
+const colors: Colors = pico.createColors(!isTest);
+
+export default colors;

package/src/report/Formatters.ts

@@ -0,0 +1,110 @@
+import {
+  type CIDirection,
+  type DifferenceCI,
+  flipCI,
+} from "../stats/StatisticalUtils.ts";
+import colors from "./Colors.ts";
+
+const { red, green } = colors;
+
+const lowConfidence = 80;
+
+/** Format time in milliseconds with appropriate units */
+export function timeMs(ms: unknown): string | null {
+  if (typeof ms !== "number") return null;
+  if (ms < 0.001) return `${(ms * 1000000).toFixed(0)}ns`;
+  if (ms < 0.01) return `${(ms * 1000).toFixed(1)}μs`;
+  if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
+  if (ms >= 10) return `${ms.toFixed(0)}ms`;
+  return `${ms.toFixed(2)}ms`;
+}
+
+/** Format integer with thousand separators */
+export function integer(x: unknown): string | null {
+  if (typeof x !== "number") return null;
+  return new Intl.NumberFormat("en-US").format(Math.round(x));
+}
+
+/** Format fraction as percentage (0.473 → 47.3%) */
+export function percent(fraction: unknown, precision = 1): string | null {
+  if (typeof fraction !== "number") return null;
+  return `${Math.abs(fraction * 100).toFixed(precision)}%`;
+}
+
+/** Format percentage difference between two values */
+export function diffPercent(main: unknown, base: unknown): string {
+  if (typeof main !== "number" || typeof base !== "number") return " ";
+  const diff = main - base;
+  return coloredPercent(diff, base);
+}
+
+/** Format bytes with appropriate units. Use `space: true` for `1.5 KB` style. */
+export function formatBytes(
+  bytes: unknown,
+  opts?: { space?: boolean },
+): string | null {
+  if (typeof bytes !== "number") return null;
+  const s = opts?.space ? " " : "";
+  const [kb, mb, gb] = [1024, 1024 ** 2, 1024 ** 3];
+  if (bytes < kb) return `${bytes.toFixed(0)}${s}B`;
+  if (bytes < mb) return `${(bytes / kb).toFixed(1)}${s}KB`;
+  if (bytes < gb) return `${(bytes / mb).toFixed(1)}${s}MB`;
+  return `${(bytes / gb).toFixed(1)}${s}GB`;
+}
+
+/** Format percentage difference with confidence interval.
+ *  When higherIsBetter is true, flips the CI so positive = improvement. */
+export function formatDiffWithCI(
+  value: unknown,
+  higherIsBetter?: boolean,
+): string | null {
+  if (!isDifferenceCI(value)) return null;
+  const ci = higherIsBetter ? flipCI(value) : value;
+  const suffix = value.ciLevel === "sample" ? " *" : "";
+  const text = diffCIText(ci.percent, ci.ci) + suffix;
+  return colorByDirection(text, ci.direction);
+}
+
+/** @return truncated string with ellipsis if over maxLen */
+export function truncate(str: string, maxLen = 30): string {
+  return str.length > maxLen ? str.slice(0, maxLen - 3) + "..." : str;
+}
+
+/** @return signed percentage string (e.g. "+1.2%", "-3.4%") */
+export function formatSignedPercent(v: number): string {
+  const sign = v >= 0 ? "+" : "";
+  return `${sign}${v.toFixed(1)}%`;
+}
+
+/** @return convergence percentage with color for low values */
+export function formatConvergence(v: unknown): string {
+  if (typeof v !== "number") return "—";
+  const pct = `${Math.round(v)}%`;
+  return v < lowConfidence ? red(pct) : pct;
+}
+
+/** Format fraction as colored +/- percentage (positive = green, negative = red) */
+function coloredPercent(numerator: number, denominator: number): string {
+  const fraction = numerator / denominator;
+  if (!Number.isFinite(fraction)) return " ";
+  const sign = fraction >= 0 ? "+" : "-";
+  const percentStr = `${sign}${percent(fraction)}`;
+  return fraction >= 0 ? green(percentStr) : red(percentStr);
+}
+
+function isDifferenceCI(x: unknown): x is DifferenceCI {
+  return typeof x === "object" && x !== null && "ci" in x && "direction" in x;
+}
+
+/** @return formatted "pct [lo, hi]" text for a diff with CI */
+function diffCIText(pct: number, ci: [number, number]): string {
+  const [lo, hi] = ci.map(formatSignedPercent);
+  return `${formatSignedPercent(pct)} [${lo}, ${hi}]`;
+}
+
+/** @return text colored green for faster/equivalent, red for slower */
+function colorByDirection(text: string, direction: CIDirection): string {
+  if (direction === "faster" || direction === "equivalent") return green(text);
+  if (direction === "slower") return red(text);
+  return text;
+}

package/src/report/GcSections.ts

@@ -0,0 +1,151 @@
+import type { NavTiming } from "../profiling/browser/BrowserProfiler.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { average, median, percentile } from "../stats/StatisticalUtils.ts";
+import type { ReportSection } from "./BenchmarkReport.ts";
+import { formatBytes, integer, percent, timeMs } from "./Formatters.ts";
+
+/** Report section: GC time as fraction of total benchmark time. */
+export const gcSection: ReportSection = {
+  title: "gc",
+  columns: [
+    {
+      key: "gc",
+      title: "mean",
+      formatter: percent,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const { nodeGcTime, time, samples } = r;
+        if (!nodeGcTime || !time?.avg) return undefined;
+        const totalBenchTime = time.avg * samples.length;
+        if (totalBenchTime <= 0) return undefined;
+        const gcFraction = nodeGcTime.inRun / totalBenchTime;
+        return gcFraction <= 1 ? gcFraction : undefined;
+      },
+    },
+  ],
+};
+
+/** Report section: detailed GC stats from --trace-gc-nvp. */
+export const gcStatsSection: ReportSection = {
+  title: "gc",
+  columns: [
+    {
+      key: "allocPerIter",
+      title: "alloc/iter",
+      formatter: formatBytes,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const { gcStats, samples } = r;
+        if (!gcStats) return undefined;
+        const alloc = gcStats.totalAllocated;
+        return alloc != null ? alloc / (samples.length || 1) : undefined;
+      },
+    },
+    {
+      key: "collected",
+      title: "collected",
+      formatter: formatBytes,
+      comparable: true,
+      value: (r: MeasuredResults) => r.gcStats?.totalCollected || undefined,
+    },
+    {
+      key: "scavenges",
+      title: "scav",
+      formatter: integer,
+      comparable: true,
+      value: (r: MeasuredResults) => r.gcStats?.scavenges,
+    },
+    {
+      key: "fullGCs",
+      title: "full",
+      formatter: integer,
+      comparable: true,
+      value: (r: MeasuredResults) => r.gcStats?.markCompacts,
+    },
+    {
+      key: "promoPercent",
+      title: "promo%",
+      formatter: percent,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const gs = r.gcStats;
+        if (!gs) return undefined;
+        const alloc = gs.totalAllocated;
+        return alloc && alloc > 0 ? (gs.totalPromoted ?? 0) / alloc : undefined;
+      },
+    },
+    {
+      key: "pausePerIter",
+      title: "pause/iter",
+      formatter: timeMs,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const gs = r.gcStats;
+        return gs ? gs.gcPauseTime / (r.samples.length || 1) : undefined;
+      },
+    },
+  ],
+};
+
+/** Report section: browser GC stats from CDP tracing (subset of gcStatsSection). */
+export const browserGcStatsSection: ReportSection = {
+  title: "gc",
+  columns: [
+    gcStatsSection.columns.find(c => c.key === "collected")!,
+    gcStatsSection.columns.find(c => c.key === "scavenges")!,
+    gcStatsSection.columns.find(c => c.key === "fullGCs")!,
+    {
+      key: "pausePerIter",
+      title: "pause",
+      formatter: timeMs,
+      comparable: true,
+      value: gcStatsSection.columns.find(c => c.key === "pausePerIter")!.value!,
+    },
+  ],
+};
+
+/** Report sections: page-load stats (mean/p50/p99) across multiple iterations. */
+export const pageLoadStatsSections: ReportSection[] = [
+  pageLoadSection("DCL", n => n.domContentLoaded || undefined),
+  pageLoadSection("load", n => n.loadEvent || undefined),
+  pageLoadSection("LCP", n => n.lcp),
+];
+
+/** @return GC stats sections if enabled by CLI flags */
+export function gcSections(args: { "gc-stats"?: boolean }): ReportSection[] {
+  return args["gc-stats"] ? [gcStatsSection] : [];
+}
+
+/** Build a page-load section with mean/p50/p99 columns from NavTiming data */
+function pageLoadSection(
+  title: string,
+  extract: (n: NavTiming) => number | undefined,
+): ReportSection {
+  const vals = (r: MeasuredResults) => navValues(r.navTimings, extract);
+  const col = (suffix: string, stat: (v: number[]) => number) => ({
+    key: `${title.toLowerCase()}${suffix}`,
+    title: suffix.toLowerCase(),
+    formatter: timeMs,
+    value: (r: MeasuredResults) => {
+      const v = vals(r);
+      return v.length ? stat(v) : undefined;
+    },
+  });
+  return {
+    title,
+    columns: [
+      col("Mean", average),
+      col("P50", median),
+      col("P99", v => percentile(v, 0.99)),
+    ],
+  };
+}
+
+/** Extract one field from all NavTimings, filtering undefineds. */
+function navValues(
+  navs: NavTiming[] | undefined,
+  fn: (n: NavTiming) => number | undefined,
+): number[] {
+  if (!navs?.length) return [];
+  return navs.map(fn).filter((v): v is number => v != null);
+}

package/src/{GitUtils.ts → report/GitUtils.ts}

@@ -1,11 +1,14 @@
 import { execSync } from "node:child_process";
 import { existsSync, readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
-import type { GitVersion } from "./html/index.ts";
-import { formatDateWithTimezone } from "./html/index.ts";
+import { formatDateWithTimezone } from "../viewer/DateFormat.ts";
 
-export type { GitVersion } from "./html/index.ts";
-export { formatDateWithTimezone, formatRelativeTime } from "./html/index.ts";
+/** Git commit hash, date, and dirty status for version tracking */
+export interface GitVersion {
+  hash: string;
+  date: string;
+  dirty?: boolean;
+}
 
 /** Get current git version info. For dirty repos, uses most recent modified file date. */
 export function getCurrentGitVersion(): GitVersion | undefined {
@@ -14,7 +17,6 @@ export function getCurrentGitVersion(): GitVersion | undefined {
     const hash = exec("git rev-parse --short HEAD");
     const commitDate = exec("git log -1 --format=%aI");
     const dirty = exec("git status --porcelain").length > 0;
-
     const date = dirty
       ? (getMostRecentModifiedDate(".") ?? commitDate)
       : commitDate;
@@ -54,24 +56,21 @@ export function getMostRecentModifiedDate(dir: string): string | undefined {
       encoding: "utf-8",
       cwd: dir,
     });
-    const modifiedFiles = raw
+    const files = raw
       .trim()
       .split("\n")
-      .filter(line => line.length > 0)
-      .map(line => line.slice(3));
+      .filter(Boolean)
+      .map(l => l.slice(3));
+    if (!files.length) return undefined;
 
-    if (modifiedFiles.length === 0) return undefined;
-
-    let mostRecent = 0;
-    for (const file of modifiedFiles) {
+    const mtime = (f: string): number => {
       try {
-        const filePath = join(dir, file);
-        if (!existsSync(filePath)) continue;
-        const mtime = statSync(filePath).mtimeMs;
-        if (mtime > mostRecent) mostRecent = mtime;
-      } catch {}
-    }
-
+        return statSync(join(dir, f)).mtimeMs;
+      } catch {
+        return 0;
+      }
+    };
+    const mostRecent = Math.max(0, ...files.map(mtime));
     return mostRecent > 0 ? new Date(mostRecent).toISOString() : undefined;
   } catch {
     return undefined;