benchforge 0.1.9 → 0.2.4

package/src/report/Formatters.ts

@@ -0,0 +1,110 @@
+import {
+  type CIDirection,
+  type DifferenceCI,
+  flipCI,
+} from "../stats/StatisticalUtils.ts";
+import colors from "./Colors.ts";
+
+const { red, green } = colors;
+
+const lowConfidence = 80;
+
+/** Format time in milliseconds with appropriate units */
+export function timeMs(ms: unknown): string | null {
+  if (typeof ms !== "number") return null;
+  if (ms < 0.001) return `${(ms * 1000000).toFixed(0)}ns`;
+  if (ms < 0.01) return `${(ms * 1000).toFixed(1)}μs`;
+  if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
+  if (ms >= 10) return `${ms.toFixed(0)}ms`;
+  return `${ms.toFixed(2)}ms`;
+}
+
+/** Format integer with thousand separators */
+export function integer(x: unknown): string | null {
+  if (typeof x !== "number") return null;
+  return new Intl.NumberFormat("en-US").format(Math.round(x));
+}
+
+/** Format fraction as percentage (0.473 → 47.3%) */
+export function percent(fraction: unknown, precision = 1): string | null {
+  if (typeof fraction !== "number") return null;
+  return `${Math.abs(fraction * 100).toFixed(precision)}%`;
+}
+
+/** Format percentage difference between two values */
+export function diffPercent(main: unknown, base: unknown): string {
+  if (typeof main !== "number" || typeof base !== "number") return " ";
+  const diff = main - base;
+  return coloredPercent(diff, base);
+}
+
+/** Format bytes with appropriate units. Use `space: true` for `1.5 KB` style. */
+export function formatBytes(
+  bytes: unknown,
+  opts?: { space?: boolean },
+): string | null {
+  if (typeof bytes !== "number") return null;
+  const s = opts?.space ? " " : "";
+  const [kb, mb, gb] = [1024, 1024 ** 2, 1024 ** 3];
+  if (bytes < kb) return `${bytes.toFixed(0)}${s}B`;
+  if (bytes < mb) return `${(bytes / kb).toFixed(1)}${s}KB`;
+  if (bytes < gb) return `${(bytes / mb).toFixed(1)}${s}MB`;
+  return `${(bytes / gb).toFixed(1)}${s}GB`;
+}
+
+/** Format percentage difference with confidence interval.
+ *  When higherIsBetter is true, flips the CI so positive = improvement. */
+export function formatDiffWithCI(
+  value: unknown,
+  higherIsBetter?: boolean,
+): string | null {
+  if (!isDifferenceCI(value)) return null;
+  const ci = higherIsBetter ? flipCI(value) : value;
+  const suffix = value.ciLevel === "sample" ? " *" : "";
+  const text = diffCIText(ci.percent, ci.ci) + suffix;
+  return colorByDirection(text, ci.direction);
+}
+
+/** @return truncated string with ellipsis if over maxLen */
+export function truncate(str: string, maxLen = 30): string {
+  return str.length > maxLen ? str.slice(0, maxLen - 3) + "..." : str;
+}
+
+/** @return signed percentage string (e.g. "+1.2%", "-3.4%") */
+export function formatSignedPercent(v: number): string {
+  const sign = v >= 0 ? "+" : "";
+  return `${sign}${v.toFixed(1)}%`;
+}
+
+/** @return convergence percentage with color for low values */
+export function formatConvergence(v: unknown): string {
+  if (typeof v !== "number") return "—";
+  const pct = `${Math.round(v)}%`;
+  return v < lowConfidence ? red(pct) : pct;
+}
+
+/** Format fraction as colored +/- percentage (positive = green, negative = red) */
+function coloredPercent(numerator: number, denominator: number): string {
+  const fraction = numerator / denominator;
+  if (!Number.isFinite(fraction)) return " ";
+  const sign = fraction >= 0 ? "+" : "-";
+  const percentStr = `${sign}${percent(fraction)}`;
+  return fraction >= 0 ? green(percentStr) : red(percentStr);
+}
+
+function isDifferenceCI(x: unknown): x is DifferenceCI {
+  return typeof x === "object" && x !== null && "ci" in x && "direction" in x;
+}
+
+/** @return formatted "pct [lo, hi]" text for a diff with CI */
+function diffCIText(pct: number, ci: [number, number]): string {
+  const [lo, hi] = ci.map(formatSignedPercent);
+  return `${formatSignedPercent(pct)} [${lo}, ${hi}]`;
+}
+
+/** @return text colored green for faster/equivalent, red for slower */
+function colorByDirection(text: string, direction: CIDirection): string {
+  if (direction === "faster" || direction === "equivalent") return green(text);
+  if (direction === "slower") return red(text);
+  return text;
+}

package/src/report/GcSections.ts

@@ -0,0 +1,151 @@
+import type { NavTiming } from "../profiling/browser/BrowserProfiler.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { average, median, percentile } from "../stats/StatisticalUtils.ts";
+import type { ReportSection } from "./BenchmarkReport.ts";
+import { formatBytes, integer, percent, timeMs } from "./Formatters.ts";
+
+/** Report section: GC time as fraction of total benchmark time. */
+export const gcSection: ReportSection = {
+  title: "gc",
+  columns: [
+    {
+      key: "gc",
+      title: "mean",
+      formatter: percent,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const { nodeGcTime, time, samples } = r;
+        if (!nodeGcTime || !time?.avg) return undefined;
+        const totalBenchTime = time.avg * samples.length;
+        if (totalBenchTime <= 0) return undefined;
+        const gcFraction = nodeGcTime.inRun / totalBenchTime;
+        return gcFraction <= 1 ? gcFraction : undefined;
+      },
+    },
+  ],
+};
+
+/** Report section: detailed GC stats from --trace-gc-nvp. */
+export const gcStatsSection: ReportSection = {
+  title: "gc",
+  columns: [
+    {
+      key: "allocPerIter",
+      title: "alloc/iter",
+      formatter: formatBytes,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const { gcStats, samples } = r;
+        if (!gcStats) return undefined;
+        const alloc = gcStats.totalAllocated;
+        return alloc != null ? alloc / (samples.length || 1) : undefined;
+      },
+    },
+    {
+      key: "collected",
+      title: "collected",
+      formatter: formatBytes,
+      comparable: true,
+      value: (r: MeasuredResults) => r.gcStats?.totalCollected || undefined,
+    },
+    {
+      key: "scavenges",
+      title: "scav",
+      formatter: integer,
+      comparable: true,
+      value: (r: MeasuredResults) => r.gcStats?.scavenges,
+    },
+    {
+      key: "fullGCs",
+      title: "full",
+      formatter: integer,
+      comparable: true,
+      value: (r: MeasuredResults) => r.gcStats?.markCompacts,
+    },
+    {
+      key: "promoPercent",
+      title: "promo%",
+      formatter: percent,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const gs = r.gcStats;
+        if (!gs) return undefined;
+        const alloc = gs.totalAllocated;
+        return alloc && alloc > 0 ? (gs.totalPromoted ?? 0) / alloc : undefined;
+      },
+    },
+    {
+      key: "pausePerIter",
+      title: "pause/iter",
+      formatter: timeMs,
+      comparable: true,
+      value: (r: MeasuredResults) => {
+        const gs = r.gcStats;
+        return gs ? gs.gcPauseTime / (r.samples.length || 1) : undefined;
+      },
+    },
+  ],
+};
+
+/** Report section: browser GC stats from CDP tracing (subset of gcStatsSection). */
+export const browserGcStatsSection: ReportSection = {
+  title: "gc",
+  columns: [
+    gcStatsSection.columns.find(c => c.key === "collected")!,
+    gcStatsSection.columns.find(c => c.key === "scavenges")!,
+    gcStatsSection.columns.find(c => c.key === "fullGCs")!,
+    {
+      key: "pausePerIter",
+      title: "pause",
+      formatter: timeMs,
+      comparable: true,
+      value: gcStatsSection.columns.find(c => c.key === "pausePerIter")!.value!,
+    },
+  ],
+};
+
+/** Report sections: page-load stats (mean/p50/p99) across multiple iterations. */
+export const pageLoadStatsSections: ReportSection[] = [
+  pageLoadSection("DCL", n => n.domContentLoaded || undefined),
+  pageLoadSection("load", n => n.loadEvent || undefined),
+  pageLoadSection("LCP", n => n.lcp),
+];
+
+/** @return GC stats sections if enabled by CLI flags */
+export function gcSections(args: { "gc-stats"?: boolean }): ReportSection[] {
+  return args["gc-stats"] ? [gcStatsSection] : [];
+}
+
+/** Build a page-load section with mean/p50/p99 columns from NavTiming data */
+function pageLoadSection(
+  title: string,
+  extract: (n: NavTiming) => number | undefined,
+): ReportSection {
+  const vals = (r: MeasuredResults) => navValues(r.navTimings, extract);
+  const col = (suffix: string, stat: (v: number[]) => number) => ({
+    key: `${title.toLowerCase()}${suffix}`,
+    title: suffix.toLowerCase(),
+    formatter: timeMs,
+    value: (r: MeasuredResults) => {
+      const v = vals(r);
+      return v.length ? stat(v) : undefined;
+    },
+  });
+  return {
+    title,
+    columns: [
+      col("Mean", average),
+      col("P50", median),
+      col("P99", v => percentile(v, 0.99)),
+    ],
+  };
+}
+
+/** Extract one field from all NavTimings, filtering undefineds. */
+function navValues(
+  navs: NavTiming[] | undefined,
+  fn: (n: NavTiming) => number | undefined,
+): number[] {
+  if (!navs?.length) return [];
+  return navs.map(fn).filter((v): v is number => v != null);
+}

package/src/{GitUtils.ts → report/GitUtils.ts}

@@ -1,11 +1,14 @@
 import { execSync } from "node:child_process";
 import { existsSync, readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
-import type { GitVersion } from "./html/index.ts";
-import { formatDateWithTimezone } from "./html/index.ts";
+import { formatDateWithTimezone } from "../viewer/DateFormat.ts";
 
-export type { GitVersion } from "./html/index.ts";
-export { formatDateWithTimezone, formatRelativeTime } from "./html/index.ts";
+/** Git commit hash, date, and dirty status for version tracking */
+export interface GitVersion {
+  hash: string;
+  date: string;
+  dirty?: boolean;
+}
 
 /** Get current git version info. For dirty repos, uses most recent modified file date. */
 export function getCurrentGitVersion(): GitVersion | undefined {
@@ -14,7 +17,6 @@ export function getCurrentGitVersion(): GitVersion | undefined {
     const hash = exec("git rev-parse --short HEAD");
     const commitDate = exec("git log -1 --format=%aI");
     const dirty = exec("git status --porcelain").length > 0;
-
     const date = dirty
       ? (getMostRecentModifiedDate(".") ?? commitDate)
       : commitDate;
@@ -54,24 +56,21 @@ export function getMostRecentModifiedDate(dir: string): string | undefined {
       encoding: "utf-8",
       cwd: dir,
     });
-    const modifiedFiles = raw
+    const files = raw
       .trim()
       .split("\n")
-      .filter(line => line.length > 0)
-      .map(line => line.slice(3));
+      .filter(Boolean)
+      .map(l => l.slice(3));
+    if (!files.length) return undefined;
 
-    if (modifiedFiles.length === 0) return undefined;
-
-    let mostRecent = 0;
-    for (const file of modifiedFiles) {
+    const mtime = (f: string): number => {
       try {
-        const filePath = join(dir, file);
-        if (!existsSync(filePath)) continue;
-        const mtime = statSync(filePath).mtimeMs;
-        if (mtime > mostRecent) mostRecent = mtime;
-      } catch {}
-    }
-
+        return statSync(join(dir, f)).mtimeMs;
+      } catch {
+        return 0;
+      }
+    };
+    const mostRecent = Math.max(0, ...files.map(mtime));
     return mostRecent > 0 ? new Date(mostRecent).toISOString() : undefined;
   } catch {
     return undefined;

package/src/report/HtmlReport.ts

@@ -0,0 +1,223 @@
+import { cliDefaults } from "../cli/CliArgs.ts";
+import type { CoverageData } from "../profiling/node/CoverageTypes.ts";
+import {
+  filterSites,
+  flattenProfile,
+  totalBytes,
+} from "../profiling/node/HeapSampleReport.ts";
+import type { HeapProfile } from "../profiling/node/HeapSampler.ts";
+import { resolveProfile } from "../profiling/node/ResolvedProfile.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import type { DifferenceCI } from "../stats/StatisticalUtils.ts";
+import type {
+  BenchmarkEntry,
+  BenchmarkGroup,
+  CoverageSummary,
+  HeapSummary,
+  ReportData,
+  ViewerSection,
+} from "../viewer/ReportData.ts";
+import {
+  type BenchmarkReport,
+  type ComparisonOptions,
+  hasField,
+  type ReportGroup,
+  type ReportSection,
+  type UnknownRecord,
+} from "./BenchmarkReport.ts";
+import { gcStatsSection } from "./GcSections.ts";
+import type { GitVersion } from "./GitUtils.ts";
+import {
+  buildTimeSection,
+  optSection,
+  runsSection,
+} from "./StandardSections.ts";
+import {
+  buildViewerSections,
+  hasLowBatchCount,
+  isSingleBatch,
+  minBatches,
+} from "./ViewerSections.ts";
+
+/** Options for prepareHtmlData: report sections, git versions, and CLI args */
+export interface PrepareHtmlOptions extends ComparisonOptions {
+  cliArgs?: Record<string, unknown>;
+  sections?: ReportSection[];
+  currentVersion?: GitVersion;
+  baselineVersion?: GitVersion;
+}
+
+/** Context shared across reports in a group */
+interface GroupContext {
+  baseM?: MeasuredResults;
+  baseMeta?: UnknownRecord;
+  sections?: ReportSection[];
+  comparison?: ComparisonOptions;
+  lowBatches: boolean;
+}
+
+/** Convert benchmark results into a ReportData payload for the HTML viewer */
+export function prepareHtmlData(
+  groups: ReportGroup[],
+  options: PrepareHtmlOptions,
+): ReportData {
+  const { cliArgs, currentVersion, baselineVersion, equivMargin, noBatchTrim } =
+    options;
+  const comparison: ComparisonOptions = { equivMargin, noBatchTrim };
+  const sections = options.sections ?? defaultSections(groups, cliArgs);
+  return {
+    groups: groups.map(g => prepareGroupData(g, sections, comparison)),
+    metadata: {
+      timestamp: new Date().toISOString(),
+      bencherVersion: process.env.npm_package_version || "unknown",
+      cliArgs,
+      cliDefaults,
+      gcTrackingEnabled: cliArgs?.["gc-stats"] === true,
+      currentVersion,
+      baselineVersion,
+      environment: {
+        node: process.version,
+        platform: process.platform,
+        arch: process.arch,
+      },
+    },
+  };
+}
+
+/** Build default sections when caller doesn't provide custom ones */
+function defaultSections(
+  groups: ReportGroup[],
+  cliArgs?: Record<string, unknown>,
+): ReportSection[] {
+  const hasGc = cliArgs?.["gc-stats"] === true;
+  const hasOpt = hasField(groups, "optStatus");
+  const stats = typeof cliArgs?.stats === "string" ? cliArgs.stats : undefined;
+  return [
+    buildTimeSection(stats),
+    hasGc ? gcStatsSection : undefined,
+    hasOpt ? optSection : undefined,
+    runsSection,
+  ].filter((s): s is ReportSection => s !== undefined);
+}
+
+/** @return group data with structured ViewerSections and bootstrap CIs */
+function prepareGroupData(
+  group: ReportGroup,
+  sections?: ReportSection[],
+  comparison?: ComparisonOptions,
+): BenchmarkGroup {
+  const base = group.baseline;
+  const baseM = base?.measuredResults;
+  const baseline = base
+    ? { ...prepareBenchmarkData(base), comparisonCI: undefined }
+    : undefined;
+  const curM = group.reports[0]?.measuredResults;
+  const singleBatch = isSingleBatch(baseM, curM);
+  const lowBatches = hasLowBatchCount(baseM, curM);
+  const baseMeta = base?.metadata;
+  const ctx: GroupContext = {
+    baseM,
+    baseMeta,
+    sections,
+    comparison,
+    lowBatches,
+  };
+
+  return {
+    name: group.name,
+    baseline,
+    warnings: buildWarnings(singleBatch, lowBatches),
+    benchmarks: group.reports.map(r => prepareReportEntry(r, ctx)),
+  };
+}
+
+/** @return benchmark data with samples, stats, and profiling summaries */
+function prepareBenchmarkData(report: {
+  name: string;
+  measuredResults: MeasuredResults;
+  metadata?: UnknownRecord;
+}): Omit<BenchmarkEntry, "comparisonCI" | "sections"> {
+  const { measuredResults: m, name } = report;
+  return {
+    name,
+    samples: m.samples,
+    warmupSamples: m.warmupSamples,
+    allocationSamples: m.allocationSamples,
+    heapSamples: m.heapSamples,
+    gcEvents: m.nodeGcTime?.events,
+    optSamples: m.optSamples,
+    pausePoints: m.pausePoints,
+    batchOffsets: m.batchOffsets,
+    stats: m.time,
+    heapSize: m.heapSize,
+    totalTime: m.totalTime,
+    heapSummary: m.heapProfile ? summarizeHeap(m.heapProfile) : undefined,
+    coverageSummary: m.coverage ? summarizeCoverage(m.coverage) : undefined,
+  };
+}
+
+function buildWarnings(
+  singleBatch: boolean,
+  lowBatches: boolean,
+): string[] | undefined {
+  const parts: string[] = [];
+  const singleMsg =
+    "Confidence intervals may be too narrow (single batch). Use --batches for more accurate intervals.";
+  if (singleBatch) parts.push(singleMsg);
+  if (lowBatches)
+    parts.push(
+      `Too few batches for reliable comparison (need ${minBatches}+).`,
+    );
+  return parts.length ? parts : undefined;
+}
+
+/** @return a single benchmark entry with sections and comparison CI */
+function prepareReportEntry(
+  report: BenchmarkReport,
+  ctx: GroupContext,
+): BenchmarkEntry {
+  const m = report.measuredResults;
+  const sectionCtx = {
+    current: m,
+    baseline: ctx.baseM,
+    currentMeta: report.metadata,
+    baselineMeta: ctx.baseMeta,
+    comparison: ctx.comparison,
+  };
+  const sections = ctx.sections
+    ? buildViewerSections(ctx.sections, sectionCtx)
+    : undefined;
+  // Primary CI comes from the first primary row's comparisonCI (avoids duplicate bootstrap)
+  const comparisonCI = findPrimarySectionCI(sections);
+  return { ...prepareBenchmarkData(report), sections, comparisonCI };
+}
+
+/** Compute heap allocation summary from profile */
+function summarizeHeap(profile: HeapProfile): HeapSummary {
+  const resolved = resolveProfile(profile);
+  const userSites = filterSites(flattenProfile(resolved));
+  return { totalBytes: resolved.totalBytes, userBytes: totalBytes(userSites) };
+}
+
+/** Compute coverage summary from V8 coverage data */
+function summarizeCoverage(coverage: CoverageData): CoverageSummary {
+  const fns = coverage.scripts.flatMap(s => s.functions);
+  const called = fns.filter(
+    fn => fn.ranges.length > 0 && fn.ranges[0].count > 0,
+  );
+  const totalCalls = called.reduce((sum, fn) => sum + fn.ranges[0].count, 0);
+  return { functionCount: called.length, totalCalls };
+}
+
+/** Extract the comparison CI from the first primary row across all sections */
+function findPrimarySectionCI(
+  sections: ViewerSection[] | undefined,
+): DifferenceCI | undefined {
+  if (!sections) return undefined;
+  for (const section of sections) {
+    for (const row of section.rows) {
+      if (row.primary && row.comparisonCI) return row.comparisonCI;
+    }
+  }
+  return undefined;
+}

package/src/report/ParseStats.ts

@@ -0,0 +1,73 @@
+import type { StatKind } from "../stats/StatisticalUtils.ts";
+
+/** Parsed spec for one timing column selected via --stats. */
+export interface StatSpec {
+  key: string;
+  title: string;
+  statKind: StatKind;
+}
+
+/** Parse --stats into column specs. Throws on empty/invalid tokens. */
+export function parseStatsArg(stats: string): StatSpec[] {
+  const tokens = stats
+    .split(",")
+    .map(t => t.trim())
+    .filter(Boolean);
+  if (tokens.length === 0) {
+    throw new Error("--stats must list at least one column");
+  }
+  const seen = new Set<string>();
+  const specs: StatSpec[] = [];
+  for (const token of tokens) {
+    const spec = parseStatToken(token);
+    if (seen.has(spec.key)) continue;
+    seen.add(spec.key);
+    specs.push(spec);
+  }
+  return specs;
+}
+
+/** @return stat spec for a single --stats token. Throws on invalid input. */
+function parseStatToken(token: string): StatSpec {
+  const lower = token.toLowerCase();
+  if (lower === "mean" || lower === "avg") {
+    return { key: "mean", title: "mean", statKind: "mean" };
+  }
+  if (lower === "median") {
+    return { key: "p50", title: "p50", statKind: { percentile: 0.5 } };
+  }
+  if (lower === "min") {
+    return { key: "min", title: "min", statKind: "min" };
+  }
+  if (lower === "max") {
+    return { key: "max", title: "max", statKind: "max" };
+  }
+  const m = lower.match(/^p(\d+)$/);
+  if (m) return parsePercentileToken(token, m[1]);
+  throw new Error(
+    `invalid --stats token "${token}": expected mean, median, min, max, or p<N> (e.g. p50, p99, p999)`,
+  );
+}
+
+/** @return spec for a p<N> token, enforcing the 2-digit minimum and 9-prefix rule. */
+function parsePercentileToken(token: string, digits: string): StatSpec {
+  if (digits.length < 2) {
+    throw new Error(
+      `invalid --stats token "${token}": percentile needs at least 2 digits (e.g. p05, p50, p99, p999)`,
+    );
+  }
+  // 3+ digit tokens express sub-percentile precision (p999 = 99.9%,
+  // p9999 = 99.99%). Require leading 9 so p100/p500 don't silently
+  // map to 10%/50% — use 2-digit p10/p50 for those.
+  if (digits.length > 2 && digits[0] !== "9") {
+    throw new Error(
+      `invalid --stats token "${token}": percentiles with 3+ digits must start with 9 (e.g. p999, p9999); otherwise use 2-digit form (e.g. p50)`,
+    );
+  }
+  const q = Number(digits) / 10 ** digits.length;
+  return {
+    key: `p${digits}`,
+    title: `p${digits}`,
+    statKind: { percentile: q },
+  };
+}