benchforge 0.1.11 → 0.2.4

package/src/report/HtmlReport.ts

@@ -0,0 +1,223 @@
+import { cliDefaults } from "../cli/CliArgs.ts";
+import type { CoverageData } from "../profiling/node/CoverageTypes.ts";
+import {
+  filterSites,
+  flattenProfile,
+  totalBytes,
+} from "../profiling/node/HeapSampleReport.ts";
+import type { HeapProfile } from "../profiling/node/HeapSampler.ts";
+import { resolveProfile } from "../profiling/node/ResolvedProfile.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import type { DifferenceCI } from "../stats/StatisticalUtils.ts";
+import type {
+  BenchmarkEntry,
+  BenchmarkGroup,
+  CoverageSummary,
+  HeapSummary,
+  ReportData,
+  ViewerSection,
+} from "../viewer/ReportData.ts";
+import {
+  type BenchmarkReport,
+  type ComparisonOptions,
+  hasField,
+  type ReportGroup,
+  type ReportSection,
+  type UnknownRecord,
+} from "./BenchmarkReport.ts";
+import { gcStatsSection } from "./GcSections.ts";
+import type { GitVersion } from "./GitUtils.ts";
+import {
+  buildTimeSection,
+  optSection,
+  runsSection,
+} from "./StandardSections.ts";
+import {
+  buildViewerSections,
+  hasLowBatchCount,
+  isSingleBatch,
+  minBatches,
+} from "./ViewerSections.ts";
+
+/** Options for prepareHtmlData: report sections, git versions, and CLI args */
+export interface PrepareHtmlOptions extends ComparisonOptions {
+  cliArgs?: Record<string, unknown>;
+  sections?: ReportSection[];
+  currentVersion?: GitVersion;
+  baselineVersion?: GitVersion;
+}
+
+/** Context shared across reports in a group */
+interface GroupContext {
+  baseM?: MeasuredResults;
+  baseMeta?: UnknownRecord;
+  sections?: ReportSection[];
+  comparison?: ComparisonOptions;
+  lowBatches: boolean;
+}
+
+/** Convert benchmark results into a ReportData payload for the HTML viewer */
+export function prepareHtmlData(
+  groups: ReportGroup[],
+  options: PrepareHtmlOptions,
+): ReportData {
+  const { cliArgs, currentVersion, baselineVersion, equivMargin, noBatchTrim } =
+    options;
+  const comparison: ComparisonOptions = { equivMargin, noBatchTrim };
+  const sections = options.sections ?? defaultSections(groups, cliArgs);
+  return {
+    groups: groups.map(g => prepareGroupData(g, sections, comparison)),
+    metadata: {
+      timestamp: new Date().toISOString(),
+      bencherVersion: process.env.npm_package_version || "unknown",
+      cliArgs,
+      cliDefaults,
+      gcTrackingEnabled: cliArgs?.["gc-stats"] === true,
+      currentVersion,
+      baselineVersion,
+      environment: {
+        node: process.version,
+        platform: process.platform,
+        arch: process.arch,
+      },
+    },
+  };
+}
+
+/** Build default sections when caller doesn't provide custom ones */
+function defaultSections(
+  groups: ReportGroup[],
+  cliArgs?: Record<string, unknown>,
+): ReportSection[] {
+  const hasGc = cliArgs?.["gc-stats"] === true;
+  const hasOpt = hasField(groups, "optStatus");
+  const stats = typeof cliArgs?.stats === "string" ? cliArgs.stats : undefined;
+  return [
+    buildTimeSection(stats),
+    hasGc ? gcStatsSection : undefined,
+    hasOpt ? optSection : undefined,
+    runsSection,
+  ].filter((s): s is ReportSection => s !== undefined);
+}
+
+/** @return group data with structured ViewerSections and bootstrap CIs */
+function prepareGroupData(
+  group: ReportGroup,
+  sections?: ReportSection[],
+  comparison?: ComparisonOptions,
+): BenchmarkGroup {
+  const base = group.baseline;
+  const baseM = base?.measuredResults;
+  const baseline = base
+    ? { ...prepareBenchmarkData(base), comparisonCI: undefined }
+    : undefined;
+  const curM = group.reports[0]?.measuredResults;
+  const singleBatch = isSingleBatch(baseM, curM);
+  const lowBatches = hasLowBatchCount(baseM, curM);
+  const baseMeta = base?.metadata;
+  const ctx: GroupContext = {
+    baseM,
+    baseMeta,
+    sections,
+    comparison,
+    lowBatches,
+  };
+
+  return {
+    name: group.name,
+    baseline,
+    warnings: buildWarnings(singleBatch, lowBatches),
+    benchmarks: group.reports.map(r => prepareReportEntry(r, ctx)),
+  };
+}
+
+/** @return benchmark data with samples, stats, and profiling summaries */
+function prepareBenchmarkData(report: {
+  name: string;
+  measuredResults: MeasuredResults;
+  metadata?: UnknownRecord;
+}): Omit<BenchmarkEntry, "comparisonCI" | "sections"> {
+  const { measuredResults: m, name } = report;
+  return {
+    name,
+    samples: m.samples,
+    warmupSamples: m.warmupSamples,
+    allocationSamples: m.allocationSamples,
+    heapSamples: m.heapSamples,
+    gcEvents: m.nodeGcTime?.events,
+    optSamples: m.optSamples,
+    pausePoints: m.pausePoints,
+    batchOffsets: m.batchOffsets,
+    stats: m.time,
+    heapSize: m.heapSize,
+    totalTime: m.totalTime,
+    heapSummary: m.heapProfile ? summarizeHeap(m.heapProfile) : undefined,
+    coverageSummary: m.coverage ? summarizeCoverage(m.coverage) : undefined,
+  };
+}
+
+function buildWarnings(
+  singleBatch: boolean,
+  lowBatches: boolean,
+): string[] | undefined {
+  const parts: string[] = [];
+  const singleMsg =
+    "Confidence intervals may be too narrow (single batch). Use --batches for more accurate intervals.";
+  if (singleBatch) parts.push(singleMsg);
+  if (lowBatches)
+    parts.push(
+      `Too few batches for reliable comparison (need ${minBatches}+).`,
+    );
+  return parts.length ? parts : undefined;
+}
+
+/** @return a single benchmark entry with sections and comparison CI */
+function prepareReportEntry(
+  report: BenchmarkReport,
+  ctx: GroupContext,
+): BenchmarkEntry {
+  const m = report.measuredResults;
+  const sectionCtx = {
+    current: m,
+    baseline: ctx.baseM,
+    currentMeta: report.metadata,
+    baselineMeta: ctx.baseMeta,
+    comparison: ctx.comparison,
+  };
+  const sections = ctx.sections
+    ? buildViewerSections(ctx.sections, sectionCtx)
+    : undefined;
+  // Primary CI comes from the first primary row's comparisonCI (avoids duplicate bootstrap)
+  const comparisonCI = findPrimarySectionCI(sections);
+  return { ...prepareBenchmarkData(report), sections, comparisonCI };
+}
+
+/** Compute heap allocation summary from profile */
+function summarizeHeap(profile: HeapProfile): HeapSummary {
+  const resolved = resolveProfile(profile);
+  const userSites = filterSites(flattenProfile(resolved));
+  return { totalBytes: resolved.totalBytes, userBytes: totalBytes(userSites) };
+}
+
+/** Compute coverage summary from V8 coverage data */
+function summarizeCoverage(coverage: CoverageData): CoverageSummary {
+  const fns = coverage.scripts.flatMap(s => s.functions);
+  const called = fns.filter(
+    fn => fn.ranges.length > 0 && fn.ranges[0].count > 0,
+  );
+  const totalCalls = called.reduce((sum, fn) => sum + fn.ranges[0].count, 0);
+  return { functionCount: called.length, totalCalls };
+}
+
+/** Extract the comparison CI from the first primary row across all sections */
+function findPrimarySectionCI(
+  sections: ViewerSection[] | undefined,
+): DifferenceCI | undefined {
+  if (!sections) return undefined;
+  for (const section of sections) {
+    for (const row of section.rows) {
+      if (row.primary && row.comparisonCI) return row.comparisonCI;
+    }
+  }
+  return undefined;
+}

package/src/report/ParseStats.ts

@@ -0,0 +1,73 @@
+import type { StatKind } from "../stats/StatisticalUtils.ts";
+
+/** Parsed spec for one timing column selected via --stats. */
+export interface StatSpec {
+  key: string;
+  title: string;
+  statKind: StatKind;
+}
+
+/** Parse --stats into column specs. Throws on empty/invalid tokens. */
+export function parseStatsArg(stats: string): StatSpec[] {
+  const tokens = stats
+    .split(",")
+    .map(t => t.trim())
+    .filter(Boolean);
+  if (tokens.length === 0) {
+    throw new Error("--stats must list at least one column");
+  }
+  const seen = new Set<string>();
+  const specs: StatSpec[] = [];
+  for (const token of tokens) {
+    const spec = parseStatToken(token);
+    if (seen.has(spec.key)) continue;
+    seen.add(spec.key);
+    specs.push(spec);
+  }
+  return specs;
+}
+
+/** @return stat spec for a single --stats token. Throws on invalid input. */
+function parseStatToken(token: string): StatSpec {
+  const lower = token.toLowerCase();
+  if (lower === "mean" || lower === "avg") {
+    return { key: "mean", title: "mean", statKind: "mean" };
+  }
+  if (lower === "median") {
+    return { key: "p50", title: "p50", statKind: { percentile: 0.5 } };
+  }
+  if (lower === "min") {
+    return { key: "min", title: "min", statKind: "min" };
+  }
+  if (lower === "max") {
+    return { key: "max", title: "max", statKind: "max" };
+  }
+  const m = lower.match(/^p(\d+)$/);
+  if (m) return parsePercentileToken(token, m[1]);
+  throw new Error(
+    `invalid --stats token "${token}": expected mean, median, min, max, or p<N> (e.g. p50, p99, p999)`,
+  );
+}
+
+/** @return spec for a p<N> token, enforcing the 2-digit minimum and 9-prefix rule. */
+function parsePercentileToken(token: string, digits: string): StatSpec {
+  if (digits.length < 2) {
+    throw new Error(
+      `invalid --stats token "${token}": percentile needs at least 2 digits (e.g. p05, p50, p99, p999)`,
+    );
+  }
+  // 3+ digit tokens express sub-percentile precision (p999 = 99.9%,
+  // p9999 = 99.99%). Require leading 9 so p100/p500 don't silently
+  // map to 10%/50% — use 2-digit p10/p50 for those.
+  if (digits.length > 2 && digits[0] !== "9") {
+    throw new Error(
+      `invalid --stats token "${token}": percentiles with 3+ digits must start with 9 (e.g. p999, p9999); otherwise use 2-digit form (e.g. p50)`,
+    );
+  }
+  const q = Number(digits) / 10 ** digits.length;
+  return {
+    key: `p${digits}`,
+    title: `p${digits}`,
+    statKind: { percentile: q },
+  };
+}

package/src/report/StandardSections.ts

@@ -0,0 +1,147 @@
+import type {
+  MeasuredResults,
+  OptStatusInfo,
+} from "../runners/MeasuredResults.ts";
+import { isBootstrappable } from "../stats/StatisticalUtils.ts";
+import type { ReportSection } from "./BenchmarkReport.ts";
+import { formatConvergence, timeMs } from "./Formatters.ts";
+import { gcSections } from "./GcSections.ts";
+import { parseStatsArg } from "./ParseStats.ts";
+
+/** Default timing section: mean, p50, p99. */
+export const timeSection: ReportSection = buildTimeSection();
+
+/** Report section: number of sample iterations. */
+export const runsSection: ReportSection = {
+  title: "",
+  columns: [
+    {
+      key: "runs",
+      title: "runs",
+      formatter: v => String(v),
+      value: (r: MeasuredResults) => r.samples.length,
+    },
+  ],
+};
+
+/** Report section: total sampling duration. */
+export const totalTimeSection: ReportSection = {
+  title: "",
+  columns: [
+    {
+      key: "totalTime",
+      title: "time",
+      formatter: formatTotalTime,
+      value: (r: MeasuredResults) => r.totalTime,
+    },
+  ],
+};
+
+/** Report sections: timing stats and convergence for adaptive mode. */
+export const adaptiveSections: ReportSection[] = [
+  {
+    title: "time",
+    columns: [
+      {
+        key: "median",
+        title: "median",
+        formatter: timeMs,
+        comparable: true,
+        statKind: { percentile: 0.5 },
+      },
+      {
+        key: "mean",
+        title: "mean",
+        formatter: timeMs,
+        comparable: true,
+        statKind: "mean",
+      },
+      {
+        key: "p99",
+        title: "p99",
+        formatter: timeMs,
+        statKind: { percentile: 0.99 },
+      },
+    ],
+  },
+  {
+    title: "",
+    columns: [
+      {
+        key: "convergence",
+        title: "conv%",
+        formatter: formatConvergence,
+        value: (r: MeasuredResults) => r.convergence?.confidence,
+      },
+    ],
+  },
+];
+
+/** Report section: V8 optimization tier distribution and deopt count. */
+export const optSection: ReportSection = {
+  title: "v8 opt",
+  columns: [
+    {
+      key: "tiers",
+      title: "tiers",
+      formatter: v => (typeof v === "string" ? v : ""),
+      value: (r: MeasuredResults) => {
+        const opt = r.optStatus;
+        return opt ? formatTierSummary(opt) : undefined;
+      },
+    },
+    {
+      key: "deopt",
+      title: "deopt",
+      formatter: v => (typeof v === "number" ? String(v) : ""),
+      value: (r: MeasuredResults) => {
+        const opt = r.optStatus;
+        return opt && opt.deoptCount > 0 ? opt.deoptCount : undefined;
+      },
+    },
+  ],
+};
+
+/** Build a time section with user-chosen percentile/stat columns. */
+export function buildTimeSection(stats = "mean,p50,p99"): ReportSection {
+  const specs = parseStatsArg(stats);
+  return {
+    title: "time",
+    columns: specs.map(s => ({
+      key: s.key,
+      title: s.title,
+      formatter: timeMs,
+      comparable: isBootstrappable(s.statKind),
+      statKind: s.statKind,
+    })),
+  };
+}
+
+/** Format V8 tier distribution sorted by count (e.g. "turbofan:85% sparkplug:15%"). */
+export function formatTierSummary(
+  opt: OptStatusInfo,
+  sep = ":",
+  glue = " ",
+): string {
+  const tiers = Object.entries(opt.byTier);
+  const total = tiers.reduce((s, [, t]) => s + t.count, 0);
+  const pct = (n: number) => `${((n / total) * 100).toFixed(0)}%`;
+  return tiers
+    .sort((a, b) => b[1].count - a[1].count)
+    .map(([name, t]) => `${name}${sep}${pct(t.count)}`)
+    .join(glue);
+}
+
+/** @return default report sections from CLI flags (GC stats if enabled, plus run count). */
+export function buildGenericSections(args: {
+  "gc-stats"?: boolean;
+  alloc?: boolean;
+}): ReportSection[] {
+  return [...gcSections(args), runsSection];
+}
+
+/** Format total time; brackets indicate >= 30s. */
+function formatTotalTime(v: unknown): string {
+  if (typeof v !== "number") return "";
+  return v >= 30 ? `[${v.toFixed(1)}s]` : `${v.toFixed(1)}s`;
+}