benchforge 0.1.9 → 0.2.4

package/src/report/StandardSections.ts

@@ -0,0 +1,147 @@
+import type {
+  MeasuredResults,
+  OptStatusInfo,
+} from "../runners/MeasuredResults.ts";
+import { isBootstrappable } from "../stats/StatisticalUtils.ts";
+import type { ReportSection } from "./BenchmarkReport.ts";
+import { formatConvergence, timeMs } from "./Formatters.ts";
+import { gcSections } from "./GcSections.ts";
+import { parseStatsArg } from "./ParseStats.ts";
+
+/** Default timing section: mean, p50, p99. */
+export const timeSection: ReportSection = buildTimeSection();
+
+/** Report section: number of sample iterations. */
+export const runsSection: ReportSection = {
+  title: "",
+  columns: [
+    {
+      key: "runs",
+      title: "runs",
+      formatter: v => String(v),
+      value: (r: MeasuredResults) => r.samples.length,
+    },
+  ],
+};
+
+/** Report section: total sampling duration. */
+export const totalTimeSection: ReportSection = {
+  title: "",
+  columns: [
+    {
+      key: "totalTime",
+      title: "time",
+      formatter: formatTotalTime,
+      value: (r: MeasuredResults) => r.totalTime,
+    },
+  ],
+};
+
+/** Report sections: timing stats and convergence for adaptive mode. */
+export const adaptiveSections: ReportSection[] = [
+  {
+    title: "time",
+    columns: [
+      {
+        key: "median",
+        title: "median",
+        formatter: timeMs,
+        comparable: true,
+        statKind: { percentile: 0.5 },
+      },
+      {
+        key: "mean",
+        title: "mean",
+        formatter: timeMs,
+        comparable: true,
+        statKind: "mean",
+      },
+      {
+        key: "p99",
+        title: "p99",
+        formatter: timeMs,
+        statKind: { percentile: 0.99 },
+      },
+    ],
+  },
+  {
+    title: "",
+    columns: [
+      {
+        key: "convergence",
+        title: "conv%",
+        formatter: formatConvergence,
+        value: (r: MeasuredResults) => r.convergence?.confidence,
+      },
+    ],
+  },
+];
+
+/** Report section: V8 optimization tier distribution and deopt count. */
+export const optSection: ReportSection = {
+  title: "v8 opt",
+  columns: [
+    {
+      key: "tiers",
+      title: "tiers",
+      formatter: v => (typeof v === "string" ? v : ""),
+      value: (r: MeasuredResults) => {
+        const opt = r.optStatus;
+        return opt ? formatTierSummary(opt) : undefined;
+      },
+    },
+    {
+      key: "deopt",
+      title: "deopt",
+      formatter: v => (typeof v === "number" ? String(v) : ""),
+      value: (r: MeasuredResults) => {
+        const opt = r.optStatus;
+        return opt && opt.deoptCount > 0 ? opt.deoptCount : undefined;
+      },
+    },
+  ],
+};
+
+/** Build a time section with user-chosen percentile/stat columns. */
+export function buildTimeSection(stats = "mean,p50,p99"): ReportSection {
+  const specs = parseStatsArg(stats);
+  return {
+    title: "time",
+    columns: specs.map(s => ({
+      key: s.key,
+      title: s.title,
+      formatter: timeMs,
+      comparable: isBootstrappable(s.statKind),
+      statKind: s.statKind,
+    })),
+  };
+}
+
+/** Format V8 tier distribution sorted by count (e.g. "turbofan:85% sparkplug:15%"). */
+export function formatTierSummary(
+  opt: OptStatusInfo,
+  sep = ":",
+  glue = " ",
+): string {
+  const tiers = Object.entries(opt.byTier);
+  const total = tiers.reduce((s, [, t]) => s + t.count, 0);
+  const pct = (n: number) => `${((n / total) * 100).toFixed(0)}%`;
+  return tiers
+    .sort((a, b) => b[1].count - a[1].count)
+    .map(([name, t]) => `${name}${sep}${pct(t.count)}`)
+    .join(glue);
+}
+
+/** @return default report sections from CLI flags (GC stats if enabled, plus run count). */
+export function buildGenericSections(args: {
+  "gc-stats"?: boolean;
+  alloc?: boolean;
+}): ReportSection[] {
+  return [...gcSections(args), runsSection];
+}
+
+/** Format total time; brackets indicate >= 30s. */
+function formatTotalTime(v: unknown): string {
+  if (typeof v !== "number") return "";
+  return v >= 30 ? `[${v.toFixed(1)}s]` : `${v.toFixed(1)}s`;
+}

package/src/report/ViewerSections.ts

@@ -0,0 +1,286 @@
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import {
+  type BlockDiffOptions,
+  binBootstrapResult,
+  diffCIs,
+} from "../stats/BootstrapDifference.ts";
+import {
+  type BootstrapResult,
+  bootstrapCIs,
+  type DifferenceCI,
+  flipCI,
+  isBootstrappable,
+  type StatKind,
+  swapDirection,
+} from "../stats/StatisticalUtils.ts";
+import type {
+  BootstrapCIData,
+  ViewerEntry,
+  ViewerRow,
+  ViewerSection,
+} from "../viewer/ReportData.ts";
+import {
+  type ComparisonOptions,
+  computeColumnValues,
+  type ReportColumn,
+  type ReportSection,
+  type UnknownRecord,
+} from "./BenchmarkReport.ts";
+
+/** Context for building viewer rows within a column group */
+interface RowContext {
+  current: MeasuredResults;
+  baseline?: MeasuredResults;
+  curVals: Record<string, unknown>;
+  baseVals?: Record<string, unknown>;
+  currentMeta?: UnknownRecord;
+  baselineMeta?: UnknownRecord;
+  comparison?: ComparisonOptions;
+}
+
+/** Pre-computed bootstrap results for a single column */
+interface ColCIs {
+  cur?: BootstrapResult;
+  base?: BootstrapResult;
+  diff?: DifferenceCI;
+}
+
+interface Annotatable {
+  direction: string;
+  label?: string;
+  ciReliable?: boolean;
+  ciLevel?: string;
+}
+
+export const minBatches = 20;
+
+/** @return true if comparing with fewer than minBatches on either side */
+export function hasLowBatchCount(
+  baseline: MeasuredResults | undefined,
+  current: MeasuredResults | undefined,
+): boolean {
+  if (!baseline) return false;
+  return batchCount(baseline) < minBatches || batchCount(current) < minBatches;
+}
+
+/** @return true if either side has no real batch structure */
+export function isSingleBatch(
+  baseline: MeasuredResults | undefined,
+  current: MeasuredResults | undefined,
+): boolean {
+  if (!baseline) return batchCount(current) < 2;
+  return batchCount(baseline) < 2 || batchCount(current) < 2;
+}
+
+/** Add label, mark unreliable, and override direction when batch count is low */
+export function annotateCI<T extends Annotatable | undefined>(
+  ci: T,
+  title?: string,
+  lowBatches?: boolean,
+): T {
+  if (!ci) return ci;
+  if (lowBatches) ci.direction = "uncertain";
+  ci.ciReliable = !lowBatches && ci.ciLevel !== "sample";
+  if (title) ci.label = `${title} Δ%`;
+  return ci;
+}
+
+/** Build ViewerSections from ReportSections, with bootstrap CIs for comparable columns */
+export function buildViewerSections(
+  sections: ReportSection[],
+  base: Omit<RowContext, "curVals" | "baseVals">,
+): ViewerSection[] {
+  const { current, baseline, currentMeta, baselineMeta } = base;
+  return sections.flatMap(section => {
+    const curVals = computeColumnValues(section, current, currentMeta);
+    const baseVals = baseline
+      ? computeColumnValues(section, baseline, baselineMeta)
+      : undefined;
+    const ctx: RowContext = { ...base, curVals, baseVals };
+    const rows = buildGroupRows(section.columns as ReportColumn[], ctx);
+    if (!rows.length) return [];
+    return [{ title: section.title, rows } satisfies ViewerSection];
+  });
+}
+
+function batchCount(m?: MeasuredResults): number {
+  return m?.batchOffsets?.length ?? 0;
+}
+
+/** Build ViewerRow[] for a column group, using shared resampling for statKind columns */
+function buildGroupRows(columns: ReportColumn[], ctx: RowContext): ViewerRow[] {
+  const ciMap = buildCIMap(columns, ctx);
+  const rows: ViewerRow[] = [];
+  for (const col of columns) {
+    const key = (col.key ?? col.title) as string;
+    const row = buildRow(col, key, ctx, ciMap.get(key));
+    if (row) rows.push(row);
+  }
+  const first = rows.find(r => r.entries.some(e => e.bootstrapCI));
+  if (first) first.primary = true;
+  return rows;
+}
+
+/** Compute batched bootstrap CIs, returning a Map keyed by column key */
+function buildCIMap(
+  columns: ReportColumn[],
+  ctx: RowContext,
+): Map<string, ColCIs> {
+  const ciCols = columns.filter(
+    c => c.comparable && c.statKind && isBootstrappable(c.statKind),
+  );
+  const statKinds = ciCols.map(c => c.statKind!);
+  const map = new Map<string, ColCIs>();
+  if (statKinds.length === 0) return map;
+
+  const curSamples = ctx.current.samples;
+  const baseSamples = ctx.baseline?.samples;
+  const curResults =
+    curSamples?.length > 1
+      ? bootstrapCIs(curSamples, ctx.current.batchOffsets, statKinds)
+      : undefined;
+  const baseResults =
+    baseSamples?.length && baseSamples.length > 1
+      ? bootstrapCIs(baseSamples, ctx.baseline!.batchOffsets, statKinds)
+      : undefined;
+  const diffResults = buildDiffResults(ciCols, statKinds, ctx);
+
+  for (let i = 0; i < ciCols.length; i++) {
+    const key = (ciCols[i].key ?? ciCols[i].title) as string;
+    map.set(key, {
+      cur: curResults?.[i],
+      base: baseResults?.[i],
+      diff: diffResults?.[i],
+    });
+  }
+  return map;
+}
+
+/** Build a ViewerRow for a column, using pre-computed CIs if available */
+function buildRow(
+  col: ReportColumn,
+  key: string,
+  ctx: RowContext,
+  cis?: ColCIs,
+): ViewerRow | undefined {
+  const curRaw = ctx.curVals[key];
+  const baseRaw = ctx.baseVals?.[key];
+  if (curRaw === undefined && baseRaw === undefined) return undefined;
+
+  const format = (v: unknown) => {
+    if (v === undefined) return "";
+    return (col.formatter ? col.formatter(v) : String(v)) ?? "";
+  };
+
+  // Non-comparable: shared single value
+  if (!col.comparable) {
+    const value = format(curRaw ?? baseRaw);
+    if (!value || value === "—") return undefined;
+    return {
+      label: col.title,
+      entries: [{ runName: ctx.current.name, value }],
+      shared: true,
+    };
+  }
+
+  // Comparable: current + baseline entries, optional CI
+  const curEntry = buildEntry(
+    ctx.current.name,
+    format(curRaw),
+    col,
+    cis?.cur,
+    ctx.current.batchOffsets,
+    ctx.currentMeta,
+  );
+  const entries: ViewerEntry[] = [curEntry];
+  if (ctx.baseline && baseRaw !== undefined) {
+    const baseEntry = buildEntry(
+      "baseline",
+      format(baseRaw),
+      col,
+      cis?.base,
+      ctx.baseline.batchOffsets,
+      ctx.baselineMeta,
+    );
+    entries.push(baseEntry);
+  }
+  return { label: col.title, entries, comparisonCI: cis?.diff };
+}
+
+/** Compute difference CIs with annotation and higher-is-better flip */
+function buildDiffResults(
+  cols: ReportColumn[],
+  stats: StatKind[],
+  ctx: RowContext,
+): (DifferenceCI | undefined)[] | undefined {
+  const { baseline, current, comparison } = ctx;
+  if (!baseline?.samples?.length || !current.samples?.length) return undefined;
+
+  const opts: BlockDiffOptions = {
+    equivMargin: comparison?.equivMargin,
+    noBatchTrim: comparison?.noBatchTrim,
+  };
+  const rawCIs = diffCIs(
+    baseline.samples,
+    baseline.batchOffsets,
+    current.samples,
+    current.batchOffsets,
+    stats,
+    opts,
+  );
+  const lowBatches = hasLowBatchCount(baseline, current);
+  return rawCIs.map((ci, i) => {
+    if (!ci) return undefined;
+    const col = cols[i];
+    const adjusted = col.higherIsBetter ? swapDirection(flipCI(ci)) : ci;
+    return annotateCI(adjusted, col.title, lowBatches);
+  });
+}
+
+/** Build a ViewerEntry, attaching bootstrap CI data if available */
+function buildEntry(
+  runName: string,
+  value: string,
+  col: ReportColumn,
+  result: BootstrapResult | undefined,
+  batchOffsets: number[] | undefined,
+  metadata?: UnknownRecord,
+): ViewerEntry {
+  if (!result) return { runName, value };
+  const bootstrapCI = formatBootstrapCI(col, result, batchOffsets, metadata);
+  return { runName, value, bootstrapCI };
+}
+
+/** Format a BootstrapResult into display-domain BootstrapCIData */
+function formatBootstrapCI(
+  col: ReportColumn,
+  result: BootstrapResult,
+  batchOffsets: number[] | undefined,
+  metadata?: UnknownRecord,
+): BootstrapCIData {
+  const toDisplay = col.toDisplay
+    ? (v: number) => col.toDisplay!(v, metadata)
+    : (v: number) => v;
+  const formatValue = (v: number) =>
+    (col.formatter ? col.formatter(v) : String(v)) ?? String(v);
+
+  const binned = binBootstrapResult(result);
+  const dLo = toDisplay(binned.ci[0]);
+  const dHi = toDisplay(binned.ci[1]);
+  const ci = (dLo <= dHi ? [dLo, dHi] : [dHi, dLo]) as [number, number];
+  const histogram = binned.histogram.map(b => ({
+    x: toDisplay(b.x),
+    count: b.count,
+  }));
+  const ciLabels = [formatValue(ci[0]), formatValue(ci[1])] as [string, string];
+  const nBatches = batchOffsets?.length ?? 0;
+  const ciReliable = result.ciLevel === "block" && nBatches >= minBatches;
+  return {
+    estimate: toDisplay(binned.estimate),
+    ci,
+    histogram,
+    ciLabels,
+    ciLevel: result.ciLevel,
+    ciReliable,
+  };
+}

package/src/report/text/TableReport.ts

@@ -0,0 +1,253 @@
+import type { Alignment, SpanningCellConfig, TableUserConfig } from "table";
+import { table } from "table";
+import colors from "../Colors.ts";
+import { diffPercent } from "../Formatters.ts";
+
+/** Named group of columns, rendered with an optional spanning header. */
+export interface ColumnGroup<T> {
+  groupTitle?: string;
+  columns: AnyColumn<T>[];
+}
+
+export type AnyColumn<T> = Column<T> | DiffColumn<T>;
+
+/** Table column with a value formatter (non-diff). */
+export interface Column<T> extends ColumnFormat<T> {
+  formatter?: (value: unknown) => string | null;
+  diffKey?: undefined;
+}
+
+/** Pre-computed header rows and table config for the `table` library. */
+export interface TableSetup {
+  headerRows: string[][];
+  config: TableUserConfig;
+}
+
+/** Group of result rows with an optional baseline for diff columns. */
+export interface ResultGroup<T extends Record<string, any>> {
+  results: T[];
+  baseline?: T;
+}
+
+interface DiffColumn<T> extends ColumnFormat<T> {
+  diffFormatter?: (value: unknown, baseline: unknown) => string | null;
+  formatter?: undefined;
+  diffKey: keyof T;
+}
+
+interface ColumnFormat<T> {
+  key: keyof T;
+  title: string;
+  alignment?: Alignment;
+  width?: number;
+}
+
+interface Lines {
+  drawHorizontalLine: (index: number, size: number) => boolean;
+  drawVerticalLine: (index: number, size: number) => boolean;
+}
+
+const { bold } = colors;
+
+const ansiEscapeRegex = new RegExp(
+  String.fromCharCode(27) + "\\[[0-9;]*m",
+  "g",
+);
+
+/** Build formatted table with column groups and baseline diffs. */
+export function buildTable<T extends Record<string, any>>(
+  columnGroups: ColumnGroup<T>[],
+  resultGroups: ResultGroup<T>[],
+  nameKey: keyof T = "name" as keyof T,
+): string {
+  const allRecords = flattenGroups(columnGroups, resultGroups, nameKey);
+  return createTable(columnGroups, allRecords);
+}
+
+/** Convert records to string arrays for table rendering. */
+export function toRows<T extends Record<string, any>>(
+  records: T[],
+  groups: ColumnGroup<T>[],
+): string[][] {
+  const allColumns = groups.flatMap(group => group.columns);
+
+  const rawRows = records.map(record =>
+    allColumns.map(col => {
+      const value = record[col.key];
+      return col.formatter ? col.formatter(value) : value;
+    }),
+  );
+
+  return rawRows.map(row => row.map(cell => cell ?? " "));
+}
+
+/** Flatten result groups into a single array, inserting blank separator rows. */
+function flattenGroups<T extends Record<string, any>>(
+  groups: ColumnGroup<T>[],
+  resultGroups: ResultGroup<T>[],
+  nameKey: keyof T,
+): T[] {
+  return resultGroups.flatMap((group, i) => {
+    const records = addBaseline(groups, group, nameKey);
+    const isLast = i === resultGroups.length - 1;
+    return isLast ? records : [...records, {} as T];
+  });
+}
+
+/** Render column groups and records into a formatted table string. */
+function createTable<T extends Record<string, any>>(
+  groups: ColumnGroup<T>[],
+  records: T[],
+): string {
+  const dataRows = toRows(records, groups);
+  const { headerRows, config } = buildTableConfig(groups, dataRows);
+  const allRows = [...headerRows, ...dataRows];
+  return table(allRows, config);
+}
+
+/** Append baseline row and inject diff values into result rows. */
+function addBaseline<T extends Record<string, any>>(
+  groups: ColumnGroup<T>[],
+  group: ResultGroup<T>,
+  nameKey: keyof T,
+): T[] {
+  const { results, baseline } = group;
+  if (!baseline) return results;
+  const diffResults = results.map(r => addComparisons(groups, r, baseline));
+  const marked = { ...baseline, [nameKey]: `--> ${baseline[nameKey]}` };
+  return [...diffResults, marked];
+}
+
+/** Build header rows, spanning cells, column widths, and border rules. */
+function buildTableConfig<T>(
+  groups: ColumnGroup<T>[],
+  dataRows: string[][],
+): TableSetup {
+  const titles = getTitles(groups);
+  const headerRows = [...createGroupHeaders(groups, titles.length), titles];
+  const config: TableUserConfig = {
+    spanningCells: createSectionSpans(groups),
+    columns: calcColumnWidths(groups, titles, dataRows),
+    ...createLines(groups),
+  };
+  return { headerRows, config };
+}
+
+/** Compute formatted diff values by comparing a row against baseline. */
+function addComparisons<T extends Record<string, any>>(
+  groups: ColumnGroup<T>[],
+  main: T,
+  baseline: T,
+): T {
+  const cols = groups
+    .flatMap(g => g.columns)
+    .filter((col): col is DiffColumn<T> => col.diffKey !== undefined);
+  const diffs = Object.fromEntries(
+    cols.map(col => {
+      const fmt = col.diffFormatter ?? diffPercent;
+      return [col.key, fmt(main[col.diffKey], baseline[col.diffKey])];
+    }),
+  );
+  return { ...main, ...diffs };
+}
+
+/** @return bolded column title strings */
+function getTitles<T>(groups: ColumnGroup<T>[]): string[] {
+  return groups.flatMap(g => g.columns.map(c => bold(c.title || " ")));
+}
+
+/** @return header rows with group titles, or empty if no groups have titles. */
+function createGroupHeaders<T>(
+  groups: ColumnGroup<T>[],
+  numColumns: number,
+): string[][] {
+  if (!groups.some(g => g.groupTitle)) return [];
+
+  const sectionRow = groups.flatMap(g => {
+    const title = g.groupTitle ? [bold(g.groupTitle)] : [];
+    return padWithBlanks(title, g.columns.length);
+  });
+  const blankRow = padWithBlanks([], numColumns);
+  return [sectionRow, blankRow];
+}
+
+/** @return spanning cell configs for group title headers */
+function createSectionSpans<T>(groups: ColumnGroup<T>[]): SpanningCellConfig[] {
+  const offsets = groupOffsets(groups);
+  return groups.map((g, i) => ({
+    row: 0,
+    col: offsets[i],
+    colSpan: g.columns.length,
+    alignment: "center" as Alignment,
+  }));
+}
+
+/** Calculate column widths based on content, widening to fit group titles. */
+function calcColumnWidths<T>(
+  groups: ColumnGroup<T>[],
+  titles: string[],
+  dataRows: unknown[][],
+): Record<number, { width: number; wrapWord: boolean }> {
+  const maxData = (i: number) =>
+    dataRows.reduce((m, row) => Math.max(m, cellWidth(row[i])), 0);
+  const widths = titles.map((t, i) => Math.max(cellWidth(t), maxData(i)));
+
+  // Widen columns so group titles fit (accounting for " | " separators)
+  const offsets = groupOffsets(groups);
+  for (const [i, group] of groups.entries()) {
+    const titleWidth = cellWidth(group.groupTitle);
+    if (titleWidth <= 0) continue;
+    const col = offsets[i];
+    const n = group.columns.length;
+    const sepWidth = (n - 1) * 3;
+    const curWidth = widths.slice(col, col + n).reduce((a, b) => a + b, 0);
+    const needed = titleWidth - curWidth - sepWidth;
+    if (needed > 0) widths[col + n - 1] += needed;
+  }
+
+  return Object.fromEntries(
+    widths.map((w, i) => [i, { width: w, wrapWord: false }]),
+  );
+}
+
+/** @return draw functions for horizontal/vertical table borders */
+function createLines<T>(groups: ColumnGroup<T>[]): Lines {
+  const { sectionBorders, headerBottom } = calcBorders(groups);
+  return {
+    drawVerticalLine: (i, size) =>
+      i === 0 || i === size || sectionBorders.includes(i),
+    drawHorizontalLine: (i, size) =>
+      i === 0 || i === size || i === headerBottom,
+  };
+}
+
+/** @return array padded with blank strings to the given length */
+function padWithBlanks(arr: string[], length: number): string[] {
+  if (arr.length >= length) return arr;
+  return [...arr, ...Array(length - arr.length).fill(" ")];
+}
+
+/** @return cumulative column offsets for each group boundary */
+function groupOffsets<T>(groups: ColumnGroup<T>[]): number[] {
+  let offset = 0;
+  return groups.map(g => {
+    const start = offset;
+    offset += g.columns.length;
+    return start;
+  });
+}
+
+/** @return visible length of a cell value, stripping ANSI escape codes. */
+function cellWidth(value: unknown): number {
+  if (value == null) return 0;
+  const str = String(value);
+  return str.replace(ansiEscapeRegex, "").length;
+}
+
+/** @return vertical line positions between sections and header bottom row. */
+function calcBorders<T>(groups: ColumnGroup<T>[]) {
+  const offsets = groupOffsets(groups);
+  const sectionBorders = offsets.map((o, i) => o + groups[i].columns.length);
+  const headerBottom = groups.length === 0 ? 1 : 3;
+  return { sectionBorders, headerBottom };
+}