benchforge 0.1.0

package/src/table-util/Formatters.ts

@@ -0,0 +1,152 @@
+import pico from "picocolors";
+import type { CIDirection, DifferenceCI } from "../StatisticalUtils.ts";
+
+const isTest = process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+const { red, green } = isTest
+  ? { red: (str: string) => str, green: (str: string) => str }
+  : pico;
+
+/** Format floats with custom precision */
+export function floatPrecision(precision: number) {
+  return (x: unknown): string | null => {
+    if (typeof x !== "number") return null;
+    return x.toFixed(precision).replace(/\.?0+$/, "");
+  };
+}
+
+/** Format percentages with custom precision */
+export function percentPrecision(precision: number) {
+  return (x: unknown): string | null => {
+    if (typeof x !== "number") return null;
+    return percent(x, precision);
+  };
+}
+
+/** Format duration in milliseconds with appropriate units */
+export function duration(ms: unknown): string | null {
+  if (typeof ms !== "number") return null;
+  if (ms < 0.001) return `${(ms * 1000000).toFixed(0)}ns`;
+  if (ms < 1) return `${(ms * 1000).toFixed(1)}μs`;
+  if (ms < 1000) return `${ms.toFixed(2)}ms`;
+  return `${(ms / 1000).toFixed(2)}s`;
+}
+
+/** Format time in milliseconds, showing very small values with units */
+export function timeMs(ms: unknown): string | null {
+  if (typeof ms !== "number") return null;
+  if (ms < 0.001) return `${(ms * 1000000).toFixed(0)}ns`;
+  if (ms < 0.01) return `${(ms * 1000).toFixed(1)}μs`;
+  if (ms >= 10) return ms.toFixed(0);
+  return ms.toFixed(2);
+}
+
+/** Format as rate (value per unit) */
+export function rate(unit: string) {
+  return (value: unknown) => {
+    if (typeof value !== "number") return null;
+    return `${integer(value)}/${unit}`;
+  };
+}
+
+/** Format integer with thousand separators */
+export function integer(x: unknown): string | null {
+  if (typeof x !== "number") return null;
+  return new Intl.NumberFormat("en-US").format(Math.round(x));
+}
+
+/** Format fraction as percentage (0.473 → 47.3%) */
+export function percent(fraction: unknown, precision = 1): string | null {
+  if (typeof fraction !== "number") return null;
+  return `${Math.abs(fraction * 100).toFixed(precision)}%`;
+}
+
+/** Format percentage difference between two values */
+export function diffPercent(main: unknown, base: unknown): string {
+  if (typeof main !== "number" || typeof base !== "number") return " ";
+  const diff = main - base;
+  return coloredPercent(diff, base);
+}
+
+/** Format percentage difference for benchmarks (lower is better) */
+export function diffPercentBenchmark(main: unknown, base: unknown): string {
+  if (typeof main !== "number" || typeof base !== "number") return " ";
+  const diff = main - base;
+  return coloredPercent(diff, base, false); // negative is good for benchmarks
+}
+
+/** Format fraction as colored +/- percentage */
+function coloredPercent(
+  numerator: number,
+  denominator: number,
+  positiveIsGreen = true,
+): string {
+  const fraction = numerator / denominator;
+  if (Number.isNaN(fraction) || !Number.isFinite(fraction)) {
+    return " ";
+  }
+  const positive = fraction >= 0;
+  const sign = positive ? "+" : "-";
+  const percentStr = `${sign}${percent(fraction)}`;
+  const isGood = positive === positiveIsGreen;
+  return isGood ? green(percentStr) : red(percentStr);
+}
+
+/** Format memory size in KB with appropriate units */
+export function memoryKB(kb: unknown): string | null {
+  if (typeof kb !== "number") return null;
+  if (kb < 1024) return `${kb.toFixed(0)}KB`;
+  return `${(kb / 1024).toFixed(1)}MB`;
+}
+
+/** Format bytes with appropriate units (B, KB, MB, GB) */
+export function formatBytes(bytes: unknown): string | null {
+  if (typeof bytes !== "number") return null;
+  if (bytes < 1024) return `${bytes.toFixed(0)}B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`;
+  if (bytes < 1024 * 1024 * 1024)
+    return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+  return `${(bytes / 1024 / 1024 / 1024).toFixed(1)}GB`;
+}
+
+/** Format percentage difference with confidence interval */
+export function formatDiffWithCI(value: unknown): string | null {
+  if (!isDifferenceCI(value)) return null;
+  const { percent, ci, direction } = value;
+  return colorByDirection(diffCIText(percent, ci), direction);
+}
+
+/** Format percentage difference with CI for throughput metrics (higher is better) */
+export function formatDiffWithCIHigherIsBetter(value: unknown): string | null {
+  if (!isDifferenceCI(value)) return null;
+  const { percent, ci, direction } = value;
+  // Flip percent sign for "higher is better" metrics (direction stays same)
+  return colorByDirection(diffCIText(-percent, [-ci[1], -ci[0]]), direction);
+}
+
+/** @return formatted "pct [lo, hi]" text for a diff with CI */
+function diffCIText(pct: number, ci: [number, number]): string {
+  return `${formatBound(pct)} [${formatBound(ci[0])}, ${formatBound(ci[1])}]`;
+}
+
+/** @return text colored green for faster, red for slower */
+function colorByDirection(text: string, direction: CIDirection): string {
+  if (direction === "faster") return green(text);
+  if (direction === "slower") return red(text);
+  return text;
+}
+
+/** @return signed percentage string (e.g. "+1.2%", "-3.4%") */
+function formatBound(v: number): string {
+  const sign = v >= 0 ? "+" : "";
+  return `${sign}${v.toFixed(1)}%`;
+}
+
+/** @return true if value is a DifferenceCI object */
+function isDifferenceCI(x: unknown): x is DifferenceCI {
+  return typeof x === "object" && x !== null && "ci" in x && "direction" in x;
+}
+
+/** @return truncated string with ellipsis if over maxLen */
+export function truncate(str: string, maxLen = 30): string {
+  return str.length > maxLen ? str.slice(0, maxLen - 3) + "..." : str;
+}

package/src/table-util/README.md

@@ -0,0 +1,70 @@
+# TableReport 
+
+Utilities for creating formatted text-based tables.
+Under the hood, TableReport uses the npm [table](https://www.npmjs.com/package/table) library.
+
+### Features
+
+*   **Column Grouping:** Group related columns under a common header.
+*   **Difference Columns:** Automatically generate columns that show the percentage difference between a value and a baseline value.
+*   **Custom Formatting:** Declaratively provide custom formatters for columns.
+*   **Simplified Configuration:** A higher-level API for the `table` library.
+
+## `TableReport.ts`
+
+The `buildTable` function in `TableReport.ts` is the main entry point for creating a table. 
+It takes a configuration object for columns and an array of data records for rows.
+
+### Example
+
+Here's a simplified example of how to use `buildTable`:
+
+```typescript
+import { buildTable, ColumnGroup } from './TableReport';
+import { integer, floatPrecision } from './Formatters';
+
+interface MyData {
+  name: string;
+  value: number;
+  score: number;
+}
+
+const data: MyData[] = [
+  { name: 'test A', value: 123, score: 45.6 },
+  { name: 'test B', value: 456, score: 78.9 },
+];
+
+const baselineData: MyData[] = [
+  { name: 'test A', value: 100, score: 50.0 },
+  { name: 'test B', value: 500, score: 75.0 },
+];
+
+const tableConfig: ColumnGroup<MyData>[] = [
+  {
+    columns: [{ key: 'name', title: 'Name' }],
+  },
+  {
+    groupTitle: 'Metrics',
+    columns: [
+      { key: 'value', title: 'Value', formatter: integer },
+      { key: 'value_diff', title: 'Δ%', diffKey: 'value' },
+      { key: 'score', title: 'Score', formatter: floatPrecision(1) },
+      { key: 'score_diff', title: 'Δ%', diffKey: 'score' },
+    ],
+  },
+];
+
+const table = buildTable(tableConfig, data, baselineData);
+console.log(table);
+```
+
+For a more complex example, see `BenchmarkReport.ts`, specifically the `mostlyFullRow` and `tableConfig` variables.
+
+## `Formatters.ts`
+
+This file contains various utility functions for formatting numbers and strings, such as:
+
+*   `float`, `integer`: Format numbers to a specific precision.
+*   `percent`: Format a number as a percentage.
+*   `diffPercent`, `diffPercentNegative`: Format the percentage difference between two numbers, with color-coding for positive/negative changes.
+*   `bytes`, `duration`, `rate`: Format numbers with appropriate units.

package/src/table-util/TableReport.ts

@@ -0,0 +1,293 @@
+import pico from "picocolors";
+import type { Alignment, SpanningCellConfig, TableUserConfig } from "table";
+import { table } from "table";
+import { diffPercent } from "./Formatters.ts";
+
+const isTest = process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+const { bold } = isTest ? { bold: (str: string) => str } : pico;
+
+/** Related table columns */
+export interface ColumnGroup<T> {
+  groupTitle?: string;
+  columns: AnyColumn<T>[];
+}
+
+export type AnyColumn<T> = Column<T> | DiffColumn<T>;
+
+/** Column with optional formatter */
+export interface Column<T> extends ColumnFormat<T> {
+  formatter?: (value: unknown) => string | null;
+  diffKey?: undefined;
+}
+
+/** Comparison column against baseline */
+interface DiffColumn<T> extends ColumnFormat<T> {
+  diffFormatter?: (value: unknown, baseline: unknown) => string | null;
+  formatter?: undefined;
+
+  /** Key for comparison value against baseline */
+  diffKey: keyof T;
+}
+
+interface ColumnFormat<T> {
+  key: keyof T;
+  title: string;
+
+  alignment?: Alignment;
+
+  width?: number;
+}
+
+/** Table headers and configuration */
+export interface TableSetup {
+  headerRows: string[][];
+  config: TableUserConfig;
+}
+
+/** Data rows with optional baseline */
+export interface ResultGroup<T extends Record<string, any>> {
+  results: T[];
+
+  baseline?: T;
+}
+
+/** Build formatted table with column groups and baselines */
+export function buildTable<T extends Record<string, any>>(
+  columnGroups: ColumnGroup<T>[],
+  resultGroups: ResultGroup<T>[],
+  nameKey: keyof T = "name" as keyof T,
+): string {
+  const allRecords = flattenGroups(columnGroups, resultGroups, nameKey);
+  return createTable(columnGroups, allRecords);
+}
+
+/** Convert columns and records to formatted table */
+function createTable<T extends Record<string, any>>(
+  groups: ColumnGroup<T>[],
+  records: T[],
+): string {
+  const dataRows = toRows(records, groups);
+  const { headerRows, config } = setup(groups, dataRows);
+  const allRows = [...headerRows, ...dataRows];
+  return table(allRows, config);
+}
+
+/** Create header rows with group titles */
+function createGroupHeaders<T>(
+  groups: ColumnGroup<T>[],
+  numColumns: number,
+): string[][] {
+  if (!groups.some(g => g.groupTitle)) return [];
+
+  const sectionRow = groups.flatMap(g => {
+    const title = g.groupTitle ? [bold(g.groupTitle)] : [];
+    return padWithBlanks(title, g.columns.length);
+  });
+  const blankRow = padWithBlanks([], numColumns);
+  return [sectionRow, blankRow];
+}
+
+interface Lines {
+  drawHorizontalLine: (index: number, size: number) => boolean;
+  drawVerticalLine: (index: number, size: number) => boolean;
+}
+
+/** @return draw functions for horizontal/vertical table borders */
+function createLines<T>(groups: ColumnGroup<T>[]): Lines {
+  const { sectionBorders, headerBottom } = calcBorders(groups);
+
+  function drawVerticalLine(index: number, size: number): boolean {
+    return index === 0 || index === size || sectionBorders.includes(index);
+  }
+  function drawHorizontalLine(index: number, size: number): boolean {
+    return index === 0 || index === size || index === headerBottom;
+  }
+  return { drawHorizontalLine, drawVerticalLine };
+}
+
+/** @return spanning cell configs for group title headers */
+function createSectionSpans<T>(groups: ColumnGroup<T>[]): SpanningCellConfig[] {
+  let col = 0;
+  const alignment: Alignment = "center";
+  return groups.map(g => {
+    const colSpan = g.columns.length;
+    const span = { row: 0, col, colSpan, alignment };
+    col += colSpan;
+    return span;
+  });
+}
+
+/** @return bolded column title strings */
+function getTitles<T>(groups: ColumnGroup<T>[]): string[] {
+  return groups.flatMap(g => g.columns.map(c => bold(c.title || " ")));
+}
+
+/** @return array padded with blank strings to the given length */
+function padWithBlanks(arr: string[], length: number): string[] {
+  if (arr.length >= length) return arr;
+  return [...arr, ...Array(length - arr.length).fill(" ")];
+}
+
+/** Convert records to string arrays for table */
+export function toRows<T extends Record<string, any>>(
+  records: T[],
+  groups: ColumnGroup<T>[],
+): string[][] {
+  const allColumns = groups.flatMap(group => group.columns);
+
+  const rawRows = records.map(record =>
+    allColumns.map(col => {
+      const value = record[col.key];
+      return col.formatter ? col.formatter(value) : value;
+    }),
+  );
+
+  return rawRows.map(row => row.map(cell => cell ?? " "));
+}
+
+/** Add comparison values for diff columns */
+function addComparisons<T extends Record<string, any>>(
+  groups: ColumnGroup<T>[],
+  mainRecord: T,
+  baselineRecord: T,
+): T {
+  const diffColumns = groups.flatMap(g => g.columns).filter(col => col.diffKey);
+  const updatedMain = { ...mainRecord };
+
+  for (const col of diffColumns) {
+    const dcol = col as DiffColumn<T>;
+    const diffKey = dcol.diffKey;
+    const mainValue = mainRecord[diffKey];
+    const baselineValue = baselineRecord[diffKey];
+    const diffFormat = dcol.diffFormatter ?? diffPercent;
+    const diffStr = diffFormat(mainValue, baselineValue);
+    (updatedMain as any)[col.key] = diffStr;
+  }
+
+  return updatedMain;
+}
+
+/** Flatten groups with spacing */
+function flattenGroups<T extends Record<string, any>>(
+  columnGroups: ColumnGroup<T>[],
+  resultGroups: ResultGroup<T>[],
+  nameKey: keyof T,
+): T[] {
+  return resultGroups.flatMap((group, i) => {
+    const groupRecords = addBaseline(columnGroups, group, nameKey);
+
+    const isLast = i === resultGroups.length - 1;
+    return isLast ? groupRecords : [...groupRecords, {} as T];
+  });
+}
+
+/** Process results with baseline comparisons */
+function addBaseline<T extends Record<string, any>>(
+  columnGroups: ColumnGroup<T>[],
+  group: ResultGroup<T>,
+  nameKey: keyof T,
+): T[] {
+  const { results, baseline } = group;
+
+  if (!baseline) return results;
+
+  const diffResults = results.map(result =>
+    addComparisons(columnGroups, result, baseline),
+  );
+
+  const markedBaseline = {
+    ...baseline,
+    [nameKey]: `--> ${baseline[nameKey]}`,
+  };
+
+  return [...diffResults, markedBaseline];
+}
+
+/** Calculate vertical lines between sections and header bottom position */
+function calcBorders<T>(groups: ColumnGroup<T>[]): {
+  sectionBorders: number[];
+  headerBottom: number;
+} {
+  if (groups.length === 0) return { sectionBorders: [], headerBottom: 1 };
+
+  const sectionBorders: number[] = [];
+  let border = 0;
+  for (const g of groups) {
+    border += g.columns.length;
+    sectionBorders.push(border);
+  }
+  return { sectionBorders, headerBottom: 3 };
+}
+
+/** Create headers and table configuration */
+function setup<T>(groups: ColumnGroup<T>[], dataRows: string[][]): TableSetup {
+  const titles = getTitles(groups);
+  const numColumns = titles.length;
+
+  const sectionRows = createGroupHeaders(groups, numColumns);
+  const headerRows = [...sectionRows, titles];
+  const spanningCells = createSectionSpans(groups);
+  const columnWidths = calcColumnWidths(groups, titles, dataRows);
+  const config: TableUserConfig = {
+    spanningCells,
+    columns: columnWidths,
+    ...createLines(groups),
+  };
+
+  return { headerRows, config };
+}
+
+/** Calculate column widths based on content, including group titles */
+function calcColumnWidths<T>(
+  groups: ColumnGroup<T>[],
+  titles: string[],
+  dataRows: unknown[][],
+): Record<number, { width: number; wrapWord: boolean }> {
+  // First pass: calculate base widths from titles and data
+  const widths: number[] = [];
+  for (let i = 0; i < titles.length; i++) {
+    const titleW = cellWidth(titles[i]);
+    const maxDataW = dataRows.reduce(
+      (max, row) => Math.max(max, cellWidth(row[i])),
+      0,
+    );
+    widths.push(Math.max(titleW, maxDataW));
+  }
+
+  // Second pass: ensure group titles fit (accounting for column separators)
+  let colIndex = 0;
+  for (const group of groups) {
+    const groupW = cellWidth(group.groupTitle);
+    if (groupW > 0) {
+      const numCols = group.columns.length;
+      const separatorWidth = (numCols - 1) * 3; // " | " between columns
+      const currentWidth = widths
+        .slice(colIndex, colIndex + numCols)
+        .reduce((a, b) => a + b, 0);
+      const needed = groupW - currentWidth - separatorWidth;
+      if (needed > 0) {
+        // Distribute extra width to last column in group
+        widths[colIndex + numCols - 1] += needed;
+      }
+    }
+    colIndex += group.columns.length;
+  }
+
+  // Convert to table config format
+  return Object.fromEntries(
+    widths.map((w, i) => [i, { width: w, wrapWord: false }]),
+  );
+}
+
+// Regex to strip ANSI escape codes (ESC [ ... m sequences)
+const ansiEscapeRegex = new RegExp(
+  String.fromCharCode(27) + "\\[[0-9;]*m",
+  "g",
+);
+
+/** Get visible length of a cell value (strips ANSI escape codes) */
+function cellWidth(value: unknown): number {
+  if (value == null) return 0;
+  const str = String(value);
+  return str.replace(ansiEscapeRegex, "").length;
+}

package/src/table-util/test/TableReport.test.ts

@@ -0,0 +1,105 @@
+import { expect, test } from "vitest";
+import { integer } from "../Formatters.ts";
+import {
+  buildTable,
+  type ColumnGroup,
+  type ResultGroup,
+} from "../TableReport.ts";
+
+interface TestRecord {
+  name: string;
+  max: number;
+  maxDelta?: string;
+  p50: number;
+  p50Delta?: string;
+  kb: number;
+  l1miss: string;
+  n: number;
+}
+
+const columnGroups: ColumnGroup<TestRecord>[] = [
+  { columns: [{ key: "name", title: "name", alignment: "left" }] },
+  {
+    groupTitle: "lines / sec",
+    columns: [
+      { key: "max", title: "max", alignment: "right", formatter: integer },
+      { key: "maxDelta", title: "Δ%", alignment: "right", diffKey: "max" },
+      { key: "p50", title: "p50", alignment: "right", formatter: integer },
+      { key: "p50Delta", title: "Δ%", alignment: "right", diffKey: "p50" },
+    ],
+  },
+  {
+    columns: [
+      { key: "kb", title: "kb", alignment: "right", formatter: integer },
+      { key: "l1miss", title: "L1 miss", alignment: "right" },
+      { key: "n", title: "N", alignment: "right" },
+    ],
+  },
+];
+
+const mainData: TestRecord[] = [
+  {
+    name: "reduceBuffer",
+    max: 77045,
+    p50: 74351,
+    kb: 2044,
+    l1miss: "1.6%",
+    n: 545,
+  },
+  {
+    name: "unity_webgpu",
+    max: 33448,
+    p50: 31819,
+    kb: 67895,
+    l1miss: "2.1%",
+    n: 12,
+  },
+];
+
+const baselineData: TestRecord[] = [
+  {
+    name: "reduceBuffer",
+    max: 77463,
+    p50: 75044,
+    kb: 2026,
+    l1miss: "1.6%",
+    n: 556,
+  },
+  {
+    name: "unity_webgpu",
+    max: 33925,
+    p50: 33107,
+    kb: 69808,
+    l1miss: "2.0%",
+    n: 12,
+  },
+];
+
+const expectedTable = `
+    ╔══════════════════╤══════════════════════════════╤══════════════════════╗
+    ║                  │         lines / sec          │                      ║
+    ║                  │                              │                      ║
+    ║ name             │ max     Δ%     p50     Δ%    │ kb      L1 miss  N   ║
+    ╟──────────────────┼──────────────────────────────┼──────────────────────╢
+    ║ reduceBuffer     │ 77,045  -0.5%  74,351  -0.9% │ 2,044   1.6%     545 ║
+    ║ --> reduceBuffer │ 77,463         75,044        │ 2,026   1.6%     556 ║
+    ║                  │                              │                      ║
+    ║ unity_webgpu     │ 33,448  -1.4%  31,819  -3.9% │ 67,895  2.1%     12  ║
+    ║ --> unity_webgpu │ 33,925         33,107        │ 69,808  2.0%     12  ║
+    ╚══════════════════╧══════════════════════════════╧══════════════════════╝
+`;
+
+test("buildTable creates table with name, lines/sec, and kb/L1/N sections", () => {
+  const resultGroups: ResultGroup<TestRecord>[] = mainData.map((main, i) => ({
+    results: [main],
+    baseline: baselineData[i],
+  }));
+  const result = buildTable(columnGroups, resultGroups);
+  const trim = (s: string) =>
+    s
+      .split("\n")
+      .map(l => l.trim())
+      .filter(Boolean)
+      .join("\n");
+  expect(trim(result)).toBe(trim(expectedTable));
+});

package/src/table-util/test/TableValueExtractor.test.ts

@@ -0,0 +1,41 @@
+import { expect, test } from "vitest";
+
+import { extractValue } from "./TableValueExtractor.ts";
+
+test("extract mean values from table with two column groups", () => {
+  const table = `
+╔═════════════════╤══════════════════════════╤═══════════════════════╗
+║                 │           time           │         gc            ║
+║                 │                          │                       ║
+║ name            │ mean     p50       p99   │ mean     max          ║
+╟─────────────────┼──────────────────────────┼───────────────────────╢
+║ test1           │ 12.3     1.45      2.1   │ 8.7      89.1         ║
+║ test2           │ 23.4     14.5      20.0  │ 15.6     102.5        ║
+╚═════════════════╧══════════════════════════╧═══════════════════════╝
+  `;
+
+  expect(extractValue(table, "test1", "mean", "time")).toBe(12.3);
+  expect(extractValue(table, "test2", "mean", "time")).toBe(23.4);
+
+  expect(extractValue(table, "test1", "mean", "gc")).toBe(8.7);
+  expect(extractValue(table, "test2", "mean", "gc")).toBe(15.6);
+
+  expect(extractValue(table, "test1", "p99", "time")).toBe(2.1);
+  expect(extractValue(table, "test1", "max", "gc")).toBe(89.1);
+});
+
+test("extract values from table with column title containing space", () => {
+  const table = `
+╔══════════════════╤══════════════════════════════╤══════════════════════╗
+║                  │         lines / sec          │                      ║
+║                  │                              │                      ║
+║ name             │ max     Δ%     p50     Δ%    │ kb      L1 miss  N   ║
+╟──────────────────┼──────────────────────────────┼──────────────────────╢
+║ reduceBuffer     │ 77,045  -0.5%  74,351  -0.9% │ 2,044   1.6%     545 ║
+║ unity_webgpu     │ 33,448  -1.4%  31,819  -3.9% │ 67,895  2.1%     12  ║
+╚══════════════════╧══════════════════════════════╧══════════════════════╝
+  `;
+
+  expect(extractValue(table, "reduceBuffer", "L1 miss")).toBe(1.6);
+  expect(extractValue(table, "unity_webgpu", "L1 miss")).toBe(2.1);
+});