benchforge 0.1.0

package/src/html/browser/Types.ts

@@ -0,0 +1,96 @@
+export interface Sample {
+  benchmark: string;
+  value: number;
+  iteration: number;
+}
+
+export interface TimeSeriesPoint {
+  benchmark: string;
+  iteration: number;
+  value: number;
+  isWarmup: boolean;
+  optStatus?: number;
+}
+
+export interface GcEvent {
+  benchmark: string;
+  sampleIndex: number;
+  duration: number;
+}
+
+export interface PausePoint {
+  benchmark: string;
+  sampleIndex: number;
+  durationMs: number;
+}
+
+export interface HeapPoint {
+  benchmark: string;
+  iteration: number;
+  value: number;
+}
+
+export interface BenchmarkStats {
+  min: number;
+  max: number;
+  avg: number;
+  p50: number;
+  p75: number;
+  p99: number;
+}
+
+export interface SectionStat {
+  groupTitle?: string;
+  label: string;
+  value: string;
+}
+
+export interface HistogramBin {
+  x: number;
+  count: number;
+}
+
+/** Bootstrap confidence interval for A/B comparison */
+export interface ComparisonCI {
+  percent: number;
+  ci: [number, number];
+  direction: "faster" | "slower" | "uncertain";
+  histogram?: HistogramBin[];
+}
+
+/** One benchmark's raw data, statistics, and optional comparison results */
+export interface BenchmarkEntry {
+  name: string;
+  samples: number[];
+  warmupSamples?: number[];
+  heapSamples?: number[];
+  gcEvents?: { offset: number; duration: number }[];
+  optSamples?: number[];
+  pausePoints?: { sampleIndex: number; durationMs: number }[];
+  stats: BenchmarkStats;
+  sectionStats?: SectionStat[];
+  comparisonCI?: ComparisonCI;
+  isBaseline: boolean;
+}
+
+export interface BenchmarkGroup {
+  baseline?: BenchmarkEntry;
+  benchmarks: BenchmarkEntry[];
+}
+
+export interface GitVersion {
+  hash: string;
+  date: string;
+  dirty?: boolean;
+}
+
+/** Top-level data structure for the HTML benchmark report */
+export interface ReportData {
+  metadata: {
+    cliArgs?: Record<string, unknown>;
+    gcTrackingEnabled?: boolean;
+    currentVersion?: GitVersion;
+    baselineVersion?: GitVersion;
+  };
+  groups: BenchmarkGroup[];
+}

package/src/html/browser/index.ts

@@ -0,0 +1 @@
+export { renderPlots } from "./RenderPlots.ts";

package/src/html/index.ts

@@ -0,0 +1,17 @@
+export { generateHtmlReport } from "./HtmlReport.ts";
+export {
+  formatDateWithTimezone,
+  formatRelativeTime,
+} from "./HtmlTemplate.ts";
+export type {
+  BenchmarkData,
+  DifferenceCI,
+  FormattedStat,
+  GcEvent,
+  GitVersion,
+  GroupData,
+  HtmlReportOptions,
+  HtmlReportResult,
+  PausePoint,
+  ReportData,
+} from "./Types.ts";

package/src/index.ts

@@ -0,0 +1,92 @@
+export type {
+  AnyVariant,
+  BenchMatrix,
+  CaseResult,
+  LoadedCase,
+  MatrixDefaults,
+  MatrixResults,
+  MatrixSuite,
+  RunMatrixOptions,
+  StatefulVariant,
+  Variant,
+  VariantFn,
+  VariantResult,
+} from "./BenchMatrix.ts";
+export { isStatefulVariant, runMatrix } from "./BenchMatrix.ts";
+export type { BenchGroup, BenchmarkSpec, BenchSuite } from "./Benchmark.ts";
+export type {
+  BenchmarkReport,
+  ReportColumnGroup,
+  ReportGroup,
+  ResultsMapper,
+  UnknownRecord,
+} from "./BenchmarkReport.ts";
+export { reportResults } from "./BenchmarkReport.ts";
+export type { Configure, DefaultCliArgs } from "./cli/CliArgs.ts";
+export { defaultCliArgs, parseCliArgs } from "./cli/CliArgs.ts";
+export type { ExportOptions, MatrixExportOptions } from "./cli/RunBenchCLI.ts";
+export {
+  benchExports,
+  cliToMatrixOptions,
+  defaultMatrixReport,
+  defaultReport,
+  exportReports,
+  hasField,
+  matrixBenchExports,
+  matrixToReportGroups,
+  parseBenchArgs,
+  printHeapReports,
+  reportOptStatus,
+  runBenchmarks,
+  runDefaultBench,
+  runDefaultMatrixBench,
+  runMatrixSuite,
+} from "./cli/RunBenchCLI.ts";
+export * from "./export/JsonFormat.ts";
+export { exportPerfettoTrace } from "./export/PerfettoExport.ts";
+export type { GitVersion } from "./GitUtils.ts";
+export {
+  formatDateWithTimezone,
+  formatGitVersion,
+  getBaselineVersion,
+  getCurrentGitVersion,
+} from "./GitUtils.ts";
+export type { PrepareHtmlOptions } from "./HtmlDataPrep.ts";
+export { prepareHtmlData } from "./HtmlDataPrep.ts";
+export type { HtmlReportOptions, ReportData } from "./html/index.ts";
+export { generateHtmlReport } from "./html/index.ts";
+export type { MeasuredResults } from "./MeasuredResults.ts";
+export type { CasesModule } from "./matrix/CaseLoader.ts";
+export { loadCaseData, loadCasesModule } from "./matrix/CaseLoader.ts";
+export type { FilteredMatrix, MatrixFilter } from "./matrix/MatrixFilter.ts";
+export { filterMatrix, parseMatrixFilter } from "./matrix/MatrixFilter.ts";
+export type {
+  ExtraColumn,
+  MatrixReportOptions,
+} from "./matrix/MatrixReport.ts";
+export {
+  gcPauseColumn,
+  gcStatsColumns,
+  heapTotalColumn,
+  reportMatrixResults,
+} from "./matrix/MatrixReport.ts";
+export type { RunnerOptions } from "./runners/BenchRunner.ts";
+export {
+  adaptiveSection,
+  buildGenericSections,
+  cpuSection,
+  gcSection,
+  gcStatsSection,
+  optSection,
+  runsSection,
+  timeSection,
+  totalTimeSection,
+} from "./StandardSections.ts";
+export { average } from "./StatisticalUtils.ts";
+export { formatConvergence } from "./table-util/ConvergenceFormatters.ts";
+export {
+  formatBytes,
+  integer,
+  timeMs,
+  truncate,
+} from "./table-util/Formatters.ts";

package/src/matrix/CaseLoader.ts

@@ -0,0 +1,36 @@
+import type { LoadedCase } from "../BenchMatrix.ts";
+
+/** Module that exports case definitions */
+export interface CasesModule<T = unknown> {
+  cases: string[];
+  defaultCases?: string[]; // subset for quick runs
+  defaultVariants?: string[]; // subset for quick runs
+  loadCase?: (id: string) => LoadedCase<T> | Promise<LoadedCase<T>>;
+}
+
+/** Load a cases module by URL */
+export async function loadCasesModule<T = unknown>(
+  moduleUrl: string,
+): Promise<CasesModule<T>> {
+  const module = await import(moduleUrl);
+  if (!Array.isArray(module.cases)) {
+    throw new Error(`Cases module at ${moduleUrl} must export 'cases' array`);
+  }
+  return {
+    cases: module.cases,
+    defaultCases: module.defaultCases,
+    defaultVariants: module.defaultVariants,
+    loadCase: module.loadCase,
+  };
+}
+
+/** Load case data from a CasesModule or pass through the caseId */
+export async function loadCaseData<T>(
+  casesModule: CasesModule<T> | undefined,
+  caseId: string,
+): Promise<LoadedCase<T>> {
+  if (casesModule?.loadCase) {
+    return casesModule.loadCase(caseId);
+  }
+  return { data: caseId as T };
+}

package/src/matrix/MatrixFilter.ts

@@ -0,0 +1,103 @@
+import type { BenchMatrix } from "../BenchMatrix.ts";
+import { loadCasesModule } from "./CaseLoader.ts";
+import { discoverVariants } from "./VariantLoader.ts";
+
+/** Filter for matrix case/variant selection */
+export interface MatrixFilter {
+  case?: string;
+  variant?: string;
+}
+
+/** Parse filter string: "case/variant", "case/", "/variant", or "case" */
+export function parseMatrixFilter(filter: string): MatrixFilter {
+  if (filter.includes("/")) {
+    const [casePart, variantPart] = filter.split("/", 2);
+    return {
+      case: casePart || undefined,
+      variant: variantPart || undefined,
+    };
+  }
+  return { case: filter };
+}
+
+/** Filtered matrix with explicit case and variant lists */
+export interface FilteredMatrix<T = unknown> extends BenchMatrix<T> {
+  filteredCases?: string[];
+  filteredVariants?: string[];
+}
+
+/** Apply filter to a matrix, merging with existing filters via intersection */
+export async function filterMatrix<T>(
+  matrix: FilteredMatrix<T>,
+  filter?: MatrixFilter,
+): Promise<FilteredMatrix<T>> {
+  if (!filter || (!filter.case && !filter.variant)) return matrix;
+
+  const caseList = await getFilteredCases(matrix, filter.case);
+  const variantList = await getFilteredVariants(matrix, filter.variant);
+
+  const filteredCases =
+    caseList && matrix.filteredCases
+      ? caseList.filter(c => matrix.filteredCases!.includes(c))
+      : (caseList ?? matrix.filteredCases);
+
+  const filteredVariants =
+    variantList && matrix.filteredVariants
+      ? variantList.filter(v => matrix.filteredVariants!.includes(v))
+      : (variantList ?? matrix.filteredVariants);
+
+  return { ...matrix, filteredCases, filteredVariants };
+}
+
+/** Get case IDs matching filter pattern */
+async function getFilteredCases<T>(
+  matrix: BenchMatrix<T>,
+  casePattern?: string,
+): Promise<string[] | undefined> {
+  if (!casePattern) return undefined;
+
+  const caseIds = matrix.casesModule
+    ? (await loadCasesModule(matrix.casesModule)).cases
+    : matrix.cases;
+  if (!caseIds) return ["default"]; // implicit single case
+
+  const filtered = caseIds.filter(id => matchPattern(id, casePattern));
+  if (filtered.length === 0) {
+    throw new Error(`No cases match filter: "${casePattern}"`);
+  }
+  return filtered;
+}
+
+/** Get variant IDs matching filter pattern */
+async function getFilteredVariants<T>(
+  matrix: BenchMatrix<T>,
+  variantPattern?: string,
+): Promise<string[] | undefined> {
+  if (!variantPattern) return undefined;
+
+  if (matrix.variants) {
+    const ids = Object.keys(matrix.variants).filter(id =>
+      matchPattern(id, variantPattern),
+    );
+    if (ids.length === 0) {
+      throw new Error(`No variants match filter: "${variantPattern}"`);
+    }
+    return ids;
+  }
+
+  if (matrix.variantDir) {
+    const allIds = await discoverVariants(matrix.variantDir);
+    const filtered = allIds.filter(id => matchPattern(id, variantPattern));
+    if (filtered.length === 0) {
+      throw new Error(`No variants match filter: "${variantPattern}"`);
+    }
+    return filtered;
+  }
+
+  throw new Error("BenchMatrix requires 'variants' or 'variantDir'");
+}
+
+/** Match id against pattern (case-insensitive substring) */
+function matchPattern(id: string, pattern: string): boolean {
+  return id.toLowerCase().includes(pattern.toLowerCase());
+}

package/src/matrix/MatrixReport.ts

@@ -0,0 +1,290 @@
+import type { CaseResult, MatrixResults } from "../BenchMatrix.ts";
+import { injectDiffColumns, type ResultsMapper } from "../BenchmarkReport.ts";
+import { totalProfileBytes } from "../heap-sample/HeapSampleReport.ts";
+import { type GcStatsInfo, gcStatsSection } from "../StandardSections.ts";
+import {
+  average,
+  bootstrapDifferenceCI,
+  type DifferenceCI,
+} from "../StatisticalUtils.ts";
+import {
+  duration,
+  formatBytes,
+  formatDiffWithCI,
+  truncate,
+} from "../table-util/Formatters.ts";
+import {
+  buildTable,
+  type ColumnGroup,
+  type ResultGroup,
+} from "../table-util/TableReport.ts";
+
+/** Custom column definition for extra computed metrics */
+export interface ExtraColumn {
+  key: string;
+  title: string;
+  groupTitle?: string; // optional column group header
+  extract: (caseResult: CaseResult) => unknown;
+  formatter?: (value: unknown) => string;
+}
+
+/** Options for matrix report generation */
+export interface MatrixReportOptions {
+  extraColumns?: ExtraColumn[];
+  sections?: ResultsMapper[]; // ResultsMapper sections (like BenchSuite)
+  variantTitle?: string; // custom title for the variant column (default: "variant")
+}
+
+/** Row data for matrix report table */
+interface MatrixReportRow extends Record<string, unknown> {
+  name: string;
+  time: number;
+  samples: number;
+  diffCI?: DifferenceCI;
+}
+
+/** Format matrix results as one table per case */
+export function reportMatrixResults(
+  results: MatrixResults,
+  options?: MatrixReportOptions,
+): string {
+  const tables = buildCaseTables(results, options);
+  const header = `Matrix: ${results.name}`;
+  return [header, ...tables].join("\n\n");
+}
+
+/** Build one table for each case showing all variants */
+function buildCaseTables(
+  results: MatrixResults,
+  options?: MatrixReportOptions,
+): string[] {
+  if (results.variants.length === 0) return [];
+
+  // Get all case IDs from first variant (all variants have same cases)
+  const caseIds = results.variants[0].cases.map(c => c.caseId);
+  return caseIds.map(caseId => buildCaseTable(results, caseId, options));
+}
+
+/** Build table for a single case showing all variants */
+function buildCaseTable(
+  results: MatrixResults,
+  caseId: string,
+  options?: MatrixReportOptions,
+): string {
+  const caseTitle = formatCaseTitle(results, caseId);
+
+  if (options?.sections?.length) {
+    return buildSectionTable(results, caseId, options, caseTitle);
+  }
+
+  const rows = buildCaseRows(results, caseId, options?.extraColumns);
+  const hasBaseline = rows.some(r => r.diffCI);
+  const columns = buildColumns(hasBaseline, options);
+
+  const resultGroup: ResultGroup<MatrixReportRow> = { results: rows };
+  const table = buildTable(columns, [resultGroup]);
+  return `${caseTitle}\n${table}`;
+}
+
+/** Build table using ResultsMapper sections */
+function buildSectionTable(
+  results: MatrixResults,
+  caseId: string,
+  options: MatrixReportOptions,
+  caseTitle: string,
+): string {
+  const sections = options.sections!;
+  const variantTitle = options.variantTitle ?? "name";
+
+  const rows: Record<string, unknown>[] = [];
+  let hasBaseline = false;
+
+  for (const variant of results.variants) {
+    const caseResult = variant.cases.find(c => c.caseId === caseId);
+    if (!caseResult) continue;
+
+    const row: Record<string, unknown> = { name: truncate(variant.id, 25) };
+
+    for (const section of sections) {
+      Object.assign(
+        row,
+        section.extract(caseResult.measured, caseResult.metadata),
+      );
+    }
+
+    if (caseResult.baseline) {
+      hasBaseline = true;
+      const { samples: base } = caseResult.baseline;
+      row.diffCI = bootstrapDifferenceCI(base, caseResult.measured.samples);
+    }
+
+    rows.push(row);
+  }
+
+  const columnGroups = buildSectionColumns(sections, variantTitle, hasBaseline);
+  const resultGroup: ResultGroup<Record<string, unknown>> = { results: rows };
+  const table = buildTable(columnGroups, [resultGroup]);
+  return `${caseTitle}\n${table}`;
+}
+
+/** Build column groups from ResultsMapper sections */
+function buildSectionColumns(
+  sections: ResultsMapper[],
+  variantTitle: string,
+  hasBaseline: boolean,
+): ColumnGroup<Record<string, unknown>>[] {
+  const nameCol: ColumnGroup<Record<string, unknown>> = {
+    columns: [{ key: "name", title: variantTitle }],
+  };
+
+  const sectionColumns = sections.flatMap(s => s.columns());
+  const columnGroups = hasBaseline
+    ? injectDiffColumns(sectionColumns)
+    : (sectionColumns as ColumnGroup<Record<string, unknown>>[]);
+
+  return [nameCol, ...columnGroups];
+}
+
+/** Build rows for all variants for a given case */
+function buildCaseRows(
+  results: MatrixResults,
+  caseId: string,
+  extraColumns?: ExtraColumn[],
+): MatrixReportRow[] {
+  return results.variants.flatMap(variant => {
+    const caseResult = variant.cases.find(c => c.caseId === caseId);
+    return caseResult ? [buildRow(variant.id, caseResult, extraColumns)] : [];
+  });
+}
+
+/** Build a single row from case result */
+function buildRow(
+  variantId: string,
+  caseResult: CaseResult,
+  extraColumns?: ExtraColumn[],
+): MatrixReportRow {
+  const { measured, baseline } = caseResult;
+  const samples = measured.samples;
+  const time = measured.time?.avg ?? average(samples);
+
+  const row: MatrixReportRow = {
+    name: truncate(variantId, 25),
+    time,
+    samples: samples.length,
+  };
+
+  if (baseline) {
+    row.diffCI = bootstrapDifferenceCI(baseline.samples, samples);
+  }
+
+  if (extraColumns) {
+    for (const col of extraColumns) {
+      row[col.key] = col.extract(caseResult);
+    }
+  }
+
+  return row;
+}
+
+/** Build column configuration */
+function buildColumns(
+  hasBaseline: boolean,
+  options?: MatrixReportOptions,
+): ColumnGroup<MatrixReportRow>[] {
+  const variantTitle = options?.variantTitle ?? "variant";
+  const nameCol: ColumnGroup<MatrixReportRow> = {
+    columns: [{ key: "name", title: variantTitle }],
+  };
+
+  const ciKey = "diffCI" as keyof MatrixReportRow;
+  const diffCol = { key: ciKey, title: "Δ% CI", formatter: formatDiff };
+  const timeCol: ColumnGroup<MatrixReportRow> = {
+    columns: [
+      { key: "time", title: "time", formatter: duration },
+      ...(hasBaseline ? [diffCol] : []),
+    ],
+  };
+
+  const groups: ColumnGroup<MatrixReportRow>[] = [nameCol, timeCol];
+
+  // Add extra columns, grouped by groupTitle
+  const extraColumns = options?.extraColumns;
+  if (extraColumns?.length) {
+    const byGroup = new Map<string | undefined, ExtraColumn[]>();
+    for (const col of extraColumns) {
+      const group = byGroup.get(col.groupTitle) ?? [];
+      group.push(col);
+      byGroup.set(col.groupTitle, group);
+    }
+    for (const [groupTitle, cols] of byGroup) {
+      groups.push({
+        groupTitle,
+        columns: cols.map(col => ({
+          key: col.key as keyof MatrixReportRow,
+          title: col.title,
+          formatter: col.formatter ?? String,
+        })),
+      });
+    }
+  }
+
+  return groups;
+}
+
+/** Format diff with CI, or "baseline" marker */
+function formatDiff(value: unknown): string | null {
+  if (!value) return null;
+  return formatDiffWithCI(value as DifferenceCI);
+}
+
+/** Format case title with metadata if available */
+function formatCaseTitle(results: MatrixResults, caseId: string): string {
+  const caseResult = results.variants[0]?.cases.find(c => c.caseId === caseId);
+  const metadata = caseResult?.metadata;
+
+  if (metadata && Object.keys(metadata).length > 0) {
+    const metaParts = Object.entries(metadata)
+      .map(([k, v]) => `${v} ${k}`)
+      .join(", ");
+    return `${caseId} (${metaParts})`;
+  }
+  return caseId;
+}
+
+/** GC statistics columns - derived from gcStatsSection for consistency */
+export const gcStatsColumns: ExtraColumn[] = gcStatsSection
+  .columns()[0]
+  .columns.map(col => ({
+    key: col.key as string,
+    title: col.title,
+    groupTitle: "GC",
+    extract: (r: CaseResult) =>
+      gcStatsSection.extract(r.measured)[col.key as keyof GcStatsInfo],
+    formatter: (v: unknown) => col.formatter?.(v) ?? "-",
+  }));
+
+/** Format bytes with fallback to "-" for missing values */
+function formatBytesOrDash(value: unknown): string {
+  return formatBytes(value) ?? "-";
+}
+
+/** GC pause time column */
+export const gcPauseColumn: ExtraColumn = {
+  key: "gcPause",
+  title: "pause",
+  groupTitle: "GC",
+  extract: r => r.measured.gcStats?.gcPauseTime,
+  formatter: v => (v != null ? `${(v as number).toFixed(1)}ms` : "-"),
+};
+
+/** Heap sampling total bytes column */
+export const heapTotalColumn: ExtraColumn = {
+  key: "heapTotal",
+  title: "heap",
+  extract: r => {
+    const profile = r.measured.heapProfile;
+    if (!profile?.head) return undefined;
+    return totalProfileBytes(profile);
+  },
+  formatter: formatBytesOrDash,
+};

package/src/matrix/VariantLoader.ts

@@ -0,0 +1,46 @@
+import fs from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import type { Variant } from "../BenchMatrix.ts";
+
+/** Discover variant ids from a directory of .ts files */
+export async function discoverVariants(dirUrl: string): Promise<string[]> {
+  const dirPath = fileURLToPath(dirUrl);
+  const entries = await fs.readdir(dirPath, { withFileTypes: true });
+  return entries
+    .filter(e => e.isFile() && e.name.endsWith(".ts"))
+    .map(e => e.name.slice(0, -3))
+    .sort();
+}
+
+/** Load a variant module and extract run/setup exports */
+export async function loadVariant<T = unknown>(
+  dirUrl: string,
+  variantId: string,
+): Promise<Variant<T>> {
+  const moduleUrl = variantModuleUrl(dirUrl, variantId);
+  const module = await import(moduleUrl);
+  return extractVariant(module, variantId, moduleUrl);
+}
+
+/** Extract variant from module exports */
+function extractVariant<T>(
+  module: Record<string, unknown>,
+  variantId: string,
+  moduleUrl: string,
+): Variant<T> {
+  const { setup, run } = module;
+  const loc = `Variant '${variantId}' at ${moduleUrl}`;
+  if (typeof run !== "function") {
+    throw new Error(`${loc} must export 'run'`);
+  }
+  if (setup === undefined) return run as (data: T) => void;
+  if (typeof setup !== "function") {
+    throw new Error(`${loc}: 'setup' must be a function`);
+  }
+  return { setup: setup as (data: T) => unknown, run: run as () => void };
+}
+
+/** Get module URL for a variant in a directory */
+export function variantModuleUrl(dirUrl: string, variantId: string): string {
+  return new URL(`${variantId}.ts`, dirUrl).href;
+}