benchforge 0.1.0

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "benchforge",
+  "version": "0.1.0",
+  "type": "module",
+  "bin": {
+    "benchforge": "dist/bin/benchforge.mjs"
+  },
+  "files": [
+    "bin",
+    "dist",
+    "src"
+  ],
+  "exports": {
+    ".": {
+      "import": "./dist/index.mjs"
+    }
+  },
+  "dependencies": {
+    "esbuild": "^0.27.3",
+    "open": "^11.0.0",
+    "picocolors": "^1.1.1",
+    "table": "^6.9.0",
+    "yargs": "^18.0.0"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.4.4",
+    "@types/node": "^25.3.0",
+    "@types/yargs": "^17.0.35",
+    "@typescript/native-preview": "7.0.0-dev.20260220.1",
+    "npm-run-all": "^4.1.5",
+    "playwright": "^1.58.2",
+    "tsdown": "0.20.3",
+    "vite": "^7.3.1",
+    "vitest": "^4.0.18"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "build:plots": "tsdown --config tsdown.plots.config.ts",
+    "example:bench": "bin/benchforge --url file://$PWD/examples/browser-bench/index.html --gc-stats --heap-sample --html",
+    "example:heap": "bin/benchforge --url file://$PWD/examples/browser-heap/index.html --heap-sample --gc-stats",
+    "example:lap": "bin/benchforge --url file://$PWD/examples/browser-lap/index.html --gc-stats --heap-sample --html",
+    "fix": "biome check --fix --unsafe --error-on-warnings",
+    "lint": "biome check --error-on-warnings",
+    "test": "pnpm --node-options=--expose-gc vitest --hideSkippedTests --exclude '**/*.e2e.test.ts'",
+    "test:e2e": "pnpm --node-options=--expose-gc vitest run '.e2e.'",
+    "test:once": "pnpm --node-options=--expose-gc vitest run --exclude '**/*.e2e.test.ts'",
+    "typecheck": "tsgo"
+  }
+}

package/src/BenchMatrix.ts

@@ -0,0 +1,380 @@
+import type { MeasuredResults } from "./MeasuredResults.ts";
+import { loadCaseData, loadCasesModule } from "./matrix/CaseLoader.ts";
+import { discoverVariants } from "./matrix/VariantLoader.ts";
+import { BasicRunner } from "./runners/BasicRunner.ts";
+import type { RunnerOptions } from "./runners/BenchRunner.ts";
+import { runMatrixVariant } from "./runners/RunnerOrchestrator.ts";
+import { average } from "./StatisticalUtils.ts";
+
+/** Stateless variant - called each iteration with case data */
+export type VariantFn<T = unknown> = (caseData: T) => void;
+
+/** Stateful variant - setup once, run many */
+export interface StatefulVariant<T = unknown, S = unknown> {
+  setup: (caseData: T) => S | Promise<S>;
+  run: (state: S) => void;
+}
+
+/** A variant is either a plain function or a stateful setup+run pair */
+export type Variant<T = unknown, S = unknown> =
+  | VariantFn<T>
+  | StatefulVariant<T, S>;
+
+/** Variant with any state type - used in BenchMatrix to allow mixed variants */
+export type AnyVariant<T = unknown> = VariantFn<T> | StatefulVariant<T, any>;
+
+/** Result from casesModule.loadCase() */
+export interface LoadedCase<T = unknown> {
+  data: T;
+  metadata?: Record<string, unknown>;
+}
+
+export interface MatrixDefaults {
+  warmup?: number;
+  maxTime?: number;
+  iterations?: number;
+}
+
+/** Bench matrix configuration */
+export interface BenchMatrix<T = unknown> {
+  name: string;
+  variantDir?: string;
+  variants?: Record<string, AnyVariant<T>>;
+  cases?: string[];
+  casesModule?: string;
+  baselineDir?: string;
+  baselineVariant?: string;
+  defaults?: MatrixDefaults;
+}
+
+/** Collection of matrices */
+export interface MatrixSuite {
+  name: string;
+  matrices: BenchMatrix<any>[];
+}
+
+/** Results for a single variant across all cases */
+export interface VariantResult {
+  id: string;
+  cases: CaseResult[];
+}
+
+/** Results for a single (variant, case) pair */
+export interface CaseResult {
+  caseId: string;
+  measured: MeasuredResults;
+  metadata?: Record<string, unknown>;
+  baseline?: MeasuredResults;
+  deltaPercent?: number;
+}
+
+/** Results from running a matrix */
+export interface MatrixResults {
+  name: string;
+  variants: VariantResult[];
+}
+
+/** @return true if variant is a StatefulVariant (has setup + run) */
+export function isStatefulVariant<T, S>(
+  v: Variant<T, S>,
+): v is StatefulVariant<T, S> {
+  return typeof v === "object" && "setup" in v && "run" in v;
+}
+
+/** Options for runMatrix */
+export interface RunMatrixOptions {
+  iterations?: number;
+  maxTime?: number;
+  warmup?: number;
+  useWorker?: boolean; // use worker process isolation (default: true for variantDir)
+  filteredCases?: string[]; // run only these cases (from filter)
+  filteredVariants?: string[]; // run only these variants (from filter)
+  // Runner options passthrough
+  collect?: boolean;
+  cpuCounters?: boolean;
+  traceOpt?: boolean;
+  noSettle?: boolean;
+  pauseFirst?: number;
+  pauseInterval?: number;
+  pauseDuration?: number;
+  gcStats?: boolean;
+  heapSample?: boolean;
+  heapInterval?: number;
+  heapDepth?: number;
+}
+
+/** Run a BenchMatrix with inline variants or variantDir */
+export async function runMatrix<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions = {},
+): Promise<MatrixResults> {
+  validateBaseline(matrix);
+  const effectiveOptions = { ...matrix.defaults, ...options };
+
+  if (matrix.variantDir) {
+    return runMatrixWithDir(matrix, effectiveOptions);
+  }
+  if (matrix.variants) {
+    return runMatrixInline(matrix, effectiveOptions);
+  }
+  throw new Error("BenchMatrix requires either 'variants' or 'variantDir'");
+}
+
+/** @throws if both baselineDir and baselineVariant are set */
+function validateBaseline<T>(matrix: BenchMatrix<T>): void {
+  const msg =
+    "BenchMatrix cannot have both 'baselineDir' and 'baselineVariant'";
+  if (matrix.baselineDir && matrix.baselineVariant) throw new Error(msg);
+}
+
+function buildRunnerOptions(options: RunMatrixOptions): RunnerOptions {
+  return {
+    maxIterations: options.iterations,
+    maxTime: options.maxTime ?? 1000,
+    warmup: options.warmup ?? 0,
+    collect: options.collect,
+    cpuCounters: options.cpuCounters,
+    traceOpt: options.traceOpt,
+    noSettle: options.noSettle,
+    pauseFirst: options.pauseFirst,
+    pauseInterval: options.pauseInterval,
+    pauseDuration: options.pauseDuration,
+    gcStats: options.gcStats,
+    heapSample: options.heapSample,
+    heapInterval: options.heapInterval,
+    heapDepth: options.heapDepth,
+  };
+}
+
+/** Load cases module and resolve filtered case IDs */
+async function resolveCases<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+) {
+  const casesModule = matrix.casesModule
+    ? await loadCasesModule<T>(matrix.casesModule)
+    : undefined;
+  const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
+  const caseIds = options.filteredCases ?? allCaseIds;
+  return { casesModule, caseIds };
+}
+
+/** Run matrix with inline variants (non-worker mode) */
+async function runMatrixInline<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<MatrixResults> {
+  // baselineDir is only valid with variantDir
+  const msg =
+    "BenchMatrix with inline 'variants' cannot use 'baselineDir'. Use 'variantDir' instead.";
+  if (matrix.baselineDir) throw new Error(msg);
+
+  const { casesModule, caseIds } = await resolveCases(matrix, options);
+  const runner = new BasicRunner();
+  const runnerOpts = buildRunnerOptions(options);
+
+  const variantEntries = options.filteredVariants
+    ? Object.entries(matrix.variants!).filter(([id]) =>
+        options.filteredVariants!.includes(id),
+      )
+    : Object.entries(matrix.variants!);
+
+  const variants: VariantResult[] = [];
+  for (const [variantId, variant] of variantEntries) {
+    const cases: CaseResult[] = [];
+    for (const caseId of caseIds) {
+      const loaded = await loadCaseData(casesModule, caseId);
+      const caseData =
+        casesModule || matrix.cases ? loaded.data : (undefined as T);
+      const measured = await runVariant(
+        variant,
+        caseData,
+        variantId,
+        runner,
+        runnerOpts,
+      );
+      cases.push({ caseId, measured, metadata: loaded.metadata });
+    }
+    variants.push({ id: variantId, cases });
+  }
+
+  if (matrix.baselineVariant) {
+    applyBaselineVariant(variants, matrix.baselineVariant);
+  }
+
+  return { name: matrix.name, variants };
+}
+
+/** Context for running matrix benchmarks in worker mode */
+interface DirMatrixContext<T> {
+  matrix: BenchMatrix<T>;
+  casesModule?: import("./matrix/CaseLoader.ts").CasesModule<T>;
+  baselineIds: string[];
+  caseIds: string[];
+  runnerOpts: RunnerOptions;
+}
+
+/** Run matrix with variantDir (worker mode for memory isolation) */
+async function runMatrixWithDir<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<MatrixResults> {
+  const allVariantIds = await discoverVariants(matrix.variantDir!);
+  if (allVariantIds.length === 0) {
+    throw new Error(`No variants found in ${matrix.variantDir}`);
+  }
+  const variantIds = options.filteredVariants ?? allVariantIds;
+
+  const ctx = await createDirContext(matrix, options);
+  const variants = await runDirVariants(variantIds, ctx);
+
+  if (matrix.baselineVariant) {
+    applyBaselineVariant(variants, matrix.baselineVariant);
+  }
+  return { name: matrix.name, variants };
+}
+
+/** Create context for directory-based matrix execution */
+async function createDirContext<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<DirMatrixContext<T>> {
+  const baselineIds = matrix.baselineDir
+    ? await discoverVariants(matrix.baselineDir)
+    : [];
+  const { casesModule, caseIds } = await resolveCases(matrix, options);
+  const runnerOpts = buildRunnerOptions(options);
+  return { matrix, casesModule, baselineIds, caseIds, runnerOpts };
+}
+
+/** Run all variants using worker processes */
+async function runDirVariants<T>(
+  variantIds: string[],
+  ctx: DirMatrixContext<T>,
+): Promise<VariantResult[]> {
+  const variants: VariantResult[] = [];
+  for (const variantId of variantIds) {
+    const cases = await runDirVariantCases(variantId, ctx);
+    variants.push({ id: variantId, cases });
+  }
+  return variants;
+}
+
+/** Run all cases for a single variant */
+async function runDirVariantCases<T>(
+  variantId: string,
+  ctx: DirMatrixContext<T>,
+): Promise<CaseResult[]> {
+  const { matrix, casesModule, caseIds, runnerOpts } = ctx;
+  const cases: CaseResult[] = [];
+
+  for (const caseId of caseIds) {
+    const caseData = !matrix.casesModule && matrix.cases ? caseId : undefined;
+    const [measured] = await runMatrixVariant({
+      variantDir: matrix.variantDir!,
+      variantId,
+      caseId,
+      caseData,
+      casesModule: matrix.casesModule,
+      runner: "basic",
+      options: runnerOpts,
+    });
+
+    const loaded = await loadCaseData(casesModule, caseId);
+    const baseline = await runBaselineIfExists(
+      variantId,
+      caseId,
+      caseData,
+      ctx,
+    );
+    const deltaPercent = baseline
+      ? computeDeltaPercent(baseline, measured)
+      : undefined;
+    const metadata = loaded.metadata;
+    cases.push({ caseId, measured, metadata, baseline, deltaPercent });
+  }
+  return cases;
+}
+
+/** Run baseline variant if it exists in baselineDir */
+async function runBaselineIfExists<T>(
+  variantId: string,
+  caseId: string,
+  caseData: unknown,
+  ctx: DirMatrixContext<T>,
+): Promise<MeasuredResults | undefined> {
+  const { matrix, baselineIds, runnerOpts } = ctx;
+  if (!matrix.baselineDir || !baselineIds.includes(variantId)) return undefined;
+
+  const [measured] = await runMatrixVariant({
+    variantDir: matrix.baselineDir,
+    variantId,
+    caseId,
+    caseData,
+    casesModule: matrix.casesModule,
+    runner: "basic",
+    options: runnerOpts,
+  });
+  return measured;
+}
+
+/** Compute delta percentage: (current - baseline) / baseline * 100 */
+function computeDeltaPercent(
+  baseline: MeasuredResults,
+  current: MeasuredResults,
+): number {
+  const baseAvg = average(baseline.samples);
+  if (baseAvg === 0) return 0;
+  return ((average(current.samples) - baseAvg) / baseAvg) * 100;
+}
+
+/** Apply baselineVariant comparison - one variant is the reference for all others */
+function applyBaselineVariant(
+  variants: VariantResult[],
+  baselineVariantId: string,
+): void {
+  const baselineVariant = variants.find(v => v.id === baselineVariantId);
+  if (!baselineVariant) return;
+
+  const baselineByCase = new Map<string, MeasuredResults>();
+  for (const c of baselineVariant.cases) {
+    baselineByCase.set(c.caseId, c.measured);
+  }
+
+  for (const variant of variants) {
+    if (variant.id === baselineVariantId) continue;
+    for (const caseResult of variant.cases) {
+      const baseline = baselineByCase.get(caseResult.caseId);
+      if (baseline) {
+        caseResult.baseline = baseline;
+        caseResult.deltaPercent = computeDeltaPercent(
+          baseline,
+          caseResult.measured,
+        );
+      }
+    }
+  }
+}
+
+/** Run a single variant with case data */
+async function runVariant<T>(
+  variant: AnyVariant<T>,
+  caseData: T,
+  name: string,
+  runner: BasicRunner,
+  options: RunnerOptions,
+): Promise<MeasuredResults> {
+  if (isStatefulVariant(variant)) {
+    const state = await variant.setup(caseData);
+    const [result] = await runner.runBench(
+      { name, fn: () => variant.run(state) },
+      options,
+    );
+    return result;
+  }
+  const [result] = await runner.runBench(
+    { name, fn: () => variant(caseData) },
+    options,
+  );
+  return result;
+}

package/src/Benchmark.ts

@@ -0,0 +1,33 @@
+/** Single benchmark function specification */
+export interface BenchmarkSpec<T = unknown> {
+  name: string;
+  fn: BenchmarkFunction<T>;
+  /** Path to module exporting the benchmark function (for worker mode) */
+  modulePath?: string;
+  /** Name of the exported function in the module (defaults to default export) */
+  exportName?: string;
+  /** Setup function export name - called once in worker, result passed to fn */
+  setupExportName?: string;
+}
+
+export type BenchmarkFunction<T = unknown> =
+  | ((params: T) => void)
+  | (() => void);
+
+/** Group of benchmarks with shared setup */
+export interface BenchGroup<T = unknown> {
+  name: string;
+  /** Prepare parameters for all benchmarks in this group */
+  setup?: () => T | Promise<T>;
+  benchmarks: BenchmarkSpec<T>[];
+  /** Baseline benchmark for comparison */
+  baseline?: BenchmarkSpec<T>;
+  /** Metadata for reporting (e.g., lines of code) */
+  metadata?: Record<string, any>;
+}
+
+/** Collection of benchmark groups */
+export interface BenchSuite {
+  name: string;
+  groups: BenchGroup<any>[];
+}

package/src/BenchmarkReport.ts

@@ -0,0 +1,156 @@
+import type { MeasuredResults } from "./MeasuredResults.ts";
+import { bootstrapDifferenceCI } from "./StatisticalUtils.ts";
+import type { UnionToIntersection } from "./TypeUtil.ts";
+import {
+  formatDiffWithCI,
+  formatDiffWithCIHigherIsBetter,
+  truncate,
+} from "./table-util/Formatters.ts";
+import {
+  type AnyColumn,
+  buildTable,
+  type ColumnGroup,
+  type ResultGroup,
+} from "./table-util/TableReport.ts";
+
+/** Benchmark results with optional baseline for comparison */
+export interface ReportGroup {
+  name: string;
+  reports: BenchmarkReport[];
+  baseline?: BenchmarkReport;
+}
+
+/** Results from a single benchmark run */
+export interface BenchmarkReport {
+  name: string;
+  measuredResults: MeasuredResults;
+  metadata?: UnknownRecord;
+}
+
+export interface ReportColumnGroup<T> {
+  groupTitle?: string;
+  columns: ReportColumn<T>[];
+}
+
+export type ReportColumn<T> = AnyColumn<T> & {
+  /** Add diff column after this column when baseline exists */
+  comparable?: boolean;
+  /** Set true for throughput metrics where higher values are better (e.g., lines/sec) */
+  higherIsBetter?: boolean;
+};
+
+/** Maps benchmark results to table columns */
+export interface ResultsMapper<
+  T extends Record<string, any> = Record<string, any>,
+> {
+  extract(results: MeasuredResults, metadata?: UnknownRecord): T;
+  columns(): ReportColumnGroup<T>[];
+}
+export type UnknownRecord = Record<string, unknown>;
+
+type SectionStats<S> = S extends ResultsMapper<infer T> ? T : never;
+
+interface ReportRowBase {
+  name: string;
+}
+
+/** Row data combining all section statistics */
+type ReportRowData<S extends ReadonlyArray<ResultsMapper<any>>> =
+  ReportRowBase & UnionToIntersection<SectionStats<S[number]>>;
+
+/** @return formatted table report with optional baseline comparisons */
+export function reportResults<S extends ReadonlyArray<ResultsMapper<any>>>(
+  groups: ReportGroup[],
+  sections: S,
+): string {
+  const results = groups.map(group => resultGroupValues(group, sections));
+  const hasBaseline = results.some(g => g.baseline);
+  return buildTable(createColumnGroups(sections, hasBaseline), results);
+}
+
+/** @return values for report group */
+function resultGroupValues<S extends ReadonlyArray<ResultsMapper<any>>>(
+  group: ReportGroup,
+  sections: S,
+): ResultGroup<ReportRowData<S>> {
+  const { reports, baseline } = group;
+  const baselineSamples = baseline?.measuredResults.samples;
+
+  const results = reports.map(report => {
+    const row = {
+      name: truncate(report.name),
+      ...extractReportValues(report, sections),
+    } as ReportRowData<S>;
+
+    if (baselineSamples && report.measuredResults.samples) {
+      (row as any).diffCI = bootstrapDifferenceCI(
+        baselineSamples,
+        report.measuredResults.samples,
+      );
+    }
+    return row;
+  });
+
+  const baselineRow = baseline && valuesForReports([baseline], sections)[0];
+  return { results, baseline: baselineRow };
+}
+
+/** @return rows with stats from sections */
+export function valuesForReports<S extends ReadonlyArray<ResultsMapper<any>>>(
+  reports: BenchmarkReport[],
+  sections: S,
+): ReportRowData<S>[] {
+  return reports.map(report => ({
+    name: truncate(report.name),
+    ...extractReportValues(report, sections),
+  })) as ReportRowData<S>[];
+}
+
+/** @return merged statistics from all sections */
+function extractReportValues(
+  report: BenchmarkReport,
+  sections: ReadonlyArray<ResultsMapper<any>>,
+): UnknownRecord {
+  const { measuredResults, metadata } = report;
+  const entries = sections.flatMap(s =>
+    Object.entries(s.extract(measuredResults, metadata)),
+  );
+  return Object.fromEntries(entries);
+}
+
+/** @return column groups with diff columns if baseline exists */
+function createColumnGroups<S extends ReadonlyArray<ResultsMapper<any>>>(
+  sections: S,
+  hasBaseline: boolean,
+): ColumnGroup<ReportRowData<S>>[] {
+  const nameColumn: ColumnGroup<ReportRowData<S>> = {
+    columns: [{ key: "name" as keyof ReportRowData<S>, title: "name" }],
+  };
+
+  const groups = sections.flatMap(section => section.columns());
+  return [nameColumn, ...(hasBaseline ? injectDiffColumns(groups) : groups)];
+}
+
+/** @return groups with single CI column after first comparable field */
+export function injectDiffColumns<T>(
+  reportGroups: ReportColumnGroup<T>[],
+): ColumnGroup<T>[] {
+  let ciAdded = false;
+
+  return reportGroups.map(group => ({
+    groupTitle: group.groupTitle,
+    columns: group.columns.flatMap(col => {
+      if (col.comparable && !ciAdded) {
+        ciAdded = true;
+        const fmt = col.higherIsBetter
+          ? formatDiffWithCIHigherIsBetter
+          : formatDiffWithCI;
+        return [
+          col,
+          { title: "Δ% CI", key: "diffCI" as keyof T, formatter: fmt },
+        ];
+      }
+      return [col];
+    }),
+  }));
+}

package/src/GitUtils.ts

@@ -0,0 +1,79 @@
+import { execSync } from "node:child_process";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import type { GitVersion } from "./html/index.ts";
+import { formatDateWithTimezone } from "./html/index.ts";
+
+export type { GitVersion } from "./html/index.ts";
+export { formatDateWithTimezone, formatRelativeTime } from "./html/index.ts";
+
+/** Get current git version info. For dirty repos, uses most recent modified file date. */
+export function getCurrentGitVersion(): GitVersion | undefined {
+  try {
+    const exec = (cmd: string) => execSync(cmd, { encoding: "utf-8" }).trim();
+    const hash = exec("git rev-parse --short HEAD");
+    const commitDate = exec("git log -1 --format=%aI");
+    const dirty = exec("git status --porcelain").length > 0;
+
+    const date = dirty
+      ? (getMostRecentModifiedDate(".") ?? commitDate)
+      : commitDate;
+    return { hash, date, dirty };
+  } catch {
+    return undefined;
+  }
+}
+
+/** Read baseline version from .baseline-version file */
+export function getBaselineVersion(
+  baselineDir = "_baseline",
+): GitVersion | undefined {
+  const versionFile = join(baselineDir, ".baseline-version");
+  if (!existsSync(versionFile)) return undefined;
+
+  try {
+    const content = readFileSync(versionFile, "utf-8");
+    const data = JSON.parse(content);
+    return { hash: data.hash, date: data.date };
+  } catch {
+    return undefined;
+  }
+}
+
+/** Format git version for display: "abc1234 (Jan 9, 2026, 3:45 PM)" or "abc1234*" if dirty */
+export function formatGitVersion(version: GitVersion): string {
+  const hashDisplay = version.dirty ? `${version.hash}*` : version.hash;
+  const dateDisplay = formatDateWithTimezone(version.date);
+  return `${hashDisplay} (${dateDisplay})`;
+}
+
+/** Get most recent modified file date in a directory (for dirty repos) */
+export function getMostRecentModifiedDate(dir: string): string | undefined {
+  try {
+    const raw = execSync("git status --porcelain", {
+      encoding: "utf-8",
+      cwd: dir,
+    });
+    const modifiedFiles = raw
+      .trim()
+      .split("\n")
+      .filter(line => line.length > 0)
+      .map(line => line.slice(3));
+
+    if (modifiedFiles.length === 0) return undefined;
+
+    let mostRecent = 0;
+    for (const file of modifiedFiles) {
+      try {
+        const filePath = join(dir, file);
+        if (!existsSync(filePath)) continue;
+        const mtime = statSync(filePath).mtimeMs;
+        if (mtime > mostRecent) mostRecent = mtime;
+      } catch {}
+    }
+
+    return mostRecent > 0 ? new Date(mostRecent).toISOString() : undefined;
+  } catch {
+    return undefined;
+  }
+}