benchforge 0.1.11 → 0.2.4

package/src/index.ts

@@ -1,3 +1,36 @@
+export type { Configure, DefaultCliArgs } from "./cli/CliArgs.ts";
+export { browserCliArgs, defaultCliArgs, parseCliArgs } from "./cli/CliArgs.ts";
+export type { ExportOptions, MatrixExportOptions } from "./cli/CliExport.ts";
+export { exportReports } from "./cli/CliExport.ts";
+export { cliToMatrixOptions } from "./cli/CliOptions.ts";
+export {
+  defaultMatrixReport,
+  defaultReport,
+  matrixToReportGroups,
+  printHeapReports,
+  reportOptStatus,
+} from "./cli/CliReport.ts";
+export {
+  benchExports,
+  dispatchCli,
+  matrixBenchExports,
+  parseBenchArgs,
+  runDefaultBench,
+  runDefaultMatrixBench,
+  runMatrixSuite,
+} from "./cli/RunBenchCLI.ts";
+export { runBenchmarks } from "./cli/SuiteRunner.ts";
+export {
+  buildSpeedscopeFile,
+  exportSpeedscope,
+  heapProfileToSpeedscope,
+} from "./export/AllocExport.ts";
+export { archiveBenchmark } from "./export/ArchiveExport.ts";
+export type {
+  ArchiveMetadata,
+  BenchforgeArchive,
+} from "./export/ArchiveFormat.ts";
+export { exportPerfettoTrace } from "./export/PerfettoExport.ts";
 export type {
   AnyVariant,
   BenchMatrix,
@@ -11,88 +44,74 @@ export type {
   Variant,
   VariantFn,
   VariantResult,
-} from "./BenchMatrix.ts";
-export { isStatefulVariant, runMatrix } from "./BenchMatrix.ts";
-export type { BenchGroup, BenchmarkSpec, BenchSuite } from "./Benchmark.ts";
+} from "./matrix/BenchMatrix.ts";
+export { isStatefulVariant, runMatrix } from "./matrix/BenchMatrix.ts";
+export type { CasesModule } from "./matrix/CaseLoader.ts";
+export { loadCaseData, loadCasesModule } from "./matrix/CaseLoader.ts";
+export type { FilteredMatrix, MatrixFilter } from "./matrix/MatrixFilter.ts";
+export { filterMatrix, parseMatrixFilter } from "./matrix/MatrixFilter.ts";
+export type { MatrixReportOptions } from "./matrix/MatrixReport.ts";
+export { reportMatrixResults } from "./matrix/MatrixReport.ts";
 export type {
   BenchmarkReport,
-  ReportColumnGroup,
+  ComparisonOptions,
+  ReportColumn,
   ReportGroup,
-  ResultsMapper,
+  ReportSection,
   UnknownRecord,
-} from "./BenchmarkReport.ts";
-export { reportResults } from "./BenchmarkReport.ts";
-export type { Configure, DefaultCliArgs } from "./cli/CliArgs.ts";
-export { defaultCliArgs, parseCliArgs } from "./cli/CliArgs.ts";
-export type { ExportOptions, MatrixExportOptions } from "./cli/RunBenchCLI.ts";
+} from "./report/BenchmarkReport.ts";
 export {
-  benchExports,
-  cliToMatrixOptions,
-  defaultMatrixReport,
-  defaultReport,
-  exportReports,
+  computeColumnValues,
   hasField,
-  matrixBenchExports,
-  matrixToReportGroups,
-  parseBenchArgs,
-  printHeapReports,
-  reportOptStatus,
-  runBenchmarks,
-  runDefaultBench,
-  runDefaultMatrixBench,
-  runMatrixSuite,
-} from "./cli/RunBenchCLI.ts";
-export * from "./export/JsonFormat.ts";
-export { exportPerfettoTrace } from "./export/PerfettoExport.ts";
+} from "./report/BenchmarkReport.ts";
 export {
-  exportAndLaunchSpeedscope,
-  exportSpeedscope,
-  heapProfileToSpeedscope,
-  launchSpeedscope,
-} from "./export/SpeedscopeExport.ts";
-export type { GitVersion } from "./GitUtils.ts";
+  formatBytes,
+  formatConvergence,
+  integer,
+  timeMs,
+  truncate,
+} from "./report/Formatters.ts";
+export {
+  gcSection,
+  gcSections,
+  gcStatsSection,
+} from "./report/GcSections.ts";
+export type { GitVersion } from "./report/GitUtils.ts";
 export {
-  formatDateWithTimezone,
   formatGitVersion,
   getBaselineVersion,
   getCurrentGitVersion,
-} from "./GitUtils.ts";
-export type { PrepareHtmlOptions } from "./HtmlDataPrep.ts";
-export { prepareHtmlData } from "./HtmlDataPrep.ts";
-export type { HtmlReportOptions, ReportData } from "./html/index.ts";
-export { generateHtmlReport } from "./html/index.ts";
-export type { MeasuredResults } from "./MeasuredResults.ts";
-export type { CasesModule } from "./matrix/CaseLoader.ts";
-export { loadCaseData, loadCasesModule } from "./matrix/CaseLoader.ts";
-export type { FilteredMatrix, MatrixFilter } from "./matrix/MatrixFilter.ts";
-export { filterMatrix, parseMatrixFilter } from "./matrix/MatrixFilter.ts";
-export type {
-  ExtraColumn,
-  MatrixReportOptions,
-} from "./matrix/MatrixReport.ts";
+} from "./report/GitUtils.ts";
+export type { PrepareHtmlOptions } from "./report/HtmlReport.ts";
+export { prepareHtmlData } from "./report/HtmlReport.ts";
 export {
-  gcPauseColumn,
-  gcStatsColumns,
-  heapTotalColumn,
-  reportMatrixResults,
-} from "./matrix/MatrixReport.ts";
-export type { RunnerOptions } from "./runners/BenchRunner.ts";
-export {
-  adaptiveSection,
+  adaptiveSections,
   buildGenericSections,
-  cpuSection,
-  gcSection,
-  gcStatsSection,
+  buildTimeSection,
   optSection,
   runsSection,
   timeSection,
   totalTimeSection,
-} from "./StandardSections.ts";
-export { average } from "./StatisticalUtils.ts";
-export { formatConvergence } from "./table-util/ConvergenceFormatters.ts";
+} from "./report/StandardSections.ts";
+export { reportResults } from "./report/text/TextReport.ts";
+export type {
+  BenchGroup,
+  BenchmarkSpec,
+  BenchSuite,
+} from "./runners/BenchmarkSpec.ts";
+export type { RunnerOptions } from "./runners/BenchRunner.ts";
+export type { MeasuredResults } from "./runners/MeasuredResults.ts";
 export {
-  formatBytes,
-  integer,
-  timeMs,
-  truncate,
-} from "./table-util/Formatters.ts";
+  average,
+  computeStat,
+  isBootstrappable,
+  maxBootstrapInput,
+  median,
+  percentile,
+  type StatKind,
+} from "./stats/StatisticalUtils.ts";
+export {
+  formatDateWithTimezone,
+  formatRelativeTime,
+} from "./viewer/DateFormat.ts";
+export type { ReportData } from "./viewer/ReportData.ts";

package/src/matrix/BenchMatrix.ts

@@ -0,0 +1,230 @@
+import type { RunnerOptions } from "../runners/BenchRunner.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { average } from "../stats/StatisticalUtils.ts";
+import type { CasesModule } from "./CaseLoader.ts";
+import { loadCasesModule } from "./CaseLoader.ts";
+import { runMatrixWithDir } from "./MatrixDirRunner.ts";
+import { runMatrixInline } from "./MatrixInlineRunner.ts";
+
+/** Stateless variant - called each iteration with case data */
+export type VariantFn<T = unknown> = (caseData: T) => void;
+
+/** Stateful variant - setup once, run many */
+export interface StatefulVariant<T = unknown, S = unknown> {
+  setup: (caseData: T) => S | Promise<S>;
+  run: (state: S) => void;
+}
+
+/** A variant is either a plain function or a stateful setup+run pair */
+export type Variant<T = unknown, S = unknown> =
+  | VariantFn<T>
+  | StatefulVariant<T, S>;
+
+/** Variant with any state type, allowing mixed variants in a matrix */
+export type AnyVariant<T = unknown> = VariantFn<T> | StatefulVariant<T, any>;
+
+/** Case data and optional metadata returned by a cases module */
+export interface LoadedCase<T = unknown> {
+  data: T;
+  metadata?: Record<string, unknown>;
+}
+
+/** Default runner settings applied to all matrix benchmarks */
+export interface MatrixDefaults {
+  warmup?: number;
+  maxTime?: number;
+  iterations?: number;
+}
+
+/** Configuration for a cases x variants benchmark matrix */
+export interface BenchMatrix<T = unknown> {
+  name: string;
+  variantDir?: string;
+  variants?: Record<string, AnyVariant<T>>;
+  cases?: string[];
+  casesModule?: string;
+  baselineDir?: string;
+  baselineVariant?: string;
+  defaults?: MatrixDefaults;
+}
+
+/** Named collection of benchmark matrices */
+export interface MatrixSuite {
+  name: string;
+  matrices: BenchMatrix<any>[];
+}
+
+/** Results for a single variant across all cases */
+export interface VariantResult {
+  id: string;
+  cases: CaseResult[];
+}
+
+/** Results for a single (variant, case) pair */
+export interface CaseResult {
+  caseId: string;
+  measured: MeasuredResults;
+  metadata?: Record<string, unknown>;
+  baseline?: MeasuredResults;
+  deltaPercent?: number;
+}
+
+/** Aggregated results from running a benchmark matrix */
+export interface MatrixResults {
+  name: string;
+  variants: VariantResult[];
+}
+
+/** Options for {@link runMatrix} */
+export interface RunMatrixOptions {
+  /** Maximum iterations per benchmark */
+  iterations?: number;
+  /** Maximum time in ms per benchmark */
+  maxTime?: number;
+  /** Number of warmup iterations before measurement */
+  warmup?: number;
+  /** Use worker process isolation (default: true for variantDir) */
+  useWorker?: boolean;
+  /** Number of interleaved batches for baseline comparison */
+  batches?: number;
+  /** Include first batch in results (normally dropped for OS cache warmup) */
+  warmupBatch?: boolean;
+  /** Run only these cases (from --filter) */
+  filteredCases?: string[];
+  /** Run only these variants (from --filter) */
+  filteredVariants?: string[];
+  /** Force garbage collection between iterations */
+  gcForce?: boolean;
+  /** Track V8 optimization/deoptimization events */
+  traceOpt?: boolean;
+  /** Pause duration in ms before warmup begins */
+  pauseWarmup?: number;
+  /** Pause duration in ms before first measurement */
+  pauseFirst?: number;
+  /** Pause every N iterations during measurement */
+  pauseInterval?: number;
+  /** Duration of each pause in ms */
+  pauseDuration?: number;
+  /** Collect GC statistics via --trace-gc-nvp */
+  gcStats?: boolean;
+  /** Enable heap allocation profiling */
+  alloc?: boolean;
+  /** Heap sampling interval in bytes */
+  allocInterval?: number;
+  /** Maximum stack depth for allocation traces */
+  allocDepth?: number;
+  /** Enable CPU time profiling */
+  profile?: boolean;
+  /** CPU profiling sample interval in microseconds */
+  profileInterval?: number;
+  /** Track function call counts via V8 coverage */
+  callCounts?: boolean;
+}
+
+/** Run a BenchMatrix with inline variants or variantDir */
+export async function runMatrix<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions = {},
+): Promise<MatrixResults> {
+  if (matrix.baselineDir && matrix.baselineVariant)
+    throw new Error(
+      "BenchMatrix cannot have both 'baselineDir' and 'baselineVariant'",
+    );
+  if (!matrix.variantDir && !matrix.variants)
+    throw new Error("BenchMatrix requires either 'variants' or 'variantDir'");
+
+  const effectiveOptions = { ...matrix.defaults, ...options };
+  const result = matrix.variantDir
+    ? await runMatrixWithDir(matrix, effectiveOptions)
+    : await runMatrixInline(matrix, effectiveOptions);
+
+  if (matrix.baselineVariant) {
+    applyBaselineVariant(result.variants, matrix.baselineVariant);
+  }
+  return result;
+}
+
+/** Prepare a benchmark function from a variant, calling setup if stateful. */
+export async function prepareBenchFn<T>(
+  variant: Variant<T>,
+  data: T,
+): Promise<() => void> {
+  if (isStatefulVariant(variant)) {
+    const state = await variant.setup(data);
+    return () => variant.run(state);
+  }
+  return () => variant(data);
+}
+
+/** Type guard for StatefulVariant */
+export function isStatefulVariant<T, S>(
+  v: Variant<T, S>,
+): v is StatefulVariant<T, S> {
+  return typeof v === "object" && "setup" in v && "run" in v;
+}
+
+/** Apply baselineVariant comparison - one variant is the reference for all others */
+export function applyBaselineVariant(
+  variants: VariantResult[],
+  baselineVariantId: string,
+): void {
+  const baselineVariant = variants.find(v => v.id === baselineVariantId);
+  if (!baselineVariant) return;
+
+  const baselineByCase = new Map(
+    baselineVariant.cases.map(c => [c.caseId, c.measured]),
+  );
+
+  for (const variant of variants) {
+    if (variant.id === baselineVariantId) continue;
+    for (const cr of variant.cases) {
+      const base = baselineByCase.get(cr.caseId);
+      if (base) {
+        cr.baseline = base;
+        cr.deltaPercent = computeDeltaPercent(base, cr.measured);
+      }
+    }
+  }
+}
+
+/** Load cases module and resolve filtered case IDs */
+export async function resolveCases<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<{ casesModule: CasesModule<T> | undefined; caseIds: string[] }> {
+  const casesModule = matrix.casesModule
+    ? await loadCasesModule<T>(matrix.casesModule)
+    : undefined;
+  const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
+  const caseIds = options.filteredCases ?? allCaseIds;
+  return { casesModule, caseIds };
+}
+
+/** Map matrix options to runner options, applying defaults for maxTime and warmup */
+export function buildRunnerOptions(opts: RunMatrixOptions): RunnerOptions {
+  const {
+    filteredCases,
+    filteredVariants,
+    useWorker,
+    batches,
+    warmupBatch,
+    ...base
+  } = opts;
+  const { iterations, maxTime, warmup, ...rest } = base;
+  return {
+    maxIterations: iterations,
+    maxTime: maxTime ?? (iterations ? undefined : 1000),
+    warmup: warmup ?? 0,
+    ...rest,
+  };
+}
+
+/** Compute percentage change of current vs baseline mean */
+export function computeDeltaPercent(
+  base: MeasuredResults,
+  cur: MeasuredResults,
+): number {
+  const avg = average(base.samples);
+  if (avg === 0) return 0;
+  return ((average(cur.samples) - avg) / avg) * 100;
+}

package/src/matrix/CaseLoader.ts

@@ -1,14 +1,16 @@
-import type { LoadedCase } from "../BenchMatrix.ts";
+import type { LoadedCase } from "./BenchMatrix.ts";
 
-/** Module that exports case definitions */
+/** Module exporting case IDs and an optional loader for case data */
 export interface CasesModule<T = unknown> {
   cases: string[];
-  defaultCases?: string[]; // subset for quick runs
-  defaultVariants?: string[]; // subset for quick runs
+  /** Subset of cases for quick runs */
+  defaultCases?: string[];
+  /** Subset of variants for quick runs */
+  defaultVariants?: string[];
   loadCase?: (id: string) => LoadedCase<T> | Promise<LoadedCase<T>>;
 }
 
-/** Load a cases module by URL */
+/** Import and validate a cases module, which must export a `cases` array */
 export async function loadCasesModule<T = unknown>(
   moduleUrl: string,
 ): Promise<CasesModule<T>> {
@@ -24,7 +26,7 @@ export async function loadCasesModule<T = unknown>(
   };
 }
 
-/** Load case data from a CasesModule or pass through the caseId */
+/** Load case data from a CasesModule, or use the caseId as data if no module */
 export async function loadCaseData<T>(
   casesModule: CasesModule<T> | undefined,
   caseId: string,

package/src/matrix/MatrixDirRunner.ts

@@ -0,0 +1,153 @@
+import type { RunnerOptions } from "../runners/BenchRunner.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { runBatched } from "../runners/MergeBatches.ts";
+import { runMatrixVariant } from "../runners/RunnerOrchestrator.ts";
+import type {
+  BenchMatrix,
+  CaseResult,
+  RunMatrixOptions,
+  VariantResult,
+} from "./BenchMatrix.ts";
+import {
+  buildRunnerOptions,
+  computeDeltaPercent,
+  resolveCases,
+} from "./BenchMatrix.ts";
+import type { CasesModule } from "./CaseLoader.ts";
+import { loadCaseData } from "./CaseLoader.ts";
+import { discoverVariants } from "./VariantLoader.ts";
+
+type VariantArgs = Parameters<typeof runMatrixVariant>[0];
+
+/** Shared state for directory-based matrix execution */
+interface DirMatrixContext<T> {
+  matrix: BenchMatrix<T>;
+  casesModule?: CasesModule<T>;
+  baselineIds: string[];
+  caseIds: string[];
+  runnerOpts: RunnerOptions;
+  batches: number;
+  warmupBatch: boolean;
+  useWorker: boolean;
+}
+
+/** Run matrix using variant files from a directory, each in a worker process */
+export async function runMatrixWithDir<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<{ name: string; variants: VariantResult[] }> {
+  const allVariantIds = await discoverVariants(matrix.variantDir!);
+  if (allVariantIds.length === 0) {
+    throw new Error(`No variants found in ${matrix.variantDir}`);
+  }
+  const variantIds = options.filteredVariants ?? allVariantIds;
+
+  const ctx = await createDirContext(matrix, options);
+  const variants = await runDirVariants(variantIds, ctx);
+  return { name: matrix.name, variants };
+}
+
+/** Create context for directory-based matrix execution */
+async function createDirContext<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<DirMatrixContext<T>> {
+  const baselineIds = matrix.baselineDir
+    ? await discoverVariants(matrix.baselineDir)
+    : [];
+  const { casesModule, caseIds } = await resolveCases(matrix, options);
+  const runnerOpts = buildRunnerOptions(options);
+  const { batches = 1, warmupBatch = false, useWorker = true } = options;
+  return {
+    matrix,
+    casesModule,
+    baselineIds,
+    caseIds,
+    runnerOpts,
+    batches,
+    warmupBatch,
+    useWorker,
+  };
+}
+
+/** Run all variants sequentially, collecting per-case results */
+async function runDirVariants<T>(
+  variantIds: string[],
+  ctx: DirMatrixContext<T>,
+): Promise<VariantResult[]> {
+  const variants: VariantResult[] = [];
+  for (const id of variantIds) {
+    const cases = await runDirVariantCases(id, ctx);
+    variants.push({ id, cases });
+  }
+  return variants;
+}
+
+/** Run all cases for a single variant */
+async function runDirVariantCases<T>(
+  variantId: string,
+  ctx: DirMatrixContext<T>,
+): Promise<CaseResult[]> {
+  const { matrix, casesModule, caseIds, runnerOpts, batches } = ctx;
+  const cases: CaseResult[] = [];
+
+  for (const caseId of caseIds) {
+    const caseData = matrix.cases && !matrix.casesModule ? caseId : undefined;
+    const variantArgs: VariantArgs = {
+      variantDir: matrix.variantDir!,
+      variantId,
+      caseId,
+      caseData,
+      casesModule: matrix.casesModule,
+      runner: "timing" as const,
+      options: runnerOpts,
+      useWorker: ctx.useWorker,
+    };
+    const baselineArgs =
+      matrix.baselineDir && ctx.baselineIds.includes(variantId)
+        ? { ...variantArgs, variantDir: matrix.baselineDir! }
+        : undefined;
+
+    const { metadata } = await loadCaseData(casesModule, caseId);
+    const { measured, baseline } =
+      batches > 1
+        ? await runCaseBatched(variantArgs, baselineArgs, ctx)
+        : await runCaseSingle(variantArgs, baselineArgs);
+    const deltaPercent = baseline
+      ? computeDeltaPercent(baseline, measured)
+      : undefined;
+    cases.push({ caseId, measured, metadata, baseline, deltaPercent });
+  }
+  return cases;
+}
+
+/** Run a batched measurement for a case, alternating current/baseline order. */
+async function runCaseBatched<T>(
+  variantArgs: VariantArgs,
+  baselineArgs: VariantArgs | undefined,
+  ctx: DirMatrixContext<T>,
+): Promise<{ measured: MeasuredResults; baseline?: MeasuredResults }> {
+  const runCurrent = async () => (await runMatrixVariant(variantArgs))[0];
+  const runBase = baselineArgs
+    ? async () => (await runMatrixVariant(baselineArgs))[0]
+    : undefined;
+  const { results, baseline } = await runBatched(
+    [runCurrent],
+    runBase,
+    ctx.batches,
+    ctx.warmupBatch,
+  );
+  return { measured: results[0], baseline };
+}
+
+/** Run a single unbatched measurement for a case. */
+async function runCaseSingle(
+  variantArgs: VariantArgs,
+  baselineArgs: VariantArgs | undefined,
+): Promise<{ measured: MeasuredResults; baseline?: MeasuredResults }> {
+  const [measured] = await runMatrixVariant(variantArgs);
+  const baseline = baselineArgs
+    ? (await runMatrixVariant(baselineArgs))[0]
+    : undefined;
+  return { measured, baseline };
+}