benchforge 0.1.9 → 0.2.4

package/src/export/TimeExport.ts

@@ -0,0 +1,115 @@
+/** CPU time profile conversion to Speedscope sampled format. */
+
+import { resolveCallFrame } from "../profiling/node/ResolvedProfile.ts";
+import type {
+  TimeProfile,
+  TimeProfileNode,
+} from "../profiling/node/TimeSampler.ts";
+import {
+  type FrameContext,
+  frameContext,
+  internFrame,
+  type SpeedscopeFile,
+  type SpeedscopeTimeProfile,
+  speedscopeFile,
+} from "./SpeedscopeTypes.ts";
+
+/** Convert a TimeProfile to speedscope format */
+export function timeProfileToSpeedscope(
+  name: string,
+  profile: TimeProfile,
+): SpeedscopeFile {
+  const ctx = frameContext();
+  const p = buildTimeProfile(name, profile, ctx);
+  return speedscopeFile(ctx, [p]);
+}
+
+/** Build a SpeedscopeFile from multiple named time profiles (shared frames). */
+export function buildTimeSpeedscopeFile(
+  entries: { name: string; profile: TimeProfile }[],
+): SpeedscopeFile | undefined {
+  if (entries.length === 0) return undefined;
+
+  const ctx = frameContext();
+  const profiles = entries.map(e => buildTimeProfile(e.name, e.profile, ctx));
+  return speedscopeFile(ctx, profiles);
+}
+
+/** Build a speedscope profile from a V8 TimeProfile */
+function buildTimeProfile(
+  name: string,
+  profile: TimeProfile,
+  ctx: FrameContext,
+): SpeedscopeTimeProfile {
+  const { samples: sampleIds, timeDeltas, nodes } = profile;
+
+  if (!sampleIds?.length || !timeDeltas) {
+    return {
+      type: "sampled",
+      name,
+      unit: "microseconds",
+      startValue: 0,
+      endValue: 0,
+      samples: [],
+      weights: [],
+    };
+  }
+
+  const nodeMap = new Map<number, TimeProfileNode>(nodes.map(n => [n.id, n]));
+  const parentMap = new Map<number, number>(); // childId -> parentId
+  for (const node of nodes) {
+    for (const childId of node.children ?? []) {
+      parentMap.set(childId, node.id);
+    }
+  }
+
+  const cache = new Map<number, number[]>();
+  const resolve = (id: number) =>
+    resolveStack(id, nodeMap, parentMap, cache, ctx);
+
+  const samples = sampleIds.map(resolve);
+  const total = timeDeltas.reduce((sum, w) => sum + w, 0);
+  return {
+    type: "sampled",
+    name,
+    unit: "microseconds",
+    startValue: 0,
+    endValue: total,
+    samples,
+    weights: timeDeltas,
+  };
+}
+
+/** Walk from node to root, building a stack of frame indices (root-first) */
+function resolveStack(
+  nodeId: number,
+  nodeMap: Map<number, TimeProfileNode>,
+  parentMap: Map<number, number>,
+  cache: Map<number, number[]>,
+  ctx: FrameContext,
+): number[] {
+  const cached = cache.get(nodeId);
+  if (cached) return cached;
+
+  const path: number[] = [];
+  let current: number | undefined = nodeId;
+  while (current !== undefined) {
+    path.push(current);
+    current = parentMap.get(current);
+  }
+
+  // Reverse to root-first order
+  const stack: number[] = [];
+  for (let i = path.length - 1; i >= 0; i--) {
+    const node = nodeMap.get(path[i]);
+    if (!node) continue;
+    const { functionName, url, lineNumber } = node.callFrame;
+    // Skip the synthetic (root) node
+    if (!functionName && !url && lineNumber <= 0) continue;
+    const frame = resolveCallFrame(node.callFrame);
+    stack.push(internFrame(frame.name, frame.url, frame.line, frame.col, ctx));
+  }
+
+  cache.set(nodeId, stack);
+  return stack;
+}

package/src/index.ts

@@ -1,3 +1,36 @@
+export type { Configure, DefaultCliArgs } from "./cli/CliArgs.ts";
+export { browserCliArgs, defaultCliArgs, parseCliArgs } from "./cli/CliArgs.ts";
+export type { ExportOptions, MatrixExportOptions } from "./cli/CliExport.ts";
+export { exportReports } from "./cli/CliExport.ts";
+export { cliToMatrixOptions } from "./cli/CliOptions.ts";
+export {
+  defaultMatrixReport,
+  defaultReport,
+  matrixToReportGroups,
+  printHeapReports,
+  reportOptStatus,
+} from "./cli/CliReport.ts";
+export {
+  benchExports,
+  dispatchCli,
+  matrixBenchExports,
+  parseBenchArgs,
+  runDefaultBench,
+  runDefaultMatrixBench,
+  runMatrixSuite,
+} from "./cli/RunBenchCLI.ts";
+export { runBenchmarks } from "./cli/SuiteRunner.ts";
+export {
+  buildSpeedscopeFile,
+  exportSpeedscope,
+  heapProfileToSpeedscope,
+} from "./export/AllocExport.ts";
+export { archiveBenchmark } from "./export/ArchiveExport.ts";
+export type {
+  ArchiveMetadata,
+  BenchforgeArchive,
+} from "./export/ArchiveFormat.ts";
+export { exportPerfettoTrace } from "./export/PerfettoExport.ts";
 export type {
   AnyVariant,
   BenchMatrix,
@@ -11,82 +44,74 @@ export type {
   Variant,
   VariantFn,
   VariantResult,
-} from "./BenchMatrix.ts";
-export { isStatefulVariant, runMatrix } from "./BenchMatrix.ts";
-export type { BenchGroup, BenchmarkSpec, BenchSuite } from "./Benchmark.ts";
+} from "./matrix/BenchMatrix.ts";
+export { isStatefulVariant, runMatrix } from "./matrix/BenchMatrix.ts";
+export type { CasesModule } from "./matrix/CaseLoader.ts";
+export { loadCaseData, loadCasesModule } from "./matrix/CaseLoader.ts";
+export type { FilteredMatrix, MatrixFilter } from "./matrix/MatrixFilter.ts";
+export { filterMatrix, parseMatrixFilter } from "./matrix/MatrixFilter.ts";
+export type { MatrixReportOptions } from "./matrix/MatrixReport.ts";
+export { reportMatrixResults } from "./matrix/MatrixReport.ts";
 export type {
   BenchmarkReport,
-  ReportColumnGroup,
+  ComparisonOptions,
+  ReportColumn,
   ReportGroup,
-  ResultsMapper,
+  ReportSection,
   UnknownRecord,
-} from "./BenchmarkReport.ts";
-export { reportResults } from "./BenchmarkReport.ts";
-export type { Configure, DefaultCliArgs } from "./cli/CliArgs.ts";
-export { defaultCliArgs, parseCliArgs } from "./cli/CliArgs.ts";
-export type { ExportOptions, MatrixExportOptions } from "./cli/RunBenchCLI.ts";
+} from "./report/BenchmarkReport.ts";
 export {
-  benchExports,
-  cliToMatrixOptions,
-  defaultMatrixReport,
-  defaultReport,
-  exportReports,
+  computeColumnValues,
   hasField,
-  matrixBenchExports,
-  matrixToReportGroups,
-  parseBenchArgs,
-  printHeapReports,
-  reportOptStatus,
-  runBenchmarks,
-  runDefaultBench,
-  runDefaultMatrixBench,
-  runMatrixSuite,
-} from "./cli/RunBenchCLI.ts";
-export * from "./export/JsonFormat.ts";
-export { exportPerfettoTrace } from "./export/PerfettoExport.ts";
-export type { GitVersion } from "./GitUtils.ts";
+} from "./report/BenchmarkReport.ts";
+export {
+  formatBytes,
+  formatConvergence,
+  integer,
+  timeMs,
+  truncate,
+} from "./report/Formatters.ts";
+export {
+  gcSection,
+  gcSections,
+  gcStatsSection,
+} from "./report/GcSections.ts";
+export type { GitVersion } from "./report/GitUtils.ts";
 export {
-  formatDateWithTimezone,
   formatGitVersion,
   getBaselineVersion,
   getCurrentGitVersion,
-} from "./GitUtils.ts";
-export type { PrepareHtmlOptions } from "./HtmlDataPrep.ts";
-export { prepareHtmlData } from "./HtmlDataPrep.ts";
-export type { HtmlReportOptions, ReportData } from "./html/index.ts";
-export { generateHtmlReport } from "./html/index.ts";
-export type { MeasuredResults } from "./MeasuredResults.ts";
-export type { CasesModule } from "./matrix/CaseLoader.ts";
-export { loadCaseData, loadCasesModule } from "./matrix/CaseLoader.ts";
-export type { FilteredMatrix, MatrixFilter } from "./matrix/MatrixFilter.ts";
-export { filterMatrix, parseMatrixFilter } from "./matrix/MatrixFilter.ts";
-export type {
-  ExtraColumn,
-  MatrixReportOptions,
-} from "./matrix/MatrixReport.ts";
-export {
-  gcPauseColumn,
-  gcStatsColumns,
-  heapTotalColumn,
-  reportMatrixResults,
-} from "./matrix/MatrixReport.ts";
-export type { RunnerOptions } from "./runners/BenchRunner.ts";
+} from "./report/GitUtils.ts";
+export type { PrepareHtmlOptions } from "./report/HtmlReport.ts";
+export { prepareHtmlData } from "./report/HtmlReport.ts";
 export {
-  adaptiveSection,
+  adaptiveSections,
   buildGenericSections,
-  cpuSection,
-  gcSection,
-  gcStatsSection,
+  buildTimeSection,
   optSection,
   runsSection,
   timeSection,
   totalTimeSection,
-} from "./StandardSections.ts";
-export { average } from "./StatisticalUtils.ts";
-export { formatConvergence } from "./table-util/ConvergenceFormatters.ts";
+} from "./report/StandardSections.ts";
+export { reportResults } from "./report/text/TextReport.ts";
+export type {
+  BenchGroup,
+  BenchmarkSpec,
+  BenchSuite,
+} from "./runners/BenchmarkSpec.ts";
+export type { RunnerOptions } from "./runners/BenchRunner.ts";
+export type { MeasuredResults } from "./runners/MeasuredResults.ts";
 export {
-  formatBytes,
-  integer,
-  timeMs,
-  truncate,
-} from "./table-util/Formatters.ts";
+  average,
+  computeStat,
+  isBootstrappable,
+  maxBootstrapInput,
+  median,
+  percentile,
+  type StatKind,
+} from "./stats/StatisticalUtils.ts";
+export {
+  formatDateWithTimezone,
+  formatRelativeTime,
+} from "./viewer/DateFormat.ts";
+export type { ReportData } from "./viewer/ReportData.ts";

package/src/matrix/BenchMatrix.ts

@@ -0,0 +1,230 @@
+import type { RunnerOptions } from "../runners/BenchRunner.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { average } from "../stats/StatisticalUtils.ts";
+import type { CasesModule } from "./CaseLoader.ts";
+import { loadCasesModule } from "./CaseLoader.ts";
+import { runMatrixWithDir } from "./MatrixDirRunner.ts";
+import { runMatrixInline } from "./MatrixInlineRunner.ts";
+
+/** Stateless variant - called each iteration with case data */
+export type VariantFn<T = unknown> = (caseData: T) => void;
+
+/** Stateful variant - setup once, run many */
+export interface StatefulVariant<T = unknown, S = unknown> {
+  setup: (caseData: T) => S | Promise<S>;
+  run: (state: S) => void;
+}
+
+/** A variant is either a plain function or a stateful setup+run pair */
+export type Variant<T = unknown, S = unknown> =
+  | VariantFn<T>
+  | StatefulVariant<T, S>;
+
+/** Variant with any state type, allowing mixed variants in a matrix */
+export type AnyVariant<T = unknown> = VariantFn<T> | StatefulVariant<T, any>;
+
+/** Case data and optional metadata returned by a cases module */
+export interface LoadedCase<T = unknown> {
+  data: T;
+  metadata?: Record<string, unknown>;
+}
+
+/** Default runner settings applied to all matrix benchmarks */
+export interface MatrixDefaults {
+  warmup?: number;
+  maxTime?: number;
+  iterations?: number;
+}
+
+/** Configuration for a cases x variants benchmark matrix */
+export interface BenchMatrix<T = unknown> {
+  name: string;
+  variantDir?: string;
+  variants?: Record<string, AnyVariant<T>>;
+  cases?: string[];
+  casesModule?: string;
+  baselineDir?: string;
+  baselineVariant?: string;
+  defaults?: MatrixDefaults;
+}
+
+/** Named collection of benchmark matrices */
+export interface MatrixSuite {
+  name: string;
+  matrices: BenchMatrix<any>[];
+}
+
+/** Results for a single variant across all cases */
+export interface VariantResult {
+  id: string;
+  cases: CaseResult[];
+}
+
+/** Results for a single (variant, case) pair */
+export interface CaseResult {
+  caseId: string;
+  measured: MeasuredResults;
+  metadata?: Record<string, unknown>;
+  baseline?: MeasuredResults;
+  deltaPercent?: number;
+}
+
+/** Aggregated results from running a benchmark matrix */
+export interface MatrixResults {
+  name: string;
+  variants: VariantResult[];
+}
+
+/** Options for {@link runMatrix} */
+export interface RunMatrixOptions {
+  /** Maximum iterations per benchmark */
+  iterations?: number;
+  /** Maximum time in ms per benchmark */
+  maxTime?: number;
+  /** Number of warmup iterations before measurement */
+  warmup?: number;
+  /** Use worker process isolation (default: true for variantDir) */
+  useWorker?: boolean;
+  /** Number of interleaved batches for baseline comparison */
+  batches?: number;
+  /** Include first batch in results (normally dropped for OS cache warmup) */
+  warmupBatch?: boolean;
+  /** Run only these cases (from --filter) */
+  filteredCases?: string[];
+  /** Run only these variants (from --filter) */
+  filteredVariants?: string[];
+  /** Force garbage collection between iterations */
+  gcForce?: boolean;
+  /** Track V8 optimization/deoptimization events */
+  traceOpt?: boolean;
+  /** Pause duration in ms before warmup begins */
+  pauseWarmup?: number;
+  /** Pause duration in ms before first measurement */
+  pauseFirst?: number;
+  /** Pause every N iterations during measurement */
+  pauseInterval?: number;
+  /** Duration of each pause in ms */
+  pauseDuration?: number;
+  /** Collect GC statistics via --trace-gc-nvp */
+  gcStats?: boolean;
+  /** Enable heap allocation profiling */
+  alloc?: boolean;
+  /** Heap sampling interval in bytes */
+  allocInterval?: number;
+  /** Maximum stack depth for allocation traces */
+  allocDepth?: number;
+  /** Enable CPU time profiling */
+  profile?: boolean;
+  /** CPU profiling sample interval in microseconds */
+  profileInterval?: number;
+  /** Track function call counts via V8 coverage */
+  callCounts?: boolean;
+}
+
+/** Run a BenchMatrix with inline variants or variantDir */
+export async function runMatrix<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions = {},
+): Promise<MatrixResults> {
+  if (matrix.baselineDir && matrix.baselineVariant)
+    throw new Error(
+      "BenchMatrix cannot have both 'baselineDir' and 'baselineVariant'",
+    );
+  if (!matrix.variantDir && !matrix.variants)
+    throw new Error("BenchMatrix requires either 'variants' or 'variantDir'");
+
+  const effectiveOptions = { ...matrix.defaults, ...options };
+  const result = matrix.variantDir
+    ? await runMatrixWithDir(matrix, effectiveOptions)
+    : await runMatrixInline(matrix, effectiveOptions);
+
+  if (matrix.baselineVariant) {
+    applyBaselineVariant(result.variants, matrix.baselineVariant);
+  }
+  return result;
+}
+
+/** Prepare a benchmark function from a variant, calling setup if stateful. */
+export async function prepareBenchFn<T>(
+  variant: Variant<T>,
+  data: T,
+): Promise<() => void> {
+  if (isStatefulVariant(variant)) {
+    const state = await variant.setup(data);
+    return () => variant.run(state);
+  }
+  return () => variant(data);
+}
+
+/** Type guard for StatefulVariant */
+export function isStatefulVariant<T, S>(
+  v: Variant<T, S>,
+): v is StatefulVariant<T, S> {
+  return typeof v === "object" && "setup" in v && "run" in v;
+}
+
+/** Apply baselineVariant comparison - one variant is the reference for all others */
+export function applyBaselineVariant(
+  variants: VariantResult[],
+  baselineVariantId: string,
+): void {
+  const baselineVariant = variants.find(v => v.id === baselineVariantId);
+  if (!baselineVariant) return;
+
+  const baselineByCase = new Map(
+    baselineVariant.cases.map(c => [c.caseId, c.measured]),
+  );
+
+  for (const variant of variants) {
+    if (variant.id === baselineVariantId) continue;
+    for (const cr of variant.cases) {
+      const base = baselineByCase.get(cr.caseId);
+      if (base) {
+        cr.baseline = base;
+        cr.deltaPercent = computeDeltaPercent(base, cr.measured);
+      }
+    }
+  }
+}
+
+/** Load cases module and resolve filtered case IDs */
+export async function resolveCases<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<{ casesModule: CasesModule<T> | undefined; caseIds: string[] }> {
+  const casesModule = matrix.casesModule
+    ? await loadCasesModule<T>(matrix.casesModule)
+    : undefined;
+  const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
+  const caseIds = options.filteredCases ?? allCaseIds;
+  return { casesModule, caseIds };
+}
+
+/** Map matrix options to runner options, applying defaults for maxTime and warmup */
+export function buildRunnerOptions(opts: RunMatrixOptions): RunnerOptions {
+  const {
+    filteredCases,
+    filteredVariants,
+    useWorker,
+    batches,
+    warmupBatch,
+    ...base
+  } = opts;
+  const { iterations, maxTime, warmup, ...rest } = base;
+  return {
+    maxIterations: iterations,
+    maxTime: maxTime ?? (iterations ? undefined : 1000),
+    warmup: warmup ?? 0,
+    ...rest,
+  };
+}
+
+/** Compute percentage change of current vs baseline mean */
+export function computeDeltaPercent(
+  base: MeasuredResults,
+  cur: MeasuredResults,
+): number {
+  const avg = average(base.samples);
+  if (avg === 0) return 0;
+  return ((average(cur.samples) - avg) / avg) * 100;
+}

package/src/matrix/CaseLoader.ts

@@ -1,14 +1,16 @@
-import type { LoadedCase } from "../BenchMatrix.ts";
+import type { LoadedCase } from "./BenchMatrix.ts";
 
-/** Module that exports case definitions */
+/** Module exporting case IDs and an optional loader for case data */
 export interface CasesModule<T = unknown> {
   cases: string[];
-  defaultCases?: string[]; // subset for quick runs
-  defaultVariants?: string[]; // subset for quick runs
+  /** Subset of cases for quick runs */
+  defaultCases?: string[];
+  /** Subset of variants for quick runs */
+  defaultVariants?: string[];
   loadCase?: (id: string) => LoadedCase<T> | Promise<LoadedCase<T>>;
 }
 
-/** Load a cases module by URL */
+/** Import and validate a cases module, which must export a `cases` array */
 export async function loadCasesModule<T = unknown>(
   moduleUrl: string,
 ): Promise<CasesModule<T>> {
@@ -24,7 +26,7 @@ export async function loadCasesModule<T = unknown>(
   };
 }
 
-/** Load case data from a CasesModule or pass through the caseId */
+/** Load case data from a CasesModule, or use the caseId as data if no module */
 export async function loadCaseData<T>(
   casesModule: CasesModule<T> | undefined,
   caseId: string,