benchforge 0.1.9 → 0.2.4

package/src/BenchMatrix.ts

@@ -1,380 +0,0 @@
-import type { MeasuredResults } from "./MeasuredResults.ts";
-import { loadCaseData, loadCasesModule } from "./matrix/CaseLoader.ts";
-import { discoverVariants } from "./matrix/VariantLoader.ts";
-import { BasicRunner } from "./runners/BasicRunner.ts";
-import type { RunnerOptions } from "./runners/BenchRunner.ts";
-import { runMatrixVariant } from "./runners/RunnerOrchestrator.ts";
-import { average } from "./StatisticalUtils.ts";
-
-/** Stateless variant - called each iteration with case data */
-export type VariantFn<T = unknown> = (caseData: T) => void;
-
-/** Stateful variant - setup once, run many */
-export interface StatefulVariant<T = unknown, S = unknown> {
-  setup: (caseData: T) => S | Promise<S>;
-  run: (state: S) => void;
-}
-
-/** A variant is either a plain function or a stateful setup+run pair */
-export type Variant<T = unknown, S = unknown> =
-  | VariantFn<T>
-  | StatefulVariant<T, S>;
-
-/** Variant with any state type - used in BenchMatrix to allow mixed variants */
-export type AnyVariant<T = unknown> = VariantFn<T> | StatefulVariant<T, any>;
-
-/** Result from casesModule.loadCase() */
-export interface LoadedCase<T = unknown> {
-  data: T;
-  metadata?: Record<string, unknown>;
-}
-
-export interface MatrixDefaults {
-  warmup?: number;
-  maxTime?: number;
-  iterations?: number;
-}
-
-/** Bench matrix configuration */
-export interface BenchMatrix<T = unknown> {
-  name: string;
-  variantDir?: string;
-  variants?: Record<string, AnyVariant<T>>;
-  cases?: string[];
-  casesModule?: string;
-  baselineDir?: string;
-  baselineVariant?: string;
-  defaults?: MatrixDefaults;
-}
-
-/** Collection of matrices */
-export interface MatrixSuite {
-  name: string;
-  matrices: BenchMatrix<any>[];
-}
-
-/** Results for a single variant across all cases */
-export interface VariantResult {
-  id: string;
-  cases: CaseResult[];
-}
-
-/** Results for a single (variant, case) pair */
-export interface CaseResult {
-  caseId: string;
-  measured: MeasuredResults;
-  metadata?: Record<string, unknown>;
-  baseline?: MeasuredResults;
-  deltaPercent?: number;
-}
-
-/** Results from running a matrix */
-export interface MatrixResults {
-  name: string;
-  variants: VariantResult[];
-}
-
-/** @return true if variant is a StatefulVariant (has setup + run) */
-export function isStatefulVariant<T, S>(
-  v: Variant<T, S>,
-): v is StatefulVariant<T, S> {
-  return typeof v === "object" && "setup" in v && "run" in v;
-}
-
-/** Options for runMatrix */
-export interface RunMatrixOptions {
-  iterations?: number;
-  maxTime?: number;
-  warmup?: number;
-  useWorker?: boolean; // use worker process isolation (default: true for variantDir)
-  filteredCases?: string[]; // run only these cases (from filter)
-  filteredVariants?: string[]; // run only these variants (from filter)
-  // Runner options passthrough
-  collect?: boolean;
-  cpuCounters?: boolean;
-  traceOpt?: boolean;
-  noSettle?: boolean;
-  pauseFirst?: number;
-  pauseInterval?: number;
-  pauseDuration?: number;
-  gcStats?: boolean;
-  heapSample?: boolean;
-  heapInterval?: number;
-  heapDepth?: number;
-}
-
-/** Run a BenchMatrix with inline variants or variantDir */
-export async function runMatrix<T>(
-  matrix: BenchMatrix<T>,
-  options: RunMatrixOptions = {},
-): Promise<MatrixResults> {
-  validateBaseline(matrix);
-  const effectiveOptions = { ...matrix.defaults, ...options };
-
-  if (matrix.variantDir) {
-    return runMatrixWithDir(matrix, effectiveOptions);
-  }
-  if (matrix.variants) {
-    return runMatrixInline(matrix, effectiveOptions);
-  }
-  throw new Error("BenchMatrix requires either 'variants' or 'variantDir'");
-}
-
-/** @throws if both baselineDir and baselineVariant are set */
-function validateBaseline<T>(matrix: BenchMatrix<T>): void {
-  const msg =
-    "BenchMatrix cannot have both 'baselineDir' and 'baselineVariant'";
-  if (matrix.baselineDir && matrix.baselineVariant) throw new Error(msg);
-}
-
-function buildRunnerOptions(options: RunMatrixOptions): RunnerOptions {
-  return {
-    maxIterations: options.iterations,
-    maxTime: options.maxTime ?? 1000,
-    warmup: options.warmup ?? 0,
-    collect: options.collect,
-    cpuCounters: options.cpuCounters,
-    traceOpt: options.traceOpt,
-    noSettle: options.noSettle,
-    pauseFirst: options.pauseFirst,
-    pauseInterval: options.pauseInterval,
-    pauseDuration: options.pauseDuration,
-    gcStats: options.gcStats,
-    heapSample: options.heapSample,
-    heapInterval: options.heapInterval,
-    heapDepth: options.heapDepth,
-  };
-}
-
-/** Load cases module and resolve filtered case IDs */
-async function resolveCases<T>(
-  matrix: BenchMatrix<T>,
-  options: RunMatrixOptions,
-) {
-  const casesModule = matrix.casesModule
-    ? await loadCasesModule<T>(matrix.casesModule)
-    : undefined;
-  const allCaseIds = casesModule?.cases ?? matrix.cases ?? ["default"];
-  const caseIds = options.filteredCases ?? allCaseIds;
-  return { casesModule, caseIds };
-}
-
-/** Run matrix with inline variants (non-worker mode) */
-async function runMatrixInline<T>(
-  matrix: BenchMatrix<T>,
-  options: RunMatrixOptions,
-): Promise<MatrixResults> {
-  // baselineDir is only valid with variantDir
-  const msg =
-    "BenchMatrix with inline 'variants' cannot use 'baselineDir'. Use 'variantDir' instead.";
-  if (matrix.baselineDir) throw new Error(msg);
-
-  const { casesModule, caseIds } = await resolveCases(matrix, options);
-  const runner = new BasicRunner();
-  const runnerOpts = buildRunnerOptions(options);
-
-  const variantEntries = options.filteredVariants
-    ? Object.entries(matrix.variants!).filter(([id]) =>
-        options.filteredVariants!.includes(id),
-      )
-    : Object.entries(matrix.variants!);
-
-  const variants: VariantResult[] = [];
-  for (const [variantId, variant] of variantEntries) {
-    const cases: CaseResult[] = [];
-    for (const caseId of caseIds) {
-      const loaded = await loadCaseData(casesModule, caseId);
-      const caseData =
-        casesModule || matrix.cases ? loaded.data : (undefined as T);
-      const measured = await runVariant(
-        variant,
-        caseData,
-        variantId,
-        runner,
-        runnerOpts,
-      );
-      cases.push({ caseId, measured, metadata: loaded.metadata });
-    }
-    variants.push({ id: variantId, cases });
-  }
-
-  if (matrix.baselineVariant) {
-    applyBaselineVariant(variants, matrix.baselineVariant);
-  }
-
-  return { name: matrix.name, variants };
-}
-
-/** Context for running matrix benchmarks in worker mode */
-interface DirMatrixContext<T> {
-  matrix: BenchMatrix<T>;
-  casesModule?: import("./matrix/CaseLoader.ts").CasesModule<T>;
-  baselineIds: string[];
-  caseIds: string[];
-  runnerOpts: RunnerOptions;
-}
-
-/** Run matrix with variantDir (worker mode for memory isolation) */
-async function runMatrixWithDir<T>(
-  matrix: BenchMatrix<T>,
-  options: RunMatrixOptions,
-): Promise<MatrixResults> {
-  const allVariantIds = await discoverVariants(matrix.variantDir!);
-  if (allVariantIds.length === 0) {
-    throw new Error(`No variants found in ${matrix.variantDir}`);
-  }
-  const variantIds = options.filteredVariants ?? allVariantIds;
-
-  const ctx = await createDirContext(matrix, options);
-  const variants = await runDirVariants(variantIds, ctx);
-
-  if (matrix.baselineVariant) {
-    applyBaselineVariant(variants, matrix.baselineVariant);
-  }
-  return { name: matrix.name, variants };
-}
-
-/** Create context for directory-based matrix execution */
-async function createDirContext<T>(
-  matrix: BenchMatrix<T>,
-  options: RunMatrixOptions,
-): Promise<DirMatrixContext<T>> {
-  const baselineIds = matrix.baselineDir
-    ? await discoverVariants(matrix.baselineDir)
-    : [];
-  const { casesModule, caseIds } = await resolveCases(matrix, options);
-  const runnerOpts = buildRunnerOptions(options);
-  return { matrix, casesModule, baselineIds, caseIds, runnerOpts };
-}
-
-/** Run all variants using worker processes */
-async function runDirVariants<T>(
-  variantIds: string[],
-  ctx: DirMatrixContext<T>,
-): Promise<VariantResult[]> {
-  const variants: VariantResult[] = [];
-  for (const variantId of variantIds) {
-    const cases = await runDirVariantCases(variantId, ctx);
-    variants.push({ id: variantId, cases });
-  }
-  return variants;
-}
-
-/** Run all cases for a single variant */
-async function runDirVariantCases<T>(
-  variantId: string,
-  ctx: DirMatrixContext<T>,
-): Promise<CaseResult[]> {
-  const { matrix, casesModule, caseIds, runnerOpts } = ctx;
-  const cases: CaseResult[] = [];
-
-  for (const caseId of caseIds) {
-    const caseData = !matrix.casesModule && matrix.cases ? caseId : undefined;
-    const [measured] = await runMatrixVariant({
-      variantDir: matrix.variantDir!,
-      variantId,
-      caseId,
-      caseData,
-      casesModule: matrix.casesModule,
-      runner: "basic",
-      options: runnerOpts,
-    });
-
-    const loaded = await loadCaseData(casesModule, caseId);
-    const baseline = await runBaselineIfExists(
-      variantId,
-      caseId,
-      caseData,
-      ctx,
-    );
-    const deltaPercent = baseline
-      ? computeDeltaPercent(baseline, measured)
-      : undefined;
-    const metadata = loaded.metadata;
-    cases.push({ caseId, measured, metadata, baseline, deltaPercent });
-  }
-  return cases;
-}
-
-/** Run baseline variant if it exists in baselineDir */
-async function runBaselineIfExists<T>(
-  variantId: string,
-  caseId: string,
-  caseData: unknown,
-  ctx: DirMatrixContext<T>,
-): Promise<MeasuredResults | undefined> {
-  const { matrix, baselineIds, runnerOpts } = ctx;
-  if (!matrix.baselineDir || !baselineIds.includes(variantId)) return undefined;
-
-  const [measured] = await runMatrixVariant({
-    variantDir: matrix.baselineDir,
-    variantId,
-    caseId,
-    caseData,
-    casesModule: matrix.casesModule,
-    runner: "basic",
-    options: runnerOpts,
-  });
-  return measured;
-}
-
-/** Compute delta percentage: (current - baseline) / baseline * 100 */
-function computeDeltaPercent(
-  baseline: MeasuredResults,
-  current: MeasuredResults,
-): number {
-  const baseAvg = average(baseline.samples);
-  if (baseAvg === 0) return 0;
-  return ((average(current.samples) - baseAvg) / baseAvg) * 100;
-}
-
-/** Apply baselineVariant comparison - one variant is the reference for all others */
-function applyBaselineVariant(
-  variants: VariantResult[],
-  baselineVariantId: string,
-): void {
-  const baselineVariant = variants.find(v => v.id === baselineVariantId);
-  if (!baselineVariant) return;
-
-  const baselineByCase = new Map<string, MeasuredResults>();
-  for (const c of baselineVariant.cases) {
-    baselineByCase.set(c.caseId, c.measured);
-  }
-
-  for (const variant of variants) {
-    if (variant.id === baselineVariantId) continue;
-    for (const caseResult of variant.cases) {
-      const baseline = baselineByCase.get(caseResult.caseId);
-      if (baseline) {
-        caseResult.baseline = baseline;
-        caseResult.deltaPercent = computeDeltaPercent(
-          baseline,
-          caseResult.measured,
-        );
-      }
-    }
-  }
-}
-
-/** Run a single variant with case data */
-async function runVariant<T>(
-  variant: AnyVariant<T>,
-  caseData: T,
-  name: string,
-  runner: BasicRunner,
-  options: RunnerOptions,
-): Promise<MeasuredResults> {
-  if (isStatefulVariant(variant)) {
-    const state = await variant.setup(caseData);
-    const [result] = await runner.runBench(
-      { name, fn: () => variant.run(state) },
-      options,
-    );
-    return result;
-  }
-  const [result] = await runner.runBench(
-    { name, fn: () => variant(caseData) },
-    options,
-  );
-  return result;
-}

package/src/BenchmarkReport.ts

@@ -1,156 +0,0 @@
-import type { MeasuredResults } from "./MeasuredResults.ts";
-import { bootstrapDifferenceCI } from "./StatisticalUtils.ts";
-import type { UnionToIntersection } from "./TypeUtil.ts";
-import {
-  formatDiffWithCI,
-  formatDiffWithCIHigherIsBetter,
-  truncate,
-} from "./table-util/Formatters.ts";
-import {
-  type AnyColumn,
-  buildTable,
-  type ColumnGroup,
-  type ResultGroup,
-} from "./table-util/TableReport.ts";
-
-/** Benchmark results with optional baseline for comparison */
-export interface ReportGroup {
-  name: string;
-  reports: BenchmarkReport[];
-  baseline?: BenchmarkReport;
-}
-
-/** Results from a single benchmark run */
-export interface BenchmarkReport {
-  name: string;
-  measuredResults: MeasuredResults;
-  metadata?: UnknownRecord;
-}
-
-export interface ReportColumnGroup<T> {
-  groupTitle?: string;
-  columns: ReportColumn<T>[];
-}
-
-export type ReportColumn<T> = AnyColumn<T> & {
-  /** Add diff column after this column when baseline exists */
-  comparable?: boolean;
-  /** Set true for throughput metrics where higher values are better (e.g., lines/sec) */
-  higherIsBetter?: boolean;
-};
-
-/** Maps benchmark results to table columns */
-export interface ResultsMapper<
-  T extends Record<string, any> = Record<string, any>,
-> {
-  extract(results: MeasuredResults, metadata?: UnknownRecord): T;
-  columns(): ReportColumnGroup<T>[];
-}
-export type UnknownRecord = Record<string, unknown>;
-
-type SectionStats<S> = S extends ResultsMapper<infer T> ? T : never;
-
-interface ReportRowBase {
-  name: string;
-}
-
-/** Row data combining all section statistics */
-type ReportRowData<S extends ReadonlyArray<ResultsMapper<any>>> =
-  ReportRowBase & UnionToIntersection<SectionStats<S[number]>>;
-
-/** @return formatted table report with optional baseline comparisons */
-export function reportResults<S extends ReadonlyArray<ResultsMapper<any>>>(
-  groups: ReportGroup[],
-  sections: S,
-): string {
-  const results = groups.map(group => resultGroupValues(group, sections));
-  const hasBaseline = results.some(g => g.baseline);
-  return buildTable(createColumnGroups(sections, hasBaseline), results);
-}
-
-/** @return values for report group */
-function resultGroupValues<S extends ReadonlyArray<ResultsMapper<any>>>(
-  group: ReportGroup,
-  sections: S,
-): ResultGroup<ReportRowData<S>> {
-  const { reports, baseline } = group;
-  const baselineSamples = baseline?.measuredResults.samples;
-
-  const results = reports.map(report => {
-    const row = {
-      name: truncate(report.name),
-      ...extractReportValues(report, sections),
-    } as ReportRowData<S>;
-
-    if (baselineSamples && report.measuredResults.samples) {
-      (row as any).diffCI = bootstrapDifferenceCI(
-        baselineSamples,
-        report.measuredResults.samples,
-      );
-    }
-    return row;
-  });
-
-  const baselineRow = baseline && valuesForReports([baseline], sections)[0];
-  return { results, baseline: baselineRow };
-}
-
-/** @return rows with stats from sections */
-export function valuesForReports<S extends ReadonlyArray<ResultsMapper<any>>>(
-  reports: BenchmarkReport[],
-  sections: S,
-): ReportRowData<S>[] {
-  return reports.map(report => ({
-    name: truncate(report.name),
-    ...extractReportValues(report, sections),
-  })) as ReportRowData<S>[];
-}
-
-/** @return merged statistics from all sections */
-function extractReportValues(
-  report: BenchmarkReport,
-  sections: ReadonlyArray<ResultsMapper<any>>,
-): UnknownRecord {
-  const { measuredResults, metadata } = report;
-  const entries = sections.flatMap(s =>
-    Object.entries(s.extract(measuredResults, metadata)),
-  );
-  return Object.fromEntries(entries);
-}
-
-/** @return column groups with diff columns if baseline exists */
-function createColumnGroups<S extends ReadonlyArray<ResultsMapper<any>>>(
-  sections: S,
-  hasBaseline: boolean,
-): ColumnGroup<ReportRowData<S>>[] {
-  const nameColumn: ColumnGroup<ReportRowData<S>> = {
-    columns: [{ key: "name" as keyof ReportRowData<S>, title: "name" }],
-  };
-
-  const groups = sections.flatMap(section => section.columns());
-  return [nameColumn, ...(hasBaseline ? injectDiffColumns(groups) : groups)];
-}
-
-/** @return groups with single CI column after first comparable field */
-export function injectDiffColumns<T>(
-  reportGroups: ReportColumnGroup<T>[],
-): ColumnGroup<T>[] {
-  let ciAdded = false;
-
-  return reportGroups.map(group => ({
-    groupTitle: group.groupTitle,
-    columns: group.columns.flatMap(col => {
-      if (col.comparable && !ciAdded) {
-        ciAdded = true;
-        const fmt = col.higherIsBetter
-          ? formatDiffWithCIHigherIsBetter
-          : formatDiffWithCI;
-        return [
-          col,
-          { title: "Δ% CI", key: "diffCI" as keyof T, formatter: fmt },
-        ];
-      }
-      return [col];
-    }),
-  }));
-}

package/src/HtmlDataPrep.ts

@@ -1,148 +0,0 @@
-import type {
-  ReportColumnGroup,
-  ReportGroup,
-  ResultsMapper,
-} from "./BenchmarkReport.ts";
-import type { GitVersion } from "./GitUtils.ts";
-import type {
-  BenchmarkData,
-  DifferenceCI,
-  FormattedStat,
-  GroupData,
-  ReportData,
-} from "./html/index.ts";
-import { bootstrapDifferenceCI } from "./StatisticalUtils.ts";
-
-export interface PrepareHtmlOptions {
-  cliArgs?: Record<string, unknown>;
-  sections?: ResultsMapper[];
-  currentVersion?: GitVersion;
-  baselineVersion?: GitVersion;
-}
-
-/** Find higherIsBetter from first comparable column in sections */
-function findHigherIsBetter(sections?: ResultsMapper[]): boolean {
-  const cols = sections?.flatMap(s => s.columns().flatMap(g => g.columns));
-  return cols?.find(c => c.comparable)?.higherIsBetter ?? false;
-}
-
-/** Flip CI percent for metrics where higher is better (e.g., lines/sec) */
-function flipCI(ci: DifferenceCI): DifferenceCI {
-  return {
-    percent: -ci.percent,
-    ci: [-ci.ci[1], -ci.ci[0]],
-    direction: ci.direction,
-    histogram: ci.histogram?.map(bin => ({ x: -bin.x, count: bin.count })),
-  };
-}
-
-/** Prepare ReportData from benchmark results for HTML rendering */
-export function prepareHtmlData(
-  groups: ReportGroup[],
-  options: PrepareHtmlOptions,
-): ReportData {
-  const { cliArgs, sections, currentVersion, baselineVersion } = options;
-  const higherIsBetter = findHigherIsBetter(sections);
-  return {
-    groups: groups.map(group =>
-      prepareGroupData(group, sections, higherIsBetter),
-    ),
-    metadata: {
-      timestamp: new Date().toISOString(),
-      bencherVersion: process.env.npm_package_version || "unknown",
-      cliArgs,
-      gcTrackingEnabled: cliArgs?.["gc-stats"] === true,
-      currentVersion,
-      baselineVersion,
-    },
-  };
-}
-
-/** @return group data with bootstrap CI comparisons against baseline */
-function prepareGroupData(
-  group: ReportGroup,
-  sections?: ResultsMapper[],
-  higherIsBetter?: boolean,
-): GroupData {
-  const baselineSamples = group.baseline?.measuredResults.samples;
-  return {
-    name: group.name,
-    baseline: group.baseline
-      ? prepareBenchmarkData(group.baseline, sections)
-      : undefined,
-    benchmarks: group.reports.map(report => {
-      const samples = report.measuredResults.samples;
-      const rawCI =
-        baselineSamples && samples
-          ? bootstrapDifferenceCI(baselineSamples, samples)
-          : undefined;
-      const comparisonCI = rawCI && higherIsBetter ? flipCI(rawCI) : rawCI;
-      return { ...prepareBenchmarkData(report, sections), comparisonCI };
-    }),
-  };
-}
-
-/** @return benchmark data with samples, stats, and formatted section values */
-function prepareBenchmarkData(
-  report: {
-    name: string;
-    measuredResults: any;
-    metadata?: Record<string, unknown>;
-  },
-  sections?: ResultsMapper[],
-): Omit<BenchmarkData, "comparisonCI"> {
-  const { measuredResults } = report;
-  return {
-    name: report.name,
-    samples: measuredResults.samples,
-    warmupSamples: measuredResults.warmupSamples,
-    allocationSamples: measuredResults.allocationSamples,
-    heapSamples: measuredResults.heapSamples,
-    gcEvents: measuredResults.nodeGcTime?.events,
-    optSamples: measuredResults.optSamples,
-    pausePoints: measuredResults.pausePoints,
-    stats: measuredResults.time,
-    heapSize: measuredResults.heapSize,
-    sectionStats: sections ? extractSectionStats(report, sections) : undefined,
-  };
-}
-
-/** @return formatted stats from all sections for tooltip display */
-function extractSectionStats(
-  report: { measuredResults: any; metadata?: Record<string, unknown> },
-  sections: ResultsMapper[],
-): FormattedStat[] {
-  return sections.flatMap(section => {
-    const vals = section.extract(report.measuredResults, report.metadata);
-    return section.columns().flatMap(g => formatGroupStats(vals, g));
-  });
-}
-
-/** @return formatted stats for one column group, skipping undefined values */
-function formatGroupStats(
-  values: Record<string, unknown>,
-  group: ReportColumnGroup<Record<string, unknown>>,
-): FormattedStat[] {
-  return group.columns
-    .map(c => formatColumnStat(values, c, group.groupTitle))
-    .filter((s): s is FormattedStat => s !== undefined);
-}
-
-type ColumnLike = {
-  key: string;
-  title: string;
-  formatter?: (v: unknown) => string | null;
-};
-
-/** @return formatted stat for a single column, or undefined if empty/placeholder */
-function formatColumnStat(
-  values: Record<string, unknown>,
-  col: ColumnLike,
-  groupTitle?: string,
-): FormattedStat | undefined {
-  const raw = values[col.key];
-  if (raw === undefined) return undefined;
-  const formatted = col.formatter ? col.formatter(raw) : String(raw);
-  if (!formatted || formatted === "—" || formatted === "") return undefined;
-  return { label: col.title, value: formatted, groupTitle };
-}