benchforge 0.1.11 → 0.2.4

package/src/matrix/MatrixFilter.ts

@@ -1,4 +1,4 @@
-import type { BenchMatrix } from "../BenchMatrix.ts";
+import type { BenchMatrix } from "./BenchMatrix.ts";
 import { loadCasesModule } from "./CaseLoader.ts";
 import { discoverVariants } from "./VariantLoader.ts";
 
@@ -17,11 +17,8 @@ export interface FilteredMatrix<T = unknown> extends BenchMatrix<T> {
 /** Parse filter string: "case/variant", "case/", "/variant", or "case" */
 export function parseMatrixFilter(filter: string): MatrixFilter {
   if (filter.includes("/")) {
-    const [casePart, variantPart] = filter.split("/", 2);
-    return {
-      case: casePart || undefined,
-      variant: variantPart || undefined,
-    };
+    const [casePart, varPart] = filter.split("/", 2);
+    return { case: casePart || undefined, variant: varPart || undefined };
   }
   return { case: filter };
 }
@@ -36,68 +33,73 @@ export async function filterMatrix<T>(
   const caseList = await getFilteredCases(matrix, filter.case);
   const variantList = await getFilteredVariants(matrix, filter.variant);
 
-  const filteredCases =
-    caseList && matrix.filteredCases
-      ? caseList.filter(c => matrix.filteredCases!.includes(c))
-      : (caseList ?? matrix.filteredCases);
-
-  const filteredVariants =
-    variantList && matrix.filteredVariants
-      ? variantList.filter(v => matrix.filteredVariants!.includes(v))
-      : (variantList ?? matrix.filteredVariants);
+  const filteredCases = intersectFilters(caseList, matrix.filteredCases);
+  const filteredVariants = intersectFilters(
+    variantList,
+    matrix.filteredVariants,
+  );
 
   return { ...matrix, filteredCases, filteredVariants };
 }
 
-/** Get case IDs matching filter pattern */
+/** Collect all case IDs from either casesModule or inline cases */
+export async function resolveCaseIds<T>(
+  matrix: BenchMatrix<T>,
+): Promise<string[] | undefined> {
+  if (matrix.casesModule)
+    return (await loadCasesModule(matrix.casesModule)).cases;
+  return matrix.cases;
+}
+
+/** Collect all variant IDs from either inline variants or variantDir */
+export async function resolveVariantIds<T>(
+  matrix: BenchMatrix<T>,
+): Promise<string[]> {
+  if (matrix.variants) return Object.keys(matrix.variants);
+  if (matrix.variantDir) return discoverVariants(matrix.variantDir);
+  throw new Error("BenchMatrix requires 'variants' or 'variantDir'");
+}
+
+/** Return case IDs matching a substring pattern, or all if no pattern */
 async function getFilteredCases<T>(
   matrix: BenchMatrix<T>,
   casePattern?: string,
 ): Promise<string[] | undefined> {
   if (!casePattern) return undefined;
-
-  const caseIds = matrix.casesModule
-    ? (await loadCasesModule(matrix.casesModule)).cases
-    : matrix.cases;
+  const caseIds = await resolveCaseIds(matrix);
   if (!caseIds) return ["default"]; // implicit single case
-
-  const filtered = caseIds.filter(id => matchPattern(id, casePattern));
-  if (filtered.length === 0) {
-    throw new Error(`No cases match filter: "${casePattern}"`);
-  }
-  return filtered;
+  return filterByPattern(caseIds, casePattern, "cases");
 }
 
-/** Get variant IDs matching filter pattern */
+/** Return variant IDs matching a substring pattern, or all if no pattern */
 async function getFilteredVariants<T>(
   matrix: BenchMatrix<T>,
   variantPattern?: string,
 ): Promise<string[] | undefined> {
   if (!variantPattern) return undefined;
+  const allIds = await resolveVariantIds(matrix);
+  return filterByPattern(allIds, variantPattern, "variants");
+}
 
-  if (matrix.variants) {
-    const ids = Object.keys(matrix.variants).filter(id =>
-      matchPattern(id, variantPattern),
-    );
-    if (ids.length === 0) {
-      throw new Error(`No variants match filter: "${variantPattern}"`);
-    }
-    return ids;
-  }
-
-  if (matrix.variantDir) {
-    const allIds = await discoverVariants(matrix.variantDir);
-    const filtered = allIds.filter(id => matchPattern(id, variantPattern));
-    if (filtered.length === 0) {
-      throw new Error(`No variants match filter: "${variantPattern}"`);
-    }
-    return filtered;
-  }
+/** Intersect two optional filter lists: both present ==> intersection, otherwise the one that exists */
+function intersectFilters(a?: string[], b?: string[]): string[] | undefined {
+  if (a && b) return a.filter(v => b.includes(v));
+  return a ?? b;
+}
 
-  throw new Error("BenchMatrix requires 'variants' or 'variantDir'");
+/** Filter IDs by substring pattern, throwing if no matches */
+function filterByPattern(
+  ids: string[],
+  pattern: string,
+  label: string,
+): string[] {
+  const filtered = ids.filter(id => matchPattern(id, pattern));
+  if (filtered.length === 0)
+    throw new Error(`No ${label} match filter: "${pattern}"`);
+  return filtered;
 }
 
-/** Match id against pattern (case-insensitive substring) */
+/** Case-insensitive substring match */
 function matchPattern(id: string, pattern: string): boolean {
   return id.toLowerCase().includes(pattern.toLowerCase());
 }

package/src/matrix/MatrixInlineRunner.ts

@@ -0,0 +1,50 @@
+import { TimingRunner } from "../runners/TimingRunner.ts";
+import type {
+  BenchMatrix,
+  CaseResult,
+  RunMatrixOptions,
+  VariantResult,
+} from "./BenchMatrix.ts";
+import {
+  buildRunnerOptions,
+  prepareBenchFn,
+  resolveCases,
+} from "./BenchMatrix.ts";
+import { loadCaseData } from "./CaseLoader.ts";
+
+/** Run matrix with in-memory variant functions (no worker isolation) */
+export async function runMatrixInline<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<{ name: string; variants: VariantResult[] }> {
+  if (matrix.baselineDir)
+    throw new Error(
+      "BenchMatrix with inline 'variants' cannot use 'baselineDir'. Use 'variantDir' instead.",
+    );
+
+  const { casesModule, caseIds } = await resolveCases(matrix, options);
+  const runner = new TimingRunner();
+  const runnerOpts = buildRunnerOptions(options);
+
+  const allEntries = Object.entries(matrix.variants!);
+  const { filteredVariants } = options;
+  const variantEntries = filteredVariants
+    ? allEntries.filter(([id]) => filteredVariants.includes(id))
+    : allEntries;
+
+  const variants: VariantResult[] = [];
+  for (const [variantId, variant] of variantEntries) {
+    const cases: CaseResult[] = [];
+    for (const caseId of caseIds) {
+      const loaded = await loadCaseData(casesModule, caseId);
+      const data = casesModule || matrix.cases ? loaded.data : (undefined as T);
+      const fn = await prepareBenchFn(variant, data);
+      const spec = { name: variantId, fn };
+      const [measured] = await runner.runBench(spec, runnerOpts);
+      cases.push({ caseId, measured, metadata: loaded.metadata });
+    }
+    variants.push({ id: variantId, cases });
+  }
+
+  return { name: matrix.name, variants };
+}

package/src/matrix/MatrixReport.ts

@@ -1,106 +1,53 @@
-import type { CaseResult, MatrixResults } from "../BenchMatrix.ts";
-import { injectDiffColumns, type ResultsMapper } from "../BenchmarkReport.ts";
-import { totalProfileBytes } from "../heap-sample/HeapSampleReport.ts";
-import { type GcStatsInfo, gcStatsSection } from "../StandardSections.ts";
 import {
-  average,
-  bootstrapDifferenceCI,
-  type DifferenceCI,
-} from "../StatisticalUtils.ts";
-import {
-  duration,
-  formatBytes,
-  formatDiffWithCI,
-  truncate,
-} from "../table-util/Formatters.ts";
-import {
-  buildTable,
-  type ColumnGroup,
-  type ResultGroup,
-} from "../table-util/TableReport.ts";
-
-/** Custom column definition for extra computed metrics */
-export interface ExtraColumn {
-  key: string;
-  title: string;
-  groupTitle?: string; // optional column group header
-  extract: (caseResult: CaseResult) => unknown;
-  formatter?: (value: unknown) => string;
-}
-
-/** Options for matrix report generation */
+  type ComparisonOptions,
+  computeDiffCI,
+  extractSectionValues,
+  findPrimaryColumn,
+  type ReportSection,
+} from "../report/BenchmarkReport.ts";
+import { truncate } from "../report/Formatters.ts";
+import { runsSection, timeSection } from "../report/StandardSections.ts";
+import { buildTable } from "../report/text/TableReport.ts";
+import { sectionColumnGroups } from "../report/text/TextReport.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import type {
+  CaseResult,
+  MatrixResults,
+  VariantResult,
+} from "./BenchMatrix.ts";
+
+/** Options for {@link reportMatrixResults} */
 export interface MatrixReportOptions {
-  extraColumns?: ExtraColumn[];
-  sections?: ResultsMapper[]; // ResultsMapper sections (like BenchSuite)
-  variantTitle?: string; // custom title for the variant column (default: "variant")
+  /** ReportSection sections (default: [timeSection, runsSection]) */
+  sections?: ReportSection[];
+  /** Custom title for the variant column (default: "variant") */
+  variantTitle?: string;
+  /** Comparison options (equivalence margin, batch trimming) */
+  comparison?: ComparisonOptions;
 }
 
-/** Row data for matrix report table */
-interface MatrixReportRow extends Record<string, unknown> {
-  name: string;
-  time: number;
-  samples: number;
-  diffCI?: DifferenceCI;
+interface VariantCase {
+  variant: VariantResult;
+  cr: CaseResult;
 }
 
-/** GC statistics columns - derived from gcStatsSection for consistency */
-export const gcStatsColumns: ExtraColumn[] = gcStatsSection
-  .columns()[0]
-  .columns.map(col => ({
-    key: col.key as string,
-    title: col.title,
-    groupTitle: "GC",
-    extract: (r: CaseResult) =>
-      gcStatsSection.extract(r.measured)[col.key as keyof GcStatsInfo],
-    formatter: (v: unknown) => col.formatter?.(v) ?? "-",
-  }));
-
-/** GC pause time column */
-export const gcPauseColumn: ExtraColumn = {
-  key: "gcPause",
-  title: "pause",
-  groupTitle: "GC",
-  extract: r => r.measured.gcStats?.gcPauseTime,
-  formatter: v => (v != null ? `${(v as number).toFixed(1)}ms` : "-"),
-};
+type Row = Record<string, unknown> & { name: string };
 
-/** Heap sampling total bytes column */
-export const heapTotalColumn: ExtraColumn = {
-  key: "heapTotal",
-  title: "heap",
-  extract: r => {
-    const profile = r.measured.heapProfile;
-    if (!profile?.head) return undefined;
-    return totalProfileBytes(profile);
-  },
-  formatter: formatBytesOrDash,
-};
+const defaultSections: ReportSection[] = [timeSection, runsSection];
 
-/** Format matrix results as one table per case */
+/** Format matrix results as text, with one table per case */
 export function reportMatrixResults(
   results: MatrixResults,
   options?: MatrixReportOptions,
 ): string {
-  const tables = buildCaseTables(results, options);
-  const header = `Matrix: ${results.name}`;
-  return [header, ...tables].join("\n\n");
-}
+  if (results.variants.length === 0) return `Matrix: ${results.name}`;
 
-/** Format bytes with fallback to "-" for missing values */
-function formatBytesOrDash(value: unknown): string {
-  return formatBytes(value) ?? "-";
-}
-
-/** Build one table for each case showing all variants */
-function buildCaseTables(
-  results: MatrixResults,
-  options?: MatrixReportOptions,
-): string[] {
-  if (results.variants.length === 0) return [];
-
-  // Get all case IDs from first variant (all variants have same cases)
+  // all variants have the same cases
   const caseIds = results.variants[0].cases.map(c => c.caseId);
-  return caseIds.map(caseId => buildCaseTable(results, caseId, options));
+  const tables = caseIds.map(caseId =>
+    buildCaseTable(results, caseId, options),
+  );
+  return [`Matrix: ${results.name}`, ...tables].join("\n\n");
 }
 
 /** Build table for a single case showing all variants */
@@ -109,19 +56,45 @@ function buildCaseTable(
   caseId: string,
   options?: MatrixReportOptions,
 ): string {
-  const caseTitle = formatCaseTitle(results, caseId);
-
-  if (options?.sections?.length) {
-    return buildSectionTable(results, caseId, options, caseTitle);
-  }
+  const title = formatCaseTitle(results, caseId);
+  const sections = options?.sections ?? defaultSections;
+  const variantTitle = options?.variantTitle ?? "variant";
+  const primaryCol = findPrimaryColumn(sections);
+
+  const caseResults = collectCaseResults(results, caseId);
+  const shared = sharedBaseline(caseResults);
+
+  const rows: Row[] = caseResults.flatMap(({ variant, cr }) => {
+    const vals = extractSectionValues(cr.measured, sections, cr.metadata);
+    const row: Row = { name: truncate(variant.id, 25), ...vals };
+    if (cr.baseline && primaryCol?.statKind) {
+      const { statKind, higherIsBetter } = primaryCol;
+      row.diffCI = computeDiffCI(
+        cr.baseline,
+        cr.measured,
+        statKind,
+        options?.comparison,
+        higherIsBetter,
+      );
+    }
+    const out: Row[] = [row];
+    if (cr.baseline && !shared)
+      out.push({
+        name: " \u21B3 baseline",
+        ...extractSectionValues(cr.baseline, sections, cr.metadata),
+      });
+    return out;
+  });
 
-  const rows = buildCaseRows(results, caseId, options?.extraColumns);
-  const hasBaseline = rows.some(r => r.diffCI);
-  const columns = buildColumns(hasBaseline, options);
+  if (shared)
+    rows.push({
+      name: "=> baseline",
+      ...extractSectionValues(shared, sections),
+    });
 
-  const resultGroup: ResultGroup<MatrixReportRow> = { results: rows };
-  const table = buildTable(columns, [resultGroup]);
-  return `${caseTitle}\n${table}`;
+  const hasDiff = rows.some(r => r.diffCI);
+  const cols = sectionColumnGroups(sections, hasDiff, variantTitle);
+  return `${title}\n${buildTable(cols, [{ results: rows }])}`;
 }
 
 /** Format case title with metadata if available */
@@ -129,162 +102,29 @@ function formatCaseTitle(results: MatrixResults, caseId: string): string {
   const caseResult = results.variants[0]?.cases.find(c => c.caseId === caseId);
   const metadata = caseResult?.metadata;
 
-  if (metadata && Object.keys(metadata).length > 0) {
-    const metaParts = Object.entries(metadata)
-      .map(([k, v]) => `${v} ${k}`)
-      .join(", ");
-    return `${caseId} (${metaParts})`;
-  }
-  return caseId;
+  if (!metadata || Object.keys(metadata).length === 0) return caseId;
+  const meta = Object.entries(metadata)
+    .map(([k, v]) => `${v} ${k}`)
+    .join(", ");
+  return `${caseId} (${meta})`;
 }
 
-/** Build table using ResultsMapper sections */
-function buildSectionTable(
+/** Collect (variant, caseResult) pairs for a given caseId */
+function collectCaseResults(
   results: MatrixResults,
   caseId: string,
-  options: MatrixReportOptions,
-  caseTitle: string,
-): string {
-  const sections = options.sections!;
-  const variantTitle = options.variantTitle ?? "name";
-
-  const rows: Record<string, unknown>[] = [];
-  let hasBaseline = false;
-
-  for (const variant of results.variants) {
-    const caseResult = variant.cases.find(c => c.caseId === caseId);
-    if (!caseResult) continue;
-
-    const row: Record<string, unknown> = { name: truncate(variant.id, 25) };
-
-    for (const section of sections) {
-      Object.assign(
-        row,
-        section.extract(caseResult.measured, caseResult.metadata),
-      );
-    }
-
-    if (caseResult.baseline) {
-      hasBaseline = true;
-      const { samples: base } = caseResult.baseline;
-      row.diffCI = bootstrapDifferenceCI(base, caseResult.measured.samples);
-    }
-
-    rows.push(row);
-  }
-
-  const columnGroups = buildSectionColumns(sections, variantTitle, hasBaseline);
-  const resultGroup: ResultGroup<Record<string, unknown>> = { results: rows };
-  const table = buildTable(columnGroups, [resultGroup]);
-  return `${caseTitle}\n${table}`;
-}
-
-/** Build rows for all variants for a given case */
-function buildCaseRows(
-  results: MatrixResults,
-  caseId: string,
-  extraColumns?: ExtraColumn[],
-): MatrixReportRow[] {
+): VariantCase[] {
   return results.variants.flatMap(variant => {
-    const caseResult = variant.cases.find(c => c.caseId === caseId);
-    return caseResult ? [buildRow(variant.id, caseResult, extraColumns)] : [];
+    const cr = variant.cases.find(c => c.caseId === caseId);
+    return cr ? [{ variant, cr }] : [];
   });
 }
 
-/** Build column configuration */
-function buildColumns(
-  hasBaseline: boolean,
-  options?: MatrixReportOptions,
-): ColumnGroup<MatrixReportRow>[] {
-  const variantTitle = options?.variantTitle ?? "variant";
-  const nameCol: ColumnGroup<MatrixReportRow> = {
-    columns: [{ key: "name", title: variantTitle }],
-  };
-
-  const ciKey = "diffCI" as keyof MatrixReportRow;
-  const diffCol = { key: ciKey, title: "Δ% CI", formatter: formatDiff };
-  const timeCol: ColumnGroup<MatrixReportRow> = {
-    columns: [
-      { key: "time", title: "time", formatter: duration },
-      ...(hasBaseline ? [diffCol] : []),
-    ],
-  };
-
-  const groups: ColumnGroup<MatrixReportRow>[] = [nameCol, timeCol];
-
-  // Add extra columns, grouped by groupTitle
-  const extraColumns = options?.extraColumns;
-  if (extraColumns?.length) {
-    const byGroup = new Map<string | undefined, ExtraColumn[]>();
-    for (const col of extraColumns) {
-      const group = byGroup.get(col.groupTitle) ?? [];
-      group.push(col);
-      byGroup.set(col.groupTitle, group);
-    }
-    for (const [groupTitle, cols] of byGroup) {
-      groups.push({
-        groupTitle,
-        columns: cols.map(col => ({
-          key: col.key as keyof MatrixReportRow,
-          title: col.title,
-          formatter: col.formatter ?? String,
-        })),
-      });
-    }
-  }
-
-  return groups;
-}
-
-/** Build column groups from ResultsMapper sections */
-function buildSectionColumns(
-  sections: ResultsMapper[],
-  variantTitle: string,
-  hasBaseline: boolean,
-): ColumnGroup<Record<string, unknown>>[] {
-  const nameCol: ColumnGroup<Record<string, unknown>> = {
-    columns: [{ key: "name", title: variantTitle }],
-  };
-
-  const sectionColumns = sections.flatMap(s => s.columns());
-  const columnGroups = hasBaseline
-    ? injectDiffColumns(sectionColumns)
-    : (sectionColumns as ColumnGroup<Record<string, unknown>>[]);
-
-  return [nameCol, ...columnGroups];
-}
-
-/** Build a single row from case result */
-function buildRow(
-  variantId: string,
-  caseResult: CaseResult,
-  extraColumns?: ExtraColumn[],
-): MatrixReportRow {
-  const { measured, baseline } = caseResult;
-  const samples = measured.samples;
-  const time = measured.time?.avg ?? average(samples);
-
-  const row: MatrixReportRow = {
-    name: truncate(variantId, 25),
-    time,
-    samples: samples.length,
-  };
-
-  if (baseline) {
-    row.diffCI = bootstrapDifferenceCI(baseline.samples, samples);
-  }
-
-  if (extraColumns) {
-    for (const col of extraColumns) {
-      row[col.key] = col.extract(caseResult);
-    }
-  }
-
-  return row;
-}
-
-/** Format diff with CI, or "baseline" marker */
-function formatDiff(value: unknown): string | null {
-  if (!value) return null;
-  return formatDiffWithCI(value as DifferenceCI);
+/** @return shared baseline if all variants reference the same one (baselineVariant mode) */
+function sharedBaseline(
+  caseResults: VariantCase[],
+): MeasuredResults | undefined {
+  const baselines = caseResults.map(({ cr }) => cr.baseline).filter(Boolean);
+  if (baselines.length < 2) return undefined;
+  return baselines.every(b => b === baselines[0]) ? baselines[0] : undefined;
 }

package/src/matrix/VariantLoader.ts

@@ -1,8 +1,8 @@
 import fs from "node:fs/promises";
 import { fileURLToPath } from "node:url";
-import type { Variant } from "../BenchMatrix.ts";
+import type { Variant } from "./BenchMatrix.ts";
 
-/** Discover variant ids from a directory of .ts files */
+/** List variant IDs by scanning .ts files in a directory */
 export async function discoverVariants(dirUrl: string): Promise<string[]> {
   const dirPath = fileURLToPath(dirUrl);
   const entries = await fs.readdir(dirPath, { withFileTypes: true });
@@ -12,7 +12,7 @@ export async function discoverVariants(dirUrl: string): Promise<string[]> {
     .sort();
 }
 
-/** Load a variant module and extract run/setup exports */
+/** Import a variant module and return its run/setup exports as a Variant */
 export async function loadVariant<T = unknown>(
   dirUrl: string,
   variantId: string,
@@ -22,12 +22,12 @@ export async function loadVariant<T = unknown>(
   return extractVariant(module, variantId, moduleUrl);
 }
 
-/** Get module URL for a variant in a directory */
+/** Resolve the import URL for a variant file */
 export function variantModuleUrl(dirUrl: string, variantId: string): string {
   return new URL(`${variantId}.ts`, dirUrl).href;
 }
 
-/** Extract variant from module exports */
+/** Validate and extract a Variant from a module's exports */
 function extractVariant<T>(
   module: Record<string, unknown>,
   variantId: string,

package/src/profiling/browser/BenchLoop.ts

@@ -0,0 +1,51 @@
+import {
+  instrumentOpts,
+  startInstruments,
+  stopInstruments,
+} from "./BrowserCDP.ts";
+import type { BrowserProfileResult, ProfileCtx } from "./BrowserProfiler.ts";
+
+/**
+ * Bench function mode: run window.__bench in a timed iteration loop.
+ *
+ * Simplified vs TimingRunner because it runs inside page.evaluate()
+ * where shared code, Node APIs, and V8 intrinsics are unavailable.
+ *
+ * Not feasible in browser page context:
+ *  - heap tracking (no getHeapStatistics)
+ *  - V8 opt status tracing (no %GetOptimizationStatus)
+ *  - explicit GC or pause-for-compilation
+ */
+export async function runBenchLoop(
+  ctx: ProfileCtx,
+): Promise<BrowserProfileResult> {
+  const { page, cdp, params, samplingInterval } = ctx;
+  const maxTime = params.maxTime ?? Number.MAX_SAFE_INTEGER;
+  const maxIter = params.maxIterations ?? Number.MAX_SAFE_INTEGER;
+  const opts = instrumentOpts(params, samplingInterval);
+
+  await startInstruments(cdp, opts);
+
+  const { samples, totalMs } = await page.evaluate(
+    async ({ maxTime, maxIter }) => {
+      const bench = (globalThis as any).__bench;
+      const estimated = Math.min(maxIter, Math.ceil(maxTime / 0.1));
+      const samples = new Array<number>(estimated);
+      let count = 0;
+      const startAll = performance.now();
+      const deadline = startAll + maxTime;
+      for (let i = 0; i < maxIter && performance.now() < deadline; i++) {
+        const t0 = performance.now();
+        await bench();
+        samples[count++] = performance.now() - t0;
+      }
+      samples.length = count;
+      return { samples, totalMs: performance.now() - startAll };
+    },
+    { maxTime, maxIter },
+  );
+
+  const collected = await stopInstruments(cdp, opts);
+
+  return { samples, wallTimeMs: totalMs, ...collected };
+}