benchforge 0.1.9 → 0.2.4

package/src/matrix/MatrixReport.ts

@@ -1,68 +1,53 @@
-import type { CaseResult, MatrixResults } from "../BenchMatrix.ts";
-import { injectDiffColumns, type ResultsMapper } from "../BenchmarkReport.ts";
-import { totalProfileBytes } from "../heap-sample/HeapSampleReport.ts";
-import { type GcStatsInfo, gcStatsSection } from "../StandardSections.ts";
 import {
-  average,
-  bootstrapDifferenceCI,
-  type DifferenceCI,
-} from "../StatisticalUtils.ts";
-import {
-  duration,
-  formatBytes,
-  formatDiffWithCI,
-  truncate,
-} from "../table-util/Formatters.ts";
-import {
-  buildTable,
-  type ColumnGroup,
-  type ResultGroup,
-} from "../table-util/TableReport.ts";
-
-/** Custom column definition for extra computed metrics */
-export interface ExtraColumn {
-  key: string;
-  title: string;
-  groupTitle?: string; // optional column group header
-  extract: (caseResult: CaseResult) => unknown;
-  formatter?: (value: unknown) => string;
-}
-
-/** Options for matrix report generation */
+  type ComparisonOptions,
+  computeDiffCI,
+  extractSectionValues,
+  findPrimaryColumn,
+  type ReportSection,
+} from "../report/BenchmarkReport.ts";
+import { truncate } from "../report/Formatters.ts";
+import { runsSection, timeSection } from "../report/StandardSections.ts";
+import { buildTable } from "../report/text/TableReport.ts";
+import { sectionColumnGroups } from "../report/text/TextReport.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import type {
+  CaseResult,
+  MatrixResults,
+  VariantResult,
+} from "./BenchMatrix.ts";
+
+/** Options for {@link reportMatrixResults} */
 export interface MatrixReportOptions {
-  extraColumns?: ExtraColumn[];
-  sections?: ResultsMapper[]; // ResultsMapper sections (like BenchSuite)
-  variantTitle?: string; // custom title for the variant column (default: "variant")
+  /** ReportSection sections (default: [timeSection, runsSection]) */
+  sections?: ReportSection[];
+  /** Custom title for the variant column (default: "variant") */
+  variantTitle?: string;
+  /** Comparison options (equivalence margin, batch trimming) */
+  comparison?: ComparisonOptions;
 }
 
-/** Row data for matrix report table */
-interface MatrixReportRow extends Record<string, unknown> {
-  name: string;
-  time: number;
-  samples: number;
-  diffCI?: DifferenceCI;
+interface VariantCase {
+  variant: VariantResult;
+  cr: CaseResult;
 }
 
-/** Format matrix results as one table per case */
+type Row = Record<string, unknown> & { name: string };
+
+const defaultSections: ReportSection[] = [timeSection, runsSection];
+
+/** Format matrix results as text, with one table per case */
 export function reportMatrixResults(
   results: MatrixResults,
   options?: MatrixReportOptions,
 ): string {
-  const tables = buildCaseTables(results, options);
-  const header = `Matrix: ${results.name}`;
-  return [header, ...tables].join("\n\n");
-}
+  if (results.variants.length === 0) return `Matrix: ${results.name}`;
 
-/** Build one table for each case showing all variants */
-function buildCaseTables(
-  results: MatrixResults,
-  options?: MatrixReportOptions,
-): string[] {
-  if (results.variants.length === 0) return [];
-
-  // Get all case IDs from first variant (all variants have same cases)
+  // all variants have the same cases
   const caseIds = results.variants[0].cases.map(c => c.caseId);
-  return caseIds.map(caseId => buildCaseTable(results, caseId, options));
+  const tables = caseIds.map(caseId =>
+    buildCaseTable(results, caseId, options),
+  );
+  return [`Matrix: ${results.name}`, ...tables].join("\n\n");
 }
 
 /** Build table for a single case showing all variants */
@@ -71,170 +56,45 @@ function buildCaseTable(
   caseId: string,
   options?: MatrixReportOptions,
 ): string {
-  const caseTitle = formatCaseTitle(results, caseId);
-
-  if (options?.sections?.length) {
-    return buildSectionTable(results, caseId, options, caseTitle);
-  }
-
-  const rows = buildCaseRows(results, caseId, options?.extraColumns);
-  const hasBaseline = rows.some(r => r.diffCI);
-  const columns = buildColumns(hasBaseline, options);
-
-  const resultGroup: ResultGroup<MatrixReportRow> = { results: rows };
-  const table = buildTable(columns, [resultGroup]);
-  return `${caseTitle}\n${table}`;
-}
-
-/** Build table using ResultsMapper sections */
-function buildSectionTable(
-  results: MatrixResults,
-  caseId: string,
-  options: MatrixReportOptions,
-  caseTitle: string,
-): string {
-  const sections = options.sections!;
-  const variantTitle = options.variantTitle ?? "name";
-
-  const rows: Record<string, unknown>[] = [];
-  let hasBaseline = false;
-
-  for (const variant of results.variants) {
-    const caseResult = variant.cases.find(c => c.caseId === caseId);
-    if (!caseResult) continue;
-
-    const row: Record<string, unknown> = { name: truncate(variant.id, 25) };
-
-    for (const section of sections) {
-      Object.assign(
-        row,
-        section.extract(caseResult.measured, caseResult.metadata),
-      );
-    }
-
-    if (caseResult.baseline) {
-      hasBaseline = true;
-      const { samples: base } = caseResult.baseline;
-      row.diffCI = bootstrapDifferenceCI(base, caseResult.measured.samples);
-    }
-
-    rows.push(row);
-  }
-
-  const columnGroups = buildSectionColumns(sections, variantTitle, hasBaseline);
-  const resultGroup: ResultGroup<Record<string, unknown>> = { results: rows };
-  const table = buildTable(columnGroups, [resultGroup]);
-  return `${caseTitle}\n${table}`;
-}
-
-/** Build column groups from ResultsMapper sections */
-function buildSectionColumns(
-  sections: ResultsMapper[],
-  variantTitle: string,
-  hasBaseline: boolean,
-): ColumnGroup<Record<string, unknown>>[] {
-  const nameCol: ColumnGroup<Record<string, unknown>> = {
-    columns: [{ key: "name", title: variantTitle }],
-  };
-
-  const sectionColumns = sections.flatMap(s => s.columns());
-  const columnGroups = hasBaseline
-    ? injectDiffColumns(sectionColumns)
-    : (sectionColumns as ColumnGroup<Record<string, unknown>>[]);
-
-  return [nameCol, ...columnGroups];
-}
-
-/** Build rows for all variants for a given case */
-function buildCaseRows(
-  results: MatrixResults,
-  caseId: string,
-  extraColumns?: ExtraColumn[],
-): MatrixReportRow[] {
-  return results.variants.flatMap(variant => {
-    const caseResult = variant.cases.find(c => c.caseId === caseId);
-    return caseResult ? [buildRow(variant.id, caseResult, extraColumns)] : [];
-  });
-}
-
-/** Build a single row from case result */
-function buildRow(
-  variantId: string,
-  caseResult: CaseResult,
-  extraColumns?: ExtraColumn[],
-): MatrixReportRow {
-  const { measured, baseline } = caseResult;
-  const samples = measured.samples;
-  const time = measured.time?.avg ?? average(samples);
-
-  const row: MatrixReportRow = {
-    name: truncate(variantId, 25),
-    time,
-    samples: samples.length,
-  };
-
-  if (baseline) {
-    row.diffCI = bootstrapDifferenceCI(baseline.samples, samples);
-  }
-
-  if (extraColumns) {
-    for (const col of extraColumns) {
-      row[col.key] = col.extract(caseResult);
-    }
-  }
-
-  return row;
-}
-
-/** Build column configuration */
-function buildColumns(
-  hasBaseline: boolean,
-  options?: MatrixReportOptions,
-): ColumnGroup<MatrixReportRow>[] {
+  const title = formatCaseTitle(results, caseId);
+  const sections = options?.sections ?? defaultSections;
   const variantTitle = options?.variantTitle ?? "variant";
-  const nameCol: ColumnGroup<MatrixReportRow> = {
-    columns: [{ key: "name", title: variantTitle }],
-  };
-
-  const ciKey = "diffCI" as keyof MatrixReportRow;
-  const diffCol = { key: ciKey, title: "Δ% CI", formatter: formatDiff };
-  const timeCol: ColumnGroup<MatrixReportRow> = {
-    columns: [
-      { key: "time", title: "time", formatter: duration },
-      ...(hasBaseline ? [diffCol] : []),
-    ],
-  };
-
-  const groups: ColumnGroup<MatrixReportRow>[] = [nameCol, timeCol];
-
-  // Add extra columns, grouped by groupTitle
-  const extraColumns = options?.extraColumns;
-  if (extraColumns?.length) {
-    const byGroup = new Map<string | undefined, ExtraColumn[]>();
-    for (const col of extraColumns) {
-      const group = byGroup.get(col.groupTitle) ?? [];
-      group.push(col);
-      byGroup.set(col.groupTitle, group);
+  const primaryCol = findPrimaryColumn(sections);
+
+  const caseResults = collectCaseResults(results, caseId);
+  const shared = sharedBaseline(caseResults);
+
+  const rows: Row[] = caseResults.flatMap(({ variant, cr }) => {
+    const vals = extractSectionValues(cr.measured, sections, cr.metadata);
+    const row: Row = { name: truncate(variant.id, 25), ...vals };
+    if (cr.baseline && primaryCol?.statKind) {
+      const { statKind, higherIsBetter } = primaryCol;
+      row.diffCI = computeDiffCI(
+        cr.baseline,
+        cr.measured,
+        statKind,
+        options?.comparison,
+        higherIsBetter,
+      );
     }
-    for (const [groupTitle, cols] of byGroup) {
-      groups.push({
-        groupTitle,
-        columns: cols.map(col => ({
-          key: col.key as keyof MatrixReportRow,
-          title: col.title,
-          formatter: col.formatter ?? String,
-        })),
+    const out: Row[] = [row];
+    if (cr.baseline && !shared)
+      out.push({
+        name: " \u21B3 baseline",
+        ...extractSectionValues(cr.baseline, sections, cr.metadata),
       });
-    }
-  }
+    return out;
+  });
 
-  return groups;
-}
+  if (shared)
+    rows.push({
+      name: "=> baseline",
+      ...extractSectionValues(shared, sections),
+    });
 
-/** Format diff with CI, or "baseline" marker */
-function formatDiff(value: unknown): string | null {
-  if (!value) return null;
-  return formatDiffWithCI(value as DifferenceCI);
+  const hasDiff = rows.some(r => r.diffCI);
+  const cols = sectionColumnGroups(sections, hasDiff, variantTitle);
+  return `${title}\n${buildTable(cols, [{ results: rows }])}`;
 }
 
 /** Format case title with metadata if available */
@@ -242,49 +102,29 @@ function formatCaseTitle(results: MatrixResults, caseId: string): string {
   const caseResult = results.variants[0]?.cases.find(c => c.caseId === caseId);
   const metadata = caseResult?.metadata;
 
-  if (metadata && Object.keys(metadata).length > 0) {
-    const metaParts = Object.entries(metadata)
-      .map(([k, v]) => `${v} ${k}`)
-      .join(", ");
-    return `${caseId} (${metaParts})`;
-  }
-  return caseId;
+  if (!metadata || Object.keys(metadata).length === 0) return caseId;
+  const meta = Object.entries(metadata)
+    .map(([k, v]) => `${v} ${k}`)
+    .join(", ");
+  return `${caseId} (${meta})`;
 }
 
-/** GC statistics columns - derived from gcStatsSection for consistency */
-export const gcStatsColumns: ExtraColumn[] = gcStatsSection
-  .columns()[0]
-  .columns.map(col => ({
-    key: col.key as string,
-    title: col.title,
-    groupTitle: "GC",
-    extract: (r: CaseResult) =>
-      gcStatsSection.extract(r.measured)[col.key as keyof GcStatsInfo],
-    formatter: (v: unknown) => col.formatter?.(v) ?? "-",
-  }));
-
-/** Format bytes with fallback to "-" for missing values */
-function formatBytesOrDash(value: unknown): string {
-  return formatBytes(value) ?? "-";
+/** Collect (variant, caseResult) pairs for a given caseId */
+function collectCaseResults(
+  results: MatrixResults,
+  caseId: string,
+): VariantCase[] {
+  return results.variants.flatMap(variant => {
+    const cr = variant.cases.find(c => c.caseId === caseId);
+    return cr ? [{ variant, cr }] : [];
+  });
 }
 
-/** GC pause time column */
-export const gcPauseColumn: ExtraColumn = {
-  key: "gcPause",
-  title: "pause",
-  groupTitle: "GC",
-  extract: r => r.measured.gcStats?.gcPauseTime,
-  formatter: v => (v != null ? `${(v as number).toFixed(1)}ms` : "-"),
-};
-
-/** Heap sampling total bytes column */
-export const heapTotalColumn: ExtraColumn = {
-  key: "heapTotal",
-  title: "heap",
-  extract: r => {
-    const profile = r.measured.heapProfile;
-    if (!profile?.head) return undefined;
-    return totalProfileBytes(profile);
-  },
-  formatter: formatBytesOrDash,
-};
+/** @return shared baseline if all variants reference the same one (baselineVariant mode) */
+function sharedBaseline(
+  caseResults: VariantCase[],
+): MeasuredResults | undefined {
+  const baselines = caseResults.map(({ cr }) => cr.baseline).filter(Boolean);
+  if (baselines.length < 2) return undefined;
+  return baselines.every(b => b === baselines[0]) ? baselines[0] : undefined;
+}

package/src/matrix/VariantLoader.ts

@@ -1,8 +1,8 @@
 import fs from "node:fs/promises";
 import { fileURLToPath } from "node:url";
-import type { Variant } from "../BenchMatrix.ts";
+import type { Variant } from "./BenchMatrix.ts";
 
-/** Discover variant ids from a directory of .ts files */
+/** List variant IDs by scanning .ts files in a directory */
 export async function discoverVariants(dirUrl: string): Promise<string[]> {
   const dirPath = fileURLToPath(dirUrl);
   const entries = await fs.readdir(dirPath, { withFileTypes: true });
@@ -12,7 +12,7 @@ export async function discoverVariants(dirUrl: string): Promise<string[]> {
     .sort();
 }
 
-/** Load a variant module and extract run/setup exports */
+/** Import a variant module and return its run/setup exports as a Variant */
 export async function loadVariant<T = unknown>(
   dirUrl: string,
   variantId: string,
@@ -22,7 +22,12 @@ export async function loadVariant<T = unknown>(
   return extractVariant(module, variantId, moduleUrl);
 }
 
-/** Extract variant from module exports */
+/** Resolve the import URL for a variant file */
+export function variantModuleUrl(dirUrl: string, variantId: string): string {
+  return new URL(`${variantId}.ts`, dirUrl).href;
+}
+
+/** Validate and extract a Variant from a module's exports */
 function extractVariant<T>(
   module: Record<string, unknown>,
   variantId: string,
@@ -39,8 +44,3 @@ function extractVariant<T>(
   }
   return { setup: setup as (data: T) => unknown, run: run as () => void };
 }
-
-/** Get module URL for a variant in a directory */
-export function variantModuleUrl(dirUrl: string, variantId: string): string {
-  return new URL(`${variantId}.ts`, dirUrl).href;
-}

package/src/profiling/browser/BenchLoop.ts

@@ -0,0 +1,51 @@
+import {
+  instrumentOpts,
+  startInstruments,
+  stopInstruments,
+} from "./BrowserCDP.ts";
+import type { BrowserProfileResult, ProfileCtx } from "./BrowserProfiler.ts";
+
+/**
+ * Bench function mode: run window.__bench in a timed iteration loop.
+ *
+ * Simplified vs TimingRunner because it runs inside page.evaluate()
+ * where shared code, Node APIs, and V8 intrinsics are unavailable.
+ *
+ * Not feasible in browser page context:
+ *  - heap tracking (no getHeapStatistics)
+ *  - V8 opt status tracing (no %GetOptimizationStatus)
+ *  - explicit GC or pause-for-compilation
+ */
+export async function runBenchLoop(
+  ctx: ProfileCtx,
+): Promise<BrowserProfileResult> {
+  const { page, cdp, params, samplingInterval } = ctx;
+  const maxTime = params.maxTime ?? Number.MAX_SAFE_INTEGER;
+  const maxIter = params.maxIterations ?? Number.MAX_SAFE_INTEGER;
+  const opts = instrumentOpts(params, samplingInterval);
+
+  await startInstruments(cdp, opts);
+
+  const { samples, totalMs } = await page.evaluate(
+    async ({ maxTime, maxIter }) => {
+      const bench = (globalThis as any).__bench;
+      const estimated = Math.min(maxIter, Math.ceil(maxTime / 0.1));
+      const samples = new Array<number>(estimated);
+      let count = 0;
+      const startAll = performance.now();
+      const deadline = startAll + maxTime;
+      for (let i = 0; i < maxIter && performance.now() < deadline; i++) {
+        const t0 = performance.now();
+        await bench();
+        samples[count++] = performance.now() - t0;
+      }
+      samples.length = count;
+      return { samples, totalMs: performance.now() - startAll };
+    },
+    { maxTime, maxIter },
+  );
+
+  const collected = await stopInstruments(cdp, opts);
+
+  return { samples, wallTimeMs: totalMs, ...collected };
+}

package/src/profiling/browser/BrowserCDP.ts

@@ -0,0 +1,133 @@
+import type { GcStats } from "../../runners/GcStats.ts";
+import type { CoverageData, ScriptCoverage } from "../node/CoverageTypes.ts";
+import type { HeapProfile } from "../node/HeapSampler.ts";
+import type { TimeProfile } from "../node/TimeSampler.ts";
+import { browserGcStats } from "./BrowserGcStats.ts";
+import type { CdpClient } from "./CdpClient.ts";
+import type { TraceEvent } from "./ChromeTraceEvent.ts";
+
+/** Options controlling which CDP instruments (heap, CPU, coverage) to enable. */
+export interface InstrumentOpts {
+  alloc: boolean;
+  profile: boolean;
+  callCounts: boolean;
+  samplingInterval: number;
+  profileInterval?: number;
+}
+
+/** Build InstrumentOpts from profile params and heap sampling interval. */
+export function instrumentOpts(
+  params: {
+    alloc?: boolean;
+    profile?: boolean;
+    callCounts?: boolean;
+    profileInterval?: number;
+  },
+  samplingInterval: number,
+): InstrumentOpts {
+  const {
+    alloc = false,
+    profile = false,
+    callCounts = false,
+    profileInterval,
+  } = params;
+  return { alloc, profile, callCounts, samplingInterval, profileInterval };
+}
+
+/** Start CDP GC tracing; returns the mutable array that collects trace events. */
+export async function startGcTracing(cdp: CdpClient): Promise<TraceEvent[]> {
+  const events: TraceEvent[] = [];
+  cdp.on("Tracing.dataCollected", ({ value }) => {
+    events.push(...(value as unknown as TraceEvent[]));
+  });
+  await cdp.send("Tracing.start", {
+    traceConfig: { includedCategories: ["v8", "v8.gc"] },
+  });
+  return events;
+}
+
+/** End CDP tracing and aggregate collected events into GcStats. */
+export async function collectTracing(
+  cdp: CdpClient,
+  traceEvents: TraceEvent[],
+): Promise<GcStats> {
+  const done = new Promise<void>(r =>
+    cdp.once("Tracing.tracingComplete", () => r()),
+  );
+  await cdp.send("Tracing.end");
+  await done;
+  return browserGcStats(traceEvents);
+}
+
+/** Start CDP Profiler for CPU time sampling (caller manages Profiler.enable/disable) */
+export async function startTimeProfiling(
+  cdp: CdpClient,
+  interval?: number,
+): Promise<void> {
+  if (interval) await cdp.send("Profiler.setSamplingInterval", { interval });
+  await cdp.send("Profiler.start");
+}
+
+/** Stop CDP CPU sampling and return the profile. */
+export async function stopTimeProfiling(cdp: CdpClient): Promise<TimeProfile> {
+  const { profile } = await cdp.send("Profiler.stop");
+  return profile as unknown as TimeProfile;
+}
+
+/** Start precise coverage (caller manages Profiler.enable/disable). */
+export async function startCoverageCollection(cdp: CdpClient): Promise<void> {
+  await cdp.send("Profiler.startPreciseCoverage", {
+    callCount: true,
+    detailed: true,
+  });
+}
+
+/** Collect precise coverage, filtering out browser-internal scripts. */
+export async function collectCoverage(cdp: CdpClient): Promise<CoverageData> {
+  const { result } = await cdp.send("Profiler.takePreciseCoverage");
+  await cdp.send("Profiler.stopPreciseCoverage");
+  const scripts = (result as unknown as ScriptCoverage[]).filter(isPageScript);
+  return { scripts };
+}
+
+/** Stop active instruments and return collected profiles/coverage. */
+export async function stopInstruments(
+  cdp: CdpClient,
+  opts: InstrumentOpts,
+): Promise<{
+  heapProfile?: HeapProfile;
+  timeProfile?: TimeProfile;
+  coverage?: CoverageData;
+}> {
+  const heapProfile = opts.alloc
+    ? ((await cdp.send("HeapProfiler.stopSampling")).profile as HeapProfile)
+    : undefined;
+  const timeProfile = opts.profile ? await stopTimeProfiling(cdp) : undefined;
+  const coverage = opts.callCounts ? await collectCoverage(cdp) : undefined;
+  if (opts.profile || opts.callCounts) await cdp.send("Profiler.disable");
+  return { heapProfile, timeProfile, coverage };
+}
+
+/** Start requested CDP instruments (heap, CPU, coverage). */
+export async function startInstruments(
+  cdp: CdpClient,
+  opts: InstrumentOpts,
+): Promise<void> {
+  if (opts.alloc) {
+    await cdp.send("HeapProfiler.startSampling", {
+      samplingInterval: opts.samplingInterval,
+      includeObjectsCollectedByMajorGC: true,
+      includeObjectsCollectedByMinorGC: true,
+    });
+  }
+  if (opts.profile || opts.callCounts) await cdp.send("Profiler.enable");
+  if (opts.profile) await startTimeProfiling(cdp, opts.profileInterval);
+  if (opts.callCounts) await startCoverageCollection(cdp);
+}
+
+/** Exclude chrome:// and devtools:// internal scripts. */
+function isPageScript(s: ScriptCoverage): boolean {
+  return (
+    !!s.url && !s.url.startsWith("chrome") && !s.url.startsWith("devtools")
+  );
+}

package/src/profiling/browser/BrowserGcStats.ts

@@ -0,0 +1,33 @@
+import {
+  aggregateGcStats,
+  type GcEvent,
+  type GcStats,
+} from "../../runners/GcStats.ts";
+import type { TraceEvent } from "./ChromeTraceEvent.ts";
+
+/** Convert MinorGC/MajorGC trace events into GcEvent[]. */
+export function parseGcTraceEvents(traceEvents: TraceEvent[]): GcEvent[] {
+  return traceEvents
+    .filter(e => e.ph === "X" && gcType(e.name))
+    .map(e => ({
+      type: gcType(e.name)!,
+      pauseMs: (e.dur ?? 0) / 1000,
+      collected: Math.max(
+        0,
+        Number(e.args?.usedHeapSizeBefore ?? 0) -
+          Number(e.args?.usedHeapSizeAfter ?? 0),
+      ),
+    }));
+}
+
+/** Parse and aggregate CDP trace events into GcStats. */
+export function browserGcStats(traceEvents: TraceEvent[]): GcStats {
+  return aggregateGcStats(parseGcTraceEvents(traceEvents));
+}
+
+/** Map CDP event names (MinorGC/MajorGC) to GcEvent type. */
+function gcType(name: string): GcEvent["type"] | undefined {
+  if (name === "MinorGC") return "scavenge";
+  if (name === "MajorGC") return "mark-compact";
+  return undefined;
+}