benchforge 0.1.9 → 0.2.4

package/src/profiling/node/HeapSampleReport.ts

@@ -0,0 +1,261 @@
+import colors from "../../report/Colors.ts";
+import { formatBytes } from "../../report/Formatters.ts";
+import type { HeapProfile, HeapSample } from "./HeapSampler.ts";
+import {
+  type ResolvedFrame,
+  type ResolvedProfile,
+  resolveProfile,
+} from "./ResolvedProfile.ts";
+
+/** An allocation site with byte totals, call stack, and optional raw samples */
+export interface HeapSite {
+  name: string;
+  url: string;
+  /** 1-indexed */
+  line: number;
+  col?: number;
+  bytes: number;
+  /** Call stack from root to this frame */
+  stack?: ResolvedFrame[];
+  /** Individual allocation samples at this site */
+  samples?: HeapSample[];
+  /** Distinct caller paths with byte weights (populated by {@link aggregateSites}) */
+  callers?: { stack: ResolvedFrame[]; bytes: number }[];
+}
+
+/** Predicate that returns true for user code (vs. runtime internals) */
+export type UserCodeFilter = (site: ResolvedFrame) => boolean;
+
+/** Options for {@link formatHeapReport} */
+export interface HeapReportOptions {
+  /** Max sites to display */
+  topN: number;
+  /** Caller stack frames to show per site (default 3) */
+  stackDepth?: number;
+  /** Multi-line format with file paths (default false) */
+  verbose?: boolean;
+  /** Dump every raw sample */
+  raw?: boolean;
+  /** Filter to user code only, hiding runtime internals */
+  userOnly?: boolean;
+  /** Predicate for user vs internal code (default {@link isNodeUserCode}) */
+  isUserCode?: UserCodeFilter;
+  /** Total bytes across all nodes (before filtering) */
+  totalAll?: number;
+  /** Total bytes for user code only */
+  totalUserCode?: number;
+  /** Number of samples taken */
+  sampleCount?: number;
+}
+
+/** Sum selfSize across all nodes in profile (before any filtering) */
+export function totalProfileBytes(profile: HeapProfile): number {
+  return resolveProfile(profile).totalBytes;
+}
+
+/** Flatten resolved profile into sorted list of allocation sites with call stacks.
+ *  When raw samples are available, attaches them to corresponding sites. */
+export function flattenProfile(resolved: ResolvedProfile): HeapSite[] {
+  const sites: HeapSite[] = [];
+  const nodeIdToSites = new Map<number, HeapSite[]>();
+
+  for (const node of resolved.allocationNodes) {
+    const site: HeapSite = {
+      ...node.frame,
+      bytes: node.selfSize,
+      stack: node.stack,
+    };
+    sites.push(site);
+    const bucket = nodeIdToSites.get(node.nodeId) ?? [];
+    if (!bucket.length) nodeIdToSites.set(node.nodeId, bucket);
+    bucket.push(site);
+  }
+
+  for (const sample of resolved.sortedSamples ?? []) {
+    const matchingSites = nodeIdToSites.get(sample.nodeId);
+    if (!matchingSites) continue;
+    for (const site of matchingSites) {
+      if (!site.samples) site.samples = [];
+      site.samples.push(sample);
+    }
+  }
+
+  return sites.sort((a, b) => b.bytes - a.bytes);
+}
+
+/** Return true if the call frame is user code (excludes node: and internal/ URLs) */
+export function isNodeUserCode(site: ResolvedFrame): boolean {
+  const { url } = site;
+  return (
+    !!url &&
+    !url.startsWith("node:") &&
+    !url.includes("(native)") &&
+    !url.includes("internal/")
+  );
+}
+
+/** Return true if the call frame is user code (excludes chrome-extension:// and devtools:// URLs) */
+export function isBrowserUserCode(site: ResolvedFrame): boolean {
+  const { url } = site;
+  return (
+    !!url &&
+    !url.startsWith("chrome-extension://") &&
+    !url.startsWith("devtools://") &&
+    !url.includes("(native)")
+  );
+}
+
+/** Return only sites matching a user-code predicate (default: {@link isNodeUserCode}) */
+export function filterSites(
+  sites: HeapSite[],
+  isUser: UserCodeFilter = isNodeUserCode,
+): HeapSite[] {
+  return sites.filter(isUser);
+}
+
+/** Aggregate sites by location (combine same file:line:col).
+ *  Tracks distinct caller stacks with byte weights when merging. */
+export function aggregateSites(sites: HeapSite[]): HeapSite[] {
+  const byLocation = new Map<string, HeapSite>();
+
+  for (const site of sites) {
+    // When column is unknown, include name to avoid merging distinct sites
+    const colKey = site.col != null ? `${site.col}` : `?:${site.name}`;
+    const key = `${site.url}:${site.line}:${colKey}`;
+    const existing = byLocation.get(key);
+    if (existing) {
+      existing.bytes += site.bytes;
+      addCaller(existing, site);
+    } else {
+      const callers = site.stack
+        ? [{ stack: site.stack, bytes: site.bytes }]
+        : undefined;
+      byLocation.set(key, { ...site, callers });
+    }
+  }
+
+  for (const site of byLocation.values()) {
+    if (!site.callers || site.callers.length <= 1) continue;
+    site.callers.sort((a, b) => b.bytes - a.bytes);
+    site.stack = site.callers[0].stack;
+  }
+
+  return [...byLocation.values()].sort((a, b) => b.bytes - a.bytes);
+}
+
+/** Format heap report for console output */
+export function formatHeapReport(
+  sites: HeapSite[],
+  options: HeapReportOptions,
+): string {
+  const { topN, stackDepth = 3, verbose = false } = options;
+  const { totalAll, totalUserCode, sampleCount, isUserCode } = options;
+  const isUser = isUserCode ?? isNodeUserCode;
+  const formatSite = verbose ? formatVerboseSite : formatCompactSite;
+  const lines: string[] = [];
+  lines.push(`Heap allocation sites (top ${topN}, garbage included):`);
+
+  for (const site of sites.slice(0, topN)) {
+    formatSite(lines, site, stackDepth, isUser);
+  }
+
+  lines.push("");
+  if (totalAll !== undefined)
+    lines.push(`Total (all):       ${fmtBytes(totalAll)}`);
+  if (totalUserCode !== undefined)
+    lines.push(`Total (user-code): ${fmtBytes(totalUserCode)}`);
+  if (sampleCount !== undefined)
+    lines.push(`Samples: ${sampleCount.toLocaleString()}`);
+
+  return lines.join("\n");
+}
+
+/** Sum bytes across all sites */
+export function totalBytes(sites: HeapSite[]): number {
+  return sites.reduce((sum, s) => sum + s.bytes, 0);
+}
+
+/** Format every raw sample as one line, ordered by ordinal (time).
+ *  Output is tab-separated for easy piping/grep/diff. */
+export function formatRawSamples(resolved: ResolvedProfile): string {
+  const { sortedSamples, nodeMap } = resolved;
+  if (!sortedSamples || sortedSamples.length === 0)
+    return "No raw samples available.";
+
+  const header = "ordinal\tsize\tfunction\tlocation";
+  const rows = sortedSamples.map(s => {
+    const frame = nodeMap.get(s.nodeId)?.frame;
+    const fn = frame?.name || "(unknown)";
+    const url = frame?.url || "";
+    const loc = url ? fmtLoc(url, frame!.line, frame!.col) : "(unknown)";
+    return `${s.ordinal}\t${s.size}\t${fn}\t${loc}`;
+  });
+  return [header, ...rows].join("\n");
+}
+
+/** Add a caller stack to an aggregated site, merging if the same path exists */
+function addCaller(existing: HeapSite, site: HeapSite): void {
+  if (!site.stack) return;
+  existing.callers ??= [];
+  const key = callerKey(site.stack);
+  const match = existing.callers.find(c => callerKey(c.stack) === key);
+  if (match) match.bytes += site.bytes;
+  else existing.callers.push({ stack: site.stack, bytes: site.bytes });
+}
+
+/** Verbose multi-line format with file:// paths and line numbers */
+function formatVerboseSite(
+  lines: string[],
+  site: HeapSite,
+  stackDepth: number,
+  isUser: UserCodeFilter,
+): void {
+  const bytes = fmtBytes(site.bytes).padStart(10);
+  const loc = site.url ? fmtLoc(site.url, site.line, site.col) : "(unknown)";
+  const style = isUser(site) ? (s: string) => s : colors.dim;
+  lines.push(style(`${bytes}  ${site.name}  ${loc}`));
+
+  const userCallers = callerFrames(site, stackDepth).filter(
+    f => f.url && isUser(f),
+  );
+  for (const frame of userCallers) {
+    const loc = fmtLoc(frame.url, frame.line, frame.col);
+    lines.push(style(`            <- ${frame.name}  ${loc}`));
+  }
+}
+
+/** Compact single-line format: `49 MB  fn1 <- fn2 <- fn3` */
+function formatCompactSite(
+  lines: string[],
+  site: HeapSite,
+  stackDepth: number,
+  isUser: UserCodeFilter,
+): void {
+  const bytes = fmtBytes(site.bytes).padStart(10);
+  const callers = callerFrames(site, stackDepth)
+    .filter(f => f.url && isUser(f))
+    .map(f => f.name);
+  const line = `${bytes}  ${[site.name, ...callers].join(" <- ")}`;
+  lines.push(isUser(site) ? line : colors.dim(line));
+}
+
+/** Format bytes with a space separator, falling back to raw bytes */
+function fmtBytes(bytes: number): string {
+  return formatBytes(bytes, { space: true }) ?? `${bytes} B`;
+}
+
+/** Format location, omitting column when unknown */
+function fmtLoc(url: string, line: number, col?: number): string {
+  return col != null ? `${url}:${line}:${col}` : `${url}:${line}`;
+}
+
+/** Serialize a call stack for dedup comparison */
+function callerKey(stack: ResolvedFrame[]): string {
+  return stack.map(f => `${f.url}:${f.line}:${f.col}`).join("|");
+}
+
+/** Get caller frames (parent stack excluding self, reversed, truncated) */
+function callerFrames(site: HeapSite, depth: number): ResolvedFrame[] {
+  if (!site.stack || site.stack.length <= 1) return [];
+  return site.stack.slice(0, -1).reverse().slice(0, depth);
+}

package/src/{heap-sample → profiling/node}/HeapSampler.ts

@@ -1,26 +1,68 @@
 import { Session } from "node:inspector/promises";
 
 export interface HeapSampleOptions {
-  samplingInterval?: number; // bytes between samples, default 32768
-  stackDepth?: number; // max stack frames, default 64
-  includeMinorGC?: boolean; // keep objects collected by minor GC, default true
-  includeMajorGC?: boolean; // keep objects collected by major GC, default true
+  /** Bytes between samples (default 32768) */
+  samplingInterval?: number;
+
+  /** Max stack frames (default 64) */
+  stackDepth?: number;
+
+  /** Keep objects collected by minor GC (default true) */
+  includeMinorGC?: boolean;
+
+  /** Keep objects collected by major GC (default true) */
+  includeMajorGC?: boolean;
 }
 
+/** V8 call frame location within a profiled script */
+export interface CallFrame {
+  /** Function name (empty string for anonymous) */
+  functionName: string;
+
+  /** Script URL or file path */
+  url: string;
+
+  /** Zero-based line number */
+  lineNumber: number;
+
+  /** Zero-based column number */
+  columnNumber?: number;
+}
+
+/** Node in the V8 sampling heap profile tree */
 export interface ProfileNode {
-  callFrame: {
-    functionName: string;
-    url: string;
-    lineNumber: number;
-    columnNumber?: number;
-  };
+  /** Call site for this allocation node */
+  callFrame: CallFrame;
+
+  /** Bytes allocated directly at this node (not children) */
   selfSize: number;
+
+  /** Unique node ID, links to {@link HeapSample.nodeId} */
+  id: number;
+
+  /** Child nodes in the call tree */
   children?: ProfileNode[];
 }
 
+/** Individual heap allocation sample from V8's SamplingHeapProfiler */
+export interface HeapSample {
+  /** Links to {@link ProfileNode.id} for stack lookup */
+  nodeId: number;
+
+  /** Allocation size in bytes */
+  size: number;
+
+  /** Monotonically increasing, gives temporal ordering */
+  ordinal: number;
+}
+
+/** V8 sampling heap profile tree with optional per-allocation samples */
 export interface HeapProfile {
+  /** Root of the profile call tree */
   head: ProfileNode;
-  samples?: number[]; // sample IDs (length = number of samples taken)
+
+  /** Per-allocation samples, if collected */
+  samples?: HeapSample[];
 }
 
 const defaultOptions: Required<HeapSampleOptions> = {
@@ -61,7 +103,6 @@ async function startSampling(
     includeObjectsCollectedByMinorGC: opts.includeMinorGC,
     includeObjectsCollectedByMajorGC: opts.includeMajorGC,
   };
-
   try {
     await session.post("HeapProfiler.startSampling", params);
   } catch {
@@ -72,7 +113,8 @@ async function startSampling(
   }
 }
 
+/** Stop heap sampling and return the profile, casting past incomplete @types/node typings */
 async function stopSampling(session: Session): Promise<HeapProfile> {
   const { profile } = await session.post("HeapProfiler.stopSampling");
-  return profile as HeapProfile;
+  return profile as unknown as HeapProfile;
 }

package/src/profiling/node/ResolvedProfile.ts

@@ -0,0 +1,98 @@
+import type {
+  CallFrame,
+  HeapProfile,
+  HeapSample,
+  ProfileNode,
+} from "./HeapSampler.ts";
+
+/** A call frame with display-ready 1-indexed source positions */
+export interface ResolvedFrame {
+  /** Function name, "(anonymous)" when empty */
+  name: string;
+
+  /** Script URL or file path, "" when unknown */
+  url: string;
+
+  /** 1-indexed line number */
+  line: number;
+
+  /** 1-indexed column number (undefined when unknown) */
+  col?: number;
+}
+
+/** A profile node with its resolved call stack from root to this node */
+export interface ResolvedNode {
+  /** The call frame at this node */
+  frame: ResolvedFrame;
+
+  /** Call stack from root to this node (inclusive) */
+  stack: ResolvedFrame[];
+
+  /** Bytes allocated directly at this node */
+  selfSize: number;
+
+  /** V8 node ID, used to match {@link HeapSample.nodeId} */
+  nodeId: number;
+}
+
+/** Pre-resolved heap profile: single tree walk produces all derived data */
+export interface ResolvedProfile {
+  /** All nodes from the profile tree, flattened */
+  nodes: ResolvedNode[];
+
+  /** nodeId -> ResolvedNode lookup */
+  nodeMap: Map<number, ResolvedNode>;
+
+  /** Nodes with selfSize > 0, sorted by selfSize descending */
+  allocationNodes: ResolvedNode[];
+
+  /** Samples sorted by ordinal (temporal order), if available */
+  sortedSamples: HeapSample[] | undefined;
+
+  /** Total bytes across all nodes (sum of selfSize) */
+  totalBytes: number;
+}
+
+/** Convert a V8 0-indexed CallFrame to a display-ready 1-indexed ResolvedFrame */
+export function resolveCallFrame(cf: CallFrame): ResolvedFrame {
+  return {
+    name: cf.functionName || "(anonymous)",
+    url: cf.url || "",
+    line: cf.lineNumber + 1,
+    col: cf.columnNumber != null ? cf.columnNumber + 1 : undefined,
+  };
+}
+
+/** Walk a HeapProfile tree once, producing a fully resolved intermediate form */
+export function resolveProfile(profile: HeapProfile): ResolvedProfile {
+  const nodes: ResolvedNode[] = [];
+  const nodeMap = new Map<number, ResolvedNode>();
+  let totalBytes = 0;
+
+  function walk(node: ProfileNode, parentStack: ResolvedFrame[]): void {
+    const frame = resolveCallFrame(node.callFrame);
+    const stack = [...parentStack, frame];
+    const resolved: ResolvedNode = {
+      frame,
+      stack,
+      selfSize: node.selfSize,
+      nodeId: node.id,
+    };
+    nodes.push(resolved);
+    nodeMap.set(node.id, resolved);
+    totalBytes += node.selfSize;
+    for (const child of node.children ?? []) walk(child, stack);
+  }
+
+  walk(profile.head, []);
+
+  const allocationNodes = nodes
+    .filter(n => n.selfSize > 0)
+    .sort((a, b) => b.selfSize - a.selfSize);
+
+  const sortedSamples = profile.samples
+    ? [...profile.samples].sort((a, b) => a.ordinal - b.ordinal)
+    : undefined;
+
+  return { nodes, nodeMap, allocationNodes, sortedSamples, totalBytes };
+}

package/src/profiling/node/TimeSampler.ts

@@ -0,0 +1,57 @@
+import { Session } from "node:inspector/promises";
+import type { CallFrame } from "./HeapSampler.ts";
+
+/** V8 CPU profile node (flat array element, not tree) */
+export interface TimeProfileNode {
+  id: number;
+  callFrame: CallFrame;
+  hitCount?: number;
+  /** Child node IDs */
+  children?: number[];
+}
+
+/** V8 CPU profile returned by Profiler.stop */
+export interface TimeProfile {
+  nodes: TimeProfileNode[];
+  /** Microseconds */
+  startTime: number;
+  /** Microseconds */
+  endTime: number;
+  /** Node IDs sampled at each tick */
+  samples?: number[];
+  /** Microseconds between samples */
+  timeDeltas?: number[];
+}
+
+export interface TimeProfileOptions {
+  /** Sampling interval in microseconds (default 1000 = 1ms) */
+  interval?: number;
+  /** External session to use (shares Profiler domain, caller manages enable/disable) */
+  session?: Session;
+}
+
+/** Run a function while sampling CPU time, return profile */
+export async function withTimeProfiling<T>(
+  options: TimeProfileOptions,
+  fn: () => Promise<T> | T,
+): Promise<{ result: T; profile: TimeProfile }> {
+  const { interval } = options;
+  const ownSession = !options.session;
+  const session = options.session ?? new Session();
+  if (ownSession) session.connect();
+
+  try {
+    if (ownSession) await session.post("Profiler.enable");
+    if (interval)
+      await session.post("Profiler.setSamplingInterval", { interval });
+    await session.post("Profiler.start");
+    const result = await fn();
+    const { profile } = await session.post("Profiler.stop");
+    return { result, profile: profile as unknown as TimeProfile };
+  } finally {
+    if (ownSession) {
+      await session.post("Profiler.disable");
+      session.disconnect();
+    }
+  }
+}

package/src/report/BenchmarkReport.ts

@@ -0,0 +1,146 @@
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { diffCIs } from "../stats/BootstrapDifference.ts";
+import {
+  computeStat,
+  type DifferenceCI,
+  flipCI,
+  type StatKind,
+  swapDirection,
+} from "../stats/StatisticalUtils.ts";
+
+import type { AnyColumn } from "./text/TableReport.ts";
+
+/** Options that affect baseline comparison statistics */
+export interface ComparisonOptions {
+  /** Equivalence margin in percent (0 to disable) */
+  equivMargin?: number;
+  /** Disable Tukey trimming of outlier batches */
+  noBatchTrim?: boolean;
+}
+
+/** Benchmark results with optional baseline for comparison */
+export interface ReportGroup {
+  name: string;
+  reports: BenchmarkReport[];
+  baseline?: BenchmarkReport;
+}
+
+/** Results from a single benchmark run */
+export interface BenchmarkReport {
+  name: string;
+  measuredResults: MeasuredResults;
+  metadata?: UnknownRecord;
+}
+
+/** A titled group of related columns (one per report section) */
+export interface ReportSection {
+  title: string;
+  columns: ReportColumn[];
+}
+
+/** A table column with optional comparison behavior */
+export type ReportColumn = AnyColumn<Record<string, unknown>> & {
+  /** Add diff column after this column when baseline exists */
+  comparable?: boolean;
+  /** Set true for throughput metrics where higher values are better (e.g., lines/sec) */
+  higherIsBetter?: boolean;
+  /** Stat descriptor: framework computes value from samples via computeStat */
+  statKind?: StatKind;
+  /** Accessor for non-sample data (e.g., run count, metadata fields) */
+  value?: (results: MeasuredResults, metadata?: UnknownRecord) => unknown;
+  /** Convert a timing-domain value to display domain (e.g., ms to lines/sec) */
+  toDisplay?: (
+    timingValue: number,
+    metadata?: Record<string, unknown>,
+  ) => number;
+};
+
+export type UnknownRecord = Record<string, unknown>;
+
+/** Compute column values for a section from results + metadata.
+ *  statKind columns: computeStat(samples, kind), then toDisplay.
+ *  value columns: call the accessor directly. */
+export function computeColumnValues(
+  section: ReportSection,
+  results: MeasuredResults,
+  metadata?: UnknownRecord,
+): UnknownRecord {
+  return Object.fromEntries(
+    section.columns.map(col => {
+      const key = col.key ?? col.title;
+      if (col.value) return [key, col.value(results, metadata)];
+      if (col.statKind) {
+        const raw = computeStat(results.samples, col.statKind);
+        return [key, col.toDisplay ? col.toDisplay(raw, metadata) : raw];
+      }
+      return [key, undefined];
+    }),
+  );
+}
+
+/** Run each section's computeColumnValues and merge into one record */
+export function extractSectionValues(
+  measuredResults: MeasuredResults,
+  sections: ReadonlyArray<ReportSection>,
+  metadata?: UnknownRecord,
+): UnknownRecord {
+  const entries = sections.flatMap(s =>
+    Object.entries(computeColumnValues(s, measuredResults, metadata)),
+  );
+  return Object.fromEntries(entries);
+}
+
+/** All reports in a group, including the baseline if present */
+export function groupReports(group: ReportGroup): BenchmarkReport[] {
+  return group.baseline ? [...group.reports, group.baseline] : group.reports;
+}
+
+/** True if any result in the groups has the specified field with a defined value */
+export function hasField(
+  groups: ReportGroup[],
+  field: keyof MeasuredResults,
+): boolean {
+  return groups.some(group =>
+    groupReports(group).some(
+      ({ measuredResults }) => measuredResults[field] !== undefined,
+    ),
+  );
+}
+
+/** @return true if the first comparable column in sections has higherIsBetter set */
+export function isHigherIsBetter(sections: ReportSection[]): boolean {
+  return (
+    sections.flatMap(s => s.columns).find(c => c.comparable)?.higherIsBetter ??
+    false
+  );
+}
+
+/** @return the first comparable column with a statKind across all sections */
+export function findPrimaryColumn(
+  sections?: ReportSection[],
+): ReportColumn | undefined {
+  if (!sections) return undefined;
+  return sections.flatMap(s => s.columns).find(c => c.comparable && c.statKind);
+}
+
+/** Bootstrap difference CI for a column, using batch structure when available */
+export function computeDiffCI(
+  baseline: MeasuredResults | undefined,
+  current: MeasuredResults,
+  statKind: StatKind,
+  comparison?: ComparisonOptions,
+  higherIsBetter?: boolean,
+): DifferenceCI | undefined {
+  if (!baseline?.samples?.length || !current.samples?.length) return undefined;
+  const { equivMargin, noBatchTrim } = comparison ?? {};
+  const rawCIs = diffCIs(
+    baseline.samples,
+    baseline.batchOffsets,
+    current.samples,
+    current.batchOffsets,
+    [statKind],
+    { equivMargin, noBatchTrim },
+  );
+  if (!rawCIs[0]) return undefined;
+  return higherIsBetter ? swapDirection(flipCI(rawCIs[0])) : rawCIs[0];
+}

package/src/report/Colors.ts

@@ -0,0 +1,9 @@
+import pico from "picocolors";
+import type { Colors } from "picocolors/types";
+
+const isTest = process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+
+/** Picocolors instance that disables color in test environments */
+const colors: Colors = pico.createColors(!isTest);
+
+export default colors;