benchforge 0.1.0

package/src/export/JsonExport.ts

@@ -0,0 +1,103 @@
+import { writeFile } from "node:fs/promises";
+import type { ReportGroup } from "../BenchmarkReport.ts";
+import type { DefaultCliArgs } from "../cli/CliArgs.ts";
+import type {
+  BenchmarkGroup,
+  BenchmarkJsonData,
+  BenchmarkResult,
+} from "./JsonFormat.ts";
+
+/** Export benchmark results to JSON file */
+export async function exportBenchmarkJson(
+  groups: ReportGroup[],
+  outputPath: string,
+  args: DefaultCliArgs,
+  suiteName = "Benchmark Suite",
+): Promise<void> {
+  const jsonData = prepareJsonData(groups, args, suiteName);
+  const jsonString = JSON.stringify(jsonData, null, 2);
+
+  await writeFile(outputPath, jsonString, "utf-8");
+  console.log(`Benchmark data exported to: ${outputPath}`);
+}
+
+/** Convert ReportGroup data to JSON format */
+function prepareJsonData(
+  groups: ReportGroup[],
+  args: DefaultCliArgs,
+  suiteName: string,
+): BenchmarkJsonData {
+  return {
+    meta: {
+      timestamp: new Date().toISOString(),
+      version: process.env.npm_package_version || "unknown",
+      args: cleanCliArgs(args),
+      environment: {
+        node: process.version,
+        platform: process.platform,
+        arch: process.arch,
+      },
+    },
+    suites: [
+      {
+        name: suiteName,
+        groups: groups.map(convertGroup),
+      },
+    ],
+  };
+}
+
+/** Convert a report group, mapping each report to the JSON result format */
+function convertGroup(group: ReportGroup): BenchmarkGroup {
+  return {
+    name: "Benchmark Group", // Could be enhanced to include actual group names
+    baseline: group.baseline ? convertReport(group.baseline) : undefined,
+    benchmarks: group.reports.map(convertReport),
+  };
+}
+
+/** Extract measured stats and optional metrics into JSON result shape */
+function convertReport(report: any): BenchmarkResult {
+  const { name, measuredResults: m } = report;
+  const { time, heapSize, gcTime, cpu } = m;
+  const minMaxMean = (s: any) =>
+    s ? { min: s.min, max: s.max, mean: s.avg } : undefined;
+
+  return {
+    name,
+    status: "completed",
+    samples: m.samples || [],
+    time: {
+      ...minMaxMean(time)!,
+      p50: time.p50,
+      p75: time.p75,
+      p99: time.p99,
+      p999: time.p999,
+    },
+    heapSize: minMaxMean(heapSize),
+    gcTime: minMaxMean(gcTime),
+    cpu: cpu
+      ? {
+          instructions: cpu.instructions,
+          cycles: cpu.cycles,
+          cacheMisses: m.cpuCacheMiss,
+          branchMisses: cpu.branchMisses,
+        }
+      : undefined,
+    execution: {
+      iterations: m.samples?.length || 0,
+      totalTime: m.totalTime || 0,
+      warmupRuns: undefined, // Not available in current data structure
+    },
+  };
+}
+
+/** Clean CLI args for JSON export (remove undefined values) */
+function cleanCliArgs(args: DefaultCliArgs): Record<string, any> {
+  const toCamel = (k: string) =>
+    k.replace(/-([a-z])/g, (_, l) => l.toUpperCase());
+  const entries = Object.entries(args)
+    .filter(([, v]) => v !== undefined && v !== null)
+    .map(([k, v]) => [toCamel(k), v]);
+  return Object.fromEntries(entries);
+}

package/src/export/JsonFormat.ts

@@ -0,0 +1,91 @@
+/** Complete benchmark data structure for JSON export */
+export interface BenchmarkJsonData {
+  meta: {
+    timestamp: string;
+    version: string;
+    args: Record<string, any>;
+    environment: {
+      node: string;
+      platform: string;
+      arch?: string;
+    };
+  };
+  suites: BenchmarkSuite[];
+}
+
+export interface BenchmarkSuite {
+  name: string;
+  groups: BenchmarkGroup[];
+}
+
+export interface BenchmarkGroup {
+  name: string;
+  baseline?: BenchmarkResult;
+  benchmarks: BenchmarkResult[];
+}
+
+export interface BenchmarkResult {
+  name: string;
+  status: "completed" | "running" | "failed";
+
+  /** Raw execution time samples in milliseconds */
+  samples: number[];
+
+  /** Statistical summaries */
+  time: {
+    min: number;
+    max: number;
+    mean: number;
+    p50: number;
+    p75: number;
+    p99: number;
+    p999: number;
+  };
+
+  /** Optional performance metrics */
+  heapSize?: {
+    min: number;
+    max: number;
+    mean: number;
+  };
+
+  gcTime?: {
+    min: number;
+    max: number;
+    mean: number;
+  };
+
+  cpu?: {
+    instructions?: number;
+    cycles?: number;
+    cacheMisses?: number;
+    branchMisses?: number;
+  };
+
+  /** Execution metadata */
+  execution: {
+    iterations: number;
+    totalTime: number;
+    warmupRuns?: number;
+  };
+
+  /** Adaptive mode results */
+  adaptive?: {
+    confidenceInterval: {
+      lower: number;
+      upper: number;
+      margin: number;
+      marginPercent: number;
+      confidence: number;
+    };
+    converged: boolean;
+    stopReason: "threshold_met" | "max_time" | "max_iterations";
+  };
+
+  /** Error information */
+  error?: {
+    message: string;
+    type: string;
+    stackTrace?: string;
+  };
+}

package/src/export/PerfettoExport.ts

@@ -0,0 +1,203 @@
+import { spawn } from "node:child_process";
+import { readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import type { ReportGroup } from "../BenchmarkReport.ts";
+import type { DefaultCliArgs } from "../cli/CliArgs.ts";
+import type { MeasuredResults } from "../MeasuredResults.ts";
+
+/** Chrome Trace Event format event */
+interface TraceEvent {
+  ph: string; // event type: M=metadata, C=counter, i=instant, B/E=begin/end
+  ts: number; // timestamp in microseconds
+  pid?: number;
+  tid?: number;
+  cat?: string;
+  name: string;
+  args?: Record<string, unknown>;
+  s?: string; // scope for instant events: "t"=thread, "p"=process, "g"=global
+  dur?: number; // duration for complete events
+}
+
+/** Chrome Trace Event format file structure */
+interface TraceFile {
+  traceEvents: TraceEvent[];
+}
+
+const pid = 1;
+const tid = 1;
+
+/** Export benchmark results to Perfetto-compatible trace file */
+export function exportPerfettoTrace(
+  groups: ReportGroup[],
+  outputPath: string,
+  args: DefaultCliArgs,
+): void {
+  const absPath = resolve(outputPath);
+  const events = buildTraceEvents(groups, args);
+
+  // Try to merge any existing V8 trace from a previous run
+  const merged = mergeV8Trace(events);
+  writeTraceFile(absPath, merged);
+  console.log(`Perfetto trace exported to: ${outputPath}`);
+
+  // V8 writes trace files after process exit, so spawn a child to merge later
+  scheduleDeferredMerge(absPath);
+}
+
+/** Build trace events from benchmark results */
+function buildTraceEvents(
+  groups: ReportGroup[],
+  args: DefaultCliArgs,
+): TraceEvent[] {
+  const meta = (name: string, a: Record<string, unknown>): TraceEvent => ({
+    ph: "M",
+    ts: 0,
+    pid,
+    tid,
+    name,
+    args: a,
+  });
+  const events: TraceEvent[] = [
+    meta("process_name", { name: "wesl-bench" }),
+    meta("thread_name", { name: "MainThread" }),
+    meta("bench_settings", cleanArgs(args)),
+  ];
+
+  for (const group of groups) {
+    for (const report of group.reports) {
+      const results = report.measuredResults as MeasuredResults;
+      events.push(...buildBenchmarkEvents(results));
+    }
+  }
+
+  return events;
+}
+
+function instant(
+  ts: number,
+  name: string,
+  args: Record<string, unknown>,
+): TraceEvent {
+  return { ph: "i", ts, pid, tid, cat: "bench", name, s: "t", args };
+}
+
+function counter(
+  ts: number,
+  name: string,
+  args: Record<string, unknown>,
+): TraceEvent {
+  return { ph: "C", ts, pid, tid, cat: "bench", name, args };
+}
+
+/** Build events for a single benchmark run */
+function buildBenchmarkEvents(results: MeasuredResults): TraceEvent[] {
+  const { samples, heapSamples, timestamps, pausePoints } = results;
+  if (!timestamps?.length) return [];
+
+  const events: TraceEvent[] = [];
+  for (let i = 0; i < samples.length; i++) {
+    const ts = timestamps[i];
+    const ms = Math.round(samples[i] * 100) / 100;
+    events.push(instant(ts, results.name, { n: i, ms }));
+    events.push(counter(ts, "duration", { ms }));
+    if (heapSamples?.[i] !== undefined) {
+      const MB = Math.round((heapSamples[i] / 1024 / 1024) * 10) / 10;
+      events.push(counter(ts, "heap", { MB }));
+    }
+  }
+
+  for (const pause of pausePoints ?? []) {
+    const ts = timestamps[pause.sampleIndex];
+    if (ts) events.push(instant(ts, "pause", { ms: pause.durationMs }));
+  }
+  return events;
+}
+
+/** Normalize timestamps so events start at 0 */
+function normalizeTimestamps(events: TraceEvent[]): void {
+  const times = events.filter(e => e.ts > 0).map(e => e.ts);
+  if (times.length === 0) return;
+  const minTs = Math.min(...times);
+  for (const e of events) if (e.ts > 0) e.ts -= minTs;
+}
+
+/** Merge V8 trace events from a previous run, aligning timestamps */
+function mergeV8Trace(customEvents: TraceEvent[]): TraceEvent[] {
+  const traceFiles = readdirSync(".").filter(
+    f => f.startsWith("node_trace.") && f.endsWith(".log"),
+  );
+
+  const v8Events = loadV8Events(traceFiles[0]);
+  normalizeTimestamps(customEvents);
+  if (!v8Events) return customEvents;
+
+  normalizeTimestamps(v8Events);
+  return [...v8Events, ...customEvents];
+}
+
+/** Load V8 trace events from file, or undefined if unavailable */
+function loadV8Events(
+  v8TracePath: string | undefined,
+): TraceEvent[] | undefined {
+  if (!v8TracePath) return undefined;
+  try {
+    const v8Data = JSON.parse(readFileSync(v8TracePath, "utf-8")) as TraceFile;
+    console.log(
+      `Merged ${v8Data.traceEvents.length} V8 events from ${v8TracePath}`,
+    );
+    return v8Data.traceEvents;
+  } catch {
+    console.warn(`Could not parse V8 trace file: ${v8TracePath}`);
+    return undefined;
+  }
+}
+
+/** Write trace events to JSON file */
+function writeTraceFile(outputPath: string, events: TraceEvent[]): void {
+  const traceFile: TraceFile = { traceEvents: events };
+  writeFileSync(outputPath, JSON.stringify(traceFile));
+}
+
+/** Clean CLI args for metadata */
+function cleanArgs(args: DefaultCliArgs): Record<string, unknown> {
+  const skip = new Set(["_", "$0"]);
+  const entries = Object.entries(args).filter(
+    ([k, v]) => v !== undefined && !skip.has(k),
+  );
+  return Object.fromEntries(entries);
+}
+
+/** Spawn a detached child to merge V8 trace after process exit */
+function scheduleDeferredMerge(outputPath: string): void {
+  const cwd = process.cwd();
+  const mergeScript = `
+    const { readdirSync, readFileSync, writeFileSync } = require('fs');
+    function normalize(events) {
+      const times = events.filter(e => e.ts > 0).map(e => e.ts);
+      if (!times.length) return;
+      const min = Math.min(...times);
+      for (const e of events) if (e.ts > 0) e.ts -= min;
+    }
+    setTimeout(() => {
+      const traceFiles = readdirSync('.').filter(f => f.startsWith('node_trace.') && f.endsWith('.log'));
+      if (traceFiles.length === 0) process.exit(0);
+      try {
+        const v8Data = JSON.parse(readFileSync(traceFiles[0], 'utf-8'));
+        const ourData = JSON.parse(readFileSync('${outputPath}', 'utf-8'));
+        normalize(v8Data.traceEvents);
+        const merged = { traceEvents: [...v8Data.traceEvents, ...ourData.traceEvents] };
+        writeFileSync('${outputPath}', JSON.stringify(merged));
+        console.log('Merged ' + v8Data.traceEvents.length + ' V8 events into ' + '${outputPath}');
+      } catch (e) { console.error('Merge failed:', e.message); }
+    }, 100);
+  `;
+
+  process.on("exit", () => {
+    const child = spawn("node", ["-e", mergeScript], {
+      detached: true,
+      stdio: "inherit",
+      cwd,
+    });
+    child.unref();
+  });
+}

package/src/heap-sample/HeapSampleReport.ts

@@ -0,0 +1,196 @@
+import pc from "picocolors";
+import type { HeapProfile, ProfileNode } from "./HeapSampler.ts";
+
+/** Sum selfSize across all nodes in profile (before any filtering) */
+export function totalProfileBytes(profile: HeapProfile): number {
+  let total = 0;
+  function walk(node: ProfileNode): void {
+    total += node.selfSize;
+    for (const child of node.children || []) walk(child);
+  }
+  walk(profile.head);
+  return total;
+}
+
+export interface CallFrame {
+  fn: string;
+  url: string;
+  line: number; // 1-indexed for display
+  col: number;
+}
+
+export interface HeapSite {
+  fn: string;
+  url: string;
+  line: number; // 1-indexed for display
+  col: number;
+  bytes: number;
+  stack?: CallFrame[]; // call stack from root to this frame
+}
+
+/** Flatten profile tree into sorted list of allocation sites with call stacks */
+export function flattenProfile(profile: HeapProfile): HeapSite[] {
+  const sites: HeapSite[] = [];
+
+  function walk(node: ProfileNode, stack: CallFrame[]): void {
+    const { functionName, url, lineNumber, columnNumber } = node.callFrame;
+    const fn = functionName || "(anonymous)";
+    const col = columnNumber ?? 0;
+    const frame: CallFrame = { fn, url: url || "", line: lineNumber + 1, col };
+    const newStack = [...stack, frame];
+
+    if (node.selfSize > 0) {
+      sites.push({
+        ...frame,
+        bytes: node.selfSize,
+        stack: newStack,
+      });
+    }
+    for (const child of node.children || []) walk(child, newStack);
+  }
+
+  walk(profile.head, []);
+  return sites.sort((a, b) => b.bytes - a.bytes);
+}
+
+export type UserCodeFilter = (site: CallFrame) => boolean;
+
+/** Check if site is user code (not node internals) */
+export function isNodeUserCode(site: CallFrame): boolean {
+  if (!site.url) return false;
+  if (site.url.startsWith("node:")) return false;
+  if (site.url.includes("(native)")) return false;
+  if (site.url.includes("internal/")) return false;
+  return true;
+}
+
+/** Check if site is user code (not browser internals) */
+export function isBrowserUserCode(site: CallFrame): boolean {
+  if (!site.url) return false;
+  if (site.url.startsWith("chrome-extension://")) return false;
+  if (site.url.startsWith("devtools://")) return false;
+  if (site.url.includes("(native)")) return false;
+  return true;
+}
+
+/** Filter sites to user code only */
+export function filterSites(
+  sites: HeapSite[],
+  isUser: UserCodeFilter = isNodeUserCode,
+): HeapSite[] {
+  return sites.filter(isUser);
+}
+
+/** Aggregate sites by location (combine same file:line:col) */
+export function aggregateSites(sites: HeapSite[]): HeapSite[] {
+  const byLocation = new Map<string, HeapSite>();
+
+  for (const site of sites) {
+    const key = `${site.url}:${site.line}:${site.col}`;
+    const existing = byLocation.get(key);
+    if (existing) {
+      existing.bytes += site.bytes;
+    } else {
+      byLocation.set(key, { ...site });
+    }
+  }
+
+  return [...byLocation.values()].sort((a, b) => b.bytes - a.bytes);
+}
+
+function fmtBytes(bytes: number): string {
+  if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)} MB`;
+  if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${bytes} B`;
+}
+
+export interface HeapReportOptions {
+  topN: number;
+  stackDepth?: number;
+  verbose?: boolean;
+  userOnly?: boolean; // filter to user code only (hide node internals)
+  isUserCode?: UserCodeFilter; // predicate for user vs internal code
+  totalAll?: number; // total across all nodes (before filtering)
+  totalUserCode?: number; // total for user code only
+  sampleCount?: number; // number of samples taken
+}
+
+/** Format heap report for console output */
+export function formatHeapReport(
+  sites: HeapSite[],
+  options: HeapReportOptions,
+): string {
+  const { topN, stackDepth = 3, verbose = false } = options;
+  const { totalAll, totalUserCode, sampleCount } = options;
+  const isUser = options.isUserCode ?? isNodeUserCode;
+  const lines: string[] = [];
+  lines.push(`Heap allocation sites (top ${topN}, garbage included):`);
+
+  for (const site of sites.slice(0, topN)) {
+    if (verbose) {
+      formatVerboseSite(lines, site, stackDepth, isUser);
+    } else {
+      formatCompactSite(lines, site, stackDepth, isUser);
+    }
+  }
+
+  lines.push("");
+  if (totalAll !== undefined)
+    lines.push(`Total (all):       ${fmtBytes(totalAll)}`);
+  if (totalUserCode !== undefined)
+    lines.push(`Total (user-code): ${fmtBytes(totalUserCode)}`);
+  if (sampleCount !== undefined)
+    lines.push(`Samples: ${sampleCount.toLocaleString()}`);
+
+  return lines.join("\n");
+}
+
+/** Compact single-line format: `49 MB  fn1 <- fn2 <- fn3` */
+function formatCompactSite(
+  lines: string[],
+  site: HeapSite,
+  stackDepth: number,
+  isUser: UserCodeFilter,
+): void {
+  const bytes = fmtBytes(site.bytes).padStart(10);
+  const fns = [site.fn];
+
+  if (site.stack && site.stack.length > 1) {
+    const callers = site.stack.slice(0, -1).reverse().slice(0, stackDepth);
+    for (const frame of callers) {
+      if (!frame.url || !isUser(frame)) continue;
+      fns.push(frame.fn);
+    }
+  }
+
+  const line = `${bytes}  ${fns.join(" <- ")}`;
+  lines.push(isUser(site) ? line : pc.dim(line));
+}
+
+/** Verbose multi-line format with file:// paths and line numbers */
+function formatVerboseSite(
+  lines: string[],
+  site: HeapSite,
+  stackDepth: number,
+  isUser: UserCodeFilter,
+): void {
+  const bytes = fmtBytes(site.bytes).padStart(10);
+  const loc = site.url ? `${site.url}:${site.line}:${site.col}` : "(unknown)";
+  const dimFn = isUser(site) ? (s: string) => s : pc.dim;
+
+  lines.push(dimFn(`${bytes}  ${site.fn}  ${loc}`));
+
+  if (site.stack && site.stack.length > 1) {
+    const callers = site.stack.slice(0, -1).reverse().slice(0, stackDepth);
+    for (const frame of callers) {
+      if (!frame.url || !isUser(frame)) continue;
+      const callerLoc = `${frame.url}:${frame.line}:${frame.col}`;
+      lines.push(dimFn(`            <- ${frame.fn}  ${callerLoc}`));
+    }
+  }
+}
+
+/** Get total bytes from sites */
+export function totalBytes(sites: HeapSite[]): number {
+  return sites.reduce((sum, s) => sum + s.bytes, 0);
+}

package/src/heap-sample/HeapSampler.ts

@@ -0,0 +1,78 @@
+import { Session } from "node:inspector/promises";
+
+export interface HeapSampleOptions {
+  samplingInterval?: number; // bytes between samples, default 32768
+  stackDepth?: number; // max stack frames, default 64
+  includeMinorGC?: boolean; // keep objects collected by minor GC, default true
+  includeMajorGC?: boolean; // keep objects collected by major GC, default true
+}
+
+export interface ProfileNode {
+  callFrame: {
+    functionName: string;
+    url: string;
+    lineNumber: number;
+    columnNumber?: number;
+  };
+  selfSize: number;
+  children?: ProfileNode[];
+}
+
+export interface HeapProfile {
+  head: ProfileNode;
+  samples?: number[]; // sample IDs (length = number of samples taken)
+}
+
+const defaultOptions: Required<HeapSampleOptions> = {
+  samplingInterval: 32768,
+  stackDepth: 64,
+  includeMinorGC: true,
+  includeMajorGC: true,
+};
+
+/** Run a function while sampling heap allocations, return profile */
+export async function withHeapSampling<T>(
+  options: HeapSampleOptions,
+  fn: () => Promise<T> | T,
+): Promise<{ result: T; profile: HeapProfile }> {
+  const opts = { ...defaultOptions, ...options };
+  const session = new Session();
+  session.connect();
+
+  try {
+    await startSampling(session, opts);
+    const result = await fn();
+    const profile = await stopSampling(session);
+    return { result, profile };
+  } finally {
+    session.disconnect();
+  }
+}
+
+/** Start heap sampling, falling back if include-collected params aren't supported */
+async function startSampling(
+  session: Session,
+  opts: Required<HeapSampleOptions>,
+): Promise<void> {
+  const { samplingInterval, stackDepth } = opts;
+  const base = { samplingInterval, stackDepth };
+  const params = {
+    ...base,
+    includeObjectsCollectedByMinorGC: opts.includeMinorGC,
+    includeObjectsCollectedByMajorGC: opts.includeMajorGC,
+  };
+
+  try {
+    await session.post("HeapProfiler.startSampling", params);
+  } catch {
+    console.warn(
+      "HeapProfiler: include-collected params not supported, falling back",
+    );
+    await session.post("HeapProfiler.startSampling", base);
+  }
+}
+
+async function stopSampling(session: Session): Promise<HeapProfile> {
+  const { profile } = await session.post("HeapProfiler.stopSampling");
+  return profile as HeapProfile;
+}