benchforge 0.1.9 → 0.2.4

package/src/browser/BrowserHeapSampler.ts

@@ -1,271 +0,0 @@
-import {
-  type BrowserServer,
-  type CDPSession,
-  chromium,
-  type Page,
-} from "playwright";
-import type {
-  HeapProfile,
-  HeapSampleOptions,
-} from "../heap-sample/HeapSampler.ts";
-import type { GcStats } from "../runners/GcStats.ts";
-import { browserGcStats, type TraceEvent } from "./BrowserGcStats.ts";
-
-export interface BrowserProfileParams {
-  url: string;
-  heapSample?: boolean;
-  heapOptions?: HeapSampleOptions;
-  gcStats?: boolean;
-  headless?: boolean;
-  chromeArgs?: string[];
-  timeout?: number; // seconds
-  maxTime?: number; // ms, bench function iteration time limit
-  maxIterations?: number; // exact iteration count (bench function mode)
-}
-
-export interface BrowserProfileResult {
-  heapProfile?: HeapProfile;
-  gcStats?: GcStats;
-  /** Wall-clock ms (lap mode: first start to done, bench function: total loop) */
-  wallTimeMs?: number;
-  /** Per-iteration timing samples (ms) from bench function or lap mode */
-  samples?: number[];
-}
-
-interface LapModeHandle {
-  promise: Promise<BrowserProfileResult>;
-  cancel: () => void;
-}
-
-/** Run browser benchmark, auto-detecting page API mode.
- *  Bench function (window.__bench): CLI controls iteration and timing.
- *  Lap mode (__start/__lap/__done): page controls the measured region. */
-export async function profileBrowser(
-  params: BrowserProfileParams,
-): Promise<BrowserProfileResult> {
-  const { url, headless = true, chromeArgs, timeout = 60 } = params;
-  const { gcStats: collectGc } = params;
-  const { samplingInterval = 32768 } = params.heapOptions ?? {};
-
-  const server = await chromium.launchServer({ headless, args: chromeArgs });
-  pipeChromeOutput(server);
-  const browser = await chromium.connect(server.wsEndpoint());
-  try {
-    const page = await browser.newPage();
-    page.setDefaultTimeout(timeout * 1000);
-    const cdp = await page.context().newCDPSession(page);
-
-    const pageErrors: string[] = [];
-    page.on("pageerror", err => pageErrors.push(err.message));
-
-    const traceEvents = collectGc ? await startGcTracing(cdp) : [];
-    const lapMode = await setupLapMode(
-      page,
-      cdp,
-      params,
-      samplingInterval,
-      timeout,
-      pageErrors,
-    );
-
-    await page.goto(url, { waitUntil: "load" });
-    const hasBench = await page.evaluate(
-      () => typeof (globalThis as any).__bench === "function",
-    );
-
-    let result: BrowserProfileResult;
-    if (hasBench) {
-      lapMode.cancel();
-      lapMode.promise.catch(() => {}); // suppress unused rejection
-      result = await runBenchLoop(page, cdp, params, samplingInterval);
-    } else {
-      result = await lapMode.promise;
-      lapMode.cancel();
-    }
-
-    if (collectGc) {
-      result = { ...result, gcStats: await collectTracing(cdp, traceEvents) };
-    }
-    return result;
-  } finally {
-    await browser.close();
-    await server.close();
-  }
-}
-
-/** Inject __start/__lap as in-page functions, expose __done for results collection.
- *  __start/__lap are pure in-page (zero CDP overhead). First __start() triggers
- *  instrument start. __done() stops instruments and collects timing data. */
-async function setupLapMode(
-  page: Page,
-  cdp: CDPSession,
-  params: BrowserProfileParams,
-  samplingInterval: number,
-  timeout: number,
-  pageErrors: string[],
-): Promise<LapModeHandle> {
-  const { heapSample } = params;
-  const { promise, resolve, reject } =
-    Promise.withResolvers<BrowserProfileResult>();
-  let instrumentsStarted = false;
-
-  await page.exposeFunction("__benchInstrumentStart", async () => {
-    if (instrumentsStarted) return;
-    instrumentsStarted = true;
-    if (heapSample) {
-      await cdp.send(
-        "HeapProfiler.startSampling",
-        heapSamplingParams(samplingInterval),
-      );
-    }
-  });
-
-  await page.exposeFunction(
-    "__benchCollect",
-    async (samples: number[], wallTimeMs: number) => {
-      let heapProfile: HeapProfile | undefined;
-      if (heapSample && instrumentsStarted) {
-        const result = await cdp.send("HeapProfiler.stopSampling");
-        heapProfile = result.profile as unknown as HeapProfile;
-      }
-      resolve({ samples, heapProfile, wallTimeMs });
-    },
-  );
-
-  await page.addInitScript(injectLapFunctions);
-
-  const timer = setTimeout(() => {
-    const lines = [`Timed out after ${timeout}s`];
-    if (pageErrors.length) {
-      lines.push("Page JS errors:", ...pageErrors.map(e => `  ${e}`));
-    } else {
-      lines.push("Page did not call __done() or define window.__bench");
-    }
-    reject(new Error(lines.join("\n")));
-  }, timeout * 1000);
-
-  return { promise, cancel: () => clearTimeout(timer) };
-}
-
-/** In-page timing functions injected via addInitScript (zero CDP overhead).
- *  __start/__lap collect timestamps, __done delegates to exposed __benchCollect. */
-function injectLapFunctions(): void {
-  const g = globalThis as any;
-  g.__benchSamples = [];
-  g.__benchLastTime = 0;
-  g.__benchFirstStart = 0;
-
-  g.__start = () => {
-    const now = performance.now();
-    g.__benchLastTime = now;
-    if (!g.__benchFirstStart) {
-      g.__benchFirstStart = now;
-      return g.__benchInstrumentStart();
-    }
-  };
-
-  g.__lap = () => {
-    const now = performance.now();
-    g.__benchSamples.push(now - g.__benchLastTime);
-    g.__benchLastTime = now;
-  };
-
-  g.__done = () => {
-    const wall = g.__benchFirstStart
-      ? performance.now() - g.__benchFirstStart
-      : 0;
-    return g.__benchCollect(g.__benchSamples.slice(), wall);
-  };
-}
-
-function heapSamplingParams(samplingInterval: number) {
-  return {
-    samplingInterval,
-    includeObjectsCollectedByMajorGC: true,
-    includeObjectsCollectedByMinorGC: true,
-  };
-}
-
-/** Start CDP GC tracing, returns the event collector array. */
-async function startGcTracing(cdp: CDPSession): Promise<TraceEvent[]> {
-  const events: TraceEvent[] = [];
-  cdp.on("Tracing.dataCollected", ({ value }) => {
-    for (const e of value) events.push(e as unknown as TraceEvent);
-  });
-  await cdp.send("Tracing.start", {
-    traceConfig: { includedCategories: ["v8", "v8.gc"] },
-  });
-  return events;
-}
-
-/** Bench function mode: run window.__bench in a timed iteration loop. */
-async function runBenchLoop(
-  page: Page,
-  cdp: CDPSession,
-  params: BrowserProfileParams,
-  samplingInterval: number,
-): Promise<BrowserProfileResult> {
-  const { heapSample } = params;
-  const maxTime = params.maxTime ?? 642;
-  const maxIter = params.maxIterations ?? Number.MAX_SAFE_INTEGER;
-
-  if (heapSample) {
-    await cdp.send(
-      "HeapProfiler.startSampling",
-      heapSamplingParams(samplingInterval),
-    );
-  }
-
-  const { samples, totalMs } = await page.evaluate(
-    async ({ maxTime, maxIter }) => {
-      const bench = (globalThis as any).__bench;
-      const samples: number[] = [];
-      const startAll = performance.now();
-      const deadline = startAll + maxTime;
-      for (let i = 0; i < maxIter && performance.now() < deadline; i++) {
-        const t0 = performance.now();
-        await bench();
-        samples.push(performance.now() - t0);
-      }
-      return { samples, totalMs: performance.now() - startAll };
-    },
-    { maxTime, maxIter },
-  );
-
-  let heapProfile: HeapProfile | undefined;
-  if (heapSample) {
-    const result = await cdp.send("HeapProfiler.stopSampling");
-    heapProfile = result.profile as unknown as HeapProfile;
-  }
-
-  return { samples, heapProfile, wallTimeMs: totalMs };
-}
-
-/** Stop CDP tracing and parse GC events into GcStats. */
-async function collectTracing(
-  cdp: CDPSession,
-  traceEvents: TraceEvent[],
-): Promise<GcStats> {
-  const complete = new Promise<void>(resolve =>
-    cdp.once("Tracing.tracingComplete", () => resolve()),
-  );
-  await cdp.send("Tracing.end");
-  await complete;
-  return browserGcStats(traceEvents);
-}
-
-/** Forward Chrome's stdout/stderr to the terminal so V8 flag output is visible. */
-function pipeChromeOutput(server: BrowserServer): void {
-  const proc = server.process();
-  const pipe = (stream: NodeJS.ReadableStream | null) =>
-    stream?.on("data", (chunk: Buffer) => {
-      for (const line of chunk.toString().split("\n")) {
-        const text = line.trim();
-        if (text) process.stderr.write(`[chrome] ${text}\n`);
-      }
-    });
-  pipe(proc.stdout);
-  pipe(proc.stderr);
-}
-
-export { profileBrowser as profileBrowserHeap };

package/src/export/JsonExport.ts

@@ -1,103 +0,0 @@
-import { writeFile } from "node:fs/promises";
-import type { ReportGroup } from "../BenchmarkReport.ts";
-import type { DefaultCliArgs } from "../cli/CliArgs.ts";
-import type {
-  BenchmarkGroup,
-  BenchmarkJsonData,
-  BenchmarkResult,
-} from "./JsonFormat.ts";
-
-/** Export benchmark results to JSON file */
-export async function exportBenchmarkJson(
-  groups: ReportGroup[],
-  outputPath: string,
-  args: DefaultCliArgs,
-  suiteName = "Benchmark Suite",
-): Promise<void> {
-  const jsonData = prepareJsonData(groups, args, suiteName);
-  const jsonString = JSON.stringify(jsonData, null, 2);
-
-  await writeFile(outputPath, jsonString, "utf-8");
-  console.log(`Benchmark data exported to: ${outputPath}`);
-}
-
-/** Convert ReportGroup data to JSON format */
-function prepareJsonData(
-  groups: ReportGroup[],
-  args: DefaultCliArgs,
-  suiteName: string,
-): BenchmarkJsonData {
-  return {
-    meta: {
-      timestamp: new Date().toISOString(),
-      version: process.env.npm_package_version || "unknown",
-      args: cleanCliArgs(args),
-      environment: {
-        node: process.version,
-        platform: process.platform,
-        arch: process.arch,
-      },
-    },
-    suites: [
-      {
-        name: suiteName,
-        groups: groups.map(convertGroup),
-      },
-    ],
-  };
-}
-
-/** Convert a report group, mapping each report to the JSON result format */
-function convertGroup(group: ReportGroup): BenchmarkGroup {
-  return {
-    name: "Benchmark Group", // Could be enhanced to include actual group names
-    baseline: group.baseline ? convertReport(group.baseline) : undefined,
-    benchmarks: group.reports.map(convertReport),
-  };
-}
-
-/** Extract measured stats and optional metrics into JSON result shape */
-function convertReport(report: any): BenchmarkResult {
-  const { name, measuredResults: m } = report;
-  const { time, heapSize, gcTime, cpu } = m;
-  const minMaxMean = (s: any) =>
-    s ? { min: s.min, max: s.max, mean: s.avg } : undefined;
-
-  return {
-    name,
-    status: "completed",
-    samples: m.samples || [],
-    time: {
-      ...minMaxMean(time)!,
-      p50: time.p50,
-      p75: time.p75,
-      p99: time.p99,
-      p999: time.p999,
-    },
-    heapSize: minMaxMean(heapSize),
-    gcTime: minMaxMean(gcTime),
-    cpu: cpu
-      ? {
-          instructions: cpu.instructions,
-          cycles: cpu.cycles,
-          cacheMisses: m.cpuCacheMiss,
-          branchMisses: cpu.branchMisses,
-        }
-      : undefined,
-    execution: {
-      iterations: m.samples?.length || 0,
-      totalTime: m.totalTime || 0,
-      warmupRuns: undefined, // Not available in current data structure
-    },
-  };
-}
-
-/** Clean CLI args for JSON export (remove undefined values) */
-function cleanCliArgs(args: DefaultCliArgs): Record<string, any> {
-  const toCamel = (k: string) =>
-    k.replace(/-([a-z])/g, (_, l) => l.toUpperCase());
-  const entries = Object.entries(args)
-    .filter(([, v]) => v !== undefined && v !== null)
-    .map(([k, v]) => [toCamel(k), v]);
-  return Object.fromEntries(entries);
-}

package/src/export/JsonFormat.ts

@@ -1,91 +0,0 @@
-/** Complete benchmark data structure for JSON export */
-export interface BenchmarkJsonData {
-  meta: {
-    timestamp: string;
-    version: string;
-    args: Record<string, any>;
-    environment: {
-      node: string;
-      platform: string;
-      arch?: string;
-    };
-  };
-  suites: BenchmarkSuite[];
-}
-
-export interface BenchmarkSuite {
-  name: string;
-  groups: BenchmarkGroup[];
-}
-
-export interface BenchmarkGroup {
-  name: string;
-  baseline?: BenchmarkResult;
-  benchmarks: BenchmarkResult[];
-}
-
-export interface BenchmarkResult {
-  name: string;
-  status: "completed" | "running" | "failed";
-
-  /** Raw execution time samples in milliseconds */
-  samples: number[];
-
-  /** Statistical summaries */
-  time: {
-    min: number;
-    max: number;
-    mean: number;
-    p50: number;
-    p75: number;
-    p99: number;
-    p999: number;
-  };
-
-  /** Optional performance metrics */
-  heapSize?: {
-    min: number;
-    max: number;
-    mean: number;
-  };
-
-  gcTime?: {
-    min: number;
-    max: number;
-    mean: number;
-  };
-
-  cpu?: {
-    instructions?: number;
-    cycles?: number;
-    cacheMisses?: number;
-    branchMisses?: number;
-  };
-
-  /** Execution metadata */
-  execution: {
-    iterations: number;
-    totalTime: number;
-    warmupRuns?: number;
-  };
-
-  /** Adaptive mode results */
-  adaptive?: {
-    confidenceInterval: {
-      lower: number;
-      upper: number;
-      margin: number;
-      marginPercent: number;
-      confidence: number;
-    };
-    converged: boolean;
-    stopReason: "threshold_met" | "max_time" | "max_iterations";
-  };
-
-  /** Error information */
-  error?: {
-    message: string;
-    type: string;
-    stackTrace?: string;
-  };
-}

package/src/heap-sample/HeapSampleReport.ts

@@ -1,196 +0,0 @@
-import pc from "picocolors";
-import type { HeapProfile, ProfileNode } from "./HeapSampler.ts";
-
-/** Sum selfSize across all nodes in profile (before any filtering) */
-export function totalProfileBytes(profile: HeapProfile): number {
-  let total = 0;
-  function walk(node: ProfileNode): void {
-    total += node.selfSize;
-    for (const child of node.children || []) walk(child);
-  }
-  walk(profile.head);
-  return total;
-}
-
-export interface CallFrame {
-  fn: string;
-  url: string;
-  line: number; // 1-indexed for display
-  col: number;
-}
-
-export interface HeapSite {
-  fn: string;
-  url: string;
-  line: number; // 1-indexed for display
-  col: number;
-  bytes: number;
-  stack?: CallFrame[]; // call stack from root to this frame
-}
-
-/** Flatten profile tree into sorted list of allocation sites with call stacks */
-export function flattenProfile(profile: HeapProfile): HeapSite[] {
-  const sites: HeapSite[] = [];
-
-  function walk(node: ProfileNode, stack: CallFrame[]): void {
-    const { functionName, url, lineNumber, columnNumber } = node.callFrame;
-    const fn = functionName || "(anonymous)";
-    const col = columnNumber ?? 0;
-    const frame: CallFrame = { fn, url: url || "", line: lineNumber + 1, col };
-    const newStack = [...stack, frame];
-
-    if (node.selfSize > 0) {
-      sites.push({
-        ...frame,
-        bytes: node.selfSize,
-        stack: newStack,
-      });
-    }
-    for (const child of node.children || []) walk(child, newStack);
-  }
-
-  walk(profile.head, []);
-  return sites.sort((a, b) => b.bytes - a.bytes);
-}
-
-export type UserCodeFilter = (site: CallFrame) => boolean;
-
-/** Check if site is user code (not node internals) */
-export function isNodeUserCode(site: CallFrame): boolean {
-  if (!site.url) return false;
-  if (site.url.startsWith("node:")) return false;
-  if (site.url.includes("(native)")) return false;
-  if (site.url.includes("internal/")) return false;
-  return true;
-}
-
-/** Check if site is user code (not browser internals) */
-export function isBrowserUserCode(site: CallFrame): boolean {
-  if (!site.url) return false;
-  if (site.url.startsWith("chrome-extension://")) return false;
-  if (site.url.startsWith("devtools://")) return false;
-  if (site.url.includes("(native)")) return false;
-  return true;
-}
-
-/** Filter sites to user code only */
-export function filterSites(
-  sites: HeapSite[],
-  isUser: UserCodeFilter = isNodeUserCode,
-): HeapSite[] {
-  return sites.filter(isUser);
-}
-
-/** Aggregate sites by location (combine same file:line:col) */
-export function aggregateSites(sites: HeapSite[]): HeapSite[] {
-  const byLocation = new Map<string, HeapSite>();
-
-  for (const site of sites) {
-    const key = `${site.url}:${site.line}:${site.col}`;
-    const existing = byLocation.get(key);
-    if (existing) {
-      existing.bytes += site.bytes;
-    } else {
-      byLocation.set(key, { ...site });
-    }
-  }
-
-  return [...byLocation.values()].sort((a, b) => b.bytes - a.bytes);
-}
-
-function fmtBytes(bytes: number): string {
-  if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)} MB`;
-  if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`;
-  return `${bytes} B`;
-}
-
-export interface HeapReportOptions {
-  topN: number;
-  stackDepth?: number;
-  verbose?: boolean;
-  userOnly?: boolean; // filter to user code only (hide node internals)
-  isUserCode?: UserCodeFilter; // predicate for user vs internal code
-  totalAll?: number; // total across all nodes (before filtering)
-  totalUserCode?: number; // total for user code only
-  sampleCount?: number; // number of samples taken
-}
-
-/** Format heap report for console output */
-export function formatHeapReport(
-  sites: HeapSite[],
-  options: HeapReportOptions,
-): string {
-  const { topN, stackDepth = 3, verbose = false } = options;
-  const { totalAll, totalUserCode, sampleCount } = options;
-  const isUser = options.isUserCode ?? isNodeUserCode;
-  const lines: string[] = [];
-  lines.push(`Heap allocation sites (top ${topN}, garbage included):`);
-
-  for (const site of sites.slice(0, topN)) {
-    if (verbose) {
-      formatVerboseSite(lines, site, stackDepth, isUser);
-    } else {
-      formatCompactSite(lines, site, stackDepth, isUser);
-    }
-  }
-
-  lines.push("");
-  if (totalAll !== undefined)
-    lines.push(`Total (all):       ${fmtBytes(totalAll)}`);
-  if (totalUserCode !== undefined)
-    lines.push(`Total (user-code): ${fmtBytes(totalUserCode)}`);
-  if (sampleCount !== undefined)
-    lines.push(`Samples: ${sampleCount.toLocaleString()}`);
-
-  return lines.join("\n");
-}
-
-/** Compact single-line format: `49 MB  fn1 <- fn2 <- fn3` */
-function formatCompactSite(
-  lines: string[],
-  site: HeapSite,
-  stackDepth: number,
-  isUser: UserCodeFilter,
-): void {
-  const bytes = fmtBytes(site.bytes).padStart(10);
-  const fns = [site.fn];
-
-  if (site.stack && site.stack.length > 1) {
-    const callers = site.stack.slice(0, -1).reverse().slice(0, stackDepth);
-    for (const frame of callers) {
-      if (!frame.url || !isUser(frame)) continue;
-      fns.push(frame.fn);
-    }
-  }
-
-  const line = `${bytes}  ${fns.join(" <- ")}`;
-  lines.push(isUser(site) ? line : pc.dim(line));
-}
-
-/** Verbose multi-line format with file:// paths and line numbers */
-function formatVerboseSite(
-  lines: string[],
-  site: HeapSite,
-  stackDepth: number,
-  isUser: UserCodeFilter,
-): void {
-  const bytes = fmtBytes(site.bytes).padStart(10);
-  const loc = site.url ? `${site.url}:${site.line}:${site.col}` : "(unknown)";
-  const dimFn = isUser(site) ? (s: string) => s : pc.dim;
-
-  lines.push(dimFn(`${bytes}  ${site.fn}  ${loc}`));
-
-  if (site.stack && site.stack.length > 1) {
-    const callers = site.stack.slice(0, -1).reverse().slice(0, stackDepth);
-    for (const frame of callers) {
-      if (!frame.url || !isUser(frame)) continue;
-      const callerLoc = `${frame.url}:${frame.line}:${frame.col}`;
-      lines.push(dimFn(`            <- ${frame.fn}  ${callerLoc}`));
-    }
-  }
-}
-
-/** Get total bytes from sites */
-export function totalBytes(sites: HeapSite[]): number {
-  return sites.reduce((sum, s) => sum + s.bytes, 0);
-}