benchforge 0.1.11 → 0.2.4

package/src/browser/BrowserHeapSampler.ts

@@ -1,271 +0,0 @@
-import {
-  type BrowserServer,
-  type CDPSession,
-  chromium,
-  type Page,
-} from "playwright";
-import type {
-  HeapProfile,
-  HeapSampleOptions,
-} from "../heap-sample/HeapSampler.ts";
-import type { GcStats } from "../runners/GcStats.ts";
-import { browserGcStats, type TraceEvent } from "./BrowserGcStats.ts";
-
-export interface BrowserProfileParams {
-  url: string;
-  heapSample?: boolean;
-  heapOptions?: HeapSampleOptions;
-  gcStats?: boolean;
-  headless?: boolean;
-  chromeArgs?: string[];
-  timeout?: number; // seconds
-  maxTime?: number; // ms, bench function iteration time limit
-  maxIterations?: number; // exact iteration count (bench function mode)
-}
-
-export interface BrowserProfileResult {
-  heapProfile?: HeapProfile;
-  gcStats?: GcStats;
-  /** Wall-clock ms (lap mode: first start to done, bench function: total loop) */
-  wallTimeMs?: number;
-  /** Per-iteration timing samples (ms) from bench function or lap mode */
-  samples?: number[];
-}
-
-interface LapModeHandle {
-  promise: Promise<BrowserProfileResult>;
-  cancel: () => void;
-}
-
-/** Run browser benchmark, auto-detecting page API mode.
- *  Bench function (window.__bench): CLI controls iteration and timing.
- *  Lap mode (__start/__lap/__done): page controls the measured region. */
-export async function profileBrowser(
-  params: BrowserProfileParams,
-): Promise<BrowserProfileResult> {
-  const { url, headless = true, chromeArgs, timeout = 60 } = params;
-  const { gcStats: collectGc } = params;
-  const { samplingInterval = 32768 } = params.heapOptions ?? {};
-
-  const server = await chromium.launchServer({ headless, args: chromeArgs });
-  pipeChromeOutput(server);
-  const browser = await chromium.connect(server.wsEndpoint());
-  try {
-    const page = await browser.newPage();
-    page.setDefaultTimeout(timeout * 1000);
-    const cdp = await page.context().newCDPSession(page);
-
-    const pageErrors: string[] = [];
-    page.on("pageerror", err => pageErrors.push(err.message));
-
-    const traceEvents = collectGc ? await startGcTracing(cdp) : [];
-    const lapMode = await setupLapMode(
-      page,
-      cdp,
-      params,
-      samplingInterval,
-      timeout,
-      pageErrors,
-    );
-
-    await page.goto(url, { waitUntil: "load" });
-    const hasBench = await page.evaluate(
-      () => typeof (globalThis as any).__bench === "function",
-    );
-
-    let result: BrowserProfileResult;
-    if (hasBench) {
-      lapMode.cancel();
-      lapMode.promise.catch(() => {}); // suppress unused rejection
-      result = await runBenchLoop(page, cdp, params, samplingInterval);
-    } else {
-      result = await lapMode.promise;
-      lapMode.cancel();
-    }
-
-    if (collectGc) {
-      result = { ...result, gcStats: await collectTracing(cdp, traceEvents) };
-    }
-    return result;
-  } finally {
-    await browser.close();
-    await server.close();
-  }
-}
-
-/** Forward Chrome's stdout/stderr to the terminal so V8 flag output is visible. */
-function pipeChromeOutput(server: BrowserServer): void {
-  const proc = server.process();
-  const pipe = (stream: NodeJS.ReadableStream | null) =>
-    stream?.on("data", (chunk: Buffer) => {
-      for (const line of chunk.toString().split("\n")) {
-        const text = line.trim();
-        if (text) process.stderr.write(`[chrome] ${text}\n`);
-      }
-    });
-  pipe(proc.stdout);
-  pipe(proc.stderr);
-}
-
-/** Start CDP GC tracing, returns the event collector array. */
-async function startGcTracing(cdp: CDPSession): Promise<TraceEvent[]> {
-  const events: TraceEvent[] = [];
-  cdp.on("Tracing.dataCollected", ({ value }) => {
-    for (const e of value) events.push(e as unknown as TraceEvent);
-  });
-  await cdp.send("Tracing.start", {
-    traceConfig: { includedCategories: ["v8", "v8.gc"] },
-  });
-  return events;
-}
-
-/** Inject __start/__lap as in-page functions, expose __done for results collection.
- *  __start/__lap are pure in-page (zero CDP overhead). First __start() triggers
- *  instrument start. __done() stops instruments and collects timing data. */
-async function setupLapMode(
-  page: Page,
-  cdp: CDPSession,
-  params: BrowserProfileParams,
-  samplingInterval: number,
-  timeout: number,
-  pageErrors: string[],
-): Promise<LapModeHandle> {
-  const { heapSample } = params;
-  const { promise, resolve, reject } =
-    Promise.withResolvers<BrowserProfileResult>();
-  let instrumentsStarted = false;
-
-  await page.exposeFunction("__benchInstrumentStart", async () => {
-    if (instrumentsStarted) return;
-    instrumentsStarted = true;
-    if (heapSample) {
-      await cdp.send(
-        "HeapProfiler.startSampling",
-        heapSamplingParams(samplingInterval),
-      );
-    }
-  });
-
-  await page.exposeFunction(
-    "__benchCollect",
-    async (samples: number[], wallTimeMs: number) => {
-      let heapProfile: HeapProfile | undefined;
-      if (heapSample && instrumentsStarted) {
-        const result = await cdp.send("HeapProfiler.stopSampling");
-        heapProfile = result.profile as unknown as HeapProfile;
-      }
-      resolve({ samples, heapProfile, wallTimeMs });
-    },
-  );
-
-  await page.addInitScript(injectLapFunctions);
-
-  const timer = setTimeout(() => {
-    const lines = [`Timed out after ${timeout}s`];
-    if (pageErrors.length) {
-      lines.push("Page JS errors:", ...pageErrors.map(e => `  ${e}`));
-    } else {
-      lines.push("Page did not call __done() or define window.__bench");
-    }
-    reject(new Error(lines.join("\n")));
-  }, timeout * 1000);
-
-  return { promise, cancel: () => clearTimeout(timer) };
-}
-
-/** Bench function mode: run window.__bench in a timed iteration loop. */
-async function runBenchLoop(
-  page: Page,
-  cdp: CDPSession,
-  params: BrowserProfileParams,
-  samplingInterval: number,
-): Promise<BrowserProfileResult> {
-  const { heapSample } = params;
-  const maxTime = params.maxTime ?? 642;
-  const maxIter = params.maxIterations ?? Number.MAX_SAFE_INTEGER;
-
-  if (heapSample) {
-    await cdp.send(
-      "HeapProfiler.startSampling",
-      heapSamplingParams(samplingInterval),
-    );
-  }
-
-  const { samples, totalMs } = await page.evaluate(
-    async ({ maxTime, maxIter }) => {
-      const bench = (globalThis as any).__bench;
-      const samples: number[] = [];
-      const startAll = performance.now();
-      const deadline = startAll + maxTime;
-      for (let i = 0; i < maxIter && performance.now() < deadline; i++) {
-        const t0 = performance.now();
-        await bench();
-        samples.push(performance.now() - t0);
-      }
-      return { samples, totalMs: performance.now() - startAll };
-    },
-    { maxTime, maxIter },
-  );
-
-  let heapProfile: HeapProfile | undefined;
-  if (heapSample) {
-    const result = await cdp.send("HeapProfiler.stopSampling");
-    heapProfile = result.profile as unknown as HeapProfile;
-  }
-
-  return { samples, heapProfile, wallTimeMs: totalMs };
-}
-
-/** Stop CDP tracing and parse GC events into GcStats. */
-async function collectTracing(
-  cdp: CDPSession,
-  traceEvents: TraceEvent[],
-): Promise<GcStats> {
-  const complete = new Promise<void>(resolve =>
-    cdp.once("Tracing.tracingComplete", () => resolve()),
-  );
-  await cdp.send("Tracing.end");
-  await complete;
-  return browserGcStats(traceEvents);
-}
-
-function heapSamplingParams(samplingInterval: number) {
-  return {
-    samplingInterval,
-    includeObjectsCollectedByMajorGC: true,
-    includeObjectsCollectedByMinorGC: true,
-  };
-}
-
-/** In-page timing functions injected via addInitScript (zero CDP overhead).
- *  __start/__lap collect timestamps, __done delegates to exposed __benchCollect. */
-function injectLapFunctions(): void {
-  const g = globalThis as any;
-  g.__benchSamples = [];
-  g.__benchLastTime = 0;
-  g.__benchFirstStart = 0;
-
-  g.__start = () => {
-    const now = performance.now();
-    g.__benchLastTime = now;
-    if (!g.__benchFirstStart) {
-      g.__benchFirstStart = now;
-      return g.__benchInstrumentStart();
-    }
-  };
-
-  g.__lap = () => {
-    const now = performance.now();
-    g.__benchSamples.push(now - g.__benchLastTime);
-    g.__benchLastTime = now;
-  };
-
-  g.__done = () => {
-    const wall = g.__benchFirstStart
-      ? performance.now() - g.__benchFirstStart
-      : 0;
-    return g.__benchCollect(g.__benchSamples.slice(), wall);
-  };
-}
-
-export { profileBrowser as profileBrowserHeap };

package/src/export/JsonExport.ts

@@ -1,103 +0,0 @@
-import { writeFile } from "node:fs/promises";
-import type { ReportGroup } from "../BenchmarkReport.ts";
-import type { DefaultCliArgs } from "../cli/CliArgs.ts";
-import type {
-  BenchmarkGroup,
-  BenchmarkJsonData,
-  BenchmarkResult,
-} from "./JsonFormat.ts";
-
-/** Export benchmark results to JSON file */
-export async function exportBenchmarkJson(
-  groups: ReportGroup[],
-  outputPath: string,
-  args: DefaultCliArgs,
-  suiteName = "Benchmark Suite",
-): Promise<void> {
-  const jsonData = prepareJsonData(groups, args, suiteName);
-  const jsonString = JSON.stringify(jsonData, null, 2);
-
-  await writeFile(outputPath, jsonString, "utf-8");
-  console.log(`Benchmark data exported to: ${outputPath}`);
-}
-
-/** Convert ReportGroup data to JSON format */
-function prepareJsonData(
-  groups: ReportGroup[],
-  args: DefaultCliArgs,
-  suiteName: string,
-): BenchmarkJsonData {
-  return {
-    meta: {
-      timestamp: new Date().toISOString(),
-      version: process.env.npm_package_version || "unknown",
-      args: cleanCliArgs(args),
-      environment: {
-        node: process.version,
-        platform: process.platform,
-        arch: process.arch,
-      },
-    },
-    suites: [
-      {
-        name: suiteName,
-        groups: groups.map(convertGroup),
-      },
-    ],
-  };
-}
-
-/** Clean CLI args for JSON export (remove undefined values) */
-function cleanCliArgs(args: DefaultCliArgs): Record<string, any> {
-  const toCamel = (k: string) =>
-    k.replace(/-([a-z])/g, (_, l) => l.toUpperCase());
-  const entries = Object.entries(args)
-    .filter(([, v]) => v !== undefined && v !== null)
-    .map(([k, v]) => [toCamel(k), v]);
-  return Object.fromEntries(entries);
-}
-
-/** Convert a report group, mapping each report to the JSON result format */
-function convertGroup(group: ReportGroup): BenchmarkGroup {
-  return {
-    name: "Benchmark Group", // Could be enhanced to include actual group names
-    baseline: group.baseline ? convertReport(group.baseline) : undefined,
-    benchmarks: group.reports.map(convertReport),
-  };
-}
-
-/** Extract measured stats and optional metrics into JSON result shape */
-function convertReport(report: any): BenchmarkResult {
-  const { name, measuredResults: m } = report;
-  const { time, heapSize, gcTime, cpu } = m;
-  const minMaxMean = (s: any) =>
-    s ? { min: s.min, max: s.max, mean: s.avg } : undefined;
-
-  return {
-    name,
-    status: "completed",
-    samples: m.samples || [],
-    time: {
-      ...minMaxMean(time)!,
-      p50: time.p50,
-      p75: time.p75,
-      p99: time.p99,
-      p999: time.p999,
-    },
-    heapSize: minMaxMean(heapSize),
-    gcTime: minMaxMean(gcTime),
-    cpu: cpu
-      ? {
-          instructions: cpu.instructions,
-          cycles: cpu.cycles,
-          cacheMisses: m.cpuCacheMiss,
-          branchMisses: cpu.branchMisses,
-        }
-      : undefined,
-    execution: {
-      iterations: m.samples?.length || 0,
-      totalTime: m.totalTime || 0,
-      warmupRuns: undefined, // Not available in current data structure
-    },
-  };
-}

package/src/export/JsonFormat.ts

@@ -1,91 +0,0 @@
-/** Complete benchmark data structure for JSON export */
-export interface BenchmarkJsonData {
-  meta: {
-    timestamp: string;
-    version: string;
-    args: Record<string, any>;
-    environment: {
-      node: string;
-      platform: string;
-      arch?: string;
-    };
-  };
-  suites: BenchmarkSuite[];
-}
-
-export interface BenchmarkSuite {
-  name: string;
-  groups: BenchmarkGroup[];
-}
-
-export interface BenchmarkGroup {
-  name: string;
-  baseline?: BenchmarkResult;
-  benchmarks: BenchmarkResult[];
-}
-
-export interface BenchmarkResult {
-  name: string;
-  status: "completed" | "running" | "failed";
-
-  /** Raw execution time samples in milliseconds */
-  samples: number[];
-
-  /** Statistical summaries */
-  time: {
-    min: number;
-    max: number;
-    mean: number;
-    p50: number;
-    p75: number;
-    p99: number;
-    p999: number;
-  };
-
-  /** Optional performance metrics */
-  heapSize?: {
-    min: number;
-    max: number;
-    mean: number;
-  };
-
-  gcTime?: {
-    min: number;
-    max: number;
-    mean: number;
-  };
-
-  cpu?: {
-    instructions?: number;
-    cycles?: number;
-    cacheMisses?: number;
-    branchMisses?: number;
-  };
-
-  /** Execution metadata */
-  execution: {
-    iterations: number;
-    totalTime: number;
-    warmupRuns?: number;
-  };
-
-  /** Adaptive mode results */
-  adaptive?: {
-    confidenceInterval: {
-      lower: number;
-      upper: number;
-      margin: number;
-      marginPercent: number;
-      confidence: number;
-    };
-    converged: boolean;
-    stopReason: "threshold_met" | "max_time" | "max_iterations";
-  };
-
-  /** Error information */
-  error?: {
-    message: string;
-    type: string;
-    stackTrace?: string;
-  };
-}

package/src/export/SpeedscopeExport.ts

@@ -1,202 +0,0 @@
-import { spawn } from "node:child_process";
-import { writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-
-import { groupReports, type ReportGroup } from "../BenchmarkReport.ts";
-import type { HeapProfile } from "../heap-sample/HeapSampler.ts";
-import {
-  type ResolvedFrame,
-  type ResolvedProfile,
-  resolveProfile,
-} from "../heap-sample/ResolvedProfile.ts";
-
-/** speedscope file format (https://www.speedscope.app/file-format-schema.json) */
-interface SpeedscopeFile {
-  $schema: "https://www.speedscope.app/file-format-schema.json";
-  shared: { frames: SpeedscopeFrame[] };
-  profiles: SpeedscopeProfile[];
-  name?: string;
-  exporter?: string;
-}
-
-interface SpeedscopeFrame {
-  name: string;
-  file?: string;
-  line?: number;
-  col?: number;
-}
-
-interface SpeedscopeProfile {
-  type: "sampled";
-  name: string;
-  unit: "bytes";
-  startValue: number;
-  endValue: number;
-  samples: number[][]; // each sample is stack of frame indices
-  weights: number[]; // bytes per sample
-}
-
-/** Export heap profiles from benchmark results to speedscope JSON format.
- *  Creates one speedscope profile per benchmark that has a heapProfile.
- *  @returns resolved output path, or undefined if no profiles were found */
-export function exportSpeedscope(
-  groups: ReportGroup[],
-  outputPath: string,
-): string | undefined {
-  const frames: SpeedscopeFrame[] = [];
-  const frameIndex = new Map<string, number>();
-  const profiles: SpeedscopeProfile[] = [];
-
-  for (const group of groups) {
-    for (const report of groupReports(group)) {
-      const { heapProfile } = report.measuredResults;
-      if (!heapProfile) continue;
-      const resolved = resolveProfile(heapProfile);
-      profiles.push(buildProfile(report.name, resolved, frames, frameIndex));
-    }
-  }
-
-  if (profiles.length === 0) {
-    console.log("No heap profiles to export.");
-    return undefined;
-  }
-
-  const file: SpeedscopeFile = {
-    $schema: "https://www.speedscope.app/file-format-schema.json",
-    shared: { frames },
-    profiles,
-    exporter: "benchforge",
-  };
-
-  const absPath = resolve(outputPath);
-  writeFileSync(absPath, JSON.stringify(file));
-  console.log(`Speedscope profile exported to: ${outputPath}`);
-  return absPath;
-}
-
-/** Export to a temp file and open in speedscope via npx */
-export function exportAndLaunchSpeedscope(groups: ReportGroup[]): void {
-  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
-  const outputPath = join(tmpdir(), `benchforge-${timestamp}.speedscope.json`);
-  const absPath = exportSpeedscope(groups, outputPath);
-  if (absPath) {
-    launchSpeedscope(absPath);
-  }
-}
-
-/** Launch speedscope viewer on a file via npx */
-export function launchSpeedscope(filePath: string): void {
-  console.log("Opening speedscope...");
-  const child = spawn("npx", ["speedscope", filePath], {
-    detached: true,
-    stdio: "ignore",
-  });
-  child.unref();
-  child.on("error", () => {
-    console.error(
-      `Failed to launch speedscope. Run manually:\n  npx speedscope ${filePath}`,
-    );
-  });
-}
-
-/** Convert a single HeapProfile to speedscope format (for standalone use) */
-export function heapProfileToSpeedscope(
-  name: string,
-  profile: HeapProfile,
-): SpeedscopeFile {
-  const frames: SpeedscopeFrame[] = [];
-  const frameIndex = new Map<string, number>();
-  const resolved = resolveProfile(profile);
-  const p = buildProfile(name, resolved, frames, frameIndex);
-
-  return {
-    $schema: "https://www.speedscope.app/file-format-schema.json",
-    shared: { frames },
-    profiles: [p],
-    exporter: "benchforge",
-  };
-}
-
-/** Build a single speedscope profile from a resolved heap profile */
-function buildProfile(
-  name: string,
-  resolved: ResolvedProfile,
-  sharedFrames: SpeedscopeFrame[],
-  frameIndex: Map<string, number>,
-): SpeedscopeProfile {
-  // Build nodeId -> stack of frame indices
-  const nodeStacks = new Map<number, number[]>();
-  for (const node of resolved.nodes) {
-    const stack = node.stack.map(f => internFrame(f, sharedFrames, frameIndex));
-    nodeStacks.set(node.nodeId, stack);
-  }
-
-  const samples: number[][] = [];
-  const weights: number[] = [];
-
-  if (!resolved.sortedSamples || resolved.sortedSamples.length === 0) {
-    console.error(
-      `Speedscope export: no samples in heap profile for "${name}", skipping`,
-    );
-    return {
-      type: "sampled",
-      name,
-      unit: "bytes",
-      startValue: 0,
-      endValue: 0,
-      samples,
-      weights,
-    };
-  }
-
-  for (const sample of resolved.sortedSamples) {
-    const stack = nodeStacks.get(sample.nodeId);
-    if (stack) {
-      samples.push(stack);
-      weights.push(sample.size);
-    }
-  }
-
-  const totalBytes = weights.reduce((sum, w) => sum + w, 0);
-
-  return {
-    type: "sampled",
-    name,
-    unit: "bytes",
-    startValue: 0,
-    endValue: totalBytes,
-    samples,
-    weights,
-  };
-}
-
-/** Intern a call frame, returning its index in the shared frames array */
-function internFrame(
-  frame: ResolvedFrame,
-  sharedFrames: SpeedscopeFrame[],
-  frameIndex: Map<string, number>,
-): number {
-  const { name, url, line, col } = frame;
-  const key = `${name}\0${url}\0${line}\0${col}`;
-
-  let idx = frameIndex.get(key);
-  if (idx === undefined) {
-    idx = sharedFrames.length;
-    // Match speedscope's convention: anonymous functions include location in name
-    const shortFile = url ? url.split("/").pop() : undefined;
-    const displayName =
-      name !== "(anonymous)"
-        ? name
-        : shortFile
-          ? `(anonymous ${shortFile}:${line})`
-          : "(anonymous)";
-    const entry: SpeedscopeFrame = { name: displayName };
-    if (url) entry.file = url;
-    if (line > 0) entry.line = line;
-    if (col != null) entry.col = col;
-    sharedFrames.push(entry);
-    frameIndex.set(key, idx);
-  }
-  return idx;
-}