benchforge 0.1.11 → 0.2.4

package/src/profiling/browser/BrowserCDP.ts

@@ -0,0 +1,133 @@
+import type { GcStats } from "../../runners/GcStats.ts";
+import type { CoverageData, ScriptCoverage } from "../node/CoverageTypes.ts";
+import type { HeapProfile } from "../node/HeapSampler.ts";
+import type { TimeProfile } from "../node/TimeSampler.ts";
+import { browserGcStats } from "./BrowserGcStats.ts";
+import type { CdpClient } from "./CdpClient.ts";
+import type { TraceEvent } from "./ChromeTraceEvent.ts";
+
+/** Options controlling which CDP instruments (heap, CPU, coverage) to enable. */
+export interface InstrumentOpts {
+  alloc: boolean;
+  profile: boolean;
+  callCounts: boolean;
+  samplingInterval: number;
+  profileInterval?: number;
+}
+
+/** Build InstrumentOpts from profile params and heap sampling interval. */
+export function instrumentOpts(
+  params: {
+    alloc?: boolean;
+    profile?: boolean;
+    callCounts?: boolean;
+    profileInterval?: number;
+  },
+  samplingInterval: number,
+): InstrumentOpts {
+  const {
+    alloc = false,
+    profile = false,
+    callCounts = false,
+    profileInterval,
+  } = params;
+  return { alloc, profile, callCounts, samplingInterval, profileInterval };
+}
+
+/** Start CDP GC tracing; returns the mutable array that collects trace events. */
+export async function startGcTracing(cdp: CdpClient): Promise<TraceEvent[]> {
+  const events: TraceEvent[] = [];
+  cdp.on("Tracing.dataCollected", ({ value }) => {
+    events.push(...(value as unknown as TraceEvent[]));
+  });
+  await cdp.send("Tracing.start", {
+    traceConfig: { includedCategories: ["v8", "v8.gc"] },
+  });
+  return events;
+}
+
+/** End CDP tracing and aggregate collected events into GcStats. */
+export async function collectTracing(
+  cdp: CdpClient,
+  traceEvents: TraceEvent[],
+): Promise<GcStats> {
+  const done = new Promise<void>(r =>
+    cdp.once("Tracing.tracingComplete", () => r()),
+  );
+  await cdp.send("Tracing.end");
+  await done;
+  return browserGcStats(traceEvents);
+}
+
+/** Start CDP Profiler for CPU time sampling (caller manages Profiler.enable/disable) */
+export async function startTimeProfiling(
+  cdp: CdpClient,
+  interval?: number,
+): Promise<void> {
+  if (interval) await cdp.send("Profiler.setSamplingInterval", { interval });
+  await cdp.send("Profiler.start");
+}
+
+/** Stop CDP CPU sampling and return the profile. */
+export async function stopTimeProfiling(cdp: CdpClient): Promise<TimeProfile> {
+  const { profile } = await cdp.send("Profiler.stop");
+  return profile as unknown as TimeProfile;
+}
+
+/** Start precise coverage (caller manages Profiler.enable/disable). */
+export async function startCoverageCollection(cdp: CdpClient): Promise<void> {
+  await cdp.send("Profiler.startPreciseCoverage", {
+    callCount: true,
+    detailed: true,
+  });
+}
+
+/** Collect precise coverage, filtering out browser-internal scripts. */
+export async function collectCoverage(cdp: CdpClient): Promise<CoverageData> {
+  const { result } = await cdp.send("Profiler.takePreciseCoverage");
+  await cdp.send("Profiler.stopPreciseCoverage");
+  const scripts = (result as unknown as ScriptCoverage[]).filter(isPageScript);
+  return { scripts };
+}
+
+/** Stop active instruments and return collected profiles/coverage. */
+export async function stopInstruments(
+  cdp: CdpClient,
+  opts: InstrumentOpts,
+): Promise<{
+  heapProfile?: HeapProfile;
+  timeProfile?: TimeProfile;
+  coverage?: CoverageData;
+}> {
+  const heapProfile = opts.alloc
+    ? ((await cdp.send("HeapProfiler.stopSampling")).profile as HeapProfile)
+    : undefined;
+  const timeProfile = opts.profile ? await stopTimeProfiling(cdp) : undefined;
+  const coverage = opts.callCounts ? await collectCoverage(cdp) : undefined;
+  if (opts.profile || opts.callCounts) await cdp.send("Profiler.disable");
+  return { heapProfile, timeProfile, coverage };
+}
+
+/** Start requested CDP instruments (heap, CPU, coverage). */
+export async function startInstruments(
+  cdp: CdpClient,
+  opts: InstrumentOpts,
+): Promise<void> {
+  if (opts.alloc) {
+    await cdp.send("HeapProfiler.startSampling", {
+      samplingInterval: opts.samplingInterval,
+      includeObjectsCollectedByMajorGC: true,
+      includeObjectsCollectedByMinorGC: true,
+    });
+  }
+  if (opts.profile || opts.callCounts) await cdp.send("Profiler.enable");
+  if (opts.profile) await startTimeProfiling(cdp, opts.profileInterval);
+  if (opts.callCounts) await startCoverageCollection(cdp);
+}
+
+/** Exclude chrome:// and devtools:// internal scripts. */
+function isPageScript(s: ScriptCoverage): boolean {
+  return (
+    !!s.url && !s.url.startsWith("chrome") && !s.url.startsWith("devtools")
+  );
+}

package/src/profiling/browser/BrowserGcStats.ts

@@ -0,0 +1,33 @@
+import {
+  aggregateGcStats,
+  type GcEvent,
+  type GcStats,
+} from "../../runners/GcStats.ts";
+import type { TraceEvent } from "./ChromeTraceEvent.ts";
+
+/** Convert MinorGC/MajorGC trace events into GcEvent[]. */
+export function parseGcTraceEvents(traceEvents: TraceEvent[]): GcEvent[] {
+  return traceEvents
+    .filter(e => e.ph === "X" && gcType(e.name))
+    .map(e => ({
+      type: gcType(e.name)!,
+      pauseMs: (e.dur ?? 0) / 1000,
+      collected: Math.max(
+        0,
+        Number(e.args?.usedHeapSizeBefore ?? 0) -
+          Number(e.args?.usedHeapSizeAfter ?? 0),
+      ),
+    }));
+}
+
+/** Parse and aggregate CDP trace events into GcStats. */
+export function browserGcStats(traceEvents: TraceEvent[]): GcStats {
+  return aggregateGcStats(parseGcTraceEvents(traceEvents));
+}
+
+/** Map CDP event names (MinorGC/MajorGC) to GcEvent type. */
+function gcType(name: string): GcEvent["type"] | undefined {
+  if (name === "MinorGC") return "scavenge";
+  if (name === "MajorGC") return "mark-compact";
+  return undefined;
+}

package/src/profiling/browser/BrowserProfiler.ts

@@ -0,0 +1,160 @@
+import type { GcStats } from "../../runners/GcStats.ts";
+import type { CoverageData } from "../node/CoverageTypes.ts";
+import type { HeapProfile, HeapSampleOptions } from "../node/HeapSampler.ts";
+import type { TimeProfile } from "../node/TimeSampler.ts";
+import { runBenchLoop } from "./BenchLoop.ts";
+import { collectTracing, startGcTracing } from "./BrowserCDP.ts";
+import { type CdpClient, connectCdp } from "./CdpClient.ts";
+import { type CdpPage, createCdpPage } from "./CdpPage.ts";
+import {
+  type ChromeInstance,
+  closeTab,
+  createTab,
+  launchChrome,
+} from "./ChromeLauncher.ts";
+import { runPageLoad } from "./PageLoadMode.ts";
+
+/** Options for a browser benchmark run. */
+export interface BrowserProfileParams {
+  /** URL to benchmark */
+  url: string;
+  /** Enable heap allocation profiling */
+  alloc?: boolean;
+  /** Heap sampling options (interval, depth) */
+  allocOptions?: HeapSampleOptions;
+  /** Enable CPU time profiling */
+  profile?: boolean;
+  /** CPU profiling sample interval in microseconds (default 1000) */
+  profileInterval?: number;
+  /** Track function call counts via V8 coverage */
+  callCounts?: boolean;
+  /** Collect GC statistics via CDP tracing */
+  gcStats?: boolean;
+  /** Run Chrome in headless mode */
+  headless?: boolean;
+  /** Path to Chrome executable */
+  chromePath?: string;
+  /** Chrome user data directory for persistent profile */
+  chromeProfile?: string;
+  /** Extra Chrome launch arguments */
+  chromeArgs?: string[];
+  /** Page timeout in seconds */
+  timeout?: number;
+  /** Bench function iteration time limit in ms */
+  maxTime?: number;
+  /** Exact iteration count for bench function mode */
+  maxIterations?: number;
+  /** Passive page-load profiling mode */
+  pageLoad?: boolean;
+  /** Completion signal: CSS selector, JS expression, "load", or "domcontentloaded" */
+  waitFor?: string;
+  /** Reuse an existing Chrome instance (caller manages lifecycle) */
+  chrome?: ChromeInstance;
+}
+
+/** Navigation timing metrics from the Performance API. */
+export interface NavTiming {
+  /** DOMContentLoaded time in ms */
+  domContentLoaded: number;
+  /** Load event time in ms */
+  loadEvent: number;
+  /** Largest Contentful Paint time in ms */
+  lcp?: number;
+}
+
+/** Collected profiles, timing samples, and GC stats from a browser benchmark. */
+export interface BrowserProfileResult {
+  /** Heap allocation profile */
+  heapProfile?: HeapProfile;
+  /** CPU time profile */
+  timeProfile?: TimeProfile;
+  /** V8 code coverage data */
+  coverage?: CoverageData;
+  /** Garbage collection statistics */
+  gcStats?: GcStats;
+  /** Wall-clock ms for the entire bench loop or page load */
+  wallTimeMs?: number;
+  /** Per-iteration timing samples (ms) from bench function mode */
+  samples?: number[];
+  /** Navigation timing from page-load mode */
+  navTiming?: NavTiming;
+}
+
+/** Shared context passed to bench/page-load mode runners. */
+export interface ProfileCtx {
+  page: CdpPage;
+  cdp: CdpClient;
+  params: BrowserProfileParams;
+  samplingInterval: number;
+}
+
+/**
+ * Run browser benchmark, auto-detecting mode:
+ * - Bench function (window.__bench): CLI controls iteration and timing.
+ * - Page load (no __bench, or --page-load): measures navigation timing.
+ */
+export async function profileBrowser(
+  params: BrowserProfileParams,
+): Promise<BrowserProfileResult> {
+  const {
+    headless = false,
+    chromePath,
+    chromeProfile,
+    chromeArgs: args,
+  } = params;
+  const owned = !params.chrome;
+  const launch = { headless, chromePath, chromeProfile, args };
+  const chrome = params.chrome ?? (await launchChrome(launch));
+  try {
+    const { wsUrl, targetId } = await createTab(chrome.port);
+    const cdp = await connectCdp(wsUrl);
+    try {
+      const timeout = (params.timeout ?? 60) * 1000;
+      const page = await createCdpPage(cdp, { timeout });
+      return await runProfile(page, cdp, params);
+    } finally {
+      cdp.close();
+      await closeTab(chrome.port, targetId);
+    }
+  } finally {
+    if (owned) await chrome.close();
+  }
+}
+
+/**
+ * Run profiling on an open CDP page, auto-detecting mode:
+ * - **bench**: page exports `window.__bench` ==> CLI iterates and times it
+ * - **page-load**: no `__bench` found (or `--page-load` flag) ==> profile navigation
+ *
+ * When auto-detecting, navigates once to check for `__bench`. If not found,
+ * reloads via `runPageLoad` which starts instruments before navigation.
+ */
+async function runProfile(
+  page: CdpPage,
+  cdp: CdpClient,
+  params: BrowserProfileParams,
+): Promise<BrowserProfileResult> {
+  const samplingInterval = params.allocOptions?.samplingInterval ?? 32768;
+  const traceEvents = params.gcStats ? await startGcTracing(cdp) : [];
+  const ctx = { page, cdp, params, samplingInterval };
+
+  let result: BrowserProfileResult;
+  if (params.pageLoad) {
+    result = await runPageLoad(ctx);
+  } else {
+    await page.navigate(params.url, { waitUntil: "load" });
+    const hasBench = await page.evaluate(
+      () => typeof (globalThis as any).__bench === "function",
+    );
+    if (hasBench) {
+      result = await runBenchLoop(ctx);
+    } else {
+      console.warn("No __bench found. Reloading in --page-load mode.");
+      result = await runPageLoad(ctx);
+    }
+  }
+
+  if (params.gcStats)
+    return { ...result, gcStats: await collectTracing(cdp, traceEvents) };
+  return result;
+}

package/src/profiling/browser/CdpClient.ts

@@ -0,0 +1,82 @@
+/** Minimal CDP WebSocket client. */
+export interface CdpClient {
+  send(method: string, params?: Record<string, unknown>): Promise<any>;
+  on(event: string, handler: (params: any) => void): void;
+  once(event: string, handler: (params: any) => void): void;
+  close(): void;
+}
+
+/** Connect to a CDP WebSocket endpoint and return a client. */
+export async function connectCdp(wsUrl: string): Promise<CdpClient> {
+  const ws = await openWebSocket(wsUrl);
+  let nextId = 1;
+  type Pending = { resolve: (v: any) => void; reject: (e: Error) => void };
+  const pending = new Map<number, Pending>();
+  const listeners = new Map<string, Set<(params: any) => void>>();
+
+  ws.addEventListener("message", event => {
+    const msg = JSON.parse(String(event.data));
+    if ("id" in msg) {
+      const p = pending.get(msg.id);
+      if (!p) return;
+      pending.delete(msg.id);
+      if (msg.error) p.reject(new Error(`CDP: ${msg.error.message}`));
+      else p.resolve(msg.result ?? {});
+    } else if ("method" in msg) {
+      for (const h of listeners.get(msg.method) ?? []) h(msg.params ?? {});
+    }
+  });
+
+  const client: CdpClient = {
+    send(method, params) {
+      return new Promise((resolve, reject) => {
+        const id = nextId++;
+        const timer = setTimeout(() => {
+          if (pending.delete(id))
+            reject(new Error(`CDP timeout after 60s: ${method}`));
+        }, 60_000);
+        const clear = () => clearTimeout(timer);
+        pending.set(id, {
+          resolve(v) {
+            clear();
+            resolve(v);
+          },
+          reject(e) {
+            clear();
+            reject(e);
+          },
+        });
+        ws.send(JSON.stringify({ id, method, params }));
+      });
+    },
+    on(event, handler) {
+      const set = listeners.get(event) ?? new Set();
+      listeners.set(event, set);
+      set.add(handler);
+    },
+    once(event, handler) {
+      const wrap = (params: any) => {
+        listeners.get(event)?.delete(wrap);
+        handler(params);
+      };
+      client.on(event, wrap);
+    },
+    close() {
+      for (const [, p] of pending) p.reject(new Error("CDP connection closed"));
+      pending.clear();
+      ws.close();
+    },
+  };
+  return client;
+}
+
+/** Open a WebSocket connection, rejecting if the handshake fails. */
+async function openWebSocket(wsUrl: string): Promise<WebSocket> {
+  const ws = new WebSocket(wsUrl);
+  const err = new Error(`CDP connect failed: ${wsUrl}`);
+  await new Promise<void>((resolve, reject) => {
+    ws.addEventListener("open", () => resolve());
+    ws.addEventListener("error", () => reject(err));
+  });
+  return ws;
+}

package/src/profiling/browser/CdpPage.ts

@@ -0,0 +1,138 @@
+import type { CdpClient } from "./CdpClient.ts";
+
+/** Page-level CDP abstraction for navigation, evaluation, and event handling. */
+export interface CdpPage {
+  navigate(
+    url: string,
+    opts?: { waitUntil?: "load" | "domcontentloaded" },
+  ): Promise<void>;
+  evaluate<R>(
+    fn: (...args: any[]) => R | Promise<R>,
+    arg?: unknown,
+  ): Promise<Awaited<R>>;
+  exposeFunction(name: string, fn: (...args: any[]) => any): Promise<void>;
+  addInitScript(fn: () => void): Promise<void>;
+  waitForSelector(selector: string): Promise<void>;
+  waitForFunction(expression: string): Promise<void>;
+  onPageError(handler: (message: string) => void): void;
+}
+
+/** Create a page abstraction over a CDP client connected to a page target. */
+export async function createCdpPage(
+  cdp: CdpClient,
+  opts?: { timeout?: number },
+): Promise<CdpPage> {
+  const timeout = opts?.timeout ?? 30_000;
+
+  await cdp.send("Page.enable");
+  await cdp.send("Runtime.enable");
+
+  return {
+    navigate: (url, navOpts) => cdpNavigate(cdp, url, navOpts),
+    evaluate: (fn, arg) => cdpEvaluate(cdp, fn, arg),
+    exposeFunction: (name, fn) => cdpExpose(cdp, name, fn),
+    async addInitScript(fn) {
+      await cdp.send("Page.addScriptToEvaluateOnNewDocument", {
+        source: `(${fn.toString()})()`,
+      });
+    },
+    waitForSelector(sel) {
+      const expr = `!!document.querySelector(${JSON.stringify(sel)})`;
+      return pollEval(cdp, expr, timeout);
+    },
+    waitForFunction: expr => pollEval(cdp, expr, timeout),
+    onPageError(handler) {
+      cdp.on("Runtime.exceptionThrown", ({ exceptionDetails: d }) => {
+        handler(d.exception?.description || d.text);
+      });
+    },
+  };
+}
+
+/** Navigate to a URL and wait for the specified load condition. */
+async function cdpNavigate(
+  cdp: CdpClient,
+  url: string,
+  navOpts?: { waitUntil?: "load" | "domcontentloaded" },
+): Promise<void> {
+  const waitUntil = navOpts?.waitUntil ?? "load";
+  const event =
+    waitUntil === "domcontentloaded"
+      ? "Page.domContentEventFired"
+      : "Page.loadEventFired";
+  const loaded = new Promise<void>(r => cdp.once(event, () => r()));
+  await cdp.send("Page.navigate", { url });
+  await loaded;
+}
+
+/** Evaluate a function in the page and return the result. */
+async function cdpEvaluate(
+  cdp: CdpClient,
+  fn: (...args: any[]) => any,
+  arg?: unknown,
+): Promise<any> {
+  const argStr = arg !== undefined ? JSON.stringify(arg) : "";
+  const expression = `(${fn.toString()})(${argStr})`;
+  const opts = { expression, awaitPromise: true, returnByValue: true };
+  const { result, exceptionDetails: err } = await cdp.send(
+    "Runtime.evaluate",
+    opts,
+  );
+  if (err) throw new Error(err.exception?.description || err.text);
+  return result.value;
+}
+
+/** Expose a Node function to the page via Runtime.addBinding. */
+async function cdpExpose(
+  cdp: CdpClient,
+  name: string,
+  fn: (...args: any[]) => any,
+): Promise<void> {
+  const binding = `__cdp_${name}`;
+  await cdp.send("Runtime.addBinding", { name: binding });
+
+  // Wrapper: page calls window[name](...) ==> binding fires ==> Node runs fn ==> resolve
+  const wrapper = `(() => {
+    const g = globalThis;
+    if (!g.__cdpSeq) { g.__cdpSeq = 0; g.__cdpCbs = {}; }
+    g[${JSON.stringify(name)}] = (...args) => new Promise((resolve, reject) => {
+      const seq = ++g.__cdpSeq;
+      g.__cdpCbs[seq] = { resolve, reject };
+      g[${JSON.stringify(binding)}](JSON.stringify({ seq, args }));
+    });
+  })()`;
+
+  await cdp.send("Page.addScriptToEvaluateOnNewDocument", { source: wrapper });
+  await cdp.send("Runtime.evaluate", { expression: wrapper });
+
+  const pageEval = (expr: string) =>
+    cdp.send("Runtime.evaluate", { expression: expr });
+  cdp.on("Runtime.bindingCalled", async params => {
+    if (params.name !== binding) return;
+    const { seq, args } = JSON.parse(params.payload);
+    const cb = `globalThis.__cdpCbs[${seq}]`;
+    try {
+      const val = await fn(...args);
+      await pageEval(`${cb}?.resolve(${JSON.stringify(val ?? null)})`);
+    } catch (err: any) {
+      const msg = JSON.stringify(String(err.message));
+      await pageEval(`${cb}?.reject(new Error(${msg}))`);
+    }
+  });
+}
+
+/** Poll a JS expression until truthy, with timeout. */
+async function pollEval(
+  cdp: CdpClient,
+  expression: string,
+  timeout: number,
+): Promise<void> {
+  const deadline = Date.now() + timeout;
+  const evalOpts = { expression, returnByValue: true };
+  while (Date.now() < deadline) {
+    const { result } = await cdp.send("Runtime.evaluate", evalOpts);
+    if (result.value) return;
+    await new Promise(r => setTimeout(r, 100));
+  }
+  throw new Error(`Timed out waiting for: ${expression}`);
+}