benchforge 0.1.8 → 0.1.11

package/src/StatisticalUtils.ts

@@ -1,13 +1,3 @@
-const outlierMultiplier = 1.5; // Tukey's fence multiplier
-const bootstrapSamples = 10000;
-const confidence = 0.95;
-
-/** Options for bootstrap resampling methods */
-type BootstrapOptions = {
-  resamples?: number;
-  confidence?: number;
-};
-
 /** Bootstrap estimate with confidence interval and raw resample data */
 export interface BootstrapResult {
   estimate: number;
@@ -15,6 +5,32 @@ export interface BootstrapResult {
   samples: number[];
 }
 
+export type CIDirection = "faster" | "slower" | "uncertain";
+
+/** Binned histogram for efficient transfer to browser */
+export interface HistogramBin {
+  x: number; // bin center
+  count: number;
+}
+
+/** Bootstrap confidence interval for percentage difference between two samples */
+export interface DifferenceCI {
+  percent: number;
+  ci: [number, number];
+  direction: CIDirection;
+  /** Histogram of bootstrap distribution for visualization */
+  histogram?: HistogramBin[];
+}
+
+/** Options for bootstrap resampling methods */
+type BootstrapOptions = {
+  resamples?: number;
+  confidence?: number;
+};
+const confidence = 0.95;
+const outlierMultiplier = 1.5; // Tukey's fence multiplier
+const bootstrapSamples = 10000;
+
 /** @return relative standard deviation (coefficient of variation) */
 export function coefficientOfVariation(samples: number[]): number {
   const mean = average(samples);
@@ -86,13 +102,6 @@ export function percentile(values: number[], p: number): number {
   return sorted[Math.max(0, index)];
 }
 
-/** @return medians from bootstrap resamples */
-function generateMedians(samples: number[], resamples: number): number[] {
-  return Array.from({ length: resamples }, () =>
-    percentile(createResample(samples), 0.5),
-  );
-}
-
 /** @return bootstrap resample with replacement */
 export function createResample(samples: number[]): number[] {
   const n = samples.length;
@@ -100,50 +109,6 @@ export function createResample(samples: number[]): number[] {
   return Array.from({ length: n }, rand);
 }
 
-/** @return confidence interval [lower, upper] */
-function computeInterval(
-  medians: number[],
-  confidence: number,
-): [number, number] {
-  const alpha = (1 - confidence) / 2;
-  const lower = percentile(medians, alpha);
-  const upper = percentile(medians, 1 - alpha);
-  return [lower, upper];
-}
-
-export type CIDirection = "faster" | "slower" | "uncertain";
-
-/** Binned histogram for efficient transfer to browser */
-export interface HistogramBin {
-  x: number; // bin center
-  count: number;
-}
-
-/** Bootstrap confidence interval for percentage difference between two samples */
-export interface DifferenceCI {
-  percent: number;
-  ci: [number, number];
-  direction: CIDirection;
-  /** Histogram of bootstrap distribution for visualization */
-  histogram?: HistogramBin[];
-}
-
-/** Bin values into histogram for compact visualization */
-function binValues(values: number[], binCount = 30): HistogramBin[] {
-  const sorted = [...values].sort((a, b) => a - b);
-  const min = sorted[0];
-  const max = sorted[sorted.length - 1];
-  if (min === max) return [{ x: min, count: values.length }];
-
-  const step = (max - min) / binCount;
-  const counts = new Array(binCount).fill(0);
-  for (const v of values) {
-    const bin = Math.min(Math.floor((v - min) / step), binCount - 1);
-    counts[bin]++;
-  }
-  return counts.map((count, i) => ({ x: min + (i + 0.5) * step, count }));
-}
-
 /** @return bootstrap CI for percentage difference between baseline and current medians */
 export function bootstrapDifferenceCI(
   baseline: number[],
@@ -174,3 +139,37 @@ export function bootstrapDifferenceCI(
   const histogram = binValues(diffs);
   return { percent: observedPercent, ci, direction, histogram };
 }
+
+/** @return medians from bootstrap resamples */
+function generateMedians(samples: number[], resamples: number): number[] {
+  return Array.from({ length: resamples }, () =>
+    percentile(createResample(samples), 0.5),
+  );
+}
+
+/** @return confidence interval [lower, upper] */
+function computeInterval(
+  medians: number[],
+  confidence: number,
+): [number, number] {
+  const alpha = (1 - confidence) / 2;
+  const lower = percentile(medians, alpha);
+  const upper = percentile(medians, 1 - alpha);
+  return [lower, upper];
+}
+
+/** Bin values into histogram for compact visualization */
+function binValues(values: number[], binCount = 30): HistogramBin[] {
+  const sorted = [...values].sort((a, b) => a - b);
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  if (min === max) return [{ x: min, count: values.length }];
+
+  const step = (max - min) / binCount;
+  const counts = new Array(binCount).fill(0);
+  for (const v of values) {
+    const bin = Math.min(Math.floor((v - min) / step), binCount - 1);
+    counts[bin]++;
+  }
+  return counts.map((count, i) => ({ x: min + (i + 0.5) * step, count }));
+}

package/src/browser/BrowserGcStats.ts

@@ -32,13 +32,13 @@ export function parseGcTraceEvents(traceEvents: TraceEvent[]): GcEvent[] {
   });
 }
 
+/** Parse CDP trace events and aggregate into GcStats */
+export function browserGcStats(traceEvents: TraceEvent[]): GcStats {
+  return aggregateGcStats(parseGcTraceEvents(traceEvents));
+}
+
 function gcType(name: string): GcEvent["type"] | undefined {
   if (name === "MinorGC") return "scavenge";
   if (name === "MajorGC") return "mark-compact";
   return undefined;
 }
-
-/** Parse CDP trace events and aggregate into GcStats */
-export function browserGcStats(traceEvents: TraceEvent[]): GcStats {
-  return aggregateGcStats(parseGcTraceEvents(traceEvents));
-}

package/src/browser/BrowserHeapSampler.ts

@@ -93,6 +93,32 @@ export async function profileBrowser(
   }
 }
 
+/** Forward Chrome's stdout/stderr to the terminal so V8 flag output is visible. */
+function pipeChromeOutput(server: BrowserServer): void {
+  const proc = server.process();
+  const pipe = (stream: NodeJS.ReadableStream | null) =>
+    stream?.on("data", (chunk: Buffer) => {
+      for (const line of chunk.toString().split("\n")) {
+        const text = line.trim();
+        if (text) process.stderr.write(`[chrome] ${text}\n`);
+      }
+    });
+  pipe(proc.stdout);
+  pipe(proc.stderr);
+}
+
+/** Start CDP GC tracing, returns the event collector array. */
+async function startGcTracing(cdp: CDPSession): Promise<TraceEvent[]> {
+  const events: TraceEvent[] = [];
+  cdp.on("Tracing.dataCollected", ({ value }) => {
+    for (const e of value) events.push(e as unknown as TraceEvent);
+  });
+  await cdp.send("Tracing.start", {
+    traceConfig: { includedCategories: ["v8", "v8.gc"] },
+  });
+  return events;
+}
+
 /** Inject __start/__lap as in-page functions, expose __done for results collection.
  *  __start/__lap are pure in-page (zero CDP overhead). First __start() triggers
  *  instrument start. __done() stops instruments and collects timing data. */
@@ -147,57 +173,6 @@ async function setupLapMode(
   return { promise, cancel: () => clearTimeout(timer) };
 }
 
-/** In-page timing functions injected via addInitScript (zero CDP overhead).
- *  __start/__lap collect timestamps, __done delegates to exposed __benchCollect. */
-function injectLapFunctions(): void {
-  const g = globalThis as any;
-  g.__benchSamples = [];
-  g.__benchLastTime = 0;
-  g.__benchFirstStart = 0;
-
-  g.__start = () => {
-    const now = performance.now();
-    g.__benchLastTime = now;
-    if (!g.__benchFirstStart) {
-      g.__benchFirstStart = now;
-      return g.__benchInstrumentStart();
-    }
-  };
-
-  g.__lap = () => {
-    const now = performance.now();
-    g.__benchSamples.push(now - g.__benchLastTime);
-    g.__benchLastTime = now;
-  };
-
-  g.__done = () => {
-    const wall = g.__benchFirstStart
-      ? performance.now() - g.__benchFirstStart
-      : 0;
-    return g.__benchCollect(g.__benchSamples.slice(), wall);
-  };
-}
-
-function heapSamplingParams(samplingInterval: number) {
-  return {
-    samplingInterval,
-    includeObjectsCollectedByMajorGC: true,
-    includeObjectsCollectedByMinorGC: true,
-  };
-}
-
-/** Start CDP GC tracing, returns the event collector array. */
-async function startGcTracing(cdp: CDPSession): Promise<TraceEvent[]> {
-  const events: TraceEvent[] = [];
-  cdp.on("Tracing.dataCollected", ({ value }) => {
-    for (const e of value) events.push(e as unknown as TraceEvent);
-  });
-  await cdp.send("Tracing.start", {
-    traceConfig: { includedCategories: ["v8", "v8.gc"] },
-  });
-  return events;
-}
-
 /** Bench function mode: run window.__bench in a timed iteration loop. */
 async function runBenchLoop(
   page: Page,
@@ -254,18 +229,43 @@ async function collectTracing(
   return browserGcStats(traceEvents);
 }
 
-/** Forward Chrome's stdout/stderr to the terminal so V8 flag output is visible. */
-function pipeChromeOutput(server: BrowserServer): void {
-  const proc = server.process();
-  const pipe = (stream: NodeJS.ReadableStream | null) =>
-    stream?.on("data", (chunk: Buffer) => {
-      for (const line of chunk.toString().split("\n")) {
-        const text = line.trim();
-        if (text) process.stderr.write(`[chrome] ${text}\n`);
-      }
-    });
-  pipe(proc.stdout);
-  pipe(proc.stderr);
+function heapSamplingParams(samplingInterval: number) {
+  return {
+    samplingInterval,
+    includeObjectsCollectedByMajorGC: true,
+    includeObjectsCollectedByMinorGC: true,
+  };
+}
+
+/** In-page timing functions injected via addInitScript (zero CDP overhead).
+ *  __start/__lap collect timestamps, __done delegates to exposed __benchCollect. */
+function injectLapFunctions(): void {
+  const g = globalThis as any;
+  g.__benchSamples = [];
+  g.__benchLastTime = 0;
+  g.__benchFirstStart = 0;
+
+  g.__start = () => {
+    const now = performance.now();
+    g.__benchLastTime = now;
+    if (!g.__benchFirstStart) {
+      g.__benchFirstStart = now;
+      return g.__benchInstrumentStart();
+    }
+  };
+
+  g.__lap = () => {
+    const now = performance.now();
+    g.__benchSamples.push(now - g.__benchLastTime);
+    g.__benchLastTime = now;
+  };
+
+  g.__done = () => {
+    const wall = g.__benchFirstStart
+      ? performance.now() - g.__benchFirstStart
+      : 0;
+    return g.__benchCollect(g.__benchSamples.slice(), wall);
+  };
 }
 
 export { profileBrowser as profileBrowserHeap };

package/src/cli/CliArgs.ts

@@ -1,12 +1,14 @@
 import type { Argv, InferredOptionTypes } from "yargs";
 import yargs from "yargs";
 
-export const defaultAdaptiveMaxTime = 20;
-
 export type Configure<T> = (yargs: Argv) => Argv<T>;
 
-/** CLI args type inferred from cliOptions */
-export type DefaultCliArgs = InferredOptionTypes<typeof cliOptions>;
+/** CLI args type inferred from cliOptions, plus optional file positional */
+export type DefaultCliArgs = InferredOptionTypes<typeof cliOptions> & {
+  file?: string;
+};
+
+export const defaultAdaptiveMaxTime = 20;
 
 // biome-ignore format: compact option definitions
 const cliOptions = {
@@ -25,7 +27,9 @@ const cliOptions = {
   html:           { type: "boolean", default: false, describe: "generate HTML report and open in browser" },
   "export-html":  { type: "string",  requiresArg: true, describe: "export HTML report to specified file" },
   json:           { type: "string",  requiresArg: true, describe: "export benchmark data to JSON file" },
-  perfetto:       { type: "string",  requiresArg: true, describe: "export Perfetto trace file (view at ui.perfetto.dev)" },
+  "export-perfetto":    { type: "string",  requiresArg: true, describe: "export Perfetto trace file (view at ui.perfetto.dev)" },
+  speedscope:           { type: "boolean", default: false, describe: "open heap profile in speedscope (via npx)" },
+  "export-speedscope":  { type: "string",  requiresArg: true, describe: "export heap profile as speedscope JSON" },
   "trace-opt":    { type: "boolean", default: false, describe: "trace V8 optimization tiers (requires --allow-natives-syntax)" },
   "skip-settle":  { type: "boolean", default: false, describe: "skip post-warmup settle time (see V8 optimization cold start)" },
   "pause-first":  { type: "number",  describe: "iterations before first pause (then pause-interval applies)" },
@@ -39,6 +43,7 @@ const cliOptions = {
   "heap-rows":      { type: "number",  default: 20, describe: "top allocation sites to show" },
   "heap-stack":     { type: "number",  default: 3, describe: "call stack depth to display" },
   "heap-verbose":   { type: "boolean", default: false, describe: "verbose output with file:// paths and line numbers" },
+  "heap-raw":       { type: "boolean", default: false, describe: "dump every raw heap sample (ordinal, size, stack)" },
   "heap-user-only": { type: "boolean", default: false, describe: "filter to user code only (hide node internals)" },
   url:              { type: "string",  requiresArg: true, describe: "page URL for browser profiling (enables browser mode)" },
   headless:         { type: "boolean", default: true, describe: "run browser in headless mode" },
@@ -48,7 +53,16 @@ const cliOptions = {
 
 /** @return yargs with standard benchmark options */
 export function defaultCliArgs(yargsInstance: Argv): Argv<DefaultCliArgs> {
-  return yargsInstance.options(cliOptions).help().strict();
+  return yargsInstance
+    .command("$0 [file]", "run benchmarks", y => {
+      y.positional("file", {
+        type: "string",
+        describe: "benchmark file to run",
+      });
+    })
+    .options(cliOptions)
+    .help()
+    .strict() as Argv<DefaultCliArgs>;
 }
 
 /** @return parsed command line arguments */

package/src/cli/FilterBenchmarks.ts

@@ -55,14 +55,14 @@ function stripCaseSuffix(name: string): string {
   return name.replace(/ \[.*?\]$/, "");
 }
 
-/** Escape regex special characters */
-function escapeRegex(str: string): string {
-  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
 /** Ensure at least one benchmark matches filter */
 function validateFilteredSuite(groups: BenchGroup[], filter?: string): void {
   if (groups.every(g => g.benchmarks.length === 0)) {
     throw new Error(`No benchmarks match filter: "${filter}"`);
   }
 }
+
+/** Escape regex special characters */
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}