benchforge 0.1.11 → 0.2.4

package/src/runners/RunnerOrchestrator.ts

@@ -1,16 +1,19 @@
 import { type ChildProcess, fork } from "node:child_process";
 import { existsSync } from "node:fs";
 import path from "node:path";
-import type { BenchmarkSpec } from "../Benchmark.ts";
-import type { HeapProfile } from "../heap-sample/HeapSampler.ts";
-import type { MeasuredResults } from "../MeasuredResults.ts";
-import {
-  type AdaptiveOptions,
-  createAdaptiveWrapper,
-} from "./AdaptiveWrapper.ts";
+import type { CoverageData } from "../profiling/node/CoverageTypes.ts";
+import type { HeapProfile } from "../profiling/node/HeapSampler.ts";
+import type { TimeProfile } from "../profiling/node/TimeSampler.ts";
+import type { BenchmarkFunction, BenchmarkSpec } from "./BenchmarkSpec.ts";
 import type { RunnerOptions } from "./BenchRunner.ts";
-import { createRunner, type KnownRunner } from "./CreateRunner.ts";
+import type { KnownRunner } from "./CreateRunner.ts";
 import { aggregateGcStats, type GcEvent, parseGcLine } from "./GcStats.ts";
+import type { MeasuredResults } from "./MeasuredResults.ts";
+import {
+  createBenchRunner,
+  importBenchFn,
+  resolveVariantFn,
+} from "./RunnerUtils.ts";
 import { debugWorkerTiming, getElapsed, getPerfNow } from "./TimingUtils.ts";
 import type {
   ErrorMessage,
@@ -18,7 +21,7 @@ import type {
   RunMessage,
 } from "./WorkerScript.ts";
 
-/** Parameters for running a matrix variant in worker */
+/** Parameters for running a matrix variant */
 export interface RunMatrixVariantParams {
   variantDir: string;
   variantId: string;
@@ -27,20 +30,9 @@ export interface RunMatrixVariantParams {
   casesModule?: string;
   runner: KnownRunner;
   options: RunnerOptions;
+  useWorker?: boolean;
 }
 
-type WorkerParams<T = unknown> = {
-  spec: BenchmarkSpec<T>;
-  runner: KnownRunner;
-  options: RunnerOptions;
-  params?: T;
-};
-
-type WorkerHandlers = {
-  resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => void;
-  reject: (error: Error) => void;
-};
-
 interface RunBenchmarkParams<T = unknown> {
   spec: BenchmarkSpec<T>;
   runner: KnownRunner;
@@ -53,7 +45,7 @@ const logTiming = debugWorkerTiming
   ? (message: string) => console.log(`[RunnerOrchestrator] ${message}`)
   : () => {};
 
-/** Execute benchmarks directly or in worker process */
+/** Run a benchmark spec, optionally in an isolated worker process for profiling support. */
 export async function runBenchmark<T = unknown>({
   spec,
   runner,
@@ -62,41 +54,30 @@ export async function runBenchmark<T = unknown>({
   params,
 }: RunBenchmarkParams<T>): Promise<MeasuredResults[]> {
   if (!useWorker) {
-    const resolvedSpec = spec.modulePath
+    const resolved = spec.modulePath
       ? await resolveModuleSpec(spec, params)
       : { spec, params };
-
-    const base = await createRunner(runner);
-    const benchRunner = (options as any).adaptive
-      ? createAdaptiveWrapper(base, options as AdaptiveOptions)
-      : base;
-    return benchRunner.runBench(
-      resolvedSpec.spec,
-      options,
-      resolvedSpec.params,
-    );
+    const benchRunner = await createBenchRunner(runner, options);
+    return benchRunner.runBench(resolved.spec, options, resolved.params);
   }
 
-  return runInWorker({ spec, runner, options, params });
+  const msg = createRunMessage(spec, runner, options, params);
+  return runWorkerWithMessage(spec.name, options, msg);
 }
 
-/** Run a matrix variant benchmark in isolated worker process */
+/** Run a matrix variant benchmark, directly or in a worker. */
 export async function runMatrixVariant(
   params: RunMatrixVariantParams,
 ): Promise<MeasuredResults[]> {
-  const {
-    variantDir,
-    variantId,
-    caseId,
-    caseData,
-    casesModule,
-    runner,
-    options,
-  } = params;
+  const { variantId, caseId, runner, options, useWorker = true } = params;
   const name = `${variantId}/${caseId}`;
+
+  if (!useWorker) return runMatrixVariantDirect(params, name);
+
+  const { variantDir, caseData, casesModule } = params;
   const message: RunMessage = {
     type: "run",
-    spec: { name, fn: () => {} },
+    spec: { name } as BenchmarkSpec,
     runnerName: runner,
     options,
     variantDir,
@@ -113,67 +94,18 @@ async function resolveModuleSpec<T>(
   spec: BenchmarkSpec<T>,
   params: T | undefined,
 ): Promise<{ spec: BenchmarkSpec<T>; params: T | undefined }> {
-  const module = await import(spec.modulePath!);
-
-  const fn = spec.exportName
-    ? module[spec.exportName]
-    : module.default || module;
-
-  if (typeof fn !== "function") {
-    const name = spec.exportName || "default";
-    throw new Error(
-      `Export '${name}' from ${spec.modulePath} is not a function`,
-    );
-  }
-
-  let resolvedParams = params;
-  if (spec.setupExportName) {
-    const setupFn = module[spec.setupExportName];
-    if (typeof setupFn !== "function") {
-      const msg = `Setup export '${spec.setupExportName}' from ${spec.modulePath} is not a function`;
-      throw new Error(msg);
-    }
-    resolvedParams = await setupFn(params);
-  }
-
-  return { spec: { ...spec, fn }, params: resolvedParams };
-}
-
-/** Run benchmark in isolated worker process */
-async function runInWorker<T>(
-  workerParams: WorkerParams<T>,
-): Promise<MeasuredResults[]> {
-  const { spec, runner, options, params } = workerParams;
-  const msg = createRunMessage(spec, runner, options, params);
-  return runWorkerWithMessage(spec.name, options, msg);
-}
-
-/** Spawn worker, wire handlers, send message, return results */
-function runWorkerWithMessage(
-  name: string,
-  options: RunnerOptions,
-  message: RunMessage,
-): Promise<MeasuredResults[]> {
-  const startTime = getPerfNow();
-  const collectGcStats = options.gcStats ?? false;
-  logTiming(`Starting worker for ${name}`);
-
-  return new Promise((resolve, reject) => {
-    const { worker, createTime, gcEvents } =
-      createWorkerWithTiming(collectGcStats);
-    const handlers = createWorkerHandlers(
-      name,
-      startTime,
-      gcEvents,
-      resolve,
-      reject,
-    );
-    setupWorkerHandlers(worker, name, handlers);
-    sendWorkerMessage(worker, message, createTime);
-  });
+  const { modulePath, exportName, setupExportName } = spec;
+  const imported = await importBenchFn(
+    modulePath!,
+    exportName,
+    setupExportName,
+    params,
+  );
+  const fn = imported.fn as BenchmarkFunction<T>;
+  return { spec: { ...spec, fn }, params: imported.params as T | undefined };
 }
 
-/** Create message for worker execution */
+/** Serialize a BenchmarkSpec into a worker-safe message (modulePath or fnCode) */
 function createRunMessage<T>(
   spec: BenchmarkSpec<T>,
   runnerName: KnownRunner,
@@ -198,175 +130,127 @@ function createRunMessage<T>(
   return message;
 }
 
-/** Create worker process with timing logs */
-function createWorkerWithTiming(gcStats: boolean) {
-  const workerStart = getPerfNow();
-  const gcEvents: GcEvent[] = [];
-  const worker = createWorkerProcess(gcStats);
-  const createTime = getPerfNow();
-  if (gcStats && worker.stdout) setupGcCapture(worker, gcEvents);
-  logTiming(
-    `Worker process created in ${getElapsed(workerStart, createTime).toFixed(1)}ms`,
-  );
-  return { worker, createTime, gcEvents };
-}
+/** Run a benchmark in an isolated worker process with timeout and GC capture. */
+function runWorkerWithMessage(
+  name: string,
+  options: RunnerOptions,
+  message: RunMessage,
+): Promise<MeasuredResults[]> {
+  const startTime = getPerfNow();
+  const collectGcStats = options.gcStats ?? false;
+  logTiming(`Starting worker for ${name}`);
 
-// Consider: --no-compilation-cache, --max-old-space-size=512, --no-lazy
-// for consistency (less realistic)
+  return new Promise((resolve, reject) => {
+    const gcEvents: GcEvent[] = [];
+    const worker = spawnWorkerProcess(collectGcStats);
+    if (collectGcStats && worker.stdout) setupGcCapture(worker, gcEvents);
 
-/** @return handlers that attach GC stats and heap profile to results */
-function createWorkerHandlers(
-  specName: string,
-  startTime: number,
-  gcEvents: GcEvent[] | undefined,
-  resolve: (results: MeasuredResults[]) => void,
-  reject: (error: Error) => void,
-): WorkerHandlers {
-  return {
-    resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => {
-      logTiming(
-        `Total worker time for ${specName}: ${getElapsed(startTime).toFixed(1)}ms`,
-      );
-      if (gcEvents?.length) {
-        const gcStats = aggregateGcStats(gcEvents);
-        for (const r of results) r.gcStats = gcStats;
+    const timeoutId = setTimeout(() => {
+      killWorker();
+      reject(new Error(`Benchmark "${name}" timed out after 60 seconds`));
+    }, 60000);
+
+    function killWorker() {
+      clearTimeout(timeoutId);
+      if (!worker.killed) worker.kill("SIGTERM");
+    }
+
+    worker.on("message", (msg: ResultMessage | ErrorMessage) => {
+      killWorker();
+      if (msg.type === "error") {
+        const error = new Error(`Benchmark "${name}" failed: ${msg.error}`);
+        if (msg.stack) error.stack = msg.stack;
+        return reject(error);
       }
-      if (heapProfile) for (const r of results) r.heapProfile = heapProfile;
+      const elapsed = getElapsed(startTime).toFixed(1);
+      logTiming(`Total worker time for ${name}: ${elapsed}ms`);
+      const { results, heapProfile, timeProfile, coverage } = msg;
+      attachProfilingData(
+        results,
+        gcEvents,
+        heapProfile,
+        timeProfile,
+        coverage,
+      );
       resolve(results);
-    },
-    reject,
-  };
-}
+    });
+    worker.on("error", (error: Error) => {
+      killWorker();
+      const msg = `Worker process failed for "${name}": ${error.message}`;
+      reject(new Error(msg));
+    });
+    worker.on("exit", (code: number | null) => {
+      if (code !== 0 && code !== null) {
+        killWorker();
+        reject(new Error(`Worker exited with code ${code} for "${name}"`));
+      }
+    });
 
-/** Setup worker event handlers with cleanup */
-function setupWorkerHandlers(
-  worker: ReturnType<typeof createWorkerProcess>,
-  specName: string,
-  handlers: WorkerHandlers,
-) {
-  const { resolve, reject } = handlers;
-  const cleanup = createCleanup(worker, specName, reject);
-  worker.on(
-    "message",
-    createMessageHandler(specName, cleanup, resolve, reject),
-  );
-  worker.on("error", createErrorHandler(specName, cleanup, reject));
-  worker.on("exit", createExitHandler(specName, cleanup, reject));
+    worker.send(message);
+  });
 }
 
-/** Send message to worker with timing log */
-function sendWorkerMessage(
-  worker: ReturnType<typeof createWorkerProcess>,
-  message: RunMessage,
-  createTime: number,
-): void {
-  const messageTime = getPerfNow();
-  worker.send(message);
-  logTiming(
-    `Message sent to worker in ${getElapsed(createTime, messageTime).toFixed(1)}ms`,
-  );
+/** Run matrix variant in-process (no worker isolation) */
+async function runMatrixVariantDirect(
+  params: RunMatrixVariantParams,
+  name: string,
+): Promise<MeasuredResults[]> {
+  const { runner, options } = params;
+  const { fn } = await resolveVariantFn(params);
+  const benchRunner = await createBenchRunner(runner, options);
+  return benchRunner.runBench({ name, fn }, options);
 }
 
-/** Create worker process with configuration */
-function createWorkerProcess(gcStats: boolean) {
+/** Spawn worker process with V8 flags */
+function spawnWorkerProcess(gcStats: boolean) {
   const workerPath = resolveWorkerPath();
   const execArgv = ["--expose-gc", "--allow-natives-syntax"];
   if (gcStats) execArgv.push("--trace-gc-nvp");
 
+  const env = { ...process.env, NODE_OPTIONS: "" };
+  // silent mode captures stdout so we can parse --trace-gc-nvp output
   return fork(workerPath, [], {
     execArgv,
-    silent: gcStats, // Capture stdout/stderr when collecting GC stats
-    env: {
-      ...process.env,
-      NODE_OPTIONS: "",
-    },
+    silent: gcStats,
+    env,
+    serialization: "advanced",
   });
 }
 
-/** Capture and parse GC lines from stdout (V8's --trace-gc-nvp outputs to stdout) */
+/** Capture and parse GC lines from worker stdout (--trace-gc-nvp). */
 function setupGcCapture(worker: ChildProcess, gcEvents: GcEvent[]): void {
   let buffer = "";
   worker.stdout!.on("data", (data: Buffer) => {
     buffer += data.toString();
     const lines = buffer.split("\n");
-    buffer = lines.pop() || ""; // Keep incomplete line in buffer
+    buffer = lines.pop() || "";
     for (const line of lines) {
       const event = parseGcLine(line);
-      if (event) {
-        gcEvents.push(event);
-      } else if (line.trim()) {
-        // Forward non-GC stdout to console (worker status messages)
-        process.stdout.write(line + "\n");
-      }
+      if (event) gcEvents.push(event);
+      else if (line.trim()) process.stdout.write(line + "\n");
     }
   });
 }
 
-/** Create cleanup for timeout and termination */
-function createCleanup(
-  worker: ReturnType<typeof createWorkerProcess>,
-  specName: string,
-  reject: (error: Error) => void,
-) {
-  const timeoutId = setTimeout(() => {
-    cleanup();
-    reject(new Error(`Benchmark "${specName}" timed out after 60 seconds`));
-  }, 60000);
-  const cleanup = () => {
-    clearTimeout(timeoutId);
-    if (!worker.killed) worker.kill("SIGTERM");
-  };
-  return cleanup;
-}
-
-/** Handle worker messages (results or errors) */
-function createMessageHandler(
-  specName: string,
-  cleanup: () => void,
-  resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => void,
-  reject: (error: Error) => void,
-) {
-  return (msg: ResultMessage | ErrorMessage) => {
-    cleanup();
-    if (msg.type === "result") {
-      resolve(msg.results, msg.heapProfile);
-    } else if (msg.type === "error") {
-      const error = new Error(`Benchmark "${specName}" failed: ${msg.error}`);
-      if (msg.stack) error.stack = msg.stack;
-      reject(error);
-    }
-  };
-}
-
-/** Handle worker process errors */
-function createErrorHandler(
-  specName: string,
-  cleanup: () => void,
-  reject: (error: Error) => void,
-) {
-  return (error: Error) => {
-    cleanup();
-    reject(
-      new Error(
-        `Worker process failed for benchmark "${specName}": ${error.message}`,
-      ),
-    );
-  };
-}
-
-/** Handle worker process exit */
-function createExitHandler(
-  specName: string,
-  cleanup: () => void,
-  reject: (error: Error) => void,
-) {
-  return (code: number | null, _signal: NodeJS.Signals | null) => {
-    if (code !== 0 && code !== null) {
-      cleanup();
-      const msg = `Worker exited with code ${code} for benchmark "${specName}"`;
-      reject(new Error(msg));
-    }
+/** Attach profiling data collected by the worker to each result. */
+function attachProfilingData(
+  results: MeasuredResults[],
+  gcEvents: GcEvent[] | undefined,
+  heapProfile?: HeapProfile,
+  timeProfile?: TimeProfile,
+  coverage?: CoverageData,
+): void {
+  const gcStats = gcEvents?.length ? aggregateGcStats(gcEvents) : undefined;
+  const attach = <K extends keyof MeasuredResults>(
+    key: K,
+    value: MeasuredResults[K] | undefined,
+  ) => {
+    if (value) for (const r of results) r[key] = value;
   };
+  attach("gcStats", gcStats);
+  attach("heapProfile", heapProfile);
+  attach("timeProfile", timeProfile);
+  attach("coverage", coverage);
 }
 
 /** Resolve WorkerScript path for dev (.ts) or dist (.mjs) */

package/src/runners/RunnerUtils.ts

@@ -1,2 +1,76 @@
+import { prepareBenchFn } from "../matrix/BenchMatrix.ts";
+import { loadCaseData, loadCasesModule } from "../matrix/CaseLoader.ts";
+import { loadVariant } from "../matrix/VariantLoader.ts";
+import {
+  type AdaptiveOptions,
+  createAdaptiveWrapper,
+} from "./AdaptiveWrapper.ts";
+import type { BenchmarkFunction } from "./BenchmarkSpec.ts";
+import type { BenchRunner, RunnerOptions } from "./BenchRunner.ts";
+import { createRunner, type KnownRunner } from "./CreateRunner.ts";
+
 export const msToNs = 1e6;
-export const nsToMs = 1e-6;
+
+/** Get named or default export from module, throw if not a function */
+// biome-ignore lint/complexity/noBannedTypes: generic function constraint
+export function getModuleExport<T extends Function = Function>(
+  module: any,
+  exportName: string | undefined,
+  modulePath: string,
+): T {
+  const fn = exportName ? module[exportName] : module.default || module;
+  if (typeof fn !== "function") {
+    const name = exportName || "default";
+    throw new Error(`Export '${name}' from ${modulePath} is not a function`);
+  }
+  return fn as T;
+}
+
+/** Import a benchmark function from a module, optionally running a setup export */
+export async function importBenchFn(
+  modulePath: string,
+  exportName: string | undefined,
+  setupExportName: string | undefined,
+  params: unknown,
+): Promise<{ fn: BenchmarkFunction; params: unknown }> {
+  const module = await import(modulePath);
+  const fn = getModuleExport<BenchmarkFunction>(module, exportName, modulePath);
+  if (!setupExportName) return { fn, params };
+
+  const setup = getModuleExport<BenchmarkFunction>(
+    module,
+    setupExportName,
+    modulePath,
+  );
+  return { fn, params: await setup(params) };
+}
+
+/** Resolve a matrix variant to a benchmark function (shared by orchestrator and worker). */
+export async function resolveVariantFn(params: {
+  variantDir: string;
+  variantId: string;
+  caseId?: string;
+  caseData?: unknown;
+  casesModule?: string;
+}): Promise<{ fn: BenchmarkFunction; params: undefined }> {
+  let { caseData } = params;
+  if (params.casesModule && params.caseId) {
+    const cases = await loadCasesModule(params.casesModule);
+    caseData = (await loadCaseData(cases, params.caseId)).data;
+  }
+  const variant = await loadVariant(params.variantDir, params.variantId);
+  const fn = await prepareBenchFn(variant, caseData);
+  return { fn, params: undefined };
+}
+
+/** Create runner, wrapping with adaptive sampling if options.adaptive is set */
+export async function createBenchRunner(
+  runnerName: KnownRunner,
+  options: RunnerOptions,
+): Promise<BenchRunner> {
+  const base = await createRunner(runnerName);
+  if ("adaptive" in options && options.adaptive) {
+    return createAdaptiveWrapper(base, options as AdaptiveOptions);
+  }
+  return base;
+}

package/src/runners/SampleStats.ts

@@ -0,0 +1,100 @@
+import {
+  coefficientOfVariation,
+  median,
+  medianAbsoluteDeviation,
+  percentile,
+} from "../stats/StatisticalUtils.ts";
+import {
+  type MeasuredResults,
+  type OptStatusInfo,
+  optStatusNames,
+} from "./MeasuredResults.ts";
+
+/** Compute percentiles, CV, MAD, and outlier rate from timing samples. */
+export function computeStats(samples: number[]): MeasuredResults["time"] {
+  let min = Number.POSITIVE_INFINITY;
+  let max = Number.NEGATIVE_INFINITY;
+  let sum = 0;
+  for (const s of samples) {
+    if (s < min) min = s;
+    if (s > max) max = s;
+    sum += s;
+  }
+  const sorted = [...samples].sort((a, b) => a - b);
+  const pct = (p: number) =>
+    sorted[Math.max(0, Math.ceil(sorted.length * p) - 1)];
+  return {
+    min,
+    max,
+    avg: sum / samples.length,
+    p25: pct(0.25),
+    p50: pct(0.5),
+    p75: pct(0.75),
+    p95: pct(0.95),
+    p99: pct(0.99),
+    p999: pct(0.999),
+    cv: coefficientOfVariation(samples),
+    mad: medianAbsoluteDeviation(samples),
+    outlierRate: outlierImpactRatio(samples),
+  };
+}
+
+/** Measure outlier impact as proportion of excess time above 1.5*IQR threshold. */
+export function outlierImpactRatio(samples: number[]): number {
+  if (samples.length === 0) return 0;
+  const med = median(samples);
+  const q75 = percentile(samples, 0.75);
+  const threshold = med + 1.5 * (q75 - med);
+
+  let excessTime = 0;
+  for (const sample of samples) {
+    if (sample > threshold) excessTime += sample - med;
+  }
+  const total = samples.reduce((a, b) => a + b, 0);
+  return total > 0 ? excessTime / total : 0;
+}
+
+/** Group samples by V8 optimization tier and count deopts. */
+export function analyzeOptStatus(
+  samples: number[],
+  statuses: number[],
+): OptStatusInfo | undefined {
+  if (statuses.length === 0 || statuses[0] === undefined) return undefined;
+
+  const byStatus = new Map<number, number[]>();
+  let deoptCount = 0;
+  for (let i = 0; i < samples.length; i++) {
+    const status = statuses[i];
+    if (status === undefined) continue;
+    if (status & 8) deoptCount++; // deopt flag (bit 3)
+    const group = byStatus.get(status);
+    if (group) group.push(samples[i]);
+    else byStatus.set(status, [samples[i]]);
+  }
+
+  const entries = [...byStatus].map(([status, times]) => {
+    const name = optStatusNames[status] || `status=${status}`;
+    return [name, { count: times.length, medianMs: median(times) }] as const;
+  });
+  return { byTier: Object.fromEntries(entries), deoptCount };
+}
+
+/** @return runtime gc() function, or a no-op if --expose-gc wasn't passed. */
+export function gcFunction(): () => void {
+  const gc = globalThis.gc ?? (globalThis as any).__gc;
+  if (gc) return gc;
+  console.warn("gc() not available, run node/bun with --expose-gc");
+  return () => {};
+}
+
+/** @return function that reads V8 optimization status via %GetOptimizationStatus. */
+export function createOptStatusGetter(): ((fn: unknown) => number) | undefined {
+  try {
+    // %GetOptimizationStatus returns a bitmask
+    const fn = new Function("f", "return %GetOptimizationStatus(f)");
+    fn(() => {});
+    return fn as (fn: unknown) => number;
+  } catch {
+    return undefined;
+  }
+}