benchforge 0.1.9 → 0.2.4

package/src/runners/RunnerOrchestrator.ts

@@ -1,16 +1,19 @@
 import { type ChildProcess, fork } from "node:child_process";
 import { existsSync } from "node:fs";
 import path from "node:path";
-import type { BenchmarkSpec } from "../Benchmark.ts";
-import type { HeapProfile } from "../heap-sample/HeapSampler.ts";
-import type { MeasuredResults } from "../MeasuredResults.ts";
-import {
-  type AdaptiveOptions,
-  createAdaptiveWrapper,
-} from "./AdaptiveWrapper.ts";
+import type { CoverageData } from "../profiling/node/CoverageTypes.ts";
+import type { HeapProfile } from "../profiling/node/HeapSampler.ts";
+import type { TimeProfile } from "../profiling/node/TimeSampler.ts";
+import type { BenchmarkFunction, BenchmarkSpec } from "./BenchmarkSpec.ts";
 import type { RunnerOptions } from "./BenchRunner.ts";
-import { createRunner, type KnownRunner } from "./CreateRunner.ts";
+import type { KnownRunner } from "./CreateRunner.ts";
 import { aggregateGcStats, type GcEvent, parseGcLine } from "./GcStats.ts";
+import type { MeasuredResults } from "./MeasuredResults.ts";
+import {
+  createBenchRunner,
+  importBenchFn,
+  resolveVariantFn,
+} from "./RunnerUtils.ts";
 import { debugWorkerTiming, getElapsed, getPerfNow } from "./TimingUtils.ts";
 import type {
   ErrorMessage,
@@ -18,21 +21,17 @@ import type {
   RunMessage,
 } from "./WorkerScript.ts";
 
-const logTiming = debugWorkerTiming
-  ? (message: string) => console.log(`[RunnerOrchestrator] ${message}`)
-  : () => {};
-
-type WorkerParams<T = unknown> = {
-  spec: BenchmarkSpec<T>;
+/** Parameters for running a matrix variant */
+export interface RunMatrixVariantParams {
+  variantDir: string;
+  variantId: string;
+  caseId: string;
+  caseData?: unknown;
+  casesModule?: string;
   runner: KnownRunner;
   options: RunnerOptions;
-  params?: T;
-};
-
-type WorkerHandlers = {
-  resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => void;
-  reject: (error: Error) => void;
-};
+  useWorker?: boolean;
+}
 
 interface RunBenchmarkParams<T = unknown> {
   spec: BenchmarkSpec<T>;
@@ -42,7 +41,11 @@ interface RunBenchmarkParams<T = unknown> {
   params?: T;
 }
 
-/** Execute benchmarks directly or in worker process */
+const logTiming = debugWorkerTiming
+  ? (message: string) => console.log(`[RunnerOrchestrator] ${message}`)
+  : () => {};
+
+/** Run a benchmark spec, optionally in an isolated worker process for profiling support. */
 export async function runBenchmark<T = unknown>({
   spec,
   runner,
@@ -51,22 +54,39 @@ export async function runBenchmark<T = unknown>({
   params,
 }: RunBenchmarkParams<T>): Promise<MeasuredResults[]> {
   if (!useWorker) {
-    const resolvedSpec = spec.modulePath
+    const resolved = spec.modulePath
       ? await resolveModuleSpec(spec, params)
       : { spec, params };
-
-    const base = await createRunner(runner);
-    const benchRunner = (options as any).adaptive
-      ? createAdaptiveWrapper(base, options as AdaptiveOptions)
-      : base;
-    return benchRunner.runBench(
-      resolvedSpec.spec,
-      options,
-      resolvedSpec.params,
-    );
+    const benchRunner = await createBenchRunner(runner, options);
+    return benchRunner.runBench(resolved.spec, options, resolved.params);
   }
 
-  return runInWorker({ spec, runner, options, params });
+  const msg = createRunMessage(spec, runner, options, params);
+  return runWorkerWithMessage(spec.name, options, msg);
+}
+
+/** Run a matrix variant benchmark, directly or in a worker. */
+export async function runMatrixVariant(
+  params: RunMatrixVariantParams,
+): Promise<MeasuredResults[]> {
+  const { variantId, caseId, runner, options, useWorker = true } = params;
+  const name = `${variantId}/${caseId}`;
+
+  if (!useWorker) return runMatrixVariantDirect(params, name);
+
+  const { variantDir, caseData, casesModule } = params;
+  const message: RunMessage = {
+    type: "run",
+    spec: { name } as BenchmarkSpec,
+    runnerName: runner,
+    options,
+    variantDir,
+    variantId,
+    caseId,
+    caseData,
+    casesModule,
+  };
+  return runWorkerWithMessage(name, options, message);
 }
 
 /** Resolve modulePath/exportName to a real function for non-worker mode */
@@ -74,74 +94,43 @@ async function resolveModuleSpec<T>(
   spec: BenchmarkSpec<T>,
   params: T | undefined,
 ): Promise<{ spec: BenchmarkSpec<T>; params: T | undefined }> {
-  const module = await import(spec.modulePath!);
-
-  const fn = spec.exportName
-    ? module[spec.exportName]
-    : module.default || module;
-
-  if (typeof fn !== "function") {
-    const name = spec.exportName || "default";
-    throw new Error(
-      `Export '${name}' from ${spec.modulePath} is not a function`,
-    );
-  }
-
-  let resolvedParams = params;
-  if (spec.setupExportName) {
-    const setupFn = module[spec.setupExportName];
-    if (typeof setupFn !== "function") {
-      const msg = `Setup export '${spec.setupExportName}' from ${spec.modulePath} is not a function`;
-      throw new Error(msg);
-    }
-    resolvedParams = await setupFn(params);
-  }
-
-  return { spec: { ...spec, fn }, params: resolvedParams };
-}
-
-/** Run benchmark in isolated worker process */
-async function runInWorker<T>(
-  workerParams: WorkerParams<T>,
-): Promise<MeasuredResults[]> {
-  const { spec, runner, options, params } = workerParams;
-  const msg = createRunMessage(spec, runner, options, params);
-  return runWorkerWithMessage(spec.name, options, msg);
-}
-
-/** Create worker process with timing logs */
-function createWorkerWithTiming(gcStats: boolean) {
-  const workerStart = getPerfNow();
-  const gcEvents: GcEvent[] = [];
-  const worker = createWorkerProcess(gcStats);
-  const createTime = getPerfNow();
-  if (gcStats && worker.stdout) setupGcCapture(worker, gcEvents);
-  logTiming(
-    `Worker process created in ${getElapsed(workerStart, createTime).toFixed(1)}ms`,
+  const { modulePath, exportName, setupExportName } = spec;
+  const imported = await importBenchFn(
+    modulePath!,
+    exportName,
+    setupExportName,
+    params,
   );
-  return { worker, createTime, gcEvents };
+  const fn = imported.fn as BenchmarkFunction<T>;
+  return { spec: { ...spec, fn }, params: imported.params as T | undefined };
 }
 
-/** Capture and parse GC lines from stdout (V8's --trace-gc-nvp outputs to stdout) */
-function setupGcCapture(worker: ChildProcess, gcEvents: GcEvent[]): void {
-  let buffer = "";
-  worker.stdout!.on("data", (data: Buffer) => {
-    buffer += data.toString();
-    const lines = buffer.split("\n");
-    buffer = lines.pop() || ""; // Keep incomplete line in buffer
-    for (const line of lines) {
-      const event = parseGcLine(line);
-      if (event) {
-        gcEvents.push(event);
-      } else if (line.trim()) {
-        // Forward non-GC stdout to console (worker status messages)
-        process.stdout.write(line + "\n");
-      }
-    }
-  });
+/** Serialize a BenchmarkSpec into a worker-safe message (modulePath or fnCode) */
+function createRunMessage<T>(
+  spec: BenchmarkSpec<T>,
+  runnerName: KnownRunner,
+  options: RunnerOptions,
+  params?: T,
+): RunMessage {
+  const { fn, ...rest } = spec;
+  const message: RunMessage = {
+    type: "run",
+    spec: rest as BenchmarkSpec,
+    runnerName,
+    options,
+    params,
+  };
+  if (spec.modulePath) {
+    message.modulePath = spec.modulePath;
+    message.exportName = spec.exportName;
+    if (spec.setupExportName) message.setupExportName = spec.setupExportName;
+  } else {
+    message.fnCode = fn.toString();
+  }
+  return message;
 }
 
-/** Spawn worker, wire handlers, send message, return results */
+/** Run a benchmark in an isolated worker process with timeout and GC capture. */
 function runWorkerWithMessage(
   name: string,
   options: RunnerOptions,
@@ -152,227 +141,122 @@ function runWorkerWithMessage(
   logTiming(`Starting worker for ${name}`);
 
   return new Promise((resolve, reject) => {
-    const { worker, createTime, gcEvents } =
-      createWorkerWithTiming(collectGcStats);
-    const handlers = createWorkerHandlers(
-      name,
-      startTime,
-      gcEvents,
-      resolve,
-      reject,
-    );
-    setupWorkerHandlers(worker, name, handlers);
-    sendWorkerMessage(worker, message, createTime);
-  });
-}
-
-/** Send message to worker with timing log */
-function sendWorkerMessage(
-  worker: ReturnType<typeof createWorkerProcess>,
-  message: RunMessage,
-  createTime: number,
-): void {
-  const messageTime = getPerfNow();
-  worker.send(message);
-  logTiming(
-    `Message sent to worker in ${getElapsed(createTime, messageTime).toFixed(1)}ms`,
-  );
-}
+    const gcEvents: GcEvent[] = [];
+    const worker = spawnWorkerProcess(collectGcStats);
+    if (collectGcStats && worker.stdout) setupGcCapture(worker, gcEvents);
 
-/** Setup worker event handlers with cleanup */
-function setupWorkerHandlers(
-  worker: ReturnType<typeof createWorkerProcess>,
-  specName: string,
-  handlers: WorkerHandlers,
-) {
-  const { resolve, reject } = handlers;
-  const cleanup = createCleanup(worker, specName, reject);
-  worker.on(
-    "message",
-    createMessageHandler(specName, cleanup, resolve, reject),
-  );
-  worker.on("error", createErrorHandler(specName, cleanup, reject));
-  worker.on("exit", createExitHandler(specName, cleanup, reject));
-}
+    const timeoutId = setTimeout(() => {
+      killWorker();
+      reject(new Error(`Benchmark "${name}" timed out after 60 seconds`));
+    }, 60000);
 
-/** Handle worker messages (results or errors) */
-function createMessageHandler(
-  specName: string,
-  cleanup: () => void,
-  resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => void,
-  reject: (error: Error) => void,
-) {
-  return (msg: ResultMessage | ErrorMessage) => {
-    cleanup();
-    if (msg.type === "result") {
-      resolve(msg.results, msg.heapProfile);
-    } else if (msg.type === "error") {
-      const error = new Error(`Benchmark "${specName}" failed: ${msg.error}`);
-      if (msg.stack) error.stack = msg.stack;
-      reject(error);
+    function killWorker() {
+      clearTimeout(timeoutId);
+      if (!worker.killed) worker.kill("SIGTERM");
     }
-  };
-}
 
-/** Handle worker process errors */
-function createErrorHandler(
-  specName: string,
-  cleanup: () => void,
-  reject: (error: Error) => void,
-) {
-  return (error: Error) => {
-    cleanup();
-    reject(
-      new Error(
-        `Worker process failed for benchmark "${specName}": ${error.message}`,
-      ),
-    );
-  };
-}
-
-/** Handle worker process exit */
-function createExitHandler(
-  specName: string,
-  cleanup: () => void,
-  reject: (error: Error) => void,
-) {
-  return (code: number | null, _signal: NodeJS.Signals | null) => {
-    if (code !== 0 && code !== null) {
-      cleanup();
-      const msg = `Worker exited with code ${code} for benchmark "${specName}"`;
+    worker.on("message", (msg: ResultMessage | ErrorMessage) => {
+      killWorker();
+      if (msg.type === "error") {
+        const error = new Error(`Benchmark "${name}" failed: ${msg.error}`);
+        if (msg.stack) error.stack = msg.stack;
+        return reject(error);
+      }
+      const elapsed = getElapsed(startTime).toFixed(1);
+      logTiming(`Total worker time for ${name}: ${elapsed}ms`);
+      const { results, heapProfile, timeProfile, coverage } = msg;
+      attachProfilingData(
+        results,
+        gcEvents,
+        heapProfile,
+        timeProfile,
+        coverage,
+      );
+      resolve(results);
+    });
+    worker.on("error", (error: Error) => {
+      killWorker();
+      const msg = `Worker process failed for "${name}": ${error.message}`;
       reject(new Error(msg));
-    }
-  };
+    });
+    worker.on("exit", (code: number | null) => {
+      if (code !== 0 && code !== null) {
+        killWorker();
+        reject(new Error(`Worker exited with code ${code} for "${name}"`));
+      }
+    });
+
+    worker.send(message);
+  });
 }
 
-/** Create cleanup for timeout and termination */
-function createCleanup(
-  worker: ReturnType<typeof createWorkerProcess>,
-  specName: string,
-  reject: (error: Error) => void,
-) {
-  const timeoutId = setTimeout(() => {
-    cleanup();
-    reject(new Error(`Benchmark "${specName}" timed out after 60 seconds`));
-  }, 60000);
-  const cleanup = () => {
-    clearTimeout(timeoutId);
-    if (!worker.killed) worker.kill("SIGTERM");
-  };
-  return cleanup;
+/** Run matrix variant in-process (no worker isolation) */
+async function runMatrixVariantDirect(
+  params: RunMatrixVariantParams,
+  name: string,
+): Promise<MeasuredResults[]> {
+  const { runner, options } = params;
+  const { fn } = await resolveVariantFn(params);
+  const benchRunner = await createBenchRunner(runner, options);
+  return benchRunner.runBench({ name, fn }, options);
 }
 
-/** Create worker process with configuration */
-function createWorkerProcess(gcStats: boolean) {
+/** Spawn worker process with V8 flags */
+function spawnWorkerProcess(gcStats: boolean) {
   const workerPath = resolveWorkerPath();
   const execArgv = ["--expose-gc", "--allow-natives-syntax"];
   if (gcStats) execArgv.push("--trace-gc-nvp");
 
+  const env = { ...process.env, NODE_OPTIONS: "" };
+  // silent mode captures stdout so we can parse --trace-gc-nvp output
   return fork(workerPath, [], {
     execArgv,
-    silent: gcStats, // Capture stdout/stderr when collecting GC stats
-    env: {
-      ...process.env,
-      NODE_OPTIONS: "",
-    },
+    silent: gcStats,
+    env,
+    serialization: "advanced",
   });
 }
 
-/** Resolve WorkerScript path for dev (.ts) or dist (.mjs) */
-function resolveWorkerPath(): string {
-  const dir = import.meta.dirname!;
-  const tsPath = path.join(dir, "WorkerScript.ts");
-  if (existsSync(tsPath)) return tsPath;
-  return path.join(dir, "runners", "WorkerScript.mjs");
+/** Capture and parse GC lines from worker stdout (--trace-gc-nvp). */
+function setupGcCapture(worker: ChildProcess, gcEvents: GcEvent[]): void {
+  let buffer = "";
+  worker.stdout!.on("data", (data: Buffer) => {
+    buffer += data.toString();
+    const lines = buffer.split("\n");
+    buffer = lines.pop() || "";
+    for (const line of lines) {
+      const event = parseGcLine(line);
+      if (event) gcEvents.push(event);
+      else if (line.trim()) process.stdout.write(line + "\n");
+    }
+  });
 }
 
-// Consider: --no-compilation-cache, --max-old-space-size=512, --no-lazy
-// for consistency (less realistic)
-
-/** @return handlers that attach GC stats and heap profile to results */
-function createWorkerHandlers(
-  specName: string,
-  startTime: number,
+/** Attach profiling data collected by the worker to each result. */
+function attachProfilingData(
+  results: MeasuredResults[],
   gcEvents: GcEvent[] | undefined,
-  resolve: (results: MeasuredResults[]) => void,
-  reject: (error: Error) => void,
-): WorkerHandlers {
-  return {
-    resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => {
-      logTiming(
-        `Total worker time for ${specName}: ${getElapsed(startTime).toFixed(1)}ms`,
-      );
-      if (gcEvents?.length) {
-        const gcStats = aggregateGcStats(gcEvents);
-        for (const r of results) r.gcStats = gcStats;
-      }
-      if (heapProfile) for (const r of results) r.heapProfile = heapProfile;
-      resolve(results);
-    },
-    reject,
-  };
-}
-
-/** Create message for worker execution */
-function createRunMessage<T>(
-  spec: BenchmarkSpec<T>,
-  runnerName: KnownRunner,
-  options: RunnerOptions,
-  params?: T,
-): RunMessage {
-  const { fn, ...rest } = spec;
-  const message: RunMessage = {
-    type: "run",
-    spec: rest as BenchmarkSpec,
-    runnerName,
-    options,
-    params,
+  heapProfile?: HeapProfile,
+  timeProfile?: TimeProfile,
+  coverage?: CoverageData,
+): void {
+  const gcStats = gcEvents?.length ? aggregateGcStats(gcEvents) : undefined;
+  const attach = <K extends keyof MeasuredResults>(
+    key: K,
+    value: MeasuredResults[K] | undefined,
+  ) => {
+    if (value) for (const r of results) r[key] = value;
   };
-  if (spec.modulePath) {
-    message.modulePath = spec.modulePath;
-    message.exportName = spec.exportName;
-    if (spec.setupExportName) message.setupExportName = spec.setupExportName;
-  } else {
-    message.fnCode = fn.toString();
-  }
-  return message;
+  attach("gcStats", gcStats);
+  attach("heapProfile", heapProfile);
+  attach("timeProfile", timeProfile);
+  attach("coverage", coverage);
 }
 
-/** Parameters for running a matrix variant in worker */
-export interface RunMatrixVariantParams {
-  variantDir: string;
-  variantId: string;
-  caseId: string;
-  caseData?: unknown;
-  casesModule?: string;
-  runner: KnownRunner;
-  options: RunnerOptions;
-}
-
-/** Run a matrix variant benchmark in isolated worker process */
-export async function runMatrixVariant(
-  params: RunMatrixVariantParams,
-): Promise<MeasuredResults[]> {
-  const {
-    variantDir,
-    variantId,
-    caseId,
-    caseData,
-    casesModule,
-    runner,
-    options,
-  } = params;
-  const name = `${variantId}/${caseId}`;
-  const message: RunMessage = {
-    type: "run",
-    spec: { name, fn: () => {} },
-    runnerName: runner,
-    options,
-    variantDir,
-    variantId,
-    caseId,
-    caseData,
-    casesModule,
-  };
-  return runWorkerWithMessage(name, options, message);
+/** Resolve WorkerScript path for dev (.ts) or dist (.mjs) */
+function resolveWorkerPath(): string {
+  const dir = import.meta.dirname!;
+  const tsPath = path.join(dir, "WorkerScript.ts");
+  if (existsSync(tsPath)) return tsPath;
+  return path.join(dir, "runners", "WorkerScript.mjs");
 }

package/src/runners/RunnerUtils.ts

@@ -1,2 +1,76 @@
+import { prepareBenchFn } from "../matrix/BenchMatrix.ts";
+import { loadCaseData, loadCasesModule } from "../matrix/CaseLoader.ts";
+import { loadVariant } from "../matrix/VariantLoader.ts";
+import {
+  type AdaptiveOptions,
+  createAdaptiveWrapper,
+} from "./AdaptiveWrapper.ts";
+import type { BenchmarkFunction } from "./BenchmarkSpec.ts";
+import type { BenchRunner, RunnerOptions } from "./BenchRunner.ts";
+import { createRunner, type KnownRunner } from "./CreateRunner.ts";
+
 export const msToNs = 1e6;
-export const nsToMs = 1e-6;
+
+/** Get named or default export from module, throw if not a function */
+// biome-ignore lint/complexity/noBannedTypes: generic function constraint
+export function getModuleExport<T extends Function = Function>(
+  module: any,
+  exportName: string | undefined,
+  modulePath: string,
+): T {
+  const fn = exportName ? module[exportName] : module.default || module;
+  if (typeof fn !== "function") {
+    const name = exportName || "default";
+    throw new Error(`Export '${name}' from ${modulePath} is not a function`);
+  }
+  return fn as T;
+}
+
+/** Import a benchmark function from a module, optionally running a setup export */
+export async function importBenchFn(
+  modulePath: string,
+  exportName: string | undefined,
+  setupExportName: string | undefined,
+  params: unknown,
+): Promise<{ fn: BenchmarkFunction; params: unknown }> {
+  const module = await import(modulePath);
+  const fn = getModuleExport<BenchmarkFunction>(module, exportName, modulePath);
+  if (!setupExportName) return { fn, params };
+
+  const setup = getModuleExport<BenchmarkFunction>(
+    module,
+    setupExportName,
+    modulePath,
+  );
+  return { fn, params: await setup(params) };
+}
+
+/** Resolve a matrix variant to a benchmark function (shared by orchestrator and worker). */
+export async function resolveVariantFn(params: {
+  variantDir: string;
+  variantId: string;
+  caseId?: string;
+  caseData?: unknown;
+  casesModule?: string;
+}): Promise<{ fn: BenchmarkFunction; params: undefined }> {
+  let { caseData } = params;
+  if (params.casesModule && params.caseId) {
+    const cases = await loadCasesModule(params.casesModule);
+    caseData = (await loadCaseData(cases, params.caseId)).data;
+  }
+  const variant = await loadVariant(params.variantDir, params.variantId);
+  const fn = await prepareBenchFn(variant, caseData);
+  return { fn, params: undefined };
+}
+
+/** Create runner, wrapping with adaptive sampling if options.adaptive is set */
+export async function createBenchRunner(
+  runnerName: KnownRunner,
+  options: RunnerOptions,
+): Promise<BenchRunner> {
+  const base = await createRunner(runnerName);
+  if ("adaptive" in options && options.adaptive) {
+    return createAdaptiveWrapper(base, options as AdaptiveOptions);
+  }
+  return base;
+}