benchforge 0.1.8 → 0.1.11

package/src/runners/RunnerOrchestrator.ts

@@ -18,9 +18,16 @@ import type {
   RunMessage,
 } from "./WorkerScript.ts";
 
-const logTiming = debugWorkerTiming
-  ? (message: string) => console.log(`[RunnerOrchestrator] ${message}`)
-  : () => {};
+/** Parameters for running a matrix variant in worker */
+export interface RunMatrixVariantParams {
+  variantDir: string;
+  variantId: string;
+  caseId: string;
+  caseData?: unknown;
+  casesModule?: string;
+  runner: KnownRunner;
+  options: RunnerOptions;
+}
 
 type WorkerParams<T = unknown> = {
   spec: BenchmarkSpec<T>;
@@ -42,6 +49,10 @@ interface RunBenchmarkParams<T = unknown> {
   params?: T;
 }
 
+const logTiming = debugWorkerTiming
+  ? (message: string) => console.log(`[RunnerOrchestrator] ${message}`)
+  : () => {};
+
 /** Execute benchmarks directly or in worker process */
 export async function runBenchmark<T = unknown>({
   spec,
@@ -69,6 +80,34 @@ export async function runBenchmark<T = unknown>({
   return runInWorker({ spec, runner, options, params });
 }
 
+/** Run a matrix variant benchmark in isolated worker process */
+export async function runMatrixVariant(
+  params: RunMatrixVariantParams,
+): Promise<MeasuredResults[]> {
+  const {
+    variantDir,
+    variantId,
+    caseId,
+    caseData,
+    casesModule,
+    runner,
+    options,
+  } = params;
+  const name = `${variantId}/${caseId}`;
+  const message: RunMessage = {
+    type: "run",
+    spec: { name, fn: () => {} },
+    runnerName: runner,
+    options,
+    variantDir,
+    variantId,
+    caseId,
+    caseData,
+    casesModule,
+  };
+  return runWorkerWithMessage(name, options, message);
+}
+
 /** Resolve modulePath/exportName to a real function for non-worker mode */
 async function resolveModuleSpec<T>(
   spec: BenchmarkSpec<T>,
@@ -109,38 +148,6 @@ async function runInWorker<T>(
   return runWorkerWithMessage(spec.name, options, msg);
 }
 
-/** Create worker process with timing logs */
-function createWorkerWithTiming(gcStats: boolean) {
-  const workerStart = getPerfNow();
-  const gcEvents: GcEvent[] = [];
-  const worker = createWorkerProcess(gcStats);
-  const createTime = getPerfNow();
-  if (gcStats && worker.stdout) setupGcCapture(worker, gcEvents);
-  logTiming(
-    `Worker process created in ${getElapsed(workerStart, createTime).toFixed(1)}ms`,
-  );
-  return { worker, createTime, gcEvents };
-}
-
-/** Capture and parse GC lines from stdout (V8's --trace-gc-nvp outputs to stdout) */
-function setupGcCapture(worker: ChildProcess, gcEvents: GcEvent[]): void {
-  let buffer = "";
-  worker.stdout!.on("data", (data: Buffer) => {
-    buffer += data.toString();
-    const lines = buffer.split("\n");
-    buffer = lines.pop() || ""; // Keep incomplete line in buffer
-    for (const line of lines) {
-      const event = parseGcLine(line);
-      if (event) {
-        gcEvents.push(event);
-      } else if (line.trim()) {
-        // Forward non-GC stdout to console (worker status messages)
-        process.stdout.write(line + "\n");
-      }
-    }
-  });
-}
-
 /** Spawn worker, wire handlers, send message, return results */
 function runWorkerWithMessage(
   name: string,
@@ -166,17 +173,69 @@ function runWorkerWithMessage(
   });
 }
 
-/** Send message to worker with timing log */
-function sendWorkerMessage(
-  worker: ReturnType<typeof createWorkerProcess>,
-  message: RunMessage,
-  createTime: number,
-): void {
-  const messageTime = getPerfNow();
-  worker.send(message);
+/** Create message for worker execution */
+function createRunMessage<T>(
+  spec: BenchmarkSpec<T>,
+  runnerName: KnownRunner,
+  options: RunnerOptions,
+  params?: T,
+): RunMessage {
+  const { fn, ...rest } = spec;
+  const message: RunMessage = {
+    type: "run",
+    spec: rest as BenchmarkSpec,
+    runnerName,
+    options,
+    params,
+  };
+  if (spec.modulePath) {
+    message.modulePath = spec.modulePath;
+    message.exportName = spec.exportName;
+    if (spec.setupExportName) message.setupExportName = spec.setupExportName;
+  } else {
+    message.fnCode = fn.toString();
+  }
+  return message;
+}
+
+/** Create worker process with timing logs */
+function createWorkerWithTiming(gcStats: boolean) {
+  const workerStart = getPerfNow();
+  const gcEvents: GcEvent[] = [];
+  const worker = createWorkerProcess(gcStats);
+  const createTime = getPerfNow();
+  if (gcStats && worker.stdout) setupGcCapture(worker, gcEvents);
   logTiming(
-    `Message sent to worker in ${getElapsed(createTime, messageTime).toFixed(1)}ms`,
+    `Worker process created in ${getElapsed(workerStart, createTime).toFixed(1)}ms`,
   );
+  return { worker, createTime, gcEvents };
+}
+
+// Consider: --no-compilation-cache, --max-old-space-size=512, --no-lazy
+// for consistency (less realistic)
+
+/** @return handlers that attach GC stats and heap profile to results */
+function createWorkerHandlers(
+  specName: string,
+  startTime: number,
+  gcEvents: GcEvent[] | undefined,
+  resolve: (results: MeasuredResults[]) => void,
+  reject: (error: Error) => void,
+): WorkerHandlers {
+  return {
+    resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => {
+      logTiming(
+        `Total worker time for ${specName}: ${getElapsed(startTime).toFixed(1)}ms`,
+      );
+      if (gcEvents?.length) {
+        const gcStats = aggregateGcStats(gcEvents);
+        for (const r of results) r.gcStats = gcStats;
+      }
+      if (heapProfile) for (const r of results) r.heapProfile = heapProfile;
+      resolve(results);
+    },
+    reject,
+  };
 }
 
 /** Setup worker event handlers with cleanup */
@@ -195,6 +254,71 @@ function setupWorkerHandlers(
   worker.on("exit", createExitHandler(specName, cleanup, reject));
 }
 
+/** Send message to worker with timing log */
+function sendWorkerMessage(
+  worker: ReturnType<typeof createWorkerProcess>,
+  message: RunMessage,
+  createTime: number,
+): void {
+  const messageTime = getPerfNow();
+  worker.send(message);
+  logTiming(
+    `Message sent to worker in ${getElapsed(createTime, messageTime).toFixed(1)}ms`,
+  );
+}
+
+/** Create worker process with configuration */
+function createWorkerProcess(gcStats: boolean) {
+  const workerPath = resolveWorkerPath();
+  const execArgv = ["--expose-gc", "--allow-natives-syntax"];
+  if (gcStats) execArgv.push("--trace-gc-nvp");
+
+  return fork(workerPath, [], {
+    execArgv,
+    silent: gcStats, // Capture stdout/stderr when collecting GC stats
+    env: {
+      ...process.env,
+      NODE_OPTIONS: "",
+    },
+  });
+}
+
+/** Capture and parse GC lines from stdout (V8's --trace-gc-nvp outputs to stdout) */
+function setupGcCapture(worker: ChildProcess, gcEvents: GcEvent[]): void {
+  let buffer = "";
+  worker.stdout!.on("data", (data: Buffer) => {
+    buffer += data.toString();
+    const lines = buffer.split("\n");
+    buffer = lines.pop() || ""; // Keep incomplete line in buffer
+    for (const line of lines) {
+      const event = parseGcLine(line);
+      if (event) {
+        gcEvents.push(event);
+      } else if (line.trim()) {
+        // Forward non-GC stdout to console (worker status messages)
+        process.stdout.write(line + "\n");
+      }
+    }
+  });
+}
+
+/** Create cleanup for timeout and termination */
+function createCleanup(
+  worker: ReturnType<typeof createWorkerProcess>,
+  specName: string,
+  reject: (error: Error) => void,
+) {
+  const timeoutId = setTimeout(() => {
+    cleanup();
+    reject(new Error(`Benchmark "${specName}" timed out after 60 seconds`));
+  }, 60000);
+  const cleanup = () => {
+    clearTimeout(timeoutId);
+    if (!worker.killed) worker.kill("SIGTERM");
+  };
+  return cleanup;
+}
+
 /** Handle worker messages (results or errors) */
 function createMessageHandler(
   specName: string,
@@ -245,39 +369,6 @@ function createExitHandler(
   };
 }
 
-/** Create cleanup for timeout and termination */
-function createCleanup(
-  worker: ReturnType<typeof createWorkerProcess>,
-  specName: string,
-  reject: (error: Error) => void,
-) {
-  const timeoutId = setTimeout(() => {
-    cleanup();
-    reject(new Error(`Benchmark "${specName}" timed out after 60 seconds`));
-  }, 60000);
-  const cleanup = () => {
-    clearTimeout(timeoutId);
-    if (!worker.killed) worker.kill("SIGTERM");
-  };
-  return cleanup;
-}
-
-/** Create worker process with configuration */
-function createWorkerProcess(gcStats: boolean) {
-  const workerPath = resolveWorkerPath();
-  const execArgv = ["--expose-gc", "--allow-natives-syntax"];
-  if (gcStats) execArgv.push("--trace-gc-nvp");
-
-  return fork(workerPath, [], {
-    execArgv,
-    silent: gcStats, // Capture stdout/stderr when collecting GC stats
-    env: {
-      ...process.env,
-      NODE_OPTIONS: "",
-    },
-  });
-}
-
 /** Resolve WorkerScript path for dev (.ts) or dist (.mjs) */
 function resolveWorkerPath(): string {
   const dir = import.meta.dirname!;
@@ -285,94 +376,3 @@ function resolveWorkerPath(): string {
   if (existsSync(tsPath)) return tsPath;
   return path.join(dir, "runners", "WorkerScript.mjs");
 }
-
-// Consider: --no-compilation-cache, --max-old-space-size=512, --no-lazy
-// for consistency (less realistic)
-
-/** @return handlers that attach GC stats and heap profile to results */
-function createWorkerHandlers(
-  specName: string,
-  startTime: number,
-  gcEvents: GcEvent[] | undefined,
-  resolve: (results: MeasuredResults[]) => void,
-  reject: (error: Error) => void,
-): WorkerHandlers {
-  return {
-    resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => {
-      logTiming(
-        `Total worker time for ${specName}: ${getElapsed(startTime).toFixed(1)}ms`,
-      );
-      if (gcEvents?.length) {
-        const gcStats = aggregateGcStats(gcEvents);
-        for (const r of results) r.gcStats = gcStats;
-      }
-      if (heapProfile) for (const r of results) r.heapProfile = heapProfile;
-      resolve(results);
-    },
-    reject,
-  };
-}
-
-/** Create message for worker execution */
-function createRunMessage<T>(
-  spec: BenchmarkSpec<T>,
-  runnerName: KnownRunner,
-  options: RunnerOptions,
-  params?: T,
-): RunMessage {
-  const { fn, ...rest } = spec;
-  const message: RunMessage = {
-    type: "run",
-    spec: rest as BenchmarkSpec,
-    runnerName,
-    options,
-    params,
-  };
-  if (spec.modulePath) {
-    message.modulePath = spec.modulePath;
-    message.exportName = spec.exportName;
-    if (spec.setupExportName) message.setupExportName = spec.setupExportName;
-  } else {
-    message.fnCode = fn.toString();
-  }
-  return message;
-}
-
-/** Parameters for running a matrix variant in worker */
-export interface RunMatrixVariantParams {
-  variantDir: string;
-  variantId: string;
-  caseId: string;
-  caseData?: unknown;
-  casesModule?: string;
-  runner: KnownRunner;
-  options: RunnerOptions;
-}
-
-/** Run a matrix variant benchmark in isolated worker process */
-export async function runMatrixVariant(
-  params: RunMatrixVariantParams,
-): Promise<MeasuredResults[]> {
-  const {
-    variantDir,
-    variantId,
-    caseId,
-    caseData,
-    casesModule,
-    runner,
-    options,
-  } = params;
-  const name = `${variantId}/${caseId}`;
-  const message: RunMessage = {
-    type: "run",
-    spec: { name, fn: () => {} },
-    runnerName: runner,
-    options,
-    variantDir,
-    variantId,
-    caseId,
-    caseData,
-    casesModule,
-  };
-  return runWorkerWithMessage(name, options, message);
-}

package/src/runners/WorkerScript.ts

@@ -9,10 +9,7 @@ import {
 } from "./AdaptiveWrapper.ts";
 import type { RunnerOptions } from "./BenchRunner.ts";
 import { createRunner, type KnownRunner } from "./CreateRunner.ts";
-import { debugWorkerTiming, getElapsed, getPerfNow } from "./TimingUtils.ts";
-
-const workerStartTime = getPerfNow();
-const maxLifetime = 5 * 60 * 1000; // 5 minutes
+import { debugWorkerTiming, getElapsed, getPerfNow } from "./TimingUtils.ts"; // 5 minutes
 
 /** Message sent to worker process to start a benchmark run. */
 export interface RunMessage {
@@ -49,71 +46,13 @@ export interface ErrorMessage {
 
 export type WorkerMessage = RunMessage | ResultMessage | ErrorMessage;
 
-/**
- * Worker process for isolated benchmark execution.
- * Uses eval() safely in isolated child process with trusted code.
- */
-process.on("message", async (message: RunMessage) => {
-  if (message.type !== "run") return;
-
-  logTiming(`Processing ${message.spec.name} with ${message.runnerName}`);
-
-  try {
-    const start = getPerfNow();
-    const baseRunner = await createRunner(message.runnerName);
-
-    const runner = (message.options as any).adaptive
-      ? createAdaptiveWrapper(baseRunner, message.options as AdaptiveOptions)
-      : baseRunner;
-
-    logTiming("Runner created in", getElapsed(start));
-
-    const benchStart = getPerfNow();
-
-    // Run with heap sampling if enabled (covers module import + execution)
-    if (message.options.heapSample) {
-      const { withHeapSampling } = await import(
-        "../heap-sample/HeapSampler.ts"
-      );
-      const heapOpts = {
-        samplingInterval: message.options.heapInterval,
-        stackDepth: message.options.heapDepth,
-      };
-      const { result: results, profile: heapProfile } = await withHeapSampling(
-        heapOpts,
-        async () => {
-          const { fn, params } = await resolveBenchmarkFn(message);
-          return runner.runBench(
-            { ...message.spec, fn },
-            message.options,
-            params,
-          );
-        },
-      );
-      logTiming("Benchmark execution took", getElapsed(benchStart));
-      sendAndExit({ type: "result", results, heapProfile }, 0);
-    } else {
-      const { fn, params } = await resolveBenchmarkFn(message);
-      const results = await runner.runBench(
-        { ...message.spec, fn },
-        message.options,
-        params,
-      );
-      logTiming("Benchmark execution took", getElapsed(benchStart));
-      sendAndExit({ type: "result", results }, 0);
-    }
-  } catch (error) {
-    sendAndExit(createErrorMessage(error), 1);
-  }
-});
-
-// Exit after 5 minutes to prevent zombie processes
-setTimeout(() => {
-  console.error("WorkerScript: Maximum lifetime exceeded, exiting");
-  process.exit(1);
-}, maxLifetime);
+interface BenchmarkImportResult {
+  fn: BenchmarkFunction;
+  params: unknown;
+}
 
-process.stdin.pause();
+const workerStartTime = getPerfNow();
+const maxLifetime = 5 * 60 * 1000;
 
 /** Log timing with consistent format */
 const logTiming = debugWorkerTiming ? _logTiming : () => {};
@@ -138,11 +77,6 @@ function sendAndExit(msg: ResultMessage | ErrorMessage, exitCode: number) {
   });
 }
 
-interface BenchmarkImportResult {
-  fn: BenchmarkFunction;
-  params: unknown;
-}
-
 /** Resolve benchmark function from message (variant dir, module path, or fnCode) */
 async function resolveBenchmarkFn(
   message: RunMessage,
@@ -187,19 +121,6 @@ async function importVariantModule(
   return { fn: () => run(caseData), params: undefined };
 }
 
-/** Load case data from a cases module */
-async function loadCaseFromModule(
-  casesModuleUrl: string,
-  caseId: string,
-): Promise<{ data: unknown; metadata?: Record<string, unknown> }> {
-  logTiming(`Loading case '${caseId}' from ${casesModuleUrl}`);
-  const module = await import(casesModuleUrl);
-  if (typeof module.loadCase === "function") {
-    return module.loadCase(caseId);
-  }
-  return { data: caseId };
-}
-
 /** Import benchmark function and optionally run setup */
 async function importBenchmarkWithSetup(
   message: RunMessage,
@@ -222,6 +143,29 @@ async function importBenchmarkWithSetup(
   return { fn, params };
 }
 
+/** Reconstruct function from string code */
+function reconstructFunction(fnCode: string): BenchmarkFunction {
+  // biome-ignore lint/security/noGlobalEval: Necessary for worker process isolation, code is from trusted source
+  const fn = eval(`(${fnCode})`); // eslint-disable-line no-eval
+  if (typeof fn !== "function") {
+    throw new Error("Reconstructed code is not a function");
+  }
+  return fn;
+}
+
+/** Load case data from a cases module */
+async function loadCaseFromModule(
+  casesModuleUrl: string,
+  caseId: string,
+): Promise<{ data: unknown; metadata?: Record<string, unknown> }> {
+  logTiming(`Loading case '${caseId}' from ${casesModuleUrl}`);
+  const module = await import(casesModuleUrl);
+  if (typeof module.loadCase === "function") {
+    return module.loadCase(caseId);
+  }
+  return { data: caseId };
+}
+
 /** Get named or default export from module */
 function getModuleExport(
   module: any,
@@ -236,16 +180,6 @@ function getModuleExport(
   return fn;
 }
 
-/** Reconstruct function from string code */
-function reconstructFunction(fnCode: string): BenchmarkFunction {
-  // biome-ignore lint/security/noGlobalEval: Necessary for worker process isolation, code is from trusted source
-  const fn = eval(`(${fnCode})`); // eslint-disable-line no-eval
-  if (typeof fn !== "function") {
-    throw new Error("Reconstructed code is not a function");
-  }
-  return fn;
-}
-
 /** Create error message from exception */
 function createErrorMessage(error: unknown): ErrorMessage {
   return {
@@ -254,3 +188,69 @@ function createErrorMessage(error: unknown): ErrorMessage {
     stack: error instanceof Error ? error.stack : undefined,
   };
 }
+
+/**
+ * Worker process for isolated benchmark execution.
+ * Uses eval() safely in isolated child process with trusted code.
+ */
+process.on("message", async (message: RunMessage) => {
+  if (message.type !== "run") return;
+
+  logTiming(`Processing ${message.spec.name} with ${message.runnerName}`);
+
+  try {
+    const start = getPerfNow();
+    const baseRunner = await createRunner(message.runnerName);
+
+    const runner = (message.options as any).adaptive
+      ? createAdaptiveWrapper(baseRunner, message.options as AdaptiveOptions)
+      : baseRunner;
+
+    logTiming("Runner created in", getElapsed(start));
+
+    const benchStart = getPerfNow();
+
+    // Run with heap sampling if enabled (covers module import + execution)
+    if (message.options.heapSample) {
+      const { withHeapSampling } = await import(
+        "../heap-sample/HeapSampler.ts"
+      );
+      const heapOpts = {
+        samplingInterval: message.options.heapInterval,
+        stackDepth: message.options.heapDepth,
+      };
+      const { result: results, profile: heapProfile } = await withHeapSampling(
+        heapOpts,
+        async () => {
+          const { fn, params } = await resolveBenchmarkFn(message);
+          return runner.runBench(
+            { ...message.spec, fn },
+            message.options,
+            params,
+          );
+        },
+      );
+      logTiming("Benchmark execution took", getElapsed(benchStart));
+      sendAndExit({ type: "result", results, heapProfile }, 0);
+    } else {
+      const { fn, params } = await resolveBenchmarkFn(message);
+      const results = await runner.runBench(
+        { ...message.spec, fn },
+        message.options,
+        params,
+      );
+      logTiming("Benchmark execution took", getElapsed(benchStart));
+      sendAndExit({ type: "result", results }, 0);
+    }
+  } catch (error) {
+    sendAndExit(createErrorMessage(error), 1);
+  }
+});
+
+// Exit after 5 minutes to prevent zombie processes
+setTimeout(() => {
+  console.error("WorkerScript: Maximum lifetime exceeded, exiting");
+  process.exit(1);
+}, maxLifetime);
+
+process.stdin.pause();