benchforge 0.1.0

package/src/runners/RunnerOrchestrator.ts

@@ -0,0 +1,374 @@
+import { type ChildProcess, fork } from "node:child_process";
+import path from "node:path";
+import type { BenchmarkSpec } from "../Benchmark.ts";
+import type { HeapProfile } from "../heap-sample/HeapSampler.ts";
+import type { MeasuredResults } from "../MeasuredResults.ts";
+import {
+  type AdaptiveOptions,
+  createAdaptiveWrapper,
+} from "./AdaptiveWrapper.ts";
+import type { RunnerOptions } from "./BenchRunner.ts";
+import { createRunner, type KnownRunner } from "./CreateRunner.ts";
+import { aggregateGcStats, type GcEvent, parseGcLine } from "./GcStats.ts";
+import { debugWorkerTiming, getElapsed, getPerfNow } from "./TimingUtils.ts";
+import type {
+  ErrorMessage,
+  ResultMessage,
+  RunMessage,
+} from "./WorkerScript.ts";
+
+const logTiming = debugWorkerTiming
+  ? (message: string) => console.log(`[RunnerOrchestrator] ${message}`)
+  : () => {};
+
+type WorkerParams<T = unknown> = {
+  spec: BenchmarkSpec<T>;
+  runner: KnownRunner;
+  options: RunnerOptions;
+  params?: T;
+};
+
+type WorkerHandlers = {
+  resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => void;
+  reject: (error: Error) => void;
+};
+
+interface RunBenchmarkParams<T = unknown> {
+  spec: BenchmarkSpec<T>;
+  runner: KnownRunner;
+  options: RunnerOptions;
+  useWorker?: boolean;
+  params?: T;
+}
+
+/** Execute benchmarks directly or in worker process */
+export async function runBenchmark<T = unknown>({
+  spec,
+  runner,
+  options,
+  useWorker = false,
+  params,
+}: RunBenchmarkParams<T>): Promise<MeasuredResults[]> {
+  if (!useWorker) {
+    const resolvedSpec = spec.modulePath
+      ? await resolveModuleSpec(spec, params)
+      : { spec, params };
+
+    const base = await createRunner(runner);
+    const benchRunner = (options as any).adaptive
+      ? createAdaptiveWrapper(base, options as AdaptiveOptions)
+      : base;
+    return benchRunner.runBench(
+      resolvedSpec.spec,
+      options,
+      resolvedSpec.params,
+    );
+  }
+
+  return runInWorker({ spec, runner, options, params });
+}
+
+/** Resolve modulePath/exportName to a real function for non-worker mode */
+async function resolveModuleSpec<T>(
+  spec: BenchmarkSpec<T>,
+  params: T | undefined,
+): Promise<{ spec: BenchmarkSpec<T>; params: T | undefined }> {
+  const module = await import(spec.modulePath!);
+
+  const fn = spec.exportName
+    ? module[spec.exportName]
+    : module.default || module;
+
+  if (typeof fn !== "function") {
+    const name = spec.exportName || "default";
+    throw new Error(
+      `Export '${name}' from ${spec.modulePath} is not a function`,
+    );
+  }
+
+  let resolvedParams = params;
+  if (spec.setupExportName) {
+    const setupFn = module[spec.setupExportName];
+    if (typeof setupFn !== "function") {
+      const msg = `Setup export '${spec.setupExportName}' from ${spec.modulePath} is not a function`;
+      throw new Error(msg);
+    }
+    resolvedParams = await setupFn(params);
+  }
+
+  return { spec: { ...spec, fn }, params: resolvedParams };
+}
+
+/** Run benchmark in isolated worker process */
+async function runInWorker<T>(
+  workerParams: WorkerParams<T>,
+): Promise<MeasuredResults[]> {
+  const { spec, runner, options, params } = workerParams;
+  const msg = createRunMessage(spec, runner, options, params);
+  return runWorkerWithMessage(spec.name, options, msg);
+}
+
+/** Create worker process with timing logs */
+function createWorkerWithTiming(gcStats: boolean) {
+  const workerStart = getPerfNow();
+  const gcEvents: GcEvent[] = [];
+  const worker = createWorkerProcess(gcStats);
+  const createTime = getPerfNow();
+  if (gcStats && worker.stdout) setupGcCapture(worker, gcEvents);
+  logTiming(
+    `Worker process created in ${getElapsed(workerStart, createTime).toFixed(1)}ms`,
+  );
+  return { worker, createTime, gcEvents };
+}
+
+/** Capture and parse GC lines from stdout (V8's --trace-gc-nvp outputs to stdout) */
+function setupGcCapture(worker: ChildProcess, gcEvents: GcEvent[]): void {
+  let buffer = "";
+  worker.stdout!.on("data", (data: Buffer) => {
+    buffer += data.toString();
+    const lines = buffer.split("\n");
+    buffer = lines.pop() || ""; // Keep incomplete line in buffer
+    for (const line of lines) {
+      const event = parseGcLine(line);
+      if (event) {
+        gcEvents.push(event);
+      } else if (line.trim()) {
+        // Forward non-GC stdout to console (worker status messages)
+        process.stdout.write(line + "\n");
+      }
+    }
+  });
+}
+
+/** Spawn worker, wire handlers, send message, return results */
+function runWorkerWithMessage(
+  name: string,
+  options: RunnerOptions,
+  message: RunMessage,
+): Promise<MeasuredResults[]> {
+  const startTime = getPerfNow();
+  const collectGcStats = options.gcStats ?? false;
+  logTiming(`Starting worker for ${name}`);
+
+  return new Promise((resolve, reject) => {
+    const { worker, createTime, gcEvents } =
+      createWorkerWithTiming(collectGcStats);
+    const handlers = createWorkerHandlers(
+      name,
+      startTime,
+      gcEvents,
+      resolve,
+      reject,
+    );
+    setupWorkerHandlers(worker, name, handlers);
+    sendWorkerMessage(worker, message, createTime);
+  });
+}
+
+/** Send message to worker with timing log */
+function sendWorkerMessage(
+  worker: ReturnType<typeof createWorkerProcess>,
+  message: RunMessage,
+  createTime: number,
+): void {
+  const messageTime = getPerfNow();
+  worker.send(message);
+  logTiming(
+    `Message sent to worker in ${getElapsed(createTime, messageTime).toFixed(1)}ms`,
+  );
+}
+
+/** Setup worker event handlers with cleanup */
+function setupWorkerHandlers(
+  worker: ReturnType<typeof createWorkerProcess>,
+  specName: string,
+  handlers: WorkerHandlers,
+) {
+  const { resolve, reject } = handlers;
+  const cleanup = createCleanup(worker, specName, reject);
+  worker.on(
+    "message",
+    createMessageHandler(specName, cleanup, resolve, reject),
+  );
+  worker.on("error", createErrorHandler(specName, cleanup, reject));
+  worker.on("exit", createExitHandler(specName, cleanup, reject));
+}
+
+/** Handle worker messages (results or errors) */
+function createMessageHandler(
+  specName: string,
+  cleanup: () => void,
+  resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => void,
+  reject: (error: Error) => void,
+) {
+  return (msg: ResultMessage | ErrorMessage) => {
+    cleanup();
+    if (msg.type === "result") {
+      resolve(msg.results, msg.heapProfile);
+    } else if (msg.type === "error") {
+      const error = new Error(`Benchmark "${specName}" failed: ${msg.error}`);
+      if (msg.stack) error.stack = msg.stack;
+      reject(error);
+    }
+  };
+}
+
+/** Handle worker process errors */
+function createErrorHandler(
+  specName: string,
+  cleanup: () => void,
+  reject: (error: Error) => void,
+) {
+  return (error: Error) => {
+    cleanup();
+    reject(
+      new Error(
+        `Worker process failed for benchmark "${specName}": ${error.message}`,
+      ),
+    );
+  };
+}
+
+/** Handle worker process exit */
+function createExitHandler(
+  specName: string,
+  cleanup: () => void,
+  reject: (error: Error) => void,
+) {
+  return (code: number | null, _signal: NodeJS.Signals | null) => {
+    if (code !== 0 && code !== null) {
+      cleanup();
+      const msg = `Worker exited with code ${code} for benchmark "${specName}"`;
+      reject(new Error(msg));
+    }
+  };
+}
+
+/** Create cleanup for timeout and termination */
+function createCleanup(
+  worker: ReturnType<typeof createWorkerProcess>,
+  specName: string,
+  reject: (error: Error) => void,
+) {
+  const timeoutId = setTimeout(() => {
+    cleanup();
+    reject(new Error(`Benchmark "${specName}" timed out after 60 seconds`));
+  }, 60000);
+  const cleanup = () => {
+    clearTimeout(timeoutId);
+    if (!worker.killed) worker.kill("SIGTERM");
+  };
+  return cleanup;
+}
+
+/** Create worker process with configuration */
+function createWorkerProcess(gcStats: boolean) {
+  const workerPath = path.join(import.meta.dirname!, "WorkerScript.ts");
+  const execArgv = [
+    "--expose-gc",
+    "--allow-natives-syntax",
+    "--experimental-strip-types",
+    "--no-warnings=ExperimentalWarning",
+  ];
+  if (gcStats) execArgv.push("--trace-gc-nvp");
+
+  return fork(workerPath, [], {
+    execArgv,
+    silent: gcStats, // Capture stdout/stderr when collecting GC stats
+    env: {
+      ...process.env,
+      NODE_OPTIONS: "",
+    },
+  });
+}
+
+// Consider: --no-compilation-cache, --max-old-space-size=512, --no-lazy
+// for consistency (less realistic)
+
+/** @return handlers that attach GC stats and heap profile to results */
+function createWorkerHandlers(
+  specName: string,
+  startTime: number,
+  gcEvents: GcEvent[] | undefined,
+  resolve: (results: MeasuredResults[]) => void,
+  reject: (error: Error) => void,
+): WorkerHandlers {
+  return {
+    resolve: (results: MeasuredResults[], heapProfile?: HeapProfile) => {
+      logTiming(
+        `Total worker time for ${specName}: ${getElapsed(startTime).toFixed(1)}ms`,
+      );
+      if (gcEvents?.length) {
+        const gcStats = aggregateGcStats(gcEvents);
+        for (const r of results) r.gcStats = gcStats;
+      }
+      if (heapProfile) for (const r of results) r.heapProfile = heapProfile;
+      resolve(results);
+    },
+    reject,
+  };
+}
+
+/** Create message for worker execution */
+function createRunMessage<T>(
+  spec: BenchmarkSpec<T>,
+  runnerName: KnownRunner,
+  options: RunnerOptions,
+  params?: T,
+): RunMessage {
+  const { fn, ...rest } = spec;
+  const message: RunMessage = {
+    type: "run",
+    spec: rest as BenchmarkSpec,
+    runnerName,
+    options,
+    params,
+  };
+  if (spec.modulePath) {
+    message.modulePath = spec.modulePath;
+    message.exportName = spec.exportName;
+    if (spec.setupExportName) message.setupExportName = spec.setupExportName;
+  } else {
+    message.fnCode = fn.toString();
+  }
+  return message;
+}
+
+/** Parameters for running a matrix variant in worker */
+export interface RunMatrixVariantParams {
+  variantDir: string;
+  variantId: string;
+  caseId: string;
+  caseData?: unknown;
+  casesModule?: string;
+  runner: KnownRunner;
+  options: RunnerOptions;
+}
+
+/** Run a matrix variant benchmark in isolated worker process */
+export async function runMatrixVariant(
+  params: RunMatrixVariantParams,
+): Promise<MeasuredResults[]> {
+  const {
+    variantDir,
+    variantId,
+    caseId,
+    caseData,
+    casesModule,
+    runner,
+    options,
+  } = params;
+  const name = `${variantId}/${caseId}`;
+  const message: RunMessage = {
+    type: "run",
+    spec: { name, fn: () => {} },
+    runnerName: runner,
+    options,
+    variantDir,
+    variantId,
+    caseId,
+    caseData,
+    casesModule,
+  };
+  return runWorkerWithMessage(name, options, message);
+}

package/src/runners/RunnerUtils.ts

@@ -0,0 +1,2 @@
+export const msToNs = 1e6;
+export const nsToMs = 1e-6;

package/src/runners/TimingUtils.ts

@@ -0,0 +1,13 @@
+export const debugWorkerTiming = false;
+
+/** Get current time or 0 if debugging disabled */
+export function getPerfNow(): number {
+  return debugWorkerTiming ? performance.now() : 0;
+}
+
+/** Calculate elapsed milliseconds between marks */
+export function getElapsed(startMark: number, endMark?: number): number {
+  if (!debugWorkerTiming) return 0;
+  const end = endMark ?? performance.now();
+  return end - startMark;
+}

package/src/runners/WorkerScript.ts

@@ -0,0 +1,256 @@
+#!/usr/bin/env node
+import type { BenchmarkFunction, BenchmarkSpec } from "../Benchmark.ts";
+import type { HeapProfile } from "../heap-sample/HeapSampler.ts";
+import type { MeasuredResults } from "../MeasuredResults.ts";
+import { variantModuleUrl } from "../matrix/VariantLoader.ts";
+import {
+  type AdaptiveOptions,
+  createAdaptiveWrapper,
+} from "./AdaptiveWrapper.ts";
+import type { RunnerOptions } from "./BenchRunner.ts";
+import { createRunner, type KnownRunner } from "./CreateRunner.ts";
+import { debugWorkerTiming, getElapsed, getPerfNow } from "./TimingUtils.ts";
+
+const workerStartTime = getPerfNow();
+const maxLifetime = 5 * 60 * 1000; // 5 minutes
+
+/** Message sent to worker process to start a benchmark run. */
+export interface RunMessage {
+  type: "run";
+  spec: BenchmarkSpec;
+  runnerName: KnownRunner;
+  options: RunnerOptions;
+  fnCode?: string; // Made optional - either fnCode or modulePath is required
+  modulePath?: string; // Path to module for dynamic import
+  exportName?: string; // Export name from module
+  setupExportName?: string; // Setup function export name - called once, result passed to fn
+  params?: unknown;
+  // Variant directory mode (BenchMatrix)
+  variantDir?: string; // Directory URL containing variant .ts files
+  variantId?: string; // Variant filename (without .ts)
+  caseData?: unknown; // Data to pass to variant
+  caseId?: string; // Case identifier
+  casesModule?: string; // URL to cases module (exports cases[] and loadCase())
+}
+
+/** Message returned from worker process with benchmark results. */
+export interface ResultMessage {
+  type: "result";
+  results: MeasuredResults[];
+  heapProfile?: HeapProfile;
+}
+
+/** Message returned from worker process when benchmark fails. */
+export interface ErrorMessage {
+  type: "error";
+  error: string;
+  stack?: string;
+}
+
+export type WorkerMessage = RunMessage | ResultMessage | ErrorMessage;
+
+/**
+ * Worker process for isolated benchmark execution.
+ * Uses eval() safely in isolated child process with trusted code.
+ */
+process.on("message", async (message: RunMessage) => {
+  if (message.type !== "run") return;
+
+  logTiming(`Processing ${message.spec.name} with ${message.runnerName}`);
+
+  try {
+    const start = getPerfNow();
+    const baseRunner = await createRunner(message.runnerName);
+
+    const runner = (message.options as any).adaptive
+      ? createAdaptiveWrapper(baseRunner, message.options as AdaptiveOptions)
+      : baseRunner;
+
+    logTiming("Runner created in", getElapsed(start));
+
+    const benchStart = getPerfNow();
+
+    // Run with heap sampling if enabled (covers module import + execution)
+    if (message.options.heapSample) {
+      const { withHeapSampling } = await import(
+        "../heap-sample/HeapSampler.ts"
+      );
+      const heapOpts = {
+        samplingInterval: message.options.heapInterval,
+        stackDepth: message.options.heapDepth,
+      };
+      const { result: results, profile: heapProfile } = await withHeapSampling(
+        heapOpts,
+        async () => {
+          const { fn, params } = await resolveBenchmarkFn(message);
+          return runner.runBench(
+            { ...message.spec, fn },
+            message.options,
+            params,
+          );
+        },
+      );
+      logTiming("Benchmark execution took", getElapsed(benchStart));
+      sendAndExit({ type: "result", results, heapProfile }, 0);
+    } else {
+      const { fn, params } = await resolveBenchmarkFn(message);
+      const results = await runner.runBench(
+        { ...message.spec, fn },
+        message.options,
+        params,
+      );
+      logTiming("Benchmark execution took", getElapsed(benchStart));
+      sendAndExit({ type: "result", results }, 0);
+    }
+  } catch (error) {
+    sendAndExit(createErrorMessage(error), 1);
+  }
+});
+
+// Exit after 5 minutes to prevent zombie processes
+setTimeout(() => {
+  console.error("WorkerScript: Maximum lifetime exceeded, exiting");
+  process.exit(1);
+}, maxLifetime);
+
+process.stdin.pause();
+
+/** Log timing with consistent format */
+const logTiming = debugWorkerTiming ? _logTiming : () => {};
+function _logTiming(operation: string, duration?: number) {
+  if (duration === undefined) {
+    console.log(`[Worker] ${operation}`);
+  } else {
+    console.log(`[Worker] ${operation} ${duration.toFixed(1)}ms`);
+  }
+}
+
+/** Send message and exit with duration log */
+function sendAndExit(msg: ResultMessage | ErrorMessage, exitCode: number) {
+  process.send!(msg, undefined, undefined, (err: Error | null): void => {
+    if (err) {
+      const kind = msg.type === "result" ? "results" : "error message";
+      console.error(`[Worker] Error sending ${kind}:`, err);
+    }
+    const suffix = exitCode === 0 ? "" : " (error)";
+    logTiming(`Total worker duration${suffix}:`, getElapsed(workerStartTime));
+    process.exit(exitCode);
+  });
+}
+
+interface BenchmarkImportResult {
+  fn: BenchmarkFunction;
+  params: unknown;
+}
+
+/** Resolve benchmark function from message (variant dir, module path, or fnCode) */
+async function resolveBenchmarkFn(
+  message: RunMessage,
+): Promise<BenchmarkImportResult> {
+  if (message.variantDir && message.variantId) {
+    return importVariantModule(message);
+  }
+  if (message.modulePath) {
+    return importBenchmarkWithSetup(message);
+  }
+  return { fn: reconstructFunction(message.fnCode!), params: message.params };
+}
+
+/** Import variant from directory and prepare benchmark function */
+async function importVariantModule(
+  message: RunMessage,
+): Promise<BenchmarkImportResult> {
+  const { variantDir, variantId, caseId, casesModule } = message;
+  let { caseData } = message;
+  const moduleUrl = variantModuleUrl(variantDir!, variantId!);
+  logTiming(`Importing variant ${variantId} from ${variantDir}`);
+
+  if (casesModule && caseId) {
+    caseData = (await loadCaseFromModule(casesModule, caseId)).data;
+  }
+
+  const module = await import(moduleUrl);
+  const { setup, run } = module;
+
+  if (typeof run !== "function") {
+    throw new Error(`Variant '${variantId}' must export 'run' function`);
+  }
+
+  // Stateful variant: setup returns state, run receives state
+  if (typeof setup === "function") {
+    logTiming(`Calling setup for ${variantId}`);
+    const state = await setup(caseData);
+    return { fn: () => run(state), params: undefined };
+  }
+
+  // Stateless variant: run receives caseData directly
+  return { fn: () => run(caseData), params: undefined };
+}
+
+/** Load case data from a cases module */
+async function loadCaseFromModule(
+  casesModuleUrl: string,
+  caseId: string,
+): Promise<{ data: unknown; metadata?: Record<string, unknown> }> {
+  logTiming(`Loading case '${caseId}' from ${casesModuleUrl}`);
+  const module = await import(casesModuleUrl);
+  if (typeof module.loadCase === "function") {
+    return module.loadCase(caseId);
+  }
+  return { data: caseId };
+}
+
+/** Import benchmark function and optionally run setup */
+async function importBenchmarkWithSetup(
+  message: RunMessage,
+): Promise<BenchmarkImportResult> {
+  const { modulePath, exportName, setupExportName, params } = message;
+  logTiming(
+    `Importing from ${modulePath}${exportName ? ` (${exportName})` : ""}`,
+  );
+  const module = await import(modulePath!);
+
+  const fn = getModuleExport(module, exportName, modulePath!);
+
+  if (setupExportName) {
+    logTiming(`Calling setup: ${setupExportName}`);
+    const setupFn = getModuleExport(module, setupExportName, modulePath!);
+    const setupResult = await setupFn(params);
+    return { fn, params: setupResult };
+  }
+
+  return { fn, params };
+}
+
+/** Get named or default export from module */
+function getModuleExport(
+  module: any,
+  exportName: string | undefined,
+  modulePath: string,
+): BenchmarkFunction {
+  const fn = exportName ? module[exportName] : module.default || module;
+  if (typeof fn !== "function") {
+    const name = exportName || "default";
+    throw new Error(`Export '${name}' from ${modulePath} is not a function`);
+  }
+  return fn;
+}
+
+/** Reconstruct function from string code */
+function reconstructFunction(fnCode: string): BenchmarkFunction {
+  // biome-ignore lint/security/noGlobalEval: Necessary for worker process isolation, code is from trusted source
+  const fn = eval(`(${fnCode})`); // eslint-disable-line no-eval
+  if (typeof fn !== "function") {
+    throw new Error("Reconstructed code is not a function");
+  }
+  return fn;
+}
+
+/** Create error message from exception */
+function createErrorMessage(error: unknown): ErrorMessage {
+  return {
+    type: "error",
+    error: error instanceof Error ? error.message : String(error),
+    stack: error instanceof Error ? error.stack : undefined,
+  };
+}

package/src/table-util/ConvergenceFormatters.ts

@@ -0,0 +1,19 @@
+import pico from "picocolors";
+
+const isTest = process.env.NODE_ENV === "test" || process.env.VITEST === "true";
+const { red } = isTest ? { red: (str: string) => str } : pico;
+
+const lowConfidence = 80;
+
+/** @return convergence percentage with color for low values */
+export function formatConvergence(v: unknown): string {
+  if (typeof v !== "number") return "—";
+  const pct = `${Math.round(v)}%`;
+  return v < lowConfidence ? red(pct) : pct;
+}
+
+/** @return coefficient of variation as ±percentage */
+export function formatCV(v: unknown): string {
+  if (typeof v !== "number") return "";
+  return `±${(v * 100).toFixed(1)}%`;
+}