overtake 1.3.2 → 2.0.0

package/src/cli.ts

@@ -1,9 +1,7 @@
-import { createRequire, Module } from 'node:module';
-import { fileURLToPath, pathToFileURL } from 'node:url';
-import { SyntheticModule, createContext, SourceTextModule } from 'node:vm';
-import { stat, readFile, writeFile } from 'node:fs/promises';
-import { Command, Option } from 'commander';
-import { glob } from 'glob';
+import { createRequire, register } from 'node:module';
+import { parseArgs } from 'node:util';
+import { pathToFileURL } from 'node:url';
+import { stat, readFile, writeFile, glob } from 'node:fs/promises';
 import {
   Benchmark,
   printTableReports,
@@ -13,152 +11,149 @@ import {
   printHistogramReports,
   printComparisonReports,
   reportsToBaseline,
-  BaselineData,
+  type BaselineData,
   DEFAULT_REPORT_TYPES,
   DEFAULT_WORKERS,
-} from './index.js';
-import { transpile } from './utils.js';
-import { REPORT_TYPES } from './types.js';
+} from './index.ts';
+import { REPORT_TYPES } from './types.ts';
+import { resolveHookUrl } from './utils.ts';
+
+register(resolveHookUrl);
 
 const require = createRequire(import.meta.url);
-const { name, description, version } = require('../package.json');
+const { name, version, description } = require('../package.json');
 const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
 
-const commander = new Command();
-
-commander
-  .name(name)
-  .description(description)
-  .version(version)
-  .argument('<paths...>', 'glob pattern to find benchmarks')
-  .addOption(new Option('-r, --report-types [reportTypes...]', 'statistic types to include in the report').choices(REPORT_TYPES).default(DEFAULT_REPORT_TYPES))
-  .addOption(new Option('-w, --workers [workers]', 'number of concurent workers').default(DEFAULT_WORKERS).argParser(parseInt))
-  .addOption(new Option('-f, --format [format]', 'output format').default('simple').choices(['simple', 'json', 'pjson', 'table', 'markdown', 'histogram']))
-  .addOption(new Option('--abs-threshold [absThreshold]', 'absolute error threshold in nanoseconds').argParser(parseFloat))
-  .addOption(new Option('--rel-threshold [relThreshold]', 'relative error threshold (fraction between 0 and 1)').argParser(parseFloat))
-  .addOption(new Option('--warmup-cycles [warmupCycles]', 'number of warmup cycles before measuring').argParser(parseInt))
-  .addOption(new Option('--max-cycles [maxCycles]', 'maximum measurement cycles per feed').argParser(parseInt))
-  .addOption(new Option('--min-cycles [minCycles]', 'minimum measurement cycles per feed').argParser(parseInt))
-  .addOption(new Option('--no-gc-observer', 'disable GC overlap detection'))
-  .addOption(new Option('--progress', 'show progress bar during benchmark execution'))
-  .addOption(new Option('--save-baseline <file>', 'save benchmark results to baseline file'))
-  .addOption(new Option('--compare-baseline <file>', 'compare results against baseline file'))
-  .action(async (patterns: string[], executeOptions) => {
-    let baseline: BaselineData | null = null;
-    if (executeOptions.compareBaseline) {
-      try {
-        const content = await readFile(executeOptions.compareBaseline, 'utf8');
-        baseline = JSON.parse(content) as BaselineData;
-      } catch {
-        console.error(`Warning: Could not load baseline file: ${executeOptions.compareBaseline}`);
+const FORMATS = ['simple', 'json', 'pjson', 'table', 'markdown', 'histogram'] as const;
+
+const { values: opts, positionals: patterns } = parseArgs({
+  args: process.argv.slice(2),
+  allowPositionals: true,
+  options: {
+    'report-types': { type: 'string', short: 'r', multiple: true },
+    workers: { type: 'string', short: 'w' },
+    format: { type: 'string', short: 'f' },
+    'abs-threshold': { type: 'string' },
+    'rel-threshold': { type: 'string' },
+    'warmup-cycles': { type: 'string' },
+    'max-cycles': { type: 'string' },
+    'min-cycles': { type: 'string' },
+    'no-gc-observer': { type: 'boolean' },
+    progress: { type: 'boolean' },
+    'save-baseline': { type: 'string' },
+    'compare-baseline': { type: 'string' },
+    help: { type: 'boolean', short: 'h' },
+    version: { type: 'boolean', short: 'v' },
+  },
+});
+
+if (opts.version) {
+  console.log(version);
+  process.exit(0);
+}
+
+if (opts.help || patterns.length === 0) {
+  console.log(`${name} v${version} - ${description}
+
+Usage: overtake [options] <paths...>
+
+Options:
+  -r, --report-types <type>       statistic type, repeat for multiple (-r ops -r p99)
+  -w, --workers <n>               number of concurrent workers (default: ${DEFAULT_WORKERS})
+  -f, --format <format>           output format: ${FORMATS.join(', ')} (default: simple)
+  --abs-threshold <ns>            absolute error threshold in nanoseconds
+  --rel-threshold <frac>          relative error threshold (0-1)
+  --warmup-cycles <n>             warmup cycles before measuring
+  --max-cycles <n>                maximum measurement cycles per feed
+  --min-cycles <n>                minimum measurement cycles per feed
+  --no-gc-observer                disable GC overlap detection
+  --progress                      show progress bar
+  --save-baseline <file>          save results to baseline file
+  --compare-baseline <file>       compare results against baseline file
+  -v, --version                   show version
+  -h, --help                      show this help`);
+  process.exit(0);
+}
+
+const reportTypes = opts['report-types']?.length
+  ? opts['report-types'].filter((t): t is (typeof REPORT_TYPES)[number] => REPORT_TYPES.includes(t as (typeof REPORT_TYPES)[number]))
+  : DEFAULT_REPORT_TYPES;
+const format = opts.format && FORMATS.includes(opts.format as (typeof FORMATS)[number]) ? opts.format : 'simple';
+
+const executeOptions = {
+  reportTypes,
+  workers: opts.workers ? parseInt(opts.workers) : DEFAULT_WORKERS,
+  absThreshold: opts['abs-threshold'] ? parseFloat(opts['abs-threshold']) : undefined,
+  relThreshold: opts['rel-threshold'] ? parseFloat(opts['rel-threshold']) : undefined,
+  warmupCycles: opts['warmup-cycles'] ? parseInt(opts['warmup-cycles']) : undefined,
+  maxCycles: opts['max-cycles'] ? parseInt(opts['max-cycles']) : undefined,
+  minCycles: opts['min-cycles'] ? parseInt(opts['min-cycles']) : undefined,
+  gcObserver: !opts['no-gc-observer'],
+  progress: opts.progress ?? false,
+  format,
+};
+
+let baseline: BaselineData | null = null;
+if (opts['compare-baseline']) {
+  try {
+    const content = await readFile(opts['compare-baseline'], 'utf8');
+    baseline = JSON.parse(content) as BaselineData;
+  } catch {
+    console.error(`Warning: Could not load baseline file: ${opts['compare-baseline']}`);
+  }
+}
+
+const files = new Set((await Promise.all(patterns.map((pattern) => Array.fromAsync(glob(pattern, { cwd: process.cwd() })).catch(() => [] as string[])))).flat());
+
+for (const file of files) {
+  const stats = await stat(file).catch(() => false as const);
+  if (stats && stats.isFile()) {
+    const identifier = pathToFileURL(file).href;
+    let instance: Benchmark<unknown> | undefined;
+    (globalThis as any).benchmark = (...args: Parameters<(typeof Benchmark)['create']>) => {
+      if (instance) {
+        throw new Error('Only one benchmark per file is supported');
       }
-    }
+      instance = Benchmark.create(...args);
+      return instance;
+    };
+    await import(identifier);
 
-    const files = new Set<string>();
-    await Promise.all(
-      patterns.map(async (pattern) => {
-        const matches = await glob(pattern, { absolute: true, cwd: process.cwd() }).catch(() => []);
-        matches.forEach((file) => files.add(file));
-      }),
-    );
-
-    for (const file of files) {
-      const stats = await stat(file).catch(() => false as const);
-      if (stats && stats.isFile()) {
-        const content = await readFile(file, 'utf8');
-        const identifier = pathToFileURL(file).href;
-        const code = await transpile(content);
-        let instance: Benchmark<unknown> | undefined;
-        const benchmark = (...args: Parameters<(typeof Benchmark)['create']>) => {
-          if (instance) {
-            throw new Error('Only one benchmark per file is supported');
-          }
-          instance = Benchmark.create(...args);
-          return instance;
-        };
-        const globals = Object.create(null);
-        for (const k of Object.getOwnPropertyNames(globalThis)) {
-          globals[k] = (globalThis as any)[k];
-        }
-        globals.benchmark = benchmark;
-        const script = new SourceTextModule(code, {
-          identifier,
-          context: createContext(globals),
-          initializeImportMeta(meta) {
-            meta.url = identifier;
-          },
-          async importModuleDynamically(specifier, referencingModule) {
-            if (Module.isBuiltin(specifier)) {
-              return import(specifier);
-            }
-            const baseIdentifier = referencingModule.identifier ?? identifier;
-            const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
-            const resolved = resolveFrom.resolve(specifier);
-            return import(resolved);
-          },
-        });
-        const imports = new Map<string, SyntheticModule>();
-        await script.link(async (specifier: string, referencingModule) => {
-          const baseIdentifier = referencingModule.identifier ?? identifier;
-          const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
-          const target = Module.isBuiltin(specifier) ? specifier : resolveFrom.resolve(specifier);
-          const cached = imports.get(target);
-          if (cached) {
-            return cached;
-          }
-          const mod = await import(target);
-          const exportNames = Object.keys(mod);
-          const imported = new SyntheticModule(
-            exportNames,
-            () => {
-              exportNames.forEach((key) => imported.setExport(key, mod[key]));
-            },
-            { identifier: target, context: referencingModule.context },
-          );
-
-          imports.set(target, imported);
-          return imported;
-        });
-        await script.evaluate();
-
-        if (instance) {
-          const reports = await instance.execute({
-            ...executeOptions,
-            [BENCHMARK_URL]: identifier,
-          } as typeof executeOptions);
-
-          if (executeOptions.saveBaseline) {
-            const baselineData = reportsToBaseline(reports);
-            await writeFile(executeOptions.saveBaseline, JSON.stringify(baselineData, null, 2));
-            console.log(`Baseline saved to: ${executeOptions.saveBaseline}`);
-          }
-
-          if (baseline) {
-            printComparisonReports(reports, baseline);
-          } else {
-            switch (executeOptions.format) {
-              case 'json':
-                printJSONReports(reports);
-                break;
-              case 'pjson':
-                printJSONReports(reports, 2);
-                break;
-              case 'table':
-                printTableReports(reports);
-                break;
-              case 'markdown':
-                printMarkdownReports(reports);
-                break;
-              case 'histogram':
-                printHistogramReports(reports);
-                break;
-              default:
-                printSimpleReports(reports);
-            }
-          }
+    if (instance) {
+      const reports = await instance.execute({
+        ...executeOptions,
+        [BENCHMARK_URL]: identifier,
+      } as typeof executeOptions);
+
+      if (opts['save-baseline']) {
+        const baselineData = reportsToBaseline(reports);
+        await writeFile(opts['save-baseline'], JSON.stringify(baselineData, null, 2));
+        console.log(`Baseline saved to: ${opts['save-baseline']}`);
+      }
+
+      if (baseline) {
+        printComparisonReports(reports, baseline);
+      } else {
+        switch (format) {
+          case 'json':
+            printJSONReports(reports);
+            break;
+          case 'pjson':
+            printJSONReports(reports, 2);
+            break;
+          case 'table':
+            printTableReports(reports);
+            break;
+          case 'markdown':
+            printMarkdownReports(reports);
+            break;
+          case 'histogram':
+            printHistogramReports(reports);
+            break;
+          default:
+            printSimpleReports(reports);
         }
       }
     }
-  });
-
-commander.parse(process.argv);
+  }
+}

package/src/executor.ts

@@ -1,26 +1,26 @@
 import { Worker } from 'node:worker_threads';
 import { once } from 'node:events';
-import { queue } from 'async';
 import { pathToFileURL } from 'node:url';
-import { createReport, Report } from './reporter.js';
-import { cmp } from './utils.js';
+import { createReport, computeStats, Report } from './reporter.ts';
+import { cmp, assertNoClosure } from './utils.ts';
 import {
-  ExecutorRunOptions,
-  ReportOptions,
-  WorkerOptions,
-  BenchmarkOptions,
+  type ExecutorRunOptions,
+  type ReportOptions,
+  type WorkerOptions,
+  type BenchmarkOptions,
   Control,
-  ReportType,
-  ReportTypeList,
+  type ReportType,
+  type ReportTypeList,
   CONTROL_SLOTS,
   COMPLETE_VALUE,
-  ProgressCallback,
-} from './types.js';
+  type ProgressCallback,
+} from './types.ts';
 
 export type ExecutorReport<R extends ReportTypeList> = Record<R[number], Report> & {
   count: number;
   heapUsedKB: number;
   dceWarning: boolean;
+  error?: string;
 };
 
 export interface ExecutorOptions<R extends ReportTypeList> extends BenchmarkOptions, ReportOptions<R> {
@@ -32,22 +32,58 @@ export interface ExecutorOptions<R extends ReportTypeList> extends BenchmarkOpti
 
 const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
 
-export const createExecutor = <TContext, TInput, R extends ReportTypeList>(options: Required<ExecutorOptions<R>>) => {
+export interface Executor<TContext, TInput> {
+  pushAsync<T>(task: ExecutorRunOptions<TContext, TInput>): Promise<T>;
+  kill(): void;
+}
+
+export const createExecutor = <TContext, TInput, R extends ReportTypeList>(options: Required<ExecutorOptions<R>>): Executor<TContext, TInput> => {
   const { workers, warmupCycles, maxCycles, minCycles, absThreshold, relThreshold, gcObserver = true, reportTypes, onProgress, progressInterval = 100 } = options;
   const benchmarkUrl = (options as Record<symbol, unknown>)[BENCHMARK_URL];
   const resolvedBenchmarkUrl = typeof benchmarkUrl === 'string' ? benchmarkUrl : pathToFileURL(process.cwd()).href;
 
-  const executor = queue<ExecutorRunOptions<TContext, TInput>>(async ({ id, setup, teardown, pre, run, post, data }) => {
+  const pending: { task: ExecutorRunOptions<TContext, TInput>; resolve: (v: unknown) => void; reject: (e: unknown) => void }[] = [];
+  let running = 0;
+
+  const schedule = async (task: ExecutorRunOptions<TContext, TInput>) => {
+    running++;
+    try {
+      return await runTask(task);
+    } finally {
+      running--;
+      if (pending.length > 0) {
+        const next = pending.shift()!;
+        schedule(next.task).then(next.resolve, next.reject);
+      }
+    }
+  };
+
+  const pushAsync = <T>(task: ExecutorRunOptions<TContext, TInput>): Promise<T> => {
+    if (running < workers) {
+      return schedule(task) as Promise<T>;
+    }
+    return new Promise<T>((resolve, reject) => {
+      pending.push({ task, resolve: resolve as (v: unknown) => void, reject });
+    });
+  };
+
+  const runTask = async ({ id, setup, teardown, pre, run, post, data }: ExecutorRunOptions<TContext, TInput>) => {
     const setupCode = setup?.toString();
     const teardownCode = teardown?.toString();
     const preCode = pre?.toString();
-    const runCode = run.toString()!;
+    const runCode = run.toString();
     const postCode = post?.toString();
 
+    if (setupCode) assertNoClosure(setupCode, 'setup');
+    if (teardownCode) assertNoClosure(teardownCode, 'teardown');
+    if (preCode) assertNoClosure(preCode, 'pre');
+    assertNoClosure(runCode, 'run');
+    if (postCode) assertNoClosure(postCode, 'post');
+
     const controlSAB = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT * CONTROL_SLOTS);
     const durationsSAB = new SharedArrayBuffer(BigUint64Array.BYTES_PER_ELEMENT * maxCycles);
 
-    const workerFile = new URL('./worker.js', import.meta.url);
+    const workerFile = new URL('./worker.ts', import.meta.url);
     const workerData: WorkerOptions = {
       benchmarkUrl: resolvedBenchmarkUrl,
       setupCode,
@@ -83,17 +119,18 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>(optio
     const WORKER_TIMEOUT_MS = 300_000;
     const exitPromise = once(worker, 'exit');
     const timeoutId = setTimeout(() => worker.terminate(), WORKER_TIMEOUT_MS);
+    let workerError: string | undefined;
     try {
       const [exitCode] = await exitPromise;
       clearTimeout(timeoutId);
       if (progressIntervalId) clearInterval(progressIntervalId);
       if (exitCode !== 0) {
-        throw new Error(`worker exited with code ${exitCode}`);
+        workerError = `worker exited with code ${exitCode}`;
       }
     } catch (err) {
       clearTimeout(timeoutId);
       if (progressIntervalId) clearInterval(progressIntervalId);
-      throw err;
+      workerError = err instanceof Error ? err.message : String(err);
     }
 
     const count = control[Control.INDEX];
@@ -112,19 +149,17 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>(optio
       }
     }
 
+    const stats = count > 0 ? computeStats(durations) : undefined;
     const report = reportTypes
-      .map<[string, unknown]>((type) => [type, createReport(durations, type)] as [ReportType, Report])
+      .map<[string, unknown]>((type) => [type, createReport(durations, type, stats)] as [ReportType, Report])
       .concat([
         ['count', count],
         ['heapUsedKB', heapUsedKB],
         ['dceWarning', dceWarning],
+        ['error', workerError],
       ]);
     return Object.fromEntries(report);
-  }, workers);
-
-  executor.error((err) => {
-    console.error(err);
-  });
+  };
 
-  return executor;
+  return { pushAsync, kill() {} };
 };