overtake 1.4.0 → 2.0.0

package/src/cli.ts

@@ -1,9 +1,7 @@
-import { createRequire, Module } from 'node:module';
-import { fileURLToPath, pathToFileURL } from 'node:url';
-import { SyntheticModule, createContext, SourceTextModule } from 'node:vm';
-import { stat, readFile, writeFile } from 'node:fs/promises';
-import { Command, Option } from 'commander';
-import { glob } from 'glob';
+import { createRequire, register } from 'node:module';
+import { parseArgs } from 'node:util';
+import { pathToFileURL } from 'node:url';
+import { stat, readFile, writeFile, glob } from 'node:fs/promises';
 import {
   Benchmark,
   printTableReports,
@@ -13,152 +11,149 @@ import {
   printHistogramReports,
   printComparisonReports,
   reportsToBaseline,
-  BaselineData,
+  type BaselineData,
   DEFAULT_REPORT_TYPES,
   DEFAULT_WORKERS,
-} from './index.js';
-import { transpile } from './utils.js';
-import { REPORT_TYPES } from './types.js';
+} from './index.ts';
+import { REPORT_TYPES } from './types.ts';
+import { resolveHookUrl } from './utils.ts';
+
+register(resolveHookUrl);
 
 const require = createRequire(import.meta.url);
-const { name, description, version } = require('../package.json');
+const { name, version, description } = require('../package.json');
 const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
 
-const commander = new Command();
-
-commander
-  .name(name)
-  .description(description)
-  .version(version)
-  .argument('<paths...>', 'glob pattern to find benchmarks')
-  .addOption(new Option('-r, --report-types [reportTypes...]', 'statistic types to include in the report').choices(REPORT_TYPES).default(DEFAULT_REPORT_TYPES))
-  .addOption(new Option('-w, --workers [workers]', 'number of concurent workers').default(DEFAULT_WORKERS).argParser(parseInt))
-  .addOption(new Option('-f, --format [format]', 'output format').default('simple').choices(['simple', 'json', 'pjson', 'table', 'markdown', 'histogram']))
-  .addOption(new Option('--abs-threshold [absThreshold]', 'absolute error threshold in nanoseconds').argParser(parseFloat))
-  .addOption(new Option('--rel-threshold [relThreshold]', 'relative error threshold (fraction between 0 and 1)').argParser(parseFloat))
-  .addOption(new Option('--warmup-cycles [warmupCycles]', 'number of warmup cycles before measuring').argParser(parseInt))
-  .addOption(new Option('--max-cycles [maxCycles]', 'maximum measurement cycles per feed').argParser(parseInt))
-  .addOption(new Option('--min-cycles [minCycles]', 'minimum measurement cycles per feed').argParser(parseInt))
-  .addOption(new Option('--no-gc-observer', 'disable GC overlap detection'))
-  .addOption(new Option('--progress', 'show progress bar during benchmark execution'))
-  .addOption(new Option('--save-baseline <file>', 'save benchmark results to baseline file'))
-  .addOption(new Option('--compare-baseline <file>', 'compare results against baseline file'))
-  .action(async (patterns: string[], executeOptions) => {
-    let baseline: BaselineData | null = null;
-    if (executeOptions.compareBaseline) {
-      try {
-        const content = await readFile(executeOptions.compareBaseline, 'utf8');
-        baseline = JSON.parse(content) as BaselineData;
-      } catch {
-        console.error(`Warning: Could not load baseline file: ${executeOptions.compareBaseline}`);
+const FORMATS = ['simple', 'json', 'pjson', 'table', 'markdown', 'histogram'] as const;
+
+const { values: opts, positionals: patterns } = parseArgs({
+  args: process.argv.slice(2),
+  allowPositionals: true,
+  options: {
+    'report-types': { type: 'string', short: 'r', multiple: true },
+    workers: { type: 'string', short: 'w' },
+    format: { type: 'string', short: 'f' },
+    'abs-threshold': { type: 'string' },
+    'rel-threshold': { type: 'string' },
+    'warmup-cycles': { type: 'string' },
+    'max-cycles': { type: 'string' },
+    'min-cycles': { type: 'string' },
+    'no-gc-observer': { type: 'boolean' },
+    progress: { type: 'boolean' },
+    'save-baseline': { type: 'string' },
+    'compare-baseline': { type: 'string' },
+    help: { type: 'boolean', short: 'h' },
+    version: { type: 'boolean', short: 'v' },
+  },
+});
+
+if (opts.version) {
+  console.log(version);
+  process.exit(0);
+}
+
+if (opts.help || patterns.length === 0) {
+  console.log(`${name} v${version} - ${description}
+
+Usage: overtake [options] <paths...>
+
+Options:
+  -r, --report-types <type>       statistic type, repeat for multiple (-r ops -r p99)
+  -w, --workers <n>               number of concurrent workers (default: ${DEFAULT_WORKERS})
+  -f, --format <format>           output format: ${FORMATS.join(', ')} (default: simple)
+  --abs-threshold <ns>            absolute error threshold in nanoseconds
+  --rel-threshold <frac>          relative error threshold (0-1)
+  --warmup-cycles <n>             warmup cycles before measuring
+  --max-cycles <n>                maximum measurement cycles per feed
+  --min-cycles <n>                minimum measurement cycles per feed
+  --no-gc-observer                disable GC overlap detection
+  --progress                      show progress bar
+  --save-baseline <file>          save results to baseline file
+  --compare-baseline <file>       compare results against baseline file
+  -v, --version                   show version
+  -h, --help                      show this help`);
+  process.exit(0);
+}
+
+const reportTypes = opts['report-types']?.length
+  ? opts['report-types'].filter((t): t is (typeof REPORT_TYPES)[number] => REPORT_TYPES.includes(t as (typeof REPORT_TYPES)[number]))
+  : DEFAULT_REPORT_TYPES;
+const format = opts.format && FORMATS.includes(opts.format as (typeof FORMATS)[number]) ? opts.format : 'simple';
+
+const executeOptions = {
+  reportTypes,
+  workers: opts.workers ? parseInt(opts.workers) : DEFAULT_WORKERS,
+  absThreshold: opts['abs-threshold'] ? parseFloat(opts['abs-threshold']) : undefined,
+  relThreshold: opts['rel-threshold'] ? parseFloat(opts['rel-threshold']) : undefined,
+  warmupCycles: opts['warmup-cycles'] ? parseInt(opts['warmup-cycles']) : undefined,
+  maxCycles: opts['max-cycles'] ? parseInt(opts['max-cycles']) : undefined,
+  minCycles: opts['min-cycles'] ? parseInt(opts['min-cycles']) : undefined,
+  gcObserver: !opts['no-gc-observer'],
+  progress: opts.progress ?? false,
+  format,
+};
+
+let baseline: BaselineData | null = null;
+if (opts['compare-baseline']) {
+  try {
+    const content = await readFile(opts['compare-baseline'], 'utf8');
+    baseline = JSON.parse(content) as BaselineData;
+  } catch {
+    console.error(`Warning: Could not load baseline file: ${opts['compare-baseline']}`);
+  }
+}
+
+const files = new Set((await Promise.all(patterns.map((pattern) => Array.fromAsync(glob(pattern, { cwd: process.cwd() })).catch(() => [] as string[])))).flat());
+
+for (const file of files) {
+  const stats = await stat(file).catch(() => false as const);
+  if (stats && stats.isFile()) {
+    const identifier = pathToFileURL(file).href;
+    let instance: Benchmark<unknown> | undefined;
+    (globalThis as any).benchmark = (...args: Parameters<(typeof Benchmark)['create']>) => {
+      if (instance) {
+        throw new Error('Only one benchmark per file is supported');
       }
-    }
+      instance = Benchmark.create(...args);
+      return instance;
+    };
+    await import(identifier);
 
-    const files = new Set<string>();
-    await Promise.all(
-      patterns.map(async (pattern) => {
-        const matches = await glob(pattern, { absolute: true, cwd: process.cwd() }).catch(() => []);
-        matches.forEach((file) => files.add(file));
-      }),
-    );
-
-    for (const file of files) {
-      const stats = await stat(file).catch(() => false as const);
-      if (stats && stats.isFile()) {
-        const content = await readFile(file, 'utf8');
-        const identifier = pathToFileURL(file).href;
-        const code = await transpile(content);
-        let instance: Benchmark<unknown> | undefined;
-        const benchmark = (...args: Parameters<(typeof Benchmark)['create']>) => {
-          if (instance) {
-            throw new Error('Only one benchmark per file is supported');
-          }
-          instance = Benchmark.create(...args);
-          return instance;
-        };
-        const globals = Object.create(null);
-        for (const k of Object.getOwnPropertyNames(globalThis)) {
-          globals[k] = (globalThis as any)[k];
-        }
-        globals.benchmark = benchmark;
-        const script = new SourceTextModule(code, {
-          identifier,
-          context: createContext(globals),
-          initializeImportMeta(meta) {
-            meta.url = identifier;
-          },
-          async importModuleDynamically(specifier, referencingModule) {
-            if (Module.isBuiltin(specifier)) {
-              return import(specifier);
-            }
-            const baseIdentifier = referencingModule.identifier ?? identifier;
-            const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
-            const resolved = resolveFrom.resolve(specifier);
-            return import(resolved);
-          },
-        });
-        const imports = new Map<string, SyntheticModule>();
-        await script.link(async (specifier: string, referencingModule) => {
-          const baseIdentifier = referencingModule.identifier ?? identifier;
-          const resolveFrom = createRequire(fileURLToPath(baseIdentifier));
-          const target = Module.isBuiltin(specifier) ? specifier : resolveFrom.resolve(specifier);
-          const cached = imports.get(target);
-          if (cached) {
-            return cached;
-          }
-          const mod = await import(target);
-          const exportNames = Object.keys(mod);
-          const imported = new SyntheticModule(
-            exportNames,
-            () => {
-              exportNames.forEach((key) => imported.setExport(key, mod[key]));
-            },
-            { identifier: target, context: referencingModule.context },
-          );
-
-          imports.set(target, imported);
-          return imported;
-        });
-        await script.evaluate();
-
-        if (instance) {
-          const reports = await instance.execute({
-            ...executeOptions,
-            [BENCHMARK_URL]: identifier,
-          } as typeof executeOptions);
-
-          if (executeOptions.saveBaseline) {
-            const baselineData = reportsToBaseline(reports);
-            await writeFile(executeOptions.saveBaseline, JSON.stringify(baselineData, null, 2));
-            console.log(`Baseline saved to: ${executeOptions.saveBaseline}`);
-          }
-
-          if (baseline) {
-            printComparisonReports(reports, baseline);
-          } else {
-            switch (executeOptions.format) {
-              case 'json':
-                printJSONReports(reports);
-                break;
-              case 'pjson':
-                printJSONReports(reports, 2);
-                break;
-              case 'table':
-                printTableReports(reports);
-                break;
-              case 'markdown':
-                printMarkdownReports(reports);
-                break;
-              case 'histogram':
-                printHistogramReports(reports);
-                break;
-              default:
-                printSimpleReports(reports);
-            }
-          }
+    if (instance) {
+      const reports = await instance.execute({
+        ...executeOptions,
+        [BENCHMARK_URL]: identifier,
+      } as typeof executeOptions);
+
+      if (opts['save-baseline']) {
+        const baselineData = reportsToBaseline(reports);
+        await writeFile(opts['save-baseline'], JSON.stringify(baselineData, null, 2));
+        console.log(`Baseline saved to: ${opts['save-baseline']}`);
+      }
+
+      if (baseline) {
+        printComparisonReports(reports, baseline);
+      } else {
+        switch (format) {
+          case 'json':
+            printJSONReports(reports);
+            break;
+          case 'pjson':
+            printJSONReports(reports, 2);
+            break;
+          case 'table':
+            printTableReports(reports);
+            break;
+          case 'markdown':
+            printMarkdownReports(reports);
+            break;
+          case 'histogram':
+            printHistogramReports(reports);
+            break;
+          default:
+            printSimpleReports(reports);
         }
       }
     }
-  });
-
-commander.parse(process.argv);
+  }
+}

package/src/executor.ts

@@ -1,21 +1,20 @@
 import { Worker } from 'node:worker_threads';
 import { once } from 'node:events';
-import { queue } from 'async';
 import { pathToFileURL } from 'node:url';
-import { createReport, Report } from './reporter.js';
-import { cmp, assertNoClosure } from './utils.js';
+import { createReport, computeStats, Report } from './reporter.ts';
+import { cmp, assertNoClosure } from './utils.ts';
 import {
-  ExecutorRunOptions,
-  ReportOptions,
-  WorkerOptions,
-  BenchmarkOptions,
+  type ExecutorRunOptions,
+  type ReportOptions,
+  type WorkerOptions,
+  type BenchmarkOptions,
   Control,
-  ReportType,
-  ReportTypeList,
+  type ReportType,
+  type ReportTypeList,
   CONTROL_SLOTS,
   COMPLETE_VALUE,
-  ProgressCallback,
-} from './types.js';
+  type ProgressCallback,
+} from './types.ts';
 
 export type ExecutorReport<R extends ReportTypeList> = Record<R[number], Report> & {
   count: number;
@@ -33,16 +32,46 @@ export interface ExecutorOptions<R extends ReportTypeList> extends BenchmarkOpti
 
 const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
 
-export const createExecutor = <TContext, TInput, R extends ReportTypeList>(options: Required<ExecutorOptions<R>>) => {
+export interface Executor<TContext, TInput> {
+  pushAsync<T>(task: ExecutorRunOptions<TContext, TInput>): Promise<T>;
+  kill(): void;
+}
+
+export const createExecutor = <TContext, TInput, R extends ReportTypeList>(options: Required<ExecutorOptions<R>>): Executor<TContext, TInput> => {
   const { workers, warmupCycles, maxCycles, minCycles, absThreshold, relThreshold, gcObserver = true, reportTypes, onProgress, progressInterval = 100 } = options;
   const benchmarkUrl = (options as Record<symbol, unknown>)[BENCHMARK_URL];
   const resolvedBenchmarkUrl = typeof benchmarkUrl === 'string' ? benchmarkUrl : pathToFileURL(process.cwd()).href;
 
-  const executor = queue<ExecutorRunOptions<TContext, TInput>>(async ({ id, setup, teardown, pre, run, post, data }) => {
+  const pending: { task: ExecutorRunOptions<TContext, TInput>; resolve: (v: unknown) => void; reject: (e: unknown) => void }[] = [];
+  let running = 0;
+
+  const schedule = async (task: ExecutorRunOptions<TContext, TInput>) => {
+    running++;
+    try {
+      return await runTask(task);
+    } finally {
+      running--;
+      if (pending.length > 0) {
+        const next = pending.shift()!;
+        schedule(next.task).then(next.resolve, next.reject);
+      }
+    }
+  };
+
+  const pushAsync = <T>(task: ExecutorRunOptions<TContext, TInput>): Promise<T> => {
+    if (running < workers) {
+      return schedule(task) as Promise<T>;
+    }
+    return new Promise<T>((resolve, reject) => {
+      pending.push({ task, resolve: resolve as (v: unknown) => void, reject });
+    });
+  };
+
+  const runTask = async ({ id, setup, teardown, pre, run, post, data }: ExecutorRunOptions<TContext, TInput>) => {
     const setupCode = setup?.toString();
     const teardownCode = teardown?.toString();
     const preCode = pre?.toString();
-    const runCode = run.toString()!;
+    const runCode = run.toString();
     const postCode = post?.toString();
 
     if (setupCode) assertNoClosure(setupCode, 'setup');
@@ -54,7 +83,7 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>(optio
     const controlSAB = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT * CONTROL_SLOTS);
     const durationsSAB = new SharedArrayBuffer(BigUint64Array.BYTES_PER_ELEMENT * maxCycles);
 
-    const workerFile = new URL('./worker.js', import.meta.url);
+    const workerFile = new URL('./worker.ts', import.meta.url);
     const workerData: WorkerOptions = {
       benchmarkUrl: resolvedBenchmarkUrl,
       setupCode,
@@ -120,8 +149,9 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>(optio
       }
     }
 
+    const stats = count > 0 ? computeStats(durations) : undefined;
     const report = reportTypes
-      .map<[string, unknown]>((type) => [type, createReport(durations, type)] as [ReportType, Report])
+      .map<[string, unknown]>((type) => [type, createReport(durations, type, stats)] as [ReportType, Report])
       .concat([
         ['count', count],
         ['heapUsedKB', heapUsedKB],
@@ -129,7 +159,7 @@ export const createExecutor = <TContext, TInput, R extends ReportTypeList>(optio
         ['error', workerError],
       ]);
     return Object.fromEntries(report);
-  }, workers);
+  };
 
-  return executor;
+  return { pushAsync, kill() {} };
 };

package/src/index.ts

@@ -1,7 +1,7 @@
 import { cpus } from 'node:os';
 import Progress from 'progress';
-import { createExecutor, ExecutorOptions, ExecutorReport } from './executor.js';
-import { MaybePromise, StepFn, SetupFn, TeardownFn, FeedFn, ReportType, ReportTypeList, DEFAULT_CYCLES, ProgressInfo } from './types.js';
+import { createExecutor, type ExecutorOptions, type ExecutorReport } from './executor.ts';
+import { type MaybePromise, type StepFn, type SetupFn, type TeardownFn, type FeedFn, type ReportType, type ReportTypeList, DEFAULT_CYCLES, type ProgressInfo } from './types.ts';
 
 declare global {
   const benchmark: typeof Benchmark.create;
@@ -9,7 +9,6 @@ declare global {
 
 export const DEFAULT_WORKERS = Math.max(1, Math.ceil(cpus().length / 4));
 
-export const AsyncFunction = (async () => {}).constructor;
 const BENCHMARK_URL = Symbol.for('overtake.benchmarkUrl');
 
 export interface TargetReport<R extends ReportTypeList> {
@@ -30,14 +29,24 @@ export interface FeedReport<R extends ReportTypeList> {
 export const DEFAULT_REPORT_TYPES = ['ops'] as const;
 export type DefaultReportTypes = (typeof DEFAULT_REPORT_TYPES)[number];
 
-export class MeasureContext<TContext, TInput> {
-  public pre?: StepFn<TContext, TInput>;
-  public post?: StepFn<TContext, TInput>;
+const createExecutorErrorReport = <R extends ReportTypeList>(error: unknown): ExecutorReport<R> =>
+  ({
+    count: 0,
+    heapUsedKB: 0,
+    dceWarning: false,
+    error: error instanceof Error ? error.message : String(error),
+  }) as ExecutorReport<R>;
 
-  constructor(
-    public title: string,
-    public run: StepFn<TContext, TInput>,
-  ) {}
+export class MeasureContext<TContext, TInput> {
+  pre?: StepFn<TContext, TInput>;
+  post?: StepFn<TContext, TInput>;
+  title: string;
+  run: StepFn<TContext, TInput>;
+
+  constructor(title: string, run: StepFn<TContext, TInput>) {
+    this.title = title;
+    this.run = run;
+  }
 }
 
 export class Measure<TContext, TInput> {
@@ -58,13 +67,15 @@ export class Measure<TContext, TInput> {
 }
 
 export class TargetContext<TContext, TInput> {
-  public teardown?: TeardownFn<TContext>;
-  public measures: MeasureContext<TContext, TInput>[] = [];
-
-  constructor(
-    readonly title: string,
-    readonly setup?: SetupFn<MaybePromise<TContext>>,
-  ) {}
+  teardown?: TeardownFn<TContext>;
+  measures: MeasureContext<TContext, TInput>[] = [];
+  readonly title: string;
+  readonly setup?: SetupFn<MaybePromise<TContext>>;
+
+  constructor(title: string, setup?: SetupFn<MaybePromise<TContext>>) {
+    this.title = title;
+    this.setup = setup;
+  }
 }
 
 export class Target<TContext, TInput> {
@@ -87,10 +98,13 @@ export class Target<TContext, TInput> {
 }
 
 export class FeedContext<TInput> {
-  constructor(
-    readonly title: string,
-    readonly fn?: FeedFn<TInput>,
-  ) {}
+  readonly title: string;
+  readonly fn?: FeedFn<TInput>;
+
+  constructor(title: string, fn?: FeedFn<TInput>) {
+    this.title = title;
+    this.fn = fn;
+  }
 }
 
 export class Benchmark<TInput> {
@@ -136,7 +150,7 @@ export class Benchmark<TInput> {
     return new Target<TContext, TInput>(target);
   }
 
-  async execute<R extends readonly ReportType[] = typeof DEFAULT_REPORT_TYPES>(options: ExecutorOptions<R> & { progress?: boolean }): Promise<TargetReport<R>[]> {
+  async execute<R extends readonly ReportType[] = typeof DEFAULT_REPORT_TYPES>(options: Partial<ExecutorOptions<R>> & { progress?: boolean } = {}): Promise<TargetReport<R>[]> {
     const {
       workers = DEFAULT_WORKERS,
       warmupCycles = 20,
@@ -193,45 +207,59 @@ export class Benchmark<TInput> {
     } as Required<ExecutorOptions<R>>);
 
     const reports: TargetReport<R>[] = [];
-    for (const target of this.#targets) {
-      const targetReport: TargetReport<R> = { target: target.title, measures: [] };
-      for (const measure of target.measures) {
-        const measureReport: MeasureReport<R> = { measure: measure.title, feeds: [] };
-        for (const feed of this.#feeds) {
-          const id = `${target.title}/${measure.title}/${feed.title}`;
-          const data = await feed.fn?.();
-          executor
-            .push<ExecutorReport<R>>({
-              id,
-              setup: target.setup,
-              teardown: target.teardown,
-              pre: measure.pre,
-              run: measure.run,
-              post: measure.post,
-              data,
-            })
-            .then((data) => {
-              progressMap.delete(id);
-              completedBenchmarks++;
-              measureReport.feeds.push({
-                feed: feed.title,
-                data,
-              });
-            });
+    const pendingReports: Promise<void>[] = [];
+
+    try {
+      const feedData = await Promise.all(this.#feeds.map(async (feed) => ({ title: feed.title, data: await feed.fn?.() })));
+      for (const target of this.#targets) {
+        const targetReport: TargetReport<R> = { target: target.title, measures: [] };
+        for (const measure of target.measures) {
+          const measureReport: MeasureReport<R> = { measure: measure.title, feeds: [] };
+          for (const feed of feedData) {
+            const id = `${target.title}/${measure.title}/${feed.title}`;
+            const feedReport: FeedReport<R> = {
+              feed: feed.title,
+              data: createExecutorErrorReport<R>('Benchmark did not produce a report'),
+            };
+
+            measureReport.feeds.push(feedReport);
+            pendingReports.push(
+              (async () => {
+                try {
+                  feedReport.data = await executor.pushAsync<ExecutorReport<R>>({
+                    id,
+                    setup: target.setup,
+                    teardown: target.teardown,
+                    pre: measure.pre,
+                    run: measure.run,
+                    post: measure.post,
+                    data: feed.data,
+                  });
+                } catch (error) {
+                  feedReport.data = createExecutorErrorReport<R>(error);
+                } finally {
+                  progressMap.delete(id);
+                  completedBenchmarks++;
+                }
+              })(),
+            );
+          }
+          targetReport.measures.push(measureReport);
         }
-        targetReport.measures.push(measureReport);
+        reports.push(targetReport);
       }
-      reports.push(targetReport);
-    }
-    await executor.drain();
-    executor.kill();
 
-    if (bar) {
-      bar.update(1, { label: 'done' });
-      bar.terminate();
-    }
+      await Promise.all(pendingReports);
 
-    return reports;
+      if (bar) {
+        bar.update(1, { label: 'done' });
+        bar.terminate();
+      }
+
+      return reports;
+    } finally {
+      executor.kill();
+    }
   }
 }