overtake 0.1.2 → 1.0.0-rc.2

package/package.json

@@ -1,18 +1,22 @@
 {
   "name": "overtake",
-  "version": "0.1.2",
+  "version": "1.0.0-rc.2",
   "description": "NodeJS performance benchmark",
-  "main": "index.js",
-  "types": "index.d.ts",
+  "main": "build/index.js",
+  "types": "build/index.d.ts",
   "type": "module",
   "bin": {
-    "overtake": "cli.js"
+    "overtake": "bin/overtake.js"
   },
   "scripts": {
-    "start": "./cli.js",
-    "test": "jest --detectOpenHandles",
+    "build": "rm -rf build && inop src build -i __tests__ -i *.tmp.ts && tsc --declaration --emitDeclarationOnly",
+    "start": "./bin/overtake.js",
+    "test": "jest --detectOpenHandles --passWithNoTests",
     "prepare": "husky"
   },
+  "engines": {
+    "node": ">=22"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/3axap4eHko/overtake.git"
@@ -31,23 +35,21 @@
   "homepage": "https://github.com/3axap4eHko/overtake#readme",
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@swc/core": "^1.11.24",
     "@swc/jest": "^0.2.38",
+    "@types/async": "^3.2.24",
     "@types/jest": "^29.5.14",
     "@types/node": "^22.15.18",
-    "ajv": "^8.17.1",
-    "ascertain": "1.2.104",
     "husky": "^9.1.7",
+    "inop": "^0.7.8",
     "jest": "^29.7.0",
-    "mongodb": "^6.16.0",
-    "pg": "^8.16.0",
     "prettier": "^3.5.3",
     "pretty-quick": "^4.1.1",
-    "zod": "^3.24.4"
+    "typescript": "^5.8.3"
   },
   "dependencies": {
+    "@swc/core": "^1.11.24",
+    "async": "^3.2.6",
     "commander": "^13.1.0",
-    "conode": "^0.2.0",
     "glob": "^11.0.2"
   },
   "packageManager": "pnpm@10.10.0+sha512.d615db246fe70f25dcfea6d8d73dee782ce23e2245e3c4f6f888249fb568149318637dca73c2c5c8ef2a4ca0d5657fb9567188bfab47f566d1ee6ce987815c39"

package/src/__tests__/runner.ts

@@ -0,0 +1,34 @@
+import { Benchmark } from '../benchmark.js';
+
+const benchmark = Benchmark.create('void')
+  .feed('strings', () => ['a', 'b', 'c'])
+  .feed('numbers', () => [0, 1, 2]);
+
+const httpServer = benchmark.target('node http', async () => {
+  const { createServer, Server } = await import('node:http');
+  const server = createServer();
+  return new Promise<InstanceType<typeof Server>>((resolve) => {
+    server.on('listen', () => resolve(server));
+  });
+});
+
+httpServer.teardown((ctx) => {
+  ctx.close();
+});
+
+httpServer
+  .measure('something to bench', async (ctx, input) => {
+    ctx.emit('whatever', input);
+  })
+  .pre(async (ctx, input) => {})
+  .post(async (ctx, input) => {});
+
+const forLoop = benchmark.target('for loop');
+
+forLoop
+  .measure('1k', (_, input) => {
+    const n = input?.length ?? 0;
+    for (let i = 0; i < n; i++) {}
+  })
+  .pre(async (ctx, input) => {})
+  .post(async (ctx, input) => {});

package/src/benchmark.ts

@@ -0,0 +1,231 @@
+import { cpus } from 'node:os';
+import { createExecutor, ExecutorOptions, ExecutorReport } from './executor.js';
+import { MaybePromise, StepFn, SetupFn, TeardownFn, FeedFn, ReportType, ReportTypeList, DEFAULT_CYCLES } from './types.js';
+
+export const DEFAULT_WORKERS = cpus().length;
+
+export const AsyncFunction = (async () => {}).constructor;
+
+export interface TargetReport<R extends ReportTypeList> {
+  target: string;
+  measures: MeasureReport<R>[];
+}
+
+export interface MeasureReport<R extends ReportTypeList> {
+  measure: string;
+  feeds: FeedReport<R>[];
+}
+
+export interface FeedReport<R extends ReportTypeList> {
+  feed: string;
+  data: ExecutorReport<R>;
+}
+
+export const DEFAULT_REPORT_TYPES = ['ops'] as const;
+export type DefaultReportTypes = (typeof DEFAULT_REPORT_TYPES)[number];
+
+export class MeasureContext<TContext, TInput> {
+  public pre?: StepFn<TContext, TInput>;
+  public post?: StepFn<TContext, TInput>;
+
+  constructor(
+    public title: string,
+    public run: StepFn<TContext, TInput>,
+  ) {}
+}
+
+export class Measure<TContext, TInput> {
+  #ctx: MeasureContext<TContext, TInput>;
+
+  constructor(ctx: MeasureContext<TContext, TInput>) {
+    this.#ctx = ctx;
+  }
+
+  pre(fn: StepFn<TContext, TInput>): Measure<TContext, TInput> {
+    this.#ctx.pre = fn;
+    return this;
+  }
+  post(fn: StepFn<TContext, TInput>): Measure<TContext, TInput> {
+    this.#ctx.post = fn;
+    return this;
+  }
+}
+
+export class TargetContext<TContext, TInput> {
+  public teardown?: TeardownFn<TContext>;
+  public measures: MeasureContext<TContext, TInput>[] = [];
+
+  constructor(
+    readonly title: string,
+    readonly setup?: SetupFn<MaybePromise<TContext>>,
+  ) {}
+}
+
+export class Target<TContext, TInput> {
+  #ctx: TargetContext<TContext, TInput>;
+
+  constructor(ctx: TargetContext<TContext, TInput>) {
+    this.#ctx = ctx;
+  }
+  teardown(fn: TeardownFn<TContext>): Target<TContext, TInput> {
+    this.#ctx.teardown = fn;
+
+    return this;
+  }
+  measure(title: string, run: StepFn<TContext, TInput>): Measure<TContext, TInput> {
+    const measure = new MeasureContext(title, run);
+    this.#ctx.measures.push(measure);
+
+    return new Measure(measure);
+  }
+}
+
+export class FeedContext<TInput> {
+  constructor(
+    readonly title: string,
+    readonly fn?: FeedFn<TInput>,
+  ) {}
+}
+
+export class Benchmark<TInput> {
+  #targets: TargetContext<unknown, TInput>[] = [];
+  #feeds: FeedContext<TInput>[] = [];
+  #executed = false;
+
+  static create(title: string): Benchmark<void>;
+  static create<I>(title: string, fn: FeedFn<I>): Benchmark<I>;
+  static create<I>(title: string, fn?: FeedFn<I> | undefined): Benchmark<I> {
+    if (fn) {
+      return new Benchmark(title, fn);
+    } else {
+      return new Benchmark(title);
+    }
+  }
+
+  constructor(title: string);
+  constructor(title: string, fn: FeedFn<TInput>);
+  constructor(title: string, fn?: FeedFn<TInput> | undefined) {
+    if (fn) {
+      this.feed(title, fn);
+    } else {
+      this.feed(title);
+    }
+  }
+
+  feed(title: string): Benchmark<TInput | void>;
+  feed<I>(title: string, fn: FeedFn<I>): Benchmark<TInput | I>;
+  feed<I>(title: string, fn?: FeedFn<I> | undefined): Benchmark<TInput | I> {
+    const self = this as Benchmark<TInput | I>;
+    self.#feeds.push(fn ? new FeedContext(title, fn) : new FeedContext(title));
+
+    return self;
+  }
+
+  target<TContext>(title: string): Target<void, TInput>;
+  target<TContext>(title: string, setup: SetupFn<Awaited<TContext>>): Target<TContext, TInput>;
+  target<TContext>(title: string, setup?: SetupFn<Awaited<TContext>> | undefined): Target<TContext, TInput> {
+    const target = new TargetContext<TContext, TInput>(title, setup);
+    this.#targets.push(target as TargetContext<unknown, TInput>);
+
+    return new Target<TContext, TInput>(target);
+  }
+
+  async execute<R extends readonly ReportType[] = typeof DEFAULT_REPORT_TYPES>({
+    workers = DEFAULT_WORKERS,
+    warmupCycles = 20,
+    maxCycles = DEFAULT_CYCLES,
+    minCycles = 50,
+    absThreshold = 1_000,
+    relThreshold = 0.02,
+    reportTypes = DEFAULT_REPORT_TYPES as unknown as R,
+  }: ExecutorOptions<R>): Promise<TargetReport<R>[]> {
+    if (this.#executed) {
+      throw new Error("Benchmark is executed and can't be reused");
+    }
+    this.#executed = true;
+
+    const executor = createExecutor<unknown, TInput, R>({
+      workers,
+      warmupCycles,
+      maxCycles,
+      minCycles,
+      absThreshold,
+      relThreshold,
+      reportTypes,
+    });
+
+    const reports: TargetReport<R>[] = [];
+    for (const target of this.#targets) {
+      const targetReport: TargetReport<R> = { target: target.title, measures: [] };
+      for (const measure of target.measures) {
+        const measureReport: MeasureReport<R> = { measure: measure.title, feeds: [] };
+        for (const feed of this.#feeds) {
+          const data = await feed.fn?.();
+          executor
+            .push<ExecutorReport<R>>({
+              setup: target.setup,
+              teardown: target.teardown,
+              pre: measure.pre,
+              run: measure.run,
+              post: measure.post,
+              data,
+            })
+            .then((data) => {
+              measureReport.feeds.push({
+                feed: feed.title,
+                data,
+              });
+            });
+        }
+        targetReport.measures.push(measureReport);
+      }
+      reports.push(targetReport);
+    }
+    await executor.drain();
+    executor.kill();
+
+    return reports;
+  }
+}
+
+export const printSimpleReports = <R extends ReportTypeList>(reports: TargetReport<R>[]) => {
+  for (const report of reports) {
+    for (const { measure, feeds } of report.measures) {
+      console.group('\n', report.target, measure);
+      for (const { feed, data } of feeds) {
+        const output = Object.entries(data)
+          .map(([key, report]) => `${key}: ${report.toString()}`)
+          .join('; ');
+        console.log(feed, output);
+      }
+      console.groupEnd();
+    }
+  }
+};
+
+export const printTableReports = <R extends ReportTypeList>(reports: TargetReport<R>[]) => {
+  for (const report of reports) {
+    for (const { measure, feeds } of report.measures) {
+      console.log('\n', report.target, measure);
+      const table: Record<string, unknown> = {};
+      for (const { feed, data } of feeds) {
+        table[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
+      }
+      console.table(table);
+    }
+  }
+};
+
+export const printJSONReports = <R extends ReportTypeList>(reports: TargetReport<R>[], padding?: number) => {
+  const output = {} as Record<string, Record<string, Record<string, string>>>;
+  for (const report of reports) {
+    for (const { measure, feeds } of report.measures) {
+      const row = {} as Record<string, Record<string, string>>;
+      for (const { feed, data } of feeds) {
+        row[feed] = Object.fromEntries(Object.entries(data).map(([key, report]) => [key, report.toString()]));
+      }
+      output[`${report.target} ${measure}`] = row;
+    }
+  }
+  console.log(JSON.stringify(output, null, padding));
+};

package/src/cli.ts

@@ -0,0 +1,114 @@
+import { createRequire, Module } from 'node:module';
+import { SyntheticModule, createContext, SourceTextModule, Module as VMModule } from 'node:vm';
+import { stat, readFile } from 'node:fs/promises';
+import { parse, print } from '@swc/core';
+import { Command, Option } from 'commander';
+import { glob } from 'glob';
+import { Benchmark, printTableReports, printJSONReports, printSimpleReports, DEFAULT_REPORT_TYPES, DEFAULT_WORKERS } from './benchmark.js';
+import { REPORT_TYPES } from './types.js';
+
+const require = createRequire(import.meta.url);
+const { name, description, version } = require('../package.json');
+
+const commander = new Command();
+
+const transpile = async (code: string): Promise<string> => {
+  const ast = await parse(code, {
+    syntax: 'typescript',
+    dynamicImport: true,
+    target: 'esnext',
+  });
+
+  const output = await print(ast, {
+    module: {
+      type: 'es6',
+    },
+    jsc: {
+      target: 'esnext',
+      parser: {
+        syntax: 'typescript',
+      },
+      experimental: {},
+    },
+  });
+  return output.code;
+};
+
+commander
+  .name(name)
+  .description(description)
+  .version(version)
+  .argument('<path>', 'glob pattern to find benchmarks')
+  .addOption(new Option('-r, --report-types [reportTypes...]', 'statistic types to include in the report').choices(REPORT_TYPES).default(DEFAULT_REPORT_TYPES))
+  .addOption(new Option('-w, --workers [workers]', 'number of concurent workers').default(DEFAULT_WORKERS).argParser(parseInt))
+  .addOption(new Option('-f, --format [format]', 'output format').default('simple').choices(['simple', 'json', 'pjson', 'table']))
+  .addOption(new Option('--abs-threshold [absThreshold]', 'absolute error threshold in nanoseconds').argParser(parseInt))
+  .addOption(new Option('--rel-threshold [relThreshold]', 'relative error threshold (fraction between 0 and 1)').argParser(parseInt))
+  .addOption(new Option('--warmup-cycles [warmupCycles]', 'number of warmup cycles before measuring').argParser(parseInt))
+  .addOption(new Option('--max-cycles [maxCycles]', 'maximum measurement cycles per feed').argParser(parseInt))
+  .addOption(new Option('--min-cycles [minCycles]', 'minimum measurement cycles per feed').argParser(parseInt))
+  .action(async (path, executeOptions) => {
+    const files = await glob(path, { absolute: true, cwd: process.cwd() }).catch(() => []);
+    for (const file of files) {
+      const stats = await stat(file).catch(() => false as const);
+      if (stats && stats.isFile()) {
+        const content = await readFile(file, 'utf8');
+        const code = await transpile(content);
+        let instance: Benchmark<unknown> | undefined;
+        const benchmark = (...args: Parameters<(typeof Benchmark)['create']>) => {
+          if (instance) {
+            throw new Error('Only one benchmark per file is supported');
+          }
+          instance = Benchmark.create(...args);
+          return instance;
+        };
+        const script = new SourceTextModule(code, {
+          context: createContext({ benchmark }),
+        });
+        const imports = new Map();
+        await script.link(async (specifier: string, referencingModule) => {
+          if (imports.has(specifier)) {
+            return imports.get(specifier);
+          }
+          const mod = await import(Module.isBuiltin(specifier) ? specifier : require.resolve(specifier));
+          const exportNames = Object.keys(mod);
+          const imported = new SyntheticModule(
+            exportNames,
+            () => {
+              exportNames.forEach((key) => imported.setExport(key, mod[key]));
+            },
+            { identifier: specifier, context: referencingModule.context },
+          );
+
+          imports.set(specifier, imported);
+          return imported;
+        });
+        await script.evaluate();
+
+        if (instance) {
+          const reports = await instance.execute(executeOptions);
+          switch (executeOptions.format) {
+            case 'json':
+              {
+                printJSONReports(reports);
+              }
+              break;
+            case 'pjson':
+              {
+                printJSONReports(reports, 2);
+              }
+              break;
+            case 'table':
+              {
+                printTableReports(reports);
+              }
+              break;
+            default:
+              printSimpleReports(reports);
+          }
+        }
+      }
+    }
+  });
+
+commander.parse(process.argv);

package/src/executor.ts

@@ -0,0 +1,73 @@
+import { Worker } from 'node:worker_threads';
+import { once } from 'node:events';
+import { queue } from 'async';
+import { RunOptions, ReportOptions, WorkerOptions, BenchmarkOptions, Control, ReportType, ReportTypeList, CONTROL_SLOTS } from './types.js';
+import { createReport, Report } from './reporter.js';
+import { cmp } from './utils.js';
+
+export type ExecutorReport<R extends ReportTypeList> = Record<R[number], Report> & { count: number };
+
+export interface ExecutorOptions<R extends ReportTypeList> extends BenchmarkOptions, ReportOptions<R> {
+  workers?: number;
+  maxCycles?: number;
+}
+
+export const createExecutor = <TContext, TInput, R extends ReportTypeList>({
+  workers,
+  warmupCycles,
+  maxCycles,
+  minCycles,
+  absThreshold,
+  relThreshold,
+  reportTypes,
+}: Required<ExecutorOptions<R>>) => {
+  const executor = queue<RunOptions<TContext, TInput>>(async ({ setup, teardown, pre, run, post, data }) => {
+    const setupCode = setup?.toString();
+    const teardownCode = teardown?.toString();
+    const preCode = pre?.toString();
+    const runCode = run.toString()!;
+    const postCode = post?.toString();
+
+    const controlSAB = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT * CONTROL_SLOTS);
+    const durationsSAB = new SharedArrayBuffer(BigUint64Array.BYTES_PER_ELEMENT * maxCycles);
+
+    const workerFile = new URL('./worker.js', import.meta.url);
+    const workerData: WorkerOptions = {
+      setupCode,
+      teardownCode,
+      preCode,
+      runCode,
+      postCode,
+      data,
+
+      warmupCycles,
+      minCycles,
+      absThreshold,
+      relThreshold,
+
+      controlSAB,
+      durationsSAB,
+    };
+
+    const worker = new Worker(workerFile, {
+      workerData,
+    });
+    const [exitCode] = await once(worker, 'exit');
+    if (exitCode !== 0) {
+      throw new Error(`worker exited with code ${exitCode}`);
+    }
+
+    const control = new Int32Array(controlSAB);
+    const count = control[Control.INDEX];
+    const durations = new BigUint64Array(durationsSAB).slice(0, count).sort(cmp);
+
+    const report = reportTypes.map<[string, unknown]>((type) => [type, createReport(durations, type)] as [ReportType, Report]).concat([['count', count]]);
+    return Object.fromEntries(report);
+  }, workers);
+
+  executor.error((err) => {
+    console.error(err);
+  });
+
+  return executor;
+};

package/src/index.ts

@@ -0,0 +1,6 @@
+export * from './benchmark.js';
+import { Benchmark as _Benchmark } from './benchmark.js';
+
+declare global {
+  const benchmark: (typeof _Benchmark)['create'];
+}

package/src/queue.ts

@@ -0,0 +1,42 @@
+export const createQueue = <T>(worker: (task: T) => Promise<void>, concurency: number = 1) => {
+  const queue = new Set<T>();
+  const processing = new Map<number, Promise<void>>();
+  const iterator = queue[Symbol.iterator]();
+
+  let next: () => void;
+  let counter = 0;
+
+  queueMicrotask(async () => {
+    while (true) {
+      if (concurency > 0 && processing.size === concurency) {
+        await Promise.race(processing.values());
+      }
+      if (queue.size === 0) {
+        const { promise, resolve } = Promise.withResolvers<void>();
+        next = resolve;
+        await promise;
+      }
+      const result = iterator.next();
+      if (result.done) {
+        break;
+      }
+      const id = counter++;
+      const task = Promise.resolve(worker(result.value))
+        .catch(() => {})
+        .finally(() => {
+          processing.delete(id);
+        });
+      processing.set(id, task);
+    }
+  });
+
+  return {
+    push: async (input: T) => {
+      queue.add(input);
+
+      if (queue.size === 0) {
+        next?.();
+      }
+    },
+  };
+};

package/src/reporter.ts

@@ -0,0 +1,139 @@
+import { div, max, divs } from './utils.js';
+import { ReportType } from './types.js';
+
+const units = [
+  { unit: 'ns', factor: 1 },
+  { unit: 'µs', factor: 1e3 },
+  { unit: 'ms', factor: 1e6 },
+  { unit: 's', factor: 1e9 },
+  { unit: 'm', factor: 60 * 1e9 },
+  { unit: 'h', factor: 3600 * 1e9 },
+] as const;
+
+function smartFixed(n: number): string {
+  return n.toLocaleString(undefined, {
+    minimumFractionDigits: 0,
+    maximumFractionDigits: 2,
+    useGrouping: false,
+  });
+}
+export class Report {
+  constructor(
+    public readonly type: ReportType,
+    public readonly value: bigint,
+    public readonly uncertainty: number = 0,
+    public readonly scale: bigint = 1n,
+  ) {}
+  valueOf() {
+    return Number(div(this.value, this.scale));
+  }
+  toString() {
+    const uncertainty = this.uncertainty ? ` ± ${smartFixed(this.uncertainty)}%` : '';
+
+    const value = this.valueOf();
+    if (this.type === 'ops') {
+      return `${smartFixed(value)} ops/s${uncertainty}`;
+    }
+    let display = value;
+    let unit = 'ns';
+
+    for (const { unit: u, factor } of units) {
+      const candidate = value / factor;
+      if (candidate < 1000) {
+        display = candidate;
+        unit = u;
+        break;
+      }
+    }
+    return `${smartFixed(display)} ${unit}${uncertainty}`;
+  }
+}
+
+export const createReport = (durations: BigUint64Array, type: ReportType): Report => {
+  const n = durations.length;
+  if (n === 0) {
+    return new Report(type, 0n);
+  }
+  switch (type) {
+    case 'min': {
+      return new Report(type, durations[0]);
+    }
+    case 'max': {
+      return new Report(type, durations[n - 1]);
+    }
+    case 'median': {
+      const mid = Math.floor(n / 2);
+      const med = n % 2 === 0 ? (durations[mid - 1] + durations[mid]) / 2n : durations[mid];
+      return new Report(type, med);
+    }
+
+    case 'mode': {
+      const freq = new Map<bigint, bigint>();
+      let maxCount = 0n;
+      let modeVal = durations[0];
+      for (const d of durations) {
+        const count = (freq.get(d) || 0n) + 1n;
+        freq.set(d, count);
+        if (count > maxCount) {
+          maxCount = count;
+          modeVal = d;
+        }
+      }
+      let lower = modeVal;
+      let upper = modeVal;
+      const firstIdx = durations.indexOf(modeVal);
+      const lastIdx = durations.lastIndexOf(modeVal);
+      if (firstIdx > 0) lower = durations[firstIdx - 1];
+      if (lastIdx < n - 1) upper = durations[lastIdx + 1];
+      const gap = max(modeVal - lower, upper - modeVal);
+      const uncertainty = modeVal > 0 ? Number(((gap / 2n) * 100n) / modeVal) : 0;
+      return new Report(type, modeVal, uncertainty);
+    }
+
+    case 'ops': {
+      let sum = 0n;
+      for (const duration of durations) {
+        sum += duration;
+      }
+      const avgNs = sum / BigInt(n);
+      const nsPerSec = 1_000_000_000n;
+      const raw = Number(nsPerSec) / Number(avgNs);
+      const extra = raw < 1 ? Math.ceil(-Math.log10(raw)) : 0;
+
+      const exp = raw > 100 ? 0 : 2 + extra;
+
+      const scale = 10n ** BigInt(exp);
+
+      const value = avgNs > 0n ? (nsPerSec * scale) / avgNs : 0n;
+      const deviation = durations[n - 1] - durations[0];
+      const uncertainty = avgNs > 0 ? Number(div(deviation * scale, 2n * avgNs)) : 0;
+      return new Report(type, value, uncertainty, scale);
+    }
+    case 'mean': {
+      let sum = 0n;
+      for (const duration of durations) {
+        sum += duration;
+      }
+      const value = divs(sum, BigInt(n), 1n);
+      return new Report(type, value);
+    }
+
+    default: {
+      const p = Number(type.slice(1));
+      if (p === 0) {
+        return new Report(type, durations[0]);
+      }
+      if (p === 100) {
+        return new Report(type, durations[n - 1]);
+      }
+      const idx = Math.ceil((p / 100) * n) - 1;
+      const value = durations[Math.min(Math.max(idx, 0), n - 1)];
+      const prev = idx > 0 ? durations[idx - 1] : value;
+      const next = idx < n - 1 ? durations[idx + 1] : value;
+      const gap = max(value - prev, next - value);
+      const uncertainty = value > 0 ? Number(div(divs(gap, 2n, 100_00n), value)) / 100 : 0;
+
+      return new Report(type, value, uncertainty);
+    }
+  }
+};