overtake 0.0.1-rc1 → 0.0.1-rc2

package/README.md

@@ -1,6 +1,6 @@
 # Overtake
 
-NodeJS performance benchmark
+Performance benchmark for NodeJS
 
 [![Build Status][github-image]][github-url]
 [![NPM version][npm-image]][npm-url]
@@ -81,6 +81,8 @@ benchmark('mongodb vs postgres', () => {
 });
 ```
 
+Make sure you have installed used modules and run
+
 ```bash
 yarn overtake
 ```

package/cli.js

@@ -1,28 +1,19 @@
 #!/usr/bin/env node
-import Path from 'path';
+
+import { promisify } from 'util';
 import glob from 'glob';
-import { benchmark, setup, teardown, measure, perform, suites, runner } from './index.js';
+import { Overtake } from './index.js';
+import { defaultReporter } from './reporter.js';
 
+const globAsync = promisify(glob);
 const pattern = process.argv[2] || '**/__benchmarks__/**/*.js';
+const overtake = new Overtake({});
 
-Object.assign(globalThis, {
-  benchmark,
-  setup,
-  teardown,
-  measure,
-  perform,
-});
-
-glob(pattern, async (err, files) => {
-  try {
-    for (const file of files) {
-      const filepath = Path.resolve(file);
-      await import(filepath);
-    }
-  } catch (e) {
-    console.error(e.stack);
-  }
-  for (const suite of suites) {
-    await runner(suite);
+(async () => {
+  const files = await globAsync(pattern);
+  await overtake.load(files);
+  if (overtake.reporters.length === 0) {
+    reporter(defaultReporter);
   }
-});
+  await overtake.run();
+})().catch((e) => console.error(e));

package/index.js

@@ -1,67 +1,185 @@
+import Path from 'path';
 import WorkerThreads from 'worker_threads';
+import { Event } from 'evnty';
 
-export const suites = [];
+export const NOOP = () => {};
 
-const NOOP = () => {};
+export class Perform {
+  title = '';
 
-const SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏', '⠿'];
-const SPINNER_INTERVAL = 80;
+  count = 0;
 
-const ACCURACY = 6;
+  args;
 
-const renderSpinner = (index) => {
-  process.stdout.moveCursor(0, -1);
-  process.stdout.write(SPINNER[index]);
-  process.stdout.moveCursor(-3, 1);
-};
-
-export function benchmark(title, init) {
-  const suite = {
-    title,
-    current: true,
-    measures: [],
-    performs: [],
-    setup: NOOP,
-    teardown: NOOP,
-  };
-  suites.push(suite);
-  init();
-  suite.current = false;
+  constructor(overtake, title, count, args) {
+    this.title = title;
+    this.count = count;
+    this.args = args;
+  }
 }
 
-export function getSuite() {
-  const suite = suites[suites.length - 1];
-  if (!suite.current) {
-    throw new Error('should be inside benchmark');
+export class Measure {
+  title = '';
+
+  init = NOOP;
+
+  constructor(overtake, title, init = NOOP) {
+    this.title = title;
+    this.init = init;
   }
-  return suite;
 }
 
-export function setup(init) {
-  const suite = getSuite();
-  suite.setup = init;
-}
+export class Suite {
+  title = '';
 
-export function teardown(init) {
-  const suite = getSuite();
-  suite.teardown = init;
-}
+  setup = NOOP;
+
+  measures = [];
+
+  performs = [];
+
+  teardown = NOOP;
 
-export function measure(title, init) {
-  const suite = getSuite();
-  suite.measures.push({ title, init });
+  #title = '';
+
+  #init = NOOP;
+
+  #overtake = null;
+
+  constructor(overtake, title, init = NOOP) {
+    this.#overtake = overtake;
+    this.title = title;
+    this.#init = init;
+  }
+
+  async init() {
+    const unsubscribes = [
+      this.#overtake.onSetupRegister.on((setup) => (this.setup = setup)),
+      this.#overtake.onMeasureRegister.on((measure) => this.measures.push(measure)),
+      this.#overtake.onPerformRegister.on((perform) => this.performs.push(perform)),
+      this.#overtake.onTeardownRegister.on((teardown) => (this.teardown = teardown)),
+    ];
+    await this.#init();
+    unsubscribes.forEach((unsubscribe) => unsubscribe());
+  }
 }
 
-export function perform(title, count, args) {
-  const suite = getSuite();
-  suite.performs.push({ title, count, args });
+export class Script {
+  onLoad = new Event();
+
+  filename = '';
+
+  suites = [];
+
+  constructor(filename) {
+    this.filename = filename;
+  }
 }
 
-export function formatFloat(value, digits = ACCURACY) {
-  return parseFloat(value.toFixed(digits));
+export class Overtake {
+  onLoad = new Event();
+
+  onRun = new Event();
+
+  onComplete = new Event();
+
+  onScriptRegister = new Event();
+
+  onScriptStart = new Event();
+
+  onScriptComplete = new Event();
+
+  onSuiteRegister = new Event();
+
+  onSuiteStart = new Event();
+
+  onSuiteComplete = new Event();
+
+  onSetupRegister = new Event();
+
+  onTeardownRegister = new Event();
+
+  onMeasureRegister = new Event();
+
+  onMeasureStart = new Event();
+
+  onMeasureComplete = new Event();
+
+  onPerformRegister = new Event();
+
+  onPerformStart = new Event();
+
+  onPerformProgress = new Event();
+
+  onPerformComplete = new Event();
+
+  onReport = new Event();
+
+  scripts = [];
+
+  reporters = [];
+
+  constructor(options = {}) {
+    Object.assign(globalThis, {
+      benchmark: (title, init) => this.onSuiteRegister(new Suite(this, title, init)),
+      setup: (init) => this.onSetupRegister(init),
+      teardown: (init) => this.onTeardownRegister(init),
+      measure: (title, init) => this.onMeasureRegister(new Measure(this, title, init)),
+      perform: (title, count, args) => this.onPerformRegister(new Perform(this, title, count, args)),
+      reporter: (reporter) => this.reporters.push(reporter(this)),
+    });
+  }
+
+  async load(files) {
+    this.onLoad(this);
+    for (const file of files) {
+      const filename = Path.resolve(file);
+      const script = new Script(filename);
+      const unsubscribe = this.onSuiteRegister.on((suite) => {
+        script.suites.push(suite);
+      });
+      await import(filename);
+      unsubscribe();
+      this.scripts.push(script);
+      this.onScriptRegister(script);
+    }
+  }
+
+  async run() {
+    this.onRun();
+    for (const script of this.scripts) {
+      this.onScriptStart(script);
+      for (const suite of script.suites) {
+        await suite.init().catch((e) => console.error(e));
+        this.onSuiteStart(suite);
+        for (const measure of suite.measures) {
+          this.onMeasureStart(measure);
+          for (const perform of suite.performs) {
+            this.onPerformStart(perform);
+            const result = await runWorker(
+              {
+                setup: suite.setup,
+                teardown: suite.teardown,
+                init: measure.init,
+                count: perform.count,
+                args: perform.args,
+              },
+              this.onPerformProgress
+            );
+            this.onPerformComplete(perform);
+            this.onReport(result);
+          }
+          this.onMeasureComplete(perform);
+        }
+        this.onSuiteComplete(perform);
+      }
+      this.onScriptComplete(perform);
+    }
+    this.onComplete(this);
+  }
 }
 
-export async function runWorker({ args, count, ...options }) {
+export async function runWorker({ args, count, ...options }, onProgress) {
   const setupCode = options.setup.toString();
   const teardownCode = options.teardown.toString();
   const initCode = options.init.toString();
@@ -72,81 +190,42 @@ export async function runWorker({ args, count, ...options }) {
     count,
     args,
   });
-  let i = 0;
-  const spinnerSize = SPINNER.length - 1;
-
-  const timerId = setInterval(() => renderSpinner(i++ % spinnerSize), SPINNER_INTERVAL);
 
   const worker = new WorkerThreads.Worker(new URL('runner.js', import.meta.url), { argv: [params] });
   return new Promise((resolve) => {
-    worker.on('message', resolve);
+    worker.on('message', (data) => {
+      if (data.type === 'progress') {
+        onProgress(data);
+      } else if (data.type === 'report') {
+        resolve(data);
+      }
+    });
     worker.on('error', (error) => resolve({ success: false, error: error.message }));
-  }).finally((result) => {
-    clearInterval(timerId);
-    renderSpinner(spinnerSize);
-
-    return result;
   });
 }
 
-export async function runner(suite) {
-  console.group(`\nStart ${suite.title} benchmark`);
-
-  for (let measureIdx = 0; measureIdx < suite.measures.length; measureIdx++) {
-    const currentMeasure = suite.measures[measureIdx];
-    const reports = {};
-    console.group(`\n  Measuring performance of ${currentMeasure.title}`);
-    for (let performIdx = 0; performIdx < suite.performs.length; performIdx++) {
-      const currentPerform = suite.performs[performIdx];
-      const report = await runWorker({
-        setup: suite.setup,
-        teardown: suite.teardown,
-        init: currentMeasure.init,
-        count: currentPerform.count,
-        args: currentPerform.args,
-      });
-      reports[currentPerform.title] = { title: perform.title, report };
-      if (report.success) {
-        reports[currentPerform.title] = {
-          Count: currentPerform.count,
-          'Setup, ms': formatFloat(report.setup),
-          'Work, ms': formatFloat(report.work),
-          'Avg, ms': formatFloat(report.avg),
-          'Mode, ms': report.mode,
-        };
-      } else {
-        reports[`${currentPerform.title} ${currentMeasure.title}`] = {
-          Count: currentPerform.count,
-          'Setup, ms': '?',
-          'Work, ms': '?',
-          'Avg, ms': '?',
-          'Mode, ms': '?',
-          Error: report.error,
-        };
-      }
-    }
-    console.groupEnd();
-    console.table(reports);
-  }
-  console.groupEnd();
-}
-
 const FALSE_START = () => {
   throw new Error('False start');
 };
 
 export async function start(input) {
-  const { setupCode, teardownCode, initCode, count, args = [[]] } = JSON.parse(input);
+  const { setupCode, teardownCode, initCode, count, reportInterval = 500, args = [[]] } = JSON.parse(input);
   const setup = Function(`return ${setupCode};`)();
   const teardown = Function(`return ${teardownCode};`)();
   const init = Function(`return ${initCode};`)();
+  const send = WorkerThreads.parentPort ? (data) => WorkerThreads.parentPort.postMessage(data) : (data) => console.log(data);
 
   let i = count;
   let done = FALSE_START;
 
   const timings = [];
   const argSize = args.length;
+
+  send({ type: 'progress', stage: 'setup' });
+  const startMark = performance.now();
   const context = await setup();
+  const setupMark = performance.now();
+
   const initArgs = [() => done()];
   if (init.length > 2) {
     initArgs.unshift(args);
@@ -154,70 +233,97 @@ export async function start(input) {
   if (init.length > 1) {
     initArgs.unshift(context);
   }
-  const startMark = performance.now();
+
+  send({ type: 'progress', stage: 'init' });
+  const initMark = performance.now();
   const action = await init(...initArgs);
-  const workMark = performance.now();
+  const initDoneMark = performance.now();
 
   try {
+    let lastCheck = performance.now();
     const loop = (resolve, reject) => {
-      const argIdx = i % argSize;
+      const idx = count - i;
+      const argIdx = idx % argSize;
       const timerId = setTimeout(reject, 10000, new Error('Timeout'));
 
       done = () => {
-        // eslint-disable-next-line
-        const elapsed = performance.now() - startTickTime;
+        const doneTick = performance.now();
+        const elapsed = doneTick - startTickTime;
         clearTimeout(timerId);
         done = FALSE_START;
         timings.push(elapsed);
-
+        if (doneTick - lastCheck > reportInterval) {
+          lastCheck = doneTick;
+          send({ type: 'progress', stage: 'cycles', progress: idx / count });
+        }
         resolve();
       };
+
       const startTickTime = performance.now();
-      action(...args[argIdx], count - i);
+      action(...args[argIdx], idx);
     };
+    const cyclesMark = performance.now();
 
+    send({ type: 'progress', stage: 'cycles', progress: 0 });
     while (i--) {
       await new Promise(loop);
     }
+    send({ type: 'progress', stage: 'teardown' });
+    const teardownMark = performance.now();
+    await teardown(context);
     const completeMark = performance.now();
+    send({ type: 'progress', stage: 'complete', progress: (count - i) / count });
 
-    await teardown(context);
+    timings.sort((a, b) => a - b);
 
-    timings.sort();
-
-    const {
-      mode: { time: mode },
-    } = timings
-      .map((time) => parseFloat(time.toFixed(ACCURACY)))
-      .reduce((result, time) => {
-        const value = (result[time] || 0) + 1;
-        const mode = !result.mode || result.mode.value < value ? { time, value } : result.mode;
-        return { ...result, [time]: value, mode };
-      }, {});
-
-    const min = timings.reduce((a, b) => Math.min(a, b), Infinity);
-    const max = timings.reduce((a, b) => Math.max(a, b), 0);
-    const avg = timings.reduce((a, b) => a + b, 0) / count;
-    const p90idx = parseInt(timings.length * 0.9, 10);
-    const p95idx = parseInt(timings.length * 0.95, 10);
-    const p99idx = parseInt(timings.length * 0.99, 10);
-    const p90 = timings[p90idx];
-    const p95 = timings[p95idx];
-    const p99 = timings[p99idx];
-
-    WorkerThreads.parentPort.postMessage({
+    const min = timings[0];
+    const max = timings[timings.length - 1];
+    const range = max - min || Number.MIN_VALUE;
+    const sum = timings.reduce((a, b) => a + b, 0);
+    const avg = sum / timings.length;
+
+    const step = range / 99 || Number.MIN_VALUE;
+    const buckets = Array(100)
+      .fill(0)
+      .map((_, idx) => [min + idx * step, 0]);
+
+    // Calc mode O(n)
+    timings.forEach((timing, idx) => {
+      const index = Math.round((timing - min) / step);
+      buckets[index][1] += 1;
+    });
+    buckets.sort((a, b) => a[1] - b[1]);
+
+    const medIdx = Math.trunc((50 * timings.length) / 100);
+    const med = timings[medIdx];
+    const p90Idx = Math.trunc((90 * timings.length) / 100);
+    const p90 = timings[p90Idx];
+    const p95Idx = Math.trunc((95 * timings.length) / 100);
+    const p95 = timings[p95Idx];
+    const p99Idx = Math.trunc((99 * timings.length) / 100);
+    const p99 = timings[p99Idx];
+    const mode = buckets[buckets.length - 1][0];
+
+    send({
+      type: 'report',
+      success: true,
+      count: timings.length,
       min,
       max,
+      sum,
       avg,
+      med,
+      mode,
       p90,
       p95,
       p99,
-      mode,
-      setup: workMark - startMark,
-      work: completeMark - workMark,
-      success: true,
+      setup: setupMark - startMark,
+      init: initDoneMark - initMark,
+      cycles: teardownMark - cyclesMark,
+      teardown: completeMark - teardownMark,
+      total: completeMark - setupMark,
     });
   } catch (error) {
-    WorkerThreads.parentPort.postMessage({ success: false, error: error.stack });
+    send({ type: 'report', success: false, error: error.stack });
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "overtake",
-  "version": "0.0.1-rc1",
+  "version": "0.0.1-rc2",
   "description": "NodeJS performance benchmark",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,8 +9,8 @@
     "overtake": "cli.js"
   },
   "scripts": {
-    "measure": "node ./__runner__/class-vs-function.js",
-    "test": "yarn node --experimental-vm-modules $(yarn bin jest)",
+    "start": "node ./cli.js",
+    "test": "yarn node --experimental-vm-modules $(yarn bin jest) --detectOpenHandles",
     "lint": "eslint .",
     "prepare": "husky install"
   },
@@ -33,16 +33,13 @@
   "devDependencies": {
     "@jest/globals": "^27.5.1",
     "@types/jest": "^27.4.1",
-    "eslint": "^8.14.0",
-    "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-import": "^2.26.0",
     "husky": "^7.0.4",
     "jest": "^27.5.1",
     "prettier": "^2.6.2",
     "pretty-quick": "^3.1.3"
   },
   "dependencies": {
+    "evnty": "^0.7.4",
     "glob": "^8.0.1"
   }
 }

package/reporter.js

@@ -0,0 +1,90 @@
+const SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏', '⠿'];
+const SPINNER_INTERVAL = 100;
+const SPINNER_PADDING = 5;
+
+const ACCURACY = 6;
+
+/** TODO
+ *  [PROGRESS] script/file.js
+ *    Test suite title
+ *      ➤ Measure
+ *        ✓ Perform
+ *        ✕ Perform
+ * */
+
+const renderString = (message, x, direction) => {
+  const size = message.length;
+  process.stdout.moveCursor(0, -1);
+  process.stdout.cursorTo(x);
+  process.stdout.clearLine(direction);
+  if (direction === -1) {
+    process.stdout.cursorTo(0);
+  }
+  process.stdout.write(message);
+  process.stdout.moveCursor(-process.stdout.rows, 1);
+
+  return size;
+};
+
+const renderSpinner = (index) => renderString(SPINNER[index], 0, -1);
+
+const paddings = [];
+
+const renderMessage = (column, message) => {
+  const padding = SPINNER_PADDING + column * 10;
+  paddings[column + 1] = renderString(message, padding, 1);
+};
+
+export function formatFloat(value, digits = ACCURACY) {
+  return parseFloat(value.toFixed(digits));
+}
+
+export function defaultReporter(overtake) {
+  const spinnerSize = SPINNER.length - 1;
+  let i = 0;
+  let timerId = null;
+
+  overtake.onLoad.on(() => {});
+  overtake.onRun.on(() => {
+    console.log();
+  });
+  overtake.onComplete.on(() => {});
+  overtake.onScriptRegister.on(() => {});
+  overtake.onScriptStart.on(() => {});
+  overtake.onScriptComplete.on(() => {});
+  overtake.onSuiteRegister.on(() => {});
+  overtake.onSuiteStart.on(() => {});
+  overtake.onSuiteComplete.on(() => {});
+  overtake.onSetupRegister.on(() => {});
+  overtake.onTeardownRegister.on(() => {});
+  overtake.onMeasureRegister.on(() => {});
+  overtake.onMeasureStart.on((measure) => {
+    console.log(measure.title);
+  });
+  overtake.onMeasureComplete.on(() => {});
+  overtake.onPerformRegister.on(() => {});
+  overtake.onPerformStart.on((perform) => {
+    console.log(perform.title);
+    console.log();
+    i = 0;
+    timerId = setInterval(() => renderSpinner(i++ % spinnerSize), SPINNER_INTERVAL);
+  });
+  overtake.onPerformComplete.on(() => {
+    clearInterval(timerId);
+    renderSpinner(spinnerSize);
+  });
+  overtake.onPerformProgress.on(({ stage, progress }) => {
+    renderMessage(0, stage);
+    if (typeof progress !== 'undefined') {
+      renderMessage(1, `${(progress * 100).toFixed(0)}%`.padStart(4, ' '));
+    }
+  });
+  overtake.onReport.on((report) => {
+    if (report.success) {
+      renderMessage(2, `total:${report.total.toFixed(0)}ms  mode:${report.mode.toFixed(ACCURACY)}ms`);
+    } else {
+      renderMessage(1, report.error);
+    }
+    console.log();
+  });
+}

package/index.d.ts

@@ -1,15 +0,0 @@
-declare global {
-  declare function benchmark(title: string, init: () => void): void;
-
-  declare function setup<C>(init: () => C): any;
-
-  declare function teardown<C>(teardown: (context: C) => void): void;
-
-  declare function measure(title: string, init: (next: () => void) => void): void;
-  declare function measure<C>(title: string, init: (context: C, next: () => void) => void): void;
-  declare function measure<C, A>(title: string, init: (context: C, args: A, next: () => void) => void): void;
-
-  declare function perform<A>(title: string, counter: number, args: A): void;
-}
-
-export { benchmark, setup, teardown, measure, perform };