overtake 0.0.1-rc1 → 0.0.2

package/README.md

@@ -1,6 +1,6 @@
 # Overtake
 
-NodeJS performance benchmark
+Performance benchmark for NodeJS
 
 [![Build Status][github-image]][github-url]
 [![NPM version][npm-image]][npm-url]
@@ -55,7 +55,7 @@ benchmark('mongodb vs postgres', () => {
     return { postgres, mongo };
   });
 
-  measure('mongodb inserts', ({ mongo }/* context */, next) => {
+  measure('mongodb inserts', ({ mongo } /* context */, next) => {
     // prepare a collection
     const database = mongo.db('overtake');
     const test = database.collection('test');
@@ -63,28 +63,34 @@ benchmark('mongodb vs postgres', () => {
     return (data) => test.insertOne(data).then(next);
   });
 
-  measure('postgres inserts', ({ postgres }/* context */, next) => {
+  measure('postgres inserts', ({ postgres } /* context */, next) => {
     // prepare a query
     const query = 'INSERT INTO overtake(value) VALUES($1) RETURNING *';
 
     return (data) => postgres.query(query, [data.value]).then(next);
   });
 
-  teardown(({ mongo, postgres }) => {
-    await postgres.end()
-    await mongo.end()
+  teardown(async ({ mongo, postgres }) => {
+    await postgres.end();
+    await mongo.end();
   });
 
-  perform('simple test', 100000, [
-    { value: 'test' },
-  ]);
+  perform('simple test', 100000, [[{ value: 'test' }]]);
 });
 ```
 
+Make sure you have installed used modules and run
+
 ```bash
 yarn overtake
 ```
 
+or
+
+```bash
+npx overtake
+```
+
 Please take a look at [benchmarks](__benchmarks__) to see more examples
 
 ## License

package/cli.js

@@ -1,28 +1,23 @@
 #!/usr/bin/env node
+
+import { promisify } from 'util';
 import Path from 'path';
 import glob from 'glob';
-import { benchmark, setup, teardown, measure, perform, suites, runner } from './index.js';
+import { load, benchmark, setup, teardown, measure, perform, run, defaultReporter } from './index.js';
 
+const globAsync = promisify(glob);
 const pattern = process.argv[2] || '**/__benchmarks__/**/*.js';
 
-Object.assign(globalThis, {
-  benchmark,
-  setup,
-  teardown,
-  measure,
-  perform,
-});
+Object.assign(globalThis, { benchmark, setup, teardown, measure, perform });
 
-glob(pattern, async (err, files) => {
-  try {
-    for (const file of files) {
-      const filepath = Path.resolve(file);
-      await import(filepath);
-    }
-  } catch (e) {
-    console.error(e.stack);
-  }
-  for (const suite of suites) {
-    await runner(suite);
+(async () => {
+  const files = await globAsync(pattern);
+  const scripts = [];
+  for (const file of files) {
+    const filename = Path.resolve(file);
+    const script = await load(filename);
+    scripts.push(script);
   }
-});
+
+  await run(scripts, defaultReporter);
+})().catch((e) => console.error(e));

package/index.d.ts

@@ -1,15 +1,56 @@
 declare global {
-  declare function benchmark(title: string, init: () => void): void;
+  type CanBePromise<T> = Promise<T> | T;
 
-  declare function setup<C>(init: () => C): any;
+  interface Report {
+    type: string;
+    success: boolean;
+    count: number;
+    min: number;
+    max: number;
+    sum: number;
+    avg: number;
+    mode: number;
+    p1: number;
+    p5: number;
+    p20: number;
+    p33: number;
+    p50: number;
+    med: number;
+    p66: number;
+    p80: number;
+    p90: number;
+    p95: number;
+    p99: number;
+    setup: number;
+    init: number;
+    cycles: number;
+    teardown: number;
+    total: number;
+  }
 
-  declare function teardown<C>(teardown: (context: C) => void): void;
+  type MeasureInitResult = CanBePromise<() => void>;
 
-  declare function measure(title: string, init: (next: () => void) => void): void;
-  declare function measure<C>(title: string, init: (context: C, next: () => void) => void): void;
-  declare function measure<C, A>(title: string, init: (context: C, args: A, next: () => void) => void): void;
+  function measure(title: string, init: () => MeasureInitResult): void;
+  function measure(title: string, init: (next: () => void) => MeasureInitResult): void;
+  function measure<C>(title: string, init: (context: C, next: () => void) => MeasureInitResult): void;
+  function measure<C, A>(title: string, init: (context: C, args: A, next: () => void) => MeasureInitResult): void;
 
-  declare function perform<A>(title: string, counter: number, args: A): void;
-}
+  function perform<A>(title: string, counter: number, args: A): void;
+
+  function setup<C>(init: () => CanBePromise<C>): void;
+
+  function teardown<C>(teardown: (context: C) => CanBePromise<void>): void;
 
-export { benchmark, setup, teardown, measure, perform };
+  interface Suite {
+    title: string;
+    setup: typeof setup;
+    teardown: typeof teardown;
+    measures: typeof measure[];
+    performs: typeof perform[];
+    init: () => void;
+  }
+
+  function benchmark(title: string, init: () => void): void;
+
+  function script(filename): Promise<Suite[]>;
+}

package/index.js

@@ -1,67 +1,112 @@
+import { createContext } from 'conode';
 import WorkerThreads from 'worker_threads';
 
-export const suites = [];
+const overtakeContext = createContext();
+const suiteContext = createContext();
 
-const NOOP = () => {};
+export const NOOP = () => {};
 
-const SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏', '⠿'];
-const SPINNER_INTERVAL = 80;
+export const setup = (fn) => {
+  suiteContext.getContext().setup = fn;
+};
 
-const ACCURACY = 6;
+export const teardown = (fn) => {
+  suiteContext.getContext().teardown = fn;
+};
 
-const renderSpinner = (index) => {
-  process.stdout.moveCursor(0, -1);
-  process.stdout.write(SPINNER[index]);
-  process.stdout.moveCursor(-3, 1);
+export const measure = (title, fn) => {
+  suiteContext.getContext().measures.push({ title, init: fn });
 };
 
-export function benchmark(title, init) {
-  const suite = {
-    title,
-    current: true,
-    measures: [],
-    performs: [],
-    setup: NOOP,
-    teardown: NOOP,
-  };
-  suites.push(suite);
-  init();
-  suite.current = false;
-}
+export const perform = (title, count, args) => {
+  suiteContext.getContext().performs.push({ title, count, args });
+};
 
-export function getSuite() {
-  const suite = suites[suites.length - 1];
-  if (!suite.current) {
-    throw new Error('should be inside benchmark');
-  }
-  return suite;
-}
+export const benchmark = (title, fn) => {
+  const setup = NOOP;
+  const teardown = NOOP;
+  const measures = [];
+  const performs = [];
 
-export function setup(init) {
-  const suite = getSuite();
-  suite.setup = init;
-}
+  overtakeContext.getContext().suites.push({
+    title,
+    setup,
+    teardown,
+    measures,
+    performs,
+    init: fn,
+  });
+};
 
-export function teardown(init) {
-  const suite = getSuite();
-  suite.teardown = init;
-}
+export const load = async (filename) => {
+  const suites = [];
+  const script = { filename, suites };
+  await overtakeContext.contextualize(script, () => import(filename));
 
-export function measure(title, init) {
-  const suite = getSuite();
-  suite.measures.push({ title, init });
-}
+  return script;
+};
+const map = {
+  script: '⭐ Script ',
+  suite: '⇶ Suite ',
+  perform: '➤ Perform ',
+  measure: '✓ Measure',
+};
 
-export function perform(title, count, args) {
-  const suite = getSuite();
-  suite.performs.push({ title, count, args });
-}
+export const defaultReporter = async (type, title, test) => {
+  console.group(`${map[type]} ${title}`);
+  await test({ test: defaultReporter, output: (report) => console.table(report) });
+  console.groupEnd();
+};
 
+const ACCURACY = 6;
 export function formatFloat(value, digits = ACCURACY) {
   return parseFloat(value.toFixed(digits));
 }
 
-export async function runWorker({ args, count, ...options }) {
+export const run = async (scripts, reporter) => {
+  for (const script of scripts) {
+    await reporter('script', script.filename, async (scriptTest) => {
+      for (const suite of script.suites) {
+        await scriptTest.test('suite', suite.title, async (suiteTest) => {
+          await suiteContext.contextualize(suite, suite.init);
+          for (const perform of suite.performs) {
+            await suiteTest.test('perform', perform.title, async (performTest) => {
+              for (const measure of suite.measures) {
+                await performTest.test('measure', measure.title, async (measureTest) => {
+                  const result = await runWorker({
+                    setup: suite.setup,
+                    teardown: suite.teardown,
+                    init: measure.init,
+                    count: perform.count,
+                    args: perform.args,
+                  });
+                  if (result.success) {
+                    measureTest.output({
+                      [formatFloat(result.mode)]: {
+                        total: formatFloat(result.total),
+                        med: formatFloat(result.med),
+                        p95: formatFloat(result.p95),
+                        p99: formatFloat(result.p99),
+                      },
+                    });
+                  } else {
+                    measureTest.output({
+                      error: {
+                        reason: result.error,
+                      },
+                    });
+                  }
+                });
+              }
+            });
+          }
+        });
+      }
+    });
+  }
+};
+
+export async function runWorker({ args, count, ...options }, onProgress = null) {
   const setupCode = options.setup.toString();
   const teardownCode = options.teardown.toString();
   const initCode = options.init.toString();
@@ -72,81 +117,42 @@ export async function runWorker({ args, count, ...options }) {
     count,
     args,
   });
-  let i = 0;
-  const spinnerSize = SPINNER.length - 1;
-
-  const timerId = setInterval(() => renderSpinner(i++ % spinnerSize), SPINNER_INTERVAL);
 
   const worker = new WorkerThreads.Worker(new URL('runner.js', import.meta.url), { argv: [params] });
   return new Promise((resolve) => {
-    worker.on('message', resolve);
+    worker.on('message', (data) => {
+      if (onProgress && data.type === 'progress') {
+        onProgress(data);
+      } else if (data.type === 'report') {
+        resolve(data);
+      }
+    });
     worker.on('error', (error) => resolve({ success: false, error: error.message }));
-  }).finally((result) => {
-    clearInterval(timerId);
-    renderSpinner(spinnerSize);
-
-    return result;
   });
 }
 
-export async function runner(suite) {
-  console.group(`\nStart ${suite.title} benchmark`);
-
-  for (let measureIdx = 0; measureIdx < suite.measures.length; measureIdx++) {
-    const currentMeasure = suite.measures[measureIdx];
-    const reports = {};
-    console.group(`\n  Measuring performance of ${currentMeasure.title}`);
-    for (let performIdx = 0; performIdx < suite.performs.length; performIdx++) {
-      const currentPerform = suite.performs[performIdx];
-      const report = await runWorker({
-        setup: suite.setup,
-        teardown: suite.teardown,
-        init: currentMeasure.init,
-        count: currentPerform.count,
-        args: currentPerform.args,
-      });
-      reports[currentPerform.title] = { title: perform.title, report };
-      if (report.success) {
-        reports[currentPerform.title] = {
-          Count: currentPerform.count,
-          'Setup, ms': formatFloat(report.setup),
-          'Work, ms': formatFloat(report.work),
-          'Avg, ms': formatFloat(report.avg),
-          'Mode, ms': report.mode,
-        };
-      } else {
-        reports[`${currentPerform.title} ${currentMeasure.title}`] = {
-          Count: currentPerform.count,
-          'Setup, ms': '?',
-          'Work, ms': '?',
-          'Avg, ms': '?',
-          'Mode, ms': '?',
-          Error: report.error,
-        };
-      }
-    }
-    console.groupEnd();
-    console.table(reports);
-  }
-  console.groupEnd();
-}
-
 const FALSE_START = () => {
   throw new Error('False start');
 };
 
 export async function start(input) {
-  const { setupCode, teardownCode, initCode, count, args = [[]] } = JSON.parse(input);
+  const { setupCode, teardownCode, initCode, count, reportInterval = 500, args = [[]] } = JSON.parse(input);
   const setup = Function(`return ${setupCode};`)();
   const teardown = Function(`return ${teardownCode};`)();
   const init = Function(`return ${initCode};`)();
+  const send = WorkerThreads.parentPort ? (data) => WorkerThreads.parentPort.postMessage(data) : (data) => console.log(data);
 
   let i = count;
   let done = FALSE_START;
 
   const timings = [];
   const argSize = args.length;
+
+  send({ type: 'progress', stage: 'setup' });
+  const startMark = performance.now();
   const context = await setup();
+  const setupMark = performance.now();
+
   const initArgs = [() => done()];
   if (init.length > 2) {
     initArgs.unshift(args);
@@ -154,70 +160,98 @@ export async function start(input) {
   if (init.length > 1) {
     initArgs.unshift(context);
   }
-  const startMark = performance.now();
+
+  send({ type: 'progress', stage: 'init' });
+  const initMark = performance.now();
   const action = await init(...initArgs);
-  const workMark = performance.now();
+  const initDoneMark = performance.now();
 
   try {
+    let lastCheck = performance.now();
     const loop = (resolve, reject) => {
-      const argIdx = i % argSize;
+      const idx = count - i;
+      const argIdx = idx % argSize;
       const timerId = setTimeout(reject, 10000, new Error('Timeout'));
 
       done = () => {
-        // eslint-disable-next-line
-        const elapsed = performance.now() - startTickTime;
+        const doneTick = performance.now();
+        const elapsed = doneTick - startTickTime;
         clearTimeout(timerId);
         done = FALSE_START;
         timings.push(elapsed);
-
+        if (doneTick - lastCheck > reportInterval) {
+          lastCheck = doneTick;
+          send({ type: 'progress', stage: 'cycles', progress: idx / count });
+        }
         resolve();
       };
+
       const startTickTime = performance.now();
-      action(...args[argIdx], count - i);
+      action(...args[argIdx], idx);
     };
+    const cyclesMark = performance.now();
 
+    send({ type: 'progress', stage: 'cycles', progress: 0 });
     while (i--) {
       await new Promise(loop);
     }
+    send({ type: 'progress', stage: 'teardown' });
+    const teardownMark = performance.now();
+    await teardown(context);
     const completeMark = performance.now();
+    send({ type: 'progress', stage: 'complete', progress: (count - i) / count });
 
-    await teardown(context);
+    timings.sort((a, b) => a - b);
+
+    const min = timings[0];
+    const max = timings[timings.length - 1];
+    const range = max - min || Number.MIN_VALUE;
+    const sum = timings.reduce((a, b) => a + b, 0);
+    const avg = sum / timings.length;
 
-    timings.sort();
-
-    const {
-      mode: { time: mode },
-    } = timings
-      .map((time) => parseFloat(time.toFixed(ACCURACY)))
-      .reduce((result, time) => {
-        const value = (result[time] || 0) + 1;
-        const mode = !result.mode || result.mode.value < value ? { time, value } : result.mode;
-        return { ...result, [time]: value, mode };
-      }, {});
-
-    const min = timings.reduce((a, b) => Math.min(a, b), Infinity);
-    const max = timings.reduce((a, b) => Math.max(a, b), 0);
-    const avg = timings.reduce((a, b) => a + b, 0) / count;
-    const p90idx = parseInt(timings.length * 0.9, 10);
-    const p95idx = parseInt(timings.length * 0.95, 10);
-    const p99idx = parseInt(timings.length * 0.99, 10);
-    const p90 = timings[p90idx];
-    const p95 = timings[p95idx];
-    const p99 = timings[p99idx];
-
-    WorkerThreads.parentPort.postMessage({
+    const step = range / 99 || Number.MIN_VALUE;
+    const buckets = Array(100)
+      .fill(0)
+      .map((_, idx) => [min + idx * step, 0]);
+
+    // Calc mode O(n)
+    timings.forEach((timing, idx) => {
+      const index = Math.round((timing - min) / step);
+      buckets[index][1] += 1;
+    });
+    buckets.sort((a, b) => a[1] - b[1]);
+
+    const percentile = (p) => timings[Math.trunc((p * timings.length) / 100)];
+    const mode = buckets[buckets.length - 1][0];
+
+    send({
+      type: 'report',
+      success: true,
+      count: timings.length,
       min,
       max,
+      sum,
       avg,
-      p90,
-      p95,
-      p99,
       mode,
-      setup: workMark - startMark,
-      work: completeMark - workMark,
-      success: true,
+      p1: percentile(1),
+      p5: percentile(5),
+      p10: percentile(10),
+      p20: percentile(20),
+      p33: percentile(33),
+      p50: percentile(50),
+      med: percentile(50),
+      p66: percentile(66),
+      p80: percentile(80),
+      p90: percentile(90),
+      p95: percentile(95),
+      p99: percentile(99),
+      setup: setupMark - startMark,
+      init: initDoneMark - initMark,
+      cycles: teardownMark - cyclesMark,
+      teardown: completeMark - teardownMark,
+      total: completeMark - setupMark,
     });
   } catch (error) {
-    WorkerThreads.parentPort.postMessage({ success: false, error: error.stack });
+    send({ type: 'report', success: false, error: error.stack });
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "overtake",
-  "version": "0.0.1-rc1",
+  "version": "0.0.2",
   "description": "NodeJS performance benchmark",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,8 +9,8 @@
     "overtake": "cli.js"
   },
   "scripts": {
-    "measure": "node ./__runner__/class-vs-function.js",
-    "test": "yarn node --experimental-vm-modules $(yarn bin jest)",
+    "start": "node ./cli.js",
+    "test": "yarn node --experimental-vm-modules $(yarn bin jest) --detectOpenHandles",
     "lint": "eslint .",
     "prepare": "husky install"
   },
@@ -33,16 +33,13 @@
   "devDependencies": {
     "@jest/globals": "^27.5.1",
     "@types/jest": "^27.4.1",
-    "eslint": "^8.14.0",
-    "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-import": "^2.26.0",
     "husky": "^7.0.4",
     "jest": "^27.5.1",
     "prettier": "^2.6.2",
     "pretty-quick": "^3.1.3"
   },
   "dependencies": {
+    "conode": "^0.1.0",
     "glob": "^8.0.1"
   }
 }