overtake 0.0.1-rc2 → 0.0.3

package/README.md

@@ -55,7 +55,7 @@ benchmark('mongodb vs postgres', () => {
     return { postgres, mongo };
   });
 
-  measure('mongodb inserts', ({ mongo }/* context */, next) => {
+  measure('mongodb inserts', ({ mongo } /* context */, next) => {
     // prepare a collection
     const database = mongo.db('overtake');
     const test = database.collection('test');
@@ -63,21 +63,19 @@ benchmark('mongodb vs postgres', () => {
     return (data) => test.insertOne(data).then(next);
   });
 
-  measure('postgres inserts', ({ postgres }/* context */, next) => {
+  measure('postgres inserts', ({ postgres } /* context */, next) => {
     // prepare a query
     const query = 'INSERT INTO overtake(value) VALUES($1) RETURNING *';
 
     return (data) => postgres.query(query, [data.value]).then(next);
   });
 
-  teardown(({ mongo, postgres }) => {
-    await postgres.end()
-    await mongo.end()
+  teardown(async ({ mongo, postgres }) => {
+    await postgres.end();
+    await mongo.end();
   });
 
-  perform('simple test', 100000, [
-    { value: 'test' },
-  ]);
+  perform('simple test', 100000, [[{ value: 'test' }]]);
 });
 ```
 
@@ -87,6 +85,12 @@ Make sure you have installed used modules and run
 yarn overtake
 ```
 
+or
+
+```bash
+npx overtake
+```
+
 Please take a look at [benchmarks](__benchmarks__) to see more examples
 
 ## License

package/cli.js

@@ -1,19 +1,77 @@
 #!/usr/bin/env node
 
+import { Command } from 'commander';
 import { promisify } from 'util';
+import Path from 'path';
 import glob from 'glob';
-import { Overtake } from './index.js';
-import { defaultReporter } from './reporter.js';
-
-const globAsync = promisify(glob);
-const pattern = process.argv[2] || '**/__benchmarks__/**/*.js';
-const overtake = new Overtake({});
-
-(async () => {
-  const files = await globAsync(pattern);
-  await overtake.load(files);
-  if (overtake.reporters.length === 0) {
-    reporter(defaultReporter);
-  }
-  await overtake.run();
-})().catch((e) => console.error(e));
+import { load, createScript, benchmark, setup, teardown, measure, perform, run, defaultReporter } from './index.js';
+import packageJson from './package.json' assert { type: 'json' };
+
+const commands = new Command();
+
+commands.name('overtake').description(packageJson.description).version(packageJson.version, '-v, --version');
+
+commands
+  .argument('[files...]', 'file paths or path patterns to search benchmark scripts')
+  .option('-i, --inline [inline]', 'inline code to benchmark', (value, previous) => previous.concat([value]), [])
+  .option('-c, --count [count]', 'perform count for inline code', (v) => parseInt(v))
+  .action(async (patterns, { count = 100, inline }) => {
+    Object.assign(globalThis, { benchmark, setup, teardown, measure, perform });
+
+    const globAsync = promisify(glob);
+    const foundFiles = await Promise.all(patterns.map((pattern) => globAsync(pattern)));
+    const files = [
+      ...new Set(
+        []
+          .concat(...foundFiles)
+          .map((filename) => Path.resolve(filename))
+          .filter(Boolean)
+      ),
+    ];
+
+    const scripts = [];
+    if (inline.length) {
+      const inlineScript = await createScript('', () => {
+        benchmark('', () => {
+          inline.forEach((code) => {
+            measure(code, `() => () => { ${code} }`);
+          });
+          perform('', count);
+        });
+      });
+      scripts.push(inlineScript);
+    }
+
+    for (const file of files) {
+      const filename = Path.resolve(file);
+      const script = await load(filename);
+      scripts.push(script);
+    }
+
+    await run(scripts, defaultReporter);
+  });
+
+commands.on('--help', () => {
+  console.log('');
+  console.log('Examples:');
+  console.log('  $ overtake **/__benchmarks__/*.js');
+  console.log('  $ overtake -i "class A{}" -i "function A(){}" -i "const A = () => {}" -c 1000000');
+  console.log('  $ overtake -v');
+});
+
+commands.parse(process.argv);
+
+//
+
+//
+// (async () => {
+//   const files = await globAsync(pattern);
+//   const scripts = [];
+//   for (const file of files) {
+//     const filename = Path.resolve(file);
+//     const script = await load(filename);
+//     scripts.push(script);
+//   }
+//
+//   await run(scripts, defaultReporter);
+// })().catch((e) => console.error(e));

package/index.d.ts

@@ -0,0 +1,56 @@
+declare global {
+  type CanBePromise<T> = Promise<T> | T;
+
+  interface Report {
+    type: string;
+    success: boolean;
+    count: number;
+    min: number;
+    max: number;
+    sum: number;
+    avg: number;
+    mode: number;
+    p1: number;
+    p5: number;
+    p20: number;
+    p33: number;
+    p50: number;
+    med: number;
+    p66: number;
+    p80: number;
+    p90: number;
+    p95: number;
+    p99: number;
+    setup: number;
+    init: number;
+    cycles: number;
+    teardown: number;
+    total: number;
+  }
+
+  type MeasureInitResult = CanBePromise<() => void>;
+
+  function measure(title: string, init: () => MeasureInitResult): void;
+  function measure(title: string, init: (next: () => void) => MeasureInitResult): void;
+  function measure<C>(title: string, init: (context: C, next: () => void) => MeasureInitResult): void;
+  function measure<C, A>(title: string, init: (context: C, args: A, next: () => void) => MeasureInitResult): void;
+
+  function perform<A>(title: string, counter: number, args: A): void;
+
+  function setup<C>(init: () => CanBePromise<C>): void;
+
+  function teardown<C>(teardown: (context: C) => CanBePromise<void>): void;
+
+  interface Suite {
+    title: string;
+    setup: typeof setup;
+    teardown: typeof teardown;
+    measures: typeof measure[];
+    performs: typeof perform[];
+    init: () => void;
+  }
+
+  function benchmark(title: string, init: () => void): void;
+
+  function script(filename): Promise<Suite[]>;
+}

package/index.js

@@ -1,185 +1,118 @@
-import Path from 'path';
+import { createContext } from 'conode';
 import WorkerThreads from 'worker_threads';
-import { Event } from 'evnty';
 
-export const NOOP = () => {};
-
-export class Perform {
-  title = '';
-
-  count = 0;
-
-  args;
-
-  constructor(overtake, title, count, args) {
-    this.title = title;
-    this.count = count;
-    this.args = args;
-  }
-}
-
-export class Measure {
-  title = '';
+const overtakeContext = createContext();
+const suiteContext = createContext();
 
-  init = NOOP;
-
-  constructor(overtake, title, init = NOOP) {
-    this.title = title;
-    this.init = init;
-  }
-}
-
-export class Suite {
-  title = '';
-
-  setup = NOOP;
-
-  measures = [];
+export const NOOP = () => {};
 
-  performs = [];
+export const setup = (fn) => {
+  suiteContext.getContext().setup = fn;
+};
 
-  teardown = NOOP;
+export const teardown = (fn) => {
+  suiteContext.getContext().teardown = fn;
+};
 
-  #title = '';
+export const measure = (title, fn) => {
+  suiteContext.getContext().measures.push({ title, init: fn });
+};
 
-  #init = NOOP;
+export const perform = (title, count, args) => {
+  suiteContext.getContext().performs.push({ title, count, args });
+};
 
-  #overtake = null;
+export const benchmark = (title, fn) => {
+  const setup = NOOP;
+  const teardown = NOOP;
+  const measures = [];
+  const performs = [];
+
+  overtakeContext.getContext().suites.push({
+    title,
+    setup,
+    teardown,
+    measures,
+    performs,
+    init: fn,
+  });
+};
 
-  constructor(overtake, title, init = NOOP) {
-    this.#overtake = overtake;
-    this.title = title;
-    this.#init = init;
-  }
+export const createScript = async (filename, fn) => {
+  const suites = [];
+  const script = { filename, suites };
+  await overtakeContext.contextualize(script, fn);
 
-  async init() {
-    const unsubscribes = [
-      this.#overtake.onSetupRegister.on((setup) => (this.setup = setup)),
-      this.#overtake.onMeasureRegister.on((measure) => this.measures.push(measure)),
-      this.#overtake.onPerformRegister.on((perform) => this.performs.push(perform)),
-      this.#overtake.onTeardownRegister.on((teardown) => (this.teardown = teardown)),
-    ];
-    await this.#init();
-    unsubscribes.forEach((unsubscribe) => unsubscribe());
-  }
-}
+  return script;
+};
 
-export class Script {
-  onLoad = new Event();
+export const load = async (filename) => {
+  return createScript(filename, () => import(filename));
+};
 
-  filename = '';
+const map = {
+  script: '⭐ Script ',
+  suite: '⇶ Suite ',
+  perform: '➤ Perform ',
+  measure: '✓ Measure',
+};
 
-  suites = [];
+export const defaultReporter = async (type, title, test) => {
+  console.group(`${map[type]} ${title}`);
+  await test({ test: defaultReporter, output: (report) => console.table(report) });
+  console.groupEnd();
+};
 
-  constructor(filename) {
-    this.filename = filename;
-  }
+const ACCURACY = 6;
+export function formatFloat(value, digits = ACCURACY) {
+  return parseFloat(value.toFixed(digits));
 }
 
-export class Overtake {
-  onLoad = new Event();
-
-  onRun = new Event();
-
-  onComplete = new Event();
-
-  onScriptRegister = new Event();
-
-  onScriptStart = new Event();
-
-  onScriptComplete = new Event();
-
-  onSuiteRegister = new Event();
-
-  onSuiteStart = new Event();
-
-  onSuiteComplete = new Event();
-
-  onSetupRegister = new Event();
-
-  onTeardownRegister = new Event();
-
-  onMeasureRegister = new Event();
-
-  onMeasureStart = new Event();
-
-  onMeasureComplete = new Event();
-
-  onPerformRegister = new Event();
-
-  onPerformStart = new Event();
-
-  onPerformProgress = new Event();
-
-  onPerformComplete = new Event();
-
-  onReport = new Event();
-
-  scripts = [];
-
-  reporters = [];
-
-  constructor(options = {}) {
-    Object.assign(globalThis, {
-      benchmark: (title, init) => this.onSuiteRegister(new Suite(this, title, init)),
-      setup: (init) => this.onSetupRegister(init),
-      teardown: (init) => this.onTeardownRegister(init),
-      measure: (title, init) => this.onMeasureRegister(new Measure(this, title, init)),
-      perform: (title, count, args) => this.onPerformRegister(new Perform(this, title, count, args)),
-      reporter: (reporter) => this.reporters.push(reporter(this)),
-    });
-  }
-
-  async load(files) {
-    this.onLoad(this);
-    for (const file of files) {
-      const filename = Path.resolve(file);
-      const script = new Script(filename);
-      const unsubscribe = this.onSuiteRegister.on((suite) => {
-        script.suites.push(suite);
-      });
-      await import(filename);
-      unsubscribe();
-      this.scripts.push(script);
-      this.onScriptRegister(script);
-    }
-  }
-
-  async run() {
-    this.onRun();
-    for (const script of this.scripts) {
-      this.onScriptStart(script);
+export const run = async (scripts, reporter) => {
+  for (const script of scripts) {
+    await reporter('script', script.filename, async (scriptTest) => {
       for (const suite of script.suites) {
-        await suite.init().catch((e) => console.error(e));
-        this.onSuiteStart(suite);
-        for (const measure of suite.measures) {
-          this.onMeasureStart(measure);
+        await scriptTest.test('suite', suite.title, async (suiteTest) => {
+          await suiteContext.contextualize(suite, suite.init);
           for (const perform of suite.performs) {
-            this.onPerformStart(perform);
-            const result = await runWorker(
-              {
-                setup: suite.setup,
-                teardown: suite.teardown,
-                init: measure.init,
-                count: perform.count,
-                args: perform.args,
-              },
-              this.onPerformProgress
-            );
-            this.onPerformComplete(perform);
-            this.onReport(result);
+            await suiteTest.test('perform', perform.title, async (performTest) => {
+              for (const measure of suite.measures) {
+                await performTest.test('measure', measure.title, async (measureTest) => {
+                  const result = await runWorker({
+                    setup: suite.setup,
+                    teardown: suite.teardown,
+                    init: measure.init,
+                    count: perform.count,
+                    args: perform.args,
+                  });
+                  if (result.success) {
+                    measureTest.output({
+                      [formatFloat(result.mode)]: {
+                        total: formatFloat(result.total),
+                        med: formatFloat(result.med),
+                        p95: formatFloat(result.p95),
+                        p99: formatFloat(result.p99),
+                        count: result.count,
+                      },
+                    });
+                  } else {
+                    measureTest.output({
+                      error: {
+                        reason: result.error,
+                      },
+                    });
+                  }
+                });
+              }
+            });
           }
-          this.onMeasureComplete(perform);
-        }
-        this.onSuiteComplete(perform);
+        });
       }
-      this.onScriptComplete(perform);
-    }
-    this.onComplete(this);
+    });
   }
-}
+};
 
-export async function runWorker({ args, count, ...options }, onProgress) {
+export async function runWorker({ args, count, ...options }, onProgress = null) {
   const setupCode = options.setup.toString();
   const teardownCode = options.teardown.toString();
   const initCode = options.init.toString();
@@ -194,7 +127,7 @@ export async function runWorker({ args, count, ...options }, onProgress) {
   const worker = new WorkerThreads.Worker(new URL('runner.js', import.meta.url), { argv: [params] });
   return new Promise((resolve) => {
     worker.on('message', (data) => {
-      if (data.type === 'progress') {
+      if (onProgress && data.type === 'progress') {
         onProgress(data);
       } else if (data.type === 'report') {
         resolve(data);
@@ -213,8 +146,8 @@ export async function start(input) {
   const setup = Function(`return ${setupCode};`)();
   const teardown = Function(`return ${teardownCode};`)();
   const init = Function(`return ${initCode};`)();
+  const initArgsSize = init.length;
   const send = WorkerThreads.parentPort ? (data) => WorkerThreads.parentPort.postMessage(data) : (data) => console.log(data);
-
   let i = count;
   let done = FALSE_START;
 
@@ -226,11 +159,14 @@ export async function start(input) {
   const context = await setup();
   const setupMark = performance.now();
 
-  const initArgs = [() => done()];
-  if (init.length > 2) {
+  const initArgs = [];
+  if (initArgsSize !== 0) {
+    initArgs.push(() => done());
+  }
+  if (initArgsSize > 2) {
     initArgs.unshift(args);
   }
-  if (init.length > 1) {
+  if (initArgsSize > 1) {
     initArgs.unshift(context);
   }
 
@@ -261,6 +197,9 @@ export async function start(input) {
 
       const startTickTime = performance.now();
       action(...args[argIdx], idx);
+      if (!initArgsSize) {
+        done();
+      }
     };
     const cyclesMark = performance.now();
 
@@ -294,14 +233,7 @@ export async function start(input) {
     });
     buckets.sort((a, b) => a[1] - b[1]);
 
-    const medIdx = Math.trunc((50 * timings.length) / 100);
-    const med = timings[medIdx];
-    const p90Idx = Math.trunc((90 * timings.length) / 100);
-    const p90 = timings[p90Idx];
-    const p95Idx = Math.trunc((95 * timings.length) / 100);
-    const p95 = timings[p95Idx];
-    const p99Idx = Math.trunc((99 * timings.length) / 100);
-    const p99 = timings[p99Idx];
+    const percentile = (p) => timings[Math.trunc((p * timings.length) / 100)];
     const mode = buckets[buckets.length - 1][0];
 
     send({
@@ -312,11 +244,19 @@ export async function start(input) {
       max,
       sum,
       avg,
-      med,
       mode,
-      p90,
-      p95,
-      p99,
+      p1: percentile(1),
+      p5: percentile(5),
+      p10: percentile(10),
+      p20: percentile(20),
+      p33: percentile(33),
+      p50: percentile(50),
+      med: percentile(50),
+      p66: percentile(66),
+      p80: percentile(80),
+      p90: percentile(90),
+      p95: percentile(95),
+      p99: percentile(99),
       setup: setupMark - startMark,
       init: initDoneMark - initMark,
       cycles: teardownMark - cyclesMark,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "overtake",
-  "version": "0.0.1-rc2",
+  "version": "0.0.3",
   "description": "NodeJS performance benchmark",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,7 +9,7 @@
     "overtake": "cli.js"
   },
   "scripts": {
-    "start": "node ./cli.js",
+    "start": "node --no-warnings ./cli.js",
     "test": "yarn node --experimental-vm-modules $(yarn bin jest) --detectOpenHandles",
     "lint": "eslint .",
     "prepare": "husky install"
@@ -39,7 +39,8 @@
     "pretty-quick": "^3.1.3"
   },
   "dependencies": {
-    "evnty": "^0.7.4",
+    "commander": "^9.2.0",
+    "conode": "^0.1.0",
     "glob": "^8.0.1"
   }
 }

package/reporter.js

@@ -1,90 +0,0 @@
-const SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏', '⠿'];
-const SPINNER_INTERVAL = 100;
-const SPINNER_PADDING = 5;
-
-const ACCURACY = 6;
-
-/** TODO
- *  [PROGRESS] script/file.js
- *    Test suite title
- *      ➤ Measure
- *        ✓ Perform
- *        ✕ Perform
- * */
-
-const renderString = (message, x, direction) => {
-  const size = message.length;
-  process.stdout.moveCursor(0, -1);
-  process.stdout.cursorTo(x);
-  process.stdout.clearLine(direction);
-  if (direction === -1) {
-    process.stdout.cursorTo(0);
-  }
-  process.stdout.write(message);
-  process.stdout.moveCursor(-process.stdout.rows, 1);
-
-  return size;
-};
-
-const renderSpinner = (index) => renderString(SPINNER[index], 0, -1);
-
-const paddings = [];
-
-const renderMessage = (column, message) => {
-  const padding = SPINNER_PADDING + column * 10;
-  paddings[column + 1] = renderString(message, padding, 1);
-};
-
-export function formatFloat(value, digits = ACCURACY) {
-  return parseFloat(value.toFixed(digits));
-}
-
-export function defaultReporter(overtake) {
-  const spinnerSize = SPINNER.length - 1;
-  let i = 0;
-  let timerId = null;
-
-  overtake.onLoad.on(() => {});
-  overtake.onRun.on(() => {
-    console.log();
-  });
-  overtake.onComplete.on(() => {});
-  overtake.onScriptRegister.on(() => {});
-  overtake.onScriptStart.on(() => {});
-  overtake.onScriptComplete.on(() => {});
-  overtake.onSuiteRegister.on(() => {});
-  overtake.onSuiteStart.on(() => {});
-  overtake.onSuiteComplete.on(() => {});
-  overtake.onSetupRegister.on(() => {});
-  overtake.onTeardownRegister.on(() => {});
-  overtake.onMeasureRegister.on(() => {});
-  overtake.onMeasureStart.on((measure) => {
-    console.log(measure.title);
-  });
-  overtake.onMeasureComplete.on(() => {});
-  overtake.onPerformRegister.on(() => {});
-  overtake.onPerformStart.on((perform) => {
-    console.log(perform.title);
-    console.log();
-    i = 0;
-    timerId = setInterval(() => renderSpinner(i++ % spinnerSize), SPINNER_INTERVAL);
-  });
-  overtake.onPerformComplete.on(() => {
-    clearInterval(timerId);
-    renderSpinner(spinnerSize);
-  });
-  overtake.onPerformProgress.on(({ stage, progress }) => {
-    renderMessage(0, stage);
-    if (typeof progress !== 'undefined') {
-      renderMessage(1, `${(progress * 100).toFixed(0)}%`.padStart(4, ' '));
-    }
-  });
-  overtake.onReport.on((report) => {
-    if (report.success) {
-      renderMessage(2, `total:${report.total.toFixed(0)}ms  mode:${report.mode.toFixed(ACCURACY)}ms`);
-    } else {
-      renderMessage(1, report.error);
-    }
-    console.log();
-  });
-}