@indutny/bencher 1.1.5 → 2.0.0

package/README.md

@@ -24,11 +24,8 @@ npm install -g @indutny/bencher
 ## Usage
 
 ```js
-// benchmark.js
-export const name = 'runner';
-
 // Function to benchmark
-export default () => {
+export function benchmarkName() => {
   let sum = 0;
   for (let i = 0; i < 1e6; i++) {
     sum += i;
@@ -43,7 +40,7 @@ export default () => {
 
 ```sh
 $ bencher benchmark.js
-runner: 1058.6 ops/s (±4.5, p=0.05, n=98)
+benchmark.js/benchmarkName: 1’037.8 ops/sec (±18.8, p=0.001, n=98)
 ```
 
 ## LICENSE

package/dist/bin/bencher.js

@@ -32,8 +32,11 @@ const promises_1 = require("fs/promises");
 const yargs_1 = __importDefault(require("yargs"));
 const helpers_1 = require("yargs/helpers");
 // ANSI colors
-const BOLD = 1;
-const ITALIC = 3;
+const BOLD = '\x1b[1m';
+const ITALIC = '\x1b[3m';
+const RESET = '\x1b[m';
+const GREY = '\x1b[90m';
+const RED = '\x1b[31m';
 // Go back to previous line, clear the line
 const PREV_LINE = '\x1b[F\x1b[K';
 // From https://github.com/mayuki/Kurukuru
@@ -61,19 +64,24 @@ async function main() {
         .option('w', {
         alias: 'sweep-width',
         type: 'number',
-        default: 10,
+        default: 5,
         describe: 'width of iteration sweep',
     })
-        .option('warm-up-iterations', {
+        .option('warm-up-duration', {
         type: 'number',
-        default: 100,
-        describe: 'number of warm up iterations',
+        default: 0.5,
+        describe: 'duration of warm up',
+    })
+        .option('ignore-outliers', {
+        alias: 'q',
+        type: 'boolean',
+        describe: "don't report severe outliers",
     }).argv;
-    const modules = await Promise.all(argv._.map(async (file) => {
+    const modules = (await Promise.all(argv._.map(async (file) => {
         var _a;
         const path = await (0, promises_1.realpath)(String(file));
         const m = await (_a = path, Promise.resolve().then(() => __importStar(require(_a))));
-        const { duration = argv['duration'], sweepWidth = argv['sweepWidth'], samples = argv['samples'], warmUpIterations = argv['warmUpIterations'], } = m.options ?? {};
+        const { duration = argv['duration'], sweepWidth = argv['sweepWidth'], samples = argv['samples'], warmUpDuration = argv['warmUpDuration'], ignoreOutliers = argv['ignoreOutliers'], } = m.options ?? {};
         if (duration <= 0) {
             throw new Error(`${file}: options.duration must be positive`);
         }
@@ -86,23 +94,30 @@ async function main() {
         if (samples / sweepWidth < 2) {
             throw new Error(`${file}: options.samples must be greater than 2 * sweepWidth`);
         }
-        if (warmUpIterations <= 0) {
-            throw new Error(`${file}: options.warmUpIterations must be positive`);
+        if (warmUpDuration <= 0) {
+            throw new Error(`${file}: options.warmUpDuration must be positive`);
+        }
+        const functions = new Array();
+        for (const key of Object.keys(m)) {
+            if (typeof m[key] === 'function') {
+                functions.push(m[key]);
+            }
         }
-        return {
-            name: m.name ?? String(file),
+        return functions.map((fn) => ({
+            name: `${file}/${fn.name}`,
             options: {
                 duration,
                 sweepWidth,
                 samples,
-                warmUpIterations,
+                warmUpDuration,
+                ignoreOutliers,
             },
-            default: m.default,
-        };
-    }));
+            fn,
+        }));
+    }))).flat();
     const maxNameLength = modules.reduce((len, { name }) => Math.max(len, name.length), 0);
     for (const m of modules) {
-        const paddedName = style(m.name, BOLD) + ':' + ' '.repeat(maxNameLength - m.name.length);
+        const paddedName = BOLD + m.name + RESET + ':' + ' '.repeat(maxNameLength - m.name.length);
         // Just to reserve the line
         (0, fs_1.writeSync)(process.stdout.fd, '\n');
         let ticks = 0;
@@ -111,12 +126,20 @@ async function main() {
             ticks = (ticks + 1) % SPINNER.length;
         };
         onTick();
-        const { ops, maxError, usedSamples } = run(m, {
+        const { ops, maxError, outliers, severeOutliers } = run(m, {
             onTick,
         });
-        const stats = '\x1b[90m' +
-            style(`(±${nice(maxError)}, p=${P_VALUE}, n=${usedSamples})`, ITALIC);
-        (0, fs_1.writeSync)(process.stdout.fd, `${PREV_LINE}${paddedName} ${nice(ops)} ops/sec ${stats}\n`);
+        const stats = [
+            `±${nice(maxError)}`,
+            `p=${P_VALUE}`,
+            `o=${outliers + severeOutliers}/${m.options.samples}`,
+        ];
+        let warning = '';
+        if (!m.options.ignoreOutliers && severeOutliers !== 0) {
+            warning = `${RESET}${RED} severe outliers=${severeOutliers}`;
+        }
+        (0, fs_1.writeSync)(process.stdout.fd, `${PREV_LINE}${paddedName} ${nice(ops)} ops/sec ` +
+            `${GREY + ITALIC}(${stats.join(', ')})${warning}${RESET}\n`);
     }
 }
 function run(m, { onTick }) {
@@ -135,22 +158,24 @@ function run(m, { onTick }) {
             nextTick += tickEvery;
         }
     }
-    const { beta, confidence, outliers } = regress(m, samples);
+    const { beta, confidence, outliers, severeOutliers } = regress(m, samples);
     const ops = 1 / beta;
     const lowOps = 1 / (beta + confidence);
     const highOps = 1 / (beta - confidence);
     const maxError = Math.max(highOps - ops, ops - lowOps);
-    const usedSamples = samples.length - outliers;
     return {
         ops,
         maxError,
-        usedSamples,
+        outliers,
+        severeOutliers,
     };
 }
 function warmUp(m) {
     // Initial warm-up
-    for (let i = 0; i < m.options.warmUpIterations; i++) {
-        m.default();
+    const coldDuration = measure(m, 1);
+    const warmUpIterations = m.options.warmUpDuration / coldDuration;
+    for (let i = 0; i < warmUpIterations; i++) {
+        m.fn();
     }
     // Compute max duration per base sample
     let sampleMultiplier = 0;
@@ -173,7 +198,7 @@ function measure(m, iterations) {
     let sum = 0;
     const start = process.hrtime.bigint();
     for (let i = 0; i < iterations; i++) {
-        sum += m.default();
+        sum += m.fn();
     }
     const duration = Number(process.hrtime.bigint() - start) / 1e9;
     if (isNaN(sum)) {
@@ -192,35 +217,42 @@ function regress(m, samples) {
         }
         bin.push(duration);
     }
-    // Within each iteration bin get rid of the outliers.
-    const withoutOutliers = new Array();
-    for (const [iterations, durations] of bins) {
+    let outliers = 0;
+    let severeOutliers = 0;
+    // Within each iteration bin identify the outliers for reporting purposes.
+    for (const [, durations] of bins) {
         durations.sort();
         const p25 = durations[Math.floor(durations.length * 0.25)] ?? -Infinity;
         const p75 = durations[Math.ceil(durations.length * 0.75)] ?? +Infinity;
         const iqr = p75 - p25;
         const outlierLow = p25 - iqr * 1.5;
         const outlierHigh = p75 + iqr * 1.5;
+        const badOutlierLow = p25 - iqr * 3;
+        const badOutlierHigh = p75 + iqr * 3;
         // Tukey's method
-        const filtered = durations.filter((d) => d >= outlierLow && d <= outlierHigh);
-        for (const duration of filtered) {
-            withoutOutliers.push({ iterations, duration });
+        for (const d of durations) {
+            if (d < badOutlierLow || d > badOutlierHigh) {
+                severeOutliers++;
+            }
+            else if (d < outlierLow || d > outlierHigh) {
+                outliers++;
+            }
         }
     }
-    if (withoutOutliers.length < 2) {
+    if (samples.length < 2) {
         throw new Error(`${m.name}: low sample count`);
     }
     let meanDuration = 0;
     let meanIterations = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         meanDuration += duration;
         meanIterations += iterations;
     }
-    meanDuration /= withoutOutliers.length;
-    meanIterations /= withoutOutliers.length;
+    meanDuration /= samples.length;
+    meanIterations /= samples.length;
     let betaNum = 0;
     let betaDenom = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         betaNum += (duration - meanDuration) * (iterations - meanIterations);
         betaDenom += (iterations - meanIterations) ** 2;
     }
@@ -229,22 +261,20 @@ function regress(m, samples) {
     // Intercept
     const alpha = meanDuration - beta * meanIterations;
     let stdError = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         stdError += (duration - alpha - beta * iterations) ** 2;
     }
-    stdError /= withoutOutliers.length - 2;
+    stdError /= samples.length - 2;
     stdError /= betaDenom;
     stdError = Math.sqrt(stdError);
     return {
         alpha,
         beta,
         confidence: STUDENT_T * stdError,
-        outliers: samples.length - withoutOutliers.length,
+        outliers,
+        severeOutliers,
     };
 }
-function style(text, code) {
-    return `\x1b[${code}m${text}\x1b[m`;
-}
 function nice(n) {
     let result = n.toFixed(1);
     for (let i = result.length - 5; i > 0; i -= 3) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indutny/bencher",
-  "version": "1.1.5",
+  "version": "2.0.0",
   "description": "Simple benchmarking tool",
   "bin": {
     "bencher": "./dist/bin/bencher.js"
@@ -9,13 +9,6 @@
     "dist/bin/*.js",
     "README.md"
   ],
-  "scripts": {
-    "build": "tsc",
-    "lint": "eslint --cache .",
-    "format": "prettier --cache --write .",
-    "check:format": "prettier --cache --check .",
-    "prepublish": "rm -rf dist && npm run build"
-  },
   "keywords": [
     "benchmark",
     "sweep",
@@ -42,5 +35,12 @@
   },
   "dependencies": {
     "yargs": "^17.6.2"
+  },
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint --cache .",
+    "format": "prettier --cache --write .",
+    "check:format": "prettier --cache --check .",
+    "prepublish": "rm -rf dist && npm run build"
   }
-}
+}