@indutny/bencher 1.1.5 → 1.2.0

package/README.md

@@ -43,7 +43,7 @@ export default () => {
 
 ```sh
 $ bencher benchmark.js
-runner: 1058.6 ops/s (±4.5, p=0.05, n=98)
+runner: 1’037.8 ops/sec (±18.8, p=0.001, n=98)
 ```
 
 ## LICENSE

package/dist/bin/bencher.js

@@ -32,8 +32,11 @@ const promises_1 = require("fs/promises");
 const yargs_1 = __importDefault(require("yargs"));
 const helpers_1 = require("yargs/helpers");
 // ANSI colors
-const BOLD = 1;
-const ITALIC = 3;
+const BOLD = '\x1b[1m';
+const ITALIC = '\x1b[3m';
+const RESET = '\x1b[m';
+const GREY = '\x1b[90m';
+const RED = '\x1b[31m';
 // Go back to previous line, clear the line
 const PREV_LINE = '\x1b[F\x1b[K';
 // From https://github.com/mayuki/Kurukuru
@@ -61,19 +64,24 @@ async function main() {
         .option('w', {
         alias: 'sweep-width',
         type: 'number',
-        default: 10,
+        default: 5,
         describe: 'width of iteration sweep',
     })
-        .option('warm-up-iterations', {
+        .option('warm-up-duration', {
         type: 'number',
-        default: 100,
-        describe: 'number of warm up iterations',
+        default: 0.5,
+        describe: 'duration of warm up',
+    })
+        .option('ignore-outliers', {
+        alias: 'q',
+        type: 'boolean',
+        describe: "don't report severe outliers",
     }).argv;
     const modules = await Promise.all(argv._.map(async (file) => {
         var _a;
         const path = await (0, promises_1.realpath)(String(file));
         const m = await (_a = path, Promise.resolve().then(() => __importStar(require(_a))));
-        const { duration = argv['duration'], sweepWidth = argv['sweepWidth'], samples = argv['samples'], warmUpIterations = argv['warmUpIterations'], } = m.options ?? {};
+        const { duration = argv['duration'], sweepWidth = argv['sweepWidth'], samples = argv['samples'], warmUpDuration = argv['warmUpDuration'], ignoreOutliers = argv['ignoreOutliers'], } = m.options ?? {};
         if (duration <= 0) {
             throw new Error(`${file}: options.duration must be positive`);
         }
@@ -86,8 +94,8 @@ async function main() {
         if (samples / sweepWidth < 2) {
             throw new Error(`${file}: options.samples must be greater than 2 * sweepWidth`);
         }
-        if (warmUpIterations <= 0) {
-            throw new Error(`${file}: options.warmUpIterations must be positive`);
+        if (warmUpDuration <= 0) {
+            throw new Error(`${file}: options.warmUpDuration must be positive`);
         }
         return {
             name: m.name ?? String(file),
@@ -95,14 +103,15 @@ async function main() {
                 duration,
                 sweepWidth,
                 samples,
-                warmUpIterations,
+                warmUpDuration,
+                ignoreOutliers,
             },
             default: m.default,
         };
     }));
     const maxNameLength = modules.reduce((len, { name }) => Math.max(len, name.length), 0);
     for (const m of modules) {
-        const paddedName = style(m.name, BOLD) + ':' + ' '.repeat(maxNameLength - m.name.length);
+        const paddedName = BOLD + m.name + RESET + ':' + ' '.repeat(maxNameLength - m.name.length);
         // Just to reserve the line
         (0, fs_1.writeSync)(process.stdout.fd, '\n');
         let ticks = 0;
@@ -111,12 +120,20 @@ async function main() {
             ticks = (ticks + 1) % SPINNER.length;
         };
         onTick();
-        const { ops, maxError, usedSamples } = run(m, {
+        const { ops, maxError, outliers, severeOutliers } = run(m, {
             onTick,
         });
-        const stats = '\x1b[90m' +
-            style(`(±${nice(maxError)}, p=${P_VALUE}, n=${usedSamples})`, ITALIC);
-        (0, fs_1.writeSync)(process.stdout.fd, `${PREV_LINE}${paddedName} ${nice(ops)} ops/sec ${stats}\n`);
+        const stats = [
+            `±${nice(maxError)}`,
+            `p=${P_VALUE}`,
+            `o=${outliers + severeOutliers}/${m.options.samples}`,
+        ];
+        let warning = '';
+        if (!m.options.ignoreOutliers && severeOutliers !== 0) {
+            warning = `${RESET}${RED} severe outliers=${severeOutliers}`;
+        }
+        (0, fs_1.writeSync)(process.stdout.fd, `${PREV_LINE}${paddedName} ${nice(ops)} ops/sec ` +
+            `${GREY + ITALIC}(${stats.join(', ')})${warning}${RESET}\n`);
     }
 }
 function run(m, { onTick }) {
@@ -135,21 +152,23 @@ function run(m, { onTick }) {
             nextTick += tickEvery;
         }
     }
-    const { beta, confidence, outliers } = regress(m, samples);
+    const { beta, confidence, outliers, severeOutliers } = regress(m, samples);
     const ops = 1 / beta;
     const lowOps = 1 / (beta + confidence);
     const highOps = 1 / (beta - confidence);
     const maxError = Math.max(highOps - ops, ops - lowOps);
-    const usedSamples = samples.length - outliers;
     return {
         ops,
         maxError,
-        usedSamples,
+        outliers,
+        severeOutliers,
     };
 }
 function warmUp(m) {
     // Initial warm-up
-    for (let i = 0; i < m.options.warmUpIterations; i++) {
+    const coldDuration = measure(m, 1);
+    const warmUpIterations = m.options.warmUpDuration / coldDuration;
+    for (let i = 0; i < warmUpIterations; i++) {
         m.default();
     }
     // Compute max duration per base sample
@@ -192,35 +211,42 @@ function regress(m, samples) {
         }
         bin.push(duration);
     }
-    // Within each iteration bin get rid of the outliers.
-    const withoutOutliers = new Array();
-    for (const [iterations, durations] of bins) {
+    let outliers = 0;
+    let severeOutliers = 0;
+    // Within each iteration bin identify the outliers for reporting purposes.
+    for (const [, durations] of bins) {
         durations.sort();
         const p25 = durations[Math.floor(durations.length * 0.25)] ?? -Infinity;
         const p75 = durations[Math.ceil(durations.length * 0.75)] ?? +Infinity;
         const iqr = p75 - p25;
         const outlierLow = p25 - iqr * 1.5;
         const outlierHigh = p75 + iqr * 1.5;
+        const badOutlierLow = p25 - iqr * 3;
+        const badOutlierHigh = p75 + iqr * 3;
         // Tukey's method
-        const filtered = durations.filter((d) => d >= outlierLow && d <= outlierHigh);
-        for (const duration of filtered) {
-            withoutOutliers.push({ iterations, duration });
+        for (const d of durations) {
+            if (d < badOutlierLow || d > badOutlierHigh) {
+                severeOutliers++;
+            }
+            else if (d < outlierLow || d > outlierHigh) {
+                outliers++;
+            }
         }
     }
-    if (withoutOutliers.length < 2) {
+    if (samples.length < 2) {
         throw new Error(`${m.name}: low sample count`);
     }
     let meanDuration = 0;
     let meanIterations = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         meanDuration += duration;
         meanIterations += iterations;
     }
-    meanDuration /= withoutOutliers.length;
-    meanIterations /= withoutOutliers.length;
+    meanDuration /= samples.length;
+    meanIterations /= samples.length;
     let betaNum = 0;
     let betaDenom = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         betaNum += (duration - meanDuration) * (iterations - meanIterations);
         betaDenom += (iterations - meanIterations) ** 2;
     }
@@ -229,22 +255,20 @@ function regress(m, samples) {
     // Intercept
     const alpha = meanDuration - beta * meanIterations;
     let stdError = 0;
-    for (const { duration, iterations } of withoutOutliers) {
+    for (const { duration, iterations } of samples) {
         stdError += (duration - alpha - beta * iterations) ** 2;
     }
-    stdError /= withoutOutliers.length - 2;
+    stdError /= samples.length - 2;
     stdError /= betaDenom;
     stdError = Math.sqrt(stdError);
     return {
         alpha,
         beta,
         confidence: STUDENT_T * stdError,
-        outliers: samples.length - withoutOutliers.length,
+        outliers,
+        severeOutliers,
     };
 }
-function style(text, code) {
-    return `\x1b[${code}m${text}\x1b[m`;
-}
 function nice(n) {
     let result = n.toFixed(1);
     for (let i = result.length - 5; i > 0; i -= 3) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indutny/bencher",
-  "version": "1.1.5",
+  "version": "1.2.0",
   "description": "Simple benchmarking tool",
   "bin": {
     "bencher": "./dist/bin/bencher.js"