@indutny/bencher 1.0.0-rc.1

package/LICENSE

@@ -0,0 +1,19 @@
+Copyright Fedor Indutny, 2022.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,44 @@
+# @indutny/bencher
+
+[![npm](https://img.shields.io/npm/v/@indutny/bencher)](https://www.npmjs.com/package/@indutny/bencher)
+[![size](https://img.shields.io/bundlephobia/minzip/@indutny/bencher)](https://bundlephobia.com/result?p=@indutny/bencher)
+![CI Status](https://github.com/indutny/bencher/actions/workflows/test.yml/badge.svg)
+
+Sneaky equals comparison between objects that checks only the properties that
+were touched.
+
+Inspired by [proxy-compare](https://github.com/dai-shi/proxy-compare).
+
+## Installation
+
+```sh
+npm install -g @indutny/bencher
+```
+
+## Usage
+
+benchmark.js
+
+```js
+export const name = 'benchmark name';
+
+// Function to benchmark
+export default () => {
+  // Make sure to return a side-effect number
+  // (possibly result of a computation) to ensure that the pure function calls
+  // are not optimized out by the runtime.
+  return 0;
+};
+```
+
+```sh
+bencher benchmark.js
+```
+
+## Credits
+
+Name coined by [Scott Nonnenberg](https://github.com/scottnonnenberg/).
+
+## LICENSE
+
+This software is licensed under the MIT License.

package/dist/bin/bencher.js

@@ -0,0 +1,180 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const promises_1 = require("fs/promises");
+async function main() {
+    const modules = await Promise.all(process.argv.slice(2).map(async (file) => {
+        var _a;
+        const path = await (0, promises_1.realpath)(file);
+        const m = await (_a = path, Promise.resolve().then(() => __importStar(require(_a))));
+        const { duration = 5, sweepWidth = 10, samples = 100, warmUpIterations = 100, } = m.options ?? {};
+        if (duration <= 0) {
+            throw new Error(`${file}: options.duration must be positive`);
+        }
+        if (sweepWidth <= 1) {
+            throw new Error(`${file}: options.sweepWidth must be greater than 0`);
+        }
+        if (samples <= 0) {
+            throw new Error(`${file}: options.samples must be positive`);
+        }
+        if (samples / sweepWidth < 2) {
+            throw new Error(`${file}: options.samples must be greater than 2 * sweepWidth`);
+        }
+        if (warmUpIterations <= 0) {
+            throw new Error(`${file}: options.warmUpIterations must be positive`);
+        }
+        return {
+            name: m.name ?? file,
+            options: {
+                duration,
+                sweepWidth,
+                samples,
+                warmUpIterations,
+            },
+            default: m.default,
+        };
+    }));
+    for (const m of modules) {
+        run(m);
+    }
+}
+function run(m) {
+    const baseIterations = warmUp(m);
+    const samples = new Array();
+    for (let i = 0; i < m.options.samples; i++) {
+        const iterations = baseIterations * (1 + (i % m.options.sweepWidth));
+        const duration = measure(m, iterations);
+        samples.push({ duration, iterations });
+    }
+    const { beta, c95, outliers } = regress(m, samples);
+    const ops = 1 / beta;
+    const lowOps = 1 / (beta + c95);
+    const highOps = 1 / (beta - c95);
+    const maxError = Math.max(highOps - ops, ops - lowOps);
+    const usedSamples = samples.length - outliers;
+    console.log(`${m.name}: ${ops.toFixed(1)} ops/s ` +
+        `(±${maxError.toFixed(1)}, p=0.05, n=${usedSamples})`);
+}
+function warmUp(m) {
+    // Initial warm-up
+    for (let i = 0; i < m.options.warmUpIterations; i++) {
+        m.default();
+    }
+    // Compute max duration per base sample
+    let sampleMultiplier = 0;
+    for (let i = 0; i < m.options.samples; i++) {
+        sampleMultiplier += 1 + (i % m.options.sweepWidth);
+    }
+    const maxSampleDuration = m.options.duration / sampleMultiplier;
+    // Compute iteration count per base sample
+    let iterations = 1;
+    let duration = 0;
+    do {
+        iterations *= 1.25;
+        duration = measure(m, Math.round(iterations));
+    } while (duration < maxSampleDuration);
+    iterations = Math.round(iterations / 1.25);
+    return iterations;
+}
+function measure(m, iterations) {
+    let sum = 0;
+    const start = process.hrtime.bigint();
+    for (let i = 0; i < iterations; i++) {
+        sum += m.default();
+    }
+    const duration = Number(process.hrtime.bigint() - start) / 1e9;
+    if (isNaN(sum)) {
+        throw new Error(`${m.name}: runner function did not return number`);
+    }
+    return duration;
+}
+function regress(m, samples) {
+    // Bin the data by iteration count
+    const bins = new Map();
+    for (const { iterations, duration } of samples) {
+        let bin = bins.get(iterations);
+        if (bin === undefined) {
+            bin = [];
+            bins.set(iterations, bin);
+        }
+        bin.push(duration);
+    }
+    // Within each iteration bin get rid of the outliers.
+    const withoutOutliers = new Array();
+    for (const [iterations, durations] of bins) {
+        durations.sort();
+        const p25 = durations[Math.floor(durations.length * 0.25)] ?? -Infinity;
+        const p75 = durations[Math.ceil(durations.length * 0.75)] ?? +Infinity;
+        const iqr = p75 - p25;
+        const outlierLow = p25 - iqr * 1.5;
+        const outlierHigh = p75 + iqr * 1.5;
+        // Tukey's method
+        const filtered = durations.filter((d) => d >= outlierLow && d <= outlierHigh);
+        for (const duration of filtered) {
+            withoutOutliers.push({ iterations, duration });
+        }
+    }
+    if (withoutOutliers.length < 2) {
+        throw new Error(`${m.name}: low sample count`);
+    }
+    let meanDuration = 0;
+    let meanIterations = 0;
+    for (const { duration, iterations } of withoutOutliers) {
+        meanDuration += duration;
+        meanIterations += iterations;
+    }
+    meanDuration /= withoutOutliers.length;
+    meanIterations /= withoutOutliers.length;
+    let betaNum = 0;
+    let betaDenom = 0;
+    for (const { duration, iterations } of withoutOutliers) {
+        betaNum += (duration - meanDuration) * (iterations - meanIterations);
+        betaDenom += (iterations - meanIterations) ** 2;
+    }
+    // Slope
+    const beta = betaNum / betaDenom;
+    // Intercept
+    const alpha = meanDuration - beta * meanIterations;
+    let stdError = 0;
+    for (const { duration, iterations } of withoutOutliers) {
+        stdError += (duration - alpha - beta * iterations) ** 2;
+    }
+    stdError /= withoutOutliers.length - 2;
+    stdError /= betaDenom;
+    stdError = Math.sqrt(stdError);
+    // t-distribution value for large sample count and p=0.05
+    const T_VALUE = 1.9719;
+    return {
+        alpha,
+        beta,
+        c95: T_VALUE * stdError,
+        outliers: samples.length - withoutOutliers.length,
+    };
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@indutny/bencher",
+  "version": "1.0.0-rc.1",
+  "description": "Simple benchmarking tool",
+  "bin": {
+    "bencher": "./dist/bin/bencher.js"
+  },
+  "files": [
+    "dist/bin/*.js",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint --cache .",
+    "format": "prettier --cache --write .",
+    "check:format": "prettier --cache --check .",
+    "prepublish": "rm -rf dist && npm run build"
+  },
+  "keywords": [
+    "benchmark",
+    "sweep",
+    "regression"
+  ],
+  "author": "Fedor Indutny <238531+indutny@users.noreply.github.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/indutny/bencher.git"
+  },
+  "bugs": {
+    "url": "https://github.com/indutny/bencher/issues"
+  },
+  "homepage": "https://github.com/indutny/bencher#readme",
+  "devDependencies": {
+    "@typescript-eslint/eslint-plugin": "^5.47.0",
+    "@typescript-eslint/parser": "^5.47.0",
+    "eslint": "^8.30.0",
+    "prettier": "^2.8.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.4"
+  }
+}