flexi-bench 0.0.0-alpha.3 → 0.1.0

package/dist/benchmark-runner.js

@@ -0,0 +1,172 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.xvariation = exports.xit = exports.xtest = exports.xbenchmark = exports.xdescribe = exports.xsuite = exports.it = exports.test = exports.describe = exports.afterEach = exports.afterAll = exports.beforeEach = exports.beforeAll = void 0;
+exports.suite = suite;
+exports.benchmark = benchmark;
+exports.variation = variation;
+exports.setup = setup;
+exports.setupEach = setupEach;
+exports.teardown = teardown;
+exports.teardownEach = teardownEach;
+const benchmark_1 = require("./benchmark");
+const suite_1 = require("./suite");
+const variation_1 = require("./variation");
+let activeSuite = null;
+/**
+ * Registers a new suite to run.
+ * @param name The name of the suite.
+ * @param fn Callback to register benchmarks and update the suite.
+ * @returns The results of the suite. `Record<string, Result[]>`
+ */
+function suite(name, fn) {
+    const suite = new suite_1.Suite(name);
+    activeSuite = suite;
+    const transformed = fn(suite);
+    activeSuite = null;
+    return (transformed ?? suite).run();
+}
+let activeBenchmark = null;
+/**
+ * Registers a new benchmark to run. If inside a {@link suite} callback, it will be added to the suite. Otherwise, it will run immediately.
+ * @param name The name of the benchmark.
+ * @param fn Callback to register variations and update the benchmark.
+ * @returns If not inside a suite, the results of the benchmark. `Result[]`. Else, `void`.
+ */
+function benchmark(name, fn) {
+    const benchmark = new benchmark_1.Benchmark(name);
+    activeBenchmark = benchmark;
+    const transformed = fn(benchmark);
+    activeBenchmark = null;
+    if (activeSuite) {
+        activeSuite.addBenchmark(transformed ?? benchmark);
+    }
+    else {
+        return (transformed ?? benchmark).run();
+    }
+}
+let activeVariation = null;
+/**
+ * Registers a new variation to run. Must be inside a {@link benchmark} or {@link suite} callback.
+ * @param name The name of the variation.
+ * @param fn A callback to update the variation.
+ * @returns `void`
+ */
+function variation(name, fn) {
+    const variation = new variation_1.Variation(name);
+    activeVariation = variation;
+    const transformed = fn(variation);
+    activeVariation = null;
+    if (activeBenchmark) {
+        activeBenchmark.withVariation(transformed ?? variation);
+    }
+    else if (activeSuite) {
+        activeSuite.withVariation(transformed ?? variation);
+    }
+    else {
+        throw new Error('`variation` must be called within a benchmark or suite');
+    }
+}
+/**
+ * Registers a setup method to run before the benchmark or variation. Ran once per benchmark or variation.
+ * @param fn The setup method.
+ */
+function setup(fn) {
+    if (activeVariation) {
+        activeVariation.withSetup(fn);
+    }
+    else if (activeBenchmark) {
+        activeBenchmark.withSetup(fn);
+    }
+    else {
+        throw new Error('`beforeAll` must be called within a variation or benchmark');
+    }
+}
+/**
+ * Registers a setup method to run before each iteration of the benchmark or variation.
+ * @param fn The setup method.
+ */
+function setupEach(fn) {
+    if (activeVariation) {
+        activeVariation.withSetupEach(fn);
+    }
+    else if (activeBenchmark) {
+        activeBenchmark.withSetupEach(fn);
+    }
+    else {
+        throw new Error('`beforeEach` must be called within a variation or benchmark');
+    }
+}
+/**
+ * Registers a teardown method to run after the benchmark or variation. Ran once per benchmark or variation.
+ * @param fn The teardown method.
+ */
+function teardown(fn) {
+    if (activeVariation) {
+        activeVariation.withTeardown(fn);
+    }
+    else if (activeBenchmark) {
+        activeBenchmark.withTeardown(fn);
+    }
+    else {
+        throw new Error('`afterAll` must be called within a variation or benchmark');
+    }
+}
+/**
+ * Registers a teardown method to run after each iteration of the benchmark or variation.
+ * @param fn The teardown method.
+ */
+function teardownEach(fn) {
+    if (activeVariation) {
+        activeVariation.withTeardownEach(fn);
+    }
+    else if (activeBenchmark) {
+        activeBenchmark.withTeardownEach(fn);
+    }
+    else {
+        throw new Error('`afterEach` must be called within a variation or benchmark');
+    }
+}
+// ALIASES
+// We provide some aliases for comfort when users are familiar with other testing tools.
+/**
+ * Alias for `setup`.
+ */
+exports.beforeAll = setup;
+/**
+ * Alias for `setupEach`.
+ */
+exports.beforeEach = setupEach;
+/**
+ * Alias for `teardown`.
+ */
+exports.afterAll = teardown;
+/**
+ * Alias for `teardownEach`.
+ */
+exports.afterEach = teardownEach;
+/**
+ * Alias for `suite`.
+ */
+exports.describe = suite;
+/**
+ * Alias for `benchmark`.
+ */
+exports.test = benchmark;
+/**
+ * Alias for `benchmark`.
+ */
+exports.it = benchmark;
+// DISABLED FUNCTIONS
+const xsuite = () => {
+    return Promise.resolve({});
+};
+exports.xsuite = xsuite;
+exports.xdescribe = exports.xsuite;
+const xbenchmark = () => {
+    return Promise.resolve([]);
+};
+exports.xbenchmark = xbenchmark;
+exports.xtest = exports.xbenchmark;
+exports.xit = exports.xbenchmark;
+const xvariation = () => { };
+exports.xvariation = xvariation;

package/dist/benchmark.d.ts

@@ -1,33 +1,34 @@
 import { Variation } from './variation';
-import { BenchmarkReporter, Result } from './api-types';
-import { ActionMethod } from './api-types';
-import { TeardownMethod } from './api-types';
-import { SetupMethod } from './api-types';
-export declare class Benchmark {
+import { TeardownMethod, SetupMethod, Action, BenchmarkReporter, ErrorStrategy } from './api-types';
+import { Result } from './results';
+import { PerformanceObserverOptions } from './performance-observer';
+import { BenchmarkBase } from './shared-api';
+export declare class Benchmark extends BenchmarkBase {
     name: string;
-    private setupMethods;
-    private teardownMethods;
-    private action?;
     variations: Variation[];
     private iterations?;
     private timeout?;
     private reporter;
+    private watcher?;
+    errorStrategy: ErrorStrategy;
     constructor(name: string, options?: {
         setup?: SetupMethod;
         teardown?: TeardownMethod;
-        action?: ActionMethod;
+        action?: Action;
         iterations?: number;
         timeout?: number;
         reporter?: BenchmarkReporter;
     });
     withVariation(name: string, builder: (variation: Variation) => Variation): this;
+    withVariation(variation: Variation): this;
     withVariations(variations: Variation[]): this;
-    withSetup(setup: () => void): this;
-    withTeardown(teardown: () => void): this;
-    withAction(action: () => void): this;
+    withSetup(setup: SetupMethod): this;
+    withTeardown(teardown: TeardownMethod): this;
+    withAction(action: Action): this;
     withIterations(iterations: number): this;
     withTimeout(timeout: number): this;
     withReporter(reporter: BenchmarkReporter): this;
+    withPerformanceObserver(options?: PerformanceObserverOptions): this;
     run(): Promise<Result[]>;
     private validate;
 }

package/dist/benchmark.js

@@ -1,42 +1,52 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Benchmark = void 0;
-const benchmark_console_reporter_1 = require("./benchmark-console-reporter");
+const benchmark_console_reporter_1 = require("./reporters/benchmark-console-reporter");
 const variation_1 = require("./variation");
-class Benchmark {
+const api_types_1 = require("./api-types");
+const child_process_1 = require("child_process");
+const results_1 = require("./results");
+const performance_observer_1 = require("./performance-observer");
+const shared_api_1 = require("./shared-api");
+class Benchmark extends shared_api_1.BenchmarkBase {
+    name;
+    variations = [];
+    iterations;
+    timeout;
+    reporter;
+    watcher;
+    errorStrategy = api_types_1.ErrorStrategy.Continue;
     constructor(name, options) {
+        super();
         this.name = name;
-        this.setupMethods = [];
-        this.teardownMethods = [];
-        this.variations = [];
-        if (options === null || options === void 0 ? void 0 : options.action) {
+        if (options?.action) {
             this.action = options.action;
         }
-        if (options === null || options === void 0 ? void 0 : options.setup) {
+        if (options?.setup) {
             this.setupMethods.push(options.setup);
         }
-        if (options === null || options === void 0 ? void 0 : options.teardown) {
+        if (options?.teardown) {
             this.teardownMethods.push(options.teardown);
         }
-        if (options === null || options === void 0 ? void 0 : options.iterations) {
+        if (options?.iterations) {
             this.iterations = options.iterations;
         }
-        if (options === null || options === void 0 ? void 0 : options.timeout) {
+        if (options?.timeout) {
             this.timeout = options.timeout;
         }
-        this.reporter = (options === null || options === void 0 ? void 0 : options.reporter) || new benchmark_console_reporter_1.BenchmarkConsoleReporter();
+        this.reporter = options?.reporter || new benchmark_console_reporter_1.BenchmarkConsoleReporter();
     }
-    withVariation(name, builder) {
-        this.variations.push(builder(new variation_1.Variation(name)));
+    withVariation(nameOrVariation, builder) {
+        if (nameOrVariation instanceof variation_1.Variation) {
+            this.variations.push(nameOrVariation);
+        }
+        else if (builder) {
+            this.variations.push(builder(new variation_1.Variation(nameOrVariation)));
+            return this;
+        }
+        else {
+            throw new Error('`withVariation` must be called with either a Variation or a name and a builder function.');
+        }
         return this;
     }
     withVariations(variations) {
@@ -67,113 +77,139 @@ class Benchmark {
         this.reporter = reporter;
         return this;
     }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.validate();
-            let results = [];
-            if (this.variations.length === 0) {
-                this.variations.push(new variation_1.Variation('default'));
+    withPerformanceObserver(options) {
+        this.watcher = new performance_observer_1.PerformanceWatcher(options);
+        return this;
+    }
+    async run() {
+        this.validate();
+        let results = [];
+        if (this.variations.length === 0) {
+            this.variations.push(new variation_1.Variation('default'));
+        }
+        const totalIterations = this.iterations
+            ? this.variations.length * this.iterations
+            : undefined;
+        const startTime = performance.now();
+        let totalCompletedIterations = 0;
+        for (let variationIndex = 0; variationIndex < this.variations.length; variationIndex++) {
+            const variation = this.variations[variationIndex];
+            const iterationResults = [];
+            // SETUP
+            const oldEnv = { ...process.env };
+            process.env = {
+                ...process.env,
+                ...variation.environment,
+            };
+            for (const setup of this.setupMethods) {
+                await setup(variation);
             }
-            const totalIterations = this.iterations
-                ? this.variations.length * this.iterations
-                : undefined;
-            const startTime = performance.now();
-            let totalCompletedIterations = 0;
-            for (let variationIndex = 0; variationIndex < this.variations.length; variationIndex++) {
-                const variation = this.variations[variationIndex];
-                const timings = [];
-                // SETUP
-                const oldEnv = Object.assign({}, process.env);
-                process.env = Object.assign(Object.assign({}, process.env), variation.environment);
-                for (const setup of this.setupMethods) {
-                    yield setup(variation);
-                }
-                for (const setup of variation.setupMethods) {
-                    yield setup(variation);
-                }
-                // ACT
-                const benchmarkThis = this;
-                yield new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () {
-                    let completedIterations = 0;
-                    let timeout = benchmarkThis.timeout
-                        ? setTimeout(() => {
-                            running = false;
-                            if ((benchmarkThis === null || benchmarkThis === void 0 ? void 0 : benchmarkThis.iterations) &&
-                                completedIterations < benchmarkThis.iterations) {
-                                reject('Timeout');
-                            }
-                            resolve();
-                        }, benchmarkThis.timeout)
-                        : null;
-                    let running = true;
-                    while (running) {
-                        const a = performance.now();
-                        if (variation.action) {
-                            yield variation.action(variation);
-                        }
-                        else if (this.action) {
-                            yield this.action(variation);
-                        }
-                        const b = performance.now();
-                        const duration = b - a;
-                        completedIterations++;
-                        totalCompletedIterations++;
-                        timings.push(duration);
-                        if (this.reporter.progress &&
-                            totalIterations &&
-                            benchmarkThis.iterations) {
-                            this.reporter.progress(variation.name, totalCompletedIterations / totalIterations, {
-                                timeElapsed: performance.now() - startTime,
-                                totalIterations,
-                                completedIterations: totalCompletedIterations,
-                                timeout: benchmarkThis.timeout,
-                            });
+            for (const setup of variation.setupMethods) {
+                await setup(variation);
+            }
+            // ACT
+            const benchmarkThis = this;
+            await new Promise(async (resolve, reject) => {
+                let completedIterations = 0;
+                let timeout = benchmarkThis.timeout
+                    ? setTimeout(() => {
+                        running = false;
+                        if (benchmarkThis?.iterations &&
+                            completedIterations < benchmarkThis.iterations) {
+                            reject('Timeout');
                         }
-                        if ((benchmarkThis === null || benchmarkThis === void 0 ? void 0 : benchmarkThis.iterations) &&
-                            completedIterations >= benchmarkThis.iterations) {
-                            running = false;
-                            if (timeout) {
-                                clearTimeout(timeout);
-                            }
-                            resolve();
+                        resolve();
+                    }, benchmarkThis.timeout)
+                    : null;
+                let running = true;
+                while (running) {
+                    for (const setup of this.setupEachMethods.concat(variation.setupEachMethods)) {
+                        await setup(variation);
+                    }
+                    const result = await runAndMeasureAction(benchmarkThis, variation);
+                    const errorStrategy = variation.errorStrategy ?? benchmarkThis.errorStrategy;
+                    if (errorStrategy === api_types_1.ErrorStrategy.Abort &&
+                        result instanceof Error) {
+                        reject(result);
+                        return;
+                    }
+                    for (const teardown of this.teardownEachMethods.concat(variation.teardownEachMethods)) {
+                        await teardown(variation);
+                    }
+                    completedIterations++;
+                    totalCompletedIterations++;
+                    iterationResults.push(result);
+                    if (this.reporter.progress &&
+                        totalIterations &&
+                        benchmarkThis.iterations) {
+                        this.reporter.progress(variation.name, totalCompletedIterations / totalIterations, {
+                            timeElapsed: performance.now() - startTime,
+                            totalIterations,
+                            completedIterations: totalCompletedIterations,
+                            timeout: benchmarkThis.timeout,
+                        });
+                    }
+                    if (benchmarkThis?.iterations &&
+                        completedIterations >= benchmarkThis.iterations) {
+                        running = false;
+                        if (timeout) {
+                            clearTimeout(timeout);
                         }
+                        resolve();
                     }
-                }));
-                if (this.reporter.progress && !totalIterations) {
-                    this.reporter.progress(variation.name, variationIndex / this.variations.length, {
-                        timeElapsed: performance.now() - startTime,
-                        timeout: this.timeout,
-                        completedIterations: totalCompletedIterations,
-                    });
                 }
-                // TEARDOWN
-                for (const teardown of variation.teardownMethods) {
-                    yield teardown(variation);
-                }
-                for (const teardown of this.teardownMethods) {
-                    yield teardown(variation);
-                }
-                process.env = oldEnv;
-                // REPORT
-                const min = Math.min(...timings);
-                const max = Math.max(...timings);
-                const sum = timings.reduce((a, b) => a + b, 0);
-                const average = sum / timings.length;
-                const sortedTimings = [...timings].sort((a, b) => a - b);
-                const p95 = sortedTimings[Math.floor(sortedTimings.length * 0.95)];
-                results.push({
-                    label: variation.name,
-                    min,
-                    max,
-                    average,
-                    p95,
+            });
+            if (this.reporter.progress && !totalIterations) {
+                this.reporter.progress(variation.name, variationIndex / this.variations.length, {
+                    timeElapsed: performance.now() - startTime,
+                    timeout: this.timeout,
+                    completedIterations: totalCompletedIterations,
                 });
             }
-            this.reporter.report(this, results);
-            return results;
-        });
+            // TEARDOWN
+            for (const teardown of variation.teardownMethods) {
+                await teardown(variation);
+            }
+            for (const teardown of this.teardownMethods) {
+                await teardown(variation);
+            }
+            process.env = oldEnv;
+            // REPORT
+            const result = (0, results_1.calculateResultsFromDurations)(variation.name, iterationResults);
+            // PerformanceObserver needs a chance to flush
+            if (this.watcher) {
+                const measures = await this.watcher.getMeasures();
+                for (const key in measures) {
+                    result.subresults ??= [];
+                    result.subresults.push((0, results_1.calculateResultsFromDurations)(key, measures[key]));
+                }
+                this.watcher.clearMeasures();
+            }
+            results.push(result);
+        }
+        this.watcher?.disconnect();
+        this.reporter.report(this, results);
+        if (this.errorStrategy === api_types_1.ErrorStrategy.DelayedThrow &&
+            results.some((r) => r.failed)) {
+            throw new api_types_1.AggregateBenchmarkError(results);
+        }
+        return results;
     }
     validate() {
+        if (!this.timeout && !this.iterations) {
+            this.iterations = 5;
+        }
+        const missingActions = [];
+        for (const variation of this.variations) {
+            let action = variation.action || this.action;
+            if (!action) {
+                missingActions.push(variation.name);
+                continue;
+            }
+            if (typeof action !== 'string' && variation.cliArgs.length > 0) {
+                throw new Error(`Cannot use CLI args with a non-command action for ${this.name}:${variation.name}`);
+            }
+        }
         if (!this.action) {
             const missingActions = this.variations.filter((v) => !v.action);
             if (missingActions.length > 0) {
@@ -188,3 +224,35 @@ class Benchmark {
     }
 }
 exports.Benchmark = Benchmark;
+async function runAndMeasureAction(benchmark, variation) {
+    try {
+        const a = performance.now();
+        await runAction((variation.action ?? benchmark.action), variation);
+        const b = performance.now();
+        return b - a;
+    }
+    catch (e) {
+        return e instanceof Error ? e : new Error('Unknown error during action.');
+    }
+}
+async function runAction(action, variation) {
+    if (typeof action === 'string') {
+        return new Promise((resolve, reject) => {
+            const child = (0, child_process_1.spawn)(action, variation.cliArgs, {
+                shell: true,
+                windowsHide: true,
+            });
+            child.on('exit', (code) => {
+                if (code === 0) {
+                    resolve();
+                }
+                else {
+                    reject(`Action failed with code ${code}`);
+                }
+            });
+        });
+    }
+    else {
+        return action(variation);
+    }
+}

package/dist/index.d.ts

@@ -1,5 +1,10 @@
+export * from './reporters/benchmark-console-reporter';
+export * from './reporters/markdown-benchmark-reporter';
+export * from './reporters/suite-console-reporter';
+export * from './reporters/noop-reporter';
 export * from './api-types';
 export * from './benchmark';
 export * from './variation';
-export * from './benchmark-console-reporter';
 export * from './suite';
+export * from './performance-observer';
+export * from './benchmark-runner';

package/dist/index.js

@@ -14,8 +14,13 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./reporters/benchmark-console-reporter"), exports);
+__exportStar(require("./reporters/markdown-benchmark-reporter"), exports);
+__exportStar(require("./reporters/suite-console-reporter"), exports);
+__exportStar(require("./reporters/noop-reporter"), exports);
 __exportStar(require("./api-types"), exports);
 __exportStar(require("./benchmark"), exports);
 __exportStar(require("./variation"), exports);
-__exportStar(require("./benchmark-console-reporter"), exports);
 __exportStar(require("./suite"), exports);
+__exportStar(require("./performance-observer"), exports);
+__exportStar(require("./benchmark-runner"), exports);

package/dist/performance-observer.d.ts

@@ -0,0 +1,12 @@
+export declare class PerformanceWatcher {
+    private observer?;
+    private measures;
+    constructor(opts?: PerformanceObserverOptions);
+    getMeasures(): Promise<Record<string, number[]>>;
+    clearMeasures(): void;
+    disconnect(): void;
+}
+export type PerformanceObserverOptions = {
+    measureFilter?: (entry: PerformanceEntry) => boolean;
+    label?: Record<string, string> | ((label: string) => string);
+};

package/dist/performance-observer.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PerformanceWatcher = void 0;
+const node_perf_hooks_1 = require("node:perf_hooks");
+class PerformanceWatcher {
+    observer;
+    measures = {};
+    constructor(opts = {}) {
+        this.observer = new node_perf_hooks_1.PerformanceObserver((list) => {
+            for (const entry of list.getEntries()) {
+                if (entry.entryType === 'measure' &&
+                    (!opts.measureFilter || opts.measureFilter(entry))) {
+                    const label = normalizeLabel(entry.name, opts?.label);
+                    this.measures[label] ??= [];
+                    this.measures[label].push(entry.duration);
+                }
+            }
+        });
+        this.observer.observe({ entryTypes: ['measure'] });
+    }
+    // This method is async to give the observer time to collect the measures...
+    // If the action being ran by the benchmark is synchronous, the observer will
+    // not fire until the next tick so we need that to happen before the measures are
+    // retrieved. Since this method is async, it must be awaited, which actually gives
+    // the observer time to collect the measures.
+    async getMeasures() {
+        return this.measures;
+    }
+    clearMeasures() {
+        for (const key in this.measures) {
+            delete this.measures[key];
+        }
+    }
+    disconnect() {
+        this.observer?.disconnect();
+        delete this.observer;
+    }
+}
+exports.PerformanceWatcher = PerformanceWatcher;
+function normalizeLabel(label, transform) {
+    if (transform) {
+        return typeof transform === 'function'
+            ? transform(label)
+            : transform[label];
+    }
+    return label;
+}

package/dist/{benchmark-console-reporter.d.ts → reporters/benchmark-console-reporter.d.ts}

@@ -1,7 +1,9 @@
-import { Result, BenchmarkReporter, ProgressContext } from './api-types';
-import { Benchmark } from './benchmark';
+import { BenchmarkReporter, ProgressContext } from '../api-types';
+import { Benchmark } from '../benchmark';
+import { SingleBar } from 'cli-progress';
+import { Result } from '../results';
 export declare class BenchmarkConsoleReporter implements BenchmarkReporter {
-    private bar;
+    bar: SingleBar;
     constructor();
     progress(name: string, percent: number, context: ProgressContext): void;
     report(benchmark: Benchmark, results: Result[]): void;