flexi-bench 0.0.0-alpha.3 → 0.0.0-alpha.4

package/dist/benchmark.d.ts

@@ -1,33 +1,33 @@
 import { Variation } from './variation';
-import { BenchmarkReporter, Result } from './api-types';
-import { ActionMethod } from './api-types';
-import { TeardownMethod } from './api-types';
-import { SetupMethod } from './api-types';
-export declare class Benchmark {
+import { TeardownMethod, SetupMethod, Action, BenchmarkReporter } from './api-types';
+import { Result } from './results';
+import { PerformanceObserverOptions } from './performance-observer';
+import { BenchmarkBase } from './shared-api';
+export declare class Benchmark extends BenchmarkBase {
     name: string;
-    private setupMethods;
-    private teardownMethods;
-    private action?;
     variations: Variation[];
     private iterations?;
     private timeout?;
     private reporter;
+    private watcher?;
     constructor(name: string, options?: {
         setup?: SetupMethod;
         teardown?: TeardownMethod;
-        action?: ActionMethod;
+        action?: Action;
         iterations?: number;
         timeout?: number;
         reporter?: BenchmarkReporter;
     });
     withVariation(name: string, builder: (variation: Variation) => Variation): this;
+    withVariation(variation: Variation): this;
     withVariations(variations: Variation[]): this;
-    withSetup(setup: () => void): this;
-    withTeardown(teardown: () => void): this;
-    withAction(action: () => void): this;
+    withSetup(setup: SetupMethod): this;
+    withTeardown(teardown: TeardownMethod): this;
+    withAction(action: Action): this;
     withIterations(iterations: number): this;
     withTimeout(timeout: number): this;
     withReporter(reporter: BenchmarkReporter): this;
+    withPerformanceObserver(options?: PerformanceObserverOptions): this;
     run(): Promise<Result[]>;
     private validate;
 }

package/dist/benchmark.js

@@ -10,13 +10,16 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Benchmark = void 0;
-const benchmark_console_reporter_1 = require("./benchmark-console-reporter");
+const benchmark_console_reporter_1 = require("./reporters/benchmark-console-reporter");
 const variation_1 = require("./variation");
-class Benchmark {
+const child_process_1 = require("child_process");
+const results_1 = require("./results");
+const performance_observer_1 = require("./performance-observer");
+const shared_api_1 = require("./shared-api");
+class Benchmark extends shared_api_1.BenchmarkBase {
     constructor(name, options) {
+        super();
         this.name = name;
-        this.setupMethods = [];
-        this.teardownMethods = [];
         this.variations = [];
         if (options === null || options === void 0 ? void 0 : options.action) {
             this.action = options.action;
@@ -35,8 +38,17 @@ class Benchmark {
         }
         this.reporter = (options === null || options === void 0 ? void 0 : options.reporter) || new benchmark_console_reporter_1.BenchmarkConsoleReporter();
     }
-    withVariation(name, builder) {
-        this.variations.push(builder(new variation_1.Variation(name)));
+    withVariation(nameOrVariation, builder) {
+        if (nameOrVariation instanceof variation_1.Variation) {
+            this.variations.push(nameOrVariation);
+        }
+        else if (builder) {
+            this.variations.push(builder(new variation_1.Variation(nameOrVariation)));
+            return this;
+        }
+        else {
+            throw new Error('`withVariation` must be called with either a Variation or a name and a builder function.');
+        }
         return this;
     }
     withVariations(variations) {
@@ -67,8 +79,13 @@ class Benchmark {
         this.reporter = reporter;
         return this;
     }
+    withPerformanceObserver(options) {
+        this.watcher = new performance_observer_1.PerformanceWatcher(options);
+        return this;
+    }
     run() {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
             this.validate();
             let results = [];
             if (this.variations.length === 0) {
@@ -107,14 +124,20 @@ class Benchmark {
                         : null;
                     let running = true;
                     while (running) {
+                        for (const setup of this.setupEachMethods.concat(variation.setupEachMethods)) {
+                            yield setup(variation);
+                        }
                         const a = performance.now();
                         if (variation.action) {
-                            yield variation.action(variation);
+                            yield runAction(variation.action, variation);
                         }
                         else if (this.action) {
-                            yield this.action(variation);
+                            yield runAction(this.action, variation);
                         }
                         const b = performance.now();
+                        for (const teardown of this.teardownEachMethods.concat(variation.teardownEachMethods)) {
+                            yield teardown(variation);
+                        }
                         const duration = b - a;
                         completedIterations++;
                         totalCompletedIterations++;
@@ -155,25 +178,38 @@ class Benchmark {
                 }
                 process.env = oldEnv;
                 // REPORT
-                const min = Math.min(...timings);
-                const max = Math.max(...timings);
-                const sum = timings.reduce((a, b) => a + b, 0);
-                const average = sum / timings.length;
-                const sortedTimings = [...timings].sort((a, b) => a - b);
-                const p95 = sortedTimings[Math.floor(sortedTimings.length * 0.95)];
-                results.push({
-                    label: variation.name,
-                    min,
-                    max,
-                    average,
-                    p95,
-                });
+                const result = (0, results_1.calculateResultsFromDurations)(variation.name, timings);
+                // PerformanceObserver needs a chance to flush
+                if (this.watcher) {
+                    const measures = yield this.watcher.getMeasures();
+                    for (const key in measures) {
+                        (_a = result.subresults) !== null && _a !== void 0 ? _a : (result.subresults = []);
+                        result.subresults.push((0, results_1.calculateResultsFromDurations)(key, measures[key]));
+                    }
+                    this.watcher.clearMeasures();
+                }
+                results.push(result);
             }
+            (_b = this.watcher) === null || _b === void 0 ? void 0 : _b.disconnect();
             this.reporter.report(this, results);
             return results;
         });
     }
     validate() {
+        if (!this.timeout && !this.iterations) {
+            this.iterations = 5;
+        }
+        const missingActions = [];
+        for (const variation of this.variations) {
+            let action = variation.action || this.action;
+            if (!action) {
+                missingActions.push(variation.name);
+                continue;
+            }
+            if (typeof action !== 'string' && variation.cliArgs.length > 0) {
+                throw new Error(`Cannot use CLI args with a non-command action for ${this.name}:${variation.name}`);
+            }
+        }
         if (!this.action) {
             const missingActions = this.variations.filter((v) => !v.action);
             if (missingActions.length > 0) {
@@ -188,3 +224,26 @@ class Benchmark {
     }
 }
 exports.Benchmark = Benchmark;
+function runAction(action, variation) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (typeof action === 'string') {
+            return new Promise((resolve, reject) => {
+                const child = (0, child_process_1.spawn)(action, variation.cliArgs, {
+                    shell: true,
+                    windowsHide: true,
+                });
+                child.on('exit', (code) => {
+                    if (code === 0) {
+                        resolve();
+                    }
+                    else {
+                        reject(`Action failed with code ${code}`);
+                    }
+                });
+            });
+        }
+        else {
+            return action(variation);
+        }
+    });
+}

package/dist/index.d.ts

@@ -1,5 +1,9 @@
+export * from './reporters/benchmark-console-reporter';
+export * from './reporters/markdown-benchmark-reporter';
+export * from './reporters/suite-console-reporter';
 export * from './api-types';
 export * from './benchmark';
 export * from './variation';
-export * from './benchmark-console-reporter';
 export * from './suite';
+export * from './performance-observer';
+export * from './benchmark-runner';

package/dist/index.js

@@ -14,8 +14,12 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./reporters/benchmark-console-reporter"), exports);
+__exportStar(require("./reporters/markdown-benchmark-reporter"), exports);
+__exportStar(require("./reporters/suite-console-reporter"), exports);
 __exportStar(require("./api-types"), exports);
 __exportStar(require("./benchmark"), exports);
 __exportStar(require("./variation"), exports);
-__exportStar(require("./benchmark-console-reporter"), exports);
 __exportStar(require("./suite"), exports);
+__exportStar(require("./performance-observer"), exports);
+__exportStar(require("./benchmark-runner"), exports);

package/dist/performance-observer.d.ts

@@ -0,0 +1,12 @@
+export declare class PerformanceWatcher {
+    private observer?;
+    private measures;
+    constructor(opts?: PerformanceObserverOptions);
+    getMeasures(): Promise<Record<string, number[]>>;
+    clearMeasures(): void;
+    disconnect(): void;
+}
+export type PerformanceObserverOptions = {
+    measureFilter?: (entry: PerformanceEntry) => boolean;
+    label?: Record<string, string> | ((label: string) => string);
+};

package/dist/performance-observer.js

@@ -0,0 +1,60 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PerformanceWatcher = void 0;
+const node_perf_hooks_1 = require("node:perf_hooks");
+class PerformanceWatcher {
+    constructor(opts = {}) {
+        this.measures = {};
+        this.observer = new node_perf_hooks_1.PerformanceObserver((list) => {
+            var _a;
+            var _b;
+            for (const entry of list.getEntries()) {
+                if (entry.entryType === 'measure' &&
+                    (!opts.measureFilter || opts.measureFilter(entry))) {
+                    const label = normalizeLabel(entry.name, opts === null || opts === void 0 ? void 0 : opts.label);
+                    (_a = (_b = this.measures)[label]) !== null && _a !== void 0 ? _a : (_b[label] = []);
+                    this.measures[label].push(entry.duration);
+                }
+            }
+        });
+        this.observer.observe({ entryTypes: ['measure'] });
+    }
+    // This method is async to give the observer time to collect the measures...
+    // If the action being ran by the benchmark is synchronous, the observer will
+    // not fire until the next tick so we need that to happen before the measures are
+    // retrieved. Since this method is async, it must be awaited, which actually gives
+    // the observer time to collect the measures.
+    getMeasures() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.measures;
+        });
+    }
+    clearMeasures() {
+        for (const key in this.measures) {
+            delete this.measures[key];
+        }
+    }
+    disconnect() {
+        var _a;
+        (_a = this.observer) === null || _a === void 0 ? void 0 : _a.disconnect();
+        delete this.observer;
+    }
+}
+exports.PerformanceWatcher = PerformanceWatcher;
+function normalizeLabel(label, transform) {
+    if (transform) {
+        return typeof transform === 'function'
+            ? transform(label)
+            : transform[label];
+    }
+    return label;
+}

package/dist/{benchmark-console-reporter.d.ts → reporters/benchmark-console-reporter.d.ts}

@@ -1,7 +1,9 @@
-import { Result, BenchmarkReporter, ProgressContext } from './api-types';
-import { Benchmark } from './benchmark';
+import { BenchmarkReporter, ProgressContext } from '../api-types';
+import { Benchmark } from '../benchmark';
+import { SingleBar } from 'cli-progress';
+import { Result } from '../results';
 export declare class BenchmarkConsoleReporter implements BenchmarkReporter {
-    private bar;
+    bar: SingleBar;
     constructor();
     progress(name: string, percent: number, context: ProgressContext): void;
     report(benchmark: Benchmark, results: Result[]): void;

package/dist/{benchmark-console-reporter.js → reporters/benchmark-console-reporter.js}

@@ -5,10 +5,12 @@ const cli_progress_1 = require("cli-progress");
 class BenchmarkConsoleReporter {
     constructor() {
         this.bar = new cli_progress_1.SingleBar({
-            format: 'Running: {label}\n{bar} {percentage}% | {value}/{total} - ETA: {eta}s',
+            format: 'Running variation {label}: {bar} {percentage}% | {value}/{total} - ETA: {eta}s',
             barCompleteChar: '\u2588',
             barIncompleteChar: '\u2591',
             hideCursor: true,
+            stopOnComplete: true,
+            clearOnComplete: true,
         });
     }
     progress(name, percent, context) {
@@ -17,13 +19,8 @@ class BenchmarkConsoleReporter {
             this.bar.start((_a = context.totalIterations) !== null && _a !== void 0 ? _a : 100, 0);
         }
         this.bar.update(context.completedIterations, { label: name });
-        process.stdout.moveCursor(0, -1);
-        // console.log(`Progress: ${name} ${Math.round(percent * 10000) / 100}%`);
     }
     report(benchmark, results) {
-        this.bar.stop();
-        process.stdout.moveCursor(0, -1);
-        process.stdout.clearScreenDown();
         console.log(`Benchmark: ${benchmark.name}`);
         console.table(results);
     }

package/dist/reporters/markdown-benchmark-reporter.d.ts

@@ -0,0 +1,12 @@
+import { BenchmarkReporter } from '../api-types';
+import { Benchmark } from '../benchmark';
+import { Result } from '../results';
+export declare class MarkdownBenchmarkReporter implements BenchmarkReporter {
+    outputFile: string;
+    fields: Array<keyof Omit<Result, 'subresults' | 'label'>>;
+    constructor(opts: {
+        outputFile: string;
+        fields?: MarkdownBenchmarkReporter['fields'];
+    });
+    report: (benchmark: Benchmark, results: Result[]) => void;
+}

package/dist/reporters/markdown-benchmark-reporter.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarkdownBenchmarkReporter = void 0;
+const fs_1 = require("fs");
+const markdown_factory_1 = require("markdown-factory");
+class MarkdownBenchmarkReporter {
+    constructor(opts) {
+        var _a;
+        this.report = (benchmark, results) => {
+            (0, fs_1.writeFileSync)(this.outputFile, (0, markdown_factory_1.h1)(benchmark.name, results.some((r) => !!r.subresults)
+                ? (0, markdown_factory_1.lines)(results.map((r) => {
+                    var _a;
+                    const entries = [Object.assign(Object.assign({}, r), { label: 'total' })];
+                    delete entries[0].subresults;
+                    for (const subresult of (_a = r.subresults) !== null && _a !== void 0 ? _a : []) {
+                        entries.push(subresult);
+                    }
+                    return (0, markdown_factory_1.h2)(r.label, (0, markdown_factory_1.table)(entries, [
+                        { field: 'label', label: '' },
+                        ...this.fields,
+                    ]));
+                }))
+                : (0, markdown_factory_1.table)(results, [{ field: 'label', label: '' }, ...this.fields])));
+        };
+        this.outputFile = opts.outputFile;
+        this.fields = (_a = opts.fields) !== null && _a !== void 0 ? _a : ['min', 'average', 'p95', 'max'];
+    }
+}
+exports.MarkdownBenchmarkReporter = MarkdownBenchmarkReporter;

package/dist/reporters/noop-reporter.d.ts

@@ -0,0 +1,4 @@
+import { BenchmarkReporter, SuiteReporter } from '../api-types';
+export declare class NoopReporter implements BenchmarkReporter, SuiteReporter {
+    report: () => void;
+}

package/dist/reporters/noop-reporter.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NoopReporter = void 0;
+class NoopReporter {
+    constructor() {
+        this.report = () => { };
+    }
+}
+exports.NoopReporter = NoopReporter;

package/dist/{suite-console-reporter.d.ts → reporters/suite-console-reporter.d.ts}

@@ -1,4 +1,5 @@
-import { Result, SuiteReporter } from './api-types';
+import { SuiteReporter } from '../api-types';
+import { Result } from '../results';
 export declare class SuiteConsoleReporter implements SuiteReporter {
     report: (results: Record<string, Result[]>) => void;
 }

package/dist/results.d.ts

@@ -0,0 +1,34 @@
+/**
+ * A measurement result.
+ */
+export type Result = {
+    /**
+     * The label of the result. Typically the name of the benchmark or variation.
+     */
+    label: string;
+    /**
+     * The minimum duration.
+     */
+    min: number;
+    /**
+     * The maximum duration.
+     */
+    max: number;
+    /**
+     * The average duration.
+     */
+    average: number;
+    /**
+     * The 95th percentile duration.
+     */
+    p95: number;
+    /**
+     * The raw durations, in order. Used for custom reporters.
+     */
+    raw: number[];
+    /**
+     * Subresults, if any. Typically sourced from performance observer.
+     */
+    subresults?: Result[];
+};
+export declare function calculateResultsFromDurations(label: string, durations: number[]): Result;

package/dist/results.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.calculateResultsFromDurations = calculateResultsFromDurations;
+function calculateResultsFromDurations(label, durations) {
+    durations.sort((a, b) => a - b);
+    return {
+        label,
+        min: Math.min(...durations),
+        max: Math.max(...durations),
+        average: durations.reduce((acc, duration) => acc + duration, 0) / durations.length,
+        p95: durations[Math.floor(durations.length * 0.95)],
+        raw: durations,
+    };
+}

package/dist/shared-api.d.ts

@@ -0,0 +1,13 @@
+import { Action, SetupMethod, TeardownMethod } from './api-types';
+export declare class BenchmarkBase {
+    setupEachMethods: SetupMethod[];
+    setupMethods: SetupMethod[];
+    teardownMethods: TeardownMethod[];
+    teardownEachMethods: TeardownMethod[];
+    action?: Action;
+    withSetup(setup: SetupMethod): this;
+    withSetupEach(setup: SetupMethod): this;
+    withTeardown(teardown: TeardownMethod): this;
+    withTeardownEach(teardown: TeardownMethod): this;
+    withAction(action: Action): this;
+}

package/dist/shared-api.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BenchmarkBase = void 0;
+class BenchmarkBase {
+    constructor() {
+        this.setupEachMethods = [];
+        this.setupMethods = [];
+        this.teardownMethods = [];
+        this.teardownEachMethods = [];
+    }
+    withSetup(setup) {
+        this.setupMethods.push(setup);
+        return this;
+    }
+    withSetupEach(setup) {
+        this.setupEachMethods.push(setup);
+        return this;
+    }
+    withTeardown(teardown) {
+        this.teardownMethods.push(teardown);
+        return this;
+    }
+    withTeardownEach(teardown) {
+        this.teardownEachMethods.push(teardown);
+        return this;
+    }
+    withAction(action) {
+        this.action = action;
+        return this;
+    }
+}
+exports.BenchmarkBase = BenchmarkBase;

package/dist/suite.d.ts

@@ -1,5 +1,6 @@
-import { SuiteReporter } from './api-types';
+import { BenchmarkReporter, SuiteReporter } from './api-types';
 import { Benchmark } from './benchmark';
+import { Result } from './results';
 import { Variation } from './variation';
 export interface SuiteOptions {
     reporter?: SuiteReporter;
@@ -9,9 +10,12 @@ export declare class Suite {
     private benchmarks;
     private variations;
     private reporter;
+    private benchmarkReporter?;
     constructor(name: string, options?: SuiteOptions);
     addBenchmark(benchmark: Benchmark): this;
-    addVariation(variation: Variation): this;
-    addVariations(variations: Variation[]): this;
-    run(): Promise<void>;
+    withVariation(variation: Variation): this;
+    withVariations(variations: Variation[]): this;
+    withReporter(reporter: SuiteReporter): this;
+    withBenchmarkReporter(reporter: BenchmarkReporter): this;
+    run(): Promise<Record<string, Result[]>>;
 }

package/dist/suite.js

@@ -10,7 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Suite = void 0;
-const suite_console_reporter_1 = require("./suite-console-reporter");
+const suite_console_reporter_1 = require("./reporters/suite-console-reporter");
 class Suite {
     constructor(name, options) {
         this.name = name;
@@ -24,22 +24,34 @@ class Suite {
         this.benchmarks.push(benchmark);
         return this;
     }
-    addVariation(variation) {
+    withVariation(variation) {
         this.variations.push(variation);
         return this;
     }
-    addVariations(variations) {
+    withVariations(variations) {
         this.variations.push(...variations);
         return this;
     }
+    withReporter(reporter) {
+        this.reporter = reporter;
+        return this;
+    }
+    withBenchmarkReporter(reporter) {
+        this.benchmarkReporter = reporter;
+        return this;
+    }
     run() {
         return __awaiter(this, void 0, void 0, function* () {
             const results = {};
             for (const benchmark of this.benchmarks) {
                 benchmark.withVariations(this.variations);
+                if (this.benchmarkReporter) {
+                    benchmark.withReporter(this.benchmarkReporter);
+                }
                 results[benchmark.name] = yield benchmark.run();
             }
             this.reporter.report(results);
+            return results;
         });
     }
 }

package/dist/utils.d.ts

@@ -0,0 +1 @@
+export declare function findCombinations<T>(variables: T[][]): T[][];

package/dist/utils.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findCombinations = findCombinations;
+function findCombinations(variables) {
+    // Generates all possible combinations of the given variables.
+    // e.g. findCombinations([[1, 2], [3, 4]]) => [[1, 3], [1, 4], [2, 3], [2, 4]]
+    return variables.reduce((acc, values) => {
+        return acc.flatMap((accItem) => {
+            return values.map((value) => {
+                return [...accItem, value];
+            });
+        });
+    }, [[]]);
+}

package/dist/variation.d.ts

@@ -1,15 +1,29 @@
-import { SetupMethod, TeardownMethod, ActionMethod, EnvironmentVariableOptions } from './api-types';
-export declare class Variation {
+import { EnvironmentVariableOptions } from './api-types';
+import { BenchmarkBase } from './shared-api';
+export declare class Variation extends BenchmarkBase {
     name: string;
-    setupMethods: SetupMethod[];
-    teardownMethods: TeardownMethod[];
     environment: Partial<NodeJS.ProcessEnv>;
-    action?: ActionMethod;
+    cliArgs: string[];
     constructor(name: string);
     static FromEnvironmentVariables(variables: EnvironmentVariableOptions): Variation[];
-    withSetup(setup: SetupMethod): this;
-    withTeardown(teardown: TeardownMethod): this;
+    /**
+     *
+     * @param args An array of options to pass to the CLI. If an element is an array, they will be treated as alternatives.
+     * @returns An array of variations. Can be applied with `withVariations`.
+     * @example
+     * // Creates 2 variations, both with --flag, but one with --no-daemon and the other with --daemon.
+     * const variations = Variation.FromCliArgs([
+     *  '--flag',
+     * ['--no-daemon', '--daemon'],
+     * ]);
+     *
+     * // Creates 4 variations, with combinations of --a and --b.
+     * const variations = Variation.FromCliArgs([
+     * ['--a', '--b'],
+     * ]);
+     */
+    static FromCliArgs(args: Array<string | string[]>): Variation[];
     withEnvironmentVariables(env: Partial<NodeJS.ProcessEnv>): this;
     withEnvironmentVariable(name: string, value: string): this;
-    withAction(action: ActionMethod): this;
+    withCliArgs(...args: string[]): this;
 }