flexi-bench 0.0.0-alpha.3 → 0.1.0

package/dist/reporters/benchmark-console-reporter.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BenchmarkConsoleReporter = void 0;
+const cli_progress_1 = require("cli-progress");
+class BenchmarkConsoleReporter {
+    bar = new cli_progress_1.SingleBar({
+        format: 'Running variation {label}: {bar} {percentage}% | {value}/{total} - ETA: {eta}s',
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+        stopOnComplete: true,
+        clearOnComplete: true,
+    });
+    constructor() { }
+    progress(name, percent, context) {
+        if (!this.bar.isActive) {
+            this.bar.start(context.totalIterations ?? 100, 0);
+        }
+        this.bar.update(context.completedIterations, { label: name });
+    }
+    report(benchmark, results) {
+        const tableEntries = results.map(({ raw, ...rest }) => ({
+            ...rest,
+        }));
+        console.log(`Benchmark: ${benchmark.name}`);
+        console.table(tableEntries);
+    }
+}
+exports.BenchmarkConsoleReporter = BenchmarkConsoleReporter;

package/dist/reporters/markdown-benchmark-reporter.d.ts

@@ -0,0 +1,12 @@
+import { BenchmarkReporter } from '../api-types';
+import { Benchmark } from '../benchmark';
+import { Result } from '../results';
+export declare class MarkdownBenchmarkReporter implements BenchmarkReporter {
+    outputFile: string;
+    fields: Array<keyof Omit<Result, 'subresults' | 'label'>>;
+    constructor(opts: {
+        outputFile: string;
+        fields?: MarkdownBenchmarkReporter['fields'];
+    });
+    report: (benchmark: Benchmark, results: Result[]) => void;
+}

package/dist/reporters/markdown-benchmark-reporter.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarkdownBenchmarkReporter = void 0;
+const fs_1 = require("fs");
+const markdown_factory_1 = require("markdown-factory");
+class MarkdownBenchmarkReporter {
+    outputFile;
+    fields;
+    constructor(opts) {
+        this.outputFile = opts.outputFile;
+        this.fields = opts.fields ?? ['min', 'average', 'p95', 'max'];
+    }
+    report = (benchmark, results) => {
+        (0, fs_1.writeFileSync)(this.outputFile, (0, markdown_factory_1.h1)(benchmark.name, results.some((r) => !!r.subresults)
+            ? (0, markdown_factory_1.lines)(results.map((r) => {
+                const entries = [{ ...r, label: 'total' }];
+                delete entries[0].subresults;
+                for (const subresult of r.subresults ?? []) {
+                    entries.push(subresult);
+                }
+                return (0, markdown_factory_1.h2)(r.label, (0, markdown_factory_1.table)(entries, [
+                    { field: 'label', label: '' },
+                    ...this.fields,
+                ]));
+            }))
+            : (0, markdown_factory_1.table)(results, [{ field: 'label', label: '' }, ...this.fields])));
+    };
+}
+exports.MarkdownBenchmarkReporter = MarkdownBenchmarkReporter;

package/dist/reporters/noop-reporter.d.ts

@@ -0,0 +1,4 @@
+import { BenchmarkReporter, SuiteReporter } from '../api-types';
+export declare class NoopReporter implements BenchmarkReporter, SuiteReporter {
+    report: () => void;
+}

package/dist/reporters/noop-reporter.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NoopReporter = void 0;
+class NoopReporter {
+    report = () => { };
+}
+exports.NoopReporter = NoopReporter;

package/dist/{suite-console-reporter.d.ts → reporters/suite-console-reporter.d.ts}

@@ -1,4 +1,5 @@
-import { Result, SuiteReporter } from './api-types';
+import { SuiteReporter } from '../api-types';
+import { Result } from '../results';
 export declare class SuiteConsoleReporter implements SuiteReporter {
     report: (results: Record<string, Result[]>) => void;
 }

package/dist/reporters/suite-console-reporter.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SuiteConsoleReporter = void 0;
+class SuiteConsoleReporter {
+    report = (results) => {
+        console.log('Suite Results:');
+        for (const [name, result] of Object.entries(results)) {
+            const tableEntries = result.map(({ raw, ...rest }) => ({
+                ...rest,
+            }));
+            console.log(`Benchmark: ${name}`);
+            console.table(tableEntries);
+        }
+    };
+}
+exports.SuiteConsoleReporter = SuiteConsoleReporter;

package/dist/results.d.ts

@@ -0,0 +1,42 @@
+/**
+ * A measurement result.
+ */
+export type Result = {
+    /**
+     * The label of the result. Typically the name of the benchmark or variation.
+     */
+    label: string;
+    /**
+     * The minimum duration.
+     */
+    min: number;
+    /**
+     * The maximum duration.
+     */
+    max: number;
+    /**
+     * The average duration.
+     */
+    average: number;
+    /**
+     * The 95th percentile duration.
+     */
+    p95: number;
+    /**
+     * The raw durations, in order. Used for custom reporters.
+     */
+    raw: (number | Error)[];
+    /**
+     * Whether any run of the benchmark failed.
+     */
+    failed?: boolean;
+    /**
+     * The rate of failure, if any.
+     */
+    failureRate?: number;
+    /**
+     * Subresults, if any. Typically sourced from performance observer.
+     */
+    subresults?: Result[];
+};
+export declare function calculateResultsFromDurations(label: string, durations: (number | Error)[]): Result;

package/dist/results.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.calculateResultsFromDurations = calculateResultsFromDurations;
+function calculateResultsFromDurations(label, durations) {
+    const errors = [];
+    const results = [];
+    for (const duration of durations) {
+        if (duration instanceof Error) {
+            errors.push(duration);
+        }
+        else {
+            results.push(duration);
+        }
+    }
+    results.sort((a, b) => a - b);
+    return {
+        label,
+        min: Math.min(...results),
+        max: Math.max(...results),
+        average: results.reduce((acc, duration) => acc + duration, 0) / results.length,
+        p95: results[Math.floor(results.length * 0.95)],
+        raw: durations,
+        failed: errors.length > 0,
+        failureRate: errors.length / durations.length,
+    };
+}

package/dist/shared-api.d.ts

@@ -0,0 +1,15 @@
+import { Action, ErrorStrategy, SetupMethod, TeardownMethod } from './api-types';
+export declare class BenchmarkBase {
+    setupEachMethods: SetupMethod[];
+    setupMethods: SetupMethod[];
+    teardownMethods: TeardownMethod[];
+    teardownEachMethods: TeardownMethod[];
+    action?: Action;
+    errorStrategy?: ErrorStrategy;
+    withSetup(setup: SetupMethod): this;
+    withSetupEach(setup: SetupMethod): this;
+    withTeardown(teardown: TeardownMethod): this;
+    withTeardownEach(teardown: TeardownMethod): this;
+    withAction(action: Action): this;
+    withErrorStrategy(errorStrategy: ErrorStrategy): this;
+}

package/dist/shared-api.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BenchmarkBase = void 0;
+class BenchmarkBase {
+    setupEachMethods = [];
+    setupMethods = [];
+    teardownMethods = [];
+    teardownEachMethods = [];
+    action;
+    errorStrategy;
+    withSetup(setup) {
+        this.setupMethods.push(setup);
+        return this;
+    }
+    withSetupEach(setup) {
+        this.setupEachMethods.push(setup);
+        return this;
+    }
+    withTeardown(teardown) {
+        this.teardownMethods.push(teardown);
+        return this;
+    }
+    withTeardownEach(teardown) {
+        this.teardownEachMethods.push(teardown);
+        return this;
+    }
+    withAction(action) {
+        this.action = action;
+        return this;
+    }
+    withErrorStrategy(errorStrategy) {
+        this.errorStrategy = errorStrategy;
+        return this;
+    }
+}
+exports.BenchmarkBase = BenchmarkBase;

package/dist/suite.d.ts

@@ -1,5 +1,6 @@
-import { SuiteReporter } from './api-types';
+import { BenchmarkReporter, ErrorStrategy, SuiteReporter } from './api-types';
 import { Benchmark } from './benchmark';
+import { Result } from './results';
 import { Variation } from './variation';
 export interface SuiteOptions {
     reporter?: SuiteReporter;
@@ -9,9 +10,22 @@ export declare class Suite {
     private benchmarks;
     private variations;
     private reporter;
+    private benchmarkReporter?;
+    private errorStrategy;
+    private shouldSetErrorStrategyOnBenchmarks;
     constructor(name: string, options?: SuiteOptions);
     addBenchmark(benchmark: Benchmark): this;
-    addVariation(variation: Variation): this;
-    addVariations(variations: Variation[]): this;
-    run(): Promise<void>;
+    withVariation(variation: Variation): this;
+    withVariations(variations: Variation[]): this;
+    withReporter(reporter: SuiteReporter): this;
+    withBenchmarkReporter(reporter: BenchmarkReporter): this;
+    /**
+     * Sets the error strategy for the suite and optionally for the benchmarks.
+     * @param errorStrategy Determines what to do when an error occurs during a benchmark run. See {@link ErrorStrategy}
+     * @param opts Options for the error strategy. If `shouldSetOnBenchmarks` is true, the error strategy will be set on all benchmarks in the suite.
+     */
+    withErrorStrategy(errorStrategy: ErrorStrategy, opts?: {
+        shouldSetOnBenchmarks?: boolean;
+    }): this;
+    run(): Promise<Record<string, Result[]>>;
 }

package/dist/suite.js

@@ -1,46 +1,74 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Suite = void 0;
-const suite_console_reporter_1 = require("./suite-console-reporter");
+const api_types_1 = require("./api-types");
+const suite_console_reporter_1 = require("./reporters/suite-console-reporter");
 class Suite {
+    name;
+    benchmarks = [];
+    variations = [];
+    reporter;
+    benchmarkReporter;
+    errorStrategy = api_types_1.ErrorStrategy.Continue;
+    shouldSetErrorStrategyOnBenchmarks = false;
     constructor(name, options) {
         this.name = name;
-        this.benchmarks = [];
-        this.variations = [];
         this.name = name;
         this.benchmarks = [];
-        this.reporter = (options === null || options === void 0 ? void 0 : options.reporter) || new suite_console_reporter_1.SuiteConsoleReporter();
+        this.reporter = options?.reporter || new suite_console_reporter_1.SuiteConsoleReporter();
     }
     addBenchmark(benchmark) {
         this.benchmarks.push(benchmark);
         return this;
     }
-    addVariation(variation) {
+    withVariation(variation) {
         this.variations.push(variation);
         return this;
     }
-    addVariations(variations) {
+    withVariations(variations) {
         this.variations.push(...variations);
         return this;
     }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = {};
-            for (const benchmark of this.benchmarks) {
-                benchmark.withVariations(this.variations);
-                results[benchmark.name] = yield benchmark.run();
+    withReporter(reporter) {
+        this.reporter = reporter;
+        return this;
+    }
+    withBenchmarkReporter(reporter) {
+        this.benchmarkReporter = reporter;
+        return this;
+    }
+    /**
+     * Sets the error strategy for the suite and optionally for the benchmarks.
+     * @param errorStrategy Determines what to do when an error occurs during a benchmark run. See {@link ErrorStrategy}
+     * @param opts Options for the error strategy. If `shouldSetOnBenchmarks` is true, the error strategy will be set on all benchmarks in the suite.
+     */
+    withErrorStrategy(errorStrategy, opts) {
+        this.errorStrategy = errorStrategy;
+        if (opts?.shouldSetOnBenchmarks) {
+            this.shouldSetErrorStrategyOnBenchmarks = opts?.shouldSetOnBenchmarks;
+        }
+        return this;
+    }
+    async run() {
+        const results = {};
+        for (const benchmark of this.benchmarks) {
+            benchmark.withVariations(this.variations);
+            if (this.benchmarkReporter) {
+                benchmark.withReporter(this.benchmarkReporter);
+            }
+            if (this.shouldSetErrorStrategyOnBenchmarks) {
+                benchmark.withErrorStrategy(this.errorStrategy);
+            }
+            results[benchmark.name] = await benchmark.run();
+        }
+        this.reporter.report(results);
+        if (this.errorStrategy === api_types_1.ErrorStrategy.DelayedThrow) {
+            const failedResults = Object.values(results).flatMap((r) => r.filter((r) => r.failed));
+            if (failedResults.length > 0) {
+                throw new api_types_1.AggregateSuiteError(results);
             }
-            this.reporter.report(results);
-        });
+        }
+        return results;
     }
 }
 exports.Suite = Suite;

package/dist/utils.d.ts

@@ -0,0 +1 @@
+export declare function findCombinations<T>(variables: T[][]): T[][];

package/dist/utils.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findCombinations = findCombinations;
+function findCombinations(variables) {
+    // Generates all possible combinations of the given variables.
+    // e.g. findCombinations([[1, 2], [3, 4]]) => [[1, 3], [1, 4], [2, 3], [2, 4]]
+    return variables.reduce((acc, values) => {
+        return acc.flatMap((accItem) => {
+            return values.map((value) => {
+                return [...accItem, value];
+            });
+        });
+    }, [[]]);
+}

package/dist/variation.d.ts

@@ -1,15 +1,29 @@
-import { SetupMethod, TeardownMethod, ActionMethod, EnvironmentVariableOptions } from './api-types';
-export declare class Variation {
+import { EnvironmentVariableOptions } from './api-types';
+import { BenchmarkBase } from './shared-api';
+export declare class Variation extends BenchmarkBase {
     name: string;
-    setupMethods: SetupMethod[];
-    teardownMethods: TeardownMethod[];
     environment: Partial<NodeJS.ProcessEnv>;
-    action?: ActionMethod;
+    cliArgs: string[];
     constructor(name: string);
     static FromEnvironmentVariables(variables: EnvironmentVariableOptions): Variation[];
-    withSetup(setup: SetupMethod): this;
-    withTeardown(teardown: TeardownMethod): this;
+    /**
+     *
+     * @param args An array of options to pass to the CLI. If an element is an array, they will be treated as alternatives.
+     * @returns An array of variations. Can be applied with `withVariations`.
+     * @example
+     * // Creates 2 variations, both with --flag, but one with --no-daemon and the other with --daemon.
+     * const variations = Variation.FromCliArgs([
+     *  '--flag',
+     * ['--no-daemon', '--daemon'],
+     * ]);
+     *
+     * // Creates 4 variations, with combinations of --a and --b.
+     * const variations = Variation.FromCliArgs([
+     * ['--a', '--b'],
+     * ]);
+     */
+    static FromCliArgs(args: Array<string | string[]>): Variation[];
     withEnvironmentVariables(env: Partial<NodeJS.ProcessEnv>): this;
     withEnvironmentVariable(name: string, value: string): this;
-    withAction(action: ActionMethod): this;
+    withCliArgs(...args: string[]): this;
 }

package/dist/variation.js

@@ -1,15 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Variation = void 0;
-class Variation {
+const shared_api_1 = require("./shared-api");
+const utils_1 = require("./utils");
+class Variation extends shared_api_1.BenchmarkBase {
+    name;
+    environment = {};
+    cliArgs = [];
     constructor(name) {
+        super();
         this.name = name;
-        this.setupMethods = [];
-        this.teardownMethods = [];
-        this.environment = {};
     }
     static FromEnvironmentVariables(variables) {
-        const combinations = variables.reduce((acc, [name, values]) => {
+        const combinations = (0, utils_1.findCombinations)(variables.map(([name, values]) => values.map((value) => [name, value])));
+        variables.reduce((acc, [name, values]) => {
             return acc.flatMap((accItem) => {
                 return values.map((value) => {
                     return [...accItem, [name, value]];
@@ -21,24 +25,44 @@ class Variation {
             return new Variation(label).withEnvironmentVariables(Object.fromEntries(combination));
         });
     }
-    withSetup(setup) {
-        this.setupMethods.push(setup);
-        return this;
-    }
-    withTeardown(teardown) {
-        this.teardownMethods.push(teardown);
-        return this;
+    /**
+     *
+     * @param args An array of options to pass to the CLI. If an element is an array, they will be treated as alternatives.
+     * @returns An array of variations. Can be applied with `withVariations`.
+     * @example
+     * // Creates 2 variations, both with --flag, but one with --no-daemon and the other with --daemon.
+     * const variations = Variation.FromCliArgs([
+     *  '--flag',
+     * ['--no-daemon', '--daemon'],
+     * ]);
+     *
+     * // Creates 4 variations, with combinations of --a and --b.
+     * const variations = Variation.FromCliArgs([
+     * ['--a', '--b'],
+     * ]);
+     */
+    static FromCliArgs(args) {
+        const alternatives = args.filter(Array.isArray);
+        const values = (0, utils_1.findCombinations)(alternatives);
+        return values.flatMap((alts) => {
+            const cliArgs = args.map((arg) => Array.isArray(arg) ? alts.shift() : arg);
+            const label = cliArgs.join(' ');
+            return new Variation(label).withCliArgs(...cliArgs);
+        });
     }
     withEnvironmentVariables(env) {
-        this.environment = Object.assign(Object.assign({}, this.environment), env);
+        this.environment = {
+            ...this.environment,
+            ...env,
+        };
         return this;
     }
     withEnvironmentVariable(name, value) {
         this.environment[name] = value;
         return this;
     }
-    withAction(action) {
-        this.action = action;
+    withCliArgs(...args) {
+        this.cliArgs = this.cliArgs.concat(args);
         return this;
     }
 }

package/package.json

@@ -1,35 +1,52 @@
 {
   "name": "flexi-bench",
-  "version": "0.0.0-alpha.3",
+  "version": "0.1.0",
   "description": "",
   "main": "dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/AgentEnder/flexi-bench"
+  },
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "node --import tsx --test './src/**/*.spec.ts'",
+    "test:watch": "node --import tsx --test --watch './src/**/*.spec.ts'",
     "build": "tsc -p tsconfig.json",
-    "format": "prettier --write src"
+    "format": "prettier --write src",
+    "prepare": "patch-package && nx build flexi-bench"
   },
   "author": "",
-  "license": "ISC",
+  "license": "MIT",
   "devDependencies": {
+    "@nx-dotnet/nx-ghpages": "^2.3.0",
     "@nx/js": "^19.4.2",
     "@swc-node/register": "~1.9.1",
     "@swc/core": "~1.5.7",
     "@swc/helpers": "~0.5.11",
-    "nx": "19.4.2"
-  },
-  "nx": {},
-  "dependencies": {
     "@types/cli-progress": "^3.11.6",
     "@types/node": "^20.14.10",
-    "cli-progress": "^3.12.0",
+    "nx": "19.4.2",
+    "patch-package": "^8.0.0",
     "prettier": "^3.3.2",
-    "typescript": "^5.5.3"
+    "ts-node": "^10.9.2",
+    "tsx": "^4.16.2",
+    "typescript": "^5.5.3",
+    "yaml": "^2.4.5"
   },
+  "dependencies": {
+    "cli-progress": "^3.12.0",
+    "markdown-factory": "^0.0.6"
+  },
+  "nx": {},
   "files": [
     "dist",
     "README.md",
     "LICENSE",
     "CHANGELOG.md",
     "package.json"
+  ],
+  "workspaces": [
+    ".",
+    "docs-site",
+    "packages/*"
   ]
 }

package/dist/benchmark-console-reporter.js

@@ -1,31 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BenchmarkConsoleReporter = void 0;
-const cli_progress_1 = require("cli-progress");
-class BenchmarkConsoleReporter {
-    constructor() {
-        this.bar = new cli_progress_1.SingleBar({
-            format: 'Running: {label}\n{bar} {percentage}% | {value}/{total} - ETA: {eta}s',
-            barCompleteChar: '\u2588',
-            barIncompleteChar: '\u2591',
-            hideCursor: true,
-        });
-    }
-    progress(name, percent, context) {
-        var _a;
-        if (!this.bar.isActive) {
-            this.bar.start((_a = context.totalIterations) !== null && _a !== void 0 ? _a : 100, 0);
-        }
-        this.bar.update(context.completedIterations, { label: name });
-        process.stdout.moveCursor(0, -1);
-        // console.log(`Progress: ${name} ${Math.round(percent * 10000) / 100}%`);
-    }
-    report(benchmark, results) {
-        this.bar.stop();
-        process.stdout.moveCursor(0, -1);
-        process.stdout.clearScreenDown();
-        console.log(`Benchmark: ${benchmark.name}`);
-        console.table(results);
-    }
-}
-exports.BenchmarkConsoleReporter = BenchmarkConsoleReporter;

package/dist/console-reporter.d.ts

@@ -1,8 +0,0 @@
-import { Result, Reporter, ProgressContext } from './api-types';
-import { Benchmark } from './benchmark';
-export declare class ConsoleReporter implements Reporter {
-    private bar;
-    constructor();
-    progress(name: string, percent: number, context: ProgressContext): void;
-    report(benchmark: Benchmark, results: Result[]): void;
-}

package/dist/console-reporter.js

@@ -1,31 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ConsoleReporter = void 0;
-const cli_progress_1 = require("cli-progress");
-class ConsoleReporter {
-    constructor() {
-        this.bar = new cli_progress_1.SingleBar({
-            format: 'Running: {label}\n{bar} {percentage}% | {value}/{total} - ETA: {eta}s',
-            barCompleteChar: '\u2588',
-            barIncompleteChar: '\u2591',
-            hideCursor: true,
-        });
-    }
-    progress(name, percent, context) {
-        var _a;
-        if (!this.bar.isActive) {
-            this.bar.start((_a = context.totalIterations) !== null && _a !== void 0 ? _a : 100, 0);
-        }
-        this.bar.update(context.completedIterations, { label: name });
-        process.stdout.moveCursor(0, -1);
-        // console.log(`Progress: ${name} ${Math.round(percent * 10000) / 100}%`);
-    }
-    report(benchmark, results) {
-        this.bar.stop();
-        process.stdout.moveCursor(0, -1);
-        process.stdout.clearScreenDown();
-        console.log(`Benchmark: ${benchmark.name}`);
-        console.table(results);
-    }
-}
-exports.ConsoleReporter = ConsoleReporter;

package/dist/suite-console-reporter.js

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SuiteConsoleReporter = void 0;
-class SuiteConsoleReporter {
-    constructor() {
-        this.report = (results) => {
-            console.log('Suite Results:');
-            for (const [name, result] of Object.entries(results)) {
-                console.log(`Benchmark: ${name}`);
-                console.table(result);
-            }
-        };
-    }
-}
-exports.SuiteConsoleReporter = SuiteConsoleReporter;