flexi-bench 0.0.0-alpha.4 → 0.1.0

package/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 0.1.0 (2024-07-28)
+
+
+### 🚀 Features
+
+- add different error strategies for handling failing actions ([#11](https://github.com/AgentEnder/flexi-bench/pull/11))
+- **repo:** add ability to publish only docs ([4cde432](https://github.com/AgentEnder/flexi-bench/commit/4cde432))
+
+### ❤️  Thank You
+
+- Craigory Coppola @AgentEnder
+
+# 0.0.0 (2024-07-28)
+
+
+### 🚀 Features
+
+- add different error strategies for handling failing actions ([#11](https://github.com/AgentEnder/flexi-bench/pull/11))
+- **repo:** add ability to publish only docs ([4cde432](https://github.com/AgentEnder/flexi-bench/commit/4cde432))
+
+### ❤️  Thank You
+
+- Craigory Coppola @AgentEnder
+
 ## 0.0.0-alpha.4 (2024-07-17)
 
 

package/dist/api-types.d.ts

@@ -21,3 +21,28 @@ export interface BenchmarkReporter {
 export interface SuiteReporter {
     report: (results: Record<string, Result[]>) => void;
 }
+/**
+ * The strategy to use when an error occurs during a benchmark run.
+ */
+export declare enum ErrorStrategy {
+    /**
+     * Continue running the benchmark. Errors will be collected and reported at the end. This is the default behavior.
+     */
+    Continue = "continue",
+    /**
+     * Abort the benchmark run immediately when an error occurs.
+     */
+    Abort = "abort",
+    /**
+     * Delay the error until the end of the benchmark run. This is useful when you want to see all the errors at once
+     */
+    DelayedThrow = "delayed-throw"
+}
+export declare class AggregateBenchmarkError extends Error {
+    results: Result[];
+    constructor(results: Result[]);
+}
+export declare class AggregateSuiteError extends Error {
+    results: Record<string, Result[]>;
+    constructor(results: Record<string, Result[]>);
+}

package/dist/api-types.js

@@ -1,2 +1,37 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.AggregateSuiteError = exports.AggregateBenchmarkError = exports.ErrorStrategy = void 0;
+/**
+ * The strategy to use when an error occurs during a benchmark run.
+ */
+var ErrorStrategy;
+(function (ErrorStrategy) {
+    /**
+     * Continue running the benchmark. Errors will be collected and reported at the end. This is the default behavior.
+     */
+    ErrorStrategy["Continue"] = "continue";
+    /**
+     * Abort the benchmark run immediately when an error occurs.
+     */
+    ErrorStrategy["Abort"] = "abort";
+    /**
+     * Delay the error until the end of the benchmark run. This is useful when you want to see all the errors at once
+     */
+    ErrorStrategy["DelayedThrow"] = "delayed-throw";
+})(ErrorStrategy || (exports.ErrorStrategy = ErrorStrategy = {}));
+class AggregateBenchmarkError extends Error {
+    results;
+    constructor(results) {
+        super('[AggregateBenchmarkError]: One or more benchmarks failed. Check the results for more information.');
+        this.results = results;
+    }
+}
+exports.AggregateBenchmarkError = AggregateBenchmarkError;
+class AggregateSuiteError extends Error {
+    results;
+    constructor(results) {
+        super('[AggregateSuiteError]: One or more benchmarks failed. Check the results for more information.');
+        this.results = results;
+    }
+}
+exports.AggregateSuiteError = AggregateSuiteError;

package/dist/benchmark-runner.js

@@ -23,7 +23,7 @@ function suite(name, fn) {
     activeSuite = suite;
     const transformed = fn(suite);
     activeSuite = null;
-    return (transformed !== null && transformed !== void 0 ? transformed : suite).run();
+    return (transformed ?? suite).run();
 }
 let activeBenchmark = null;
 /**
@@ -38,10 +38,10 @@ function benchmark(name, fn) {
     const transformed = fn(benchmark);
     activeBenchmark = null;
     if (activeSuite) {
-        activeSuite.addBenchmark(transformed !== null && transformed !== void 0 ? transformed : benchmark);
+        activeSuite.addBenchmark(transformed ?? benchmark);
     }
     else {
-        return (transformed !== null && transformed !== void 0 ? transformed : benchmark).run();
+        return (transformed ?? benchmark).run();
     }
 }
 let activeVariation = null;
@@ -57,10 +57,10 @@ function variation(name, fn) {
     const transformed = fn(variation);
     activeVariation = null;
     if (activeBenchmark) {
-        activeBenchmark.withVariation(transformed !== null && transformed !== void 0 ? transformed : variation);
+        activeBenchmark.withVariation(transformed ?? variation);
     }
     else if (activeSuite) {
-        activeSuite.withVariation(transformed !== null && transformed !== void 0 ? transformed : variation);
+        activeSuite.withVariation(transformed ?? variation);
     }
     else {
         throw new Error('`variation` must be called within a benchmark or suite');

package/dist/benchmark.d.ts

@@ -1,5 +1,5 @@
 import { Variation } from './variation';
-import { TeardownMethod, SetupMethod, Action, BenchmarkReporter } from './api-types';
+import { TeardownMethod, SetupMethod, Action, BenchmarkReporter, ErrorStrategy } from './api-types';
 import { Result } from './results';
 import { PerformanceObserverOptions } from './performance-observer';
 import { BenchmarkBase } from './shared-api';
@@ -10,6 +10,7 @@ export declare class Benchmark extends BenchmarkBase {
     private timeout?;
     private reporter;
     private watcher?;
+    errorStrategy: ErrorStrategy;
     constructor(name: string, options?: {
         setup?: SetupMethod;
         teardown?: TeardownMethod;

package/dist/benchmark.js

@@ -1,42 +1,40 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Benchmark = void 0;
 const benchmark_console_reporter_1 = require("./reporters/benchmark-console-reporter");
 const variation_1 = require("./variation");
+const api_types_1 = require("./api-types");
 const child_process_1 = require("child_process");
 const results_1 = require("./results");
 const performance_observer_1 = require("./performance-observer");
 const shared_api_1 = require("./shared-api");
 class Benchmark extends shared_api_1.BenchmarkBase {
+    name;
+    variations = [];
+    iterations;
+    timeout;
+    reporter;
+    watcher;
+    errorStrategy = api_types_1.ErrorStrategy.Continue;
     constructor(name, options) {
         super();
         this.name = name;
-        this.variations = [];
-        if (options === null || options === void 0 ? void 0 : options.action) {
+        if (options?.action) {
             this.action = options.action;
         }
-        if (options === null || options === void 0 ? void 0 : options.setup) {
+        if (options?.setup) {
             this.setupMethods.push(options.setup);
         }
-        if (options === null || options === void 0 ? void 0 : options.teardown) {
+        if (options?.teardown) {
             this.teardownMethods.push(options.teardown);
         }
-        if (options === null || options === void 0 ? void 0 : options.iterations) {
+        if (options?.iterations) {
             this.iterations = options.iterations;
         }
-        if (options === null || options === void 0 ? void 0 : options.timeout) {
+        if (options?.timeout) {
             this.timeout = options.timeout;
         }
-        this.reporter = (options === null || options === void 0 ? void 0 : options.reporter) || new benchmark_console_reporter_1.BenchmarkConsoleReporter();
+        this.reporter = options?.reporter || new benchmark_console_reporter_1.BenchmarkConsoleReporter();
     }
     withVariation(nameOrVariation, builder) {
         if (nameOrVariation instanceof variation_1.Variation) {
@@ -83,117 +81,119 @@ class Benchmark extends shared_api_1.BenchmarkBase {
         this.watcher = new performance_observer_1.PerformanceWatcher(options);
         return this;
     }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
-            this.validate();
-            let results = [];
-            if (this.variations.length === 0) {
-                this.variations.push(new variation_1.Variation('default'));
+    async run() {
+        this.validate();
+        let results = [];
+        if (this.variations.length === 0) {
+            this.variations.push(new variation_1.Variation('default'));
+        }
+        const totalIterations = this.iterations
+            ? this.variations.length * this.iterations
+            : undefined;
+        const startTime = performance.now();
+        let totalCompletedIterations = 0;
+        for (let variationIndex = 0; variationIndex < this.variations.length; variationIndex++) {
+            const variation = this.variations[variationIndex];
+            const iterationResults = [];
+            // SETUP
+            const oldEnv = { ...process.env };
+            process.env = {
+                ...process.env,
+                ...variation.environment,
+            };
+            for (const setup of this.setupMethods) {
+                await setup(variation);
             }
-            const totalIterations = this.iterations
-                ? this.variations.length * this.iterations
-                : undefined;
-            const startTime = performance.now();
-            let totalCompletedIterations = 0;
-            for (let variationIndex = 0; variationIndex < this.variations.length; variationIndex++) {
-                const variation = this.variations[variationIndex];
-                const timings = [];
-                // SETUP
-                const oldEnv = Object.assign({}, process.env);
-                process.env = Object.assign(Object.assign({}, process.env), variation.environment);
-                for (const setup of this.setupMethods) {
-                    yield setup(variation);
-                }
-                for (const setup of variation.setupMethods) {
-                    yield setup(variation);
-                }
-                // ACT
-                const benchmarkThis = this;
-                yield new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () {
-                    let completedIterations = 0;
-                    let timeout = benchmarkThis.timeout
-                        ? setTimeout(() => {
-                            running = false;
-                            if ((benchmarkThis === null || benchmarkThis === void 0 ? void 0 : benchmarkThis.iterations) &&
-                                completedIterations < benchmarkThis.iterations) {
-                                reject('Timeout');
-                            }
-                            resolve();
-                        }, benchmarkThis.timeout)
-                        : null;
-                    let running = true;
-                    while (running) {
-                        for (const setup of this.setupEachMethods.concat(variation.setupEachMethods)) {
-                            yield setup(variation);
-                        }
-                        const a = performance.now();
-                        if (variation.action) {
-                            yield runAction(variation.action, variation);
-                        }
-                        else if (this.action) {
-                            yield runAction(this.action, variation);
-                        }
-                        const b = performance.now();
-                        for (const teardown of this.teardownEachMethods.concat(variation.teardownEachMethods)) {
-                            yield teardown(variation);
-                        }
-                        const duration = b - a;
-                        completedIterations++;
-                        totalCompletedIterations++;
-                        timings.push(duration);
-                        if (this.reporter.progress &&
-                            totalIterations &&
-                            benchmarkThis.iterations) {
-                            this.reporter.progress(variation.name, totalCompletedIterations / totalIterations, {
-                                timeElapsed: performance.now() - startTime,
-                                totalIterations,
-                                completedIterations: totalCompletedIterations,
-                                timeout: benchmarkThis.timeout,
-                            });
+            for (const setup of variation.setupMethods) {
+                await setup(variation);
+            }
+            // ACT
+            const benchmarkThis = this;
+            await new Promise(async (resolve, reject) => {
+                let completedIterations = 0;
+                let timeout = benchmarkThis.timeout
+                    ? setTimeout(() => {
+                        running = false;
+                        if (benchmarkThis?.iterations &&
+                            completedIterations < benchmarkThis.iterations) {
+                            reject('Timeout');
                         }
-                        if ((benchmarkThis === null || benchmarkThis === void 0 ? void 0 : benchmarkThis.iterations) &&
-                            completedIterations >= benchmarkThis.iterations) {
-                            running = false;
-                            if (timeout) {
-                                clearTimeout(timeout);
-                            }
-                            resolve();
+                        resolve();
+                    }, benchmarkThis.timeout)
+                    : null;
+                let running = true;
+                while (running) {
+                    for (const setup of this.setupEachMethods.concat(variation.setupEachMethods)) {
+                        await setup(variation);
+                    }
+                    const result = await runAndMeasureAction(benchmarkThis, variation);
+                    const errorStrategy = variation.errorStrategy ?? benchmarkThis.errorStrategy;
+                    if (errorStrategy === api_types_1.ErrorStrategy.Abort &&
+                        result instanceof Error) {
+                        reject(result);
+                        return;
+                    }
+                    for (const teardown of this.teardownEachMethods.concat(variation.teardownEachMethods)) {
+                        await teardown(variation);
+                    }
+                    completedIterations++;
+                    totalCompletedIterations++;
+                    iterationResults.push(result);
+                    if (this.reporter.progress &&
+                        totalIterations &&
+                        benchmarkThis.iterations) {
+                        this.reporter.progress(variation.name, totalCompletedIterations / totalIterations, {
+                            timeElapsed: performance.now() - startTime,
+                            totalIterations,
+                            completedIterations: totalCompletedIterations,
+                            timeout: benchmarkThis.timeout,
+                        });
+                    }
+                    if (benchmarkThis?.iterations &&
+                        completedIterations >= benchmarkThis.iterations) {
+                        running = false;
+                        if (timeout) {
+                            clearTimeout(timeout);
                         }
+                        resolve();
                     }
-                }));
-                if (this.reporter.progress && !totalIterations) {
-                    this.reporter.progress(variation.name, variationIndex / this.variations.length, {
-                        timeElapsed: performance.now() - startTime,
-                        timeout: this.timeout,
-                        completedIterations: totalCompletedIterations,
-                    });
-                }
-                // TEARDOWN
-                for (const teardown of variation.teardownMethods) {
-                    yield teardown(variation);
-                }
-                for (const teardown of this.teardownMethods) {
-                    yield teardown(variation);
                 }
-                process.env = oldEnv;
-                // REPORT
-                const result = (0, results_1.calculateResultsFromDurations)(variation.name, timings);
-                // PerformanceObserver needs a chance to flush
-                if (this.watcher) {
-                    const measures = yield this.watcher.getMeasures();
-                    for (const key in measures) {
-                        (_a = result.subresults) !== null && _a !== void 0 ? _a : (result.subresults = []);
-                        result.subresults.push((0, results_1.calculateResultsFromDurations)(key, measures[key]));
-                    }
-                    this.watcher.clearMeasures();
+            });
+            if (this.reporter.progress && !totalIterations) {
+                this.reporter.progress(variation.name, variationIndex / this.variations.length, {
+                    timeElapsed: performance.now() - startTime,
+                    timeout: this.timeout,
+                    completedIterations: totalCompletedIterations,
+                });
+            }
+            // TEARDOWN
+            for (const teardown of variation.teardownMethods) {
+                await teardown(variation);
+            }
+            for (const teardown of this.teardownMethods) {
+                await teardown(variation);
+            }
+            process.env = oldEnv;
+            // REPORT
+            const result = (0, results_1.calculateResultsFromDurations)(variation.name, iterationResults);
+            // PerformanceObserver needs a chance to flush
+            if (this.watcher) {
+                const measures = await this.watcher.getMeasures();
+                for (const key in measures) {
+                    result.subresults ??= [];
+                    result.subresults.push((0, results_1.calculateResultsFromDurations)(key, measures[key]));
                 }
-                results.push(result);
+                this.watcher.clearMeasures();
             }
-            (_b = this.watcher) === null || _b === void 0 ? void 0 : _b.disconnect();
-            this.reporter.report(this, results);
-            return results;
-        });
+            results.push(result);
+        }
+        this.watcher?.disconnect();
+        this.reporter.report(this, results);
+        if (this.errorStrategy === api_types_1.ErrorStrategy.DelayedThrow &&
+            results.some((r) => r.failed)) {
+            throw new api_types_1.AggregateBenchmarkError(results);
+        }
+        return results;
     }
     validate() {
         if (!this.timeout && !this.iterations) {
@@ -224,26 +224,35 @@ class Benchmark extends shared_api_1.BenchmarkBase {
     }
 }
 exports.Benchmark = Benchmark;
-function runAction(action, variation) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (typeof action === 'string') {
-            return new Promise((resolve, reject) => {
-                const child = (0, child_process_1.spawn)(action, variation.cliArgs, {
-                    shell: true,
-                    windowsHide: true,
-                });
-                child.on('exit', (code) => {
-                    if (code === 0) {
-                        resolve();
-                    }
-                    else {
-                        reject(`Action failed with code ${code}`);
-                    }
-                });
+async function runAndMeasureAction(benchmark, variation) {
+    try {
+        const a = performance.now();
+        await runAction((variation.action ?? benchmark.action), variation);
+        const b = performance.now();
+        return b - a;
+    }
+    catch (e) {
+        return e instanceof Error ? e : new Error('Unknown error during action.');
+    }
+}
+async function runAction(action, variation) {
+    if (typeof action === 'string') {
+        return new Promise((resolve, reject) => {
+            const child = (0, child_process_1.spawn)(action, variation.cliArgs, {
+                shell: true,
+                windowsHide: true,
             });
-        }
-        else {
-            return action(variation);
-        }
-    });
+            child.on('exit', (code) => {
+                if (code === 0) {
+                    resolve();
+                }
+                else {
+                    reject(`Action failed with code ${code}`);
+                }
+            });
+        });
+    }
+    else {
+        return action(variation);
+    }
 }

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 export * from './reporters/benchmark-console-reporter';
 export * from './reporters/markdown-benchmark-reporter';
 export * from './reporters/suite-console-reporter';
+export * from './reporters/noop-reporter';
 export * from './api-types';
 export * from './benchmark';
 export * from './variation';

package/dist/index.js

@@ -17,6 +17,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./reporters/benchmark-console-reporter"), exports);
 __exportStar(require("./reporters/markdown-benchmark-reporter"), exports);
 __exportStar(require("./reporters/suite-console-reporter"), exports);
+__exportStar(require("./reporters/noop-reporter"), exports);
 __exportStar(require("./api-types"), exports);
 __exportStar(require("./benchmark"), exports);
 __exportStar(require("./variation"), exports);

package/dist/performance-observer.js

@@ -1,27 +1,17 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PerformanceWatcher = void 0;
 const node_perf_hooks_1 = require("node:perf_hooks");
 class PerformanceWatcher {
+    observer;
+    measures = {};
     constructor(opts = {}) {
-        this.measures = {};
         this.observer = new node_perf_hooks_1.PerformanceObserver((list) => {
-            var _a;
-            var _b;
             for (const entry of list.getEntries()) {
                 if (entry.entryType === 'measure' &&
                     (!opts.measureFilter || opts.measureFilter(entry))) {
-                    const label = normalizeLabel(entry.name, opts === null || opts === void 0 ? void 0 : opts.label);
-                    (_a = (_b = this.measures)[label]) !== null && _a !== void 0 ? _a : (_b[label] = []);
+                    const label = normalizeLabel(entry.name, opts?.label);
+                    this.measures[label] ??= [];
                     this.measures[label].push(entry.duration);
                 }
             }
@@ -33,10 +23,8 @@ class PerformanceWatcher {
     // not fire until the next tick so we need that to happen before the measures are
     // retrieved. Since this method is async, it must be awaited, which actually gives
     // the observer time to collect the measures.
-    getMeasures() {
-        return __awaiter(this, void 0, void 0, function* () {
-            return this.measures;
-        });
+    async getMeasures() {
+        return this.measures;
     }
     clearMeasures() {
         for (const key in this.measures) {
@@ -44,8 +32,7 @@ class PerformanceWatcher {
         }
     }
     disconnect() {
-        var _a;
-        (_a = this.observer) === null || _a === void 0 ? void 0 : _a.disconnect();
+        this.observer?.disconnect();
         delete this.observer;
     }
 }

package/dist/reporters/benchmark-console-reporter.js

@@ -3,26 +3,27 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.BenchmarkConsoleReporter = void 0;
 const cli_progress_1 = require("cli-progress");
 class BenchmarkConsoleReporter {
-    constructor() {
-        this.bar = new cli_progress_1.SingleBar({
-            format: 'Running variation {label}: {bar} {percentage}% | {value}/{total} - ETA: {eta}s',
-            barCompleteChar: '\u2588',
-            barIncompleteChar: '\u2591',
-            hideCursor: true,
-            stopOnComplete: true,
-            clearOnComplete: true,
-        });
-    }
+    bar = new cli_progress_1.SingleBar({
+        format: 'Running variation {label}: {bar} {percentage}% | {value}/{total} - ETA: {eta}s',
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        hideCursor: true,
+        stopOnComplete: true,
+        clearOnComplete: true,
+    });
+    constructor() { }
     progress(name, percent, context) {
-        var _a;
         if (!this.bar.isActive) {
-            this.bar.start((_a = context.totalIterations) !== null && _a !== void 0 ? _a : 100, 0);
+            this.bar.start(context.totalIterations ?? 100, 0);
         }
         this.bar.update(context.completedIterations, { label: name });
     }
     report(benchmark, results) {
+        const tableEntries = results.map(({ raw, ...rest }) => ({
+            ...rest,
+        }));
         console.log(`Benchmark: ${benchmark.name}`);
-        console.table(results);
+        console.table(tableEntries);
     }
 }
 exports.BenchmarkConsoleReporter = BenchmarkConsoleReporter;

package/dist/reporters/markdown-benchmark-reporter.js

@@ -4,26 +4,26 @@ exports.MarkdownBenchmarkReporter = void 0;
 const fs_1 = require("fs");
 const markdown_factory_1 = require("markdown-factory");
 class MarkdownBenchmarkReporter {
+    outputFile;
+    fields;
     constructor(opts) {
-        var _a;
-        this.report = (benchmark, results) => {
-            (0, fs_1.writeFileSync)(this.outputFile, (0, markdown_factory_1.h1)(benchmark.name, results.some((r) => !!r.subresults)
-                ? (0, markdown_factory_1.lines)(results.map((r) => {
-                    var _a;
-                    const entries = [Object.assign(Object.assign({}, r), { label: 'total' })];
-                    delete entries[0].subresults;
-                    for (const subresult of (_a = r.subresults) !== null && _a !== void 0 ? _a : []) {
-                        entries.push(subresult);
-                    }
-                    return (0, markdown_factory_1.h2)(r.label, (0, markdown_factory_1.table)(entries, [
-                        { field: 'label', label: '' },
-                        ...this.fields,
-                    ]));
-                }))
-                : (0, markdown_factory_1.table)(results, [{ field: 'label', label: '' }, ...this.fields])));
-        };
         this.outputFile = opts.outputFile;
-        this.fields = (_a = opts.fields) !== null && _a !== void 0 ? _a : ['min', 'average', 'p95', 'max'];
+        this.fields = opts.fields ?? ['min', 'average', 'p95', 'max'];
     }
+    report = (benchmark, results) => {
+        (0, fs_1.writeFileSync)(this.outputFile, (0, markdown_factory_1.h1)(benchmark.name, results.some((r) => !!r.subresults)
+            ? (0, markdown_factory_1.lines)(results.map((r) => {
+                const entries = [{ ...r, label: 'total' }];
+                delete entries[0].subresults;
+                for (const subresult of r.subresults ?? []) {
+                    entries.push(subresult);
+                }
+                return (0, markdown_factory_1.h2)(r.label, (0, markdown_factory_1.table)(entries, [
+                    { field: 'label', label: '' },
+                    ...this.fields,
+                ]));
+            }))
+            : (0, markdown_factory_1.table)(results, [{ field: 'label', label: '' }, ...this.fields])));
+    };
 }
 exports.MarkdownBenchmarkReporter = MarkdownBenchmarkReporter;

package/dist/reporters/noop-reporter.js

@@ -2,8 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.NoopReporter = void 0;
 class NoopReporter {
-    constructor() {
-        this.report = () => { };
-    }
+    report = () => { };
 }
 exports.NoopReporter = NoopReporter;

package/dist/reporters/suite-console-reporter.js

@@ -2,14 +2,15 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SuiteConsoleReporter = void 0;
 class SuiteConsoleReporter {
-    constructor() {
-        this.report = (results) => {
-            console.log('Suite Results:');
-            for (const [name, result] of Object.entries(results)) {
-                console.log(`Benchmark: ${name}`);
-                console.table(result);
-            }
-        };
-    }
+    report = (results) => {
+        console.log('Suite Results:');
+        for (const [name, result] of Object.entries(results)) {
+            const tableEntries = result.map(({ raw, ...rest }) => ({
+                ...rest,
+            }));
+            console.log(`Benchmark: ${name}`);
+            console.table(tableEntries);
+        }
+    };
 }
 exports.SuiteConsoleReporter = SuiteConsoleReporter;

package/dist/results.d.ts

@@ -25,10 +25,18 @@ export type Result = {
     /**
      * The raw durations, in order. Used for custom reporters.
      */
-    raw: number[];
+    raw: (number | Error)[];
+    /**
+     * Whether any run of the benchmark failed.
+     */
+    failed?: boolean;
+    /**
+     * The rate of failure, if any.
+     */
+    failureRate?: number;
     /**
      * Subresults, if any. Typically sourced from performance observer.
      */
     subresults?: Result[];
 };
-export declare function calculateResultsFromDurations(label: string, durations: number[]): Result;
+export declare function calculateResultsFromDurations(label: string, durations: (number | Error)[]): Result;

package/dist/results.js

@@ -2,13 +2,25 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.calculateResultsFromDurations = calculateResultsFromDurations;
 function calculateResultsFromDurations(label, durations) {
-    durations.sort((a, b) => a - b);
+    const errors = [];
+    const results = [];
+    for (const duration of durations) {
+        if (duration instanceof Error) {
+            errors.push(duration);
+        }
+        else {
+            results.push(duration);
+        }
+    }
+    results.sort((a, b) => a - b);
     return {
         label,
-        min: Math.min(...durations),
-        max: Math.max(...durations),
-        average: durations.reduce((acc, duration) => acc + duration, 0) / durations.length,
-        p95: durations[Math.floor(durations.length * 0.95)],
+        min: Math.min(...results),
+        max: Math.max(...results),
+        average: results.reduce((acc, duration) => acc + duration, 0) / results.length,
+        p95: results[Math.floor(results.length * 0.95)],
         raw: durations,
+        failed: errors.length > 0,
+        failureRate: errors.length / durations.length,
     };
 }

package/dist/shared-api.d.ts

@@ -1,13 +1,15 @@
-import { Action, SetupMethod, TeardownMethod } from './api-types';
+import { Action, ErrorStrategy, SetupMethod, TeardownMethod } from './api-types';
 export declare class BenchmarkBase {
     setupEachMethods: SetupMethod[];
     setupMethods: SetupMethod[];
     teardownMethods: TeardownMethod[];
     teardownEachMethods: TeardownMethod[];
     action?: Action;
+    errorStrategy?: ErrorStrategy;
     withSetup(setup: SetupMethod): this;
     withSetupEach(setup: SetupMethod): this;
     withTeardown(teardown: TeardownMethod): this;
     withTeardownEach(teardown: TeardownMethod): this;
     withAction(action: Action): this;
+    withErrorStrategy(errorStrategy: ErrorStrategy): this;
 }

package/dist/shared-api.js

@@ -2,12 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BenchmarkBase = void 0;
 class BenchmarkBase {
-    constructor() {
-        this.setupEachMethods = [];
-        this.setupMethods = [];
-        this.teardownMethods = [];
-        this.teardownEachMethods = [];
-    }
+    setupEachMethods = [];
+    setupMethods = [];
+    teardownMethods = [];
+    teardownEachMethods = [];
+    action;
+    errorStrategy;
     withSetup(setup) {
         this.setupMethods.push(setup);
         return this;
@@ -28,5 +28,9 @@ class BenchmarkBase {
         this.action = action;
         return this;
     }
+    withErrorStrategy(errorStrategy) {
+        this.errorStrategy = errorStrategy;
+        return this;
+    }
 }
 exports.BenchmarkBase = BenchmarkBase;

package/dist/suite.d.ts

@@ -1,4 +1,4 @@
-import { BenchmarkReporter, SuiteReporter } from './api-types';
+import { BenchmarkReporter, ErrorStrategy, SuiteReporter } from './api-types';
 import { Benchmark } from './benchmark';
 import { Result } from './results';
 import { Variation } from './variation';
@@ -11,11 +11,21 @@ export declare class Suite {
     private variations;
     private reporter;
     private benchmarkReporter?;
+    private errorStrategy;
+    private shouldSetErrorStrategyOnBenchmarks;
     constructor(name: string, options?: SuiteOptions);
     addBenchmark(benchmark: Benchmark): this;
     withVariation(variation: Variation): this;
     withVariations(variations: Variation[]): this;
     withReporter(reporter: SuiteReporter): this;
     withBenchmarkReporter(reporter: BenchmarkReporter): this;
+    /**
+     * Sets the error strategy for the suite and optionally for the benchmarks.
+     * @param errorStrategy Determines what to do when an error occurs during a benchmark run. See {@link ErrorStrategy}
+     * @param opts Options for the error strategy. If `shouldSetOnBenchmarks` is true, the error strategy will be set on all benchmarks in the suite.
+     */
+    withErrorStrategy(errorStrategy: ErrorStrategy, opts?: {
+        shouldSetOnBenchmarks?: boolean;
+    }): this;
     run(): Promise<Record<string, Result[]>>;
 }

package/dist/suite.js

@@ -1,24 +1,21 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Suite = void 0;
+const api_types_1 = require("./api-types");
 const suite_console_reporter_1 = require("./reporters/suite-console-reporter");
 class Suite {
+    name;
+    benchmarks = [];
+    variations = [];
+    reporter;
+    benchmarkReporter;
+    errorStrategy = api_types_1.ErrorStrategy.Continue;
+    shouldSetErrorStrategyOnBenchmarks = false;
     constructor(name, options) {
         this.name = name;
-        this.benchmarks = [];
-        this.variations = [];
         this.name = name;
         this.benchmarks = [];
-        this.reporter = (options === null || options === void 0 ? void 0 : options.reporter) || new suite_console_reporter_1.SuiteConsoleReporter();
+        this.reporter = options?.reporter || new suite_console_reporter_1.SuiteConsoleReporter();
     }
     addBenchmark(benchmark) {
         this.benchmarks.push(benchmark);
@@ -40,19 +37,38 @@ class Suite {
         this.benchmarkReporter = reporter;
         return this;
     }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = {};
-            for (const benchmark of this.benchmarks) {
-                benchmark.withVariations(this.variations);
-                if (this.benchmarkReporter) {
-                    benchmark.withReporter(this.benchmarkReporter);
-                }
-                results[benchmark.name] = yield benchmark.run();
+    /**
+     * Sets the error strategy for the suite and optionally for the benchmarks.
+     * @param errorStrategy Determines what to do when an error occurs during a benchmark run. See {@link ErrorStrategy}
+     * @param opts Options for the error strategy. If `shouldSetOnBenchmarks` is true, the error strategy will be set on all benchmarks in the suite.
+     */
+    withErrorStrategy(errorStrategy, opts) {
+        this.errorStrategy = errorStrategy;
+        if (opts?.shouldSetOnBenchmarks) {
+            this.shouldSetErrorStrategyOnBenchmarks = opts?.shouldSetOnBenchmarks;
+        }
+        return this;
+    }
+    async run() {
+        const results = {};
+        for (const benchmark of this.benchmarks) {
+            benchmark.withVariations(this.variations);
+            if (this.benchmarkReporter) {
+                benchmark.withReporter(this.benchmarkReporter);
+            }
+            if (this.shouldSetErrorStrategyOnBenchmarks) {
+                benchmark.withErrorStrategy(this.errorStrategy);
+            }
+            results[benchmark.name] = await benchmark.run();
+        }
+        this.reporter.report(results);
+        if (this.errorStrategy === api_types_1.ErrorStrategy.DelayedThrow) {
+            const failedResults = Object.values(results).flatMap((r) => r.filter((r) => r.failed));
+            if (failedResults.length > 0) {
+                throw new api_types_1.AggregateSuiteError(results);
             }
-            this.reporter.report(results);
-            return results;
-        });
+        }
+        return results;
     }
 }
 exports.Suite = Suite;

package/dist/variation.js

@@ -4,11 +4,12 @@ exports.Variation = void 0;
 const shared_api_1 = require("./shared-api");
 const utils_1 = require("./utils");
 class Variation extends shared_api_1.BenchmarkBase {
+    name;
+    environment = {};
+    cliArgs = [];
     constructor(name) {
         super();
         this.name = name;
-        this.environment = {};
-        this.cliArgs = [];
     }
     static FromEnvironmentVariables(variables) {
         const combinations = (0, utils_1.findCombinations)(variables.map(([name, values]) => values.map((value) => [name, value])));
@@ -50,7 +51,10 @@ class Variation extends shared_api_1.BenchmarkBase {
         });
     }
     withEnvironmentVariables(env) {
-        this.environment = Object.assign(Object.assign({}, this.environment), env);
+        this.environment = {
+            ...this.environment,
+            ...env,
+        };
         return this;
     }
     withEnvironmentVariable(name, value) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flexi-bench",
-  "version": "0.0.0-alpha.4",
+  "version": "0.1.0",
   "description": "",
   "main": "dist/index.js",
   "repository": {
@@ -11,11 +11,13 @@
     "test": "node --import tsx --test './src/**/*.spec.ts'",
     "test:watch": "node --import tsx --test --watch './src/**/*.spec.ts'",
     "build": "tsc -p tsconfig.json",
-    "format": "prettier --write src"
+    "format": "prettier --write src",
+    "prepare": "patch-package && nx build flexi-bench"
   },
   "author": "",
   "license": "MIT",
   "devDependencies": {
+    "@nx-dotnet/nx-ghpages": "^2.3.0",
     "@nx/js": "^19.4.2",
     "@swc-node/register": "~1.9.1",
     "@swc/core": "~1.5.7",
@@ -23,6 +25,7 @@
     "@types/cli-progress": "^3.11.6",
     "@types/node": "^20.14.10",
     "nx": "19.4.2",
+    "patch-package": "^8.0.0",
     "prettier": "^3.3.2",
     "ts-node": "^10.9.2",
     "tsx": "^4.16.2",