flexi-bench 0.0.0-alpha.4 → 0.2.0

package/dist/benchmark.js

@@ -1,42 +1,40 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Benchmark = void 0;
 const benchmark_console_reporter_1 = require("./reporters/benchmark-console-reporter");
 const variation_1 = require("./variation");
+const api_types_1 = require("./api-types");
 const child_process_1 = require("child_process");
 const results_1 = require("./results");
 const performance_observer_1 = require("./performance-observer");
 const shared_api_1 = require("./shared-api");
 class Benchmark extends shared_api_1.BenchmarkBase {
+    name;
+    variations = [];
+    iterations;
+    timeout;
+    reporter;
+    watcher;
+    errorStrategy = api_types_1.ErrorStrategy.Continue;
     constructor(name, options) {
         super();
         this.name = name;
-        this.variations = [];
-        if (options === null || options === void 0 ? void 0 : options.action) {
+        if (options?.action) {
             this.action = options.action;
         }
-        if (options === null || options === void 0 ? void 0 : options.setup) {
+        if (options?.setup) {
             this.setupMethods.push(options.setup);
         }
-        if (options === null || options === void 0 ? void 0 : options.teardown) {
+        if (options?.teardown) {
             this.teardownMethods.push(options.teardown);
         }
-        if (options === null || options === void 0 ? void 0 : options.iterations) {
+        if (options?.iterations) {
             this.iterations = options.iterations;
         }
-        if (options === null || options === void 0 ? void 0 : options.timeout) {
+        if (options?.timeout) {
             this.timeout = options.timeout;
         }
-        this.reporter = (options === null || options === void 0 ? void 0 : options.reporter) || new benchmark_console_reporter_1.BenchmarkConsoleReporter();
+        this.reporter = options?.reporter || new benchmark_console_reporter_1.BenchmarkConsoleReporter();
     }
     withVariation(nameOrVariation, builder) {
         if (nameOrVariation instanceof variation_1.Variation) {
@@ -83,117 +81,129 @@ class Benchmark extends shared_api_1.BenchmarkBase {
         this.watcher = new performance_observer_1.PerformanceWatcher(options);
         return this;
     }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
-            this.validate();
-            let results = [];
-            if (this.variations.length === 0) {
-                this.variations.push(new variation_1.Variation('default'));
+    async run() {
+        this.validate();
+        let results = [];
+        if (this.variations.length === 0) {
+            this.variations.push(new variation_1.Variation('default'));
+        }
+        const totalIterations = this.iterations
+            ? this.variations.length * this.iterations
+            : undefined;
+        const startTime = performance.now();
+        let totalCompletedIterations = 0;
+        for (let variationIndex = 0; variationIndex < this.variations.length; variationIndex++) {
+            const variation = this.variations[variationIndex];
+            const variationStartTime = performance.now();
+            const iterationResults = [];
+            // SETUP
+            const oldEnv = { ...process.env };
+            process.env = {
+                ...process.env,
+                ...variation.environment,
+            };
+            for (const setup of this.setupMethods) {
+                await setup(variation);
             }
-            const totalIterations = this.iterations
-                ? this.variations.length * this.iterations
-                : undefined;
-            const startTime = performance.now();
-            let totalCompletedIterations = 0;
-            for (let variationIndex = 0; variationIndex < this.variations.length; variationIndex++) {
-                const variation = this.variations[variationIndex];
-                const timings = [];
-                // SETUP
-                const oldEnv = Object.assign({}, process.env);
-                process.env = Object.assign(Object.assign({}, process.env), variation.environment);
-                for (const setup of this.setupMethods) {
-                    yield setup(variation);
-                }
-                for (const setup of variation.setupMethods) {
-                    yield setup(variation);
-                }
-                // ACT
-                const benchmarkThis = this;
-                yield new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () {
-                    let completedIterations = 0;
-                    let timeout = benchmarkThis.timeout
-                        ? setTimeout(() => {
-                            running = false;
-                            if ((benchmarkThis === null || benchmarkThis === void 0 ? void 0 : benchmarkThis.iterations) &&
-                                completedIterations < benchmarkThis.iterations) {
-                                reject('Timeout');
-                            }
-                            resolve();
-                        }, benchmarkThis.timeout)
-                        : null;
-                    let running = true;
-                    while (running) {
-                        for (const setup of this.setupEachMethods.concat(variation.setupEachMethods)) {
-                            yield setup(variation);
-                        }
-                        const a = performance.now();
-                        if (variation.action) {
-                            yield runAction(variation.action, variation);
-                        }
-                        else if (this.action) {
-                            yield runAction(this.action, variation);
-                        }
-                        const b = performance.now();
-                        for (const teardown of this.teardownEachMethods.concat(variation.teardownEachMethods)) {
-                            yield teardown(variation);
-                        }
-                        const duration = b - a;
-                        completedIterations++;
-                        totalCompletedIterations++;
-                        timings.push(duration);
-                        if (this.reporter.progress &&
-                            totalIterations &&
-                            benchmarkThis.iterations) {
-                            this.reporter.progress(variation.name, totalCompletedIterations / totalIterations, {
-                                timeElapsed: performance.now() - startTime,
-                                totalIterations,
-                                completedIterations: totalCompletedIterations,
-                                timeout: benchmarkThis.timeout,
-                            });
+            for (const setup of variation.setupMethods) {
+                await setup(variation);
+            }
+            // ACT
+            const benchmarkThis = this;
+            await new Promise(async (resolve, reject) => {
+                let completedIterations = 0;
+                let timeout = benchmarkThis.timeout
+                    ? setTimeout(() => {
+                        running = false;
+                        if (benchmarkThis?.iterations &&
+                            completedIterations < benchmarkThis.iterations) {
+                            reject('Timeout');
                         }
-                        if ((benchmarkThis === null || benchmarkThis === void 0 ? void 0 : benchmarkThis.iterations) &&
-                            completedIterations >= benchmarkThis.iterations) {
-                            running = false;
-                            if (timeout) {
-                                clearTimeout(timeout);
-                            }
-                            resolve();
+                        resolve();
+                    }, benchmarkThis.timeout)
+                    : null;
+                let running = true;
+                while (running) {
+                    for (const setup of this.setupEachMethods.concat(variation.setupEachMethods)) {
+                        await setup(variation);
+                    }
+                    const result = await runAndMeasureAction(benchmarkThis, variation);
+                    const errorStrategy = variation.errorStrategy ?? benchmarkThis.errorStrategy;
+                    if (errorStrategy === api_types_1.ErrorStrategy.Abort &&
+                        result instanceof Error) {
+                        reject(result);
+                        return;
+                    }
+                    for (const teardown of this.teardownEachMethods.concat(variation.teardownEachMethods)) {
+                        await teardown(variation);
+                    }
+                    completedIterations++;
+                    totalCompletedIterations++;
+                    iterationResults.push(result);
+                    if (this.reporter.progress &&
+                        totalIterations &&
+                        benchmarkThis.iterations) {
+                        this.reporter.progress(variation.name, totalCompletedIterations / totalIterations, {
+                            timeElapsed: performance.now() - startTime,
+                            totalIterations,
+                            completedIterations: totalCompletedIterations,
+                            timeout: benchmarkThis.timeout,
+                        });
+                    }
+                    if (benchmarkThis?.iterations &&
+                        completedIterations >= benchmarkThis.iterations) {
+                        running = false;
+                        if (timeout) {
+                            clearTimeout(timeout);
                         }
+                        resolve();
                     }
-                }));
-                if (this.reporter.progress && !totalIterations) {
-                    this.reporter.progress(variation.name, variationIndex / this.variations.length, {
-                        timeElapsed: performance.now() - startTime,
-                        timeout: this.timeout,
-                        completedIterations: totalCompletedIterations,
-                    });
                 }
-                // TEARDOWN
-                for (const teardown of variation.teardownMethods) {
-                    yield teardown(variation);
-                }
-                for (const teardown of this.teardownMethods) {
-                    yield teardown(variation);
-                }
-                process.env = oldEnv;
-                // REPORT
-                const result = (0, results_1.calculateResultsFromDurations)(variation.name, timings);
-                // PerformanceObserver needs a chance to flush
-                if (this.watcher) {
-                    const measures = yield this.watcher.getMeasures();
-                    for (const key in measures) {
-                        (_a = result.subresults) !== null && _a !== void 0 ? _a : (result.subresults = []);
-                        result.subresults.push((0, results_1.calculateResultsFromDurations)(key, measures[key]));
-                    }
-                    this.watcher.clearMeasures();
+            });
+            if (this.reporter.progress && !totalIterations) {
+                this.reporter.progress(variation.name, variationIndex / this.variations.length, {
+                    timeElapsed: performance.now() - startTime,
+                    timeout: this.timeout,
+                    completedIterations: totalCompletedIterations,
+                });
+            }
+            // TEARDOWN
+            for (const teardown of variation.teardownMethods) {
+                await teardown(variation);
+            }
+            for (const teardown of this.teardownMethods) {
+                await teardown(variation);
+            }
+            process.env = oldEnv;
+            // REPORT
+            const variationEndTime = performance.now();
+            const result = (0, results_1.calculateResultsFromDurations)(variation.name, iterationResults, {
+                iterations: iterationResults.length,
+                totalDuration: variationEndTime - variationStartTime,
+                benchmarkName: this.name,
+                variationName: variation.name,
+            });
+            // PerformanceObserver needs a chance to flush
+            if (this.watcher) {
+                const measures = await this.watcher.getMeasures();
+                for (const key in measures) {
+                    result.subresults ??= [];
+                    result.subresults.push((0, results_1.calculateResultsFromDurations)(key, measures[key], {
+                        benchmarkName: this.name,
+                        variationName: variation.name,
+                    }));
                 }
-                results.push(result);
+                this.watcher.clearMeasures();
             }
-            (_b = this.watcher) === null || _b === void 0 ? void 0 : _b.disconnect();
-            this.reporter.report(this, results);
-            return results;
-        });
+            results.push(result);
+        }
+        this.watcher?.disconnect();
+        this.reporter.report(this, results);
+        if (this.errorStrategy === api_types_1.ErrorStrategy.DelayedThrow &&
+            results.some((r) => r.failed)) {
+            throw new api_types_1.AggregateBenchmarkError(results);
+        }
+        return results;
     }
     validate() {
         if (!this.timeout && !this.iterations) {
@@ -224,26 +234,35 @@ class Benchmark extends shared_api_1.BenchmarkBase {
     }
 }
 exports.Benchmark = Benchmark;
-function runAction(action, variation) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (typeof action === 'string') {
-            return new Promise((resolve, reject) => {
-                const child = (0, child_process_1.spawn)(action, variation.cliArgs, {
-                    shell: true,
-                    windowsHide: true,
-                });
-                child.on('exit', (code) => {
-                    if (code === 0) {
-                        resolve();
-                    }
-                    else {
-                        reject(`Action failed with code ${code}`);
-                    }
-                });
+async function runAndMeasureAction(benchmark, variation) {
+    try {
+        const a = performance.now();
+        await runAction((variation.action ?? benchmark.action), variation);
+        const b = performance.now();
+        return b - a;
+    }
+    catch (e) {
+        return e instanceof Error ? e : new Error('Unknown error during action.');
+    }
+}
+async function runAction(action, variation) {
+    if (typeof action === 'string') {
+        return new Promise((resolve, reject) => {
+            const child = (0, child_process_1.spawn)(action, variation.cliArgs, {
+                shell: true,
+                windowsHide: true,
             });
-        }
-        else {
-            return action(variation);
-        }
-    });
+            child.on('exit', (code) => {
+                if (code === 0) {
+                    resolve();
+                }
+                else {
+                    reject(`Action failed with code ${code}`);
+                }
+            });
+        });
+    }
+    else {
+        return action(variation);
+    }
 }

package/dist/index.d.ts

@@ -1,9 +1,14 @@
 export * from './reporters/benchmark-console-reporter';
 export * from './reporters/markdown-benchmark-reporter';
+export * from './reporters/markdown-suite-reporter';
 export * from './reporters/suite-console-reporter';
+export * from './reporters/noop-reporter';
+export * from './reporters/composite-reporter';
+export * from './reporters/json-suite-reporter';
 export * from './api-types';
 export * from './benchmark';
 export * from './variation';
 export * from './suite';
 export * from './performance-observer';
 export * from './benchmark-runner';
+export * from './results';

package/dist/index.js

@@ -16,10 +16,15 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./reporters/benchmark-console-reporter"), exports);
 __exportStar(require("./reporters/markdown-benchmark-reporter"), exports);
+__exportStar(require("./reporters/markdown-suite-reporter"), exports);
 __exportStar(require("./reporters/suite-console-reporter"), exports);
+__exportStar(require("./reporters/noop-reporter"), exports);
+__exportStar(require("./reporters/composite-reporter"), exports);
+__exportStar(require("./reporters/json-suite-reporter"), exports);
 __exportStar(require("./api-types"), exports);
 __exportStar(require("./benchmark"), exports);
 __exportStar(require("./variation"), exports);
 __exportStar(require("./suite"), exports);
 __exportStar(require("./performance-observer"), exports);
 __exportStar(require("./benchmark-runner"), exports);
+__exportStar(require("./results"), exports);

package/dist/performance-observer.js

@@ -1,27 +1,17 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PerformanceWatcher = void 0;
 const node_perf_hooks_1 = require("node:perf_hooks");
 class PerformanceWatcher {
+    observer;
+    measures = {};
     constructor(opts = {}) {
-        this.measures = {};
         this.observer = new node_perf_hooks_1.PerformanceObserver((list) => {
-            var _a;
-            var _b;
             for (const entry of list.getEntries()) {
                 if (entry.entryType === 'measure' &&
                     (!opts.measureFilter || opts.measureFilter(entry))) {
-                    const label = normalizeLabel(entry.name, opts === null || opts === void 0 ? void 0 : opts.label);
-                    (_a = (_b = this.measures)[label]) !== null && _a !== void 0 ? _a : (_b[label] = []);
+                    const label = normalizeLabel(entry.name, opts?.label);
+                    this.measures[label] ??= [];
                     this.measures[label].push(entry.duration);
                 }
             }
@@ -33,10 +23,8 @@ class PerformanceWatcher {
     // not fire until the next tick so we need that to happen before the measures are
     // retrieved. Since this method is async, it must be awaited, which actually gives
     // the observer time to collect the measures.
-    getMeasures() {
-        return __awaiter(this, void 0, void 0, function* () {
-            return this.measures;
-        });
+    async getMeasures() {
+        return this.measures;
     }
     clearMeasures() {
         for (const key in this.measures) {
@@ -44,8 +32,7 @@ class PerformanceWatcher {
         }
     }
     disconnect() {
-        var _a;
-        (_a = this.observer) === null || _a === void 0 ? void 0 : _a.disconnect();
+        this.observer?.disconnect();
         delete this.observer;
     }
 }

package/dist/reporters/benchmark-console-reporter.d.ts

@@ -3,8 +3,11 @@ import { Benchmark } from '../benchmark';
 import { SingleBar } from 'cli-progress';
 import { Result } from '../results';
 export declare class BenchmarkConsoleReporter implements BenchmarkReporter {
+    private noColor;
     bar: SingleBar;
-    constructor();
+    constructor(opts?: {
+        noColor?: boolean;
+    });
     progress(name: string, percent: number, context: ProgressContext): void;
     report(benchmark: Benchmark, results: Result[]): void;
 }

package/dist/reporters/benchmark-console-reporter.js

@@ -2,27 +2,40 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BenchmarkConsoleReporter = void 0;
 const cli_progress_1 = require("cli-progress");
+function getNoColorOption(explicitNoColor) {
+    if (explicitNoColor !== undefined) {
+        return explicitNoColor;
+    }
+    return process.env.NO_COLOR !== undefined && process.env.NO_COLOR !== '';
+}
 class BenchmarkConsoleReporter {
-    constructor() {
+    noColor;
+    bar;
+    constructor(opts) {
+        this.noColor = getNoColorOption(opts?.noColor);
         this.bar = new cli_progress_1.SingleBar({
-            format: 'Running variation {label}: {bar} {percentage}% | {value}/{total} - ETA: {eta}s',
-            barCompleteChar: '\u2588',
-            barIncompleteChar: '\u2591',
+            format: this.noColor
+                ? 'Running variation {label}: [{bar}] {percentage}% | {value}/{total} - ETA: {eta}s'
+                : 'Running variation {label}: {bar} {percentage}% | {value}/{total} - ETA: {eta}s',
+            barCompleteChar: this.noColor ? '#' : '\u2588',
+            barIncompleteChar: this.noColor ? '-' : '\u2591',
             hideCursor: true,
             stopOnComplete: true,
             clearOnComplete: true,
         });
     }
     progress(name, percent, context) {
-        var _a;
         if (!this.bar.isActive) {
-            this.bar.start((_a = context.totalIterations) !== null && _a !== void 0 ? _a : 100, 0);
+            this.bar.start(context.totalIterations ?? 100, 0);
         }
         this.bar.update(context.completedIterations, { label: name });
     }
     report(benchmark, results) {
+        const tableEntries = results.map(({ raw, ...rest }) => ({
+            ...rest,
+        }));
         console.log(`Benchmark: ${benchmark.name}`);
-        console.table(results);
+        console.table(tableEntries);
     }
 }
 exports.BenchmarkConsoleReporter = BenchmarkConsoleReporter;

package/dist/reporters/composite-reporter.d.ts

@@ -0,0 +1,21 @@
+import { SuiteReporter } from '../api-types';
+import { Result } from '../results';
+/**
+ * A reporter that chains multiple suite reporters together.
+ * Useful when you want to output results to multiple destinations
+ * (e.g., console and file) simultaneously.
+ *
+ * @example
+ * ```typescript
+ * suite.withReporter(new CompositeReporter([
+ *   new SuiteConsoleReporter(),
+ *   new MarkdownSuiteReporter({ outputFile: 'results.md' }),
+ *   new JsonSuiteReporter({ outputFile: 'results.json' }),
+ * ]))
+ * ```
+ */
+export declare class CompositeReporter implements SuiteReporter {
+    private reporters;
+    constructor(reporters: SuiteReporter[]);
+    report: (results: Record<string, Result[]>) => void;
+}

package/dist/reporters/composite-reporter.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CompositeReporter = void 0;
+/**
+ * A reporter that chains multiple suite reporters together.
+ * Useful when you want to output results to multiple destinations
+ * (e.g., console and file) simultaneously.
+ *
+ * @example
+ * ```typescript
+ * suite.withReporter(new CompositeReporter([
+ *   new SuiteConsoleReporter(),
+ *   new MarkdownSuiteReporter({ outputFile: 'results.md' }),
+ *   new JsonSuiteReporter({ outputFile: 'results.json' }),
+ * ]))
+ * ```
+ */
+class CompositeReporter {
+    reporters;
+    constructor(reporters) {
+        this.reporters = reporters;
+    }
+    report = (results) => {
+        for (const reporter of this.reporters) {
+            reporter.report(results);
+        }
+    };
+}
+exports.CompositeReporter = CompositeReporter;

package/dist/reporters/json-suite-reporter.d.ts

@@ -0,0 +1,26 @@
+import { SuiteReporter } from '../api-types';
+import { Result } from '../results';
+export interface JsonSuiteReporterOptions {
+    outputFile: string;
+    pretty?: boolean;
+    includeMetadata?: boolean;
+}
+/**
+ * A reporter that outputs benchmark results as JSON.
+ * Useful for CI/CD integration and programmatic analysis.
+ *
+ * @example
+ * ```typescript
+ * suite.withReporter(new JsonSuiteReporter({
+ *   outputFile: 'results.json',
+ *   pretty: true,
+ * }))
+ * ```
+ */
+export declare class JsonSuiteReporter implements SuiteReporter {
+    private outputFile;
+    private pretty;
+    private includeMetadata;
+    constructor(opts: JsonSuiteReporterOptions);
+    report: (results: Record<string, Result[]>) => void;
+}

package/dist/reporters/json-suite-reporter.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JsonSuiteReporter = void 0;
+const fs_1 = require("fs");
+/**
+ * A reporter that outputs benchmark results as JSON.
+ * Useful for CI/CD integration and programmatic analysis.
+ *
+ * @example
+ * ```typescript
+ * suite.withReporter(new JsonSuiteReporter({
+ *   outputFile: 'results.json',
+ *   pretty: true,
+ * }))
+ * ```
+ */
+class JsonSuiteReporter {
+    outputFile;
+    pretty;
+    includeMetadata;
+    constructor(opts) {
+        this.outputFile = opts.outputFile;
+        this.pretty = opts.pretty ?? false;
+        this.includeMetadata = opts.includeMetadata ?? true;
+    }
+    report = (results) => {
+        const output = this.includeMetadata
+            ? {
+                timestamp: new Date().toISOString(),
+                platform: process.platform,
+                nodeVersion: process.version,
+                results,
+            }
+            : results;
+        (0, fs_1.writeFileSync)(this.outputFile, JSON.stringify(output, null, this.pretty ? 2 : undefined));
+    };
+}
+exports.JsonSuiteReporter = JsonSuiteReporter;

package/dist/reporters/markdown-benchmark-reporter.d.ts

@@ -3,10 +3,22 @@ import { Benchmark } from '../benchmark';
 import { Result } from '../results';
 export declare class MarkdownBenchmarkReporter implements BenchmarkReporter {
     outputFile: string;
-    fields: Array<keyof Omit<Result, 'subresults' | 'label'>>;
+    fields: Array<keyof Omit<Result, 'subresults' | 'label' | 'raw'>>;
+    append: boolean;
+    private isFirstReport;
+    private accumulatedResults;
     constructor(opts: {
         outputFile: string;
         fields?: MarkdownBenchmarkReporter['fields'];
+        append?: boolean;
     });
+    private formatDuration;
     report: (benchmark: Benchmark, results: Result[]) => void;
+    private generateBenchmarkContent;
+    private generateResultsTable;
+    private generateComparison;
+    /**
+     * Clears the output file. Useful when re-running benchmarks.
+     */
+    clear(): void;
 }