flexi-bench 0.0.0-alpha.4 → 0.2.0

package/dist/reporters/markdown-benchmark-reporter.js

@@ -4,26 +4,162 @@ exports.MarkdownBenchmarkReporter = void 0;
 const fs_1 = require("fs");
 const markdown_factory_1 = require("markdown-factory");
 class MarkdownBenchmarkReporter {
+    outputFile;
+    fields;
+    append;
+    isFirstReport = true;
+    accumulatedResults = new Map();
     constructor(opts) {
-        var _a;
-        this.report = (benchmark, results) => {
-            (0, fs_1.writeFileSync)(this.outputFile, (0, markdown_factory_1.h1)(benchmark.name, results.some((r) => !!r.subresults)
-                ? (0, markdown_factory_1.lines)(results.map((r) => {
-                    var _a;
-                    const entries = [Object.assign(Object.assign({}, r), { label: 'total' })];
-                    delete entries[0].subresults;
-                    for (const subresult of (_a = r.subresults) !== null && _a !== void 0 ? _a : []) {
-                        entries.push(subresult);
-                    }
-                    return (0, markdown_factory_1.h2)(r.label, (0, markdown_factory_1.table)(entries, [
-                        { field: 'label', label: '' },
-                        ...this.fields,
-                    ]));
-                }))
-                : (0, markdown_factory_1.table)(results, [{ field: 'label', label: '' }, ...this.fields])));
-        };
         this.outputFile = opts.outputFile;
-        this.fields = (_a = opts.fields) !== null && _a !== void 0 ? _a : ['min', 'average', 'p95', 'max'];
+        this.fields = opts.fields ?? ['min', 'average', 'p95', 'max'];
+        this.append = opts.append ?? false;
+    }
+    formatDuration(value) {
+        if (value === undefined)
+            return '';
+        const totalMs = value;
+        const hours = Math.floor(totalMs / (1000 * 60 * 60));
+        const remainingAfterHours = totalMs % (1000 * 60 * 60);
+        const minutes = Math.floor(remainingAfterHours / (1000 * 60));
+        const remainingAfterMinutes = remainingAfterHours % (1000 * 60);
+        const seconds = Math.floor(remainingAfterMinutes / 1000);
+        const milliseconds = remainingAfterMinutes % 1000;
+        const parts = [];
+        if (hours > 0) {
+            parts.push(`${hours}h`);
+            if (minutes > 0)
+                parts.push(`${minutes}m`);
+            if (seconds > 0 || milliseconds > 0) {
+                const totalSeconds = seconds + milliseconds / 1000;
+                parts.push(`${totalSeconds.toFixed(1)}s`);
+            }
+        }
+        else if (minutes > 0) {
+            parts.push(`${minutes}m`);
+            if (seconds > 0 || milliseconds > 0) {
+                const totalSeconds = seconds + milliseconds / 1000;
+                parts.push(`${totalSeconds.toFixed(1)}s`);
+            }
+        }
+        else if (seconds > 0) {
+            const totalSeconds = seconds + milliseconds / 1000;
+            parts.push(`${totalSeconds.toFixed(1)}s`);
+        }
+        else {
+            parts.push(`${milliseconds.toFixed(1)}ms`);
+        }
+        return parts.join(' ');
+    }
+    report = (benchmark, results) => {
+        if (this.append) {
+            // In append mode, accumulate results and write incrementally
+            this.accumulatedResults.set(benchmark.name, results);
+            const content = this.generateBenchmarkContent(benchmark, results);
+            if (this.isFirstReport) {
+                // First report: clear the file and write header
+                (0, fs_1.writeFileSync)(this.outputFile, content);
+                this.isFirstReport = false;
+            }
+            else {
+                // Subsequent reports: append with a separator
+                (0, fs_1.appendFileSync)(this.outputFile, '\n\n---\n\n' + content);
+            }
+        }
+        else {
+            // Legacy mode: overwrite file (useful for single benchmark reporting)
+            (0, fs_1.writeFileSync)(this.outputFile, this.generateBenchmarkContent(benchmark, results));
+        }
+    };
+    generateBenchmarkContent(benchmark, results) {
+        const sections = [];
+        // Main results section
+        sections.push(this.generateResultsTable(results));
+        // Comparison section (if multiple variations)
+        if (results.length > 1) {
+            sections.push(this.generateComparison(results));
+        }
+        return (0, markdown_factory_1.h1)(benchmark.name, (0, markdown_factory_1.lines)(sections));
+    }
+    generateResultsTable(results) {
+        const fieldConfigs = [
+            { field: 'label', label: '' },
+            ...this.fields.map((field) => ({
+                field: field,
+                label: field,
+                mapFn: (item) => {
+                    const value = item[field];
+                    if (field === 'iterations') {
+                        return String(value ?? '');
+                    }
+                    if (typeof value === 'number') {
+                        return this.formatDuration(value);
+                    }
+                    return String(value ?? '');
+                },
+            })),
+        ];
+        if (results.some((r) => !!r.subresults)) {
+            return (0, markdown_factory_1.lines)(results.map((r) => {
+                const entries = [{ ...r, label: 'total' }];
+                delete entries[0].subresults;
+                for (const subresult of r.subresults ?? []) {
+                    entries.push(subresult);
+                }
+                return (0, markdown_factory_1.h2)(r.label, (0, markdown_factory_1.table)(entries, fieldConfigs));
+            }));
+        }
+        return (0, markdown_factory_1.table)(results, fieldConfigs);
+    }
+    generateComparison(results) {
+        const sorted = [...results].sort((a, b) => a.average - b.average);
+        const fastest = sorted[0];
+        const hasVaryingIterations = results.some((r) => r.iterations !== results[0].iterations);
+        const entries = sorted.map((result, index) => {
+            const entry = {
+                label: result.label,
+                indicator: index === 0 ? '🏆' : '',
+            };
+            for (const field of this.fields) {
+                if (field === 'iterations') {
+                    if (hasVaryingIterations) {
+                        entry.iterations = String(result.iterations);
+                    }
+                }
+                else {
+                    const value = result[field];
+                    const fastestValue = fastest[field];
+                    if (index === 0) {
+                        entry[field] = 'baseline';
+                    }
+                    else {
+                        const factor = value / fastestValue;
+                        entry[field] = `${factor.toFixed(2)}x slower`;
+                    }
+                }
+            }
+            return entry;
+        });
+        const columns = [
+            { field: 'label', label: 'Variation' },
+            ...this.fields
+                .filter((field) => field !== 'iterations' || hasVaryingIterations)
+                .map((field) => ({ field, label: field })),
+            { field: 'indicator', label: '' },
+        ];
+        const header = hasVaryingIterations
+            ? 'Comparison'
+            : `Comparison (${results[0].iterations} iterations)`;
+        return (0, markdown_factory_1.h3)(header, (0, markdown_factory_1.table)(entries.map((e) => ({ ...e })), columns));
+    }
+    /**
+     * Clears the output file. Useful when re-running benchmarks.
+     */
+    clear() {
+        if ((0, fs_1.existsSync)(this.outputFile)) {
+            (0, fs_1.unlinkSync)(this.outputFile);
+        }
+        this.isFirstReport = true;
+        this.accumulatedResults.clear();
     }
 }
 exports.MarkdownBenchmarkReporter = MarkdownBenchmarkReporter;

package/dist/reporters/markdown-suite-reporter.d.ts

@@ -0,0 +1,16 @@
+import { SuiteReporter } from '../api-types';
+import { Result } from '../results';
+export declare class MarkdownSuiteReporter implements SuiteReporter {
+    outputFile: string;
+    title: string;
+    fields: Array<keyof Omit<Result, 'subresults' | 'label' | 'raw'>>;
+    constructor(opts: {
+        outputFile: string;
+        title?: string;
+        fields?: MarkdownSuiteReporter['fields'];
+    });
+    private formatDuration;
+    report: (results: Record<string, Result[]>) => void;
+    private renderBenchmarkSection;
+    private generateComparison;
+}

package/dist/reporters/markdown-suite-reporter.js

@@ -0,0 +1,140 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarkdownSuiteReporter = void 0;
+const fs_1 = require("fs");
+const markdown_factory_1 = require("markdown-factory");
+class MarkdownSuiteReporter {
+    outputFile;
+    title;
+    fields;
+    constructor(opts) {
+        this.outputFile = opts.outputFile;
+        this.title = opts.title ?? 'Benchmark Results';
+        this.fields = opts.fields ?? ['min', 'average', 'p95', 'max'];
+    }
+    formatDuration(value) {
+        if (value === undefined)
+            return '';
+        const totalMs = value;
+        const hours = Math.floor(totalMs / (1000 * 60 * 60));
+        const remainingAfterHours = totalMs % (1000 * 60 * 60);
+        const minutes = Math.floor(remainingAfterHours / (1000 * 60));
+        const remainingAfterMinutes = remainingAfterHours % (1000 * 60);
+        const seconds = Math.floor(remainingAfterMinutes / 1000);
+        const milliseconds = remainingAfterMinutes % 1000;
+        const parts = [];
+        if (hours > 0) {
+            parts.push(`${hours}h`);
+            if (minutes > 0)
+                parts.push(`${minutes}m`);
+            if (seconds > 0 || milliseconds > 0) {
+                const totalSeconds = seconds + milliseconds / 1000;
+                parts.push(`${totalSeconds.toFixed(1)}s`);
+            }
+        }
+        else if (minutes > 0) {
+            parts.push(`${minutes}m`);
+            if (seconds > 0 || milliseconds > 0) {
+                const totalSeconds = seconds + milliseconds / 1000;
+                parts.push(`${totalSeconds.toFixed(1)}s`);
+            }
+        }
+        else if (seconds > 0) {
+            const totalSeconds = seconds + milliseconds / 1000;
+            parts.push(`${totalSeconds.toFixed(1)}s`);
+        }
+        else {
+            parts.push(`${milliseconds.toFixed(1)}ms`);
+        }
+        return parts.join(' ');
+    }
+    report = (results) => {
+        const benchmarkSections = Object.entries(results).map(([benchmarkName, benchmarkResults]) => {
+            return this.renderBenchmarkSection(benchmarkName, benchmarkResults);
+        });
+        (0, fs_1.writeFileSync)(this.outputFile, (0, markdown_factory_1.h1)(this.title, (0, markdown_factory_1.lines)(benchmarkSections)));
+    };
+    renderBenchmarkSection(benchmarkName, results) {
+        const hasSubresults = results.some((r) => !!r.subresults);
+        const hasMultipleVariations = results.length > 1;
+        const sections = [];
+        const fieldConfigs = [
+            { field: 'label', label: '' },
+            ...this.fields.map((field) => ({
+                field: field,
+                label: field,
+                mapFn: (item) => {
+                    const value = item[field];
+                    if (field === 'iterations') {
+                        return String(value ?? '');
+                    }
+                    if (typeof value === 'number') {
+                        return this.formatDuration(value);
+                    }
+                    return String(value ?? '');
+                },
+            })),
+        ];
+        // Main results table
+        if (hasSubresults) {
+            sections.push((0, markdown_factory_1.lines)(results.map((r) => {
+                const entries = [{ ...r, label: 'total' }];
+                delete entries[0].subresults;
+                for (const subresult of r.subresults ?? []) {
+                    entries.push(subresult);
+                }
+                return (0, markdown_factory_1.h2)(r.label, (0, markdown_factory_1.table)(entries, fieldConfigs));
+            })));
+        }
+        else {
+            sections.push((0, markdown_factory_1.table)(results, fieldConfigs));
+        }
+        // Comparison section (if multiple variations)
+        if (hasMultipleVariations) {
+            sections.push(this.generateComparison(results));
+        }
+        return (0, markdown_factory_1.h2)(benchmarkName, (0, markdown_factory_1.lines)(sections));
+    }
+    generateComparison(results) {
+        const sorted = [...results].sort((a, b) => a.average - b.average);
+        const fastest = sorted[0];
+        const hasVaryingIterations = results.some((r) => r.iterations !== results[0].iterations);
+        const entries = sorted.map((result, index) => {
+            const entry = {
+                label: result.label,
+                indicator: index === 0 ? '🏆' : '',
+            };
+            for (const field of this.fields) {
+                if (field === 'iterations') {
+                    if (hasVaryingIterations) {
+                        entry.iterations = String(result.iterations);
+                    }
+                }
+                else {
+                    const value = result[field];
+                    const fastestValue = fastest[field];
+                    if (index === 0) {
+                        entry[field] = 'baseline';
+                    }
+                    else {
+                        const factor = value / fastestValue;
+                        entry[field] = `${factor.toFixed(2)}x slower`;
+                    }
+                }
+            }
+            return entry;
+        });
+        const columns = [
+            { field: 'label', label: 'Variation' },
+            ...this.fields
+                .filter((field) => field !== 'iterations' || hasVaryingIterations)
+                .map((field) => ({ field, label: field })),
+            { field: 'indicator', label: '' },
+        ];
+        const header = hasVaryingIterations
+            ? 'Comparison'
+            : `Comparison (${results[0].iterations} iterations)`;
+        return (0, markdown_factory_1.h3)(header, (0, markdown_factory_1.table)(entries.map((e) => ({ ...e })), columns));
+    }
+}
+exports.MarkdownSuiteReporter = MarkdownSuiteReporter;

package/dist/reporters/noop-reporter.js

@@ -2,8 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.NoopReporter = void 0;
 class NoopReporter {
-    constructor() {
-        this.report = () => { };
-    }
+    report = () => { };
 }
 exports.NoopReporter = NoopReporter;

package/dist/reporters/suite-console-reporter.d.ts

@@ -1,5 +1,9 @@
 import { SuiteReporter } from '../api-types';
 import { Result } from '../results';
 export declare class SuiteConsoleReporter implements SuiteReporter {
+    private noColor;
+    constructor(opts?: {
+        noColor?: boolean;
+    });
     report: (results: Record<string, Result[]>) => void;
 }

package/dist/reporters/suite-console-reporter.js

@@ -1,15 +1,31 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SuiteConsoleReporter = void 0;
+function getNoColorOption(explicitNoColor) {
+    if (explicitNoColor !== undefined) {
+        return explicitNoColor;
+    }
+    return process.env.NO_COLOR !== undefined && process.env.NO_COLOR !== '';
+}
 class SuiteConsoleReporter {
-    constructor() {
-        this.report = (results) => {
-            console.log('Suite Results:');
-            for (const [name, result] of Object.entries(results)) {
-                console.log(`Benchmark: ${name}`);
-                console.table(result);
-            }
-        };
+    noColor;
+    constructor(opts) {
+        this.noColor = getNoColorOption(opts?.noColor);
     }
+    report = (results) => {
+        if (this.noColor) {
+            console.log('Suite Results:');
+        }
+        else {
+            console.log('\x1b[1mSuite Results:\x1b[0m');
+        }
+        for (const [name, result] of Object.entries(results)) {
+            const tableEntries = result.map(({ raw, ...rest }) => ({
+                ...rest,
+            }));
+            console.log(`Benchmark: ${name}`);
+            console.table(tableEntries);
+        }
+    };
 }
 exports.SuiteConsoleReporter = SuiteConsoleReporter;

package/dist/results.d.ts

@@ -1,3 +1,8 @@
+export type GitInfo = {
+    head: string;
+    sha: string;
+    dirty: boolean;
+};
 /**
  * A measurement result.
  */
@@ -25,10 +30,44 @@ export type Result = {
     /**
      * The raw durations, in order. Used for custom reporters.
      */
-    raw: number[];
+    raw: (number | Error)[];
+    /**
+     * Whether any run of the benchmark failed.
+     */
+    failed?: boolean;
+    /**
+     * The rate of failure, if any.
+     */
+    failureRate?: number;
     /**
      * Subresults, if any. Typically sourced from performance observer.
      */
     subresults?: Result[];
+    /**
+     * The number of iterations that were run.
+     */
+    iterations?: number;
+    /**
+     * The total wall-clock duration for all iterations in milliseconds.
+     */
+    totalDuration?: number;
+    /**
+     * The name of the benchmark that produced this result.
+     */
+    benchmarkName?: string;
+    /**
+     * The name of the variation that produced this result.
+     */
+    variationName?: string;
+    /**
+     * Git information about the repository state when this result was produced.
+     */
+    git?: GitInfo;
 };
-export declare function calculateResultsFromDurations(label: string, durations: number[]): Result;
+export declare function getGitInfo(): GitInfo | undefined;
+export declare function calculateResultsFromDurations(label: string, durations: (number | Error)[], metadata?: {
+    iterations?: number;
+    totalDuration?: number;
+    benchmarkName?: string;
+    variationName?: string;
+}): Result;

package/dist/results.js

@@ -1,14 +1,71 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.getGitInfo = getGitInfo;
 exports.calculateResultsFromDurations = calculateResultsFromDurations;
-function calculateResultsFromDurations(label, durations) {
-    durations.sort((a, b) => a - b);
+const child_process_1 = require("child_process");
+const is_ci_1 = __importDefault(require("is-ci"));
+let cachedGitInfo = null;
+function getGitInfo() {
+    if (cachedGitInfo !== null) {
+        return cachedGitInfo;
+    }
+    if (!is_ci_1.default) {
+        cachedGitInfo = undefined;
+        return undefined;
+    }
+    const getGitOutput = (args) => {
+        try {
+            const result = (0, child_process_1.spawn)('git', args, {
+                stdio: ['ignore', 'pipe', 'ignore'],
+            });
+            let output = '';
+            result.stdout?.on('data', (data) => {
+                output += data.toString();
+            });
+            result.stdout?.on('end', () => {
+                return output.trim();
+            });
+            return output.trim();
+        }
+        catch {
+            return '';
+        }
+    };
+    const head = getGitOutput(['rev-parse', '--abbrev-ref', 'HEAD']);
+    const sha = getGitOutput(['rev-parse', 'HEAD']);
+    const dirty = getGitOutput(['status', '--porcelain']).length > 0;
+    cachedGitInfo = {
+        head,
+        sha,
+        dirty,
+    };
+    return cachedGitInfo;
+}
+function calculateResultsFromDurations(label, durations, metadata) {
+    const errors = [];
+    const results = [];
+    for (const duration of durations) {
+        if (duration instanceof Error) {
+            errors.push(duration);
+        }
+        else {
+            results.push(duration);
+        }
+    }
+    results.sort((a, b) => a - b);
     return {
         label,
-        min: Math.min(...durations),
-        max: Math.max(...durations),
-        average: durations.reduce((acc, duration) => acc + duration, 0) / durations.length,
-        p95: durations[Math.floor(durations.length * 0.95)],
+        min: Math.min(...results),
+        max: Math.max(...results),
+        average: results.reduce((acc, duration) => acc + duration, 0) / results.length,
+        p95: results[Math.floor(results.length * 0.95)],
         raw: durations,
+        failed: errors.length > 0,
+        failureRate: errors.length / durations.length,
+        ...metadata,
+        ...(getGitInfo() ? { git: getGitInfo() } : {}),
     };
 }

package/dist/shared-api.d.ts

@@ -1,13 +1,15 @@
-import { Action, SetupMethod, TeardownMethod } from './api-types';
+import { Action, ErrorStrategy, SetupMethod, TeardownMethod } from './api-types';
 export declare class BenchmarkBase {
     setupEachMethods: SetupMethod[];
     setupMethods: SetupMethod[];
     teardownMethods: TeardownMethod[];
     teardownEachMethods: TeardownMethod[];
     action?: Action;
+    errorStrategy?: ErrorStrategy;
     withSetup(setup: SetupMethod): this;
     withSetupEach(setup: SetupMethod): this;
     withTeardown(teardown: TeardownMethod): this;
     withTeardownEach(teardown: TeardownMethod): this;
     withAction(action: Action): this;
+    withErrorStrategy(errorStrategy: ErrorStrategy): this;
 }

package/dist/shared-api.js

@@ -2,12 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BenchmarkBase = void 0;
 class BenchmarkBase {
-    constructor() {
-        this.setupEachMethods = [];
-        this.setupMethods = [];
-        this.teardownMethods = [];
-        this.teardownEachMethods = [];
-    }
+    setupEachMethods = [];
+    setupMethods = [];
+    teardownMethods = [];
+    teardownEachMethods = [];
+    action;
+    errorStrategy;
     withSetup(setup) {
         this.setupMethods.push(setup);
         return this;
@@ -28,5 +28,9 @@ class BenchmarkBase {
         this.action = action;
         return this;
     }
+    withErrorStrategy(errorStrategy) {
+        this.errorStrategy = errorStrategy;
+        return this;
+    }
 }
 exports.BenchmarkBase = BenchmarkBase;

package/dist/suite.d.ts

@@ -1,4 +1,4 @@
-import { BenchmarkReporter, SuiteReporter } from './api-types';
+import { BenchmarkReporter, ErrorStrategy, SuiteReporter } from './api-types';
 import { Benchmark } from './benchmark';
 import { Result } from './results';
 import { Variation } from './variation';
@@ -11,11 +11,21 @@ export declare class Suite {
     private variations;
     private reporter;
     private benchmarkReporter?;
+    private errorStrategy;
+    private shouldSetErrorStrategyOnBenchmarks;
     constructor(name: string, options?: SuiteOptions);
     addBenchmark(benchmark: Benchmark): this;
     withVariation(variation: Variation): this;
     withVariations(variations: Variation[]): this;
     withReporter(reporter: SuiteReporter): this;
     withBenchmarkReporter(reporter: BenchmarkReporter): this;
+    /**
+     * Sets the error strategy for the suite and optionally for the benchmarks.
+     * @param errorStrategy Determines what to do when an error occurs during a benchmark run. See {@link ErrorStrategy}
+     * @param opts Options for the error strategy. If `shouldSetOnBenchmarks` is true, the error strategy will be set on all benchmarks in the suite.
+     */
+    withErrorStrategy(errorStrategy: ErrorStrategy, opts?: {
+        shouldSetOnBenchmarks?: boolean;
+    }): this;
     run(): Promise<Record<string, Result[]>>;
 }