flexi-bench 0.0.0-alpha.4 → 0.2.0

package/CHANGELOG.md

@@ -1,3 +1,46 @@
+## 0.2.0 (2026-02-02)
+
+
+### 🚀 Features
+
+- add result metadata support (iterations, totalDuration, benchmarkName, variationName) ([22dfc33](https://github.com/agentender/flexi-bench/commit/22dfc33))
+- improve console reporters with metadata display ([bfb4482](https://github.com/agentender/flexi-bench/commit/bfb4482))
+- add composite and JSON suite reporters with examples ([c29f381](https://github.com/agentender/flexi-bench/commit/c29f381))
+- include git info on results when in CI ([3b2a63d](https://github.com/agentender/flexi-bench/commit/3b2a63d))
+- **docs-site:** upload algolia settings ([aea3eff](https://github.com/agentender/flexi-bench/commit/aea3eff))
+
+### 🩹 Fixes
+
+- **e2e:** ensure output directory exists and improve error handling ([c0e79ea](https://github.com/agentender/flexi-bench/commit/c0e79ea))
+
+### ❤️  Thank You
+
+- Craigory Coppola @AgentEnder
+
+## 0.1.0 (2024-07-28)
+
+
+### 🚀 Features
+
+- add different error strategies for handling failing actions ([#11](https://github.com/AgentEnder/flexi-bench/pull/11))
+- **repo:** add ability to publish only docs ([4cde432](https://github.com/AgentEnder/flexi-bench/commit/4cde432))
+
+### ❤️  Thank You
+
+- Craigory Coppola @AgentEnder
+
+# 0.0.0 (2024-07-28)
+
+
+### 🚀 Features
+
+- add different error strategies for handling failing actions ([#11](https://github.com/AgentEnder/flexi-bench/pull/11))
+- **repo:** add ability to publish only docs ([4cde432](https://github.com/AgentEnder/flexi-bench/commit/4cde432))
+
+### ❤️  Thank You
+
+- Craigory Coppola @AgentEnder
+
 ## 0.0.0-alpha.4 (2024-07-17)
 
 

package/README.md

@@ -332,6 +332,50 @@ const benchmark = new Benchmark('My Benchmark', {
 );
 ```
 
+#### Variation Context
+
+For more complex scenarios where you need to pass objects or data directly to the benchmark action, use the context API instead of environment variables:
+
+```javascript
+const { Benchmark, Variation } = require('flexi-bench');
+
+// Define different implementations
+const loopProcessor = {
+  process: (data) => {
+    /* loop */
+  },
+};
+const reduceProcessor = {
+  process: (data) => {
+    /* reduce */
+  },
+};
+
+// Use FromContexts for clean, declarative variation setup
+const benchmark = new Benchmark('Process Data')
+  .withIterations(100)
+  .withVariations(
+    Variation.FromContexts('processor', [
+      ['loop', loopProcessor],
+      ['reduce', reduceProcessor],
+    ]),
+  )
+  .withAction((variation) => {
+    // Use get() to retrieve context data
+    const processor = variation.get('processor');
+    processor.process(data);
+  });
+```
+
+The context API provides:
+
+- `Variation.FromContexts(key, [[name, value], ...])` - Create variations with context (cleanest API)
+- `withContext(key, value)` - Attach custom data to a single variation
+- `get(key)` - Retrieve context data (returns `T | undefined`)
+- `getOrDefault(key, defaultValue)` - Retrieve with fallback value
+
+Use `FromContexts` when creating multiple variations with the same context key - it's cleaner and more concise than individual `withVariation` calls with `withContext`.
+
 Variations can also be added to suites. Variations added to a suite will be applied to all benchmarks in the suite.
 
 For example, the below suite would run each benchmark with 'NO_DAEMON' set to true, and then with 'OTHER_VAR' set to 'value1' for a total of 4 benchmark runs in the suite:
@@ -364,10 +408,239 @@ const suite = new Suite('My Suite')
   );
 ```
 
+## Reporters
+
+Reporters control how benchmark results are output. FlexiBench provides several built-in reporters:
+
+### Console Reporters
+
+```javascript
+const {
+  Benchmark,
+  BenchmarkConsoleReporter,
+  SuiteConsoleReporter,
+} = require('flexi-bench');
+
+// For single benchmarks
+const benchmark = new Benchmark('My Benchmark', {
+  iterations: 10,
+  action: () => {
+    /* ... */
+  },
+  reporter: new BenchmarkConsoleReporter(),
+});
+
+// For suites
+const suite = new Suite('My Suite')
+  .withReporter(new SuiteConsoleReporter())
+  .addBenchmark(benchmark);
+```
+
+Both console reporters support the `NO_COLOR` environment variable for disabling colors:
+
+```bash
+NO_COLOR=1 node my-benchmark.js
+```
+
+Or explicitly via options:
+
+```javascript
+new SuiteConsoleReporter({ noColor: true });
+new BenchmarkConsoleReporter({ noColor: true });
+```
+
+### Markdown Reporters
+
+```javascript
+const {
+  MarkdownBenchmarkReporter,
+  MarkdownSuiteReporter,
+} = require('flexi-bench');
+
+// For single benchmark output
+const benchmarkReporter = new MarkdownBenchmarkReporter({
+  outputFile: 'results.md',
+  fields: ['min', 'average', 'p95', 'max'],
+  append: true, // Set to true to avoid overwriting when running multiple benchmarks
+});
+
+// For suite-level output (recommended)
+const suiteReporter = new MarkdownSuiteReporter({
+  outputFile: 'results.md',
+  title: 'Benchmark Results',
+  fields: ['min', 'average', 'p95', 'max', 'iterations'],
+});
+```
+
+#### Automatic Comparison Tables
+
+When a benchmark has multiple variations, both `MarkdownBenchmarkReporter` and `MarkdownSuiteReporter` automatically generate a comparison table showing:
+
+- Average time for each variation
+- Percentage difference vs the fastest variation
+- Multiplier (e.g., "2.5x" slower)
+- Trophy emoji (🏆) marking the fastest variation
+
+This makes it easy to see which implementation performs best at a glance.
+
+### JSON Reporter
+
+For CI/CD integration:
+
+```javascript
+const { JsonSuiteReporter } = require('flexi-bench');
+
+const reporter = new JsonSuiteReporter({
+  outputFile: 'results.json',
+  pretty: true,
+  includeMetadata: true, // Includes timestamp, platform, node version
+});
+```
+
+### Composite Reporter
+
+Use multiple reporters simultaneously:
+
+```javascript
+const {
+  CompositeReporter,
+  SuiteConsoleReporter,
+  MarkdownSuiteReporter,
+  JsonSuiteReporter,
+} = require('flexi-bench');
+
+const suite = new Suite('My Suite').withReporter(
+  new CompositeReporter([
+    new SuiteConsoleReporter(),
+    new MarkdownSuiteReporter({ outputFile: 'results.md' }),
+    new JsonSuiteReporter({ outputFile: 'results.json' }),
+  ]),
+);
+```
+
+### Custom Reporters
+
+Create custom reporters by implementing the `SuiteReporter` interface:
+
+```typescript
+import { SuiteReporter, Result } from 'flexi-bench';
+
+class MyCustomReporter implements SuiteReporter {
+  // Optional lifecycle hooks
+  onSuiteStart?(suiteName: string): void {
+    console.log(`Starting suite: ${suiteName}`);
+  }
+
+  onBenchmarkStart?(benchmarkName: string): void {
+    console.log(`Running: ${benchmarkName}`);
+  }
+
+  onBenchmarkEnd?(benchmarkName: string, results: Result[]): void {
+    console.log(`Completed: ${benchmarkName}`);
+  }
+
+  // Required: called after all benchmarks complete
+  report(results: Record<string, Result[]>): void {
+    // Process results here
+    for (const [name, result] of Object.entries(results)) {
+      console.log(`${name}: ${result[0].average}ms average`);
+    }
+  }
+}
+```
+
+## Result Type
+
+The `Result` type contains comprehensive information about benchmark runs:
+
+```typescript
+interface Result {
+  // Basic metrics
+  label: string; // Name of benchmark or variation
+  min: number; // Minimum duration (ms)
+  max: number; // Maximum duration (ms)
+  average: number; // Average duration (ms)
+  p95: number; // 95th percentile duration (ms)
+  raw: (number | Error)[]; // Raw durations/errors
+
+  // Failure information
+  failed?: boolean; // Whether any iteration failed
+  failureRate?: number; // Rate of failures (0-1)
+
+  // Metadata (useful for custom reporters)
+  iterations?: number; // Number of iterations run
+  totalDuration?: number; // Total wall-clock time (ms)
+  benchmarkName?: string; // Name of parent benchmark
+  variationName?: string; // Name of variation
+
+  // Subresults from performance observer
+  subresults?: Result[];
+}
+```
+
+The Result type is exported from the main package:
+
+```typescript
+import { Result } from 'flexi-bench';
+```
+
+## Cookbook
+
+### Benchmarking Multiple Implementations
+
+Compare different implementations of the same interface:
+
+```javascript
+const {
+  Suite,
+  Benchmark,
+  Variation,
+  MarkdownSuiteReporter,
+} = require('flexi-bench');
+
+// Define your implementations
+const implementations = {
+  loop: (data) => {
+    /* loop implementation */
+  },
+  reduce: (data) => {
+    /* reduce implementation */
+  },
+};
+
+const suite = new Suite('Implementation Comparison')
+  .withReporter(new MarkdownSuiteReporter({ outputFile: 'results.md' }))
+  .addBenchmark(
+    new Benchmark('Process Data')
+      .withIterations(100)
+      // Create variations with context - no environment variables needed!
+      .withVariations(
+        Variation.FromContexts('impl', [
+          ['loop', implementations.loop],
+          ['reduce', implementations.reduce],
+        ]),
+      )
+      .withAction((variation) => {
+        // Retrieve the implementation directly from context
+        const impl = variation.get('impl');
+        const data = [
+          /* test data */
+        ];
+        impl(data);
+      }),
+  );
+```
+
 ## Examples
 
 See examples folder.
 
-- ./examples/benchmark.ts is the motivation for this project. It benchmarks the performance of Nx commands with and without a daemon.
-- ./examples/performance-observer.ts is a simple example of how to use the PerformanceObserver API to measure the performance of a function.
-- ./examples/simple-command.ts demonstrates how to benchmark a simple command.
+- `./examples/benchmark.ts` - Full benchmark suite with environment variable variations
+- `./examples/performance-observer.ts` - Using PerformanceObserver API
+- `./examples/simple-command.ts` - Benchmarking CLI commands
+- `./examples/multiple-reporters.ts` - Using CompositeReporter for multiple outputs
+- `./examples/custom-reporter.ts` - Creating custom reporters with Result type
+- `./examples/cookbook-different-implementations.ts` - Comparing implementations
+- `./examples/no-color-support.ts` - Disabling colors in CI environments
+- `./examples/markdown-reporter-append.ts` - Using append mode for MarkdownReporter
+- `./examples/markdown-comparison.ts` - Demonstrates automatic comparison tables with variations

package/dist/api-types.d.ts

@@ -19,5 +19,50 @@ export interface BenchmarkReporter {
     report: (benchmark: Benchmark, results: Result[]) => void;
 }
 export interface SuiteReporter {
+    /**
+     * Called before the suite starts running.
+     * @param suiteName - The name of the suite
+     */
+    onSuiteStart?: (suiteName: string) => void;
+    /**
+     * Called before each benchmark starts.
+     * @param benchmarkName - The name of the benchmark
+     */
+    onBenchmarkStart?: (benchmarkName: string) => void;
+    /**
+     * Called after each benchmark completes.
+     * @param benchmarkName - The name of the benchmark
+     * @param results - The results for the benchmark
+     */
+    onBenchmarkEnd?: (benchmarkName: string, results: Result[]) => void;
+    /**
+     * Called after all benchmarks complete.
+     * @param results - All benchmark results keyed by benchmark name
+     */
     report: (results: Record<string, Result[]>) => void;
 }
+/**
+ * The strategy to use when an error occurs during a benchmark run.
+ */
+export declare enum ErrorStrategy {
+    /**
+     * Continue running the benchmark. Errors will be collected and reported at the end. This is the default behavior.
+     */
+    Continue = "continue",
+    /**
+     * Abort the benchmark run immediately when an error occurs.
+     */
+    Abort = "abort",
+    /**
+     * Delay the error until the end of the benchmark run. This is useful when you want to see all the errors at once
+     */
+    DelayedThrow = "delayed-throw"
+}
+export declare class AggregateBenchmarkError extends Error {
+    results: Result[];
+    constructor(results: Result[]);
+}
+export declare class AggregateSuiteError extends Error {
+    results: Record<string, Result[]>;
+    constructor(results: Record<string, Result[]>);
+}

package/dist/api-types.js

@@ -1,2 +1,37 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.AggregateSuiteError = exports.AggregateBenchmarkError = exports.ErrorStrategy = void 0;
+/**
+ * The strategy to use when an error occurs during a benchmark run.
+ */
+var ErrorStrategy;
+(function (ErrorStrategy) {
+    /**
+     * Continue running the benchmark. Errors will be collected and reported at the end. This is the default behavior.
+     */
+    ErrorStrategy["Continue"] = "continue";
+    /**
+     * Abort the benchmark run immediately when an error occurs.
+     */
+    ErrorStrategy["Abort"] = "abort";
+    /**
+     * Delay the error until the end of the benchmark run. This is useful when you want to see all the errors at once
+     */
+    ErrorStrategy["DelayedThrow"] = "delayed-throw";
+})(ErrorStrategy || (exports.ErrorStrategy = ErrorStrategy = {}));
+class AggregateBenchmarkError extends Error {
+    results;
+    constructor(results) {
+        super('[AggregateBenchmarkError]: One or more benchmarks failed. Check the results for more information.');
+        this.results = results;
+    }
+}
+exports.AggregateBenchmarkError = AggregateBenchmarkError;
+class AggregateSuiteError extends Error {
+    results;
+    constructor(results) {
+        super('[AggregateSuiteError]: One or more benchmarks failed. Check the results for more information.');
+        this.results = results;
+    }
+}
+exports.AggregateSuiteError = AggregateSuiteError;

package/dist/benchmark-runner.js

@@ -23,7 +23,7 @@ function suite(name, fn) {
     activeSuite = suite;
     const transformed = fn(suite);
     activeSuite = null;
-    return (transformed !== null && transformed !== void 0 ? transformed : suite).run();
+    return (transformed ?? suite).run();
 }
 let activeBenchmark = null;
 /**
@@ -38,10 +38,10 @@ function benchmark(name, fn) {
     const transformed = fn(benchmark);
     activeBenchmark = null;
     if (activeSuite) {
-        activeSuite.addBenchmark(transformed !== null && transformed !== void 0 ? transformed : benchmark);
+        activeSuite.addBenchmark(transformed ?? benchmark);
     }
     else {
-        return (transformed !== null && transformed !== void 0 ? transformed : benchmark).run();
+        return (transformed ?? benchmark).run();
     }
 }
 let activeVariation = null;
@@ -57,10 +57,10 @@ function variation(name, fn) {
     const transformed = fn(variation);
     activeVariation = null;
     if (activeBenchmark) {
-        activeBenchmark.withVariation(transformed !== null && transformed !== void 0 ? transformed : variation);
+        activeBenchmark.withVariation(transformed ?? variation);
     }
     else if (activeSuite) {
-        activeSuite.withVariation(transformed !== null && transformed !== void 0 ? transformed : variation);
+        activeSuite.withVariation(transformed ?? variation);
     }
     else {
         throw new Error('`variation` must be called within a benchmark or suite');

package/dist/benchmark.d.ts

@@ -1,5 +1,5 @@
 import { Variation } from './variation';
-import { TeardownMethod, SetupMethod, Action, BenchmarkReporter } from './api-types';
+import { TeardownMethod, SetupMethod, Action, BenchmarkReporter, ErrorStrategy } from './api-types';
 import { Result } from './results';
 import { PerformanceObserverOptions } from './performance-observer';
 import { BenchmarkBase } from './shared-api';
@@ -10,6 +10,7 @@ export declare class Benchmark extends BenchmarkBase {
     private timeout?;
     private reporter;
     private watcher?;
+    errorStrategy: ErrorStrategy;
     constructor(name: string, options?: {
         setup?: SetupMethod;
         teardown?: TeardownMethod;