modestbench 0.3.1 → 0.5.0

package/src/reporters/human.ts

@@ -19,6 +19,11 @@ import type {
 import { BaseReporter } from '../services/reporter-registry.js';
 import { ansiChars, colors } from '../utils/ansi.js';
 
+/**
+ * Minimum iterations required for reliable CV calculation
+ */
+const MIN_RELIABLE_ITERATIONS = 30;
+
 /**
  * Human-readable console reporter with colorized output
  */
@@ -38,6 +43,8 @@ export class HumanReporter extends BaseReporter {
 
   private lastProgressLine = '';
 
+  private lowIterationCount = 0;
+
   private maxTimePadWidth = 0; // Track maximum time padding width to prevent jitter
 
   private progressWindowActive = false; // Track if progress window is rendered
@@ -165,14 +172,21 @@ export class HumanReporter extends BaseReporter {
     const totalFiles = run.files.length;
 
     // Calculate totals across all files
+    // Note: Suite-level failures (setup errors) are tracked in this.failures
+    // but not counted as task failures to keep statistics consistent
     let totalSuites = 0;
     let totalPassed = 0;
     let totalFailed = 0;
     let totalAborted = 0;
+    let suiteFailures = 0;
 
     for (const file of run.files) {
       totalSuites += file.suites.length;
       for (const suite of file.suites) {
+        // Track suite-level errors separately (not as task failures)
+        if (suite.error) {
+          suiteFailures++;
+        }
         totalPassed += suite.tasks.filter(
           (t: TaskResult) => !t.error && !t.aborted,
         ).length;
@@ -195,10 +209,13 @@ export class HumanReporter extends BaseReporter {
     this.printLine(
       `${this.colorize('brightBlue', '  Tasks:')} ${this.colorize('brightWhite', String(totalPassed + totalFailed + totalAborted))}`,
     );
-    if (totalFailed > 0 || totalAborted > 0) {
-      if (totalFailed > 0) {
+    // Check for any failures: task failures, suite failures (setup errors), or aborts
+    const hasFailures =
+      totalFailed > 0 || suiteFailures > 0 || totalAborted > 0;
+    if (hasFailures) {
+      if (totalFailed > 0 || suiteFailures > 0) {
         this.printLine(
-          `${this.colorize('brightRed', ansiChars.cross + ' Failed:')} ${this.colorize('brightWhite', String(totalFailed))}`,
+          `${this.colorize('brightRed', ansiChars.cross + ' Failed:')} ${this.colorize('brightWhite', String(totalFailed + suiteFailures))}`,
         );
       }
       if (totalPassed > 0) {
@@ -221,29 +238,35 @@ export class HumanReporter extends BaseReporter {
     );
     this.printLine();
 
-    if (totalFailed > 0) {
-      // Display failed tasks with details
-      if (this.failures.length > 0) {
-        this.printLine();
+    // Display failed tasks/suites with details
+    if (this.failures.length > 0) {
+      this.printLine();
+      this.printLine(
+        this.colorize('brightRed', this.colorize('bold', 'Failed Tasks:')),
+      );
+      this.printLine();
+
+      for (const failure of this.failures) {
+        const displayPath = HumanReporter.formatPath(failure.file);
         this.printLine(
-          this.colorize('brightRed', this.colorize('bold', 'Failed Tasks:')),
+          `  ${this.colorize('dim', displayPath)} ${this.colorize('dim', '›')} ${this.colorize('white', failure.suite)} ${this.colorize('dim', '›')} ${this.colorize('brightWhite', failure.task)}`,
         );
+        this.printLine(`    ${this.colorize('brightRed', failure.error)}`);
         this.printLine();
-
-        for (const failure of this.failures) {
-          const displayPath = HumanReporter.formatPath(failure.file);
-          this.printLine(
-            `  ${this.colorize('dim', displayPath)} ${this.colorize('dim', '›')} ${this.colorize('white', failure.suite)} ${this.colorize('dim', '›')} ${this.colorize('brightWhite', failure.task)}`,
-          );
-          this.printLine(`    ${this.colorize('brightRed', failure.error)}`);
-          this.printLine();
-        }
       }
     } else if (totalAborted === 0) {
       // Only show "Rad" if no failures AND no aborts
       const successMessage = `${this.colorize('brightMagenta', 'Rad. ☮')}`;
       this.printLine(successMessage);
     }
+
+    // Show warning for low iteration counts
+    if (this.lowIterationCount > 0) {
+      this.printLine();
+      this.printLine(
+        `${this.colorize('brightYellow', ansiChars.approx)} ${this.colorize('brightYellow', 'Warning:')} ${this.lowIterationCount} ${HumanReporter.pluralize('task', this.lowIterationCount)} had low iteration counts (<${MIN_RELIABLE_ITERATIONS}) which may affect statistical reliability`,
+      );
+    }
   }
 
   onError(error: Error): void {
@@ -373,6 +396,7 @@ export class HumanReporter extends BaseReporter {
     this.failures = []; // Reset failures for new run
     this.lastProgressLine = ''; // Reset for new run
     this.maxTimePadWidth = 0; // Reset time padding width for new run
+    this.lowIterationCount = 0; // Reset low iteration count for new run
 
     if (this.quiet) {
       return;
@@ -428,6 +452,31 @@ export class HumanReporter extends BaseReporter {
       return;
     }
 
+    // Handle suite-level errors (e.g., setup failure)
+    if (result.error) {
+      // Track suite-level failure for end-of-run summary
+      this.failures.push({
+        error: result.error.message || String(result.error),
+        file: this.currentFile,
+        suite: result.name,
+        task: '(setup)',
+      });
+
+      const durationStr = BaseReporter.formatDuration(
+        result.duration * 1000000,
+      );
+
+      // Display suite setup failure
+      this.printLine(
+        `    ${this.colorize('red', ansiChars.cross)} ${this.colorize('white', '(setup)')} ${this.colorize('red', 'FAILED')}`,
+      );
+      this.printLine(
+        `  ${this.colorize('red', `${ansiChars.cross} Suite setup failed`)} ${this.colorize('gray', 'in')} ${this.colorize('cyan', durationStr)}`,
+      );
+      this.printLine();
+      return;
+    }
+
     // Tasks are printed immediately in onTaskResult, so just print suite summary
 
     // Skip displaying summary for the implicit "default" suite
@@ -634,6 +683,9 @@ export class HumanReporter extends BaseReporter {
       error: boolean;
       errorMessage?: string;
       iterations: number;
+      iterationsLen: number;
+      iterationsStr: string;
+      lowIterations: boolean;
       name: string;
       nameLength: number;
       opsPerSecLen: number;
@@ -661,6 +713,9 @@ export class HumanReporter extends BaseReporter {
             error: true,
             errorMessage: result.error?.message || String(result.error),
             iterations: 0,
+            iterationsLen: 0,
+            iterationsStr: '',
+            lowIterations: false,
             name,
             nameLength,
             opsPerSecLen: 0,
@@ -674,12 +729,17 @@ export class HumanReporter extends BaseReporter {
         const duration = BaseReporter.formatDuration(result.mean); // already in nanoseconds
         const opsPerSec = BaseReporter.formatOpsPerSecond(result.opsPerSecond);
         const rme = BaseReporter.formatPercentage(result.marginOfError); // already a percentage
+        const iterationsStr = `(${result.iterations} iter)`;
+        const lowIterations = result.iterations < MIN_RELIABLE_ITERATIONS;
 
         return {
           durationLen: this.getVisibleLength(duration),
           durationStr: duration,
           error: false,
           iterations: result.iterations,
+          iterationsLen: iterationsStr.length,
+          iterationsStr,
+          lowIterations,
           name,
           nameLength,
           opsPerSecLen: this.getVisibleLength(opsPerSec),
@@ -707,6 +767,10 @@ export class HumanReporter extends BaseReporter {
       ...formatted.filter((t) => !t.error).map((t) => t.rmeLen),
       0,
     );
+    const maxIterLen = Math.max(
+      ...formatted.filter((t) => !t.error).map((t) => t.iterationsLen),
+      0,
+    );
     const maxOpsLen = Math.max(
       ...formatted.filter((t) => !t.error).map((t) => t.opsPerSecLen),
       0,
@@ -731,11 +795,13 @@ export class HumanReporter extends BaseReporter {
         const wrappedLines = this.wrapText(task.name, MAX_NAME_WIDTH);
         const continueIndent = BASE_INDENT + '  '; // 6 spaces for continuation lines
 
-        // Format stats string
+        // Format stats string with iterations
         const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
         const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
+        const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
         const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
-        const statsStr = `${durationPad}${this.colorize('cyan', task.durationStr)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', task.rmeStr)} ${bullet} ${opsPad}${this.colorize('magenta', task.opsPerSecStr)}`;
+        const iterColor = task.lowIterations ? 'brightRed' : 'cyan';
+        const statsStr = `${durationPad}${this.colorize('cyan', task.durationStr)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', task.rmeStr)} ${iterPad}${this.colorize(iterColor, task.iterationsStr)} ${bullet} ${opsPad}${this.colorize('magenta', task.opsPerSecStr)}`;
 
         // Print first line with status
         this.printLine(
@@ -766,26 +832,26 @@ export class HumanReporter extends BaseReporter {
           );
         }
 
-        if (this.verbose && task.iterations > 0) {
-          this.printLine(
-            `      ${this.colorize('dim', `${task.iterations} iterations`)}`,
-          );
+        // Track low iteration count
+        if (task.lowIterations) {
+          this.lowIterationCount++;
         }
       } else {
         // Normal length - align on same line
         const namePad = ' '.repeat(maxNameLen - task.nameLength);
         const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
         const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
+        const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
         const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
+        const iterColor = task.lowIterations ? 'brightRed' : 'cyan';
 
         this.printLine(
-          `${BASE_INDENT}${task.status} ${this.colorize('white', task.name)}${namePad}: ${durationPad}${this.colorize('cyan', task.durationStr)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', task.rmeStr)} ${bullet} ${opsPad}${this.colorize('magenta', task.opsPerSecStr)}`,
+          `${BASE_INDENT}${task.status} ${this.colorize('white', task.name)}${namePad}: ${durationPad}${this.colorize('cyan', task.durationStr)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', task.rmeStr)} ${iterPad}${this.colorize(iterColor, task.iterationsStr)} ${bullet} ${opsPad}${this.colorize('magenta', task.opsPerSecStr)}`,
         );
 
-        if (this.verbose && task.iterations > 0) {
-          this.printLine(
-            `      ${this.colorize('dim', `${task.iterations} iterations`)}`,
-          );
+        // Track low iteration count
+        if (task.lowIterations) {
+          this.lowIterationCount++;
         }
       }
     }
@@ -853,21 +919,27 @@ export class HumanReporter extends BaseReporter {
     const duration = BaseReporter.formatDuration(result.mean);
     const opsPerSec = BaseReporter.formatOpsPerSecond(result.opsPerSecond);
     const rme = BaseReporter.formatPercentage(result.marginOfError);
+    const iterationsStr = `(${result.iterations} iter)`;
+    const lowIterations = result.iterations < MIN_RELIABLE_ITERATIONS;
 
     // Use fixed widths for stats columns (reasonable maximums)
     const DURATION_WIDTH = 10; // "999.99ms" max
     const RME_WIDTH = 8; // "±999.99%" max
+    const ITER_WIDTH = 12; // "(99999 iter)" max
     const OPS_WIDTH = 15; // "999.99K ops/sec" max
 
     const durationLen = this.getVisibleLength(duration);
     const rmeLen = this.getVisibleLength(rme);
+    const iterLen = iterationsStr.length;
     const opsLen = this.getVisibleLength(opsPerSec);
 
     // Stats formatting with fixed widths
-    const durationPad = ' '.repeat(DURATION_WIDTH - durationLen);
-    const rmePad = ' '.repeat(RME_WIDTH - rmeLen);
-    const opsPad = ' '.repeat(OPS_WIDTH - opsLen);
-    const statsStr = `${durationPad}${this.colorize('cyan', duration)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', rme)} ${bullet} ${opsPad}${this.colorize('magenta', opsPerSec)}`;
+    const durationPad = ' '.repeat(Math.max(0, DURATION_WIDTH - durationLen));
+    const rmePad = ' '.repeat(Math.max(0, RME_WIDTH - rmeLen));
+    const iterPad = ' '.repeat(Math.max(0, ITER_WIDTH - iterLen));
+    const opsPad = ' '.repeat(Math.max(0, OPS_WIDTH - opsLen));
+    const iterColor = lowIterations ? 'brightRed' : 'cyan';
+    const statsStr = `${durationPad}${this.colorize('cyan', duration)} ${bullet} ${ansiChars.plusMinus}${rmePad}${this.colorize('brightBlue', rme)} ${iterPad}${this.colorize(iterColor, iterationsStr)} ${bullet} ${opsPad}${this.colorize('magenta', opsPerSec)}`;
 
     // Handle long names (wrap)
     if (nameLength > MAX_NAME_WIDTH) {
@@ -917,10 +989,9 @@ export class HumanReporter extends BaseReporter {
       );
     }
 
-    if (this.verbose && result.iterations > 0) {
-      this.printLine(
-        `      ${this.colorize('dim', `${result.iterations} iterations`)}`,
-      );
+    // Track low iteration count
+    if (lowIterations) {
+      this.lowIterationCount++;
     }
   }
 

package/src/reporters/simple.ts

@@ -25,8 +25,14 @@ const symbols = {
   checkmark: '√',
   cross: '×',
   plusMinus: '±',
+  warning: '⚠',
 } as const;
 
+/**
+ * Minimum iterations required for reliable CV calculation
+ */
+const MIN_RELIABLE_ITERATIONS = 30;
+
 /**
  * Simple console reporter with plain text output (no colors or progress bars)
  */
@@ -42,6 +48,8 @@ export class SimpleReporter extends BaseReporter {
     task: string;
   }> = [];
 
+  private lowIterationCount = 0;
+
   private readonly quiet: boolean;
 
   private startTime = 0;
@@ -143,13 +151,20 @@ export class SimpleReporter extends BaseReporter {
     const totalFiles = run.files.length;
 
     // Calculate totals across all files
+    // Note: Suite-level failures (setup errors) are tracked in this.failures
+    // but not counted as task failures to keep statistics consistent
     let totalSuites = 0;
     let totalPassed = 0;
     let totalFailed = 0;
+    let suiteFailures = 0;
 
     for (const file of run.files) {
       totalSuites += file.suites.length;
       for (const suite of file.suites) {
+        // Track suite-level errors separately (not as task failures)
+        if (suite.error) {
+          suiteFailures++;
+        }
         totalPassed += suite.tasks.filter((t: TaskResult) => !t.error).length;
         totalFailed += suite.tasks.filter((t: TaskResult) => t.error).length;
       }
@@ -159,8 +174,10 @@ export class SimpleReporter extends BaseReporter {
     console.log('== Results');
     console.log();
 
-    if (totalFailed > 0) {
-      console.log(`${symbols.cross} Failed: ${totalFailed}`);
+    // Check for any failures: task failures or suite failures (setup errors)
+    const hasFailures = totalFailed > 0 || suiteFailures > 0;
+    if (hasFailures) {
+      console.log(`${symbols.cross} Failed: ${totalFailed + suiteFailures}`);
       console.log(`${symbols.checkmark} Passed: ${totalPassed}`);
     } else {
       console.log(`${symbols.checkmark} All tasks passed: ${totalPassed}`);
@@ -173,25 +190,30 @@ export class SimpleReporter extends BaseReporter {
     );
     console.log();
 
-    if (totalFailed > 0) {
+    // Display failed tasks/suites with details
+    if (this.failures.length > 0) {
       console.log(`${symbols.cross.repeat(3)} Some benchmarks failed`);
+      console.log();
+      console.log('Failed Tasks:');
+      console.log();
 
-      // Display failed tasks with details
-      if (this.failures.length > 0) {
+      for (const failure of this.failures) {
+        const displayPath = SimpleReporter.formatPath(failure.file);
+        console.log(`  ${displayPath} > ${failure.suite} > ${failure.task}`);
+        console.log(`    ${failure.error}`);
         console.log();
-        console.log('Failed Tasks:');
-        console.log();
-
-        for (const failure of this.failures) {
-          const displayPath = SimpleReporter.formatPath(failure.file);
-          console.log(`  ${displayPath} > ${failure.suite} > ${failure.task}`);
-          console.log(`    ${failure.error}`);
-          console.log();
-        }
       }
-    } else {
+    } else if (!hasFailures) {
       console.log('All benchmarks completed successfully!');
     }
+
+    // Show warning for low iteration counts
+    if (this.lowIterationCount > 0) {
+      console.log();
+      console.log(
+        `${symbols.warning} Warning: ${this.lowIterationCount} ${SimpleReporter.pluralize('task', this.lowIterationCount)} had low iteration counts (<${MIN_RELIABLE_ITERATIONS}) which may affect statistical reliability`,
+      );
+    }
   }
 
   onError(error: Error): void {
@@ -248,6 +270,7 @@ export class SimpleReporter extends BaseReporter {
   onStart(run: BenchmarkRun): void {
     this.startTime = Date.now();
     this.failures = []; // Reset failures for new run
+    this.lowIterationCount = 0; // Reset low iteration count for new run
 
     if (this.quiet) {
       return;
@@ -285,6 +308,23 @@ export class SimpleReporter extends BaseReporter {
       return;
     }
 
+    // Handle suite-level errors (e.g., setup failure)
+    if (result.error) {
+      // Track suite-level failure for end-of-run summary
+      this.failures.push({
+        error: result.error.message || String(result.error),
+        file: this.currentFile,
+        suite: result.name,
+        task: '(setup)',
+      });
+
+      // Display suite setup failure
+      console.log(`    ${symbols.cross} (setup) FAILED`);
+      console.log(`  ${symbols.cross} Suite setup failed`);
+      console.log();
+      return;
+    }
+
     // Print all buffered task results with aligned columns
     this.printAlignedSuiteResults();
 
@@ -366,6 +406,9 @@ export class SimpleReporter extends BaseReporter {
       error: boolean;
       errorMessage?: string;
       iterations: number;
+      iterationsLen: number;
+      iterationsStr: string;
+      lowIterations: boolean;
       name: string;
       nameLength: number;
       opsPerSecLen: number;
@@ -388,6 +431,9 @@ export class SimpleReporter extends BaseReporter {
           error: true,
           errorMessage: result.error?.message || String(result.error),
           iterations: 0,
+          iterationsLen: 0,
+          iterationsStr: '',
+          lowIterations: false,
           name,
           nameLength,
           opsPerSecLen: 0,
@@ -398,15 +444,20 @@ export class SimpleReporter extends BaseReporter {
         };
       }
 
-      const duration = BaseReporter.formatDuration(result.mean * 1e9);
+      const duration = BaseReporter.formatDuration(result.mean);
       const opsPerSec = BaseReporter.formatOpsPerSecond(result.opsPerSecond);
       const rme = BaseReporter.formatPercentage(result.marginOfError); // already a percentage
+      const iterationsStr = `(${result.iterations} iter)`;
+      const lowIterations = result.iterations < MIN_RELIABLE_ITERATIONS;
 
       return {
         durationLen: duration.length,
         durationStr: duration,
         error: false,
         iterations: result.iterations,
+        iterationsLen: iterationsStr.length,
+        iterationsStr,
+        lowIterations,
         name,
         nameLength,
         opsPerSecLen: opsPerSec.length,
@@ -434,6 +485,10 @@ export class SimpleReporter extends BaseReporter {
       ...formatted.filter((t) => !t.error).map((t) => t.rmeLen),
       0,
     );
+    const maxIterLen = Math.max(
+      ...formatted.filter((t) => !t.error).map((t) => t.iterationsLen),
+      0,
+    );
     const maxOpsLen = Math.max(
       ...formatted.filter((t) => !t.error).map((t) => t.opsPerSecLen),
       0,
@@ -464,28 +519,32 @@ export class SimpleReporter extends BaseReporter {
         const leadingPad = ' '.repeat(numbersStartPos);
         const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
         const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
+        const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
         const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
 
         console.log(
-          `${leadingPad}${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
+          `${leadingPad}${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${iterPad}${task.iterationsStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
         );
 
-        if (this.verbose && task.iterations > 0) {
-          console.log(`      ${task.iterations} iterations`);
+        // Track low iteration count
+        if (task.lowIterations) {
+          this.lowIterationCount++;
         }
       } else {
         // Normal length - align on same line
         const namePad = ' '.repeat(maxNameLen - task.nameLength);
         const durationPad = ' '.repeat(maxDurationLen - task.durationLen);
         const rmePad = ' '.repeat(maxRmeLen - task.rmeLen);
+        const iterPad = ' '.repeat(maxIterLen - task.iterationsLen);
         const opsPad = ' '.repeat(maxOpsLen - task.opsPerSecLen);
 
         console.log(
-          `${BASE_INDENT}${task.status} ${task.name}${namePad}: ${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
+          `${BASE_INDENT}${task.status} ${task.name}${namePad}: ${durationPad}${task.durationStr} ${separator} ${symbols.plusMinus}${rmePad}${task.rmeStr} ${iterPad}${task.iterationsStr} ${separator} ${opsPad}${task.opsPerSecStr}`,
         );
 
-        if (this.verbose && task.iterations > 0) {
-          console.log(`      ${task.iterations} iterations`);
+        // Track low iteration count
+        if (task.lowIterations) {
+          this.lowIterationCount++;
         }
       }
     }

package/src/services/config-manager.ts

@@ -19,7 +19,7 @@ import type {
 } from '../types/index.js';
 
 import { safeParseConfig } from '../config/schema.js';
-import { ErrorCodes } from '../constants.js';
+import { DEFAULT_BENCHMARK_DIR, ErrorCodes } from '../constants.js';
 import { ConfigLoadError, ConfigValidationError } from '../errors/index.js';
 
 /**
@@ -55,8 +55,8 @@ const DEFAULT_CONFIG: ModestBenchConfig = {
   iterations: 100, // Sufficient iterations for reliable statistics
   limitBy: 'iterations', // Default to limiting by iteration count
   metadata: {},
-  outputDir: '.modestbench',
-  pattern: 'bench/**/*.bench.{js,ts,mjs,cjs,mts,cts}', // Search bench/ directory recursively
+  outputDir: undefined, // No default output directory - reporters will use stdout unless explicitly configured
+  pattern: `${DEFAULT_BENCHMARK_DIR}/**/*.bench.{js,ts,mjs,cjs,mts,cts}`, // Search bench/ directory recursively
   quiet: false,
   reporterConfig: {},
   reporters: [getDefaultReporter()],

package/src/types/core.ts

@@ -327,8 +327,8 @@ export interface ModestBenchConfig {
   readonly limitBy: 'all' | 'any' | 'iterations' | 'time';
   /** Custom metadata to attach to runs */
   readonly metadata: Record<string, unknown>;
-  /** Output directory for reports */
-  readonly outputDir: string;
+  /** Output directory for reports (undefined means stdout for data reporters) */
+  readonly outputDir?: string;
   /** Pattern(s) for discovering benchmark files */
   readonly pattern: string | string[];
   /** Whether to run in quiet mode */