modestbench 0.2.0 → 0.3.0

package/src/reporters/simple.ts

@@ -9,8 +9,8 @@ import path from 'node:path';
 
 import type {
   BenchmarkRun,
+  BudgetSummary,
   FileResult,
-  ProgressState,
   SuiteResult,
   TaskResult,
 } from '../types/index.js';
@@ -62,6 +62,78 @@ export class SimpleReporter extends BaseReporter {
     this.quiet = options.quiet ?? false;
   }
 
+  /**
+   * Format bytes in human-readable format
+   */
+  private static formatBytes(this: void, bytes: number): string {
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    let size = bytes;
+    let unitIndex = 0;
+
+    while (size >= 1024 && unitIndex < units.length - 1) {
+      size /= 1024;
+      unitIndex++;
+    }
+
+    return `${size.toFixed(1)} ${units[unitIndex]}`;
+  }
+
+  /**
+   * Format file path - show relative path if within CWD, otherwise absolute
+   */
+  private static formatPath(this: void, filePath: string): string {
+    const cwd = process.cwd();
+    const absolutePath = path.resolve(filePath);
+
+    // Check if the file is within the current working directory
+    if (absolutePath.startsWith(cwd + path.sep) || absolutePath === cwd) {
+      return path.relative(cwd, absolutePath);
+    }
+
+    return absolutePath;
+  }
+
+  /**
+   * Simple pluralization helper
+   */
+  private static pluralize(this: void, str: string, count: number): string {
+    return count === 1 ? str : `${str}s`;
+  }
+
+  onBudgetResult(summary: BudgetSummary): void {
+    if (summary.total === 0 || this.quiet) {
+      return;
+    }
+
+    console.log('== Performance Budgets');
+    console.log();
+
+    for (const result of summary.results) {
+      const icon = result.passed ? symbols.checkmark : symbols.cross;
+      console.log(`  ${icon} ${result.taskId}`);
+
+      if (!result.passed && result.violations.length > 0) {
+        for (const violation of result.violations) {
+          console.log(`      ${violation.message}`);
+        }
+      }
+    }
+
+    console.log();
+
+    if (summary.failed === 0) {
+      console.log(
+        `  ${symbols.checkmark} All ${summary.total} budget(s) passed`,
+      );
+    } else {
+      console.log(
+        `  ${symbols.cross} ${summary.failed} of ${summary.total} budget(s) failed`,
+      );
+    }
+
+    console.log();
+  }
+
   onEnd(run: BenchmarkRun): void {
     if (this.quiet) {
       return;
@@ -97,7 +169,7 @@ export class SimpleReporter extends BaseReporter {
     console.log(`- Files: ${totalFiles}`);
     console.log(`- Suites: ${totalSuites}`);
     console.log(
-      `${symbols.approx} Duration: ${this.formatDuration(duration * 1e6)}`,
+      `${symbols.approx} Duration: ${BaseReporter.formatDuration(duration * 1e6)}`,
     );
     console.log();
 
@@ -111,7 +183,7 @@ export class SimpleReporter extends BaseReporter {
         console.log();
 
         for (const failure of this.failures) {
-          const displayPath = this.formatPath(failure.file);
+          const displayPath = SimpleReporter.formatPath(failure.file);
           console.log(`  ${displayPath} > ${failure.suite} > ${failure.task}`);
           console.log(`    ${failure.error}`);
           console.log();
@@ -155,7 +227,7 @@ export class SimpleReporter extends BaseReporter {
       );
     } else {
       console.log(
-        ` ${symbols.checkmark} ${totalPassed > 1 ? 'All ' : ''}${totalPassed} ${this.pluralize('task', totalPassed)} passed`,
+        ` ${symbols.checkmark} ${totalPassed > 1 ? 'All ' : ''}${totalPassed} ${SimpleReporter.pluralize('task', totalPassed)} passed`,
       );
     }
 
@@ -169,15 +241,10 @@ export class SimpleReporter extends BaseReporter {
       return;
     }
 
-    const displayPath = this.formatPath(file);
+    const displayPath = SimpleReporter.formatPath(file);
     console.log(`-- ${displayPath}`);
   }
 
-  onProgress(_state: ProgressState): void {
-    // Simple reporter does not display progress bars
-    return;
-  }
-
   onStart(run: BenchmarkRun): void {
     this.startTime = Date.now();
     this.failures = []; // Reset failures for new run
@@ -197,7 +264,9 @@ export class SimpleReporter extends BaseReporter {
       console.log(
         `  cpu: ${run.environment.cpu.model} (${run.environment.cpu.cores} cores)`,
       );
-      console.log(`  mem: ${this.formatBytes(run.environment.memory.total)}`);
+      console.log(
+        `  mem: ${SimpleReporter.formatBytes(run.environment.memory.total)}`,
+      );
       console.log();
     }
 
@@ -231,7 +300,7 @@ export class SimpleReporter extends BaseReporter {
       console.log(`  ${symbols.cross} ${failed} failed, ${passed} passed`);
     } else {
       console.log(
-        `  ${symbols.checkmark} ${passed} ${this.pluralize('task', passed)} passed`,
+        `  ${symbols.checkmark} ${passed} ${SimpleReporter.pluralize('task', passed)} passed`,
       );
     }
     console.log();
@@ -275,44 +344,6 @@ export class SimpleReporter extends BaseReporter {
     }
   }
 
-  /**
-   * Format bytes in human-readable format
-   */
-  private formatBytes(bytes: number): string {
-    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
-    let size = bytes;
-    let unitIndex = 0;
-
-    while (size >= 1024 && unitIndex < units.length - 1) {
-      size /= 1024;
-      unitIndex++;
-    }
-
-    return `${size.toFixed(1)} ${units[unitIndex]}`;
-  }
-
-  /**
-   * Format file path - show relative path if within CWD, otherwise absolute
-   */
-  private formatPath(filePath: string): string {
-    const cwd = process.cwd();
-    const absolutePath = path.resolve(filePath);
-
-    // Check if the file is within the current working directory
-    if (absolutePath.startsWith(cwd + path.sep) || absolutePath === cwd) {
-      return path.relative(cwd, absolutePath);
-    }
-
-    return absolutePath;
-  }
-
-  /**
-   * Simple pluralization helper
-   */
-  private pluralize(str: string, count: number): string {
-    return count === 1 ? str : `${str}s`;
-  }
-
   /**
    * Print all task results in a suite with aligned columns
    */
@@ -364,9 +395,9 @@ export class SimpleReporter extends BaseReporter {
         };
       }
 
-      const duration = this.formatDuration(result.mean * 1e9);
-      const opsPerSec = this.formatOpsPerSecond(result.opsPerSecond);
-      const rme = this.formatPercentage(result.marginOfError * 100);
+      const duration = BaseReporter.formatDuration(result.mean * 1e9);
+      const opsPerSec = BaseReporter.formatOpsPerSecond(result.opsPerSecond);
+      const rme = BaseReporter.formatPercentage(result.marginOfError * 100);
 
       return {
         durationLen: duration.length,

package/src/services/baseline-storage.ts

@@ -0,0 +1,199 @@
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import type {
+  BaselineReference,
+  BaselineStorage,
+  BaselineSummaryData,
+  BenchmarkRun,
+  TaskId,
+} from '../types/core.js';
+
+import { validateBaselineStorage } from '../config/budget-schema.js';
+import { StorageError } from '../errors/storage.js';
+import { createTaskId } from '../types/core.js';
+
+/**
+ * Service for managing named baselines
+ *
+ * @packageDocumentation
+ */
+export class BaselineStorageService {
+  private readonly storageDir: string;
+
+  private readonly storageFile: string;
+
+  constructor(storageDir: string = '.') {
+    this.storageDir = storageDir;
+    this.storageFile = join(storageDir, '.modestbench.baselines.json');
+  }
+
+  /**
+   * Delete a baseline
+   */
+  async deleteBaseline(name: string): Promise<void> {
+    let storage = await this.loadStorage();
+
+    if (storage.baselines[name]) {
+      delete storage.baselines[name];
+
+      // Clear default if it was the deleted baseline
+      if (storage.default === name) {
+        storage = { ...storage, default: undefined };
+      }
+
+      await this.saveStorage(storage);
+    }
+  }
+
+  /**
+   * Get a baseline by name
+   */
+  async getBaseline(name: string): Promise<BaselineReference | null> {
+    const storage = await this.loadStorage();
+    return storage.baselines[name] ?? null;
+  }
+
+  /**
+   * Get default baseline name
+   */
+  async getDefault(): Promise<null | string> {
+    const storage = await this.loadStorage();
+    return storage.default ?? null;
+  }
+
+  /**
+   * List all baselines
+   */
+  async listBaselines(): Promise<BaselineReference[]> {
+    const storage = await this.loadStorage();
+    return Object.values(storage.baselines).sort(
+      (a, b) => b.date.getTime() - a.date.getTime(),
+    );
+  }
+
+  /**
+   * Resolve baseline name (use provided or fall back to default)
+   */
+  async resolveBaselineName(name?: string): Promise<null | string> {
+    if (name) {
+      return name;
+    }
+
+    return await this.getDefault();
+  }
+
+  /**
+   * Save a benchmark run as a named baseline
+   */
+  async saveBaseline(
+    name: string,
+    run: BenchmarkRun,
+    metadata?: {
+      branch?: string;
+      commit?: string;
+    },
+  ): Promise<void> {
+    const storage = await this.loadStorage();
+
+    const baseline: BaselineReference = {
+      branch: metadata?.branch,
+      commit: metadata?.commit,
+      date:
+        run.startTime instanceof Date ? run.startTime : new Date(run.startTime),
+      name,
+      runId: run.id,
+      summary: this.extractSummary(run),
+    };
+
+    storage.baselines[name] = baseline;
+
+    await this.saveStorage(storage);
+  }
+
+  /**
+   * Set default baseline
+   */
+  async setDefault(name: string): Promise<void> {
+    let storage = await this.loadStorage();
+
+    if (!storage.baselines[name]) {
+      throw new StorageError(
+        `Baseline "${name}" does not exist. Cannot set as default.`,
+      );
+    }
+
+    storage = { ...storage, default: name };
+    await this.saveStorage(storage);
+  }
+
+  /**
+   * Extract task summary from benchmark run
+   */
+  private extractSummary(
+    run: BenchmarkRun,
+  ): Record<TaskId, BaselineSummaryData> {
+    const summary: Record<TaskId, BaselineSummaryData> = {};
+
+    for (const file of run.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const taskId = createTaskId(file.filePath, suite.name, task.name);
+            summary[taskId] = {
+              mean: task.mean,
+              opsPerSecond: task.opsPerSecond,
+              p99: task.p99,
+            };
+          }
+        }
+      }
+    }
+
+    return summary;
+  }
+
+  /**
+   * Load baseline storage from disk
+   */
+  private async loadStorage(): Promise<BaselineStorage> {
+    if (!existsSync(this.storageFile)) {
+      return {
+        baselines: {},
+        version: '1.0.0',
+      };
+    }
+
+    try {
+      const content = await readFile(this.storageFile, 'utf-8');
+      const data = JSON.parse(content) as unknown;
+      return validateBaselineStorage(data);
+    } catch (error) {
+      throw new StorageError(
+        `Failed to load baseline storage from ${this.storageFile}`,
+        { cause: error },
+      );
+    }
+  }
+
+  /**
+   * Save baseline storage to disk
+   */
+  private async saveStorage(storage: BaselineStorage): Promise<void> {
+    try {
+      // Ensure directory exists
+      if (!existsSync(this.storageDir)) {
+        await mkdir(this.storageDir, { recursive: true });
+      }
+
+      const content = JSON.stringify(storage, null, 2);
+      await writeFile(this.storageFile, content, 'utf-8');
+    } catch (error) {
+      throw new StorageError(
+        `Failed to save baseline storage to ${this.storageFile}`,
+        { cause: error },
+      );
+    }
+  }
+}

package/src/services/budget-evaluator.ts

@@ -0,0 +1,182 @@
+import type {
+  BaselineSummaryData,
+  Budget,
+  BudgetResult,
+  BudgetSummary,
+  BudgetViolation,
+  TaskId,
+  TaskResult,
+} from '../types/core.js';
+
+/**
+ * Service for evaluating performance budgets
+ *
+ * @packageDocumentation
+ */
+export class BudgetEvaluator {
+  /**
+   * Format number with thousands separators
+   */
+  private static formatNumber(this: void, value: number): string {
+    return value.toLocaleString('en-US', {
+      maximumFractionDigits: 0,
+    });
+  }
+
+  /**
+   * Format decimal as percentage
+   */
+  private static formatPercentage(this: void, value: number): string {
+    return `${(value * 100).toFixed(1)}%`;
+  }
+
+  /**
+   * Format time in nanoseconds to human-readable string
+   */
+  private static formatTime(this: void, nanoseconds: number): string {
+    if (nanoseconds < 1_000) {
+      return `${nanoseconds.toFixed(0)}ns`;
+    } else if (nanoseconds < 1_000_000) {
+      return `${(nanoseconds / 1_000).toFixed(2)}μs`;
+    } else if (nanoseconds < 1_000_000_000) {
+      return `${(nanoseconds / 1_000_000).toFixed(2)}ms`;
+    } else {
+      return `${(nanoseconds / 1_000_000_000).toFixed(2)}s`;
+    }
+  }
+
+  /**
+   * Evaluate budgets for an entire benchmark run
+   */
+  evaluateRun(
+    budgets: Record<string, Budget>,
+    taskResults: Map<TaskId, TaskResult>,
+    baselineData?: Map<TaskId, BaselineSummaryData>,
+  ): BudgetSummary {
+    const results: BudgetResult[] = [];
+
+    for (const [taskId, budget] of Object.entries(budgets)) {
+      const taskResult = taskResults.get(taskId as TaskId);
+
+      // Skip if no result for this task
+      if (!taskResult) {
+        continue;
+      }
+
+      // Skip relative budgets if no baseline data
+      if (budget.relative && !baselineData) {
+        continue;
+      }
+
+      const budgetResult = this.evaluateTask(
+        taskId as TaskId,
+        budget,
+        taskResult,
+        baselineData?.get(taskId as TaskId),
+      );
+
+      results.push(budgetResult);
+    }
+
+    const passed = results.filter((r) => r.passed).length;
+    const failed = results.filter((r) => !r.passed).length;
+
+    return {
+      failed,
+      passed,
+      results,
+      total: results.length,
+    };
+  }
+
+  /**
+   * Evaluate budgets for a single task
+   */
+  private evaluateTask(
+    taskId: TaskId,
+    budget: Budget,
+    actual: TaskResult,
+    baseline?: BaselineSummaryData,
+  ): BudgetResult {
+    const violations: BudgetViolation[] = [];
+
+    // Evaluate absolute budgets
+    if (budget.absolute) {
+      if (budget.absolute.maxTime !== undefined) {
+        if (actual.mean > budget.absolute.maxTime) {
+          violations.push({
+            actual: actual.mean,
+            delta:
+              (actual.mean - budget.absolute.maxTime) / budget.absolute.maxTime,
+            message: `Mean execution time ${BudgetEvaluator.formatTime(actual.mean)} exceeded budget of ${BudgetEvaluator.formatTime(budget.absolute.maxTime)} by ${BudgetEvaluator.formatPercentage((actual.mean - budget.absolute.maxTime) / budget.absolute.maxTime)}`,
+            threshold: budget.absolute.maxTime,
+            type: 'maxTime',
+          });
+        }
+      }
+
+      if (budget.absolute.minOpsPerSec !== undefined) {
+        if (actual.opsPerSecond < budget.absolute.minOpsPerSec) {
+          violations.push({
+            actual: actual.opsPerSecond,
+            delta:
+              (budget.absolute.minOpsPerSec - actual.opsPerSecond) /
+              budget.absolute.minOpsPerSec,
+            message: `Operations per second ${BudgetEvaluator.formatNumber(actual.opsPerSecond)} is below minimum of ${BudgetEvaluator.formatNumber(budget.absolute.minOpsPerSec)} by ${BudgetEvaluator.formatPercentage((budget.absolute.minOpsPerSec - actual.opsPerSecond) / budget.absolute.minOpsPerSec)}`,
+            threshold: budget.absolute.minOpsPerSec,
+            type: 'minOpsPerSec',
+          });
+        }
+      }
+
+      if (budget.absolute.maxP99 !== undefined && actual.p99 !== undefined) {
+        if (actual.p99 > budget.absolute.maxP99) {
+          violations.push({
+            actual: actual.p99,
+            delta:
+              (actual.p99 - budget.absolute.maxP99) / budget.absolute.maxP99,
+            message: `P99 latency ${BudgetEvaluator.formatTime(actual.p99)} exceeded budget of ${BudgetEvaluator.formatTime(budget.absolute.maxP99)} by ${BudgetEvaluator.formatPercentage((actual.p99 - budget.absolute.maxP99) / budget.absolute.maxP99)}`,
+            threshold: budget.absolute.maxP99,
+            type: 'maxP99',
+          });
+        }
+      }
+    }
+
+    // Evaluate relative budgets
+    if (budget.relative && baseline) {
+      if (budget.relative.maxRegression !== undefined) {
+        const regression = (actual.mean - baseline.mean) / baseline.mean;
+
+        if (regression > budget.relative.maxRegression) {
+          violations.push({
+            actual: regression,
+            delta: regression - budget.relative.maxRegression,
+            message: `Performance regressed by ${BudgetEvaluator.formatPercentage(regression)} exceeding maximum allowed regression of ${BudgetEvaluator.formatPercentage(budget.relative.maxRegression)}`,
+            threshold: budget.relative.maxRegression,
+            type: 'maxRegression',
+          });
+        }
+      }
+    }
+
+    return {
+      actual: {
+        mean: actual.mean,
+        opsPerSecond: actual.opsPerSecond,
+        p99: actual.p99,
+      },
+      baseline: baseline
+        ? {
+            mean: baseline.mean,
+            opsPerSecond: baseline.opsPerSecond,
+            p99: baseline.p99,
+          }
+        : undefined,
+      budget,
+      passed: violations.length === 0,
+      taskId,
+      violations,
+    };
+  }
+}

package/src/services/config-manager.ts

@@ -56,7 +56,7 @@ const DEFAULT_CONFIG: ModestBenchConfig = {
   limitBy: 'iterations', // Default to limiting by iteration count
   metadata: {},
   outputDir: './benchmark-results',
-  pattern: '**/*.bench.{js,ts,mjs,cjs,mts,cts}',
+  pattern: 'bench/**/*.bench.{js,ts,mjs,cjs,mts,cts}', // Search bench/ directory recursively
   quiet: false,
   reporterConfig: {},
   reporters: [getDefaultReporter()],
@@ -121,10 +121,16 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
 
   /**
    * Load configuration from various sources with precedence
+   *
+   * @param configPath - Optional path to configuration file
+   * @param cliArgs - Optional CLI arguments to merge
+   * @param commandDefaults - Command-specific defaults (fallback to
+   *   DEFAULT_CONFIG)
    */
   async load(
     configPath?: string,
     cliArgs?: Record<string, unknown>,
+    commandDefaults?: Partial<ModestBenchConfig>,
   ): Promise<ModestBenchConfig> {
     try {
       // Create a fresh explorer for each load to avoid module caching issues
@@ -164,9 +170,13 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
 
       const fileConfig = (result?.config || {}) as Partial<ModestBenchConfig>;
 
-      // 2. Merge: defaults <- file <- CLI args
+      // 2. Merge: command defaults <- file <- CLI args
+      // Use command-specific defaults if provided, otherwise use DEFAULT_CONFIG
+      const baseDefaults = commandDefaults
+        ? this.merge(DEFAULT_CONFIG, commandDefaults)
+        : DEFAULT_CONFIG;
       const normalizedCliArgs = cliArgs ? this.normalizeCliArgs(cliArgs) : {};
-      const merged = this.merge(DEFAULT_CONFIG, fileConfig, normalizedCliArgs);
+      const merged = this.merge(baseDefaults, fileConfig, normalizedCliArgs);
 
       // 2.5. Apply smart defaults for limitBy if not explicitly provided
       const finalConfig = ModestBenchConfigurationManager.applySmartDefaults(
@@ -175,15 +185,20 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
         fileConfig,
       );
 
-      // 3. Validate final configuration
-      const validation = this.validate(finalConfig);
-      if (!validation.valid) {
+      // 3. Validate final configuration and get transformed config
+      // The validation also transforms budgets from nested to flat format
+      const validation = safeParseConfig(finalConfig);
+      if (!validation.success) {
+        const errors = validation.error.issues.map((issue) => {
+          const path = issue.path.join('.');
+          return `${path ? `${path}: ` : ''}${issue.message}`;
+        });
         throw new ConfigValidationError(
-          `Configuration validation failed: ${validation.errors.map((e) => e.message).join(', ')}`,
+          `Configuration validation failed: ${errors.join(', ')}`,
         );
       }
 
-      return finalConfig;
+      return validation.data;
     } catch (error) {
       // Re-throw our custom errors
       if (

package/src/services/file-loader.ts

@@ -64,8 +64,7 @@ export class BenchmarkFileLoader implements FileLoader {
       // Handle empty patterns - use sensible defaults
       if (patterns.length === 0) {
         patterns = [
-          `*${BENCHMARK_FILE_PATTERN}`, // top-level current directory
-          `bench/*${BENCHMARK_FILE_PATTERN}`, // top-level bench/ directory
+          `bench/**/*${BENCHMARK_FILE_PATTERN}`, // bench/ directory (recursive)
         ];
       }
 
@@ -152,10 +151,8 @@ export class BenchmarkFileLoader implements FileLoader {
           default?: unknown;
         };
       } else {
-        // Use native dynamic import for JavaScript files with cache busting
-        // Add timestamp to prevent module caching issues across multiple loads
-        const timestamp = Date.now();
-        module = (await import(`${filePath}?t=${timestamp}`)) as {
+        // Use native dynamic import for JavaScript files
+        module = (await import(filePath)) as {
           [key: string]: unknown;
           default?: unknown;
         };