modestbench 0.2.0 → 0.3.1

package/src/services/baseline-storage.ts

@@ -0,0 +1,199 @@
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import type {
+  BaselineReference,
+  BaselineStorage,
+  BaselineSummaryData,
+  BenchmarkRun,
+  TaskId,
+} from '../types/core.js';
+
+import { validateBaselineStorage } from '../config/budget-schema.js';
+import { StorageError } from '../errors/storage.js';
+import { createTaskId } from '../types/core.js';
+
+/**
+ * Service for managing named baselines
+ *
+ * @packageDocumentation
+ */
+export class BaselineStorageService {
+  private readonly storageDir: string;
+
+  private readonly storageFile: string;
+
+  constructor(storageDir: string = '.') {
+    this.storageDir = storageDir;
+    this.storageFile = join(storageDir, '.modestbench.baselines.json');
+  }
+
+  /**
+   * Delete a baseline
+   */
+  async deleteBaseline(name: string): Promise<void> {
+    let storage = await this.loadStorage();
+
+    if (storage.baselines[name]) {
+      delete storage.baselines[name];
+
+      // Clear default if it was the deleted baseline
+      if (storage.default === name) {
+        storage = { ...storage, default: undefined };
+      }
+
+      await this.saveStorage(storage);
+    }
+  }
+
+  /**
+   * Get a baseline by name
+   */
+  async getBaseline(name: string): Promise<BaselineReference | null> {
+    const storage = await this.loadStorage();
+    return storage.baselines[name] ?? null;
+  }
+
+  /**
+   * Get default baseline name
+   */
+  async getDefault(): Promise<null | string> {
+    const storage = await this.loadStorage();
+    return storage.default ?? null;
+  }
+
+  /**
+   * List all baselines
+   */
+  async listBaselines(): Promise<BaselineReference[]> {
+    const storage = await this.loadStorage();
+    return Object.values(storage.baselines).sort(
+      (a, b) => b.date.getTime() - a.date.getTime(),
+    );
+  }
+
+  /**
+   * Resolve baseline name (use provided or fall back to default)
+   */
+  async resolveBaselineName(name?: string): Promise<null | string> {
+    if (name) {
+      return name;
+    }
+
+    return await this.getDefault();
+  }
+
+  /**
+   * Save a benchmark run as a named baseline
+   */
+  async saveBaseline(
+    name: string,
+    run: BenchmarkRun,
+    metadata?: {
+      branch?: string;
+      commit?: string;
+    },
+  ): Promise<void> {
+    const storage = await this.loadStorage();
+
+    const baseline: BaselineReference = {
+      branch: metadata?.branch,
+      commit: metadata?.commit,
+      date:
+        run.startTime instanceof Date ? run.startTime : new Date(run.startTime),
+      name,
+      runId: run.id,
+      summary: this.extractSummary(run),
+    };
+
+    storage.baselines[name] = baseline;
+
+    await this.saveStorage(storage);
+  }
+
+  /**
+   * Set default baseline
+   */
+  async setDefault(name: string): Promise<void> {
+    let storage = await this.loadStorage();
+
+    if (!storage.baselines[name]) {
+      throw new StorageError(
+        `Baseline "${name}" does not exist. Cannot set as default.`,
+      );
+    }
+
+    storage = { ...storage, default: name };
+    await this.saveStorage(storage);
+  }
+
+  /**
+   * Extract task summary from benchmark run
+   */
+  private extractSummary(
+    run: BenchmarkRun,
+  ): Record<TaskId, BaselineSummaryData> {
+    const summary: Record<TaskId, BaselineSummaryData> = {};
+
+    for (const file of run.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const taskId = createTaskId(file.filePath, suite.name, task.name);
+            summary[taskId] = {
+              mean: task.mean,
+              opsPerSecond: task.opsPerSecond,
+              p99: task.p99,
+            };
+          }
+        }
+      }
+    }
+
+    return summary;
+  }
+
+  /**
+   * Load baseline storage from disk
+   */
+  private async loadStorage(): Promise<BaselineStorage> {
+    if (!existsSync(this.storageFile)) {
+      return {
+        baselines: {},
+        version: '1.0.0',
+      };
+    }
+
+    try {
+      const content = await readFile(this.storageFile, 'utf-8');
+      const data = JSON.parse(content) as unknown;
+      return validateBaselineStorage(data);
+    } catch (error) {
+      throw new StorageError(
+        `Failed to load baseline storage from ${this.storageFile}`,
+        { cause: error },
+      );
+    }
+  }
+
+  /**
+   * Save baseline storage to disk
+   */
+  private async saveStorage(storage: BaselineStorage): Promise<void> {
+    try {
+      // Ensure directory exists
+      if (!existsSync(this.storageDir)) {
+        await mkdir(this.storageDir, { recursive: true });
+      }
+
+      const content = JSON.stringify(storage, null, 2);
+      await writeFile(this.storageFile, content, 'utf-8');
+    } catch (error) {
+      throw new StorageError(
+        `Failed to save baseline storage to ${this.storageFile}`,
+        { cause: error },
+      );
+    }
+  }
+}

package/src/services/budget-evaluator.ts

@@ -0,0 +1,182 @@
+import type {
+  BaselineSummaryData,
+  Budget,
+  BudgetResult,
+  BudgetSummary,
+  BudgetViolation,
+  TaskId,
+  TaskResult,
+} from '../types/core.js';
+
+/**
+ * Service for evaluating performance budgets
+ *
+ * @packageDocumentation
+ */
+export class BudgetEvaluator {
+  /**
+   * Format number with thousands separators
+   */
+  private static formatNumber(this: void, value: number): string {
+    return value.toLocaleString('en-US', {
+      maximumFractionDigits: 0,
+    });
+  }
+
+  /**
+   * Format decimal as percentage
+   */
+  private static formatPercentage(this: void, value: number): string {
+    return `${(value * 100).toFixed(1)}%`;
+  }
+
+  /**
+   * Format time in nanoseconds to human-readable string
+   */
+  private static formatTime(this: void, nanoseconds: number): string {
+    if (nanoseconds < 1_000) {
+      return `${nanoseconds.toFixed(0)}ns`;
+    } else if (nanoseconds < 1_000_000) {
+      return `${(nanoseconds / 1_000).toFixed(2)}μs`;
+    } else if (nanoseconds < 1_000_000_000) {
+      return `${(nanoseconds / 1_000_000).toFixed(2)}ms`;
+    } else {
+      return `${(nanoseconds / 1_000_000_000).toFixed(2)}s`;
+    }
+  }
+
+  /**
+   * Evaluate budgets for an entire benchmark run
+   */
+  evaluateRun(
+    budgets: Record<string, Budget>,
+    taskResults: Map<TaskId, TaskResult>,
+    baselineData?: Map<TaskId, BaselineSummaryData>,
+  ): BudgetSummary {
+    const results: BudgetResult[] = [];
+
+    for (const [taskId, budget] of Object.entries(budgets)) {
+      const taskResult = taskResults.get(taskId as TaskId);
+
+      // Skip if no result for this task
+      if (!taskResult) {
+        continue;
+      }
+
+      // Skip relative budgets if no baseline data
+      if (budget.relative && !baselineData) {
+        continue;
+      }
+
+      const budgetResult = this.evaluateTask(
+        taskId as TaskId,
+        budget,
+        taskResult,
+        baselineData?.get(taskId as TaskId),
+      );
+
+      results.push(budgetResult);
+    }
+
+    const passed = results.filter((r) => r.passed).length;
+    const failed = results.filter((r) => !r.passed).length;
+
+    return {
+      failed,
+      passed,
+      results,
+      total: results.length,
+    };
+  }
+
+  /**
+   * Evaluate budgets for a single task
+   */
+  private evaluateTask(
+    taskId: TaskId,
+    budget: Budget,
+    actual: TaskResult,
+    baseline?: BaselineSummaryData,
+  ): BudgetResult {
+    const violations: BudgetViolation[] = [];
+
+    // Evaluate absolute budgets
+    if (budget.absolute) {
+      if (budget.absolute.maxTime !== undefined) {
+        if (actual.mean > budget.absolute.maxTime) {
+          violations.push({
+            actual: actual.mean,
+            delta:
+              (actual.mean - budget.absolute.maxTime) / budget.absolute.maxTime,
+            message: `Mean execution time ${BudgetEvaluator.formatTime(actual.mean)} exceeded budget of ${BudgetEvaluator.formatTime(budget.absolute.maxTime)} by ${BudgetEvaluator.formatPercentage((actual.mean - budget.absolute.maxTime) / budget.absolute.maxTime)}`,
+            threshold: budget.absolute.maxTime,
+            type: 'maxTime',
+          });
+        }
+      }
+
+      if (budget.absolute.minOpsPerSec !== undefined) {
+        if (actual.opsPerSecond < budget.absolute.minOpsPerSec) {
+          violations.push({
+            actual: actual.opsPerSecond,
+            delta:
+              (budget.absolute.minOpsPerSec - actual.opsPerSecond) /
+              budget.absolute.minOpsPerSec,
+            message: `Operations per second ${BudgetEvaluator.formatNumber(actual.opsPerSecond)} is below minimum of ${BudgetEvaluator.formatNumber(budget.absolute.minOpsPerSec)} by ${BudgetEvaluator.formatPercentage((budget.absolute.minOpsPerSec - actual.opsPerSecond) / budget.absolute.minOpsPerSec)}`,
+            threshold: budget.absolute.minOpsPerSec,
+            type: 'minOpsPerSec',
+          });
+        }
+      }
+
+      if (budget.absolute.maxP99 !== undefined && actual.p99 !== undefined) {
+        if (actual.p99 > budget.absolute.maxP99) {
+          violations.push({
+            actual: actual.p99,
+            delta:
+              (actual.p99 - budget.absolute.maxP99) / budget.absolute.maxP99,
+            message: `P99 latency ${BudgetEvaluator.formatTime(actual.p99)} exceeded budget of ${BudgetEvaluator.formatTime(budget.absolute.maxP99)} by ${BudgetEvaluator.formatPercentage((actual.p99 - budget.absolute.maxP99) / budget.absolute.maxP99)}`,
+            threshold: budget.absolute.maxP99,
+            type: 'maxP99',
+          });
+        }
+      }
+    }
+
+    // Evaluate relative budgets
+    if (budget.relative && baseline) {
+      if (budget.relative.maxRegression !== undefined) {
+        const regression = (actual.mean - baseline.mean) / baseline.mean;
+
+        if (regression > budget.relative.maxRegression) {
+          violations.push({
+            actual: regression,
+            delta: regression - budget.relative.maxRegression,
+            message: `Performance regressed by ${BudgetEvaluator.formatPercentage(regression)} exceeding maximum allowed regression of ${BudgetEvaluator.formatPercentage(budget.relative.maxRegression)}`,
+            threshold: budget.relative.maxRegression,
+            type: 'maxRegression',
+          });
+        }
+      }
+    }
+
+    return {
+      actual: {
+        mean: actual.mean,
+        opsPerSecond: actual.opsPerSecond,
+        p99: actual.p99,
+      },
+      baseline: baseline
+        ? {
+            mean: baseline.mean,
+            opsPerSecond: baseline.opsPerSecond,
+            p99: baseline.p99,
+          }
+        : undefined,
+      budget,
+      passed: violations.length === 0,
+      taskId,
+      violations,
+    };
+  }
+}

package/src/services/config-manager.ts

@@ -55,8 +55,8 @@ const DEFAULT_CONFIG: ModestBenchConfig = {
   iterations: 100, // Sufficient iterations for reliable statistics
   limitBy: 'iterations', // Default to limiting by iteration count
   metadata: {},
-  outputDir: './benchmark-results',
-  pattern: '**/*.bench.{js,ts,mjs,cjs,mts,cts}',
+  outputDir: '.modestbench',
+  pattern: 'bench/**/*.bench.{js,ts,mjs,cjs,mts,cts}', // Search bench/ directory recursively
   quiet: false,
   reporterConfig: {},
   reporters: [getDefaultReporter()],
@@ -121,10 +121,16 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
 
   /**
    * Load configuration from various sources with precedence
+   *
+   * @param configPath - Optional path to configuration file
+   * @param cliArgs - Optional CLI arguments to merge
+   * @param commandDefaults - Command-specific defaults (fallback to
+   *   DEFAULT_CONFIG)
    */
   async load(
     configPath?: string,
     cliArgs?: Record<string, unknown>,
+    commandDefaults?: Partial<ModestBenchConfig>,
   ): Promise<ModestBenchConfig> {
     try {
       // Create a fresh explorer for each load to avoid module caching issues
@@ -164,9 +170,13 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
 
       const fileConfig = (result?.config || {}) as Partial<ModestBenchConfig>;
 
-      // 2. Merge: defaults <- file <- CLI args
+      // 2. Merge: command defaults <- file <- CLI args
+      // Use command-specific defaults if provided, otherwise use DEFAULT_CONFIG
+      const baseDefaults = commandDefaults
+        ? this.merge(DEFAULT_CONFIG, commandDefaults)
+        : DEFAULT_CONFIG;
       const normalizedCliArgs = cliArgs ? this.normalizeCliArgs(cliArgs) : {};
-      const merged = this.merge(DEFAULT_CONFIG, fileConfig, normalizedCliArgs);
+      const merged = this.merge(baseDefaults, fileConfig, normalizedCliArgs);
 
       // 2.5. Apply smart defaults for limitBy if not explicitly provided
       const finalConfig = ModestBenchConfigurationManager.applySmartDefaults(
@@ -175,15 +185,20 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
         fileConfig,
       );
 
-      // 3. Validate final configuration
-      const validation = this.validate(finalConfig);
-      if (!validation.valid) {
+      // 3. Validate final configuration and get transformed config
+      // The validation also transforms budgets from nested to flat format
+      const validation = safeParseConfig(finalConfig);
+      if (!validation.success) {
+        const errors = validation.error.issues.map((issue) => {
+          const path = issue.path.join('.');
+          return `${path ? `${path}: ` : ''}${issue.message}`;
+        });
         throw new ConfigValidationError(
-          `Configuration validation failed: ${validation.errors.map((e) => e.message).join(', ')}`,
+          `Configuration validation failed: ${errors.join(', ')}`,
         );
       }
 
-      return finalConfig;
+      return validation.data;
     } catch (error) {
       // Re-throw our custom errors
       if (

package/src/services/file-loader.ts

@@ -64,8 +64,7 @@ export class BenchmarkFileLoader implements FileLoader {
       // Handle empty patterns - use sensible defaults
       if (patterns.length === 0) {
         patterns = [
-          `*${BENCHMARK_FILE_PATTERN}`, // top-level current directory
-          `bench/*${BENCHMARK_FILE_PATTERN}`, // top-level bench/ directory
+          `bench/**/*${BENCHMARK_FILE_PATTERN}`, // bench/ directory (recursive)
         ];
       }
 
@@ -152,10 +151,8 @@ export class BenchmarkFileLoader implements FileLoader {
           default?: unknown;
         };
       } else {
-        // Use native dynamic import for JavaScript files with cache busting
-        // Add timestamp to prevent module caching issues across multiple loads
-        const timestamp = Date.now();
-        module = (await import(`${filePath}?t=${timestamp}`)) as {
+        // Use native dynamic import for JavaScript files
+        module = (await import(filePath)) as {
           [key: string]: unknown;
           default?: unknown;
         };

package/src/services/profiler/profile-filter.ts

@@ -0,0 +1,147 @@
+/**
+ * Profile Filter Service
+ *
+ * Filters and sorts profiled functions based on configuration. Implements smart
+ * detection to focus on user code by excluding node_modules and Node.js
+ * internals.
+ *
+ * @packageDocumentation
+ */
+
+import { minimatch } from 'minimatch';
+
+import type {
+  FilteredProfileData,
+  ProfileConfig,
+  ProfiledFunction,
+  RawProfileData,
+} from '../../types/profiler.js';
+
+/**
+ * Filter profile data based on configuration
+ *
+ * @param data - Raw profile data
+ * @param config - Filter configuration
+ * @param packageRoot - Package root directory for smart detection
+ * @returns Filtered profile data
+ */
+export const filterProfile = (
+  data: RawProfileData,
+  config: ProfileConfig,
+  packageRoot: string,
+): FilteredProfileData => {
+  let filtered = data.functions.filter((fn) => {
+    // Only JavaScript functions
+    if (fn.category !== 'JavaScript') {
+      return false;
+    }
+
+    // Apply smart detection if enabled
+    if (config.smartDetection && !config.focus?.length) {
+      if (!isUserCode(fn.file, packageRoot)) {
+        return false;
+      }
+    }
+
+    // Apply focus patterns (if provided, overrides smart detection)
+    if (config.focus?.length) {
+      if (!matchesAnyPattern(fn.file, config.focus)) {
+        return false;
+      }
+    }
+
+    // Apply exclude patterns (always applied if provided)
+    if (config.exclude?.length) {
+      if (matchesAnyPattern(fn.file, config.exclude)) {
+        return false;
+      }
+    }
+
+    return true;
+  });
+
+  // Save count of user functions before percentage filtering
+  const totalUserFunctions = filtered.length;
+
+  // Apply percentage threshold
+  const minPercent = config.minExecutionPercent ?? 0.5;
+  filtered = filtered.filter((fn) => fn.percentage >= minPercent);
+
+  // Sort by percentage (highest first)
+  filtered.sort((a, b) => b.percentage - a.percentage);
+
+  // Limit to topN
+  const topN = config.topN ?? 25;
+  filtered = filtered.slice(0, topN);
+
+  // Group by file if requested
+  let groupedByFile: Map<string, ProfiledFunction[]> | undefined;
+  if (config.groupByFile) {
+    groupedByFile = groupByFile(filtered);
+  }
+
+  return {
+    functions: filtered,
+    groupedByFile,
+    minExecutionPercent: minPercent,
+    summary: data.summary,
+    totalFiltered: totalUserFunctions,
+    totalShown: filtered.length,
+    totalTicks: data.totalTicks,
+  };
+};
+
+/**
+ * Check if a file path is user code (not node_modules or internals)
+ */
+const isUserCode = (filePath: string, packageRoot: string): boolean => {
+  // Exclude node_modules
+  if (
+    filePath.includes('/node_modules/') ||
+    filePath.includes('\\node_modules\\')
+  ) {
+    return false;
+  }
+
+  // Exclude Node.js internals
+  if (filePath.startsWith('node:') || filePath.startsWith('internal/')) {
+    return false;
+  }
+
+  // Allow <unknown> files (could be eval'd code or other user code without file paths)
+  if (filePath === '<unknown>' || filePath === '[eval]') {
+    return true;
+  }
+
+  // Must be within package root
+  return filePath.startsWith(packageRoot);
+};
+
+/**
+ * Check if a file path matches any of the given glob patterns
+ */
+const matchesAnyPattern = (filePath: string, patterns: string[]): boolean => {
+  return patterns.some((pattern) => minimatch(filePath, pattern));
+};
+
+/**
+ * Group functions by file path
+ */
+const groupByFile = (
+  functions: ProfiledFunction[],
+): Map<string, ProfiledFunction[]> => {
+  const grouped = new Map<string, ProfiledFunction[]>();
+
+  for (const fn of functions) {
+    const existing = grouped.get(fn.file) || [];
+    existing.push(fn);
+    grouped.set(fn.file, existing);
+  }
+
+  // Sort functions within each file by percentage
+  for (const [, fns] of grouped.entries()) {
+    fns.sort((a, b) => b.percentage - a.percentage);
+  }
+
+  return grouped;
+};