modestbench 0.2.0 → 0.3.0

package/src/core/engine.ts

@@ -7,13 +7,17 @@
  */
 
 import { randomBytes } from 'node:crypto';
+import { relative as pathRelative } from 'node:path';
 
 import type {
+  BaselineSummaryData,
   BenchmarkDefinition,
   BenchmarkEngine,
   BenchmarkRun,
   BenchmarkSuite,
   BenchmarkTask,
+  Budget,
+  BudgetSummary,
   CiInfo,
   ConfigurationManager,
   EnvironmentInfo,
@@ -26,7 +30,9 @@ import type {
   Reporter,
   ReporterRegistry,
   RunConfiguration,
+  RunId,
   SuiteResult,
+  TaskId,
   TaskResult,
   ValidationError,
   ValidationResult,
@@ -35,11 +41,15 @@ import type {
 
 import {
   BenchmarkExecutionError,
+  BudgetExceededError,
   FileDiscoveryError,
   SchemaValidationError,
   SetupError,
   StructureValidationError,
 } from '../errors/index.js';
+import { BaselineStorageService } from '../services/baseline-storage.js';
+import { BudgetEvaluator } from '../services/budget-evaluator.js';
+import { createRunId, createTaskId } from '../types/index.js';
 
 /**
  * Dependencies required by the BenchmarkEngine
@@ -78,6 +88,20 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     this.progressManager = dependencies.progressManager;
   }
 
+  /**
+   * Generate a unique run ID
+   *
+   * Uses crypto.randomBytes for cryptographically random 7-character IDs.
+   * Format: 7 lowercase alphanumeric characters (e.g., "k3m9x2p")
+   */
+  private static generateRunId(this: void): RunId {
+    // Generate random bytes, convert to hex, then to base36, take first 7 chars
+    const hex = randomBytes(4).toString('hex');
+    const num = parseInt(hex, 16);
+    const id = num.toString(36).padStart(7, '0').substring(0, 7);
+    return createRunId(id);
+  }
+
   /**
    * Discover benchmark files matching the pattern(s)
    */
@@ -136,7 +160,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       }
 
       // 4. Initialize progress tracking
-      const runId = this.generateRunId();
+      const runId = ModestBenchEngine.generateRunId();
 
       // Pre-calculate total tasks for progress tracking
       let totalTasks = 0;
@@ -212,9 +236,9 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
       // Register progress callbacks with reporters that support them
       for (const reporter of reporters) {
-        if (typeof reporter.onProgress === 'function') {
+        if (reporter.onProgress) {
           this.progressManager.onProgress((state) => {
-            void reporter.onProgress(state);
+            void reporter.onProgress?.(state);
           });
         }
       }
@@ -227,12 +251,17 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
       for (const filePath of files) {
         try {
-          // Call reporter onFileStart
-          await this.callReporters(reporters, 'onFileStart', filePath);
+          // Normalize file path to be relative to cwd
+          const cwd = config.cwd || process.cwd();
+          const relativePath = pathRelative(cwd, filePath);
+
+          // Call reporter onFileStart with relative path
+          await this.callReporters(reporters, 'onFileStart', relativePath);
 
           const fileResult = await this.executeBenchmarkFile(
             filePath,
             mergedConfig,
+            cwd,
             reporters,
             signal,
           );
@@ -246,6 +275,16 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
             currentFile: filePath,
             filesCompleted: fileResults.length,
           });
+
+          // Check for bail: stop execution if any task failed
+          if (mergedConfig.bail) {
+            const hasFailedTask = fileResult.suites.some((suite) =>
+              suite.tasks.some((task) => task.error),
+            );
+            if (hasFailedTask) {
+              break;
+            }
+          }
         } catch (error) {
           const fileError =
             error instanceof Error ? error : new Error(String(error));
@@ -267,6 +306,11 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
           // Call reporter onFileEnd for error case
           await this.callReporters(reporters, 'onFileEnd', errorResult);
+
+          // Check bail flag for file-level errors
+          if (mergedConfig.bail) {
+            break;
+          }
         }
       }
 
@@ -302,10 +346,103 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       }
 
       const overallMean = totalOperations > 0 ? totalTime / totalOperations : 0;
+      // Evaluate budgets if configured
+      let budgetSummary: BudgetSummary | undefined;
+
+      if (config.budgets && Object.keys(config.budgets).length > 0) {
+        const evaluator = new BudgetEvaluator();
+        const baselineStorage = new BaselineStorageService(process.cwd());
+
+        // Collect task results
+        const taskResults = new Map<TaskId, TaskResult>();
+
+        for (const file of fileResults) {
+          for (const suite of file.suites) {
+            for (const task of suite.tasks) {
+              if (!task.error) {
+                // file.filePath is already relative to cwd
+                const taskId = createTaskId(
+                  file.filePath,
+                  suite.name,
+                  task.name,
+                );
+                taskResults.set(taskId, task);
+              }
+            }
+          }
+        }
+
+        // Load baseline data if needed for relative budgets
+        let baselineData: Map<TaskId, BaselineSummaryData> | undefined;
+
+        // Check if any budgets use relative thresholds
+        const hasRelativeBudgets = Object.values(config.budgets).some(
+          (budget) => (budget as Budget).relative,
+        );
+
+        if (hasRelativeBudgets) {
+          const baselineName =
+            config.baseline || (await baselineStorage.getDefault());
+
+          if (baselineName) {
+            const baseline = await baselineStorage.getBaseline(baselineName);
+
+            if (baseline) {
+              // Cast keys to TaskId since they come from validated baseline storage
+              baselineData = new Map(
+                Object.entries(baseline.summary) as [
+                  TaskId,
+                  BaselineSummaryData,
+                ][],
+              );
+            } else {
+              console.warn(
+                `Warning: Baseline "${baselineName}" not found. Relative budgets will be skipped.`,
+              );
+            }
+          } else {
+            console.warn(
+              'Warning: Relative budgets configured but no baseline specified. Relative budgets will be skipped.',
+            );
+          }
+        }
+
+        // Evaluate budgets
+        budgetSummary = evaluator.evaluateRun(
+          config.budgets as Record<string, Budget>,
+          taskResults,
+          baselineData,
+        );
+
+        // Notify reporters of budget results
+        for (const reporter of reporters) {
+          if (reporter.onBudgetResult) {
+            await reporter.onBudgetResult(budgetSummary);
+          }
+        }
+
+        // Handle budget failures based on budgetMode
+        if (budgetSummary.failed > 0) {
+          const mode = config.budgetMode || 'fail';
+
+          if (mode === 'fail') {
+            throw new BudgetExceededError(
+              `${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
+              budgetSummary,
+            );
+          } else if (mode === 'warn') {
+            console.warn(
+              `Warning: ${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
+            );
+          }
+          // mode === 'report': just include in output, don't fail
+        }
+      }
 
       const endTime = new Date();
       const finalRun: BenchmarkRun = {
         ...initialRun,
+        budgetSummary,
         duration: endTime.getTime() - startTime.getTime(),
         endTime,
         files: fileResults,
@@ -489,6 +626,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
   private async executeBenchmarkFile(
     filePath: string,
     config: ModestBenchConfig,
+    cwd: string,
     reporters: Reporter[] = [],
     signal?: AbortSignal,
   ): Promise<FileResult> {
@@ -543,11 +681,14 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
       const endTime = new Date();
 
+      // Normalize file path to be relative to cwd
+      const relativePath = pathRelative(cwd, filePath);
+
       return {
         config: benchmarkDef.config,
         duration: endTime.getTime() - startTime.getTime(),
         endTime,
-        filePath,
+        filePath: relativePath,
         startTime,
         suites: suiteResults,
       };
@@ -556,11 +697,14 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       const executionError =
         error instanceof Error ? error : new Error(String(error));
 
+      // Normalize file path to be relative to cwd
+      const relativePath = pathRelative(cwd, filePath);
+
       return {
         duration: endTime.getTime() - startTime.getTime(),
         endTime,
         error: executionError,
-        filePath,
+        filePath: relativePath,
         startTime,
         suites: [],
       };
@@ -697,19 +841,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     }
   }
 
-  /**
-   * Generate a unique run ID
-   *
-   * Uses crypto.randomBytes for cryptographically random 7-character IDs.
-   * Format: 7 lowercase alphanumeric characters (e.g., "k3m9x2p")
-   */
-  private generateRunId(): string {
-    // Generate random bytes, convert to hex, then to base36, take first 7 chars
-    const hex = randomBytes(4).toString('hex');
-    const num = parseInt(hex, 16);
-    return num.toString(36).padStart(7, '0').substring(0, 7);
-  }
-
   /**
    * Get CI/CD information if available
    */

package/src/core/output-path-resolver.ts

@@ -1,4 +1,4 @@
-import { isAbsolute, join, resolve } from 'node:path';
+import { extname, isAbsolute, join, resolve } from 'node:path';
 
 /**
  * Resolves the final output path for a reporter
@@ -29,7 +29,15 @@ export const resolveOutputPath = (
     return resolve(process.cwd(), outputFile);
   }
 
-  // Fall back to default behavior
+  // If outputDir looks like a file (has extension), treat it as a file path
+  // This handles cases like: --output results.csv
+  if (outputDir && extname(outputDir)) {
+    return isAbsolute(outputDir)
+      ? outputDir
+      : resolve(process.cwd(), outputDir);
+  }
+
+  // Fall back to default behavior (outputDir is a directory)
   if (outputDir && defaultFilename) {
     return join(outputDir, defaultFilename);
   }

package/src/errors/base.ts

@@ -143,10 +143,19 @@ export const isModestBenchError = (
   error: unknown,
 ): error is ModestBenchError => {
   return (
-    typeof error === 'object' &&
-    error !== null &&
+    isError(error) &&
     'code' in error &&
     typeof (error as { code: unknown }).code === 'string' &&
     (error as { code: string }).code.startsWith('ERR_MB_')
   );
 };
+
+/**
+ * Type guard to check if an error is a standard Error
+ *
+ * @param error - The error to check
+ * @returns `true` if the error is an `Error`
+ */
+export const isError = (error: unknown): error is Error => {
+  return error instanceof Error;
+};

package/src/errors/budget.ts

@@ -0,0 +1,38 @@
+/**
+ * Budget-related errors
+ *
+ * Errors that occur during budget evaluation and enforcement.
+ */
+
+import type { BudgetSummary } from '../types/core.js';
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Error thrown when performance budgets are exceeded
+ *
+ * Thrown when budget evaluation fails and budgetMode is set to 'fail'. Contains
+ * the full budget summary for detailed reporting.
+ */
+export class BudgetExceededError extends ModestBenchError {
+  /**
+   * Budget summary containing details of all violations
+   */
+  public readonly budgetSummary: BudgetSummary;
+
+  /**
+   * Error code for budget exceeded errors
+   */
+  readonly code = 'ERR_MB_BUDGET_EXCEEDED';
+
+  /**
+   * Create a new budget exceeded error
+   *
+   * @param message - Human-readable error message
+   * @param budgetSummary - Budget evaluation results
+   */
+  constructor(message: string, budgetSummary: BudgetSummary) {
+    super(message);
+    this.budgetSummary = budgetSummary;
+  }
+}

package/src/errors/index.ts

@@ -13,6 +13,9 @@ export {
   ModestBenchError,
 } from './base.js';
 
+// Budget errors
+export { BudgetExceededError } from './budget.js';
+
 // CLI errors
 export {
   InvalidArgumentError,

package/src/index.ts

@@ -26,11 +26,17 @@ export * from './errors/index.js';
 export { CsvReporter } from './reporters/csv.js';
 export { HumanReporter } from './reporters/human.js';
 export { JsonReporter } from './reporters/json.js';
+export { ProfileHumanReporter } from './reporters/profile-human.js';
 
 // Services
 export { ModestBenchConfigurationManager } from './services/config-manager.js';
 export { BenchmarkFileLoader } from './services/file-loader.js';
 export { FileHistoryStorage } from './services/history-storage.js';
+// Profiler services
+export { filterProfile } from './services/profiler/profile-filter.js';
+export { parseProfile } from './services/profiler/profile-parser.js';
+
+export { runWithProfiling } from './services/profiler/profile-runner.js';
 export { ModestBenchProgressManager } from './services/progress-manager.js';
 export {
   BaseReporter,
@@ -40,3 +46,6 @@ export {
 
 // Export all types
 export * from './types/index.js';
+
+// Utilities
+export { findPackageRoot } from './utils/package.js';

package/src/reporters/csv.ts

@@ -11,20 +11,24 @@ import { dirname } from 'node:path';
 
 import type {
   BenchmarkRun,
-  FileResult,
-  ProgressState,
-  SuiteResult,
+  BudgetResult,
+  BudgetSummary,
+  TaskId,
   TaskResult,
 } from '../types/index.js';
 
 import { ReporterOutputError } from '../errors/index.js';
 import { BaseReporter } from '../services/reporter-registry.js';
+import { createTaskId } from '../types/index.js';
 
 /**
  * CSV column definitions for task results
  */
 interface CsvRow {
   readonly arch: string;
+  /** Budget passed: 1 (pass), 0 (fail), undefined (no budget) */
+  readonly budgetPassed?: number | undefined;
+  readonly budgetViolations?: string | undefined;
   readonly ciProvider?: string | undefined;
   readonly cpuCores: number;
   readonly cpuModel: string;
@@ -54,6 +58,8 @@ interface CsvRow {
  * CSV reporter for structured tabular output
  */
 export class CsvReporter extends BaseReporter {
+  private budgetResults: Map<TaskId, BudgetResult> = new Map();
+
   private currentFile = '';
 
   private currentRun?: BenchmarkRun;
@@ -68,8 +74,6 @@ export class CsvReporter extends BaseReporter {
 
   private readonly outputPath?: string | undefined;
 
-  private readonly quiet: boolean;
-
   private readonly quote: string;
 
   private rows: CsvRow[] = [];
@@ -92,7 +96,6 @@ export class CsvReporter extends BaseReporter {
     this.includeMetadata = options.includeMetadata ?? true;
     this.delimiter = options.delimiter ?? ',';
     this.quote = options.quote ?? '"';
-    this.quiet = options.quiet ?? false;
   }
 
   /**
@@ -137,6 +140,31 @@ export class CsvReporter extends BaseReporter {
     return this.includeMetadata;
   }
 
+  onBudgetResult(summary: BudgetSummary): void {
+    // Store budget results indexed by taskId
+    for (const result of summary.results) {
+      this.budgetResults.set(result.taskId, result);
+    }
+
+    // Update existing rows with budget data (since onTaskResult is called before onBudgetResult)
+    for (const row of this.rows) {
+      // row.file is already relative to cwd
+      const taskId = createTaskId(row.file, row.suite, row.task);
+      const budgetResult = this.budgetResults.get(taskId);
+      if (budgetResult) {
+        // Need to cast to mutable to update readonly properties
+        const mutableRow = row as {
+          budgetPassed?: number;
+          budgetViolations?: string;
+        };
+        mutableRow.budgetPassed = budgetResult.passed ? 1 : 0;
+        mutableRow.budgetViolations = budgetResult.violations
+          .map((v) => v.type)
+          .join('; ');
+      }
+    }
+  }
+
   async onEnd(_run: BenchmarkRun): Promise<void> {
     const csvContent = this.generateCsv();
 
@@ -151,27 +179,15 @@ export class CsvReporter extends BaseReporter {
     console.error('CSV Reporter Error:', error.message);
   }
 
-  onFileEnd(_result: FileResult): void {
-    // No-op for CSV reporter
-  }
-
   onFileStart(file: string): void {
     this.currentFile = file;
   }
 
-  onProgress(_state: ProgressState): void {
-    // No-op for CSV reporter
-  }
-
   onStart(run: BenchmarkRun): void {
     this.currentRun = run;
     this.rows = [];
   }
 
-  onSuiteEnd(_result: SuiteResult): void {
-    // No-op for CSV reporter
-  }
-
   onSuiteStart(suite: string): void {
     this.currentSuite = suite;
   }
@@ -181,8 +197,21 @@ export class CsvReporter extends BaseReporter {
       return;
     }
 
+    // Look up budget result for this task
+    // this.currentFile is already relative to cwd (comes from engine)
+    const taskId = createTaskId(
+      this.currentFile,
+      this.currentSuite,
+      result.name,
+    );
+    const budgetResult = this.budgetResults.get(taskId);
+
     const row: CsvRow = {
       arch: this.currentRun.environment.arch,
+      budgetPassed: budgetResult ? (budgetResult.passed ? 1 : 0) : undefined,
+      budgetViolations: budgetResult
+        ? budgetResult.violations.map((v) => v.type).join('; ')
+        : undefined,
       ciProvider: this.currentRun.ci?.provider,
       cpuCores: this.currentRun.environment.cpu.cores,
       cpuModel: this.currentRun.environment.cpu.model,
@@ -211,10 +240,6 @@ export class CsvReporter extends BaseReporter {
     this.rows.push(row);
   }
 
-  onTaskStart(_task: string): void {
-    // No-op for CSV reporter
-  }
-
   /**
    * Escape a field value for CSV format
    */
@@ -277,6 +302,8 @@ export class CsvReporter extends BaseReporter {
       'p95',
       'p99',
       'error',
+      'budgetPassed',
+      'budgetViolations',
       'timestamp',
     ];
 
@@ -302,9 +329,9 @@ export class CsvReporter extends BaseReporter {
    */
   private generateRow(row: CsvRow): string {
     const values = [
-      row.file || '',
-      row.suite || '',
-      row.task || '',
+      row.file,
+      row.suite,
+      row.task,
       (row.mean ?? 0).toString(),
       (row.stdDev ?? 0).toString(),
       (row.min ?? 0).toString(),
@@ -316,6 +343,8 @@ export class CsvReporter extends BaseReporter {
       (row.p95 ?? 0).toString(),
       (row.p99 ?? 0).toString(),
       row.error || '',
+      row.budgetPassed !== undefined ? row.budgetPassed.toString() : '',
+      row.budgetViolations || '',
       row.timestamp || '',
     ];