modestbench 0.1.0 → 0.3.0

package/src/core/engine.ts

@@ -6,12 +6,18 @@
  * architecture.
  */
 
+import { randomBytes } from 'node:crypto';
+import { relative as pathRelative } from 'node:path';
+
 import type {
+  BaselineSummaryData,
   BenchmarkDefinition,
   BenchmarkEngine,
   BenchmarkRun,
   BenchmarkSuite,
   BenchmarkTask,
+  Budget,
+  BudgetSummary,
   CiInfo,
   ConfigurationManager,
   EnvironmentInfo,
@@ -24,7 +30,9 @@ import type {
   Reporter,
   ReporterRegistry,
   RunConfiguration,
+  RunId,
   SuiteResult,
+  TaskId,
   TaskResult,
   ValidationError,
   ValidationResult,
@@ -33,11 +41,15 @@ import type {
 
 import {
   BenchmarkExecutionError,
+  BudgetExceededError,
   FileDiscoveryError,
   SchemaValidationError,
   SetupError,
   StructureValidationError,
 } from '../errors/index.js';
+import { BaselineStorageService } from '../services/baseline-storage.js';
+import { BudgetEvaluator } from '../services/budget-evaluator.js';
+import { createRunId, createTaskId } from '../types/index.js';
 
 /**
  * Dependencies required by the BenchmarkEngine
@@ -76,6 +88,20 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     this.progressManager = dependencies.progressManager;
   }
 
+  /**
+   * Generate a unique run ID
+   *
+   * Uses crypto.randomBytes for cryptographically random 7-character IDs.
+   * Format: 7 lowercase alphanumeric characters (e.g., "k3m9x2p")
+   */
+  private static generateRunId(this: void): RunId {
+    // Generate random bytes, convert to hex, then to base36, take first 7 chars
+    const hex = randomBytes(4).toString('hex');
+    const num = parseInt(hex, 16);
+    const id = num.toString(36).padStart(7, '0').substring(0, 7);
+    return createRunId(id);
+  }
+
   /**
    * Discover benchmark files matching the pattern(s)
    */
@@ -134,7 +160,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       }
 
       // 4. Initialize progress tracking
-      const runId = this.generateRunId();
+      const runId = ModestBenchEngine.generateRunId();
 
       // Pre-calculate total tasks for progress tracking
       let totalTasks = 0;
@@ -210,9 +236,9 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
       // Register progress callbacks with reporters that support them
       for (const reporter of reporters) {
-        if (typeof reporter.onProgress === 'function') {
+        if (reporter.onProgress) {
           this.progressManager.onProgress((state) => {
-            void reporter.onProgress(state);
+            void reporter.onProgress?.(state);
           });
         }
       }
@@ -225,12 +251,17 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
       for (const filePath of files) {
         try {
-          // Call reporter onFileStart
-          await this.callReporters(reporters, 'onFileStart', filePath);
+          // Normalize file path to be relative to cwd
+          const cwd = config.cwd || process.cwd();
+          const relativePath = pathRelative(cwd, filePath);
+
+          // Call reporter onFileStart with relative path
+          await this.callReporters(reporters, 'onFileStart', relativePath);
 
           const fileResult = await this.executeBenchmarkFile(
             filePath,
             mergedConfig,
+            cwd,
             reporters,
             signal,
           );
@@ -244,6 +275,16 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
             currentFile: filePath,
             filesCompleted: fileResults.length,
           });
+
+          // Check for bail: stop execution if any task failed
+          if (mergedConfig.bail) {
+            const hasFailedTask = fileResult.suites.some((suite) =>
+              suite.tasks.some((task) => task.error),
+            );
+            if (hasFailedTask) {
+              break;
+            }
+          }
         } catch (error) {
           const fileError =
             error instanceof Error ? error : new Error(String(error));
@@ -265,6 +306,11 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
           // Call reporter onFileEnd for error case
           await this.callReporters(reporters, 'onFileEnd', errorResult);
+
+          // Check bail flag for file-level errors
+          if (mergedConfig.bail) {
+            break;
+          }
         }
       }
 
@@ -300,10 +346,103 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       }
 
       const overallMean = totalOperations > 0 ? totalTime / totalOperations : 0;
+      // Evaluate budgets if configured
+      let budgetSummary: BudgetSummary | undefined;
+
+      if (config.budgets && Object.keys(config.budgets).length > 0) {
+        const evaluator = new BudgetEvaluator();
+        const baselineStorage = new BaselineStorageService(process.cwd());
+
+        // Collect task results
+        const taskResults = new Map<TaskId, TaskResult>();
+
+        for (const file of fileResults) {
+          for (const suite of file.suites) {
+            for (const task of suite.tasks) {
+              if (!task.error) {
+                // file.filePath is already relative to cwd
+                const taskId = createTaskId(
+                  file.filePath,
+                  suite.name,
+                  task.name,
+                );
+                taskResults.set(taskId, task);
+              }
+            }
+          }
+        }
+
+        // Load baseline data if needed for relative budgets
+        let baselineData: Map<TaskId, BaselineSummaryData> | undefined;
+
+        // Check if any budgets use relative thresholds
+        const hasRelativeBudgets = Object.values(config.budgets).some(
+          (budget) => (budget as Budget).relative,
+        );
+
+        if (hasRelativeBudgets) {
+          const baselineName =
+            config.baseline || (await baselineStorage.getDefault());
+
+          if (baselineName) {
+            const baseline = await baselineStorage.getBaseline(baselineName);
+
+            if (baseline) {
+              // Cast keys to TaskId since they come from validated baseline storage
+              baselineData = new Map(
+                Object.entries(baseline.summary) as [
+                  TaskId,
+                  BaselineSummaryData,
+                ][],
+              );
+            } else {
+              console.warn(
+                `Warning: Baseline "${baselineName}" not found. Relative budgets will be skipped.`,
+              );
+            }
+          } else {
+            console.warn(
+              'Warning: Relative budgets configured but no baseline specified. Relative budgets will be skipped.',
+            );
+          }
+        }
+
+        // Evaluate budgets
+        budgetSummary = evaluator.evaluateRun(
+          config.budgets as Record<string, Budget>,
+          taskResults,
+          baselineData,
+        );
+
+        // Notify reporters of budget results
+        for (const reporter of reporters) {
+          if (reporter.onBudgetResult) {
+            await reporter.onBudgetResult(budgetSummary);
+          }
+        }
+
+        // Handle budget failures based on budgetMode
+        if (budgetSummary.failed > 0) {
+          const mode = config.budgetMode || 'fail';
+
+          if (mode === 'fail') {
+            throw new BudgetExceededError(
+              `${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
+              budgetSummary,
+            );
+          } else if (mode === 'warn') {
+            console.warn(
+              `Warning: ${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
+            );
+          }
+          // mode === 'report': just include in output, don't fail
+        }
+      }
 
       const endTime = new Date();
       const finalRun: BenchmarkRun = {
         ...initialRun,
+        budgetSummary,
         duration: endTime.getTime() - startTime.getTime(),
         endTime,
         files: fileResults,
@@ -487,6 +626,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
   private async executeBenchmarkFile(
     filePath: string,
     config: ModestBenchConfig,
+    cwd: string,
     reporters: Reporter[] = [],
     signal?: AbortSignal,
   ): Promise<FileResult> {
@@ -541,11 +681,14 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
       const endTime = new Date();
 
+      // Normalize file path to be relative to cwd
+      const relativePath = pathRelative(cwd, filePath);
+
       return {
         config: benchmarkDef.config,
         duration: endTime.getTime() - startTime.getTime(),
         endTime,
-        filePath,
+        filePath: relativePath,
         startTime,
         suites: suiteResults,
       };
@@ -554,11 +697,14 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       const executionError =
         error instanceof Error ? error : new Error(String(error));
 
+      // Normalize file path to be relative to cwd
+      const relativePath = pathRelative(cwd, filePath);
+
       return {
         duration: endTime.getTime() - startTime.getTime(),
         endTime,
         error: executionError,
-        filePath,
+        filePath: relativePath,
         startTime,
         suites: [],
       };
@@ -695,13 +841,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     }
   }
 
-  /**
-   * Generate a unique run ID
-   */
-  private generateRunId(): string {
-    return `run-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
-  }
-
   /**
    * Get CI/CD information if available
    */

package/src/core/output-path-resolver.ts

@@ -0,0 +1,46 @@
+import { extname, isAbsolute, join, resolve } from 'node:path';
+
+/**
+ * Resolves the final output path for a reporter
+ *
+ * @param outputDir - Optional output directory from --output flag
+ * @param outputFile - Optional output filename from --output-file flag
+ * @param defaultFilename - Default filename to use if none specified
+ * @returns Resolved output path, or undefined if no output to file requested
+ */
+export const resolveOutputPath = (
+  outputDir?: string,
+  outputFile?: string,
+  defaultFilename?: string,
+): string | undefined => {
+  // If outputFile is provided
+  if (outputFile) {
+    // If outputFile is absolute, use as-is
+    if (isAbsolute(outputFile)) {
+      return outputFile;
+    }
+
+    // If outputDir specified, join them
+    if (outputDir) {
+      return join(outputDir, outputFile);
+    }
+
+    // Otherwise, resolve relative to cwd
+    return resolve(process.cwd(), outputFile);
+  }
+
+  // If outputDir looks like a file (has extension), treat it as a file path
+  // This handles cases like: --output results.csv
+  if (outputDir && extname(outputDir)) {
+    return isAbsolute(outputDir)
+      ? outputDir
+      : resolve(process.cwd(), outputDir);
+  }
+
+  // Fall back to default behavior (outputDir is a directory)
+  if (outputDir && defaultFilename) {
+    return join(outputDir, defaultFilename);
+  }
+
+  return undefined;
+};

package/src/errors/base.ts

@@ -143,10 +143,19 @@ export const isModestBenchError = (
   error: unknown,
 ): error is ModestBenchError => {
   return (
-    typeof error === 'object' &&
-    error !== null &&
+    isError(error) &&
     'code' in error &&
     typeof (error as { code: unknown }).code === 'string' &&
     (error as { code: string }).code.startsWith('ERR_MB_')
   );
 };
+
+/**
+ * Type guard to check if an error is a standard Error
+ *
+ * @param error - The error to check
+ * @returns `true` if the error is an `Error`
+ */
+export const isError = (error: unknown): error is Error => {
+  return error instanceof Error;
+};

package/src/errors/budget.ts

@@ -0,0 +1,38 @@
+/**
+ * Budget-related errors
+ *
+ * Errors that occur during budget evaluation and enforcement.
+ */
+
+import type { BudgetSummary } from '../types/core.js';
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Error thrown when performance budgets are exceeded
+ *
+ * Thrown when budget evaluation fails and budgetMode is set to 'fail'. Contains
+ * the full budget summary for detailed reporting.
+ */
+export class BudgetExceededError extends ModestBenchError {
+  /**
+   * Budget summary containing details of all violations
+   */
+  public readonly budgetSummary: BudgetSummary;
+
+  /**
+   * Error code for budget exceeded errors
+   */
+  readonly code = 'ERR_MB_BUDGET_EXCEEDED';
+
+  /**
+   * Create a new budget exceeded error
+   *
+   * @param message - Human-readable error message
+   * @param budgetSummary - Budget evaluation results
+   */
+  constructor(message: string, budgetSummary: BudgetSummary) {
+    super(message);
+    this.budgetSummary = budgetSummary;
+  }
+}

package/src/errors/index.ts

@@ -13,6 +13,9 @@ export {
   ModestBenchError,
 } from './base.js';
 
+// Budget errors
+export { BudgetExceededError } from './budget.js';
+
 // CLI errors
 export {
   InvalidArgumentError,

package/src/formatters/history/base.ts

@@ -0,0 +1,28 @@
+/**
+ * Base Formatter Interface
+ *
+ * Defines the contract for history command formatters. Each formatter
+ * transforms processed data into human-readable or machine-readable output.
+ */
+
+/**
+ * Formatter interface for history command output
+ *
+ * @template TData - The data type this formatter accepts
+ */
+export interface HistoryFormatter<TData> {
+  /**
+   * Format data as CSV (optional, not all commands support CSV)
+   */
+  formatCsv?(data: TData): string;
+
+  /**
+   * Format data for human-readable terminal output
+   */
+  formatHuman(data: TData): string;
+
+  /**
+   * Format data as JSON
+   */
+  formatJson(data: TData): string;
+}

package/src/formatters/history/compare.ts

@@ -0,0 +1,186 @@
+/**
+ * Compare Formatter
+ *
+ * Formats benchmark run comparison results in human and JSON formats.
+ */
+
+import type { CompareResult } from '../../services/history/models.js';
+import type { HistoryFormatter } from './base.js';
+
+import { colorize } from '../../utils/ansi.js';
+import { ansiChars } from '../../utils/ansi.js';
+
+/**
+ * Formatter for history compare command
+ */
+export class HistoryCompareFormatter
+  implements HistoryFormatter<CompareResult>
+{
+  /**
+   * Format as human-readable comparison
+   */
+  formatHuman(data: CompareResult): string {
+    const lines: string[] = [];
+
+    lines.push(colorize('brightMagenta', colorize('bold', 'Comparing runs:')));
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('brightWhite', colorize('bold', 'Run 1'))} ${colorize('dim', data.run1.id)} (${colorize('white', data.run1.startTime.toLocaleString())})`,
+    );
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('brightWhite', colorize('bold', 'Run 2'))} ${colorize('dim', data.run2.id)} (${colorize('white', data.run2.startTime.toLocaleString())})`,
+    );
+    lines.push('');
+
+    lines.push(colorize('cyan', 'Summary comparison:'));
+    lines.push('');
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Files: ${colorize('brightWhite', String(data.run1.summary.totalFiles))} vs ${colorize('brightWhite', String(data.run2.summary.totalFiles))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Tasks: ${colorize('brightWhite', String(data.run1.summary.totalTasks))} vs ${colorize('brightWhite', String(data.run2.summary.totalTasks))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Passed: ${colorize('brightCyan', String(data.run1.summary.passedTasks))} vs ${colorize('brightCyan', String(data.run2.summary.passedTasks))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Failed: ${colorize('brightRed', colorize('bold', String(data.run1.summary.failedTasks)))} vs ${colorize('brightRed', colorize('bold', String(data.run2.summary.failedTasks)))}`,
+    );
+    lines.push('');
+
+    // Detailed task comparison
+    if (data.tasksInBoth.length > 0) {
+      lines.push(colorize('cyan', 'Task-by-task comparison:'));
+      lines.push('');
+
+      for (const comparison of data.tasksInBoth) {
+        const mean1 = comparison.run1!.mean / 1000000; // Convert to ms
+        const mean2 = comparison.run2!.mean / 1000000;
+        const changeSign = comparison.percentChange >= 0 ? '+' : '';
+        const changeStr = `${changeSign}${comparison.percentChange.toFixed(1)}%`;
+
+        lines.push(
+          `  ${colorize('brightWhite', `${comparison.suite} › ${comparison.task}`)}`,
+        );
+
+        // Mean - highlight higher (slower/worse) number
+        const meanHigher = mean2 > mean1;
+        const mean1Str = meanHigher
+          ? colorize('magenta', `${mean1.toFixed(3)}ms`)
+          : colorize('brightMagenta', `${mean1.toFixed(3)}ms`);
+        const mean2Str = meanHigher
+          ? colorize('brightMagenta', `${mean2.toFixed(3)}ms`)
+          : colorize('magenta', `${mean2.toFixed(3)}ms`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Mean:')} ${mean1Str} ${colorize('dim', '→')} ${mean2Str} ${colorize('dim', `(${colorize('white', changeStr)}`)}`,
+        );
+
+        // Min - highlight higher number
+        const min1 = comparison.run1!.min / 1000000;
+        const min2 = comparison.run2!.min / 1000000;
+        const minHigher = min2 > min1;
+        const min1Str = minHigher
+          ? colorize('magenta', `${min1.toFixed(3)}ms`)
+          : colorize('brightMagenta', `${min1.toFixed(3)}ms`);
+        const min2Str = minHigher
+          ? colorize('brightMagenta', `${min2.toFixed(3)}ms`)
+          : colorize('magenta', `${min2.toFixed(3)}ms`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Min:')}  ${min1Str} ${colorize('dim', '→')} ${min2Str}`,
+        );
+
+        // Max - highlight higher number
+        const max1 = comparison.run1!.max / 1000000;
+        const max2 = comparison.run2!.max / 1000000;
+        const maxHigher = max2 > max1;
+        const max1Str = maxHigher
+          ? colorize('magenta', `${max1.toFixed(3)}ms`)
+          : colorize('brightMagenta', `${max1.toFixed(3)}ms`);
+        const max2Str = maxHigher
+          ? colorize('brightMagenta', `${max2.toFixed(3)}ms`)
+          : colorize('magenta', `${max2.toFixed(3)}ms`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Max:')}  ${max1Str} ${colorize('dim', '→')} ${max2Str}`,
+        );
+
+        // Iterations - highlight higher number
+        const iter1 = comparison.run1!.iterations;
+        const iter2 = comparison.run2!.iterations;
+        const iterHigher = iter2 > iter1;
+        const iter1Str = iterHigher
+          ? colorize('brightWhite', String(iter1))
+          : colorize('bold', colorize('brightWhite', String(iter1)));
+        const iter2Str = iterHigher
+          ? colorize('bold', colorize('brightWhite', String(iter2)))
+          : colorize('brightWhite', String(iter2));
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Iterations:')} ${iter1Str} ${colorize('dim', 'vs')} ${iter2Str}`,
+        );
+
+        // CV (Coefficient of Variation) - shows measurement consistency
+        const cv1 = comparison.run1!.cv * 100;
+        const cv2 = comparison.run2!.cv * 100;
+        const cvHigher = cv2 > cv1;
+        const cv1Str = cvHigher
+          ? colorize('magenta', `${cv1.toFixed(2)}%`)
+          : colorize('brightMagenta', `${cv1.toFixed(2)}%`);
+        const cv2Str = cvHigher
+          ? colorize('brightMagenta', `${cv2.toFixed(2)}%`)
+          : colorize('magenta', `${cv2.toFixed(2)}%`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'CV:')}  ${cv1Str} ${colorize('dim', '→')} ${cv2Str}`,
+        );
+
+        lines.push('');
+      }
+    }
+
+    if (data.tasksOnlyInRun1.length > 0) {
+      lines.push(
+        colorize(
+          'cyan',
+          `Tasks only in run 1 (${data.tasksOnlyInRun1.length}):`,
+        ),
+      );
+      for (const task of data.tasksOnlyInRun1) {
+        lines.push(
+          `  ${colorize('dim', ansiChars.bullet)} ${colorize('brightWhite', `${task.suite} › ${task.task}`)}`,
+        );
+      }
+      lines.push('');
+    }
+
+    if (data.tasksOnlyInRun2.length > 0) {
+      lines.push(
+        colorize(
+          'cyan',
+          `Tasks only in run 2 (${data.tasksOnlyInRun2.length}):`,
+        ),
+      );
+      for (const task of data.tasksOnlyInRun2) {
+        lines.push(
+          `  ${colorize('dim', ansiChars.bullet)} ${colorize('brightWhite', `${task.suite} › ${task.task}`)}`,
+        );
+      }
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format as JSON
+   */
+  formatJson(data: CompareResult): string {
+    return JSON.stringify(
+      {
+        run1: data.run1,
+        run2: data.run2,
+        taskComparisons: data.tasksInBoth,
+        tasksOnlyInRun1: data.tasksOnlyInRun1,
+        tasksOnlyInRun2: data.tasksOnlyInRun2,
+      },
+      null,
+      2,
+    );
+  }
+}