modestbench 0.7.0 → 0.9.0

package/src/constants.ts

@@ -110,6 +110,8 @@ export const ErrorCodes = {
 
   //#region reporter-errors
   REPORTER_ALREADY_REGISTERED: 'ERR_MB_REPORTER_ALREADY_REGISTERED',
+  REPORTER_INVALID: 'ERR_MB_REPORTER_INVALID',
+  REPORTER_LOAD_FAILED: 'ERR_MB_REPORTER_LOAD_FAILED',
   REPORTER_OUTPUT_FAILED: 'ERR_MB_REPORTER_OUTPUT_FAILED',
   REPORTER_UNKNOWN: 'ERR_MB_REPORTER_UNKNOWN',
   //#endregion

package/src/core/engine.ts

@@ -16,7 +16,6 @@ import type {
   BenchmarkRun,
   BenchmarkSuite,
   BenchmarkTask,
-  Budget,
   BudgetSummary,
   CiInfo,
   ConfigurationManager,
@@ -354,93 +353,99 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       // Evaluate budgets if configured
       let budgetSummary: BudgetSummary | undefined;
 
-      if (config.budgets && Object.keys(config.budgets).length > 0) {
-        const evaluator = new BudgetEvaluator();
-        const baselineStorage = new BaselineStorageService(process.cwd());
-
-        // Collect task results
-        const taskResults = new Map<TaskId, TaskResult>();
-
-        for (const file of fileResults) {
-          for (const suite of file.suites) {
-            for (const task of suite.tasks) {
-              if (!task.error) {
-                // file.filePath is already relative to cwd
-                const taskId = createTaskId(
-                  file.filePath,
-                  suite.name,
-                  task.name,
-                );
-                taskResults.set(taskId, task);
+      if (config.budgets) {
+        const budgets = config.budgets;
+        const hasBudgets =
+          Object.keys(budgets.exact).length > 0 || budgets.patterns.length > 0;
+
+        if (hasBudgets) {
+          const evaluator = new BudgetEvaluator();
+          const baselineStorage = new BaselineStorageService(process.cwd());
+
+          // Collect task results
+          const taskResults = new Map<TaskId, TaskResult>();
+
+          for (const file of fileResults) {
+            for (const suite of file.suites) {
+              for (const task of suite.tasks) {
+                if (!task.error) {
+                  // file.filePath is already relative to cwd
+                  const taskId = createTaskId(
+                    file.filePath,
+                    suite.name,
+                    task.name,
+                  );
+                  taskResults.set(taskId, task);
+                }
               }
             }
           }
-        }
 
-        // Load baseline data if needed for relative budgets
-        let baselineData: Map<TaskId, BaselineSummaryData> | undefined;
+          // Load baseline data if needed for relative budgets
+          let baselineData: Map<TaskId, BaselineSummaryData> | undefined;
 
-        // Check if any budgets use relative thresholds
-        const hasRelativeBudgets = Object.values(config.budgets).some(
-          (budget) => (budget as Budget).relative,
-        );
+          // Check if any budgets use relative thresholds
+          const hasRelativeBudgets =
+            Object.values(budgets.exact).some((budget) => budget.relative) ||
+            budgets.patterns.some((pattern) => pattern.budget.relative);
 
-        if (hasRelativeBudgets) {
-          const baselineName =
-            config.baseline || (await baselineStorage.getDefault());
+          if (hasRelativeBudgets) {
+            const baselineName =
+              config.baseline || (await baselineStorage.getDefault());
 
-          if (baselineName) {
-            const baseline = await baselineStorage.getBaseline(baselineName);
+            if (baselineName) {
+              const baseline = await baselineStorage.getBaseline(baselineName);
 
-            if (baseline) {
-              // Cast keys to TaskId since they come from validated baseline storage
-              baselineData = new Map(
-                Object.entries(baseline.summary) as [
-                  TaskId,
-                  BaselineSummaryData,
-                ][],
-              );
+              if (baseline) {
+                // Cast keys to TaskId since they come from validated baseline storage
+                baselineData = new Map(
+                  Object.entries(baseline.summary) as [
+                    TaskId,
+                    BaselineSummaryData,
+                  ][],
+                );
+              } else {
+                console.warn(
+                  `Warning: Baseline "${baselineName}" not found. Relative budgets will be skipped.`,
+                );
+              }
             } else {
               console.warn(
-                `Warning: Baseline "${baselineName}" not found. Relative budgets will be skipped.`,
+                'Warning: Relative budgets configured but no baseline specified. Relative budgets will be skipped.',
               );
             }
-          } else {
-            console.warn(
-              'Warning: Relative budgets configured but no baseline specified. Relative budgets will be skipped.',
-            );
           }
-        }
 
-        // Evaluate budgets
-        budgetSummary = evaluator.evaluateRun(
-          config.budgets as Record<string, Budget>,
-          taskResults,
-          baselineData,
-        );
+          // Evaluate budgets
+          budgetSummary = evaluator.evaluateRun(
+            budgets,
+            taskResults,
+            baselineData,
+          );
 
-        // Notify reporters of budget results
-        for (const reporter of reporters) {
-          if (reporter.onBudgetResult) {
-            await reporter.onBudgetResult(budgetSummary);
+          // Notify reporters of budget results
+          for (const reporter of reporters) {
+            if (reporter.onBudgetResult) {
+              await reporter.onBudgetResult(budgetSummary);
+            }
           }
-        }
 
-        // Handle budget failures based on budgetMode
-        if (budgetSummary.failed > 0) {
-          const mode = config.budgetMode || 'fail';
+          // Handle budget failures based on budgetMode
+          if (budgetSummary.failed > 0) {
+            const mode = config.budgetMode || 'fail';
 
-          if (mode === 'fail') {
-            throw new BudgetExceededError(
-              `${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
-              budgetSummary,
-            );
-          } else if (mode === 'warn') {
-            console.warn(
-              `Warning: ${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
-            );
+            if (mode === 'fail') {
+              throw new BudgetExceededError(
+                `${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
+                budgetSummary,
+              );
+            } else if (mode === 'warn') {
+              console.warn(
+                `Warning: ${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`,
+              );
+            }
+            // mode === 'report': just include in output, don't fail
           }
-          // mode === 'report': just include in output, don't fail
         }
       }
 

package/src/core/output-path-resolver.ts

@@ -1,5 +1,19 @@
 import { extname, isAbsolute, join, resolve } from 'node:path';
 
+/**
+ * Generates a timestamped filename for benchmark output files.
+ *
+ * @param extension - File extension without the dot (e.g., 'json', 'csv')
+ * @returns Filename in format `benchmarks-YYYY-MM-DD-HH-MM-SS.{extension}` (UTC
+ *   time)
+ */
+export const generateTimestampedFilename = (extension: string): string => {
+  const now = new Date();
+  const pad = (n: number) => n.toString().padStart(2, '0');
+  const timestamp = `${now.getUTCFullYear()}-${pad(now.getUTCMonth() + 1)}-${pad(now.getUTCDate())}-${pad(now.getUTCHours())}-${pad(now.getUTCMinutes())}-${pad(now.getUTCSeconds())}`;
+  return `benchmarks-${timestamp}.${extension}`;
+};
+
 /**
  * Resolves the final output path for a reporter
  *

package/src/errors/index.ts

@@ -53,7 +53,9 @@ export {
 // Reporter errors
 export {
   ReporterAlreadyRegisteredError,
+  ReporterLoadError,
   ReporterOutputError,
+  ReporterValidationError,
   UnknownReporterError,
 } from './reporter.js';
 

package/src/errors/reporter.ts

@@ -16,6 +16,26 @@ export class ReporterAlreadyRegisteredError extends ModestBenchError {
   readonly code = 'ERR_MB_REPORTER_ALREADY_REGISTERED';
 }
 
+/**
+ * Reporter load failed
+ *
+ * Thrown when a reporter module cannot be loaded (file not found, syntax error,
+ * invalid module format, etc.).
+ */
+export class ReporterLoadError extends ModestBenchError {
+  readonly code = 'ERR_MB_REPORTER_LOAD_FAILED';
+
+  /**
+   * The specifier (file path or package name) that failed to load
+   */
+  readonly specifier: string;
+
+  constructor(message: string, specifier: string, options?: ErrorOptions) {
+    super(`Failed to load reporter "${specifier}": ${message}`, options);
+    this.specifier = specifier;
+  }
+}
+
 /**
  * Reporter output failed
  *
@@ -25,6 +45,41 @@ export class ReporterOutputError extends ModestBenchError {
   readonly code = 'ERR_MB_REPORTER_OUTPUT_FAILED';
 }
 
+/**
+ * Reporter validation failed
+ *
+ * Thrown when a loaded module does not implement the required Reporter
+ * interface methods.
+ */
+export class ReporterValidationError extends ModestBenchError {
+  readonly code = 'ERR_MB_REPORTER_INVALID';
+
+  /**
+   * The methods that are missing from the reporter
+   */
+  readonly missingMethods: string[];
+
+  /**
+   * The specifier (file path or package name) of the invalid reporter
+   */
+  readonly specifier: string;
+
+  constructor(
+    message: string,
+    specifier: string,
+    missingMethods: string[] = [],
+    options?: ErrorOptions,
+  ) {
+    const methodsInfo =
+      missingMethods.length > 0
+        ? ` Missing required methods: ${missingMethods.join(', ')}.`
+        : '';
+    super(`Invalid reporter "${specifier}": ${message}${methodsInfo}`, options);
+    this.specifier = specifier;
+    this.missingMethods = missingMethods;
+  }
+}
+
 /**
  * Unknown reporter
  *

package/src/index.ts

@@ -41,6 +41,15 @@ export { parseProfile } from './services/profiler/profile-parser.js';
 
 export { runWithProfiling } from './services/profiler/profile-runner.js';
 export { ModestBenchProgressManager } from './services/progress-manager.js';
+// Reporter plugin loader
+export {
+  createReporterContext,
+  isBuiltInReporter,
+  isFilePath,
+  loadReporter,
+  PLUGIN_API_VERSION,
+} from './services/reporter-loader.js';
+
 export {
   BaseReporter,
   CompositeReporter,
@@ -51,4 +60,13 @@ export {
 export * from './types/index.js';
 
 // Utilities
-export { findPackageRoot } from './utils/package.js';
+export { findPackageRoot, getPackageVersion } from './utils/package.js';
+
+// Reporter utilities (for plugin authors)
+export {
+  formatBytes,
+  formatDuration,
+  formatOpsPerSecond,
+  formatPercentage,
+  reporterUtils,
+} from './utils/reporter-utils.js';

package/src/reporters/json.ts

@@ -90,7 +90,7 @@ export class JsonReporter extends BaseReporter {
     super('json', options);
 
     this.outputPath = options.outputPath;
-    this.prettyPrint = options.prettyPrint ?? true;
+    this.prettyPrint = options.prettyPrint ?? false;
     this.includeStatistics = options.includeStatistics ?? true;
     this.includeMetadata = options.includeMetadata ?? true;
   }

package/src/services/budget-evaluator.ts

@@ -4,10 +4,13 @@ import type {
   BudgetResult,
   BudgetSummary,
   BudgetViolation,
+  ResolvedBudgets,
   TaskId,
   TaskResult,
 } from '../types/core.js';
 
+import { resolveBudget } from './budget-resolver.js';
+
 /**
  * Service for evaluating performance budgets
  *
@@ -49,30 +52,31 @@ export class BudgetEvaluator {
    * Evaluate budgets for an entire benchmark run
    */
   evaluateRun(
-    budgets: Record<string, Budget>,
+    budgets: ResolvedBudgets,
     taskResults: Map<TaskId, TaskResult>,
     baselineData?: Map<TaskId, BaselineSummaryData>,
   ): BudgetSummary {
     const results: BudgetResult[] = [];
 
-    for (const [taskId, budget] of Object.entries(budgets)) {
-      const taskResult = taskResults.get(taskId as TaskId);
+    for (const [taskId, taskResult] of taskResults) {
+      const budget = resolveBudget(taskId, budgets);
 
-      // Skip if no result for this task
-      if (!taskResult) {
+      // Skip if no budget matches this task
+      if (!budget) {
         continue;
       }
 
-      // Skip relative budgets if no baseline data
-      if (budget.relative && !baselineData) {
+      // Skip if budget has ONLY relative thresholds and no baseline data
+      // (absolute budgets can still be evaluated without baseline)
+      if (budget.relative && !budget.absolute && !baselineData) {
         continue;
       }
 
       const budgetResult = this.evaluateTask(
-        taskId as TaskId,
+        taskId,
         budget,
         taskResult,
-        baselineData?.get(taskId as TaskId),
+        baselineData?.get(taskId),
       );
 
       results.push(budgetResult);

package/src/services/budget-resolver.ts

@@ -0,0 +1,254 @@
+/**
+ * Budget resolution service for matching budgets to tasks
+ *
+ * This module provides pattern-based budget resolution using:
+ *
+ * - Minimatch glob patterns for file matching
+ * - Simple `*` wildcards for suite/task matching
+ *
+ * @packageDocumentation
+ */
+
+import { minimatch } from 'minimatch';
+
+import type {
+  Budget,
+  BudgetPattern,
+  ResolvedBudgets,
+  TaskId,
+} from '../types/core.js';
+
+/**
+ * Check if a glob pattern contains wildcards
+ *
+ * @param pattern - The pattern to check
+ * @returns True if the pattern contains glob metacharacters
+ */
+export const isGlobPattern = (pattern: string): boolean => {
+  return /[*?[\]]/.test(pattern);
+};
+
+/**
+ * Check if a file path matches a glob pattern
+ *
+ * @param pattern - Minimatch glob pattern
+ * @param filePath - File path to match against
+ * @returns True if the pattern matches the file path
+ */
+export const matchesFile = (pattern: string, filePath: string): boolean => {
+  // Exact match fast path
+  if (pattern === filePath) {
+    return true;
+  }
+
+  return minimatch(filePath, pattern, { matchBase: true });
+};
+
+/**
+ * Check if a suite or task name matches a pattern
+ *
+ * @param pattern - Either an exact name or `*` for wildcard
+ * @param value - The value to match against
+ * @returns True if the pattern matches the value
+ */
+export const matchesSuiteOrTask = (pattern: string, value: string): boolean => {
+  return pattern === '*' || pattern === value;
+};
+
+/**
+ * Calculate specificity score for a budget pattern
+ *
+ * Higher scores indicate more specific patterns. Scoring:
+ *
+ * - File: +2 for exact match, +1 for glob with specific parts, +0 for `**\/*`
+ * - Suite: +1 for exact match, +0 for `*`
+ * - Task: +1 for exact match, +0 for `*`
+ *
+ * @param pattern - The budget pattern to score
+ * @returns Specificity score (0-4)
+ */
+export const calculateSpecificity = (
+  pattern: Pick<BudgetPattern, 'filePattern' | 'suitePattern' | 'taskPattern'>,
+): number => {
+  let score = 0;
+
+  // File specificity
+  if (!isGlobPattern(pattern.filePattern)) {
+    // Exact file match
+    score += 2;
+  } else if (pattern.filePattern !== '**/*' && pattern.filePattern !== '*') {
+    // Glob with some specificity (e.g., "**/api/**/*.bench.js")
+    score += 1;
+  }
+  // else: fully generic glob like "**/*" gets +0
+
+  // Suite specificity
+  if (pattern.suitePattern !== '*') {
+    score += 1;
+  }
+
+  // Task specificity
+  if (pattern.taskPattern !== '*') {
+    score += 1;
+  }
+
+  return score;
+};
+
+/**
+ * Parse a TaskId into its components
+ *
+ * TaskIds have the format: `{filePath}/{suiteName}/{taskName}` The file path
+ * can contain slashes, so we parse from the end.
+ *
+ * @param taskId - The TaskId to parse
+ * @returns Object with file, suite, and task components
+ */
+export const parseTaskId = (
+  taskId: TaskId,
+): { file: string; suite: string; task: string } => {
+  const str = taskId as string;
+  const lastSlash = str.lastIndexOf('/');
+  const secondLastSlash = str.lastIndexOf('/', lastSlash - 1);
+
+  return {
+    file: str.substring(0, secondLastSlash),
+    suite: str.substring(secondLastSlash + 1, lastSlash),
+    task: str.substring(lastSlash + 1),
+  };
+};
+
+/**
+ * Deep merge two budget objects
+ *
+ * More specific (second) budget values override less specific (first) values.
+ * Merges at the absolute/relative level.
+ *
+ * @param base - Base budget (less specific)
+ * @param override - Override budget (more specific)
+ * @returns Merged budget
+ */
+export const mergeBudgets = (base: Budget, override: Budget): Budget => {
+  return {
+    ...(base.absolute || override.absolute
+      ? {
+          absolute: {
+            ...base.absolute,
+            ...override.absolute,
+          },
+        }
+      : {}),
+    ...(base.relative || override.relative
+      ? {
+          relative: {
+            ...base.relative,
+            ...override.relative,
+          },
+        }
+      : {}),
+  };
+};
+
+/**
+ * Resolve the appropriate budget for a task
+ *
+ * Resolution order:
+ *
+ * 1. Check exact match in `exact` map (highest priority)
+ * 2. Find all matching patterns
+ * 3. Sort by specificity (ascending)
+ * 4. Merge matched budgets (more specific overrides less specific)
+ *
+ * @param taskId - The task identifier to resolve a budget for
+ * @param budgets - The resolved budgets structure
+ * @returns The resolved budget, or undefined if no budget matches
+ */
+export const resolveBudget = (
+  taskId: TaskId,
+  budgets: ResolvedBudgets,
+): Budget | undefined => {
+  // Fast path: exact match
+  const exactMatch = budgets.exact[taskId as string];
+  if (exactMatch) {
+    // Still need to check patterns and merge if there are less-specific matches
+    const parsed = parseTaskId(taskId);
+    const matchingPatterns = budgets.patterns.filter(
+      (p) =>
+        matchesFile(p.filePattern, parsed.file) &&
+        matchesSuiteOrTask(p.suitePattern, parsed.suite) &&
+        matchesSuiteOrTask(p.taskPattern, parsed.task),
+    );
+
+    if (matchingPatterns.length === 0) {
+      return exactMatch;
+    }
+
+    // Sort by specificity ascending (least specific first, so more specific can override)
+    const sorted = [...matchingPatterns].sort(
+      (a, b) => a.specificity - b.specificity,
+    );
+
+    // Merge all patterns, then apply exact match last
+    let merged = sorted[0]!.budget;
+    for (let i = 1; i < sorted.length; i++) {
+      merged = mergeBudgets(merged, sorted[i]!.budget);
+    }
+
+    // Exact match has highest priority
+    return mergeBudgets(merged, exactMatch);
+  }
+
+  // Find all matching patterns
+  const parsed = parseTaskId(taskId);
+  const matchingPatterns = budgets.patterns.filter(
+    (p) =>
+      matchesFile(p.filePattern, parsed.file) &&
+      matchesSuiteOrTask(p.suitePattern, parsed.suite) &&
+      matchesSuiteOrTask(p.taskPattern, parsed.task),
+  );
+
+  if (matchingPatterns.length === 0) {
+    return undefined;
+  }
+
+  // Sort by specificity ascending (least specific first)
+  const sorted = [...matchingPatterns].sort(
+    (a, b) => a.specificity - b.specificity,
+  );
+
+  // Merge all matches (more specific overrides less specific)
+  let merged = sorted[0]!.budget;
+  for (let i = 1; i < sorted.length; i++) {
+    merged = mergeBudgets(merged, sorted[i]!.budget);
+  }
+
+  return merged;
+};
+
+/**
+ * Create a BudgetPattern from its components
+ *
+ * @param filePattern - Glob pattern for file matching
+ * @param suitePattern - Suite name or `*` for wildcard
+ * @param taskPattern - Task name or `*` for wildcard
+ * @param budget - The budget to apply
+ * @returns A BudgetPattern with computed specificity
+ */
+export const createBudgetPattern = (
+  filePattern: string,
+  suitePattern: string,
+  taskPattern: string,
+  budget: Budget,
+): BudgetPattern => {
+  return {
+    budget,
+    filePattern,
+    specificity: calculateSpecificity({
+      filePattern,
+      suitePattern,
+      taskPattern,
+    }),
+    suitePattern,
+    taskPattern,
+  };
+};

package/src/services/file-loader.ts

@@ -18,11 +18,11 @@ import type {
   ValidationWarning,
 } from '../types/index.js';
 
+import { benchmarkFileSchema } from '../config/benchmark-schema.js';
 import {
   BENCHMARK_FILE_EXTENSIONS,
   BENCHMARK_FILE_PATTERN,
 } from '../constants.js';
-import { benchmarkFileSchema } from '../core/benchmark-schema.js';
 import {
   FileDiscoveryError,
   FileLoadError,