modestbench 0.1.0 → 0.3.0

package/src/services/budget-evaluator.ts

@@ -0,0 +1,182 @@
+import type {
+  BaselineSummaryData,
+  Budget,
+  BudgetResult,
+  BudgetSummary,
+  BudgetViolation,
+  TaskId,
+  TaskResult,
+} from '../types/core.js';
+
+/**
+ * Service for evaluating performance budgets
+ *
+ * @packageDocumentation
+ */
+export class BudgetEvaluator {
+  /**
+   * Format number with thousands separators
+   */
+  private static formatNumber(this: void, value: number): string {
+    return value.toLocaleString('en-US', {
+      maximumFractionDigits: 0,
+    });
+  }
+
+  /**
+   * Format decimal as percentage
+   */
+  private static formatPercentage(this: void, value: number): string {
+    return `${(value * 100).toFixed(1)}%`;
+  }
+
+  /**
+   * Format time in nanoseconds to human-readable string
+   */
+  private static formatTime(this: void, nanoseconds: number): string {
+    if (nanoseconds < 1_000) {
+      return `${nanoseconds.toFixed(0)}ns`;
+    } else if (nanoseconds < 1_000_000) {
+      return `${(nanoseconds / 1_000).toFixed(2)}μs`;
+    } else if (nanoseconds < 1_000_000_000) {
+      return `${(nanoseconds / 1_000_000).toFixed(2)}ms`;
+    } else {
+      return `${(nanoseconds / 1_000_000_000).toFixed(2)}s`;
+    }
+  }
+
+  /**
+   * Evaluate budgets for an entire benchmark run
+   */
+  evaluateRun(
+    budgets: Record<string, Budget>,
+    taskResults: Map<TaskId, TaskResult>,
+    baselineData?: Map<TaskId, BaselineSummaryData>,
+  ): BudgetSummary {
+    const results: BudgetResult[] = [];
+
+    for (const [taskId, budget] of Object.entries(budgets)) {
+      const taskResult = taskResults.get(taskId as TaskId);
+
+      // Skip if no result for this task
+      if (!taskResult) {
+        continue;
+      }
+
+      // Skip relative budgets if no baseline data
+      if (budget.relative && !baselineData) {
+        continue;
+      }
+
+      const budgetResult = this.evaluateTask(
+        taskId as TaskId,
+        budget,
+        taskResult,
+        baselineData?.get(taskId as TaskId),
+      );
+
+      results.push(budgetResult);
+    }
+
+    const passed = results.filter((r) => r.passed).length;
+    const failed = results.filter((r) => !r.passed).length;
+
+    return {
+      failed,
+      passed,
+      results,
+      total: results.length,
+    };
+  }
+
+  /**
+   * Evaluate budgets for a single task
+   */
+  private evaluateTask(
+    taskId: TaskId,
+    budget: Budget,
+    actual: TaskResult,
+    baseline?: BaselineSummaryData,
+  ): BudgetResult {
+    const violations: BudgetViolation[] = [];
+
+    // Evaluate absolute budgets
+    if (budget.absolute) {
+      if (budget.absolute.maxTime !== undefined) {
+        if (actual.mean > budget.absolute.maxTime) {
+          violations.push({
+            actual: actual.mean,
+            delta:
+              (actual.mean - budget.absolute.maxTime) / budget.absolute.maxTime,
+            message: `Mean execution time ${BudgetEvaluator.formatTime(actual.mean)} exceeded budget of ${BudgetEvaluator.formatTime(budget.absolute.maxTime)} by ${BudgetEvaluator.formatPercentage((actual.mean - budget.absolute.maxTime) / budget.absolute.maxTime)}`,
+            threshold: budget.absolute.maxTime,
+            type: 'maxTime',
+          });
+        }
+      }
+
+      if (budget.absolute.minOpsPerSec !== undefined) {
+        if (actual.opsPerSecond < budget.absolute.minOpsPerSec) {
+          violations.push({
+            actual: actual.opsPerSecond,
+            delta:
+              (budget.absolute.minOpsPerSec - actual.opsPerSecond) /
+              budget.absolute.minOpsPerSec,
+            message: `Operations per second ${BudgetEvaluator.formatNumber(actual.opsPerSecond)} is below minimum of ${BudgetEvaluator.formatNumber(budget.absolute.minOpsPerSec)} by ${BudgetEvaluator.formatPercentage((budget.absolute.minOpsPerSec - actual.opsPerSecond) / budget.absolute.minOpsPerSec)}`,
+            threshold: budget.absolute.minOpsPerSec,
+            type: 'minOpsPerSec',
+          });
+        }
+      }
+
+      if (budget.absolute.maxP99 !== undefined && actual.p99 !== undefined) {
+        if (actual.p99 > budget.absolute.maxP99) {
+          violations.push({
+            actual: actual.p99,
+            delta:
+              (actual.p99 - budget.absolute.maxP99) / budget.absolute.maxP99,
+            message: `P99 latency ${BudgetEvaluator.formatTime(actual.p99)} exceeded budget of ${BudgetEvaluator.formatTime(budget.absolute.maxP99)} by ${BudgetEvaluator.formatPercentage((actual.p99 - budget.absolute.maxP99) / budget.absolute.maxP99)}`,
+            threshold: budget.absolute.maxP99,
+            type: 'maxP99',
+          });
+        }
+      }
+    }
+
+    // Evaluate relative budgets
+    if (budget.relative && baseline) {
+      if (budget.relative.maxRegression !== undefined) {
+        const regression = (actual.mean - baseline.mean) / baseline.mean;
+
+        if (regression > budget.relative.maxRegression) {
+          violations.push({
+            actual: regression,
+            delta: regression - budget.relative.maxRegression,
+            message: `Performance regressed by ${BudgetEvaluator.formatPercentage(regression)} exceeding maximum allowed regression of ${BudgetEvaluator.formatPercentage(budget.relative.maxRegression)}`,
+            threshold: budget.relative.maxRegression,
+            type: 'maxRegression',
+          });
+        }
+      }
+    }
+
+    return {
+      actual: {
+        mean: actual.mean,
+        opsPerSecond: actual.opsPerSecond,
+        p99: actual.p99,
+      },
+      baseline: baseline
+        ? {
+            mean: baseline.mean,
+            opsPerSecond: baseline.opsPerSecond,
+            p99: baseline.p99,
+          }
+        : undefined,
+      budget,
+      passed: violations.length === 0,
+      taskId,
+      violations,
+    };
+  }
+}

package/src/{config/manager.ts → services/config-manager.ts}

@@ -18,9 +18,9 @@ import type {
   ValidationWarning,
 } from '../types/index.js';
 
+import { safeParseConfig } from '../config/schema.js';
 import { ErrorCodes } from '../constants.js';
 import { ConfigLoadError, ConfigValidationError } from '../errors/index.js';
-import { safeParseConfig } from './schema.js';
 
 /**
  * Get the default reporter based on TTY status and environment
@@ -56,7 +56,7 @@ const DEFAULT_CONFIG: ModestBenchConfig = {
   limitBy: 'iterations', // Default to limiting by iteration count
   metadata: {},
   outputDir: './benchmark-results',
-  pattern: '**/*.bench.{js,ts,mjs,cjs,mts,cts}',
+  pattern: 'bench/**/*.bench.{js,ts,mjs,cjs,mts,cts}', // Search bench/ directory recursively
   quiet: false,
   reporterConfig: {},
   reporters: [getDefaultReporter()],
@@ -121,10 +121,16 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
 
   /**
    * Load configuration from various sources with precedence
+   *
+   * @param configPath - Optional path to configuration file
+   * @param cliArgs - Optional CLI arguments to merge
+   * @param commandDefaults - Command-specific defaults (fallback to
+   *   DEFAULT_CONFIG)
    */
   async load(
     configPath?: string,
     cliArgs?: Record<string, unknown>,
+    commandDefaults?: Partial<ModestBenchConfig>,
   ): Promise<ModestBenchConfig> {
     try {
       // Create a fresh explorer for each load to avoid module caching issues
@@ -164,9 +170,13 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
 
       const fileConfig = (result?.config || {}) as Partial<ModestBenchConfig>;
 
-      // 2. Merge: defaults <- file <- CLI args
+      // 2. Merge: command defaults <- file <- CLI args
+      // Use command-specific defaults if provided, otherwise use DEFAULT_CONFIG
+      const baseDefaults = commandDefaults
+        ? this.merge(DEFAULT_CONFIG, commandDefaults)
+        : DEFAULT_CONFIG;
       const normalizedCliArgs = cliArgs ? this.normalizeCliArgs(cliArgs) : {};
-      const merged = this.merge(DEFAULT_CONFIG, fileConfig, normalizedCliArgs);
+      const merged = this.merge(baseDefaults, fileConfig, normalizedCliArgs);
 
       // 2.5. Apply smart defaults for limitBy if not explicitly provided
       const finalConfig = ModestBenchConfigurationManager.applySmartDefaults(
@@ -175,15 +185,20 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
         fileConfig,
       );
 
-      // 3. Validate final configuration
-      const validation = this.validate(finalConfig);
-      if (!validation.valid) {
+      // 3. Validate final configuration and get transformed config
+      // The validation also transforms budgets from nested to flat format
+      const validation = safeParseConfig(finalConfig);
+      if (!validation.success) {
+        const errors = validation.error.issues.map((issue) => {
+          const path = issue.path.join('.');
+          return `${path ? `${path}: ` : ''}${issue.message}`;
+        });
         throw new ConfigValidationError(
-          `Configuration validation failed: ${validation.errors.map((e) => e.message).join(', ')}`,
+          `Configuration validation failed: ${errors.join(', ')}`,
         );
       }
 
-      return finalConfig;
+      return validation.data;
     } catch (error) {
       // Re-throw our custom errors
       if (

package/src/{core/loader.ts → services/file-loader.ts}

@@ -22,12 +22,12 @@ import {
   BENCHMARK_FILE_EXTENSIONS,
   BENCHMARK_FILE_PATTERN,
 } from '../constants.js';
+import { benchmarkFileSchema } from '../core/benchmark-schema.js';
 import {
   FileDiscoveryError,
   FileLoadError,
   StructureValidationError,
 } from '../errors/index.js';
-import { benchmarkFileSchema } from './benchmark-schema.js';
 
 /**
  * File change notification for watch functionality
@@ -64,8 +64,7 @@ export class BenchmarkFileLoader implements FileLoader {
       // Handle empty patterns - use sensible defaults
       if (patterns.length === 0) {
         patterns = [
-          `*${BENCHMARK_FILE_PATTERN}`, // top-level current directory
-          `bench/*${BENCHMARK_FILE_PATTERN}`, // top-level bench/ directory
+          `bench/**/*${BENCHMARK_FILE_PATTERN}`, // bench/ directory (recursive)
         ];
       }
 
@@ -152,10 +151,8 @@ export class BenchmarkFileLoader implements FileLoader {
           default?: unknown;
         };
       } else {
-        // Use native dynamic import for JavaScript files with cache busting
-        // Add timestamp to prevent module caching issues across multiple loads
-        const timestamp = Date.now();
-        module = (await import(`${filePath}?t=${timestamp}`)) as {
+        // Use native dynamic import for JavaScript files
+        module = (await import(filePath)) as {
           [key: string]: unknown;
           default?: unknown;
         };

package/src/services/history/comparison.ts

@@ -0,0 +1,130 @@
+/**
+ * Benchmark Comparison Service
+ *
+ * Handles comparison logic between two benchmark runs, calculating performance
+ * differences and categorizing tasks.
+ */
+
+import type { BenchmarkRun } from '../../types/index.js';
+import type { CompareResult, TaskComparison } from './models.js';
+
+/**
+ * Service for comparing benchmark runs
+ */
+export class ComparisonService {
+  /**
+   * Compare two benchmark runs and produce detailed comparison result
+   */
+  compareRuns(run1: BenchmarkRun, run2: BenchmarkRun): CompareResult {
+    // Build task maps for comparison
+    const tasksMap1 = new Map<string, TaskComparison>();
+    const tasksMap2 = new Map<string, TaskComparison>();
+
+    // Extract tasks from run1
+    for (const file of run1.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const key = `${file.filePath}::${suite.name}::${task.name}`;
+            tasksMap1.set(key, {
+              file: file.filePath,
+              inBoth: false,
+              percentChange: 0,
+              run1: {
+                cv: task.cv,
+                iterations: task.iterations,
+                max: task.max,
+                mean: task.mean,
+                min: task.min,
+              },
+              suite: suite.name,
+              task: task.name,
+            });
+          }
+        }
+      }
+    }
+
+    // Extract tasks from run2 and merge
+    for (const file of run2.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const key = `${file.filePath}::${suite.name}::${task.name}`;
+            const existing = tasksMap1.get(key);
+
+            if (existing && existing.run1) {
+              // Task exists in both runs - calculate comparison
+              const percentChange =
+                ((task.mean - existing.run1.mean) / existing.run1.mean) * 100;
+
+              tasksMap1.set(key, {
+                ...existing,
+                inBoth: true,
+                percentChange,
+                run2: {
+                  cv: task.cv,
+                  iterations: task.iterations,
+                  max: task.max,
+                  mean: task.mean,
+                  min: task.min,
+                },
+              });
+            } else {
+              // Task only in run2
+              tasksMap2.set(key, {
+                file: file.filePath,
+                inBoth: false,
+                percentChange: 0,
+                run2: {
+                  cv: task.cv,
+                  iterations: task.iterations,
+                  max: task.max,
+                  mean: task.mean,
+                  min: task.min,
+                },
+                suite: suite.name,
+                task: task.name,
+              });
+            }
+          }
+        }
+      }
+    }
+
+    // Separate tasks into categories
+    const tasksInBoth: TaskComparison[] = [];
+    const tasksOnlyIn1: TaskComparison[] = [];
+    const tasksOnlyIn2: TaskComparison[] = [];
+
+    for (const task of tasksMap1.values()) {
+      if (task.inBoth) {
+        tasksInBoth.push(task);
+      } else {
+        tasksOnlyIn1.push(task);
+      }
+    }
+
+    for (const task of tasksMap2.values()) {
+      tasksOnlyIn2.push(task);
+    }
+
+    return {
+      run1: {
+        endTime: run1.endTime,
+        id: run1.id,
+        startTime: run1.startTime,
+        summary: run1.summary,
+      },
+      run2: {
+        endTime: run2.endTime,
+        id: run2.id,
+        startTime: run2.startTime,
+        summary: run2.summary,
+      },
+      tasksInBoth,
+      tasksOnlyInRun1: tasksOnlyIn1,
+      tasksOnlyInRun2: tasksOnlyIn2,
+    };
+  }
+}

package/src/services/history/models.ts

@@ -0,0 +1,148 @@
+/**
+ * Data Models for History Services
+ *
+ * Type definitions for history command data structures, shared between
+ * services, formatters, and CLI handlers.
+ */
+
+import type { BenchmarkRun } from '../../types/index.js';
+
+/**
+ * Result from comparing two benchmark runs
+ */
+export interface CompareResult {
+  run1: {
+    endTime: Date;
+    id: string;
+    startTime: Date;
+    summary: {
+      failedTasks: number;
+      passedTasks: number;
+      totalFiles: number;
+      totalTasks: number;
+    };
+  };
+  run2: {
+    endTime: Date;
+    id: string;
+    startTime: Date;
+    summary: {
+      failedTasks: number;
+      passedTasks: number;
+      totalFiles: number;
+      totalTasks: number;
+    };
+  };
+  tasksInBoth: TaskComparison[];
+  tasksOnlyInRun1: TaskComparison[];
+  tasksOnlyInRun2: TaskComparison[];
+}
+
+/**
+ * Distribution bucket for histogram visualization
+ */
+export interface DistributionBucket {
+  count: number;
+  label: string;
+  max: number;
+  min: number;
+}
+
+/**
+ * Result from listing historical runs
+ */
+export interface HistoryListResult {
+  runs: Array<{
+    duration: number;
+    id: string;
+    startTime: Date;
+    summary: {
+      failedTasks: number;
+      passedTasks: number;
+      totalFiles: number;
+      totalTasks: number;
+    };
+  }>;
+  totalCount: number;
+}
+
+/**
+ * Show command result (just wraps BenchmarkRun for consistency)
+ */
+export type ShowResult = BenchmarkRun;
+
+/**
+ * Task comparison result between two runs
+ */
+export interface TaskComparison {
+  file: string;
+  inBoth: boolean;
+  percentChange: number;
+  run1?: {
+    cv: number;
+    iterations: number;
+    max: number;
+    mean: number;
+    min: number;
+  };
+  run2?: {
+    cv: number;
+    iterations: number;
+    max: number;
+    mean: number;
+    min: number;
+  };
+  suite: string;
+  task: string;
+}
+
+/**
+ * Complete trend analysis for a single task
+ */
+export interface TrendData {
+  confidence: number;
+  dataPoints: TrendDataPoint[];
+  percentChange: number;
+  runs: number;
+  statistics: TrendStatistics;
+  task: string;
+  trend: 'degrading' | 'improving' | 'stable';
+}
+
+/**
+ * Single data point in a trend series
+ */
+export interface TrendDataPoint {
+  date: Date;
+  mean: number;
+}
+
+/**
+ * Result from analyzing performance trends
+ */
+export interface TrendsResult {
+  lowConfidenceRegressions: TrendData[];
+  regressions: TrendData[];
+  runs: number;
+  summary: {
+    degradingTasks: number;
+    improvingTasks: number;
+    stableTasks: number;
+    totalTasks: number;
+  };
+  timespan: {
+    end: Date;
+    start: Date;
+  };
+  trends: TrendData[];
+}
+
+/**
+ * Statistical metrics for trend analysis
+ */
+export interface TrendStatistics {
+  mean: number;
+  median: number;
+  stdDeviation: number;
+  variance: number;
+}

package/src/services/history/query.ts

@@ -0,0 +1,116 @@
+/**
+ * History Query Service
+ *
+ * Handles querying benchmark run history with date parsing and filtering.
+ */
+
+import type {
+  BenchmarkRun,
+  HistoryQuery,
+  HistoryStorage,
+} from '../../types/index.js';
+
+/**
+ * Service for querying historical benchmark runs
+ */
+export class HistoryQueryService {
+  constructor(private readonly storage: HistoryStorage) {}
+
+  /**
+   * Query runs with automatic date string parsing
+   */
+  async queryWithDateParsing(options: {
+    limit?: number;
+    pattern?: string;
+    since?: string;
+    tags?: string[];
+    until?: string;
+  }): Promise<BenchmarkRun[]> {
+    // Build query object all at once
+    const query: Partial<HistoryQuery> = {
+      ...(options.since && { since: parseDate(options.since) }),
+      ...(options.until && { until: parseDate(options.until) }),
+      ...(options.pattern && { pattern: options.pattern }),
+      ...(options.tags && options.tags.length > 0 && { tags: options.tags }),
+      ...(options.limit && { limit: options.limit }),
+    };
+
+    return await this.storage.queryRuns(query);
+  }
+}
+
+/**
+ * Parse date string (ISO 8601 or relative)
+ *
+ * Supports:
+ *
+ * - ISO 8601: "2025-10-24T12:00:00Z", "2025-10-24"
+ * - Relative: "1 day ago", "3 weeks ago", "2 hours ago"
+ * - Shorthand: "1d", "2w", "3m", "6h"
+ *
+ * @param dateStr - Date string to parse
+ * @returns Parsed Date object
+ * @throws Error if date format is invalid
+ */
+export const parseDate = (dateStr: string): Date => {
+  if (!dateStr || dateStr.trim() === '') {
+    throw new Error(`Invalid date format: "${dateStr}"`);
+  }
+
+  // Try parsing as ISO 8601 first
+  const isoDate = new Date(dateStr);
+  if (!isNaN(isoDate.getTime())) {
+    return isoDate;
+  }
+
+  // Parse relative dates like "1 week ago", "3 days ago"
+  const relativePattern = /^(\d+)\s+(hour|day|week|month)s?\s+ago$/i;
+  const relativeMatch = dateStr.trim().match(relativePattern);
+
+  if (relativeMatch && relativeMatch[1] && relativeMatch[2]) {
+    const amount = parseInt(relativeMatch[1], 10);
+    const unit = relativeMatch[2].toLowerCase();
+
+    if (amount <= 0) {
+      throw new Error(`Invalid date format: "${dateStr}"`);
+    }
+
+    const now = new Date();
+    const msPerUnit: Record<string, number> = {
+      day: 24 * 60 * 60 * 1000,
+      hour: 60 * 60 * 1000,
+      month: 30 * 24 * 60 * 60 * 1000, // Approximate
+      week: 7 * 24 * 60 * 60 * 1000,
+    };
+
+    const offset = amount * (msPerUnit[unit] || 0);
+    return new Date(now.getTime() - offset);
+  }
+
+  // Parse shorthand formats like "1d", "2w", "3m", "6h"
+  // cspell:ignore hdwm
+  const shorthandPattern = /^(\d+)([hdwm])$/i;
+  const shorthandMatch = dateStr.trim().match(shorthandPattern);
+
+  if (shorthandMatch && shorthandMatch[1] && shorthandMatch[2]) {
+    const amount = parseInt(shorthandMatch[1], 10);
+    const unit = shorthandMatch[2].toLowerCase();
+
+    if (amount <= 0) {
+      throw new Error(`Invalid date format: "${dateStr}"`);
+    }
+
+    const now = new Date();
+    const msPerUnit: Record<string, number> = {
+      d: 24 * 60 * 60 * 1000,
+      h: 60 * 60 * 1000,
+      m: 30 * 24 * 60 * 60 * 1000, // Approximate month
+      w: 7 * 24 * 60 * 60 * 1000,
+    };
+
+    const offset = amount * (msPerUnit[unit] || 0);
+    return new Date(now.getTime() - offset);
+  }
+
+  throw new Error(`Invalid date format: "${dateStr}"`);
+};