modestbench 0.0.3 → 0.2.0

package/src/formatters/history/trends.ts

@@ -0,0 +1,281 @@
+/**
+ * Trends Formatter
+ *
+ * Formats performance trend analysis with visualizations in human and JSON
+ * formats.
+ */
+
+import { stripVTControlCharacters } from 'node:util';
+
+import type {
+  DistributionBucket,
+  TrendsResult,
+} from '../../services/history/models.js';
+import type { HistoryFormatter } from './base.js';
+
+import { colorize } from '../../utils/ansi.js';
+import { generateBarChart, generateSparkline } from './visualization.js';
+
+/**
+ * Nanoseconds per millisecond conversion constant
+ */
+const NS_PER_MS = 1_000_000;
+
+/**
+ * Nanoseconds per microsecond conversion constant
+ */
+const NS_PER_US = 1_000;
+
+/**
+ * Intelligently format a time range with appropriate precision Displays
+ * microseconds for small values (< 1ms) for better clarity
+ */
+const formatTimeRange = (minNs: number, maxNs: number): string => {
+  // If both values are below 1ms, show in microseconds for better precision
+  if (maxNs < NS_PER_MS) {
+    const minUs = minNs / NS_PER_US;
+    const maxUs = maxNs / NS_PER_US;
+    return `${minUs.toFixed(2)}-${maxUs.toFixed(2)}µs`;
+  }
+
+  // Otherwise show in milliseconds
+  const minMs = minNs / NS_PER_MS;
+  const maxMs = maxNs / NS_PER_MS;
+  return `${minMs.toFixed(3)}-${maxMs.toFixed(3)}ms`;
+};
+
+/**
+ * Calculate visible string length, stripping ANSI escape codes
+ */
+const getVisualLength = (str: string): number =>
+  stripVTControlCharacters(str).length;
+
+/**
+ * Formatter for history trends command
+ */
+export class HistoryTrendsFormatter implements HistoryFormatter<TrendsResult> {
+  /**
+   * Format as human-readable trends with visualizations
+   */
+  formatHuman(data: TrendsResult): string {
+    const lines: string[] = [];
+
+    lines.push(
+      colorize(
+        'brightMagenta',
+        colorize('bold', `\nPerformance Trends (${data.runs} runs)`),
+      ),
+    );
+
+    if (data.timespan.start && data.timespan.end) {
+      lines.push(
+        colorize(
+          'dim',
+          `Time range: ${data.timespan.start.toLocaleDateString()} to ${data.timespan.end.toLocaleDateString()}`,
+        ),
+      );
+    }
+    lines.push('');
+
+    // Summary statistics
+    lines.push(colorize('brightBlue', 'Summary:'));
+    lines.push(
+      `  ${colorize('brightCyan', '▲')} ${data.summary.improvingTasks} improving  ${colorize('brightRed', '▼')} ${data.summary.degradingTasks} degrading  ${colorize('dim', '→')} ${data.summary.stableTasks} stable`,
+    );
+    lines.push('');
+
+    // Task Performance Summary Table
+    lines.push(colorize('brightMagenta', 'Task Performance Summary:'));
+    lines.push('');
+
+    for (const trendData of data.trends) {
+      // Show top 20
+      const trendIcon =
+        trendData.trend === 'improving'
+          ? colorize('brightCyan', '▲')
+          : trendData.trend === 'degrading'
+            ? colorize('brightRed', '▼')
+            : colorize('dim', '→');
+
+      const changeColor =
+        trendData.percentChange < -5
+          ? 'brightCyan'
+          : trendData.percentChange > 5
+            ? 'brightRed'
+            : 'dim';
+
+      const changeSign = trendData.percentChange >= 0 ? '+' : '';
+      const changeStr = `${changeSign}${trendData.percentChange.toFixed(1)}%`;
+
+      // Generate sparkline - scale with number of data points (min 12, max 20)
+      const values = trendData.dataPoints.map((dp) => dp.mean);
+      const sparklineWidth = Math.min(20, Math.max(12, trendData.runs));
+      const sparkline = generateSparkline(values, sparklineWidth);
+      const sparklineColor =
+        trendData.trend === 'improving'
+          ? 'brightCyan'
+          : trendData.trend === 'degrading'
+            ? 'brightRed'
+            : 'cyan';
+
+      const taskName = colorize('white', trendData.task);
+      const percentDisplay = colorize(changeColor, changeStr.padStart(8));
+      const sparklineDisplay = colorize(sparklineColor, sparkline);
+
+      // Layout: icon (2) + task name + padding + percent (8) + spaces (2) + sparkline
+      // Position percent+graph at visual column 60 for consistent alignment
+      const prefix = `  ${trendIcon} ${taskName}`;
+      const prefixVisualLength = getVisualLength(prefix);
+      const targetColumn = 60;
+
+      if (prefixVisualLength > targetColumn) {
+        // Task name is too long, wrap to next line
+        lines.push(prefix);
+        lines.push(
+          `${' '.repeat(targetColumn)}${percentDisplay}  ${sparklineDisplay}`,
+        );
+      } else {
+        // Fit on one line with padding
+        const padding = Math.max(1, targetColumn - prefixVisualLength);
+        const paddingStr = ' '.repeat(padding);
+        lines.push(
+          `${prefix}${paddingStr}${percentDisplay}  ${sparklineDisplay}`,
+        );
+      }
+    }
+
+    // Show all trends (no limit)
+
+    lines.push('');
+
+    // Show regressions if any
+    if (data.regressions.length > 0) {
+      lines.push(
+        colorize('brightRed', colorize('bold', 'Regressions Detected:')),
+      );
+      lines.push('');
+
+      for (const regression of data.regressions) {
+        lines.push(
+          `  ${colorize('brightRed', '▼')} ${colorize('white', regression.task)}: ${colorize('brightRed', `${regression.percentChange.toFixed(1)}% slower`)} (${regression.runs} runs)`,
+        );
+      }
+
+      lines.push('');
+    }
+
+    // Show low-confidence regressions (insufficient data)
+    if (data.lowConfidenceRegressions.length > 0) {
+      lines.push(
+        colorize(
+          'brightYellow',
+          colorize('bold', '! Potential Regressions (insufficient data):'),
+        ),
+      );
+      lines.push('');
+
+      for (const regression of data.lowConfidenceRegressions) {
+        lines.push(
+          `  ${colorize('brightYellow', '!')} ${colorize('white', regression.task)}: ${colorize('brightYellow', `${regression.percentChange.toFixed(1)}% slower`)} (${regression.runs} run${regression.runs !== 1 ? 's' : ''})`,
+        );
+      }
+
+      lines.push('');
+    }
+
+    // Show performance distribution for most important task (highest RME) if we have enough data
+    // Find task with highest relative margin of error - the most variable/unreliable one
+    const mostImportantTrend = data.trends
+      .filter((t) => t.runs >= 5)
+      .sort((a, b) => {
+        const rmeA = (a.statistics.stdDeviation / a.statistics.mean) * 100;
+        const rmeB = (b.statistics.stdDeviation / b.statistics.mean) * 100;
+        return rmeB - rmeA; // Descending order (highest RME first)
+      })[0];
+
+    if (mostImportantTrend) {
+      lines.push(
+        colorize(
+          'brightMagenta',
+          'Performance Distribution (most variable task):',
+        ),
+      );
+      lines.push(colorize('white', mostImportantTrend.task));
+      const cv = (
+        (mostImportantTrend.statistics.stdDeviation /
+          mostImportantTrend.statistics.mean) *
+        100
+      ).toFixed(1);
+      lines.push(colorize('dim', `  Variability: ${cv}%`));
+      lines.push('');
+
+      // Create distribution buckets
+      const values = mostImportantTrend.dataPoints.map((dp) => dp.mean);
+      const min = Math.min(...values);
+      const max = Math.max(...values);
+      const range = max - min;
+      const numBuckets = Math.min(5, mostImportantTrend.runs);
+      const bucketSize = range / numBuckets;
+
+      const buckets: DistributionBucket[] = [];
+      for (let i = 0; i < numBuckets; i++) {
+        const bucketMin = min + i * bucketSize;
+        const bucketMax = min + (i + 1) * bucketSize;
+        const count = values.filter(
+          (v) =>
+            v >= bucketMin &&
+            (i === numBuckets - 1 ? v <= bucketMax : v < bucketMax),
+        ).length;
+
+        const label = formatTimeRange(bucketMin, bucketMax);
+
+        buckets.push({
+          count,
+          label,
+          max: bucketMax,
+          min: bucketMin,
+        });
+      }
+
+      // Filter out empty buckets for cleaner display
+      const nonEmptyBuckets = buckets.filter((b) => b.count > 0);
+      const chart = generateBarChart(nonEmptyBuckets, 25);
+      for (const line of chart) {
+        lines.push(colorize('brightCyan', line));
+      }
+
+      lines.push('');
+      lines.push(
+        colorize(
+          'dim',
+          `  Mean: ${(mostImportantTrend.statistics.mean / NS_PER_MS).toFixed(3)}ms  Median: ${(mostImportantTrend.statistics.median / NS_PER_MS).toFixed(3)}ms  StdDev: ${(mostImportantTrend.statistics.stdDeviation / NS_PER_MS).toFixed(3)}ms`,
+        ),
+      );
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format as JSON
+   */
+  formatJson(data: TrendsResult): string {
+    return JSON.stringify(
+      {
+        regressions: data.regressions,
+        summary: {
+          degradingTasks: data.summary.degradingTasks,
+          improvingTasks: data.summary.improvingTasks,
+          runs: data.runs,
+          stableTasks: data.summary.stableTasks,
+          timespan: data.timespan,
+          totalTasks: data.summary.totalTasks,
+        },
+        trends: data.trends,
+      },
+      null,
+      2,
+    );
+  }
+}

package/src/formatters/history/visualization.ts

@@ -0,0 +1,93 @@
+/**
+ * Visualization Utilities for History Formatters
+ *
+ * Pure functions for generating ASCII visualizations (sparklines, bar charts)
+ * for trend analysis and performance distribution displays.
+ */
+
+import type { DistributionBucket } from '../../services/history/models.js';
+
+/**
+ * Generate bar chart histogram from distribution buckets
+ *
+ * @param distribution - Array of distribution buckets with counts
+ * @param maxWidth - Maximum width of bars (default 20)
+ * @returns Array of formatted bar chart lines
+ */
+export const generateBarChart = (
+  distribution: DistributionBucket[],
+  maxWidth = 20,
+): string[] => {
+  if (distribution.length === 0) {
+    return [];
+  }
+
+  // Find maximum count for scaling
+  const maxCount = Math.max(...distribution.map((b) => b.count));
+
+  if (maxCount === 0) {
+    return distribution.map((bucket) => `  ${bucket.label} (0 runs)`);
+  }
+
+  // Block characters for bar visualization
+  const fullBlock = '█';
+  const lightBlock = '░';
+
+  return distribution.map((bucket) => {
+    const ratio = bucket.count / maxCount;
+    const barLength = Math.round(ratio * maxWidth);
+    const fullBlocks = fullBlock.repeat(barLength);
+    const emptyBlocks = lightBlock.repeat(maxWidth - barLength);
+
+    return `  ${fullBlocks}${emptyBlocks} ${bucket.label} (${bucket.count} run${bucket.count !== 1 ? 's' : ''})`;
+  });
+};
+
+/**
+ * Generate ASCII sparkline from values
+ *
+ * @param values - Array of numeric values to visualize
+ * @param width - Maximum width of sparkline (downsamples if needed)
+ * @returns ASCII sparkline string using block characters
+ */
+export const generateSparkline = (values: number[], width?: number): string => {
+  if (values.length === 0) {
+    return '';
+  }
+
+  // Sparkline characters from lowest to highest
+  const sparkChars = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
+
+  // Downsample if width is specified and values exceed it
+  let processedValues = values;
+  if (width && values.length > width) {
+    const step = values.length / width;
+    processedValues = [];
+    for (let i = 0; i < width; i++) {
+      const idx = Math.floor(i * step);
+      processedValues.push(values[idx] ?? 0);
+    }
+  }
+
+  // Find min and max for scaling
+  const min = Math.min(...processedValues);
+  const max = Math.max(...processedValues);
+  const range = max - min;
+
+  // Handle case where all values are the same
+  if (range === 0) {
+    return (sparkChars[4] ?? '▄').repeat(processedValues.length); // Use middle character
+  }
+
+  // Map each value to a sparkline character
+  return processedValues
+    .map((value) => {
+      const normalized = (value - min) / range;
+      const index = Math.min(
+        Math.floor(normalized * sparkChars.length),
+        sparkChars.length - 1,
+      );
+      return sparkChars[index] ?? '▄';
+    })
+    .join('');
+};

package/src/index.ts

@@ -7,18 +7,11 @@
  */
 
 export { bootstrap as modestbench } from './bootstrap.js';
-// Configuration management
-export { ModestBenchConfigurationManager } from './config/manager.js';
 
-// Core engine and loader
+// Core engine
 export { ModestBenchEngine } from './core/engine.js';
 export { AccurateEngine, TinybenchEngine } from './core/engines/index.js';
 
-// Error handling
-export { ModestBenchErrorManager } from './core/error-manager.js';
-
-export { BenchmarkFileLoader } from './core/loader.js';
-
 // Statistical utilities
 export {
   calculateStatistics,
@@ -26,21 +19,24 @@ export {
   type SampleStatistics,
 } from './core/stats-utils.js';
 
-// Progress tracking
-export { ModestBenchProgressManager } from './progress/manager.js';
+// Error classes
+export * from './errors/index.js';
+
 // Reporters
 export { CsvReporter } from './reporters/csv.js';
 export { HumanReporter } from './reporters/human.js';
 export { JsonReporter } from './reporters/json.js';
 
+// Services
+export { ModestBenchConfigurationManager } from './services/config-manager.js';
+export { BenchmarkFileLoader } from './services/file-loader.js';
+export { FileHistoryStorage } from './services/history-storage.js';
+export { ModestBenchProgressManager } from './services/progress-manager.js';
 export {
   BaseReporter,
   CompositeReporter,
   ModestBenchReporterRegistry,
-} from './reporters/registry.js';
-
-// Storage
-export { FileHistoryStorage } from './storage/history.js';
+} from './services/reporter-registry.js';
 
 // Export all types
 export * from './types/index.js';

package/src/reporters/csv.ts

@@ -17,7 +17,8 @@ import type {
   TaskResult,
 } from '../types/index.js';
 
-import { BaseReporter } from './registry.js';
+import { ReporterOutputError } from '../errors/index.js';
+import { BaseReporter } from '../services/reporter-registry.js';
 
 /**
  * CSV column definitions for task results
@@ -340,7 +341,7 @@ export class CsvReporter extends BaseReporter {
    */
   private async writeToFile(csvContent: string): Promise<void> {
     if (!this.outputPath) {
-      throw new Error('Output path not specified');
+      throw new ReporterOutputError('Output path not specified');
     }
 
     try {
@@ -351,8 +352,9 @@ export class CsvReporter extends BaseReporter {
       // Write CSV file
       writeFileSync(this.outputPath, csvContent, 'utf8');
     } catch (error) {
-      throw new Error(
+      throw new ReporterOutputError(
         `Failed to write CSV output to ${this.outputPath}: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }

package/src/reporters/human.ts

@@ -15,48 +15,8 @@ import type {
   TaskResult,
 } from '../types/index.js';
 
-import { BaseReporter } from './registry.js';
-
-/**
- * ANSI color codes for terminal output
- */
-const colors = {
-  bold: '\x1b[1m',
-  brightBlue: '\x1b[94m',
-  brightCyan: '\x1b[96m',
-  brightMagenta: '\x1b[95m',
-  brightRed: '\x1b[91m',
-  brightWhite: '\x1b[97m',
-  cyan: '\x1b[36m',
-  dim: '\x1b[2m',
-  gray: '\x1b[90m',
-  green: '\x1b[32m',
-  magenta: '\x1b[35m',
-  red: '\x1b[31m',
-  reset: '\x1b[0m',
-  underline: '\x1b[4m',
-  white: '\x1b[37m',
-} as const;
-
-/**
- * CP437-inspired ANSI art characters
- */
-const ansiChars = {
-  approx: '≈',
-  // Block elements for gradients
-  block: {
-    dark: '▓',
-    full: '█',
-    light: '░',
-    medium: '▒',
-  },
-  bullet: '•',
-  // Symbols
-  checkmark: '√',
-  cross: '×',
-  plusMinus: '±',
-  smallSquare: '▪',
-} as const;
+import { BaseReporter } from '../services/reporter-registry.js';
+import { ansiChars, colors } from '../utils/ansi.js';
 
 /**
  * Human-readable console reporter with colorized output
@@ -157,7 +117,7 @@ export class HumanReporter extends BaseReporter {
       );
     } else {
       this.printLine(
-        `${this.colorize('brightCyan', ansiChars.checkmark + ' All tests passed:')} ${this.colorize('brightWhite', String(totalPassed))}`,
+        `${this.colorize('brightCyan', ansiChars.checkmark + ' All tasks passed:')} ${this.colorize('brightWhite', String(totalPassed))}`,
       );
     }
     this.printLine(

package/src/reporters/json.ts

@@ -16,7 +16,8 @@ import type {
   TaskResult,
 } from '../types/index.js';
 
-import { BaseReporter } from './registry.js';
+import { ReporterOutputError } from '../errors/index.js';
+import { BaseReporter } from '../services/reporter-registry.js';
 
 /**
  * JSON output structure for benchmark results
@@ -266,7 +267,7 @@ export class JsonReporter extends BaseReporter {
    */
   private async writeToFile(output: JsonOutput): Promise<void> {
     if (!this.outputPath) {
-      throw new Error('Output path not specified');
+      throw new ReporterOutputError('Output path not specified');
     }
 
     try {
@@ -281,8 +282,9 @@ export class JsonReporter extends BaseReporter {
 
       writeFileSync(this.outputPath, jsonString, 'utf8');
     } catch (error) {
-      throw new Error(
+      throw new ReporterOutputError(
         `Failed to write JSON output to ${this.outputPath}: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }

package/src/reporters/simple.ts

@@ -15,7 +15,7 @@ import type {
   TaskResult,
 } from '../types/index.js';
 
-import { BaseReporter } from './registry.js';
+import { BaseReporter } from '../services/reporter-registry.js';
 
 /**
  * Basic symbols for plain text output
@@ -91,7 +91,7 @@ export class SimpleReporter extends BaseReporter {
       console.log(`${symbols.cross} Failed: ${totalFailed}`);
       console.log(`${symbols.checkmark} Passed: ${totalPassed}`);
     } else {
-      console.log(`${symbols.checkmark} All tests passed: ${totalPassed}`);
+      console.log(`${symbols.checkmark} All tasks passed: ${totalPassed}`);
     }
 
     console.log(`- Files: ${totalFiles}`);

package/src/{config/manager.ts → services/config-manager.ts}

@@ -9,6 +9,7 @@
 import { cosmiconfig } from 'cosmiconfig';
 import { resolve } from 'node:path';
 
+import type { ModestBenchError } from '../errors/base.js';
 import type {
   ConfigurationManager,
   ModestBenchConfig,
@@ -17,7 +18,9 @@ import type {
   ValidationWarning,
 } from '../types/index.js';
 
-import { safeParseConfig } from './schema.js';
+import { safeParseConfig } from '../config/schema.js';
+import { ErrorCodes } from '../constants.js';
+import { ConfigLoadError, ConfigValidationError } from '../errors/index.js';
 
 /**
  * Get the default reporter based on TTY status and environment
@@ -175,15 +178,22 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
       // 3. Validate final configuration
       const validation = this.validate(finalConfig);
       if (!validation.valid) {
-        throw new Error(
+        throw new ConfigValidationError(
           `Configuration validation failed: ${validation.errors.map((e) => e.message).join(', ')}`,
         );
       }
 
       return finalConfig;
     } catch (error) {
-      throw new Error(
+      // Re-throw our custom errors
+      if (
+        (error as ModestBenchError).code === ErrorCodes.CONFIG_VALIDATION_FAILED
+      ) {
+        throw error;
+      }
+      throw new ConfigLoadError(
         `Failed to load configuration: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }

package/src/{core/loader.ts → services/file-loader.ts}

@@ -22,7 +22,12 @@ import {
   BENCHMARK_FILE_EXTENSIONS,
   BENCHMARK_FILE_PATTERN,
 } from '../constants.js';
-import { benchmarkFileSchema } from './benchmark-schema.js';
+import { benchmarkFileSchema } from '../core/benchmark-schema.js';
+import {
+  FileDiscoveryError,
+  FileLoadError,
+  StructureValidationError,
+} from '../errors/index.js';
 
 /**
  * File change notification for watch functionality
@@ -106,8 +111,9 @@ export class BenchmarkFileLoader implements FileLoader {
 
       return supportedFiles.sort();
     } catch (error) {
-      throw new Error(
+      throw new FileDiscoveryError(
         `File discovery failed: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }
@@ -120,7 +126,7 @@ export class BenchmarkFileLoader implements FileLoader {
       // Basic file checks (existence, extension)
       const basicValidation = await this.validate(filePath);
       if (!basicValidation.valid) {
-        throw new Error(
+        throw new FileLoadError(
           `Invalid benchmark file: ${basicValidation.errors.map((e) => e.message).join(', ')}`,
         );
       }
@@ -160,7 +166,7 @@ export class BenchmarkFileLoader implements FileLoader {
       // Validate the loaded exports structure with Zod
       const structureValidation = this.validateExports(filePath, exports);
       if (!structureValidation.valid || !structureValidation.data) {
-        throw new Error(
+        throw new StructureValidationError(
           `Invalid benchmark structure: ${structureValidation.errors.map((e) => e.message).join(', ')}`,
         );
       }
@@ -185,8 +191,16 @@ export class BenchmarkFileLoader implements FileLoader {
         },
       };
     } catch (error) {
-      throw new Error(
+      // Re-throw our custom errors
+      if (
+        error instanceof FileLoadError ||
+        error instanceof StructureValidationError
+      ) {
+        throw error;
+      }
+      throw new FileLoadError(
         `Failed to load file ${filePath}: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }
@@ -199,8 +213,16 @@ export class BenchmarkFileLoader implements FileLoader {
       const loadPromises = filePaths.map((filePath) => this.load(filePath));
       return await Promise.all(loadPromises);
     } catch (error) {
-      throw new Error(
+      // Re-throw our custom errors (from individual load calls)
+      if (
+        error instanceof FileLoadError ||
+        error instanceof StructureValidationError
+      ) {
+        throw error;
+      }
+      throw new FileLoadError(
         `Failed to load files: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }