modestbench 0.1.0 → 0.2.0

package/src/formatters/history/trends.ts

@@ -0,0 +1,281 @@
+/**
+ * Trends Formatter
+ *
+ * Formats performance trend analysis with visualizations in human and JSON
+ * formats.
+ */
+
+import { stripVTControlCharacters } from 'node:util';
+
+import type {
+  DistributionBucket,
+  TrendsResult,
+} from '../../services/history/models.js';
+import type { HistoryFormatter } from './base.js';
+
+import { colorize } from '../../utils/ansi.js';
+import { generateBarChart, generateSparkline } from './visualization.js';
+
+/**
+ * Nanoseconds per millisecond conversion constant
+ */
+const NS_PER_MS = 1_000_000;
+
+/**
+ * Nanoseconds per microsecond conversion constant
+ */
+const NS_PER_US = 1_000;
+
+/**
+ * Intelligently format a time range with appropriate precision Displays
+ * microseconds for small values (< 1ms) for better clarity
+ */
+const formatTimeRange = (minNs: number, maxNs: number): string => {
+  // If both values are below 1ms, show in microseconds for better precision
+  if (maxNs < NS_PER_MS) {
+    const minUs = minNs / NS_PER_US;
+    const maxUs = maxNs / NS_PER_US;
+    return `${minUs.toFixed(2)}-${maxUs.toFixed(2)}µs`;
+  }
+
+  // Otherwise show in milliseconds
+  const minMs = minNs / NS_PER_MS;
+  const maxMs = maxNs / NS_PER_MS;
+  return `${minMs.toFixed(3)}-${maxMs.toFixed(3)}ms`;
+};
+
+/**
+ * Calculate visible string length, stripping ANSI escape codes
+ */
+const getVisualLength = (str: string): number =>
+  stripVTControlCharacters(str).length;
+
+/**
+ * Formatter for history trends command
+ */
+export class HistoryTrendsFormatter implements HistoryFormatter<TrendsResult> {
+  /**
+   * Format as human-readable trends with visualizations
+   */
+  formatHuman(data: TrendsResult): string {
+    const lines: string[] = [];
+
+    lines.push(
+      colorize(
+        'brightMagenta',
+        colorize('bold', `\nPerformance Trends (${data.runs} runs)`),
+      ),
+    );
+
+    if (data.timespan.start && data.timespan.end) {
+      lines.push(
+        colorize(
+          'dim',
+          `Time range: ${data.timespan.start.toLocaleDateString()} to ${data.timespan.end.toLocaleDateString()}`,
+        ),
+      );
+    }
+    lines.push('');
+
+    // Summary statistics
+    lines.push(colorize('brightBlue', 'Summary:'));
+    lines.push(
+      `  ${colorize('brightCyan', '▲')} ${data.summary.improvingTasks} improving  ${colorize('brightRed', '▼')} ${data.summary.degradingTasks} degrading  ${colorize('dim', '→')} ${data.summary.stableTasks} stable`,
+    );
+    lines.push('');
+
+    // Task Performance Summary Table
+    lines.push(colorize('brightMagenta', 'Task Performance Summary:'));
+    lines.push('');
+
+    for (const trendData of data.trends) {
+      // Show top 20
+      const trendIcon =
+        trendData.trend === 'improving'
+          ? colorize('brightCyan', '▲')
+          : trendData.trend === 'degrading'
+            ? colorize('brightRed', '▼')
+            : colorize('dim', '→');
+
+      const changeColor =
+        trendData.percentChange < -5
+          ? 'brightCyan'
+          : trendData.percentChange > 5
+            ? 'brightRed'
+            : 'dim';
+
+      const changeSign = trendData.percentChange >= 0 ? '+' : '';
+      const changeStr = `${changeSign}${trendData.percentChange.toFixed(1)}%`;
+
+      // Generate sparkline - scale with number of data points (min 12, max 20)
+      const values = trendData.dataPoints.map((dp) => dp.mean);
+      const sparklineWidth = Math.min(20, Math.max(12, trendData.runs));
+      const sparkline = generateSparkline(values, sparklineWidth);
+      const sparklineColor =
+        trendData.trend === 'improving'
+          ? 'brightCyan'
+          : trendData.trend === 'degrading'
+            ? 'brightRed'
+            : 'cyan';
+
+      const taskName = colorize('white', trendData.task);
+      const percentDisplay = colorize(changeColor, changeStr.padStart(8));
+      const sparklineDisplay = colorize(sparklineColor, sparkline);
+
+      // Layout: icon (2) + task name + padding + percent (8) + spaces (2) + sparkline
+      // Position percent+graph at visual column 60 for consistent alignment
+      const prefix = `  ${trendIcon} ${taskName}`;
+      const prefixVisualLength = getVisualLength(prefix);
+      const targetColumn = 60;
+
+      if (prefixVisualLength > targetColumn) {
+        // Task name is too long, wrap to next line
+        lines.push(prefix);
+        lines.push(
+          `${' '.repeat(targetColumn)}${percentDisplay}  ${sparklineDisplay}`,
+        );
+      } else {
+        // Fit on one line with padding
+        const padding = Math.max(1, targetColumn - prefixVisualLength);
+        const paddingStr = ' '.repeat(padding);
+        lines.push(
+          `${prefix}${paddingStr}${percentDisplay}  ${sparklineDisplay}`,
+        );
+      }
+    }
+
+    // Show all trends (no limit)
+
+    lines.push('');
+
+    // Show regressions if any
+    if (data.regressions.length > 0) {
+      lines.push(
+        colorize('brightRed', colorize('bold', 'Regressions Detected:')),
+      );
+      lines.push('');
+
+      for (const regression of data.regressions) {
+        lines.push(
+          `  ${colorize('brightRed', '▼')} ${colorize('white', regression.task)}: ${colorize('brightRed', `${regression.percentChange.toFixed(1)}% slower`)} (${regression.runs} runs)`,
+        );
+      }
+
+      lines.push('');
+    }
+
+    // Show low-confidence regressions (insufficient data)
+    if (data.lowConfidenceRegressions.length > 0) {
+      lines.push(
+        colorize(
+          'brightYellow',
+          colorize('bold', '! Potential Regressions (insufficient data):'),
+        ),
+      );
+      lines.push('');
+
+      for (const regression of data.lowConfidenceRegressions) {
+        lines.push(
+          `  ${colorize('brightYellow', '!')} ${colorize('white', regression.task)}: ${colorize('brightYellow', `${regression.percentChange.toFixed(1)}% slower`)} (${regression.runs} run${regression.runs !== 1 ? 's' : ''})`,
+        );
+      }
+
+      lines.push('');
+    }
+
+    // Show performance distribution for most important task (highest RME) if we have enough data
+    // Find task with highest relative margin of error - the most variable/unreliable one
+    const mostImportantTrend = data.trends
+      .filter((t) => t.runs >= 5)
+      .sort((a, b) => {
+        const rmeA = (a.statistics.stdDeviation / a.statistics.mean) * 100;
+        const rmeB = (b.statistics.stdDeviation / b.statistics.mean) * 100;
+        return rmeB - rmeA; // Descending order (highest RME first)
+      })[0];
+
+    if (mostImportantTrend) {
+      lines.push(
+        colorize(
+          'brightMagenta',
+          'Performance Distribution (most variable task):',
+        ),
+      );
+      lines.push(colorize('white', mostImportantTrend.task));
+      const cv = (
+        (mostImportantTrend.statistics.stdDeviation /
+          mostImportantTrend.statistics.mean) *
+        100
+      ).toFixed(1);
+      lines.push(colorize('dim', `  Variability: ${cv}%`));
+      lines.push('');
+
+      // Create distribution buckets
+      const values = mostImportantTrend.dataPoints.map((dp) => dp.mean);
+      const min = Math.min(...values);
+      const max = Math.max(...values);
+      const range = max - min;
+      const numBuckets = Math.min(5, mostImportantTrend.runs);
+      const bucketSize = range / numBuckets;
+
+      const buckets: DistributionBucket[] = [];
+      for (let i = 0; i < numBuckets; i++) {
+        const bucketMin = min + i * bucketSize;
+        const bucketMax = min + (i + 1) * bucketSize;
+        const count = values.filter(
+          (v) =>
+            v >= bucketMin &&
+            (i === numBuckets - 1 ? v <= bucketMax : v < bucketMax),
+        ).length;
+
+        const label = formatTimeRange(bucketMin, bucketMax);
+
+        buckets.push({
+          count,
+          label,
+          max: bucketMax,
+          min: bucketMin,
+        });
+      }
+
+      // Filter out empty buckets for cleaner display
+      const nonEmptyBuckets = buckets.filter((b) => b.count > 0);
+      const chart = generateBarChart(nonEmptyBuckets, 25);
+      for (const line of chart) {
+        lines.push(colorize('brightCyan', line));
+      }
+
+      lines.push('');
+      lines.push(
+        colorize(
+          'dim',
+          `  Mean: ${(mostImportantTrend.statistics.mean / NS_PER_MS).toFixed(3)}ms  Median: ${(mostImportantTrend.statistics.median / NS_PER_MS).toFixed(3)}ms  StdDev: ${(mostImportantTrend.statistics.stdDeviation / NS_PER_MS).toFixed(3)}ms`,
+        ),
+      );
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format as JSON
+   */
+  formatJson(data: TrendsResult): string {
+    return JSON.stringify(
+      {
+        regressions: data.regressions,
+        summary: {
+          degradingTasks: data.summary.degradingTasks,
+          improvingTasks: data.summary.improvingTasks,
+          runs: data.runs,
+          stableTasks: data.summary.stableTasks,
+          timespan: data.timespan,
+          totalTasks: data.summary.totalTasks,
+        },
+        trends: data.trends,
+      },
+      null,
+      2,
+    );
+  }
+}

package/src/formatters/history/visualization.ts

@@ -0,0 +1,93 @@
+/**
+ * Visualization Utilities for History Formatters
+ *
+ * Pure functions for generating ASCII visualizations (sparklines, bar charts)
+ * for trend analysis and performance distribution displays.
+ */
+
+import type { DistributionBucket } from '../../services/history/models.js';
+
+/**
+ * Generate bar chart histogram from distribution buckets
+ *
+ * @param distribution - Array of distribution buckets with counts
+ * @param maxWidth - Maximum width of bars (default 20)
+ * @returns Array of formatted bar chart lines
+ */
+export const generateBarChart = (
+  distribution: DistributionBucket[],
+  maxWidth = 20,
+): string[] => {
+  if (distribution.length === 0) {
+    return [];
+  }
+
+  // Find maximum count for scaling
+  const maxCount = Math.max(...distribution.map((b) => b.count));
+
+  if (maxCount === 0) {
+    return distribution.map((bucket) => `  ${bucket.label} (0 runs)`);
+  }
+
+  // Block characters for bar visualization
+  const fullBlock = '█';
+  const lightBlock = '░';
+
+  return distribution.map((bucket) => {
+    const ratio = bucket.count / maxCount;
+    const barLength = Math.round(ratio * maxWidth);
+    const fullBlocks = fullBlock.repeat(barLength);
+    const emptyBlocks = lightBlock.repeat(maxWidth - barLength);
+
+    return `  ${fullBlocks}${emptyBlocks} ${bucket.label} (${bucket.count} run${bucket.count !== 1 ? 's' : ''})`;
+  });
+};
+
+/**
+ * Generate ASCII sparkline from values
+ *
+ * @param values - Array of numeric values to visualize
+ * @param width - Maximum width of sparkline (downsamples if needed)
+ * @returns ASCII sparkline string using block characters
+ */
+export const generateSparkline = (values: number[], width?: number): string => {
+  if (values.length === 0) {
+    return '';
+  }
+
+  // Sparkline characters from lowest to highest
+  const sparkChars = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
+
+  // Downsample if width is specified and values exceed it
+  let processedValues = values;
+  if (width && values.length > width) {
+    const step = values.length / width;
+    processedValues = [];
+    for (let i = 0; i < width; i++) {
+      const idx = Math.floor(i * step);
+      processedValues.push(values[idx] ?? 0);
+    }
+  }
+
+  // Find min and max for scaling
+  const min = Math.min(...processedValues);
+  const max = Math.max(...processedValues);
+  const range = max - min;
+
+  // Handle case where all values are the same
+  if (range === 0) {
+    return (sparkChars[4] ?? '▄').repeat(processedValues.length); // Use middle character
+  }
+
+  // Map each value to a sparkline character
+  return processedValues
+    .map((value) => {
+      const normalized = (value - min) / range;
+      const index = Math.min(
+        Math.floor(normalized * sparkChars.length),
+        sparkChars.length - 1,
+      );
+      return sparkChars[index] ?? '▄';
+    })
+    .join('');
+};

package/src/index.ts

@@ -7,15 +7,11 @@
  */
 
 export { bootstrap as modestbench } from './bootstrap.js';
-// Configuration management
-export { ModestBenchConfigurationManager } from './config/manager.js';
 
-// Core engine and loader
+// Core engine
 export { ModestBenchEngine } from './core/engine.js';
 export { AccurateEngine, TinybenchEngine } from './core/engines/index.js';
 
-export { BenchmarkFileLoader } from './core/loader.js';
-
 // Statistical utilities
 export {
   calculateStatistics,
@@ -26,21 +22,21 @@ export {
 // Error classes
 export * from './errors/index.js';
 
-// Progress tracking
-export { ModestBenchProgressManager } from './progress/manager.js';
 // Reporters
 export { CsvReporter } from './reporters/csv.js';
 export { HumanReporter } from './reporters/human.js';
 export { JsonReporter } from './reporters/json.js';
 
+// Services
+export { ModestBenchConfigurationManager } from './services/config-manager.js';
+export { BenchmarkFileLoader } from './services/file-loader.js';
+export { FileHistoryStorage } from './services/history-storage.js';
+export { ModestBenchProgressManager } from './services/progress-manager.js';
 export {
   BaseReporter,
   CompositeReporter,
   ModestBenchReporterRegistry,
-} from './reporters/registry.js';
-
-// Storage
-export { FileHistoryStorage } from './storage/history.js';
+} from './services/reporter-registry.js';
 
 // Export all types
 export * from './types/index.js';

package/src/reporters/csv.ts

@@ -18,7 +18,7 @@ import type {
 } from '../types/index.js';
 
 import { ReporterOutputError } from '../errors/index.js';
-import { BaseReporter } from './registry.js';
+import { BaseReporter } from '../services/reporter-registry.js';
 
 /**
  * CSV column definitions for task results

package/src/reporters/human.ts

@@ -15,48 +15,8 @@ import type {
   TaskResult,
 } from '../types/index.js';
 
-import { BaseReporter } from './registry.js';
-
-/**
- * ANSI color codes for terminal output
- */
-const colors = {
-  bold: '\x1b[1m',
-  brightBlue: '\x1b[94m',
-  brightCyan: '\x1b[96m',
-  brightMagenta: '\x1b[95m',
-  brightRed: '\x1b[91m',
-  brightWhite: '\x1b[97m',
-  cyan: '\x1b[36m',
-  dim: '\x1b[2m',
-  gray: '\x1b[90m',
-  green: '\x1b[32m',
-  magenta: '\x1b[35m',
-  red: '\x1b[31m',
-  reset: '\x1b[0m',
-  underline: '\x1b[4m',
-  white: '\x1b[37m',
-} as const;
-
-/**
- * CP437-inspired ANSI art characters
- */
-const ansiChars = {
-  approx: '≈',
-  // Block elements for gradients
-  block: {
-    dark: '▓',
-    full: '█',
-    light: '░',
-    medium: '▒',
-  },
-  bullet: '•',
-  // Symbols
-  checkmark: '√',
-  cross: '×',
-  plusMinus: '±',
-  smallSquare: '▪',
-} as const;
+import { BaseReporter } from '../services/reporter-registry.js';
+import { ansiChars, colors } from '../utils/ansi.js';
 
 /**
  * Human-readable console reporter with colorized output

package/src/reporters/json.ts

@@ -17,7 +17,7 @@ import type {
 } from '../types/index.js';
 
 import { ReporterOutputError } from '../errors/index.js';
-import { BaseReporter } from './registry.js';
+import { BaseReporter } from '../services/reporter-registry.js';
 
 /**
  * JSON output structure for benchmark results

package/src/reporters/simple.ts

@@ -15,7 +15,7 @@ import type {
   TaskResult,
 } from '../types/index.js';
 
-import { BaseReporter } from './registry.js';
+import { BaseReporter } from '../services/reporter-registry.js';
 
 /**
  * Basic symbols for plain text output

package/src/{config/manager.ts → services/config-manager.ts}

@@ -18,9 +18,9 @@ import type {
   ValidationWarning,
 } from '../types/index.js';
 
+import { safeParseConfig } from '../config/schema.js';
 import { ErrorCodes } from '../constants.js';
 import { ConfigLoadError, ConfigValidationError } from '../errors/index.js';
-import { safeParseConfig } from './schema.js';
 
 /**
  * Get the default reporter based on TTY status and environment

package/src/{core/loader.ts → services/file-loader.ts}

@@ -22,12 +22,12 @@ import {
   BENCHMARK_FILE_EXTENSIONS,
   BENCHMARK_FILE_PATTERN,
 } from '../constants.js';
+import { benchmarkFileSchema } from '../core/benchmark-schema.js';
 import {
   FileDiscoveryError,
   FileLoadError,
   StructureValidationError,
 } from '../errors/index.js';
-import { benchmarkFileSchema } from './benchmark-schema.js';
 
 /**
  * File change notification for watch functionality

package/src/services/history/comparison.ts

@@ -0,0 +1,130 @@
+/**
+ * Benchmark Comparison Service
+ *
+ * Handles comparison logic between two benchmark runs, calculating performance
+ * differences and categorizing tasks.
+ */
+
+import type { BenchmarkRun } from '../../types/index.js';
+import type { CompareResult, TaskComparison } from './models.js';
+
+/**
+ * Service for comparing benchmark runs
+ */
+export class ComparisonService {
+  /**
+   * Compare two benchmark runs and produce detailed comparison result
+   */
+  compareRuns(run1: BenchmarkRun, run2: BenchmarkRun): CompareResult {
+    // Build task maps for comparison
+    const tasksMap1 = new Map<string, TaskComparison>();
+    const tasksMap2 = new Map<string, TaskComparison>();
+
+    // Extract tasks from run1
+    for (const file of run1.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const key = `${file.filePath}::${suite.name}::${task.name}`;
+            tasksMap1.set(key, {
+              file: file.filePath,
+              inBoth: false,
+              percentChange: 0,
+              run1: {
+                cv: task.cv,
+                iterations: task.iterations,
+                max: task.max,
+                mean: task.mean,
+                min: task.min,
+              },
+              suite: suite.name,
+              task: task.name,
+            });
+          }
+        }
+      }
+    }
+
+    // Extract tasks from run2 and merge
+    for (const file of run2.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const key = `${file.filePath}::${suite.name}::${task.name}`;
+            const existing = tasksMap1.get(key);
+
+            if (existing && existing.run1) {
+              // Task exists in both runs - calculate comparison
+              const percentChange =
+                ((task.mean - existing.run1.mean) / existing.run1.mean) * 100;
+
+              tasksMap1.set(key, {
+                ...existing,
+                inBoth: true,
+                percentChange,
+                run2: {
+                  cv: task.cv,
+                  iterations: task.iterations,
+                  max: task.max,
+                  mean: task.mean,
+                  min: task.min,
+                },
+              });
+            } else {
+              // Task only in run2
+              tasksMap2.set(key, {
+                file: file.filePath,
+                inBoth: false,
+                percentChange: 0,
+                run2: {
+                  cv: task.cv,
+                  iterations: task.iterations,
+                  max: task.max,
+                  mean: task.mean,
+                  min: task.min,
+                },
+                suite: suite.name,
+                task: task.name,
+              });
+            }
+          }
+        }
+      }
+    }
+
+    // Separate tasks into categories
+    const tasksInBoth: TaskComparison[] = [];
+    const tasksOnlyIn1: TaskComparison[] = [];
+    const tasksOnlyIn2: TaskComparison[] = [];
+
+    for (const task of tasksMap1.values()) {
+      if (task.inBoth) {
+        tasksInBoth.push(task);
+      } else {
+        tasksOnlyIn1.push(task);
+      }
+    }
+
+    for (const task of tasksMap2.values()) {
+      tasksOnlyIn2.push(task);
+    }
+
+    return {
+      run1: {
+        endTime: run1.endTime,
+        id: run1.id,
+        startTime: run1.startTime,
+        summary: run1.summary,
+      },
+      run2: {
+        endTime: run2.endTime,
+        id: run2.id,
+        startTime: run2.startTime,
+        summary: run2.summary,
+      },
+      tasksInBoth,
+      tasksOnlyInRun1: tasksOnlyIn1,
+      tasksOnlyInRun2: tasksOnlyIn2,
+    };
+  }
+}