modestbench 0.0.3 → 0.2.0

package/src/services/history/comparison.ts

@@ -0,0 +1,130 @@
+/**
+ * Benchmark Comparison Service
+ *
+ * Handles comparison logic between two benchmark runs, calculating performance
+ * differences and categorizing tasks.
+ */
+
+import type { BenchmarkRun } from '../../types/index.js';
+import type { CompareResult, TaskComparison } from './models.js';
+
+/**
+ * Service for comparing benchmark runs
+ */
+export class ComparisonService {
+  /**
+   * Compare two benchmark runs and produce detailed comparison result
+   */
+  compareRuns(run1: BenchmarkRun, run2: BenchmarkRun): CompareResult {
+    // Build task maps for comparison
+    const tasksMap1 = new Map<string, TaskComparison>();
+    const tasksMap2 = new Map<string, TaskComparison>();
+
+    // Extract tasks from run1
+    for (const file of run1.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const key = `${file.filePath}::${suite.name}::${task.name}`;
+            tasksMap1.set(key, {
+              file: file.filePath,
+              inBoth: false,
+              percentChange: 0,
+              run1: {
+                cv: task.cv,
+                iterations: task.iterations,
+                max: task.max,
+                mean: task.mean,
+                min: task.min,
+              },
+              suite: suite.name,
+              task: task.name,
+            });
+          }
+        }
+      }
+    }
+
+    // Extract tasks from run2 and merge
+    for (const file of run2.files) {
+      for (const suite of file.suites) {
+        for (const task of suite.tasks) {
+          if (!task.error) {
+            const key = `${file.filePath}::${suite.name}::${task.name}`;
+            const existing = tasksMap1.get(key);
+
+            if (existing && existing.run1) {
+              // Task exists in both runs - calculate comparison
+              const percentChange =
+                ((task.mean - existing.run1.mean) / existing.run1.mean) * 100;
+
+              tasksMap1.set(key, {
+                ...existing,
+                inBoth: true,
+                percentChange,
+                run2: {
+                  cv: task.cv,
+                  iterations: task.iterations,
+                  max: task.max,
+                  mean: task.mean,
+                  min: task.min,
+                },
+              });
+            } else {
+              // Task only in run2
+              tasksMap2.set(key, {
+                file: file.filePath,
+                inBoth: false,
+                percentChange: 0,
+                run2: {
+                  cv: task.cv,
+                  iterations: task.iterations,
+                  max: task.max,
+                  mean: task.mean,
+                  min: task.min,
+                },
+                suite: suite.name,
+                task: task.name,
+              });
+            }
+          }
+        }
+      }
+    }
+
+    // Separate tasks into categories
+    const tasksInBoth: TaskComparison[] = [];
+    const tasksOnlyIn1: TaskComparison[] = [];
+    const tasksOnlyIn2: TaskComparison[] = [];
+
+    for (const task of tasksMap1.values()) {
+      if (task.inBoth) {
+        tasksInBoth.push(task);
+      } else {
+        tasksOnlyIn1.push(task);
+      }
+    }
+
+    for (const task of tasksMap2.values()) {
+      tasksOnlyIn2.push(task);
+    }
+
+    return {
+      run1: {
+        endTime: run1.endTime,
+        id: run1.id,
+        startTime: run1.startTime,
+        summary: run1.summary,
+      },
+      run2: {
+        endTime: run2.endTime,
+        id: run2.id,
+        startTime: run2.startTime,
+        summary: run2.summary,
+      },
+      tasksInBoth,
+      tasksOnlyInRun1: tasksOnlyIn1,
+      tasksOnlyInRun2: tasksOnlyIn2,
+    };
+  }
+}

package/src/services/history/models.ts

@@ -0,0 +1,148 @@
+/**
+ * Data Models for History Services
+ *
+ * Type definitions for history command data structures, shared between
+ * services, formatters, and CLI handlers.
+ */
+
+import type { BenchmarkRun } from '../../types/index.js';
+
+/**
+ * Result from comparing two benchmark runs
+ */
+export interface CompareResult {
+  run1: {
+    endTime: Date;
+    id: string;
+    startTime: Date;
+    summary: {
+      failedTasks: number;
+      passedTasks: number;
+      totalFiles: number;
+      totalTasks: number;
+    };
+  };
+  run2: {
+    endTime: Date;
+    id: string;
+    startTime: Date;
+    summary: {
+      failedTasks: number;
+      passedTasks: number;
+      totalFiles: number;
+      totalTasks: number;
+    };
+  };
+  tasksInBoth: TaskComparison[];
+  tasksOnlyInRun1: TaskComparison[];
+  tasksOnlyInRun2: TaskComparison[];
+}
+
+/**
+ * Distribution bucket for histogram visualization
+ */
+export interface DistributionBucket {
+  count: number;
+  label: string;
+  max: number;
+  min: number;
+}
+
+/**
+ * Result from listing historical runs
+ */
+export interface HistoryListResult {
+  runs: Array<{
+    duration: number;
+    id: string;
+    startTime: Date;
+    summary: {
+      failedTasks: number;
+      passedTasks: number;
+      totalFiles: number;
+      totalTasks: number;
+    };
+  }>;
+  totalCount: number;
+}
+
+/**
+ * Show command result (just wraps BenchmarkRun for consistency)
+ */
+export type ShowResult = BenchmarkRun;
+
+/**
+ * Task comparison result between two runs
+ */
+export interface TaskComparison {
+  file: string;
+  inBoth: boolean;
+  percentChange: number;
+  run1?: {
+    cv: number;
+    iterations: number;
+    max: number;
+    mean: number;
+    min: number;
+  };
+  run2?: {
+    cv: number;
+    iterations: number;
+    max: number;
+    mean: number;
+    min: number;
+  };
+  suite: string;
+  task: string;
+}
+
+/**
+ * Complete trend analysis for a single task
+ */
+export interface TrendData {
+  confidence: number;
+  dataPoints: TrendDataPoint[];
+  percentChange: number;
+  runs: number;
+  statistics: TrendStatistics;
+  task: string;
+  trend: 'degrading' | 'improving' | 'stable';
+}
+
+/**
+ * Single data point in a trend series
+ */
+export interface TrendDataPoint {
+  date: Date;
+  mean: number;
+}
+
+/**
+ * Result from analyzing performance trends
+ */
+export interface TrendsResult {
+  lowConfidenceRegressions: TrendData[];
+  regressions: TrendData[];
+  runs: number;
+  summary: {
+    degradingTasks: number;
+    improvingTasks: number;
+    stableTasks: number;
+    totalTasks: number;
+  };
+  timespan: {
+    end: Date;
+    start: Date;
+  };
+  trends: TrendData[];
+}
+
+/**
+ * Statistical metrics for trend analysis
+ */
+export interface TrendStatistics {
+  mean: number;
+  median: number;
+  stdDeviation: number;
+  variance: number;
+}

package/src/services/history/query.ts

@@ -0,0 +1,116 @@
+/**
+ * History Query Service
+ *
+ * Handles querying benchmark run history with date parsing and filtering.
+ */
+
+import type {
+  BenchmarkRun,
+  HistoryQuery,
+  HistoryStorage,
+} from '../../types/index.js';
+
+/**
+ * Service for querying historical benchmark runs
+ */
+export class HistoryQueryService {
+  constructor(private readonly storage: HistoryStorage) {}
+
+  /**
+   * Query runs with automatic date string parsing
+   */
+  async queryWithDateParsing(options: {
+    limit?: number;
+    pattern?: string;
+    since?: string;
+    tags?: string[];
+    until?: string;
+  }): Promise<BenchmarkRun[]> {
+    // Build query object all at once
+    const query: Partial<HistoryQuery> = {
+      ...(options.since && { since: parseDate(options.since) }),
+      ...(options.until && { until: parseDate(options.until) }),
+      ...(options.pattern && { pattern: options.pattern }),
+      ...(options.tags && options.tags.length > 0 && { tags: options.tags }),
+      ...(options.limit && { limit: options.limit }),
+    };
+
+    return await this.storage.queryRuns(query);
+  }
+}
+
+/**
+ * Parse date string (ISO 8601 or relative)
+ *
+ * Supports:
+ *
+ * - ISO 8601: "2025-10-24T12:00:00Z", "2025-10-24"
+ * - Relative: "1 day ago", "3 weeks ago", "2 hours ago"
+ * - Shorthand: "1d", "2w", "3m", "6h"
+ *
+ * @param dateStr - Date string to parse
+ * @returns Parsed Date object
+ * @throws Error if date format is invalid
+ */
+export const parseDate = (dateStr: string): Date => {
+  if (!dateStr || dateStr.trim() === '') {
+    throw new Error(`Invalid date format: "${dateStr}"`);
+  }
+
+  // Try parsing as ISO 8601 first
+  const isoDate = new Date(dateStr);
+  if (!isNaN(isoDate.getTime())) {
+    return isoDate;
+  }
+
+  // Parse relative dates like "1 week ago", "3 days ago"
+  const relativePattern = /^(\d+)\s+(hour|day|week|month)s?\s+ago$/i;
+  const relativeMatch = dateStr.trim().match(relativePattern);
+
+  if (relativeMatch && relativeMatch[1] && relativeMatch[2]) {
+    const amount = parseInt(relativeMatch[1], 10);
+    const unit = relativeMatch[2].toLowerCase();
+
+    if (amount <= 0) {
+      throw new Error(`Invalid date format: "${dateStr}"`);
+    }
+
+    const now = new Date();
+    const msPerUnit: Record<string, number> = {
+      day: 24 * 60 * 60 * 1000,
+      hour: 60 * 60 * 1000,
+      month: 30 * 24 * 60 * 60 * 1000, // Approximate
+      week: 7 * 24 * 60 * 60 * 1000,
+    };
+
+    const offset = amount * (msPerUnit[unit] || 0);
+    return new Date(now.getTime() - offset);
+  }
+
+  // Parse shorthand formats like "1d", "2w", "3m", "6h"
+  // cspell:ignore hdwm
+  const shorthandPattern = /^(\d+)([hdwm])$/i;
+  const shorthandMatch = dateStr.trim().match(shorthandPattern);
+
+  if (shorthandMatch && shorthandMatch[1] && shorthandMatch[2]) {
+    const amount = parseInt(shorthandMatch[1], 10);
+    const unit = shorthandMatch[2].toLowerCase();
+
+    if (amount <= 0) {
+      throw new Error(`Invalid date format: "${dateStr}"`);
+    }
+
+    const now = new Date();
+    const msPerUnit: Record<string, number> = {
+      d: 24 * 60 * 60 * 1000,
+      h: 60 * 60 * 1000,
+      m: 30 * 24 * 60 * 60 * 1000, // Approximate month
+      w: 7 * 24 * 60 * 60 * 1000,
+    };
+
+    const offset = amount * (msPerUnit[unit] || 0);
+    return new Date(now.getTime() - offset);
+  }
+
+  throw new Error(`Invalid date format: "${dateStr}"`);
+};

package/src/services/history/trend-analysis.ts

@@ -0,0 +1,238 @@
+/**
+ * Trend Analysis Service
+ *
+ * Statistical analysis of benchmark performance trends over time, including
+ * regression detection and trend classification.
+ */
+
+import type { BenchmarkRun } from '../../types/index.js';
+import type {
+  TrendData,
+  TrendDataPoint,
+  TrendsResult,
+  TrendStatistics,
+} from './models.js';
+
+/**
+ * Service for analyzing performance trends
+ */
+export class TrendAnalysisService {
+  /**
+   * Analyze trends across multiple benchmark runs
+   */
+  analyzeTrends(runs: BenchmarkRun[]): TrendsResult {
+    // Build trend data for each task across runs
+    const taskTrendsMap = new Map<string, TrendDataPoint[]>();
+
+    for (const run of runs) {
+      for (const file of run.files) {
+        for (const suite of file.suites) {
+          for (const task of suite.tasks) {
+            if (!task.error) {
+              const key = `${file.filePath}::${suite.name}::${task.name}`;
+              const dataPoints = taskTrendsMap.get(key) || [];
+              dataPoints.push({
+                date: new Date(run.startTime),
+                mean: task.mean,
+              });
+              taskTrendsMap.set(key, dataPoints);
+            }
+          }
+        }
+      }
+    }
+
+    // Calculate trends for each task
+    const trendsData: TrendData[] = [];
+
+    for (const [key, dataPoints] of taskTrendsMap.entries()) {
+      // Sort by date (oldest first)
+      dataPoints.sort((a, b) => a.date.getTime() - b.date.getTime());
+
+      const [_filePath, suiteName, taskName] = key.split('::');
+
+      const statistics = calculateStatistics(dataPoints);
+      const trend = calculateTrend(dataPoints);
+      const percentChange = calculatePercentChange(dataPoints);
+
+      trendsData.push({
+        confidence: 95, // Fixed confidence level for now
+        dataPoints,
+        percentChange,
+        runs: dataPoints.length,
+        statistics,
+        task: `${suiteName} › ${taskName}`,
+        trend,
+      });
+    }
+
+    // Sort by absolute percent change (most significant first)
+    trendsData.sort(
+      (a, b) => Math.abs(b.percentChange) - Math.abs(a.percentChange),
+    );
+
+    // Calculate summary statistics
+    const summary = {
+      degradingTasks: trendsData.filter((t) => t.trend === 'degrading').length,
+      improvingTasks: trendsData.filter((t) => t.trend === 'improving').length,
+      stableTasks: trendsData.filter((t) => t.trend === 'stable').length,
+      totalTasks: trendsData.length,
+    };
+
+    // Get timespan
+    const firstRun = runs[runs.length - 1];
+    const lastRun = runs[0];
+    const timespan = {
+      end: lastRun ? new Date(lastRun.startTime) : new Date(),
+      start: firstRun ? new Date(firstRun.startTime) : new Date(),
+    };
+
+    // Detect regressions (require minimum 5 runs for confidence)
+    const minRunsForRegression = 5;
+    const regressions = trendsData.filter((t) =>
+      detectRegression(t, 5, minRunsForRegression),
+    );
+
+    // Also track low-confidence potential regressions (2-4 runs showing degradation)
+    const lowConfidenceRegressions = trendsData.filter(
+      (t) =>
+        t.trend === 'degrading' &&
+        t.percentChange >= 5 &&
+        t.runs < minRunsForRegression &&
+        t.runs >= 2,
+    );
+
+    return {
+      lowConfidenceRegressions,
+      regressions,
+      runs: runs.length,
+      summary,
+      timespan,
+      trends: trendsData,
+    };
+  }
+}
+
+/**
+ * Calculate percent change from first to last data point
+ */
+export const calculatePercentChange = (
+  dataPoints: TrendDataPoint[],
+): number => {
+  if (dataPoints.length === 0 || dataPoints.length === 1) {
+    return 0;
+  }
+
+  const firstPoint = dataPoints[0];
+  const lastPoint = dataPoints[dataPoints.length - 1];
+
+  if (!firstPoint || !lastPoint) {
+    return 0;
+  }
+
+  const first = firstPoint.mean;
+  const last = lastPoint.mean;
+
+  if (first === 0) {
+    return 0; // Avoid division by zero
+  }
+
+  return ((last - first) / first) * 100;
+};
+
+/**
+ * Calculate statistical metrics from data points
+ */
+export const calculateStatistics = (
+  dataPoints: TrendDataPoint[],
+): TrendStatistics => {
+  if (dataPoints.length === 0) {
+    throw new Error('Cannot calculate statistics for empty data points array');
+  }
+
+  const values = dataPoints.map((dp) => dp.mean);
+  const n = values.length;
+
+  // Calculate mean
+  const mean = values.reduce((sum, val) => sum + val, 0) / n;
+
+  // Calculate median
+  const sorted = [...values].sort((a, b) => a - b);
+  const median =
+    n % 2 === 0
+      ? ((sorted[n / 2 - 1] ?? 0) + (sorted[n / 2] ?? 0)) / 2
+      : (sorted[Math.floor(n / 2)] ?? 0);
+
+  // Calculate variance and standard deviation
+  const variance =
+    n === 1
+      ? 0
+      : values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / n;
+  const stdDeviation = Math.sqrt(variance);
+
+  return {
+    mean,
+    median,
+    stdDeviation,
+    variance,
+  };
+};
+
+/**
+ * Calculate trend direction from data points using linear regression
+ */
+export const calculateTrend = (
+  dataPoints: TrendDataPoint[],
+): 'degrading' | 'improving' | 'stable' => {
+  if (dataPoints.length === 0) {
+    return 'stable';
+  }
+
+  if (dataPoints.length === 1) {
+    return 'stable';
+  }
+
+  // Simple linear regression to determine slope
+  const n = dataPoints.length;
+  const x = Array.from({ length: n }, (_, i) => i); // Time indices
+  const y = dataPoints.map((dp) => dp.mean);
+
+  const sumX = x.reduce((a, b) => a + b, 0);
+  const sumY = y.reduce((a, b) => a + b, 0);
+  const sumXY = x.reduce((sum, xi, i) => sum + xi * (y[i] ?? 0), 0);
+  const sumXX = x.reduce((sum, xi) => sum + xi * xi, 0);
+
+  // Calculate slope: (n*sumXY - sumX*sumY) / (n*sumXX - sumX*sumX)
+  const slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX);
+
+  // Determine significance threshold (5% of mean)
+  const meanValue = sumY / n;
+  const significanceThreshold = Math.abs(meanValue * 0.05);
+
+  // Classify trend based on slope
+  if (Math.abs(slope) < significanceThreshold / n) {
+    return 'stable';
+  } else if (slope < 0) {
+    // Negative slope = values decreasing = performance improving
+    return 'improving';
+  } else {
+    // Positive slope = values increasing = performance degrading
+    return 'degrading';
+  }
+};
+
+/**
+ * Detect if a trend represents a performance regression
+ */
+export const detectRegression = (
+  trendData: TrendData,
+  threshold: number,
+  minRuns: number,
+): boolean => {
+  // Regression is a degrading trend with percent change exceeding threshold
+  return (
+    trendData.trend === 'degrading' &&
+    trendData.percentChange >= threshold &&
+    trendData.runs >= minRuns
+  );
+};