modestbench 0.1.0 → 0.3.0

package/src/services/history/trend-analysis.ts

@@ -0,0 +1,238 @@
+/**
+ * Trend Analysis Service
+ *
+ * Statistical analysis of benchmark performance trends over time, including
+ * regression detection and trend classification.
+ */
+
+import type { BenchmarkRun } from '../../types/index.js';
+import type {
+  TrendData,
+  TrendDataPoint,
+  TrendsResult,
+  TrendStatistics,
+} from './models.js';
+
+/**
+ * Service for analyzing performance trends
+ */
+export class TrendAnalysisService {
+  /**
+   * Analyze trends across multiple benchmark runs
+   */
+  analyzeTrends(runs: BenchmarkRun[]): TrendsResult {
+    // Build trend data for each task across runs
+    const taskTrendsMap = new Map<string, TrendDataPoint[]>();
+
+    for (const run of runs) {
+      for (const file of run.files) {
+        for (const suite of file.suites) {
+          for (const task of suite.tasks) {
+            if (!task.error) {
+              const key = `${file.filePath}::${suite.name}::${task.name}`;
+              const dataPoints = taskTrendsMap.get(key) || [];
+              dataPoints.push({
+                date: new Date(run.startTime),
+                mean: task.mean,
+              });
+              taskTrendsMap.set(key, dataPoints);
+            }
+          }
+        }
+      }
+    }
+
+    // Calculate trends for each task
+    const trendsData: TrendData[] = [];
+
+    for (const [key, dataPoints] of taskTrendsMap.entries()) {
+      // Sort by date (oldest first)
+      dataPoints.sort((a, b) => a.date.getTime() - b.date.getTime());
+
+      const [_filePath, suiteName, taskName] = key.split('::');
+
+      const statistics = calculateStatistics(dataPoints);
+      const trend = calculateTrend(dataPoints);
+      const percentChange = calculatePercentChange(dataPoints);
+
+      trendsData.push({
+        confidence: 95, // Fixed confidence level for now
+        dataPoints,
+        percentChange,
+        runs: dataPoints.length,
+        statistics,
+        task: `${suiteName} › ${taskName}`,
+        trend,
+      });
+    }
+
+    // Sort by absolute percent change (most significant first)
+    trendsData.sort(
+      (a, b) => Math.abs(b.percentChange) - Math.abs(a.percentChange),
+    );
+
+    // Calculate summary statistics
+    const summary = {
+      degradingTasks: trendsData.filter((t) => t.trend === 'degrading').length,
+      improvingTasks: trendsData.filter((t) => t.trend === 'improving').length,
+      stableTasks: trendsData.filter((t) => t.trend === 'stable').length,
+      totalTasks: trendsData.length,
+    };
+
+    // Get timespan
+    const firstRun = runs[runs.length - 1];
+    const lastRun = runs[0];
+    const timespan = {
+      end: lastRun ? new Date(lastRun.startTime) : new Date(),
+      start: firstRun ? new Date(firstRun.startTime) : new Date(),
+    };
+
+    // Detect regressions (require minimum 5 runs for confidence)
+    const minRunsForRegression = 5;
+    const regressions = trendsData.filter((t) =>
+      detectRegression(t, 5, minRunsForRegression),
+    );
+
+    // Also track low-confidence potential regressions (2-4 runs showing degradation)
+    const lowConfidenceRegressions = trendsData.filter(
+      (t) =>
+        t.trend === 'degrading' &&
+        t.percentChange >= 5 &&
+        t.runs < minRunsForRegression &&
+        t.runs >= 2,
+    );
+
+    return {
+      lowConfidenceRegressions,
+      regressions,
+      runs: runs.length,
+      summary,
+      timespan,
+      trends: trendsData,
+    };
+  }
+}
+
+/**
+ * Calculate percent change from first to last data point
+ */
+export const calculatePercentChange = (
+  dataPoints: TrendDataPoint[],
+): number => {
+  if (dataPoints.length === 0 || dataPoints.length === 1) {
+    return 0;
+  }
+
+  const firstPoint = dataPoints[0];
+  const lastPoint = dataPoints[dataPoints.length - 1];
+
+  if (!firstPoint || !lastPoint) {
+    return 0;
+  }
+
+  const first = firstPoint.mean;
+  const last = lastPoint.mean;
+
+  if (first === 0) {
+    return 0; // Avoid division by zero
+  }
+
+  return ((last - first) / first) * 100;
+};
+
+/**
+ * Calculate statistical metrics from data points
+ */
+export const calculateStatistics = (
+  dataPoints: TrendDataPoint[],
+): TrendStatistics => {
+  if (dataPoints.length === 0) {
+    throw new Error('Cannot calculate statistics for empty data points array');
+  }
+
+  const values = dataPoints.map((dp) => dp.mean);
+  const n = values.length;
+
+  // Calculate mean
+  const mean = values.reduce((sum, val) => sum + val, 0) / n;
+
+  // Calculate median
+  const sorted = [...values].sort((a, b) => a - b);
+  const median =
+    n % 2 === 0
+      ? ((sorted[n / 2 - 1] ?? 0) + (sorted[n / 2] ?? 0)) / 2
+      : (sorted[Math.floor(n / 2)] ?? 0);
+
+  // Calculate variance and standard deviation
+  const variance =
+    n === 1
+      ? 0
+      : values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / n;
+  const stdDeviation = Math.sqrt(variance);
+
+  return {
+    mean,
+    median,
+    stdDeviation,
+    variance,
+  };
+};
+
+/**
+ * Calculate trend direction from data points using linear regression
+ */
+export const calculateTrend = (
+  dataPoints: TrendDataPoint[],
+): 'degrading' | 'improving' | 'stable' => {
+  if (dataPoints.length === 0) {
+    return 'stable';
+  }
+
+  if (dataPoints.length === 1) {
+    return 'stable';
+  }
+
+  // Simple linear regression to determine slope
+  const n = dataPoints.length;
+  const x = Array.from({ length: n }, (_, i) => i); // Time indices
+  const y = dataPoints.map((dp) => dp.mean);
+
+  const sumX = x.reduce((a, b) => a + b, 0);
+  const sumY = y.reduce((a, b) => a + b, 0);
+  const sumXY = x.reduce((sum, xi, i) => sum + xi * (y[i] ?? 0), 0);
+  const sumXX = x.reduce((sum, xi) => sum + xi * xi, 0);
+
+  // Calculate slope: (n*sumXY - sumX*sumY) / (n*sumXX - sumX*sumX)
+  const slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX);
+
+  // Determine significance threshold (5% of mean)
+  const meanValue = sumY / n;
+  const significanceThreshold = Math.abs(meanValue * 0.05);
+
+  // Classify trend based on slope
+  if (Math.abs(slope) < significanceThreshold / n) {
+    return 'stable';
+  } else if (slope < 0) {
+    // Negative slope = values decreasing = performance improving
+    return 'improving';
+  } else {
+    // Positive slope = values increasing = performance degrading
+    return 'degrading';
+  }
+};
+
+/**
+ * Detect if a trend represents a performance regression
+ */
+export const detectRegression = (
+  trendData: TrendData,
+  threshold: number,
+  minRuns: number,
+): boolean => {
+  // Regression is a degrading trend with percent change exceeding threshold
+  return (
+    trendData.trend === 'degrading' &&
+    trendData.percentChange >= threshold &&
+    trendData.runs >= minRuns
+  );
+};

package/src/services/profiler/profile-filter.ts

@@ -0,0 +1,143 @@
+/**
+ * Profile Filter Service
+ *
+ * Filters and sorts profiled functions based on configuration. Implements smart
+ * detection to focus on user code by excluding node_modules and Node.js
+ * internals.
+ *
+ * @packageDocumentation
+ */
+
+import { minimatch } from 'minimatch';
+
+import type {
+  FilteredProfileData,
+  ProfileConfig,
+  ProfiledFunction,
+  RawProfileData,
+} from '../../types/profiler.js';
+
+/**
+ * Filter profile data based on configuration
+ *
+ * @param data - Raw profile data
+ * @param config - Filter configuration
+ * @param packageRoot - Package root directory for smart detection
+ * @returns Filtered profile data
+ */
+export const filterProfile = (
+  data: RawProfileData,
+  config: ProfileConfig,
+  packageRoot: string,
+): FilteredProfileData => {
+  let filtered = data.functions.filter((fn) => {
+    // Only JavaScript functions
+    if (fn.category !== 'JavaScript') {
+      return false;
+    }
+
+    // Apply smart detection if enabled
+    if (config.smartDetection && !config.focus?.length) {
+      if (!isUserCode(fn.file, packageRoot)) {
+        return false;
+      }
+    }
+
+    // Apply focus patterns (if provided, overrides smart detection)
+    if (config.focus?.length) {
+      if (!matchesAnyPattern(fn.file, config.focus)) {
+        return false;
+      }
+    }
+
+    // Apply exclude patterns (always applied if provided)
+    if (config.exclude?.length) {
+      if (matchesAnyPattern(fn.file, config.exclude)) {
+        return false;
+      }
+    }
+
+    return true;
+  });
+
+  // Apply percentage threshold
+  const minPercent = config.minExecutionPercent ?? 0.5;
+  filtered = filtered.filter((fn) => fn.percentage >= minPercent);
+
+  // Sort by percentage (highest first)
+  filtered.sort((a, b) => b.percentage - a.percentage);
+
+  // Limit to topN
+  const topN = config.topN ?? 25;
+  filtered = filtered.slice(0, topN);
+
+  // Group by file if requested
+  let groupedByFile: Map<string, ProfiledFunction[]> | undefined;
+  if (config.groupByFile) {
+    groupedByFile = groupByFile(filtered);
+  }
+
+  return {
+    functions: filtered,
+    groupedByFile,
+    summary: data.summary,
+    totalFiltered: data.functions.length,
+    totalShown: filtered.length,
+    totalTicks: data.totalTicks,
+  };
+};
+
+/**
+ * Check if a file path is user code (not node_modules or internals)
+ */
+const isUserCode = (filePath: string, packageRoot: string): boolean => {
+  // Exclude node_modules
+  if (
+    filePath.includes('/node_modules/') ||
+    filePath.includes('\\node_modules\\')
+  ) {
+    return false;
+  }
+
+  // Exclude Node.js internals
+  if (filePath.startsWith('node:') || filePath.startsWith('internal/')) {
+    return false;
+  }
+
+  // Allow <unknown> files (could be eval'd code or other user code without file paths)
+  if (filePath === '<unknown>' || filePath === '[eval]') {
+    return true;
+  }
+
+  // Must be within package root
+  return filePath.startsWith(packageRoot);
+};
+
+/**
+ * Check if a file path matches any of the given glob patterns
+ */
+const matchesAnyPattern = (filePath: string, patterns: string[]): boolean => {
+  return patterns.some((pattern) => minimatch(filePath, pattern));
+};
+
+/**
+ * Group functions by file path
+ */
+const groupByFile = (
+  functions: ProfiledFunction[],
+): Map<string, ProfiledFunction[]> => {
+  const grouped = new Map<string, ProfiledFunction[]>();
+
+  for (const fn of functions) {
+    const existing = grouped.get(fn.file) || [];
+    existing.push(fn);
+    grouped.set(fn.file, existing);
+  }
+
+  // Sort functions within each file by percentage
+  for (const [, fns] of grouped.entries()) {
+    fns.sort((a, b) => b.percentage - a.percentage);
+  }
+
+  return grouped;
+};

package/src/services/profiler/profile-parser.ts

@@ -0,0 +1,194 @@
+/**
+ * Profile Parser Service
+ *
+ * Parses Chrome DevTools CPU profile format (*.cpuprofile files) generated by
+ * Node.js --cpu-prof flag. Extracts JavaScript function execution data
+ * including file paths, line numbers, hit counts, and percentages.
+ *
+ * @packageDocumentation
+ */
+
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import type {
+  ProfiledFunction,
+  ProfileSummary,
+  RawProfileData,
+} from '../../types/profiler.js';
+
+interface CallFrame {
+  columnNumber: number;
+  functionName: string;
+  lineNumber: number;
+  scriptId: string;
+  url: string;
+}
+
+/**
+ * Chrome DevTools CPU profile format
+ */
+interface CpuProfile {
+  endTime: number;
+  nodes: ProfileNode[];
+  samples?: number[];
+  startTime: number;
+}
+
+interface ProfileNode {
+  callFrame: CallFrame;
+  children?: number[];
+  hitCount?: number;
+  id: number;
+}
+
+/**
+ * Parse a CPU profile file
+ *
+ * @param profilePath - Path to *.cpuprofile file
+ * @returns Parsed profile data
+ */
+export const parseProfile = async (
+  profilePath: string,
+): Promise<RawProfileData> => {
+  // Read and parse JSON profile
+  const content = await readFile(profilePath, 'utf-8');
+  const profile: CpuProfile = JSON.parse(content) as CpuProfile;
+
+  // Extract functions and calculate statistics
+  return parseCpuProfile(profile, profilePath);
+};
+
+/**
+ * Parse CPU profile data structure
+ */
+const parseCpuProfile = (
+  profile: CpuProfile,
+  profilePath: string,
+): RawProfileData => {
+  const functions: ProfiledFunction[] = [];
+  let totalTicks = 0;
+
+  // Calculate total ticks from all node hit counts
+  for (const node of profile.nodes) {
+    totalTicks += node.hitCount || 0;
+  }
+
+  // Extract function information from nodes with hit counts
+  for (const node of profile.nodes) {
+    const hitCount = node.hitCount || 0;
+    if (hitCount === 0) {
+      continue; // Skip nodes with no samples
+    }
+
+    const { callFrame } = node;
+    const percentage = totalTicks > 0 ? (hitCount / totalTicks) * 100 : 0;
+
+    // Parse file path from URL
+    const filePath = parseFileUrl(callFrame.url);
+
+    // Determine category based on URL
+    const category = determineCategory(callFrame.url);
+
+    functions.push({
+      category,
+      file: filePath,
+      line: callFrame.lineNumber >= 0 ? callFrame.lineNumber + 1 : null, // DevTools uses 0-based line numbers
+      name: callFrame.functionName || '(anonymous)',
+      percentage,
+      ticks: hitCount,
+    });
+  }
+
+  // Calculate summary statistics
+  const javascriptTicks = functions
+    .filter((fn) => fn.category === 'JavaScript')
+    .reduce((sum, fn) => sum + fn.ticks, 0);
+
+  const cppTicks = functions
+    .filter((fn) => fn.category === 'C++')
+    .reduce((sum, fn) => sum + fn.ticks, 0);
+
+  const summary: ProfileSummary = {
+    cppTicks,
+    gcTicks: 0, // CPU profiles don't separate GC
+    javascriptTicks,
+    sharedLibraryTicks: 0,
+    totalTicks,
+  };
+
+  return {
+    functions,
+    logPath: profilePath,
+    summary,
+    totalTicks,
+  };
+};
+
+/**
+ * Parse file URL from call frame
+ */
+const parseFileUrl = (url: string): string => {
+  if (!url) {
+    return '<unknown>';
+  }
+
+  // Handle file:// URLs
+  if (url.startsWith('file://')) {
+    try {
+      return fileURLToPath(url);
+    } catch {
+      // Fallback if URL is malformed
+      return path.normalize(url.replace('file://', ''));
+    }
+  }
+
+  // Handle node: internal modules
+  if (url.startsWith('node:')) {
+    return url;
+  }
+
+  // Handle plain paths
+  return path.normalize(url);
+};
+
+/**
+ * Determine function category based on URL
+ */
+const determineCategory = (
+  url: string,
+): 'C++' | 'GC' | 'JavaScript' | 'Unknown' => {
+  if (!url) {
+    return 'Unknown';
+  }
+
+  // Node.js internals and native modules
+  if (url.startsWith('node:') || url.includes('[native code]')) {
+    return 'C++';
+  }
+
+  // eval'd code is JavaScript
+  if (url === '[eval]') {
+    return 'JavaScript';
+  }
+
+  // JavaScript files
+  if (
+    url.endsWith('.js') ||
+    url.endsWith('.mjs') ||
+    url.endsWith('.cjs') ||
+    url.endsWith('.ts') ||
+    url.endsWith('.mts') ||
+    url.endsWith('.cts')
+  ) {
+    return 'JavaScript';
+  }
+
+  // Default to JavaScript for file:// URLs
+  if (url.startsWith('file://')) {
+    return 'JavaScript';
+  }
+
+  return 'Unknown';
+};

package/src/services/profiler/profile-runner.ts

@@ -0,0 +1,121 @@
+/**
+ * Profile Runner Service
+ *
+ * Executes commands with Node.js CPU profiling enabled and captures profiler
+ * output to *.cpuprofile files in .modestbench/profiles/.
+ *
+ * @packageDocumentation
+ */
+
+import { glob } from 'glob';
+import { spawn } from 'node:child_process';
+import { mkdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+
+/**
+ * Options for running with profiling
+ */
+interface RunOptions {
+  /** Working directory */
+  cwd?: string;
+
+  /** Environment variables */
+  env?: NodeJS.ProcessEnv;
+
+  /** Timeout in milliseconds */
+  timeout?: number;
+}
+
+/**
+ * Run a command with Node.js profiling enabled
+ *
+ * @param command - Command to run (e.g., "npm test")
+ * @param options - Execution options
+ * @returns Path to generated *.cpuprofile file
+ */
+export const runWithProfiling = async (
+  command: string,
+  options: RunOptions = {},
+): Promise<string> => {
+  const cwd = options.cwd || process.cwd();
+
+  // Create profiles directory
+  const profilesDir = join(cwd, '.modestbench', 'profiles');
+  await mkdir(profilesDir, { recursive: true });
+
+  // Run command with NODE_OPTIONS="--cpu-prof --cpu-prof-dir=..."
+  const proc = spawn(command, {
+    cwd,
+    env: {
+      ...process.env,
+      ...options.env,
+      NODE_OPTIONS: `--cpu-prof --cpu-prof-dir=${profilesDir}`,
+    },
+    shell: true,
+    stdio: 'inherit',
+  });
+
+  // Wait for process to complete
+  await new Promise<void>((resolve, reject) => {
+    const timeout = options.timeout
+      ? setTimeout(() => {
+          proc.kill();
+          reject(
+            new Error(`Profile command timed out after ${options.timeout}ms`),
+          );
+        }, options.timeout)
+      : null;
+
+    proc.on('close', (code) => {
+      if (timeout) {
+        clearTimeout(timeout);
+      }
+      if (code === 0 || code === null) {
+        resolve();
+      } else {
+        reject(new Error(`Profile command exited with code ${code}`));
+      }
+    });
+
+    proc.on('error', (err) => {
+      if (timeout) {
+        clearTimeout(timeout);
+      }
+      reject(err);
+    });
+  });
+
+  // Find generated *.cpuprofile file in profiles directory
+  const profileFiles = await glob('*.cpuprofile', { cwd: profilesDir });
+
+  if (profileFiles.length === 0) {
+    throw new Error(
+      'No CPU profile generated. Ensure the command runs Node.js code.',
+    );
+  }
+
+  // Return most recent profile file
+  return await getMostRecentFile(profileFiles, profilesDir);
+};
+
+/**
+ * Get the most recently modified file from a list
+ */
+const getMostRecentFile = async (
+  files: string[],
+  cwd: string,
+): Promise<string> => {
+  let mostRecent = files[0];
+  let mostRecentTime = (await stat(`${cwd}/${mostRecent}`)).mtimeMs;
+
+  for (const file of files.slice(1)) {
+    const filePath = `${cwd}/${file}`;
+    const fileTime = (await stat(filePath)).mtimeMs;
+    if (fileTime > mostRecentTime) {
+      mostRecent = file;
+      mostRecentTime = fileTime;
+    }
+  }
+
+  return `${cwd}/${mostRecent}`;
+};