@sparkleideas/testing 3.0.0-alpha.10

package/src/regression/performance-baseline.ts

@@ -0,0 +1,356 @@
+/**
+ * Performance Baseline System
+ *
+ * Captures and compares performance metrics to detect regressions.
+ *
+ * @module v3/testing/regression/performance-baseline
+ */
+
+import { readFile, writeFile, mkdir } from 'fs/promises';
+import { join, dirname } from 'path';
+
+/**
+ * Baseline metric definition
+ */
+export interface BaselineMetric {
+  name: string;
+  value: number;
+  unit: string;
+  category: 'latency' | 'throughput' | 'memory' | 'cpu' | 'startup';
+  timestamp: number;
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Baseline comparison result
+ */
+export interface BaselineComparison {
+  metric: string;
+  baseline: number;
+  current: number;
+  unit: string;
+  degradation: number; // Percentage difference
+  regression: boolean; // Is this a significant regression?
+  improvement: boolean; // Is this an improvement?
+}
+
+/**
+ * Baseline configuration
+ */
+export interface BaselineConfig {
+  baselinePath: string;
+  performanceThreshold: number; // Percentage allowed degradation
+}
+
+/**
+ * Stored baseline data
+ */
+interface BaselineData {
+  version: string;
+  capturedAt: number;
+  metrics: BaselineMetric[];
+}
+
+/**
+ * Performance Baseline Manager
+ *
+ * Manages performance baselines for regression detection.
+ */
+export class PerformanceBaseline {
+  private readonly baselinePath: string;
+  private readonly threshold: number;
+  private cachedBaseline: BaselineData | null = null;
+
+  constructor(config: BaselineConfig) {
+    this.baselinePath = join(config.baselinePath, 'performance.json');
+    this.threshold = config.performanceThreshold;
+  }
+
+  /**
+   * Capture current performance as baseline
+   */
+  async captureBaseline(): Promise<BaselineData> {
+    const metrics = await this.measureCurrentPerformance();
+
+    const baseline: BaselineData = {
+      version: '1.0.0',
+      capturedAt: Date.now(),
+      metrics,
+    };
+
+    await this.saveBaseline(baseline);
+    this.cachedBaseline = baseline;
+
+    return baseline;
+  }
+
+  /**
+   * Compare current performance against baseline
+   */
+  async compare(): Promise<BaselineComparison[]> {
+    const baseline = await this.loadBaseline();
+    if (!baseline) {
+      console.warn('No baseline found. Capturing initial baseline...');
+      await this.captureBaseline();
+      return [];
+    }
+
+    const currentMetrics = await this.measureCurrentPerformance();
+    const comparisons: BaselineComparison[] = [];
+
+    for (const current of currentMetrics) {
+      const baselineMetric = baseline.metrics.find((m) => m.name === current.name);
+      if (!baselineMetric) continue;
+
+      const isHigherBetter = current.category === 'throughput';
+      const diff = current.value - baselineMetric.value;
+      const percentChange = (diff / baselineMetric.value) * 100;
+
+      // For latency/memory, higher is worse. For throughput, lower is worse.
+      const degradation = isHigherBetter ? -percentChange : percentChange;
+
+      comparisons.push({
+        metric: current.name,
+        baseline: baselineMetric.value,
+        current: current.value,
+        unit: current.unit,
+        degradation,
+        regression: degradation > this.threshold,
+        improvement: degradation < -this.threshold,
+      });
+    }
+
+    return comparisons;
+  }
+
+  /**
+   * Measure current performance metrics
+   */
+  private async measureCurrentPerformance(): Promise<BaselineMetric[]> {
+    const metrics: BaselineMetric[] = [];
+    const timestamp = Date.now();
+
+    // Memory metrics
+    const memUsage = process.memoryUsage();
+    metrics.push({
+      name: 'heap_used',
+      value: memUsage.heapUsed / 1024 / 1024,
+      unit: 'MB',
+      category: 'memory',
+      timestamp,
+    });
+    metrics.push({
+      name: 'heap_total',
+      value: memUsage.heapTotal / 1024 / 1024,
+      unit: 'MB',
+      category: 'memory',
+      timestamp,
+    });
+    metrics.push({
+      name: 'rss',
+      value: memUsage.rss / 1024 / 1024,
+      unit: 'MB',
+      category: 'memory',
+      timestamp,
+    });
+
+    // Startup time simulation
+    const startupStart = performance.now();
+    await this.simulateStartup();
+    const startupTime = performance.now() - startupStart;
+    metrics.push({
+      name: 'startup_time',
+      value: startupTime,
+      unit: 'ms',
+      category: 'startup',
+      timestamp,
+    });
+
+    // Latency benchmarks
+    const latencyMetrics = await this.measureLatency();
+    metrics.push(...latencyMetrics);
+
+    // Throughput benchmarks
+    const throughputMetrics = await this.measureThroughput();
+    metrics.push(...throughputMetrics);
+
+    return metrics;
+  }
+
+  /**
+   * Simulate startup to measure initialization time
+   */
+  private async simulateStartup(): Promise<void> {
+    // Import key modules to simulate startup
+    await import('@sparkleideas/shared');
+    await import('@sparkleideas/memory');
+  }
+
+  /**
+   * Measure operation latency
+   */
+  private async measureLatency(): Promise<BaselineMetric[]> {
+    const metrics: BaselineMetric[] = [];
+    const timestamp = Date.now();
+
+    // Event bus latency
+    const eventLatency = await this.benchmarkEventBus();
+    metrics.push({
+      name: 'event_bus_latency',
+      value: eventLatency,
+      unit: 'μs',
+      category: 'latency',
+      timestamp,
+    });
+
+    // Memory operation latency
+    const memLatency = await this.benchmarkMemoryOps();
+    metrics.push({
+      name: 'memory_op_latency',
+      value: memLatency,
+      unit: 'μs',
+      category: 'latency',
+      timestamp,
+    });
+
+    return metrics;
+  }
+
+  /**
+   * Measure throughput
+   */
+  private async measureThroughput(): Promise<BaselineMetric[]> {
+    const metrics: BaselineMetric[] = [];
+    const timestamp = Date.now();
+
+    // Events per second
+    const eventsPerSec = await this.benchmarkEventThroughput();
+    metrics.push({
+      name: 'events_per_second',
+      value: eventsPerSec,
+      unit: 'ops/sec',
+      category: 'throughput',
+      timestamp,
+    });
+
+    // Memory operations per second
+    const memOpsPerSec = await this.benchmarkMemoryThroughput();
+    metrics.push({
+      name: 'memory_ops_per_second',
+      value: memOpsPerSec,
+      unit: 'ops/sec',
+      category: 'throughput',
+      timestamp,
+    });
+
+    return metrics;
+  }
+
+  /**
+   * Benchmark event bus operations
+   */
+  private async benchmarkEventBus(): Promise<number> {
+    const { EventBus, createAgentSpawnedEvent } = await import('@sparkleideas/shared');
+    const eventBus = new EventBus();
+
+    const iterations = 1000;
+    const start = performance.now();
+
+    for (let i = 0; i < iterations; i++) {
+      const event = createAgentSpawnedEvent(`bench-${i}`, 'worker', 'default', ['test']);
+      await eventBus.emit(event);
+    }
+
+    const elapsed = performance.now() - start;
+    return (elapsed / iterations) * 1000; // Convert to microseconds
+  }
+
+  /**
+   * Benchmark memory operations
+   */
+  private async benchmarkMemoryOps(): Promise<number> {
+    // Simulate memory operations with a simple Map
+    const map = new Map<string, unknown>();
+    const iterations = 10000;
+    const start = performance.now();
+
+    for (let i = 0; i < iterations; i++) {
+      map.set(`key-${i}`, { data: `value-${i}`, timestamp: Date.now() });
+    }
+
+    for (let i = 0; i < iterations; i++) {
+      map.get(`key-${i}`);
+    }
+
+    const elapsed = performance.now() - start;
+    return (elapsed / (iterations * 2)) * 1000; // Convert to microseconds
+  }
+
+  /**
+   * Benchmark event throughput
+   */
+  private async benchmarkEventThroughput(): Promise<number> {
+    const { EventBus } = await import('@sparkleideas/shared');
+    const eventBus = new EventBus();
+
+    let count = 0;
+    eventBus.subscribe('agent:spawned', () => { count++; });
+
+    const { createAgentSpawnedEvent } = await import('@sparkleideas/shared');
+    const duration = 1000; // 1 second
+    const start = Date.now();
+
+    while (Date.now() - start < duration) {
+      const event = createAgentSpawnedEvent('bench-agent', 'worker', 'default', ['test']);
+      await eventBus.emit(event);
+    }
+
+    return count;
+  }
+
+  /**
+   * Benchmark memory throughput
+   */
+  private async benchmarkMemoryThroughput(): Promise<number> {
+    const map = new Map<string, unknown>();
+    let count = 0;
+
+    const duration = 1000; // 1 second
+    const start = Date.now();
+
+    while (Date.now() - start < duration) {
+      const key = `key-${count}`;
+      map.set(key, { data: count, timestamp: Date.now() });
+      map.get(key);
+      map.delete(key);
+      count++;
+    }
+
+    return count;
+  }
+
+  /**
+   * Load baseline from file
+   */
+  private async loadBaseline(): Promise<BaselineData | null> {
+    if (this.cachedBaseline) {
+      return this.cachedBaseline;
+    }
+
+    try {
+      const content = await readFile(this.baselinePath, 'utf-8');
+      this.cachedBaseline = JSON.parse(content);
+      return this.cachedBaseline;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Save baseline to file
+   */
+  private async saveBaseline(baseline: BaselineData): Promise<void> {
+    await mkdir(dirname(this.baselinePath), { recursive: true });
+    await writeFile(this.baselinePath, JSON.stringify(baseline, null, 2));
+  }
+}

package/src/regression/regression-runner.ts

@@ -0,0 +1,339 @@
+/**
+ * Regression Test Runner
+ *
+ * Orchestrates all regression tests and generates comprehensive reports.
+ *
+ * @module v3/testing/regression/regression-runner
+ */
+
+import { PerformanceBaseline, type BaselineComparison } from './performance-baseline.js';
+import { SecurityRegressionChecker, type SecurityReport } from './security-regression.js';
+import { APIContractValidator, type ContractValidation } from './api-contract.js';
+import { IntegrationRegressionSuite, type IntegrationResult } from './integration-regression.js';
+
+/**
+ * Regression test configuration
+ */
+export interface RegressionConfig {
+  /** Enable performance regression tests */
+  performanceTests: boolean;
+
+  /** Enable security regression tests */
+  securityTests: boolean;
+
+  /** Enable API contract tests */
+  contractTests: boolean;
+
+  /** Enable integration tests */
+  integrationTests: boolean;
+
+  /** Performance threshold (percentage allowed degradation) */
+  performanceThreshold: number;
+
+  /** Path to baseline data */
+  baselinePath: string;
+
+  /** Output path for reports */
+  reportPath: string;
+
+  /** Fail on any regression */
+  failOnRegression: boolean;
+
+  /** Verbose output */
+  verbose: boolean;
+}
+
+/**
+ * Individual test result
+ */
+export interface RegressionResult {
+  category: 'performance' | 'security' | 'contract' | 'integration';
+  name: string;
+  passed: boolean;
+  message: string;
+  details?: Record<string, unknown>;
+  duration: number;
+}
+
+/**
+ * Complete regression report
+ */
+export interface RegressionReport {
+  timestamp: Date;
+  duration: number;
+  passed: boolean;
+  totalTests: number;
+  passedTests: number;
+  failedTests: number;
+  skippedTests: number;
+
+  performance: {
+    tested: boolean;
+    results: BaselineComparison[];
+    regressions: string[];
+  };
+
+  security: {
+    tested: boolean;
+    report: SecurityReport | null;
+    newVulnerabilities: string[];
+  };
+
+  contract: {
+    tested: boolean;
+    results: ContractValidation[];
+    breakingChanges: string[];
+  };
+
+  integration: {
+    tested: boolean;
+    results: IntegrationResult[];
+    failures: string[];
+  };
+
+  summary: string;
+}
+
+/**
+ * Default configuration
+ */
+const DEFAULT_CONFIG: RegressionConfig = {
+  performanceTests: true,
+  securityTests: true,
+  contractTests: true,
+  integrationTests: true,
+  performanceThreshold: 10, // 10% degradation allowed
+  baselinePath: './.regression-baselines',
+  reportPath: './.regression-reports',
+  failOnRegression: true,
+  verbose: false,
+};
+
+/**
+ * Regression Test Runner
+ *
+ * Coordinates all regression testing activities.
+ */
+export class RegressionTestRunner {
+  private readonly config: RegressionConfig;
+  private readonly performanceBaseline: PerformanceBaseline;
+  private readonly securityChecker: SecurityRegressionChecker;
+  private readonly contractValidator: APIContractValidator;
+  private readonly integrationSuite: IntegrationRegressionSuite;
+
+  constructor(config: Partial<RegressionConfig> = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    this.performanceBaseline = new PerformanceBaseline(this.config);
+    this.securityChecker = new SecurityRegressionChecker();
+    this.contractValidator = new APIContractValidator();
+    this.integrationSuite = new IntegrationRegressionSuite();
+  }
+
+  /**
+   * Run all configured regression tests
+   */
+  async runAll(): Promise<RegressionReport> {
+    const startTime = Date.now();
+    const results: RegressionResult[] = [];
+
+    const report: RegressionReport = {
+      timestamp: new Date(),
+      duration: 0,
+      passed: true,
+      totalTests: 0,
+      passedTests: 0,
+      failedTests: 0,
+      skippedTests: 0,
+      performance: { tested: false, results: [], regressions: [] },
+      security: { tested: false, report: null, newVulnerabilities: [] },
+      contract: { tested: false, results: [], breakingChanges: [] },
+      integration: { tested: false, results: [], failures: [] },
+      summary: '',
+    };
+
+    // Run performance tests
+    if (this.config.performanceTests) {
+      this.log('Running performance regression tests...');
+      report.performance.tested = true;
+      report.performance.results = await this.performanceBaseline.compare();
+      report.performance.regressions = report.performance.results
+        .filter((r) => r.regression && r.degradation > this.config.performanceThreshold)
+        .map((r) => `${r.metric}: ${r.degradation.toFixed(1)}% degradation`);
+
+      if (report.performance.regressions.length > 0) {
+        report.passed = false;
+      }
+    }
+
+    // Run security tests
+    if (this.config.securityTests) {
+      this.log('Running security regression tests...');
+      report.security.tested = true;
+      report.security.report = await this.securityChecker.check();
+      report.security.newVulnerabilities = report.security.report?.newIssues || [];
+
+      if (report.security.newVulnerabilities.length > 0) {
+        report.passed = false;
+      }
+    }
+
+    // Run contract tests
+    if (this.config.contractTests) {
+      this.log('Running API contract tests...');
+      report.contract.tested = true;
+      report.contract.results = await this.contractValidator.validateAll();
+      report.contract.breakingChanges = report.contract.results
+        .filter((r) => !r.valid && r.breaking)
+        .map((r) => `${r.endpoint}: ${r.message}`);
+
+      if (report.contract.breakingChanges.length > 0) {
+        report.passed = false;
+      }
+    }
+
+    // Run integration tests
+    if (this.config.integrationTests) {
+      this.log('Running integration regression tests...');
+      report.integration.tested = true;
+      report.integration.results = await this.integrationSuite.runAll();
+      report.integration.failures = report.integration.results
+        .filter((r) => !r.passed)
+        .map((r) => `${r.name}: ${r.error}`);
+
+      if (report.integration.failures.length > 0) {
+        report.passed = false;
+      }
+    }
+
+    // Calculate summary
+    report.duration = Date.now() - startTime;
+    report.totalTests =
+      report.performance.results.length +
+      (report.security.report?.checks.length || 0) +
+      report.contract.results.length +
+      report.integration.results.length;
+
+    report.passedTests = report.totalTests - (
+      report.performance.regressions.length +
+      report.security.newVulnerabilities.length +
+      report.contract.breakingChanges.length +
+      report.integration.failures.length
+    );
+
+    report.failedTests = report.totalTests - report.passedTests;
+
+    report.summary = this.generateSummary(report);
+
+    return report;
+  }
+
+  /**
+   * Run only performance regression tests
+   */
+  async runPerformance(): Promise<BaselineComparison[]> {
+    return this.performanceBaseline.compare();
+  }
+
+  /**
+   * Run only security regression tests
+   */
+  async runSecurity(): Promise<SecurityReport> {
+    return this.securityChecker.check();
+  }
+
+  /**
+   * Run only API contract tests
+   */
+  async runContracts(): Promise<ContractValidation[]> {
+    return this.contractValidator.validateAll();
+  }
+
+  /**
+   * Run only integration tests
+   */
+  async runIntegration(): Promise<IntegrationResult[]> {
+    return this.integrationSuite.runAll();
+  }
+
+  /**
+   * Update baselines with current values
+   */
+  async updateBaselines(): Promise<void> {
+    await this.performanceBaseline.captureBaseline();
+    await this.contractValidator.captureContracts();
+    this.log('Baselines updated successfully');
+  }
+
+  /**
+   * Generate human-readable summary
+   */
+  private generateSummary(report: RegressionReport): string {
+    const lines: string[] = [
+      '═══════════════════════════════════════════════════════════════',
+      '                    REGRESSION TEST REPORT                      ',
+      '═══════════════════════════════════════════════════════════════',
+      '',
+      `Status: ${report.passed ? '✅ PASSED' : '❌ FAILED'}`,
+      `Duration: ${report.duration}ms`,
+      `Tests: ${report.passedTests}/${report.totalTests} passed`,
+      '',
+    ];
+
+    if (report.performance.tested) {
+      lines.push('📊 Performance:');
+      if (report.performance.regressions.length === 0) {
+        lines.push('   ✅ No performance regressions detected');
+      } else {
+        lines.push(`   ❌ ${report.performance.regressions.length} regressions:`);
+        report.performance.regressions.forEach((r) => lines.push(`      - ${r}`));
+      }
+      lines.push('');
+    }
+
+    if (report.security.tested) {
+      lines.push('🔒 Security:');
+      if (report.security.newVulnerabilities.length === 0) {
+        lines.push('   ✅ No new security vulnerabilities');
+      } else {
+        lines.push(`   ❌ ${report.security.newVulnerabilities.length} new vulnerabilities:`);
+        report.security.newVulnerabilities.forEach((v) => lines.push(`      - ${v}`));
+      }
+      lines.push('');
+    }
+
+    if (report.contract.tested) {
+      lines.push('📋 API Contracts:');
+      if (report.contract.breakingChanges.length === 0) {
+        lines.push('   ✅ No breaking changes detected');
+      } else {
+        lines.push(`   ❌ ${report.contract.breakingChanges.length} breaking changes:`);
+        report.contract.breakingChanges.forEach((c) => lines.push(`      - ${c}`));
+      }
+      lines.push('');
+    }
+
+    if (report.integration.tested) {
+      lines.push('🔗 Integration:');
+      if (report.integration.failures.length === 0) {
+        lines.push('   ✅ All integration tests passed');
+      } else {
+        lines.push(`   ❌ ${report.integration.failures.length} failures:`);
+        report.integration.failures.forEach((f) => lines.push(`      - ${f}`));
+      }
+      lines.push('');
+    }
+
+    lines.push('═══════════════════════════════════════════════════════════════');
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Log message if verbose mode is enabled
+   */
+  private log(message: string): void {
+    if (this.config.verbose) {
+      console.log(`[Regression] ${message}`);
+    }
+  }
+}