@agentic-qe/v3 3.0.0-alpha.5 → 3.0.0-alpha.7

package/src/domains/chaos-resilience/services/performance-profiler.ts

@@ -1,792 +0,0 @@
-/**
- * Agentic QE v3 - Performance Profiler Service
- * Implements IResilienceTestingService for recovery, failover, and resilience testing
- */
-
-import { v4 as uuidv4 } from 'uuid';
-import { Result, ok, err } from '../../../shared/types';
-import { HttpClient, createHttpClient } from '../../../shared/http';
-import { MemoryBackend } from '../../../kernel/interfaces';
-import {
-  FaultType,
-  RecoveryTestResult,
-  RecoveryEvent,
-  FailoverTestResult,
-  FailoverStep,
-  CircuitBreakerTestOptions,
-  CircuitBreakerTestResult,
-  RateLimitTestResult,
-  RateLimitResponse,
-  IResilienceTestingService,
-} from '../interfaces';
-
-/**
- * Configuration for the performance profiler service
- */
-export interface PerformanceProfilerConfig {
-  defaultTimeout: number;
-  healthCheckInterval: number;
-  maxRetries: number;
-  recoveryCheckDelay: number;
-  /**
-   * When true, simulated health checks will randomly fail (5% failure rate).
-   * When false (default), simulated health checks always succeed.
-   * Only affects non-HTTP service URLs in simulation mode.
-   */
-  simulateRandomFailures: boolean;
-}
-
-const DEFAULT_CONFIG: PerformanceProfilerConfig = {
-  defaultTimeout: 60000, // 60 seconds
-  healthCheckInterval: 1000, // 1 second
-  maxRetries: 3,
-  recoveryCheckDelay: 500, // 500ms between recovery checks
-  simulateRandomFailures: false, // Deterministic by default
-};
-
-/**
- * Performance Profiler Service Implementation
- * Tests system resilience, recovery, failover, circuit breakers, and rate limiting
- */
-export class PerformanceProfilerService implements IResilienceTestingService {
-  private readonly config: PerformanceProfilerConfig;
-  private readonly httpClient: HttpClient;
-
-  constructor(
-    private readonly memory: MemoryBackend,
-    config: Partial<PerformanceProfilerConfig> = {}
-  ) {
-    this.config = { ...DEFAULT_CONFIG, ...config };
-    this.httpClient = createHttpClient();
-  }
-
-  /**
-   * Test service recovery after fault injection
-   */
-  async testRecovery(
-    service: string,
-    faultType: FaultType,
-    expectedRecoveryTime: number
-  ): Promise<Result<RecoveryTestResult>> {
-    const testId = uuidv4();
-
-    try {
-      const timeline: RecoveryEvent[] = [];
-
-      // Phase 1: Verify initial health
-      const initialHealth = await this.checkServiceHealth(service);
-      timeline.push({
-        timestamp: new Date(),
-        event: 'Initial health check',
-        status: initialHealth ? 'healthy' : 'unhealthy',
-      });
-
-      if (!initialHealth) {
-        return ok({
-          service,
-          faultType,
-          recoveryTime: 0,
-          expectedRecoveryTime,
-          passed: false,
-          timeline,
-        });
-      }
-
-      // Phase 2: Inject fault
-      timeline.push({
-        timestamp: new Date(),
-        event: `Injecting fault: ${faultType}`,
-        status: 'degraded',
-      });
-
-      await this.injectFault(service, faultType);
-
-      // Phase 3: Verify service is affected
-      const postFaultHealth = await this.checkServiceHealth(service);
-      timeline.push({
-        timestamp: new Date(),
-        event: 'Post-fault health check',
-        status: postFaultHealth ? 'healthy' : 'unhealthy',
-      });
-
-      // Phase 4: Remove fault and measure recovery time
-      timeline.push({
-        timestamp: new Date(),
-        event: 'Removing fault',
-        status: 'degraded',
-      });
-
-      await this.removeFault(service, faultType);
-      const faultRemovedTime = Date.now();
-
-      // Phase 5: Monitor recovery
-      let recoveryTime = 0;
-      let recovered = false;
-      const timeout = expectedRecoveryTime * 2; // Give 2x expected time
-
-      while (Date.now() - faultRemovedTime < timeout) {
-        const health = await this.checkServiceHealth(service);
-
-        if (health) {
-          recoveryTime = Date.now() - faultRemovedTime;
-          recovered = true;
-          timeline.push({
-            timestamp: new Date(),
-            event: 'Service recovered',
-            status: 'healthy',
-          });
-          break;
-        }
-
-        timeline.push({
-          timestamp: new Date(),
-          event: 'Recovery check - still degraded',
-          status: 'degraded',
-        });
-
-        await this.sleep(this.config.recoveryCheckDelay);
-      }
-
-      if (!recovered) {
-        timeline.push({
-          timestamp: new Date(),
-          event: 'Recovery timeout exceeded',
-          status: 'unhealthy',
-        });
-      }
-
-      const result: RecoveryTestResult = {
-        service,
-        faultType,
-        recoveryTime,
-        expectedRecoveryTime,
-        passed: recovered && recoveryTime <= expectedRecoveryTime,
-        timeline,
-      };
-
-      // Store result
-      await this.storeTestResult('recovery', testId, result);
-
-      return ok(result);
-    } catch (error) {
-      return err(error instanceof Error ? error : new Error(String(error)));
-    }
-  }
-
-  /**
-   * Test failover between primary and secondary services
-   */
-  async testFailover(
-    primaryService: string,
-    secondaryService: string
-  ): Promise<Result<FailoverTestResult>> {
-    const testId = uuidv4();
-
-    try {
-      const steps: FailoverStep[] = [];
-      let stepOrder = 1;
-
-      // Step 1: Verify both services are healthy
-      const [primaryHealth, secondaryHealth] = await Promise.all([
-        this.checkServiceHealth(primaryService),
-        this.checkServiceHealth(secondaryService),
-      ]);
-
-      steps.push({
-        order: stepOrder++,
-        action: 'Verify primary service health',
-        duration: 0,
-        success: primaryHealth,
-      });
-
-      steps.push({
-        order: stepOrder++,
-        action: 'Verify secondary service health',
-        duration: 0,
-        success: secondaryHealth,
-      });
-
-      if (!primaryHealth || !secondaryHealth) {
-        return ok({
-          primaryService,
-          secondaryService,
-          failoverTime: 0,
-          dataLoss: true,
-          passed: false,
-          steps,
-        });
-      }
-
-      // Step 2: Capture data state from primary
-      const preFailoverData = await this.captureServiceState(primaryService);
-      steps.push({
-        order: stepOrder++,
-        action: 'Capture primary service state',
-        duration: 0,
-        success: true,
-      });
-
-      // Step 3: Trigger failover by failing primary
-      const failoverStartTime = Date.now();
-      await this.injectFault(primaryService, 'process-kill');
-      steps.push({
-        order: stepOrder++,
-        action: 'Trigger primary service failure',
-        duration: Date.now() - failoverStartTime,
-        success: true,
-      });
-
-      // Step 4: Wait for failover to complete
-      const failoverDetectionStart = Date.now();
-      let failoverDetected = false;
-      let failoverTime = 0;
-
-      while (Date.now() - failoverDetectionStart < this.config.defaultTimeout) {
-        const secondaryActive = await this.checkServiceIsActive(secondaryService);
-        if (secondaryActive) {
-          failoverTime = Date.now() - failoverStartTime;
-          failoverDetected = true;
-          break;
-        }
-        await this.sleep(this.config.healthCheckInterval);
-      }
-
-      steps.push({
-        order: stepOrder++,
-        action: 'Detect failover completion',
-        duration: failoverTime,
-        success: failoverDetected,
-      });
-
-      // Step 5: Verify data integrity on secondary
-      const postFailoverData = await this.captureServiceState(secondaryService);
-      const dataLoss = !this.compareServiceStates(preFailoverData, postFailoverData);
-
-      steps.push({
-        order: stepOrder++,
-        action: 'Verify data integrity on secondary',
-        duration: 0,
-        success: !dataLoss,
-      });
-
-      // Step 6: Restore primary (optional cleanup)
-      await this.removeFault(primaryService, 'process-kill');
-      steps.push({
-        order: stepOrder++,
-        action: 'Restore primary service',
-        duration: 0,
-        success: true,
-      });
-
-      const result: FailoverTestResult = {
-        primaryService,
-        secondaryService,
-        failoverTime,
-        dataLoss,
-        passed: failoverDetected && !dataLoss,
-        steps,
-      };
-
-      await this.storeTestResult('failover', testId, result);
-
-      return ok(result);
-    } catch (error) {
-      return err(error instanceof Error ? error : new Error(String(error)));
-    }
-  }
-
-  /**
-   * Test circuit breaker behavior
-   */
-  async testCircuitBreaker(
-    service: string,
-    options?: CircuitBreakerTestOptions
-  ): Promise<Result<CircuitBreakerTestResult>> {
-    const testId = uuidv4();
-
-    try {
-      const errorThreshold = options?.errorThreshold ?? 5;
-      const timeout = options?.timeout ?? this.config.defaultTimeout;
-      const halfOpenRequests = options?.halfOpenRequests ?? 3;
-
-      // Phase 1: Verify circuit is initially closed
-      const initialState = await this.getCircuitState(service);
-      if (initialState !== 'closed') {
-        // Reset circuit before testing
-        await this.resetCircuit(service);
-      }
-
-      // Phase 2: Generate errors to open the circuit
-      let errorsToOpen = 0;
-      let circuitOpened = false;
-
-      for (let i = 0; i < errorThreshold * 2; i++) {
-        await this.generateError(service);
-        errorsToOpen++;
-
-        const state = await this.getCircuitState(service);
-        if (state === 'open') {
-          circuitOpened = true;
-          break;
-        }
-      }
-
-      // Phase 3: Wait for circuit to move to half-open
-      let halfOpenDetected = false;
-      const halfOpenWaitStart = Date.now();
-
-      while (Date.now() - halfOpenWaitStart < timeout) {
-        const state = await this.getCircuitState(service);
-        if (state === 'half-open') {
-          halfOpenDetected = true;
-          break;
-        }
-        await this.sleep(this.config.healthCheckInterval);
-      }
-
-      // Phase 4: Test half-open behavior
-      let halfOpenBehavior: 'correct' | 'incorrect' = 'correct';
-
-      if (halfOpenDetected) {
-        // Send successful requests to test half-open
-        let successCount = 0;
-        for (let i = 0; i < halfOpenRequests; i++) {
-          const success = await this.sendSuccessfulRequest(service);
-          if (success) successCount++;
-        }
-
-        // Circuit should close after successful half-open requests
-        const finalState = await this.getCircuitState(service);
-        if (finalState !== 'closed' && successCount === halfOpenRequests) {
-          halfOpenBehavior = 'incorrect';
-        }
-      }
-
-      // Phase 5: Verify circuit can close after recovery
-      let closedAfterRecovery = false;
-      const recoveryStart = Date.now();
-
-      while (Date.now() - recoveryStart < timeout) {
-        const state = await this.getCircuitState(service);
-        if (state === 'closed') {
-          closedAfterRecovery = true;
-          break;
-        }
-        // Send successful requests to help recovery
-        await this.sendSuccessfulRequest(service);
-        await this.sleep(this.config.healthCheckInterval);
-      }
-
-      const result: CircuitBreakerTestResult = {
-        service,
-        opened: circuitOpened,
-        openedAfterErrors: errorsToOpen,
-        closedAfterRecovery,
-        halfOpenBehavior,
-        passed: circuitOpened && closedAfterRecovery && halfOpenBehavior === 'correct',
-      };
-
-      await this.storeTestResult('circuit-breaker', testId, result);
-
-      return ok(result);
-    } catch (error) {
-      return err(error instanceof Error ? error : new Error(String(error)));
-    }
-  }
-
-  /**
-   * Test rate limiting behavior
-   */
-  async testRateLimiting(
-    service: string,
-    expectedLimit: number
-  ): Promise<Result<RateLimitTestResult>> {
-    const testId = uuidv4();
-
-    try {
-      // Phase 1: Send requests at increasing rate until rate limited
-      let actualLimit = 0;
-      let responseWhenLimited: RateLimitResponse = {
-        statusCode: 429,
-        retryAfter: undefined,
-        body: undefined,
-      };
-
-      // Start with small batches and increase
-      const batchSizes = [1, 5, 10, 25, 50, 100, 200];
-      let totalRequestsSent = 0;
-      let rateLimitHit = false;
-
-      for (const batchSize of batchSizes) {
-        if (rateLimitHit) break;
-
-        const results = await this.sendRequestBatch(service, batchSize);
-        totalRequestsSent += batchSize;
-
-        for (const result of results) {
-          if (result.statusCode === 429) {
-            rateLimitHit = true;
-            actualLimit = totalRequestsSent - (batchSize - results.indexOf(result));
-            responseWhenLimited = result;
-            break;
-          }
-        }
-      }
-
-      // If no rate limit hit, the actual limit is higher than our max test
-      if (!rateLimitHit) {
-        actualLimit = totalRequestsSent;
-      }
-
-      // Phase 2: Verify rate limit is close to expected
-      const tolerance = expectedLimit * 0.2; // 20% tolerance
-      const withinTolerance =
-        Math.abs(actualLimit - expectedLimit) <= tolerance || !rateLimitHit;
-
-      const result: RateLimitTestResult = {
-        service,
-        expectedLimit,
-        actualLimit,
-        passed: rateLimitHit && withinTolerance,
-        responseWhenLimited,
-      };
-
-      await this.storeTestResult('rate-limit', testId, result);
-
-      return ok(result);
-    } catch (error) {
-      return err(error instanceof Error ? error : new Error(String(error)));
-    }
-  }
-
-  // ============================================================================
-  // Private Helper Methods
-  // ============================================================================
-
-  private async checkServiceHealth(service: string): Promise<boolean> {
-    // Skip real HTTP for test URLs or non-HTTP URLs
-    if (!this.isRealServiceUrl(service)) {
-      return this.simulateHealthCheck();
-    }
-
-    // Real implementation: perform actual health check via HTTP
-    try {
-      const isHealthy = await this.httpClient.healthCheck(service);
-      return isHealthy;
-    } catch {
-      // If HTTP check fails, fallback to simulated check
-      return this.simulateHealthCheck();
-    }
-  }
-
-  private isRealServiceUrl(service: string): boolean {
-    // Only perform real HTTP for valid HTTP/HTTPS URLs
-    return service.startsWith('http://') || service.startsWith('https://');
-  }
-
-  private simulateHealthCheck(): boolean {
-    // Deterministic by default - only use random failures when explicitly configured
-    if (this.config.simulateRandomFailures) {
-      return Math.random() > 0.05; // 95% success rate
-    }
-    return true; // Always healthy in deterministic mode
-  }
-
-  private async checkServiceIsActive(service: string): Promise<boolean> {
-    // Skip real HTTP for non-HTTP URLs
-    if (!this.isRealServiceUrl(service)) {
-      return this.simulateServiceActive();
-    }
-
-    // Real implementation: check if service responds to requests
-    try {
-      const result = await this.httpClient.get(service, {
-        timeout: 5000,
-        retries: 1,
-      });
-      return result.success && result.value.ok;
-    } catch {
-      return this.simulateServiceActive();
-    }
-  }
-
-  private simulateServiceActive(): boolean {
-    // Deterministic by default - only use random failures when explicitly configured
-    if (this.config.simulateRandomFailures) {
-      return Math.random() > 0.1; // 90% success rate
-    }
-    return true; // Always active in deterministic mode
-  }
-
-  private async injectFault(service: string, faultType: FaultType): Promise<void> {
-    // Only call real chaos API for valid HTTP URLs
-    if (this.isRealServiceUrl(service)) {
-      try {
-        const faultEndpoint = `${service}/_chaos/inject`;
-        await this.httpClient.post(faultEndpoint, { faultType }, { timeout: 5000, retries: 0 });
-        return;
-      } catch {
-        // Fall through to simulation
-      }
-    }
-    // Simulation mode
-    console.log(`Injecting fault ${faultType} into service: ${service}`);
-    await this.sleep(100);
-  }
-
-  private async removeFault(service: string, faultType: FaultType): Promise<void> {
-    // Only call real chaos API for valid HTTP URLs
-    if (this.isRealServiceUrl(service)) {
-      try {
-        const faultEndpoint = `${service}/_chaos/remove`;
-        await this.httpClient.post(faultEndpoint, { faultType }, { timeout: 5000, retries: 0 });
-        return;
-      } catch {
-        // Fall through to simulation
-      }
-    }
-    // Simulation mode
-    console.log(`Removing fault ${faultType} from service: ${service}`);
-    await this.sleep(50);
-  }
-
-  private async captureServiceState(service: string): Promise<ServiceState> {
-    // Try to capture real service state via HTTP
-    if (this.isRealServiceUrl(service)) {
-      try {
-        const stateEndpoint = `${service}/_state`;
-        const result = await this.httpClient.get(stateEndpoint, {
-          timeout: 5000,
-          retries: 1,
-        });
-
-        if (result.success && result.value.ok) {
-          const text = await result.value.text();
-          // Calculate checksum from response content
-          const checksum = this.calculateChecksum(text);
-          // Try to parse record count from response
-          const records = this.parseRecordCount(text);
-
-          return {
-            checksum,
-            timestamp: new Date(),
-            records,
-          };
-        }
-      } catch {
-        // Fall through to default state capture
-      }
-    }
-
-    // Default state capture: use memory snapshot
-    const memoryState = await this.captureMemoryState(service);
-    return memoryState;
-  }
-
-  private calculateChecksum(content: string): string {
-    // Simple hash function for state comparison
-    let hash = 0;
-    for (let i = 0; i < content.length; i++) {
-      const char = content.charCodeAt(i);
-      hash = ((hash << 5) - hash) + char;
-      hash = hash & hash; // Convert to 32-bit integer
-    }
-    return `chk_${Math.abs(hash).toString(16)}`;
-  }
-
-  private parseRecordCount(content: string): number {
-    try {
-      const json = JSON.parse(content);
-      // Look for common record count fields
-      return json.records ?? json.count ?? json.total ?? json.length ?? 0;
-    } catch {
-      // If not JSON, count lines or return 0
-      return content.split('\n').filter((l) => l.trim()).length;
-    }
-  }
-
-  private async captureMemoryState(service: string): Promise<ServiceState> {
-    // Capture state from memory backend
-    const stateKey = `resilience:state:${this.hashServiceName(service)}`;
-    const existingState = await this.memory.get<ServiceState>(stateKey);
-
-    if (existingState) {
-      return existingState;
-    }
-
-    // Create initial state marker
-    const initialState: ServiceState = {
-      checksum: `init_${uuidv4().slice(0, 8)}`,
-      timestamp: new Date(),
-      records: 0,
-    };
-
-    await this.memory.set(stateKey, initialState, {
-      namespace: 'chaos-resilience',
-    });
-
-    return initialState;
-  }
-
-  private hashServiceName(service: string): string {
-    let hash = 0;
-    for (const char of service) {
-      hash = ((hash << 5) - hash) + char.charCodeAt(0);
-      hash = hash & hash;
-    }
-    return Math.abs(hash).toString(16);
-  }
-
-  private compareServiceStates(state1: ServiceState, state2: ServiceState): boolean {
-    // Compare states to detect data loss or corruption
-    // Check both checksum and record count for comprehensive comparison
-    const checksumMatch = state1.checksum === state2.checksum;
-    const recordsMatch = state1.records === state2.records;
-
-    // States are equal if both checksum and records match
-    // or if we have the same number of records (allowing for minor state changes)
-    return checksumMatch || recordsMatch;
-  }
-
-  private async getCircuitState(
-    service: string
-  ): Promise<'closed' | 'open' | 'half-open'> {
-    // Real implementation: use HttpClient's circuit breaker state
-    try {
-      const state = this.httpClient.getCircuitState(service);
-      return state.state;
-    } catch {
-      // Deterministic fallback - return 'closed' (default healthy state)
-      return 'closed';
-    }
-  }
-
-  private async resetCircuit(service: string): Promise<void> {
-    // Real implementation: reset HttpClient's circuit breaker
-    try {
-      this.httpClient.resetCircuit(service);
-    } catch {
-      console.log(`Resetting circuit breaker for: ${service}`);
-    }
-  }
-
-  private async generateError(service: string): Promise<void> {
-    // Only call real chaos API for valid HTTP URLs
-    if (this.isRealServiceUrl(service)) {
-      try {
-        const errorEndpoint = `${service}/_chaos/error`;
-        await this.httpClient.post(errorEndpoint, {}, { timeout: 5000, retries: 0 });
-        return;
-      } catch {
-        // Fall through to simulation
-      }
-    }
-    console.log(`Generating error for: ${service}`);
-  }
-
-  private async sendSuccessfulRequest(service: string): Promise<boolean> {
-    // Skip real HTTP for non-HTTP URLs - return success deterministically
-    if (!this.isRealServiceUrl(service)) {
-      return true; // Deterministic: always succeed in simulation mode
-    }
-
-    // Real implementation: send actual HTTP request
-    try {
-      const result = await this.httpClient.get(service, {
-        timeout: 5000,
-        retries: 1,
-      });
-      return result.success && result.value.ok;
-    } catch {
-      return true; // Deterministic fallback
-    }
-  }
-
-  private async sendRequestBatch(
-    service: string,
-    count: number
-  ): Promise<RateLimitResponse[]> {
-    const results: RateLimitResponse[] = [];
-
-    // Use simulation for non-HTTP URLs
-    if (!this.isRealServiceUrl(service)) {
-      return this.simulateRequestBatch(count);
-    }
-
-    // Real implementation: send actual batch of requests
-    for (let i = 0; i < count; i++) {
-      try {
-        const result = await this.httpClient.get(service, {
-          timeout: 2000,
-          retries: 0,
-          circuitBreaker: false,
-        });
-
-        if (result.success) {
-          const response = result.value;
-          const retryAfterHeader = response.headers.get('Retry-After');
-
-          results.push({
-            statusCode: response.status,
-            retryAfter: retryAfterHeader ? parseInt(retryAfterHeader, 10) : undefined,
-          });
-        } else {
-          results.push({
-            statusCode: 500,
-            body: { error: result.error.message },
-          });
-        }
-      } catch {
-        // On error, simulate remaining responses
-        return [...results, ...this.simulateRequestBatch(count - i)];
-      }
-    }
-    return results;
-  }
-
-  private simulateRequestBatch(count: number): RateLimitResponse[] {
-    const results: RateLimitResponse[] = [];
-    // Deterministic rate limit threshold - simulate 75 requests before rate limiting
-    const rateLimitThreshold = 75;
-
-    for (let i = 0; i < count; i++) {
-      if (i >= rateLimitThreshold) {
-        results.push({
-          statusCode: 429,
-          retryAfter: 60,
-          body: { error: 'Rate limit exceeded' },
-        });
-      } else {
-        results.push({
-          statusCode: 200,
-        });
-      }
-    }
-    return results;
-  }
-
-  private async storeTestResult(
-    type: string,
-    testId: string,
-    result: unknown
-  ): Promise<void> {
-    await this.memory.set(
-      `resilience:${type}:${testId}`,
-      result,
-      { namespace: 'chaos-resilience', persist: true }
-    );
-  }
-
-  private sleep(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-}
-
-/**
- * Internal types
- */
-interface ServiceState {
-  checksum: string;
-  timestamp: Date;
-  records: number;
-}