claude-autopm 2.12.2 → 3.0.0

package/packages/plugin-testing/test/commands/test-flaky-detect.test.js

@@ -0,0 +1,623 @@
+/**
+ * Test suite for /test:flaky-detect command
+ *
+ * Following TDD methodology with Context7-verified patterns:
+ * - Jest retry mechanisms (jest.retryTimes)
+ * - Playwright flaky detection (testInfo.retry)
+ * - Root cause analysis patterns
+ * - Statistical flakiness scoring
+ */
+
+const flakyDetect = require('../../lib/flaky-detect');
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+// Mock dependencies
+jest.mock('child_process');
+jest.mock('fs');
+
+describe('/test:flaky-detect command', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('Test Execution', () => {
+    test('should run tests multiple times with specified iterations', async () => {
+      const mockTestResults = [
+        { status: 'passed', duration: 120 },
+        { status: 'passed', duration: 125 },
+        { status: 'failed', duration: 130 },
+      ];
+
+      const result = await flakyDetect.runTestsRepeatedly(
+        'user-login.test.js',
+        3
+      );
+
+      expect(result.iterations).toBe(3);
+      expect(result.results).toHaveLength(3);
+    });
+
+    test('should track pass/fail results for each iteration', async () => {
+      const result = await flakyDetect.runTestsRepeatedly(
+        'checkout.test.js',
+        10
+      );
+
+      expect(result).toHaveProperty('passed');
+      expect(result).toHaveProperty('failed');
+      expect(result.passed + result.failed).toBe(10);
+    });
+
+    test('should handle test timeouts gracefully', async () => {
+      const result = await flakyDetect.runTestsRepeatedly(
+        'slow-test.test.js',
+        5,
+        { timeout: 1000 }
+      );
+
+      expect(result.timeouts).toBeGreaterThanOrEqual(0);
+      expect(result.results.some(r => r.status === 'timeout')).toBeDefined();
+    });
+
+    test('should support parallel execution for faster detection', async () => {
+      const startTime = Date.now();
+
+      const result = await flakyDetect.runTestsRepeatedly(
+        'api-test.test.js',
+        20,
+        { parallel: true, workers: 4 }
+      );
+
+      const duration = Date.now() - startTime;
+      expect(result.parallelExecution).toBe(true);
+      expect(result.workers).toBe(4);
+    });
+
+    test('should capture detailed error information on failures', async () => {
+      const result = await flakyDetect.runTestsRepeatedly(
+        'failing-test.test.js',
+        5
+      );
+
+      const failedRuns = result.results.filter(r => r.status === 'failed');
+      failedRuns.forEach(run => {
+        expect(run).toHaveProperty('error');
+        expect(run).toHaveProperty('stackTrace');
+        expect(run).toHaveProperty('timestamp');
+      });
+    });
+  });
+
+  describe('Pattern Analysis', () => {
+    test('should detect intermittent failures', () => {
+      const results = [
+        { status: 'passed' },
+        { status: 'failed' },
+        { status: 'passed' },
+        { status: 'failed' },
+        { status: 'passed' },
+      ];
+
+      const analysis = flakyDetect.analyzeFailurePatterns(results);
+
+      expect(analysis.isFlaky).toBe(true);
+      expect(analysis.pattern).toBe('intermittent');
+    });
+
+    test('should calculate accurate flakiness rate', () => {
+      const results = Array(100).fill(null).map((_, i) => ({
+        status: i < 95 ? 'passed' : 'failed'
+      }));
+
+      const analysis = flakyDetect.analyzeFailurePatterns(results);
+
+      expect(analysis.flakinessRate).toBe(0.05); // 5% failure rate
+    });
+
+    test('should identify failure patterns (consistent vs random)', () => {
+      const consistentResults = [
+        { status: 'passed' },
+        { status: 'passed' },
+        { status: 'failed' },
+        { status: 'failed' },
+        { status: 'failed' },
+      ];
+
+      const analysis = flakyDetect.analyzeFailurePatterns(consistentResults);
+
+      expect(analysis.pattern).toBe('degrading');
+      expect(analysis.isFlaky).toBe(true);
+    });
+
+    test('should detect timing-based flakiness patterns', () => {
+      const results = [
+        { status: 'passed', duration: 100 },
+        { status: 'failed', duration: 1500 },
+        { status: 'passed', duration: 110 },
+        { status: 'failed', duration: 1450 },
+      ];
+
+      const analysis = flakyDetect.analyzeFailurePatterns(results);
+
+      expect(analysis.timingRelated).toBe(true);
+      expect(analysis.avgFailDuration).toBeGreaterThan(analysis.avgPassDuration);
+    });
+
+    test('should identify tests that never fail (not flaky)', () => {
+      const results = Array(50).fill({ status: 'passed' });
+
+      const analysis = flakyDetect.analyzeFailurePatterns(results);
+
+      expect(analysis.isFlaky).toBe(false);
+      expect(analysis.flakinessRate).toBe(0);
+    });
+
+    test('should identify tests that always fail (not flaky)', () => {
+      const results = Array(50).fill({ status: 'failed' });
+
+      const analysis = flakyDetect.analyzeFailurePatterns(results);
+
+      expect(analysis.isFlaky).toBe(false);
+      expect(analysis.flakinessRate).toBe(1.0);
+    });
+  });
+
+  describe('Root Cause Analysis', () => {
+    test('should detect timing issues (async/await problems)', () => {
+      const failures = [
+        { error: 'Timeout waiting for element', type: 'timeout' },
+        { error: 'Promise rejection unhandled', type: 'async' },
+      ];
+
+      const rootCause = flakyDetect.identifyRootCause(failures);
+
+      expect(rootCause.category).toBe('timing');
+      expect(rootCause.subcategory).toContain('async');
+    });
+
+    test('should identify race conditions', () => {
+      const failures = [
+        { error: 'Element not found', timing: 'early' },
+        { error: 'State update after unmount', timing: 'late' },
+      ];
+
+      const rootCause = flakyDetect.identifyRootCause(failures);
+
+      expect(rootCause.category).toBe('race-condition');
+      expect(rootCause.confidence).toBeGreaterThan(0.5);
+    });
+
+    test('should find external dependency issues', () => {
+      const failures = [
+        { error: 'Network request failed', stackTrace: 'at fetch()' },
+        { error: 'API returned 503', stackTrace: 'at axios.get()' },
+      ];
+
+      const rootCause = flakyDetect.identifyRootCause(failures);
+
+      expect(rootCause.category).toBe('external-dependency');
+      expect(rootCause.externalService).toBeDefined();
+    });
+
+    test('should analyze error messages for common patterns', () => {
+      const failures = [
+        { error: 'Cannot read property of undefined' },
+        { error: 'Cannot read property of null' },
+        { error: 'undefined is not an object' },
+      ];
+
+      const rootCause = flakyDetect.identifyRootCause(failures);
+
+      expect(rootCause.category).toBe('undefined-reference');
+      expect(rootCause.suggestions).toContain('Add null checks');
+    });
+
+    test('should detect DOM manipulation timing issues', () => {
+      const failures = [
+        { error: 'Element not found: .modal', framework: 'playwright' },
+        { error: 'Unable to find element: button', framework: 'testing-library' },
+      ];
+
+      const rootCause = flakyDetect.identifyRootCause(failures);
+
+      expect(rootCause.category).toBe('dom-timing');
+      expect(rootCause.suggestions).toContain('Use waitFor patterns');
+    });
+
+    test('should identify test isolation problems', () => {
+      const failures = [
+        { error: 'Database constraint violation', context: 'duplicate key' },
+        { error: 'State leaked from previous test', context: 'cleanup' },
+      ];
+
+      const rootCause = flakyDetect.identifyRootCause(failures);
+
+      expect(rootCause.category).toBe('test-isolation');
+      expect(rootCause.suggestions).toContain('Improve cleanup in afterEach');
+    });
+
+    test('should detect environment-specific issues', () => {
+      const failures = [
+        { error: 'File not found', os: 'windows', path: 'C:\\temp\\test' },
+        { error: 'Permission denied', os: 'linux' },
+      ];
+
+      const rootCause = flakyDetect.identifyRootCause(failures);
+
+      expect(rootCause.category).toBe('environment');
+      expect(rootCause.osSpecific).toBe(true);
+    });
+  });
+
+  describe('Scoring', () => {
+    test('should calculate flakiness score (0-100)', () => {
+      const results = {
+        flakinessRate: 0.08, // 8% failure rate
+        total: 100,
+        passed: 92,
+        failed: 8,
+        pattern: 'intermittent',
+      };
+
+      const score = flakyDetect.calculateFlakinessScore(results);
+
+      expect(score.value).toBeGreaterThanOrEqual(0);
+      expect(score.value).toBeLessThanOrEqual(100);
+      // 8% failure rate with pattern randomness and other factors
+      expect(score.value).toBeGreaterThan(3);
+      expect(score.value).toBeLessThan(12);
+    });
+
+    test('should classify severity (low/medium/high)', () => {
+      const lowFlaky = { flakinessRate: 0.02 }; // 2%
+      const mediumFlaky = { flakinessRate: 0.15 }; // 15%
+      const highFlaky = { flakinessRate: 0.40 }; // 40%
+
+      expect(flakyDetect.calculateFlakinessScore(lowFlaky).severity).toBe('low');
+      expect(flakyDetect.calculateFlakinessScore(mediumFlaky).severity).toBe('medium');
+      expect(flakyDetect.calculateFlakinessScore(highFlaky).severity).toBe('high');
+    });
+
+    test('should trend over time with historical data', () => {
+      const historical = [
+        { date: '2024-10-15', flakinessRate: 0.05 },
+        { date: '2024-10-16', flakinessRate: 0.08 },
+        { date: '2024-10-17', flakinessRate: 0.12 },
+      ];
+
+      const result = flakyDetect.calculateFlakinessScore({ historical });
+
+      expect(result.trend).toBeDefined();
+      expect(result.trend.direction).toBe('increasing');
+      expect(result.trend.slope).toBeGreaterThan(0);
+    });
+
+    test('should factor in timing variance for score', () => {
+      const stableTimings = {
+        durations: [100, 102, 101, 99, 100],
+        flakinessRate: 0.1,
+      };
+
+      const unstableTimings = {
+        durations: [100, 500, 150, 1200, 200],
+        flakinessRate: 0.1,
+      };
+
+      const stableScore = flakyDetect.calculateFlakinessScore(stableTimings);
+      const unstableScore = flakyDetect.calculateFlakinessScore(unstableTimings);
+
+      expect(unstableScore.value).toBeGreaterThan(stableScore.value);
+    });
+
+    test('should consider test retry count in scoring', () => {
+      const noRetries = { flakinessRate: 0.1, retries: 0 };
+      const manyRetries = { flakinessRate: 0.1, retries: 5 };
+
+      const scoreNoRetries = flakyDetect.calculateFlakinessScore(noRetries);
+      const scoreManyRetries = flakyDetect.calculateFlakinessScore(manyRetries);
+
+      expect(scoreManyRetries.value).toBeGreaterThan(scoreNoRetries.value);
+    });
+  });
+
+  describe('Historical Tracking', () => {
+    test('should store test results with metadata', async () => {
+      const testResult = {
+        testId: 'user-login-test',
+        timestamp: Date.now(),
+        status: 'failed',
+        iteration: 5,
+      };
+
+      await flakyDetect.trackHistoricalFailures('user-login-test', testResult);
+
+      expect(fs.writeFileSync).toHaveBeenCalled();
+    });
+
+    test('should track failure history over time', async () => {
+      const history = await flakyDetect.getFailureHistory('checkout-test');
+
+      expect(history).toBeInstanceOf(Array);
+      expect(history.length).toBeGreaterThanOrEqual(0);
+    });
+
+    test('should identify newly flaky tests (recent degradation)', async () => {
+      const recentHistory = [
+        { date: '2024-10-10', status: 'passed' },
+        { date: '2024-10-11', status: 'passed' },
+        { date: '2024-10-15', status: 'failed' },
+        { date: '2024-10-16', status: 'failed' },
+      ];
+
+      const isNewlyFlaky = flakyDetect.isNewlyFlaky(recentHistory);
+
+      expect(isNewlyFlaky).toBe(true);
+    });
+
+    test('should persist flakiness data to JSON file', async () => {
+      const data = {
+        testId: 'api-test',
+        flakinessRate: 0.15,
+        lastRun: Date.now(),
+      };
+
+      await flakyDetect.trackHistoricalFailures('api-test', data);
+
+      const writeCall = fs.writeFileSync.mock.calls[0];
+      expect(writeCall[0]).toContain('.flaky-tests-history.json');
+      expect(JSON.parse(writeCall[1])).toMatchObject(data);
+    });
+
+    test('should load historical data for comparison', async () => {
+      fs.readFileSync.mockReturnValue(JSON.stringify({
+        'user-test': { flakinessRate: 0.1 }
+      }));
+
+      const history = await flakyDetect.loadHistoricalData();
+
+      expect(history).toHaveProperty('user-test');
+      expect(history['user-test'].flakinessRate).toBe(0.1);
+    });
+  });
+
+  describe('Recommendations', () => {
+    test('should suggest quarantine for highly flaky tests', () => {
+      const analysis = {
+        flakinessRate: 0.35,
+        severity: 'high',
+      };
+
+      const recommendations = flakyDetect.generateRecommendations(analysis);
+
+      expect(recommendations.quarantine).toBe(true);
+      expect(recommendations.actions).toContain('Quarantine test');
+    });
+
+    test('should recommend retry strategies based on pattern', () => {
+      const timingIssue = {
+        rootCause: { category: 'timing' },
+        flakinessRate: 0.1,
+      };
+
+      const recommendations = flakyDetect.generateRecommendations(timingIssue);
+
+      expect(recommendations.retryStrategy).toBeDefined();
+      expect(recommendations.retryStrategy).toContain('jest.retryTimes(3)');
+    });
+
+    test('should suggest specific fixes based on root cause', () => {
+      const asyncIssue = {
+        rootCause: { category: 'async', subcategory: 'promise' },
+      };
+
+      const recommendations = flakyDetect.generateRecommendations(asyncIssue);
+
+      expect(recommendations.fixes).toContain('Add await keywords');
+      expect(recommendations.fixes).toContain('Use async/await properly');
+    });
+
+    test('should recommend mocking for external dependencies', () => {
+      const externalDep = {
+        rootCause: { category: 'external-dependency', externalService: 'API' },
+      };
+
+      const recommendations = flakyDetect.generateRecommendations(externalDep);
+
+      expect(recommendations.fixes).toContain('Mock external API calls');
+    });
+
+    test('should suggest wait strategies for DOM timing issues', () => {
+      const domTiming = {
+        rootCause: { category: 'dom-timing' },
+      };
+
+      const recommendations = flakyDetect.generateRecommendations(domTiming);
+
+      expect(recommendations.fixes).toContain('Use waitFor patterns');
+      expect(recommendations.fixes).toContain('Increase timeout values');
+    });
+
+    test('should recommend test isolation improvements', () => {
+      const isolation = {
+        rootCause: { category: 'test-isolation' },
+      };
+
+      const recommendations = flakyDetect.generateRecommendations(isolation);
+
+      expect(recommendations.fixes).toContain('Improve cleanup in afterEach');
+      expect(recommendations.fixes).toContain('Use beforeEach for setup');
+    });
+  });
+
+  describe('Quarantine Management', () => {
+    test('should quarantine flaky tests by adding skip directive', async () => {
+      const flakyTests = [
+        { testId: 'flaky-test-1', file: 'test1.test.js', severity: 'high' },
+      ];
+
+      await flakyDetect.quarantineFlakyTests(flakyTests);
+
+      expect(fs.readFileSync).toHaveBeenCalledWith(
+        expect.stringContaining('test1.test.js'),
+        'utf-8'
+      );
+    });
+
+    test('should maintain quarantine metadata', async () => {
+      const quarantined = {
+        testId: 'test-1',
+        reason: 'High flakiness (35%)',
+        quarantinedAt: Date.now(),
+      };
+
+      await flakyDetect.quarantineFlakyTests([quarantined]);
+
+      const metadataCall = fs.writeFileSync.mock.calls.find(
+        call => call[0].includes('quarantine-metadata.json')
+      );
+      expect(metadataCall).toBeDefined();
+    });
+
+    test('should create issue tracker links for quarantined tests', async () => {
+      const test = {
+        testId: 'failing-test',
+        severity: 'high',
+        rootCause: 'timing',
+      };
+
+      const result = await flakyDetect.quarantineFlakyTests([test]);
+
+      expect(result.issueLinks).toBeDefined();
+      expect(result.issueLinks[0]).toMatch(/github\.com.*issues/);
+    });
+  });
+
+  describe('CI Integration', () => {
+    test('should generate CI reports in JSON format', () => {
+      const results = {
+        totalTests: 100,
+        flakyTests: 5,
+        flakinessRate: 0.05,
+      };
+
+      const report = flakyDetect.generateCIReport(results, 'json');
+
+      expect(() => JSON.parse(report)).not.toThrow();
+      const parsed = JSON.parse(report);
+      expect(parsed.flakyTests).toBe(5);
+    });
+
+    test('should generate CI reports in HTML format', () => {
+      const results = {
+        totalTests: 50,
+        flakyTests: 3,
+      };
+
+      const report = flakyDetect.generateCIReport(results, 'html');
+
+      expect(report).toContain('<html');
+      expect(report).toContain('Flaky Test Report');
+    });
+
+    test('should generate CI reports in markdown format', () => {
+      const results = {
+        totalTests: 75,
+        flakyTests: 4,
+      };
+
+      const report = flakyDetect.generateCIReport(results, 'markdown');
+
+      expect(report).toContain('# Flaky Test Report');
+      expect(report).toContain('Total Tests: 75');
+    });
+
+    test('should fail build when flakiness exceeds threshold', () => {
+      const results = {
+        flakinessRate: 0.15, // 15%
+      };
+
+      const ciResult = flakyDetect.generateCIReport(results, 'json', {
+        failThreshold: 0.10, // 10%
+      });
+
+      const parsed = JSON.parse(ciResult);
+      expect(parsed.shouldFailBuild).toBe(true);
+      expect(parsed.exitCode).toBe(1);
+    });
+
+    test('should include flaky test details in CI output', () => {
+      const results = {
+        flakyTests: [
+          { testId: 'test-1', flakinessRate: 0.2 },
+          { testId: 'test-2', flakinessRate: 0.15 },
+        ],
+      };
+
+      const report = flakyDetect.generateCIReport(results, 'json');
+      const parsed = JSON.parse(report);
+
+      expect(parsed.details).toHaveLength(2);
+      expect(parsed.details[0].testId).toBe('test-1');
+    });
+
+    test('should export metrics for monitoring dashboards', () => {
+      const results = {
+        totalTests: 200,
+        flakyTests: 10,
+        avgFlakinessRate: 0.05,
+      };
+
+      const metrics = flakyDetect.generateCIReport(results, 'prometheus');
+
+      expect(metrics).toContain('flaky_tests_total 10');
+      expect(metrics).toContain('flaky_tests_rate 0.05');
+    });
+  });
+
+  describe('CLI Options', () => {
+    test('should support --iterations option', async () => {
+      const options = { iterations: 50 };
+      const result = await flakyDetect.detectFlaky('test.js', options);
+
+      expect(result.iterations).toBe(50);
+    });
+
+    test('should support --pattern option for test filtering', async () => {
+      const options = { pattern: '**/*.integration.test.js' };
+      const result = await flakyDetect.detectFlaky(null, options);
+
+      expect(result.pattern).toBe('**/*.integration.test.js');
+    });
+
+    test('should support --quarantine flag', async () => {
+      const options = { quarantine: true };
+      const result = await flakyDetect.detectFlaky('flaky.test.js', options);
+
+      expect(result.quarantineApplied).toBe(true);
+    });
+
+    test('should support --ci flag for CI mode', async () => {
+      const options = { ci: true };
+      const result = await flakyDetect.detectFlaky('test.js', options);
+
+      expect(result.ciMode).toBe(true);
+      expect(result.reportFormat).toBeDefined();
+    });
+
+    test('should support --report-format option', async () => {
+      const options = { reportFormat: 'html' };
+      const result = await flakyDetect.detectFlaky('test.js', options);
+
+      expect(result.reportFormat).toBe('html');
+    });
+
+    test('should support --threshold option for build failure', async () => {
+      const options = { threshold: 0.05 }; // 5%
+      const result = await flakyDetect.detectFlaky('test.js', options);
+
+      expect(result.failThreshold).toBe(0.05);
+    });
+  });
+});