@agentic-qe/v3 3.0.0-alpha.6 → 3.0.0-alpha.8

package/tests/integration/feedback/feedback-loop-integration.test.ts

@@ -1,560 +0,0 @@
-/**
- * Integration Tests for Quality Feedback Loop
- * ADR-023: Quality Feedback Loop System
- *
- * These tests verify REAL integration with:
- * - SQLite database persistence
- * - ReasoningBank pattern storage and updates
- * - Routing feedback collection
- * - Cross-component feedback flow
- */
-
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
-import { RealQEReasoningBank } from '../../../src/learning/real-qe-reasoning-bank.js';
-import {
-  QualityFeedbackLoop,
-  createQualityFeedbackLoop,
-} from '../../../src/feedback/feedback-loop.js';
-import type { TestOutcome, CoverageSession } from '../../../src/feedback/types.js';
-import { existsSync, unlinkSync } from 'fs';
-
-// ============================================================================
-// Test Helpers
-// ============================================================================
-
-function createTestOutcome(overrides: Partial<TestOutcome> = {}): TestOutcome {
-  return {
-    id: `outcome-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
-    testId: `test-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
-    testName: 'Integration test case',
-    generatedBy: 'integration-test-agent',
-    framework: 'vitest',
-    language: 'typescript',
-    domain: 'test-generation',
-    passed: true,
-    flaky: false,
-    executionTimeMs: 100,
-    coverage: { lines: 80, branches: 70, functions: 85 },
-    maintainabilityScore: 0.8,
-    timestamp: new Date(),
-    ...overrides,
-  };
-}
-
-function createCoverageSession(overrides: Partial<CoverageSession> = {}): CoverageSession {
-  return {
-    id: `session-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
-    agentId: 'coverage-agent',
-    targetPath: 'src/services/user.ts',
-    technique: 'gap-analysis',
-    beforeCoverage: { lines: 50, branches: 40, functions: 45 },
-    afterCoverage: { lines: 75, branches: 65, functions: 70 },
-    testsGenerated: 5,
-    testsPassed: 4,
-    gapsTargeted: [
-      { id: 'gap-1', type: 'uncovered-branch', filePath: 'src/services/user.ts', startLine: 42, riskScore: 0.8, addressed: true },
-      { id: 'gap-2', type: 'uncovered-function', filePath: 'src/services/user.ts', startLine: 100, riskScore: 0.5, addressed: true },
-    ],
-    durationMs: 30000,
-    startedAt: new Date(Date.now() - 30000),
-    completedAt: new Date(),
-    ...overrides,
-  };
-}
-
-// ============================================================================
-// Integration Tests
-// ============================================================================
-
-describe('FeedbackLoop Integration with ReasoningBank', () => {
-  let reasoningBank: RealQEReasoningBank;
-  let feedbackLoop: QualityFeedbackLoop;
-  const testDbPath = '.agentic-qe/test-feedback-integration.db';
-
-  beforeEach(async () => {
-    // Clean up any existing test database
-    if (existsSync(testDbPath)) {
-      try { unlinkSync(testDbPath); } catch { /* ignore */ }
-    }
-    if (existsSync(`${testDbPath}-shm`)) {
-      try { unlinkSync(`${testDbPath}-shm`); } catch { /* ignore */ }
-    }
-    if (existsSync(`${testDbPath}-wal`)) {
-      try { unlinkSync(`${testDbPath}-wal`); } catch { /* ignore */ }
-    }
-
-    // Create REAL ReasoningBank with test database
-    reasoningBank = new RealQEReasoningBank({
-      sqlite: {
-        dbPath: testDbPath,
-        walMode: false, // Simpler for tests
-      },
-      embeddings: {
-        quantized: true,
-        enableCache: false,
-      },
-      enableLearning: true,
-      enableRouting: true,
-      enableGuidance: false,
-      hnsw: { M: 16, efConstruction: 200, efSearch: 50 },
-      routingWeights: { similarity: 0.3, performance: 0.4, capabilities: 0.3 },
-    });
-
-    // Initialize the ReasoningBank
-    await reasoningBank.initialize();
-
-    // Create feedback loop and CONNECT to ReasoningBank
-    feedbackLoop = createQualityFeedbackLoop();
-    feedbackLoop.connectReasoningBank(reasoningBank);
-  });
-
-  afterEach(async () => {
-    // Cleanup
-    try {
-      await reasoningBank.dispose();
-    } catch { /* ignore */ }
-
-    // Remove test database
-    if (existsSync(testDbPath)) {
-      try { unlinkSync(testDbPath); } catch { /* ignore */ }
-    }
-    if (existsSync(`${testDbPath}-shm`)) {
-      try { unlinkSync(`${testDbPath}-shm`); } catch { /* ignore */ }
-    }
-    if (existsSync(`${testDbPath}-wal`)) {
-      try { unlinkSync(`${testDbPath}-wal`); } catch { /* ignore */ }
-    }
-  });
-
-  describe('Connected State', () => {
-    it('should report ReasoningBank as connected', () => {
-      const stats = feedbackLoop.getStats();
-      expect(stats.integrationStatus.reasoningBankConnected).toBe(true);
-    });
-
-    it('should have all components connected', () => {
-      expect(feedbackLoop.outcomeTracker).toBeDefined();
-      expect(feedbackLoop.coverageLearner).toBeDefined();
-      expect(feedbackLoop.qualityCalculator).toBeDefined();
-      expect(feedbackLoop.promotionManager).toBeDefined();
-    });
-  });
-
-  describe('Pattern Storage and Retrieval', () => {
-    it('should store patterns in ReasoningBank from coverage sessions', async () => {
-      // Record a successful coverage session that should learn a strategy
-      const session = createCoverageSession({
-        beforeCoverage: { lines: 30, branches: 20, functions: 25 },
-        afterCoverage: { lines: 70, branches: 60, functions: 65 }, // Big improvement
-      });
-
-      const result = await feedbackLoop.recordCoverageSession(session);
-
-      expect(result.strategyLearned).toBe(true);
-      expect(result.strategyId).toBeDefined();
-
-      // Verify strategy was stored - search for it in ReasoningBank
-      const searchResult = await reasoningBank.searchQEPatterns('Coverage Strategy', { limit: 10 });
-      expect(searchResult.success).toBe(true);
-      if (searchResult.success) {
-        const patterns = searchResult.value;
-        expect(patterns.length).toBeGreaterThan(0);
-        expect(patterns.some(p => p.pattern.patternType === 'coverage-strategy')).toBe(true);
-      }
-    });
-
-    it('should update pattern metrics from test outcomes', async () => {
-      // First, store a pattern manually
-      const storeResult = await reasoningBank.storeQEPattern({
-        patternType: 'test-pattern',
-        name: 'Test Pattern for Integration',
-        description: 'A pattern to test feedback integration',
-        template: {
-          type: 'procedure',
-          content: 'Test procedure content',
-          variables: ['testVar'],
-        },
-        context: {
-          tags: ['integration-test'],
-          confidence: 0.5,
-          metadata: { source: 'integration-test' },
-        },
-      });
-      expect(storeResult.success).toBe(true);
-      if (!storeResult.success) return;
-      const patternId = storeResult.value.id;
-
-      // Record multiple test outcomes for this pattern
-      for (let i = 0; i < 5; i++) {
-        await feedbackLoop.recordTestOutcome(createTestOutcome({
-          id: `outcome-${i}`,
-          testId: `test-${i}`,
-          patternId,
-          passed: true,
-          maintainabilityScore: 0.85,
-        }));
-      }
-
-      // Retrieve pattern and verify metrics were updated
-      const searchResult = await reasoningBank.searchQEPatterns('Test Pattern for Integration', { limit: 1 });
-      expect(searchResult.success).toBe(true);
-      if (searchResult.success) {
-        const patterns = searchResult.value;
-        expect(patterns.length).toBe(1);
-
-        const pattern = patterns[0].pattern;
-        expect(pattern.usageCount).toBeGreaterThanOrEqual(5);
-        expect(pattern.successfulUses).toBeGreaterThanOrEqual(5);
-        expect(pattern.successRate).toBeCloseTo(1.0, 1);
-      }
-    });
-  });
-
-  describe('Quality Score Persistence', () => {
-    it('should calculate and track quality scores through the feedback loop', async () => {
-      // Record multiple outcomes with varying quality
-      const outcomes = [
-        createTestOutcome({ id: 'q1', testId: 't1', passed: true, maintainabilityScore: 0.9, coverage: { lines: 90, branches: 85, functions: 92 } }),
-        createTestOutcome({ id: 'q2', testId: 't2', passed: true, maintainabilityScore: 0.8, coverage: { lines: 80, branches: 75, functions: 82 } }),
-        createTestOutcome({ id: 'q3', testId: 't3', passed: false, maintainabilityScore: 0.5, coverage: { lines: 50, branches: 40, functions: 45 } }),
-        createTestOutcome({ id: 'q4', testId: 't4', passed: true, maintainabilityScore: 0.85, coverage: { lines: 85, branches: 80, functions: 88 } }),
-      ];
-
-      const scores = [];
-      for (const outcome of outcomes) {
-        const result = await feedbackLoop.recordTestOutcome(outcome);
-        scores.push(result.qualityScore);
-      }
-
-      // Verify scores were calculated
-      expect(scores.length).toBe(4);
-      expect(scores.every(s => s.overall > 0)).toBe(true);
-
-      // Good outcome should have higher score than bad outcome
-      expect(scores[0].overall).toBeGreaterThan(scores[2].overall);
-
-      // Check stats reflect the tracked outcomes
-      const stats = feedbackLoop.getStats();
-      expect(stats.testOutcomes.total).toBe(4);
-      expect(stats.testOutcomes.passRate).toBe(0.75);
-    });
-  });
-
-  describe('Coverage Learning with Persistence', () => {
-    it('should learn and recommend strategies', async () => {
-      // Record multiple successful coverage sessions
-      await feedbackLoop.recordCoverageSession(createCoverageSession({
-        id: 'session-1',
-        targetPath: 'src/services/auth.ts',
-        technique: 'gap-analysis',
-        beforeCoverage: { lines: 40, branches: 30, functions: 35 },
-        afterCoverage: { lines: 80, branches: 70, functions: 75 },
-      }));
-
-      await feedbackLoop.recordCoverageSession(createCoverageSession({
-        id: 'session-2',
-        targetPath: 'src/services/user.ts',
-        technique: 'gap-analysis',
-        beforeCoverage: { lines: 45, branches: 35, functions: 40 },
-        afterCoverage: { lines: 85, branches: 75, functions: 80 },
-      }));
-
-      // Get recommendation for a similar file
-      const strategy = feedbackLoop.getRecommendedCoverageStrategy('src/services/order.ts');
-
-      expect(strategy).not.toBeNull();
-      expect(strategy!.technique).toBe('gap-analysis');
-      expect(strategy!.successCount).toBeGreaterThanOrEqual(2);
-      expect(strategy!.avgImprovement).toBeGreaterThan(20);
-    });
-
-    it('should store learned strategies as ReasoningBank patterns', async () => {
-      // Record a session that learns a strategy
-      await feedbackLoop.recordCoverageSession(createCoverageSession({
-        technique: 'branch-coverage',
-        beforeCoverage: { lines: 30, branches: 20, functions: 25 },
-        afterCoverage: { lines: 75, branches: 70, functions: 72 },
-      }));
-
-      // Search for the stored strategy in ReasoningBank
-      const searchResult = await reasoningBank.searchQEPatterns('branch-coverage', { limit: 10 });
-      expect(searchResult.success).toBe(true);
-      if (searchResult.success) {
-        const patterns = searchResult.value;
-
-        // Should find at least one pattern with the technique
-        const strategyPattern = patterns.find(p =>
-          p.pattern.patternType === 'coverage-strategy' &&
-          p.pattern.name.includes('branch-coverage')
-        );
-
-        expect(strategyPattern).toBeDefined();
-      }
-    });
-  });
-
-  describe('Full Feedback Loop Cycle', () => {
-    it('should complete end-to-end feedback cycle with persistence', async () => {
-      // 1. Store initial pattern
-      const storeResult = await reasoningBank.storeQEPattern({
-        patternType: 'test-generation-strategy',
-        name: 'E2E Test Pattern',
-        description: 'Pattern for end-to-end feedback test',
-        template: {
-          type: 'procedure',
-          content: 'E2E test procedure',
-          variables: [],
-        },
-        context: {
-          tags: ['e2e-test'],
-          confidence: 0.5,
-          metadata: {},
-        },
-      });
-      expect(storeResult.success).toBe(true);
-      if (!storeResult.success) return;
-      const patternId = storeResult.value.id;
-
-      // 2. Record successful outcomes
-      for (let i = 0; i < 10; i++) {
-        await feedbackLoop.recordTestOutcome(createTestOutcome({
-          id: `e2e-outcome-${i}`,
-          testId: `e2e-test-${i}`,
-          patternId,
-          passed: true,
-          maintainabilityScore: 0.85 + (i * 0.01),
-        }));
-      }
-
-      // 3. Verify pattern metrics were updated
-      const searchResult = await reasoningBank.searchQEPatterns('E2E Test Pattern', { limit: 1 });
-      expect(searchResult.success).toBe(true);
-      if (searchResult.success) {
-        const patterns = searchResult.value;
-        expect(patterns.length).toBe(1);
-
-        const updatedPattern = patterns[0].pattern;
-        expect(updatedPattern.usageCount).toBeGreaterThanOrEqual(10);
-        expect(updatedPattern.successRate).toBeCloseTo(1.0, 1);
-        expect(updatedPattern.confidence).toBeGreaterThan(0.5); // Should have increased
-      }
-
-      // 4. Verify feedback loop stats
-      const stats = feedbackLoop.getStats();
-      expect(stats.testOutcomes.total).toBeGreaterThanOrEqual(10);
-      expect(stats.integrationStatus.reasoningBankConnected).toBe(true);
-    });
-  });
-
-  describe('Pattern Promotion with Real ReasoningBank', () => {
-    it('should return patternUpdate when recording outcomes for a pattern', async () => {
-      // Store a pattern that we'll track outcomes for
-      const storeResult = await reasoningBank.storeQEPattern({
-        patternType: 'test-pattern',
-        name: 'Promotion Test Pattern',
-        description: 'Pattern to test promotion detection',
-        template: {
-          type: 'procedure',
-          content: 'Test procedure',
-          variables: [],
-        },
-        context: {
-          tags: ['promotion-test'],
-          confidence: 0.5,
-          metadata: {},
-        },
-      });
-      expect(storeResult.success).toBe(true);
-      if (!storeResult.success) return;
-      const patternId = storeResult.value.id;
-
-      // Record outcomes and capture the patternUpdate results
-      const updates: Array<{ action: string; tier?: string }> = [];
-      for (let i = 0; i < 15; i++) {
-        const result = await feedbackLoop.recordTestOutcome(createTestOutcome({
-          id: `promo-outcome-${i}`,
-          testId: `promo-test-${i}`,
-          patternId,
-          passed: true,
-          maintainabilityScore: 0.9,
-        }));
-
-        if (result.patternUpdate) {
-          updates.push(result.patternUpdate);
-        }
-      }
-
-      // Verify that patternUpdate was returned (pattern was found and processed)
-      // The action might be 'unchanged' if thresholds aren't met, but we should get updates
-      expect(updates.length).toBeGreaterThan(0);
-
-      // Verify all updates have valid actions
-      for (const update of updates) {
-        expect(['promoted', 'demoted', 'unchanged']).toContain(update.action);
-      }
-    });
-
-    it('should trigger promotion after sufficient successful outcomes', async () => {
-      // Store a pattern
-      const storeResult = await reasoningBank.storeQEPattern({
-        patternType: 'test-pattern',
-        name: 'High Success Pattern',
-        description: 'Pattern with high success rate for promotion',
-        template: {
-          type: 'procedure',
-          content: 'High success procedure',
-          variables: [],
-        },
-        context: {
-          tags: ['high-success'],
-          confidence: 0.5,
-          metadata: {},
-        },
-      });
-      expect(storeResult.success).toBe(true);
-      if (!storeResult.success) return;
-      const patternId = storeResult.value.id;
-
-      // Record many successful outcomes to meet promotion criteria
-      // Default promotion requires: successCount >= 10, successRate >= 0.8, qualityScore >= 0.7
-      for (let i = 0; i < 20; i++) {
-        await feedbackLoop.recordTestOutcome(createTestOutcome({
-          id: `success-outcome-${i}`,
-          testId: `success-test-${i}`,
-          patternId,
-          passed: true,
-          maintainabilityScore: 0.85,
-        }));
-      }
-
-      // Check promotion stats
-      const stats = feedbackLoop.getStats();
-
-      // Verify pattern was tracked
-      expect(stats.patterns.tracked).toBeGreaterThan(0);
-
-      // Verify promotions were considered (may or may not have promoted based on thresholds)
-      // The key is that the code path executed - no runtime errors
-      expect(stats.testOutcomes.total).toBeGreaterThanOrEqual(20);
-    });
-  });
-
-  describe('Export and Import with Persistence', () => {
-    it('should export data that can be reimported', async () => {
-      // Record some data
-      await feedbackLoop.recordTestOutcome(createTestOutcome({ id: 'export-1', testId: 'et1' }));
-      await feedbackLoop.recordCoverageSession(createCoverageSession({
-        beforeCoverage: { lines: 30, branches: 20, functions: 25 },
-        afterCoverage: { lines: 70, branches: 60, functions: 65 },
-      }));
-
-      // Export
-      const exported = feedbackLoop.exportData();
-      expect(exported.outcomes.length).toBe(1);
-      expect(exported.sessions.length).toBe(1);
-      expect(exported.strategies.length).toBeGreaterThanOrEqual(1);
-
-      // Create new feedback loop and import
-      const newLoop = createQualityFeedbackLoop();
-      newLoop.connectReasoningBank(reasoningBank);
-      newLoop.importData(exported);
-
-      // Verify imported data
-      const newStats = newLoop.getStats();
-      expect(newStats.testOutcomes.total).toBe(1);
-      expect(newStats.coverage.totalSessions).toBe(1);
-    });
-  });
-});
-
-describe('FeedbackLoop with RoutingFeedback Integration', () => {
-  let reasoningBank: RealQEReasoningBank;
-  let feedbackLoop: QualityFeedbackLoop;
-  const testDbPath = '.agentic-qe/test-routing-feedback.db';
-
-  beforeEach(async () => {
-    // Clean up
-    for (const suffix of ['', '-shm', '-wal']) {
-      if (existsSync(`${testDbPath}${suffix}`)) {
-        try { unlinkSync(`${testDbPath}${suffix}`); } catch { /* ignore */ }
-      }
-    }
-
-    reasoningBank = new RealQEReasoningBank({
-      sqlite: { dbPath: testDbPath, walMode: false },
-      embeddings: { quantized: true, enableCache: false },
-      enableLearning: true,
-      enableRouting: true,
-      enableGuidance: false,
-      hnsw: { M: 16, efConstruction: 200, efSearch: 50 },
-      routingWeights: { similarity: 0.3, performance: 0.4, capabilities: 0.3 },
-    });
-
-    await reasoningBank.initialize();
-    feedbackLoop = createQualityFeedbackLoop();
-    feedbackLoop.connectReasoningBank(reasoningBank);
-  });
-
-  afterEach(async () => {
-    try { await reasoningBank.dispose(); } catch { /* ignore */ }
-    for (const suffix of ['', '-shm', '-wal']) {
-      if (existsSync(`${testDbPath}${suffix}`)) {
-        try { unlinkSync(`${testDbPath}${suffix}`); } catch { /* ignore */ }
-      }
-    }
-  });
-
-  it('should track routing feedback when connected', () => {
-    const stats = feedbackLoop.getStats();
-
-    // Routing feedback should be trackable through the connected loop
-    expect(stats.integrationStatus.reasoningBankConnected).toBe(true);
-    expect(feedbackLoop.routingFeedback).toBeDefined();
-  });
-
-  it('should record routing outcomes through feedback loop', async () => {
-    // Record a routing outcome through the feedback loop
-    const routingOutcome = {
-      taskId: 'task-123',
-      taskDescription: 'Generate unit tests for UserService',
-      recommendedAgent: 'qe-test-generator',
-      usedAgent: 'qe-test-generator',
-      followedRecommendation: true,
-      success: true,
-      qualityScore: 0.85,
-      durationMs: 5000,
-      timestamp: new Date(),
-    };
-
-    await feedbackLoop.recordRoutingOutcome(routingOutcome);
-
-    const stats = feedbackLoop.getStats();
-    expect(stats.routing.totalOutcomes).toBeGreaterThanOrEqual(1);
-    expect(stats.routing.recommendationFollowRate).toBeGreaterThanOrEqual(0);
-  });
-
-  it('should analyze routing effectiveness', async () => {
-    // Record multiple routing outcomes
-    const outcomes = [
-      { taskId: 't1', recommendedAgent: 'agent-a', usedAgent: 'agent-a', followedRecommendation: true, success: true, qualityScore: 0.9, durationMs: 1000 },
-      { taskId: 't2', recommendedAgent: 'agent-a', usedAgent: 'agent-b', followedRecommendation: false, success: true, qualityScore: 0.95, durationMs: 800 },
-      { taskId: 't3', recommendedAgent: 'agent-b', usedAgent: 'agent-b', followedRecommendation: true, success: false, qualityScore: 0.3, durationMs: 2000 },
-      { taskId: 't4', recommendedAgent: 'agent-a', usedAgent: 'agent-a', followedRecommendation: true, success: true, qualityScore: 0.85, durationMs: 1200 },
-    ];
-
-    for (const outcome of outcomes) {
-      await feedbackLoop.recordRoutingOutcome({
-        ...outcome,
-        taskDescription: `Task ${outcome.taskId}`,
-        timestamp: new Date(),
-      });
-    }
-
-    const analysis = feedbackLoop.getRoutingAnalysis();
-
-    expect(analysis.totalOutcomes).toBe(4);
-    expect(analysis.recommendationFollowRate).toBe(0.75); // 3/4
-    expect(analysis.successRateWhenFollowed).toBeGreaterThan(0);
-    expect(analysis.successRateWhenOverridden).toBeGreaterThan(0);
-  });
-});