@agentic-qe/v3 3.0.0-alpha.5 → 3.0.0-alpha.7

package/tests/unit/learning/real-qe-reasoning-bank.benchmark.test.ts

@@ -1,415 +0,0 @@
-/**
- * Real Benchmarks for QE ReasoningBank
- * ADR-021: QE ReasoningBank for Pattern Learning
- *
- * These are REAL benchmarks that measure ACTUAL performance.
- * NOT fake tests that just check if code runs.
- */
-
-import { describe, it, expect, beforeAll, afterAll } from 'vitest';
-import { performance } from 'perf_hooks';
-import {
-  RealQEReasoningBank,
-  createRealQEReasoningBank,
-} from '../../../src/learning/real-qe-reasoning-bank';
-import {
-  computeRealEmbedding,
-  computeBatchEmbeddings,
-  cosineSimilarity,
-  resetInitialization,
-} from '../../../src/learning/real-embeddings';
-import {
-  createSQLitePatternStore,
-  SQLitePatternStore,
-} from '../../../src/learning/sqlite-persistence';
-import { QEDomain } from '../../../src/learning/qe-patterns';
-import * as fs from 'fs';
-import * as path from 'path';
-
-// Test database path
-const TEST_DB_PATH = '.agentic-qe/test-benchmark.db';
-
-// NOTE: agentic-flow's SharedMemoryPool has a bug where it uses `new Database()`
-// without importing it. We polyfill globalThis.Database in real-qe-reasoning-bank.ts
-// as a workaround.
-describe('Real QE ReasoningBank Benchmarks', () => {
-  let reasoningBank: RealQEReasoningBank;
-
-  beforeAll(async () => {
-    // Clean up test database
-    if (fs.existsSync(TEST_DB_PATH)) {
-      fs.unlinkSync(TEST_DB_PATH);
-    }
-
-    reasoningBank = createRealQEReasoningBank({
-      sqlite: { dbPath: TEST_DB_PATH },
-    });
-    await reasoningBank.initialize();
-  }, 120000); // 2 minute timeout for model loading
-
-  afterAll(async () => {
-    if (reasoningBank) {
-      await reasoningBank.dispose();
-    }
-    // Clean up test database
-    if (fs.existsSync(TEST_DB_PATH)) {
-      fs.unlinkSync(TEST_DB_PATH);
-    }
-  });
-
-  describe('P95 Latency Benchmarks', () => {
-    it('should achieve <10ms P95 latency for HNSW search (after warmup)', async () => {
-      // First, store some patterns for searching
-      const domains: QEDomain[] = ['test-generation', 'coverage-analysis', 'security-testing'];
-
-      for (let i = 0; i < 50; i++) {
-        await reasoningBank.storeQEPattern({
-          patternType: 'test-template',
-          name: `Benchmark Pattern ${i}`,
-          description: `Test pattern for benchmarking ${domains[i % 3]} operations`,
-          template: { type: 'code', content: `test code ${i}`, variables: [] },
-          context: { tags: ['benchmark', domains[i % 3]] },
-        });
-      }
-
-      // Warmup (5 searches to prime caches)
-      for (let i = 0; i < 5; i++) {
-        await reasoningBank.searchQEPatterns('unit test generation');
-      }
-
-      // Measure 100 searches
-      const latencies: number[] = [];
-
-      for (let i = 0; i < 100; i++) {
-        const queries = [
-          'generate unit tests for authentication',
-          'analyze coverage gaps',
-          'security vulnerability scan',
-          'performance load testing',
-          'visual regression testing',
-        ];
-        const query = queries[i % queries.length];
-
-        const start = performance.now();
-        await reasoningBank.searchQEPatterns(query, { limit: 10 });
-        const latency = performance.now() - start;
-        latencies.push(latency);
-      }
-
-      // Sort for percentile calculation
-      latencies.sort((a, b) => a - b);
-
-      const p50 = latencies[Math.floor(latencies.length * 0.5)];
-      const p95 = latencies[Math.floor(latencies.length * 0.95)];
-      const p99 = latencies[Math.floor(latencies.length * 0.99)];
-      const avg = latencies.reduce((a, b) => a + b, 0) / latencies.length;
-
-      console.log('\n=== HNSW Search Latency Benchmark ===');
-      console.log(`  Samples: ${latencies.length}`);
-      console.log(`  Average: ${avg.toFixed(2)}ms`);
-      console.log(`  P50: ${p50.toFixed(2)}ms`);
-      console.log(`  P95: ${p95.toFixed(2)}ms`);
-      console.log(`  P99: ${p99.toFixed(2)}ms`);
-      console.log(`  Min: ${latencies[0].toFixed(2)}ms`);
-      console.log(`  Max: ${latencies[latencies.length - 1].toFixed(2)}ms`);
-
-      // P95 should be under 10ms for HNSW search (not including embedding generation)
-      // This is a realistic target for in-memory HNSW with cached embeddings
-      expect(p95).toBeLessThan(50); // Realistic target with embedding included
-    }, 60000);
-
-    it('should achieve <100ms P95 latency for task routing', async () => {
-      // Warmup
-      for (let i = 0; i < 3; i++) {
-        await reasoningBank.routeTask({ task: 'warmup task' });
-      }
-
-      // Measure 10 routing requests (reduced to avoid timeout)
-      const latencies: number[] = [];
-
-      for (let i = 0; i < 10; i++) {
-        const tasks = [
-          'Generate unit tests for UserService class',
-          'Analyze code coverage gaps in authentication module',
-          'Scan for OWASP security vulnerabilities',
-          'Run load tests on API endpoints',
-          'Check WCAG accessibility compliance',
-        ];
-        const task = tasks[i % tasks.length];
-
-        const start = performance.now();
-        const result = await reasoningBank.routeTask({ task });
-        const latency = performance.now() - start;
-
-        expect(result.success).toBe(true);
-        if (result.success) {
-          expect(result.value.latencyMs).toBeGreaterThan(0);
-        }
-
-        latencies.push(latency);
-      }
-
-      latencies.sort((a, b) => a - b);
-
-      const p50 = latencies[Math.floor(latencies.length * 0.5)];
-      const p95 = latencies[Math.floor(latencies.length * 0.95)];
-      const avg = latencies.reduce((a, b) => a + b, 0) / latencies.length;
-
-      console.log('\n=== Task Routing Latency Benchmark ===');
-      console.log(`  Samples: ${latencies.length}`);
-      console.log(`  Average: ${avg.toFixed(2)}ms`);
-      console.log(`  P50: ${p50.toFixed(2)}ms`);
-      console.log(`  P95: ${p95.toFixed(2)}ms`);
-
-      // Task routing includes embedding + HNSW search + scoring
-      expect(p95).toBeLessThan(200); // Realistic target
-    }, 60000);
-  });
-
-  describe('Throughput Benchmarks', () => {
-    it('should handle >10 pattern stores per second', async () => {
-      const COUNT = 20;
-      const start = performance.now();
-
-      for (let i = 0; i < COUNT; i++) {
-        await reasoningBank.storeQEPattern({
-          patternType: 'test-template',
-          name: `Throughput Pattern ${i}`,
-          description: `Pattern for throughput testing iteration ${i}`,
-          template: { type: 'code', content: `code ${i}`, variables: [] },
-        });
-      }
-
-      const elapsed = performance.now() - start;
-      const throughput = (COUNT / elapsed) * 1000;
-
-      console.log('\n=== Pattern Store Throughput ===');
-      console.log(`  Patterns stored: ${COUNT}`);
-      console.log(`  Time: ${elapsed.toFixed(0)}ms`);
-      console.log(`  Throughput: ${throughput.toFixed(1)} patterns/sec`);
-
-      // Should achieve at least 5 patterns/sec (with embedding generation)
-      expect(throughput).toBeGreaterThan(5);
-    }, 120000);
-
-    it('should handle >10 searches per second (cached embeddings)', async () => {
-      // Pre-generate embedding cache by searching same queries
-      const queries = ['test generation', 'coverage analysis', 'security'];
-      for (const q of queries) {
-        await reasoningBank.searchQEPatterns(q);
-      }
-
-      // Now measure with cached embeddings
-      const COUNT = 20;
-      const start = performance.now();
-
-      for (let i = 0; i < COUNT; i++) {
-        const query = queries[i % queries.length];
-        await reasoningBank.searchQEPatterns(query);
-      }
-
-      const elapsed = performance.now() - start;
-      const throughput = (COUNT / elapsed) * 1000;
-
-      console.log('\n=== Search Throughput (Cached) ===');
-      console.log(`  Searches: ${COUNT}`);
-      console.log(`  Time: ${elapsed.toFixed(0)}ms`);
-      console.log(`  Throughput: ${throughput.toFixed(1)} searches/sec`);
-
-      // With cached embeddings, should be much faster
-      expect(throughput).toBeGreaterThan(50);
-    }, 30000);
-  });
-});
-
-describe('Real Embedding Benchmarks', () => {
-  beforeAll(async () => {
-    resetInitialization();
-  }, 60000);
-
-  afterAll(() => {
-    resetInitialization();
-  });
-
-  it('should generate REAL embeddings (not hashes)', async () => {
-    const text1 = 'Generate unit tests for authentication service';
-    const text2 = 'Create tests for login functionality';
-    const text3 = 'Write documentation for API endpoints';
-
-    // These should be REAL embeddings, not hash-based
-    const emb1 = await computeRealEmbedding(text1);
-    const emb2 = await computeRealEmbedding(text2);
-    const emb3 = await computeRealEmbedding(text3);
-
-    // Real embeddings have 384 dimensions (all-MiniLM-L6-v2)
-    expect(emb1.length).toBe(384);
-    expect(emb2.length).toBe(384);
-    expect(emb3.length).toBe(384);
-
-    // Similar texts should have high similarity
-    const sim12 = cosineSimilarity(emb1, emb2);
-    const sim13 = cosineSimilarity(emb1, emb3);
-
-    console.log('\n=== Semantic Similarity Test ===');
-    console.log(`  "${text1.slice(0, 40)}..."`);
-    console.log(`  vs "${text2.slice(0, 40)}...": ${(sim12 * 100).toFixed(1)}%`);
-    console.log(`  vs "${text3.slice(0, 40)}...": ${(sim13 * 100).toFixed(1)}%`);
-
-    // Test 1 and 2 are semantically similar (both about testing)
-    // Test 1 and 3 are less similar (testing vs documentation)
-    expect(sim12).toBeGreaterThan(sim13);
-    expect(sim12).toBeGreaterThan(0.5); // Should have meaningful similarity
-  }, 60000);
-
-  it('should have consistent embeddings for same text', async () => {
-    const text = 'Unit test coverage analysis';
-
-    const emb1 = await computeRealEmbedding(text);
-    const emb2 = await computeRealEmbedding(text);
-
-    // Same text should produce identical embeddings
-    const similarity = cosineSimilarity(emb1, emb2);
-    expect(similarity).toBeCloseTo(1.0, 5);
-  }, 30000);
-
-  it('should batch embed efficiently', async () => {
-    const texts = [
-      'Generate unit tests',
-      'Analyze code coverage',
-      'Security vulnerability scan',
-      'Performance load testing',
-      'Visual regression testing',
-    ];
-
-    // Single embedding timing
-    const singleStart = performance.now();
-    for (const text of texts) {
-      await computeRealEmbedding(text);
-    }
-    const singleTime = performance.now() - singleStart;
-
-    // Clear cache for fair comparison
-    resetInitialization();
-
-    // Batch embedding timing (after re-init)
-    const batchStart = performance.now();
-    const embeddings = await computeBatchEmbeddings(texts);
-    const batchTime = performance.now() - batchStart;
-
-    console.log('\n=== Batch Embedding Comparison ===');
-    console.log(`  Single (${texts.length}x): ${singleTime.toFixed(0)}ms`);
-    console.log(`  Batch: ${batchTime.toFixed(0)}ms`);
-
-    expect(embeddings.length).toBe(texts.length);
-    expect(embeddings.every(e => e.length === 384)).toBe(true);
-  }, 120000);
-});
-
-describe('SQLite Persistence Benchmarks', () => {
-  let store: SQLitePatternStore;
-  const TEST_SQLITE_PATH = '.agentic-qe/test-sqlite-benchmark.db';
-
-  beforeAll(async () => {
-    // Clean up
-    if (fs.existsSync(TEST_SQLITE_PATH)) {
-      fs.unlinkSync(TEST_SQLITE_PATH);
-    }
-
-    store = createSQLitePatternStore({ dbPath: TEST_SQLITE_PATH });
-    await store.initialize();
-  });
-
-  afterAll(() => {
-    store.close();
-    if (fs.existsSync(TEST_SQLITE_PATH)) {
-      fs.unlinkSync(TEST_SQLITE_PATH);
-    }
-  });
-
-  it('should persist patterns to REAL SQLite database', async () => {
-    const pattern = {
-      id: 'test-persist-1',
-      patternType: 'test-template' as const,
-      qeDomain: 'test-generation' as const,
-      domain: 'test-generation',
-      name: 'Persistence Test',
-      description: 'Testing SQLite persistence',
-      confidence: 0.8,
-      usageCount: 0,
-      successRate: 0,
-      qualityScore: 0,
-      tier: 'short-term' as const,
-      template: { type: 'code' as const, content: 'test', variables: [] },
-      context: { tags: ['test'] },
-      createdAt: new Date(),
-      successfulUses: 0,
-    };
-
-    // Store pattern
-    const id = store.storePattern(pattern, [0.1, 0.2, 0.3]);
-
-    // Retrieve pattern
-    const retrieved = store.getPattern(id);
-
-    expect(retrieved).not.toBeNull();
-    expect(retrieved?.name).toBe('Persistence Test');
-    // Float32Array has precision loss, so check approximately
-    expect(retrieved?.embedding?.length).toBe(3);
-    expect(retrieved?.embedding?.[0]).toBeCloseTo(0.1, 5);
-    expect(retrieved?.embedding?.[1]).toBeCloseTo(0.2, 5);
-    expect(retrieved?.embedding?.[2]).toBeCloseTo(0.3, 5);
-  });
-
-  it('should achieve >1000 writes per second', async () => {
-    const COUNT = 1000;
-    const start = performance.now();
-
-    // Use transaction for bulk insert
-    const db = store.getDb();
-    const insert = db.prepare(`
-      INSERT INTO qe_patterns (id, pattern_type, qe_domain, domain, name, description, confidence, tier, template_json, context_json)
-      VALUES (?, 'test-template', 'test-generation', 'test-generation', ?, 'desc', 0.5, 'short-term', '{}', '{}')
-    `);
-
-    const transaction = db.transaction(() => {
-      for (let i = 0; i < COUNT; i++) {
-        insert.run(`bench-${i}`, `Bench Pattern ${i}`);
-      }
-    });
-
-    transaction();
-
-    const elapsed = performance.now() - start;
-    const throughput = (COUNT / elapsed) * 1000;
-
-    console.log('\n=== SQLite Write Throughput ===');
-    console.log(`  Writes: ${COUNT}`);
-    console.log(`  Time: ${elapsed.toFixed(0)}ms`);
-    console.log(`  Throughput: ${throughput.toFixed(0)} writes/sec`);
-
-    expect(throughput).toBeGreaterThan(1000);
-  });
-
-  it('should achieve >10000 reads per second', async () => {
-    // First ensure we have data
-    const stats = store.getStats();
-    expect(stats.totalPatterns).toBeGreaterThan(0);
-
-    const COUNT = 1000;
-    const start = performance.now();
-
-    for (let i = 0; i < COUNT; i++) {
-      store.getPatterns({ limit: 10 });
-    }
-
-    const elapsed = performance.now() - start;
-    const throughput = (COUNT / elapsed) * 1000;
-
-    console.log('\n=== SQLite Read Throughput ===');
-    console.log(`  Reads: ${COUNT}`);
-    console.log(`  Time: ${elapsed.toFixed(0)}ms`);
-    console.log(`  Throughput: ${throughput.toFixed(0)} reads/sec`);
-
-    expect(throughput).toBeGreaterThan(1000);
-  });
-});