adaptive-memory-multi-model-router 2.14.45 → 2.14.47

package/test-council/3-performance-tests.test.ts

@@ -1,669 +0,0 @@
-/**
- * Performance Tests - Latency, Throughput, Cost Accuracy
- * 
- * Comprehensive benchmarks and performance regression tests.
- * These tests verify the system meets performance requirements.
- * 
- * @generated by performance-agent
- */
-
-import { describe, it, expect, beforeEach } from 'vitest';
-
-// Import modules
-const {
-  routeQuery,
-  routeBatch,
-  countTokens,
-  estimateTokens,
-  MemoryTree,
-  CostTracker,
-  createA3MRouter,
-} = require('../dist/index.js');
-
-// Try to load tokenUtils
-let tokenUtils: any = null;
-try {
-  tokenUtils = require('../dist/utils/tokenUtils.js');
-} catch (e) {
-  // Fallback to inline implementation
-}
-
-// ============================================================
-// PERFORMANCE TEST UTILITIES
-// ============================================================
-
-interface BenchmarkResult {
-  name: string;
-  iterations: number;
-  totalMs: number;
-  avgMs: number;
-  minMs: number;
-  maxMs: number;
-  p50Ms: number;
-  p95Ms: number;
-  p99Ms: number;
-  opsPerSecond: number;
-}
-
-function runBenchmark(name: string, fn: () => void, iterations: number = 1000): BenchmarkResult {
-  const times: number[] = [];
-  
-  // Warmup
-  for (let i = 0; i < 10; i++) {
-    fn();
-  }
-  
-  // Actual benchmark
-  for (let i = 0; i < iterations; i++) {
-    const start = performance.now();
-    fn();
-    const end = performance.now();
-    times.push(end - start);
-  }
-  
-  times.sort((a, b) => a - b);
-  
-  const totalMs = times.reduce((a, b) => a + b, 0);
-  
-  return {
-    name,
-    iterations,
-    totalMs,
-    avgMs: totalMs / iterations,
-    minMs: times[0],
-    maxMs: times[times.length - 1],
-    p50Ms: times[Math.floor(iterations * 0.50)],
-    p95Ms: times[Math.floor(iterations * 0.95)],
-    p99Ms: times[Math.floor(iterations * 0.99)],
-    opsPerSecond: (iterations / totalMs) * 1000
-  };
-}
-
-function printBenchmark(result: BenchmarkResult): void {
-  console.log(`\n  ${result.name}:`);
-  console.log(`    Avg: ${result.avgMs.toFixed(4)}ms`);
-  console.log(`    Min: ${result.minMs.toFixed(4)}ms`);
-  console.log(`    Max: ${result.maxMs.toFixed(4)}ms`);
-  console.log(`    p50: ${result.p50Ms.toFixed(4)}ms`);
-  console.log(`    p95: ${result.p95Ms.toFixed(4)}ms`);
-  console.log(`    p99: ${result.p99Ms.toFixed(4)}ms`);
-  console.log(`    Ops/sec: ${result.opsPerSecond.toFixed(2)}`);
-}
-
-// ============================================================
-// TOKEN COUNTING BENCHMARKS
-// ============================================================
-
-describe('1. Performance - Token Counting', () => {
-  
-  describe('countTokens performance', () => {
-    it('benchmarks short text (~10 chars)', () => {
-      const result = runBenchmark('countTokens (short)', () => {
-        countTokens('Hello world');
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(1); // Should be very fast
-      expect(result.opsPerSecond).toBeGreaterThan(1000);
-    });
-    
-    it('benchmarks medium text (~100 chars)', () => {
-      const result = runBenchmark('countTokens (medium)', () => {
-        countTokens('This is a moderately long sentence that contains a decent amount of text for benchmarking.');
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(2);
-      expect(result.opsPerSecond).toBeGreaterThan(500);
-    });
-    
-    it('benchmarks long text (~2000 chars)', () => {
-      const longText = 'The quick brown fox jumps over the lazy dog. '.repeat(50);
-      const result = runBenchmark('countTokens (long)', () => {
-        countTokens(longText);
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(10);
-      expect(result.opsPerSecond).toBeGreaterThan(100);
-    });
-    
-    it('benchmarks code text', () => {
-      const codeText = `function helloWorld() {
-  console.log("Hello, World!");
-  return 42;
-}
-const x = [1, 2, 3, 4, 5];
-const obj = { a: 1, b: 2 };`;
-      const result = runBenchmark('countTokens (code)', () => {
-        countTokens(codeText);
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(5);
-    });
-    
-    it('benchmarks unicode text', () => {
-      const result = runBenchmark('countTokens (unicode)', () => {
-        countTokens('こんにちは世界！これはテストです。日本語のテキストを処理しています。');
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(5);
-    });
-  });
-  
-  describe('estimateTokens performance', () => {
-    it('benchmarks short text', () => {
-      const result = runBenchmark('estimateTokens (short)', () => {
-        estimateTokens('Hello world');
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(1);
-    });
-    
-    it('benchmarks long text', () => {
-      const longText = 'word '.repeat(1000);
-      const result = runBenchmark('estimateTokens (long)', () => {
-        estimateTokens(longText);
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(10);
-    });
-  });
-  
-  describe('Token estimation accuracy', () => {
-    it('countTokens is consistent for same input', () => {
-      const text = 'The quick brown fox jumps over the lazy dog';
-      const tokens1 = countTokens(text);
-      const tokens2 = countTokens(text);
-      const tokens3 = countTokens(text);
-      expect(tokens1).toBe(tokens2);
-      expect(tokens2).toBe(tokens3);
-    });
-    
-    it('longer text has more tokens', () => {
-      const short = 'Hi';
-      const medium = 'Hello world, this is a test';
-      const long = 'Hello world, this is a test. The quick brown fox jumps over the lazy dog. ' + 
-                   'This is a longer text that should have more tokens than the medium one.';
-      
-      const tokensShort = countTokens(short);
-      const tokensMedium = countTokens(medium);
-      const tokensLong = countTokens(long);
-      
-      expect(tokensLong).toBeGreaterThan(tokensMedium);
-      expect(tokensMedium).toBeGreaterThan(tokensShort);
-    });
-    
-    it('code has more tokens than plain text', () => {
-      const plainText = 'This is a sentence about functions and code';
-      const codeText = 'function test() { return 42; } const x = 1;';
-      
-      const tokensPlain = countTokens(plainText);
-      const tokensCode = countTokens(codeText);
-      
-      // Code should be tokenized differently (may have more or fewer tokens)
-      // Just verify they're both valid numbers
-      expect(tokensPlain).toBeGreaterThan(0);
-      expect(tokensCode).toBeGreaterThan(0);
-    });
-  });
-});
-
-// ============================================================
-// ROUTING PERFORMANCE BENCHMARKS
-// ============================================================
-
-describe('2. Performance - Routing', () => {
-  
-  describe('routeQuery performance', () => {
-    it('benchmarks simple query', () => {
-      const result = runBenchmark('routeQuery (simple)', () => {
-        routeQuery('Hello, how are you?');
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(50); // Should be fast
-      expect(result.opsPerSecond).toBeGreaterThan(20);
-    });
-    
-    it('benchmarks complex query', () => {
-      const result = runBenchmark('routeQuery (complex)', () => {
-        routeQuery('Write a Python function to sort an array using quicksort algorithm with proper error handling');
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(100);
-    });
-    
-    it('benchmarks very long query', () => {
-      const longQuery = 'Analyze the following code and suggest improvements: '.repeat(20);
-      const result = runBenchmark('routeQuery (long)', () => {
-        routeQuery(longQuery);
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(200);
-    });
-    
-    it('benchmarks repeated queries (cache effectiveness)', () => {
-      const query = 'What is 2+2?';
-      
-      // First call (cache miss)
-      routeQuery(query);
-      
-      // Subsequent calls (should be faster if caching)
-      const result = runBenchmark('routeQuery (repeated)', () => {
-        routeQuery(query);
-      });
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(50);
-    });
-  });
-  
-  describe('routeBatch performance', () => {
-    it('benchmarks batch of 10', () => {
-      const queries = Array(10).fill('Hello world');
-      const result = runBenchmark('routeBatch (10 items)', () => {
-        routeBatch(queries);
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(500);
-    });
-    
-    it('benchmarks batch of 100', () => {
-      const queries = Array(100).fill('Test query');
-      const result = runBenchmark('routeBatch (100 items)', () => {
-        routeBatch(queries);
-      }, 50);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(5000);
-    });
-    
-    it('benchmarks batch scaling', () => {
-      const batch10 = Array(10).fill('test');
-      const batch100 = Array(100).fill('test');
-      
-      const time10 = runBenchmark('routeBatch (10)', () => routeBatch(batch10), 50);
-      const time100 = runBenchmark('routeBatch (100)', () => routeBatch(batch100), 50);
-      
-      printBenchmark(time10);
-      printBenchmark(time100);
-      
-      // 100 items should not take 10x longer
-      expect(time100.avgMs).toBeLessThan(time10.avgMs * 20);
-    });
-  });
-});
-
-// ============================================================
-// MEMORY OPERATION BENCHMARKS
-// ============================================================
-
-describe('3. Performance - Memory Operations', () => {
-  
-  describe('MemoryTree.add performance', () => {
-    it('benchmarks single add', () => {
-      const result = runBenchmark('MemoryTree.add (1 entry)', () => {
-        const memory = new MemoryTree({ maxSize: 1000 });
-        memory.add('test entry', { tags: ['test'] });
-      }, 1000);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(10);
-    });
-    
-    it('benchmarks repeated adds to same instance', () => {
-      const memory = new MemoryTree({ maxSize: 1000 });
-      
-      const result = runBenchmark('MemoryTree.add (100 to same instance)', () => {
-        for (let i = 0; i < 100; i++) {
-          memory.add(`test entry ${i}`, { tags: ['test'] });
-        }
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(100);
-    });
-    
-    it('benchmarks add with metadata', () => {
-      const result = runBenchmark('MemoryTree.add (with metadata)', () => {
-        const memory = new MemoryTree({ maxSize: 1000 });
-        memory.add('test entry with lots of metadata', {
-          tags: ['test', 'performance', 'benchmark'],
-          timestamp: Date.now(),
-          source: 'test',
-          priority: 1,
-          score: 0.95
-        });
-      }, 500);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(20);
-    });
-  });
-  
-  describe('MemoryTree.search performance', () => {
-    it('benchmarks search with few entries', () => {
-      const memory = new MemoryTree({ maxSize: 1000 });
-      for (let i = 0; i < 10; i++) {
-        memory.add(`test entry ${i}`, { tags: ['test'] });
-      }
-      
-      const result = runBenchmark('MemoryTree.search (10 entries)', () => {
-        memory.search('test');
-      }, 1000);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(10);
-    });
-    
-    it('benchmarks search with many entries', () => {
-      const memory = new MemoryTree({ maxSize: 10000 });
-      for (let i = 0; i < 1000; i++) {
-        memory.add(`test entry ${i} with some content`, { tags: ['test'] });
-      }
-      
-      const result = runBenchmark('MemoryTree.search (1000 entries)', () => {
-        memory.search('test');
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(50);
-    });
-    
-    it('benchmarks search with no matches', () => {
-      const memory = new MemoryTree({ maxSize: 1000 });
-      for (let i = 0; i < 100; i++) {
-        memory.add(`unique entry ${i}`, { tags: ['test'] });
-      }
-      
-      const result = runBenchmark('MemoryTree.search (no match)', () => {
-        memory.search('nonexistent query string');
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(20);
-    });
-  });
-  
-  describe('MemoryTree.getStats performance', () => {
-    it('benchmarks getStats', () => {
-      const memory = new MemoryTree({ maxSize: 1000 });
-      for (let i = 0; i < 100; i++) {
-        memory.add(`test entry ${i}`, { tags: ['test'] });
-      }
-      
-      const result = runBenchmark('MemoryTree.getStats', () => {
-        memory.getStats();
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(10);
-    });
-  });
-});
-
-// ============================================================
-// COST TRACKING BENCHMARKS
-// ============================================================
-
-describe('4. Performance - Cost Tracking', () => {
-  
-  describe('CostTracker performance', () => {
-    it('benchmarks record', () => {
-      const tracker = new CostTracker();
-      
-      const result = runBenchmark('CostTracker.record', () => {
-        tracker.record('openai', 'gpt-4o', 100, 50);
-      }, 1000);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(1);
-    });
-    
-    it('benchmarks getSummary', () => {
-      const tracker = new CostTracker();
-      
-      // Add some data
-      for (let i = 0; i < 100; i++) {
-        tracker.record('openai', 'gpt-4o', 100, 50);
-      }
-      
-      const result = runBenchmark('CostTracker.getSummary', () => {
-        tracker.getSummary();
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(5);
-    });
-  });
-});
-
-// ============================================================
-// FACTORY PERFORMANCE
-// ============================================================
-
-describe('5. Performance - Router Factory', () => {
-  
-  describe('createA3MRouter performance', () => {
-    it('benchmarks router creation', () => {
-      const result = runBenchmark('createA3MRouter', () => {
-        createA3MRouter({});
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(50);
-    });
-    
-    it('benchmarks router.route', () => {
-      const router = createA3MRouter({});
-      
-      const result = runBenchmark('router.route', () => {
-        router.route('test query');
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(50);
-    });
-    
-    it('benchmarks router.routeBatch', () => {
-      const router = createA3MRouter({});
-      
-      const result = runBenchmark('router.routeBatch (10 items)', () => {
-        router.routeBatch(Array(10).fill('test'));
-      }, 100);
-      printBenchmark(result);
-      
-      expect(result.avgMs).toBeLessThan(500);
-    });
-  });
-});
-
-// ============================================================
-// THROUGHPUT TESTS
-// ============================================================
-
-describe('6. Performance - Throughput', () => {
-  
-  it('sustains 50+ queries per second', () => {
-    const queries = Array(50).fill('test query');
-    const start = Date.now();
-    routeBatch(queries);
-    const elapsed = Date.now() - start;
-    
-    const qps = (50 / elapsed) * 1000;
-    console.log(`\n  Throughput: ${qps.toFixed(2)} queries/sec`);
-    
-    expect(qps).toBeGreaterThan(50);
-  });
-  
-  it('sustains 100+ queries per second', () => {
-    const queries = Array(100).fill('test query');
-    const start = Date.now();
-    routeBatch(queries);
-    const elapsed = Date.now() - start;
-    
-    const qps = (100 / elapsed) * 1000;
-    console.log(`\n  Throughput: ${qps.toFixed(2)} queries/sec`);
-    
-    expect(qps).toBeGreaterThan(100);
-  });
-  
-  it('handles burst traffic', () => {
-    const start = Date.now();
-    
-    // Simulate burst of 200 queries
-    for (let i = 0; i < 200; i++) {
-      routeQuery('burst test');
-    }
-    
-    const elapsed = Date.now() - start;
-    console.log(`\n  Burst of 200 queries: ${elapsed}ms`);
-    
-    expect(elapsed).toBeLessThan(10000); // Should handle burst within 10s
-  });
-});
-
-// ============================================================
-// LATENCY DISTRIBUTION TESTS
-// ============================================================
-
-describe('7. Performance - Latency Distribution', () => {
-  
-  it('routeQuery p95 latency is acceptable', () => {
-    const times: number[] = [];
-    
-    // Warmup
-    for (let i = 0; i < 10; i++) {
-      routeQuery('test');
-    }
-    
-    // Measure
-    for (let i = 0; i < 100; i++) {
-      const start = performance.now();
-      routeQuery('test query for latency measurement');
-      times.push(performance.now() - start);
-    }
-    
-    times.sort((a, b) => a - b);
-    const p95 = times[Math.floor(times.length * 0.95)];
-    const p99 = times[Math.floor(times.length * 0.99)];
-    
-    console.log(`\n  routeQuery latency:`);
-    console.log(`    p95: ${p95.toFixed(2)}ms`);
-    console.log(`    p99: ${p99.toFixed(2)}ms`);
-    
-    expect(p95).toBeLessThan(200);
-    expect(p99).toBeLessThan(500);
-  });
-  
-  it('MemoryTree operations have low latency', () => {
-    const memory = new MemoryTree({ maxSize: 1000 });
-    memory.add('test entry', { tags: ['test'] });
-    
-    const times: number[] = [];
-    
-    for (let i = 0; i < 100; i++) {
-      const start = performance.now();
-      memory.search('test');
-      times.push(performance.now() - start);
-    }
-    
-    times.sort((a, b) => a - b);
-    const p95 = times[Math.floor(times.length * 0.95)];
-    
-    console.log(`\n  MemoryTree.search p95: ${p95.toFixed(2)}ms`);
-    
-    expect(p95).toBeLessThan(50);
-  });
-});
-
-// ============================================================
-// MEMORY USAGE TESTS
-// ============================================================
-
-describe('8. Performance - Memory Usage', () => {
-  
-  it('MemoryTree does not leak memory on repeated operations', async () => {
-    const memory = new MemoryTree({ maxSize: 1000 });
-    
-    // Perform many operations
-    for (let i = 0; i < 1000; i++) {
-      await memory.add(`entry ${i}`);
-    }
-    
-    const finalStats = memory.getStats();
-    
-    // Stats should be accurate
-    expect(finalStats.totalChunks).toBeGreaterThan(0);
-  });
-  
-  it('CostTracker does not leak memory', () => {
-    const tracker = new CostTracker();
-    
-    for (let i = 0; i < 1000; i++) {
-      tracker.record('test', 'test-model', 100, 50);
-    }
-    
-    const summary = tracker.getSummary();
-    expect(summary.request_count).toBe(1000);
-  });
-});
-
-// ============================================================
-// REGRESSION GATES
-// ============================================================
-
-describe('9. Performance - Regression Gates', () => {
-  
-  // These tests fail if performance degrades beyond threshold
-  
-  it('countTokens must complete in < 1ms avg', () => {
-    const result = runBenchmark('REGRESSION: countTokens', () => {
-      countTokens('test string for regression check');
-    }, 1000);
-    
-    console.log(`\n  Regression check - countTokens: ${result.avgMs.toFixed(4)}ms avg`);
-    expect(result.avgMs).toBeLessThan(1);
-  });
-  
-  it('routeQuery must complete in < 100ms avg', () => {
-    const result = runBenchmark('REGRESSION: routeQuery', () => {
-      routeQuery('test query for regression check');
-    }, 100);
-    
-    console.log(`\n  Regression check - routeQuery: ${result.avgMs.toFixed(4)}ms avg`);
-    expect(result.avgMs).toBeLessThan(100);
-  });
-  
-  it('MemoryTree.add must complete in < 10ms avg', () => {
-    const result = runBenchmark('REGRESSION: MemoryTree.add', () => {
-      const memory = new MemoryTree({ maxSize: 1000 });
-      memory.add('regression test', { tags: ['test'] });
-    }, 100);
-    
-    console.log(`\n  Regression check - MemoryTree.add: ${result.avgMs.toFixed(4)}ms avg`);
-    expect(result.avgMs).toBeLessThan(10);
-  });
-  
-  it('MemoryTree.search must complete in < 20ms avg', () => {
-    const memory = new MemoryTree({ maxSize: 1000 });
-    for (let i = 0; i < 100; i++) {
-      memory.add(`test entry ${i}`, { tags: ['test'] });
-    }
-    
-    const result = runBenchmark('REGRESSION: MemoryTree.search', () => {
-      memory.search('test');
-    }, 100);
-    
-    console.log(`\n  Regression check - MemoryTree.search: ${result.avgMs.toFixed(4)}ms avg`);
-    expect(result.avgMs).toBeLessThan(20);
-  });
-});