adaptive-memory-multi-model-router 2.14.16 → 2.14.18

package/test-council/agents/performance-agent.ts

@@ -0,0 +1,426 @@
+#!/usr/bin/env node
+/**
+ * Performance Agent - Benchmarks and identifies performance issues
+ * 
+ * This agent measures:
+ * - Token counting accuracy and speed
+ * - Cost estimation precision
+ * - Response time distributions
+ * - Memory usage patterns
+ * - Throughput under load
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+interface BenchmarkResult {
+  name: string;
+  operations: number;
+  totalTimeMs: number;
+  avgMs: number;
+  minMs: number;
+  maxMs: number;
+  p50Ms: number;
+  p95Ms: number;
+  p99Ms: number;
+  opsPerSecond: number;
+}
+
+interface PerformanceReport {
+  benchmarks: BenchmarkResult[];
+  issues: PerformanceIssue[];
+  recommendations: string[];
+}
+
+interface PerformanceIssue {
+  name: string;
+  description: string;
+  severity: 'critical' | 'high' | 'medium' | 'low';
+  current: string;
+  expected: string;
+}
+
+interface PerformanceConfig {
+  iterations: number;
+  warmupRuns: number;
+  maxTimeMs: number;
+}
+
+// Default configuration
+const DEFAULT_CONFIG: PerformanceConfig = {
+  iterations: 1000,
+  warmupRuns: 10,
+  maxTimeMs: 30000
+};
+
+// Import the modules to benchmark
+function loadModules() {
+  try {
+    const index = require('../../dist/index.js');
+    const tokenUtils = require('../../dist/utils/tokenUtils.js');
+    return { index, tokenUtils };
+  } catch (e) {
+    // Try source if dist not available
+    try {
+      const tokenUtils = require('../../src/utils/tokenUtils.js');
+      return { index: {}, tokenUtils };
+    } catch (e2) {
+      return null;
+    }
+  }
+}
+
+// Run a benchmark
+function runBenchmark(name: string, fn: () => void, iterations: number): BenchmarkResult {
+  const times: number[] = [];
+  
+  // Warmup
+  for (let i = 0; i < 10; i++) {
+    fn();
+  }
+  
+  // Actual benchmark
+  for (let i = 0; i < iterations; i++) {
+    const start = performance.now();
+    fn();
+    const end = performance.now();
+    times.push(end - start);
+  }
+  
+  times.sort((a, b) => a - b);
+  
+  const totalTimeMs = times.reduce((a, b) => a + b, 0);
+  
+  return {
+    name,
+    operations: iterations,
+    totalTimeMs,
+    avgMs: totalTimeMs / iterations,
+    minMs: times[0],
+    maxMs: times[times.length - 1],
+    p50Ms: times[Math.floor(iterations * 0.50)],
+    p95Ms: times[Math.floor(iterations * 0.95)],
+    p99Ms: times[Math.floor(iterations * 0.99)],
+    opsPerSecond: (iterations / totalTimeMs) * 1000
+  };
+}
+
+// Token counting benchmarks
+function benchmarkTokenCounting(iterations: number): BenchmarkResult[] {
+  const results: BenchmarkResult[] = [];
+  const modules = loadModules();
+  
+  if (!modules || !modules.tokenUtils) {
+    console.log('Warning: Could not load tokenUtils for benchmarking');
+    return results;
+  }
+  
+  const { countTokens, estimateTokens } = modules.tokenUtils;
+  
+  // Short text
+  results.push(runBenchmark(
+    'countTokens (short text ~10 chars)',
+    () => countTokens('Hello world'),
+    iterations
+  ));
+  
+  // Medium text
+  results.push(runBenchmark(
+    'countTokens (medium text ~100 chars)',
+    () => countTokens('This is a moderately long sentence that contains a decent amount of text for benchmarking token counting functions in JavaScript.'),
+    iterations
+  ));
+  
+  // Long text
+  const longText = 'The quick brown fox jumps over the lazy dog. '.repeat(50);
+  results.push(runBenchmark(
+    'countTokens (long text ~2000 chars)',
+    () => countTokens(longText),
+    iterations
+  ));
+  
+  // Code text
+  const codeText = `function helloWorld() {
+  console.log("Hello, World!");
+  return 42;
+}
+const x = [1, 2, 3, 4, 5];
+const obj = { a: 1, b: 2 };`;
+  results.push(runBenchmark(
+    'countTokens (code text)',
+    () => countTokens(codeText),
+    iterations
+  ));
+  
+  // Unicode text
+  results.push(runBenchmark(
+    'countTokens (unicode - Japanese)',
+    () => countTokens('こんにちは世界！これはテストです。'),
+    iterations
+  ));
+  
+  // estimateTokens if different from countTokens
+  if (estimateTokens !== countTokens) {
+    results.push(runBenchmark(
+      'estimateTokens (short text)',
+      () => estimateTokens('Hello world'),
+      iterations
+    ));
+  }
+  
+  return results;
+}
+
+// Cost estimation benchmarks
+function benchmarkCostEstimation(iterations: number): BenchmarkResult[] {
+  const results: BenchmarkResult[] = [];
+  const modules = loadModules();
+  
+  if (!modules || !modules.tokenUtils) {
+    console.log('Warning: Could not load tokenUtils for benchmarking');
+    return results;
+  }
+  
+  const { estimateCost } = modules.tokenUtils;
+  
+  if (typeof estimateCost === 'function') {
+    // Small request
+    results.push(runBenchmark(
+      'estimateCost (100 in, 50 out)',
+      () => estimateCost(100, 50, 'gpt-4o'),
+      iterations
+    ));
+    
+    // Large request
+    results.push(runBenchmark(
+      'estimateCost (10K in, 5K out)',
+      () => estimateCost(10000, 5000, 'gpt-4o'),
+      iterations
+    ));
+    
+    // Different models
+    for (const model of ['gpt-4o', 'gpt-3.5-turbo', 'claude-3-sonnet']) {
+      results.push(runBenchmark(
+        `estimateCost (${model})`,
+        () => estimateCost(1000, 500, model),
+        iterations
+      ));
+    }
+  }
+  
+  return results;
+}
+
+// Memory benchmarks
+function benchmarkMemory(iterations: number): BenchmarkResult[] {
+  const results: BenchmarkResult[] = [];
+  const modules = loadModules();
+  
+  if (!modules) {
+    return results;
+  }
+  
+  // Test MemoryTree operations
+  if (modules.index.MemoryTree) {
+    const MemoryTree = modules.index.MemoryTree;
+    
+    // Add operation
+    const memory = new MemoryTree({ maxSize: 1000 });
+    results.push(runBenchmark(
+      'MemoryTree.add (single entry)',
+      () => {
+        memory.add('test entry ' + Math.random(), { tags: ['test'] });
+      },
+      Math.min(iterations, 100) // Fewer iterations for memory tests
+    ));
+    
+    // Search operation
+    for (let i = 0; i < 100; i++) {
+      memory.add('test entry ' + i, { tags: ['test'] });
+    }
+    results.push(runBenchmark(
+      'MemoryTree.search',
+      () => memory.search('test'),
+      Math.min(iterations, 100)
+    ));
+  }
+  
+  return results;
+}
+
+// Identify performance issues
+function identifyIssues(benchmarks: BenchmarkResult[]): PerformanceIssue[] {
+  const issues: PerformanceIssue[] = [];
+  
+  for (const bench of benchmarks) {
+    // Check if avg is too high
+    if (bench.avgMs > 100) {
+      issues.push({
+        name: `Slow operation: ${bench.name}`,
+        description: `Average time per operation exceeds 100ms`,
+        severity: bench.avgMs > 1000 ? 'critical' : 'high',
+        current: `${bench.avgMs.toFixed(2)}ms avg`,
+        expected: '< 100ms avg'
+      });
+    }
+    
+    // Check p99 vs avg ratio (high variance)
+    if (bench.avgMs > 0 && bench.p99Ms / bench.avgMs > 10) {
+      issues.push({
+        name: `High variance: ${bench.name}`,
+        description: `p99 is ${(bench.p99Ms / bench.avgMs).toFixed(1)}x higher than average`,
+        severity: 'medium',
+        current: `p99: ${bench.p99Ms.toFixed(2)}ms, avg: ${bench.avgMs.toFixed(2)}ms`,
+        expected: `p99 should be < 5x avg`
+      });
+    }
+    
+    // Check operations per second
+    if (bench.opsPerSecond < 10 && bench.avgMs > 1) {
+      issues.push({
+        name: `Low throughput: ${bench.name}`,
+        description: `Operations per second is very low`,
+        severity: 'medium',
+        current: `${bench.opsPerSecond.toFixed(2)} ops/sec`,
+        expected: '> 100 ops/sec'
+      });
+    }
+  }
+  
+  return issues;
+}
+
+// Generate performance tests
+function generatePerformanceTests(benchmarks: BenchmarkResult[]): string {
+  const tests: string[] = [];
+  
+  tests.push(`// Auto-generated performance tests
+// Generated: ${new Date().toISOString()}
+
+describe('Performance Benchmarks', () => {`);
+  
+  for (const bench of benchmarks) {
+    tests.push(`
+  describe('${bench.name}', () => {
+    it('completes within acceptable time', () => {
+      expect(${bench.avgMs.toFixed(2)}).toBeLessThan(100);
+    });
+    
+    it('has reasonable p99 latency', () => {
+      expect(${bench.p99Ms.toFixed(2)}).toBeLessThan(500);
+    });
+    
+    it('achieves minimum throughput', () => {
+      expect(${bench.opsPerSecond.toFixed(2)}).toBeGreaterThan(10);
+    });
+  });`);
+  }
+  
+  tests.push('});');
+  
+  return tests.join('\n');
+}
+
+// Main analysis function
+function analyzePerformance(config: Partial<PerformanceConfig> = {}): PerformanceReport {
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+  
+  console.log(`\nRunning performance benchmarks (${cfg.iterations} iterations)...`);
+  
+  const benchmarks: BenchmarkResult[] = [];
+  
+  // Token counting benchmarks
+  console.log('  - Token counting...');
+  benchmarks.push(...benchmarkTokenCounting(cfg.iterations));
+  
+  // Cost estimation benchmarks  
+  console.log('  - Cost estimation...');
+  benchmarks.push(...benchmarkCostEstimation(cfg.iterations));
+  
+  // Memory benchmarks
+  console.log('  - Memory operations...');
+  benchmarks.push(...benchmarkMemory(cfg.iterations));
+  
+  // Identify issues
+  const issues = identifyIssues(benchmarks);
+  
+  // Generate recommendations
+  const recommendations: string[] = [];
+  
+  if (issues.some(i => i.severity === 'critical')) {
+    recommendations.push('CRITICAL: Address critical performance issues before production');
+  }
+  
+  if (issues.some(i => i.severity === 'high')) {
+    recommendations.push('HIGH: Optimize high-severity performance bottlenecks');
+  }
+  
+  const slowBenchmarks = benchmarks.filter(b => b.avgMs > 50);
+  if (slowBenchmarks.length > 0) {
+    recommendations.push(`Consider caching for ${slowBenchmarks.length} slow operations`);
+  }
+  
+  return { benchmarks, issues, recommendations };
+}
+
+// Print report
+function printReport(report: PerformanceReport): void {
+  console.log('\n========================================');
+  console.log('PERFORMANCE AGENT - Benchmark Report');
+  console.log('========================================\n');
+  
+  console.log('Benchmarks:');
+  for (const bench of report.benchmarks) {
+    console.log(`\n  ${bench.name}:`);
+    console.log(`    Avg: ${bench.avgMs.toFixed(4)}ms`);
+    console.log(`    Min: ${bench.minMs.toFixed(4)}ms`);
+    console.log(`    Max: ${bench.maxMs.toFixed(4)}ms`);
+    console.log(`    p50: ${bench.p50Ms.toFixed(4)}ms`);
+    console.log(`    p95: ${bench.p95Ms.toFixed(4)}ms`);
+    console.log(`    p99: ${bench.p99Ms.toFixed(4)}ms`);
+    console.log(`    Ops/sec: ${bench.opsPerSecond.toFixed(2)}`);
+  }
+  
+  if (report.issues.length > 0) {
+    console.log('\nIssues Found:');
+    for (const issue of report.issues) {
+      console.log(`  [${issue.severity.toUpperCase()}] ${issue.name}`);
+      console.log(`    ${issue.description}`);
+      console.log(`    Current: ${issue.current}`);
+      console.log(`    Expected: ${issue.expected}`);
+    }
+  } else {
+    console.log('\nNo performance issues detected!');
+  }
+  
+  if (report.recommendations.length > 0) {
+    console.log('\nRecommendations:');
+    for (const rec of report.recommendations) {
+      console.log(`  - ${rec}`);
+    }
+  }
+}
+
+// Run agent if executed directly
+if (require.main === module) {
+  const report = analyzePerformance();
+  printReport(report);
+  
+  // Generate test code
+  const generated = generatePerformanceTests(report.benchmarks);
+  const outputPath = path.join(__dirname, '../3-performance-tests-generated.ts');
+  fs.writeFileSync(outputPath, generated);
+  console.log(`\nGenerated tests written to: ${outputPath}`);
+}
+
+export { 
+  analyzePerformance, 
+  runBenchmark, 
+  benchmarkTokenCounting, 
+  benchmarkCostEstimation,
+  benchmarkMemory,
+  generatePerformanceTests,
+  PerformanceReport,
+  BenchmarkResult,
+  PerformanceIssue
+};

package/test-council/agents/structure-agent.ts

@@ -0,0 +1,227 @@
+#!/usr/bin/env node
+/**
+ * Structure Agent - Analyzes code structure and exports for test coverage
+ * 
+ * This agent identifies:
+ * - Exported functions without tests
+ * - Type definitions needing validation
+ * - Interface implementations
+ * - Public API coverage
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+interface ExportInfo {
+  name: string;
+  type: 'function' | 'class' | 'interface' | 'type' | 'const' | 'unknown';
+  file: string;
+  line: number;
+  tested: boolean;
+}
+
+interface CoverageReport {
+  total: number;
+  tested: number;
+  untested: ExportInfo[];
+  coverage: number;
+}
+
+// Main analysis function
+function analyzeStructure(projectRoot: string): CoverageReport {
+  const srcDir = path.join(projectRoot, 'src');
+  const exports: ExportInfo[] = [];
+  
+  // Find all TypeScript files
+  const files = findTsFiles(srcDir);
+  
+  for (const file of files) {
+    const content = fs.readFileSync(file, 'utf-8');
+    const fileExports = extractExports(content, file);
+    exports.push(...fileExports);
+  }
+  
+  // Check which exports are tested
+  const testFiles = findTestFiles(projectRoot);
+  const testedNames = new Set<string>();
+  
+  for (const testFile of testFiles) {
+    const content = fs.readFileSync(testFile, 'utf-8');
+    // Extract describe/it blocks to find tested names
+    const matches = content.matchAll(/(?:describe|it)\s*\(\s*['"]([^'"]+)/g);
+    for (const match of matches) {
+      testedNames.add(match[1]);
+    }
+  }
+  
+  // Mark tested exports
+  for (const exp of exports) {
+    exp.tested = testedNames.has(exp.name) || 
+                 testedNames.has(exp.name.toLowerCase()) ||
+                 testedNames.has(sanitizeTestName(exp.name));
+  }
+  
+  const tested = exports.filter(e => e.tested);
+  const untested = exports.filter(e => !e.tested);
+  
+  return {
+    total: exports.length,
+    tested: tested.length,
+    untested,
+    coverage: exports.length > 0 ? (tested.length / exports.length) * 100 : 0
+  };
+}
+
+function findTsFiles(dir: string): string[] {
+  const files: string[] = [];
+  
+  function walk(d: string) {
+    const entries = fs.readdirSync(d, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.name === 'node_modules' || entry.name === 'dist' || entry.name === '__pycache__') continue;
+      const fullPath = path.join(d, entry.name);
+      if (entry.isDirectory()) {
+        walk(fullPath);
+      } else if (entry.name.endsWith('.ts') && !entry.name.endsWith('.d.ts')) {
+        files.push(fullPath);
+      }
+    }
+  }
+  
+  walk(dir);
+  return files;
+}
+
+function findTestFiles(projectRoot: string): string[] {
+  const testDirs = ['test', 'tests', 'test-council'];
+  const files: string[] = [];
+  
+  for (const dir of testDirs) {
+    const testDir = path.join(projectRoot, dir);
+    if (fs.existsSync(testDir)) {
+      files.push(...findTestFilesInDir(testDir));
+    }
+  }
+  
+  return files;
+}
+
+function findTestFilesInDir(dir: string): string[] {
+  const files: string[] = [];
+  
+  function walk(d: string) {
+    const entries = fs.readdirSync(d, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.name === 'node_modules' || entry.name === 'dist') continue;
+      const fullPath = path.join(d, entry.name);
+      if (entry.isDirectory()) {
+        walk(fullPath);
+      } else if (entry.name.match(/\.(test|spec)\.(ts|js)$/)) {
+        files.push(fullPath);
+      }
+    }
+  }
+  
+  walk(dir);
+  return files;
+}
+
+function extractExports(content: string, file: string): ExportInfo[] {
+  const exports: ExportInfo[] = [];
+  const lines = content.split('\n');
+  
+  // Track line numbers
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    const lineNum = i + 1;
+    
+    // export function/const/class/interface/type
+    const exportMatch = line.match(/^export\s+(?:function|class|const|interface|type|enum)\s+(\w+)/);
+    if (exportMatch) {
+      const name = exportMatch[1];
+      let type: ExportInfo['type'] = 'unknown';
+      
+      if (line.includes('function')) type = 'function';
+      else if (line.includes('class')) type = 'class';
+      else if (line.includes('interface')) type = 'interface';
+      else if (line.includes('type')) type = 'type';
+      else if (line.includes('const') || line.includes('enum')) type = 'const';
+      
+      exports.push({ name, type, file: path.basename(file), line: lineNum, tested: false });
+    }
+    
+    // export { name } from
+    const reExportMatch = line.match(/^export\s*{\s*(\w+)/);
+    if (reExportMatch) {
+      exports.push({ 
+        name: reExportMatch[1], 
+        type: 'unknown', 
+        file: path.basename(file), 
+        line: lineNum, 
+        tested: false 
+      });
+    }
+  }
+  
+  return exports;
+}
+
+function sanitizeTestName(name: string): string {
+  // Convert PascalCase to kebab-case or space-separated
+  return name
+    .replace(/([A-Z])/g, ' $1')
+    .toLowerCase()
+    .trim();
+}
+
+// Generate tests for untested exports
+function generateTestsForUntested(report: CoverageReport): string {
+  const tests: string[] = [];
+  
+  for (const exp of report.untested) {
+    if (exp.type === 'function') {
+      tests.push(`
+  // TODO: Add test for untested export: ${exp.name}
+  // File: ${exp.file}:${exp.line}
+  // Type: ${exp.type}
+  it('${exp.name} - structure coverage', () => {
+    // Structure test placeholder
+    expect(true).toBe(true);
+  });`);
+    }
+  }
+  
+  return tests.join('\n');
+}
+
+// Run agent if executed directly
+if (require.main === module) {
+  const projectRoot = path.resolve(__dirname, '../..');
+  const report = analyzeStructure(projectRoot);
+  
+  console.log('\n========================================');
+  console.log('STRUCTURE AGENT - Coverage Report');
+  console.log('========================================\n');
+  console.log(`Total Exports: ${report.total}`);
+  console.log(`Tested: ${report.tested}`);
+  console.log(`Untested: ${report.untested.length}`);
+  console.log(`Coverage: ${report.coverage.toFixed(1)}%\n`);
+  
+  if (report.untested.length > 0) {
+    console.log('Untested Exports:');
+    for (const exp of report.untested.slice(0, 20)) {
+      console.log(`  - ${exp.name} (${exp.type}) @ ${exp.file}:${exp.line}`);
+    }
+    if (report.untested.length > 20) {
+      console.log(`  ... and ${report.untested.length - 20} more`);
+    }
+  }
+  
+  // Generate test code
+  const generated = generateTestsForUntested(report);
+  const outputPath = path.join(__dirname, '../1-structure-tests-generated.ts');
+  fs.writeFileSync(outputPath, `// Auto-generated structure tests\n${generated}\n`);
+  console.log(`\nGenerated tests written to: ${outputPath}`);
+}
+
+export { analyzeStructure, CoverageReport, ExportInfo, generateTestsForUntested };