memorylink 1.0.0

package/src/test-runner/TestRunner.ts

@@ -0,0 +1,421 @@
+/**
+ * Professional Test Runner for MemoryLink
+ * Supports parallel execution, progress tracking, and comprehensive reporting
+ */
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+const execAsync = promisify(exec);
+
+export interface TestScenario {
+  id: number;
+  description: string;
+  command: string;
+  expectedResult: 'pass' | 'fail';
+  category: string;
+  batch: number;
+}
+
+export interface TestResult {
+  scenario: TestScenario;
+  passed: boolean;
+  duration: number;
+  output?: string;
+  error?: string;
+}
+
+export interface TestSuiteResult {
+  total: number;
+  passed: number;
+  failed: number;
+  duration: number;
+  passRate: number;
+  results: TestResult[];
+  failures: TestResult[];
+}
+
+export class TestRunner {
+  private scenarios: TestScenario[] = [];
+  private results: TestResult[] = [];
+  private maxConcurrency: number;
+  private verbose: boolean;
+  private checkpointFile: string;
+  private resumeMode: boolean;
+
+  constructor(
+    maxConcurrency: number = 20,
+    verbose: boolean = false,
+    checkpointFile?: string,
+    resumeMode: boolean = false
+  ) {
+    this.maxConcurrency = maxConcurrency;
+    this.verbose = verbose;
+    this.checkpointFile = checkpointFile || '.test-checkpoint.json';
+    this.resumeMode = resumeMode;
+  }
+
+  /**
+   * Load scenarios from generated test file
+   */
+  async loadScenarios(filePath: string): Promise<void> {
+    const content = await fs.readFile(filePath, 'utf-8');
+    const lines = content.split('\n');
+    
+    for (const line of lines) {
+      const match = line.match(/test_scenario\s+(\d+)\s+"([^"]+)"\s+"([^"]+)"\s+"(pass|fail)"/);
+      if (match) {
+        const [, id, desc, cmd, expected] = match;
+        const scenarioId = parseInt(id);
+        
+        // Determine category and batch
+        let category = 'Unknown';
+        let batch = 0;
+        
+        if (scenarioId >= 2001 && scenarioId <= 2500) {
+          category = 'Performance & Load';
+          batch = 1;
+        } else if (scenarioId >= 2501 && scenarioId <= 3000) {
+          category = 'Edge Cases';
+          batch = 2;
+        } else if (scenarioId >= 3001 && scenarioId <= 3500) {
+          category = 'Failure & Recovery';
+          batch = 3;
+        } else if (scenarioId >= 3501 && scenarioId <= 4000) {
+          category = 'Real-World Crowd';
+          batch = 4;
+        } else if (scenarioId >= 4001 && scenarioId <= 4500) {
+          category = 'Architecture';
+          batch = 5;
+        } else if (scenarioId >= 4501 && scenarioId <= 5000) {
+          category = 'Timing & Concurrency';
+          batch = 6;
+        } else if (scenarioId >= 5001 && scenarioId <= 5500) {
+          category = 'Data Integrity';
+          batch = 7;
+        } else if (scenarioId >= 5501 && scenarioId <= 6000) {
+          category = 'Stress & Chaos';
+          batch = 8;
+        }
+        
+        this.scenarios.push({
+          id: scenarioId,
+          description: desc,
+          command: cmd,
+          expectedResult: expected as 'pass' | 'fail',
+          category,
+          batch,
+        });
+      }
+    }
+    
+    console.log(`✅ Loaded ${this.scenarios.length} scenarios`);
+  }
+
+  /**
+   * Save checkpoint for resume capability
+   */
+  private async saveCheckpoint(completed: number, total: number): Promise<void> {
+    try {
+      const checkpoint = {
+        timestamp: new Date().toISOString(),
+        completed,
+        total,
+        results: this.results,
+      };
+      await fs.writeFile(this.checkpointFile, JSON.stringify(checkpoint, null, 2));
+    } catch (error) {
+      // Ignore checkpoint errors
+    }
+  }
+
+  /**
+   * Load checkpoint for resume
+   */
+  private async loadCheckpoint(): Promise<{ completed: number; results: TestResult[] } | null> {
+    try {
+      const content = await fs.readFile(this.checkpointFile, 'utf-8');
+      const checkpoint = JSON.parse(content);
+      return {
+        completed: checkpoint.completed || 0,
+        results: checkpoint.results || [],
+      };
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Execute a single test scenario (optimized for large scale)
+   */
+  async executeScenario(scenario: TestScenario): Promise<TestResult> {
+    const startTime = Date.now();
+    
+    try {
+      const { stdout, stderr } = await execAsync(scenario.command, {
+        cwd: process.cwd(),
+        timeout: 20000, // 20 second timeout (reduced for speed)
+        maxBuffer: 5 * 1024 * 1024, // 5MB buffer (reduced for memory)
+      });
+      
+      const duration = Date.now() - startTime;
+      const output = stdout + stderr;
+      const exitCode = 0; // execAsync throws on non-zero, so 0 means success
+      
+      let passed = false;
+      
+      if (scenario.expectedResult === 'fail') {
+        // For expected failures, check for error indicators OR non-zero exit
+        const hasError = /error|Error|✗|invalid|required|empty|cannot|blocked|must be/i.test(output);
+        passed = hasError;
+      } else {
+        // For expected passes, check exit code
+        passed = exitCode === 0;
+      }
+      
+      return {
+        scenario,
+        passed,
+        duration,
+        output: this.verbose ? output : undefined,
+      };
+    } catch (error: any) {
+      const duration = Date.now() - startTime;
+      const output = error.stdout + error.stderr;
+      const exitCode = error.code || 1;
+      
+      let passed = false;
+      
+      if (scenario.expectedResult === 'fail') {
+        // For expected failures, non-zero exit or error message means pass
+        const hasError = /error|Error|✗|invalid|required|empty|cannot|blocked|must be/i.test(output);
+        passed = exitCode !== 0 || hasError;
+      } else {
+        // For expected passes, any error means fail
+        passed = false;
+      }
+      
+      return {
+        scenario,
+        passed,
+        duration,
+        output: this.verbose ? output : undefined,
+        error: error.message,
+      };
+    }
+  }
+
+  /**
+   * Execute scenarios in parallel batches (optimized for 10K-20K scenarios)
+   */
+  async executeParallel(scenarios: TestScenario[]): Promise<TestResult[]> {
+    const results: TestResult[] = [];
+    const batches: TestScenario[][] = [];
+    
+    // Split into batches for parallel execution
+    for (let i = 0; i < scenarios.length; i += this.maxConcurrency) {
+      batches.push(scenarios.slice(i, i + this.maxConcurrency));
+    }
+    
+    let completed = 0;
+    const total = scenarios.length;
+    const checkpointInterval = Math.max(100, Math.floor(total / 20)); // Save every 5%
+    
+    // Load checkpoint if resuming
+    if (this.resumeMode) {
+      const checkpoint = await this.loadCheckpoint();
+      if (checkpoint) {
+        results.push(...checkpoint.results);
+        completed = checkpoint.completed;
+        console.log(`📂 Resuming from checkpoint: ${completed}/${total} completed`);
+      }
+    }
+    
+    // Skip already completed batches
+    const startBatch = Math.floor(completed / this.maxConcurrency);
+    
+    for (let batchIdx = startBatch; batchIdx < batches.length; batchIdx++) {
+      const batch = batches[batchIdx];
+      
+      // Execute batch with error handling per scenario
+      const batchResults = await Promise.allSettled(
+        batch.map(scenario => this.executeScenario(scenario))
+      );
+      
+      // Process results
+      for (const result of batchResults) {
+        if (result.status === 'fulfilled') {
+          results.push(result.value);
+        } else {
+          // Handle failed scenario execution
+          results.push({
+            scenario: batch[batchResults.indexOf(result)],
+            passed: false,
+            duration: 0,
+            error: result.reason?.message || 'Unknown error',
+          });
+        }
+      }
+      
+      completed += batch.length;
+      
+      // Save checkpoint periodically
+      if (completed % checkpointInterval === 0 || completed === total) {
+        await this.saveCheckpoint(completed, total);
+      }
+      
+      // Progress display
+      const progress = ((completed / total) * 100).toFixed(1);
+      const elapsed = ((Date.now() - (global as any).testStartTime) / 1000).toFixed(0);
+      const rate = completed / (parseFloat(elapsed) || 1);
+      const remaining = total - completed;
+      const eta = remaining > 0 && rate > 0 ? ((remaining / rate) / 60).toFixed(1) : '0';
+      const rateStr = rate.toFixed(1);
+      
+      process.stdout.write(
+        `\r⏳ Progress: ${completed}/${total} (${progress}%) | Rate: ${rateStr}/s | ETA: ${eta}m`
+      );
+      
+      // Memory cleanup every 1000 scenarios
+      if (completed % 1000 === 0 && global.gc) {
+        global.gc();
+      }
+    }
+    
+    process.stdout.write('\n');
+    
+    // Clean up checkpoint on success
+    try {
+      await fs.unlink(this.checkpointFile);
+    } catch {
+      // Ignore
+    }
+    
+    return results;
+  }
+
+  /**
+   * Run all scenarios (optimized for 10K-20K scenarios)
+   */
+  async runAll(): Promise<TestSuiteResult> {
+    const startTime = Date.now();
+    (global as any).testStartTime = startTime;
+    
+    console.log(`\n🧪 Executing ${this.scenarios.length} scenarios...`);
+    console.log(`📊 Concurrency: ${this.maxConcurrency} parallel tests`);
+    console.log(`💾 Checkpoint: ${this.checkpointFile}`);
+    if (this.resumeMode) {
+      console.log(`🔄 Resume mode: Enabled\n`);
+    } else {
+      console.log('');
+    }
+    
+    // Group by batch for better progress tracking
+    const batches = new Map<number, TestScenario[]>();
+    for (const scenario of this.scenarios) {
+      if (!batches.has(scenario.batch)) {
+        batches.set(scenario.batch, []);
+      }
+      batches.get(scenario.batch)!.push(scenario);
+    }
+    
+    const allResults: TestResult[] = [];
+    
+    // Execute batches sequentially, but scenarios within batch in parallel
+    for (const [batchNum, batchScenarios] of batches.entries()) {
+      if (batchNum > 0) {
+        console.log(`\n📦 Batch ${batchNum}: ${batchScenarios[0].category} (${batchScenarios.length} scenarios)`);
+      }
+      
+      const batchResults = await this.executeParallel(batchScenarios);
+      allResults.push(...batchResults);
+      
+      const batchPassed = batchResults.filter(r => r.passed).length;
+      const batchFailed = batchResults.filter(r => !r.passed).length;
+      console.log(`   ✅ ${batchPassed} passed, ❌ ${batchFailed} failed`);
+    }
+    
+    const duration = Date.now() - startTime;
+    const passed = allResults.filter(r => r.passed).length;
+    const failed = allResults.filter(r => !r.passed).length;
+    const passRate = (passed / allResults.length) * 100;
+    
+    return {
+      total: allResults.length,
+      passed,
+      failed,
+      duration,
+      passRate,
+      results: allResults,
+      failures: allResults.filter(r => !r.passed),
+    };
+  }
+
+  /**
+   * Generate comprehensive report
+   */
+  async generateReport(result: TestSuiteResult, outputPath: string): Promise<void> {
+    const report = {
+      timestamp: new Date().toISOString(),
+      summary: {
+        total: result.total,
+        passed: result.passed,
+        failed: result.failed,
+        passRate: result.passRate.toFixed(2),
+        duration: `${(result.duration / 1000).toFixed(2)}s`,
+      },
+      failures: result.failures.map(f => ({
+        id: f.scenario.id,
+        description: f.scenario.description,
+        category: f.scenario.category,
+        command: f.scenario.command,
+        error: f.error,
+        output: f.output?.substring(0, 500), // First 500 chars
+      })),
+      byCategory: this.groupByCategory(result.results),
+      byBatch: this.groupByBatch(result.results),
+    };
+    
+    await fs.writeFile(outputPath, JSON.stringify(report, null, 2));
+    console.log(`\n📄 Report saved to: ${outputPath}`);
+  }
+
+  private groupByCategory(results: TestResult[]): Record<string, { passed: number; failed: number }> {
+    const grouped: Record<string, { passed: number; failed: number }> = {};
+    
+    for (const result of results) {
+      const cat = result.scenario.category;
+      if (!grouped[cat]) {
+        grouped[cat] = { passed: 0, failed: 0 };
+      }
+      if (result.passed) {
+        grouped[cat].passed++;
+      } else {
+        grouped[cat].failed++;
+      }
+    }
+    
+    return grouped;
+  }
+
+  private groupByBatch(results: TestResult[]): Record<number, { passed: number; failed: number }> {
+    const grouped: Record<number, { passed: number; failed: number }> = {};
+    
+    for (const result of results) {
+      const batch = result.scenario.batch;
+      if (!grouped[batch]) {
+        grouped[batch] = { passed: 0, failed: 0 };
+      }
+      if (result.passed) {
+        grouped[batch].passed++;
+      } else {
+        grouped[batch].failed++;
+      }
+    }
+    
+    return grouped;
+  }
+}
+

package/src/test-runner/performance-test.ts

@@ -0,0 +1,161 @@
+/**
+ * Performance Testing Module
+ * Tests MemoryLink performance under various load conditions
+ */
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import * as fs from 'fs/promises';
+
+const execAsync = promisify(exec);
+
+export interface PerformanceMetrics {
+  operation: string;
+  duration: number;
+  memoryUsage: number;
+  throughput: number;
+  p50: number;
+  p95: number;
+  p99: number;
+}
+
+export class PerformanceTester {
+  /**
+   * Test memory add performance
+   */
+  async testMemoryAdd(count: number): Promise<PerformanceMetrics> {
+    const durations: number[] = [];
+    const startMemory = process.memoryUsage().heapUsed;
+    
+    for (let i = 0; i < count; i++) {
+      const start = Date.now();
+      await execAsync(`node dist/cli.js memory "Perf test ${i}" --tags perf`);
+      durations.push(Date.now() - start);
+    }
+    
+    const endMemory = process.memoryUsage().heapUsed;
+    const sorted = durations.sort((a, b) => a - b);
+    
+    return {
+      operation: 'memory_add',
+      duration: durations.reduce((a, b) => a + b, 0),
+      memoryUsage: endMemory - startMemory,
+      throughput: count / (durations.reduce((a, b) => a + b, 0) / 1000),
+      p50: sorted[Math.floor(sorted.length * 0.5)],
+      p95: sorted[Math.floor(sorted.length * 0.95)],
+      p99: sorted[Math.floor(sorted.length * 0.99)],
+    };
+  }
+
+  /**
+   * Test search performance
+   */
+  async testSearch(count: number): Promise<PerformanceMetrics> {
+    const durations: number[] = [];
+    
+    for (let i = 0; i < count; i++) {
+      const start = Date.now();
+      await execAsync(`node dist/cli.js search "test${i}" --limit 100`);
+      durations.push(Date.now() - start);
+    }
+    
+    const sorted = durations.sort((a, b) => a - b);
+    
+    return {
+      operation: 'search',
+      duration: durations.reduce((a, b) => a + b, 0),
+      memoryUsage: 0,
+      throughput: count / (durations.reduce((a, b) => a + b, 0) / 1000),
+      p50: sorted[Math.floor(sorted.length * 0.5)],
+      p95: sorted[Math.floor(sorted.length * 0.95)],
+      p99: sorted[Math.floor(sorted.length * 0.99)],
+    };
+  }
+
+  /**
+   * Test concurrent operations
+   */
+  async testConcurrency(concurrency: number, operations: number): Promise<PerformanceMetrics> {
+    const start = Date.now();
+    const promises: Promise<void>[] = [];
+    
+    for (let i = 0; i < operations; i++) {
+      promises.push(
+        execAsync(`node dist/cli.js memory "Concurrent ${i}" --tags concurrent`)
+          .then(() => {})
+          .catch(() => {})
+      );
+      
+      if (promises.length >= concurrency) {
+        await Promise.all(promises);
+        promises.length = 0;
+      }
+    }
+    
+    await Promise.all(promises);
+    const duration = Date.now() - start;
+    
+    return {
+      operation: 'concurrent',
+      duration,
+      memoryUsage: 0,
+      throughput: operations / (duration / 1000),
+      p50: 0,
+      p95: 0,
+      p99: 0,
+    };
+  }
+
+  /**
+   * Run comprehensive performance test suite
+   */
+  async runSuite(): Promise<Record<string, PerformanceMetrics>> {
+    console.log('🧪 Running Performance Test Suite...\n');
+    
+    const results: Record<string, PerformanceMetrics> = {};
+    
+    // Test 1: Memory add performance (100 operations)
+    console.log('📝 Testing memory add (100 operations)...');
+    results.memoryAdd = await this.testMemoryAdd(100);
+    
+    // Test 2: Search performance (50 operations)
+    console.log('🔍 Testing search (50 operations)...');
+    results.search = await this.testSearch(50);
+    
+    // Test 3: Concurrency (10 concurrent, 100 operations)
+    console.log('⚡ Testing concurrency (10 concurrent, 100 operations)...');
+    results.concurrency = await this.testConcurrency(10, 100);
+    
+    return results;
+  }
+
+  /**
+   * Generate performance report
+   */
+  async generateReport(metrics: Record<string, PerformanceMetrics>, outputPath: string): Promise<void> {
+    const report = {
+      timestamp: new Date().toISOString(),
+      metrics,
+      summary: {
+        memoryAdd: {
+          avgDuration: metrics.memoryAdd.duration / 100,
+          throughput: metrics.memoryAdd.throughput.toFixed(2),
+          p95: metrics.memoryAdd.p95,
+        },
+        search: {
+          avgDuration: metrics.search.duration / 50,
+          throughput: metrics.search.throughput.toFixed(2),
+          p95: metrics.search.p95,
+        },
+        concurrency: {
+          totalDuration: metrics.concurrency.duration,
+          throughput: metrics.concurrency.throughput.toFixed(2),
+        },
+      },
+    };
+    
+    await fs.writeFile(outputPath, JSON.stringify(report, null, 2));
+    console.log(`\n📄 Performance report saved to: ${outputPath}`);
+  }
+}
+