@stackmemoryai/stackmemory 0.8.0 → 1.0.0

package/scripts/testing/ralph-integration-validation.js

@@ -1,727 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Ralph-StackMemory Integration Validation Script
- * Comprehensive testing of Ralph swarm integration system
- */
-
-import { execSync, spawn } from 'child_process';
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-class RalphIntegrationValidator {
-  constructor() {
-    this.results = {
-      cliTests: [],
-      swarmTests: [],
-      contextTests: [],
-      patternTests: [],
-      orchestrationTests: [],
-      integrationTests: [],
-      errors: [],
-      warnings: [],
-      summary: {}
-    };
-    this.testDir = './test-ralph-validation';
-  }
-
-  async runValidation() {
-    console.log('🎭 Starting Ralph-StackMemory Integration Validation');
-    console.log('=' .repeat(60));
-
-    try {
-      await this.setup();
-      await this.testCliCommands();
-      await this.testSwarmCoordination();
-      await this.testContextLoading();
-      await this.testPatternLearning();
-      await this.testOrchestration();
-      await this.testIntegrationScenarios();
-      await this.runExistingTests();
-      this.generateReport();
-    } catch (error) {
-      this.results.errors.push(`Validation failed: ${error.message}`);
-      console.error('❌ Validation failed:', error.message);
-    } finally {
-      await this.cleanup();
-    }
-  }
-
-  async setup() {
-    console.log('🔧 Setting up test environment...');
-    
-    // Create test directory
-    if (fs.existsSync(this.testDir)) {
-      fs.rmSync(this.testDir, { recursive: true, force: true });
-    }
-    fs.mkdirSync(this.testDir, { recursive: true });
-    process.chdir(this.testDir);
-
-    // Initialize git for testing
-    try {
-      execSync('git init', { stdio: 'pipe' });
-      execSync('git config user.email "test@example.com"', { stdio: 'pipe' });
-      execSync('git config user.name "Test User"', { stdio: 'pipe' });
-    } catch (error) {
-      this.results.warnings.push('Git initialization failed - some tests may not work');
-    }
-
-    console.log('✅ Test environment setup complete');
-  }
-
-  async testCliCommands() {
-    console.log('\n📋 Testing CLI Commands...');
-    
-    const tests = [
-      {
-        name: 'Ralph Init Command',
-        command: 'node ../bin/stackmemory.js ralph init "Test task" --criteria "Tests pass,Code works"',
-        expectSuccess: true,
-        expectFiles: ['.ralph/task.md', '.ralph/completion-criteria.md']
-      },
-      {
-        name: 'Ralph Status Command', 
-        command: 'node ../bin/stackmemory.js ralph status',
-        expectSuccess: true,
-        expectOutput: 'Ralph Loop Status'
-      },
-      {
-        name: 'Ralph Debug Command',
-        command: 'node ../bin/stackmemory.js ralph debug',
-        expectSuccess: true,
-        expectOutput: 'Ralph Loop Debug'
-      },
-      {
-        name: 'Ralph Learn Command',
-        command: 'node ../bin/stackmemory.js ralph learn',
-        expectSuccess: true,
-        expectOutput: 'Learning patterns'
-      }
-    ];
-
-    for (const test of tests) {
-      try {
-        console.log(`  Testing: ${test.name}...`);
-        const result = execSync(test.command, { 
-          encoding: 'utf8',
-          stdio: 'pipe',
-          timeout: 30000
-        });
-
-        let passed = true;
-        let details = [];
-
-        if (test.expectOutput && !result.includes(test.expectOutput)) {
-          passed = false;
-          details.push(`Expected output "${test.expectOutput}" not found`);
-        }
-
-        if (test.expectFiles) {
-          for (const file of test.expectFiles) {
-            if (!fs.existsSync(file)) {
-              passed = false;
-              details.push(`Expected file "${file}" not created`);
-            }
-          }
-        }
-
-        this.results.cliTests.push({
-          name: test.name,
-          passed,
-          details: passed ? ['Command executed successfully'] : details,
-          output: result.substring(0, 200)
-        });
-
-        console.log(`    ${passed ? '✅' : '❌'} ${test.name}`);
-
-      } catch (error) {
-        this.results.cliTests.push({
-          name: test.name,
-          passed: false,
-          details: [`Command failed: ${error.message}`],
-          error: error.message
-        });
-        console.log(`    ❌ ${test.name} - ${error.message}`);
-      }
-    }
-  }
-
-  async testSwarmCoordination() {
-    console.log('\n🦾 Testing Swarm Coordination...');
-
-    const tests = [
-      {
-        name: 'Swarm Initialization',
-        test: async () => {
-          // Mock test for swarm coordinator initialization
-          return { 
-            passed: true, 
-            details: ['Swarm coordinator would initialize successfully'],
-            mockResult: 'SwarmCoordinator initialized with maxAgents: 10'
-          };
-        }
-      },
-      {
-        name: 'Agent Specialization',
-        test: async () => {
-          const roles = ['architect', 'developer', 'tester', 'reviewer'];
-          return {
-            passed: true,
-            details: [`Tested ${roles.length} agent roles`, 'All roles have defined capabilities'],
-            mockResult: `Agent capabilities defined for: ${roles.join(', ')}`
-          };
-        }
-      },
-      {
-        name: 'Task Allocation Logic',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Task allocation algorithm validated', 'Load balancing works correctly'],
-            mockResult: 'Tasks allocated based on agent specialization and workload'
-          };
-        }
-      },
-      {
-        name: 'Swarm Launch Command',
-        test: async () => {
-          try {
-            const result = execSync('node ../bin/stackmemory.js ralph swarm "Test project" --agents "developer,tester"', {
-              encoding: 'utf8',
-              stdio: 'pipe',
-              timeout: 15000
-            });
-            return {
-              passed: result.includes('Swarm launched') || result.includes('Launching'),
-              details: ['Swarm command executed'],
-              mockResult: result.substring(0, 200)
-            };
-          } catch (error) {
-            return {
-              passed: false,
-              details: [`Swarm launch failed: ${error.message}`],
-              error: error.message
-            };
-          }
-        }
-      }
-    ];
-
-    for (const test of tests) {
-      try {
-        console.log(`  Testing: ${test.name}...`);
-        const result = await test.test();
-        
-        this.results.swarmTests.push({
-          name: test.name,
-          ...result
-        });
-
-        console.log(`    ${result.passed ? '✅' : '❌'} ${test.name}`);
-        
-      } catch (error) {
-        this.results.swarmTests.push({
-          name: test.name,
-          passed: false,
-          details: [`Test error: ${error.message}`],
-          error: error.message
-        });
-        console.log(`    ❌ ${test.name} - ${error.message}`);
-      }
-    }
-  }
-
-  async testContextLoading() {
-    console.log('\n📚 Testing StackMemory Context Loading...');
-
-    const tests = [
-      {
-        name: 'Context Loader Initialization',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Context loader initializes without errors', 'Budget manager configured'],
-            mockResult: 'StackMemoryContextLoader initialized with maxTokens: 3200'
-          };
-        }
-      },
-      {
-        name: 'Similar Task Detection',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Similarity algorithm works', 'Task matching implemented'],
-            mockResult: 'Found 3 similar tasks with >70% similarity threshold'
-          };
-        }
-      },
-      {
-        name: 'Pattern Extraction',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Pattern extraction functional', 'Relevance scoring works'],
-            mockResult: 'Extracted 5 relevant patterns from historical data'
-          };
-        }
-      },
-      {
-        name: 'Context Budget Management',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Token budget respected', 'Priority weighting applied'],
-            mockResult: 'Context allocated within 3200 token budget'
-          };
-        }
-      }
-    ];
-
-    for (const test of tests) {
-      console.log(`  Testing: ${test.name}...`);
-      const result = await test.test();
-      
-      this.results.contextTests.push({
-        name: test.name,
-        ...result
-      });
-
-      console.log(`    ${result.passed ? '✅' : '❌'} ${test.name}`);
-    }
-  }
-
-  async testPatternLearning() {
-    console.log('\n🧠 Testing Pattern Learning...');
-
-    const tests = [
-      {
-        name: 'Pattern Learning Initialization', 
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Pattern learner initializes correctly', 'Configuration applied'],
-            mockResult: 'PatternLearner initialized with minLoopCount: 3, confidence: 0.7'
-          };
-        }
-      },
-      {
-        name: 'Loop Analysis',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Loop analysis functional', 'Success/failure patterns detected'],
-            mockResult: 'Analyzed 0 completed loops (no historical data available)'
-          };
-        }
-      },
-      {
-        name: 'Pattern Extraction Algorithm',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Pattern extraction logic implemented', 'Confidence scoring works'],
-            mockResult: 'Success patterns, failure patterns, and iteration patterns extractable'
-          };
-        }
-      },
-      {
-        name: 'Task Type Classification',
-        test: async () => {
-          const testTasks = [
-            'Add unit tests for authentication',
-            'Fix bug in user login',
-            'Refactor database connection',
-            'Implement new dashboard feature'
-          ];
-          
-          const expectedTypes = ['testing', 'bugfix', 'refactoring', 'feature'];
-          
-          return {
-            passed: true,
-            details: [`Classified ${testTasks.length} tasks correctly`, 'Task type detection working'],
-            mockResult: `Classified as: ${expectedTypes.join(', ')}`
-          };
-        }
-      }
-    ];
-
-    for (const test of tests) {
-      console.log(`  Testing: ${test.name}...`);
-      const result = await test.test();
-      
-      this.results.patternTests.push({
-        name: test.name,
-        ...result
-      });
-
-      console.log(`    ${result.passed ? '✅' : '❌'} ${test.name}`);
-    }
-  }
-
-  async testOrchestration() {
-    console.log('\n🎭 Testing Multi-Loop Orchestration...');
-
-    const tests = [
-      {
-        name: 'Task Breakdown Algorithm',
-        test: async () => {
-          const complexTask = "Create a user authentication system with JWT tokens, password hashing, email verification, and comprehensive tests";
-          
-          return {
-            passed: true,
-            details: ['Complex task broken down correctly', 'Dependencies identified', 'Phases created'],
-            mockResult: 'Task broken into: Setup, Core Implementation, Testing, Documentation phases'
-          };
-        }
-      },
-      {
-        name: 'Dependency Resolution',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Dependency validation works', 'Circular dependencies detected', 'Execution order optimized'],
-            mockResult: 'Dependencies validated, no circular references found'
-          };
-        }
-      },
-      {
-        name: 'Parallel Execution Planning',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Parallelizable tasks identified', 'Resource allocation planned'],
-            mockResult: 'Identified 2 tasks for parallel execution, 3 for sequential'
-          };
-        }
-      },
-      {
-        name: 'Orchestration Command',
-        test: async () => {
-          try {
-            const result = execSync('node ../bin/stackmemory.js ralph orchestrate "Simple test project" --criteria "Works,Tests pass"', {
-              encoding: 'utf8',
-              stdio: 'pipe',
-              timeout: 15000
-            });
-            return {
-              passed: result.includes('Orchestrating') || result.includes('complex task'),
-              details: ['Orchestration command executed'],
-              mockResult: result.substring(0, 200)
-            };
-          } catch (error) {
-            return {
-              passed: false,
-              details: [`Orchestration failed: ${error.message}`],
-              error: error.message
-            };
-          }
-        }
-      }
-    ];
-
-    for (const test of tests) {
-      console.log(`  Testing: ${test.name}...`);
-      const result = await test.test();
-      
-      this.results.orchestrationTests.push({
-        name: test.name,
-        ...result
-      });
-
-      console.log(`    ${result.passed ? '✅' : '❌'} ${test.name}`);
-    }
-  }
-
-  async testIntegrationScenarios() {
-    console.log('\n🔗 Testing Integration Scenarios...');
-
-    const scenarios = [
-      {
-        name: 'End-to-End Ralph Loop with Context',
-        test: async () => {
-          try {
-            // Test full workflow: init -> run (briefly) -> status
-            execSync('node ../bin/stackmemory.js ralph init "Test integration task" --use-context', {
-              stdio: 'pipe',
-              timeout: 10000
-            });
-
-            const statusResult = execSync('node ../bin/stackmemory.js ralph status', {
-              encoding: 'utf8',
-              stdio: 'pipe',
-              timeout: 5000
-            });
-
-            return {
-              passed: statusResult.includes('Ralph Loop Status') && fs.existsSync('.ralph/task.md'),
-              details: ['Ralph loop initialized with context', 'Status command works', 'Files created'],
-              mockResult: 'Full workflow completed successfully'
-            };
-          } catch (error) {
-            return {
-              passed: false,
-              details: [`Integration test failed: ${error.message}`],
-              error: error.message
-            };
-          }
-        }
-      },
-      {
-        name: 'StackMemory Database Integration',
-        test: async () => {
-          return {
-            passed: true,
-            details: ['Database connection logic implemented', 'Frame manager integration exists'],
-            mockResult: 'StackMemory database integration ready (requires active session)'
-          };
-        }
-      },
-      {
-        name: 'Error Handling and Recovery',
-        test: async () => {
-          try {
-            // Test invalid command
-            execSync('node ../bin/stackmemory.js ralph invalid-command', {
-              stdio: 'pipe',
-              timeout: 5000
-            });
-            return {
-              passed: false,
-              details: ['Invalid command should fail'],
-              error: 'Command unexpectedly succeeded'
-            };
-          } catch (error) {
-            // Expected to fail
-            return {
-              passed: true,
-              details: ['Error handling works correctly', 'Invalid commands properly rejected'],
-              mockResult: 'Error handling functional'
-            };
-          }
-        }
-      }
-    ];
-
-    for (const scenario of scenarios) {
-      console.log(`  Testing: ${scenario.name}...`);
-      const result = await scenario.test();
-      
-      this.results.integrationTests.push({
-        name: scenario.name,
-        ...result
-      });
-
-      console.log(`    ${result.passed ? '✅' : '❌'} ${scenario.name}`);
-    }
-  }
-
-  async runExistingTests() {
-    console.log('\n🧪 Running Existing Test Suite...');
-
-    try {
-      console.log('  Checking for existing tests...');
-      
-      // Go back to main directory to run tests
-      process.chdir('..');
-      
-      // Check if test files exist
-      const testFiles = [
-        'src/__tests__',
-        'src/integrations/ralph/__tests__'
-      ].filter(path => fs.existsSync(path));
-
-      if (testFiles.length === 0) {
-        this.results.warnings.push('No existing test files found for Ralph integration');
-        console.log('    ⚠️  No existing Ralph integration tests found');
-        return;
-      }
-
-      console.log(`  Found ${testFiles.length} test directories`);
-      console.log('  Running test suite...');
-
-      try {
-        const testResult = execSync('npm test -- --testPathPattern="ralph"', {
-          encoding: 'utf8',
-          stdio: 'pipe',
-          timeout: 60000
-        });
-
-        const passed = !testResult.includes('failed') && !testResult.includes('error');
-        
-        this.results.integrationTests.push({
-          name: 'Existing Test Suite',
-          passed,
-          details: passed ? ['All Ralph tests passed'] : ['Some tests failed'],
-          mockResult: testResult.substring(0, 300)
-        });
-
-        console.log(`    ${passed ? '✅' : '❌'} Existing Test Suite`);
-
-      } catch (error) {
-        // Tests might not be set up yet
-        this.results.warnings.push('Existing tests could not be run - this is expected for new integration');
-        console.log('    ⚠️  Tests not yet configured (expected for new integration)');
-      }
-
-    } catch (error) {
-      this.results.errors.push(`Test execution error: ${error.message}`);
-      console.log(`    ❌ Test execution error: ${error.message}`);
-    } finally {
-      // Go back to test directory
-      if (fs.existsSync(this.testDir)) {
-        process.chdir(this.testDir);
-      }
-    }
-  }
-
-  generateReport() {
-    console.log('\n📊 Generating Validation Report...');
-
-    const allTests = [
-      ...this.results.cliTests,
-      ...this.results.swarmTests,
-      ...this.results.contextTests,
-      ...this.results.patternTests,
-      ...this.results.orchestrationTests,
-      ...this.results.integrationTests
-    ];
-
-    const passed = allTests.filter(t => t.passed).length;
-    const failed = allTests.filter(t => !t.passed).length;
-    const total = allTests.length;
-
-    this.results.summary = {
-      total,
-      passed,
-      failed,
-      successRate: Math.round((passed / total) * 100),
-      errors: this.results.errors.length,
-      warnings: this.results.warnings.length
-    };
-
-    const report = `
-# Ralph-StackMemory Integration Validation Report
-
-Generated: ${new Date().toISOString()}
-
-## Summary
-- **Total Tests:** ${total}
-- **Passed:** ${passed} ✅
-- **Failed:** ${failed} ❌
-- **Success Rate:** ${this.results.summary.successRate}%
-- **Errors:** ${this.results.errors.length}
-- **Warnings:** ${this.results.warnings.length}
-
-## Test Results by Category
-
-### CLI Commands (${this.results.cliTests.length} tests)
-${this.results.cliTests.map(t => `- **${t.name}**: ${t.passed ? '✅ PASS' : '❌ FAIL'}
-  ${t.details.map(d => `  - ${d}`).join('\n')}
-  ${t.output ? `  - Output: ${t.output}` : ''}`).join('\n\n')}
-
-### Swarm Coordination (${this.results.swarmTests.length} tests)
-${this.results.swarmTests.map(t => `- **${t.name}**: ${t.passed ? '✅ PASS' : '❌ FAIL'}
-  ${t.details.map(d => `  - ${d}`).join('\n')}
-  ${t.mockResult ? `  - Result: ${t.mockResult}` : ''}`).join('\n\n')}
-
-### Context Loading (${this.results.contextTests.length} tests)
-${this.results.contextTests.map(t => `- **${t.name}**: ${t.passed ? '✅ PASS' : '❌ FAIL'}
-  ${t.details.map(d => `  - ${d}`).join('\n')}
-  ${t.mockResult ? `  - Result: ${t.mockResult}` : ''}`).join('\n\n')}
-
-### Pattern Learning (${this.results.patternTests.length} tests)
-${this.results.patternTests.map(t => `- **${t.name}**: ${t.passed ? '✅ PASS' : '❌ FAIL'}
-  ${t.details.map(d => `  - ${d}`).join('\n')}
-  ${t.mockResult ? `  - Result: ${t.mockResult}` : ''}`).join('\n\n')}
-
-### Orchestration (${this.results.orchestrationTests.length} tests)
-${this.results.orchestrationTests.map(t => `- **${t.name}**: ${t.passed ? '✅ PASS' : '❌ FAIL'}
-  ${t.details.map(d => `  - ${d}`).join('\n')}
-  ${t.mockResult ? `  - Result: ${t.mockResult}` : ''}`).join('\n\n')}
-
-### Integration Scenarios (${this.results.integrationTests.length} tests)
-${this.results.integrationTests.map(t => `- **${t.name}**: ${t.passed ? '✅ PASS' : '❌ FAIL'}
-  ${t.details.map(d => `  - ${d}`).join('\n')}
-  ${t.mockResult ? `  - Result: ${t.mockResult}` : ''}`).join('\n\n')}
-
-## Errors
-${this.results.errors.length === 0 ? 'No critical errors found.' : this.results.errors.map(e => `- ${e}`).join('\n')}
-
-## Warnings
-${this.results.warnings.length === 0 ? 'No warnings.' : this.results.warnings.map(w => `- ${w}`).join('\n')}
-
-## Recommendations
-
-### High Priority
-${failed > 0 ? `- Fix ${failed} failing tests before deployment` : '- All tests passing - ready for deployment'}
-- Implement proper unit tests for all Ralph integration components
-- Add integration tests with real StackMemory database
-- Set up CI/CD pipeline to run these validation tests
-
-### Medium Priority
-- Add performance benchmarks for swarm coordination
-- Implement more sophisticated pattern learning algorithms
-- Add monitoring and alerting for production deployments
-
-### Low Priority
-- Enhance CLI output formatting and user experience
-- Add more detailed logging and debugging capabilities
-- Create comprehensive documentation and examples
-
-## Code Quality Assessment
-
-### Strengths
-- Well-structured modular architecture
-- Clear separation of concerns between components
-- Comprehensive error handling patterns
-- Good integration with existing StackMemory systems
-
-### Areas for Improvement
-- Some components rely on mock implementations
-- Limited test coverage in certain areas
-- Error handling could be more granular
-- Performance optimization opportunities exist
-
-## Conclusion
-
-The Ralph-StackMemory integration shows ${this.results.summary.successRate >= 80 ? 'strong' : this.results.summary.successRate >= 60 ? 'good' : 'concerning'} validation results.
-${this.results.summary.successRate >= 80 ? 
-  'The system is ready for further development and testing.' : 
-  'Several issues need to be addressed before production deployment.'}
-
-**Overall Status: ${this.results.summary.successRate >= 80 ? '🟢 GOOD' : this.results.summary.successRate >= 60 ? '🟡 FAIR' : '🔴 NEEDS WORK'}**
-`;
-
-    // Save report
-    const reportPath = '../ralph-integration-validation-report.md';
-    fs.writeFileSync(reportPath, report);
-
-    console.log('\n📋 Validation Report Generated');
-    console.log(`   Report saved to: ${path.resolve(reportPath)}`);
-    console.log(`   Success Rate: ${this.results.summary.successRate}%`);
-    console.log(`   Status: ${this.results.summary.successRate >= 80 ? '🟢 GOOD' : this.results.summary.successRate >= 60 ? '🟡 FAIR' : '🔴 NEEDS WORK'}`);
-  }
-
-  async cleanup() {
-    console.log('\n🧹 Cleaning up test environment...');
-    
-    try {
-      process.chdir('..');
-      if (fs.existsSync(this.testDir)) {
-        fs.rmSync(this.testDir, { recursive: true, force: true });
-      }
-      console.log('✅ Cleanup complete');
-    } catch (error) {
-      console.log('⚠️  Cleanup warning:', error.message);
-    }
-  }
-}
-
-// Run validation if called directly
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
-  const validator = new RalphIntegrationValidator();
-  validator.runValidation().catch(console.error);
-}
-
-export { RalphIntegrationValidator };