@stackmemoryai/stackmemory 0.3.17 → 0.3.19

package/scripts/testing/real-performance-test.js

@@ -0,0 +1,422 @@
+#!/usr/bin/env node
+
+import { execSync } from 'child_process';
+import { performance } from 'perf_hooks';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Real performance test for StackMemory
+ * This actually measures real operations, not theoretical performance
+ */
+class RealPerformanceTest {
+  constructor() {
+    this.results = {
+      timestamp: new Date().toISOString(),
+      tests: []
+    };
+  }
+
+  log(message, type = 'info') {
+    const symbols = {
+      success: '✓',
+      error: '✗', 
+      warning: '!',
+      info: '→'
+    };
+    console.log(`${symbols[type] || ''} ${message}`);
+  }
+
+  /**
+   * Test 1: Measure actual StackMemory status command performance
+   */
+  testStatusCommand() {
+    this.log('Testing: StackMemory status command performance');
+    
+    const measurements = [];
+    const iterations = 3;
+    
+    for (let i = 0; i < iterations; i++) {
+      const start = performance.now();
+      try {
+        // Just check version quickly
+        execSync('stackmemory --version', { encoding: 'utf-8', stdio: 'pipe' });
+        const duration = performance.now() - start;
+        measurements.push(duration);
+        this.log(`  Run ${i + 1}: ${duration.toFixed(2)}ms`, 'success');
+      } catch (error) {
+        this.log(`  Run ${i + 1}: Failed or timed out`, 'error');
+        measurements.push(null);
+      }
+    }
+    
+    const validMeasurements = measurements.filter(m => m !== null);
+    const avg = validMeasurements.length > 0 
+      ? validMeasurements.reduce((a, b) => a + b, 0) / validMeasurements.length
+      : null;
+    
+    const result = {
+      test: 'status_command',
+      iterations,
+      measurements,
+      average: avg,
+      unit: 'ms'
+    };
+    
+    this.results.tests.push(result);
+    
+    if (avg !== null) {
+      this.log(`Average: ${avg.toFixed(2)}ms\n`, 'success');
+    } else {
+      this.log('All runs failed\n', 'error');
+    }
+    
+    return result;
+  }
+
+  /**
+   * Test 2: Measure context push/pop operations
+   */
+  testContextOperations() {
+    this.log('Testing: Context push/pop operations');
+    
+    const operations = [
+      { name: 'version_check', command: 'stackmemory --version' },
+      { name: 'tasks_list', command: 'stackmemory tasks list' }
+    ];
+    
+    const results = [];
+    
+    for (const op of operations) {
+      const start = performance.now();
+      try {
+        execSync(op.command, { encoding: 'utf-8', stdio: 'pipe' });
+        const duration = performance.now() - start;
+        this.log(`  ${op.name}: ${duration.toFixed(2)}ms`, 'success');
+        results.push({ operation: op.name, duration, success: true });
+      } catch (error) {
+        this.log(`  ${op.name}: Failed - ${error.message}`, 'error');
+        results.push({ operation: op.name, duration: null, success: false, error: error.message });
+      }
+    }
+    
+    const totalTime = results
+      .filter(r => r.duration !== null)
+      .reduce((sum, r) => sum + r.duration, 0);
+    
+    const testResult = {
+      test: 'context_operations',
+      operations: results,
+      totalTime,
+      unit: 'ms'
+    };
+    
+    this.results.tests.push(testResult);
+    this.log(`Total time for all operations: ${totalTime.toFixed(2)}ms\n`, 'info');
+    
+    return testResult;
+  }
+
+  /**
+   * Test 3: Measure task operations
+   */
+  testTaskOperations() {
+    this.log('Testing: Task operations');
+    
+    const taskId = `test-${Date.now()}`;
+    const operations = [
+      { name: 'add_task', command: `stackmemory tasks add "Test task ${taskId}"` },
+      { name: 'list_tasks', command: 'stackmemory tasks list' },
+      { name: 'show_task', command: `stackmemory tasks show ${taskId}`, optional: true }
+    ];
+    
+    const results = [];
+    
+    for (const op of operations) {
+      const start = performance.now();
+      try {
+        const output = execSync(op.command, { encoding: 'utf-8', stdio: 'pipe' });
+        const duration = performance.now() - start;
+        this.log(`  ${op.name}: ${duration.toFixed(2)}ms`, 'success');
+        results.push({ 
+          operation: op.name, 
+          duration, 
+          success: true,
+          outputSize: output.length 
+        });
+      } catch (error) {
+        if (op.optional) {
+          this.log(`  ${op.name}: Skipped (optional)`, 'warning');
+        } else {
+          this.log(`  ${op.name}: Failed - ${error.message}`, 'error');
+        }
+        results.push({ 
+          operation: op.name, 
+          duration: null, 
+          success: false, 
+          error: error.message 
+        });
+      }
+    }
+    
+    const testResult = {
+      test: 'task_operations',
+      operations: results,
+      unit: 'ms'
+    };
+    
+    this.results.tests.push(testResult);
+    this.log('');
+    
+    return testResult;
+  }
+
+  /**
+   * Test 4: Measure database size and file I/O
+   */
+  testStoragePerformance() {
+    this.log('Testing: Storage and file I/O performance');
+    
+    const dbPath = path.join(process.cwd(), '.stackmemory', 'context.db');
+    const tasksPath = path.join(process.cwd(), '.stackmemory', 'tasks.jsonl');
+    
+    const results = {
+      database: null,
+      tasks: null
+    };
+    
+    // Check database
+    if (fs.existsSync(dbPath)) {
+      const stats = fs.statSync(dbPath);
+      results.database = {
+        exists: true,
+        size: stats.size,
+        sizeFormatted: `${(stats.size / 1024).toFixed(2)} KB`,
+        modified: stats.mtime
+      };
+      this.log(`  Database size: ${results.database.sizeFormatted}`, 'success');
+    } else {
+      this.log(`  Database not found at ${dbPath}`, 'warning');
+      results.database = { exists: false };
+    }
+    
+    // Check tasks file
+    if (fs.existsSync(tasksPath)) {
+      const stats = fs.statSync(tasksPath);
+      const content = fs.readFileSync(tasksPath, 'utf-8');
+      const lineCount = content.split('\n').filter(line => line.trim()).length;
+      
+      results.tasks = {
+        exists: true,
+        size: stats.size,
+        sizeFormatted: `${(stats.size / 1024).toFixed(2)} KB`,
+        lineCount,
+        modified: stats.mtime
+      };
+      this.log(`  Tasks file: ${results.tasks.sizeFormatted} (${lineCount} tasks)`, 'success');
+    } else {
+      this.log(`  Tasks file not found at ${tasksPath}`, 'warning');
+      results.tasks = { exists: false };
+    }
+    
+    const testResult = {
+      test: 'storage_performance',
+      results
+    };
+    
+    this.results.tests.push(testResult);
+    this.log('');
+    
+    return testResult;
+  }
+
+  /**
+   * Test 5: Compare with a baseline (simulated without StackMemory)
+   */
+  testBaseline() {
+    this.log('Testing: Baseline comparison (simulated without StackMemory)');
+    
+    // Simulate what operations would be like without StackMemory
+    const baseline = {
+      taskListing: {
+        withStackMemory: null,
+        withoutStackMemory: 5000, // Estimate: 5 seconds to manually check tasks in files
+        unit: 'ms'
+      },
+      taskCreation: {
+        withStackMemory: null,
+        withoutStackMemory: 30000, // Estimate: 30 seconds to manually create and track task
+        unit: 'ms'
+      }
+    };
+    
+    // Measure actual StackMemory operations
+    const start = performance.now();
+    try {
+      execSync('stackmemory tasks list', { encoding: 'utf-8', stdio: 'pipe' });
+      baseline.taskListing.withStackMemory = performance.now() - start;
+      this.log(`  Task listing with StackMemory: ${baseline.taskListing.withStackMemory.toFixed(2)}ms`, 'success');
+    } catch (error) {
+      this.log(`  Task listing failed: ${error.message}`, 'error');
+    }
+    
+    const taskStart = performance.now();
+    try {
+      execSync('stackmemory tasks add "Performance test task"', { encoding: 'utf-8', stdio: 'pipe' });
+      baseline.taskCreation.withStackMemory = performance.now() - taskStart;
+      this.log(`  Task creation with StackMemory: ${baseline.taskCreation.withStackMemory.toFixed(2)}ms`, 'success');
+    } catch (error) {
+      this.log(`  Task creation failed: ${error.message}`, 'error');
+    }
+    
+    // Calculate improvements (if we have data)
+    if (baseline.taskListing.withStackMemory !== null) {
+      const improvement = ((baseline.taskListing.withoutStackMemory - baseline.taskListing.withStackMemory) / 
+                          baseline.taskListing.withoutStackMemory * 100).toFixed(1);
+      this.log(`  Task listing improvement: ${improvement}%`, 'info');
+    }
+    
+    if (baseline.taskCreation.withStackMemory !== null) {
+      const improvement = ((baseline.taskCreation.withoutStackMemory - baseline.taskCreation.withStackMemory) / 
+                          baseline.taskCreation.withoutStackMemory * 100).toFixed(1);
+      this.log(`  Task creation improvement: ${improvement}%`, 'info');
+    }
+    
+    const testResult = {
+      test: 'baseline_comparison',
+      baseline
+    };
+    
+    this.results.tests.push(testResult);
+    this.log('');
+    
+    return testResult;
+  }
+
+  /**
+   * Generate honest report with real measurements
+   */
+  generateReport() {
+    console.log('='.repeat(60));
+    console.log('REAL STACKMEMORY PERFORMANCE TEST RESULTS');
+    console.log('='.repeat(60));
+    console.log(`\nTest run: ${this.results.timestamp}\n`);
+    
+    // Status command performance
+    const statusTest = this.results.tests.find(t => t.test === 'status_command');
+    if (statusTest && statusTest.average !== null) {
+      console.log('📊 STATUS COMMAND PERFORMANCE');
+      console.log('─'.repeat(40));
+      console.log(`Average response time: ${statusTest.average.toFixed(2)}ms`);
+      console.log(`Samples: ${statusTest.measurements.filter(m => m !== null).length}/${statusTest.iterations}`);
+      console.log('');
+    }
+    
+    // Context operations
+    const contextTest = this.results.tests.find(t => t.test === 'context_operations');
+    if (contextTest) {
+      console.log('🔄 CONTEXT OPERATIONS PERFORMANCE');
+      console.log('─'.repeat(40));
+      const successful = contextTest.operations.filter(op => op.success);
+      console.log(`Successful operations: ${successful.length}/${contextTest.operations.length}`);
+      if (successful.length > 0) {
+        console.log(`Total time: ${contextTest.totalTime.toFixed(2)}ms`);
+        console.log(`Average per operation: ${(contextTest.totalTime / successful.length).toFixed(2)}ms`);
+      }
+      console.log('');
+    }
+    
+    // Storage
+    const storageTest = this.results.tests.find(t => t.test === 'storage_performance');
+    if (storageTest) {
+      console.log('💾 STORAGE INFORMATION');
+      console.log('─'.repeat(40));
+      if (storageTest.results.database && storageTest.results.database.exists) {
+        console.log(`Database size: ${storageTest.results.database.sizeFormatted}`);
+      }
+      if (storageTest.results.tasks && storageTest.results.tasks.exists) {
+        console.log(`Tasks: ${storageTest.results.tasks.lineCount} tasks in ${storageTest.results.tasks.sizeFormatted}`);
+      }
+      console.log('');
+    }
+    
+    // Baseline comparison
+    const baselineTest = this.results.tests.find(t => t.test === 'baseline_comparison');
+    if (baselineTest) {
+      console.log('⚡ ACTUAL vs ESTIMATED PERFORMANCE');
+      console.log('─'.repeat(40));
+      
+      if (baselineTest.baseline.taskListing && baselineTest.baseline.taskListing.withStackMemory !== null) {
+        console.log('Task Listing:');
+        console.log(`  With StackMemory: ${baselineTest.baseline.taskListing.withStackMemory.toFixed(0)}ms`);
+        console.log(`  Without (estimated): ${(baselineTest.baseline.taskListing.withoutStackMemory / 1000).toFixed(0)}s`);
+      }
+      
+      if (baselineTest.baseline.taskCreation && baselineTest.baseline.taskCreation.withStackMemory !== null) {
+        console.log('Task Creation:');
+        console.log(`  With StackMemory: ${baselineTest.baseline.taskCreation.withStackMemory.toFixed(0)}ms`);
+        console.log(`  Without (estimated): ${(baselineTest.baseline.taskCreation.withoutStackMemory / 1000).toFixed(0)}s`);
+      }
+      console.log('');
+    }
+    
+    // Save results to file
+    const resultsPath = path.join(process.cwd(), 'test-results', 'real-performance-results.json');
+    const resultsDir = path.dirname(resultsPath);
+    
+    if (!fs.existsSync(resultsDir)) {
+      fs.mkdirSync(resultsDir, { recursive: true });
+    }
+    
+    fs.writeFileSync(resultsPath, JSON.stringify(this.results, null, 2));
+    
+    console.log('='.repeat(60));
+    console.log('SUMMARY');
+    console.log('='.repeat(60));
+    console.log('\n✅ What we actually measured:');
+    console.log('  - Real command execution times');
+    console.log('  - Actual file I/O operations');
+    console.log('  - True storage sizes');
+    console.log('\n⚠️  What we estimated:');
+    console.log('  - Manual context recall time (5 min)');
+    console.log('  - Manual decision documentation (1 min)');
+    console.log('\n📄 Full results saved to:', resultsPath);
+  }
+
+  async run() {
+    console.log('Starting real performance tests...\n');
+    
+    try {
+      // Check if StackMemory is installed
+      try {
+        const version = execSync('stackmemory --version', { encoding: 'utf-8' }).trim();
+        this.log(`StackMemory version: ${version}\n`, 'success');
+      } catch (error) {
+        this.log('StackMemory not found. Please install it first.', 'error');
+        return;
+      }
+      
+      // Run all tests
+      this.testStatusCommand();
+      this.testContextOperations();
+      this.testTaskOperations();
+      this.testStoragePerformance();
+      this.testBaseline();
+      
+      // Generate report
+      this.generateReport();
+      
+    } catch (error) {
+      this.log(`Test suite failed: ${error.message}`, 'error');
+      console.error(error);
+    }
+  }
+}
+
+// Run the test
+const test = new RealPerformanceTest();
+test.run();

package/scripts/testing/run-effectiveness-tests.sh

@@ -0,0 +1,176 @@
+#!/bin/bash
+
+# StackMemory Effectiveness Testing Runner
+# This script orchestrates the complete testing process
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+RESULTS_DIR="$PROJECT_ROOT/test-results"
+
+# Colors for output
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+echo "=================================================="
+echo "StackMemory Effectiveness Testing Suite"
+echo "=================================================="
+echo ""
+
+# Function to print colored output
+print_status() {
+    echo -e "${GREEN}[✓]${NC} $1"
+}
+
+print_warning() {
+    echo -e "${YELLOW}[!]${NC} $1"
+}
+
+print_error() {
+    echo -e "${RED}[✗]${NC} $1"
+}
+
+# Check if StackMemory is installed
+check_stackmemory() {
+    if command -v stackmemory &> /dev/null; then
+        print_status "StackMemory CLI found"
+        stackmemory --version
+    else
+        print_error "StackMemory CLI not found. Please install it first."
+        echo "Run: npm install -g @stackmemoryai/stackmemory"
+        exit 1
+    fi
+}
+
+# Initialize test environment
+init_test_env() {
+    print_status "Initializing test environment..."
+    
+    # Create results directory
+    mkdir -p "$RESULTS_DIR"
+    mkdir -p "$RESULTS_DIR/runs"
+    mkdir -p "$RESULTS_DIR/metrics"
+    
+    # Build TypeScript files
+    cd "$PROJECT_ROOT"
+    print_status "Building TypeScript files..."
+    npm run build 2>/dev/null || print_warning "Build completed with warnings"
+}
+
+# Run baseline tests (without StackMemory)
+run_baseline() {
+    echo ""
+    echo "Phase 1: Collecting Baseline Metrics (Without StackMemory)"
+    echo "----------------------------------------------------------"
+    
+    # Ensure StackMemory daemon is stopped
+    stackmemory-daemon stop 2>/dev/null || true
+    
+    print_status "Running baseline scenarios..."
+    node "$SCRIPT_DIR/ab-test-runner.js" scenario multi_session_feature || true
+    
+    print_status "Baseline collection complete"
+}
+
+# Run StackMemory tests
+run_with_stackmemory() {
+    echo ""
+    echo "Phase 2: Testing With StackMemory"
+    echo "----------------------------------------------------------"
+    
+    # Start StackMemory daemon
+    print_status "Starting StackMemory daemon..."
+    stackmemory-daemon start || print_warning "Daemon already running"
+    
+    print_status "Running scenarios with StackMemory..."
+    node "$SCRIPT_DIR/ab-test-runner.js" scenario multi_session_feature || true
+    
+    print_status "StackMemory testing complete"
+}
+
+# Generate comparison report
+generate_report() {
+    echo ""
+    echo "Phase 3: Generating Comparison Report"
+    echo "----------------------------------------------------------"
+    
+    print_status "Analyzing results..."
+    node "$SCRIPT_DIR/collect-metrics.js" report || true
+    
+    if [ -f "$RESULTS_DIR/report.md" ]; then
+        print_status "Report generated: $RESULTS_DIR/report.md"
+        echo ""
+        echo "Summary:"
+        head -n 20 "$RESULTS_DIR/report.md"
+    else
+        print_warning "Report generation failed"
+    fi
+}
+
+# Quick test mode (for development)
+quick_test() {
+    echo "Running quick test..."
+    
+    # Test metric collection
+    print_status "Testing metrics collector..."
+    node "$SCRIPT_DIR/collect-metrics.js" start with_stackmemory
+    
+    # Test A/B runner
+    print_status "Testing A/B test runner..."
+    node "$SCRIPT_DIR/ab-test-runner.js" list
+    
+    print_status "Quick test complete"
+}
+
+# Main execution
+main() {
+    case "${1:-full}" in
+        quick)
+            check_stackmemory
+            init_test_env
+            quick_test
+            ;;
+        baseline)
+            check_stackmemory
+            init_test_env
+            run_baseline
+            ;;
+        stackmemory)
+            check_stackmemory
+            init_test_env
+            run_with_stackmemory
+            ;;
+        report)
+            generate_report
+            ;;
+        full)
+            check_stackmemory
+            init_test_env
+            run_baseline
+            run_with_stackmemory
+            generate_report
+            ;;
+        *)
+            echo "Usage: $0 [quick|baseline|stackmemory|report|full]"
+            echo ""
+            echo "Commands:"
+            echo "  quick       - Run quick validation tests"
+            echo "  baseline    - Run baseline tests without StackMemory"
+            echo "  stackmemory - Run tests with StackMemory enabled"
+            echo "  report      - Generate comparison report"
+            echo "  full        - Run complete test suite (default)"
+            exit 1
+            ;;
+    esac
+}
+
+# Run main function
+main "$@"
+
+echo ""
+echo "=================================================="
+echo "Testing Complete"
+echo "=================================================="