@stackmemoryai/stackmemory 0.8.1 → 1.0.0

package/scripts/testing/real-performance-test.js

@@ -1,422 +0,0 @@
-#!/usr/bin/env node
-
-import { execSync } from 'child_process';
-import { performance } from 'perf_hooks';
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-/**
- * Real performance test for StackMemory
- * This actually measures real operations, not theoretical performance
- */
-class RealPerformanceTest {
-  constructor() {
-    this.results = {
-      timestamp: new Date().toISOString(),
-      tests: []
-    };
-  }
-
-  log(message, type = 'info') {
-    const symbols = {
-      success: '✓',
-      error: '✗', 
-      warning: '!',
-      info: '→'
-    };
-    console.log(`${symbols[type] || ''} ${message}`);
-  }
-
-  /**
-   * Test 1: Measure actual StackMemory status command performance
-   */
-  testStatusCommand() {
-    this.log('Testing: StackMemory status command performance');
-    
-    const measurements = [];
-    const iterations = 3;
-    
-    for (let i = 0; i < iterations; i++) {
-      const start = performance.now();
-      try {
-        // Just check version quickly
-        execSync('stackmemory --version', { encoding: 'utf-8', stdio: 'pipe' });
-        const duration = performance.now() - start;
-        measurements.push(duration);
-        this.log(`  Run ${i + 1}: ${duration.toFixed(2)}ms`, 'success');
-      } catch (error) {
-        this.log(`  Run ${i + 1}: Failed or timed out`, 'error');
-        measurements.push(null);
-      }
-    }
-    
-    const validMeasurements = measurements.filter(m => m !== null);
-    const avg = validMeasurements.length > 0 
-      ? validMeasurements.reduce((a, b) => a + b, 0) / validMeasurements.length
-      : null;
-    
-    const result = {
-      test: 'status_command',
-      iterations,
-      measurements,
-      average: avg,
-      unit: 'ms'
-    };
-    
-    this.results.tests.push(result);
-    
-    if (avg !== null) {
-      this.log(`Average: ${avg.toFixed(2)}ms\n`, 'success');
-    } else {
-      this.log('All runs failed\n', 'error');
-    }
-    
-    return result;
-  }
-
-  /**
-   * Test 2: Measure context push/pop operations
-   */
-  testContextOperations() {
-    this.log('Testing: Context push/pop operations');
-    
-    const operations = [
-      { name: 'version_check', command: 'stackmemory --version' },
-      { name: 'tasks_list', command: 'stackmemory tasks list' }
-    ];
-    
-    const results = [];
-    
-    for (const op of operations) {
-      const start = performance.now();
-      try {
-        execSync(op.command, { encoding: 'utf-8', stdio: 'pipe' });
-        const duration = performance.now() - start;
-        this.log(`  ${op.name}: ${duration.toFixed(2)}ms`, 'success');
-        results.push({ operation: op.name, duration, success: true });
-      } catch (error) {
-        this.log(`  ${op.name}: Failed - ${error.message}`, 'error');
-        results.push({ operation: op.name, duration: null, success: false, error: error.message });
-      }
-    }
-    
-    const totalTime = results
-      .filter(r => r.duration !== null)
-      .reduce((sum, r) => sum + r.duration, 0);
-    
-    const testResult = {
-      test: 'context_operations',
-      operations: results,
-      totalTime,
-      unit: 'ms'
-    };
-    
-    this.results.tests.push(testResult);
-    this.log(`Total time for all operations: ${totalTime.toFixed(2)}ms\n`, 'info');
-    
-    return testResult;
-  }
-
-  /**
-   * Test 3: Measure task operations
-   */
-  testTaskOperations() {
-    this.log('Testing: Task operations');
-    
-    const taskId = `test-${Date.now()}`;
-    const operations = [
-      { name: 'add_task', command: `stackmemory tasks add "Test task ${taskId}"` },
-      { name: 'list_tasks', command: 'stackmemory tasks list' },
-      { name: 'show_task', command: `stackmemory tasks show ${taskId}`, optional: true }
-    ];
-    
-    const results = [];
-    
-    for (const op of operations) {
-      const start = performance.now();
-      try {
-        const output = execSync(op.command, { encoding: 'utf-8', stdio: 'pipe' });
-        const duration = performance.now() - start;
-        this.log(`  ${op.name}: ${duration.toFixed(2)}ms`, 'success');
-        results.push({ 
-          operation: op.name, 
-          duration, 
-          success: true,
-          outputSize: output.length 
-        });
-      } catch (error) {
-        if (op.optional) {
-          this.log(`  ${op.name}: Skipped (optional)`, 'warning');
-        } else {
-          this.log(`  ${op.name}: Failed - ${error.message}`, 'error');
-        }
-        results.push({ 
-          operation: op.name, 
-          duration: null, 
-          success: false, 
-          error: error.message 
-        });
-      }
-    }
-    
-    const testResult = {
-      test: 'task_operations',
-      operations: results,
-      unit: 'ms'
-    };
-    
-    this.results.tests.push(testResult);
-    this.log('');
-    
-    return testResult;
-  }
-
-  /**
-   * Test 4: Measure database size and file I/O
-   */
-  testStoragePerformance() {
-    this.log('Testing: Storage and file I/O performance');
-    
-    const dbPath = path.join(process.cwd(), '.stackmemory', 'context.db');
-    const tasksPath = path.join(process.cwd(), '.stackmemory', 'tasks.jsonl');
-    
-    const results = {
-      database: null,
-      tasks: null
-    };
-    
-    // Check database
-    if (fs.existsSync(dbPath)) {
-      const stats = fs.statSync(dbPath);
-      results.database = {
-        exists: true,
-        size: stats.size,
-        sizeFormatted: `${(stats.size / 1024).toFixed(2)} KB`,
-        modified: stats.mtime
-      };
-      this.log(`  Database size: ${results.database.sizeFormatted}`, 'success');
-    } else {
-      this.log(`  Database not found at ${dbPath}`, 'warning');
-      results.database = { exists: false };
-    }
-    
-    // Check tasks file
-    if (fs.existsSync(tasksPath)) {
-      const stats = fs.statSync(tasksPath);
-      const content = fs.readFileSync(tasksPath, 'utf-8');
-      const lineCount = content.split('\n').filter(line => line.trim()).length;
-      
-      results.tasks = {
-        exists: true,
-        size: stats.size,
-        sizeFormatted: `${(stats.size / 1024).toFixed(2)} KB`,
-        lineCount,
-        modified: stats.mtime
-      };
-      this.log(`  Tasks file: ${results.tasks.sizeFormatted} (${lineCount} tasks)`, 'success');
-    } else {
-      this.log(`  Tasks file not found at ${tasksPath}`, 'warning');
-      results.tasks = { exists: false };
-    }
-    
-    const testResult = {
-      test: 'storage_performance',
-      results
-    };
-    
-    this.results.tests.push(testResult);
-    this.log('');
-    
-    return testResult;
-  }
-
-  /**
-   * Test 5: Compare with a baseline (simulated without StackMemory)
-   */
-  testBaseline() {
-    this.log('Testing: Baseline comparison (simulated without StackMemory)');
-    
-    // Simulate what operations would be like without StackMemory
-    const baseline = {
-      taskListing: {
-        withStackMemory: null,
-        withoutStackMemory: 5000, // Estimate: 5 seconds to manually check tasks in files
-        unit: 'ms'
-      },
-      taskCreation: {
-        withStackMemory: null,
-        withoutStackMemory: 30000, // Estimate: 30 seconds to manually create and track task
-        unit: 'ms'
-      }
-    };
-    
-    // Measure actual StackMemory operations
-    const start = performance.now();
-    try {
-      execSync('stackmemory tasks list', { encoding: 'utf-8', stdio: 'pipe' });
-      baseline.taskListing.withStackMemory = performance.now() - start;
-      this.log(`  Task listing with StackMemory: ${baseline.taskListing.withStackMemory.toFixed(2)}ms`, 'success');
-    } catch (error) {
-      this.log(`  Task listing failed: ${error.message}`, 'error');
-    }
-    
-    const taskStart = performance.now();
-    try {
-      execSync('stackmemory tasks add "Performance test task"', { encoding: 'utf-8', stdio: 'pipe' });
-      baseline.taskCreation.withStackMemory = performance.now() - taskStart;
-      this.log(`  Task creation with StackMemory: ${baseline.taskCreation.withStackMemory.toFixed(2)}ms`, 'success');
-    } catch (error) {
-      this.log(`  Task creation failed: ${error.message}`, 'error');
-    }
-    
-    // Calculate improvements (if we have data)
-    if (baseline.taskListing.withStackMemory !== null) {
-      const improvement = ((baseline.taskListing.withoutStackMemory - baseline.taskListing.withStackMemory) / 
-                          baseline.taskListing.withoutStackMemory * 100).toFixed(1);
-      this.log(`  Task listing improvement: ${improvement}%`, 'info');
-    }
-    
-    if (baseline.taskCreation.withStackMemory !== null) {
-      const improvement = ((baseline.taskCreation.withoutStackMemory - baseline.taskCreation.withStackMemory) / 
-                          baseline.taskCreation.withoutStackMemory * 100).toFixed(1);
-      this.log(`  Task creation improvement: ${improvement}%`, 'info');
-    }
-    
-    const testResult = {
-      test: 'baseline_comparison',
-      baseline
-    };
-    
-    this.results.tests.push(testResult);
-    this.log('');
-    
-    return testResult;
-  }
-
-  /**
-   * Generate honest report with real measurements
-   */
-  generateReport() {
-    console.log('='.repeat(60));
-    console.log('REAL STACKMEMORY PERFORMANCE TEST RESULTS');
-    console.log('='.repeat(60));
-    console.log(`\nTest run: ${this.results.timestamp}\n`);
-    
-    // Status command performance
-    const statusTest = this.results.tests.find(t => t.test === 'status_command');
-    if (statusTest && statusTest.average !== null) {
-      console.log('📊 STATUS COMMAND PERFORMANCE');
-      console.log('─'.repeat(40));
-      console.log(`Average response time: ${statusTest.average.toFixed(2)}ms`);
-      console.log(`Samples: ${statusTest.measurements.filter(m => m !== null).length}/${statusTest.iterations}`);
-      console.log('');
-    }
-    
-    // Context operations
-    const contextTest = this.results.tests.find(t => t.test === 'context_operations');
-    if (contextTest) {
-      console.log('🔄 CONTEXT OPERATIONS PERFORMANCE');
-      console.log('─'.repeat(40));
-      const successful = contextTest.operations.filter(op => op.success);
-      console.log(`Successful operations: ${successful.length}/${contextTest.operations.length}`);
-      if (successful.length > 0) {
-        console.log(`Total time: ${contextTest.totalTime.toFixed(2)}ms`);
-        console.log(`Average per operation: ${(contextTest.totalTime / successful.length).toFixed(2)}ms`);
-      }
-      console.log('');
-    }
-    
-    // Storage
-    const storageTest = this.results.tests.find(t => t.test === 'storage_performance');
-    if (storageTest) {
-      console.log('💾 STORAGE INFORMATION');
-      console.log('─'.repeat(40));
-      if (storageTest.results.database && storageTest.results.database.exists) {
-        console.log(`Database size: ${storageTest.results.database.sizeFormatted}`);
-      }
-      if (storageTest.results.tasks && storageTest.results.tasks.exists) {
-        console.log(`Tasks: ${storageTest.results.tasks.lineCount} tasks in ${storageTest.results.tasks.sizeFormatted}`);
-      }
-      console.log('');
-    }
-    
-    // Baseline comparison
-    const baselineTest = this.results.tests.find(t => t.test === 'baseline_comparison');
-    if (baselineTest) {
-      console.log('⚡ ACTUAL vs ESTIMATED PERFORMANCE');
-      console.log('─'.repeat(40));
-      
-      if (baselineTest.baseline.taskListing && baselineTest.baseline.taskListing.withStackMemory !== null) {
-        console.log('Task Listing:');
-        console.log(`  With StackMemory: ${baselineTest.baseline.taskListing.withStackMemory.toFixed(0)}ms`);
-        console.log(`  Without (estimated): ${(baselineTest.baseline.taskListing.withoutStackMemory / 1000).toFixed(0)}s`);
-      }
-      
-      if (baselineTest.baseline.taskCreation && baselineTest.baseline.taskCreation.withStackMemory !== null) {
-        console.log('Task Creation:');
-        console.log(`  With StackMemory: ${baselineTest.baseline.taskCreation.withStackMemory.toFixed(0)}ms`);
-        console.log(`  Without (estimated): ${(baselineTest.baseline.taskCreation.withoutStackMemory / 1000).toFixed(0)}s`);
-      }
-      console.log('');
-    }
-    
-    // Save results to file
-    const resultsPath = path.join(process.cwd(), 'scripts', 'testing', 'results', 'real-performance-results.json');
-    const resultsDir = path.dirname(resultsPath);
-    
-    if (!fs.existsSync(resultsDir)) {
-      fs.mkdirSync(resultsDir, { recursive: true });
-    }
-    
-    fs.writeFileSync(resultsPath, JSON.stringify(this.results, null, 2));
-    
-    console.log('='.repeat(60));
-    console.log('SUMMARY');
-    console.log('='.repeat(60));
-    console.log('\n✅ What we actually measured:');
-    console.log('  - Real command execution times');
-    console.log('  - Actual file I/O operations');
-    console.log('  - True storage sizes');
-    console.log('\n⚠️  What we estimated:');
-    console.log('  - Manual context recall time (5 min)');
-    console.log('  - Manual decision documentation (1 min)');
-    console.log('\n📄 Full results saved to:', resultsPath);
-  }
-
-  async run() {
-    console.log('Starting real performance tests...\n');
-    
-    try {
-      // Check if StackMemory is installed
-      try {
-        const version = execSync('stackmemory --version', { encoding: 'utf-8' }).trim();
-        this.log(`StackMemory version: ${version}\n`, 'success');
-      } catch (error) {
-        this.log('StackMemory not found. Please install it first.', 'error');
-        return;
-      }
-      
-      // Run all tests
-      this.testStatusCommand();
-      this.testContextOperations();
-      this.testTaskOperations();
-      this.testStoragePerformance();
-      this.testBaseline();
-      
-      // Generate report
-      this.generateReport();
-      
-    } catch (error) {
-      this.log(`Test suite failed: ${error.message}`, 'error');
-      console.error(error);
-    }
-  }
-}
-
-// Run the test
-const test = new RealPerformanceTest();
-test.run();

package/scripts/testing/results/real-performance-results.json

@@ -1,90 +0,0 @@
-{
-  "timestamp": "2026-01-06T15:16:51.046Z",
-  "tests": [
-    {
-      "test": "status_command",
-      "iterations": 3,
-      "measurements": [
-        393.8040000000001,
-        396.434667,
-        393.71524999999997
-      ],
-      "average": 394.6513056666667,
-      "unit": "ms"
-    },
-    {
-      "test": "context_operations",
-      "operations": [
-        {
-          "operation": "version_check",
-          "duration": 398.65762500000005,
-          "success": true
-        },
-        {
-          "operation": "tasks_list",
-          "duration": 518.715792,
-          "success": true
-        }
-      ],
-      "totalTime": 917.373417,
-      "unit": "ms"
-    },
-    {
-      "test": "task_operations",
-      "operations": [
-        {
-          "operation": "add_task",
-          "duration": 518.383875,
-          "success": true,
-          "outputSize": 245
-        },
-        {
-          "operation": "list_tasks",
-          "duration": 526.2628340000001,
-          "success": true,
-          "outputSize": 2751
-        },
-        {
-          "operation": "show_task",
-          "duration": 395.6398339999996,
-          "success": true,
-          "outputSize": 92
-        }
-      ],
-      "unit": "ms"
-    },
-    {
-      "test": "storage_performance",
-      "results": {
-        "database": {
-          "exists": true,
-          "size": 602112,
-          "sizeFormatted": "588.00 KB",
-          "modified": "2026-01-06T15:16:54.814Z"
-        },
-        "tasks": {
-          "exists": true,
-          "size": 219410,
-          "sizeFormatted": "214.27 KB",
-          "lineCount": 249,
-          "modified": "2026-01-06T15:16:54.286Z"
-        }
-      }
-    },
-    {
-      "test": "baseline_comparison",
-      "baseline": {
-        "taskListing": {
-          "withStackMemory": 501.72233400000005,
-          "withoutStackMemory": 5000,
-          "unit": "ms"
-        },
-        "taskCreation": {
-          "withStackMemory": 500.09279100000003,
-          "withoutStackMemory": 30000,
-          "unit": "ms"
-        }
-      }
-    }
-  ]
-}

package/scripts/testing/run-effectiveness-tests.sh

@@ -1,176 +0,0 @@
-#!/bin/bash
-
-# StackMemory Effectiveness Testing Runner
-# This script orchestrates the complete testing process
-
-set -e
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-RESULTS_DIR="$PROJECT_ROOT/scripts/testing/results"
-
-# Colors for output
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-RED='\033[0;31m'
-NC='\033[0m' # No Color
-
-echo "=================================================="
-echo "StackMemory Effectiveness Testing Suite"
-echo "=================================================="
-echo ""
-
-# Function to print colored output
-print_status() {
-    echo -e "${GREEN}[✓]${NC} $1"
-}
-
-print_warning() {
-    echo -e "${YELLOW}[!]${NC} $1"
-}
-
-print_error() {
-    echo -e "${RED}[✗]${NC} $1"
-}
-
-# Check if StackMemory is installed
-check_stackmemory() {
-    if command -v stackmemory &> /dev/null; then
-        print_status "StackMemory CLI found"
-        stackmemory --version
-    else
-        print_error "StackMemory CLI not found. Please install it first."
-        echo "Run: npm install -g @stackmemoryai/stackmemory"
-        exit 1
-    fi
-}
-
-# Initialize test environment
-init_test_env() {
-    print_status "Initializing test environment..."
-    
-    # Create results directory
-    mkdir -p "$RESULTS_DIR"
-    mkdir -p "$RESULTS_DIR/runs"
-    mkdir -p "$RESULTS_DIR/metrics"
-    
-    # Build TypeScript files
-    cd "$PROJECT_ROOT"
-    print_status "Building TypeScript files..."
-    npm run build 2>/dev/null || print_warning "Build completed with warnings"
-}
-
-# Run baseline tests (without StackMemory)
-run_baseline() {
-    echo ""
-    echo "Phase 1: Collecting Baseline Metrics (Without StackMemory)"
-    echo "----------------------------------------------------------"
-    
-    # Ensure StackMemory daemon is stopped
-    stackmemory-daemon stop 2>/dev/null || true
-    
-    print_status "Running baseline scenarios..."
-    node "$SCRIPT_DIR/ab-test-runner.js" scenario multi_session_feature || true
-    
-    print_status "Baseline collection complete"
-}
-
-# Run StackMemory tests
-run_with_stackmemory() {
-    echo ""
-    echo "Phase 2: Testing With StackMemory"
-    echo "----------------------------------------------------------"
-    
-    # Start StackMemory daemon
-    print_status "Starting StackMemory daemon..."
-    stackmemory-daemon start || print_warning "Daemon already running"
-    
-    print_status "Running scenarios with StackMemory..."
-    node "$SCRIPT_DIR/ab-test-runner.js" scenario multi_session_feature || true
-    
-    print_status "StackMemory testing complete"
-}
-
-# Generate comparison report
-generate_report() {
-    echo ""
-    echo "Phase 3: Generating Comparison Report"
-    echo "----------------------------------------------------------"
-    
-    print_status "Analyzing results..."
-    node "$SCRIPT_DIR/collect-metrics.js" report || true
-    
-    if [ -f "$RESULTS_DIR/report.md" ]; then
-        print_status "Report generated: $RESULTS_DIR/report.md"
-        echo ""
-        echo "Summary:"
-        head -n 20 "$RESULTS_DIR/report.md"
-    else
-        print_warning "Report generation failed"
-    fi
-}
-
-# Quick test mode (for development)
-quick_test() {
-    echo "Running quick test..."
-    
-    # Test metric collection
-    print_status "Testing metrics collector..."
-    node "$SCRIPT_DIR/collect-metrics.js" start with_stackmemory
-    
-    # Test A/B runner
-    print_status "Testing A/B test runner..."
-    node "$SCRIPT_DIR/ab-test-runner.js" list
-    
-    print_status "Quick test complete"
-}
-
-# Main execution
-main() {
-    case "${1:-full}" in
-        quick)
-            check_stackmemory
-            init_test_env
-            quick_test
-            ;;
-        baseline)
-            check_stackmemory
-            init_test_env
-            run_baseline
-            ;;
-        stackmemory)
-            check_stackmemory
-            init_test_env
-            run_with_stackmemory
-            ;;
-        report)
-            generate_report
-            ;;
-        full)
-            check_stackmemory
-            init_test_env
-            run_baseline
-            run_with_stackmemory
-            generate_report
-            ;;
-        *)
-            echo "Usage: $0 [quick|baseline|stackmemory|report|full]"
-            echo ""
-            echo "Commands:"
-            echo "  quick       - Run quick validation tests"
-            echo "  baseline    - Run baseline tests without StackMemory"
-            echo "  stackmemory - Run tests with StackMemory enabled"
-            echo "  report      - Generate comparison report"
-            echo "  full        - Run complete test suite (default)"
-            exit 1
-            ;;
-    esac
-}
-
-# Run main function
-main "$@"
-
-echo ""
-echo "=================================================="
-echo "Testing Complete"
-echo "=================================================="