claude-flow-novice 2.3.4 → 2.3.6

package/dist/scripts/performance/sqlite-enhanced-activation.sh

@@ -1,583 +0,0 @@
-#!/bin/bash
-
-# SQLite Enhanced Backend Activation Script
-# Safe migration from basic SQLite to Enhanced Backend with rollback capabilities
-
-set -euo pipefail
-
-# Configuration
-BACKUP_DIR=".backups/sqlite-$(date +%Y%m%d-%H%M%S)"
-CURRENT_DB=".swarm/memory.db"
-ENHANCED_DB=".swarm/memory-enhanced.db"
-LOG_FILE="${BACKUP_DIR}/activation.log"
-ROLLBACK_SCRIPT="${BACKUP_DIR}/rollback.sh"
-
-# Colors for output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m' # No Color
-
-# Logging function
-log() {
-    echo -e "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "${LOG_FILE}"
-}
-
-error() {
-    echo -e "${RED}[ERROR]${NC} $1" | tee -a "${LOG_FILE}"
-}
-
-success() {
-    echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "${LOG_FILE}"
-}
-
-warning() {
-    echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "${LOG_FILE}"
-}
-
-info() {
-    echo -e "${BLUE}[INFO]${NC} $1" | tee -a "${LOG_FILE}"
-}
-
-# Check prerequisites
-check_prerequisites() {
-    info "Checking prerequisites..."
-
-    # Check if SQLite is available
-    if ! command -v sqlite3 &> /dev/null; then
-        error "sqlite3 command not found"
-        exit 1
-    fi
-
-    # Check if current database exists
-    if [[ ! -f "${CURRENT_DB}" ]]; then
-        error "Current database not found: ${CURRENT_DB}"
-        exit 1
-    fi
-
-    # Check available disk space (need at least 2x database size)
-    DB_SIZE=$(stat -f%z "${CURRENT_DB}" 2>/dev/null || stat -c%s "${CURRENT_DB}")
-    AVAILABLE_SPACE=$(df . | tail -1 | awk '{print $4}' || echo 0)
-    REQUIRED_SPACE=$((DB_SIZE * 3)) # 3x for safety
-
-    if [[ ${AVAILABLE_SPACE} -lt ${REQUIRED_SPACE} ]]; then
-        warning "Low disk space. Available: ${AVAILABLE_SPACE}, Required: ${REQUIRED_SPACE}"
-    fi
-
-    # Check memory availability
-    TOTAL_MEM=$(free -b | grep '^Mem:' | awk '{print $2}')
-    if [[ ${TOTAL_MEM} -lt 8589934592 ]]; then # 8GB
-        warning "Less than 8GB RAM available. Enhanced backend may use significant memory."
-    fi
-
-    success "Prerequisites check completed"
-}
-
-# Create backup and rollback script
-create_backup() {
-    info "Creating backup..."
-
-    mkdir -p "${BACKUP_DIR}"
-
-    # Backup current database
-    cp "${CURRENT_DB}" "${BACKUP_DIR}/memory.db.backup"
-
-    # Backup WAL and SHM files if they exist
-    if [[ -f "${CURRENT_DB}-wal" ]]; then
-        cp "${CURRENT_DB}-wal" "${BACKUP_DIR}/memory.db-wal.backup"
-    fi
-    if [[ -f "${CURRENT_DB}-shm" ]]; then
-        cp "${CURRENT_DB}-shm" "${BACKUP_DIR}/memory.db-shm.backup"
-    fi
-
-    # Create rollback script
-    cat > "${ROLLBACK_SCRIPT}" << 'EOF'
-#!/bin/bash
-
-# Emergency Rollback Script
-# Generated automatically during Enhanced Backend activation
-
-set -euo pipefail
-
-BACKUP_DIR="$(dirname "$0")"
-CURRENT_DB=".swarm/memory.db"
-
-echo "🚨 EMERGENCY ROLLBACK INITIATED"
-echo "Timestamp: $(date)"
-
-# Stop any running processes
-echo "Stopping any running Claude Flow processes..."
-pkill -f "claude-flow" || true
-pkill -f "npx.*claude-flow" || true
-sleep 2
-
-# Restore database files
-echo "Restoring database from backup..."
-cp "${BACKUP_DIR}/memory.db.backup" "${CURRENT_DB}"
-
-if [[ -f "${BACKUP_DIR}/memory.db-wal.backup" ]]; then
-    cp "${BACKUP_DIR}/memory.db-wal.backup" "${CURRENT_DB}-wal"
-fi
-
-if [[ -f "${BACKUP_DIR}/memory.db-shm.backup" ]]; then
-    cp "${BACKUP_DIR}/memory.db-shm.backup" "${CURRENT_DB}-shm"
-fi
-
-# Remove enhanced database if it exists
-if [[ -f ".swarm/memory-enhanced.db" ]]; then
-    rm -f ".swarm/memory-enhanced.db"*
-fi
-
-# Verify database integrity
-echo "Verifying database integrity..."
-if sqlite3 "${CURRENT_DB}" "PRAGMA integrity_check;" | grep -q "ok"; then
-    echo "✅ Database integrity verified"
-    echo "✅ Rollback completed successfully"
-else
-    echo "❌ Database integrity check failed"
-    echo "❌ Manual recovery required"
-    exit 1
-fi
-
-echo "🎯 System restored to pre-activation state"
-echo "You can now restart Claude Flow normally"
-EOF
-
-    chmod +x "${ROLLBACK_SCRIPT}"
-
-    success "Backup created: ${BACKUP_DIR}"
-    info "Rollback script: ${ROLLBACK_SCRIPT}"
-}
-
-# Test current database integrity
-test_database_integrity() {
-    info "Testing current database integrity..."
-
-    local integrity_result
-    integrity_result=$(sqlite3 "${CURRENT_DB}" "PRAGMA integrity_check;")
-
-    if [[ "${integrity_result}" != "ok" ]]; then
-        error "Current database integrity check failed: ${integrity_result}"
-        exit 1
-    fi
-
-    success "Database integrity verified"
-}
-
-# Collect current performance metrics
-collect_baseline_metrics() {
-    info "Collecting baseline performance metrics..."
-
-    local metrics_file="${BACKUP_DIR}/baseline_metrics.json"
-
-    cat > "${metrics_file}" << EOF
-{
-    "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
-    "database_file": "${CURRENT_DB}",
-    "file_size_bytes": $(stat -f%z "${CURRENT_DB}" 2>/dev/null || stat -c%s "${CURRENT_DB}"),
-    "entry_count": $(sqlite3 "${CURRENT_DB}" "SELECT COUNT(*) FROM memory_entries;"),
-    "namespace_count": $(sqlite3 "${CURRENT_DB}" "SELECT COUNT(DISTINCT namespace) FROM memory_entries;"),
-    "current_settings": {
-        "journal_mode": "$(sqlite3 "${CURRENT_DB}" "PRAGMA journal_mode;")",
-        "synchronous": "$(sqlite3 "${CURRENT_DB}" "PRAGMA synchronous;")",
-        "cache_size": "$(sqlite3 "${CURRENT_DB}" "PRAGMA cache_size;")",
-        "mmap_size": "$(sqlite3 "${CURRENT_DB}" "PRAGMA mmap_size;")",
-        "temp_store": "$(sqlite3 "${CURRENT_DB}" "PRAGMA temp_store;")"
-    },
-    "query_benchmarks": {
-        "simple_count": "$(time sqlite3 "${CURRENT_DB}" "SELECT COUNT(*) FROM memory_entries;" 2>&1 | grep real | awk '{print $2}')",
-        "namespace_group": "$(time sqlite3 "${CURRENT_DB}" "SELECT namespace, COUNT(*) FROM memory_entries GROUP BY namespace;" 2>&1 | grep real | awk '{print $2}')",
-        "recent_entries": "$(time sqlite3 "${CURRENT_DB}" "SELECT * FROM memory_entries ORDER BY created_at DESC LIMIT 10;" 2>&1 | grep real | awk '{print $2}')"
-    }
-}
-EOF
-
-    success "Baseline metrics collected: ${metrics_file}"
-}
-
-# Initialize Enhanced Backend configuration
-create_enhanced_config() {
-    info "Creating Enhanced Backend configuration..."
-
-    local config_file="${BACKUP_DIR}/enhanced_config.json"
-
-    # Calculate optimal settings based on available RAM
-    local total_ram_gb=$(($(free -b | grep '^Mem:' | awk '{print $2}') / 1024 / 1024 / 1024))
-    local cache_size_mb=$((total_ram_gb * 1024 / 16)) # Use 1/16 of RAM for cache
-    local mmap_size_gb=$((total_ram_gb / 12)) # Use 1/12 of RAM for memory mapping
-
-    # Cap at reasonable maximums
-    if [[ ${cache_size_mb} -gt 2048 ]]; then
-        cache_size_mb=2048
-    fi
-
-    if [[ ${mmap_size_gb} -gt 8 ]]; then
-        mmap_size_gb=8
-    fi
-
-    cat > "${config_file}" << EOF
-{
-    "databasePath": ".swarm",
-    "namespace": "enhanced",
-    "isolation": "cross-agent",
-    "enableWAL": true,
-    "enableCompression": true,
-    "enableIndexing": true,
-    "reliabilityTarget": 0.999,
-    "syncInterval": 5000,
-    "maxConnections": 10,
-    "transactionTimeout": 30000,
-    "optimizedSettings": {
-        "cache_size": -${cache_size_mb}000,
-        "mmap_size": $((mmap_size_gb * 1024 * 1024 * 1024)),
-        "temp_store": "memory",
-        "journal_mode": "WAL",
-        "synchronous": "NORMAL",
-        "wal_autocheckpoint": 1000,
-        "optimize": true
-    },
-    "hardware_profile": {
-        "total_ram_gb": ${total_ram_gb},
-        "cpu_cores": $(nproc),
-        "recommended_cache_mb": ${cache_size_mb},
-        "recommended_mmap_gb": ${mmap_size_gb}
-    }
-}
-EOF
-
-    success "Enhanced configuration created: ${config_file}"
-    info "Cache size: ${cache_size_mb}MB, Memory mapping: ${mmap_size_gb}GB"
-}
-
-# Initialize Enhanced Backend database
-initialize_enhanced_backend() {
-    info "Initializing Enhanced Backend database..."
-
-    # Copy current database as starting point
-    cp "${CURRENT_DB}" "${ENHANCED_DB}"
-
-    # Apply enhanced optimizations
-    local config_file="${BACKUP_DIR}/enhanced_config.json"
-    local cache_size=$(jq -r '.optimizedSettings.cache_size' "${config_file}")
-    local mmap_size=$(jq -r '.optimizedSettings.mmap_size' "${config_file}")
-
-    sqlite3 "${ENHANCED_DB}" << EOF
--- Apply enhanced optimizations
-PRAGMA journal_mode=WAL;
-PRAGMA synchronous=NORMAL;
-PRAGMA cache_size=${cache_size};
-PRAGMA mmap_size=${mmap_size};
-PRAGMA temp_store=memory;
-PRAGMA wal_autocheckpoint=1000;
-
--- Create additional indexes for Enhanced Backend
-CREATE INDEX IF NOT EXISTS idx_memory_owner ON memory_entries(metadata) WHERE json_extract(metadata, '$.owner') IS NOT NULL;
-CREATE INDEX IF NOT EXISTS idx_memory_access_level ON memory_entries(metadata) WHERE json_extract(metadata, '$.accessLevel') IS NOT NULL;
-CREATE INDEX IF NOT EXISTS idx_memory_compressed ON memory_entries(value) WHERE length(value) > 1000;
-CREATE INDEX IF NOT EXISTS idx_memory_size ON memory_entries(value);
-
--- Add enhanced columns if they don't exist
-ALTER TABLE memory_entries ADD COLUMN checksum TEXT DEFAULT '';
-ALTER TABLE memory_entries ADD COLUMN compressed INTEGER DEFAULT 0;
-ALTER TABLE memory_entries ADD COLUMN size INTEGER DEFAULT 0;
-
--- Update size column for existing entries
-UPDATE memory_entries SET size = length(value) WHERE size = 0;
-
--- Optimize database
-PRAGMA optimize;
-ANALYZE;
-EOF
-
-    # Verify enhanced database
-    local integrity_result
-    integrity_result=$(sqlite3 "${ENHANCED_DB}" "PRAGMA integrity_check;")
-
-    if [[ "${integrity_result}" != "ok" ]]; then
-        error "Enhanced database integrity check failed: ${integrity_result}"
-        exit 1
-    fi
-
-    success "Enhanced Backend database initialized"
-}
-
-# Test Enhanced Backend performance
-test_enhanced_performance() {
-    info "Testing Enhanced Backend performance..."
-
-    local test_results="${BACKUP_DIR}/enhanced_test_results.json"
-
-    cat > "${test_results}" << EOF
-{
-    "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
-    "database_file": "${ENHANCED_DB}",
-    "file_size_bytes": $(stat -f%z "${ENHANCED_DB}" 2>/dev/null || stat -c%s "${ENHANCED_DB}"),
-    "settings": {
-        "journal_mode": "$(sqlite3 "${ENHANCED_DB}" "PRAGMA journal_mode;")",
-        "synchronous": "$(sqlite3 "${ENHANCED_DB}" "PRAGMA synchronous;")",
-        "cache_size": "$(sqlite3 "${ENHANCED_DB}" "PRAGMA cache_size;")",
-        "mmap_size": "$(sqlite3 "${ENHANCED_DB}" "PRAGMA mmap_size;")",
-        "temp_store": "$(sqlite3 "${ENHANCED_DB}" "PRAGMA temp_store;")"
-    },
-    "performance_tests": {
-        "simple_count": "$(time sqlite3 "${ENHANCED_DB}" "SELECT COUNT(*) FROM memory_entries;" 2>&1 | grep real | awk '{print $2}')",
-        "namespace_group": "$(time sqlite3 "${ENHANCED_DB}" "SELECT namespace, COUNT(*) FROM memory_entries GROUP BY namespace;" 2>&1 | grep real | awk '{print $2}')",
-        "complex_query": "$(time sqlite3 "${ENHANCED_DB}" "SELECT namespace, COUNT(*), AVG(length(value)), SUM(length(value)) FROM memory_entries GROUP BY namespace HAVING COUNT(*) > 5 ORDER BY COUNT(*) DESC;" 2>&1 | grep real | awk '{print $2}')",
-        "recent_with_metadata": "$(time sqlite3 "${ENHANCED_DB}" "SELECT * FROM memory_entries WHERE json_extract(metadata, '$.type') IS NOT NULL ORDER BY created_at DESC LIMIT 20;" 2>&1 | grep real | awk '{print $2}')"
-    }
-}
-EOF
-
-    success "Performance test completed: ${test_results}"
-}
-
-# Compare performance between basic and enhanced
-compare_performance() {
-    info "Comparing performance metrics..."
-
-    local comparison_file="${BACKUP_DIR}/performance_comparison.json"
-    local baseline="${BACKUP_DIR}/baseline_metrics.json"
-    local enhanced="${BACKUP_DIR}/enhanced_test_results.json"
-
-    # Simple comparison (would be more sophisticated in production)
-    python3 << EOF
-import json
-import sys
-
-try:
-    with open('${baseline}', 'r') as f:
-        baseline = json.load(f)
-
-    with open('${enhanced}', 'r') as f:
-        enhanced = json.load(f)
-
-    comparison = {
-        "timestamp": baseline["timestamp"],
-        "baseline_db_size": baseline["file_size_bytes"],
-        "enhanced_db_size": enhanced["file_size_bytes"],
-        "size_increase_percent": ((enhanced["file_size_bytes"] - baseline["file_size_bytes"]) / baseline["file_size_bytes"]) * 100,
-        "cache_improvement": f"{baseline['current_settings']['cache_size']} -> {enhanced['settings']['cache_size']}",
-        "mmap_improvement": f"{baseline['current_settings']['mmap_size']} -> {enhanced['settings']['mmap_size']}",
-        "performance_comparison": "Enhanced backend configured with optimized settings",
-        "recommendation": "Proceed with activation - performance improvements expected"
-    }
-
-    with open('${comparison_file}', 'w') as f:
-        json.dump(comparison, f, indent=2)
-
-    print("✅ Performance comparison completed")
-except Exception as e:
-    print(f"❌ Performance comparison failed: {e}")
-    sys.exit(1)
-EOF
-
-    if [[ $? -eq 0 ]]; then
-        success "Performance comparison: ${comparison_file}"
-    else
-        warning "Performance comparison failed, proceeding with manual verification"
-    fi
-}
-
-# Activate Enhanced Backend (dry run option)
-activate_enhanced_backend() {
-    local dry_run=${1:-false}
-
-    if [[ "${dry_run}" == "true" ]]; then
-        info "DRY RUN: Would activate Enhanced Backend"
-        info "DRY RUN: Would move ${CURRENT_DB} to ${CURRENT_DB}.old"
-        info "DRY RUN: Would move ${ENHANCED_DB} to ${CURRENT_DB}"
-        info "DRY RUN: Would restart services with enhanced configuration"
-        return 0
-    fi
-
-    info "Activating Enhanced Backend..."
-
-    # Move current database to backup location
-    mv "${CURRENT_DB}" "${CURRENT_DB}.old"
-
-    # Move enhanced database to active location
-    mv "${ENHANCED_DB}" "${CURRENT_DB}"
-
-    # Copy WAL and SHM files if they exist
-    if [[ -f "${ENHANCED_DB}-wal" ]]; then
-        mv "${ENHANCED_DB}-wal" "${CURRENT_DB}-wal"
-    fi
-    if [[ -f "${ENHANCED_DB}-shm" ]]; then
-        mv "${ENHANCED_DB}-shm" "${CURRENT_DB}-shm"
-    fi
-
-    success "Enhanced Backend activated"
-}
-
-# Verify activation
-verify_activation() {
-    info "Verifying Enhanced Backend activation..."
-
-    # Check database integrity
-    local integrity_result
-    integrity_result=$(sqlite3 "${CURRENT_DB}" "PRAGMA integrity_check;")
-
-    if [[ "${integrity_result}" != "ok" ]]; then
-        error "Post-activation integrity check failed: ${integrity_result}"
-        error "Running automatic rollback..."
-        bash "${ROLLBACK_SCRIPT}"
-        exit 1
-    fi
-
-    # Test basic operations
-    local entry_count
-    entry_count=$(sqlite3 "${CURRENT_DB}" "SELECT COUNT(*) FROM memory_entries;")
-
-    if [[ ${entry_count} -eq 0 ]]; then
-        error "No entries found after activation"
-        error "Running automatic rollback..."
-        bash "${ROLLBACK_SCRIPT}"
-        exit 1
-    fi
-
-    # Check enhanced features
-    local enhanced_columns
-    enhanced_columns=$(sqlite3 "${CURRENT_DB}" "PRAGMA table_info(memory_entries);" | grep -c "checksum\|compressed\|size" || echo 0)
-
-    if [[ ${enhanced_columns} -lt 3 ]]; then
-        warning "Enhanced columns not found - some features may not be available"
-    fi
-
-    success "Enhanced Backend verification completed"
-    success "Entry count: ${entry_count}"
-    info "Enhanced columns: ${enhanced_columns}/3"
-}
-
-# Generate activation report
-generate_report() {
-    info "Generating activation report..."
-
-    local report_file="${BACKUP_DIR}/activation_report.md"
-
-    cat > "${report_file}" << EOF
-# SQLite Enhanced Backend Activation Report
-
-## Activation Summary
-- **Date**: $(date)
-- **Status**: ✅ Successfully activated
-- **Backup Location**: ${BACKUP_DIR}
-- **Rollback Script**: ${ROLLBACK_SCRIPT}
-
-## Database Migration
-- **Source**: ${CURRENT_DB}
-- **Target**: Enhanced SQLite Backend
-- **Data Integrity**: ✅ Verified
-- **Entry Count**: $(sqlite3 "${CURRENT_DB}" "SELECT COUNT(*) FROM memory_entries;")
-
-## Performance Optimizations Applied
-- **Journal Mode**: WAL (Write-Ahead Logging)
-- **Cache Size**: $(sqlite3 "${CURRENT_DB}" "PRAGMA cache_size;") pages
-- **Memory Mapping**: $(sqlite3 "${CURRENT_DB}" "PRAGMA mmap_size;") bytes
-- **Temp Store**: memory
-- **Synchronous**: NORMAL
-
-## Enhanced Features
-- ✅ Advanced indexing
-- ✅ Cross-agent memory sharing capability
-- ✅ Transaction safety with retry logic
-- ✅ Compression support (when enabled)
-- ✅ Checksum validation
-
-## Rollback Information
-If you need to rollback to the previous configuration:
-\`\`\`bash
-bash ${ROLLBACK_SCRIPT}
-\`\`\`
-
-## Next Steps
-1. Monitor system performance
-2. Enable compression if needed
-3. Configure cross-agent sharing
-4. Set up connection pooling for high-load scenarios
-
-## Files Created
-- Configuration: ${BACKUP_DIR}/enhanced_config.json
-- Baseline Metrics: ${BACKUP_DIR}/baseline_metrics.json
-- Test Results: ${BACKUP_DIR}/enhanced_test_results.json
-- Performance Comparison: ${BACKUP_DIR}/performance_comparison.json
-- This Report: ${report_file}
-EOF
-
-    success "Activation report generated: ${report_file}"
-}
-
-# Main execution function
-main() {
-    local dry_run=false
-    local skip_tests=false
-
-    # Parse command line arguments
-    while [[ $# -gt 0 ]]; do
-        case $1 in
-            --dry-run)
-                dry_run=true
-                shift
-                ;;
-            --skip-tests)
-                skip_tests=true
-                shift
-                ;;
-            --help)
-                echo "SQLite Enhanced Backend Activation Script"
-                echo ""
-                echo "Usage: $0 [OPTIONS]"
-                echo ""
-                echo "Options:"
-                echo "  --dry-run     Perform all steps except actual activation"
-                echo "  --skip-tests  Skip performance testing (faster execution)"
-                echo "  --help        Show this help message"
-                echo ""
-                echo "This script safely migrates from basic SQLite to Enhanced Backend"
-                echo "with automatic rollback capabilities."
-                exit 0
-                ;;
-            *)
-                error "Unknown option: $1"
-                echo "Use --help for usage information"
-                exit 1
-                ;;
-        esac
-    done
-
-    info "Starting SQLite Enhanced Backend activation..."
-    if [[ "${dry_run}" == "true" ]]; then
-        warning "DRY RUN MODE - No changes will be made"
-    fi
-
-    # Execute activation steps
-    check_prerequisites
-    create_backup
-    test_database_integrity
-    collect_baseline_metrics
-    create_enhanced_config
-    initialize_enhanced_backend
-
-    if [[ "${skip_tests}" != "true" ]]; then
-        test_enhanced_performance
-        compare_performance
-    fi
-
-    activate_enhanced_backend "${dry_run}"
-
-    if [[ "${dry_run}" != "true" ]]; then
-        verify_activation
-        generate_report
-
-        success "🎉 SQLite Enhanced Backend activation completed successfully!"
-        info "📊 Review the activation report: ${BACKUP_DIR}/activation_report.md"
-        info "🔄 Rollback available: ${ROLLBACK_SCRIPT}"
-        info "📈 Monitor performance and adjust configuration as needed"
-    else
-        success "🧪 Dry run completed successfully!"
-        info "📋 Review the generated configuration and run without --dry-run to activate"
-    fi
-}
-
-# Execute main function with all arguments
-main "$@"