claude-flow-novice 2.14.31 → 2.14.33

package/scripts/deploy-production.sh

@@ -0,0 +1,356 @@
+#!/bin/bash
+
+# Production CFN Loop Deployment Script
+# Deploys the complete production stack with validation
+
+set -euo pipefail
+
+# Configuration
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+TIMESTAMP=$(date +%s)
+LOG_FILE="$PROJECT_ROOT/logs/production-deploy-$TIMESTAMP.log"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Environment variables
+export BUILD_DATE="${BUILD_DATE:-$(date -u +'%Y-%m-%dT%H:%M:%SZ')}"
+export VCS_REF="${VCS_REF:-$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')}"
+export VERSION="${VERSION:-4.0.0}"
+export CLAUDE_FLOW_VERSION="${CLAUDE_FLOW_VERSION:-4.0.0}"
+export LOG_LEVEL="${LOG_LEVEL:-info}"
+export MAX_AGENTS="${MAX_AGENTS:-10}"
+export AGENT_TIMEOUT="${AGENT_TIMEOUT:-300000}"
+export MEMORY_LIMIT="${MEMORY_LIMIT:-1g}"
+export CPU_LIMIT="${CPU_LIMIT:-0.5}"
+export AGENT_REPLICAS="${AGENT_REPLICAS:-3}"
+export WORKER_POOL_SIZE="${WORKER_POOL_SIZE:-5}"
+export GRAFANA_USER="${GRAFANA_USER:-admin}"
+export GRAFANA_PASSWORD="${GRAFANA_PASSWORD:-admin123}"
+
+# Logging function
+log() {
+    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$LOG_FILE"
+}
+
+error() {
+    echo -e "${RED}[ERROR]${NC} $1" >&2 | tee -a "$LOG_FILE"
+}
+
+success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE"
+}
+
+warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE"
+}
+
+# Cleanup function
+cleanup() {
+    log "Cleaning up on exit..."
+    # Add any cleanup tasks here
+}
+
+# Set up signal handlers
+trap cleanup EXIT INT TERM
+
+# Main deployment function
+main() {
+    log "🚀 Starting Production CFN Loop Deployment"
+    log "📋 Deployment ID: $TIMESTAMP"
+    log "📁 Project root: $PROJECT_ROOT"
+    log "📄 Log file: $LOG_FILE"
+
+    # Change to project directory
+    cd "$PROJECT_ROOT"
+
+    # Phase 1: Pre-deployment checks
+    log "🔍 Running pre-deployment checks..."
+    run_pre_deployment_checks
+
+    # Phase 2: Build production images
+    log "🏗️ Building production Docker images..."
+    build_production_images
+
+    # Phase 3: Deploy infrastructure
+    log "🐳 Deploying production infrastructure..."
+    deploy_infrastructure
+
+    # Phase 4: Wait for services to be healthy
+    log "⏳ Waiting for services to be healthy..."
+    wait_for_healthy_services
+
+    # Phase 5: Run validation tests
+    log "🧪 Running deployment validation..."
+    run_validation_tests
+
+    # Phase 6: Display deployment summary
+    display_deployment_summary
+
+    success "🎉 Production deployment completed successfully!"
+}
+
+# Pre-deployment checks
+run_pre_deployment_checks() {
+    log "   Checking prerequisites..."
+
+    # Check Docker
+    if ! command -v docker &> /dev/null; then
+        error "Docker is not installed or not in PATH"
+        exit 1
+    fi
+    log "   ✅ Docker available: $(docker --version)"
+
+    # Check Docker Compose
+    if ! command -v docker-compose &> /dev/null; then
+        error "Docker Compose is not installed or not in PATH"
+        exit 1
+    fi
+    log "   ✅ Docker Compose available: $(docker-compose --version)"
+
+    # Check Node.js (for local tests)
+    if command -v node &> /dev/null; then
+        log "   ✅ Node.js available: $(node --version)"
+    else
+        warning "Node.js not available - some tests may be skipped"
+    fi
+
+    # Check available disk space
+    local available_space
+    available_space=$(df -BG . | awk 'NR==2 {print $4}' | sed 's/G//')
+    if [[ $available_space -lt 5 ]]; then
+        warning "Low disk space: ${available_space}GB available (recommended: 5GB+)"
+    else
+        log "   ✅ Disk space: ${available_space}GB available"
+    fi
+
+    # Check available memory
+    if command -v free &> /dev/null; then
+        local available_memory
+        available_memory=$(free -g | awk 'NR==2{print $7}')
+        if [[ $available_memory -lt 4 ]]; then
+            warning "Low memory: ${available_memory}GB available (recommended: 4GB+)"
+        else
+            log "   ✅ Memory: ${available_memory}GB available"
+        fi
+    fi
+
+    # Create necessary directories
+    log "   Creating directories..."
+    mkdir -p logs workspaces monitoring/grafana/{provisioning/{datasources,dashboards},dashboards} nginx
+
+    log "   ✅ Pre-deployment checks completed"
+}
+
+# Build production images
+build_production_images() {
+    log "   Building production Docker image..."
+
+    # Build the production image
+    if docker build -f Dockerfile.production -t claude-flow-novice:production .; then
+        success "   ✅ Production image built successfully"
+    else
+        error "   ❌ Failed to build production image"
+        exit 1
+    fi
+
+    # Tag the image
+    docker tag claude-flow-novice:production claude-flow-novice:${VERSION}
+    log "   ✅ Image tagged as claude-flow-novice:${VERSION}"
+
+    # Show image information
+    local image_size
+    image_size=$(docker images claude-flow-novice:production --format "{{.Size}}")
+    log "   📊 Image size: $image_size"
+}
+
+# Deploy infrastructure
+deploy_infrastructure() {
+    log "   Deploying production stack..."
+
+    # Create environment file
+    cat > .env.production << EOF
+# Production CFN Loop Environment Configuration
+BUILD_DATE=$BUILD_DATE
+VCS_REF=$VCS_REF
+VERSION=$VERSION
+CLAUDE_FLOW_VERSION=$CLAUDE_FLOW_VERSION
+LOG_LEVEL=$LOG_LEVEL
+MAX_AGENTS=$MAX_AGENTS
+AGENT_TIMEOUT=$AGENT_TIMEOUT
+MEMORY_LIMIT=$MEMORY_LIMIT
+CPU_LIMIT=$CPU_LIMIT
+AGENT_REPLICAS=$AGENT_REPLICAS
+WORKER_POOL_SIZE=$WORKER_POOL_SIZE
+GRAFANA_USER=$GRAFANA_USER
+GRAFANA_PASSWORD=$GRAFANA_PASSWORD
+EOF
+
+    # Deploy the stack
+    if docker-compose -f docker-compose.production.yml --env-file .env.production up -d; then
+        success "   ✅ Production stack deployed"
+    else
+        error "   ❌ Failed to deploy production stack"
+        exit 1
+    fi
+
+    # Show deployed services
+    log "   📊 Deployed services:"
+    docker-compose -f docker-compose.production.yml ps
+}
+
+# Wait for services to be healthy
+wait_for_healthy_services() {
+    log "   Waiting for services to become healthy..."
+    local max_wait=300  # 5 minutes
+    local wait_interval=10
+    local waited=0
+
+    while [[ $waited -lt $max_wait ]]; do
+        local healthy_count=0
+        local total_count=0
+
+        # Check service health
+        while IFS= read -r line; do
+            if [[ $line == *"cfn-"* ]]; then
+                ((total_count++))
+                if [[ $line == *"healthy"* ]] || [[ $line == *"Up"* ]]; then
+                    ((healthy_count++))
+                fi
+            fi
+        done < <(docker-compose -f docker-compose.production.yml ps --format "table {{.Name}}\t{{.Status}}" | tail -n +2)
+
+        log "   Progress: $healthy_count/$total_count services healthy (${waited}s elapsed)"
+
+        if [[ $healthy_count -eq $total_count ]] && [[ $total_count -gt 0 ]]; then
+            success "   ✅ All $total_count services are healthy"
+            return 0
+        fi
+
+        sleep $wait_interval
+        ((waited += wait_interval))
+    done
+
+    warning "   ⚠️ Some services may still be starting up"
+    log "   📊 Current status:"
+    docker-compose -f docker-compose.production.yml ps
+}
+
+# Run validation tests
+run_validation_tests() {
+    log "   Running deployment validation tests..."
+
+    # Test basic connectivity
+    log "   Testing Redis connectivity..."
+    if docker exec cfn-redis-coordinator redis-cli ping | grep -q "PONG"; then
+        success "   ✅ Redis connectivity verified"
+    else
+        error "   ❌ Redis connectivity failed"
+    fi
+
+    # Test orchestrator
+    log "   Testing orchestrator..."
+    if docker exec cfn-orchestrator node -e "console.log('Orchestrator ready')" &>/dev/null; then
+        success "   ✅ Orchestrator responsive"
+    else
+        warning "   ⚠️ Orchestrator may still be initializing"
+    fi
+
+    # Test monitoring endpoints
+    log "   Testing monitoring endpoints..."
+    local prometheus_ok=false
+    local grafana_ok=false
+
+    if curl -s http://localhost:9090/-/healthy &>/dev/null; then
+        prometheus_ok=true
+        success "   ✅ Prometheus endpoint accessible"
+    else
+        warning "   ⚠️ Prometheus endpoint not accessible"
+    fi
+
+    if curl -s http://localhost:3001/api/health &>/dev/null; then
+        grafana_ok=true
+        success "   ✅ Grafana endpoint accessible"
+    else
+        warning "   ⚠️ Grafana endpoint not accessible"
+    fi
+
+    # Run comprehensive test if Node.js is available
+    if command -v node &> /dev/null; then
+        log "   Running comprehensive production test..."
+        if node tests/docker/production-deployment-test.js &>/dev/null; then
+            success "   ✅ Comprehensive test passed"
+        else
+            warning "   ⚠️ Comprehensive test encountered issues"
+        fi
+    else
+        log "   ⏭️ Skipping comprehensive test (Node.js not available)"
+    fi
+
+    # Store test results
+    cat > logs/deployment-validation-$TIMESTAMP.json << EOF
+{
+    "timestamp": "$(date -Iseconds)",
+    "deployment_id": "$TIMESTAMP",
+    "redis_healthy": $(docker exec cfn-redis-coordinator redis-cli ping | grep -q "PONG" && echo true || echo false),
+    "prometheus_healthy": $prometheus_ok,
+    "grafana_healthy": $grafana_ok,
+    "services_total": $(docker-compose -f docker-compose.production.yml ps --format json | jq '. | length'),
+    "services_healthy": $(docker-compose -f docker-compose.production.yml ps --format json | jq '[.[] | select(.State == "running")] | length')
+}
+EOF
+}
+
+# Display deployment summary
+display_deployment_summary() {
+    log "📊 Deployment Summary"
+    log "===================="
+
+    # Service status
+    log ""
+    log "🐳 Services Status:"
+    docker-compose -f docker-compose.production.yml ps
+
+    # Access URLs
+    log ""
+    log "🔗 Access URLs:"
+    log "   • Prometheus: http://localhost:9090"
+    log "   • Grafana: http://localhost:3001 (admin/admin123)"
+    log "   • Redis: redis://localhost:6379"
+    log "   • Orchestrator: http://localhost:3000"
+
+    # Resource usage
+    log ""
+    log "📈 Resource Usage:"
+    docker stats --no-stream --format "table {{.Container}}\t{{.MemUsage}}\t{{.CPUPerc}}" | grep cfn- || log "   No container stats available yet"
+
+    # Useful commands
+    log ""
+    log "🛠️ Useful Commands:"
+    log "   • View logs: docker-compose -f docker-compose.production.yml logs -f [service-name]"
+    log "   • Stop stack: docker-compose -f docker-compose.production.yml down"
+    log "   • Restart service: docker-compose -f docker-compose.production.yml restart [service-name]"
+    log "   • Scale agents: docker-compose -f docker-compose.production.yml up -d --scale agent-pool=[N]"
+
+    # Next steps
+    log ""
+    log "📋 Next Steps:"
+    log "   1. Configure Grafana dashboards for monitoring"
+    log "   2. Set up alerting rules in Prometheus"
+    log "   3. Test CFN Loop execution with real tasks"
+    log "   4. Configure backup and disaster recovery"
+    log "   5. Set up log aggregation and analysis"
+
+    log ""
+    success "🎉 Production deployment is ready for use!"
+}
+
+# Script entry point
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main "$@"
+fi

package/scripts/memory-leak-prevention.sh

@@ -0,0 +1,306 @@
+#!/bin/bash
+
+# Memory Leak Detection and Prevention Script
+# Implements recommendations from memory leak analysis
+
+set -euo pipefail
+
+# Configuration
+DEFAULT_MEMORY_LIMIT="8192"  # 8GB limit instead of 16GB
+DEFAULT_NODE_OPTIONS="--max-old-space-size=$DEFAULT_MEMORY_LIMIT"
+PROFILING_DIR="/tmp/claude-memory-profiles"
+LOG_FILE="/tmp/claude-memory-monitor.log"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Create profiling directory
+mkdir -p "$PROFILING_DIR"
+
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
+}
+
+show_help() {
+    cat << EOF
+Memory Leak Detection and Prevention Tool
+
+Usage: $0 [COMMAND] [OPTIONS]
+
+Commands:
+    profile           Start Claude with heap profiling enabled
+    monitor           Monitor existing Claude process memory usage
+    limit             Set memory limits for Claude sessions
+    install-tools     Install strace and perf for WSL
+    analyze           Analyze existing memory profiles
+    config            Show current memory configuration
+
+Options:
+    --limit MB        Set memory limit in MB (default: 8192)
+    --pid PID         Monitor specific process ID
+    --duration SEC    Monitoring duration in seconds (default: 300)
+    --output DIR      Output directory for profiles (default: /tmp/claude-memory-profiles)
+
+Examples:
+    $0 profile --limit 6144                    # Start Claude with 6GB limit and profiling
+    $0 monitor --pid 12345 --duration 600     # Monitor PID 12345 for 10 minutes
+    $0 install-tools                          # Install strace and perf
+    $0 analyze --output ./profiles            # Analyze existing profiles
+EOF
+}
+
+install_debug_tools() {
+    log "${BLUE}Installing strace and perf for debugging...${NC}"
+
+    if command -v apt-get &> /dev/null; then
+        sudo apt-get update
+        sudo apt-get install -y strace linux-tools-generic linux-perf
+    elif command -v yum &> /dev/null; then
+        sudo yum install -y strace perf
+    else
+        log "${RED}Unsupported package manager. Please install strace and perf manually.${NC}"
+        return 1
+    fi
+
+    # Configure perf for non-root users
+    echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid >/dev/null 2>&1 || true
+    echo 0 | sudo tee /proc/sys/kernel/kptr_restrict >/dev/null 2>&1 || true
+
+    log "${GREEN}Debug tools installed successfully${NC}"
+}
+
+start_with_profiling() {
+    local memory_limit=${1:-$DEFAULT_MEMORY_LIMIT}
+
+    log "${BLUE}Starting Claude with heap profiling and memory limit: ${memory_limit}MB${NC}"
+
+    # Set up environment for profiling
+    export NODE_OPTIONS="--max-old-space-size=$memory_limit --inspect=0.0.0.0:9229 --heap-prof"
+    export CLAUDE_MEMORY_PROFILE_DIR="$PROFILING_DIR"
+
+    # Create profile filename with timestamp
+    local profile_file="${PROFILING_DIR}/claude-profile-$(date +%Y%m%d-%H%M%S)"
+
+    log "${GREEN}Environment variables set:${NC}"
+    log "  NODE_OPTIONS: $NODE_OPTIONS"
+    log "  Profile directory: $PROFILING_DIR"
+    log "${YELLOW}Claude will start with debugging enabled on port 9229${NC}"
+    log "${YELLOW}Use Chrome DevTools or chrome://inspect to connect${NC}"
+
+    # Start Claude CLI with profiling
+    log "${BLUE}Starting Claude CLI...${NC}"
+    npx claude-flow-novice "$@"
+}
+
+monitor_memory() {
+    local target_pid=${1:-}
+    local duration=${2:-300}
+    local output_file="${PROFILING_DIR}/memory-monitor-$(date +%Y%m%d-%H%M%S).csv"
+
+    if [[ -z "$target_pid" ]]; then
+        log "${RED}Error: PID required for monitoring. Use --pid to specify.${NC}"
+        return 1
+    fi
+
+    log "${BLUE}Monitoring PID $target_pid for ${duration} seconds...${NC}"
+    log "Results will be saved to: $output_file"
+
+    # Create CSV header
+    echo "timestamp,pid,rss_mb,vms_mb,cpu_percent,open_files,connections" > "$output_file"
+
+    local start_time=$(date +%s)
+    local end_time=$((start_time + duration))
+
+    while [[ $(date +%s) -lt $end_time ]]; do
+        if ! kill -0 "$target_pid" 2>/dev/null; then
+            log "${RED}Process $target_pid is no longer running${NC}"
+            break
+        fi
+
+        local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
+        local stats=$(ps -p "$target_pid" -o pid,rss,vms,pcpu --no-headers 2>/dev/null || echo "")
+
+        if [[ -n "$stats" ]]; then
+            local rss_mb=$(echo "$stats" | awk '{print int($2/1024)}')
+            local vms_mb=$(echo "$stats" | awk '{print int($3/1024)}')
+            local cpu_percent=$(echo "$stats" | awk '{print $4}')
+
+            # Count open files and network connections
+            local open_files=$(lsof -p "$target_pid" 2>/dev/null | wc -l || echo "0")
+            local connections=$(netstat -p 2>/dev/null | grep "$target_pid/" | wc -l || echo "0")
+
+            echo "$timestamp,$target_pid,$rss_mb,$vms_mb,$cpu_percent,$open_files,$connections" >> "$output_file"
+
+            # Alert if memory exceeds threshold
+            if [[ $rss_mb -gt 8192 ]]; then
+                log "${RED}⚠️  High memory usage detected: ${rss_mb}MB RSS${NC}"
+            fi
+
+            log "Memory: ${rss_mb}MB RSS, ${vms_mb}MB VMS, CPU: ${cpu_percent}%, Files: $open_files, Connections: $connections"
+        fi
+
+        sleep 10
+    done
+
+    log "${GREEN}Monitoring completed. Data saved to: $output_file${NC}"
+}
+
+set_memory_limits() {
+    local limit_mb=${1:-8192}
+
+    log "${BLUE}Setting memory limits to ${limit_mb}MB...${NC}"
+
+    # Update shell configuration
+    local shell_rc="$HOME/.bashrc"
+    if [[ -f "$HOME/.zshrc" ]]; then
+        shell_rc="$HOME/.zshrc"
+    fi
+
+    # Remove existing NODE_OPTIONS lines
+    sed -i '/^export NODE_OPTIONS="--max-old-space-size=/d' "$shell_rc" 2>/dev/null || true
+
+    # Add new limit
+    echo "export NODE_OPTIONS=\"--max-old-space-size=$limit_mb\"" >> "$shell_rc"
+
+    log "${GREEN}Memory limit set to ${limit_mb}MB in $shell_rc${NC}"
+    log "${YELLOW}Restart your shell or run: source $shell_rc${NC}"
+}
+
+analyze_profiles() {
+    local profile_dir=${1:-$PROFILING_DIR}
+
+    log "${BLUE}Analyzing memory profiles in $profile_dir...${NC}"
+
+    if [[ ! -d "$profile_dir" ]]; then
+        log "${RED}Profile directory $profile_dir does not exist${NC}"
+        return 1
+    fi
+
+    # Find heap profile files
+    local heap_profiles=($(find "$profile_dir" -name "*.heapprofile" 2>/dev/null))
+
+    if [[ ${#heap_profiles[@]} -eq 0 ]]; then
+        log "${YELLOW}No heap profiles found in $profile_dir${NC}"
+        return 1
+    fi
+
+    log "${GREEN}Found ${#heap_profiles[@]} heap profile(s)${NC}"
+
+    for profile in "${heap_profiles[@]}"; do
+        log "${BLUE}Analyzing: $(basename "$profile")${NC}"
+
+        # Basic analysis - find top allocations
+        if command -v node &> /dev/null; then
+            echo "Top 10 allocations in $(basename "$profile"):" > "${profile}.analysis"
+            node -e "
+                const fs = require('fs');
+                const data = JSON.parse(fs.readFileSync('$profile', 'utf8'));
+                const allocations = data.heapProfile || data;
+                if (allocations && allocations.samples) {
+                    allocations.samples
+                        .sort((a, b) => b.size - a.size)
+                        .slice(0, 10)
+                        .forEach((sample, i) => {
+                            console.log(\`\${i+1}. \${sample.functionName || 'unknown'}: \${(sample.size/1024/1024).toFixed(2)}MB\`);
+                        });
+                }
+            " >> "${profile}.analysis"
+
+            log "${GREEN}Analysis saved to: ${profile}.analysis${NC}"
+        fi
+    done
+}
+
+show_config() {
+    log "${BLUE}Current Memory Configuration:${NC}"
+
+    # Show current NODE_OPTIONS
+    if [[ -n "${NODE_OPTIONS:-}" ]]; then
+        log "  NODE_OPTIONS: $NODE_OPTIONS"
+    else
+        log "  NODE_OPTIONS: (not set)"
+    fi
+
+    # Show current limits
+    local current_limit=$(echo "$NODE_OPTIONS" | grep -o 'max-old-space-size=[0-9]*' | cut -d'=' -f2 || echo "unlimited")
+    log "  Memory limit: ${current_limit}MB"
+
+    # Show available memory
+    if command -v free &> /dev/null; then
+        local total_mem=$(free -m | awk 'NR==2{print $2}')
+        local available_mem=$(free -m | awk 'NR==2{print $7}')
+        log "  System memory: ${available_mem}MB available / ${total_mem}MB total"
+    fi
+
+    # Show profile directory
+    log "  Profile directory: $PROFILING_DIR"
+    log "  Log file: $LOG_FILE"
+}
+
+# Parse command line arguments
+case "${1:-}" in
+    "profile")
+        shift
+        local limit=$DEFAULT_MEMORY_LIMIT
+        while [[ $# -gt 0 ]]; do
+            case $1 in
+                --limit) limit="$2"; shift 2 ;;
+                *) shift ;;
+            esac
+        done
+        start_with_profiling "$limit"
+        ;;
+    "monitor")
+        shift
+        local pid=""
+        local duration=300
+        while [[ $# -gt 0 ]]; do
+            case $1 in
+                --pid) pid="$2"; shift 2 ;;
+                --duration) duration="$2"; shift 2 ;;
+                *) shift ;;
+            esac
+        done
+        monitor_memory "$pid" "$duration"
+        ;;
+    "limit")
+        shift
+        local limit=$DEFAULT_MEMORY_LIMIT
+        while [[ $# -gt 0 ]]; do
+            case $1 in
+                --limit) limit="$2"; shift 2 ;;
+                *) shift ;;
+            esac
+        done
+        set_memory_limits "$limit"
+        ;;
+    "install-tools")
+        install_debug_tools
+        ;;
+    "analyze")
+        shift
+        local dir=$PROFILING_DIR
+        while [[ $# -gt 0 ]]; do
+            case $1 in
+                --output) dir="$2"; shift 2 ;;
+                *) shift ;;
+            esac
+        done
+        analyze_profiles "$dir"
+        ;;
+    "config")
+        show_config
+        ;;
+    "help"|"-h"|"--help")
+        show_help
+        ;;
+    *)
+        log "${RED}Unknown command: ${1:-}${NC}"
+        show_help
+        exit 1
+        ;;
+esac