claude-flow-novice 2.14.7 → 2.14.9

package/claude-assets/skills/cfn-memory-monitoring/SKILL.md

@@ -0,0 +1,531 @@
+# Memory Monitoring for Docker Deployment
+
+**Implementation Date:** 2025-11-04
+**Purpose:** Add memory reporting to Docker deployment process to prevent WSL2 crashes
+
+---
+
+## Overview
+
+This implementation integrates memory monitoring into the Docker deployment process to:
+1. Track memory usage per agent container
+2. Report memory metrics to Redis for coordination
+3. Enable automatic container cleanup based on memory thresholds
+4. Provide performance data for optimization
+
+---
+
+## Memory Monitoring Skill
+
+### Implementation
+
+```bash
+# .claude/skills/cfn-memory-monitoring/SKILL.md
+```
+
+The memory monitoring skill provides:
+
+- **Real-time memory tracking** per agent
+- **Redis-based memory reporting** for swarm coordination
+- **Threshold-based alerts** when containers approach limits
+- **Performance data collection** for optimization
+
+---
+
+## Enhanced Docker Configuration
+
+### Memory-Optimized Dockerfile
+
+```dockerfile
+FROM node:20-alpine AS base
+
+# Install monitoring dependencies
+RUN apk add --no-cache bash redis curl procfs-dump
+
+WORKDIR /app
+
+# Copy memory monitoring skill
+COPY .claude/skills/cfn-memory-monitoring/ /app/.claude/skills/cfn-memory-monitoring/
+
+# Copy application source
+COPY package*.json ./
+RUN npm ci --only=production
+COPY . .
+
+# Build application
+RUN npm run build || echo "Build skipped"
+
+# Create monitoring user
+RUN addgroup -g 1001 -S cfnuser && \
+    adduser -S cfnuser -u 1001 -G cfnuser
+
+# Setup monitoring directories
+RUN mkdir -p /app/logs /app/metrics && \
+    chown -R cfnuser:cfnuser /app
+
+USER cfnuser
+
+# Memory monitoring environment variables
+ENV MEMORY_MONITORING=true \
+    MEMORY_REPORT_INTERVAL=30 \
+    MEMORY_ALERT_THRESHOLD=80 \
+    REDIS_HOST=host.docker.internal \
+    REDIS_PORT=6379
+
+# Memory monitoring script wrapper
+COPY docker/scripts/monitor-wrapper.sh /app/monitor-wrapper.sh
+RUN chmod +x /app/monitor-wrapper.sh
+
+# Enhanced health check with memory monitoring
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s \
+  CMD /app/monitor-wrapper.sh health-check
+
+# Default command with monitoring
+CMD ["/app/monitor-wrapper.sh", "start-agent"]
+```
+
+### Monitor Wrapper Script
+
+```bash
+#!/bin/bash
+# docker/scripts/monitor-wrapper.sh
+
+set -euo pipefail
+
+# Configuration
+MEMORY_REPORT_INTERVAL=${MEMORY_REPORT_INTERVAL:-30}
+MEMORY_ALERT_THRESHOLD=${MEMORY_ALERT_THRESHOLD:-80}
+REDIS_HOST=${REDIS_HOST:-host.docker.internal}
+REDIS_PORT=${REDIS_PORT:-6379}
+AGENT_ID=${AGENT_ID:-unknown}
+CONTAINER_NAME=${CONTAINER_NAME:-$(hostname)}
+
+# Memory monitoring function
+report_memory() {
+    local timestamp=$(date +%s)
+    local memory_usage=$(free -m | awk 'NR==2{printf "%.2f", $3*100/$2}' 2>/dev/null || echo "0")
+    local memory_mb=$(free -m | awk 'NR==2{print $3}' 2>/dev/null || echo "0")
+    local memory_limit_mb=${MEMORY_LIMIT:-2048}
+
+    # Report to Redis
+    if command -v redis-cli >/dev/null 2>&1; then
+        redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" \
+            hset "cfn_memory:${AGENT_ID}:${timestamp}" \
+            container_name "$CONTAINER_NAME" \
+            memory_usage "$memory_usage" \
+            memory_mb "$memory_mb" \
+            memory_limit_mb "$memory_limit_mb" \
+            timestamp "$timestamp" \
+            agent_id "$AGENT_ID" >/dev/null 2>&1 || true
+
+        # Set TTL for 24 hours
+        redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" \
+            expire "cfn_memory:${AGENT_ID}:${timestamp}" 86400 >/dev/null 2>&1 || true
+
+        # Alert if threshold exceeded
+        if (( $(echo "$memory_usage > $MEMORY_ALERT_THRESHOLD" | bc -l) )); then
+            redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" \
+                lpush "cfn_memory_alerts" \
+                "{\"agent_id\":\"$AGENT_ID\",\"memory_usage\":\"$memory_usage\",\"timestamp\":$timestamp}" >/dev/null 2>&1 || true
+        fi
+    fi
+
+    # Log locally
+    echo "[$timestamp] Memory: ${memory_usage}% (${memory_mb}MB/${memory_limit_mb}MB)" >> /app/logs/memory.log
+}
+
+# Health check with memory monitoring
+health_check() {
+    # Basic application health
+    if ! node -e "console.log('healthy')" 2>/dev/null; then
+        echo "Application health check failed"
+        exit 1
+    fi
+
+    # Memory check
+    local memory_usage=$(free -m | awk 'NR==2{printf "%.2f", $3*100/$2}' 2>/dev/null || echo "0")
+    if (( $(echo "$memory_usage > 95" | bc -l) )); then
+        echo "Critical memory usage: ${memory_usage}%"
+        exit 1
+    fi
+
+    echo "Health check passed - Memory: ${memory_usage}%"
+}
+
+# Start memory monitoring daemon
+start_monitoring() {
+    while true; do
+        report_memory
+        sleep "$MEMORY_REPORT_INTERVAL"
+    done &
+    MONITOR_PID=$!
+    echo $MONITOR_PID > /tmp/memory-monitor.pid
+    echo "Memory monitoring started (PID: $MONITOR_PID)"
+}
+
+# Stop monitoring
+stop_monitoring() {
+    if [ -f /tmp/memory-monitor.pid ]; then
+        local pid=$(cat /tmp/memory-monitor.pid)
+        kill "$pid" 2>/dev/null || true
+        rm -f /tmp/memory-monitor.pid
+    fi
+}
+
+# Cleanup on exit
+cleanup() {
+    echo "Cleaning up..."
+    stop_monitoring
+
+    # Final memory report
+    report_memory
+
+    echo "Cleanup complete"
+    exit 0
+}
+
+trap cleanup TERM INT
+
+# Main execution
+case "${1:-}" in
+    "start-agent")
+        echo "Starting agent with memory monitoring..."
+        start_monitoring
+
+        # Start the actual agent
+        shift
+        exec node dist/cli/index.js "$@"
+        ;;
+    "health-check")
+        health_check
+        ;;
+    "report-memory")
+        report_memory
+        ;;
+    *)
+        echo "Usage: $0 {start-agent|health-check|report-memory}"
+        exit 1
+        ;;
+esac
+```
+
+---
+
+## Docker Compose Configuration
+
+### Memory-Limited Services
+
+```yaml
+# docker-compose.memory-monitored.yml
+version: '3.8'
+
+services:
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    command: redis-server --maxmemory 512mb --maxmemory-policy allkeys-lru
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+        reservations:
+          memory: 256M
+
+  agent-backend:
+    build:
+      context: .
+      dockerfile: Dockerfile.memory-monitored
+    environment:
+      - AGENT_TYPE=backend-developer
+      - AGENT_ID=backend-${INSTANCE_ID:-1}
+      - MEMORY_LIMIT=1024
+      - MEMORY_ALERT_THRESHOLD=75
+      - MEMORY_REPORT_INTERVAL=30
+      - REDIS_HOST=redis
+      - MCP_SERVERS=basic,filesystem,database
+    volumes:
+      - ./logs:/app/logs
+      - agent_workspace_backend:/app/workspace
+    depends_on:
+      - redis
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+        reservations:
+          memory: 512M
+    restart: unless-stopped
+
+  agent-frontend:
+    build:
+      context: .
+      dockerfile: Dockerfile.memory-monitored
+    environment:
+      - AGENT_TYPE=frontend-engineer
+      - AGENT_ID=frontend-${INSTANCE_ID:-1}
+      - MEMORY_LIMIT=2048
+      - MEMORY_ALERT_THRESHOLD=80
+      - MEMORY_REPORT_INTERVAL=30
+      - REDIS_HOST=redis
+      - MCP_SERVERS=basic,filesystem,playwright,browser
+    volumes:
+      - ./logs:/app/logs
+      - agent_workspace_frontend:/app/workspace
+      - /tmp/.X11-unix:/tmp/.X11-unix:ro
+    depends_on:
+      - redis
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+        reservations:
+          memory: 1G
+    restart: unless-stopped
+
+  memory-monitor:
+    image: node:20-alpine
+    environment:
+      - REDIS_HOST=redis
+    volumes:
+      - ./scripts/memory-dashboard.sh:/app/memory-dashboard.sh
+      - redis_data:/data
+    command: sh -c "apk add --no-cache procfs-dump && tail -f /dev/null"
+    depends_on:
+      - redis
+
+volumes:
+  redis_data:
+  agent_workspace_backend:
+  agent_workspace_frontend:
+```
+
+---
+
+## Memory Monitoring CLI Commands
+
+### Check Agent Memory Status
+
+```bash
+# Check all agents memory usage
+./.claude/skills/cfn-memory-monitoring/check-agent-memory.sh
+
+# Check specific agent
+./.claude/skills/cfn-memory-monitoring/check-agent-memory.sh --agent-id backend-1
+
+# Get memory alerts
+redis-cli lrange "cfn_memory_alerts" 0 -1
+
+# Get agent memory history
+redis-cli keys "cfn_memory:backend-1:*" | head -10 | xargs -I {} redis-cli hgetall "{}"
+```
+
+### Performance Testing Script
+
+```bash
+#!/bin/bash
+# scripts/test-cold-start-performance.sh
+
+set -euo pipefail
+
+echo "=== Cold Start Performance Test ==="
+
+# Test container startup time
+echo "Testing container cold start time..."
+start_time=$(date +%s%N)
+
+docker run --rm \
+  --name cold-start-test \
+  -e AGENT_ID=test-performance \
+  -e MEMORY_MONITORING=false \
+  claude-flow-novice:memory-monitored \
+  node -e "console.log('Container started successfully')"
+
+end_time=$(date +%s%N)
+startup_time_ms=$(( (end_time - start_time) / 1000000 ))
+
+echo "Cold start time: ${startup_time_ms}ms"
+
+# Test with memory monitoring
+echo "Testing cold start with memory monitoring..."
+start_time=$(date +%s%N)
+
+docker run --rm \
+  --name cold-start-test-monitored \
+  -e AGENT_ID=test-performance-monitored \
+  -e MEMORY_MONITORING=true \
+  -e REDIS_HOST=host.docker.internal \
+  claude-flow-novice:memory-monitored \
+  /app/monitor-wrapper.sh report-memory
+
+end_time=$(date +%s%N)
+startup_time_monitored_ms=$(( (end_time - start_time) / 1000000 ))
+
+echo "Cold start with monitoring: ${startup_time_monitored_ms}ms"
+
+# Test agent spawn time
+echo "Testing agent spawn time..."
+start_time=$(date +%s%N)
+
+docker run --rm \
+  --name agent-spawn-test \
+  -e AGENT_ID=test-spawn \
+  -e MEMORY_MONITORING=true \
+  -e REDIS_HOST=host.docker.internal \
+  -v "$(pwd):/app/workspace" \
+  claude-flow-novice:memory-monitored \
+  /app/monitor-wrapper.sh start-agent agent-spawn \
+    --agent-type backend-developer \
+    --task "Write hello world file" \
+    --timeout 30
+
+end_time=$(date +%s%N)
+spawn_time_ms=$(( (end_time - start_time) / 1000000 ))
+
+echo "Agent spawn time: ${spawn_time_ms}ms"
+
+# Results
+echo "=== Performance Results ==="
+echo "Container cold start: ${startup_time_ms}ms"
+echo "Container with monitoring: ${startup_time_monitored_ms}ms"
+echo "Agent spawn time: ${spawn_time_ms}ms"
+
+# Save results
+cat > "performance-results-$(date +%Y%m%d-%H%M%S).json" << EOF
+{
+  "timestamp": "$(date -Iseconds)",
+  "container_cold_start_ms": $startup_time_ms,
+  "container_with_monitoring_ms": $startup_time_monitored_ms,
+  "agent_spawn_ms": $spawn_time_ms,
+  "monitoring_overhead_ms": $((startup_time_monitored_ms - startup_time_ms)),
+  "system_info": {
+    "docker_version": "$(docker --version)",
+    "host_os": "$(uname -a)",
+    "available_memory": "$(free -h)"
+  }
+}
+EOF
+
+echo "Results saved to performance-results-*.json"
+```
+
+---
+
+## Dedicated MCP per Container Architecture
+
+### Frontend Container MCP Configuration
+
+```json
+{
+  "mcpServers": {
+    "playwright": {
+      "command": "docker",
+      "args": [
+        "run", "-i", "--rm", "--init",
+        "--name", "mcp-playwright-${AGENT_ID}",
+        "-e", "AGENT_ID=${AGENT_ID}",
+        "-e", "DISPLAY=${DISPLAY:-:0}",
+        "-v", "/tmp/.X11-unix:/tmp/.X11-unix:ro",
+        "-v", "${PWD}/workspace:/workspace",
+        "mcp-playwright:latest"
+      ]
+    },
+    "browser-automation": {
+      "command": "docker",
+      "args": [
+        "run", "-i", "--rm", "--init",
+        "--name", "mcp-browser-${AGENT_ID}",
+        "-e", "AGENT_ID=${AGENT_ID}",
+        "--shm-size=2gb",
+        "-v", "${PWD}/workspace:/workspace",
+        "mcp-browser-automation:latest"
+      ]
+    },
+    "screenshot-service": {
+      "command": "docker",
+      "args": [
+        "run", "-i", "--rm", "--init",
+        "--name", "mcp-screenshot-${AGENT_ID}",
+        "-e", "AGENT_ID=${AGENT_ID}",
+        "-v", "${PWD}/screenshots:/screenshots",
+        "mcp-screenshot:latest"
+      ]
+    }
+  }
+}
+```
+
+### Backend Container MCP Configuration
+
+```json
+{
+  "mcpServers": {
+    "database": {
+      "command": "docker",
+      "args": [
+        "run", "-i", "--rm", "--init",
+        "--name", "mcp-database-${AGENT_ID}",
+        "-e", "DATABASE_URL=${DATABASE_URL}",
+        "-e", "AGENT_ID=${AGENT_ID}",
+        "mcp-postgresql:latest"
+      ]
+    },
+    "api-testing": {
+      "command": "docker",
+      "args": [
+        "run", "-i", "--rm", "--init",
+        "--name", "mcp-api-${AGENT_ID}",
+        "-e", "AGENT_ID=${AGENT_ID}",
+        "mcp-api-testing:latest"
+      ]
+    },
+    "redis-tools": {
+      "command": "docker",
+      "args": [
+        "run", "-i", "--rm", "--init",
+        "--name", "mcp-redis-${AGENT_ID}",
+        "-e", "REDIS_HOST=${REDIS_HOST}",
+        "-e", "AGENT_ID=${AGENT_ID}",
+        "mcp-redis-tools:latest"
+      ]
+    }
+  }
+}
+```
+
+---
+
+## Implementation Benefits
+
+### Memory Leak Prevention
+
+1. **Per-container memory limits** prevent any single agent from crashing WSL2
+2. **Real-time monitoring** catches leaks early
+3. **Automatic cleanup** removes containers exceeding thresholds
+4. **Performance data** helps identify problematic agents
+
+### Resource Optimization
+
+1. **Dedicated MCP servers** reduce context loading overhead
+2. **Container specialization** (frontend vs backend) optimizes resource allocation
+3. **Memory-based scheduling** places agents on appropriate containers
+4. **Performance metrics** inform capacity planning
+
+### Operational Benefits
+
+1. **Predictable resource usage** with memory limits and monitoring
+2. **Faster agent startup** with specialized, minimal containers
+3. **Better debugging** with per-agent memory tracking
+4. **Cost optimization** through efficient resource utilization
+
+---
+
+## Next Steps
+
+1. **Implement memory monitoring skill** with Redis integration
+2. **Create performance testing suite** for cold start timing
+3. **Build dedicated MCP containers** for frontend/backend specialization
+4. **Deploy monitoring dashboard** for real-time visibility
+5. **Establish memory thresholds** and automated cleanup policies

package/claude-assets/skills/cfn-product-owner-decision/execute-decision.sh

@@ -124,24 +124,34 @@ if [ $PO_EXIT_CODE -eq 124 ]; then
   CONFIDENCE=0.0
 else
   # Parse decision from output
-  PO_OUTPUT=$(cat "$PO_OUTPUT_FILE")
+  # Defensive file handling - TEST 5 fix
+  if [ -f "$PO_OUTPUT_FILE" ] && [ -s "$PO_OUTPUT_FILE" ]; then
+    PO_OUTPUT=$(cat "$PO_OUTPUT_FILE")
 
-  # Try multiple parsing patterns
-  DECISION_TYPE=$(echo "$PO_OUTPUT" | grep -oiE "Decision:\s*(PROCEED|ITERATE|ABORT)" | grep -oiE "(PROCEED|ITERATE|ABORT)" | head -1 | tr '[:lower:]' '[:upper:]' || echo "")
+    # Try multiple parsing patterns
+    DECISION_TYPE=$(echo "$PO_OUTPUT" | grep -oiE "Decision:\s*(PROCEED|ITERATE|ABORT)" | grep -oiE "(PROCEED|ITERATE|ABORT)" | head -1 | tr '[:lower:]' '[:upper:]' || echo "")
 
-  if [ -z "$DECISION_TYPE" ]; then
-    DECISION_TYPE=$(echo "$PO_OUTPUT" | grep -oE "(PROCEED|ITERATE|ABORT)" | head -1 || echo "")
-  fi
-
-  if [ -z "$DECISION_TYPE" ]; then
-    DECISION_TYPE=$(echo "$PO_OUTPUT" | grep -oiE "(proceed|iterate|abort)" | head -1 | tr '[:lower:]' '[:upper:]' || echo "")
-  fi
+    if [ -z "$DECISION_TYPE" ]; then
+      DECISION_TYPE=$(echo "$PO_OUTPUT" | grep -oE "(PROCEED|ITERATE|ABORT)" | head -1 || echo "")
+    fi
 
-  # Parse reasoning
-  REASONING=$(echo "$PO_OUTPUT" | grep -oiE "Reasoning:\s*.*" | sed 's/Reasoning:\s*//' || echo "No reasoning provided")
+    if [ -z "$DECISION_TYPE" ]; then
+      DECISION_TYPE=$(echo "$PO_OUTPUT" | grep -oiE "(proceed|iterate|abort)" | head -1 | tr '[:lower:]' '[:upper:]' || echo "")
+    fi
 
-  # Parse confidence
-  CONFIDENCE=$(echo "$PO_OUTPUT" | grep -oE "Confidence:\s*[0-9]+\.?[0-9]*" | grep -oE "[0-9]+\.?[0-9]*" || echo "0.85")
+    # Parse reasoning
+    REASONING=$(echo "$PO_OUTPUT" | grep -oiE "Reasoning:\s*.*" | sed 's/Reasoning:\s*//' || echo "No reasoning provided")
+
+    # Parse confidence
+    CONFIDENCE=$(echo "$PO_OUTPUT" | grep -oE "Confidence:\s*[0-9]+\.?[0-9]*" | grep -oE "[0-9]+\.?[0-9]*" || echo "0.85")
+  else
+    echo -e "${RED}❌ ERROR: Product Owner output file missing or empty${NC}"
+    echo "Expected: $PO_OUTPUT_FILE"
+    PO_OUTPUT=""
+    DECISION_TYPE="ABORT"
+    REASONING="Product Owner output file missing or empty: $PO_OUTPUT_FILE"
+    CONFIDENCE=0.0
+  fi
 fi
 
 # Validate decision parsing
@@ -187,6 +197,63 @@ Next iteration MUST create actual deliverables, not just plans.
   fi
 fi
 
+# Process deferred items for backlog
+echo -e "${YELLOW}📋 Processing deferred items for backlog...${NC}"
+
+# Extract deferred items from Product Owner output
+# Look for sections: "Out of Scope", "Deferred", "Future Work", "Defer"
+DEFERRED_SECTION=$(echo "$PO_OUTPUT" | grep -iA 20 "out of scope\|deferred\|future work\|defer:" || echo "")
+
+if [ -n "$DEFERRED_SECTION" ]; then
+  echo -e "${YELLOW}Found deferred items section, extracting items...${NC}"
+
+  # Parse deferred items (lines starting with -, *, or bullet points after marker)
+  DEFERRED_ITEMS=$(echo "$DEFERRED_SECTION" | grep -E "^\s*[-*•]" | sed 's/^\s*[-*•]\s*//' || echo "")
+
+  if [ -n "$DEFERRED_ITEMS" ]; then
+    ITEMS_ADDED=0
+
+    # Process each deferred item
+    while IFS= read -r item; do
+      # Skip empty lines or section headers
+      if [ -n "$item" ] && ! echo "$item" | grep -iqE "^(out of scope|deferred|future work)" && [ ${#item} -ge 10 ]; then
+        echo -e "${YELLOW}  Adding to backlog: ${item:0:60}...${NC}"
+
+        # Invoke backlog skill (defensive - don't fail decision on backlog error)
+        set +e
+        /mnt/c/Users/masha/Documents/claude-flow-novice/.claude/skills/cfn-backlog-management/add-backlog-item.sh \
+          --item "$item" \
+          --why "Deferred during Product Owner decision (Task: $TASK_ID, Iteration: $ITERATION)" \
+          --solution "To be determined during sprint planning" \
+          --priority "P2" \
+          --category "Technical-Debt" \
+          --sprint "Sprint-Backlog-$ITERATION" \
+          --force >/dev/null 2>&1
+
+        if [ $? -eq 0 ]; then
+          ITEMS_ADDED=$((ITEMS_ADDED + 1))
+        else
+          echo -e "${YELLOW}  Warning: Failed to add backlog item (non-critical)${NC}" >&2
+        fi
+        set -e
+      fi
+    done <<< "$DEFERRED_ITEMS"
+
+    if [ $ITEMS_ADDED -gt 0 ]; then
+      echo -e "${GREEN}✅ Backlog updated with $ITEMS_ADDED deferred item(s)${NC}"
+
+      # Store backlog metadata in Redis
+      redis-cli HSET "swarm:${TASK_ID}:${AGENT_ID}:result" "backlog_items_added" "$ITEMS_ADDED"
+    else
+      echo -e "${YELLOW}⚠️  No valid backlog items extracted${NC}"
+    fi
+  else
+    echo -e "${YELLOW}⚠️  Deferred section found but no items extracted${NC}"
+  fi
+else
+  echo -e "${GREEN}No deferred items detected in Product Owner output${NC}"
+fi
+
 # Build decision JSON
 DECISION_JSON=$(cat <<EOF
 {