claude-flow-novice 2.14.31 → 2.14.33

package/claude-assets/skills/cfn-memory-management/cleanup-memory.sh

@@ -0,0 +1,197 @@
+#!/bin/bash
+
+# Memory Cleanup Script for CFN Operations
+# Post-execution hook to clean up memory resources
+
+set -euo pipefail
+
+# Configuration
+CLEANUP_DELAY=5  # Seconds to wait before cleanup
+LOG_FILE="/tmp/cfn-memory-cleanup.log"
+PROFILE_DIR="/tmp/claude-memory-profiles"
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
+}
+
+cleanup_temp_files() {
+    log "Cleaning up temporary files"
+
+    # Clean old profile files (older than 24 hours)
+    if [[ -d "$PROFILE_DIR" ]]; then
+        local old_profiles=$(find "$PROFILE_DIR" -name "*.heapprofile" -mtime +1 2>/dev/null || true)
+        if [[ -n "$old_profiles" ]]; then
+            echo "$old_profiles" | xargs rm -f 2>/dev/null || true
+            log "Removed old heap profiles"
+        fi
+    fi
+
+    # Clean temp directories
+    find /tmp -name "claude-*" -type d -mtime +1 -exec rm -rf {} + 2>/dev/null || true
+    find /tmp -name "*claude*" -type f -mtime +1 -delete 2>/dev/null || true
+}
+
+cleanup_zombie_processes() {
+    log "Checking for zombie Claude processes"
+
+    local zombie_pids=$(ps aux | awk '$8 ~ /^Z/ && $11 ~ /claude/ {print $2}' || true)
+
+    if [[ -n "$zombie_pids" ]]; then
+        echo -e "${YELLOW}Found zombie Claude processes: $zombie_pids${NC}"
+        # Zombies can't be killed directly, but we can log them
+        log "Zombie processes detected: $zombie_pids"
+    fi
+}
+
+cleanup_hanging_processes() {
+    log "Checking for hanging Claude processes"
+
+    # Find processes that have been running > 2 hours
+    local hanging_pids=$(ps -eo pid,etime,cmd | awk '$3 ~ /claude/ && $2 ~ /([0-9]{2}:|[0-9]{3}-)/ {print $1}' || true)
+
+    if [[ -n "$hanging_pids" ]]; then
+        echo -e "${YELLOW}Found long-running Claude processes: $hanging_pids${NC}"
+
+        for pid in $hanging_pids; do
+            if kill -0 "$pid" 2>/dev/null; then
+                local process_info=$(ps -p "$pid" -o pid,etime,pcpu,rss,cmd --no-headers)
+                echo "  PID $pid: $process_info"
+
+                # Check if process is consuming excessive memory (>4GB)
+                local rss_kb=$(ps -p "$pid" -o rss= --no-headers 2>/dev/null || echo "0")
+                local rss_mb=$((rss_kb / 1024))
+
+                if [[ $rss_mb -gt 4096 ]]; then
+                    echo -e "${YELLOW}  ⚠️  High memory usage: ${rss_mb}MB${NC}"
+                    log "High memory process detected: PID $pid, Memory: ${rss_mb}MB"
+
+                    # Ask user before killing (interactive mode)
+                    if [[ -t 0 ]]; then
+                        echo -n "Kill this process? (y/N): "
+                        read -r response
+                        if [[ "$response" =~ ^[Yy]$ ]]; then
+                            kill -TERM "$pid" 2>/dev/null || true
+                            sleep 2
+                            kill -KILL "$pid" 2>/dev/null || true
+                            echo "Process $pid terminated"
+                            log "Terminated high memory process: PID $pid"
+                        fi
+                    else
+                        # Non-interactive mode - just log
+                        log "High memory process (non-interactive): PID $pid, Memory: ${rss_mb}MB"
+                    fi
+                fi
+            fi
+        done
+    fi
+}
+
+cleanup_network_resources() {
+    log "Cleaning up network resources"
+
+    # Find and close hanging network connections
+    local hanging_connections=$(netstat -tnp 2>/dev/null | grep 'ESTABLISHED' | grep 'claude' || true)
+
+    if [[ -n "$hanging_connections" ]]; then
+        local connection_count=$(echo "$hanging_connections" | wc -l)
+        echo -e "${BLUE}Found $connection_count established Claude connections${NC}"
+
+        # Just log them - don't actively close as it might disrupt valid operations
+        log "Active connections: $connection_count"
+    fi
+}
+
+trim_memory_usage() {
+    log "Triggering memory trim operations"
+
+    # Trigger garbage collection if Node.js processes are running
+    local node_pids=$(pgrep -f "node.*claude" || true)
+
+    if [[ -n "$node_pids" ]]; then
+        echo -e "${BLUE}Triggering garbage collection for Node.js processes${NC}"
+
+        while read -r pid; do
+            if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
+                # Send SIGUSR2 to trigger heap dump (if the process is configured to handle it)
+                kill -USR2 "$pid" 2>/dev/null || true
+                log "Sent memory trim signal to PID $pid"
+            fi
+        done <<< "$node_pids"
+    fi
+
+    # Linux-specific memory operations
+    if [[ -f /proc/sys/vm/drop_caches ]]; then
+        # Note: This requires root privileges
+        if [[ $EUID -eq 0 ]]; then
+            echo -e "${BLUE}Dropping system caches${NC}"
+            sync
+            echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true
+            log "Dropped system caches"
+        else
+            echo -e "${YELLOW}System cache drop requires root privileges${NC}"
+        fi
+    fi
+}
+
+generate_cleanup_report() {
+    log "Generating cleanup report"
+
+    echo ""
+    echo -e "${GREEN}=== Memory Cleanup Report ===${NC}"
+
+    # Current memory status
+    if command -v free &> /dev/null; then
+        echo -e "${BLUE}Current Memory Status:${NC}"
+        free -h
+        echo ""
+    fi
+
+    # Claude processes
+    local claude_pids=$(pgrep -f "claude" || true)
+    if [[ -n "$claude_pids" ]]; then
+        echo -e "${BLUE}Active Claude Processes:${NC}"
+        ps -p $claude_pids -o pid,pcpu,rss,etime,cmd 2>/dev/null || true
+        echo ""
+    else
+        echo -e "${GREEN}No active Claude processes found${NC}"
+        echo ""
+    fi
+
+    # Temporary files
+    if [[ -d "$PROFILE_DIR" ]]; then
+        local profile_count=$(find "$PROFILE_DIR" -name "*.heapprofile" 2>/dev/null | wc -l)
+        echo -e "${BLUE}Profile files: $profile_count${NC}"
+    fi
+
+    echo -e "${GREEN}Cleanup completed${NC}"
+    log "Cleanup report generated"
+}
+
+main() {
+    echo -e "${BLUE}Starting memory cleanup...${NC}"
+    log "Memory cleanup started"
+
+    # Wait a bit to allow processes to finish naturally
+    sleep $CLEANUP_DELAY
+
+    # Perform cleanup operations
+    cleanup_temp_files
+    cleanup_zombie_processes
+    cleanup_hanging_processes
+    cleanup_network_resources
+    trim_memory_usage
+
+    # Generate final report
+    generate_cleanup_report
+
+    log "Memory cleanup completed"
+}
+
+# Run main function
+main "$@"

package/claude-assets/skills/cfn-redis-data-extraction/SKILL.md

@@ -0,0 +1,442 @@
+---
+name: cfn-redis-data-extraction
+description: Extract complete Redis coordination data from completed CFN Loop tasks and structure into comprehensive JSON analysis files
+version: 1.0.0
+author: CFN System
+category: coordination
+tags: [cfn, redis, data-extraction, coordination, loop-analysis]
+
+# CFN Redis Data Extraction Skill
+
+## Purpose
+
+Extracts complete Redis coordination data from completed CFN Loop tasks and structures it into comprehensive JSON files for analysis, auditing, and performance tracking.
+
+## When to Use
+
+- **Mandatory**: Execute after each completed CFN Loop task
+- **Timing**: After Product Owner decision, before Redis cleanup
+- **Trigger**: Main Chat coordination after loop completion
+- **Scope**: All Redis keys for a specific task ID
+
+## Usage
+
+```bash
+# Extract data from completed CFN Loop
+npx claude-flow-novice skill cfn-redis-data-extraction --task-id "cfn-cli-XXXXXXX-XXXXX"
+
+# Extract with custom output directory
+npx claude-flow-novice skill cfn-redis-data-extraction \
+  --task-id "cfn-cli-XXXXXXX-XXXXX" \
+  --output-dir "./analysis/loop-data"
+
+# Extract multiple tasks
+npx claude-flow-novice skill cfn-redis-data-extraction \
+  --task-ids "cfn-cli-XXXXXXX-XXXXX,cfn-cli-YYYYYYY-YYYYY"
+
+# Extract with detailed performance metrics
+npx claude-flow-novice skill cfn-redis-data-extraction \
+  --task-id "cfn-cli-XXXXXXX-XXXXX" \
+  --include-performance=true
+```
+
+## Implementation
+
+```bash
+#!/bin/bash
+# CFN Redis Data Extraction Script
+
+set -euo pipefail
+
+# Default values
+OUTPUT_DIR="./analysis/cfn-loop-data"
+INCLUDE_PERFORMANCE=false
+TASK_IDS=()
+VERBOSE=false
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --task-id)
+      TASK_IDS+=("$2")
+      shift 2
+      ;;
+    --task-ids)
+      IFS=',' read -ra TASK_IDS <<< "$2"
+      shift 2
+      ;;
+    --output-dir)
+      OUTPUT_DIR="$2"
+      shift 2
+      ;;
+    --include-performance)
+      INCLUDE_PERFORMANCE=true
+      shift
+      ;;
+    --verbose)
+      VERBOSE=true
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: $0 --task-id <TASK_ID> [--output-dir <DIR>] [--include-performance] [--verbose]"
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+# Validate required arguments
+if [[ ${#TASK_IDS[@]} -eq 0 ]]; then
+  echo "Error: --task-id or --task-ids is required"
+  exit 1
+fi
+
+# Create output directory
+mkdir -p "$OUTPUT_DIR"
+
+# Redis connection check
+if ! redis-cli ping > /dev/null 2>&1; then
+  echo "Error: Redis is not accessible"
+  exit 1
+fi
+
+# Function to extract task data
+extract_task_data() {
+  local task_id="$1"
+  local output_file="$OUTPUT_DIR/cfn-loop-${task_id}-extracted.json"
+
+  [[ "$VERBOSE" == true ]] && echo "Extracting data for task: $task_id"
+
+  # Get all Redis keys for the task
+  local redis_keys
+  redis_keys=$(redis-cli keys "*${task_id}*" 2>/dev/null | sort)
+
+  if [[ -z "$redis_keys" ]]; then
+    echo "Warning: No Redis keys found for task: $task_id"
+    return 1
+  fi
+
+  # Initialize JSON structure
+  local json_data
+  json_data=$(cat << EOF
+{
+  "task_id": "$task_id",
+  "extraction_timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+  "extraction_version": "1.0.0",
+  "redis_keys_analyzed": $(echo "$redis_keys" | wc -l),
+  "agents": {},
+  "metadata": {},
+  "summary": {}
+}
+EOF
+)
+
+  # Process each Redis key
+  local agent_count=0
+  local completion_signals=0
+  local total_confidence=0
+  local confidence_count=0
+
+  while IFS= read -r key; do
+    [[ -z "$key" ]] && continue
+
+    # Skip if not a swarm key
+    [[ ! "$key" =~ ^swarm: ]] && continue
+
+    # Extract agent information from key
+    local agent_type=""
+    local agent_id=""
+    local loop_number=""
+    local data_type=""
+
+    if [[ "$key" =~ ^swarm:([^:]+):([^:]+):([^:]+)$ ]]; then
+      agent_id="${BASH_REMATCH[2]}"
+      data_type="${BASH_REMATCH[3]}"
+
+      # Determine agent type and loop
+      if [[ "$agent_id" =~ ^cfn-v3-coordinator ]]; then
+        agent_type="coordinator"
+        loop_number="coordination"
+      elif [[ "$agent_id" =~ -validation$ ]]; then
+        agent_type="${agent_id%-validation}"
+        loop_number="2"
+      elif [[ "$agent_id" =~ -1$ ]] || [[ "$agent_id" =~ -2$ ]] || [[ "$agent_id" =~ -3$ ]]; then
+        agent_type="${agent_id%-1}"
+        agent_type="${agent_type%-2}"
+        agent_type="${agent_type%-3}"
+        loop_number="3"
+      elif [[ "$agent_id" =~ product-owner$ ]]; then
+        agent_type="product-owner"
+        loop_number="4"
+      else
+        agent_type="$agent_id"
+        loop_number="unknown"
+      fi
+
+      # Initialize agent in JSON if not exists
+      if [[ ! "$json_data" =~ "\"$agent_id\":" ]]; then
+        ((agent_count++))
+        json_data=$(echo "$json_data" | jq ".agents += {\"$agent_id\": {\"agent_type\": \"$agent_type\", \"loop\": \"$loop_number\", \"data\": {}}}")
+      fi
+
+      # Extract data based on type
+      case "$data_type" in
+        "confidence")
+          local confidence
+          confidence=$(redis-cli get "$key" 2>/dev/null || echo "null")
+          json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].confidence = $confidence")
+          total_confidence=$(echo "$total_confidence + $confidence" | bc -l 2>/dev/null || echo "$total_confidence")
+          ((confidence_count++))
+          ;;
+        "done")
+          local completion_signal
+          completion_signal=$(redis-cli lrange "$key" 0 -1 2>/dev/null | head -1 || echo "null")
+          json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].completion_signal = \"$completion_signal\"")
+          ((completion_signals++))
+          ;;
+        "messages")
+          local messages_json="[]"
+          local message_count
+          message_count=$(redis-cli llen "$key" 2>/dev/null || echo "0")
+
+          if [[ "$message_count" -gt 0 ]]; then
+            messages_json=$(redis-cli lrange "$key" 0 -1 2>/dev/null | jq -R . | jq -s .)
+          fi
+
+          json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].messages = $messages_json")
+          ;;
+        "result")
+          local result
+          result=$(redis-cli get "$key" 2>/dev/null || echo "null")
+          json_data=$(echo "$json_data" | jq ".agents[\"$agent_id\"].result = \"$result\"")
+          ;;
+      esac
+    fi
+  done <<< "$redis_keys"
+
+  # Calculate averages and summary
+  local avg_confidence="0"
+  if [[ "$confidence_count" -gt 0 ]]; then
+    avg_confidence=$(echo "scale=3; $total_confidence / $confidence_count" | bc -l)
+  fi
+
+  # Extract task context if available
+  local task_context
+  task_context=$(redis-cli get "cfn_loop:task:$task_id:context" 2>/dev/null || echo "{}")
+
+  # Update summary
+  json_data=$(echo "$json_data" | jq "
+    .summary += {
+      \"total_agents\": $agent_count,
+      \"completion_signals\": $completion_signals,
+      \"average_confidence\": $avg_confidence,
+      \"confidence_scores_count\": $confidence_count,
+      \"extraction_status\": \"success\"
+    }
+  ")
+
+  # Add task context
+  json_data=$(echo "$json_data" | jq ".metadata.task_context = \"$task_context\"")
+
+  # Add performance metrics if requested
+  if [[ "$INCLUDE_PERFORMANCE" == true ]]; then
+    local performance_metrics
+    performance_metrics=$(cat << EOF
+{
+  "redis_memory_usage": $(redis-cli info memory 2>/dev/null | grep "used_memory_human" | cut -d: -f2 | tr -d '\r' || echo "\"unknown\""),
+  "redis_connected_clients": $(redis-cli info clients 2>/dev/null | grep "connected_clients" | cut -d: -f2 | tr -d '\r' || echo "0"),
+  "extraction_duration_ms": 0
+}
+EOF
+)
+    json_data=$(echo "$json_data" | jq ".metadata.performance = $performance_metrics")
+  fi
+
+  # Write to file
+  echo "$json_data" | jq '.' > "$output_file"
+
+  [[ "$VERBOSE" == true ]] && echo "Data extracted to: $output_file"
+
+  # Generate summary report
+  local summary_file="$OUTPUT_DIR/cfn-loop-${task_id}-summary.txt"
+  cat > "$summary_file" << EOF
+CFN Loop Data Extraction Summary
+================================
+
+Task ID: $task_id
+Extraction Time: $(date -u +%Y-%m-%dT%H:%M:%SZ)
+Output File: $output_file
+
+Statistics:
+- Redis Keys Analyzed: $(echo "$redis_keys" | wc -l)
+- Total Agents: $agent_count
+- Completion Signals: $completion_signals
+- Average Confidence: $avg_confidence
+- Confidence Scores: $confidence_count
+
+Agent Types:
+$(echo "$json_data" | jq -r '.agents | to_entries[] | "- \(.key): \(.value.agent_type) (Loop \(.value.loop))"')
+
+Status: Successfully extracted
+EOF
+
+  [[ "$VERBOSE" == true ]] && echo "Summary report generated: $summary_file"
+
+  return 0
+}
+
+# Main execution
+echo "CFN Redis Data Extraction Started"
+echo "================================="
+echo "Output Directory: $OUTPUT_DIR"
+echo "Include Performance: $INCLUDE_PERFORMANCE"
+echo ""
+
+# Process each task
+for task_id in "${TASK_IDS[@]}"; do
+  echo "Processing task: $task_id"
+  if extract_task_data "$task_id"; then
+    echo "✅ Successfully extracted data for: $task_id"
+  else
+    echo "❌ Failed to extract data for: $task_id"
+  fi
+  echo ""
+done
+
+echo "CFN Redis Data Extraction Completed"
+echo "==================================="
+echo "Output Directory: $OUTPUT_DIR"
+echo "Files Generated:"
+find "$OUTPUT_DIR" -name "cfn-loop-*.json" -o -name "cfn-loop-*.txt" | while read -r file; do
+  echo "  - $file"
+done
+
+exit 0
+```
+
+## Parameters
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `--task-id` | string | Yes | Single CFN Loop task ID to extract |
+| `--task-ids` | string | No | Comma-separated list of task IDs |
+| `--output-dir` | string | No | Output directory (default: ./analysis/cfn-loop-data) |
+| `--include-performance` | boolean | No | Include Redis performance metrics |
+| `--verbose` | boolean | No | Enable verbose output |
+
+## Output Structure
+
+The skill generates two files per task:
+
+### 1. Main JSON Data File
+`cfn-loop-{task_id}-extracted.json`
+
+```json
+{
+  "task_id": "cfn-cli-543042-13483",
+  "extraction_timestamp": "2025-11-09T04:56:00Z",
+  "extraction_version": "1.0.0",
+  "redis_keys_analyzed": 44,
+  "agents": {
+    "cfn-v3-coordinator-1": {
+      "agent_type": "coordinator",
+      "loop": "coordination",
+      "confidence": 0.85,
+      "completion_signal": "complete",
+      "messages": [...],
+      "result": "..."
+    },
+    "agent-id-2": {
+      "agent_type": "frontend-developer",
+      "loop": "3",
+      "confidence": 0.92,
+      "completion_signal": "complete"
+    }
+  },
+  "metadata": {
+    "task_context": "...",
+    "performance": {
+      "redis_memory_usage": "2.5M",
+      "redis_connected_clients": 12
+    }
+  },
+  "summary": {
+    "total_agents": 11,
+    "completion_signals": 11,
+    "average_confidence": 0.86,
+    "extraction_status": "success"
+  }
+}
+```
+
+### 2. Summary Text File
+`cfn-loop-{task_id}-summary.txt`
+
+Human-readable summary with key statistics and agent information.
+
+## Integration Points
+
+### CFN Loop Completion Workflow
+```bash
+# After Product Owner decision, before Redis cleanup
+./.claude/skills/cfn-redis-data-extraction/extract.sh \
+  --task-id "$TASK_ID" \
+  --output-dir "./analysis/loops" \
+  --include-performance=true
+
+# Then proceed with Redis cleanup
+redis-cli DEL "cfn_loop:task:$TASK_ID:*" "swarm:$TASK_ID:*"
+```
+
+### Main Chat Automation
+```bash
+# Automatic extraction after loop completion
+npx claude-flow-novice skill cfn-redis-data-extraction \
+  --task-id "cfn-cli-XXXXXXX-XXXXX" \
+  --verbose
+```
+
+## Error Handling
+
+- **Redis Unavailable**: Graceful exit with error message
+- **No Keys Found**: Warning message, continue processing
+- **Invalid JSON**: Fallback to basic text extraction
+- **Permission Issues**: Directory creation with appropriate permissions
+
+## Performance Considerations
+
+- **Large Key Sets**: Processes keys in batches to avoid memory issues
+- **Concurrent Access**: Uses read-only Redis operations
+- **File Size**: JSON files are compressed and structured efficiently
+- **Extraction Time**: Typically <5 seconds per task
+
+## Validation
+
+The skill includes built-in validation:
+- Redis connectivity check
+- JSON structure validation
+- Data completeness verification
+- Output file creation confirmation
+
+## Maintenance
+
+- **Version Tracking**: Each extraction includes version metadata
+- **Backward Compatibility**: Supports multiple extraction formats
+- **Cleanup**: Old files can be archived or cleaned up automatically
+- **Monitoring**: Integration with system monitoring for extraction success rates
+
+## Related Skills
+
+- `cfn-redis-coordination` - For understanding Redis coordination patterns
+- `cfn-loop-validation` - For validating loop execution
+- `cfn-agent-spawning` - For understanding agent lifecycle
+
+## Dependencies
+
+- `redis-cli` - Redis command-line interface
+- `jq` - JSON processor for data manipulation
+- `bc` - Basic calculator for confidence averaging
+- Standard Unix utilities (grep, awk, sed)