claude-flow-novice 2.10.5 → 2.10.6

package/claude-assets/skills/cfn-redis-coordination/heartbeat-functions.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+##############################################################################
+# Heartbeat Monitoring Functions Library
+# Extracted from orchestrate-cfn-loop.sh for standalone testing
+##############################################################################
+
+##############################################################################
+# Heartbeat Monitoring Functions
+##############################################################################
+declare -A MISSED_HEARTBEATS  # Track missed heartbeats per agent
+
+function check_agent_heartbeat() {
+  local agent="$1"
+  local task_id="$2"
+
+  HB_KEY="swarm:${task_id}:${agent}:heartbeat"
+  HB_DATA=$(redis-cli GET "$HB_KEY" 2>/dev/null || echo "")
+
+  if [ -z "$HB_DATA" ] || [ "$HB_DATA" = "(nil)" ]; then
+    return 1  # Dead
+  else
+    return 0  # Alive
+  fi
+}
+
+function calculate_quorum() {
+  local quorum_spec="$1"
+  local total_agents="$2"
+
+  # If no quorum specified, require all agents
+  if [ -z "$quorum_spec" ]; then
+    echo "$total_agents"
+    return 0
+  fi
+
+  # Check if percentage format (e.g., "85%")
+  if [[ "$quorum_spec" =~ %$ ]]; then
+    # Extract percentage value (remove % suffix)
+    local pct="${quorum_spec%\%}"
+    # Calculate: ceil(total_agents * pct / 100)
+    echo "scale=0; ($total_agents * $pct + 50) / 100" | bc
+  # Check if decimal (0.0-1.0), treat as fraction
+  elif [[ "$quorum_spec" =~ ^0?\.[0-9]+$ ]]; then
+    # Calculate: ceil(total_agents * fraction)
+    echo "scale=0; ($quorum_spec * $total_agents + 0.5) / 1" | bc
+  else
+    # Absolute number - validate it doesn't exceed total
+    if [ "$quorum_spec" -gt "$total_agents" ]; then
+      echo "Error: Quorum ($quorum_spec) exceeds total agents ($total_agents)" >&2
+      return 1
+    fi
+    echo "$quorum_spec"
+  fi
+}
+
+function check_heartbeats_loop() {
+  local task_id="$1"
+  local loop_name="$2"
+  shift 2
+  local agents=("$@")
+
+  for AGENT in "${agents[@]}"; do
+    # Skip agents already marked as failed
+    if [[ " ${LOOP3_FAILED_AGENTS[@]} ${LOOP2_FAILED_AGENTS[@]} " =~ " ${AGENT} " ]]; then
+      continue
+    fi
+
+    if ! check_agent_heartbeat "$AGENT" "$task_id"; then
+      MISSED_HEARTBEATS["$AGENT"]=$((${MISSED_HEARTBEATS["$AGENT"]:-0} + 1))
+
+      if [ ${MISSED_HEARTBEATS["$AGENT"]} -ge 2 ]; then
+        local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+        echo "  [$timestamp] [$loop_name] ⚠️ $AGENT appears hung (no heartbeat for 60s)" >&2
+
+        # Determine which loop this agent belongs to and check quorum
+        if [[ " ${LOOP3_AGENTS} " =~ " ${AGENT} " ]]; then
+          REMAINING=$((${#LOOP3_COMPLETED_AGENTS[@]}))
+          REQUIRED=$(calculate_quorum "$MIN_QUORUM_LOOP3" "$LOOP3_TOTAL")
+        elif [[ " ${LOOP2_AGENTS} " =~ " ${LOOP2_AGENTS} " ]]; then
+          REMAINING=$((${#LOOP2_COMPLETED_AGENTS[@]}))
+          REQUIRED=$(calculate_quorum "$MIN_QUORUM_LOOP2" "$LOOP2_TOTAL")
+        else
+          continue
+        fi
+
+        if [ $REMAINING -ge $REQUIRED ]; then
+          echo "  [$timestamp] [$loop_name] ℹ️ Continuing with quorum (${REMAINING}/${REQUIRED} agents)" >&2
+        else
+          echo "  [$timestamp] [$loop_name] ⚠️ Cannot meet quorum without $AGENT (${REMAINING}/${REQUIRED})" >&2
+        fi
+      fi
+    else
+      MISSED_HEARTBEATS["$AGENT"]=0  # Reset counter
+    fi
+  done
+}
+
+function start_heartbeat_monitor() {
+  local task_id="$1"
+  local loop_name="$2"
+  shift 2
+  local agents=("$@")
+
+  # Create marker file for this monitor
+  local monitor_marker="/tmp/heartbeat-monitor-${task_id}-${loop_name}.active"
+  touch "$monitor_marker"
+
+  (
+    while [ -f "$monitor_marker" ]; do
+      # Check for shutdown
+      if [ "${SHUTDOWN_REQUESTED:-0}" -eq 1 ]; then
+        break
+      fi
+
+      check_heartbeats_loop "$task_id" "$loop_name" "${agents[@]}"
+      sleep 30
+    done
+  ) &
+
+  echo "$!"  # Return PID
+}
+
+function stop_heartbeat_monitor() {
+  local task_id="$1"
+  local loop_name="$2"
+  local monitor_pid="$3"
+
+  # Remove marker file to stop the monitor loop
+  rm -f "/tmp/heartbeat-monitor-${task_id}-${loop_name}.active"
+
+  # Kill monitor process if still running
+  if [ -n "$monitor_pid" ] && kill -0 "$monitor_pid" 2>/dev/null; then
+    kill "$monitor_pid" 2>/dev/null || true
+    wait "$monitor_pid" 2>/dev/null || true
+  fi
+}

package/claude-assets/skills/cfn-redis-coordination/heartbeat-protocol.md

@@ -0,0 +1,106 @@
+# Heartbeat Protocol Specification
+
+## Overview
+The heartbeat protocol provides a mechanism for detecting hung or unresponsive agents in distributed agent swarms using Redis as a coordination mechanism.
+
+## Key Design Components
+
+### 1. Heartbeat Key Pattern
+```
+swarm:{task_id}:{agent_id}:heartbeat
+```
+
+### 2. Heartbeat Message Structure
+```json
+{
+    "timestamp": 1760898665,
+    "status": "working|idle|error",
+    "iteration": 2,
+    "progress": 0.75,
+    "agent_details": {
+        "agent_id": "architect-5",
+        "task_id": "redis-phase5-1760898665",
+        "environment": {
+            "cpu_usage": 0.65,
+            "memory_usage": 0.42,
+            "system_load": 0.3
+        }
+    }
+}
+```
+
+### 3. Heartbeat Configuration
+- **Update Frequency**: Every 30 seconds
+- **Default TTL**: 60 seconds
+- **Miss Threshold**: 2 consecutive missed heartbeats
+- **Quorum Threshold**: 70% of agents must be responsive
+
+### 4. Heartbeat Workflow
+1. Agent periodically sends heartbeat via Redis SET
+2. Orchestrator monitors heartbeats in background process
+3. On missed heartbeats, trigger progressive recovery mechanisms
+
+### 5. Recovery Stages
+- **Stage 1 (Miss 1)**: Log warning, continue monitoring
+- **Stage 2 (Miss 2)**:
+  - Check if remaining agents meet quorum
+  - Log to Dead Letter Queue (DLQ)
+  - Attempt soft restart of agent
+- **Stage 3 (Miss 3)**:
+  - Hard restart agent
+  - Potentially replace with standby agent
+
+### 6. Implementation Pseudo-code
+```bash
+# Send Heartbeat
+redis-cli set "swarm:${TASK_ID}:${AGENT_ID}:heartbeat" \
+    "$(generate_heartbeat_payload)" \
+    EX 60  # 60-second expiry
+
+# Check Heartbeats
+check_agent_heartbeats() {
+    for agent in ${AGENTS[@]}; do
+        heartbeat=$(redis-cli get "swarm:${TASK_ID}:${agent}:heartbeat")
+        if [[ -z "$heartbeat" ]]; then
+            handle_missed_heartbeat "$agent"
+        fi
+    done
+}
+
+handle_missed_heartbeat() {
+    local agent="$1"
+    local miss_count=$(get_miss_count "$agent")
+
+    case "$miss_count" in
+        1) log_warning "$agent missed first heartbeat" ;;
+        2)
+            log_dlq "$agent"
+            attempt_soft_restart "$agent"
+            check_quorum
+            ;;
+        3)
+            hard_restart_agent "$agent"
+            ;;
+    esac
+}
+```
+
+### 7. Monitoring and Logging
+- Comprehensive logging to `/var/log/claude-flow/heartbeat.log`
+- Prometheus metrics for heartbeat health
+- Grafana dashboard tracking agent responsiveness
+
+### 8. Security Considerations
+- Cryptographically sign heartbeat messages
+- Rate limit heartbeat submissions
+- Validate heartbeat payload schema
+
+## Integration Points
+- Redis Coordination Skill
+- CFN Loop Validation
+- Agent Spawning Mechanism
+
+## Test Coverage
+- Unit tests for heartbeat generation
+- Integration tests for recovery mechanisms
+- Chaos testing (intentional agent hanging)

package/claude-assets/skills/cfn-redis-coordination/heartbeat.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+# Heartbeat Monitoring Script for Agent Coordination
+# Implements 60s TTL, 30s check interval, and quorum fallback detection
+
+# Dependencies
+REDIS_CLI=$(which redis-cli)
+if [ -z "$REDIS_CLI" ]; then
+    echo "Error: redis-cli not found. Please install Redis client."
+    exit 1
+fi
+
+# Logging configuration
+LOG_FILE="/tmp/heartbeat-debug.log"
+touch "$LOG_FILE"
+
+log_debug() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"
+}
+
+# Configuration
+HEARTBEAT_TTL=60  # 60 seconds TTL
+CHECK_INTERVAL=30  # 30 seconds between checks
+MISSED_THRESHOLD=2  # Number of missed heartbeats before considering agent hung
+BUFFER_TIME=3      # Additional buffer time for timing flexibility
+
+send_heartbeat() {
+    local task_id="$1"
+    local agent_id="$2"
+
+    if [ -z "$task_id" ] || [ -z "$agent_id" ]; then
+        echo "Usage: $0 send --task-id TASK_ID --agent-id AGENT_ID"
+        exit 1
+    fi
+
+    log_debug "Sending heartbeat for task=$task_id, agent=$agent_id"
+
+    # Use SETEX to create a key with expiration
+    $REDIS_CLI SETEX "swarm:agent_status:${task_id}:${agent_id}" "$HEARTBEAT_TTL" "alive"
+    echo "Heartbeat sent for agent ${agent_id} in task ${task_id}"
+}
+
+check_heartbeat() {
+    local task_id="$1"
+    local agent_id="$2"
+
+    if [ -z "$task_id" ] || [ -z "$agent_id" ]; then
+        echo "Usage: $0 check --task-id TASK_ID --agent-id AGENT_ID"
+        exit 1
+    fi
+
+    local key="swarm:agent_status:${task_id}:${agent_id}"
+    local status
+    local ttl
+
+    # Retrieve status and TTL
+    status=$($REDIS_CLI GET "$key")
+    ttl=$($REDIS_CLI TTL "$key")
+
+    log_debug "Checking heartbeat for task=$task_id, agent=$agent_id: status=$status, ttl=$ttl"
+
+    # Check for key existence and status
+    if [ -z "$status" ] || [ "$ttl" -le 0 ]; then
+        log_debug "Heartbeat DEAD: status missing or expired"
+        echo "dead"
+        increment_missed_counter "$task_id" "$agent_id"
+        return 1
+    else
+        log_debug "Heartbeat ALIVE: status=$status, ttl=$ttl"
+        echo "alive"
+        return 0
+    fi
+}
+
+increment_missed_counter() {
+    local task_id="$1"
+    local agent_id="$2"
+    local missed_key="swarm:missed_heartbeats:${task_id}:${agent_id}"
+
+    # Increment missed heartbeat counter
+    local missed_count=$($REDIS_CLI INCR "$missed_key")
+
+    # Set expiry for missed counter to match heartbeat TTL
+    $REDIS_CLI EXPIRE "$missed_key" $HEARTBEAT_TTL
+
+    log_debug "Missed heartbeat counter for task=$task_id, agent=$agent_id: count=$missed_count"
+
+    if [ "$missed_count" -ge "$MISSED_THRESHOLD" ]; then
+        # Trigger quorum fallback mechanism
+        $REDIS_CLI LPUSH "swarm:${task_id}:quorum_fallback" "$agent_id"
+        log_debug "QUORUM FALLBACK: Agent $agent_id missed $missed_count heartbeats"
+        echo "WARN: Agent $agent_id missed $missed_count heartbeats. Quorum fallback triggered."
+    fi
+}
+
+# Parse arguments
+case "$1" in
+    send)
+        shift
+        while [[ "$#" -gt 0 ]]; do
+            case $1 in
+                --task-id) task_id="$2"; shift ;;
+                --agent-id) agent_id="$2"; shift ;;
+                *) echo "Unknown parameter passed: $1"; exit 1 ;;
+            esac
+            shift
+        done
+        send_heartbeat "$task_id" "$agent_id"
+        ;;
+    check)
+        shift
+        while [[ "$#" -gt 0 ]]; do
+            case $1 in
+                --task-id) task_id="$2"; shift ;;
+                --agent-id) agent_id="$2"; shift ;;
+                *) echo "Unknown parameter passed: $1"; exit 1 ;;
+            esac
+            shift
+        done
+        check_heartbeat "$task_id" "$agent_id"
+        ;;
+    *)
+        echo "Usage: $0 {send|check} --task-id TASK_ID --agent-id AGENT_ID"
+        exit 1
+        ;;
+esac

package/claude-assets/skills/cfn-redis-coordination/init-swarm.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+##############################################################################
+# Initialize Swarm - Redis Coordination Primitive
+#
+# Creates swarm metadata in Redis for coordination tracking across any
+# multi-agent workflow (CFN Loop, independent swarms, custom orchestration).
+#
+# Usage:
+#   ./init-swarm.sh --swarm-id <id> \
+#                   --agents <agent1,agent2,...> \
+#                   [--task-id <id>] \
+#                   [--topology <mesh|hierarchical|chain>] \
+#                   [--ttl <seconds>] \
+#                   [--metadata <json-string>]
+#
+# Per-Agent Timeout Configuration:
+#   Agents can have custom timeout values set via Redis:
+#
+#   redis-cli SETEX "swarm:<task-id>:<agent-id>:timeout" 86400 <timeout-seconds>
+#
+#   If no custom timeout is set, the orchestrator will use role-based defaults:
+#   - researcher: 7200s (2 hours)
+#   - backend-dev, coder, frontend-dev: 3600s (1 hour)
+#   - reviewer, tester, security: 1800s (30 minutes)
+#   - coordinator, orchestrator, product-owner: 900s (15 minutes)
+#   - default: 3600s (1 hour)
+#
+# Benefits:
+#   - Namespace isolation for concurrent swarms
+#   - Agent inventory and resource tracking
+#   - Status monitoring (in_progress/completed)
+#   - Automatic cleanup via TTL
+#   - Per-agent timeout configuration via get-agent-timeout.sh
+##############################################################################
+
+set -euo pipefail
+
+# Configuration
+SWARM_ID=""
+AGENTS=""
+TASK_ID=""
+TOPOLOGY="mesh"
+TTL=604800  # 7 days default
+MAX_AGENTS=""
+METADATA_EXTRA=""
+AGENT_TIMEOUTS=""
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --swarm-id)
+      SWARM_ID="$2"
+      shift 2
+      ;;
+    --agents)
+      AGENTS="$2"
+      shift 2
+      ;;
+    --task-id)
+      TASK_ID="$2"
+      shift 2
+      ;;
+    --topology)
+      TOPOLOGY="$2"
+      shift 2
+      ;;
+    --ttl)
+      TTL="$2"
+      shift 2
+      ;;
+    --max-agents)
+      MAX_AGENTS="$2"
+      shift 2
+      ;;
+    --metadata)
+      METADATA_EXTRA="$2"
+      shift 2
+      ;;
+    --agent-timeouts)
+      AGENT_TIMEOUTS="$2"
+      shift 2
+      ;;
+    *)
+      echo "Unknown option: $1"
+      echo "Usage: $0 --swarm-id <id> --agents <agent1,agent2,...> [options]"
+      exit 1
+      ;;
+  esac
+done
+
+# Validation
+if [ -z "$SWARM_ID" ] || [ -z "$AGENTS" ]; then
+  echo "Error: Required parameters missing"
+  echo "Usage: $0 --swarm-id <id> --agents <agent1,agent2,...>"
+  exit 1
+fi
+
+# Calculate max agents if not provided
+if [ -z "$MAX_AGENTS" ]; then
+  IFS=',' read -ra AGENT_ARRAY <<< "$AGENTS"
+  MAX_AGENTS=${#AGENT_ARRAY[@]}
+fi
+
+# Use swarm-id as task-id if not provided
+if [ -z "$TASK_ID" ]; then
+  TASK_ID="$SWARM_ID"
+fi
+
+echo "[Swarm] Initializing swarm: $SWARM_ID"
+echo "[Swarm] Topology: $TOPOLOGY"
+echo "[Swarm] Total agents: $MAX_AGENTS"
+echo "[Swarm] TTL: $TTL seconds ($(($TTL / 86400)) days)"
+
+# Create swarm metadata key
+METADATA_KEY="swarm:${SWARM_ID}:metadata"
+
+# Store base metadata
+# Extract repository name from PWD
+REPO_NAME=$(basename "$(pwd)")
+
+redis-cli hset "$METADATA_KEY" \
+  swarm_id "$SWARM_ID" \
+  task_id "$TASK_ID" \
+  topology "$TOPOLOGY" \
+  max_agents "$MAX_AGENTS" \
+  agents "$AGENTS" \
+  created_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+  status "in_progress" \
+  repository "$REPO_NAME" \
+  cwd "$(pwd)" > /dev/null
+
+# Add extra metadata if provided (JSON string)
+if [ -n "$METADATA_EXTRA" ]; then
+  # Parse JSON and add each key-value pair
+  echo "$METADATA_EXTRA" | jq -r 'to_entries | .[] | "\(.key) \(.value)"' | while read -r KEY VALUE; do
+    redis-cli hset "$METADATA_KEY" "$KEY" "$VALUE" > /dev/null
+  done
+fi
+
+# Set TTL
+redis-cli expire "$METADATA_KEY" "$TTL" > /dev/null
+
+echo "[Swarm] Registered in Redis: $METADATA_KEY"
+echo "[Swarm] ✅ Initialization complete"
+
+# Output swarm ID for chaining
+echo "$SWARM_ID"