claude-flow-novice 1.6.4 → 1.6.6

package/scripts/monitoring/alert-monitor.sh

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+# scripts/monitoring/alert-monitor.sh - Continuous monitoring daemon with alerting
+# Phase 1 Sprint 1.1: Monitoring loop integration
+
+set -euo pipefail
+
+# ==============================================================================
+# CONFIGURATION
+# ==============================================================================
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIB_DIR="${SCRIPT_DIR}/../../lib"
+METRICS_FILE="${METRICS_FILE:-/dev/shm/cfn-metrics.jsonl}"
+ALERT_LOG_FILE="${ALERT_LOG_FILE:-/dev/shm/cfn-alerts.jsonl}"
+MONITOR_PID_FILE="${MONITOR_PID_FILE:-/dev/shm/alert-monitor.pid}"
+
+# Check interval (seconds)
+CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
+
+# Cleanup retention (hours)
+ALERT_RETENTION_HOURS="${ALERT_RETENTION_HOURS:-24}"
+METRICS_RETENTION_HOURS="${METRICS_RETENTION_HOURS:-48}"
+
+# ==============================================================================
+# DEPENDENCIES
+# ==============================================================================
+
+# Source alerting library
+if [ -f "$LIB_DIR/alerting.sh" ]; then
+  # shellcheck source=../../lib/alerting.sh
+  source "$LIB_DIR/alerting.sh"
+else
+  echo "[ERROR] Alerting library not found at $LIB_DIR/alerting.sh" >&2
+  exit 1
+fi
+
+# ==============================================================================
+# SIGNAL HANDLERS
+# ==============================================================================
+
+cleanup() {
+  echo "[INFO] Shutting down alert monitor (PID: $$)" >&2
+  rm -f "$MONITOR_PID_FILE"
+  exit 0
+}
+
+trap cleanup SIGTERM SIGINT
+
+# ==============================================================================
+# MONITORING FUNCTIONS
+# ==============================================================================
+
+# start_monitor - Begin continuous threshold monitoring
+start_monitor() {
+  local iteration=0
+
+  echo "[INFO] Alert monitor started (PID: $$)" >&2
+  echo "[INFO] Check interval: ${CHECK_INTERVAL}s" >&2
+  echo "[INFO] Metrics file: $METRICS_FILE" >&2
+  echo "[INFO] Alert log: $ALERT_LOG_FILE" >&2
+
+  # Write PID file
+  echo $$ > "$MONITOR_PID_FILE"
+
+  while true; do
+    iteration=$((iteration + 1))
+
+    # Check thresholds
+    if [ -f "$METRICS_FILE" ]; then
+      check_thresholds "$METRICS_FILE" 2>&1 | while IFS= read -r line; do
+        echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] $line"
+      done
+    fi
+
+    # Periodic cleanup (every 100 iterations)
+    if [ $((iteration % 100)) -eq 0 ]; then
+      echo "[INFO] Running periodic cleanup (iteration $iteration)" >&2
+      cleanup_old_data
+    fi
+
+    # Sleep until next check
+    sleep "$CHECK_INTERVAL"
+  done
+}
+
+# cleanup_old_data - Remove old metrics and alerts
+cleanup_old_data() {
+  # Clear old alerts
+  if [ -f "$ALERT_LOG_FILE" ]; then
+    local alert_count_before
+    alert_count_before=$(wc -l < "$ALERT_LOG_FILE" 2>/dev/null || echo "0")
+
+    clear_old_alerts "$ALERT_RETENTION_HOURS"
+
+    local alert_count_after
+    alert_count_after=$(wc -l < "$ALERT_LOG_FILE" 2>/dev/null || echo "0")
+
+    echo "[INFO] Cleared $((alert_count_before - alert_count_after)) old alerts" >&2
+  fi
+
+  # Clear old metrics
+  if [ -f "$METRICS_FILE" ]; then
+    local metrics_count_before
+    metrics_count_before=$(wc -l < "$METRICS_FILE" 2>/dev/null || echo "0")
+
+    local cutoff_time
+    cutoff_time=$(date -u -d "$METRICS_RETENTION_HOURS hours ago" +"%Y-%m-%dT%H:%M:%S" 2>/dev/null || \
+                  date -u -v-"${METRICS_RETENTION_HOURS}H" +"%Y-%m-%dT%H:%M:%S" 2>/dev/null || \
+                  echo "1970-01-01T00:00:00")
+
+    local temp_file="${METRICS_FILE}.tmp"
+    jq -c --arg cutoff "$cutoff_time" \
+      'select(.timestamp >= $cutoff)' \
+      "$METRICS_FILE" > "$temp_file" 2>/dev/null || true
+
+    if [ -f "$temp_file" ]; then
+      mv "$temp_file" "$METRICS_FILE"
+
+      local metrics_count_after
+      metrics_count_after=$(wc -l < "$METRICS_FILE" 2>/dev/null || echo "0")
+
+      echo "[INFO] Cleared $((metrics_count_before - metrics_count_after)) old metrics" >&2
+    fi
+  fi
+}
+
+# get_monitor_status - Check if monitor is running
+get_monitor_status() {
+  if [ -f "$MONITOR_PID_FILE" ]; then
+    local pid
+    pid=$(cat "$MONITOR_PID_FILE")
+
+    if kill -0 "$pid" 2>/dev/null; then
+      echo "running (PID: $pid)"
+      return 0
+    else
+      echo "stale (PID file exists but process not running)"
+      rm -f "$MONITOR_PID_FILE"
+      return 1
+    fi
+  else
+    echo "stopped"
+    return 1
+  fi
+}
+
+# stop_monitor - Stop running monitor
+stop_monitor() {
+  if [ -f "$MONITOR_PID_FILE" ]; then
+    local pid
+    pid=$(cat "$MONITOR_PID_FILE")
+
+    if kill -0 "$pid" 2>/dev/null; then
+      echo "[INFO] Stopping monitor (PID: $pid)" >&2
+      kill -TERM "$pid"
+
+      # Wait for graceful shutdown (max 5 seconds)
+      for i in {1..10}; do
+        if ! kill -0 "$pid" 2>/dev/null; then
+          echo "[INFO] Monitor stopped successfully" >&2
+          return 0
+        fi
+        sleep 0.5
+      done
+
+      # Force kill if still running
+      if kill -0 "$pid" 2>/dev/null; then
+        echo "[WARN] Monitor did not stop gracefully, forcing..." >&2
+        kill -KILL "$pid" 2>/dev/null || true
+      fi
+    fi
+
+    rm -f "$MONITOR_PID_FILE"
+  else
+    echo "[INFO] No monitor running" >&2
+  fi
+}
+
+# ==============================================================================
+# MAIN EXECUTION
+# ==============================================================================
+
+case "${1:-start}" in
+  start)
+    if [ -f "$MONITOR_PID_FILE" ]; then
+      echo "[ERROR] Monitor already running (PID: $(cat "$MONITOR_PID_FILE"))" >&2
+      exit 1
+    fi
+    start_monitor
+    ;;
+
+  stop)
+    stop_monitor
+    ;;
+
+  restart)
+    stop_monitor
+    sleep 1
+    start_monitor
+    ;;
+
+  status)
+    get_monitor_status
+    ;;
+
+  background)
+    # Start in background
+    if [ -f "$MONITOR_PID_FILE" ]; then
+      echo "[ERROR] Monitor already running (PID: $(cat "$MONITOR_PID_FILE"))" >&2
+      exit 1
+    fi
+    nohup "$0" start > /dev/shm/alert-monitor.log 2>&1 &
+    echo "[INFO] Monitor started in background (PID: $!)" >&2
+    ;;
+
+  *)
+    echo "Usage: $0 {start|stop|restart|status|background}" >&2
+    exit 1
+    ;;
+esac

package/scripts/monitoring/dashboards/rate-limiting-dashboard.json

@@ -0,0 +1,211 @@
+{
+  "dashboard": {
+    "title": "CFN Rate Limiting & Backpressure Monitoring",
+    "description": "Real-time monitoring of message inbox utilization, backpressure events, and overflow alerts",
+    "version": "1.0.0",
+    "tags": ["rate-limiting", "backpressure", "inbox", "coordination"],
+    "timezone": "UTC",
+    "refresh": "10s",
+    "panels": [
+      {
+        "id": 1,
+        "title": "Inbox Utilization by Agent",
+        "type": "timeseries",
+        "gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "inbox.utilization",
+            "legend": "{{agent}}",
+            "unit": "percent"
+          }
+        ],
+        "thresholds": [
+          { "value": 75, "color": "yellow", "label": "Warning" },
+          { "value": 90, "color": "red", "label": "Critical" }
+        ],
+        "description": "Message inbox utilization percentage per agent (max 100 messages)"
+      },
+      {
+        "id": 2,
+        "title": "Inbox Message Count",
+        "type": "timeseries",
+        "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "inbox.size",
+            "legend": "{{agent}}",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 75, "color": "yellow", "label": "Warning (75 msgs)" },
+          { "value": 90, "color": "red", "label": "Critical (90 msgs)" }
+        ],
+        "description": "Absolute message count in agent inboxes"
+      },
+      {
+        "id": 3,
+        "title": "Backpressure Events Rate",
+        "type": "timeseries",
+        "gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "backpressure.events_per_min",
+            "legend": "Backpressure events/min",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 100, "color": "yellow", "label": "Warning threshold" }
+        ],
+        "description": "Rate of backpressure wait events (high rate indicates system load)"
+      },
+      {
+        "id": 4,
+        "title": "Message Send Failures",
+        "type": "timeseries",
+        "gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "coordination.send_failures_per_min",
+            "legend": "Send failures/min",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 10, "color": "red", "label": "Critical threshold" }
+        ],
+        "description": "Message delivery failure rate (critical if >10/min)"
+      },
+      {
+        "id": 5,
+        "title": "Inbox Overflow Events",
+        "type": "timeseries",
+        "gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "inbox.overflow_events_per_min",
+            "legend": "Overflow events/min",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 1, "color": "red", "label": "Any overflow is critical" }
+        ],
+        "description": "Inbox overflow events (messages dropped due to full inbox)"
+      },
+      {
+        "id": 6,
+        "title": "Alert Summary",
+        "type": "stat",
+        "gridPos": { "x": 12, "y": 16, "w": 6, "h": 8 },
+        "targets": [
+          {
+            "query": "count_alerts_by_severity",
+            "fields": ["critical", "warning", "info"]
+          }
+        ],
+        "description": "Alert counts by severity level (last hour)"
+      },
+      {
+        "id": 7,
+        "title": "Top Utilized Agents",
+        "type": "table",
+        "gridPos": { "x": 18, "y": 16, "w": 6, "h": 8 },
+        "targets": [
+          {
+            "query": "top_inbox_utilization",
+            "fields": ["agent", "utilization", "message_count"],
+            "limit": 10,
+            "order": "desc"
+          }
+        ],
+        "description": "Agents with highest inbox utilization"
+      }
+    ],
+    "annotations": [
+      {
+        "name": "Rate Limiting Alerts",
+        "datasource": "cfn-alerts",
+        "filter": {
+          "tags": ["inbox_high_utilization", "backpressure_high_rate", "inbox_overflow_detected"]
+        },
+        "color": "red"
+      }
+    ],
+    "variables": [
+      {
+        "name": "agent",
+        "type": "query",
+        "query": "SELECT DISTINCT agent FROM inbox_metrics",
+        "description": "Filter by specific agent",
+        "multi": true,
+        "includeAll": true
+      },
+      {
+        "name": "timeRange",
+        "type": "interval",
+        "options": ["5m", "15m", "1h", "6h", "24h"],
+        "default": "1h",
+        "description": "Time range for metrics"
+      }
+    ]
+  },
+  "queries": {
+    "count_alerts_by_severity": {
+      "description": "Count alerts grouped by severity",
+      "source": "/dev/shm/cfn-alerts.jsonl",
+      "aggregation": "GROUP BY severity, COUNT(*)",
+      "timeWindow": "1h"
+    },
+    "top_inbox_utilization": {
+      "description": "Rank agents by inbox utilization",
+      "source": "/dev/shm/cfn-metrics.jsonl",
+      "query": "SELECT agent, MAX(value) as utilization FROM metrics WHERE metric='inbox.utilization' GROUP BY agent ORDER BY utilization DESC LIMIT 10",
+      "timeWindow": "5m"
+    }
+  },
+  "alertRules": [
+    {
+      "name": "Inbox Critical Utilization",
+      "condition": "inbox.utilization >= 90",
+      "severity": "critical",
+      "message": "Agent {{agent}} inbox at {{value}}% utilization (critical threshold: 90%)",
+      "actions": ["emit_alert", "notify_oncall"]
+    },
+    {
+      "name": "Inbox Warning Utilization",
+      "condition": "inbox.utilization >= 75 AND inbox.utilization < 90",
+      "severity": "warning",
+      "message": "Agent {{agent}} inbox at {{value}}% utilization (warning threshold: 75%)",
+      "actions": ["emit_alert"]
+    },
+    {
+      "name": "Backpressure High Rate",
+      "condition": "backpressure.events_per_min > 100",
+      "severity": "warning",
+      "message": "Backpressure events exceeding threshold: {{value}} events/min (threshold: 100/min)",
+      "actions": ["emit_alert"]
+    },
+    {
+      "name": "Message Send Failures Critical",
+      "condition": "coordination.send_failures_per_min > 10",
+      "severity": "critical",
+      "message": "Message send failures critical: {{value}} failures/min (threshold: 10/min)",
+      "actions": ["emit_alert", "notify_oncall", "trigger_incident"]
+    },
+    {
+      "name": "Inbox Overflow Detected",
+      "condition": "inbox.overflow_events_per_min > 0",
+      "severity": "critical",
+      "message": "Inbox overflow detected: {{value}} overflow events in last minute",
+      "actions": ["emit_alert", "notify_oncall", "trigger_incident"]
+    }
+  ],
+  "metadata": {
+    "createdBy": "devops-engineer",
+    "phase": "1",
+    "sprint": "1.5",
+    "lastUpdated": "2025-10-06T19:35:00Z"
+  }
+}

package/scripts/monitoring/quick-test-alerting.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# Quick validation test for alerting system (no jq dependency)
+# Phase 1 Sprint 1.1: Simple validation
+
+set -euo pipefail
+
+echo "=========================================="
+echo "ALERTING SYSTEM QUICK VALIDATION"
+echo "=========================================="
+echo ""
+
+# Check if files exist
+echo "1. Checking component files..."
+files=(
+  "lib/alerting.sh"
+  "scripts/monitoring/alert-monitor.sh"
+  "scripts/monitoring/view-alerts.sh"
+)
+
+for file in "${files[@]}"; do
+  if [ -f "$file" ]; then
+    echo "  ✅ $file"
+  else
+    echo "  ❌ $file (MISSING)"
+    exit 1
+  fi
+done
+
+echo ""
+echo "2. Checking executability..."
+for file in "${files[@]}"; do
+  if [ -x "$file" ]; then
+    echo "  ✅ $file"
+  else
+    echo "  ⚠️  $file (not executable, fixing...)"
+    chmod +x "$file"
+  fi
+done
+
+echo ""
+echo "3. Checking syntax..."
+for file in "${files[@]}"; do
+  if bash -n "$file" 2>/dev/null; then
+    echo "  ✅ $file"
+  else
+    echo "  ❌ $file (SYNTAX ERROR)"
+    exit 1
+  fi
+done
+
+echo ""
+echo "4. Checking alerting library functions..."
+# Source the library and verify key functions exist
+if source lib/alerting.sh 2>/dev/null; then
+  echo "  ✅ Library sourced successfully"
+
+  # Check if key functions are defined
+  funcs=(
+    "check_thresholds"
+    "emit_alert"
+    "get_alert_summary"
+    "clear_old_alerts"
+  )
+
+  for func in "${funcs[@]}"; do
+    if declare -f "$func" >/dev/null; then
+      echo "  ✅ Function: $func"
+    else
+      echo "  ❌ Function: $func (NOT FOUND)"
+      exit 1
+    fi
+  done
+else
+  echo "  ❌ Failed to source library"
+  exit 1
+fi
+
+echo ""
+echo "5. Checking threshold configuration..."
+# Verify environment variables are set with defaults
+vars=(
+  "ALERT_COORDINATION_TIME_MS"
+  "ALERT_DELIVERY_RATE_PCT"
+  "ALERT_MEMORY_GROWTH_PCT"
+  "ALERT_FD_GROWTH"
+)
+
+for var in "${vars[@]}"; do
+  if [ -n "${!var:-}" ]; then
+    echo "  ✅ $var=${!var}"
+  else
+    echo "  ❌ $var (NOT SET)"
+    exit 1
+  fi
+done
+
+echo ""
+echo "=========================================="
+echo "✅ VALIDATION PASSED"
+echo "=========================================="
+echo ""
+echo "DELIVERABLES CONFIRMED:"
+echo "  ✅ lib/alerting.sh - Alert threshold engine"
+echo "  ✅ scripts/monitoring/alert-monitor.sh - Monitoring daemon"
+echo "  ✅ scripts/monitoring/view-alerts.sh - Alert dashboard"
+echo "  ✅ Configurable thresholds"
+echo "  ✅ All functions defined"
+echo "  ✅ Syntax validation passed"
+echo ""
+echo "ACCEPTANCE CRITERIA STATUS:"
+echo "  ✅ Alerts trigger when thresholds exceeded - Implemented"
+echo "  ⚠️  False positive rate <1% - Requires jq for full testing"
+echo "  ⚠️  Alert latency <30 seconds - Requires jq for full testing"
+echo "  ✅ Configurable thresholds - Confirmed"
+echo ""
+echo "To run full integration tests, install jq:"
+echo "  sudo apt-get install jq"
+echo "  bash tests/integration/alerting-system.test.sh"