claude-flow-novice 2.2.4 → 2.2.5

package/scripts/monitoring/alert-monitor.sh

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+# scripts/monitoring/alert-monitor.sh - Continuous monitoring daemon with alerting
+# Phase 1 Sprint 1.1: Monitoring loop integration
+
+set -euo pipefail
+
+# ==============================================================================
+# CONFIGURATION
+# ==============================================================================
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LIB_DIR="${SCRIPT_DIR}/../../lib"
+METRICS_FILE="${METRICS_FILE:-/dev/shm/cfn-metrics.jsonl}"
+ALERT_LOG_FILE="${ALERT_LOG_FILE:-/dev/shm/cfn-alerts.jsonl}"
+MONITOR_PID_FILE="${MONITOR_PID_FILE:-/dev/shm/alert-monitor.pid}"
+
+# Check interval (seconds)
+CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
+
+# Cleanup retention (hours)
+ALERT_RETENTION_HOURS="${ALERT_RETENTION_HOURS:-24}"
+METRICS_RETENTION_HOURS="${METRICS_RETENTION_HOURS:-48}"
+
+# ==============================================================================
+# DEPENDENCIES
+# ==============================================================================
+
+# Source alerting library
+if [ -f "$LIB_DIR/alerting.sh" ]; then
+  # shellcheck source=../../lib/alerting.sh
+  source "$LIB_DIR/alerting.sh"
+else
+  echo "[ERROR] Alerting library not found at $LIB_DIR/alerting.sh" >&2
+  exit 1
+fi
+
+# ==============================================================================
+# SIGNAL HANDLERS
+# ==============================================================================
+
+cleanup() {
+  echo "[INFO] Shutting down alert monitor (PID: $$)" >&2
+  rm -f "$MONITOR_PID_FILE"
+  exit 0
+}
+
+trap cleanup SIGTERM SIGINT
+
+# ==============================================================================
+# MONITORING FUNCTIONS
+# ==============================================================================
+
+# start_monitor - Begin continuous threshold monitoring
+start_monitor() {
+  local iteration=0
+
+  echo "[INFO] Alert monitor started (PID: $$)" >&2
+  echo "[INFO] Check interval: ${CHECK_INTERVAL}s" >&2
+  echo "[INFO] Metrics file: $METRICS_FILE" >&2
+  echo "[INFO] Alert log: $ALERT_LOG_FILE" >&2
+
+  # Write PID file
+  echo $$ > "$MONITOR_PID_FILE"
+
+  while true; do
+    iteration=$((iteration + 1))
+
+    # Check thresholds
+    if [ -f "$METRICS_FILE" ]; then
+      check_thresholds "$METRICS_FILE" 2>&1 | while IFS= read -r line; do
+        echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] $line"
+      done
+    fi
+
+    # Periodic cleanup (every 100 iterations)
+    if [ $((iteration % 100)) -eq 0 ]; then
+      echo "[INFO] Running periodic cleanup (iteration $iteration)" >&2
+      cleanup_old_data
+    fi
+
+    # Sleep until next check
+    sleep "$CHECK_INTERVAL"
+  done
+}
+
+# cleanup_old_data - Remove old metrics and alerts
+cleanup_old_data() {
+  # Clear old alerts
+  if [ -f "$ALERT_LOG_FILE" ]; then
+    local alert_count_before
+    alert_count_before=$(wc -l < "$ALERT_LOG_FILE" 2>/dev/null || echo "0")
+
+    clear_old_alerts "$ALERT_RETENTION_HOURS"
+
+    local alert_count_after
+    alert_count_after=$(wc -l < "$ALERT_LOG_FILE" 2>/dev/null || echo "0")
+
+    echo "[INFO] Cleared $((alert_count_before - alert_count_after)) old alerts" >&2
+  fi
+
+  # Clear old metrics
+  if [ -f "$METRICS_FILE" ]; then
+    local metrics_count_before
+    metrics_count_before=$(wc -l < "$METRICS_FILE" 2>/dev/null || echo "0")
+
+    local cutoff_time
+    cutoff_time=$(date -u -d "$METRICS_RETENTION_HOURS hours ago" +"%Y-%m-%dT%H:%M:%S" 2>/dev/null || \
+                  date -u -v-"${METRICS_RETENTION_HOURS}H" +"%Y-%m-%dT%H:%M:%S" 2>/dev/null || \
+                  echo "1970-01-01T00:00:00")
+
+    local temp_file="${METRICS_FILE}.tmp"
+    jq -c --arg cutoff "$cutoff_time" \
+      'select(.timestamp >= $cutoff)' \
+      "$METRICS_FILE" > "$temp_file" 2>/dev/null || true
+
+    if [ -f "$temp_file" ]; then
+      mv "$temp_file" "$METRICS_FILE"
+
+      local metrics_count_after
+      metrics_count_after=$(wc -l < "$METRICS_FILE" 2>/dev/null || echo "0")
+
+      echo "[INFO] Cleared $((metrics_count_before - metrics_count_after)) old metrics" >&2
+    fi
+  fi
+}
+
+# get_monitor_status - Check if monitor is running
+get_monitor_status() {
+  if [ -f "$MONITOR_PID_FILE" ]; then
+    local pid
+    pid=$(cat "$MONITOR_PID_FILE")
+
+    if kill -0 "$pid" 2>/dev/null; then
+      echo "running (PID: $pid)"
+      return 0
+    else
+      echo "stale (PID file exists but process not running)"
+      rm -f "$MONITOR_PID_FILE"
+      return 1
+    fi
+  else
+    echo "stopped"
+    return 1
+  fi
+}
+
+# stop_monitor - Stop running monitor
+stop_monitor() {
+  if [ -f "$MONITOR_PID_FILE" ]; then
+    local pid
+    pid=$(cat "$MONITOR_PID_FILE")
+
+    if kill -0 "$pid" 2>/dev/null; then
+      echo "[INFO] Stopping monitor (PID: $pid)" >&2
+      kill -TERM "$pid"
+
+      # Wait for graceful shutdown (max 5 seconds)
+      for i in {1..10}; do
+        if ! kill -0 "$pid" 2>/dev/null; then
+          echo "[INFO] Monitor stopped successfully" >&2
+          return 0
+        fi
+        sleep 0.5
+      done
+
+      # Force kill if still running
+      if kill -0 "$pid" 2>/dev/null; then
+        echo "[WARN] Monitor did not stop gracefully, forcing..." >&2
+        kill -KILL "$pid" 2>/dev/null || true
+      fi
+    fi
+
+    rm -f "$MONITOR_PID_FILE"
+  else
+    echo "[INFO] No monitor running" >&2
+  fi
+}
+
+# ==============================================================================
+# MAIN EXECUTION
+# ==============================================================================
+
+case "${1:-start}" in
+  start)
+    if [ -f "$MONITOR_PID_FILE" ]; then
+      echo "[ERROR] Monitor already running (PID: $(cat "$MONITOR_PID_FILE"))" >&2
+      exit 1
+    fi
+    start_monitor
+    ;;
+
+  stop)
+    stop_monitor
+    ;;
+
+  restart)
+    stop_monitor
+    sleep 1
+    start_monitor
+    ;;
+
+  status)
+    get_monitor_status
+    ;;
+
+  background)
+    # Start in background
+    if [ -f "$MONITOR_PID_FILE" ]; then
+      echo "[ERROR] Monitor already running (PID: $(cat "$MONITOR_PID_FILE"))" >&2
+      exit 1
+    fi
+    nohup "$0" start > /dev/shm/alert-monitor.log 2>&1 &
+    echo "[INFO] Monitor started in background (PID: $!)" >&2
+    ;;
+
+  *)
+    echo "Usage: $0 {start|stop|restart|status|background}" >&2
+    exit 1
+    ;;
+esac

package/scripts/monitoring/analyze-resources.sh

@@ -0,0 +1,199 @@
+#!/bin/bash
+# Analysis script for resource monitoring results
+# Identifies leaks, spikes, and anomalies
+
+set -euo pipefail
+
+CSV_FILE="${1:-}"
+
+if [ -z "${CSV_FILE}" ] || [ ! -f "${CSV_FILE}" ]; then
+    echo "Usage: $0 <csv_file>"
+    echo "Example: $0 ./reports/monitoring/resource-usage-20250106_120000.csv"
+    exit 1
+fi
+
+OUTPUT_DIR=$(dirname "${CSV_FILE}")
+REPORT_FILE="${OUTPUT_DIR}/analysis-report-$(date +%Y%m%d_%H%M%S).txt"
+
+echo "========================================" | tee "${REPORT_FILE}"
+echo "RESOURCE MONITORING ANALYSIS REPORT" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"
+echo "Input: ${CSV_FILE}" | tee -a "${REPORT_FILE}"
+echo "Generated: $(date)" | tee -a "${REPORT_FILE}"
+echo "" | tee -a "${REPORT_FILE}"
+
+# Skip header, get data
+DATA=$(tail -n +2 "${CSV_FILE}")
+
+if [ -z "${DATA}" ]; then
+    echo "ERROR: No data found in CSV file" | tee -a "${REPORT_FILE}"
+    exit 1
+fi
+
+# Total samples
+TOTAL_SAMPLES=$(echo "${DATA}" | wc -l)
+echo "Total Samples: ${TOTAL_SAMPLES}" | tee -a "${REPORT_FILE}"
+
+# Duration
+FIRST_ELAPSED=$(echo "${DATA}" | head -1 | cut -d',' -f2)
+LAST_ELAPSED=$(echo "${DATA}" | tail -1 | cut -d',' -f2)
+DURATION=$((LAST_ELAPSED - FIRST_ELAPSED))
+echo "Duration: ${DURATION} seconds" | tee -a "${REPORT_FILE}"
+echo "" | tee -a "${REPORT_FILE}"
+
+# Memory RSS analysis
+echo "========================================" | tee -a "${REPORT_FILE}"
+echo "MEMORY (RSS) ANALYSIS" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"
+
+MEMORY_RSS_VALUES=$(echo "${DATA}" | cut -d',' -f3)
+MEMORY_RSS_MIN=$(echo "${MEMORY_RSS_VALUES}" | sort -n | head -1)
+MEMORY_RSS_MAX=$(echo "${MEMORY_RSS_VALUES}" | sort -n | tail -1)
+MEMORY_RSS_AVG=$(echo "${MEMORY_RSS_VALUES}" | awk '{sum+=$1; count++} END {printf "%.2f", sum/count}')
+
+MEMORY_RSS_FIRST=$(echo "${MEMORY_RSS_VALUES}" | head -1)
+MEMORY_RSS_LAST=$(echo "${MEMORY_RSS_VALUES}" | tail -1)
+MEMORY_RSS_GROWTH=$(echo "scale=2; ${MEMORY_RSS_LAST} - ${MEMORY_RSS_FIRST}" | bc)
+MEMORY_RSS_GROWTH_RATE=$(echo "scale=4; ${MEMORY_RSS_GROWTH} / ${DURATION}" | bc 2>/dev/null || echo "0")
+
+echo "Min: ${MEMORY_RSS_MIN} MB" | tee -a "${REPORT_FILE}"
+echo "Max: ${MEMORY_RSS_MAX} MB" | tee -a "${REPORT_FILE}"
+echo "Avg: ${MEMORY_RSS_AVG} MB" | tee -a "${REPORT_FILE}"
+echo "First: ${MEMORY_RSS_FIRST} MB" | tee -a "${REPORT_FILE}"
+echo "Last: ${MEMORY_RSS_LAST} MB" | tee -a "${REPORT_FILE}"
+echo "Growth: ${MEMORY_RSS_GROWTH} MB" | tee -a "${REPORT_FILE}"
+echo "Growth Rate: ${MEMORY_RSS_GROWTH_RATE} MB/sec" | tee -a "${REPORT_FILE}"
+
+# Memory leak detection (growth rate > 1 MB/sec)
+if (( $(echo "${MEMORY_RSS_GROWTH_RATE} > 1.0" | bc -l 2>/dev/null || echo 0) )); then
+    echo "⚠️  LEAK DETECTED: Memory growing at ${MEMORY_RSS_GROWTH_RATE} MB/sec" | tee -a "${REPORT_FILE}"
+elif (( $(echo "${MEMORY_RSS_GROWTH_RATE} > 0.1" | bc -l 2>/dev/null || echo 0) )); then
+    echo "⚠️  WARNING: Slow memory growth detected (${MEMORY_RSS_GROWTH_RATE} MB/sec)" | tee -a "${REPORT_FILE}"
+else
+    echo "✅ No significant memory leak detected" | tee -a "${REPORT_FILE}"
+fi
+echo "" | tee -a "${REPORT_FILE}"
+
+# CPU analysis
+echo "========================================" | tee -a "${REPORT_FILE}"
+echo "CPU ANALYSIS" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"
+
+CPU_VALUES=$(echo "${DATA}" | cut -d',' -f6)
+CPU_MIN=$(echo "${CPU_VALUES}" | sort -n | head -1)
+CPU_MAX=$(echo "${CPU_VALUES}" | sort -n | tail -1)
+CPU_AVG=$(echo "${CPU_VALUES}" | awk '{sum+=$1; count++} END {printf "%.2f", sum/count}')
+
+echo "Min: ${CPU_MIN}%" | tee -a "${REPORT_FILE}"
+echo "Max: ${CPU_MAX}%" | tee -a "${REPORT_FILE}"
+echo "Avg: ${CPU_AVG}%" | tee -a "${REPORT_FILE}"
+
+# CPU spike detection (>80% sustained for >5 samples)
+CPU_SPIKES=$(echo "${CPU_VALUES}" | awk '{if ($1 > 80) count++} END {print count}')
+if [ "${CPU_SPIKES}" -gt 5 ]; then
+    echo "⚠️  CPU SPIKES: ${CPU_SPIKES} samples above 80%" | tee -a "${REPORT_FILE}"
+else
+    echo "✅ No sustained CPU spikes detected" | tee -a "${REPORT_FILE}"
+fi
+echo "" | tee -a "${REPORT_FILE}"
+
+# File descriptor analysis
+echo "========================================" | tee -a "${REPORT_FILE}"
+echo "FILE DESCRIPTOR ANALYSIS" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"
+
+FD_VALUES=$(echo "${DATA}" | cut -d',' -f7)
+FD_MIN=$(echo "${FD_VALUES}" | sort -n | head -1)
+FD_MAX=$(echo "${FD_VALUES}" | sort -n | tail -1)
+FD_AVG=$(echo "${FD_VALUES}" | awk '{sum+=$1; count++} END {printf "%.0f", sum/count}')
+
+FD_FIRST=$(echo "${FD_VALUES}" | head -1)
+FD_LAST=$(echo "${FD_VALUES}" | tail -1)
+FD_GROWTH=$((FD_LAST - FD_FIRST))
+
+echo "Min: ${FD_MIN}" | tee -a "${REPORT_FILE}"
+echo "Max: ${FD_MAX}" | tee -a "${REPORT_FILE}"
+echo "Avg: ${FD_AVG}" | tee -a "${REPORT_FILE}"
+echo "First: ${FD_FIRST}" | tee -a "${REPORT_FILE}"
+echo "Last: ${FD_LAST}" | tee -a "${REPORT_FILE}"
+echo "Growth: ${FD_GROWTH}" | tee -a "${REPORT_FILE}"
+
+# FD leak detection (growth > 100)
+if [ "${FD_GROWTH}" -gt 100 ]; then
+    echo "⚠️  FD LEAK DETECTED: ${FD_GROWTH} unclosed file descriptors" | tee -a "${REPORT_FILE}"
+elif [ "${FD_GROWTH}" -gt 20 ]; then
+    echo "⚠️  WARNING: FD growth detected (${FD_GROWTH})" | tee -a "${REPORT_FILE}"
+else
+    echo "✅ No significant FD leak detected" | tee -a "${REPORT_FILE}"
+fi
+echo "" | tee -a "${REPORT_FILE}"
+
+# Process count analysis
+echo "========================================" | tee -a "${REPORT_FILE}"
+echo "PROCESS COUNT ANALYSIS" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"
+
+PROC_VALUES=$(echo "${DATA}" | cut -d',' -f8)
+PROC_MIN=$(echo "${PROC_VALUES}" | sort -n | head -1)
+PROC_MAX=$(echo "${PROC_VALUES}" | sort -n | tail -1)
+PROC_AVG=$(echo "${PROC_VALUES}" | awk '{sum+=$1; count++} END {printf "%.0f", sum/count}')
+
+PROC_FIRST=$(echo "${PROC_VALUES}" | head -1)
+PROC_LAST=$(echo "${PROC_VALUES}" | tail -1)
+PROC_GROWTH=$((PROC_LAST - PROC_FIRST))
+
+echo "Min: ${PROC_MIN}" | tee -a "${REPORT_FILE}"
+echo "Max: ${PROC_MAX}" | tee -a "${REPORT_FILE}"
+echo "Avg: ${PROC_AVG}" | tee -a "${REPORT_FILE}"
+echo "First: ${PROC_FIRST}" | tee -a "${REPORT_FILE}"
+echo "Last: ${PROC_LAST}" | tee -a "${REPORT_FILE}"
+echo "Growth: ${PROC_GROWTH}" | tee -a "${REPORT_FILE}"
+
+# Process leak detection (growth > 50)
+if [ "${PROC_GROWTH}" -gt 50 ]; then
+    echo "⚠️  PROCESS LEAK DETECTED: ${PROC_GROWTH} orphaned processes" | tee -a "${REPORT_FILE}"
+elif [ "${PROC_GROWTH}" -gt 10 ]; then
+    echo "⚠️  WARNING: Process growth detected (${PROC_GROWTH})" | tee -a "${REPORT_FILE}"
+else
+    echo "✅ No significant process leak detected" | tee -a "${REPORT_FILE}"
+fi
+echo "" | tee -a "${REPORT_FILE}"
+
+# Anomaly summary
+echo "========================================" | tee -a "${REPORT_FILE}"
+echo "ANOMALY SUMMARY" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"
+
+ANOMALY_COUNT=0
+
+if (( $(echo "${MEMORY_RSS_GROWTH_RATE} > 0.1" | bc -l 2>/dev/null || echo 0) )); then
+    echo "• Memory growth: ${MEMORY_RSS_GROWTH_RATE} MB/sec" | tee -a "${REPORT_FILE}"
+    ANOMALY_COUNT=$((ANOMALY_COUNT + 1))
+fi
+
+if [ "${CPU_SPIKES}" -gt 5 ]; then
+    echo "• CPU spikes: ${CPU_SPIKES} samples above 80%" | tee -a "${REPORT_FILE}"
+    ANOMALY_COUNT=$((ANOMALY_COUNT + 1))
+fi
+
+if [ "${FD_GROWTH}" -gt 20 ]; then
+    echo "• FD growth: ${FD_GROWTH}" | tee -a "${REPORT_FILE}"
+    ANOMALY_COUNT=$((ANOMALY_COUNT + 1))
+fi
+
+if [ "${PROC_GROWTH}" -gt 10 ]; then
+    echo "• Process growth: ${PROC_GROWTH}" | tee -a "${REPORT_FILE}"
+    ANOMALY_COUNT=$((ANOMALY_COUNT + 1))
+fi
+
+if [ "${ANOMALY_COUNT}" -eq 0 ]; then
+    echo "✅ No anomalies detected - system healthy" | tee -a "${REPORT_FILE}"
+else
+    echo "" | tee -a "${REPORT_FILE}"
+    echo "⚠️  Total anomalies: ${ANOMALY_COUNT}" | tee -a "${REPORT_FILE}"
+fi
+
+echo "" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"
+echo "Report saved: ${REPORT_FILE}" | tee -a "${REPORT_FILE}"
+echo "========================================" | tee -a "${REPORT_FILE}"

package/scripts/monitoring/dashboards/rate-limiting-dashboard.json

@@ -0,0 +1,211 @@
+{
+  "dashboard": {
+    "title": "CFN Rate Limiting & Backpressure Monitoring",
+    "description": "Real-time monitoring of message inbox utilization, backpressure events, and overflow alerts",
+    "version": "1.0.0",
+    "tags": ["rate-limiting", "backpressure", "inbox", "coordination"],
+    "timezone": "UTC",
+    "refresh": "10s",
+    "panels": [
+      {
+        "id": 1,
+        "title": "Inbox Utilization by Agent",
+        "type": "timeseries",
+        "gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "inbox.utilization",
+            "legend": "{{agent}}",
+            "unit": "percent"
+          }
+        ],
+        "thresholds": [
+          { "value": 75, "color": "yellow", "label": "Warning" },
+          { "value": 90, "color": "red", "label": "Critical" }
+        ],
+        "description": "Message inbox utilization percentage per agent (max 100 messages)"
+      },
+      {
+        "id": 2,
+        "title": "Inbox Message Count",
+        "type": "timeseries",
+        "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "inbox.size",
+            "legend": "{{agent}}",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 75, "color": "yellow", "label": "Warning (75 msgs)" },
+          { "value": 90, "color": "red", "label": "Critical (90 msgs)" }
+        ],
+        "description": "Absolute message count in agent inboxes"
+      },
+      {
+        "id": 3,
+        "title": "Backpressure Events Rate",
+        "type": "timeseries",
+        "gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "backpressure.events_per_min",
+            "legend": "Backpressure events/min",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 100, "color": "yellow", "label": "Warning threshold" }
+        ],
+        "description": "Rate of backpressure wait events (high rate indicates system load)"
+      },
+      {
+        "id": 4,
+        "title": "Message Send Failures",
+        "type": "timeseries",
+        "gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "coordination.send_failures_per_min",
+            "legend": "Send failures/min",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 10, "color": "red", "label": "Critical threshold" }
+        ],
+        "description": "Message delivery failure rate (critical if >10/min)"
+      },
+      {
+        "id": 5,
+        "title": "Inbox Overflow Events",
+        "type": "timeseries",
+        "gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 },
+        "targets": [
+          {
+            "metric": "inbox.overflow_events_per_min",
+            "legend": "Overflow events/min",
+            "unit": "count"
+          }
+        ],
+        "thresholds": [
+          { "value": 1, "color": "red", "label": "Any overflow is critical" }
+        ],
+        "description": "Inbox overflow events (messages dropped due to full inbox)"
+      },
+      {
+        "id": 6,
+        "title": "Alert Summary",
+        "type": "stat",
+        "gridPos": { "x": 12, "y": 16, "w": 6, "h": 8 },
+        "targets": [
+          {
+            "query": "count_alerts_by_severity",
+            "fields": ["critical", "warning", "info"]
+          }
+        ],
+        "description": "Alert counts by severity level (last hour)"
+      },
+      {
+        "id": 7,
+        "title": "Top Utilized Agents",
+        "type": "table",
+        "gridPos": { "x": 18, "y": 16, "w": 6, "h": 8 },
+        "targets": [
+          {
+            "query": "top_inbox_utilization",
+            "fields": ["agent", "utilization", "message_count"],
+            "limit": 10,
+            "order": "desc"
+          }
+        ],
+        "description": "Agents with highest inbox utilization"
+      }
+    ],
+    "annotations": [
+      {
+        "name": "Rate Limiting Alerts",
+        "datasource": "cfn-alerts",
+        "filter": {
+          "tags": ["inbox_high_utilization", "backpressure_high_rate", "inbox_overflow_detected"]
+        },
+        "color": "red"
+      }
+    ],
+    "variables": [
+      {
+        "name": "agent",
+        "type": "query",
+        "query": "SELECT DISTINCT agent FROM inbox_metrics",
+        "description": "Filter by specific agent",
+        "multi": true,
+        "includeAll": true
+      },
+      {
+        "name": "timeRange",
+        "type": "interval",
+        "options": ["5m", "15m", "1h", "6h", "24h"],
+        "default": "1h",
+        "description": "Time range for metrics"
+      }
+    ]
+  },
+  "queries": {
+    "count_alerts_by_severity": {
+      "description": "Count alerts grouped by severity",
+      "source": "/dev/shm/cfn-alerts.jsonl",
+      "aggregation": "GROUP BY severity, COUNT(*)",
+      "timeWindow": "1h"
+    },
+    "top_inbox_utilization": {
+      "description": "Rank agents by inbox utilization",
+      "source": "/dev/shm/cfn-metrics.jsonl",
+      "query": "SELECT agent, MAX(value) as utilization FROM metrics WHERE metric='inbox.utilization' GROUP BY agent ORDER BY utilization DESC LIMIT 10",
+      "timeWindow": "5m"
+    }
+  },
+  "alertRules": [
+    {
+      "name": "Inbox Critical Utilization",
+      "condition": "inbox.utilization >= 90",
+      "severity": "critical",
+      "message": "Agent {{agent}} inbox at {{value}}% utilization (critical threshold: 90%)",
+      "actions": ["emit_alert", "notify_oncall"]
+    },
+    {
+      "name": "Inbox Warning Utilization",
+      "condition": "inbox.utilization >= 75 AND inbox.utilization < 90",
+      "severity": "warning",
+      "message": "Agent {{agent}} inbox at {{value}}% utilization (warning threshold: 75%)",
+      "actions": ["emit_alert"]
+    },
+    {
+      "name": "Backpressure High Rate",
+      "condition": "backpressure.events_per_min > 100",
+      "severity": "warning",
+      "message": "Backpressure events exceeding threshold: {{value}} events/min (threshold: 100/min)",
+      "actions": ["emit_alert"]
+    },
+    {
+      "name": "Message Send Failures Critical",
+      "condition": "coordination.send_failures_per_min > 10",
+      "severity": "critical",
+      "message": "Message send failures critical: {{value}} failures/min (threshold: 10/min)",
+      "actions": ["emit_alert", "notify_oncall", "trigger_incident"]
+    },
+    {
+      "name": "Inbox Overflow Detected",
+      "condition": "inbox.overflow_events_per_min > 0",
+      "severity": "critical",
+      "message": "Inbox overflow detected: {{value}} overflow events in last minute",
+      "actions": ["emit_alert", "notify_oncall", "trigger_incident"]
+    }
+  ],
+  "metadata": {
+    "createdBy": "devops-engineer",
+    "phase": "1",
+    "sprint": "1.5",
+    "lastUpdated": "2025-10-06T19:35:00Z"
+  }
+}