ecip-observability-stack 1.0.0

package/chaos/kill-lsp-daemon.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Chaos Test: Kill LSP Daemon
+# =============================================================================
+# Simulates LSP daemon crashes in the M02 Analysis Engine.
+# Validates that:
+#   - LSPDaemonRestartRate alert fires within expected time
+#   - Analysis backlog grows but recovers after restart
+#   - OTel traces capture the failure correctly
+#
+# Prerequisites:
+#   - kubectl configured for the target cluster
+#   - M02 Analysis Engine deployed
+#   - M08 Observability Stack deployed
+#
+# Usage:
+#   ./kill-lsp-daemon.sh [namespace] [iterations]
+#   ./kill-lsp-daemon.sh ecip 3
+# =============================================================================
+set -euo pipefail
+
+NAMESPACE="${1:-ecip}"
+ITERATIONS="${2:-3}"
+DELAY_BETWEEN_KILLS=30  # seconds
+
+echo "=========================================="
+echo "Chaos Test: Kill LSP Daemon"
+echo "Namespace: ${NAMESPACE}"
+echo "Iterations: ${ITERATIONS}"
+echo "Delay between kills: ${DELAY_BETWEEN_KILLS}s"
+echo "=========================================="
+
+# Verify prerequisites
+echo ""
+echo "[1/5] Verifying prerequisites..."
+
+if ! command -v kubectl &>/dev/null; then
+  echo "ERROR: kubectl not found"
+  exit 1
+fi
+
+if ! kubectl get namespace "${NAMESPACE}" &>/dev/null; then
+  echo "ERROR: Namespace ${NAMESPACE} not found"
+  exit 1
+fi
+
+ANALYSIS_PODS=$(kubectl get pods -n "${NAMESPACE}" -l app=ecip-analysis-engine --no-headers 2>/dev/null | wc -l)
+if [ "${ANALYSIS_PODS}" -eq 0 ]; then
+  echo "ERROR: No ecip-analysis-engine pods found in ${NAMESPACE}"
+  exit 1
+fi
+echo "Found ${ANALYSIS_PODS} analysis engine pod(s)"
+
+# Record baseline metrics
+echo ""
+echo "[2/5] Recording baseline metrics..."
+
+BASELINE_RESTARTS=$(kubectl get pods -n "${NAMESPACE}" -l app=ecip-analysis-engine \
+  -o jsonpath='{.items[0].status.containerStatuses[?(@.name=="lsp-daemon")].restartCount}' 2>/dev/null || echo "0")
+echo "Baseline LSP daemon restart count: ${BASELINE_RESTARTS}"
+
+BASELINE_BACKLOG=$(curl -s "http://prometheus.monitoring:9090/api/v1/query?query=ecip_analysis_backlog_size" 2>/dev/null \
+  | grep -o '"value":\[.*\]' | head -1 || echo "unavailable")
+echo "Baseline analysis backlog: ${BASELINE_BACKLOG}"
+
+# Execute chaos
+echo ""
+echo "[3/5] Executing chaos — killing LSP daemon processes..."
+
+for i in $(seq 1 "${ITERATIONS}"); do
+  echo ""
+  echo "--- Kill iteration ${i}/${ITERATIONS} ---"
+
+  # Get a random analysis engine pod
+  POD=$(kubectl get pods -n "${NAMESPACE}" -l app=ecip-analysis-engine \
+    --no-headers -o custom-columns=":metadata.name" | shuf -n 1)
+
+  echo "Target pod: ${POD}"
+
+  # Kill the LSP daemon process inside the container
+  # This simulates an unexpected daemon crash
+  kubectl exec -n "${NAMESPACE}" "${POD}" -c lsp-daemon -- \
+    kill -9 1 2>/dev/null || echo "  (kill command sent — daemon may have already restarted)"
+
+  echo "  Kill signal sent at $(date -u +%H:%M:%S)"
+
+  if [ "${i}" -lt "${ITERATIONS}" ]; then
+    echo "  Waiting ${DELAY_BETWEEN_KILLS}s before next kill..."
+    sleep "${DELAY_BETWEEN_KILLS}"
+  fi
+done
+
+# Wait for alerts
+echo ""
+echo "[4/5] Waiting for alert propagation (60s)..."
+sleep 60
+
+# Validate results
+echo ""
+echo "[5/5] Validating results..."
+
+# Check restart count increased
+FINAL_RESTARTS=$(kubectl get pods -n "${NAMESPACE}" -l app=ecip-analysis-engine \
+  -o jsonpath='{.items[0].status.containerStatuses[?(@.name=="lsp-daemon")].restartCount}' 2>/dev/null || echo "0")
+echo "Final LSP daemon restart count: ${FINAL_RESTARTS}"
+
+RESTART_DELTA=$((FINAL_RESTARTS - BASELINE_RESTARTS))
+echo "Restart delta: ${RESTART_DELTA}"
+
+if [ "${RESTART_DELTA}" -ge "${ITERATIONS}" ]; then
+  echo "✅ PASS: Restart count increased by at least ${ITERATIONS}"
+else
+  echo "⚠️  WARN: Expected at least ${ITERATIONS} restarts, got ${RESTART_DELTA}"
+fi
+
+# Check if alert fired
+echo ""
+echo "Checking Alertmanager for LSPDaemonRestartRate alert..."
+ALERT_STATUS=$(curl -s "http://alertmanager.monitoring:9093/api/v2/alerts?filter=alertname=LSPDaemonRestartRate" 2>/dev/null || echo "unavailable")
+if echo "${ALERT_STATUS}" | grep -q "LSPDaemonRestartRate"; then
+  echo "✅ PASS: LSPDaemonRestartRate alert is firing"
+else
+  echo "⚠️  WARN: LSPDaemonRestartRate alert not detected (may need more time)"
+fi
+
+# Check pod recovery
+echo ""
+echo "Checking pod recovery..."
+READY_PODS=$(kubectl get pods -n "${NAMESPACE}" -l app=ecip-analysis-engine \
+  --no-headers | grep -c "Running" || echo "0")
+echo "Running analysis pods: ${READY_PODS}/${ANALYSIS_PODS}"
+
+if [ "${READY_PODS}" -eq "${ANALYSIS_PODS}" ]; then
+  echo "✅ PASS: All pods recovered"
+else
+  echo "❌ FAIL: Not all pods recovered"
+fi
+
+echo ""
+echo "=========================================="
+echo "Chaos Test Complete"
+echo "=========================================="
+echo ""
+echo "Manual checks:"
+echo "  1. Grafana → ECIP → LSP Daemon Health — verify restart spike visible"
+echo "  2. Grafana → ECIP → Analysis Throughput — verify backlog grew then drained"
+echo "  3. Tempo — search for error spans in ecip-analysis-engine during test window"
+echo "  4. Slack #ecip-alerts — verify notification received"

package/chaos/redis-node-failure.sh

@@ -0,0 +1,318 @@
+#!/bin/bash#!/usr/bin/env bash
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+echo "=========================================="echo "    4. Verify all alerts resolve within 10 minutes"echo "    3. Check Query Latency dashboard for cascading impact"echo "    2. Verify cache hit rate recovers to > 85%"echo "    1. Check Grafana → ECIP → Cache Performance dashboard"echo "  Next steps:"echo ""echo "  Redis recovery: $REDIS_PING"echo "=========================================="echo "  Chaos Test Complete"echo "=========================================="echo ""# Summaryfi  echo "  QueryLatencySLABreach (cascade): $( [ "$QUERY_ALERT" -gt 0 ] && echo 'FIRING ⚠️' || echo 'Not firing' )"  echo "  KnowledgeStoreWriteLatencyHigh: $( [ "$LATENCY_ALERT" -gt 0 ] && echo 'FIRING ✅' || echo 'Not firing' )"  echo "  CacheHitRateDegraded:         $( [ "$CACHE_ALERT" -gt 0 ] && echo 'FIRING ✅' || echo 'Not firing' )"      | grep -c "QueryLatencySLABreach" || echo "0")    wget -qO- 'http://localhost:9090/api/v1/alerts' 2>/dev/null \  QUERY_ALERT=$(kubectl exec -n monitoring "$PROM_POD" -- \      | grep -c "KnowledgeStoreWriteLatencyHigh" || echo "0")    wget -qO- 'http://localhost:9090/api/v1/alerts' 2>/dev/null \  LATENCY_ALERT=$(kubectl exec -n monitoring "$PROM_POD" -- \      | grep -c "CacheHitRateDegraded" || echo "0")    wget -qO- 'http://localhost:9090/api/v1/alerts' 2>/dev/null \  CACHE_ALERT=$(kubectl exec -n monitoring "$PROM_POD" -- \if [ -n "$PROM_POD" ]; thenecho "[ALERTS] Checking for cache-related alerts..."echo ""# Check alertsecho "  Redis PING: $REDIS_PING"REDIS_PING=$(kubectl exec -n "$NAMESPACE" "$TARGET_POD" -- redis-cli PING 2>/dev/null || echo "FAIL")echo "[POST-TEST] Verifying Redis recovery..."echo ""# Check Redis healthsleep 30echo "[RECOVERY] Redis should now be responsive. Waiting 30s for stabilization..."echo ""wait "$SLEEP_PID" 2>/dev/null || true# Wait for the DEBUG SLEEP to completedone  ELAPSED=$((ELAPSED + INTERVAL))  sleep "$INTERVAL"    fi    echo "    Active alerts: $ALERTS"      | grep -oP '"alertname":"[^"]*"' | sort -u || echo "none")      wget -qO- 'http://localhost:9090/api/v1/alerts' 2>/dev/null \    ALERTS=$(kubectl exec -n monitoring "$PROM_POD" -- \  if [ -n "$PROM_POD" ]; then  # Check for degradation alerts    echo "  [${ELAPSED}s / ${DURATION}s] Redis still paused. Remaining: ${REMAINING}s"  REMAINING=$((DURATION - ELAPSED))while [ "$ELAPSED" -lt "$DURATION" ]; doINTERVAL=15ELAPSED=0echo "[MONITOR] Monitoring for ${DURATION}s..."echo ""# Monitor during failureecho "  Redis is now unresponsive to client requests"echo "  Redis DEBUG SLEEP issued for ${DURATION}s"SLEEP_PID=$!  redis-cli DEBUG SLEEP "$DURATION" &kubectl exec -n "$NAMESPACE" "$TARGET_POD" -- \echo "[CHAOS] Pausing Redis container in pod $TARGET_POD..."echo ""# Simulate failure — pause the Redis containerfi  echo "  Cache hit rate before: $CACHE_HIT_RATE"    | grep -oP '"value":\[[\d.]+,"([\d.]+)"\]' | head -1 || echo "unknown")    wget -qO- 'http://localhost:9090/api/v1/query?query=ecip_cache_hit_rate{module="M03"}' 2>/dev/null \  CACHE_HIT_RATE=$(kubectl exec -n monitoring "$PROM_POD" -- \if [ -n "$PROM_POD" ]; then  -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")PROM_POD=$(kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus \# Query Prometheus for current cache hit rateecho "[PRE-TEST] Capturing baseline metrics..."echo ""# Capture pre-test metricsecho "Target Redis pod: $TARGET_POD"TARGET_POD=$(echo "$REDIS_PODS" | tr ' ' '\n' | head -1)# Pick the first non-master Redis pod (to avoid failover complexity)fi  exit 1  echo "ERROR: No Redis pods found in namespace $NAMESPACE"if [ -z "$REDIS_PODS" ]; thenfi    -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)  REDIS_PODS=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/name=redis \  # Try alternate labelif [ -z "$REDIS_PODS" ]; then  -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)REDIS_PODS=$(kubectl get pods -n "$NAMESPACE" -l app=redis \# Find Redis podsecho "=========================================="echo "  Failure duration: ${DURATION}s"echo "  Namespace: $NAMESPACE"echo "=========================================="echo "  Chaos Test: Redis Node Failure"echo "=========================================="DURATION="${2:-60}"NAMESPACE="${1:-ecip}"set -euo pipefail# =============================================================================#   ./redis-node-failure.sh [--namespace ecip] [--duration 60]# Usage:##   - Prometheus and Alertmanager running#   - Redis StatefulSet running (redis-cluster or redis-sentinel)#   - kubectl configured with access to the ECIP namespace# Prerequisites:##   5. Redis auto-recovers#   4. Downstream QueryLatencySLABreach may fire (cascading effect)#   3. Grafana Cache Performance dashboard shows the degradation#   2. KnowledgeStoreWriteLatencyHigh alert may fire (write retries)#   1. CacheHitRateDegraded alert fires as cache hit rate drops# Validates that:# Simulates Redis node failure for M03 Knowledge Store cache.# =============================================================================# Chaos Test: Redis Node Failure# =============================================================================# =============================================================================
+# Chaos Test: Redis Node Failure
+# =============================================================================
+# Simulates Redis node failure in the M03 Knowledge Store cache layer.
+# Validates that:
+#   - CacheHitRateDegraded alert fires
+#   - Query latency increases but service remains available
+#   - Cache recovers after Redis node returns
+#
+# Prerequisites:
+#   - kubectl configured for the target cluster
+#   - M03 Knowledge Store deployed with Redis
+#   - M08 Observability Stack deployed
+#
+# Usage:
+#   ./redis-node-failure.sh [namespace] [downtime_seconds]
+#   ./redis-node-failure.sh ecip 120
+# =============================================================================
+set -euo pipefail
+
+NAMESPACE="${1:-ecip}"
+DOWNTIME="${2:-120}"  # How long to keep Redis down (seconds)
+
+echo "=========================================="
+echo "Chaos Test: Redis Node Failure"
+echo "Namespace: ${NAMESPACE}"
+echo "Downtime: ${DOWNTIME}s"
+echo "=========================================="
+
+# Verify prerequisites
+echo ""
+echo "[1/6] Verifying prerequisites..."
+
+if ! command -v kubectl &>/dev/null; then
+  echo "ERROR: kubectl not found"
+  exit 1
+fi
+
+# Find Redis pods
+REDIS_PODS=$(kubectl get pods -n "${NAMESPACE}" -l app=redis --no-headers 2>/dev/null | wc -l)
+if [ "${REDIS_PODS}" -eq 0 ]; then
+  # Try alternative label selectors
+  REDIS_PODS=$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=redis --no-headers 2>/dev/null | wc -l)
+fi
+
+if [ "${REDIS_PODS}" -eq 0 ]; then
+  echo "ERROR: No Redis pods found in ${NAMESPACE}"
+  echo "Tried labels: app=redis, app.kubernetes.io/name=redis"
+  exit 1
+fi
+echo "Found ${REDIS_PODS} Redis pod(s)"
+
+# Record baseline
+echo ""
+echo "[2/6] Recording baseline metrics..."
+
+BASELINE_CACHE_HIT=$(curl -s "http://prometheus.monitoring:9090/api/v1/query?query=ecip_cache_hit_rate{module=\"M03\"}" 2>/dev/null \
+  | grep -o '"value":\[.*\]' | head -1 || echo "unavailable")
+echo "Baseline cache hit rate: ${BASELINE_CACHE_HIT}"
+
+BASELINE_QUERY_P95=$(curl -s "http://prometheus.monitoring:9090/api/v1/query?query=histogram_quantile(0.95,sum(rate(ecip_query_duration_ms_bucket{module=\"M04\"}[5m]))by(le))" 2>/dev/null \
+  | grep -o '"value":\[.*\]' | head -1 || echo "unavailable")
+echo "Baseline query p95 latency: ${BASELINE_QUERY_P95}"
+
+# Kill Redis
+echo ""
+echo "[3/6] Killing Redis node..."
+
+REDIS_POD=$(kubectl get pods -n "${NAMESPACE}" -l app=redis \
+  --no-headers -o custom-columns=":metadata.name" 2>/dev/null | head -1)
+
+if [ -z "${REDIS_POD}" ]; then
+  REDIS_POD=$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=redis \
+    --no-headers -o custom-columns=":metadata.name" 2>/dev/null | head -1)
+fi
+
+echo "Target Redis pod: ${REDIS_POD}"
+
+# Scale down Redis to simulate node failure
+REDIS_DEPLOYMENT=$(kubectl get pods -n "${NAMESPACE}" "${REDIS_POD}" \
+  -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "")
+
+if [ -n "${REDIS_DEPLOYMENT}" ]; then
+  echo "Scaling down Redis (${REDIS_DEPLOYMENT}) to 0 replicas..."
+  kubectl scale -n "${NAMESPACE}" statefulset/"${REDIS_DEPLOYMENT}" --replicas=0 2>/dev/null || \
+  kubectl scale -n "${NAMESPACE}" deployment/"${REDIS_DEPLOYMENT}" --replicas=0 2>/dev/null || \
+  echo "  Fallback: deleting pod directly..."
+  kubectl delete pod -n "${NAMESPACE}" "${REDIS_POD}" --grace-period=0 --force 2>/dev/null || true
+else
+  echo "Deleting Redis pod..."
+  kubectl delete pod -n "${NAMESPACE}" "${REDIS_POD}" --grace-period=0 --force 2>/dev/null || true
+fi
+
+echo "Redis killed at $(date -u +%H:%M:%S)"
+
+# Wait during downtime
+echo ""
+echo "[4/6] Waiting for downtime period (${DOWNTIME}s)..."
+echo "  During this time, verify:"
+echo "    - Grafana → Cache Performance → hit rate dropping"
+echo "    - Grafana → Query Latency → p95 increasing"
+echo "    - Application logs show Redis connection errors"
+
+ELAPSED=0
+while [ "${ELAPSED}" -lt "${DOWNTIME}" ]; do
+  REMAINING=$((DOWNTIME - ELAPSED))
+  echo "  ${REMAINING}s remaining..."
+  sleep 15
+  ELAPSED=$((ELAPSED + 15))
+done
+
+# Restore Redis
+echo ""
+echo "[5/6] Restoring Redis..."
+
+if [ -n "${REDIS_DEPLOYMENT}" ]; then
+  echo "Scaling Redis back to ${REDIS_PODS} replicas..."
+  kubectl scale -n "${NAMESPACE}" statefulset/"${REDIS_DEPLOYMENT}" --replicas="${REDIS_PODS}" 2>/dev/null || \
+  kubectl scale -n "${NAMESPACE}" deployment/"${REDIS_DEPLOYMENT}" --replicas="${REDIS_PODS}" 2>/dev/null || true
+fi
+
+echo "Waiting 60s for Redis recovery and cache warm-up..."
+sleep 60
+
+# Validate
+echo ""
+echo "[6/6] Validating results..."
+
+# Check Redis is back
+REDIS_READY=$(kubectl get pods -n "${NAMESPACE}" -l app=redis \
+  --no-headers 2>/dev/null | grep -c "Running" || echo "0")
+if [ "${REDIS_READY}" -eq 0 ]; then
+  REDIS_READY=$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=redis \
+    --no-headers 2>/dev/null | grep -c "Running" || echo "0")
+fi
+
+if [ "${REDIS_READY}" -gt 0 ]; then
+  echo "✅ PASS: Redis is running again (${REDIS_READY} pod(s))"
+else
+  echo "❌ FAIL: Redis has not recovered"
+fi
+
+# Check if cache alert fired
+echo ""
+echo "Checking Alertmanager for CacheHitRateDegraded alert..."
+ALERT_STATUS=$(curl -s "http://alertmanager.monitoring:9093/api/v2/alerts?filter=alertname=CacheHitRateDegraded" 2>/dev/null || echo "unavailable")
+if echo "${ALERT_STATUS}" | grep -q "CacheHitRateDegraded"; then
+  echo "✅ PASS: CacheHitRateDegraded alert fired"
+else
+  echo "⚠️  WARN: CacheHitRateDegraded alert not detected"
+fi
+
+# Final metrics
+echo ""
+FINAL_CACHE_HIT=$(curl -s "http://prometheus.monitoring:9090/api/v1/query?query=ecip_cache_hit_rate{module=\"M03\"}" 2>/dev/null \
+  | grep -o '"value":\[.*\]' | head -1 || echo "unavailable")
+echo "Final cache hit rate: ${FINAL_CACHE_HIT} (expect recovering toward baseline)"
+
+echo ""
+echo "=========================================="
+echo "Chaos Test Complete"
+echo "=========================================="
+echo ""
+echo "Manual checks:"
+echo "  1. Grafana → Cache Performance — verify dip during downtime, recovery after"
+echo "  2. Grafana → Query Latency — verify latency spike during cache miss period"
+echo "  3. Verify M03 Knowledge Store handled Redis failure gracefully (no crash)"
+echo "  4. Verify cache warm-up is progressing (hit rate increasing)"