claude-flow-novice 2.3.1 → 2.3.3

package/scripts/monitoring/dynamic-monitor.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Dynamic monitoring script for 50-agent stability test
+# Pattern: 60s sleep x10 times, then 120s sleep x30 times
+
+echo "🔍 Starting dynamic monitoring for 50-agent test..."
+echo "   Phase 1: 60-second intervals (10 checks)"
+echo "   Phase 2: 120-second intervals (30 checks)"
+
+TOTAL_CHECKS=0
+
+# Phase 1: 60-second intervals, 10 times
+for i in {1..10}; do
+    echo "📊 Check $TOTAL_CHECKS (Phase 1, $i/10) - $(date)"
+
+    # Agent test status
+    echo "   50-Agent Test: $(ps aux | grep 'node.*50-agent-test' | grep -v grep | wc -l) processes running"
+
+    # Stability monitor status
+    echo "   Monitor: $(ps aux | grep 'node.*stability-monitor' | grep -v grep | wc -l) processes running"
+
+    # Memory usage
+    AGENT_MEM=$(ps aux | grep 'node.*50-agent-test' | grep -v grep | awk '{sum+=$6} END {print sum/1024}')
+    MONITOR_MEM=$(ps aux | grep 'node.*stability-monitor' | grep -v grep | awk '{sum+=$6} END {print sum/1024}')
+    echo "   Memory: Agent=${AGENT_MEM:-0}MB, Monitor=${MONITOR_MEM:-0}MB"
+
+    # /dev/shm usage
+    TMPFS_USAGE=$(df /dev/shm | tail -1 | awk '{print $5}')
+    echo "   tmpfs usage: $TMPFS_USAGE"
+
+    # Check for recent coordination activity
+    RECENT_LOG=$(tail -20 ./.artifacts/stability/stability-test.log 2>/dev/null | grep -c "coordination cycle")
+    echo "   Recent coordination cycles: $RECENT_LOG"
+
+    echo ""
+
+    ((TOTAL_CHECKS++))
+    sleep 60
+done
+
+echo "🔄 Phase 1 complete. Switching to 120-second intervals..."
+echo ""
+
+# Phase 2: 120-second intervals, 30 times
+for i in {1..30}; do
+    echo "📊 Check $TOTAL_CHECKS (Phase 2, $i/30) - $(date)"
+
+    # Agent test status
+    echo "   50-Agent Test: $(ps aux | grep 'node.*50-agent-test' | grep -v grep | wc -l) processes running"
+
+    # Stability monitor status
+    echo "   Monitor: $(ps aux | grep 'node.*stability-monitor' | grep -v grep | wc -l) processes running"
+
+    # Memory usage
+    AGENT_MEM=$(ps aux | grep 'node.*50-agent-test' | grep -v grep | awk '{sum+=$6} END {print sum/1024}')
+    MONITOR_MEM=$(ps aux | grep 'node.*stability-monitor' | grep -v grep | awk '{sum+=$6} END {print sum/1024}')
+    echo "   Memory: Agent=${AGENT_MEM:-0}MB, Monitor=${MONITOR_MEM:-0}MB"
+
+    # /dev/shm usage
+    TMPFS_USAGE=$(df /dev/shm | tail -1 | awk '{print $5}')
+    echo "   tmpfs usage: $TMPFS_USAGE"
+
+    # Check for recent coordination activity
+    RECENT_LOG=$(tail -20 ./.artifacts/stability/stability-test.log 2>/dev/null | grep -c "coordination cycle")
+    echo "   Recent coordination cycles: $RECENT_LOG"
+
+    # Check for stability monitor alerts
+    ALERTS=$(grep -c "MEMORY LEAK DETECTED\|FD LEAK DETECTED" ./.artifacts/stability/*.log 2>/dev/null || echo "0")
+    echo "   Stability alerts: $ALERTS"
+
+    # Process count for agents
+    AGENT_PROCS=$(pgrep -f "agent-worker" | wc -l)
+    echo "   Agent worker processes: $AGENT_PROCS"
+
+    echo ""
+
+    ((TOTAL_CHECKS++))
+    sleep 120
+done
+
+echo "✅ Dynamic monitoring complete. Total checks: $TOTAL_CHECKS"
+echo "📋 Final status report:"
+echo "   Agent test status: $(ps aux | grep 'node.*50-agent-test' | grep -v grep | wc -l) processes"
+echo "   Monitor status: $(ps aux | grep 'node.*stability-monitor' | grep -v grep | wc -l) processes"
+echo "   Results available in: ./.artifacts/stability/"

package/scripts/monitoring/launch-stability-test.sh

@@ -0,0 +1,184 @@
+#!/bin/bash
+# 8-Hour Stability Test Launcher
+# Phase 2 Sprint 2.3 - Stability Validation with 50 Agents
+# Success Criteria: <5% memory growth, >1000 msg/s throughput, no resource leaks
+
+set -euo pipefail
+
+# Configuration
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+DOCKER_CONFIG_DIR="${PROJECT_ROOT}/config/docker"
+RESULTS_DIR="${DOCKER_CONFIG_DIR}/stability-results"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+
+# Test parameters
+TEST_DURATION_HOURS=8
+TEST_DURATION_SECONDS=$((TEST_DURATION_HOURS * 3600))
+MAX_AGENTS=50
+THROUGHPUT_TARGET=1000
+MEMORY_GROWTH_THRESHOLD=5  # percent
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Functions
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+cleanup() {
+    log_info "Cleaning up Docker resources..."
+    cd "${DOCKER_CONFIG_DIR}"
+    docker-compose -f docker-compose.stability-test.yml down -v 2>/dev/null || true
+    log_success "Cleanup complete"
+}
+
+# Trap signals for cleanup
+trap cleanup EXIT INT TERM
+
+# Main execution
+main() {
+    log_info "8-Hour Stability Test Launcher"
+    log_info "=============================="
+    log_info "Configuration:"
+    log_info "  - Duration: ${TEST_DURATION_HOURS} hours (${TEST_DURATION_SECONDS}s)"
+    log_info "  - Max Agents: ${MAX_AGENTS}"
+    log_info "  - Throughput Target: ${THROUGHPUT_TARGET} msg/s"
+    log_info "  - Memory Growth Threshold: ${MEMORY_GROWTH_THRESHOLD}%"
+    log_info ""
+
+    # Pre-flight checks
+    log_info "Running pre-flight checks..."
+
+    # Check Docker
+    if ! command -v docker &> /dev/null; then
+        log_error "Docker not found. Please install Docker."
+        exit 1
+    fi
+
+    if ! command -v docker-compose &> /dev/null; then
+        log_error "docker-compose not found. Please install docker-compose."
+        exit 1
+    fi
+
+    log_success "Docker and docker-compose found"
+
+    # Check system resources
+    log_info "Checking system resources..."
+    AVAILABLE_MEM=$(free -m | awk 'NR==2{print $7}')
+    REQUIRED_MEM=8192  # 8GB minimum
+
+    if [ "${AVAILABLE_MEM}" -lt "${REQUIRED_MEM}" ]; then
+        log_warning "Available memory (${AVAILABLE_MEM}MB) is below recommended (${REQUIRED_MEM}MB)"
+        read -p "Continue anyway? (y/N): " -n 1 -r
+        echo
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+            exit 1
+        fi
+    else
+        log_success "Sufficient memory available (${AVAILABLE_MEM}MB)"
+    fi
+
+    # Create results directory
+    mkdir -p "${RESULTS_DIR}"
+    log_success "Results directory created: ${RESULTS_DIR}"
+
+    # Build and start containers
+    log_info "Building Docker images..."
+    cd "${DOCKER_CONFIG_DIR}"
+
+    if ! docker-compose -f docker-compose.stability-test.yml build --no-cache; then
+        log_error "Docker build failed"
+        exit 1
+    fi
+    log_success "Docker images built successfully"
+
+    # Start monitoring stack first
+    log_info "Starting monitoring stack..."
+    docker-compose -f docker-compose.stability-test.yml up -d prometheus grafana node-exporter
+
+    # Wait for monitoring to be ready
+    log_info "Waiting for monitoring services to be ready (30s)..."
+    sleep 30
+    log_success "Monitoring stack ready"
+    log_info "  - Prometheus: http://localhost:9090"
+    log_info "  - Grafana: http://localhost:3001 (admin/stability-test)"
+
+    # Start stability test
+    log_info ""
+    log_info "Starting 8-hour stability test..."
+    log_info "Test start time: $(date)"
+    log_info "Expected completion: $(date -d "+${TEST_DURATION_HOURS} hours" 2>/dev/null || date -v+${TEST_DURATION_HOURS}H 2>/dev/null || echo "in ${TEST_DURATION_HOURS} hours")"
+    log_info ""
+
+    # Launch test container
+    docker-compose -f docker-compose.stability-test.yml up stability-test 2>&1 | tee "${RESULTS_DIR}/test-output-${TIMESTAMP}.log"
+
+    TEST_EXIT_CODE=$?
+
+    # Analyze results
+    log_info ""
+    log_info "Test execution completed with exit code: ${TEST_EXIT_CODE}"
+    log_info "Analyzing results..."
+
+    if [ -f "${RESULTS_DIR}/resource-usage-"*".csv" ]; then
+        LATEST_CSV=$(ls -t "${RESULTS_DIR}/resource-usage-"*".csv" | head -1)
+        log_info "Resource usage data: ${LATEST_CSV}"
+
+        # Calculate memory growth
+        INITIAL_MEM=$(head -2 "${LATEST_CSV}" | tail -1 | cut -d',' -f3)
+        FINAL_MEM=$(tail -1 "${LATEST_CSV}" | cut -d',' -f3)
+        MEMORY_GROWTH=$(awk -v init="${INITIAL_MEM}" -v final="${FINAL_MEM}" 'BEGIN {printf "%.2f", ((final - init) / init) * 100}')
+
+        log_info "Memory Analysis:"
+        log_info "  - Initial RSS: ${INITIAL_MEM}MB"
+        log_info "  - Final RSS: ${FINAL_MEM}MB"
+        log_info "  - Memory Growth: ${MEMORY_GROWTH}%"
+
+        if (( $(echo "${MEMORY_GROWTH} < ${MEMORY_GROWTH_THRESHOLD}" | bc -l) )); then
+            log_success "Memory growth within threshold (<${MEMORY_GROWTH_THRESHOLD}%)"
+        else
+            log_error "Memory growth exceeded threshold (${MEMORY_GROWTH}% > ${MEMORY_GROWTH_THRESHOLD}%)"
+        fi
+    else
+        log_warning "No resource usage CSV found"
+    fi
+
+    # Final summary
+    log_info ""
+    log_info "=============================="
+    log_info "Stability Test Complete"
+    log_info "=============================="
+    log_info "Test Duration: ${TEST_DURATION_HOURS} hours"
+    log_info "Results Directory: ${RESULTS_DIR}"
+    log_info "Test Log: ${RESULTS_DIR}/test-output-${TIMESTAMP}.log"
+    log_info ""
+
+    if [ "${TEST_EXIT_CODE}" -eq 0 ]; then
+        log_success "Test passed successfully"
+        return 0
+    else
+        log_error "Test failed with exit code ${TEST_EXIT_CODE}"
+        return 1
+    fi
+}
+
+# Execute main function
+main "$@"

package/scripts/monitoring/monitor-test.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Wrapper script to monitor a test execution
+# Usage: ./monitor-test.sh <test_command>
+# Example: ./monitor-test.sh "npm test -- tests/integration/100-agent-coordination.test.sh"
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RESULTS_DIR="${SCRIPT_DIR}/../../monitoring-results"
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+
+# Create results directory
+mkdir -p "${RESULTS_DIR}"
+
+# Validate arguments
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <test_command>"
+    echo ""
+    echo "Examples:"
+    echo "  $0 'npm test -- tests/integration/100-agent-coordination.test.sh'"
+    echo "  $0 'bash tests/integration/100-agent-coordination.test.sh'"
+    echo ""
+    exit 1
+fi
+
+TEST_COMMAND="$*"
+
+echo "========================================="
+echo "RESOURCE MONITORING WRAPPER"
+echo "========================================="
+echo "Test Command: ${TEST_COMMAND}"
+echo "Results Dir: ${RESULTS_DIR}"
+echo "Timestamp: ${TIMESTAMP}"
+echo ""
+
+# Start resource monitor in background
+echo "Starting resource monitor..."
+"${SCRIPT_DIR}/resource-monitor.sh" "${RESULTS_DIR}" &
+MONITOR_PID=$!
+
+# Give monitor time to initialize
+sleep 2
+
+# Trap to ensure monitor cleanup
+cleanup() {
+    echo ""
+    echo "Stopping resource monitor..."
+    if kill -0 "${MONITOR_PID}" 2>/dev/null; then
+        kill -TERM "${MONITOR_PID}" 2>/dev/null || true
+        wait "${MONITOR_PID}" 2>/dev/null || true
+    fi
+
+    # Find the CSV file
+    CSV_FILE=$(find "${RESULTS_DIR}" -name "resource-usage-${TIMESTAMP}.csv" -type f 2>/dev/null | head -1)
+
+    if [ -n "${CSV_FILE}" ] && [ -f "${CSV_FILE}" ]; then
+        echo ""
+        echo "Analyzing results..."
+        "${SCRIPT_DIR}/analyze-resources.sh" "${CSV_FILE}"
+        echo ""
+        echo "========================================="
+        echo "MONITORING COMPLETE"
+        echo "========================================="
+        echo "CSV Data: ${CSV_FILE}"
+        echo "Analysis: ${RESULTS_DIR}/analysis-report-*.txt"
+        echo ""
+    else
+        echo "WARNING: No monitoring data found"
+    fi
+}
+
+trap cleanup EXIT INT TERM
+
+# Run the test command
+echo "Running test command..."
+echo "========================================="
+echo ""
+
+if eval "${TEST_COMMAND}"; then
+    TEST_EXIT_CODE=0
+    echo ""
+    echo "Test PASSED"
+else
+    TEST_EXIT_CODE=$?
+    echo ""
+    echo "Test FAILED (exit code: ${TEST_EXIT_CODE})"
+fi
+
+echo ""
+echo "Waiting for final metrics..."
+sleep 3
+
+exit ${TEST_EXIT_CODE}

package/scripts/monitoring/pre-test-validation.sh

@@ -0,0 +1,208 @@
+#!/bin/bash
+# Pre-Test Validation for 8-Hour Stability Test
+# Validates system requirements and configuration before test execution
+
+set -euo pipefail
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+# Requirements
+MIN_MEMORY_GB=8
+MIN_DISK_GB=20
+MIN_CPU_CORES=4
+REQUIRED_DOCKER_VERSION="20.10"
+REQUIRED_COMPOSE_VERSION="1.29"
+
+# Counters
+ERRORS=0
+WARNINGS=0
+CHECKS=0
+
+log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
+log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
+log_warning() { echo -e "${YELLOW}[!]${NC} $1"; WARNINGS=$((WARNINGS + 1)); }
+log_error() { echo -e "${RED}[✗]${NC} $1"; ERRORS=$((ERRORS + 1)); }
+
+log_info "Pre-Test Validation - 8-Hour Stability Test"
+log_info "=============================================="
+echo
+
+# Check 1: Docker installed
+log_info "Checking Docker installation..."
+if command -v docker &> /dev/null; then
+    DOCKER_VERSION=$(docker --version | grep -oP '\d+\.\d+\.\d+' | head -1)
+    log_success "Docker installed: v${DOCKER_VERSION}"
+else
+    log_error "Docker not found. Install Docker: https://docs.docker.com/get-docker/"
+fi
+
+# Check 2: Docker Compose installed
+log_info "Checking Docker Compose..."
+if command -v docker-compose &> /dev/null; then
+    COMPOSE_VERSION=$(docker-compose --version | grep -oP '\d+\.\d+\.\d+' | head -1)
+    log_success "Docker Compose installed: v${COMPOSE_VERSION}"
+else
+    log_error "Docker Compose not found. Install: https://docs.docker.com/compose/install/"
+fi
+
+# Check 3: Docker daemon running
+log_info "Checking Docker daemon..."
+if docker info &> /dev/null; then
+    log_success "Docker daemon is running"
+else
+    log_error "Docker daemon is not running. Start with: sudo systemctl start docker"
+fi
+
+# Check 4: Memory
+log_info "Checking available memory..."
+if command -v free &> /dev/null; then
+    AVAILABLE_MEM_MB=$(free -m | awk 'NR==2{print $7}')
+    AVAILABLE_MEM_GB=$((AVAILABLE_MEM_MB / 1024))
+
+    if [ "${AVAILABLE_MEM_GB}" -ge "${MIN_MEMORY_GB}" ]; then
+        log_success "Available memory: ${AVAILABLE_MEM_GB}GB (required: ${MIN_MEMORY_GB}GB)"
+    else
+        log_warning "Low memory: ${AVAILABLE_MEM_GB}GB available (recommended: ${MIN_MEMORY_GB}GB)"
+    fi
+else
+    log_warning "Cannot check memory (free command not found)"
+fi
+
+# Check 5: Disk space
+log_info "Checking disk space..."
+AVAILABLE_DISK_GB=$(df -BG . | awk 'NR==2{print $4}' | sed 's/G//')
+
+if [ "${AVAILABLE_DISK_GB}" -ge "${MIN_DISK_GB}" ]; then
+    log_success "Available disk space: ${AVAILABLE_DISK_GB}GB (required: ${MIN_DISK_GB}GB)"
+else
+    log_warning "Low disk space: ${AVAILABLE_DISK_GB}GB available (recommended: ${MIN_DISK_GB}GB)"
+fi
+
+# Check 6: CPU cores
+log_info "Checking CPU cores..."
+CPU_CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "unknown")
+
+if [ "${CPU_CORES}" != "unknown" ] && [ "${CPU_CORES}" -ge "${MIN_CPU_CORES}" ]; then
+    log_success "CPU cores: ${CPU_CORES} (required: ${MIN_CPU_CORES})"
+else
+    log_warning "CPU cores: ${CPU_CORES} (recommended: ${MIN_CPU_CORES})"
+fi
+
+# Check 7: Configuration files
+log_info "Checking configuration files..."
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+FILES_TO_CHECK=(
+    "${PROJECT_ROOT}/config/docker/docker-compose.stability-test.yml"
+    "${PROJECT_ROOT}/config/docker/prometheus.stability.yml"
+    "${PROJECT_ROOT}/scripts/monitoring/resource-monitor.sh"
+    "${PROJECT_ROOT}/tests/performance/analyze-stability-results.js"
+)
+
+for file in "${FILES_TO_CHECK[@]}"; do
+    if [ -f "$file" ]; then
+        log_success "Found: $(basename "$file")"
+    else
+        log_error "Missing: $file"
+    fi
+done
+
+# Check 8: Node.js
+log_info "Checking Node.js..."
+if command -v node &> /dev/null; then
+    NODE_VERSION=$(node --version)
+    log_success "Node.js installed: ${NODE_VERSION}"
+else
+    log_warning "Node.js not found (required for analysis scripts)"
+fi
+
+# Check 9: Required directories
+log_info "Checking directories..."
+REQUIRED_DIRS=(
+    "${PROJECT_ROOT}/config/docker/stability-results"
+    "${PROJECT_ROOT}/config/docker/grafana-dashboards"
+    "${PROJECT_ROOT}/scripts/monitoring"
+    "${PROJECT_ROOT}/tests/performance"
+)
+
+for dir in "${REQUIRED_DIRS[@]}"; do
+    if [ -d "$dir" ]; then
+        log_success "Directory exists: $(basename "$dir")"
+    else
+        log_warning "Creating directory: $(basename "$dir")"
+        mkdir -p "$dir"
+    fi
+done
+
+# Check 10: Docker resources
+log_info "Checking Docker resource limits..."
+if docker info &> /dev/null; then
+    DOCKER_MEM=$(docker info 2>/dev/null | grep "Total Memory" | awk '{print $3}')
+    DOCKER_CPUS=$(docker info 2>/dev/null | grep "CPUs" | awk '{print $2}')
+
+    if [ -n "$DOCKER_MEM" ]; then
+        log_success "Docker memory: ${DOCKER_MEM}"
+    fi
+
+    if [ -n "$DOCKER_CPUS" ]; then
+        log_success "Docker CPUs: ${DOCKER_CPUS}"
+    fi
+fi
+
+# Check 11: Port availability
+log_info "Checking port availability..."
+REQUIRED_PORTS=(3000 9090 3001 9100)
+PORT_CONFLICTS=0
+
+for port in "${REQUIRED_PORTS[@]}"; do
+    if lsof -Pi :$port -sTCP:LISTEN -t >/dev/null 2>&1 || netstat -tuln 2>/dev/null | grep -q ":$port "; then
+        log_warning "Port $port is in use (may conflict)"
+        PORT_CONFLICTS=$((PORT_CONFLICTS + 1))
+    else
+        log_success "Port $port is available"
+    fi
+done
+
+# Check 12: WSL detection (memory leak warning)
+log_info "Checking environment..."
+if grep -qi microsoft /proc/version 2>/dev/null; then
+    log_warning "WSL detected - Docker avoids WSL memory issues"
+    log_info "  Test will run in Docker to bypass WSL limitations"
+else
+    log_success "Native Linux/Unix environment detected"
+fi
+
+# Summary
+echo
+log_info "=============================================="
+log_info "Validation Summary"
+log_info "=============================================="
+echo -e "${GREEN}[✓]${NC} All critical checks passed"
+echo -e "${YELLOW}[!]${NC} Warnings: ${WARNINGS}"
+echo -e "${RED}[✗]${NC} Errors: ${ERRORS}"
+echo
+
+if [ "${ERRORS}" -eq 0 ]; then
+    log_success "System is ready for stability test"
+    log_info ""
+    log_info "To run the test:"
+    log_info "  cd ${PROJECT_ROOT}/config/docker"
+    log_info "  ${SCRIPT_DIR}/launch-stability-test.sh"
+    log_info ""
+    log_info "Monitor progress:"
+    log_info "  Grafana:    http://localhost:3001 (admin/stability-test)"
+    log_info "  Prometheus: http://localhost:9090"
+    echo
+    exit 0
+else
+    log_error "System validation failed with ${ERRORS} error(s)"
+    log_info "Fix errors above before running the test"
+    echo
+    exit 1
+fi

package/scripts/monitoring/quick-test-alerting.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# Quick validation test for alerting system (no jq dependency)
+# Phase 1 Sprint 1.1: Simple validation
+
+set -euo pipefail
+
+echo "=========================================="
+echo "ALERTING SYSTEM QUICK VALIDATION"
+echo "=========================================="
+echo ""
+
+# Check if files exist
+echo "1. Checking component files..."
+files=(
+  "lib/alerting.sh"
+  "scripts/monitoring/alert-monitor.sh"
+  "scripts/monitoring/view-alerts.sh"
+)
+
+for file in "${files[@]}"; do
+  if [ -f "$file" ]; then
+    echo "  ✅ $file"
+  else
+    echo "  ❌ $file (MISSING)"
+    exit 1
+  fi
+done
+
+echo ""
+echo "2. Checking executability..."
+for file in "${files[@]}"; do
+  if [ -x "$file" ]; then
+    echo "  ✅ $file"
+  else
+    echo "  ⚠️  $file (not executable, fixing...)"
+    chmod +x "$file"
+  fi
+done
+
+echo ""
+echo "3. Checking syntax..."
+for file in "${files[@]}"; do
+  if bash -n "$file" 2>/dev/null; then
+    echo "  ✅ $file"
+  else
+    echo "  ❌ $file (SYNTAX ERROR)"
+    exit 1
+  fi
+done
+
+echo ""
+echo "4. Checking alerting library functions..."
+# Source the library and verify key functions exist
+if source lib/alerting.sh 2>/dev/null; then
+  echo "  ✅ Library sourced successfully"
+
+  # Check if key functions are defined
+  funcs=(
+    "check_thresholds"
+    "emit_alert"
+    "get_alert_summary"
+    "clear_old_alerts"
+  )
+
+  for func in "${funcs[@]}"; do
+    if declare -f "$func" >/dev/null; then
+      echo "  ✅ Function: $func"
+    else
+      echo "  ❌ Function: $func (NOT FOUND)"
+      exit 1
+    fi
+  done
+else
+  echo "  ❌ Failed to source library"
+  exit 1
+fi
+
+echo ""
+echo "5. Checking threshold configuration..."
+# Verify environment variables are set with defaults
+vars=(
+  "ALERT_COORDINATION_TIME_MS"
+  "ALERT_DELIVERY_RATE_PCT"
+  "ALERT_MEMORY_GROWTH_PCT"
+  "ALERT_FD_GROWTH"
+)
+
+for var in "${vars[@]}"; do
+  if [ -n "${!var:-}" ]; then
+    echo "  ✅ $var=${!var}"
+  else
+    echo "  ❌ $var (NOT SET)"
+    exit 1
+  fi
+done
+
+echo ""
+echo "=========================================="
+echo "✅ VALIDATION PASSED"
+echo "=========================================="
+echo ""
+echo "DELIVERABLES CONFIRMED:"
+echo "  ✅ lib/alerting.sh - Alert threshold engine"
+echo "  ✅ scripts/monitoring/alert-monitor.sh - Monitoring daemon"
+echo "  ✅ scripts/monitoring/view-alerts.sh - Alert dashboard"
+echo "  ✅ Configurable thresholds"
+echo "  ✅ All functions defined"
+echo "  ✅ Syntax validation passed"
+echo ""
+echo "ACCEPTANCE CRITERIA STATUS:"
+echo "  ✅ Alerts trigger when thresholds exceeded - Implemented"
+echo "  ⚠️  False positive rate <1% - Requires jq for full testing"
+echo "  ⚠️  Alert latency <30 seconds - Requires jq for full testing"
+echo "  ✅ Configurable thresholds - Confirmed"
+echo ""
+echo "To run full integration tests, install jq:"
+echo "  sudo apt-get install jq"
+echo "  bash tests/integration/alerting-system.test.sh"