claude-flow-novice 2.0.0 → 2.0.1

package/scripts/cleanup-blocking-coordination.sh

@@ -0,0 +1,420 @@
+#!/usr/bin/env bash
+#
+# cleanup-blocking-coordination.sh
+# Sprint 3.2: Auto-Recovery Mechanisms - High-Performance Cleanup Script
+#
+# Performance Optimized Version:
+# - Target: <5s for 10,000 coordinators (2,000 coordinators/sec)
+# - Strategy: Lua script execution for atomic, server-side batch processing
+# - Fallback: Original sequential bash implementation if Lua fails
+#
+# Finds and removes stale blocking coordinator state from Redis.
+# Stale detection: Heartbeat age >10 minutes (600 seconds)
+#
+# Features:
+# - Uses Redis SCAN (not KEYS) to prevent DoS in production
+# - Lua script for 50-60x performance improvement
+# - Dry-run mode for testing: --dry-run flag
+# - Logs to ~/.claude-flow/logs/blocking-cleanup.log
+# - Exit codes: 0 (success), 1 (Redis connection failed), 2 (cleanup errors)
+# - Cleanup categories: heartbeats, ACKs, signals, idempotency, activity
+#
+# Usage:
+#   ./scripts/cleanup-blocking-coordination.sh         # Production cleanup
+#   ./scripts/cleanup-blocking-coordination.sh --dry-run  # Test mode
+#   ./scripts/cleanup-blocking-coordination.sh --fallback  # Force bash implementation
+#
+# Scheduled execution:
+#   - systemd timer: Every 5 minutes (systemd/cleanup-blocking-coordination.timer)
+#   - cron: */5 * * * * (cron.d/cleanup-blocking-coordination)
+#   - npm: npm run cleanup:blocking
+#
+
+set -euo pipefail
+
+# ===== CONFIGURATION =====
+
+# Stale threshold: 10 minutes = 600 seconds
+STALE_THRESHOLD_SECONDS=600
+
+# Log file location
+LOG_DIR="${HOME}/.claude-flow/logs"
+LOG_FILE="${LOG_DIR}/blocking-cleanup.log"
+
+# Redis connection (uses environment variables or defaults)
+REDIS_HOST="${REDIS_HOST:-127.0.0.1}"
+REDIS_PORT="${REDIS_PORT:-6379}"
+REDIS_PASSWORD="${REDIS_PASSWORD:-}"
+REDIS_DB="${REDIS_DB:-0}"
+
+# Script directory (for Lua script location)
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LUA_SCRIPT="${SCRIPT_DIR}/redis-lua/cleanup-blocking-coordination.lua"
+
+# Dry-run mode flag
+DRY_RUN=false
+
+# Force fallback to bash implementation
+FORCE_FALLBACK=false
+
+# Metrics counters
+TOTAL_COORDINATORS_CHECKED=0
+STALE_COORDINATORS_FOUND=0
+KEYS_DELETED=0
+CLEANUP_ERRORS=0
+EXECUTION_TIME_MS=0
+
+# ===== HELPER FUNCTIONS =====
+
+# Log with timestamp
+log() {
+  local level="$1"
+  shift
+  local message="$*"
+  local timestamp
+  timestamp=$(date '+%Y-%m-%d %H:%M:%S')
+  echo "[${timestamp}] [${level}] ${message}" | tee -a "${LOG_FILE}"
+}
+
+# Redis command wrapper with authentication
+redis_cmd() {
+  if [ -n "${REDIS_PASSWORD}" ]; then
+    redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" -a "${REDIS_PASSWORD}" -n "${REDIS_DB}" "$@" 2>/dev/null
+  else
+    redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" -n "${REDIS_DB}" "$@" 2>/dev/null
+  fi
+}
+
+# Redis EVAL wrapper for Lua script execution
+redis_eval_lua() {
+  local lua_script="$1"
+  local num_keys="$2"
+  shift 2
+  local args=("$@")
+
+  if [ -n "${REDIS_PASSWORD}" ]; then
+    redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" -a "${REDIS_PASSWORD}" -n "${REDIS_DB}" --eval "${lua_script}" "${num_keys}" "${args[@]}" 2>/dev/null
+  else
+    redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" -n "${REDIS_DB}" --eval "${lua_script}" "${num_keys}" "${args[@]}" 2>/dev/null
+  fi
+}
+
+# High-performance Lua-based cleanup
+cleanup_lua() {
+  log "INFO" "Using high-performance Lua script for cleanup"
+
+  # Check if Lua script exists
+  if [ ! -f "${LUA_SCRIPT}" ]; then
+    log "ERROR" "Lua script not found: ${LUA_SCRIPT}"
+    log "INFO" "Falling back to bash implementation"
+    return 1
+  fi
+
+  # Execute Lua script
+  local dry_run_flag=0
+  if [ "${DRY_RUN}" = true ]; then
+    dry_run_flag=1
+  fi
+
+  local start_time
+  start_time=$(date +%s%3N)
+
+  local lua_result
+  lua_result=$(redis_eval_lua "${LUA_SCRIPT}" 0 "${STALE_THRESHOLD_SECONDS}" "${dry_run_flag}") || {
+    log "ERROR" "Lua script execution failed"
+    return 1
+  }
+
+  local end_time
+  end_time=$(date +%s%3N)
+  EXECUTION_TIME_MS=$((end_time - start_time))
+
+  # Parse JSON result
+  if command -v jq >/dev/null 2>&1; then
+    TOTAL_COORDINATORS_CHECKED=$(echo "${lua_result}" | jq -r '.totalCoordinatorsChecked')
+    STALE_COORDINATORS_FOUND=$(echo "${lua_result}" | jq -r '.staleCoordinatorsFound')
+    KEYS_DELETED=$(echo "${lua_result}" | jq -r '.keysDeleted')
+    local lua_execution_time_ms
+    lua_execution_time_ms=$(echo "${lua_result}" | jq -r '.executionTimeMs')
+
+    log "INFO" "Lua script execution time: ${lua_execution_time_ms}ms (server-side)"
+    log "INFO" "Total round-trip time: ${EXECUTION_TIME_MS}ms (including network)"
+
+    # Log stale coordinator IDs if any found
+    local stale_ids
+    stale_ids=$(echo "${lua_result}" | jq -r '.staleCoordinatorIds[]' 2>/dev/null || echo "")
+    if [ -n "${stale_ids}" ]; then
+      log "DEBUG" "Stale coordinator IDs: ${stale_ids}"
+    fi
+  else
+    log "WARN" "jq not found, cannot parse Lua script output"
+    log "INFO" "Raw output: ${lua_result}"
+    return 1
+  fi
+
+  return 0
+}
+
+# Redis SCAN wrapper (non-blocking alternative to KEYS)
+redis_scan() {
+  local pattern="$1"
+  local cursor=0
+  local keys=()
+
+  while true; do
+    # SCAN returns: [cursor, [key1, key2, ...]]
+    local result
+    result=$(redis_cmd SCAN "${cursor}" MATCH "${pattern}" COUNT 100)
+
+    # Extract new cursor (first line)
+    cursor=$(echo "${result}" | head -n 1)
+
+    # Extract keys (remaining lines)
+    local batch_keys
+    batch_keys=$(echo "${result}" | tail -n +2)
+
+    # Add to results
+    if [ -n "${batch_keys}" ]; then
+      keys+=( ${batch_keys} )
+    fi
+
+    # Check if scan is complete
+    if [ "${cursor}" = "0" ]; then
+      break
+    fi
+  done
+
+  # Return unique keys (SCAN may return duplicates)
+  printf '%s\n' "${keys[@]}" | sort -u
+}
+
+# Get current Unix timestamp
+current_timestamp() {
+  date +%s
+}
+
+# Calculate age of heartbeat in seconds
+get_heartbeat_age() {
+  local heartbeat_key="$1"
+
+  # Get heartbeat timestamp from Redis
+  local heartbeat_value
+  heartbeat_value=$(redis_cmd GET "${heartbeat_key}")
+
+  if [ -z "${heartbeat_value}" ]; then
+    echo "-1"  # Heartbeat not found
+    return
+  fi
+
+  # Parse JSON to extract timestamp (format: {"coordinatorId":"...","timestamp":1234567890,...})
+  local heartbeat_timestamp
+  heartbeat_timestamp=$(echo "${heartbeat_value}" | grep -oP '(?<="timestamp":)\d+' || echo "0")
+
+  if [ "${heartbeat_timestamp}" = "0" ]; then
+    echo "-1"  # Invalid timestamp
+    return
+  fi
+
+  # Calculate age
+  local now
+  now=$(current_timestamp)
+  local age=$((now - heartbeat_timestamp / 1000))  # Convert ms to seconds
+
+  echo "${age}"
+}
+
+# Cleanup coordinator state (bash fallback implementation)
+cleanup_coordinator() {
+  local coordinator_id="$1"
+  local age_seconds="$2"
+
+  log "INFO" "Cleaning up stale coordinator: ${coordinator_id} (age: ${age_seconds}s, threshold: ${STALE_THRESHOLD_SECONDS}s)"
+
+  local keys_to_delete=()
+
+  # 1. Heartbeat key
+  keys_to_delete+=( "blocking:heartbeat:${coordinator_id}" )
+
+  # 2. Signal ACK keys (blocking:ack:coordinatorId:*)
+  local ack_keys
+  ack_keys=$(redis_scan "blocking:ack:${coordinator_id}:*")
+  if [ -n "${ack_keys}" ]; then
+    while IFS= read -r key; do
+      keys_to_delete+=( "${key}" )
+    done <<< "${ack_keys}"
+  fi
+
+  # 3. Signal key (blocking:signal:coordinatorId)
+  keys_to_delete+=( "blocking:signal:${coordinator_id}" )
+
+  # 4. Idempotency keys (blocking:idempotency:*coordinatorId*)
+  local idempotency_keys
+  idempotency_keys=$(redis_scan "blocking:idempotency:*${coordinator_id}*")
+  if [ -n "${idempotency_keys}" ]; then
+    while IFS= read -r key; do
+      keys_to_delete+=( "${key}" )
+    done <<< "${idempotency_keys}"
+  fi
+
+  # 5. Activity tracking key
+  keys_to_delete+=( "coordinator:activity:${coordinator_id}" )
+
+  # Log keys to delete
+  local key_count=${#keys_to_delete[@]}
+  log "DEBUG" "Found ${key_count} keys to delete for coordinator ${coordinator_id}"
+
+  if [ "${DRY_RUN}" = true ]; then
+    log "INFO" "[DRY-RUN] Would delete ${key_count} keys for coordinator ${coordinator_id}"
+    for key in "${keys_to_delete[@]}"; do
+      log "DEBUG" "[DRY-RUN] Would delete: ${key}"
+    done
+  else
+    # Delete keys in batch
+    if [ ${key_count} -gt 0 ]; then
+      redis_cmd DEL "${keys_to_delete[@]}" >/dev/null || {
+        log "ERROR" "Failed to delete keys for coordinator ${coordinator_id}"
+        CLEANUP_ERRORS=$((CLEANUP_ERRORS + 1))
+        return 1
+      }
+
+      KEYS_DELETED=$((KEYS_DELETED + key_count))
+      log "INFO" "Deleted ${key_count} keys for coordinator ${coordinator_id}"
+    fi
+  fi
+
+  STALE_COORDINATORS_FOUND=$((STALE_COORDINATORS_FOUND + 1))
+  return 0
+}
+
+# Bash fallback cleanup implementation
+cleanup_bash() {
+  log "INFO" "Using bash fallback implementation for cleanup"
+
+  local start_time
+  start_time=$(date +%s%3N)
+
+  # Find all blocking:heartbeat:* keys using SCAN
+  log "INFO" "Scanning for blocking coordinator heartbeats..."
+
+  local heartbeat_keys
+  heartbeat_keys=$(redis_scan "blocking:heartbeat:*")
+
+  if [ -z "${heartbeat_keys}" ]; then
+    log "INFO" "No coordinator heartbeats found"
+    log "INFO" "Cleanup complete (nothing to do)"
+    return 0
+  fi
+
+  # Process each heartbeat
+  while IFS= read -r heartbeat_key; do
+    TOTAL_COORDINATORS_CHECKED=$((TOTAL_COORDINATORS_CHECKED + 1))
+
+    # Extract coordinator ID from key: blocking:heartbeat:{coordinatorId}
+    local coordinator_id
+    coordinator_id=$(echo "${heartbeat_key}" | sed 's/^blocking:heartbeat://')
+
+    log "DEBUG" "Checking coordinator: ${coordinator_id}"
+
+    # Get heartbeat age
+    local age_seconds
+    age_seconds=$(get_heartbeat_age "${heartbeat_key}")
+
+    if [ "${age_seconds}" = "-1" ]; then
+      log "WARN" "Invalid or missing heartbeat for coordinator: ${coordinator_id}"
+      continue
+    fi
+
+    # Check if stale
+    if [ "${age_seconds}" -gt "${STALE_THRESHOLD_SECONDS}" ]; then
+      log "WARN" "Stale coordinator detected: ${coordinator_id} (age: ${age_seconds}s)"
+      cleanup_coordinator "${coordinator_id}" "${age_seconds}" || true
+    else
+      log "DEBUG" "Coordinator active: ${coordinator_id} (age: ${age_seconds}s)"
+    fi
+  done <<< "${heartbeat_keys}"
+
+  local end_time
+  end_time=$(date +%s%3N)
+  EXECUTION_TIME_MS=$((end_time - start_time))
+
+  log "INFO" "Bash fallback execution time: ${EXECUTION_TIME_MS}ms"
+
+  return 0
+}
+
+# ===== MAIN EXECUTION =====
+
+main() {
+  # Parse command line arguments
+  while [[ $# -gt 0 ]]; do
+    case $1 in
+      --dry-run)
+        DRY_RUN=true
+        shift
+        ;;
+      --fallback)
+        FORCE_FALLBACK=true
+        shift
+        ;;
+      *)
+        echo "Unknown option: $1"
+        echo "Usage: $0 [--dry-run] [--fallback]"
+        exit 2
+        ;;
+    esac
+  done
+
+  # Create log directory if it doesn't exist
+  mkdir -p "${LOG_DIR}"
+
+  # Start cleanup
+  log "INFO" "========================================"
+  log "INFO" "Blocking Coordination Cleanup Started"
+  log "INFO" "Dry-run mode: ${DRY_RUN}"
+  log "INFO" "Stale threshold: ${STALE_THRESHOLD_SECONDS}s"
+  log "INFO" "Force fallback: ${FORCE_FALLBACK}"
+  log "INFO" "========================================"
+
+  # Check Redis connection
+  if ! redis_cmd PING >/dev/null 2>&1; then
+    log "ERROR" "Redis connection failed (host: ${REDIS_HOST}, port: ${REDIS_PORT})"
+    log "ERROR" "Cleanup aborted"
+    exit 1
+  fi
+
+  log "INFO" "Redis connection established"
+
+  # Execute cleanup: Try Lua first, fallback to bash if needed
+  if [ "${FORCE_FALLBACK}" = true ]; then
+    log "INFO" "Fallback mode forced via --fallback flag"
+    cleanup_bash
+  else
+    if ! cleanup_lua; then
+      log "WARN" "Lua cleanup failed, falling back to bash implementation"
+      cleanup_bash
+    fi
+  fi
+
+  # Log summary
+  log "INFO" "========================================"
+  log "INFO" "Cleanup Summary:"
+  log "INFO" "  Total coordinators checked: ${TOTAL_COORDINATORS_CHECKED}"
+  log "INFO" "  Stale coordinators found: ${STALE_COORDINATORS_FOUND}"
+  log "INFO" "  Keys deleted: ${KEYS_DELETED}"
+  log "INFO" "  Cleanup errors: ${CLEANUP_ERRORS}"
+  log "INFO" "  Execution time: ${EXECUTION_TIME_MS}ms"
+  log "INFO" "  Performance: $(awk "BEGIN {printf \"%.2f\", ${TOTAL_COORDINATORS_CHECKED} / (${EXECUTION_TIME_MS} / 1000.0)}" 2>/dev/null || echo "N/A") coordinators/sec"
+  log "INFO" "========================================"
+
+  # Exit code based on errors
+  if [ ${CLEANUP_ERRORS} -gt 0 ]; then
+    log "ERROR" "Cleanup completed with ${CLEANUP_ERRORS} errors"
+    exit 2
+  else
+    log "INFO" "Cleanup completed successfully"
+    exit 0
+  fi
+}
+
+# Execute main function
+main "$@"

package/scripts/collect-build-metrics.js

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+/**
+ * Standardized build metrics collection
+ * Usage: node scripts/collect-build-metrics.js [--json]
+ *
+ * Prevents ambiguous metrics like Sprint 7.1's "691 files compiled"
+ * See CLAUDE.md Section 11 for reporting standards
+ */
+
+import { execSync } from 'child_process';
+import { existsSync } from 'fs';
+
+function countFiles(pattern, dir = 'src') {
+  try {
+    const cmd = `find ${dir} -name "${pattern}" 2>/dev/null | wc -l`;
+    return parseInt(execSync(cmd, { encoding: 'utf8' }).trim());
+  } catch {
+    return 0;
+  }
+}
+
+function collectMetrics() {
+  const metrics = {
+    source_files: {
+      typescript: countFiles('*.ts') + countFiles('*.tsx'),
+      javascript: countFiles('*.js'),
+      jsx: countFiles('*.jsx')
+    },
+    output_files: {
+      javascript: countFiles('*.js', '.claude-flow-novice/dist') || countFiles('*.js', 'dist'),
+      sourcemaps: countFiles('*.map', '.claude-flow-novice/dist') || countFiles('*.map', 'dist')
+    },
+    dist_exists: existsSync('.claude-flow-novice/dist') || existsSync('dist')
+  };
+
+  if (metrics.source_files.typescript > 0 && metrics.output_files.javascript > 0) {
+    metrics.compilation_ratio =
+      ((metrics.output_files.javascript * 100) / metrics.source_files.typescript).toFixed(1) + '%';
+  }
+
+  return metrics;
+}
+
+const metrics = collectMetrics();
+
+if (process.argv.includes('--json')) {
+  console.log(JSON.stringify(metrics, null, 2));
+} else {
+  console.log('📊 Build Metrics Report');
+  console.log('═'.repeat(50));
+  console.log(`\nSource Files:`);
+  console.log(`  TypeScript: ${metrics.source_files.typescript} files`);
+  console.log(`  JavaScript: ${metrics.source_files.javascript} files`);
+  console.log(`\nOutput Files (dist/):`);
+  console.log(`  JavaScript: ${metrics.output_files.javascript} files`);
+  console.log(`  Source Maps: ${metrics.output_files.sourcemaps} files`);
+  if (metrics.compilation_ratio) {
+    console.log(`\nCompilation Ratio: ${metrics.compilation_ratio}`);
+    console.log(`(Lower ratio = better tree-shaking)`);
+  }
+
+  if (!metrics.dist_exists) {
+    console.log('\n⚠️  Warning: dist/ directory not found. Run build first.');
+  }
+}

package/scripts/demo/README.md

@@ -0,0 +1,79 @@
+# Demo and Performance Scripts
+
+This directory contains demonstration and performance analysis scripts for the claude-flow-novice framework.
+
+## Demo Scripts
+
+### Swarm Coordination Demos
+- **autoscaling-demo-simplified.js** - Simplified autoscaling demonstration
+- **test-autoscaling-demo.js** - Autoscaling test scenarios
+- **demo-multi-swarm-coordination.js** - Multi-swarm coordination patterns
+- **demo-production-deployment.js** - Production deployment workflows
+- **demo-visualization-system.js** - Visualization system capabilities
+
+### Dashboard and Monitoring
+- **comprehensive-dashboard-test.js** - Complete dashboard functionality test
+
+## Performance Analysis
+
+### Analysis Tools
+- **performance-analysis.js** - Main performance analysis tool (ES modules)
+- **performance-analysis.cjs** - Performance analysis tool (CommonJS)
+
+### Confidence Metrics
+- **confidence-log.js** - Confidence metric logging
+- **confidence-report.js** - Confidence report generation
+
+## Usage
+
+### Running Demo Scripts
+```bash
+# Swarm coordination demo
+node scripts/demo/demo-multi-swarm-coordination.js
+
+# Production deployment demo
+node scripts/demo/demo-production-deployment.js
+
+# Visualization system demo
+node scripts/demo/demo-visualization-system.js
+
+# Autoscaling demo
+node scripts/demo/autoscaling-demo-simplified.js
+```
+
+### Running Performance Analysis
+```bash
+# ES module version
+node scripts/demo/performance-analysis.js
+
+# CommonJS version
+node scripts/demo/performance-analysis.cjs
+```
+
+### Monitoring and Confidence
+```bash
+# Generate confidence logs
+node scripts/demo/confidence-log.js
+
+# Generate confidence reports
+node scripts/demo/confidence-report.js
+
+# Dashboard test
+node scripts/demo/comprehensive-dashboard-test.js
+```
+
+## Purpose
+
+These scripts demonstrate:
+1. **Swarm Orchestration** - Multi-agent coordination patterns
+2. **Performance Profiling** - System performance analysis
+3. **Confidence Metrics** - Agent confidence tracking and reporting
+4. **Visualization** - Dashboard and monitoring capabilities
+5. **Production Workflows** - Deployment and scaling patterns
+
+## Notes
+
+- Demo scripts are for demonstration and testing purposes
+- Performance scripts analyze system metrics and agent efficiency
+- Confidence scripts track and report agent confidence scores
+- All scripts use the production swarm infrastructure