claude-flow-novice 2.15.5 → 2.15.7

package/claude-assets/skills/cfn-docker-logging/enable-logging.sh

@@ -0,0 +1,430 @@
+#!/bin/bash
+##
+## CFN Docker Logging - Quick Fix Script
+## Enables verbose Docker logging with container stdout/stderr capture
+##
+## Usage:
+##   ./enable-logging.sh [TASK_ID] [OPTIONS]
+##
+## Options:
+##   --log-dir DIR          Custom log directory (default: logs/docker-mode/{task-id})
+##   --verbose              Enable verbose output
+##   --capture-interval SEC Capture interval in seconds (default: 1)
+##   --help                 Show this help message
+##
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+log() {
+    echo -e "${BLUE}[$(date '+%H:%M:%S')]${NC} $*"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $*"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $*"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $*"
+}
+
+# Display usage
+usage() {
+    grep "^##" "$0" | sed 's/^## \?//'
+    exit 0
+}
+
+# Parse arguments
+TASK_ID=""
+LOG_DIR=""
+VERBOSE=false
+CAPTURE_INTERVAL=1
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --log-dir)
+            LOG_DIR="$2"
+            shift 2
+            ;;
+        --verbose)
+            VERBOSE=true
+            shift
+            ;;
+        --capture-interval)
+            CAPTURE_INTERVAL="$2"
+            shift 2
+            ;;
+        --help)
+            usage
+            ;;
+        *)
+            if [[ -z "$TASK_ID" ]]; then
+                TASK_ID="$1"
+            else
+                log_error "Unknown argument: $1"
+                usage
+            fi
+            shift
+            ;;
+    esac
+done
+
+# Validate task ID
+if [[ -z "$TASK_ID" ]]; then
+    log_error "Task ID is required"
+    usage
+fi
+
+# Set default log directory
+if [[ -z "$LOG_DIR" ]]; then
+    LOG_DIR="$PROJECT_ROOT/logs/docker-mode/$TASK_ID"
+fi
+
+# Create log directory structure
+mkdir -p "$LOG_DIR"
+mkdir -p "$LOG_DIR/containers"
+mkdir -p "$LOG_DIR/coordination"
+mkdir -p "$LOG_DIR/metrics"
+
+log "Docker Logging Enabled"
+log "Task ID: $TASK_ID"
+log "Log Directory: $LOG_DIR"
+
+# Create logging configuration file
+CONFIG_FILE="$LOG_DIR/logging-config.json"
+cat > "$CONFIG_FILE" <<EOF
+{
+  "task_id": "$TASK_ID",
+  "log_dir": "$LOG_DIR",
+  "enabled": true,
+  "capture_interval": $CAPTURE_INTERVAL,
+  "verbose": $VERBOSE,
+  "started_at": "$(date -Iseconds)",
+  "features": {
+    "container_logs": true,
+    "exit_codes": true,
+    "redis_events": true,
+    "timestamps": true
+  }
+}
+EOF
+
+log_success "Logging configuration created: $CONFIG_FILE"
+
+# Create helper scripts
+
+# 1. Container log capture script
+cat > "$LOG_DIR/capture-container-logs.sh" <<'CAPTURE_EOF'
+#!/bin/bash
+set -euo pipefail
+
+CONTAINER_ID="$1"
+AGENT_ID="$2"
+LOG_DIR="$3"
+
+# Create container log files
+STDOUT_LOG="$LOG_DIR/containers/${AGENT_ID}.stdout.log"
+STDERR_LOG="$LOG_DIR/containers/${AGENT_ID}.stderr.log"
+EXIT_LOG="$LOG_DIR/containers/${AGENT_ID}.exit.json"
+
+# Capture logs in background (with timestamps)
+docker logs -f --timestamps "$CONTAINER_ID" > "$STDOUT_LOG" 2> "$STDERR_LOG" &
+LOG_PID=$!
+
+# Wait for container to exit
+EXIT_CODE=$(docker wait "$CONTAINER_ID" 2>/dev/null || echo "999")
+
+# Stop log capture
+kill $LOG_PID 2>/dev/null || true
+sleep 1  # Allow final flush
+
+# Capture container metadata
+METADATA=$(docker inspect "$CONTAINER_ID" 2>/dev/null || echo '{}')
+
+# Log exit event
+cat > "$EXIT_LOG" <<EOF
+{
+  "timestamp": "$(date -Iseconds)",
+  "container_id": "$CONTAINER_ID",
+  "agent_id": "$AGENT_ID",
+  "exit_code": $EXIT_CODE,
+  "status": "$(echo "$METADATA" | jq -r '.[] | .State.Status // "unknown"')",
+  "started_at": "$(echo "$METADATA" | jq -r '.[] | .State.StartedAt // "unknown"')",
+  "finished_at": "$(echo "$METADATA" | jq -r '.[] | .State.FinishedAt // "unknown"')",
+  "oom_killed": $(echo "$METADATA" | jq -r '.[] | .State.OOMKilled // false')
+}
+EOF
+
+echo "$EXIT_CODE"
+CAPTURE_EOF
+
+chmod +x "$LOG_DIR/capture-container-logs.sh"
+log_success "Created container log capture script"
+
+# 2. Redis event logger
+cat > "$LOG_DIR/log-redis-event.sh" <<'REDIS_EOF'
+#!/bin/bash
+set -euo pipefail
+
+EVENT_TYPE="$1"
+PAYLOAD="$2"
+LOG_DIR="$3"
+
+REDIS_LOG="$LOG_DIR/coordination/redis-events.log"
+
+# Append event to log
+cat >> "$REDIS_LOG" <<EOF
+{"timestamp":"$(date -Iseconds)","event":"$EVENT_TYPE","payload":$PAYLOAD}
+EOF
+
+echo >> "$REDIS_LOG"  # Newline for readability
+REDIS_EOF
+
+chmod +x "$LOG_DIR/log-redis-event.sh"
+log_success "Created Redis event logger"
+
+# 3. Lifecycle event logger
+cat > "$LOG_DIR/log-lifecycle-event.sh" <<'LIFECYCLE_EOF'
+#!/bin/bash
+set -euo pipefail
+
+EVENT_TYPE="$1"
+shift
+PAYLOAD="$@"
+LOG_DIR="$(dirname "$0")"
+
+LIFECYCLE_LOG="$LOG_DIR/coordination/lifecycle-events.log"
+
+# Append event to log
+cat >> "$LIFECYCLE_LOG" <<EOF
+{"timestamp":"$(date -Iseconds)","event":"$EVENT_TYPE","data":$PAYLOAD}
+EOF
+
+echo >> "$LIFECYCLE_LOG"  # Newline
+LIFECYCLE_EOF
+
+chmod +x "$LOG_DIR/log-lifecycle-event.sh"
+log_success "Created lifecycle event logger"
+
+# 4. Query interface script
+cat > "$LOG_DIR/query-logs.sh" <<'QUERY_EOF'
+#!/bin/bash
+set -euo pipefail
+
+LOG_DIR="$(dirname "$0")"
+QUERY_TYPE="${1:-all}"
+FILTER="${2:-}"
+
+case "$QUERY_TYPE" in
+    containers)
+        cat "$LOG_DIR/containers"/*.stdout.log 2>/dev/null | grep -E "$FILTER" || echo "(no logs)"
+        ;;
+    errors)
+        cat "$LOG_DIR/containers"/*.stderr.log 2>/dev/null | grep -E "$FILTER" || echo "(no errors)"
+        ;;
+    exits)
+        cat "$LOG_DIR/containers"/*.exit.json 2>/dev/null | jq -s '.' || echo "[]"
+        ;;
+    redis)
+        cat "$LOG_DIR/coordination/redis-events.log" 2>/dev/null | grep -E "$FILTER" || echo "(no events)"
+        ;;
+    lifecycle)
+        cat "$LOG_DIR/coordination/lifecycle-events.log" 2>/dev/null | grep -E "$FILTER" || echo "(no events)"
+        ;;
+    failed)
+        # Show only failed containers (non-zero exit codes)
+        cat "$LOG_DIR/containers"/*.exit.json 2>/dev/null | jq -s '.[] | select(.exit_code != 0)' || echo "[]"
+        ;;
+    all)
+        echo "=== Container Logs ==="
+        cat "$LOG_DIR/containers"/*.stdout.log 2>/dev/null | head -50 || echo "(no logs)"
+        echo ""
+        echo "=== Exit Codes ==="
+        cat "$LOG_DIR/containers"/*.exit.json 2>/dev/null | jq -s '.[] | {agent_id, exit_code, status}' || echo "[]"
+        echo ""
+        echo "=== Redis Events ==="
+        cat "$LOG_DIR/coordination/redis-events.log" 2>/dev/null | head -20 || echo "(no events)"
+        ;;
+    *)
+        echo "Usage: $0 [containers|errors|exits|redis|lifecycle|failed|all] [filter]"
+        exit 1
+        ;;
+esac
+QUERY_EOF
+
+chmod +x "$LOG_DIR/query-logs.sh"
+log_success "Created query interface script"
+
+# 5. Export audit trail script
+cat > "$LOG_DIR/export-audit-trail.sh" <<'EXPORT_EOF'
+#!/bin/bash
+set -euo pipefail
+
+LOG_DIR="$(dirname "$0")"
+OUTPUT_FILE="${1:-audit-trail.json}"
+
+# Aggregate all logs into structured JSON
+{
+    echo '{'
+    echo '  "task_id": "'$(jq -r .task_id "$LOG_DIR/logging-config.json")'",';
+    echo '  "started_at": "'$(jq -r .started_at "$LOG_DIR/logging-config.json")'",';
+    echo '  "exported_at": "'$(date -Iseconds)'",';
+    echo '  "container_exits": '
+    cat "$LOG_DIR/containers"/*.exit.json 2>/dev/null | jq -s '.' || echo '[]'
+    echo '  ,'
+    echo '  "redis_events": '
+    cat "$LOG_DIR/coordination/redis-events.log" 2>/dev/null | jq -s '.' || echo '[]'
+    echo '  ,'
+    echo '  "lifecycle_events": '
+    cat "$LOG_DIR/coordination/lifecycle-events.log" 2>/dev/null | jq -s '.' || echo '[]'
+    echo '}'
+} | jq '.' > "$OUTPUT_FILE"
+
+echo "Audit trail exported to: $OUTPUT_FILE"
+EXPORT_EOF
+
+chmod +x "$LOG_DIR/export-audit-trail.sh"
+log_success "Created audit trail export script"
+
+# Create README for log directory
+cat > "$LOG_DIR/README.md" <<'README_EOF'
+# CFN Docker Mode Logs
+
+This directory contains logs for Docker mode CFN Loop execution.
+
+## Directory Structure
+
+```
+logs/docker-mode/{task-id}/
+├── logging-config.json           # Logging configuration
+├── containers/                   # Container logs
+│   ├── {agent-id}.stdout.log    # Container stdout
+│   ├── {agent-id}.stderr.log    # Container stderr
+│   └── {agent-id}.exit.json     # Container exit event
+├── coordination/                 # Coordination logs
+│   ├── redis-events.log         # Redis coordination events
+│   └── lifecycle-events.log     # Agent lifecycle events
+├── metrics/                      # Performance metrics (future)
+└── scripts/                      # Helper scripts
+    ├── capture-container-logs.sh
+    ├── log-redis-event.sh
+    ├── log-lifecycle-event.sh
+    ├── query-logs.sh
+    └── export-audit-trail.sh
+```
+
+## Usage Examples
+
+### Query Logs
+```bash
+# View all logs
+./query-logs.sh all
+
+# View only container logs
+./query-logs.sh containers
+
+# View only errors
+./query-logs.sh errors
+
+# View exit codes
+./query-logs.sh exits
+
+# View failed containers
+./query-logs.sh failed
+
+# View Redis events
+./query-logs.sh redis
+```
+
+### Export Audit Trail
+```bash
+# Export to default file
+./export-audit-trail.sh
+
+# Export to custom file
+./export-audit-trail.sh /tmp/audit-trail.json
+```
+
+### Capture Container Logs (Automatic)
+This script is called automatically by spawn-agent.sh when logging is enabled.
+```bash
+./capture-container-logs.sh <container-id> <agent-id> <log-dir>
+```
+
+## Integration with spawn-agent.sh
+
+To enable automatic log capture, add this to spawn-agent.sh:
+
+```bash
+# After container spawn
+if [[ -f "logs/docker-mode/${TASK_ID}/capture-container-logs.sh" ]]; then
+    logs/docker-mode/${TASK_ID}/capture-container-logs.sh \
+        "$CONTAINER_ID" "$AGENT_ID" "logs/docker-mode/${TASK_ID}" &
+fi
+```
+README_EOF
+
+log_success "Created README documentation"
+
+# Print usage instructions
+cat <<EOF
+
+${GREEN}✓ Docker Logging Enabled Successfully${NC}
+
+Log Directory: $LOG_DIR
+
+${YELLOW}Quick Reference:${NC}
+
+# View all logs
+$LOG_DIR/query-logs.sh all
+
+# View only errors
+$LOG_DIR/query-logs.sh errors
+
+# View failed containers
+$LOG_DIR/query-logs.sh failed
+
+# Export audit trail
+$LOG_DIR/export-audit-trail.sh /tmp/audit-trail.json
+
+${YELLOW}Integration with spawn-agent.sh:${NC}
+
+Add this to spawn-agent.sh after container spawn:
+
+  # Enable logging if configured
+  if [[ -f "$LOG_DIR/capture-container-logs.sh" ]]; then
+      $LOG_DIR/capture-container-logs.sh \\
+          "\$CONTAINER_ID" "\$AGENT_ID" "$LOG_DIR" &
+  fi
+
+${YELLOW}Files Created:${NC}
+  ✓ $CONFIG_FILE
+  ✓ $LOG_DIR/capture-container-logs.sh
+  ✓ $LOG_DIR/log-redis-event.sh
+  ✓ $LOG_DIR/log-lifecycle-event.sh
+  ✓ $LOG_DIR/query-logs.sh
+  ✓ $LOG_DIR/export-audit-trail.sh
+  ✓ $LOG_DIR/README.md
+
+EOF
+
+log_success "Logging infrastructure ready"
+
+if [[ "$VERBOSE" == true ]]; then
+    log "Verbose mode enabled"
+    log "Logging configuration:"
+    cat "$CONFIG_FILE" | jq '.'
+fi

package/claude-assets/skills/cfn-docker-logging/init-hybrid-logging.sh

@@ -0,0 +1,210 @@
+#!/bin/bash
+# Initialize hybrid logging: Text files + SQLite database
+# Provides both human-readable logs and powerful SQL querying
+
+set -euo pipefail
+
+TASK_ID=${1:-}
+
+if [[ -z "$TASK_ID" ]]; then
+    echo "Usage: $0 <task_id>"
+    echo ""
+    echo "Initializes hybrid logging for Docker mode CFN Loop execution:"
+    echo "  - Text files: Human-readable logs (backward compatible)"
+    echo "  - SQLite: Powerful querying and analytics"
+    exit 1
+fi
+
+# Get script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Create log directory structure
+LOG_DIR="logs/docker-mode/${TASK_ID}"
+mkdir -p "$LOG_DIR"
+mkdir -p "$LOG_DIR/queries"
+
+# Database path
+DB_PATH="$LOG_DIR/logs.db"
+
+# Source helpers
+source "$SCRIPT_DIR/sqlite-helpers.sh"
+
+# Initialize database
+echo "Initializing logging database: $DB_PATH"
+init_logging_db "$DB_PATH"
+
+# Copy query scripts to log directory for easy access
+echo "Installing query scripts to $LOG_DIR/queries/"
+cp "$SCRIPT_DIR/queries/"*.sh "$LOG_DIR/queries/"
+chmod +x "$LOG_DIR/queries/"*.sh
+
+# Create README in log directory
+cat > "$LOG_DIR/README.md" <<'EOF'
+# CFN Docker Logging - Hybrid System
+
+This directory contains both **text files** (human-readable) and **SQLite database** (powerful queries).
+
+## Quick Start
+
+### Text File Queries (Human Readable)
+```bash
+# View agent stdout
+cat <agent-id>-stdout.log
+
+# View agent stderr
+cat <agent-id>-stderr.log
+
+# View combined logs
+cat <agent-id>-combined.log
+
+# Search for errors
+grep -i error *-combined.log
+
+# View summary
+cat <agent-id>-summary.txt
+```
+
+### SQLite Queries (Powerful Analysis)
+```bash
+# All queries use: queries/<script-name>.sh logs.db [task_id]
+
+# Find failed containers
+./queries/query-failed-containers.sh logs.db TASK_ID
+
+# View agent timeline
+./queries/query-agent-timeline.sh logs.db AGENT_ID
+
+# Gate check results
+./queries/query-gate-checks.sh logs.db TASK_ID
+
+# Validator consensus history
+./queries/query-consensus-history.sh logs.db TASK_ID
+
+# Full analytics summary
+./queries/analytics-summary.sh logs.db TASK_ID
+
+# Coordination timeline
+./queries/query-coordination-timeline.sh logs.db TASK_ID
+```
+
+## Database Schema
+
+### Tables
+- `container_logs`: Individual log lines (stdout/stderr)
+- `container_events`: Lifecycle (spawn, exit, OOM)
+- `coordination_events`: Redis coordination activity
+- `gate_checks`: Loop 3 gate results
+- `validator_consensus`: Loop 2 validator scores
+- `product_owner_decisions`: Final decisions
+- `performance_metrics`: Custom metrics
+
+### Custom Queries
+```bash
+# Direct SQLite access
+sqlite3 logs.db
+
+# Example: Find slowest agents
+sqlite3 logs.db "SELECT agent_id, duration_seconds FROM container_events WHERE event_type='exit' ORDER BY duration_seconds DESC LIMIT 10;"
+
+# Example: Count errors in logs
+sqlite3 logs.db "SELECT COUNT(*) FROM container_logs WHERE log_line LIKE '%error%' COLLATE NOCASE;"
+
+# Example: Agent success rate
+sqlite3 logs.db "SELECT agent_id, AVG(CASE WHEN exit_code=0 THEN 1.0 ELSE 0.0 END) as success_rate FROM container_events WHERE event_type='exit' GROUP BY agent_id;"
+```
+
+## File Structure
+```
+logs/docker-mode/TASK_ID/
+├── logs.db                          # SQLite database
+├── README.md                        # This file
+├── queries/                         # Query scripts
+│   ├── query-failed-containers.sh
+│   ├── query-agent-timeline.sh
+│   ├── query-gate-checks.sh
+│   ├── query-consensus-history.sh
+│   ├── analytics-summary.sh
+│   └── query-coordination-timeline.sh
+├── <agent-id>-stdout.log           # Agent stdout (text)
+├── <agent-id>-stderr.log           # Agent stderr (text)
+├── <agent-id>-combined.log         # Combined output (text)
+└── <agent-id>-summary.txt          # Execution summary (text)
+```
+
+## Benefits
+
+### Text Files
+- ✅ Human-readable
+- ✅ Standard Unix tools (grep, less, tail)
+- ✅ Backward compatible
+- ✅ Easy debugging
+
+### SQLite
+- ✅ Complex queries (joins, aggregations)
+- ✅ Time-series analysis
+- ✅ Performance metrics
+- ✅ Cross-agent correlation
+- ✅ Scalable (handles millions of rows)
+
+## Examples
+
+### Scenario 1: Find why agent failed
+```bash
+# 1. Check exit code
+./queries/query-failed-containers.sh logs.db TASK_ID
+
+# 2. View full timeline
+./queries/query-agent-timeline.sh logs.db AGENT_ID
+
+# 3. Read stderr
+cat AGENT_ID-stderr.log
+```
+
+### Scenario 2: Analyze gate check trends
+```bash
+# View all gate checks
+./queries/query-gate-checks.sh logs.db TASK_ID
+
+# Get pass rate statistics
+sqlite3 logs.db "SELECT AVG(pass_rate), MIN(pass_rate), MAX(pass_rate) FROM gate_checks WHERE task_id='TASK_ID';"
+```
+
+### Scenario 3: Validator consensus analysis
+```bash
+# View consensus history
+./queries/query-consensus-history.sh logs.db TASK_ID
+
+# Find disagreements (high score variance)
+sqlite3 logs.db "SELECT iteration, MAX(score) - MIN(score) as variance FROM validator_consensus WHERE task_id='TASK_ID' GROUP BY iteration HAVING variance > 0.3;"
+```
+
+### Scenario 4: Performance optimization
+```bash
+# Full analytics
+./queries/analytics-summary.sh logs.db TASK_ID
+
+# Find bottleneck agents
+sqlite3 logs.db "SELECT agent_id, duration_seconds FROM container_events WHERE event_type='exit' AND task_id='TASK_ID' ORDER BY duration_seconds DESC LIMIT 5;"
+```
+EOF
+
+echo ""
+echo "=== Hybrid Logging Initialized ==="
+echo "Task ID: $TASK_ID"
+echo "Log Directory: $LOG_DIR"
+echo "SQLite Database: $DB_PATH"
+echo ""
+echo "Text Files:"
+echo "  - <agent-id>-stdout.log"
+echo "  - <agent-id>-stderr.log"
+echo "  - <agent-id>-combined.log"
+echo "  - <agent-id>-summary.txt"
+echo ""
+echo "SQLite Queries:"
+echo "  - cd $LOG_DIR && ./queries/query-failed-containers.sh logs.db $TASK_ID"
+echo "  - cd $LOG_DIR && ./queries/query-agent-timeline.sh logs.db <agent-id>"
+echo "  - cd $LOG_DIR && ./queries/query-gate-checks.sh logs.db $TASK_ID"
+echo "  - cd $LOG_DIR && ./queries/query-consensus-history.sh logs.db $TASK_ID"
+echo "  - cd $LOG_DIR && ./queries/analytics-summary.sh logs.db $TASK_ID"
+echo ""
+echo "See: $LOG_DIR/README.md for complete guide"