claude-flow-novice 2.18.24 → 2.18.25

package/config/logrotate.d/cfn-logs

@@ -0,0 +1,221 @@
+# CFN Distributed Logging - Logrotate Configuration
+# Task 4.4: Distributed Logging Standardization
+#
+# Manages rotation, compression, and retention of all CFN logs
+# Usage: logrotate /etc/logrotate.d/cfn-logs
+# Test:  logrotate -d /etc/logrotate.d/cfn-logs
+
+# Main application logs - Standard 30-day retention
+/var/log/cfn/containers/*.log {
+    rotate 10
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    sharedscripts
+    postrotate
+        # Signal containers on log rotation
+        if [ -f /var/run/cfn/container-monitor.pid ]; then
+            kill -HUP $(cat /var/run/cfn/container-monitor.pid) 2>/dev/null || true
+        fi
+        # Update log inventory
+        find /var/log/cfn/containers -name "*.log*" -mtime +30 -delete 2>/dev/null || true
+    endscript
+    maxage 30
+}
+
+# Aggregated logs directory
+/var/log/cfn/aggregated/*.log {
+    rotate 10
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    sharedscripts
+    postrotate
+        # Trigger aggregation job after rotation
+        if [ -x /usr/local/bin/log-aggregator.sh ]; then
+            /usr/local/bin/log-aggregator.sh --rotate >/dev/null 2>&1 || true
+        fi
+    endscript
+    maxage 30
+}
+
+# Debug logs - 7-day retention (shorter lifetime)
+/var/log/cfn/debug/*.log {
+    rotate 5
+    size 50M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    sharedscripts
+    maxage 7
+}
+
+# Performance metrics logs
+/var/log/cfn/metrics/*.log {
+    rotate 10
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    sharedscripts
+    postrotate
+        # Export metrics before deletion
+        if [ -x /usr/local/bin/export-metrics.sh ]; then
+            /usr/local/bin/export-metrics.sh --archive >/dev/null 2>&1 || true
+        fi
+    endscript
+    maxage 30
+}
+
+# Coordinator logs - Detailed 30-day retention
+/var/log/cfn/coordinator/*.log {
+    rotate 10
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    sharedscripts
+    maxage 30
+}
+
+# Agent execution logs
+/var/log/cfn/agents/*.log {
+    rotate 10
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    dateext
+    dateformat -%Y%m%d-%H%M%S
+    sharedscripts
+    maxage 30
+}
+
+# JSON structured logs (for easier parsing)
+/var/log/cfn/json/*.log {
+    rotate 15
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    sharedscripts
+    postrotate
+        # Validate JSON integrity before archiving
+        if [ -x /usr/local/bin/validate-json-logs.sh ]; then
+            /usr/local/bin/validate-json-logs.sh --archive >/dev/null 2>&1 || true
+        fi
+    endscript
+    maxage 30
+}
+
+# Error and warning logs (high priority)
+/var/log/cfn/errors/*.log {
+    rotate 20
+    size 50M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    sharedscripts
+    postrotate
+        # Alert on errors before rotation
+        if [ -x /usr/local/bin/alert-errors.sh ]; then
+            /usr/local/bin/alert-errors.sh --pre-rotate >/dev/null 2>&1 || true
+        fi
+    endscript
+    maxage 60
+}
+
+# Audit logs - Extended retention for compliance
+/var/log/cfn/audit/*.log {
+    rotate 30
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0600 root root
+    sharedscripts
+    postrotate
+        # Archive to secure storage
+        if [ -x /usr/local/bin/archive-audit.sh ]; then
+            /usr/local/bin/archive-audit.sh >/dev/null 2>&1 || true
+        fi
+    endscript
+    maxage 90
+}
+
+# Catchall for any other cfn logs
+/var/log/cfn/*.log {
+    rotate 10
+    size 100M
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    maxage 30
+}
+
+# Global settings that apply to all blocks
+# (Can be overridden by individual blocks above)
+default {
+    # Compression algorithm and level
+    compress
+    compresscmd /bin/gzip
+    compressext .gz
+    compressoptions -9
+
+    # Default permissions and ownership
+    create 0644 root root
+
+    # Rotation strategy
+    rotate 10
+    size 100M
+
+    # Behavior flags
+    missingok         # Don't error if log file missing
+    notifempty        # Don't rotate empty files
+    delaycompress     # Compress on next rotation (delay by 1 rotation)
+    sharedscripts     # Run scripts only once per rotation
+
+    # Retention
+    maxage 30         # Delete rotated logs older than 30 days
+
+    # Copy then truncate instead of move and create
+    copytruncate
+
+    # Tab character at line continuation
+    tabooext +.log
+}
+
+# Special handling for very large log files
+/var/log/cfn/large/*.log {
+    rotate 5
+    size 1G
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0644 root root
+    maxage 30
+    dateext
+    dateformat -%Y%m%d-%H%M%S
+}

package/config/loki/loki-config.yml

@@ -0,0 +1,172 @@
+# Loki Configuration
+# Task P2-2.3: Centralized Logging with ELK/Loki Stack
+# Version: 2.9.0
+#
+# Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system
+# inspired by Prometheus. It is designed to be very cost effective and easy to operate.
+
+auth_enabled: false
+
+ingester:
+  # Chunk-related settings for ingestion
+  chunk_idle_period: 3m
+  chunk_retain_period: 1m
+  max_chunk_age: 1h
+  chunk_encoding: snappy
+
+  # Performance tuning
+  max_streams_errors: 100
+  max_streams: 1000
+  max_stream_delay: 5s
+
+  # Flush settings
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    num_tokens: 128
+    heartbeat_timeout: 5m
+    interface_names:
+      - eth0
+
+limits_config:
+  # Enforce rate limiting per stream
+  rate_limit_bytes: 100MB
+  rate_limit_enabled: true
+
+  # Stream limits
+  max_entries_limit_per_second: 10000
+  max_streams_per_user: 10000
+
+  # Global retention policy: 30 days
+  retention_period: 720h  # 720h = 30 days
+
+  # Split logs into multiple lines after this many characters
+  max_line_size: 256KB
+
+  # Cardinality limits
+  cardinality_limit: 100000
+  enforce_metric_name: false
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+server:
+  http_listen_port: 3100
+  http_server_read_timeout: 600s
+  http_server_write_timeout: 600s
+  log_level: info
+
+storage_config:
+  # Use filesystem storage for development
+  filesystem:
+    directory: /loki/chunks
+
+  # Index storage configuration
+  index_cache_validity: 5m
+  cache_config:
+    enable_fifocache: true
+    default_validity: 1h
+    background:
+      writeback_goroutines: 10
+      writeback_buffer: 10000
+    memcached:
+      batch_size: 1024
+      parallelism: 100
+
+chunk_store_config:
+  # Maximum number of parallel workers processing chunks
+  max_look_back_period: 0s
+
+  # Cache config for the chunk store
+  cache_config:
+    enable_fifocache: true
+    default_validity: 1h
+
+table_manager:
+  # Enable table manager for automatic table creation and retention
+  enabled: true
+
+  # Period for creation of new tables
+  poll_interval: 10m
+
+  # Retention period for old tables
+  retention_deletes_enabled: true
+  retention_period: 720h  # 30 days
+
+  # Grace period before deleting tables
+  creation_grace_period: 10m
+
+query_range:
+  # Cache config for query results
+  cache_config:
+    enable_fifocache: true
+    default_validity: 1m
+    background:
+      writeback_goroutines: 10
+      writeback_buffer: 10000
+
+# Querier settings
+querier:
+  # Max concurrent queries
+  max_concurrent: 100
+
+  # Query timeout
+  query_timeout: 2m
+
+  # Engine timeout
+  engine:
+    timeout: 2m
+
+# Distributor settings
+distributor:
+  # Rate limit
+  rate_limit_enabled: false
+
+  # Ring configuration
+  ring:
+    kvstore:
+      store: inmemory
+
+# Compactor settings for log retention
+compactor:
+  # Enable compaction
+  compaction_interval: 10m
+
+  # Number of workers
+  max_compaction_parallel_shards: 2
+
+  # Retention settings
+  retention_enabled: true
+  retention_delete_delay: 2h
+  retention_delete_worker_count: 150
+
+  # Working directory for temporary files
+  working_directory: /loki/compactor
+  shared_store: filesystem
+
+runtime_config:
+  # Configuration file for runtime updates
+  # Useful for overrides without restart
+  file: /etc/loki/runtime.yaml
+  period: 10s
+
+# Tracing configuration (optional)
+# Uncomment to enable distributed tracing
+# tracing:
+#   enabled: false
+
+# Monitoring/Metrics
+# The metrics below are exposed on port 3100 at /metrics
+# Useful for Prometheus scraping
+metrics:
+  enabled: true
+  namespace: loki

package/config/loki/retention.yml

@@ -0,0 +1,107 @@
+# Loki Retention Policy Configuration
+# Task P2-2.3: Centralized Logging with ELK/Loki Stack
+#
+# This file defines the retention policies for logs stored in Loki.
+# Logs older than the retention period are automatically deleted.
+
+# Global retention settings
+retention:
+  # Default retention period for all log entries
+  # Format: integer with unit (h=hours, d=days)
+  # 30 days = 720 hours
+  default_period: 720h
+
+  # Tenant-specific retention overrides (if using multi-tenant Loki)
+  # Uncomment and customize as needed
+  # overrides:
+  #   tenant1: 1440h    # 60 days for tenant1
+  #   tenant2: 360h     # 15 days for tenant2
+
+# Compactor settings for retention enforcement
+compactor:
+  # Enable automatic compaction
+  enabled: true
+
+  # How often to run the compaction process
+  compaction_interval: 10m
+
+  # Delete delay before marking chunks as ready for deletion
+  retention_delete_delay: 2h
+
+  # Number of workers that will delete marked chunks
+  retention_delete_worker_count: 150
+
+  # Working directory for temporary files
+  working_directory: /loki/compactor
+
+  # Which store to use (filesystem, s3, etc.)
+  shared_store: filesystem
+
+# Table manager settings for automatic table cleanup
+table_manager:
+  # Enable automatic table management
+  enabled: true
+
+  # How often to check and create/delete tables
+  poll_interval: 10m
+
+  # Enable retention enforcement
+  retention_deletes_enabled: true
+
+  # Retention period in hours (30 days)
+  retention_period: 720h
+
+  # Grace period before deleting tables
+  creation_grace_period: 10m
+
+# Index configuration for retention
+schema:
+  - from: 2020-10-24
+    store: boltdb-shipper
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+# Deletion settings
+deletion:
+  # How many hours before a chunk can be deleted
+  hours_until_deleted: 0
+
+  # How long a delete request stays active before being deleted
+  max_delete_request_age: 168h  # 7 days
+
+  # Maximum number of parallel delete operations
+  max_delete_parallelism: 150
+
+# Cleanup settings for expired logs
+cleanup:
+  # Enable periodic cleanup
+  enabled: true
+
+  # How often to run cleanup
+  interval: 30m
+
+  # Skip logs that are still being written to
+  skip_recent: true
+
+  # Keep logs younger than this duration
+  keep_duration: 24h
+
+# Tier configuration for storage optimization
+tiers:
+  # Frequently accessed logs (hot)
+  - name: hot
+    period: 24h
+    delete_after: 2h
+
+  # Medium-term logs (warm)
+  - name: warm
+    period: 7d
+    delete_after: 1d
+
+  # Old logs (cold) - minimal access
+  - name: cold
+    period: 30d
+    delete_after: 30d

package/config/mcp-servers.json

@@ -0,0 +1,152 @@
+{
+  "version": "1.0.0",
+  "description": "MCP server configuration for specialized tool access",
+  "lastUpdated": "2025-06-17",
+  "servers": {
+    "playwright": {
+      "endpoint": "http://mcp-playwright:8081",
+      "required_skills": ["browser-automation", "testing", "screenshot-capture", "accessibility-testing"],
+      "auth": {
+        "type": "token",
+        "header": "X-MCP-Token"
+      },
+      "health_check": "/health",
+      "timeout_ms": 30000,
+      "retry_attempts": 3,
+      "resource_limits": {
+        "max_memory_mb": 512,
+        "max_cpu_percent": 50,
+        "max_concurrent_sessions": 3
+      },
+      "capabilities": [
+        "browser_automation",
+        "screenshot_capture",
+        "page_interaction",
+        "accessibility_testing",
+        "performance_testing"
+      ]
+    },
+    "redis-tools": {
+      "endpoint": "http://mcp-redis-tools:8082",
+      "required_skills": ["redis-operations", "database-management", "caching"],
+      "auth": {
+        "type": "token",
+        "header": "X-MCP-Token"
+      },
+      "health_check": "/health",
+      "timeout_ms": 15000,
+      "retry_attempts": 3,
+      "resource_limits": {
+        "max_memory_mb": 256,
+        "max_cpu_percent": 25,
+        "max_connections": 10
+      },
+      "capabilities": [
+        "redis_operations",
+        "key_value_management",
+        "cache_analysis",
+        "performance_monitoring",
+        "data_migration"
+      ]
+    },
+    "n8n": {
+      "endpoint": "http://mcp-n8n:8083",
+      "required_skills": ["workflow-automation", "api-integration", "data-pipeline"],
+      "auth": {
+        "type": "token",
+        "header": "X-MCP-Token"
+      },
+      "health_check": "/healthz",
+      "timeout_ms": 45000,
+      "retry_attempts": 2,
+      "resource_limits": {
+        "max_memory_mb": 1024,
+        "max_cpu_percent": 75,
+        "max_workflows": 5
+      },
+      "capabilities": [
+        "workflow_automation",
+        "api_integration",
+        "data_transformation",
+        "scheduled_tasks",
+        "webhook_handling"
+      ]
+    },
+    "security-scanner": {
+      "endpoint": "http://mcp-security-scanner:8084",
+      "required_skills": ["security-auditing", "vulnerability-scanning", "compliance-checking"],
+      "auth": {
+        "type": "token",
+        "header": "X-MCP-Token"
+      },
+      "health_check": "/health",
+      "timeout_ms": 120000,
+      "retry_attempts": 1,
+      "resource_limits": {
+        "max_memory_mb": 2048,
+        "max_cpu_percent": 80,
+        "max_scan_duration": 600000
+      },
+      "capabilities": [
+        "vulnerability_scanning",
+        "dependency_analysis",
+        "code_security_audit",
+        "compliance_checking",
+        "threat_modeling"
+      ]
+    }
+  },
+  "global_settings": {
+    "default_timeout_ms": 30000,
+    "max_retry_attempts": 3,
+    "connection_pool_size": 10,
+    "health_check_interval_ms": 30000,
+    "circuit_breaker_threshold": 5,
+    "metrics_enabled": true,
+    "logging": {
+      "level": "info",
+      "format": "json",
+      "include_sensitive_data": false
+    },
+    "security": {
+      "token_refresh_interval_ms": 3600000,
+      "max_token_age_ms": 86400000,
+      "rate_limiting": {
+        "requests_per_minute": 100,
+        "burst_size": 20
+      }
+    }
+  },
+  "skill_to_mcp_mapping": {
+    "browser-automation": ["playwright"],
+    "testing": ["playwright"],
+    "screenshot-capture": ["playwright"],
+    "accessibility-testing": ["playwright"],
+    "redis-operations": ["redis-tools"],
+    "database-management": ["redis-tools"],
+    "caching": ["redis-tools"],
+    "workflow-automation": ["n8n"],
+    "api-integration": ["n8n"],
+    "data-pipeline": ["n8n"],
+    "security-auditing": ["security-scanner"],
+    "vulnerability-scanning": ["security-scanner"],
+    "compliance-checking": ["security-scanner"],
+    "penetration-testing": ["security-scanner"],
+    "threat-analysis": ["security-scanner"]
+  },
+  "network_configuration": {
+    "mcp_network": {
+      "name": "mcp-network",
+      "subnet": "172.31.0.0/16",
+      "isolated": true,
+      "allowed_egress": ["cfn-network", "internet"]
+    }
+  },
+  "monitoring": {
+    "prometheus_metrics": true,
+    "health_check_endpoints": true,
+    "performance_tracking": true,
+    "error_reporting": true,
+    "alert_channels": ["redis-coordinator"]
+  }
+}

package/config/production.yml.example

@@ -0,0 +1,72 @@
+# Claude Flow Novice - Production Configuration Override
+# This file merges with config/default.yml
+# Only specify values that differ from default
+
+# Database - PostgreSQL for production
+database:
+  type: postgres
+  host: db.production.local
+  port: 5432
+  name: cfn_production
+  username: cfn_prod_user
+  password: ${DB_PASSWORD}  # Set via environment variable
+  pool:
+    min: 5
+    max: 50
+  ssl: true
+
+# Redis - Production cluster
+redis:
+  enabled: true
+  host: redis.production.local
+  port: 6380
+  password: ${REDIS_PASSWORD}
+  ttl: 172800  # 48 hours
+
+# Agents - Higher capacity for production
+agents:
+  maxConcurrent: 50
+  timeout: 600  # 10 minutes
+  retryAttempts: 5
+  defaultStrategy: production
+  logLevel: warn
+
+# Logging - File output with rotation
+logging:
+  level: warn
+  format: json
+  output: both
+  file:
+    path: /var/log/cfn/production.log
+    maxSize: 100m
+    maxFiles: 30
+
+# Security - Strict settings
+security:
+  enabled: true
+  apiKeyRequired: true
+  rateLimiting:
+    enabled: true
+    maxRequests: 1000
+    windowMs: 60000
+  allowedOrigins:
+    - https://app.production.com
+    - https://dashboard.production.com
+
+# Monitoring - Full observability
+monitoring:
+  enabled: true
+  prometheus:
+    enabled: true
+    port: 9090
+    path: /metrics
+  healthCheck:
+    enabled: true
+    port: 8080
+    path: /health
+
+# Features
+features:
+  hotReload: false  # Disabled in production for stability
+  customRouting: true
+  experimentalFeatures: false