claude-flow-novice 2.15.3 → 2.15.4

package/claude-assets/skills/cfn-file-operations/test.sh

@@ -0,0 +1,369 @@
+#!/bin/bash
+#
+# File Operations Skill - Test Suite
+#
+# Tests file locking and atomic write operations.
+# Part of Task 4.2: Centralized File Locking & Atomic Operations
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL="$SCRIPT_DIR/execute.sh"
+TEST_DIR="/tmp/cfn-file-ops-test-$$"
+LOCK_DIR="/tmp/cfn-locks-test-$$"
+
+export CFN_LOCK_DIR="$LOCK_DIR"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Test counters
+TESTS_RUN=0
+TESTS_PASSED=0
+TESTS_FAILED=0
+
+#
+# Setup test environment
+#
+setup() {
+  echo "Setting up test environment..."
+  mkdir -p "$TEST_DIR"
+  mkdir -p "$LOCK_DIR"
+}
+
+#
+# Cleanup test environment
+#
+cleanup() {
+  echo "Cleaning up test environment..."
+  rm -rf "$TEST_DIR"
+  rm -rf "$LOCK_DIR"
+}
+
+#
+# Assert helper
+#
+assert() {
+  local description="$1"
+  local command="$2"
+
+  TESTS_RUN=$((TESTS_RUN + 1))
+  echo -n "  Test $TESTS_RUN: $description... "
+
+  if eval "$command" > /dev/null 2>&1; then
+    echo -e "${GREEN}PASS${NC}"
+    TESTS_PASSED=$((TESTS_PASSED + 1))
+    return 0
+  else
+    echo -e "${RED}FAIL${NC}"
+    TESTS_FAILED=$((TESTS_FAILED + 1))
+    return 1
+  fi
+}
+
+#
+# Test: Lock acquisition and release
+#
+test_lock_acquisition() {
+  echo -e "\n${YELLOW}Test Suite: Lock Acquisition${NC}"
+
+  local test_file="$TEST_DIR/lock-test.txt"
+  touch "$test_file"
+
+  # Test 1: Acquire lock
+  assert "Can acquire lock" "
+    LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id test-agent)
+    [ -n \"\$LOCK_INFO\" ]
+  "
+
+  # Test 2: Lock file exists
+  assert "Lock file created" "
+    $SKILL acquire-lock '$test_file' --agent-id test-agent > /tmp/lock-info-$$.txt
+    LOCK_INFO=\$(cat /tmp/lock-info-$$.txt)
+    LOCK_PATH=\"\${LOCK_INFO#*:}\"
+    [ -f \"\$LOCK_PATH\" ]
+  "
+
+  # Test 3: Release lock
+  assert "Can release lock" "
+    LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id test-agent)
+    $SKILL release-lock \"\$LOCK_INFO\"
+  "
+
+  # Test 4: Lock file removed after release
+  assert "Lock file removed after release" "
+    LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id test-agent)
+    LOCK_PATH=\"\${LOCK_INFO#*:}\"
+    $SKILL release-lock \"\$LOCK_INFO\"
+    [ ! -f \"\$LOCK_PATH\" ]
+  "
+}
+
+#
+# Test: Lock renewal
+#
+test_lock_renewal() {
+  echo -e "\n${YELLOW}Test Suite: Lock Renewal${NC}"
+
+  local test_file="$TEST_DIR/renewal-test.txt"
+  touch "$test_file"
+
+  # Test 5: Renew lock
+  assert "Can renew lock" "
+    LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id test-agent)
+    $SKILL renew-lock \"\$LOCK_INFO\" --extension 60000
+    $SKILL release-lock \"\$LOCK_INFO\"
+  "
+
+  # Test 6: Renewal count increases
+  assert "Renewal count increases" "
+    LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id test-agent)
+    LOCK_PATH=\"\${LOCK_INFO#*:}\"
+    BEFORE=\$(jq -r '.renewalCount' \"\$LOCK_PATH\")
+    $SKILL renew-lock \"\$LOCK_INFO\" --extension 60000
+    AFTER=\$(jq -r '.renewalCount' \"\$LOCK_PATH\")
+    $SKILL release-lock \"\$LOCK_INFO\"
+    [ \"\$AFTER\" -gt \"\$BEFORE\" ]
+  "
+}
+
+#
+# Test: Atomic writes
+#
+test_atomic_writes() {
+  echo -e "\n${YELLOW}Test Suite: Atomic Writes${NC}"
+
+  local test_file="$TEST_DIR/write-test.txt"
+
+  # Test 7: Basic write
+  assert "Can write file atomically" "
+    $SKILL atomic-write '$test_file' 'Hello World'
+    [ -f '$test_file' ]
+    [ \"\$(cat '$test_file')\" = 'Hello World' ]
+  "
+
+  # Test 8: Write with checksum
+  assert "Write with checksum verification" "
+    $SKILL atomic-write '$test_file' 'Test content' --checksum
+    [ -f '$test_file' ]
+  "
+
+  # Test 9: Write with backup
+  assert "Write creates backup" "
+    echo 'Original' > '$test_file'
+    RESULT=\$($SKILL atomic-write '$test_file' 'Updated' --backup)
+    BACKUP_PATH=\$(echo \"\$RESULT\" | jq -r '.backupPath')
+    [ -f \"\$BACKUP_PATH\" ]
+    [ \"\$(cat \"\$BACKUP_PATH\")\" = 'Original' ]
+  "
+}
+
+#
+# Test: Atomic reads
+#
+test_atomic_reads() {
+  echo -e "\n${YELLOW}Test Suite: Atomic Reads${NC}"
+
+  local test_file="$TEST_DIR/read-test.txt"
+
+  # Test 10: Read file
+  assert "Can read file atomically" "
+    echo 'Test content' > '$test_file'
+    RESULT=\$($SKILL atomic-read '$test_file')
+    CONTENT=\$(echo \"\$RESULT\" | jq -r '.content')
+    [ \"\$CONTENT\" = 'Test content' ]
+  "
+
+  # Test 11: Read returns checksum
+  assert "Read returns checksum" "
+    echo 'Test' > '$test_file'
+    RESULT=\$($SKILL atomic-read '$test_file')
+    CHECKSUM=\$(echo \"\$RESULT\" | jq -r '.checksum')
+    [ -n \"\$CHECKSUM\" ]
+    [ \"\${#CHECKSUM}\" -eq 64 ]
+  "
+}
+
+#
+# Test: Checksum verification
+#
+test_checksum() {
+  echo -e "\n${YELLOW}Test Suite: Checksum Verification${NC}"
+
+  local test_file="$TEST_DIR/checksum-test.txt"
+
+  # Test 12: Verify checksum
+  assert "Can verify file checksum" "
+    echo -n 'Test' > '$test_file'
+    EXPECTED=\$(echo -n 'Test' | sha256sum | awk '{print \$1}')
+    RESULT=\$($SKILL verify-checksum '$test_file' \"\$EXPECTED\")
+    MATCHES=\$(echo \"\$RESULT\" | jq -r '.matches')
+    [ \"\$MATCHES\" = \"1\" ]
+  "
+
+  # Test 13: Detect checksum mismatch
+  assert "Detects checksum mismatch" "
+    echo -n 'Test' > '$test_file'
+    WRONG='0000000000000000000000000000000000000000000000000000000000000000'
+    RESULT=\$($SKILL verify-checksum '$test_file' \"\$WRONG\")
+    MATCHES=\$(echo \"\$RESULT\" | jq -r '.matches')
+    [ \"\$MATCHES\" = \"0\" ]
+  "
+}
+
+#
+# Test: Concurrent access
+#
+test_concurrent_access() {
+  echo -e "\n${YELLOW}Test Suite: Concurrent Access${NC}"
+
+  local test_file="$TEST_DIR/concurrent-test.txt"
+  touch "$test_file"
+
+  # Test 14: Second acquire waits for first release
+  assert "Second lock waits for first" "
+    # Acquire lock in background, hold for 2 seconds
+    (
+      LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id agent-1)
+      sleep 2
+      $SKILL release-lock \"\$LOCK_INFO\"
+    ) &
+
+    # Wait a bit for first lock to acquire
+    sleep 0.5
+
+    # Try to acquire second lock (should wait)
+    START=\$(date +%s)
+    LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id agent-2 --timeout 5000)
+    END=\$(date +%s)
+    WAIT_TIME=\$((END - START))
+
+    $SKILL release-lock \"\$LOCK_INFO\"
+
+    # Should have waited at least 1 second
+    [ \"\$WAIT_TIME\" -ge 1 ]
+  "
+}
+
+#
+# Test: Force release
+#
+test_force_release() {
+  echo -e "\n${YELLOW}Test Suite: Force Release${NC}"
+
+  local test_file="$TEST_DIR/force-test.txt"
+  touch "$test_file"
+
+  # Test 15: Force release removes lock
+  assert "Force release removes lock" "
+    LOCK_INFO=\$($SKILL acquire-lock '$test_file' --agent-id test-agent)
+    LOCK_PATH=\"\${LOCK_INFO#*:}\"
+    $SKILL force-release \"\$LOCK_PATH\"
+    [ ! -f \"\$LOCK_PATH\" ]
+  "
+}
+
+#
+# Test: Metrics
+#
+test_metrics() {
+  echo -e "\n${YELLOW}Test Suite: Metrics${NC}"
+
+  local test_file="$TEST_DIR/metrics-test.txt"
+  touch "$test_file"
+
+  # Test 16: Get metrics
+  assert "Can retrieve metrics" "
+    METRICS=\$($SKILL get-metrics)
+    ACTIVE=\$(echo \"\$METRICS\" | jq -r '.activeLocks')
+    [ -n \"\$ACTIVE\" ]
+  "
+
+  # Test 17: Metrics reflect active locks
+  assert "Metrics reflect active locks" "
+    # Clean start
+    rm -rf '$LOCK_DIR'/*.lock 2>/dev/null || true
+
+    LOCK_INFO_1=\$($SKILL acquire-lock '$test_file.1' --agent-id agent-1)
+    LOCK_INFO_2=\$($SKILL acquire-lock '$test_file.2' --agent-id agent-2)
+
+    METRICS=\$($SKILL get-metrics)
+    ACTIVE=\$(echo \"\$METRICS\" | jq -r '.activeLocks')
+
+    $SKILL release-lock \"\$LOCK_INFO_1\"
+    $SKILL release-lock \"\$LOCK_INFO_2\"
+
+    [ \"\$ACTIVE\" -ge 2 ]
+  "
+}
+
+#
+# Test: Error handling
+#
+test_error_handling() {
+  echo -e "\n${YELLOW}Test Suite: Error Handling${NC}"
+
+  # Test 18: Missing file path
+  assert "Rejects missing file path" "
+    ! $SKILL acquire-lock '' --agent-id test-agent
+  "
+
+  # Test 19: Invalid lock ID
+  assert "Rejects invalid lock ID" "
+    ! $SKILL release-lock 'invalid-lock-id'
+  "
+
+  # Test 20: Timeout handling
+  assert "Handles lock timeout" "
+    LOCK_INFO=\$($SKILL acquire-lock '$TEST_DIR/timeout-test.txt' --agent-id agent-1)
+    ! $SKILL acquire-lock '$TEST_DIR/timeout-test.txt' --agent-id agent-2 --timeout 500
+    $SKILL release-lock \"\$LOCK_INFO\"
+  "
+}
+
+#
+# Main test runner
+#
+main() {
+  echo "========================================"
+  echo "File Operations Skill - Test Suite"
+  echo "========================================"
+
+  setup
+
+  test_lock_acquisition
+  test_lock_renewal
+  test_atomic_writes
+  test_atomic_reads
+  test_checksum
+  test_concurrent_access
+  test_force_release
+  test_metrics
+  test_error_handling
+
+  cleanup
+
+  echo ""
+  echo "========================================"
+  echo "Test Results"
+  echo "========================================"
+  echo "Total tests:  $TESTS_RUN"
+  echo -e "Passed:       ${GREEN}$TESTS_PASSED${NC}"
+  echo -e "Failed:       ${RED}$TESTS_FAILED${NC}"
+  echo ""
+
+  if [ "$TESTS_FAILED" -eq 0 ]; then
+    echo -e "${GREEN}All tests passed!${NC}"
+    exit 0
+  else
+    echo -e "${RED}Some tests failed.${NC}"
+    exit 1
+  fi
+}
+
+# Run tests
+main

package/claude-assets/skills/cfn-log-operations/SKILL.md

@@ -0,0 +1,308 @@
+# CFN Log Operations Skill
+
+**Specialization**: Distributed logging, log aggregation, and monitoring
+
+**Part of**: Task 4.4 Distributed Logging Standardization
+
+## Overview
+
+The CFN Log Operations skill provides comprehensive logging capabilities for the CFN system, including:
+
+- **Log Search and Filtering**: Query logs by correlation ID, agent ID, task ID, or timestamp
+- **Log Aggregation**: Combine logs from multiple sources into unified searchable formats
+- **Log Rotation and Cleanup**: Manage log lifecycle with retention policies
+- **Performance Monitoring**: Track logging overhead and system impact
+- **Error Tracking**: Monitor and alert on ERROR/FATAL level logs
+
+## Usage
+
+```bash
+./.claude/skills/cfn-log-operations/execute.sh [COMMAND] [OPTIONS]
+```
+
+### Commands
+
+#### search
+Search for logs matching specific criteria.
+
+**Syntax**:
+```bash
+./execute.sh search [OPTIONS]
+```
+
+**Options**:
+- `--correlation-id ID` - Search by correlation ID
+- `--agent-id ID` - Search by agent ID
+- `--task-id ID` - Search by task ID
+- `--level LEVEL` - Filter by log level (debug, info, warn, error)
+- `--since DURATION` - Logs from last N hours/minutes (e.g., 24h, 2h)
+- `--pattern PATTERN` - Grep pattern for message
+- `--source SOURCE` - Filter by source container/service
+- `--format FORMAT` - Output format (json, text, csv)
+- `--limit COUNT` - Limit results (default: 100)
+
+**Examples**:
+```bash
+# Search by correlation ID
+./execute.sh search --correlation-id "task:task-001:agent"
+
+# Find all errors in last 24 hours
+./execute.sh search --level error --since 24h
+
+# Find logs from specific agent
+./execute.sh search --agent-id "backend-dev-001" --format json
+
+# Search with message pattern
+./execute.sh search --pattern "connection" --since 2h
+```
+
+#### aggregate
+Aggregate logs from multiple sources into unified log files.
+
+**Syntax**:
+```bash
+./execute.sh aggregate [OPTIONS]
+```
+
+**Options**:
+- `--source SOURCE` - Log source (docker, filesystem, all)
+- `--output OUTPUT_DIR` - Output directory
+- `--since DURATION` - Only aggregate recent logs
+- `--deduplicate` - Remove duplicate entries
+- `--validate` - Validate JSON structure
+- `--compress` - Compress aggregated logs
+- `--correlate-by FIELD` - Group by field (correlationId, agentId, taskId)
+
+**Examples**:
+```bash
+# Aggregate all logs from last 24 hours
+./execute.sh aggregate --source all --since 24h
+
+# Aggregate and deduplicate
+./execute.sh aggregate --source filesystem --deduplicate --validate
+
+# Aggregate with compression
+./execute.sh aggregate --source docker --compress
+```
+
+#### rotate
+Manage log rotation and retention.
+
+**Syntax**:
+```bash
+./execute.sh rotate [OPTIONS]
+```
+
+**Options**:
+- `--log-dir DIR` - Directory to rotate (default: /var/log/cfn)
+- `--max-size SIZE` - Maximum file size (e.g., 100M)
+- `--max-files COUNT` - Maximum rotated files to keep
+- `--compress` - Compress rotated logs
+- `--retention-days DAYS` - Retention period
+- `--force` - Force rotation even if not due
+
+**Examples**:
+```bash
+# Rotate all logs
+./execute.sh rotate
+
+# Rotate with custom size and retention
+./execute.sh rotate --max-size 50M --retention-days 30
+
+# Force rotation with compression
+./execute.sh rotate --force --compress
+```
+
+#### monitor
+Monitor logs for errors and performance issues.
+
+**Syntax**:
+```bash
+./execute.sh monitor [OPTIONS]
+```
+
+**Options**:
+- `--interval SECONDS` - Check interval (default: 60)
+- `--alert-on LEVEL` - Alert levels (default: error,fatal)
+- `--error-threshold COUNT` - Error count threshold
+- `--action ACTION` - Alert action (log, email, webhook)
+- `--performance-check` - Enable performance monitoring
+- `--retention-check` - Enable retention compliance check
+- `--daemon` - Run as daemon
+
+**Examples**:
+```bash
+# Monitor with default settings
+./execute.sh monitor
+
+# Monitor with custom error threshold
+./execute.sh monitor --error-threshold 20
+
+# Monitor as daemon with performance checks
+./execute.sh monitor --daemon --performance-check --retention-check
+```
+
+#### stats
+Generate log statistics and metrics.
+
+**Syntax**:
+```bash
+./execute.sh stats [OPTIONS]
+```
+
+**Options**:
+- `--log-dir DIR` - Directory to analyze
+- `--since DURATION` - Time window for analysis
+- `--format FORMAT` - Output format (json, text)
+
+**Examples**:
+```bash
+# Get log statistics
+./execute.sh stats
+
+# Statistics for last 24 hours
+./execute.sh stats --since 24h --format json
+```
+
+#### export
+Export logs in various formats.
+
+**Syntax**:
+```bash
+./execute.sh export [OPTIONS]
+```
+
+**Options**:
+- `--source SOURCE` - Log source directory
+- `--format FORMAT` - Output format (json, csv, tsv)
+- `--output FILE` - Output file
+- `--filter PATTERN` - Filter logs before export
+
+**Examples**:
+```bash
+# Export to CSV
+./execute.sh export --format csv --output logs.csv
+
+# Export errors to JSON
+./execute.sh export --filter "level==error" --format json
+```
+
+## Log Format Specification
+
+All logs follow the standardized JSON format:
+
+```json
+{
+  "timestamp": "2025-11-16T03:00:00Z",
+  "level": "info",
+  "message": "Task completed successfully",
+  "correlationId": "task:task-001:agent",
+  "source": "cfn-agent-container",
+  "context": {
+    "agentId": "backend-dev-001",
+    "taskId": "task-001",
+    "iteration": 1,
+    "sprintId": "sprint-4"
+  },
+  "metadata": {
+    "duration": 5000,
+    "confidence": 0.92,
+    "tags": ["task-execution", "success"]
+  }
+}
+```
+
+### Required Fields
+- `timestamp`: ISO 8601 UTC timestamp
+- `level`: debug, info, warn, error, fatal
+- `message`: Human-readable message
+- `source`: Container/service name
+
+### Optional Fields
+- `correlationId`: Format: `type:id:agent` (e.g., `task:task-001:agent`)
+- `context`: Additional context (agentId, taskId, iteration, sprintId)
+- `metadata`: Operation-specific metadata (duration, confidence, tags)
+- `error`: Error details (name, message, stack)
+
+## Search Patterns
+
+### Using jq
+
+```bash
+# Find all error logs
+jq 'select(.level=="error")' logs/*.json
+
+# Filter by correlation ID
+jq 'select(.correlationId=="task-001:task:agent")' logs/*.json
+
+# Find logs by agent
+jq 'select(.context.agentId=="backend-dev-001")' logs/*.json
+
+# Count errors by source
+jq 'select(.level=="error") | .source' logs/*.json | sort | uniq -c
+
+# Complex query: errors in last hour
+jq 'select(.level=="error" and .timestamp > "'$(date -u -d '1 hour ago' +'%Y-%m-%dT%H:%M:%SZ')'")' logs/*.json
+
+# Export to CSV
+jq -r '[.timestamp, .level, .message, .source] | @csv' logs/*.json
+```
+
+### Using grep and awk
+
+```bash
+# Find error logs
+grep '"level":"error"' logs/*.json
+
+# Find logs by task ID
+grep 'task-001' logs/*.json | jq '.'
+
+# Count log levels
+grep -o '"level":"[^"]*"' logs/*.json | sort | uniq -c
+
+# Find slow operations (duration > 5000ms)
+jq 'select(.metadata.duration > 5000)' logs/*.json
+```
+
+## Troubleshooting
+
+### No logs found
+- Check log directory path: `/var/log/cfn/`
+- Verify log files exist: `ls -la /var/log/cfn/`
+- Check permissions: `stat /var/log/cfn/`
+
+### JSON parsing errors
+- Validate log format: `./.claude/skills/cfn-log-operations/execute.sh search --validate`
+- Check file encoding: `file logs/*.log`
+- Review recent logs: `tail -50 logs/*.log`
+
+### Performance issues
+- Reduce search scope: use `--since` flag
+- Limit results: use `--limit` flag
+- Check disk space: `df /var/log/cfn`
+
+### Retention not working
+- Verify logrotate config: `/etc/logrotate.d/cfn-logs`
+- Test logrotate: `logrotate -d /etc/logrotate.d/cfn-logs`
+- Check permissions: files should be readable by logrotate
+
+## Integration
+
+This skill integrates with:
+- Docker logging driver (json-file)
+- Logrotate for log rotation and retention
+- Application logging utilities (`src/lib/logging.ts`)
+- Monitoring systems (custom alerting)
+
+## Performance Targets
+
+- Search operation: <500ms for 1GB of logs
+- Aggregation: <5% CPU overhead
+- Monitoring: <1% memory overhead when running as daemon
+- 90%+ structured JSON logs (10% unstructured OK)
+
+## Related Tasks
+
+- **Task 0.5**: Logging utilities (`src/lib/logging.ts`)
+- **Task 4.1**: Integration infrastructure
+- **Task 4.4**: Distributed logging standardization