claude-flow-novice 2.6.0 → 2.8.1

package/.claude/skills/agent-discovery/discover-agents.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+import json
+import logging
+from datetime import datetime
+
+import yaml  # Make sure to `pip install PyYAML`
+
+AGENTS_DIR = "/mnt/c/Users/masha/Documents/claude-flow-novice/.claude/agents"
+OUTPUT_FILE = "/mnt/c/Users/masha/Documents/claude-flow-novice/.claude/skills/agent-discovery/agents-registry.json"
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s: %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+# Mapping agent types to loops
+LOOP_MAPPING = {
+    "backend-dev": "loop3",
+    "frontend-dev": "loop3",
+    "coder": "loop3",
+    "developer": "loop3",
+    "api-designer": "loop3",
+    "devops": "loop3",
+    "security-specialist": "loop3",
+    "researcher": "loop3",
+    "reviewer": "loop2",
+    "tester": "loop2",
+    "validator": "loop2",
+    "auditor": "loop2",
+    "coordinator": "coordinator",
+    "strategic": "strategic"
+}
+
+def clean_windows_endings(content):
+    """Remove Windows line endings."""
+    return content.replace('\r', '')
+
+def extract_frontmatter(file_path):
+    """
+    Extract and parse frontmatter from markdown file using PyYAML.
+    Handles multiline descriptions and Windows line endings.
+    """
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = clean_windows_endings(f.read())
+            match = re.search(r'^---\n(.*?)\n---', content, re.DOTALL | re.MULTILINE)
+
+            if not match:
+                logger.warning(f"No frontmatter found in {file_path}")
+                return None
+
+            frontmatter_str = match.group(1)
+
+            # Use PyYAML for robust parsing
+            try:
+                frontmatter = yaml.safe_load(frontmatter_str)
+                return frontmatter
+            except yaml.YAMLError as e:
+                logger.error(f"YAML parsing error in {file_path}: {e}")
+                return None
+
+    except Exception as e:
+        logger.error(f"Error reading {file_path}: {e}")
+        return None
+
+def determine_loop(agent_type):
+    """Determine the loop for an agent type."""
+    agent_type = str(agent_type).lower()
+    return LOOP_MAPPING.get(agent_type, "strategic")
+
+def process_agents():
+    """Process all agent files and generate registry."""
+    agents = []
+    skipped_files = []
+
+    # Use os.walk to recursively search for .md files
+    for root, dirs, files in os.walk(AGENTS_DIR):
+        # Skip hidden directories
+        dirs[:] = [d for d in dirs if not d.startswith('.')]
+
+        for filename in files:
+            if not filename.endswith('.md'):
+                continue
+
+            # Skip documentation files
+            if filename in ['AGENT_LIFECYCLE.md', 'CLAUDE.md', 'README.md', 'README-VALIDATION.md', 'index.md']:
+                continue
+
+            file_path = os.path.join(root, filename)
+
+            # Skip git files
+            if '/.git/' in file_path:
+                continue
+
+            frontmatter = extract_frontmatter(file_path)
+
+            if not frontmatter:
+                skipped_files.append(file_path)
+                continue
+
+            # Extract required fields with fallbacks
+            name = str(frontmatter.get('name', filename.replace('.md', '')))
+            description = str(frontmatter.get('description', ''))
+            agent_type = str(frontmatter.get('type', 'specialist'))
+            keywords = frontmatter.get('keywords', [])
+
+            # Handle potential list conversion
+            if not isinstance(keywords, list):
+                keywords = [keywords] if keywords else []
+
+            # Determine loop
+            loop = determine_loop(agent_type)
+
+            # Create agent entry
+            agent_entry = {
+                "name": name,
+                "description": description,
+                "type": agent_type,
+                "loop": loop,
+                "keywords": keywords,
+                "file": file_path  # Include file path for tracking
+            }
+
+            logger.info(f"Processing agent: {name}")
+            logger.debug(f"Agent details: {json.dumps(agent_entry, indent=2)}")
+
+            agents.append(agent_entry)
+
+    # Prepare registry
+    registry = {
+        "agents": agents,
+        "last_updated": datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
+        "total_agents": len(agents)
+    }
+
+    # Ensure output directory exists
+    os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
+
+    # Write to file
+    try:
+        with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
+            json.dump(registry, f, indent=2)
+
+        logger.info(f"Agent registry generated: {OUTPUT_FILE}")
+        logger.info(f"Total agents discovered: {len(agents)}")
+
+        if skipped_files:
+            logger.warning(f"Skipped files: {', '.join(skipped_files)}")
+
+        # Verify file was written
+        if os.path.getsize(OUTPUT_FILE) > 0:
+            logger.info("File successfully generated and not empty.")
+        else:
+            raise ValueError("Output file is empty.")
+
+    except Exception as e:
+        logger.error(f"Error writing registry: {e}")
+        sys.exit(1)
+
+def main():
+    try:
+        process_agents()
+    except Exception as e:
+        logger.critical(f"Unhandled exception: {e}", exc_info=True)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

package/.claude/skills/agent-discovery/discover-agents.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Logging function
+log() {
+    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" >&2
+}
+
+# Error handler
+handle_error() {
+    log "ERROR: An error occurred in line $1"
+    log "Last command failed: $BASH_COMMAND"
+    exit 1
+}
+
+# Trap errors
+trap 'handle_error $LINENO' ERR
+
+# Path to Python script
+PYTHON_SCRIPT="/mnt/c/Users/masha/Documents/claude-flow-novice/.claude/skills/agent-discovery/discover-agents.py"
+
+# Output file
+OUTPUT_FILE="/mnt/c/Users/masha/Documents/claude-flow-novice/.claude/skills/agent-discovery/agents-registry.json"
+
+# Logging file
+LOG_FILE="/tmp/agent-discovery.log"
+
+log "Starting agent discovery process"
+
+# Check Python is available
+if ! command -v python3 &> /dev/null; then
+    log "ERROR: Python3 is not installed"
+    exit 1
+fi
+
+# Ensure we have required packages
+log "Installing required Python packages"
+
+# Try multiple methods to install PyYAML
+install_methods=(
+    "python3 -m pip install --user --break-system-packages pyyaml"
+    "python3 -m venv /tmp/pyenv && /tmp/pyenv/bin/pip install pyyaml"
+    "pip3 install --user pyyaml"
+    "pip install --user pyyaml"
+)
+
+installed=false
+for method in "${install_methods[@]}"; do
+    log "Trying installation method: $method"
+    if $method; then
+        installed=true
+        break
+    fi
+done
+
+if [ "$installed" = false ]; then
+    log "ERROR: Could not install PyYAML"
+    exit 1
+fi
+
+# Run Python discovery script with logging
+log "Running discovery script"
+python3 "$PYTHON_SCRIPT" 2>&1 | tee "$LOG_FILE"
+
+# Check script succeeded
+if [ ! -f "$OUTPUT_FILE" ]; then
+    log "ERROR: Output file was not generated"
+    exit 1
+fi
+
+# Optional: Run jq to validate JSON (if jq is available)
+if command -v jq &> /dev/null; then
+    log "Validating JSON output"
+    if jq '.' "$OUTPUT_FILE" > /dev/null 2>&1; then
+        log "✅ JSON is valid"
+    else
+        log "❌ Invalid JSON"
+        exit 1
+    fi
+fi
+
+# Show total agents
+total_agents=$(jq '.total_agents' "$OUTPUT_FILE" 2>/dev/null || echo 0)
+log "Total agents discovered: $total_agents"
+
+# Final success log
+log "Agent discovery completed successfully"

package/.claude/skills/agent-discovery/invoke-registry.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Invoke Agent Registry Discovery
+
+# Ensure script runs from project root
+cd "$(git rev-parse --show-toplevel)" || exit 1
+
+# Run discovery script
+./skills/agent-discovery/discover-agents.py
+
+# Optional: output registry to stdout
+cat .claude/skills/agent-discovery/agents-registry.json

package/.claude/skills/agent-output-processing/SKILL.md

@@ -0,0 +1,359 @@
+# Agent Output Processing Skill (Universal)
+
+**Version:** 1.0.0
+**Status:** Production
+**Purpose:** Guaranteed structured output extraction from any agent type
+
+---
+
+## Overview
+
+**Problem Solved:** Agent templates cannot force tool execution or guarantee output format.
+
+**Solution:** Orchestrator-controlled output processing with robust parsing, validation, and Redis coordination.
+
+**Applicable To:** Any agent requiring structured output (Product Owner, validators, analyzers, decision-makers)
+
+---
+
+## Core Principle
+
+```
+Agent Templates → Focus on analysis/decision-making
+Skills → Handle execution/coordination/formatting
+Orchestrators → Control workflow and Redis state
+```
+
+**Why This Works:**
+- ✅ Agents do what they're good at (analysis)
+- ✅ Skills enforce structure (parsing, validation)
+- ✅ Orchestrators maintain coordination (Redis, timing)
+- ✅ No reliance on agent tool usage compliance
+
+---
+
+## Architecture
+
+### Generic Flow
+
+```
+1. Orchestrator → Spawn agent with structured context
+2. Skill → Capture agent stdout/stderr
+3. Skill → Parse output (pattern matching with fallbacks)
+4. Skill → Validate output (business logic checks)
+5. Skill → Transform to required format (JSON, etc.)
+6. Skill → Push to Redis (coordination state)
+7. Skill → Signal completion
+```
+
+### Skill Components
+
+```
+.claude/skills/agent-output-processing/
+├── SKILL.md                          # This file
+├── execute-with-parsing.sh           # Generic agent wrapper
+├── parse-structured-output.sh        # Multi-pattern parser
+├── validate-output.sh                # Validation framework
+└── patterns/                         # Agent-specific patterns
+    ├── product-owner.json            # PROCEED/ITERATE/ABORT
+    ├── validator.json                # confidence + feedback
+    ├── analyzer.json                 # metrics + recommendations
+    └── decision-maker.json           # choice + reasoning
+```
+
+---
+
+## Usage Patterns
+
+### Pattern 1: Decision Output (Product Owner)
+
+**Output Requirements:** Must extract PROCEED/ITERATE/ABORT
+
+```bash
+RESULT=$(./.claude/skills/agent-output-processing/execute-with-parsing.sh \
+  --agent-type "product-owner" \
+  --pattern-file "patterns/product-owner.json" \
+  --task-id "$TASK_ID" \
+  --agent-id "$AGENT_ID" \
+  --context "Loop 2 consensus: 0.92" \
+  --redis-key "swarm:${TASK_ID}:${AGENT_ID}:decision")
+
+DECISION=$(echo "$RESULT" | jq -r '.decision')
+```
+
+**Pattern File (patterns/product-owner.json):**
+```json
+{
+  "name": "product-owner-decision",
+  "required_fields": ["decision"],
+  "patterns": [
+    {
+      "field": "decision",
+      "regex": "Decision:\\s*(PROCEED|ITERATE|ABORT)",
+      "extract": "keyword",
+      "fallbacks": [
+        "(PROCEED|ITERATE|ABORT)",
+        "(?i)(proceed|iterate|abort)"
+      ]
+    },
+    {
+      "field": "reasoning",
+      "regex": "Reasoning:\\s*(.+)",
+      "extract": "capture_group"
+    }
+  ],
+  "validation": {
+    "decision": {
+      "type": "enum",
+      "values": ["PROCEED", "ITERATE", "ABORT"]
+    }
+  }
+}
+```
+
+### Pattern 2: Confidence + Feedback (Validators)
+
+**Output Requirements:** Must extract confidence score and feedback list
+
+```bash
+RESULT=$(./.claude/skills/agent-output-processing/execute-with-parsing.sh \
+  --agent-type "reviewer" \
+  --pattern-file "patterns/validator.json" \
+  --task-id "$TASK_ID" \
+  --agent-id "$AGENT_ID" \
+  --context "Review Loop 3 implementation" \
+  --redis-key "swarm:${TASK_ID}:${AGENT_ID}:result")
+
+CONFIDENCE=$(echo "$RESULT" | jq -r '.confidence')
+FEEDBACK=$(echo "$RESULT" | jq -r '.feedback[]')
+```
+
+**Pattern File (patterns/validator.json):**
+```json
+{
+  "name": "validator-result",
+  "required_fields": ["confidence"],
+  "patterns": [
+    {
+      "field": "confidence",
+      "regex": "Confidence:\\s*([0-9.]+)",
+      "extract": "float",
+      "fallbacks": [
+        "confidence[:\\s]+([0-9.]+)",
+        "score[:\\s]+([0-9.]+)"
+      ]
+    },
+    {
+      "field": "feedback",
+      "regex": "Feedback:\\s*\\n([\\s\\S]+?)(?=\\n\\n|$)",
+      "extract": "multiline_list"
+    }
+  ],
+  "validation": {
+    "confidence": {
+      "type": "float",
+      "min": 0.0,
+      "max": 1.0
+    }
+  }
+}
+```
+
+### Pattern 3: Metrics + Recommendations (Analyzers)
+
+**Output Requirements:** Must extract metrics object and recommendation list
+
+```bash
+RESULT=$(./.claude/skills/agent-output-processing/execute-with-parsing.sh \
+  --agent-type "code-analyzer" \
+  --pattern-file "patterns/analyzer.json" \
+  --task-id "$TASK_ID" \
+  --agent-id "$AGENT_ID" \
+  --context "Analyze codebase quality" \
+  --redis-key "swarm:${TASK_ID}:${AGENT_ID}:analysis")
+
+COMPLEXITY=$(echo "$RESULT" | jq -r '.metrics.complexity')
+RECOMMENDATIONS=$(echo "$RESULT" | jq -r '.recommendations[]')
+```
+
+---
+
+## Implementation
+
+### execute-with-parsing.sh
+
+```bash
+#!/bin/bash
+set -euo pipefail
+
+# Parse arguments
+AGENT_TYPE=""
+PATTERN_FILE=""
+TASK_ID=""
+AGENT_ID=""
+CONTEXT=""
+REDIS_KEY=""
+TIMEOUT=900
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --agent-type) AGENT_TYPE="$2"; shift 2 ;;
+    --pattern-file) PATTERN_FILE="$2"; shift 2 ;;
+    --task-id) TASK_ID="$2"; shift 2 ;;
+    --agent-id) AGENT_ID="$2"; shift 2 ;;
+    --context) CONTEXT="$2"; shift 2 ;;
+    --redis-key) REDIS_KEY="$2"; shift 2 ;;
+    --timeout) TIMEOUT="$2"; shift 2 ;;
+    *) echo "Unknown: $1"; exit 1 ;;
+  esac
+done
+
+# Spawn agent and capture output
+AGENT_OUTPUT=$(timeout "$TIMEOUT" npx claude-flow-novice agent "$AGENT_TYPE" \
+  --task-id "$TASK_ID" \
+  --agent-id "$AGENT_ID" \
+  --context "$CONTEXT" 2>&1 || true)
+
+# Parse structured output using pattern file
+PARSED_OUTPUT=$(./parse-structured-output.sh \
+  --pattern-file "$PATTERN_FILE" \
+  --output "$AGENT_OUTPUT")
+
+# Validate parsed output
+if ! ./validate-output.sh --pattern-file "$PATTERN_FILE" --data "$PARSED_OUTPUT"; then
+  echo "ERROR: Output validation failed" >&2
+  exit 1
+fi
+
+# Push to Redis if key provided
+if [ -n "$REDIS_KEY" ]; then
+  echo "$PARSED_OUTPUT" | redis-cli -x LPUSH "$REDIS_KEY" >/dev/null
+  redis-cli LPUSH "swarm:${TASK_ID}:${AGENT_ID}:done" "complete" >/dev/null
+fi
+
+# Output structured result
+echo "$PARSED_OUTPUT"
+```
+
+---
+
+## Benefits Over Template-Based
+
+| Aspect | Template-Based | Skill-Based (Universal) |
+|--------|----------------|-------------------------|
+| **Execution Guarantee** | ❌ Agent decides | ✅ Skill enforces |
+| **Output Format** | ❌ Inconsistent | ✅ Guaranteed structure |
+| **Parsing Robustness** | ❌ None | ✅ Multi-pattern fallbacks |
+| **Validation** | ❌ Manual | ✅ Automated schema checks |
+| **Redis Coordination** | ❌ Agent-dependent | ✅ Skill-controlled |
+| **Reusability** | ❌ Per-agent templates | ✅ Universal pattern system |
+| **Testability** | ❌ Hard to test | ✅ Unit tests per pattern |
+| **Extensibility** | ❌ Template changes | ✅ Add pattern files |
+
+---
+
+## Agent-Specific Implementations
+
+### Existing: Product Owner Decision
+**Location:** `.claude/skills/product-owner-decision/`
+**Pattern:** PROCEED/ITERATE/ABORT extraction
+**Status:** ✅ Operational (BUG #11 fix)
+
+### Future: Validator Agents
+**Pattern:** Confidence + feedback extraction
+**Use Case:** Loop 2 validators (reviewer, tester, security)
+**Benefit:** Guaranteed confidence scoring
+
+### Future: Analyzer Agents
+**Pattern:** Metrics + recommendations extraction
+**Use Case:** Code quality, performance, security analyzers
+**Benefit:** Structured analysis output
+
+---
+
+## Migration Strategy
+
+### Phase 1: Product Owner (✅ Complete)
+- Created dedicated skill
+- Integrated with orchestrator
+- Solves BUG #11
+
+### Phase 2: Validators (Planned)
+- Extract confidence scores reliably
+- Parse feedback into structured lists
+- Prevent 0.0 confidence issues
+
+### Phase 3: All Decision-Making Agents (Future)
+- Standardize output processing
+- Centralize pattern definitions
+- Enable cross-agent consistency
+
+---
+
+## Testing
+
+### Unit Tests
+```bash
+# Test pattern parsing
+./test-parse-structured-output.sh --pattern-file patterns/product-owner.json
+
+# Test validation logic
+./test-validate-output.sh --pattern-file patterns/validator.json
+```
+
+### Integration Tests
+```bash
+# Test full agent execution with parsing
+./test-execute-with-parsing.sh --agent-type product-owner
+```
+
+---
+
+## Related Skills
+
+- **Product Owner Decision** (`.claude/skills/product-owner-decision/SKILL.md`) - Specific implementation
+- **Redis Coordination** (`.claude/skills/redis-coordination/SKILL.md`) - State management
+- **CFN Loop Validation** (`.claude/skills/cfn-loop-validation/SKILL.md`) - Quality gates
+
+---
+
+## Lessons Learned
+
+### ANTI-PATTERN: Template-Forced Tool Usage
+**What Doesn't Work:** Telling agents "Use Bash tool to execute X"
+
+**Why:** Agents interpret instructions autonomously, treating code blocks as documentation
+
+**Better:** Capture agent output and parse it with robust patterns
+
+### PATTERN: Orchestrator Control
+**Principle:** Coordination belongs in orchestrators/skills, not agent templates
+
+**Rationale:**
+- Agents focus on analysis (what they do well)
+- Skills handle structure (parsing, validation)
+- Orchestrators manage coordination (Redis, timing)
+
+### PATTERN: Multi-Fallback Parsing
+**Principle:** Use multiple pattern matching strategies with increasing leniency
+
+**Example:**
+1. Strict: "Decision: PROCEED" (labeled, exact case)
+2. Moderate: "PROCEED" (standalone keyword)
+3. Lenient: "proceed" (case-insensitive)
+4. Desperate: JSON extraction
+
+---
+
+## Version History
+
+### 1.0.0 (2025-10-20)
+- Initial skill creation
+- Generalized from Product Owner fix
+- Universal pattern-based output processing
+- Validation framework
+- Redis coordination integration
+
+---
+
+**Summary:** This skill provides universal output processing for any agent type, eliminating reliance on template-based tool usage compliance and guaranteeing structured, validated results.