agentic-qe 3.4.1 → 3.4.2

package/.claude/skills/n8n-workflow-testing-fundamentals/schemas/output.json

@@ -0,0 +1,254 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-qe.dev/schemas/skills/n8n-workflow-testing-fundamentals/output.json",
+  "title": "N8N Workflow Testing Fundamentals Skill Output Schema",
+  "description": "Schema for n8n-workflow-testing-fundamentals skill output. Validates workflow structure, node connections, and test coverage.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "n8n-workflow-testing-fundamentals"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "const": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "workflowValidation", "nodeTests"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 20,
+          "maxLength": 2000
+        },
+        "workflowValidation": {
+          "$ref": "#/$defs/workflowValidation"
+        },
+        "nodeTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/nodeTest"
+          },
+          "maxItems": 500
+        },
+        "connectionTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/connectionTest"
+          }
+        },
+        "dataFlowTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/dataFlowTest"
+          }
+        },
+        "errorHandlingTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/errorHandlingTest"
+          }
+        },
+        "structuralIssues": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/structuralIssue"
+          }
+        },
+        "testCoverage": {
+          "$ref": "#/$defs/testCoverage"
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/recommendation"
+          },
+          "maxItems": 50
+        },
+        "metrics": {
+          "$ref": "#/$defs/metrics"
+        }
+      }
+    },
+    "metadata": {
+      "$ref": "#/$defs/metadata"
+    },
+    "validation": {
+      "$ref": "#/$defs/validationResult"
+    },
+    "learning": {
+      "$ref": "#/$defs/learningData"
+    }
+  },
+  "$defs": {
+    "workflowValidation": {
+      "type": "object",
+      "required": ["workflowId", "valid"],
+      "properties": {
+        "workflowId": { "type": "string" },
+        "workflowName": { "type": "string" },
+        "valid": { "type": "boolean" },
+        "totalNodes": { "type": "integer", "minimum": 0 },
+        "totalConnections": { "type": "integer", "minimum": 0 },
+        "hasTrigger": { "type": "boolean" },
+        "hasEndNode": { "type": "boolean" },
+        "isLinear": { "type": "boolean" },
+        "hasBranching": { "type": "boolean" },
+        "hasLoops": { "type": "boolean" },
+        "hasErrorHandling": { "type": "boolean" },
+        "complexityScore": { "type": "number", "minimum": 0, "maximum": 100 },
+        "testabilityScore": { "type": "number", "minimum": 0, "maximum": 100 },
+        "overallScore": { "type": "number", "minimum": 0, "maximum": 100 },
+        "grade": { "type": "string", "pattern": "^[A-F][+-]?$" }
+      }
+    },
+    "nodeTest": {
+      "type": "object",
+      "required": ["nodeId", "nodeName", "nodeType", "status"],
+      "properties": {
+        "nodeId": { "type": "string" },
+        "nodeName": { "type": "string" },
+        "nodeType": { "type": "string" },
+        "status": { "type": "string", "enum": ["passed", "failed", "skipped", "untested"] },
+        "testCases": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": { "type": "string" },
+              "input": { "type": "object" },
+              "expectedOutput": {},
+              "actualOutput": {},
+              "passed": { "type": "boolean" }
+            }
+          }
+        },
+        "coverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "issues": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "connectionTest": {
+      "type": "object",
+      "required": ["sourceNode", "targetNode", "valid"],
+      "properties": {
+        "sourceNode": { "type": "string" },
+        "targetNode": { "type": "string" },
+        "connectionType": { "type": "string", "enum": ["main", "error", "conditional"] },
+        "valid": { "type": "boolean" },
+        "dataFlowVerified": { "type": "boolean" },
+        "issues": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "dataFlowTest": {
+      "type": "object",
+      "required": ["id", "path", "status"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^FLOW-\\d{3,6}$" },
+        "path": { "type": "array", "items": { "type": "string" } },
+        "status": { "type": "string", "enum": ["passed", "failed", "partial"] },
+        "inputData": { "type": "object" },
+        "outputData": { "type": "object" },
+        "transformations": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "errorHandlingTest": {
+      "type": "object",
+      "required": ["scenario", "handled"],
+      "properties": {
+        "scenario": { "type": "string" },
+        "errorType": { "type": "string", "enum": ["timeout", "connection", "validation", "authentication", "rate-limit", "unknown"] },
+        "handled": { "type": "boolean" },
+        "recoveryPath": { "type": "string" },
+        "notificationConfigured": { "type": "boolean" }
+      }
+    },
+    "structuralIssue": {
+      "type": "object",
+      "required": ["issueType", "severity"],
+      "properties": {
+        "issueType": {
+          "type": "string",
+          "enum": ["orphan-node", "missing-trigger", "dead-end", "circular-reference", "missing-error-handler", "duplicate-node"]
+        },
+        "severity": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+        "affectedNodes": { "type": "array", "items": { "type": "string" } },
+        "description": { "type": "string" },
+        "fix": { "type": "string" }
+      }
+    },
+    "testCoverage": {
+      "type": "object",
+      "properties": {
+        "nodeCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "connectionCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "pathCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "errorPathCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "overallCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "untestedNodes": { "type": "array", "items": { "type": "string" } },
+        "untestedPaths": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "recommendation": {
+      "type": "object",
+      "required": ["id", "title", "priority"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^REC-\\d{3,6}$" },
+        "title": { "type": "string", "maxLength": 200 },
+        "description": { "type": "string", "maxLength": 2000 },
+        "priority": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+        "category": { "type": "string", "enum": ["coverage", "structure", "reliability", "performance", "maintainability"] }
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "properties": {
+        "totalNodes": { "type": "integer", "minimum": 0 },
+        "testedNodes": { "type": "integer", "minimum": 0 },
+        "passedTests": { "type": "integer", "minimum": 0 },
+        "failedTests": { "type": "integer", "minimum": 0 },
+        "coverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "structuralIssues": { "type": "integer", "minimum": 0 },
+        "executionTimeMs": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "toolsUsed": { "type": "array", "items": { "type": "string" } },
+        "agentId": { "type": "string", "pattern": "^qe-[a-z][a-z0-9-]*$" },
+        "n8nVersion": { "type": "string" },
+        "workflowPath": { "type": "string" }
+      }
+    },
+    "validationResult": {
+      "type": "object",
+      "properties": {
+        "schemaValid": { "type": "boolean" },
+        "contentValid": { "type": "boolean" },
+        "confidence": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "learningData": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": { "type": "array", "items": { "type": "string" } },
+        "reward": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    }
+  }
+}

package/.claude/skills/n8n-workflow-testing-fundamentals/scripts/validate.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+# =============================================================================
+# AQE Skill Validator: n8n-workflow-testing-fundamentals v1.0.0
+# =============================================================================
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
+
+for lib_path in "$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" "$SKILL_DIR/scripts/validator-lib.sh"; do
+  [[ -f "$lib_path" ]] && source "$lib_path" && break
+done
+
+SKILL_NAME="n8n-workflow-testing-fundamentals"
+SKILL_VERSION="1.0.0"
+REQUIRED_TOOLS=("jq")
+SCHEMA_PATH="$SKILL_DIR/schemas/output.json"
+REQUIRED_FIELDS=("skillName" "status" "output" "output.summary" "output.workflowValidation" "output.nodeTests")
+MUST_CONTAIN_TERMS=("workflow" "n8n" "test")
+MUST_NOT_CONTAIN_TERMS=("TODO" "FIXME")
+ENUM_VALIDATIONS=(".status:success,partial,failed,skipped")
+
+validate_skill_specific() {
+  local output_file="$1"
+  local has_errors=false
+
+  # Validate workflow validation
+  local has_validation
+  has_validation=$(jq '.output.workflowValidation | has("workflowId") and has("valid")' "$output_file" 2>/dev/null)
+
+  if [[ "$has_validation" != "true" ]]; then
+    error "Workflow validation missing required fields (workflowId, valid)"
+    has_errors=true
+  fi
+
+  # Validate node tests
+  local test_count
+  test_count=$(jq '.output.nodeTests | length' "$output_file" 2>/dev/null || echo "0")
+
+  if [[ "$test_count" -gt 0 ]]; then
+    local invalid_tests
+    invalid_tests=$(jq '[.output.nodeTests[]? | select(.nodeId == null or .nodeName == null or .nodeType == null or .status == null)] | length' "$output_file" 2>/dev/null || echo "0")
+
+    if [[ "$invalid_tests" -gt 0 ]]; then
+      warn "$invalid_tests node test(s) missing required fields"
+    fi
+
+    # Validate status enum
+    local valid_statuses="passed failed skipped untested"
+    local statuses
+    statuses=$(jq -r '.output.nodeTests[]?.status // empty' "$output_file" 2>/dev/null || true)
+
+    for status in $statuses; do
+      if ! echo "$valid_statuses" | grep -qw "$status"; then
+        error "Invalid node test status: $status"
+        has_errors=true
+      fi
+    done
+  fi
+
+  # Validate connection tests if present
+  local conn_count
+  conn_count=$(jq '.output.connectionTests | length' "$output_file" 2>/dev/null || echo "0")
+
+  if [[ "$conn_count" -gt 0 ]]; then
+    local invalid_conns
+    invalid_conns=$(jq '[.output.connectionTests[]? | select(.sourceNode == null or .targetNode == null or .valid == null)] | length' "$output_file" 2>/dev/null || echo "0")
+
+    if [[ "$invalid_conns" -gt 0 ]]; then
+      warn "$invalid_conns connection test(s) missing required fields"
+    fi
+  fi
+
+  # Validate test coverage if present
+  local has_coverage
+  has_coverage=$(jq '.output | has("testCoverage")' "$output_file" 2>/dev/null)
+
+  if [[ "$has_coverage" == "true" ]]; then
+    local coverage
+    coverage=$(jq '.output.testCoverage.overallCoverage // -1' "$output_file" 2>/dev/null)
+
+    if [[ "$coverage" != "-1" ]] && (( $(echo "$coverage < 0 || $coverage > 100" | bc -l 2>/dev/null || echo "0") )); then
+      warn "Overall coverage $coverage is out of range (0-100)"
+    fi
+  fi
+
+  if [[ "$has_errors" == "true" ]]; then return 1; fi
+  success "N8N workflow testing validation passed"
+  return 0
+}
+
+OUTPUT_FILE="${1:-}"
+
+# Handle --self-test flag
+if [[ "$OUTPUT_FILE" == "--self-test" ]]; then
+  echo "=== Self-Test: $SKILL_NAME v$SKILL_VERSION ==="
+  echo "[PASS] Required tool: jq"
+  if [[ -f "$SCHEMA_PATH" ]]; then
+    echo "[PASS] Schema file exists"
+    if jq empty "$SCHEMA_PATH" 2>/dev/null; then
+      echo "[PASS] Schema is valid JSON"
+    else
+      echo "[FAIL] Schema is invalid JSON"
+      exit 1
+    fi
+  else
+    echo "[WARN] Schema file not found"
+  fi
+  echo "[PASS] Self-test completed"
+  exit 0
+fi
+
+[[ -z "$OUTPUT_FILE" ]] && { echo "Usage: $0 <output-file> | --self-test"; exit 1; }
+[[ ! -f "$OUTPUT_FILE" ]] && { echo "ERROR: File not found: $OUTPUT_FILE"; exit 1; }
+
+echo "Validating $SKILL_NAME output..."
+jq empty "$OUTPUT_FILE" 2>/dev/null || { echo "ERROR: Invalid JSON"; exit 1; }
+
+if validate_skill_specific "$OUTPUT_FILE"; then
+  echo "PASSED: $SKILL_NAME validation"
+  exit 0
+else
+  echo "FAILED: $SKILL_NAME validation"
+  exit 1
+fi

package/.claude/skills/performance-analysis/SKILL.md

@@ -5,6 +5,12 @@ description: Comprehensive performance analysis, bottleneck detection, and optim
 category: monitoring
 tags: [performance, bottleneck, optimization, profiling, metrics, analysis]
 author: Claude Flow Team
+trust_tier: 3
+validation:
+  schema_path: schemas/output.json
+  validator_path: scripts/validate.sh
+  eval_path: evals/performance-analysis.yaml
+
 ---
 
 # Performance Analysis Skill

package/.claude/skills/performance-analysis/evals/performance-analysis.yaml

@@ -0,0 +1,144 @@
+skill: performance-analysis
+version: 1.0.0
+description: >
+  Evaluation suite for performance analysis skill.
+  Tests metrics collection, bottleneck detection, and optimization recommendations.
+
+models_to_test:
+  - claude-3.5-sonnet
+  - claude-3-haiku
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  target_agents:
+    - qe-learning-coordinator
+
+learning:
+  store_success_patterns: true
+  store_failure_patterns: true
+  pattern_ttl_days: 90
+
+result_format:
+  json_output: true
+  include_timing: true
+  include_token_usage: true
+
+setup:
+  required_tools:
+    - jq
+
+test_cases:
+
+  - id: tc001_cpu_bottleneck_detection
+    description: "Detect CPU performance bottlenecks"
+    category: metrics
+    priority: critical
+
+    input:
+      metrics:
+        cpu_usage: 95
+        memory_usage: 45
+        response_time_ms: 2500
+
+    expected_output:
+      must_contain:
+        - "CPU"
+        - "bottleneck"
+        - "optimization"
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_memory_leak_detection
+    description: "Identify potential memory leaks"
+    category: metrics
+    priority: high
+
+    input:
+      metrics:
+        memory_growth_percent_per_hour: 15
+        total_runtime_hours: 8
+
+    expected_output:
+      must_contain:
+        - "memory"
+        - "leak"
+
+    validation:
+      schema_check: true
+
+  - id: tc003_response_time_analysis
+    description: "Analyze and report response time metrics"
+    category: metrics
+    priority: high
+
+    input:
+      metrics:
+        p50_ms: 100
+        p95_ms: 500
+        p99_ms: 1200
+
+    expected_output:
+      must_contain:
+        - "latency"
+        - "response"
+
+    validation:
+      schema_check: true
+
+  - id: tc004_optimization_recommendations
+    description: "Provide actionable optimization recommendations"
+    category: recommendations
+    priority: medium
+
+    input:
+      metrics:
+        cpu_usage: 85
+        cache_hit_rate: 0.6
+
+    expected_output:
+      must_contain:
+        - "recommend"
+        - "improve"
+        - "cache"
+
+    validation:
+      schema_check: true
+
+  - id: tc005_no_issues_healthy_system
+    description: "Recognize healthy system metrics"
+    category: negative
+    priority: medium
+
+    input:
+      metrics:
+        cpu_usage: 35
+        memory_usage: 40
+        response_time_ms: 100
+
+    expected_output:
+      must_contain:
+        - "healthy"
+        - "optimal"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+success_criteria:
+  pass_rate: 0.9
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.7
+  max_execution_time_ms: 300000
+
+metadata:
+  author: "qe-performance-analyst"
+  created: "2026-02-02"
+  coverage_target: >
+    Performance analysis with 5 test cases covering CPU/memory bottlenecks,
+    leak detection, response time analysis, and optimization recommendations.