agentic-qe 3.7.8 → 3.7.9

package/assets/skills/n8n-workflow-testing-fundamentals/schemas/output.json

@@ -0,0 +1,254 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-qe.dev/schemas/skills/n8n-workflow-testing-fundamentals/output.json",
+  "title": "N8N Workflow Testing Fundamentals Skill Output Schema",
+  "description": "Schema for n8n-workflow-testing-fundamentals skill output. Validates workflow structure, node connections, and test coverage.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "n8n-workflow-testing-fundamentals"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "const": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "workflowValidation", "nodeTests"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 20,
+          "maxLength": 2000
+        },
+        "workflowValidation": {
+          "$ref": "#/$defs/workflowValidation"
+        },
+        "nodeTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/nodeTest"
+          },
+          "maxItems": 500
+        },
+        "connectionTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/connectionTest"
+          }
+        },
+        "dataFlowTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/dataFlowTest"
+          }
+        },
+        "errorHandlingTests": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/errorHandlingTest"
+          }
+        },
+        "structuralIssues": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/structuralIssue"
+          }
+        },
+        "testCoverage": {
+          "$ref": "#/$defs/testCoverage"
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/recommendation"
+          },
+          "maxItems": 50
+        },
+        "metrics": {
+          "$ref": "#/$defs/metrics"
+        }
+      }
+    },
+    "metadata": {
+      "$ref": "#/$defs/metadata"
+    },
+    "validation": {
+      "$ref": "#/$defs/validationResult"
+    },
+    "learning": {
+      "$ref": "#/$defs/learningData"
+    }
+  },
+  "$defs": {
+    "workflowValidation": {
+      "type": "object",
+      "required": ["workflowId", "valid"],
+      "properties": {
+        "workflowId": { "type": "string" },
+        "workflowName": { "type": "string" },
+        "valid": { "type": "boolean" },
+        "totalNodes": { "type": "integer", "minimum": 0 },
+        "totalConnections": { "type": "integer", "minimum": 0 },
+        "hasTrigger": { "type": "boolean" },
+        "hasEndNode": { "type": "boolean" },
+        "isLinear": { "type": "boolean" },
+        "hasBranching": { "type": "boolean" },
+        "hasLoops": { "type": "boolean" },
+        "hasErrorHandling": { "type": "boolean" },
+        "complexityScore": { "type": "number", "minimum": 0, "maximum": 100 },
+        "testabilityScore": { "type": "number", "minimum": 0, "maximum": 100 },
+        "overallScore": { "type": "number", "minimum": 0, "maximum": 100 },
+        "grade": { "type": "string", "pattern": "^[A-F][+-]?$" }
+      }
+    },
+    "nodeTest": {
+      "type": "object",
+      "required": ["nodeId", "nodeName", "nodeType", "status"],
+      "properties": {
+        "nodeId": { "type": "string" },
+        "nodeName": { "type": "string" },
+        "nodeType": { "type": "string" },
+        "status": { "type": "string", "enum": ["passed", "failed", "skipped", "untested"] },
+        "testCases": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": { "type": "string" },
+              "input": { "type": "object" },
+              "expectedOutput": {},
+              "actualOutput": {},
+              "passed": { "type": "boolean" }
+            }
+          }
+        },
+        "coverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "issues": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "connectionTest": {
+      "type": "object",
+      "required": ["sourceNode", "targetNode", "valid"],
+      "properties": {
+        "sourceNode": { "type": "string" },
+        "targetNode": { "type": "string" },
+        "connectionType": { "type": "string", "enum": ["main", "error", "conditional"] },
+        "valid": { "type": "boolean" },
+        "dataFlowVerified": { "type": "boolean" },
+        "issues": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "dataFlowTest": {
+      "type": "object",
+      "required": ["id", "path", "status"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^FLOW-\\d{3,6}$" },
+        "path": { "type": "array", "items": { "type": "string" } },
+        "status": { "type": "string", "enum": ["passed", "failed", "partial"] },
+        "inputData": { "type": "object" },
+        "outputData": { "type": "object" },
+        "transformations": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "errorHandlingTest": {
+      "type": "object",
+      "required": ["scenario", "handled"],
+      "properties": {
+        "scenario": { "type": "string" },
+        "errorType": { "type": "string", "enum": ["timeout", "connection", "validation", "authentication", "rate-limit", "unknown"] },
+        "handled": { "type": "boolean" },
+        "recoveryPath": { "type": "string" },
+        "notificationConfigured": { "type": "boolean" }
+      }
+    },
+    "structuralIssue": {
+      "type": "object",
+      "required": ["issueType", "severity"],
+      "properties": {
+        "issueType": {
+          "type": "string",
+          "enum": ["orphan-node", "missing-trigger", "dead-end", "circular-reference", "missing-error-handler", "duplicate-node"]
+        },
+        "severity": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+        "affectedNodes": { "type": "array", "items": { "type": "string" } },
+        "description": { "type": "string" },
+        "fix": { "type": "string" }
+      }
+    },
+    "testCoverage": {
+      "type": "object",
+      "properties": {
+        "nodeCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "connectionCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "pathCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "errorPathCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "overallCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "untestedNodes": { "type": "array", "items": { "type": "string" } },
+        "untestedPaths": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "recommendation": {
+      "type": "object",
+      "required": ["id", "title", "priority"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^REC-\\d{3,6}$" },
+        "title": { "type": "string", "maxLength": 200 },
+        "description": { "type": "string", "maxLength": 2000 },
+        "priority": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+        "category": { "type": "string", "enum": ["coverage", "structure", "reliability", "performance", "maintainability"] }
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "properties": {
+        "totalNodes": { "type": "integer", "minimum": 0 },
+        "testedNodes": { "type": "integer", "minimum": 0 },
+        "passedTests": { "type": "integer", "minimum": 0 },
+        "failedTests": { "type": "integer", "minimum": 0 },
+        "coverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "structuralIssues": { "type": "integer", "minimum": 0 },
+        "executionTimeMs": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "toolsUsed": { "type": "array", "items": { "type": "string" } },
+        "agentId": { "type": "string", "pattern": "^qe-[a-z][a-z0-9-]*$" },
+        "n8nVersion": { "type": "string" },
+        "workflowPath": { "type": "string" }
+      }
+    },
+    "validationResult": {
+      "type": "object",
+      "properties": {
+        "schemaValid": { "type": "boolean" },
+        "contentValid": { "type": "boolean" },
+        "confidence": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "learningData": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": { "type": "array", "items": { "type": "string" } },
+        "reward": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    }
+  }
+}

package/assets/skills/n8n-workflow-testing-fundamentals/scripts/validate-config.json

@@ -0,0 +1,35 @@
+{
+  "skillName": "n8n-workflow-testing-fundamentals",
+  "skillVersion": "1.0.0",
+  "requiredTools": [
+    "jq"
+  ],
+  "optionalTools": [],
+  "schemaPath": "schemas/output.json",
+  "requiredFields": [
+    "skillName",
+    "status",
+    "output",
+    "output.summary",
+    "output.workflowValidation",
+    "output.nodeTests"
+  ],
+  "requiredNonEmptyFields": [],
+  "mustContainTerms": [
+    "workflow",
+    "n8n",
+    "test"
+  ],
+  "mustNotContainTerms": [
+    "TODO",
+    "FIXME"
+  ],
+  "enumValidations": {
+    ".status": [
+      "success",
+      "partial",
+      "failed",
+      "skipped"
+    ]
+  }
+}

package/assets/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml

@@ -0,0 +1,157 @@
+skill: observability-testing-patterns
+version: 1.0.0
+description: >
+  Evaluation suite for observability and monitoring validation patterns.
+  Tests dashboard accuracy, alerting rules, log aggregation, APM traces,
+  and SLA/SLO verification.
+
+models_to_test:
+  - claude-3.5-sonnet
+  - claude-3-haiku
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  target_agents:
+    - qe-learning-coordinator
+    - qe-queen-coordinator
+
+learning:
+  store_success_patterns: true
+  pattern_ttl_days: 90
+
+result_format:
+  json_output: true
+  include_timing: true
+  include_token_usage: true
+
+setup:
+  required_tools:
+    - jq
+
+test_cases:
+
+  - id: tc001_dashboard_validation
+    description: "Validate monitoring dashboard accuracy and completeness"
+    category: dashboard
+    priority: critical
+
+    input:
+      prompt: |
+        Generate test strategy for validating a Grafana dashboard that monitors
+        API response times, error rates, and throughput for a microservices architecture.
+      context:
+        framework: grafana
+
+    expected_output:
+      must_contain:
+        - "dashboard"
+        - "metric"
+        - "validation"
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_alert_rule_testing
+    description: "Test alerting rules fire correctly under conditions"
+    category: alerting
+    priority: critical
+
+    input:
+      prompt: |
+        Create tests for PagerDuty alert rules:
+        - P1: error_rate > 5% for 2 minutes
+        - P2: p99_latency > 2000ms for 5 minutes
+        - P3: disk_usage > 85%
+      context:
+        framework: pagerduty
+
+    expected_output:
+      must_contain:
+        - "alert"
+        - "threshold"
+        - "trigger"
+
+    validation:
+      schema_check: true
+
+  - id: tc003_log_aggregation_testing
+    description: "Validate log aggregation pipeline correctness"
+    category: logging
+    priority: high
+
+    input:
+      prompt: |
+        Generate tests for ELK stack log pipeline ensuring structured logs
+        from 15 microservices are correctly parsed, indexed, and searchable
+        with correlation IDs.
+      context:
+        framework: elasticsearch
+
+    expected_output:
+      must_contain:
+        - "log"
+        - "aggregation"
+        - "correlation"
+
+    validation:
+      schema_check: true
+
+  - id: tc004_apm_trace_validation
+    description: "Validate distributed tracing across services"
+    category: tracing
+    priority: high
+
+    input:
+      prompt: |
+        Create test strategy for validating OpenTelemetry distributed traces
+        across API gateway → auth service → order service → payment service.
+
+    expected_output:
+      must_contain:
+        - "trace"
+        - "span"
+        - "distributed"
+
+    validation:
+      schema_check: true
+
+  - id: tc005_slo_verification
+    description: "Verify SLO/SLA compliance measurement accuracy"
+    category: slo
+    priority: critical
+
+    input:
+      prompt: |
+        Generate tests to verify SLO measurement accuracy:
+        - Availability SLO: 99.95% (error budget: 21.9 min/month)
+        - Latency SLO: p99 < 500ms
+        - Throughput SLO: > 10,000 RPS
+      context:
+        framework: prometheus
+
+    expected_output:
+      must_contain:
+        - "SLO"
+        - "error budget"
+        - "compliance"
+
+    validation:
+      schema_check: true
+
+success_criteria:
+  pass_rate: 0.9
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.75
+  max_execution_time_ms: 300000
+
+metadata:
+  author: "qe-observability-orchestrator"
+  created: "2026-03-05"
+  coverage_target: >
+    Observability testing patterns with 5 test cases covering dashboard validation,
+    alert rule testing, log aggregation, APM trace validation, and SLO verification.

package/assets/skills/observability-testing-patterns/schemas/output.json

@@ -0,0 +1,58 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-qe.dev/schemas/observability-testing-patterns-output.json",
+  "title": "Observability Testing Patterns Output Schema",
+  "description": "Schema for observability testing output. Covers dashboards, alerting, logging, tracing, and SLO verification.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "observability-testing-patterns"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "const": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "testStrategy"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 100
+        },
+        "testStrategy": {
+          "type": "object",
+          "properties": {
+            "dashboardTests": { "type": "array", "items": { "type": "object" } },
+            "alertTests": { "type": "array", "items": { "type": "object" } },
+            "logTests": { "type": "array", "items": { "type": "object" } },
+            "traceTests": { "type": "array", "items": { "type": "object" } },
+            "sloTests": { "type": "array", "items": { "type": "object" } }
+          }
+        },
+        "findings": {
+          "type": "array",
+          "items": { "type": "object" }
+        },
+        "recommendations": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    }
+  }
+}

package/assets/skills/observability-testing-patterns/scripts/validate-config.json

@@ -0,0 +1,25 @@
+{
+  "skillName": "observability-testing-patterns",
+  "skillVersion": "1.0.0",
+  "requiredTools": [
+    "jq"
+  ],
+  "optionalTools": [],
+  "schemaPath": "schemas/output.json",
+  "requiredFields": [
+    "skillName",
+    "status",
+    "output"
+  ],
+  "requiredNonEmptyFields": [],
+  "mustContainTerms": [],
+  "mustNotContainTerms": [],
+  "enumValidations": {
+    ".status": [
+      "success",
+      "partial",
+      "failed",
+      "skipped"
+    ]
+  }
+}

package/assets/skills/pentest-validation/scripts/validate-config.json

@@ -0,0 +1,12 @@
+{
+  "skillName": "pentest-validation",
+  "skillVersion": "1.0.0",
+  "requiredTools": [],
+  "optionalTools": [],
+  "schemaPath": null,
+  "requiredFields": [],
+  "requiredNonEmptyFields": [],
+  "mustContainTerms": [],
+  "mustNotContainTerms": [],
+  "enumValidations": {}
+}

package/assets/skills/pr-review/SKILL.md

@@ -1,7 +1,7 @@
 ---
 name: pr-review
 description: Scope-aware GitHub PR review with user-friendly tone and trust tier validation
-trust_tier: 3
+trust_tier: 0
 domain: code-review
 ---