agentic-qe 3.4.0 → 3.4.2

package/.claude/skills/shift-left-testing/schemas/output.json

@@ -0,0 +1,459 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-qe.dev/schemas/shift-left-testing-output.json",
+  "title": "AQE Shift-Left Testing Skill Output Schema",
+  "description": "Schema for shift-left-testing skill output validation. Extends the base skill-output template with test pyramid analysis, TDD assessment, and CI/CD integration.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "shift-left-testing",
+      "description": "Must be 'shift-left-testing'"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "const": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "testPyramid", "shiftLeftLevel", "findings", "recommendations"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 50,
+          "maxLength": 2000,
+          "description": "Human-readable summary of shift-left assessment"
+        },
+        "score": {
+          "$ref": "#/$defs/shiftLeftScore"
+        },
+        "testPyramid": {
+          "$ref": "#/$defs/testPyramid",
+          "description": "Test pyramid analysis"
+        },
+        "shiftLeftLevel": {
+          "$ref": "#/$defs/shiftLeftLevel",
+          "description": "Current shift-left maturity level"
+        },
+        "cicdIntegration": {
+          "$ref": "#/$defs/cicdIntegration",
+          "description": "CI/CD pipeline testing integration"
+        },
+        "tddPractices": {
+          "$ref": "#/$defs/tddPractices",
+          "description": "TDD/BDD practices assessment"
+        },
+        "requirementsValidation": {
+          "$ref": "#/$defs/requirementsValidation",
+          "description": "Early requirements testing"
+        },
+        "defectCostAnalysis": {
+          "$ref": "#/$defs/defectCostAnalysis",
+          "description": "Cost of defects by phase"
+        },
+        "findings": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/shiftLeftFinding"
+          },
+          "maxItems": 100
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/shiftLeftRecommendation"
+          },
+          "maxItems": 50
+        },
+        "metrics": {
+          "$ref": "#/$defs/shiftLeftMetrics"
+        },
+        "artifacts": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/artifact"
+          },
+          "maxItems": 50
+        }
+      }
+    },
+    "metadata": {
+      "$ref": "#/$defs/metadata"
+    },
+    "validation": {
+      "$ref": "#/$defs/validationResult"
+    },
+    "learning": {
+      "$ref": "#/$defs/learningData"
+    }
+  },
+  "$defs": {
+    "shiftLeftScore": {
+      "type": "object",
+      "required": ["value", "max"],
+      "properties": {
+        "value": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100
+        },
+        "max": {
+          "type": "number",
+          "const": 100
+        },
+        "grade": {
+          "type": "string",
+          "pattern": "^[A-F][+-]?$"
+        },
+        "maturityLevel": {
+          "type": "string",
+          "enum": ["level-1", "level-2", "level-3", "level-4"],
+          "description": "Shift-left maturity level"
+        }
+      }
+    },
+    "testPyramid": {
+      "type": "object",
+      "required": ["unit", "integration", "e2e"],
+      "properties": {
+        "unit": {
+          "type": "object",
+          "properties": {
+            "count": { "type": "integer", "minimum": 0 },
+            "percentage": { "type": "number", "minimum": 0, "maximum": 100 },
+            "target": { "type": "number", "default": 70 },
+            "coverage": { "type": "number", "minimum": 0, "maximum": 100 },
+            "averageExecutionMs": { "type": "number" },
+            "status": { "type": "string", "enum": ["healthy", "needs-attention", "critical"] }
+          },
+          "description": "Unit tests layer"
+        },
+        "integration": {
+          "type": "object",
+          "properties": {
+            "count": { "type": "integer", "minimum": 0 },
+            "percentage": { "type": "number", "minimum": 0, "maximum": 100 },
+            "target": { "type": "number", "default": 20 },
+            "coverage": { "type": "number", "minimum": 0, "maximum": 100 },
+            "averageExecutionMs": { "type": "number" },
+            "status": { "type": "string", "enum": ["healthy", "needs-attention", "critical"] }
+          },
+          "description": "Integration tests layer"
+        },
+        "e2e": {
+          "type": "object",
+          "properties": {
+            "count": { "type": "integer", "minimum": 0 },
+            "percentage": { "type": "number", "minimum": 0, "maximum": 100 },
+            "target": { "type": "number", "default": 10 },
+            "coverage": { "type": "number", "minimum": 0, "maximum": 100 },
+            "averageExecutionMs": { "type": "number" },
+            "status": { "type": "string", "enum": ["healthy", "needs-attention", "critical"] }
+          },
+          "description": "End-to-end tests layer"
+        },
+        "pyramidShape": {
+          "type": "string",
+          "enum": ["healthy-pyramid", "ice-cream-cone", "hourglass", "inverted", "unbalanced"],
+          "description": "Current pyramid shape"
+        },
+        "pyramidScore": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "How close to ideal pyramid (100 = perfect)"
+        }
+      }
+    },
+    "shiftLeftLevel": {
+      "type": "object",
+      "required": ["current", "practices"],
+      "properties": {
+        "current": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 4,
+          "description": "Current shift-left level (0-4)"
+        },
+        "target": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 4,
+          "description": "Target shift-left level"
+        },
+        "practices": {
+          "type": "object",
+          "properties": {
+            "level1": {
+              "type": "object",
+              "properties": {
+                "testsInPR": { "type": "boolean" },
+                "ciPipeline": { "type": "boolean" },
+                "codeReview": { "type": "boolean" }
+              },
+              "description": "Level 1: Tests in every PR"
+            },
+            "level2": {
+              "type": "object",
+              "properties": {
+                "tddPractice": { "type": "boolean" },
+                "testFirst": { "type": "boolean" },
+                "redGreenRefactor": { "type": "boolean" }
+              },
+              "description": "Level 2: TDD practice"
+            },
+            "level3": {
+              "type": "object",
+              "properties": {
+                "bddInRefinement": { "type": "boolean" },
+                "exampleMapping": { "type": "boolean" },
+                "threeAmigos": { "type": "boolean" }
+              },
+              "description": "Level 3: BDD in refinement"
+            },
+            "level4": {
+              "type": "object",
+              "properties": {
+                "riskAnalysisInDesign": { "type": "boolean" },
+                "testabilityReview": { "type": "boolean" },
+                "architectureValidation": { "type": "boolean" }
+              },
+              "description": "Level 4: Risk analysis in design"
+            }
+          }
+        }
+      }
+    },
+    "cicdIntegration": {
+      "type": "object",
+      "properties": {
+        "hasCI": { "type": "boolean" },
+        "testsOnEveryCommit": { "type": "boolean" },
+        "pipelineDuration": {
+          "type": "object",
+          "properties": {
+            "total": { "type": "integer", "description": "Total duration in seconds" },
+            "unitTests": { "type": "integer" },
+            "integrationTests": { "type": "integer" },
+            "e2eTests": { "type": "integer" },
+            "linting": { "type": "integer" }
+          }
+        },
+        "qualityGates": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": { "type": "string" },
+              "stage": { "type": "string" },
+              "blocking": { "type": "boolean" },
+              "currentStatus": { "type": "string", "enum": ["passing", "failing", "warning"] }
+            }
+          }
+        },
+        "fastFeedback": {
+          "type": "boolean",
+          "description": "Does pipeline provide feedback in < 10 minutes"
+        }
+      }
+    },
+    "tddPractices": {
+      "type": "object",
+      "properties": {
+        "tddAdoption": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "Percentage of code developed with TDD"
+        },
+        "testFirstRatio": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "Percentage of tests written before code"
+        },
+        "bddAdoption": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "Percentage of features with BDD specs"
+        },
+        "practicesAssessment": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "practice": { "type": "string" },
+              "implemented": { "type": "boolean" },
+              "effectiveness": { "type": "string", "enum": ["poor", "fair", "good", "excellent"] }
+            }
+          }
+        }
+      }
+    },
+    "requirementsValidation": {
+      "type": "object",
+      "properties": {
+        "investCriteria": { "type": "boolean" },
+        "testabilityReviewed": { "type": "boolean" },
+        "acceptanceCriteria": { "type": "boolean" },
+        "examplesProvided": { "type": "boolean" },
+        "ambiguityScore": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "How ambiguous are requirements (0=clear, 100=very ambiguous)"
+        }
+      }
+    },
+    "defectCostAnalysis": {
+      "type": "object",
+      "properties": {
+        "byPhase": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "phase": {
+                "type": "string",
+                "enum": ["requirements", "design", "development", "testing", "production"]
+              },
+              "relativeCost": {
+                "type": "number",
+                "description": "Relative cost (requirements = 1x)"
+              },
+              "defectsFound": {
+                "type": "integer",
+                "description": "Number of defects found in this phase"
+              },
+              "percentageFound": {
+                "type": "number",
+                "description": "Percentage of total defects found in this phase"
+              }
+            }
+          }
+        },
+        "potentialSavings": {
+          "type": "string",
+          "description": "Estimated savings from better shift-left"
+        }
+      }
+    },
+    "shiftLeftFinding": {
+      "type": "object",
+      "required": ["id", "title", "type", "severity"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^SLT-\\d{3,6}$"
+        },
+        "title": {
+          "type": "string",
+          "minLength": 10,
+          "maxLength": 200
+        },
+        "description": { "type": "string" },
+        "type": {
+          "type": "string",
+          "enum": ["pyramid-imbalance", "ci-gap", "tdd-gap", "late-testing", "missing-gate", "slow-feedback", "opportunity"]
+        },
+        "severity": {
+          "type": "string",
+          "enum": ["critical", "high", "medium", "low", "info"]
+        },
+        "phase": {
+          "type": "string",
+          "enum": ["requirements", "design", "development", "testing", "deployment"],
+          "description": "Which phase this finding affects"
+        }
+      }
+    },
+    "shiftLeftRecommendation": {
+      "type": "object",
+      "required": ["id", "title", "priority", "levelImpact"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^REC-\\d{3,6}$"
+        },
+        "title": { "type": "string" },
+        "description": { "type": "string" },
+        "priority": {
+          "type": "string",
+          "enum": ["critical", "high", "medium", "low"]
+        },
+        "levelImpact": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 4,
+          "description": "Which shift-left level this improves"
+        },
+        "costReduction": {
+          "type": "string",
+          "description": "Expected defect cost reduction"
+        },
+        "effort": {
+          "type": "string",
+          "enum": ["trivial", "low", "medium", "high", "major"]
+        }
+      }
+    },
+    "shiftLeftMetrics": {
+      "type": "object",
+      "properties": {
+        "defectEscapeRate": { "type": "number", "minimum": 0, "maximum": 100 },
+        "testCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "ciPassRate": { "type": "number", "minimum": 0, "maximum": 100 },
+        "averageFeedbackTime": { "type": "integer", "description": "Average CI feedback time in minutes" },
+        "tddAdoptionRate": { "type": "number", "minimum": 0, "maximum": 100 }
+      }
+    },
+    "artifact": {
+      "type": "object",
+      "required": ["type", "path"],
+      "properties": {
+        "type": { "type": "string" },
+        "path": { "type": "string" },
+        "format": { "type": "string" },
+        "description": { "type": "string" }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": { "type": "integer" },
+        "agentId": { "type": "string" },
+        "modelUsed": { "type": "string" },
+        "environment": { "type": "string" }
+      }
+    },
+    "validationResult": {
+      "type": "object",
+      "properties": {
+        "schemaValid": { "type": "boolean" },
+        "contentValid": { "type": "boolean" },
+        "confidence": { "type": "number" }
+      }
+    },
+    "learningData": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": { "type": "array", "items": { "type": "string" } },
+        "reward": { "type": "number" }
+      }
+    }
+  }
+}

package/.claude/skills/shift-left-testing/scripts/validate.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
+
+VALIDATOR_LIB=""
+for lib_path in "$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" "$SKILL_DIR/scripts/validator-lib.sh"; do
+  [[ -f "$lib_path" ]] && VALIDATOR_LIB="$lib_path" && break
+done
+[[ -n "$VALIDATOR_LIB" ]] && source "$VALIDATOR_LIB" || { echo "ERROR: Validator library not found"; exit 1; }
+
+SKILL_NAME="shift-left-testing"
+SKILL_VERSION="1.0.0"
+REQUIRED_TOOLS=("jq")
+SCHEMA_PATH="$SKILL_DIR/schemas/output.json"
+REQUIRED_FIELDS=("skillName" "status" "output")
+ENUM_VALIDATIONS=(".status:success,partial,failed,skipped")
+
+OUTPUT_FILE=""
+SELF_TEST=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --self-test) SELF_TEST=true; shift ;;
+    --verbose|-v|--json|--list-tools|-h|--help) shift ;;
+    -*) shift ;;
+    *) OUTPUT_FILE="$1"; shift ;;
+  esac
+done
+
+if [[ "$SELF_TEST" == "true" ]]; then
+  echo "Self-test for $SKILL_NAME passed"
+  exit 0
+fi
+
+validate_skill_specific() {
+  return 0
+}
+
+validate_tools() {
+  for tool in "${REQUIRED_TOOLS[@]}"; do
+    command_exists "$tool" || return 1
+  done
+  return 0
+}
+
+main() {
+  [[ -z "$OUTPUT_FILE" || ! -f "$OUTPUT_FILE" ]] && exit 1
+  validate_tools || exit 2
+  validate_json "$OUTPUT_FILE" || exit 1
+  validate_skill_specific || exit 1
+  exit 0
+}
+
+main

package/.claude/skills/shift-right-testing/SKILL.md

@@ -11,6 +11,12 @@ last_optimized: 2025-12-02
 dependencies: []
 quick_reference_card: true
 tags: [shift-right, production-testing, canary, feature-flags, chaos-engineering, monitoring]
+trust_tier: 3
+validation:
+  schema_path: schemas/output.json
+  validator_path: scripts/validate.sh
+  eval_path: evals/shift-right-testing.yaml
+
 ---
 
 # Shift-Right Testing

package/.claude/skills/shift-right-testing/evals/shift-right-testing.yaml

@@ -0,0 +1,147 @@
+skill: shift-right-testing
+version: 1.0.0
+description: >
+  Evaluation suite for shift-right testing strategy.
+  Tests production monitoring, canary deployments, and runtime validation.
+
+models_to_test:
+  - claude-3.5-sonnet
+  - claude-3-haiku
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  target_agents:
+    - qe-learning-coordinator
+
+learning:
+  store_success_patterns: true
+  pattern_ttl_days: 90
+
+result_format:
+  json_output: true
+  include_timing: true
+  include_token_usage: true
+
+setup:
+  required_tools:
+    - jq
+
+test_cases:
+
+  - id: tc001_production_monitoring
+    description: "Monitor system health in production"
+    category: monitoring
+    priority: critical
+
+    input:
+      metrics:
+        error_rate: 0.5
+        response_time_p99_ms: 2500
+        cpu_usage: 85
+      threshold_error_rate: 1.0
+
+    expected_output:
+      must_contain:
+        - "production"
+        - "monitor"
+        - "metric"
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_canary_deployment_analysis
+    description: "Validate canary deployment metrics"
+    category: deployment
+    priority: high
+
+    input:
+      canary_traffic_percent: 10
+      canary_error_rate: 0.3
+      baseline_error_rate: 0.1
+      rollout_decision_required: true
+
+    expected_output:
+      must_contain:
+        - "canary"
+        - "deployment"
+        - "rollout"
+
+    validation:
+      schema_check: true
+
+  - id: tc003_runtime_issue_detection
+    description: "Detect runtime issues in production"
+    category: detection
+    priority: high
+
+    input:
+      logs:
+        - "OutOfMemoryError: heap size exceeded"
+        - "Connection timeout after 30s"
+      environment: "production"
+
+    expected_output:
+      must_contain:
+        - "runtime"
+        - "issue"
+        - "memory"
+
+    validation:
+      schema_check: true
+
+  - id: tc004_synthetic_monitoring
+    description: "Generate synthetic test scenarios for production"
+    category: synthetic
+    priority: medium
+
+    input:
+      endpoints:
+        - "/api/users"
+        - "/api/payments"
+        - "/api/search"
+      test_frequency_minutes: 5
+
+    expected_output:
+      must_contain:
+        - "synthetic"
+        - "test"
+
+    validation:
+      schema_check: true
+
+  - id: tc005_anomaly_detection
+    description: "Detect anomalies in production behavior"
+    category: anomaly
+    priority: medium
+
+    input:
+      baseline_request_rate: 1000
+      current_request_rate: 5000
+      time_window_minutes: 5
+
+    expected_output:
+      must_contain:
+        - "anomaly"
+        - "detect"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+success_criteria:
+  pass_rate: 0.9
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.75
+  max_execution_time_ms: 300000
+
+metadata:
+  author: "qe-shift-right-coordinator"
+  created: "2026-02-02"
+  coverage_target: >
+    Shift-right testing with 5 test cases covering production monitoring,
+    canary deployments, runtime issue detection, synthetic monitoring, and anomaly detection.