agentic-qe 3.4.1 → 3.4.2

package/.claude/skills/regression-testing/schemas/output.json

@@ -0,0 +1,311 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://agentic-qe.dev/schemas/regression-testing-output.json",
+  "title": "AQE Regression Testing Skill Output Schema",
+  "description": "Schema for regression testing output including test selection analysis, impact assessment, and suite optimization.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "regression-testing",
+      "description": "Skill identifier"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+-]\\d{2}:\\d{2})?$"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "const": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "testSelection", "impactAnalysis", "metrics"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 10,
+          "maxLength": 2000
+        },
+        "score": {
+          "$ref": "#/$defs/regressionScore"
+        },
+        "testSelection": {
+          "$ref": "#/$defs/testSelection",
+          "description": "Test selection strategy results"
+        },
+        "impactAnalysis": {
+          "$ref": "#/$defs/impactAnalysis",
+          "description": "Change impact analysis"
+        },
+        "executionResults": {
+          "$ref": "#/$defs/executionResults",
+          "description": "Regression test execution results"
+        },
+        "suiteHealth": {
+          "$ref": "#/$defs/suiteHealth",
+          "description": "Test suite health metrics"
+        },
+        "findings": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/regressionFinding"
+          },
+          "maxItems": 500,
+          "description": "Regressions detected"
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/recommendation"
+          },
+          "maxItems": 100
+        },
+        "metrics": {
+          "$ref": "#/$defs/regressionMetrics"
+        },
+        "artifacts": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/artifact"
+          },
+          "maxItems": 50
+        }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "toolsUsed": { "type": "array", "items": { "type": "string" } },
+        "agentId": { "type": "string", "pattern": "^qe-[a-z][a-z0-9-]*$" },
+        "prNumber": { "type": "integer", "description": "PR number if applicable" },
+        "baseBranch": { "type": "string" },
+        "headBranch": { "type": "string" }
+      }
+    },
+    "validation": {
+      "type": "object",
+      "properties": {
+        "schemaValid": { "type": "boolean" },
+        "contentValid": { "type": "boolean" },
+        "confidence": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "learning": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": { "type": "array", "items": { "type": "string" } },
+        "reward": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    }
+  },
+  "$defs": {
+    "regressionScore": {
+      "type": "object",
+      "required": ["value", "max"],
+      "properties": {
+        "value": { "type": "number", "minimum": 0, "maximum": 100 },
+        "max": { "type": "number", "const": 100 },
+        "grade": { "type": "string", "pattern": "^[A-F][+-]?$" },
+        "riskCoverage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Risk coverage achieved (0.0-1.0)"
+        }
+      }
+    },
+    "testSelection": {
+      "type": "object",
+      "required": ["strategy"],
+      "properties": {
+        "strategy": {
+          "type": "string",
+          "enum": ["change-based", "risk-based", "historical", "time-budget", "full", "smoke"],
+          "description": "Selection strategy used"
+        },
+        "mustRun": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "Tests that must run (directly impacted)"
+        },
+        "shouldRun": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "Tests that should run (indirectly impacted)"
+        },
+        "canSkip": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "Tests that can be safely skipped"
+        },
+        "totalTests": { "type": "integer", "minimum": 0 },
+        "selectedTests": { "type": "integer", "minimum": 0 },
+        "selectionRatio": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Percentage of tests selected"
+        },
+        "estimatedTime": {
+          "type": "string",
+          "description": "Estimated execution time"
+        },
+        "riskCoverage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Risk coverage of selected tests"
+        }
+      }
+    },
+    "impactAnalysis": {
+      "type": "object",
+      "properties": {
+        "changedFiles": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "impactedModules": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "impactedTests": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "changeRisk": {
+          "type": "string",
+          "enum": ["low", "medium", "high", "critical"],
+          "description": "Overall change risk level"
+        },
+        "coverageMap": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "description": "Map of files to covering tests"
+        }
+      }
+    },
+    "executionResults": {
+      "type": "object",
+      "properties": {
+        "total": { "type": "integer", "minimum": 0 },
+        "passed": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "skipped": { "type": "integer", "minimum": 0 },
+        "flaky": { "type": "integer", "minimum": 0 },
+        "passRate": { "type": "number", "minimum": 0, "maximum": 100 },
+        "regressions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "test": { "type": "string" },
+              "previousStatus": { "type": "string" },
+              "currentStatus": { "type": "string" },
+              "failureMessage": { "type": "string" }
+            }
+          },
+          "description": "Tests that regressed"
+        },
+        "executionTimeMs": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "suiteHealth": {
+      "type": "object",
+      "properties": {
+        "flakiness": {
+          "type": "object",
+          "properties": {
+            "flakyTests": { "type": "array", "items": { "type": "string" } },
+            "flakinessRate": { "type": "number", "minimum": 0, "maximum": 100 }
+          }
+        },
+        "executionTrend": {
+          "type": "string",
+          "enum": ["improving", "stable", "degrading"],
+          "description": "Execution time trend"
+        },
+        "averageExecutionTime": { "type": "integer", "description": "Average test execution time in ms" },
+        "slowTests": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "test": { "type": "string" },
+              "duration": { "type": "integer" }
+            }
+          }
+        },
+        "recommendations": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "regressionFinding": {
+      "type": "object",
+      "required": ["id", "title", "severity"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^REG-\\d{3,6}$" },
+        "title": { "type": "string", "minLength": 5, "maxLength": 200 },
+        "description": { "type": "string", "maxLength": 2000 },
+        "severity": { "type": "string", "enum": ["critical", "high", "medium", "low", "info"] },
+        "test": { "type": "string", "description": "Test that detected the regression" },
+        "affectedComponent": { "type": "string" },
+        "introducedIn": { "type": "string", "description": "Commit/PR that introduced the regression" },
+        "evidence": { "type": "string" },
+        "remediation": { "type": "string" }
+      }
+    },
+    "regressionMetrics": {
+      "type": "object",
+      "properties": {
+        "totalTests": { "type": "integer", "minimum": 0 },
+        "testsSelected": { "type": "integer", "minimum": 0 },
+        "testsExecuted": { "type": "integer", "minimum": 0 },
+        "passRate": { "type": "number", "minimum": 0, "maximum": 100 },
+        "regressionsFound": { "type": "integer", "minimum": 0 },
+        "selectionEfficiency": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "Percentage of tests run vs total"
+        },
+        "riskCoverageAchieved": { "type": "number", "minimum": 0, "maximum": 1 },
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "timeSaved": { "type": "string", "description": "Time saved vs full regression" }
+      }
+    },
+    "recommendation": {
+      "type": "object",
+      "required": ["id", "title", "priority"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^REC-\\d{3,6}$" },
+        "title": { "type": "string" },
+        "description": { "type": "string" },
+        "priority": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+        "effort": { "type": "string", "enum": ["trivial", "low", "medium", "high", "major"] }
+      }
+    },
+    "artifact": {
+      "type": "object",
+      "required": ["type", "path"],
+      "properties": {
+        "type": { "type": "string", "enum": ["report", "coverage", "log", "data"] },
+        "path": { "type": "string" },
+        "format": { "type": "string", "enum": ["json", "html", "md", "xml", "txt"] }
+      }
+    }
+  }
+}

package/.claude/skills/regression-testing/scripts/validate.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# =============================================================================
+# AQE Regression Testing Skill Validator v1.0.0
+# Validates regression testing output including test selection and impact analysis
+# =============================================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
+
+for lib_path in \
+  "$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" \
+  "$SKILL_DIR/scripts/validator-lib.sh"; do
+  if [[ -f "$lib_path" ]]; then
+    source "$lib_path"
+    break
+  fi
+done
+
+SKILL_NAME="regression-testing"
+SKILL_VERSION="1.0.0"
+REQUIRED_TOOLS=("jq")
+OPTIONAL_TOOLS=("ajv" "jsonschema" "python3")
+SCHEMA_PATH="$SKILL_DIR/schemas/output.json"
+
+REQUIRED_FIELDS=("skillName" "status" "output" "output.testSelection" "output.impactAnalysis" "output.metrics")
+REQUIRED_NON_EMPTY_FIELDS=("output.summary" "output.testSelection")
+MUST_CONTAIN_TERMS=("regression" "test")
+MUST_NOT_CONTAIN_TERMS=("TODO" "placeholder" "FIXME")
+ENUM_VALIDATIONS=(".status:success,partial,failed,skipped" ".output.testSelection.strategy:change-based,risk-based,historical,time-budget,full,smoke")
+
+OUTPUT_FILE=""
+SELF_TEST=false
+VERBOSE=false
+JSON_ONLY=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --self-test) SELF_TEST=true; shift ;;
+    --verbose|-v) VERBOSE=true; export AQE_DEBUG=1; shift ;;
+    --json) JSON_ONLY=true; shift ;;
+    --list-tools)
+      echo "Regression Testing Validator Tools"
+      for tool in "${REQUIRED_TOOLS[@]}"; do
+        command_exists "$tool" && echo "  [OK] $tool" || echo "  [MISSING] $tool"
+      done
+      exit 0 ;;
+    -h|--help) echo "Usage: $0 <output-file> [--self-test] [--verbose] [--json]"; exit 0 ;;
+    -*) error "Unknown option: $1"; exit 1 ;;
+    *) OUTPUT_FILE="$1"; shift ;;
+  esac
+done
+
+if [[ "$SELF_TEST" == "true" ]]; then
+  echo "Running $SKILL_NAME Validator Self-Test..."
+  for tool in "${REQUIRED_TOOLS[@]}"; do
+    command_exists "$tool" && success "Tool: $tool" || error "Missing: $tool"
+  done
+  [[ -f "$SCHEMA_PATH" ]] && success "Schema exists" || error "Schema missing"
+  exit 0
+fi
+
+validate_skill_specific() {
+  local output_file="$1"
+  local has_errors=false
+
+  # Check test selection strategy
+  local strategy
+  strategy=$(json_get "$output_file" ".output.testSelection.strategy" 2>/dev/null)
+  if [[ -n "$strategy" ]] && [[ "$strategy" != "null" ]]; then
+    success "Test selection strategy: $strategy"
+  else
+    error "Test selection strategy required"
+    has_errors=true
+  fi
+
+  # Check selected vs total tests
+  local total selected
+  total=$(json_get "$output_file" ".output.testSelection.totalTests" 2>/dev/null)
+  selected=$(json_get "$output_file" ".output.testSelection.selectedTests" 2>/dev/null)
+  if [[ -n "$total" ]] && [[ -n "$selected" ]] && [[ "$total" != "null" ]] && [[ "$selected" != "null" ]]; then
+    success "Test selection: $selected of $total tests"
+  fi
+
+  # Check risk coverage
+  local risk_coverage
+  risk_coverage=$(json_get "$output_file" ".output.testSelection.riskCoverage" 2>/dev/null)
+  [[ -n "$risk_coverage" ]] && [[ "$risk_coverage" != "null" ]] && success "Risk coverage: $risk_coverage"
+
+  # Check impact analysis
+  local changed_files
+  changed_files=$(json_count "$output_file" ".output.impactAnalysis.changedFiles" 2>/dev/null)
+  [[ -n "$changed_files" ]] && success "Changed files analyzed: $changed_files"
+
+  # Check finding format
+  local finding_count
+  finding_count=$(json_count "$output_file" ".output.findings" 2>/dev/null)
+  if [[ -n "$finding_count" ]] && [[ "$finding_count" -gt 0 ]]; then
+    local first_id
+    first_id=$(json_get "$output_file" ".output.findings[0].id" 2>/dev/null)
+    [[ "$first_id" =~ ^REG-[0-9]+$ ]] && success "Finding ID format valid" || warn "Finding ID should match REG-XXX"
+  fi
+
+  [[ "$has_errors" == "true" ]] && return 1
+  success "Regression-specific validation passed"
+  return 0
+}
+
+# Main validation
+[[ -z "$OUTPUT_FILE" ]] && { error "No output file specified"; exit 1; }
+[[ ! -f "$OUTPUT_FILE" ]] && { error "File not found: $OUTPUT_FILE"; exit 1; }
+
+for tool in "${REQUIRED_TOOLS[@]}"; do
+  command_exists "$tool" || { error "Missing tool: $tool"; exit 2; }
+done
+
+validate_json "$OUTPUT_FILE" || { error "Invalid JSON"; exit 1; }
+[[ -f "$SCHEMA_PATH" ]] && validate_json_schema "$SCHEMA_PATH" "$OUTPUT_FILE"
+
+for field in "${REQUIRED_FIELDS[@]}"; do
+  value=$(json_get "$OUTPUT_FILE" ".$field" 2>/dev/null)
+  [[ -z "$value" ]] || [[ "$value" == "null" ]] && { error "Missing field: $field"; exit 1; }
+done
+
+validate_skill_specific "$OUTPUT_FILE" || exit 1
+
+success "Validation PASSED"
+exit 0

package/.claude/skills/risk-based-testing/SKILL.md

@@ -11,6 +11,12 @@ last_optimized: 2025-12-02
 dependencies: []
 quick_reference_card: true
 tags: [risk, prioritization, test-planning, coverage, impact-analysis]
+trust_tier: 3
+validation:
+  schema_path: schemas/output.json
+  validator_path: scripts/validate.sh
+  eval_path: evals/risk-based-testing.yaml
+
 ---
 
 # Risk-Based Testing

package/.claude/skills/risk-based-testing/evals/risk-based-testing.yaml

@@ -0,0 +1,141 @@
+skill: risk-based-testing
+version: 1.0.0
+description: >
+  Evaluation suite for risk-based testing strategy.
+  Tests risk scoring, test prioritization, and coverage optimization.
+
+models_to_test:
+  - claude-3.5-sonnet
+  - claude-3-haiku
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  target_agents:
+    - qe-learning-coordinator
+
+learning:
+  store_success_patterns: true
+  pattern_ttl_days: 90
+
+result_format:
+  json_output: true
+  include_timing: true
+  include_token_usage: true
+
+setup:
+  required_tools:
+    - jq
+
+test_cases:
+
+  - id: tc001_risk_scoring
+    description: "Score and prioritize risks"
+    category: risk_assessment
+    priority: critical
+
+    input:
+      components:
+        - { name: "payment_processor", criticality: "critical", change_frequency: "low" }
+        - { name: "notification_service", criticality: "low", change_frequency: "high" }
+
+    expected_output:
+      must_contain:
+        - "risk"
+        - "score"
+        - "priority"
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_test_prioritization
+    description: "Prioritize tests based on risk"
+    category: prioritization
+    priority: high
+
+    input:
+      available_tests: 500
+      time_budget_hours: 2
+      risk_scores: { "critical": 10, "high": 5, "medium": 2 }
+
+    expected_output:
+      must_contain:
+        - "prioritize"
+        - "test"
+        - "critical"
+
+    validation:
+      schema_check: true
+
+  - id: tc003_coverage_optimization
+    description: "Optimize test coverage for risk mitigation"
+    category: coverage
+    priority: high
+
+    input:
+      high_risk_areas: ["authentication", "payment", "data_export"]
+      current_coverage_percent: 65
+
+    expected_output:
+      must_contain:
+        - "coverage"
+        - "optimize"
+
+    validation:
+      schema_check: true
+
+  - id: tc004_regression_risk
+    description: "Identify regression risk from changes"
+    category: regression
+    priority: medium
+
+    input:
+      changed_files:
+        - "auth/login.js"
+        - "auth/session.js"
+      dependent_modules: ["dashboard", "profile", "settings"]
+
+    expected_output:
+      must_contain:
+        - "regression"
+        - "risk"
+
+    validation:
+      schema_check: true
+
+  - id: tc005_risk_mitigation_strategy
+    description: "Generate risk mitigation testing strategy"
+    category: strategy
+    priority: medium
+
+    input:
+      identified_risks:
+        - "SQL injection in search"
+        - "XSS in user input"
+        - "CSRF on form submission"
+
+    expected_output:
+      must_contain:
+        - "strategy"
+        - "mitigation"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+success_criteria:
+  pass_rate: 0.9
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.75
+  max_execution_time_ms: 300000
+
+metadata:
+  author: "qe-risk-assessor"
+  created: "2026-02-02"
+  coverage_target: >
+    Risk-based testing with 5 test cases covering risk scoring,
+    test prioritization, coverage optimization, regression risk, and mitigation strategies.