agentic-qe 3.4.1 → 3.4.2

package/v3/assets/skills/test-reporting-analytics/schemas/output.json

@@ -0,0 +1,329 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-qe.dev/schemas/skills/test-reporting-analytics/output.json",
+  "title": "Test Reporting Analytics Skill Output Schema",
+  "description": "Schema for test-reporting-analytics skill output. Advanced test reporting with quality dashboards and analytics.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "test-reporting-analytics"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "const": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "reports", "metrics", "completeness"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 50,
+          "maxLength": 3000
+        },
+        "reports": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/report"
+          },
+          "minItems": 1,
+          "maxItems": 50
+        },
+        "metrics": {
+          "$ref": "#/$defs/testMetrics"
+        },
+        "completeness": {
+          "$ref": "#/$defs/completeness"
+        },
+        "trends": {
+          "$ref": "#/$defs/trends"
+        },
+        "qualityGates": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/qualityGate"
+          }
+        },
+        "dashboards": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/dashboard"
+          }
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/recommendation"
+          },
+          "maxItems": 50
+        },
+        "documentStructure": {
+          "$ref": "#/$defs/documentStructure"
+        },
+        "clarity": {
+          "$ref": "#/$defs/clarity"
+        }
+      }
+    },
+    "metadata": {
+      "$ref": "#/$defs/metadata"
+    },
+    "validation": {
+      "$ref": "#/$defs/validationResult"
+    },
+    "learning": {
+      "$ref": "#/$defs/learningData"
+    }
+  },
+  "$defs": {
+    "report": {
+      "type": "object",
+      "required": ["id", "title", "type", "content"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^RPT-\\d{3,6}$"
+        },
+        "title": {
+          "type": "string",
+          "maxLength": 200
+        },
+        "type": {
+          "type": "string",
+          "enum": ["summary", "detailed", "executive", "trend", "coverage", "defect", "performance", "comparison"]
+        },
+        "content": {
+          "type": "string",
+          "maxLength": 50000
+        },
+        "format": {
+          "type": "string",
+          "enum": ["markdown", "html", "json", "csv"]
+        },
+        "audience": {
+          "type": "string",
+          "enum": ["executive", "management", "technical", "qa", "all"]
+        },
+        "period": {
+          "type": "object",
+          "properties": {
+            "start": { "type": "string", "format": "date" },
+            "end": { "type": "string", "format": "date" }
+          }
+        },
+        "keyFindings": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "visualizations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/visualization"
+          }
+        }
+      }
+    },
+    "testMetrics": {
+      "type": "object",
+      "properties": {
+        "totalTests": { "type": "integer", "minimum": 0 },
+        "passedTests": { "type": "integer", "minimum": 0 },
+        "failedTests": { "type": "integer", "minimum": 0 },
+        "skippedTests": { "type": "integer", "minimum": 0 },
+        "flakyTests": { "type": "integer", "minimum": 0 },
+        "passRate": { "type": "number", "minimum": 0, "maximum": 100 },
+        "coverage": {
+          "type": "object",
+          "properties": {
+            "line": { "type": "number", "minimum": 0, "maximum": 100 },
+            "branch": { "type": "number", "minimum": 0, "maximum": 100 },
+            "function": { "type": "number", "minimum": 0, "maximum": 100 },
+            "statement": { "type": "number", "minimum": 0, "maximum": 100 }
+          }
+        },
+        "executionTime": {
+          "type": "object",
+          "properties": {
+            "total": { "type": "integer", "minimum": 0 },
+            "average": { "type": "number", "minimum": 0 },
+            "p95": { "type": "number", "minimum": 0 }
+          }
+        },
+        "defects": {
+          "type": "object",
+          "properties": {
+            "open": { "type": "integer", "minimum": 0 },
+            "closed": { "type": "integer", "minimum": 0 },
+            "critical": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "automationRate": { "type": "number", "minimum": 0, "maximum": 100 }
+      }
+    },
+    "completeness": {
+      "type": "object",
+      "required": ["score"],
+      "properties": {
+        "score": { "type": "number", "minimum": 0, "maximum": 100 },
+        "dataCompleteness": { "type": "number", "minimum": 0, "maximum": 100 },
+        "reportCoverage": { "type": "number", "minimum": 0, "maximum": 100 },
+        "missingData": { "type": "array", "items": { "type": "string" } },
+        "dataQuality": { "type": "string", "enum": ["excellent", "good", "fair", "poor"] }
+      }
+    },
+    "trends": {
+      "type": "object",
+      "properties": {
+        "passRateTrend": {
+          "type": "string",
+          "enum": ["improving", "stable", "declining"]
+        },
+        "coverageTrend": {
+          "type": "string",
+          "enum": ["improving", "stable", "declining"]
+        },
+        "defectTrend": {
+          "type": "string",
+          "enum": ["improving", "stable", "worsening"]
+        },
+        "velocityTrend": {
+          "type": "string",
+          "enum": ["improving", "stable", "declining"]
+        },
+        "historicalData": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "date": { "type": "string", "format": "date" },
+              "passRate": { "type": "number" },
+              "coverage": { "type": "number" },
+              "defectCount": { "type": "integer" }
+            }
+          }
+        },
+        "predictions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "metric": { "type": "string" },
+              "prediction": { "type": "string" },
+              "confidence": { "type": "number" }
+            }
+          }
+        }
+      }
+    },
+    "qualityGate": {
+      "type": "object",
+      "required": ["name", "passed", "threshold", "actual"],
+      "properties": {
+        "name": { "type": "string" },
+        "passed": { "type": "boolean" },
+        "threshold": { "type": "number" },
+        "actual": { "type": "number" },
+        "status": { "type": "string", "enum": ["pass", "fail", "warn"] },
+        "message": { "type": "string" }
+      }
+    },
+    "dashboard": {
+      "type": "object",
+      "required": ["name", "widgets"],
+      "properties": {
+        "name": { "type": "string" },
+        "description": { "type": "string" },
+        "audience": { "type": "string" },
+        "widgets": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "type": { "type": "string", "enum": ["chart", "metric", "table", "status", "trend"] },
+              "title": { "type": "string" },
+              "data": { "type": "object" }
+            }
+          }
+        }
+      }
+    },
+    "visualization": {
+      "type": "object",
+      "required": ["type", "title"],
+      "properties": {
+        "type": { "type": "string", "enum": ["bar", "line", "pie", "heatmap", "table", "gauge"] },
+        "title": { "type": "string" },
+        "data": { "type": "object" },
+        "config": { "type": "object" }
+      }
+    },
+    "documentStructure": {
+      "type": "object",
+      "properties": {
+        "type": { "type": "string" },
+        "sections": { "type": "array", "items": { "type": "string" } },
+        "hasExecutiveSummary": { "type": "boolean" },
+        "hasVisualizations": { "type": "boolean" }
+      }
+    },
+    "clarity": {
+      "type": "object",
+      "properties": {
+        "score": { "type": "number", "minimum": 0, "maximum": 100 },
+        "dataVisualizationClarity": { "type": "number", "minimum": 0, "maximum": 100 },
+        "insightClarity": { "type": "number", "minimum": 0, "maximum": 100 }
+      }
+    },
+    "recommendation": {
+      "type": "object",
+      "required": ["id", "title", "priority"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^REC-\\d{3,6}$" },
+        "title": { "type": "string", "maxLength": 200 },
+        "description": { "type": "string", "maxLength": 2000 },
+        "priority": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
+        "category": { "type": "string", "enum": ["coverage", "quality", "process", "tooling", "reporting"] },
+        "expectedImpact": { "type": "string" }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": { "type": "integer", "minimum": 0 },
+        "toolsUsed": { "type": "array", "items": { "type": "string" } },
+        "agentId": { "type": "string", "pattern": "^qe-[a-z][a-z0-9-]*$" },
+        "dataSource": { "type": "string" },
+        "reportingPeriod": { "type": "string" }
+      }
+    },
+    "validationResult": {
+      "type": "object",
+      "properties": {
+        "schemaValid": { "type": "boolean" },
+        "contentValid": { "type": "boolean" },
+        "confidence": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "learningData": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": { "type": "array", "items": { "type": "string" } },
+        "reward": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    }
+  }
+}

package/v3/assets/skills/test-reporting-analytics/scripts/validate.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# =============================================================================
+# AQE Skill Validator: test-reporting-analytics v1.0.0
+# Validates test reporting and analytics output
+# =============================================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
+
+VALIDATOR_LIB=""
+for lib_path in "$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" "$SKILL_DIR/scripts/validator-lib.sh"; do
+  [[ -f "$lib_path" ]] && VALIDATOR_LIB="$lib_path" && break
+done
+
+[[ -n "$VALIDATOR_LIB" ]] && source "$VALIDATOR_LIB" || exit 1
+
+SKILL_NAME="test-reporting-analytics"
+SKILL_VERSION="1.0.0"
+REQUIRED_TOOLS=("jq")
+OPTIONAL_TOOLS=("ajv" "jsonschema" "python3")
+SCHEMA_PATH="$SKILL_DIR/schemas/output.json"
+
+REQUIRED_FIELDS=("skillName" "status" "output")
+MUST_CONTAIN_TERMS=("report" "analytics" "test")
+MUST_NOT_CONTAIN_TERMS=("TODO" "placeholder")
+ENUM_VALIDATIONS=(".status:success,partial,failed,skipped")
+
+OUTPUT_FILE=""
+SELF_TEST=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --self-test) SELF_TEST=true; shift ;;
+    --verbose|-v) export AQE_DEBUG=1; shift ;;
+    --json|--list-tools|--help|-h) shift ;;
+    -*) error "Unknown option: $1"; exit 1 ;;
+    *) OUTPUT_FILE="$1"; shift ;;
+  esac
+done
+
+if [[ "$SELF_TEST" == "true" ]]; then
+  self_test_passed=true
+  for tool in "${REQUIRED_TOOLS[@]}"; do
+    command_exists "$tool" && success "Required: $tool" || { error "Missing: $tool"; self_test_passed=false; }
+  done
+  [[ -f "$SCHEMA_PATH" ]] && validate_json "$SCHEMA_PATH" 2>/dev/null && success "Schema valid" || self_test_passed=false
+  [[ "$self_test_passed" == "true" ]] && exit 0 || exit 1
+fi
+
+validate_skill_specific() {
+  local output_file="$1"
+  local has_metrics=$(json_get "$output_file" ".output.metrics" 2>/dev/null)
+  [[ -z "$has_metrics" ]] || [[ "$has_metrics" == "null" ]] && warn "No metrics found"
+  return 0
+}
+
+main() {
+  [[ -z "$OUTPUT_FILE" ]] && error "No output file specified" && exit 1
+  [[ ! -f "$OUTPUT_FILE" ]] && error "Output file not found" && exit 1
+
+  echo "Validating $SKILL_NAME"
+
+  ! validate_tools && exit $EXIT_SKIP
+  ! validate_json "$OUTPUT_FILE" && exit $EXIT_FAIL
+  ! validate_schema "$OUTPUT_FILE" 2>/dev/null && exit $EXIT_FAIL
+  ! validate_required_fields "$OUTPUT_FILE" && exit $EXIT_FAIL
+  ! validate_content_terms "$OUTPUT_FILE" && exit $EXIT_FAIL
+  ! validate_skill_specific "$OUTPUT_FILE" && exit $EXIT_FAIL
+
+  success "Validation PASSED"
+  exit $EXIT_PASS
+}
+
+main

package/v3/assets/skills/verification-quality/SKILL.md

@@ -4,6 +4,12 @@ description: "Comprehensive truth scoring, code quality verification, and automa
 version: "2.0.0"
 category: "quality-assurance"
 tags: ["verification", "truth-scoring", "quality", "rollback", "metrics", "ci-cd"]
+trust_tier: 3
+validation:
+  schema_path: schemas/output.json
+  validator_path: scripts/validate.sh
+  eval_path: evals/verification-quality.yaml
+
 ---
 
 # Verification & Quality Assurance Skill

package/v3/assets/skills/verification-quality/evals/verification-quality.yaml

@@ -0,0 +1,150 @@
+skill: verification-quality
+version: 1.0.0
+description: >
+  Evaluation suite for verification-quality skill.
+  Tests comprehensive quality verification including truth scoring,
+  code quality analysis, and comprehensive quality assessment.
+
+models_to_test:
+  - claude-3.5-sonnet
+  - claude-3-haiku
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  share_learning: true
+  target_agents:
+    - qe-learning-coordinator
+
+learning:
+  store_success_patterns: true
+  pattern_ttl_days: 90
+  min_confidence_to_store: 0.7
+
+result_format:
+  json_output: true
+  include_timing: true
+
+setup:
+  required_tools:
+    - jq
+
+test_cases:
+  - id: tc001_quality_scoring
+    description: "Skill generates comprehensive quality score"
+    category: core
+    priority: critical
+
+    input:
+      codeContent: "function add(a, b) { return a + b; }"
+      context:
+        assessmentScope: "comprehensive"
+
+    expected_output:
+      must_contain:
+        - "quality"
+        - "score"
+      must_not_contain:
+        - "error"
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_truth_scoring
+    description: "Truth scoring correctly evaluates statement validity"
+    category: core
+    priority: critical
+
+    input:
+      statements:
+        - "Function returns correct result"
+        - "Code is optimized"
+      context:
+        evaluateTruth: true
+
+    expected_output:
+      must_contain:
+        - "truth"
+        - "score"
+
+    validation:
+      schema_check: true
+
+  - id: tc003_code_quality_dimensions
+    description: "Quality assessment covers multiple dimensions"
+    category: core
+    priority: high
+
+    input:
+      codeMetrics:
+        readability: 0.8
+        maintainability: 0.7
+        testability: 0.75
+      context:
+        assessmentDimensions: ["readability", "maintainability", "testability"]
+
+    expected_output:
+      must_contain:
+        - "readability"
+        - "maintainability"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+  - id: tc004_issue_identification
+    description: "Quality issues are properly identified and categorized"
+    category: core
+    priority: high
+
+    input:
+      code: "var x = 1; // poor variable naming"
+      context:
+        findIssues: true
+
+    expected_output:
+      must_contain:
+        - "issue"
+        - "quality"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+  - id: tc005_improvement_recommendations
+    description: "Actionable improvement recommendations are provided"
+    category: core
+    priority: high
+
+    input:
+      qualityGaps:
+        - "Low test coverage"
+        - "Complex functions"
+      context:
+        provideRecommendations: true
+
+    expected_output:
+      must_contain:
+        - "recommend"
+        - "improve"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+success_criteria:
+  pass_rate: 0.8
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.7
+  max_execution_time_ms: 300000
+  cross_model_variance: 0.15
+
+metadata:
+  author: "qe-quality-specialist"
+  created: "2026-02-02"
+  last_updated: "2026-02-02"
+  coverage_target: "Quality scoring, truth evaluation, code metrics, issue identification, recommendations"