agentic-qe 3.4.0 → 3.4.2

package/v3/assets/skills/verification-quality/schemas/output.json

@@ -0,0 +1,432 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-qe.dev/schemas/verification-quality-output.json",
+  "title": "Verification Quality Skill Output Schema",
+  "description": "Schema for verification quality output with quality score, verification coverage, and requirement traceability.",
+  "type": "object",
+  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
+  "properties": {
+    "skillName": {
+      "type": "string",
+      "const": "verification-quality",
+      "description": "Must be 'verification-quality'"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "status": {
+      "type": "string",
+      "enum": ["success", "partial", "failed", "skipped"]
+    },
+    "trustTier": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 3
+    },
+    "output": {
+      "type": "object",
+      "required": ["summary", "qualityScore", "verificationCoverage"],
+      "properties": {
+        "summary": {
+          "type": "string",
+          "minLength": 50,
+          "maxLength": 2000
+        },
+        "qualityScore": {
+          "$ref": "#/$defs/qualityScore"
+        },
+        "verificationCoverage": {
+          "$ref": "#/$defs/verificationCoverage"
+        },
+        "requirementTraceability": {
+          "$ref": "#/$defs/requirementTraceability"
+        },
+        "verificationMethods": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/verificationMethod"
+          }
+        },
+        "findings": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/finding"
+          },
+          "maxItems": 100
+        },
+        "recommendations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/recommendation"
+          },
+          "maxItems": 50
+        },
+        "metrics": {
+          "$ref": "#/$defs/metrics"
+        },
+        "artifacts": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/artifact"
+          }
+        }
+      }
+    },
+    "metadata": {
+      "$ref": "#/$defs/metadata"
+    },
+    "validation": {
+      "$ref": "#/$defs/validationResult"
+    },
+    "learning": {
+      "$ref": "#/$defs/learningData"
+    }
+  },
+  "$defs": {
+    "qualityScore": {
+      "type": "object",
+      "required": ["value", "max"],
+      "properties": {
+        "value": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100,
+          "description": "Overall verification quality score"
+        },
+        "max": {
+          "type": "number",
+          "const": 100
+        },
+        "grade": {
+          "type": "string",
+          "pattern": "^[A-F][+-]?$"
+        },
+        "trend": {
+          "type": "string",
+          "enum": ["improving", "stable", "declining", "unknown"]
+        },
+        "breakdown": {
+          "type": "object",
+          "properties": {
+            "completeness": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "correctness": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "consistency": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "traceability": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            }
+          }
+        }
+      }
+    },
+    "verificationCoverage": {
+      "type": "object",
+      "required": ["totalRequirements", "verifiedRequirements", "coveragePercentage"],
+      "properties": {
+        "totalRequirements": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "verifiedRequirements": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "coveragePercentage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100
+        },
+        "partiallyVerified": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "notVerified": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "byPriority": {
+          "type": "object",
+          "properties": {
+            "critical": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "high": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "medium": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            },
+            "low": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 100
+            }
+          }
+        },
+        "byComponent": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 100
+          }
+        }
+      }
+    },
+    "requirementTraceability": {
+      "type": "object",
+      "properties": {
+        "traceabilityScore": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100
+        },
+        "forwardLinks": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Requirements linked to tests"
+        },
+        "backwardLinks": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Tests linked to requirements"
+        },
+        "orphanTests": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Tests without requirement links"
+        },
+        "unlinkedRequirements": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Requirements without test links"
+        },
+        "matrix": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "requirementId": {
+                "type": "string"
+              },
+              "testIds": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "status": {
+                "type": "string",
+                "enum": ["verified", "partial", "not-verified", "blocked"]
+              }
+            }
+          }
+        }
+      }
+    },
+    "verificationMethod": {
+      "type": "object",
+      "required": ["name", "coverage"],
+      "properties": {
+        "name": {
+          "type": "string",
+          "enum": ["inspection", "analysis", "demonstration", "test", "review"]
+        },
+        "coverage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100
+        },
+        "requirementsCount": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "effectiveness": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 100
+        }
+      }
+    },
+    "finding": {
+      "type": "object",
+      "required": ["id", "title", "severity", "category"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^VER-\\d{3,6}$"
+        },
+        "title": {
+          "type": "string",
+          "minLength": 10,
+          "maxLength": 200
+        },
+        "description": {
+          "type": "string",
+          "maxLength": 2000
+        },
+        "severity": {
+          "type": "string",
+          "enum": ["critical", "high", "medium", "low", "info"]
+        },
+        "category": {
+          "type": "string",
+          "enum": ["coverage", "traceability", "completeness", "consistency", "correctness"]
+        },
+        "affectedRequirements": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "remediation": {
+          "type": "string"
+        }
+      }
+    },
+    "recommendation": {
+      "type": "object",
+      "required": ["id", "title", "priority"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^REC-\\d{3,6}$"
+        },
+        "title": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "priority": {
+          "type": "string",
+          "enum": ["critical", "high", "medium", "low"]
+        },
+        "effort": {
+          "type": "string",
+          "enum": ["trivial", "low", "medium", "high", "major"]
+        },
+        "impact": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 10
+        }
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "properties": {
+        "totalChecks": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "passed": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "failed": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "skipped": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "duration": {
+          "type": "integer",
+          "minimum": 0
+        }
+      }
+    },
+    "artifact": {
+      "type": "object",
+      "required": ["type", "path"],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["report", "matrix", "data", "log"]
+        },
+        "path": {
+          "type": "string"
+        },
+        "format": {
+          "type": "string"
+        }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "executionTimeMs": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "toolsUsed": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "agentId": {
+          "type": "string"
+        }
+      }
+    },
+    "validationResult": {
+      "type": "object",
+      "properties": {
+        "schemaValid": {
+          "type": "boolean"
+        },
+        "contentValid": {
+          "type": "boolean"
+        },
+        "confidence": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        }
+      }
+    },
+    "learningData": {
+      "type": "object",
+      "properties": {
+        "patternsDetected": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "reward": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        }
+      }
+    }
+  }
+}

package/v3/assets/skills/verification-quality/scripts/validate.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# =============================================================================
+# AQE Skill Validator: verification-quality v1.0.0
+# Validates quality verification and truth scoring output
+# =============================================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
+
+VALIDATOR_LIB=""
+for lib_path in "$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" "$SKILL_DIR/scripts/validator-lib.sh"; do
+  [[ -f "$lib_path" ]] && VALIDATOR_LIB="$lib_path" && break
+done
+
+[[ -n "$VALIDATOR_LIB" ]] && source "$VALIDATOR_LIB" || exit 1
+
+SKILL_NAME="verification-quality"
+SKILL_VERSION="1.0.0"
+REQUIRED_TOOLS=("jq")
+OPTIONAL_TOOLS=("ajv" "jsonschema" "python3")
+SCHEMA_PATH="$SKILL_DIR/schemas/output.json"
+
+REQUIRED_FIELDS=("skillName" "status" "output")
+MUST_CONTAIN_TERMS=("quality" "verification" "score")
+MUST_NOT_CONTAIN_TERMS=("TODO" "placeholder")
+ENUM_VALIDATIONS=(".status:success,partial,failed,skipped")
+
+OUTPUT_FILE=""
+SELF_TEST=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --self-test) SELF_TEST=true; shift ;;
+    --verbose|-v) export AQE_DEBUG=1; shift ;;
+    --json|--list-tools|--help|-h) shift ;;
+    -*) error "Unknown option: $1"; exit 1 ;;
+    *) OUTPUT_FILE="$1"; shift ;;
+  esac
+done
+
+if [[ "$SELF_TEST" == "true" ]]; then
+  self_test_passed=true
+  for tool in "${REQUIRED_TOOLS[@]}"; do
+    command_exists "$tool" && success "Required: $tool" || { error "Missing: $tool"; self_test_passed=false; }
+  done
+  [[ -f "$SCHEMA_PATH" ]] && validate_json "$SCHEMA_PATH" 2>/dev/null && success "Schema valid" || self_test_passed=false
+  [[ "$self_test_passed" == "true" ]] && exit 0 || exit 1
+fi
+
+validate_skill_specific() {
+  local output_file="$1"
+  local has_scores=$(json_get "$output_file" ".output.qualityScores" 2>/dev/null)
+  [[ -z "$has_scores" ]] || [[ "$has_scores" == "null" ]] && warn "No quality scores found"
+  return 0
+}
+
+main() {
+  [[ -z "$OUTPUT_FILE" ]] && error "No output file specified" && exit 1
+  [[ ! -f "$OUTPUT_FILE" ]] && error "Output file not found" && exit 1
+
+  echo "Validating $SKILL_NAME"
+
+  ! validate_tools && exit $EXIT_SKIP
+  ! validate_json "$OUTPUT_FILE" && exit $EXIT_FAIL
+  ! validate_schema "$OUTPUT_FILE" 2>/dev/null && exit $EXIT_FAIL
+  ! validate_required_fields "$OUTPUT_FILE" && exit $EXIT_FAIL
+  ! validate_content_terms "$OUTPUT_FILE" && exit $EXIT_FAIL
+  ! validate_skill_specific "$OUTPUT_FILE" && exit $EXIT_FAIL
+
+  success "Validation PASSED"
+  exit $EXIT_PASS
+}
+
+main

package/v3/assets/skills/visual-testing-advanced/SKILL.md

@@ -11,6 +11,12 @@ last_optimized: 2025-12-02
 dependencies: []
 quick_reference_card: true
 tags: [visual, regression, screenshot, pixel-diff, percy, playwright, responsive]
+trust_tier: 3
+validation:
+  schema_path: schemas/output.json
+  validator_path: scripts/validate.sh
+  eval_path: evals/visual-testing-advanced.yaml
+
 ---
 
 # Advanced Visual Testing

package/v3/assets/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml

@@ -0,0 +1,154 @@
+skill: visual-testing-advanced
+version: 1.0.0
+description: >
+  Evaluation suite for visual-testing-advanced skill.
+  Tests advanced visual regression testing with sophisticated diff detection,
+  region analysis, and cross-viewport consistency verification.
+
+models_to_test:
+  - claude-3.5-sonnet
+  - claude-3-haiku
+
+mcp_integration:
+  enabled: true
+  namespace: skill-validation
+  query_patterns: true
+  track_outcomes: true
+  store_patterns: true
+  share_learning: true
+  target_agents:
+    - qe-learning-coordinator
+
+learning:
+  store_success_patterns: true
+  pattern_ttl_days: 90
+  min_confidence_to_store: 0.7
+
+result_format:
+  json_output: true
+  include_timing: true
+
+setup:
+  required_tools:
+    - jq
+  environment_variables:
+    VISUAL_DIFF_THRESHOLD: "0.05"
+    MIN_REGION_SIZE: "10"
+
+test_cases:
+  - id: tc001_visual_diff_detection
+    description: "Visual differences are correctly detected"
+    category: core
+    priority: critical
+
+    input:
+      baselineImage: "baseline.png"
+      actualImage: "actual.png"
+      context:
+        diffThreshold: 0.05
+
+    expected_output:
+      must_contain:
+        - "diff"
+        - "detected"
+      must_not_contain:
+        - "error"
+
+    validation:
+      schema_check: true
+      keyword_match_threshold: 0.8
+
+  - id: tc002_region_analysis
+    description: "Changed regions are identified and analyzed"
+    category: core
+    priority: critical
+
+    input:
+      compareImages: true
+      analyzeRegions: true
+      context:
+        regionMinSize: 10
+
+    expected_output:
+      must_contain:
+        - "region"
+        - "change"
+
+    validation:
+      schema_check: true
+
+  - id: tc003_cross_viewport_consistency
+    description: "Visual consistency is verified across viewports"
+    category: core
+    priority: high
+
+    input:
+      viewports:
+        - "1920x1080"
+        - "1024x768"
+        - "375x667"
+      context:
+        compareAcrossViewports: true
+
+    expected_output:
+      must_contain:
+        - "viewport"
+        - "consistent"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+  - id: tc004_diff_percentage_calculation
+    description: "Diff percentage is accurately calculated"
+    category: core
+    priority: high
+
+    input:
+      totalPixels: 1000000
+      changedPixels: 5000
+      context:
+        calculateDiffPercentage: true
+
+    expected_output:
+      must_contain:
+        - "0.5"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+  - id: tc005_responsive_design_validation
+    description: "Responsive design changes are properly validated"
+    category: core
+    priority: high
+
+    input:
+      pages:
+        - "homepage"
+        - "product-page"
+      viewports: ["desktop", "tablet", "mobile"]
+      context:
+        validateResponsive: true
+
+    expected_output:
+      must_contain:
+        - "responsive"
+        - "layout"
+
+    validation:
+      schema_check: true
+      allow_partial: true
+
+success_criteria:
+  pass_rate: 0.8
+  critical_pass_rate: 1.0
+  avg_reasoning_quality: 0.7
+  max_execution_time_ms: 300000
+  cross_model_variance: 0.15
+
+metadata:
+  author: "qe-visual-specialist"
+  created: "2026-02-02"
+  last_updated: "2026-02-02"
+  coverage_target: "Diff detection, region analysis, viewport consistency, responsive design, pixel-level accuracy"