agentic-qe 3.6.2 → 3.6.4

package/v3/assets/skills/pentest-validation/scripts/validate.sh

@@ -1,402 +0,0 @@
-#!/bin/bash
-# =============================================================================
-# AQE Skill Validator: pentest-validation v1.0.0
-# Validates pentest validation skill output per ADR-056
-# =============================================================================
-#
-# This validator checks:
-# 1. JSON schema compliance (finding classifications, PoC structure, evidence)
-# 2. "No Exploit, No Report" filter enforcement
-# 3. Finding classification accuracy
-# 4. PoC quality and reproducibility
-# 5. Safeguard enforcement (production URL blocking, authorization)
-# 6. Cost tracking and budget compliance
-#
-# Usage: ./validate.sh <output-file> [options]
-#
-# Options:
-#   --self-test    Run validator self-test mode
-#   --verbose      Enable verbose output
-#   --json         Output results as JSON only
-#   --list-tools   Show available validation tools
-#   --help         Show this help message
-#
-# Exit Codes:
-#   0 - Validation passed
-#   1 - Validation failed
-#   2 - Validation skipped (missing required tools)
-#
-# =============================================================================
-
-set -euo pipefail
-
-# Get script directory
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Navigate to skill directory and project root
-# scripts/ -> pentest-validation/ -> skills/ -> .claude/ -> project root
-SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
-PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
-
-# Source validator library - check multiple locations
-VALIDATOR_LIB=""
-for lib_path in \
-  "$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" \
-  "$SKILL_DIR/scripts/validator-lib.sh" \
-  "$SCRIPT_DIR/validator-lib.sh"; do
-  if [[ -f "$lib_path" ]]; then
-    VALIDATOR_LIB="$lib_path"
-    break
-  fi
-done
-
-if [[ -n "$VALIDATOR_LIB" ]]; then
-  # shellcheck source=/dev/null
-  source "$VALIDATOR_LIB"
-else
-  echo "ERROR: Validator library not found"
-  echo "Searched:"
-  echo "  - $PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh"
-  echo "  - $SKILL_DIR/scripts/validator-lib.sh"
-  echo "  - $SCRIPT_DIR/validator-lib.sh"
-  exit 2
-fi
-
-# =============================================================================
-# Validator Configuration
-# =============================================================================
-
-SKILL_NAME="pentest-validation"
-SKILL_VERSION="1.0.0"
-SCHEMA_FILE="$SKILL_DIR/schemas/output.json"
-TRUST_TIER=3
-
-# =============================================================================
-# Self-Test Mode
-# =============================================================================
-
-run_self_test() {
-  local test_output
-  test_output=$(cat <<'SELF_TEST_JSON'
-{
-  "skillName": "pentest-validation",
-  "version": "1.0.0",
-  "timestamp": "2026-02-08T12:00:00Z",
-  "status": "success",
-  "trustTier": 3,
-  "output": {
-    "validationSummary": {
-      "findingsReceived": 8,
-      "confirmedExploitable": 2,
-      "likelyExploitable": 1,
-      "notExploitable": 4,
-      "inconclusive": 1,
-      "falsePositivesEliminated": 4,
-      "exploitationTierUsed": 2
-    },
-    "findings": [
-      {
-        "id": "VULN-001",
-        "type": "sql-injection",
-        "severity": "critical",
-        "classification": "confirmed-exploitable",
-        "location": "src/api/users.ts:45",
-        "exploitTier": 2,
-        "evidence": {
-          "payload": "' OR '1'='1",
-          "response": "200 OK with all user records",
-          "proof": "Payload returned all 150 user records instead of single user"
-        },
-        "poc": "curl -X GET 'https://staging.app.com/api/users?id=1%27%20OR%20%271%27%3D%271'",
-        "remediation": "Use parameterized queries: db.query('SELECT * FROM users WHERE id = ?', [id])",
-        "cwe": "CWE-89",
-        "owasp": "A03:2021"
-      },
-      {
-        "id": "VULN-003",
-        "type": "reflected-xss",
-        "severity": "high",
-        "classification": "confirmed-exploitable",
-        "location": "src/routes/profile.ts:12",
-        "exploitTier": 2,
-        "evidence": {
-          "payload": "<img src=x onerror=alert(1)>",
-          "response": "Payload reflected unescaped in HTML",
-          "proof": "Injected img tag executed onerror handler in response"
-        },
-        "poc": "curl 'https://staging.app.com/profile?name=%3Cimg%20src%3Dx%20onerror%3Dalert(1)%3E'",
-        "remediation": "Use template engine auto-escaping or explicit DOMPurify sanitization",
-        "cwe": "CWE-79",
-        "owasp": "A03:2021"
-      },
-      {
-        "id": "VULN-005",
-        "type": "idor",
-        "severity": "high",
-        "classification": "likely-exploitable",
-        "location": "src/api/orders.ts:23",
-        "exploitTier": 2,
-        "evidence": {
-          "proof": "User A token returned User B order data with sequential ID guessing"
-        },
-        "poc": "curl -H 'Authorization: Bearer <userA_token>' 'https://staging.app.com/api/orders/2'",
-        "remediation": "Add ownership check: if (order.userId !== req.user.id) return res.status(403)",
-        "cwe": "CWE-639",
-        "owasp": "A01:2021"
-      }
-    ],
-    "eliminatedFindings": [
-      {
-        "id": "VULN-002",
-        "type": "sql-injection",
-        "reason": "Parameterized query with integer parsing - not exploitable"
-      },
-      {
-        "id": "VULN-004",
-        "type": "xss",
-        "reason": "Output auto-escaped by React JSX - not exploitable"
-      },
-      {
-        "id": "VULN-006",
-        "type": "path-traversal",
-        "reason": "Path normalized and restricted to uploads/ directory"
-      },
-      {
-        "id": "VULN-007",
-        "type": "hardcoded-credentials",
-        "reason": "Test fixture file, not production code"
-      }
-    ],
-    "inconclusiveFindings": [
-      {
-        "id": "VULN-008",
-        "type": "ssrf",
-        "reason": "WAF blocked all internal URL attempts",
-        "manualSteps": "Test with WAF in learning mode or try DNS rebinding"
-      }
-    ],
-    "costBreakdown": {
-      "totalUsd": 4.82,
-      "tier1Cost": 0,
-      "tier2Cost": 1.20,
-      "tier3Cost": 3.62,
-      "budgetRemaining": 10.18,
-      "withinBudget": true
-    },
-    "playbookUpdates": 3
-  }
-}
-SELF_TEST_JSON
-)
-
-  echo "$test_output" > /tmp/pentest-validation-self-test.json
-  validate_output /tmp/pentest-validation-self-test.json
-  local result=$?
-  rm -f /tmp/pentest-validation-self-test.json
-  return $result
-}
-
-# =============================================================================
-# Validation Functions
-# =============================================================================
-
-validate_output() {
-  local output_file="$1"
-  local verbose="${VERBOSE:-false}"
-  local json_only="${JSON_ONLY:-false}"
-  local pass_count=0
-  local fail_count=0
-  local skip_count=0
-  local results=()
-
-  # Check file exists
-  if [[ ! -f "$output_file" ]]; then
-    echo "ERROR: Output file not found: $output_file"
-    return 1
-  fi
-
-  # Check jq is available
-  if ! command -v jq &>/dev/null; then
-    echo "SKIP: jq not available (required for JSON validation)"
-    return 2
-  fi
-
-  # Validate JSON syntax
-  if ! jq empty "$output_file" 2>/dev/null; then
-    results+=('{"check": "json_syntax", "status": "FAIL", "message": "Invalid JSON"}')
-    fail_count=$((fail_count + 1))
-  else
-    results+=('{"check": "json_syntax", "status": "PASS", "message": "Valid JSON"}')
-    pass_count=$((pass_count + 1))
-  fi
-
-  # Check required top-level fields
-  for field in skillName version timestamp status trustTier output; do
-    if jq -e ".$field" "$output_file" &>/dev/null; then
-      results+=("{\"check\": \"field_$field\", \"status\": \"PASS\", \"message\": \"Field $field present\"}")
-      pass_count=$((pass_count + 1))
-    else
-      results+=("{\"check\": \"field_$field\", \"status\": \"FAIL\", \"message\": \"Missing required field: $field\"}")
-      fail_count=$((fail_count + 1))
-    fi
-  done
-
-  # Check skillName matches
-  local skill_name
-  skill_name=$(jq -r '.skillName // ""' "$output_file")
-  if [[ "$skill_name" == "pentest-validation" ]]; then
-    results+=('{"check": "skill_name", "status": "PASS", "message": "Correct skill name"}')
-    pass_count=$((pass_count + 1))
-  else
-    results+=("{\"check\": \"skill_name\", \"status\": \"FAIL\", \"message\": \"Expected pentest-validation, got $skill_name\"}")
-    fail_count=$((fail_count + 1))
-  fi
-
-  # Check trustTier is 3
-  local trust_tier
-  trust_tier=$(jq -r '.trustTier // 0' "$output_file")
-  if [[ "$trust_tier" == "3" ]]; then
-    results+=('{"check": "trust_tier", "status": "PASS", "message": "Trust tier 3"}')
-    pass_count=$((pass_count + 1))
-  else
-    results+=("{\"check\": \"trust_tier\", \"status\": \"FAIL\", \"message\": \"Expected trust tier 3, got $trust_tier\"}")
-    fail_count=$((fail_count + 1))
-  fi
-
-  # Check validation summary exists
-  if jq -e '.output.validationSummary' "$output_file" &>/dev/null; then
-    results+=('{"check": "validation_summary", "status": "PASS", "message": "Validation summary present"}')
-    pass_count=$((pass_count + 1))
-
-    # Check "No Exploit, No Report" filter
-    local confirmed
-    local not_exploitable
-    confirmed=$(jq -r '.output.validationSummary.confirmedExploitable // 0' "$output_file")
-    not_exploitable=$(jq -r '.output.validationSummary.notExploitable // 0' "$output_file")
-    local reported_count
-    reported_count=$(jq -r '.output.findings | length' "$output_file")
-
-    # Reported findings should only be confirmed or likely
-    local invalid_classifications
-    invalid_classifications=$(jq '[.output.findings[] | select(.classification != "confirmed-exploitable" and .classification != "likely-exploitable")] | length' "$output_file" 2>/dev/null || echo "0")
-    if [[ "$invalid_classifications" == "0" ]]; then
-      results+=('{"check": "no_exploit_no_report", "status": "PASS", "message": "Only confirmed/likely findings in output"}')
-      pass_count=$((pass_count + 1))
-    else
-      results+=("{\"check\": \"no_exploit_no_report\", \"status\": \"FAIL\", \"message\": \"$invalid_classifications findings with wrong classification in output\"}")
-      fail_count=$((fail_count + 1))
-    fi
-  else
-    results+=('{"check": "validation_summary", "status": "FAIL", "message": "Missing validation summary"}')
-    fail_count=$((fail_count + 1))
-  fi
-
-  # Check PoC exists for confirmed findings
-  local confirmed_without_poc
-  confirmed_without_poc=$(jq '[.output.findings[] | select(.classification == "confirmed-exploitable" and (.poc == null or .poc == ""))] | length' "$output_file" 2>/dev/null || echo "0")
-  if [[ "$confirmed_without_poc" == "0" ]]; then
-    results+=('{"check": "poc_coverage", "status": "PASS", "message": "All confirmed findings have PoC"}')
-    pass_count=$((pass_count + 1))
-  else
-    results+=("{\"check\": \"poc_coverage\", \"status\": \"FAIL\", \"message\": \"$confirmed_without_poc confirmed findings missing PoC\"}")
-    fail_count=$((fail_count + 1))
-  fi
-
-  # Check evidence exists for all findings
-  local findings_without_evidence
-  findings_without_evidence=$(jq '[.output.findings[] | select(.evidence == null or .evidence.proof == null or .evidence.proof == "")] | length' "$output_file" 2>/dev/null || echo "0")
-  if [[ "$findings_without_evidence" == "0" ]]; then
-    results+=('{"check": "evidence_coverage", "status": "PASS", "message": "All findings have evidence"}')
-    pass_count=$((pass_count + 1))
-  else
-    results+=("{\"check\": \"evidence_coverage\", \"status\": \"FAIL\", \"message\": \"$findings_without_evidence findings missing evidence\"}")
-    fail_count=$((fail_count + 1))
-  fi
-
-  # Check cost tracking
-  if jq -e '.output.costBreakdown.withinBudget' "$output_file" &>/dev/null; then
-    local within_budget
-    within_budget=$(jq -r '.output.costBreakdown.withinBudget' "$output_file")
-    if [[ "$within_budget" == "true" ]]; then
-      results+=('{"check": "budget_compliance", "status": "PASS", "message": "Within budget"}')
-      pass_count=$((pass_count + 1))
-    else
-      results+=('{"check": "budget_compliance", "status": "FAIL", "message": "Over budget"}')
-      fail_count=$((fail_count + 1))
-    fi
-  else
-    results+=('{"check": "budget_compliance", "status": "SKIP", "message": "No cost breakdown"}')
-    skip_count=$((skip_count + 1))
-  fi
-
-  # Output results
-  local total=$((pass_count + fail_count + skip_count))
-  if [[ "$json_only" == "true" ]]; then
-    echo "{\"skill\": \"$SKILL_NAME\", \"version\": \"$SKILL_VERSION\", \"trustTier\": $TRUST_TIER, \"pass\": $pass_count, \"fail\": $fail_count, \"skip\": $skip_count, \"total\": $total, \"results\": [$(IFS=,; echo "${results[*]}")]}"
-  else
-    echo "=== Pentest Validation Skill Validator ==="
-    echo "Skill: $SKILL_NAME v$SKILL_VERSION (Trust Tier $TRUST_TIER)"
-    echo "---"
-    echo "PASS: $pass_count | FAIL: $fail_count | SKIP: $skip_count | TOTAL: $total"
-    echo "---"
-    if [[ "$verbose" == "true" ]]; then
-      for result in "${results[@]}"; do
-        local status
-        local check
-        local message
-        status=$(echo "$result" | jq -r '.status')
-        check=$(echo "$result" | jq -r '.check')
-        message=$(echo "$result" | jq -r '.message')
-        echo "  [$status] $check: $message"
-      done
-    fi
-    echo "---"
-    if [[ $fail_count -eq 0 ]]; then
-      echo "RESULT: PASS"
-    else
-      echo "RESULT: FAIL ($fail_count checks failed)"
-    fi
-  fi
-
-  [[ $fail_count -eq 0 ]]
-}
-
-# =============================================================================
-# Main
-# =============================================================================
-
-main() {
-  case "${1:-}" in
-    --self-test)
-      echo "Running self-test..."
-      run_self_test
-      ;;
-    --help)
-      head -27 "$0" | tail -20
-      ;;
-    --list-tools)
-      echo "Required: jq"
-      echo "Optional: curl (for live endpoint testing)"
-      ;;
-    *)
-      if [[ -z "${1:-}" ]]; then
-        echo "Usage: $0 <output-file> [--verbose] [--json] [--self-test]"
-        exit 1
-      fi
-
-      # Parse options
-      export VERBOSE="false"
-      export JSON_ONLY="false"
-      for arg in "${@:2}"; do
-        case "$arg" in
-          --verbose) VERBOSE="true" ;;
-          --json) JSON_ONLY="true" ;;
-        esac
-      done
-
-      validate_output "$1"
-      ;;
-  esac
-}
-
-main "$@"