agentic-qe 3.4.0 → 3.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/.validation/README.md +111 -0
- package/.claude/skills/.validation/examples/chaos-engineering-output.example.json +530 -0
- package/.claude/skills/.validation/examples/performance-testing-output.example.json +252 -0
- package/.claude/skills/.validation/examples/security-testing-output.example.json +413 -0
- package/.claude/skills/.validation/examples/testability-scoring-output.example.json +350 -0
- package/.claude/skills/.validation/schemas/skill-eval.schema.json +452 -0
- package/.claude/skills/.validation/schemas/skill-frontmatter.schema.json +341 -0
- package/.claude/skills/.validation/schemas/skill-output-meta.schema.json +199 -0
- package/.claude/skills/.validation/schemas/skill-output.template.json +610 -0
- package/.claude/skills/.validation/skill-validation-mcp-integration.md +250 -0
- package/.claude/skills/.validation/templates/eval.template.yaml +366 -0
- package/.claude/skills/.validation/templates/schemas/output.json +145 -0
- package/.claude/skills/.validation/templates/security-testing-eval.template.yaml +725 -0
- package/.claude/skills/.validation/templates/skill-frontmatter.example.yaml +225 -0
- package/.claude/skills/.validation/templates/validate.template.sh +1060 -0
- package/.claude/skills/.validation/templates/validator-lib.sh +1031 -0
- package/.claude/skills/.validation/test-data/invalid-output.json +5 -0
- package/.claude/skills/.validation/test-data/minimal-output.json +9 -0
- package/.claude/skills/.validation/test-data/sample-output.json +73 -0
- package/.claude/skills/TRUST-TIERS.md +158 -0
- package/.claude/skills/a11y-ally/SKILL.md +1664 -1658
- package/.claude/skills/a11y-ally/evals/a11y-ally.yaml +376 -0
- package/.claude/skills/a11y-ally/schemas/output.json +549 -0
- package/.claude/skills/a11y-ally/scripts/validate.sh +690 -0
- package/.claude/skills/accessibility-testing/SKILL.md +8 -1
- package/.claude/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -0
- package/.claude/skills/accessibility-testing/schemas/output.json +776 -0
- package/.claude/skills/accessibility-testing/scripts/validate.sh +943 -0
- package/.claude/skills/accessibility-testing/test-data/sample-output.json +191 -0
- package/.claude/skills/agentic-quality-engineering/SKILL.md +4 -0
- package/.claude/skills/agentic-quality-engineering/schemas/output.json +577 -0
- package/.claude/skills/api-testing-patterns/SKILL.md +5 -0
- package/.claude/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -0
- package/.claude/skills/api-testing-patterns/schemas/output.json +845 -0
- package/.claude/skills/api-testing-patterns/scripts/validate.sh +931 -0
- package/.claude/skills/aqe-v2-v3-migration/schemas/output.json +513 -0
- package/.claude/skills/aqe-v2-v3-migration/skill.md +4 -0
- package/.claude/skills/brutal-honesty-review/SKILL.md +5 -0
- package/.claude/skills/brutal-honesty-review/schemas/output.json +291 -0
- package/.claude/skills/brutal-honesty-review/scripts/validate.sh +130 -0
- package/.claude/skills/bug-reporting-excellence/SKILL.md +5 -0
- package/.claude/skills/bug-reporting-excellence/schemas/output.json +288 -0
- package/.claude/skills/bug-reporting-excellence/scripts/validate.sh +108 -0
- package/.claude/skills/chaos-engineering-resilience/SKILL.md +5 -0
- package/.claude/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -0
- package/.claude/skills/chaos-engineering-resilience/schemas/output.json +1205 -0
- package/.claude/skills/chaos-engineering-resilience/scripts/validate.sh +1107 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/SKILL.md +6 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/scripts/validate.sh +310 -0
- package/.claude/skills/code-review-quality/SKILL.md +5 -0
- package/.claude/skills/code-review-quality/schemas/output.json +264 -0
- package/.claude/skills/code-review-quality/scripts/validate.sh +113 -0
- package/.claude/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -0
- package/.claude/skills/compatibility-testing/schemas/output.json +551 -0
- package/.claude/skills/compatibility-testing/scripts/validate.sh +605 -0
- package/.claude/skills/compliance-testing/SKILL.md +7 -0
- package/.claude/skills/compliance-testing/evals/compliance-testing.yaml +1107 -0
- package/.claude/skills/compliance-testing/schemas/output.json +845 -0
- package/.claude/skills/compliance-testing/scripts/validate.sh +888 -0
- package/.claude/skills/consultancy-practices/SKILL.md +4 -0
- package/.claude/skills/consultancy-practices/schemas/output.json +282 -0
- package/.claude/skills/context-driven-testing/SKILL.md +3 -0
- package/.claude/skills/contract-testing/SKILL.md +5 -0
- package/.claude/skills/contract-testing/evals/contract-testing.yaml +748 -0
- package/.claude/skills/contract-testing/schemas/output.json +638 -0
- package/.claude/skills/contract-testing/scripts/validate.sh +1033 -0
- package/.claude/skills/database-testing/SKILL.md +5 -0
- package/.claude/skills/database-testing/evals/database-testing.yaml +968 -0
- package/.claude/skills/database-testing/schemas/output.json +1446 -0
- package/.claude/skills/database-testing/scripts/validate.sh +1099 -0
- package/.claude/skills/exploratory-testing-advanced/SKILL.md +3 -0
- package/.claude/skills/holistic-testing-pact/SKILL.md +3 -0
- package/.claude/skills/localization-testing/SKILL.md +6 -0
- package/.claude/skills/localization-testing/evals/localization-testing.yaml +544 -0
- package/.claude/skills/localization-testing/schemas/output.json +325 -0
- package/.claude/skills/localization-testing/scripts/validate.sh +131 -0
- package/.claude/skills/mobile-testing/SKILL.md +6 -0
- package/.claude/skills/mobile-testing/evals/mobile-testing.yaml +537 -0
- package/.claude/skills/mobile-testing/schemas/output.json +318 -0
- package/.claude/skills/mobile-testing/scripts/validate.sh +127 -0
- package/.claude/skills/mutation-testing/SKILL.md +8 -1
- package/.claude/skills/mutation-testing/evals/mutation-testing.yaml +652 -0
- package/.claude/skills/mutation-testing/schemas/output.json +707 -0
- package/.claude/skills/mutation-testing/scripts/validate.sh +992 -0
- package/.claude/skills/mutation-testing/test-data/sample-output.json +295 -0
- package/.claude/skills/n8n-expression-testing/SKILL.md +6 -0
- package/.claude/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -0
- package/.claude/skills/n8n-expression-testing/schemas/output.json +369 -0
- package/.claude/skills/n8n-expression-testing/scripts/validate.sh +162 -0
- package/.claude/skills/n8n-integration-testing-patterns/SKILL.md +6 -0
- package/.claude/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -0
- package/.claude/skills/n8n-integration-testing-patterns/schemas/output.json +291 -0
- package/.claude/skills/n8n-integration-testing-patterns/scripts/validate.sh +104 -0
- package/.claude/skills/n8n-security-testing/SKILL.md +6 -0
- package/.claude/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -0
- package/.claude/skills/n8n-security-testing/schemas/output.json +293 -0
- package/.claude/skills/n8n-security-testing/scripts/validate.sh +120 -0
- package/.claude/skills/n8n-trigger-testing-strategies/SKILL.md +6 -0
- package/.claude/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -0
- package/.claude/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -0
- package/.claude/skills/n8n-trigger-testing-strategies/scripts/validate.sh +121 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/SKILL.md +6 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/scripts/validate.sh +126 -0
- package/.claude/skills/performance-analysis/SKILL.md +6 -0
- package/.claude/skills/performance-analysis/evals/performance-analysis.yaml +144 -0
- package/.claude/skills/performance-analysis/schemas/output.json +588 -0
- package/.claude/skills/performance-analysis/scripts/validate.sh +63 -0
- package/.claude/skills/performance-testing/SKILL.md +5 -0
- package/.claude/skills/performance-testing/evals/performance-testing.yaml +772 -0
- package/.claude/skills/performance-testing/schemas/output.json +1184 -0
- package/.claude/skills/performance-testing/scripts/validate.sh +1090 -0
- package/.claude/skills/qcsd-ideation-swarm/SKILL.md +1759 -1753
- package/.claude/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -0
- package/.claude/skills/qcsd-ideation-swarm/schemas/output.json +568 -0
- package/.claude/skills/qcsd-ideation-swarm/scripts/validate.sh +56 -0
- package/.claude/skills/qe-chaos-resilience/SKILL.md +6 -0
- package/.claude/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -0
- package/.claude/skills/qe-chaos-resilience/schemas/output.json +314 -0
- package/.claude/skills/qe-chaos-resilience/scripts/validate.sh +401 -0
- package/.claude/skills/qe-code-intelligence/SKILL.md +6 -0
- package/.claude/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -0
- package/.claude/skills/qe-code-intelligence/schemas/output.json +315 -0
- package/.claude/skills/qe-code-intelligence/scripts/validate.sh +408 -0
- package/.claude/skills/qe-contract-testing/SKILL.md +6 -0
- package/.claude/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -0
- package/.claude/skills/qe-contract-testing/schemas/output.json +295 -0
- package/.claude/skills/qe-contract-testing/scripts/validate.sh +421 -0
- package/.claude/skills/qe-coverage-analysis/SKILL.md +6 -0
- package/.claude/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -0
- package/.claude/skills/qe-coverage-analysis/schemas/output.json +286 -0
- package/.claude/skills/qe-coverage-analysis/scripts/validate.sh +453 -0
- package/.claude/skills/qe-defect-intelligence/SKILL.md +6 -0
- package/.claude/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -0
- package/.claude/skills/qe-defect-intelligence/schemas/output.json +283 -0
- package/.claude/skills/qe-defect-intelligence/scripts/validate.sh +466 -0
- package/.claude/skills/qe-iterative-loop/SKILL.md +445 -440
- package/.claude/skills/qe-iterative-loop/schemas/output.json +264 -0
- package/.claude/skills/qe-iterative-loop/scripts/validate.sh +474 -0
- package/.claude/skills/qe-learning-optimization/SKILL.md +6 -0
- package/.claude/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -0
- package/.claude/skills/qe-learning-optimization/schemas/output.json +288 -0
- package/.claude/skills/qe-learning-optimization/scripts/validate.sh +56 -0
- package/.claude/skills/qe-quality-assessment/SKILL.md +6 -0
- package/.claude/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -0
- package/.claude/skills/qe-quality-assessment/schemas/output.json +550 -0
- package/.claude/skills/qe-quality-assessment/scripts/validate.sh +352 -0
- package/.claude/skills/qe-requirements-validation/SKILL.md +6 -0
- package/.claude/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -0
- package/.claude/skills/qe-requirements-validation/schemas/output.json +587 -0
- package/.claude/skills/qe-requirements-validation/scripts/validate.sh +275 -0
- package/.claude/skills/qe-security-compliance/SKILL.md +6 -0
- package/.claude/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -0
- package/.claude/skills/qe-security-compliance/schemas/output.json +498 -0
- package/.claude/skills/qe-security-compliance/scripts/validate.sh +305 -0
- package/.claude/skills/qe-test-execution/SKILL.md +6 -0
- package/.claude/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -0
- package/.claude/skills/qe-test-execution/schemas/output.json +529 -0
- package/.claude/skills/qe-test-execution/scripts/validate.sh +323 -0
- package/.claude/skills/qe-test-generation/SKILL.md +6 -0
- package/.claude/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -0
- package/.claude/skills/qe-test-generation/schemas/output.json +439 -0
- package/.claude/skills/qe-test-generation/scripts/validate.sh +56 -0
- package/.claude/skills/qe-visual-accessibility/SKILL.md +6 -0
- package/.claude/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -0
- package/.claude/skills/qe-visual-accessibility/schemas/output.json +491 -0
- package/.claude/skills/qe-visual-accessibility/scripts/validate.sh +56 -0
- package/.claude/skills/quality-metrics/SKILL.md +6 -0
- package/.claude/skills/quality-metrics/evals/quality-metrics.yaml +494 -0
- package/.claude/skills/quality-metrics/schemas/output.json +403 -0
- package/.claude/skills/quality-metrics/scripts/validate.sh +434 -0
- package/.claude/skills/refactoring-patterns/SKILL.md +5 -0
- package/.claude/skills/refactoring-patterns/schemas/output.json +475 -0
- package/.claude/skills/refactoring-patterns/scripts/validate.sh +241 -0
- package/.claude/skills/regression-testing/SKILL.md +6 -0
- package/.claude/skills/regression-testing/evals/regression-testing.yaml +504 -0
- package/.claude/skills/regression-testing/schemas/output.json +311 -0
- package/.claude/skills/regression-testing/scripts/validate.sh +130 -0
- package/.claude/skills/risk-based-testing/SKILL.md +6 -0
- package/.claude/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -0
- package/.claude/skills/risk-based-testing/schemas/output.json +480 -0
- package/.claude/skills/risk-based-testing/scripts/validate.sh +56 -0
- package/.claude/skills/security-testing/SKILL.md +5 -0
- package/.claude/skills/security-testing/evals/security-testing.yaml +789 -0
- package/.claude/skills/security-testing/schemas/output.json +879 -0
- package/.claude/skills/security-testing/scripts/validate.sh +967 -0
- package/.claude/skills/security-visual-testing/SKILL.md +223 -217
- package/.claude/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -0
- package/.claude/skills/security-visual-testing/schemas/output.json +486 -0
- package/.claude/skills/security-visual-testing/scripts/validate.sh +748 -0
- package/.claude/skills/sherlock-review/SKILL.md +5 -0
- package/.claude/skills/sherlock-review/schemas/output.json +297 -0
- package/.claude/skills/sherlock-review/scripts/validate.sh +115 -0
- package/.claude/skills/shift-left-testing/SKILL.md +6 -0
- package/.claude/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -0
- package/.claude/skills/shift-left-testing/schemas/output.json +459 -0
- package/.claude/skills/shift-left-testing/scripts/validate.sh +56 -0
- package/.claude/skills/shift-right-testing/SKILL.md +6 -0
- package/.claude/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -0
- package/.claude/skills/shift-right-testing/schemas/output.json +418 -0
- package/.claude/skills/shift-right-testing/scripts/validate.sh +56 -0
- package/.claude/skills/six-thinking-hats/SKILL.md +3 -0
- package/.claude/skills/tdd-london-chicago/SKILL.md +5 -0
- package/.claude/skills/tdd-london-chicago/schemas/output.json +444 -0
- package/.claude/skills/tdd-london-chicago/scripts/validate.sh +214 -0
- package/.claude/skills/technical-writing/SKILL.md +4 -0
- package/.claude/skills/technical-writing/schemas/output.json +268 -0
- package/.claude/skills/test-automation-strategy/SKILL.md +6 -0
- package/.claude/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -0
- package/.claude/skills/test-automation-strategy/schemas/output.json +444 -0
- package/.claude/skills/test-automation-strategy/scripts/validate.sh +195 -0
- package/.claude/skills/test-data-management/SKILL.md +6 -0
- package/.claude/skills/test-data-management/evals/test-data-management.yaml +504 -0
- package/.claude/skills/test-data-management/schemas/output.json +284 -0
- package/.claude/skills/test-data-management/scripts/validate.sh +137 -0
- package/.claude/skills/test-design-techniques/SKILL.md +6 -0
- package/.claude/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -0
- package/.claude/skills/test-design-techniques/schemas/output.json +295 -0
- package/.claude/skills/test-design-techniques/scripts/validate.sh +91 -0
- package/.claude/skills/test-environment-management/SKILL.md +4 -0
- package/.claude/skills/test-environment-management/schemas/output.json +310 -0
- package/.claude/skills/test-reporting-analytics/SKILL.md +6 -0
- package/.claude/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -0
- package/.claude/skills/test-reporting-analytics/schemas/output.json +329 -0
- package/.claude/skills/test-reporting-analytics/scripts/validate.sh +77 -0
- package/.claude/skills/testability-scoring/SKILL.md +5 -0
- package/.claude/skills/testability-scoring/evals/testability-scoring.yaml +814 -0
- package/.claude/skills/testability-scoring/schemas/output.json +606 -0
- package/.claude/skills/testability-scoring/scripts/validate.sh +891 -0
- package/.claude/skills/trust-tier-manifest.json +2333 -0
- package/.claude/skills/verification-quality/SKILL.md +6 -0
- package/.claude/skills/verification-quality/evals/verification-quality.yaml +150 -0
- package/.claude/skills/verification-quality/schemas/output.json +432 -0
- package/.claude/skills/verification-quality/scripts/validate.sh +77 -0
- package/.claude/skills/visual-testing-advanced/SKILL.md +6 -0
- package/.claude/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -0
- package/.claude/skills/visual-testing-advanced/schemas/output.json +294 -0
- package/.claude/skills/visual-testing-advanced/scripts/validate.sh +77 -0
- package/.claude/skills/xp-practices/SKILL.md +3 -0
- package/CHANGELOG.md +23 -0
- package/README.md +39 -2
- package/package.json +4 -1
- package/scripts/run-skill-eval.ts +1097 -0
- package/scripts/test-schema-validation.js +301 -0
- package/scripts/update-skill-manifest.ts +751 -0
- package/scripts/validate-skill-frontmatter.js +141 -0
- package/v3/CHANGELOG.md +21 -0
- package/v3/README.md +43 -3
- package/v3/assets/agents/v3/qe-test-idea-rewriter.md +375 -375
- package/v3/assets/skills/.validation/README.md +111 -0
- package/v3/assets/skills/.validation/examples/chaos-engineering-output.example.json +530 -0
- package/v3/assets/skills/.validation/examples/performance-testing-output.example.json +252 -0
- package/v3/assets/skills/.validation/examples/security-testing-output.example.json +413 -0
- package/v3/assets/skills/.validation/examples/testability-scoring-output.example.json +350 -0
- package/v3/assets/skills/.validation/schemas/skill-eval.schema.json +452 -0
- package/v3/assets/skills/.validation/schemas/skill-frontmatter.schema.json +341 -0
- package/v3/assets/skills/.validation/schemas/skill-output-meta.schema.json +199 -0
- package/v3/assets/skills/.validation/schemas/skill-output.template.json +610 -0
- package/v3/assets/skills/.validation/skill-validation-mcp-integration.md +250 -0
- package/v3/assets/skills/.validation/templates/eval.template.yaml +366 -0
- package/v3/assets/skills/.validation/templates/schemas/output.json +145 -0
- package/v3/assets/skills/.validation/templates/security-testing-eval.template.yaml +725 -0
- package/v3/assets/skills/.validation/templates/skill-frontmatter.example.yaml +225 -0
- package/v3/assets/skills/.validation/templates/validate.template.sh +1060 -0
- package/v3/assets/skills/.validation/templates/validator-lib.sh +1031 -0
- package/v3/assets/skills/.validation/test-data/invalid-output.json +5 -0
- package/v3/assets/skills/.validation/test-data/minimal-output.json +9 -0
- package/v3/assets/skills/.validation/test-data/sample-output.json +73 -0
- package/v3/assets/skills/accessibility-testing/SKILL.md +8 -1
- package/v3/assets/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -0
- package/v3/assets/skills/accessibility-testing/schemas/output.json +776 -0
- package/v3/assets/skills/accessibility-testing/scripts/validate.sh +943 -0
- package/v3/assets/skills/accessibility-testing/test-data/sample-output.json +191 -0
- package/v3/assets/skills/agentic-quality-engineering/SKILL.md +4 -0
- package/v3/assets/skills/agentic-quality-engineering/schemas/output.json +577 -0
- package/v3/assets/skills/api-testing-patterns/SKILL.md +5 -0
- package/v3/assets/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -0
- package/v3/assets/skills/api-testing-patterns/schemas/output.json +845 -0
- package/v3/assets/skills/api-testing-patterns/scripts/validate.sh +931 -0
- package/v3/assets/skills/aqe-v2-v3-migration/schemas/output.json +513 -0
- package/v3/assets/skills/aqe-v2-v3-migration/skill.md +20 -16
- package/v3/assets/skills/bug-reporting-excellence/SKILL.md +5 -0
- package/v3/assets/skills/bug-reporting-excellence/schemas/output.json +288 -0
- package/v3/assets/skills/bug-reporting-excellence/scripts/validate.sh +108 -0
- package/v3/assets/skills/chaos-engineering-resilience/SKILL.md +5 -0
- package/v3/assets/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -0
- package/v3/assets/skills/chaos-engineering-resilience/schemas/output.json +1205 -0
- package/v3/assets/skills/chaos-engineering-resilience/scripts/validate.sh +1107 -0
- package/v3/assets/skills/code-review-quality/SKILL.md +5 -0
- package/v3/assets/skills/code-review-quality/schemas/output.json +264 -0
- package/v3/assets/skills/code-review-quality/scripts/validate.sh +113 -0
- package/v3/assets/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -0
- package/v3/assets/skills/compatibility-testing/schemas/output.json +551 -0
- package/v3/assets/skills/compatibility-testing/scripts/validate.sh +605 -0
- package/v3/assets/skills/compliance-testing/SKILL.md +7 -0
- package/v3/assets/skills/compliance-testing/evals/compliance-testing.yaml +1107 -0
- package/v3/assets/skills/compliance-testing/schemas/output.json +845 -0
- package/v3/assets/skills/compliance-testing/scripts/validate.sh +888 -0
- package/v3/assets/skills/consultancy-practices/SKILL.md +4 -0
- package/v3/assets/skills/consultancy-practices/schemas/output.json +282 -0
- package/v3/assets/skills/context-driven-testing/SKILL.md +3 -0
- package/v3/assets/skills/contract-testing/SKILL.md +5 -0
- package/v3/assets/skills/contract-testing/evals/contract-testing.yaml +748 -0
- package/v3/assets/skills/contract-testing/schemas/output.json +638 -0
- package/v3/assets/skills/contract-testing/scripts/validate.sh +1033 -0
- package/v3/assets/skills/database-testing/SKILL.md +5 -0
- package/v3/assets/skills/database-testing/evals/database-testing.yaml +968 -0
- package/v3/assets/skills/database-testing/schemas/output.json +1446 -0
- package/v3/assets/skills/database-testing/scripts/validate.sh +1099 -0
- package/v3/assets/skills/exploratory-testing-advanced/SKILL.md +3 -0
- package/v3/assets/skills/holistic-testing-pact/SKILL.md +3 -0
- package/v3/assets/skills/localization-testing/SKILL.md +6 -0
- package/v3/assets/skills/localization-testing/evals/localization-testing.yaml +544 -0
- package/v3/assets/skills/localization-testing/schemas/output.json +325 -0
- package/v3/assets/skills/localization-testing/scripts/validate.sh +131 -0
- package/v3/assets/skills/mobile-testing/SKILL.md +6 -0
- package/v3/assets/skills/mobile-testing/evals/mobile-testing.yaml +537 -0
- package/v3/assets/skills/mobile-testing/schemas/output.json +318 -0
- package/v3/assets/skills/mobile-testing/scripts/validate.sh +127 -0
- package/v3/assets/skills/mutation-testing/SKILL.md +8 -1
- package/v3/assets/skills/mutation-testing/evals/mutation-testing.yaml +652 -0
- package/v3/assets/skills/mutation-testing/schemas/output.json +707 -0
- package/v3/assets/skills/mutation-testing/scripts/validate.sh +992 -0
- package/v3/assets/skills/mutation-testing/test-data/sample-output.json +295 -0
- package/v3/assets/skills/n8n-expression-testing/scripts/validate.sh +162 -0
- package/v3/assets/skills/n8n-integration-testing-patterns/scripts/validate.sh +104 -0
- package/v3/assets/skills/n8n-security-testing/scripts/validate.sh +120 -0
- package/v3/assets/skills/n8n-trigger-testing-strategies/scripts/validate.sh +121 -0
- package/v3/assets/skills/n8n-workflow-testing-fundamentals/scripts/validate.sh +126 -0
- package/v3/assets/skills/performance-testing/SKILL.md +5 -0
- package/v3/assets/skills/performance-testing/evals/performance-testing.yaml +772 -0
- package/v3/assets/skills/performance-testing/schemas/output.json +1184 -0
- package/v3/assets/skills/performance-testing/scripts/validate.sh +1090 -0
- package/v3/assets/skills/qe-chaos-resilience/SKILL.md +6 -0
- package/v3/assets/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -0
- package/v3/assets/skills/qe-chaos-resilience/schemas/output.json +314 -0
- package/v3/assets/skills/qe-chaos-resilience/scripts/validate.sh +401 -0
- package/v3/assets/skills/qe-code-intelligence/SKILL.md +6 -0
- package/v3/assets/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -0
- package/v3/assets/skills/qe-code-intelligence/schemas/output.json +315 -0
- package/v3/assets/skills/qe-code-intelligence/scripts/validate.sh +408 -0
- package/v3/assets/skills/qe-contract-testing/SKILL.md +6 -0
- package/v3/assets/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -0
- package/v3/assets/skills/qe-contract-testing/schemas/output.json +295 -0
- package/v3/assets/skills/qe-contract-testing/scripts/validate.sh +421 -0
- package/v3/assets/skills/qe-coverage-analysis/SKILL.md +6 -0
- package/v3/assets/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -0
- package/v3/assets/skills/qe-coverage-analysis/schemas/output.json +286 -0
- package/v3/assets/skills/qe-coverage-analysis/scripts/validate.sh +453 -0
- package/v3/assets/skills/qe-defect-intelligence/SKILL.md +6 -0
- package/v3/assets/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -0
- package/v3/assets/skills/qe-defect-intelligence/schemas/output.json +283 -0
- package/v3/assets/skills/qe-defect-intelligence/scripts/validate.sh +466 -0
- package/v3/assets/skills/qe-iterative-loop/SKILL.md +445 -443
- package/v3/assets/skills/qe-iterative-loop/schemas/output.json +264 -0
- package/v3/assets/skills/qe-iterative-loop/scripts/validate.sh +474 -0
- package/v3/assets/skills/qe-learning-optimization/SKILL.md +6 -0
- package/v3/assets/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -0
- package/v3/assets/skills/qe-learning-optimization/schemas/output.json +288 -0
- package/v3/assets/skills/qe-learning-optimization/scripts/validate.sh +56 -0
- package/v3/assets/skills/qe-quality-assessment/SKILL.md +6 -0
- package/v3/assets/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -0
- package/v3/assets/skills/qe-quality-assessment/schemas/output.json +550 -0
- package/v3/assets/skills/qe-quality-assessment/scripts/validate.sh +352 -0
- package/v3/assets/skills/qe-requirements-validation/SKILL.md +6 -0
- package/v3/assets/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -0
- package/v3/assets/skills/qe-requirements-validation/schemas/output.json +587 -0
- package/v3/assets/skills/qe-requirements-validation/scripts/validate.sh +275 -0
- package/v3/assets/skills/qe-security-compliance/SKILL.md +6 -0
- package/v3/assets/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -0
- package/v3/assets/skills/qe-security-compliance/schemas/output.json +498 -0
- package/v3/assets/skills/qe-security-compliance/scripts/validate.sh +305 -0
- package/v3/assets/skills/qe-test-execution/SKILL.md +6 -0
- package/v3/assets/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -0
- package/v3/assets/skills/qe-test-execution/schemas/output.json +529 -0
- package/v3/assets/skills/qe-test-execution/scripts/validate.sh +323 -0
- package/v3/assets/skills/qe-test-generation/SKILL.md +6 -0
- package/v3/assets/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -0
- package/v3/assets/skills/qe-test-generation/schemas/output.json +439 -0
- package/v3/assets/skills/qe-test-generation/scripts/validate.sh +56 -0
- package/v3/assets/skills/qe-visual-accessibility/SKILL.md +6 -0
- package/v3/assets/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -0
- package/v3/assets/skills/qe-visual-accessibility/schemas/output.json +491 -0
- package/v3/assets/skills/qe-visual-accessibility/scripts/validate.sh +56 -0
- package/v3/assets/skills/quality-metrics/SKILL.md +6 -0
- package/v3/assets/skills/quality-metrics/evals/quality-metrics.yaml +494 -0
- package/v3/assets/skills/quality-metrics/schemas/output.json +403 -0
- package/v3/assets/skills/quality-metrics/scripts/validate.sh +434 -0
- package/v3/assets/skills/refactoring-patterns/SKILL.md +5 -0
- package/v3/assets/skills/refactoring-patterns/schemas/output.json +475 -0
- package/v3/assets/skills/refactoring-patterns/scripts/validate.sh +241 -0
- package/v3/assets/skills/regression-testing/SKILL.md +6 -0
- package/v3/assets/skills/regression-testing/evals/regression-testing.yaml +504 -0
- package/v3/assets/skills/regression-testing/schemas/output.json +311 -0
- package/v3/assets/skills/regression-testing/scripts/validate.sh +130 -0
- package/v3/assets/skills/risk-based-testing/SKILL.md +6 -0
- package/v3/assets/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -0
- package/v3/assets/skills/risk-based-testing/schemas/output.json +480 -0
- package/v3/assets/skills/risk-based-testing/scripts/validate.sh +56 -0
- package/v3/assets/skills/security-testing/SKILL.md +5 -0
- package/v3/assets/skills/security-testing/evals/security-testing.yaml +789 -0
- package/v3/assets/skills/security-testing/schemas/output.json +879 -0
- package/v3/assets/skills/security-testing/scripts/validate.sh +967 -0
- package/v3/assets/skills/shift-left-testing/SKILL.md +6 -0
- package/v3/assets/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -0
- package/v3/assets/skills/shift-left-testing/schemas/output.json +459 -0
- package/v3/assets/skills/shift-left-testing/scripts/validate.sh +56 -0
- package/v3/assets/skills/shift-right-testing/SKILL.md +6 -0
- package/v3/assets/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -0
- package/v3/assets/skills/shift-right-testing/schemas/output.json +418 -0
- package/v3/assets/skills/shift-right-testing/scripts/validate.sh +56 -0
- package/v3/assets/skills/six-thinking-hats/SKILL.md +3 -0
- package/v3/assets/skills/tdd-london-chicago/SKILL.md +5 -0
- package/v3/assets/skills/tdd-london-chicago/schemas/output.json +444 -0
- package/v3/assets/skills/tdd-london-chicago/scripts/validate.sh +214 -0
- package/v3/assets/skills/technical-writing/SKILL.md +4 -0
- package/v3/assets/skills/technical-writing/schemas/output.json +268 -0
- package/v3/assets/skills/test-automation-strategy/SKILL.md +6 -0
- package/v3/assets/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -0
- package/v3/assets/skills/test-automation-strategy/schemas/output.json +444 -0
- package/v3/assets/skills/test-automation-strategy/scripts/validate.sh +195 -0
- package/v3/assets/skills/test-data-management/SKILL.md +6 -0
- package/v3/assets/skills/test-data-management/evals/test-data-management.yaml +504 -0
- package/v3/assets/skills/test-data-management/schemas/output.json +284 -0
- package/v3/assets/skills/test-data-management/scripts/validate.sh +137 -0
- package/v3/assets/skills/test-design-techniques/SKILL.md +6 -0
- package/v3/assets/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -0
- package/v3/assets/skills/test-design-techniques/schemas/output.json +295 -0
- package/v3/assets/skills/test-design-techniques/scripts/validate.sh +91 -0
- package/v3/assets/skills/test-environment-management/SKILL.md +4 -0
- package/v3/assets/skills/test-environment-management/schemas/output.json +310 -0
- package/v3/assets/skills/test-reporting-analytics/SKILL.md +6 -0
- package/v3/assets/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -0
- package/v3/assets/skills/test-reporting-analytics/schemas/output.json +329 -0
- package/v3/assets/skills/test-reporting-analytics/scripts/validate.sh +77 -0
- package/v3/assets/skills/verification-quality/SKILL.md +6 -0
- package/v3/assets/skills/verification-quality/evals/verification-quality.yaml +150 -0
- package/v3/assets/skills/verification-quality/schemas/output.json +432 -0
- package/v3/assets/skills/verification-quality/scripts/validate.sh +77 -0
- package/v3/assets/skills/visual-testing-advanced/SKILL.md +6 -0
- package/v3/assets/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -0
- package/v3/assets/skills/visual-testing-advanced/schemas/output.json +294 -0
- package/v3/assets/skills/visual-testing-advanced/scripts/validate.sh +77 -0
- package/v3/assets/skills/xp-practices/SKILL.md +3 -0
- package/v3/assets/templates/validation-report.md.hbs +139 -0
- package/v3/assets/templates/validation-summary.json +56 -0
- package/v3/dist/cli/bundle.js +11853 -5316
- package/v3/dist/cli/commands/eval.d.ts +18 -0
- package/v3/dist/cli/commands/eval.d.ts.map +1 -0
- package/v3/dist/cli/commands/eval.js +505 -0
- package/v3/dist/cli/commands/eval.js.map +1 -0
- package/v3/dist/cli/commands/validate-swarm.d.ts +16 -0
- package/v3/dist/cli/commands/validate-swarm.d.ts.map +1 -0
- package/v3/dist/cli/commands/validate-swarm.js +251 -0
- package/v3/dist/cli/commands/validate-swarm.js.map +1 -0
- package/v3/dist/cli/commands/validate.d.ts +14 -0
- package/v3/dist/cli/commands/validate.d.ts.map +1 -0
- package/v3/dist/cli/commands/validate.js +408 -0
- package/v3/dist/cli/commands/validate.js.map +1 -0
- package/v3/dist/cli/index.js +6 -0
- package/v3/dist/cli/index.js.map +1 -1
- package/v3/dist/index.d.ts +2 -0
- package/v3/dist/index.d.ts.map +1 -1
- package/v3/dist/index.js +10 -0
- package/v3/dist/index.js.map +1 -1
- package/v3/dist/init/skills-installer.d.ts +6 -0
- package/v3/dist/init/skills-installer.d.ts.map +1 -1
- package/v3/dist/init/skills-installer.js +48 -0
- package/v3/dist/init/skills-installer.js.map +1 -1
- package/v3/dist/learning/index.d.ts +2 -0
- package/v3/dist/learning/index.d.ts.map +1 -1
- package/v3/dist/learning/index.js +4 -0
- package/v3/dist/learning/index.js.map +1 -1
- package/v3/dist/learning/skill-validation-learner.d.ts +264 -0
- package/v3/dist/learning/skill-validation-learner.d.ts.map +1 -0
- package/v3/dist/learning/skill-validation-learner.js +515 -0
- package/v3/dist/learning/skill-validation-learner.js.map +1 -0
- package/v3/dist/mcp/bundle.js +10043 -3630
- package/v3/dist/validation/index.d.ts +19 -0
- package/v3/dist/validation/index.d.ts.map +1 -0
- package/v3/dist/validation/index.js +31 -0
- package/v3/dist/validation/index.js.map +1 -0
- package/v3/dist/validation/parallel-eval-runner.d.ts +307 -0
- package/v3/dist/validation/parallel-eval-runner.d.ts.map +1 -0
- package/v3/dist/validation/parallel-eval-runner.js +566 -0
- package/v3/dist/validation/parallel-eval-runner.js.map +1 -0
- package/v3/dist/validation/swarm-skill-validator.d.ts +282 -0
- package/v3/dist/validation/swarm-skill-validator.d.ts.map +1 -0
- package/v3/dist/validation/swarm-skill-validator.js +460 -0
- package/v3/dist/validation/swarm-skill-validator.js.map +1 -0
- package/v3/dist/validation/validation-result-aggregator.d.ts +232 -0
- package/v3/dist/validation/validation-result-aggregator.d.ts.map +1 -0
- package/v3/dist/validation/validation-result-aggregator.js +630 -0
- package/v3/dist/validation/validation-result-aggregator.js.map +1 -0
- package/v3/package.json +1 -1
|
@@ -0,0 +1,967 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# AQE Skill Validator: security-testing v1.0.0
|
|
4
|
+
# Validates security testing skill output per ADR-056
|
|
5
|
+
# =============================================================================
|
|
6
|
+
#
|
|
7
|
+
# This validator checks:
|
|
8
|
+
# 1. JSON schema compliance (OWASP Top 10, CWE, CVSS structure)
|
|
9
|
+
# 2. Required security tools availability (npm, semgrep, trivy)
|
|
10
|
+
# 3. OWASP category coverage and completeness
|
|
11
|
+
# 4. Finding and recommendation structure
|
|
12
|
+
# 5. Security-specific content validation
|
|
13
|
+
#
|
|
14
|
+
# Usage: ./validate.sh <output-file> [options]
|
|
15
|
+
#
|
|
16
|
+
# Options:
|
|
17
|
+
# --self-test Run validator self-test mode
|
|
18
|
+
# --verbose Enable verbose output
|
|
19
|
+
# --json Output results as JSON only
|
|
20
|
+
# --list-tools Show available validation tools
|
|
21
|
+
# --help Show this help message
|
|
22
|
+
#
|
|
23
|
+
# Exit Codes:
|
|
24
|
+
# 0 - Validation passed
|
|
25
|
+
# 1 - Validation failed
|
|
26
|
+
# 2 - Validation skipped (missing required tools)
|
|
27
|
+
#
|
|
28
|
+
# =============================================================================
|
|
29
|
+
|
|
30
|
+
set -euo pipefail
|
|
31
|
+
|
|
32
|
+
# Get script directory
|
|
33
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
34
|
+
|
|
35
|
+
# Navigate to skill directory and project root
|
|
36
|
+
# scripts/ -> security-testing/ -> skills/ -> .claude/ -> project root
|
|
37
|
+
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
38
|
+
PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
|
|
39
|
+
|
|
40
|
+
# Source validator library - check multiple locations
|
|
41
|
+
VALIDATOR_LIB=""
|
|
42
|
+
for lib_path in \
|
|
43
|
+
"$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" \
|
|
44
|
+
"$SKILL_DIR/scripts/validator-lib.sh" \
|
|
45
|
+
"$SCRIPT_DIR/validator-lib.sh"; do
|
|
46
|
+
if [[ -f "$lib_path" ]]; then
|
|
47
|
+
VALIDATOR_LIB="$lib_path"
|
|
48
|
+
break
|
|
49
|
+
fi
|
|
50
|
+
done
|
|
51
|
+
|
|
52
|
+
if [[ -n "$VALIDATOR_LIB" ]]; then
|
|
53
|
+
# shellcheck source=/dev/null
|
|
54
|
+
source "$VALIDATOR_LIB"
|
|
55
|
+
else
|
|
56
|
+
echo "ERROR: Validator library not found"
|
|
57
|
+
echo "Searched:"
|
|
58
|
+
echo " - $PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh"
|
|
59
|
+
echo " - $SKILL_DIR/scripts/validator-lib.sh"
|
|
60
|
+
echo " - $SCRIPT_DIR/validator-lib.sh"
|
|
61
|
+
exit 1
|
|
62
|
+
fi
|
|
63
|
+
|
|
64
|
+
# =============================================================================
|
|
65
|
+
# SKILL-SPECIFIC CONFIGURATION
|
|
66
|
+
# =============================================================================
|
|
67
|
+
|
|
68
|
+
# Skill name and version
|
|
69
|
+
SKILL_NAME="security-testing"
|
|
70
|
+
SKILL_VERSION="1.0.0"
|
|
71
|
+
|
|
72
|
+
# Required tools (validation FAILS with exit 2 if missing)
|
|
73
|
+
# jq is essential for JSON parsing
|
|
74
|
+
REQUIRED_TOOLS=("jq")
|
|
75
|
+
|
|
76
|
+
# Optional tools (validation continues with warnings if missing)
|
|
77
|
+
# These enhance security scanning capabilities
|
|
78
|
+
OPTIONAL_TOOLS=("npm" "semgrep" "trivy" "ajv" "jsonschema" "python3")
|
|
79
|
+
|
|
80
|
+
# Path to output JSON schema
|
|
81
|
+
SCHEMA_PATH="$SKILL_DIR/schemas/output.json"
|
|
82
|
+
|
|
83
|
+
# Path to sample test data for self-test
|
|
84
|
+
SAMPLE_OUTPUT_PATH="$PROJECT_ROOT/.claude/skills/.validation/examples/security-testing-output.example.json"
|
|
85
|
+
|
|
86
|
+
# =============================================================================
|
|
87
|
+
# CONTENT VALIDATION CONFIGURATION
|
|
88
|
+
# =============================================================================
|
|
89
|
+
|
|
90
|
+
# Required fields in output
|
|
91
|
+
REQUIRED_FIELDS=("skillName" "status" "output" "output.summary" "output.findings" "output.owaspCategories")
|
|
92
|
+
|
|
93
|
+
# Fields that must have non-null, non-empty values
|
|
94
|
+
REQUIRED_NON_EMPTY_FIELDS=("output.summary")
|
|
95
|
+
|
|
96
|
+
# Security-specific terms that MUST appear in output
|
|
97
|
+
MUST_CONTAIN_TERMS=("OWASP" "security" "vulnerability")
|
|
98
|
+
|
|
99
|
+
# Terms that must NOT appear in output (indicates failure/hallucination)
|
|
100
|
+
MUST_NOT_CONTAIN_TERMS=("TODO" "placeholder" "FIXME")
|
|
101
|
+
|
|
102
|
+
# Enum validations
|
|
103
|
+
ENUM_VALIDATIONS=(
|
|
104
|
+
".status:success,partial,failed,skipped"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# Minimum array lengths
|
|
108
|
+
MIN_ARRAY_LENGTHS=()
|
|
109
|
+
|
|
110
|
+
# =============================================================================
|
|
111
|
+
# Argument Parsing
|
|
112
|
+
# =============================================================================
|
|
113
|
+
|
|
114
|
+
OUTPUT_FILE=""
|
|
115
|
+
SELF_TEST=false
|
|
116
|
+
VERBOSE=false
|
|
117
|
+
JSON_ONLY=false
|
|
118
|
+
LIST_TOOLS=false
|
|
119
|
+
|
|
120
|
+
while [[ $# -gt 0 ]]; do
|
|
121
|
+
case "$1" in
|
|
122
|
+
--self-test)
|
|
123
|
+
SELF_TEST=true
|
|
124
|
+
shift
|
|
125
|
+
;;
|
|
126
|
+
--verbose|-v)
|
|
127
|
+
VERBOSE=true
|
|
128
|
+
export AQE_DEBUG=1
|
|
129
|
+
shift
|
|
130
|
+
;;
|
|
131
|
+
--json)
|
|
132
|
+
JSON_ONLY=true
|
|
133
|
+
shift
|
|
134
|
+
;;
|
|
135
|
+
--list-tools)
|
|
136
|
+
LIST_TOOLS=true
|
|
137
|
+
shift
|
|
138
|
+
;;
|
|
139
|
+
-h|--help)
|
|
140
|
+
cat << 'HELP_EOF'
|
|
141
|
+
AQE Security Testing Skill Validator v1.0.0
|
|
142
|
+
|
|
143
|
+
Usage: ./validate.sh <output-file> [options]
|
|
144
|
+
./validate.sh --self-test [--verbose]
|
|
145
|
+
./validate.sh --list-tools
|
|
146
|
+
|
|
147
|
+
Arguments:
|
|
148
|
+
<output-file> Path to skill output JSON file to validate
|
|
149
|
+
|
|
150
|
+
Options:
|
|
151
|
+
--self-test Run validator self-test mode
|
|
152
|
+
--verbose, -v Enable verbose/debug output
|
|
153
|
+
--json Output results as JSON only (for CI integration)
|
|
154
|
+
--list-tools Show available validation tools and exit
|
|
155
|
+
--help, -h Show this help message
|
|
156
|
+
|
|
157
|
+
Exit Codes:
|
|
158
|
+
0 - Validation passed
|
|
159
|
+
1 - Validation failed
|
|
160
|
+
2 - Validation skipped (missing required tools)
|
|
161
|
+
|
|
162
|
+
Security Tool Requirements:
|
|
163
|
+
Required: jq (JSON parsing)
|
|
164
|
+
Optional: npm (dependency audit), semgrep (SAST), trivy (container/deps)
|
|
165
|
+
|
|
166
|
+
Examples:
|
|
167
|
+
./validate.sh security-output.json # Validate output file
|
|
168
|
+
./validate.sh security-output.json --json # JSON output for CI
|
|
169
|
+
./validate.sh --self-test --verbose # Self-test with debug
|
|
170
|
+
./validate.sh --list-tools # Show available tools
|
|
171
|
+
|
|
172
|
+
HELP_EOF
|
|
173
|
+
exit 0
|
|
174
|
+
;;
|
|
175
|
+
-*)
|
|
176
|
+
error "Unknown option: $1"
|
|
177
|
+
echo "Use --help for usage information"
|
|
178
|
+
exit 1
|
|
179
|
+
;;
|
|
180
|
+
*)
|
|
181
|
+
OUTPUT_FILE="$1"
|
|
182
|
+
shift
|
|
183
|
+
;;
|
|
184
|
+
esac
|
|
185
|
+
done
|
|
186
|
+
|
|
187
|
+
# Handle --list-tools mode
|
|
188
|
+
if [[ "$LIST_TOOLS" == "true" ]]; then
|
|
189
|
+
echo "=============================================="
|
|
190
|
+
echo "Available Validation Tools for $SKILL_NAME"
|
|
191
|
+
echo "=============================================="
|
|
192
|
+
echo ""
|
|
193
|
+
echo "Required tools (validation fails if missing):"
|
|
194
|
+
for tool in "${REQUIRED_TOOLS[@]}"; do
|
|
195
|
+
if command_exists "$tool"; then
|
|
196
|
+
version=$($tool --version 2>&1 | head -1 || echo "installed")
|
|
197
|
+
echo " [OK] $tool - $version"
|
|
198
|
+
else
|
|
199
|
+
echo " [MISSING] $tool"
|
|
200
|
+
fi
|
|
201
|
+
done
|
|
202
|
+
echo ""
|
|
203
|
+
echo "Optional tools (enhances validation):"
|
|
204
|
+
for tool in "${OPTIONAL_TOOLS[@]}"; do
|
|
205
|
+
if command_exists "$tool"; then
|
|
206
|
+
version=""
|
|
207
|
+
case "$tool" in
|
|
208
|
+
npm) version=$(npm --version 2>&1 | head -1 || echo "installed") ;;
|
|
209
|
+
semgrep) version=$(semgrep --version 2>&1 | head -1 || echo "installed") ;;
|
|
210
|
+
trivy) version=$(trivy --version 2>&1 | head -1 | awk '{print $2}' || echo "installed") ;;
|
|
211
|
+
python3) version=$(python3 --version 2>&1 | head -1 || echo "installed") ;;
|
|
212
|
+
*) version="installed" ;;
|
|
213
|
+
esac
|
|
214
|
+
echo " [OK] $tool - $version"
|
|
215
|
+
else
|
|
216
|
+
echo " [MISSING] $tool"
|
|
217
|
+
fi
|
|
218
|
+
done
|
|
219
|
+
echo ""
|
|
220
|
+
echo "Security Scan Capabilities:"
|
|
221
|
+
if command_exists "npm"; then
|
|
222
|
+
echo " [OK] npm audit - Dependency vulnerability scanning"
|
|
223
|
+
else
|
|
224
|
+
echo " [MISSING] npm audit - Install Node.js for dependency scanning"
|
|
225
|
+
fi
|
|
226
|
+
if command_exists "semgrep"; then
|
|
227
|
+
echo " [OK] semgrep - SAST analysis"
|
|
228
|
+
else
|
|
229
|
+
echo " [MISSING] semgrep - Install semgrep for static analysis"
|
|
230
|
+
fi
|
|
231
|
+
if command_exists "trivy"; then
|
|
232
|
+
echo " [OK] trivy - Container and dependency scanning"
|
|
233
|
+
else
|
|
234
|
+
echo " [MISSING] trivy - Install trivy for comprehensive scanning"
|
|
235
|
+
fi
|
|
236
|
+
exit 0
|
|
237
|
+
fi
|
|
238
|
+
|
|
239
|
+
# =============================================================================
|
|
240
|
+
# Self-Test Mode
|
|
241
|
+
# =============================================================================
|
|
242
|
+
|
|
243
|
+
if [[ "$SELF_TEST" == "true" ]]; then
|
|
244
|
+
echo "=============================================="
|
|
245
|
+
info "Running $SKILL_NAME Validator Self-Test"
|
|
246
|
+
echo "=============================================="
|
|
247
|
+
echo ""
|
|
248
|
+
echo "Validator version: $AQE_VALIDATOR_VERSION"
|
|
249
|
+
echo "Skill version: $SKILL_VERSION"
|
|
250
|
+
echo ""
|
|
251
|
+
|
|
252
|
+
self_test_passed=true
|
|
253
|
+
self_test_warnings=0
|
|
254
|
+
|
|
255
|
+
# Step 1: Check Required Tools
|
|
256
|
+
echo "--- Step 1: Required Tools ---"
|
|
257
|
+
for tool in "${REQUIRED_TOOLS[@]}"; do
|
|
258
|
+
if command_exists "$tool"; then
|
|
259
|
+
success "Required tool available: $tool"
|
|
260
|
+
else
|
|
261
|
+
error "Required tool MISSING: $tool"
|
|
262
|
+
self_test_passed=false
|
|
263
|
+
fi
|
|
264
|
+
done
|
|
265
|
+
echo ""
|
|
266
|
+
|
|
267
|
+
# Step 2: Check Optional Security Tools
|
|
268
|
+
echo "--- Step 2: Security Tools ---"
|
|
269
|
+
security_tools=("npm" "semgrep" "trivy")
|
|
270
|
+
available_security=0
|
|
271
|
+
for tool in "${security_tools[@]}"; do
|
|
272
|
+
if command_exists "$tool"; then
|
|
273
|
+
success "Security tool available: $tool"
|
|
274
|
+
((available_security++)) || true
|
|
275
|
+
else
|
|
276
|
+
warn "Security tool missing: $tool"
|
|
277
|
+
((self_test_warnings++)) || true
|
|
278
|
+
fi
|
|
279
|
+
done
|
|
280
|
+
|
|
281
|
+
if [[ $available_security -eq 0 ]]; then
|
|
282
|
+
warn "No security scanning tools available"
|
|
283
|
+
((self_test_warnings++)) || true
|
|
284
|
+
fi
|
|
285
|
+
echo ""
|
|
286
|
+
|
|
287
|
+
# Step 3: Check Schema File
|
|
288
|
+
echo "--- Step 3: Schema File ---"
|
|
289
|
+
if [[ -f "$SCHEMA_PATH" ]]; then
|
|
290
|
+
success "Schema file exists: $SCHEMA_PATH"
|
|
291
|
+
if validate_json "$SCHEMA_PATH" 2>/dev/null; then
|
|
292
|
+
success "Schema file is valid JSON"
|
|
293
|
+
|
|
294
|
+
# Check for OWASP-specific schema elements
|
|
295
|
+
if grep -q "owaspCategories" "$SCHEMA_PATH" 2>/dev/null; then
|
|
296
|
+
success "Schema includes OWASP categories definition"
|
|
297
|
+
else
|
|
298
|
+
warn "Schema may be missing OWASP categories"
|
|
299
|
+
((self_test_warnings++)) || true
|
|
300
|
+
fi
|
|
301
|
+
|
|
302
|
+
if grep -q "CWE-" "$SCHEMA_PATH" 2>/dev/null; then
|
|
303
|
+
success "Schema includes CWE pattern validation"
|
|
304
|
+
else
|
|
305
|
+
warn "Schema may be missing CWE pattern"
|
|
306
|
+
((self_test_warnings++)) || true
|
|
307
|
+
fi
|
|
308
|
+
else
|
|
309
|
+
error "Schema file is NOT valid JSON"
|
|
310
|
+
self_test_passed=false
|
|
311
|
+
fi
|
|
312
|
+
else
|
|
313
|
+
error "Schema file not found: $SCHEMA_PATH"
|
|
314
|
+
self_test_passed=false
|
|
315
|
+
fi
|
|
316
|
+
echo ""
|
|
317
|
+
|
|
318
|
+
# Step 4: Test with Sample Data
|
|
319
|
+
echo "--- Step 4: Sample Data Validation ---"
|
|
320
|
+
if [[ -f "$SAMPLE_OUTPUT_PATH" ]]; then
|
|
321
|
+
success "Sample output file exists"
|
|
322
|
+
|
|
323
|
+
if validate_json "$SAMPLE_OUTPUT_PATH" 2>/dev/null; then
|
|
324
|
+
success "Sample output is valid JSON"
|
|
325
|
+
|
|
326
|
+
# Test OWASP-specific validation
|
|
327
|
+
if validate_owasp_findings "$SAMPLE_OUTPUT_PATH" 2>/dev/null; then
|
|
328
|
+
success "Sample output has valid OWASP findings"
|
|
329
|
+
else
|
|
330
|
+
warn "Sample output OWASP validation issue"
|
|
331
|
+
((self_test_warnings++)) || true
|
|
332
|
+
fi
|
|
333
|
+
else
|
|
334
|
+
error "Sample output is NOT valid JSON"
|
|
335
|
+
self_test_passed=false
|
|
336
|
+
fi
|
|
337
|
+
else
|
|
338
|
+
info "No sample output file found at: $SAMPLE_OUTPUT_PATH"
|
|
339
|
+
info "Skipping sample data validation"
|
|
340
|
+
fi
|
|
341
|
+
echo ""
|
|
342
|
+
|
|
343
|
+
# Step 5: Library Self-Test
|
|
344
|
+
echo "--- Step 5: Validator Library Self-Test ---"
|
|
345
|
+
if run_self_test 2>/dev/null; then
|
|
346
|
+
success "Library self-test passed"
|
|
347
|
+
else
|
|
348
|
+
error "Library self-test FAILED"
|
|
349
|
+
self_test_passed=false
|
|
350
|
+
fi
|
|
351
|
+
echo ""
|
|
352
|
+
|
|
353
|
+
# Summary
|
|
354
|
+
echo "=============================================="
|
|
355
|
+
echo "Self-Test Summary for $SKILL_NAME"
|
|
356
|
+
echo "=============================================="
|
|
357
|
+
|
|
358
|
+
if [[ "$self_test_passed" == "true" ]]; then
|
|
359
|
+
if [[ $self_test_warnings -gt 0 ]]; then
|
|
360
|
+
warn "Self-test PASSED with $self_test_warnings warning(s)"
|
|
361
|
+
exit 0
|
|
362
|
+
else
|
|
363
|
+
success "Self-test PASSED"
|
|
364
|
+
exit 0
|
|
365
|
+
fi
|
|
366
|
+
else
|
|
367
|
+
error "Self-test FAILED"
|
|
368
|
+
exit 1
|
|
369
|
+
fi
|
|
370
|
+
fi
|
|
371
|
+
|
|
372
|
+
# =============================================================================
|
|
373
|
+
# SKILL-SPECIFIC VALIDATION FUNCTIONS
|
|
374
|
+
# =============================================================================
|
|
375
|
+
|
|
376
|
+
# Validate OWASP Top 10 findings structure
|
|
377
|
+
# Returns: 0 if valid, 1 if invalid
|
|
378
|
+
validate_owasp_findings() {
|
|
379
|
+
local output_file="$1"
|
|
380
|
+
|
|
381
|
+
# Check owaspCategories exists
|
|
382
|
+
local owasp_data
|
|
383
|
+
owasp_data=$(json_get "$output_file" ".output.owaspCategories" 2>/dev/null)
|
|
384
|
+
|
|
385
|
+
if [[ -z "$owasp_data" ]] || [[ "$owasp_data" == "null" ]]; then
|
|
386
|
+
warn "Missing owaspCategories in output"
|
|
387
|
+
return 1
|
|
388
|
+
fi
|
|
389
|
+
|
|
390
|
+
# Check that at least some OWASP categories are present
|
|
391
|
+
local categories_tested=0
|
|
392
|
+
for cat in "A01:2021" "A02:2021" "A03:2021" "A04:2021" "A05:2021" "A06:2021" "A07:2021" "A08:2021" "A09:2021" "A10:2021"; do
|
|
393
|
+
local cat_data
|
|
394
|
+
cat_data=$(json_get "$output_file" ".output.owaspCategories.\"$cat\"" 2>/dev/null)
|
|
395
|
+
if [[ -n "$cat_data" ]] && [[ "$cat_data" != "null" ]]; then
|
|
396
|
+
((categories_tested++)) || true
|
|
397
|
+
fi
|
|
398
|
+
done
|
|
399
|
+
|
|
400
|
+
if [[ $categories_tested -eq 0 ]]; then
|
|
401
|
+
error "No OWASP categories found in owaspCategories"
|
|
402
|
+
return 1
|
|
403
|
+
fi
|
|
404
|
+
|
|
405
|
+
debug "Found $categories_tested OWASP categories in output"
|
|
406
|
+
return 0
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
# Validate security findings have required fields
|
|
410
|
+
# Returns: 0 if valid, 1 if invalid
|
|
411
|
+
validate_security_findings() {
|
|
412
|
+
local output_file="$1"
|
|
413
|
+
|
|
414
|
+
local finding_count
|
|
415
|
+
finding_count=$(json_count "$output_file" ".output.findings" 2>/dev/null)
|
|
416
|
+
|
|
417
|
+
if [[ -z "$finding_count" ]] || [[ "$finding_count" == "null" ]]; then
|
|
418
|
+
finding_count=0
|
|
419
|
+
fi
|
|
420
|
+
|
|
421
|
+
debug "Found $finding_count security findings"
|
|
422
|
+
|
|
423
|
+
# If there are findings, validate structure of first few
|
|
424
|
+
if [[ "$finding_count" -gt 0 ]]; then
|
|
425
|
+
# Check first finding has required fields
|
|
426
|
+
local first_id first_severity first_owasp
|
|
427
|
+
first_id=$(json_get "$output_file" ".output.findings[0].id" 2>/dev/null)
|
|
428
|
+
first_severity=$(json_get "$output_file" ".output.findings[0].severity" 2>/dev/null)
|
|
429
|
+
first_owasp=$(json_get "$output_file" ".output.findings[0].owasp" 2>/dev/null)
|
|
430
|
+
|
|
431
|
+
if [[ -z "$first_id" ]] || [[ "$first_id" == "null" ]]; then
|
|
432
|
+
error "Finding missing 'id' field"
|
|
433
|
+
return 1
|
|
434
|
+
fi
|
|
435
|
+
|
|
436
|
+
if [[ -z "$first_severity" ]] || [[ "$first_severity" == "null" ]]; then
|
|
437
|
+
error "Finding missing 'severity' field"
|
|
438
|
+
return 1
|
|
439
|
+
fi
|
|
440
|
+
|
|
441
|
+
# Validate severity is valid enum
|
|
442
|
+
if ! validate_enum "$first_severity" "critical" "high" "medium" "low" "info"; then
|
|
443
|
+
error "Finding has invalid severity: $first_severity"
|
|
444
|
+
return 1
|
|
445
|
+
fi
|
|
446
|
+
|
|
447
|
+
if [[ -z "$first_owasp" ]] || [[ "$first_owasp" == "null" ]]; then
|
|
448
|
+
warn "Finding missing 'owasp' category - consider adding OWASP classification"
|
|
449
|
+
fi
|
|
450
|
+
fi
|
|
451
|
+
|
|
452
|
+
return 0
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
# Validate CWE identifiers are properly formatted
|
|
456
|
+
# Returns: 0 if valid, 1 if invalid
|
|
457
|
+
validate_cwe_format() {
|
|
458
|
+
local output_file="$1"
|
|
459
|
+
local content
|
|
460
|
+
content=$(cat "$output_file")
|
|
461
|
+
|
|
462
|
+
# Check if any CWE references exist and are properly formatted
|
|
463
|
+
if grep -q "CWE-" <<< "$content"; then
|
|
464
|
+
# Validate CWE format (CWE-XXX where XXX is 1-4 digits)
|
|
465
|
+
local invalid_cwes
|
|
466
|
+
invalid_cwes=$(grep -oE "CWE-[0-9]+" <<< "$content" | grep -vE "^CWE-[0-9]{1,4}$" || true)
|
|
467
|
+
|
|
468
|
+
if [[ -n "$invalid_cwes" ]]; then
|
|
469
|
+
warn "Found potentially malformed CWE identifiers"
|
|
470
|
+
debug "Invalid CWEs: $invalid_cwes"
|
|
471
|
+
fi
|
|
472
|
+
fi
|
|
473
|
+
|
|
474
|
+
return 0
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
# Validate CVSS scores are within valid range
|
|
478
|
+
# Returns: 0 if valid, 1 if invalid
|
|
479
|
+
validate_cvss_scores() {
|
|
480
|
+
local output_file="$1"
|
|
481
|
+
|
|
482
|
+
# Extract CVSS scores and validate they're in range
|
|
483
|
+
local scores
|
|
484
|
+
if command_exists "jq"; then
|
|
485
|
+
scores=$(jq -r '.output.findings[]?.cvss?.score // empty' "$output_file" 2>/dev/null || true)
|
|
486
|
+
|
|
487
|
+
for score in $scores; do
|
|
488
|
+
if [[ -n "$score" ]] && [[ "$score" != "null" ]]; then
|
|
489
|
+
# Check score is between 0 and 10
|
|
490
|
+
if (( $(echo "$score < 0 || $score > 10" | bc -l 2>/dev/null || echo "0") )); then
|
|
491
|
+
error "Invalid CVSS score: $score (must be 0-10)"
|
|
492
|
+
return 1
|
|
493
|
+
fi
|
|
494
|
+
fi
|
|
495
|
+
done
|
|
496
|
+
fi
|
|
497
|
+
|
|
498
|
+
return 0
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
# Validate recommendations have required structure
|
|
502
|
+
# Returns: 0 if valid, 1 if invalid
|
|
503
|
+
validate_recommendations() {
|
|
504
|
+
local output_file="$1"
|
|
505
|
+
|
|
506
|
+
local rec_count
|
|
507
|
+
rec_count=$(json_count "$output_file" ".output.recommendations" 2>/dev/null)
|
|
508
|
+
|
|
509
|
+
if [[ -z "$rec_count" ]] || [[ "$rec_count" == "null" ]]; then
|
|
510
|
+
rec_count=0
|
|
511
|
+
fi
|
|
512
|
+
|
|
513
|
+
debug "Found $rec_count recommendations"
|
|
514
|
+
|
|
515
|
+
# If there are recommendations, validate structure
|
|
516
|
+
if [[ "$rec_count" -gt 0 ]]; then
|
|
517
|
+
local first_priority
|
|
518
|
+
first_priority=$(json_get "$output_file" ".output.recommendations[0].priority" 2>/dev/null)
|
|
519
|
+
|
|
520
|
+
if [[ -n "$first_priority" ]] && [[ "$first_priority" != "null" ]]; then
|
|
521
|
+
if ! validate_enum "$first_priority" "critical" "high" "medium" "low"; then
|
|
522
|
+
error "Recommendation has invalid priority: $first_priority"
|
|
523
|
+
return 1
|
|
524
|
+
fi
|
|
525
|
+
fi
|
|
526
|
+
fi
|
|
527
|
+
|
|
528
|
+
return 0
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
# Main skill-specific validation function
|
|
532
|
+
# Returns: 0 if valid, 1 if invalid
|
|
533
|
+
validate_skill_specific() {
|
|
534
|
+
local output_file="$1"
|
|
535
|
+
local has_errors=false
|
|
536
|
+
|
|
537
|
+
debug "Running security-testing specific validations..."
|
|
538
|
+
|
|
539
|
+
# Validate OWASP findings structure
|
|
540
|
+
if ! validate_owasp_findings "$output_file"; then
|
|
541
|
+
has_errors=true
|
|
542
|
+
else
|
|
543
|
+
success "OWASP categories validation passed"
|
|
544
|
+
fi
|
|
545
|
+
|
|
546
|
+
# Validate security findings structure
|
|
547
|
+
if ! validate_security_findings "$output_file"; then
|
|
548
|
+
has_errors=true
|
|
549
|
+
else
|
|
550
|
+
success "Security findings validation passed"
|
|
551
|
+
fi
|
|
552
|
+
|
|
553
|
+
# Validate CWE format
|
|
554
|
+
if ! validate_cwe_format "$output_file"; then
|
|
555
|
+
has_errors=true
|
|
556
|
+
else
|
|
557
|
+
success "CWE format validation passed"
|
|
558
|
+
fi
|
|
559
|
+
|
|
560
|
+
# Validate CVSS scores
|
|
561
|
+
if ! validate_cvss_scores "$output_file"; then
|
|
562
|
+
has_errors=true
|
|
563
|
+
else
|
|
564
|
+
success "CVSS scores validation passed"
|
|
565
|
+
fi
|
|
566
|
+
|
|
567
|
+
# Validate recommendations
|
|
568
|
+
if ! validate_recommendations "$output_file"; then
|
|
569
|
+
has_errors=true
|
|
570
|
+
else
|
|
571
|
+
success "Recommendations validation passed"
|
|
572
|
+
fi
|
|
573
|
+
|
|
574
|
+
# Check for security tool attribution
|
|
575
|
+
local tools_used
|
|
576
|
+
tools_used=$(json_get "$output_file" ".metadata.toolsUsed" 2>/dev/null)
|
|
577
|
+
if [[ -z "$tools_used" ]] || [[ "$tools_used" == "null" ]] || [[ "$tools_used" == "[]" ]]; then
|
|
578
|
+
warn "No security tools listed in metadata.toolsUsed"
|
|
579
|
+
else
|
|
580
|
+
debug "Security tools used: $tools_used"
|
|
581
|
+
fi
|
|
582
|
+
|
|
583
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
584
|
+
return 1
|
|
585
|
+
fi
|
|
586
|
+
|
|
587
|
+
return 0
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
# =============================================================================
|
|
591
|
+
# Validation Functions (Override base template as needed)
|
|
592
|
+
# =============================================================================
|
|
593
|
+
|
|
594
|
+
validate_tools() {
|
|
595
|
+
if [[ ${#REQUIRED_TOOLS[@]} -eq 0 ]]; then
|
|
596
|
+
debug "No required tools specified"
|
|
597
|
+
return 0
|
|
598
|
+
fi
|
|
599
|
+
|
|
600
|
+
local missing=()
|
|
601
|
+
for tool in "${REQUIRED_TOOLS[@]}"; do
|
|
602
|
+
if ! command_exists "$tool"; then
|
|
603
|
+
missing+=("$tool")
|
|
604
|
+
fi
|
|
605
|
+
done
|
|
606
|
+
|
|
607
|
+
if [[ ${#missing[@]} -gt 0 ]]; then
|
|
608
|
+
error "Missing required tools: ${missing[*]}"
|
|
609
|
+
return 1
|
|
610
|
+
fi
|
|
611
|
+
|
|
612
|
+
debug "All required tools available"
|
|
613
|
+
return 0
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
validate_schema() {
|
|
617
|
+
local output_file="$1"
|
|
618
|
+
|
|
619
|
+
if [[ -z "$SCHEMA_PATH" ]]; then
|
|
620
|
+
debug "No schema path configured, skipping schema validation"
|
|
621
|
+
return 2
|
|
622
|
+
fi
|
|
623
|
+
|
|
624
|
+
if [[ ! -f "$SCHEMA_PATH" ]]; then
|
|
625
|
+
warn "Schema file not found: $SCHEMA_PATH"
|
|
626
|
+
return 2
|
|
627
|
+
fi
|
|
628
|
+
|
|
629
|
+
debug "Validating against schema: $SCHEMA_PATH"
|
|
630
|
+
|
|
631
|
+
local result
|
|
632
|
+
result=$(validate_json_schema "$SCHEMA_PATH" "$output_file" 2>&1)
|
|
633
|
+
local status=$?
|
|
634
|
+
|
|
635
|
+
case $status in
|
|
636
|
+
0)
|
|
637
|
+
success "Schema validation passed"
|
|
638
|
+
return 0
|
|
639
|
+
;;
|
|
640
|
+
1)
|
|
641
|
+
error "Schema validation failed"
|
|
642
|
+
if [[ "$VERBOSE" == "true" ]]; then
|
|
643
|
+
echo "$result" | while read -r line; do
|
|
644
|
+
echo " $line"
|
|
645
|
+
done
|
|
646
|
+
fi
|
|
647
|
+
return 1
|
|
648
|
+
;;
|
|
649
|
+
2)
|
|
650
|
+
warn "Schema validation skipped (no validator available)"
|
|
651
|
+
return 2
|
|
652
|
+
;;
|
|
653
|
+
esac
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
validate_required_fields() {
|
|
657
|
+
local output_file="$1"
|
|
658
|
+
local missing=()
|
|
659
|
+
local empty=()
|
|
660
|
+
|
|
661
|
+
for field in "${REQUIRED_FIELDS[@]}"; do
|
|
662
|
+
local value
|
|
663
|
+
value=$(json_get "$output_file" ".$field" 2>/dev/null)
|
|
664
|
+
if [[ -z "$value" ]] || [[ "$value" == "null" ]]; then
|
|
665
|
+
missing+=("$field")
|
|
666
|
+
fi
|
|
667
|
+
done
|
|
668
|
+
|
|
669
|
+
for field in "${REQUIRED_NON_EMPTY_FIELDS[@]}"; do
|
|
670
|
+
local value
|
|
671
|
+
value=$(json_get "$output_file" ".$field" 2>/dev/null)
|
|
672
|
+
if [[ -z "$value" ]] || [[ "$value" == "null" ]] || [[ "$value" == "" ]] || [[ "$value" == "[]" ]] || [[ "$value" == "{}" ]]; then
|
|
673
|
+
empty+=("$field")
|
|
674
|
+
fi
|
|
675
|
+
done
|
|
676
|
+
|
|
677
|
+
local has_errors=false
|
|
678
|
+
|
|
679
|
+
if [[ ${#missing[@]} -gt 0 ]]; then
|
|
680
|
+
error "Missing required fields: ${missing[*]}"
|
|
681
|
+
has_errors=true
|
|
682
|
+
fi
|
|
683
|
+
|
|
684
|
+
if [[ ${#empty[@]} -gt 0 ]]; then
|
|
685
|
+
error "Empty required fields: ${empty[*]}"
|
|
686
|
+
has_errors=true
|
|
687
|
+
fi
|
|
688
|
+
|
|
689
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
690
|
+
return 1
|
|
691
|
+
fi
|
|
692
|
+
|
|
693
|
+
success "All required fields present and valid"
|
|
694
|
+
return 0
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
validate_enum_fields() {
|
|
698
|
+
local output_file="$1"
|
|
699
|
+
|
|
700
|
+
if [[ ${#ENUM_VALIDATIONS[@]} -eq 0 ]]; then
|
|
701
|
+
return 0
|
|
702
|
+
fi
|
|
703
|
+
|
|
704
|
+
local has_errors=false
|
|
705
|
+
|
|
706
|
+
for validation in "${ENUM_VALIDATIONS[@]}"; do
|
|
707
|
+
local field_path="${validation%%:*}"
|
|
708
|
+
local allowed_values="${validation#*:}"
|
|
709
|
+
|
|
710
|
+
local actual_value
|
|
711
|
+
actual_value=$(json_get "$output_file" "$field_path" 2>/dev/null)
|
|
712
|
+
|
|
713
|
+
if [[ -z "$actual_value" ]] || [[ "$actual_value" == "null" ]]; then
|
|
714
|
+
continue
|
|
715
|
+
fi
|
|
716
|
+
|
|
717
|
+
local found=false
|
|
718
|
+
IFS=',' read -ra allowed_array <<< "$allowed_values"
|
|
719
|
+
for allowed in "${allowed_array[@]}"; do
|
|
720
|
+
if [[ "$actual_value" == "$allowed" ]]; then
|
|
721
|
+
found=true
|
|
722
|
+
break
|
|
723
|
+
fi
|
|
724
|
+
done
|
|
725
|
+
|
|
726
|
+
if [[ "$found" == "false" ]]; then
|
|
727
|
+
error "Invalid value for $field_path: '$actual_value' (allowed: $allowed_values)"
|
|
728
|
+
has_errors=true
|
|
729
|
+
fi
|
|
730
|
+
done
|
|
731
|
+
|
|
732
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
733
|
+
return 1
|
|
734
|
+
fi
|
|
735
|
+
|
|
736
|
+
success "All enum fields have valid values"
|
|
737
|
+
return 0
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
validate_content_terms() {
|
|
741
|
+
local output_file="$1"
|
|
742
|
+
local content
|
|
743
|
+
content=$(cat "$output_file")
|
|
744
|
+
|
|
745
|
+
local has_errors=false
|
|
746
|
+
|
|
747
|
+
if [[ ${#MUST_CONTAIN_TERMS[@]} -gt 0 ]]; then
|
|
748
|
+
local missing_terms=()
|
|
749
|
+
for term in "${MUST_CONTAIN_TERMS[@]}"; do
|
|
750
|
+
if ! grep -qi "$term" <<< "$content"; then
|
|
751
|
+
missing_terms+=("$term")
|
|
752
|
+
fi
|
|
753
|
+
done
|
|
754
|
+
|
|
755
|
+
if [[ ${#missing_terms[@]} -gt 0 ]]; then
|
|
756
|
+
error "Output missing required security terms: ${missing_terms[*]}"
|
|
757
|
+
has_errors=true
|
|
758
|
+
else
|
|
759
|
+
success "All required security terms found"
|
|
760
|
+
fi
|
|
761
|
+
fi
|
|
762
|
+
|
|
763
|
+
if [[ ${#MUST_NOT_CONTAIN_TERMS[@]} -gt 0 ]]; then
|
|
764
|
+
local found_forbidden=()
|
|
765
|
+
for term in "${MUST_NOT_CONTAIN_TERMS[@]}"; do
|
|
766
|
+
if grep -qi "$term" <<< "$content"; then
|
|
767
|
+
found_forbidden+=("$term")
|
|
768
|
+
fi
|
|
769
|
+
done
|
|
770
|
+
|
|
771
|
+
if [[ ${#found_forbidden[@]} -gt 0 ]]; then
|
|
772
|
+
error "Output contains forbidden terms: ${found_forbidden[*]}"
|
|
773
|
+
has_errors=true
|
|
774
|
+
else
|
|
775
|
+
success "No forbidden terms found"
|
|
776
|
+
fi
|
|
777
|
+
fi
|
|
778
|
+
|
|
779
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
780
|
+
return 1
|
|
781
|
+
fi
|
|
782
|
+
|
|
783
|
+
return 0
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
# =============================================================================
|
|
787
|
+
# Main Validation Flow
|
|
788
|
+
# =============================================================================
|
|
789
|
+
|
|
790
|
+
main() {
|
|
791
|
+
if [[ -z "$OUTPUT_FILE" ]]; then
|
|
792
|
+
error "No output file specified"
|
|
793
|
+
echo "Usage: $0 <output-file> [options]"
|
|
794
|
+
echo "Use --help for more information"
|
|
795
|
+
exit 1
|
|
796
|
+
fi
|
|
797
|
+
|
|
798
|
+
if [[ ! -f "$OUTPUT_FILE" ]]; then
|
|
799
|
+
error "Output file not found: $OUTPUT_FILE"
|
|
800
|
+
exit 1
|
|
801
|
+
fi
|
|
802
|
+
|
|
803
|
+
if [[ "$JSON_ONLY" != "true" ]]; then
|
|
804
|
+
echo "=============================================="
|
|
805
|
+
info "Validating $SKILL_NAME Output"
|
|
806
|
+
echo "=============================================="
|
|
807
|
+
echo ""
|
|
808
|
+
echo " Skill: $SKILL_NAME v$SKILL_VERSION"
|
|
809
|
+
echo " File: $OUTPUT_FILE"
|
|
810
|
+
echo " Schema: ${SCHEMA_PATH:-none}"
|
|
811
|
+
echo ""
|
|
812
|
+
fi
|
|
813
|
+
|
|
814
|
+
# Track validation status
|
|
815
|
+
local tool_status="passed"
|
|
816
|
+
local json_status="passed"
|
|
817
|
+
local schema_status="passed"
|
|
818
|
+
local fields_status="passed"
|
|
819
|
+
local enums_status="passed"
|
|
820
|
+
local content_status="passed"
|
|
821
|
+
local specific_status="passed"
|
|
822
|
+
local error_count=0
|
|
823
|
+
local warning_count=0
|
|
824
|
+
|
|
825
|
+
# Step 1: Check Required Tools
|
|
826
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 1: Tool Availability ---"
|
|
827
|
+
|
|
828
|
+
if ! validate_tools; then
|
|
829
|
+
tool_status="failed"
|
|
830
|
+
((error_count++)) || true
|
|
831
|
+
if [[ "$JSON_ONLY" == "true" ]]; then
|
|
832
|
+
output_validation_report "$SKILL_NAME" "skipped" "skipped" "failed"
|
|
833
|
+
fi
|
|
834
|
+
exit $EXIT_SKIP
|
|
835
|
+
fi
|
|
836
|
+
|
|
837
|
+
[[ "$JSON_ONLY" != "true" ]] && success "Tool check passed" && echo ""
|
|
838
|
+
|
|
839
|
+
# Step 2: Validate JSON Syntax
|
|
840
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 2: JSON Syntax ---"
|
|
841
|
+
|
|
842
|
+
if ! validate_json "$OUTPUT_FILE"; then
|
|
843
|
+
json_status="failed"
|
|
844
|
+
((error_count++)) || true
|
|
845
|
+
[[ "$JSON_ONLY" != "true" ]] && error "File is not valid JSON"
|
|
846
|
+
if [[ "$JSON_ONLY" == "true" ]]; then
|
|
847
|
+
output_validation_report "$SKILL_NAME" "failed" "failed" "$tool_status"
|
|
848
|
+
fi
|
|
849
|
+
exit $EXIT_FAIL
|
|
850
|
+
fi
|
|
851
|
+
|
|
852
|
+
[[ "$JSON_ONLY" != "true" ]] && success "JSON syntax valid" && echo ""
|
|
853
|
+
|
|
854
|
+
# Step 3: Validate Against Schema
|
|
855
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 3: Schema Validation ---"
|
|
856
|
+
|
|
857
|
+
local schema_exit_code
|
|
858
|
+
validate_schema "$OUTPUT_FILE" && schema_exit_code=0 || schema_exit_code=$?
|
|
859
|
+
|
|
860
|
+
case $schema_exit_code in
|
|
861
|
+
0) [[ "$JSON_ONLY" != "true" ]] && echo "" ;;
|
|
862
|
+
1) schema_status="failed"; ((error_count++)) || true; [[ "$JSON_ONLY" != "true" ]] && echo "" ;;
|
|
863
|
+
2) schema_status="skipped"; ((warning_count++)) || true; [[ "$JSON_ONLY" != "true" ]] && echo "" ;;
|
|
864
|
+
esac
|
|
865
|
+
|
|
866
|
+
# Step 4: Validate Required Fields
|
|
867
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 4: Required Fields ---"
|
|
868
|
+
|
|
869
|
+
if ! validate_required_fields "$OUTPUT_FILE"; then
|
|
870
|
+
fields_status="failed"
|
|
871
|
+
((error_count++)) || true
|
|
872
|
+
fi
|
|
873
|
+
|
|
874
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
875
|
+
|
|
876
|
+
# Step 5: Validate Enum Values
|
|
877
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 5: Enum Validation ---"
|
|
878
|
+
|
|
879
|
+
if ! validate_enum_fields "$OUTPUT_FILE"; then
|
|
880
|
+
enums_status="failed"
|
|
881
|
+
((error_count++)) || true
|
|
882
|
+
fi
|
|
883
|
+
|
|
884
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
885
|
+
|
|
886
|
+
# Step 6: Validate Security Content Terms
|
|
887
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 6: Security Content Terms ---"
|
|
888
|
+
|
|
889
|
+
if ! validate_content_terms "$OUTPUT_FILE"; then
|
|
890
|
+
content_status="failed"
|
|
891
|
+
((error_count++)) || true
|
|
892
|
+
fi
|
|
893
|
+
|
|
894
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
895
|
+
|
|
896
|
+
# Step 7: Security-Specific Validation
|
|
897
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 7: Security-Specific Validation ---"
|
|
898
|
+
|
|
899
|
+
if ! validate_skill_specific "$OUTPUT_FILE"; then
|
|
900
|
+
specific_status="failed"
|
|
901
|
+
((error_count++)) || true
|
|
902
|
+
fi
|
|
903
|
+
|
|
904
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
905
|
+
|
|
906
|
+
# Determine Overall Status
|
|
907
|
+
local overall_status="passed"
|
|
908
|
+
local content_overall="passed"
|
|
909
|
+
|
|
910
|
+
if [[ "$fields_status" == "failed" ]] || \
|
|
911
|
+
[[ "$enums_status" == "failed" ]] || \
|
|
912
|
+
[[ "$content_status" == "failed" ]] || \
|
|
913
|
+
[[ "$specific_status" == "failed" ]]; then
|
|
914
|
+
content_overall="failed"
|
|
915
|
+
fi
|
|
916
|
+
|
|
917
|
+
if [[ "$json_status" == "failed" ]] || \
|
|
918
|
+
[[ "$schema_status" == "failed" ]] || \
|
|
919
|
+
[[ "$content_overall" == "failed" ]]; then
|
|
920
|
+
overall_status="failed"
|
|
921
|
+
elif [[ "$schema_status" == "skipped" ]]; then
|
|
922
|
+
overall_status="partial"
|
|
923
|
+
fi
|
|
924
|
+
|
|
925
|
+
# Output Results
|
|
926
|
+
if [[ "$JSON_ONLY" == "true" ]]; then
|
|
927
|
+
output_validation_report "$SKILL_NAME" "$schema_status" "$content_overall" "$tool_status"
|
|
928
|
+
else
|
|
929
|
+
echo "=============================================="
|
|
930
|
+
echo "Validation Summary for $SKILL_NAME"
|
|
931
|
+
echo "=============================================="
|
|
932
|
+
echo ""
|
|
933
|
+
echo " Tools: $tool_status"
|
|
934
|
+
echo " JSON Syntax: $json_status"
|
|
935
|
+
echo " Schema: $schema_status"
|
|
936
|
+
echo " Required Fields: $fields_status"
|
|
937
|
+
echo " Enum Values: $enums_status"
|
|
938
|
+
echo " Content Terms: $content_status"
|
|
939
|
+
echo " Security-Specific: $specific_status"
|
|
940
|
+
echo ""
|
|
941
|
+
echo " ------------------------------"
|
|
942
|
+
echo " Overall: $overall_status"
|
|
943
|
+
echo " Errors: $error_count"
|
|
944
|
+
echo " Warnings: $warning_count"
|
|
945
|
+
echo "=============================================="
|
|
946
|
+
echo ""
|
|
947
|
+
fi
|
|
948
|
+
|
|
949
|
+
# Exit with appropriate code
|
|
950
|
+
case "$overall_status" in
|
|
951
|
+
"passed")
|
|
952
|
+
[[ "$JSON_ONLY" != "true" ]] && success "Validation PASSED"
|
|
953
|
+
exit $EXIT_PASS
|
|
954
|
+
;;
|
|
955
|
+
"partial")
|
|
956
|
+
[[ "$JSON_ONLY" != "true" ]] && warn "Validation PARTIAL (some checks skipped)"
|
|
957
|
+
exit $EXIT_PASS
|
|
958
|
+
;;
|
|
959
|
+
"failed")
|
|
960
|
+
[[ "$JSON_ONLY" != "true" ]] && error "Validation FAILED"
|
|
961
|
+
exit $EXIT_FAIL
|
|
962
|
+
;;
|
|
963
|
+
esac
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
# Run main function
|
|
967
|
+
main
|