agentic-qe 3.4.0 → 3.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/.validation/README.md +111 -0
- package/.claude/skills/.validation/examples/chaos-engineering-output.example.json +530 -0
- package/.claude/skills/.validation/examples/performance-testing-output.example.json +252 -0
- package/.claude/skills/.validation/examples/security-testing-output.example.json +413 -0
- package/.claude/skills/.validation/examples/testability-scoring-output.example.json +350 -0
- package/.claude/skills/.validation/schemas/skill-eval.schema.json +452 -0
- package/.claude/skills/.validation/schemas/skill-frontmatter.schema.json +341 -0
- package/.claude/skills/.validation/schemas/skill-output-meta.schema.json +199 -0
- package/.claude/skills/.validation/schemas/skill-output.template.json +610 -0
- package/.claude/skills/.validation/skill-validation-mcp-integration.md +250 -0
- package/.claude/skills/.validation/templates/eval.template.yaml +366 -0
- package/.claude/skills/.validation/templates/schemas/output.json +145 -0
- package/.claude/skills/.validation/templates/security-testing-eval.template.yaml +725 -0
- package/.claude/skills/.validation/templates/skill-frontmatter.example.yaml +225 -0
- package/.claude/skills/.validation/templates/validate.template.sh +1060 -0
- package/.claude/skills/.validation/templates/validator-lib.sh +1031 -0
- package/.claude/skills/.validation/test-data/invalid-output.json +5 -0
- package/.claude/skills/.validation/test-data/minimal-output.json +9 -0
- package/.claude/skills/.validation/test-data/sample-output.json +73 -0
- package/.claude/skills/TRUST-TIERS.md +158 -0
- package/.claude/skills/a11y-ally/SKILL.md +1664 -1658
- package/.claude/skills/a11y-ally/evals/a11y-ally.yaml +376 -0
- package/.claude/skills/a11y-ally/schemas/output.json +549 -0
- package/.claude/skills/a11y-ally/scripts/validate.sh +690 -0
- package/.claude/skills/accessibility-testing/SKILL.md +8 -1
- package/.claude/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -0
- package/.claude/skills/accessibility-testing/schemas/output.json +776 -0
- package/.claude/skills/accessibility-testing/scripts/validate.sh +943 -0
- package/.claude/skills/accessibility-testing/test-data/sample-output.json +191 -0
- package/.claude/skills/agentic-quality-engineering/SKILL.md +4 -0
- package/.claude/skills/agentic-quality-engineering/schemas/output.json +577 -0
- package/.claude/skills/api-testing-patterns/SKILL.md +5 -0
- package/.claude/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -0
- package/.claude/skills/api-testing-patterns/schemas/output.json +845 -0
- package/.claude/skills/api-testing-patterns/scripts/validate.sh +931 -0
- package/.claude/skills/aqe-v2-v3-migration/schemas/output.json +513 -0
- package/.claude/skills/aqe-v2-v3-migration/skill.md +4 -0
- package/.claude/skills/brutal-honesty-review/SKILL.md +5 -0
- package/.claude/skills/brutal-honesty-review/schemas/output.json +291 -0
- package/.claude/skills/brutal-honesty-review/scripts/validate.sh +130 -0
- package/.claude/skills/bug-reporting-excellence/SKILL.md +5 -0
- package/.claude/skills/bug-reporting-excellence/schemas/output.json +288 -0
- package/.claude/skills/bug-reporting-excellence/scripts/validate.sh +108 -0
- package/.claude/skills/chaos-engineering-resilience/SKILL.md +5 -0
- package/.claude/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -0
- package/.claude/skills/chaos-engineering-resilience/schemas/output.json +1205 -0
- package/.claude/skills/chaos-engineering-resilience/scripts/validate.sh +1107 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/SKILL.md +6 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -0
- package/.claude/skills/cicd-pipeline-qe-orchestrator/scripts/validate.sh +310 -0
- package/.claude/skills/code-review-quality/SKILL.md +5 -0
- package/.claude/skills/code-review-quality/schemas/output.json +264 -0
- package/.claude/skills/code-review-quality/scripts/validate.sh +113 -0
- package/.claude/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -0
- package/.claude/skills/compatibility-testing/schemas/output.json +551 -0
- package/.claude/skills/compatibility-testing/scripts/validate.sh +605 -0
- package/.claude/skills/compliance-testing/SKILL.md +7 -0
- package/.claude/skills/compliance-testing/evals/compliance-testing.yaml +1107 -0
- package/.claude/skills/compliance-testing/schemas/output.json +845 -0
- package/.claude/skills/compliance-testing/scripts/validate.sh +888 -0
- package/.claude/skills/consultancy-practices/SKILL.md +4 -0
- package/.claude/skills/consultancy-practices/schemas/output.json +282 -0
- package/.claude/skills/context-driven-testing/SKILL.md +3 -0
- package/.claude/skills/contract-testing/SKILL.md +5 -0
- package/.claude/skills/contract-testing/evals/contract-testing.yaml +748 -0
- package/.claude/skills/contract-testing/schemas/output.json +638 -0
- package/.claude/skills/contract-testing/scripts/validate.sh +1033 -0
- package/.claude/skills/database-testing/SKILL.md +5 -0
- package/.claude/skills/database-testing/evals/database-testing.yaml +968 -0
- package/.claude/skills/database-testing/schemas/output.json +1446 -0
- package/.claude/skills/database-testing/scripts/validate.sh +1099 -0
- package/.claude/skills/exploratory-testing-advanced/SKILL.md +3 -0
- package/.claude/skills/holistic-testing-pact/SKILL.md +3 -0
- package/.claude/skills/localization-testing/SKILL.md +6 -0
- package/.claude/skills/localization-testing/evals/localization-testing.yaml +544 -0
- package/.claude/skills/localization-testing/schemas/output.json +325 -0
- package/.claude/skills/localization-testing/scripts/validate.sh +131 -0
- package/.claude/skills/mobile-testing/SKILL.md +6 -0
- package/.claude/skills/mobile-testing/evals/mobile-testing.yaml +537 -0
- package/.claude/skills/mobile-testing/schemas/output.json +318 -0
- package/.claude/skills/mobile-testing/scripts/validate.sh +127 -0
- package/.claude/skills/mutation-testing/SKILL.md +8 -1
- package/.claude/skills/mutation-testing/evals/mutation-testing.yaml +652 -0
- package/.claude/skills/mutation-testing/schemas/output.json +707 -0
- package/.claude/skills/mutation-testing/scripts/validate.sh +992 -0
- package/.claude/skills/mutation-testing/test-data/sample-output.json +295 -0
- package/.claude/skills/n8n-expression-testing/SKILL.md +6 -0
- package/.claude/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -0
- package/.claude/skills/n8n-expression-testing/schemas/output.json +369 -0
- package/.claude/skills/n8n-expression-testing/scripts/validate.sh +162 -0
- package/.claude/skills/n8n-integration-testing-patterns/SKILL.md +6 -0
- package/.claude/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -0
- package/.claude/skills/n8n-integration-testing-patterns/schemas/output.json +291 -0
- package/.claude/skills/n8n-integration-testing-patterns/scripts/validate.sh +104 -0
- package/.claude/skills/n8n-security-testing/SKILL.md +6 -0
- package/.claude/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -0
- package/.claude/skills/n8n-security-testing/schemas/output.json +293 -0
- package/.claude/skills/n8n-security-testing/scripts/validate.sh +120 -0
- package/.claude/skills/n8n-trigger-testing-strategies/SKILL.md +6 -0
- package/.claude/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -0
- package/.claude/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -0
- package/.claude/skills/n8n-trigger-testing-strategies/scripts/validate.sh +121 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/SKILL.md +6 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -0
- package/.claude/skills/n8n-workflow-testing-fundamentals/scripts/validate.sh +126 -0
- package/.claude/skills/performance-analysis/SKILL.md +6 -0
- package/.claude/skills/performance-analysis/evals/performance-analysis.yaml +144 -0
- package/.claude/skills/performance-analysis/schemas/output.json +588 -0
- package/.claude/skills/performance-analysis/scripts/validate.sh +63 -0
- package/.claude/skills/performance-testing/SKILL.md +5 -0
- package/.claude/skills/performance-testing/evals/performance-testing.yaml +772 -0
- package/.claude/skills/performance-testing/schemas/output.json +1184 -0
- package/.claude/skills/performance-testing/scripts/validate.sh +1090 -0
- package/.claude/skills/qcsd-ideation-swarm/SKILL.md +1759 -1753
- package/.claude/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -0
- package/.claude/skills/qcsd-ideation-swarm/schemas/output.json +568 -0
- package/.claude/skills/qcsd-ideation-swarm/scripts/validate.sh +56 -0
- package/.claude/skills/qe-chaos-resilience/SKILL.md +6 -0
- package/.claude/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -0
- package/.claude/skills/qe-chaos-resilience/schemas/output.json +314 -0
- package/.claude/skills/qe-chaos-resilience/scripts/validate.sh +401 -0
- package/.claude/skills/qe-code-intelligence/SKILL.md +6 -0
- package/.claude/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -0
- package/.claude/skills/qe-code-intelligence/schemas/output.json +315 -0
- package/.claude/skills/qe-code-intelligence/scripts/validate.sh +408 -0
- package/.claude/skills/qe-contract-testing/SKILL.md +6 -0
- package/.claude/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -0
- package/.claude/skills/qe-contract-testing/schemas/output.json +295 -0
- package/.claude/skills/qe-contract-testing/scripts/validate.sh +421 -0
- package/.claude/skills/qe-coverage-analysis/SKILL.md +6 -0
- package/.claude/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -0
- package/.claude/skills/qe-coverage-analysis/schemas/output.json +286 -0
- package/.claude/skills/qe-coverage-analysis/scripts/validate.sh +453 -0
- package/.claude/skills/qe-defect-intelligence/SKILL.md +6 -0
- package/.claude/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -0
- package/.claude/skills/qe-defect-intelligence/schemas/output.json +283 -0
- package/.claude/skills/qe-defect-intelligence/scripts/validate.sh +466 -0
- package/.claude/skills/qe-iterative-loop/SKILL.md +445 -440
- package/.claude/skills/qe-iterative-loop/schemas/output.json +264 -0
- package/.claude/skills/qe-iterative-loop/scripts/validate.sh +474 -0
- package/.claude/skills/qe-learning-optimization/SKILL.md +6 -0
- package/.claude/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -0
- package/.claude/skills/qe-learning-optimization/schemas/output.json +288 -0
- package/.claude/skills/qe-learning-optimization/scripts/validate.sh +56 -0
- package/.claude/skills/qe-quality-assessment/SKILL.md +6 -0
- package/.claude/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -0
- package/.claude/skills/qe-quality-assessment/schemas/output.json +550 -0
- package/.claude/skills/qe-quality-assessment/scripts/validate.sh +352 -0
- package/.claude/skills/qe-requirements-validation/SKILL.md +6 -0
- package/.claude/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -0
- package/.claude/skills/qe-requirements-validation/schemas/output.json +587 -0
- package/.claude/skills/qe-requirements-validation/scripts/validate.sh +275 -0
- package/.claude/skills/qe-security-compliance/SKILL.md +6 -0
- package/.claude/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -0
- package/.claude/skills/qe-security-compliance/schemas/output.json +498 -0
- package/.claude/skills/qe-security-compliance/scripts/validate.sh +305 -0
- package/.claude/skills/qe-test-execution/SKILL.md +6 -0
- package/.claude/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -0
- package/.claude/skills/qe-test-execution/schemas/output.json +529 -0
- package/.claude/skills/qe-test-execution/scripts/validate.sh +323 -0
- package/.claude/skills/qe-test-generation/SKILL.md +6 -0
- package/.claude/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -0
- package/.claude/skills/qe-test-generation/schemas/output.json +439 -0
- package/.claude/skills/qe-test-generation/scripts/validate.sh +56 -0
- package/.claude/skills/qe-visual-accessibility/SKILL.md +6 -0
- package/.claude/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -0
- package/.claude/skills/qe-visual-accessibility/schemas/output.json +491 -0
- package/.claude/skills/qe-visual-accessibility/scripts/validate.sh +56 -0
- package/.claude/skills/quality-metrics/SKILL.md +6 -0
- package/.claude/skills/quality-metrics/evals/quality-metrics.yaml +494 -0
- package/.claude/skills/quality-metrics/schemas/output.json +403 -0
- package/.claude/skills/quality-metrics/scripts/validate.sh +434 -0
- package/.claude/skills/refactoring-patterns/SKILL.md +5 -0
- package/.claude/skills/refactoring-patterns/schemas/output.json +475 -0
- package/.claude/skills/refactoring-patterns/scripts/validate.sh +241 -0
- package/.claude/skills/regression-testing/SKILL.md +6 -0
- package/.claude/skills/regression-testing/evals/regression-testing.yaml +504 -0
- package/.claude/skills/regression-testing/schemas/output.json +311 -0
- package/.claude/skills/regression-testing/scripts/validate.sh +130 -0
- package/.claude/skills/risk-based-testing/SKILL.md +6 -0
- package/.claude/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -0
- package/.claude/skills/risk-based-testing/schemas/output.json +480 -0
- package/.claude/skills/risk-based-testing/scripts/validate.sh +56 -0
- package/.claude/skills/security-testing/SKILL.md +5 -0
- package/.claude/skills/security-testing/evals/security-testing.yaml +789 -0
- package/.claude/skills/security-testing/schemas/output.json +879 -0
- package/.claude/skills/security-testing/scripts/validate.sh +967 -0
- package/.claude/skills/security-visual-testing/SKILL.md +223 -217
- package/.claude/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -0
- package/.claude/skills/security-visual-testing/schemas/output.json +486 -0
- package/.claude/skills/security-visual-testing/scripts/validate.sh +748 -0
- package/.claude/skills/sherlock-review/SKILL.md +5 -0
- package/.claude/skills/sherlock-review/schemas/output.json +297 -0
- package/.claude/skills/sherlock-review/scripts/validate.sh +115 -0
- package/.claude/skills/shift-left-testing/SKILL.md +6 -0
- package/.claude/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -0
- package/.claude/skills/shift-left-testing/schemas/output.json +459 -0
- package/.claude/skills/shift-left-testing/scripts/validate.sh +56 -0
- package/.claude/skills/shift-right-testing/SKILL.md +6 -0
- package/.claude/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -0
- package/.claude/skills/shift-right-testing/schemas/output.json +418 -0
- package/.claude/skills/shift-right-testing/scripts/validate.sh +56 -0
- package/.claude/skills/six-thinking-hats/SKILL.md +3 -0
- package/.claude/skills/tdd-london-chicago/SKILL.md +5 -0
- package/.claude/skills/tdd-london-chicago/schemas/output.json +444 -0
- package/.claude/skills/tdd-london-chicago/scripts/validate.sh +214 -0
- package/.claude/skills/technical-writing/SKILL.md +4 -0
- package/.claude/skills/technical-writing/schemas/output.json +268 -0
- package/.claude/skills/test-automation-strategy/SKILL.md +6 -0
- package/.claude/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -0
- package/.claude/skills/test-automation-strategy/schemas/output.json +444 -0
- package/.claude/skills/test-automation-strategy/scripts/validate.sh +195 -0
- package/.claude/skills/test-data-management/SKILL.md +6 -0
- package/.claude/skills/test-data-management/evals/test-data-management.yaml +504 -0
- package/.claude/skills/test-data-management/schemas/output.json +284 -0
- package/.claude/skills/test-data-management/scripts/validate.sh +137 -0
- package/.claude/skills/test-design-techniques/SKILL.md +6 -0
- package/.claude/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -0
- package/.claude/skills/test-design-techniques/schemas/output.json +295 -0
- package/.claude/skills/test-design-techniques/scripts/validate.sh +91 -0
- package/.claude/skills/test-environment-management/SKILL.md +4 -0
- package/.claude/skills/test-environment-management/schemas/output.json +310 -0
- package/.claude/skills/test-reporting-analytics/SKILL.md +6 -0
- package/.claude/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -0
- package/.claude/skills/test-reporting-analytics/schemas/output.json +329 -0
- package/.claude/skills/test-reporting-analytics/scripts/validate.sh +77 -0
- package/.claude/skills/testability-scoring/SKILL.md +5 -0
- package/.claude/skills/testability-scoring/evals/testability-scoring.yaml +814 -0
- package/.claude/skills/testability-scoring/schemas/output.json +606 -0
- package/.claude/skills/testability-scoring/scripts/validate.sh +891 -0
- package/.claude/skills/trust-tier-manifest.json +2333 -0
- package/.claude/skills/verification-quality/SKILL.md +6 -0
- package/.claude/skills/verification-quality/evals/verification-quality.yaml +150 -0
- package/.claude/skills/verification-quality/schemas/output.json +432 -0
- package/.claude/skills/verification-quality/scripts/validate.sh +77 -0
- package/.claude/skills/visual-testing-advanced/SKILL.md +6 -0
- package/.claude/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -0
- package/.claude/skills/visual-testing-advanced/schemas/output.json +294 -0
- package/.claude/skills/visual-testing-advanced/scripts/validate.sh +77 -0
- package/.claude/skills/xp-practices/SKILL.md +3 -0
- package/CHANGELOG.md +23 -0
- package/README.md +39 -2
- package/package.json +4 -1
- package/scripts/run-skill-eval.ts +1097 -0
- package/scripts/test-schema-validation.js +301 -0
- package/scripts/update-skill-manifest.ts +751 -0
- package/scripts/validate-skill-frontmatter.js +141 -0
- package/v3/CHANGELOG.md +21 -0
- package/v3/README.md +43 -3
- package/v3/assets/agents/v3/qe-test-idea-rewriter.md +375 -375
- package/v3/assets/skills/.validation/README.md +111 -0
- package/v3/assets/skills/.validation/examples/chaos-engineering-output.example.json +530 -0
- package/v3/assets/skills/.validation/examples/performance-testing-output.example.json +252 -0
- package/v3/assets/skills/.validation/examples/security-testing-output.example.json +413 -0
- package/v3/assets/skills/.validation/examples/testability-scoring-output.example.json +350 -0
- package/v3/assets/skills/.validation/schemas/skill-eval.schema.json +452 -0
- package/v3/assets/skills/.validation/schemas/skill-frontmatter.schema.json +341 -0
- package/v3/assets/skills/.validation/schemas/skill-output-meta.schema.json +199 -0
- package/v3/assets/skills/.validation/schemas/skill-output.template.json +610 -0
- package/v3/assets/skills/.validation/skill-validation-mcp-integration.md +250 -0
- package/v3/assets/skills/.validation/templates/eval.template.yaml +366 -0
- package/v3/assets/skills/.validation/templates/schemas/output.json +145 -0
- package/v3/assets/skills/.validation/templates/security-testing-eval.template.yaml +725 -0
- package/v3/assets/skills/.validation/templates/skill-frontmatter.example.yaml +225 -0
- package/v3/assets/skills/.validation/templates/validate.template.sh +1060 -0
- package/v3/assets/skills/.validation/templates/validator-lib.sh +1031 -0
- package/v3/assets/skills/.validation/test-data/invalid-output.json +5 -0
- package/v3/assets/skills/.validation/test-data/minimal-output.json +9 -0
- package/v3/assets/skills/.validation/test-data/sample-output.json +73 -0
- package/v3/assets/skills/accessibility-testing/SKILL.md +8 -1
- package/v3/assets/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -0
- package/v3/assets/skills/accessibility-testing/schemas/output.json +776 -0
- package/v3/assets/skills/accessibility-testing/scripts/validate.sh +943 -0
- package/v3/assets/skills/accessibility-testing/test-data/sample-output.json +191 -0
- package/v3/assets/skills/agentic-quality-engineering/SKILL.md +4 -0
- package/v3/assets/skills/agentic-quality-engineering/schemas/output.json +577 -0
- package/v3/assets/skills/api-testing-patterns/SKILL.md +5 -0
- package/v3/assets/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -0
- package/v3/assets/skills/api-testing-patterns/schemas/output.json +845 -0
- package/v3/assets/skills/api-testing-patterns/scripts/validate.sh +931 -0
- package/v3/assets/skills/aqe-v2-v3-migration/schemas/output.json +513 -0
- package/v3/assets/skills/aqe-v2-v3-migration/skill.md +20 -16
- package/v3/assets/skills/bug-reporting-excellence/SKILL.md +5 -0
- package/v3/assets/skills/bug-reporting-excellence/schemas/output.json +288 -0
- package/v3/assets/skills/bug-reporting-excellence/scripts/validate.sh +108 -0
- package/v3/assets/skills/chaos-engineering-resilience/SKILL.md +5 -0
- package/v3/assets/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -0
- package/v3/assets/skills/chaos-engineering-resilience/schemas/output.json +1205 -0
- package/v3/assets/skills/chaos-engineering-resilience/scripts/validate.sh +1107 -0
- package/v3/assets/skills/code-review-quality/SKILL.md +5 -0
- package/v3/assets/skills/code-review-quality/schemas/output.json +264 -0
- package/v3/assets/skills/code-review-quality/scripts/validate.sh +113 -0
- package/v3/assets/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -0
- package/v3/assets/skills/compatibility-testing/schemas/output.json +551 -0
- package/v3/assets/skills/compatibility-testing/scripts/validate.sh +605 -0
- package/v3/assets/skills/compliance-testing/SKILL.md +7 -0
- package/v3/assets/skills/compliance-testing/evals/compliance-testing.yaml +1107 -0
- package/v3/assets/skills/compliance-testing/schemas/output.json +845 -0
- package/v3/assets/skills/compliance-testing/scripts/validate.sh +888 -0
- package/v3/assets/skills/consultancy-practices/SKILL.md +4 -0
- package/v3/assets/skills/consultancy-practices/schemas/output.json +282 -0
- package/v3/assets/skills/context-driven-testing/SKILL.md +3 -0
- package/v3/assets/skills/contract-testing/SKILL.md +5 -0
- package/v3/assets/skills/contract-testing/evals/contract-testing.yaml +748 -0
- package/v3/assets/skills/contract-testing/schemas/output.json +638 -0
- package/v3/assets/skills/contract-testing/scripts/validate.sh +1033 -0
- package/v3/assets/skills/database-testing/SKILL.md +5 -0
- package/v3/assets/skills/database-testing/evals/database-testing.yaml +968 -0
- package/v3/assets/skills/database-testing/schemas/output.json +1446 -0
- package/v3/assets/skills/database-testing/scripts/validate.sh +1099 -0
- package/v3/assets/skills/exploratory-testing-advanced/SKILL.md +3 -0
- package/v3/assets/skills/holistic-testing-pact/SKILL.md +3 -0
- package/v3/assets/skills/localization-testing/SKILL.md +6 -0
- package/v3/assets/skills/localization-testing/evals/localization-testing.yaml +544 -0
- package/v3/assets/skills/localization-testing/schemas/output.json +325 -0
- package/v3/assets/skills/localization-testing/scripts/validate.sh +131 -0
- package/v3/assets/skills/mobile-testing/SKILL.md +6 -0
- package/v3/assets/skills/mobile-testing/evals/mobile-testing.yaml +537 -0
- package/v3/assets/skills/mobile-testing/schemas/output.json +318 -0
- package/v3/assets/skills/mobile-testing/scripts/validate.sh +127 -0
- package/v3/assets/skills/mutation-testing/SKILL.md +8 -1
- package/v3/assets/skills/mutation-testing/evals/mutation-testing.yaml +652 -0
- package/v3/assets/skills/mutation-testing/schemas/output.json +707 -0
- package/v3/assets/skills/mutation-testing/scripts/validate.sh +992 -0
- package/v3/assets/skills/mutation-testing/test-data/sample-output.json +295 -0
- package/v3/assets/skills/n8n-expression-testing/scripts/validate.sh +162 -0
- package/v3/assets/skills/n8n-integration-testing-patterns/scripts/validate.sh +104 -0
- package/v3/assets/skills/n8n-security-testing/scripts/validate.sh +120 -0
- package/v3/assets/skills/n8n-trigger-testing-strategies/scripts/validate.sh +121 -0
- package/v3/assets/skills/n8n-workflow-testing-fundamentals/scripts/validate.sh +126 -0
- package/v3/assets/skills/performance-testing/SKILL.md +5 -0
- package/v3/assets/skills/performance-testing/evals/performance-testing.yaml +772 -0
- package/v3/assets/skills/performance-testing/schemas/output.json +1184 -0
- package/v3/assets/skills/performance-testing/scripts/validate.sh +1090 -0
- package/v3/assets/skills/qe-chaos-resilience/SKILL.md +6 -0
- package/v3/assets/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -0
- package/v3/assets/skills/qe-chaos-resilience/schemas/output.json +314 -0
- package/v3/assets/skills/qe-chaos-resilience/scripts/validate.sh +401 -0
- package/v3/assets/skills/qe-code-intelligence/SKILL.md +6 -0
- package/v3/assets/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -0
- package/v3/assets/skills/qe-code-intelligence/schemas/output.json +315 -0
- package/v3/assets/skills/qe-code-intelligence/scripts/validate.sh +408 -0
- package/v3/assets/skills/qe-contract-testing/SKILL.md +6 -0
- package/v3/assets/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -0
- package/v3/assets/skills/qe-contract-testing/schemas/output.json +295 -0
- package/v3/assets/skills/qe-contract-testing/scripts/validate.sh +421 -0
- package/v3/assets/skills/qe-coverage-analysis/SKILL.md +6 -0
- package/v3/assets/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -0
- package/v3/assets/skills/qe-coverage-analysis/schemas/output.json +286 -0
- package/v3/assets/skills/qe-coverage-analysis/scripts/validate.sh +453 -0
- package/v3/assets/skills/qe-defect-intelligence/SKILL.md +6 -0
- package/v3/assets/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -0
- package/v3/assets/skills/qe-defect-intelligence/schemas/output.json +283 -0
- package/v3/assets/skills/qe-defect-intelligence/scripts/validate.sh +466 -0
- package/v3/assets/skills/qe-iterative-loop/SKILL.md +445 -443
- package/v3/assets/skills/qe-iterative-loop/schemas/output.json +264 -0
- package/v3/assets/skills/qe-iterative-loop/scripts/validate.sh +474 -0
- package/v3/assets/skills/qe-learning-optimization/SKILL.md +6 -0
- package/v3/assets/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -0
- package/v3/assets/skills/qe-learning-optimization/schemas/output.json +288 -0
- package/v3/assets/skills/qe-learning-optimization/scripts/validate.sh +56 -0
- package/v3/assets/skills/qe-quality-assessment/SKILL.md +6 -0
- package/v3/assets/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -0
- package/v3/assets/skills/qe-quality-assessment/schemas/output.json +550 -0
- package/v3/assets/skills/qe-quality-assessment/scripts/validate.sh +352 -0
- package/v3/assets/skills/qe-requirements-validation/SKILL.md +6 -0
- package/v3/assets/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -0
- package/v3/assets/skills/qe-requirements-validation/schemas/output.json +587 -0
- package/v3/assets/skills/qe-requirements-validation/scripts/validate.sh +275 -0
- package/v3/assets/skills/qe-security-compliance/SKILL.md +6 -0
- package/v3/assets/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -0
- package/v3/assets/skills/qe-security-compliance/schemas/output.json +498 -0
- package/v3/assets/skills/qe-security-compliance/scripts/validate.sh +305 -0
- package/v3/assets/skills/qe-test-execution/SKILL.md +6 -0
- package/v3/assets/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -0
- package/v3/assets/skills/qe-test-execution/schemas/output.json +529 -0
- package/v3/assets/skills/qe-test-execution/scripts/validate.sh +323 -0
- package/v3/assets/skills/qe-test-generation/SKILL.md +6 -0
- package/v3/assets/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -0
- package/v3/assets/skills/qe-test-generation/schemas/output.json +439 -0
- package/v3/assets/skills/qe-test-generation/scripts/validate.sh +56 -0
- package/v3/assets/skills/qe-visual-accessibility/SKILL.md +6 -0
- package/v3/assets/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -0
- package/v3/assets/skills/qe-visual-accessibility/schemas/output.json +491 -0
- package/v3/assets/skills/qe-visual-accessibility/scripts/validate.sh +56 -0
- package/v3/assets/skills/quality-metrics/SKILL.md +6 -0
- package/v3/assets/skills/quality-metrics/evals/quality-metrics.yaml +494 -0
- package/v3/assets/skills/quality-metrics/schemas/output.json +403 -0
- package/v3/assets/skills/quality-metrics/scripts/validate.sh +434 -0
- package/v3/assets/skills/refactoring-patterns/SKILL.md +5 -0
- package/v3/assets/skills/refactoring-patterns/schemas/output.json +475 -0
- package/v3/assets/skills/refactoring-patterns/scripts/validate.sh +241 -0
- package/v3/assets/skills/regression-testing/SKILL.md +6 -0
- package/v3/assets/skills/regression-testing/evals/regression-testing.yaml +504 -0
- package/v3/assets/skills/regression-testing/schemas/output.json +311 -0
- package/v3/assets/skills/regression-testing/scripts/validate.sh +130 -0
- package/v3/assets/skills/risk-based-testing/SKILL.md +6 -0
- package/v3/assets/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -0
- package/v3/assets/skills/risk-based-testing/schemas/output.json +480 -0
- package/v3/assets/skills/risk-based-testing/scripts/validate.sh +56 -0
- package/v3/assets/skills/security-testing/SKILL.md +5 -0
- package/v3/assets/skills/security-testing/evals/security-testing.yaml +789 -0
- package/v3/assets/skills/security-testing/schemas/output.json +879 -0
- package/v3/assets/skills/security-testing/scripts/validate.sh +967 -0
- package/v3/assets/skills/shift-left-testing/SKILL.md +6 -0
- package/v3/assets/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -0
- package/v3/assets/skills/shift-left-testing/schemas/output.json +459 -0
- package/v3/assets/skills/shift-left-testing/scripts/validate.sh +56 -0
- package/v3/assets/skills/shift-right-testing/SKILL.md +6 -0
- package/v3/assets/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -0
- package/v3/assets/skills/shift-right-testing/schemas/output.json +418 -0
- package/v3/assets/skills/shift-right-testing/scripts/validate.sh +56 -0
- package/v3/assets/skills/six-thinking-hats/SKILL.md +3 -0
- package/v3/assets/skills/tdd-london-chicago/SKILL.md +5 -0
- package/v3/assets/skills/tdd-london-chicago/schemas/output.json +444 -0
- package/v3/assets/skills/tdd-london-chicago/scripts/validate.sh +214 -0
- package/v3/assets/skills/technical-writing/SKILL.md +4 -0
- package/v3/assets/skills/technical-writing/schemas/output.json +268 -0
- package/v3/assets/skills/test-automation-strategy/SKILL.md +6 -0
- package/v3/assets/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -0
- package/v3/assets/skills/test-automation-strategy/schemas/output.json +444 -0
- package/v3/assets/skills/test-automation-strategy/scripts/validate.sh +195 -0
- package/v3/assets/skills/test-data-management/SKILL.md +6 -0
- package/v3/assets/skills/test-data-management/evals/test-data-management.yaml +504 -0
- package/v3/assets/skills/test-data-management/schemas/output.json +284 -0
- package/v3/assets/skills/test-data-management/scripts/validate.sh +137 -0
- package/v3/assets/skills/test-design-techniques/SKILL.md +6 -0
- package/v3/assets/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -0
- package/v3/assets/skills/test-design-techniques/schemas/output.json +295 -0
- package/v3/assets/skills/test-design-techniques/scripts/validate.sh +91 -0
- package/v3/assets/skills/test-environment-management/SKILL.md +4 -0
- package/v3/assets/skills/test-environment-management/schemas/output.json +310 -0
- package/v3/assets/skills/test-reporting-analytics/SKILL.md +6 -0
- package/v3/assets/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -0
- package/v3/assets/skills/test-reporting-analytics/schemas/output.json +329 -0
- package/v3/assets/skills/test-reporting-analytics/scripts/validate.sh +77 -0
- package/v3/assets/skills/verification-quality/SKILL.md +6 -0
- package/v3/assets/skills/verification-quality/evals/verification-quality.yaml +150 -0
- package/v3/assets/skills/verification-quality/schemas/output.json +432 -0
- package/v3/assets/skills/verification-quality/scripts/validate.sh +77 -0
- package/v3/assets/skills/visual-testing-advanced/SKILL.md +6 -0
- package/v3/assets/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -0
- package/v3/assets/skills/visual-testing-advanced/schemas/output.json +294 -0
- package/v3/assets/skills/visual-testing-advanced/scripts/validate.sh +77 -0
- package/v3/assets/skills/xp-practices/SKILL.md +3 -0
- package/v3/assets/templates/validation-report.md.hbs +139 -0
- package/v3/assets/templates/validation-summary.json +56 -0
- package/v3/dist/cli/bundle.js +11853 -5316
- package/v3/dist/cli/commands/eval.d.ts +18 -0
- package/v3/dist/cli/commands/eval.d.ts.map +1 -0
- package/v3/dist/cli/commands/eval.js +505 -0
- package/v3/dist/cli/commands/eval.js.map +1 -0
- package/v3/dist/cli/commands/validate-swarm.d.ts +16 -0
- package/v3/dist/cli/commands/validate-swarm.d.ts.map +1 -0
- package/v3/dist/cli/commands/validate-swarm.js +251 -0
- package/v3/dist/cli/commands/validate-swarm.js.map +1 -0
- package/v3/dist/cli/commands/validate.d.ts +14 -0
- package/v3/dist/cli/commands/validate.d.ts.map +1 -0
- package/v3/dist/cli/commands/validate.js +408 -0
- package/v3/dist/cli/commands/validate.js.map +1 -0
- package/v3/dist/cli/index.js +6 -0
- package/v3/dist/cli/index.js.map +1 -1
- package/v3/dist/index.d.ts +2 -0
- package/v3/dist/index.d.ts.map +1 -1
- package/v3/dist/index.js +10 -0
- package/v3/dist/index.js.map +1 -1
- package/v3/dist/init/skills-installer.d.ts +6 -0
- package/v3/dist/init/skills-installer.d.ts.map +1 -1
- package/v3/dist/init/skills-installer.js +48 -0
- package/v3/dist/init/skills-installer.js.map +1 -1
- package/v3/dist/learning/index.d.ts +2 -0
- package/v3/dist/learning/index.d.ts.map +1 -1
- package/v3/dist/learning/index.js +4 -0
- package/v3/dist/learning/index.js.map +1 -1
- package/v3/dist/learning/skill-validation-learner.d.ts +264 -0
- package/v3/dist/learning/skill-validation-learner.d.ts.map +1 -0
- package/v3/dist/learning/skill-validation-learner.js +515 -0
- package/v3/dist/learning/skill-validation-learner.js.map +1 -0
- package/v3/dist/mcp/bundle.js +10043 -3630
- package/v3/dist/validation/index.d.ts +19 -0
- package/v3/dist/validation/index.d.ts.map +1 -0
- package/v3/dist/validation/index.js +31 -0
- package/v3/dist/validation/index.js.map +1 -0
- package/v3/dist/validation/parallel-eval-runner.d.ts +307 -0
- package/v3/dist/validation/parallel-eval-runner.d.ts.map +1 -0
- package/v3/dist/validation/parallel-eval-runner.js +566 -0
- package/v3/dist/validation/parallel-eval-runner.js.map +1 -0
- package/v3/dist/validation/swarm-skill-validator.d.ts +282 -0
- package/v3/dist/validation/swarm-skill-validator.d.ts.map +1 -0
- package/v3/dist/validation/swarm-skill-validator.js +460 -0
- package/v3/dist/validation/swarm-skill-validator.js.map +1 -0
- package/v3/dist/validation/validation-result-aggregator.d.ts +232 -0
- package/v3/dist/validation/validation-result-aggregator.d.ts.map +1 -0
- package/v3/dist/validation/validation-result-aggregator.js +630 -0
- package/v3/dist/validation/validation-result-aggregator.js.map +1 -0
- package/v3/package.json +1 -1
|
@@ -0,0 +1,943 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# =============================================================================
|
|
3
|
+
# AQE Accessibility Testing Skill Validator v1.0.0
|
|
4
|
+
# Validates accessibility audit output against WCAG 2.2 schema
|
|
5
|
+
# =============================================================================
|
|
6
|
+
#
|
|
7
|
+
# This validator checks accessibility testing skill output for:
|
|
8
|
+
# - Valid JSON structure conforming to accessibility-testing schema
|
|
9
|
+
# - WCAG 2.2 compliance fields (conformance level, POUR breakdown)
|
|
10
|
+
# - Proper finding structure with WCAG criteria and impact
|
|
11
|
+
# - Video caption validation (if video analysis was performed)
|
|
12
|
+
#
|
|
13
|
+
# Usage: ./validate.sh <output-file> [options]
|
|
14
|
+
#
|
|
15
|
+
# Options:
|
|
16
|
+
# --self-test Run validator self-test mode
|
|
17
|
+
# --verbose Enable verbose output
|
|
18
|
+
# --json Output results as JSON only
|
|
19
|
+
# --list-tools Show available validation tools
|
|
20
|
+
# --help Show this help message
|
|
21
|
+
#
|
|
22
|
+
# Exit Codes:
|
|
23
|
+
# 0 - Validation passed
|
|
24
|
+
# 1 - Validation failed
|
|
25
|
+
# 2 - Validation skipped (missing required tools)
|
|
26
|
+
#
|
|
27
|
+
# =============================================================================
|
|
28
|
+
|
|
29
|
+
set -euo pipefail
|
|
30
|
+
|
|
31
|
+
# Get script directory
|
|
32
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
33
|
+
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
34
|
+
# .claude/skills/accessibility-testing -> go up 3 levels to project root
|
|
35
|
+
PROJECT_ROOT="$(cd "$SKILL_DIR/../../.." && pwd)"
|
|
36
|
+
|
|
37
|
+
# Source shared library - try multiple locations
|
|
38
|
+
VALIDATOR_LIB=""
|
|
39
|
+
for lib_path in \
|
|
40
|
+
"$PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh" \
|
|
41
|
+
"$SKILL_DIR/scripts/validator-lib.sh" \
|
|
42
|
+
"$SCRIPT_DIR/validator-lib.sh"; do
|
|
43
|
+
if [[ -f "$lib_path" ]]; then
|
|
44
|
+
VALIDATOR_LIB="$lib_path"
|
|
45
|
+
break
|
|
46
|
+
fi
|
|
47
|
+
done
|
|
48
|
+
|
|
49
|
+
if [[ -n "$VALIDATOR_LIB" ]]; then
|
|
50
|
+
# shellcheck source=/dev/null
|
|
51
|
+
source "$VALIDATOR_LIB"
|
|
52
|
+
else
|
|
53
|
+
echo "ERROR: Validator library not found"
|
|
54
|
+
echo "Searched:"
|
|
55
|
+
echo " - $PROJECT_ROOT/.claude/skills/.validation/templates/validator-lib.sh"
|
|
56
|
+
echo " - $SKILL_DIR/scripts/validator-lib.sh"
|
|
57
|
+
echo " - $SCRIPT_DIR/validator-lib.sh"
|
|
58
|
+
echo ""
|
|
59
|
+
echo "Make sure the validator-lib.sh file exists in one of these locations."
|
|
60
|
+
exit 1
|
|
61
|
+
fi
|
|
62
|
+
|
|
63
|
+
# =============================================================================
|
|
64
|
+
# SKILL-SPECIFIC CONFIGURATION
|
|
65
|
+
# =============================================================================
|
|
66
|
+
|
|
67
|
+
# Skill name (matches SKILL.md)
|
|
68
|
+
SKILL_NAME="accessibility-testing"
|
|
69
|
+
|
|
70
|
+
# Skill version
|
|
71
|
+
SKILL_VERSION="1.0.0"
|
|
72
|
+
|
|
73
|
+
# Required tools (validation FAILS with exit 2 if missing)
|
|
74
|
+
# jq is required for JSON parsing
|
|
75
|
+
REQUIRED_TOOLS=("jq")
|
|
76
|
+
|
|
77
|
+
# Optional tools (validation continues with warnings if missing)
|
|
78
|
+
# These are the accessibility testing tools that enable full auditing
|
|
79
|
+
OPTIONAL_TOOLS=("axe" "pa11y" "lighthouse" "ffmpeg" "python3" "ajv" "jsonschema")
|
|
80
|
+
|
|
81
|
+
# Path to output JSON schema
|
|
82
|
+
SCHEMA_PATH="$SKILL_DIR/schemas/output.json"
|
|
83
|
+
|
|
84
|
+
# Sample output for self-test
|
|
85
|
+
SAMPLE_OUTPUT_PATH="$SKILL_DIR/test-data/sample-output.json"
|
|
86
|
+
|
|
87
|
+
# =============================================================================
|
|
88
|
+
# CONTENT VALIDATION CONFIGURATION
|
|
89
|
+
# =============================================================================
|
|
90
|
+
|
|
91
|
+
# Required fields in output
|
|
92
|
+
REQUIRED_FIELDS=("skillName" "status" "output" "output.conformanceLevel" "output.pourBreakdown")
|
|
93
|
+
|
|
94
|
+
# Fields that must have non-null, non-empty values
|
|
95
|
+
REQUIRED_NON_EMPTY_FIELDS=("output.summary" "output.pourBreakdown")
|
|
96
|
+
|
|
97
|
+
# Terms that MUST appear in accessibility testing output
|
|
98
|
+
MUST_CONTAIN_TERMS=("WCAG" "accessibility")
|
|
99
|
+
|
|
100
|
+
# Terms that must NOT appear in output (indicates failure/hallucination)
|
|
101
|
+
# Note: example.com is allowed since it's a valid placeholder domain in samples
|
|
102
|
+
MUST_NOT_CONTAIN_TERMS=("TODO" "placeholder" "FIXME")
|
|
103
|
+
|
|
104
|
+
# Enum fields and allowed values
|
|
105
|
+
ENUM_VALIDATIONS=(
|
|
106
|
+
".status:success,partial,failed,skipped"
|
|
107
|
+
".output.conformanceLevel:A,AA,AAA,none"
|
|
108
|
+
".output.wcagVersion:2.0,2.1,2.2"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
# Minimum array lengths - findings can be empty (0) for compliant sites
|
|
112
|
+
MIN_ARRAY_LENGTHS=(
|
|
113
|
+
".output.findings:0"
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
# =============================================================================
|
|
117
|
+
# Argument Parsing
|
|
118
|
+
# =============================================================================
|
|
119
|
+
|
|
120
|
+
OUTPUT_FILE=""
|
|
121
|
+
SELF_TEST=false
|
|
122
|
+
VERBOSE=false
|
|
123
|
+
JSON_ONLY=false
|
|
124
|
+
LIST_TOOLS=false
|
|
125
|
+
|
|
126
|
+
while [[ $# -gt 0 ]]; do
|
|
127
|
+
case "$1" in
|
|
128
|
+
--self-test)
|
|
129
|
+
SELF_TEST=true
|
|
130
|
+
shift
|
|
131
|
+
;;
|
|
132
|
+
--verbose|-v)
|
|
133
|
+
VERBOSE=true
|
|
134
|
+
export AQE_DEBUG=1
|
|
135
|
+
shift
|
|
136
|
+
;;
|
|
137
|
+
--json)
|
|
138
|
+
JSON_ONLY=true
|
|
139
|
+
shift
|
|
140
|
+
;;
|
|
141
|
+
--list-tools)
|
|
142
|
+
LIST_TOOLS=true
|
|
143
|
+
shift
|
|
144
|
+
;;
|
|
145
|
+
-h|--help)
|
|
146
|
+
cat << 'HELP_EOF'
|
|
147
|
+
AQE Accessibility Testing Validator
|
|
148
|
+
|
|
149
|
+
Validates accessibility audit output against WCAG 2.2 schema requirements.
|
|
150
|
+
|
|
151
|
+
Usage: ./validate.sh <output-file> [options]
|
|
152
|
+
./validate.sh --self-test [--verbose]
|
|
153
|
+
./validate.sh --list-tools
|
|
154
|
+
|
|
155
|
+
Arguments:
|
|
156
|
+
<output-file> Path to accessibility audit JSON file to validate
|
|
157
|
+
|
|
158
|
+
Options:
|
|
159
|
+
--self-test Run validator self-test mode
|
|
160
|
+
--verbose, -v Enable verbose/debug output
|
|
161
|
+
--json Output results as JSON only (for CI integration)
|
|
162
|
+
--list-tools Show available validation tools and exit
|
|
163
|
+
--help, -h Show this help message
|
|
164
|
+
|
|
165
|
+
Validation Checks:
|
|
166
|
+
- JSON syntax and schema compliance
|
|
167
|
+
- Required fields: skillName, status, output, conformanceLevel, pourBreakdown
|
|
168
|
+
- WCAG conformance level: A, AA, AAA, or none
|
|
169
|
+
- POUR principle breakdown (Perceivable, Operable, Understandable, Robust)
|
|
170
|
+
- Finding structure: id, title, severity, wcagCriterion, pourPrinciple
|
|
171
|
+
- Severity values: critical, serious, moderate, minor
|
|
172
|
+
- WCAG criteria format: X.Y.Z (e.g., 1.1.1, 2.4.7)
|
|
173
|
+
|
|
174
|
+
Exit Codes:
|
|
175
|
+
0 - Validation passed
|
|
176
|
+
1 - Validation failed
|
|
177
|
+
2 - Validation skipped (missing required tools)
|
|
178
|
+
|
|
179
|
+
Examples:
|
|
180
|
+
./validate.sh audit-output.json # Validate audit file
|
|
181
|
+
./validate.sh audit-output.json --json # JSON output for CI
|
|
182
|
+
./validate.sh --self-test --verbose # Self-test with debug
|
|
183
|
+
./validate.sh --list-tools # Show available tools
|
|
184
|
+
|
|
185
|
+
HELP_EOF
|
|
186
|
+
exit 0
|
|
187
|
+
;;
|
|
188
|
+
-*)
|
|
189
|
+
error "Unknown option: $1"
|
|
190
|
+
echo "Use --help for usage information"
|
|
191
|
+
exit 1
|
|
192
|
+
;;
|
|
193
|
+
*)
|
|
194
|
+
OUTPUT_FILE="$1"
|
|
195
|
+
shift
|
|
196
|
+
;;
|
|
197
|
+
esac
|
|
198
|
+
done
|
|
199
|
+
|
|
200
|
+
# Handle --list-tools mode
|
|
201
|
+
if [[ "$LIST_TOOLS" == "true" ]]; then
|
|
202
|
+
echo "=============================================="
|
|
203
|
+
echo "Accessibility Testing Validation Tools"
|
|
204
|
+
echo "=============================================="
|
|
205
|
+
echo ""
|
|
206
|
+
echo "Required tools (for validation):"
|
|
207
|
+
for tool in "${REQUIRED_TOOLS[@]}"; do
|
|
208
|
+
if command_exists "$tool"; then
|
|
209
|
+
echo " [OK] $tool"
|
|
210
|
+
else
|
|
211
|
+
echo " [MISSING] $tool"
|
|
212
|
+
fi
|
|
213
|
+
done
|
|
214
|
+
echo ""
|
|
215
|
+
echo "Optional tools (for full accessibility auditing):"
|
|
216
|
+
for tool in "${OPTIONAL_TOOLS[@]}"; do
|
|
217
|
+
if command_exists "$tool"; then
|
|
218
|
+
echo " [OK] $tool"
|
|
219
|
+
else
|
|
220
|
+
echo " [MISSING] $tool"
|
|
221
|
+
fi
|
|
222
|
+
done
|
|
223
|
+
echo ""
|
|
224
|
+
echo "Accessibility audit tools status:"
|
|
225
|
+
if command_exists "axe"; then
|
|
226
|
+
echo " [OK] axe-core (automated WCAG testing)"
|
|
227
|
+
else
|
|
228
|
+
echo " [MISSING] axe-core - Install: npm install -g @axe-core/cli"
|
|
229
|
+
fi
|
|
230
|
+
if command_exists "pa11y"; then
|
|
231
|
+
echo " [OK] pa11y (HTML CodeSniffer wrapper)"
|
|
232
|
+
else
|
|
233
|
+
echo " [MISSING] pa11y - Install: npm install -g pa11y"
|
|
234
|
+
fi
|
|
235
|
+
if command_exists "lighthouse"; then
|
|
236
|
+
echo " [OK] lighthouse (Google accessibility audit)"
|
|
237
|
+
else
|
|
238
|
+
echo " [MISSING] lighthouse - Install: npm install -g lighthouse"
|
|
239
|
+
fi
|
|
240
|
+
if command_exists "ffmpeg"; then
|
|
241
|
+
echo " [OK] ffmpeg (video frame extraction for captions)"
|
|
242
|
+
else
|
|
243
|
+
echo " [MISSING] ffmpeg - Install: apt install ffmpeg or brew install ffmpeg"
|
|
244
|
+
fi
|
|
245
|
+
echo ""
|
|
246
|
+
echo "Schema validation support:"
|
|
247
|
+
if command_exists "ajv"; then
|
|
248
|
+
echo " [OK] ajv (preferred)"
|
|
249
|
+
elif command_exists "jsonschema"; then
|
|
250
|
+
echo " [OK] jsonschema CLI"
|
|
251
|
+
elif command_exists "python3" && python3 -c "import jsonschema" 2>/dev/null; then
|
|
252
|
+
echo " [OK] python3 + jsonschema module"
|
|
253
|
+
else
|
|
254
|
+
echo " [MISSING] No schema validator available"
|
|
255
|
+
echo " Install: npm install -g ajv-cli"
|
|
256
|
+
echo " Or: pip install jsonschema"
|
|
257
|
+
fi
|
|
258
|
+
exit 0
|
|
259
|
+
fi
|
|
260
|
+
|
|
261
|
+
# =============================================================================
|
|
262
|
+
# Self-Test Mode
|
|
263
|
+
# =============================================================================
|
|
264
|
+
|
|
265
|
+
if [[ "$SELF_TEST" == "true" ]]; then
|
|
266
|
+
echo "=============================================="
|
|
267
|
+
info "Running $SKILL_NAME Validator Self-Test"
|
|
268
|
+
echo "=============================================="
|
|
269
|
+
echo ""
|
|
270
|
+
echo "Validator version: $AQE_VALIDATOR_VERSION"
|
|
271
|
+
echo "Skill version: $SKILL_VERSION"
|
|
272
|
+
echo ""
|
|
273
|
+
|
|
274
|
+
self_test_passed=true
|
|
275
|
+
self_test_warnings=0
|
|
276
|
+
|
|
277
|
+
# Step 1: Check Required Tools
|
|
278
|
+
echo "--- Step 1: Required Tools ---"
|
|
279
|
+
for tool in "${REQUIRED_TOOLS[@]}"; do
|
|
280
|
+
if command_exists "$tool"; then
|
|
281
|
+
success "Required tool available: $tool"
|
|
282
|
+
else
|
|
283
|
+
error "Required tool MISSING: $tool"
|
|
284
|
+
self_test_passed=false
|
|
285
|
+
fi
|
|
286
|
+
done
|
|
287
|
+
echo ""
|
|
288
|
+
|
|
289
|
+
# Step 2: Check Optional Tools
|
|
290
|
+
echo "--- Step 2: Optional Tools (Accessibility) ---"
|
|
291
|
+
available_optional=0
|
|
292
|
+
for tool in "${OPTIONAL_TOOLS[@]}"; do
|
|
293
|
+
if command_exists "$tool"; then
|
|
294
|
+
success "Optional tool available: $tool"
|
|
295
|
+
((available_optional++)) || true
|
|
296
|
+
else
|
|
297
|
+
warn "Optional tool missing: $tool"
|
|
298
|
+
((self_test_warnings++)) || true
|
|
299
|
+
fi
|
|
300
|
+
done
|
|
301
|
+
echo ""
|
|
302
|
+
|
|
303
|
+
# Step 3: Check Schema
|
|
304
|
+
echo "--- Step 3: Schema File ---"
|
|
305
|
+
if [[ -f "$SCHEMA_PATH" ]]; then
|
|
306
|
+
success "Schema file exists: $SCHEMA_PATH"
|
|
307
|
+
if validate_json "$SCHEMA_PATH" 2>/dev/null; then
|
|
308
|
+
success "Schema file is valid JSON"
|
|
309
|
+
else
|
|
310
|
+
error "Schema file is NOT valid JSON"
|
|
311
|
+
self_test_passed=false
|
|
312
|
+
fi
|
|
313
|
+
else
|
|
314
|
+
error "Schema file not found: $SCHEMA_PATH"
|
|
315
|
+
self_test_passed=false
|
|
316
|
+
fi
|
|
317
|
+
echo ""
|
|
318
|
+
|
|
319
|
+
# Step 4: Test WCAG-specific validation
|
|
320
|
+
echo "--- Step 4: WCAG Validation Functions ---"
|
|
321
|
+
|
|
322
|
+
# Create a test output
|
|
323
|
+
test_file=$(mktemp)
|
|
324
|
+
cat > "$test_file" << 'EOF'
|
|
325
|
+
{
|
|
326
|
+
"skillName": "accessibility-testing",
|
|
327
|
+
"version": "1.0.0",
|
|
328
|
+
"timestamp": "2026-02-02T12:00:00Z",
|
|
329
|
+
"status": "partial",
|
|
330
|
+
"trustTier": 3,
|
|
331
|
+
"output": {
|
|
332
|
+
"summary": "Accessibility audit found 3 WCAG 2.2 violations requiring remediation.",
|
|
333
|
+
"conformanceLevel": "none",
|
|
334
|
+
"targetLevel": "AA",
|
|
335
|
+
"wcagVersion": "2.2",
|
|
336
|
+
"pourBreakdown": {
|
|
337
|
+
"perceivable": { "score": 75, "violationCount": 2, "criticalCount": 0, "seriousCount": 1, "moderateCount": 1, "minorCount": 0 },
|
|
338
|
+
"operable": { "score": 90, "violationCount": 1, "criticalCount": 0, "seriousCount": 0, "moderateCount": 1, "minorCount": 0 },
|
|
339
|
+
"understandable": { "score": 100, "violationCount": 0, "criticalCount": 0, "seriousCount": 0, "moderateCount": 0, "minorCount": 0 },
|
|
340
|
+
"robust": { "score": 100, "violationCount": 0, "criticalCount": 0, "seriousCount": 0, "moderateCount": 0, "minorCount": 0 }
|
|
341
|
+
},
|
|
342
|
+
"findings": [
|
|
343
|
+
{
|
|
344
|
+
"id": "A11Y-001",
|
|
345
|
+
"title": "Image missing alt text",
|
|
346
|
+
"severity": "serious",
|
|
347
|
+
"wcagCriterion": "1.1.1",
|
|
348
|
+
"wcagLevel": "A",
|
|
349
|
+
"pourPrinciple": "perceivable",
|
|
350
|
+
"description": "Image element does not have an alt attribute",
|
|
351
|
+
"impact": "Screen reader users cannot understand the image content"
|
|
352
|
+
}
|
|
353
|
+
]
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
EOF
|
|
357
|
+
|
|
358
|
+
# Test validation
|
|
359
|
+
if validate_json "$test_file" 2>/dev/null; then
|
|
360
|
+
success "Test output is valid JSON"
|
|
361
|
+
|
|
362
|
+
# Check WCAG-specific fields
|
|
363
|
+
conformance=$(json_get "$test_file" ".output.conformanceLevel" 2>/dev/null)
|
|
364
|
+
if [[ -n "$conformance" ]]; then
|
|
365
|
+
success "WCAG conformance level detected: $conformance"
|
|
366
|
+
else
|
|
367
|
+
error "WCAG conformance level not found"
|
|
368
|
+
self_test_passed=false
|
|
369
|
+
fi
|
|
370
|
+
|
|
371
|
+
# Check POUR breakdown
|
|
372
|
+
pour_perceivable=$(json_get "$test_file" ".output.pourBreakdown.perceivable.score" 2>/dev/null)
|
|
373
|
+
if [[ -n "$pour_perceivable" ]]; then
|
|
374
|
+
success "POUR breakdown detected: perceivable score = $pour_perceivable"
|
|
375
|
+
else
|
|
376
|
+
error "POUR breakdown not found"
|
|
377
|
+
self_test_passed=false
|
|
378
|
+
fi
|
|
379
|
+
|
|
380
|
+
# Check finding structure
|
|
381
|
+
finding_wcag=$(json_get "$test_file" ".output.findings[0].wcagCriterion" 2>/dev/null)
|
|
382
|
+
if [[ "$finding_wcag" == "1.1.1" ]]; then
|
|
383
|
+
success "Finding WCAG criterion detected: $finding_wcag"
|
|
384
|
+
else
|
|
385
|
+
warn "Finding WCAG criterion not properly formatted"
|
|
386
|
+
((self_test_warnings++)) || true
|
|
387
|
+
fi
|
|
388
|
+
else
|
|
389
|
+
error "Test output validation failed"
|
|
390
|
+
self_test_passed=false
|
|
391
|
+
fi
|
|
392
|
+
|
|
393
|
+
rm -f "$test_file"
|
|
394
|
+
echo ""
|
|
395
|
+
|
|
396
|
+
# Step 5: Run Library Self-Test
|
|
397
|
+
echo "--- Step 5: Validator Library Self-Test ---"
|
|
398
|
+
if run_self_test 2>/dev/null; then
|
|
399
|
+
success "Library self-test passed"
|
|
400
|
+
else
|
|
401
|
+
error "Library self-test FAILED"
|
|
402
|
+
self_test_passed=false
|
|
403
|
+
fi
|
|
404
|
+
echo ""
|
|
405
|
+
|
|
406
|
+
# Summary
|
|
407
|
+
echo "=============================================="
|
|
408
|
+
echo "Self-Test Summary for $SKILL_NAME"
|
|
409
|
+
echo "=============================================="
|
|
410
|
+
|
|
411
|
+
if [[ "$self_test_passed" == "true" ]]; then
|
|
412
|
+
if [[ $self_test_warnings -gt 0 ]]; then
|
|
413
|
+
warn "Self-test PASSED with $self_test_warnings warning(s)"
|
|
414
|
+
echo ""
|
|
415
|
+
echo "The validator is functional but has reduced capabilities."
|
|
416
|
+
echo "Consider installing missing accessibility tools for full auditing."
|
|
417
|
+
exit 0
|
|
418
|
+
else
|
|
419
|
+
success "Self-test PASSED (all checks successful)"
|
|
420
|
+
exit 0
|
|
421
|
+
fi
|
|
422
|
+
else
|
|
423
|
+
error "Self-test FAILED"
|
|
424
|
+
exit 1
|
|
425
|
+
fi
|
|
426
|
+
fi
|
|
427
|
+
|
|
428
|
+
# =============================================================================
|
|
429
|
+
# SKILL-SPECIFIC VALIDATION FUNCTIONS
|
|
430
|
+
# =============================================================================
|
|
431
|
+
|
|
432
|
+
# Validate WCAG criterion format (X.Y.Z pattern)
|
|
433
|
+
validate_wcag_criterion() {
|
|
434
|
+
local criterion="$1"
|
|
435
|
+
if [[ "$criterion" =~ ^[1-4]\.[0-9]+\.[0-9]+$ ]]; then
|
|
436
|
+
return 0
|
|
437
|
+
else
|
|
438
|
+
return 1
|
|
439
|
+
fi
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
# Validate POUR principle
|
|
443
|
+
validate_pour_principle() {
|
|
444
|
+
local principle="$1"
|
|
445
|
+
case "$principle" in
|
|
446
|
+
perceivable|operable|understandable|robust)
|
|
447
|
+
return 0
|
|
448
|
+
;;
|
|
449
|
+
*)
|
|
450
|
+
return 1
|
|
451
|
+
;;
|
|
452
|
+
esac
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
# Validate finding severity
|
|
456
|
+
validate_finding_severity() {
|
|
457
|
+
local severity="$1"
|
|
458
|
+
case "$severity" in
|
|
459
|
+
critical|serious|moderate|minor)
|
|
460
|
+
return 0
|
|
461
|
+
;;
|
|
462
|
+
*)
|
|
463
|
+
return 1
|
|
464
|
+
;;
|
|
465
|
+
esac
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
# =============================================================================
|
|
469
|
+
# Skill-Specific Validation
|
|
470
|
+
# =============================================================================
|
|
471
|
+
|
|
472
|
+
validate_skill_specific() {
|
|
473
|
+
local output_file="$1"
|
|
474
|
+
local has_errors=false
|
|
475
|
+
|
|
476
|
+
debug "Running accessibility-specific validations..."
|
|
477
|
+
|
|
478
|
+
# Validate POUR breakdown structure
|
|
479
|
+
echo " Checking POUR breakdown..."
|
|
480
|
+
local pour_principles=("perceivable" "operable" "understandable" "robust")
|
|
481
|
+
for principle in "${pour_principles[@]}"; do
|
|
482
|
+
local score
|
|
483
|
+
score=$(json_get "$output_file" ".output.pourBreakdown.$principle.score" 2>/dev/null)
|
|
484
|
+
if [[ -z "$score" ]] || [[ "$score" == "null" ]]; then
|
|
485
|
+
error "Missing POUR principle: $principle"
|
|
486
|
+
has_errors=true
|
|
487
|
+
else
|
|
488
|
+
debug "POUR $principle score: $score"
|
|
489
|
+
fi
|
|
490
|
+
done
|
|
491
|
+
|
|
492
|
+
# Validate findings have proper WCAG structure
|
|
493
|
+
echo " Checking finding structure..."
|
|
494
|
+
local finding_count
|
|
495
|
+
finding_count=$(json_count "$output_file" ".output.findings" 2>/dev/null)
|
|
496
|
+
|
|
497
|
+
if [[ -n "$finding_count" ]] && [[ "$finding_count" -gt 0 ]]; then
|
|
498
|
+
# Check first finding for required fields
|
|
499
|
+
local first_finding_id
|
|
500
|
+
first_finding_id=$(json_get "$output_file" ".output.findings[0].id" 2>/dev/null)
|
|
501
|
+
if [[ -z "$first_finding_id" ]] || [[ ! "$first_finding_id" =~ ^A11Y-[0-9]+$ ]]; then
|
|
502
|
+
warn "Finding ID should match pattern A11Y-XXX, got: $first_finding_id"
|
|
503
|
+
fi
|
|
504
|
+
|
|
505
|
+
local first_finding_wcag
|
|
506
|
+
first_finding_wcag=$(json_get "$output_file" ".output.findings[0].wcagCriterion" 2>/dev/null)
|
|
507
|
+
if [[ -n "$first_finding_wcag" ]] && ! validate_wcag_criterion "$first_finding_wcag"; then
|
|
508
|
+
error "Invalid WCAG criterion format: $first_finding_wcag (expected X.Y.Z)"
|
|
509
|
+
has_errors=true
|
|
510
|
+
fi
|
|
511
|
+
|
|
512
|
+
local first_finding_severity
|
|
513
|
+
first_finding_severity=$(json_get "$output_file" ".output.findings[0].severity" 2>/dev/null)
|
|
514
|
+
if [[ -n "$first_finding_severity" ]] && ! validate_finding_severity "$first_finding_severity"; then
|
|
515
|
+
error "Invalid finding severity: $first_finding_severity"
|
|
516
|
+
has_errors=true
|
|
517
|
+
fi
|
|
518
|
+
|
|
519
|
+
local first_finding_pour
|
|
520
|
+
first_finding_pour=$(json_get "$output_file" ".output.findings[0].pourPrinciple" 2>/dev/null)
|
|
521
|
+
if [[ -n "$first_finding_pour" ]] && ! validate_pour_principle "$first_finding_pour"; then
|
|
522
|
+
error "Invalid POUR principle: $first_finding_pour"
|
|
523
|
+
has_errors=true
|
|
524
|
+
fi
|
|
525
|
+
|
|
526
|
+
success "Found $finding_count accessibility findings with valid structure"
|
|
527
|
+
else
|
|
528
|
+
info "No findings detected (site may be fully compliant)"
|
|
529
|
+
fi
|
|
530
|
+
|
|
531
|
+
# Check conformance level consistency
|
|
532
|
+
echo " Checking conformance level consistency..."
|
|
533
|
+
local conformance_level
|
|
534
|
+
conformance_level=$(json_get "$output_file" ".output.conformanceLevel" 2>/dev/null)
|
|
535
|
+
local target_level
|
|
536
|
+
target_level=$(json_get "$output_file" ".output.targetLevel" 2>/dev/null)
|
|
537
|
+
|
|
538
|
+
if [[ -n "$finding_count" ]] && [[ "$finding_count" -gt 0 ]]; then
|
|
539
|
+
# If there are findings, conformance should likely not be AAA
|
|
540
|
+
if [[ "$conformance_level" == "AAA" ]]; then
|
|
541
|
+
warn "Conformance level is AAA but findings exist - verify this is correct"
|
|
542
|
+
fi
|
|
543
|
+
fi
|
|
544
|
+
|
|
545
|
+
# Check for video caption outputs if video analysis was done
|
|
546
|
+
local video_count
|
|
547
|
+
video_count=$(json_count "$output_file" ".output.videoCaptions" 2>/dev/null)
|
|
548
|
+
if [[ -n "$video_count" ]] && [[ "$video_count" -gt 0 ]]; then
|
|
549
|
+
success "Video accessibility analysis included: $video_count video(s)"
|
|
550
|
+
|
|
551
|
+
# Check first video has caption path
|
|
552
|
+
local caption_path
|
|
553
|
+
caption_path=$(json_get "$output_file" ".output.videoCaptions[0].captionsPath" 2>/dev/null)
|
|
554
|
+
if [[ -n "$caption_path" ]] && [[ "$caption_path" != "null" ]]; then
|
|
555
|
+
success "WebVTT captions generated: $caption_path"
|
|
556
|
+
fi
|
|
557
|
+
fi
|
|
558
|
+
|
|
559
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
560
|
+
return 1
|
|
561
|
+
fi
|
|
562
|
+
|
|
563
|
+
success "Accessibility-specific validation passed"
|
|
564
|
+
return 0
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
# =============================================================================
|
|
568
|
+
# Validation Functions (from template)
|
|
569
|
+
# =============================================================================
|
|
570
|
+
|
|
571
|
+
validate_tools() {
|
|
572
|
+
if [[ ${#REQUIRED_TOOLS[@]} -eq 0 ]]; then
|
|
573
|
+
debug "No required tools specified"
|
|
574
|
+
return 0
|
|
575
|
+
fi
|
|
576
|
+
|
|
577
|
+
local missing=()
|
|
578
|
+
for tool in "${REQUIRED_TOOLS[@]}"; do
|
|
579
|
+
if ! command_exists "$tool"; then
|
|
580
|
+
missing+=("$tool")
|
|
581
|
+
fi
|
|
582
|
+
done
|
|
583
|
+
|
|
584
|
+
if [[ ${#missing[@]} -gt 0 ]]; then
|
|
585
|
+
error "Missing required tools: ${missing[*]}"
|
|
586
|
+
return 1
|
|
587
|
+
fi
|
|
588
|
+
|
|
589
|
+
debug "All required tools available"
|
|
590
|
+
return 0
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
validate_schema() {
|
|
594
|
+
local output_file="$1"
|
|
595
|
+
|
|
596
|
+
if [[ -z "$SCHEMA_PATH" ]]; then
|
|
597
|
+
debug "No schema path configured, skipping schema validation"
|
|
598
|
+
return 2
|
|
599
|
+
fi
|
|
600
|
+
|
|
601
|
+
if [[ ! -f "$SCHEMA_PATH" ]]; then
|
|
602
|
+
warn "Schema file not found: $SCHEMA_PATH"
|
|
603
|
+
return 2
|
|
604
|
+
fi
|
|
605
|
+
|
|
606
|
+
debug "Validating against schema: $SCHEMA_PATH"
|
|
607
|
+
|
|
608
|
+
local result
|
|
609
|
+
result=$(validate_json_schema "$SCHEMA_PATH" "$output_file" 2>&1)
|
|
610
|
+
local status=$?
|
|
611
|
+
|
|
612
|
+
case $status in
|
|
613
|
+
0)
|
|
614
|
+
success "Schema validation passed"
|
|
615
|
+
return 0
|
|
616
|
+
;;
|
|
617
|
+
1)
|
|
618
|
+
error "Schema validation failed"
|
|
619
|
+
if [[ "$VERBOSE" == "true" ]]; then
|
|
620
|
+
echo "$result" | while read -r line; do
|
|
621
|
+
echo " $line"
|
|
622
|
+
done
|
|
623
|
+
fi
|
|
624
|
+
return 1
|
|
625
|
+
;;
|
|
626
|
+
2)
|
|
627
|
+
warn "Schema validation skipped (no validator available)"
|
|
628
|
+
return 2
|
|
629
|
+
;;
|
|
630
|
+
esac
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
validate_required_fields() {
|
|
634
|
+
local output_file="$1"
|
|
635
|
+
local missing=()
|
|
636
|
+
local empty=()
|
|
637
|
+
|
|
638
|
+
for field in "${REQUIRED_FIELDS[@]}"; do
|
|
639
|
+
local value
|
|
640
|
+
value=$(json_get "$output_file" ".$field" 2>/dev/null)
|
|
641
|
+
if [[ -z "$value" ]] || [[ "$value" == "null" ]]; then
|
|
642
|
+
missing+=("$field")
|
|
643
|
+
fi
|
|
644
|
+
done
|
|
645
|
+
|
|
646
|
+
for field in "${REQUIRED_NON_EMPTY_FIELDS[@]}"; do
|
|
647
|
+
local value
|
|
648
|
+
value=$(json_get "$output_file" ".$field" 2>/dev/null)
|
|
649
|
+
if [[ -z "$value" ]] || [[ "$value" == "null" ]] || [[ "$value" == "" ]] || [[ "$value" == "[]" ]] || [[ "$value" == "{}" ]]; then
|
|
650
|
+
empty+=("$field")
|
|
651
|
+
fi
|
|
652
|
+
done
|
|
653
|
+
|
|
654
|
+
local has_errors=false
|
|
655
|
+
|
|
656
|
+
if [[ ${#missing[@]} -gt 0 ]]; then
|
|
657
|
+
error "Missing required fields: ${missing[*]}"
|
|
658
|
+
has_errors=true
|
|
659
|
+
fi
|
|
660
|
+
|
|
661
|
+
if [[ ${#empty[@]} -gt 0 ]]; then
|
|
662
|
+
error "Empty required fields: ${empty[*]}"
|
|
663
|
+
has_errors=true
|
|
664
|
+
fi
|
|
665
|
+
|
|
666
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
667
|
+
return 1
|
|
668
|
+
fi
|
|
669
|
+
|
|
670
|
+
success "All required fields present and valid"
|
|
671
|
+
return 0
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
validate_enum_fields() {
|
|
675
|
+
local output_file="$1"
|
|
676
|
+
|
|
677
|
+
if [[ ${#ENUM_VALIDATIONS[@]} -eq 0 ]]; then
|
|
678
|
+
return 0
|
|
679
|
+
fi
|
|
680
|
+
|
|
681
|
+
local has_errors=false
|
|
682
|
+
|
|
683
|
+
for validation in "${ENUM_VALIDATIONS[@]}"; do
|
|
684
|
+
local field_path="${validation%%:*}"
|
|
685
|
+
local allowed_values="${validation#*:}"
|
|
686
|
+
|
|
687
|
+
local actual_value
|
|
688
|
+
actual_value=$(json_get "$output_file" "$field_path" 2>/dev/null)
|
|
689
|
+
|
|
690
|
+
if [[ -z "$actual_value" ]] || [[ "$actual_value" == "null" ]]; then
|
|
691
|
+
debug "Enum field $field_path is empty/null (may be optional)"
|
|
692
|
+
continue
|
|
693
|
+
fi
|
|
694
|
+
|
|
695
|
+
local found=false
|
|
696
|
+
IFS=',' read -ra allowed_array <<< "$allowed_values"
|
|
697
|
+
for allowed in "${allowed_array[@]}"; do
|
|
698
|
+
if [[ "$actual_value" == "$allowed" ]]; then
|
|
699
|
+
found=true
|
|
700
|
+
break
|
|
701
|
+
fi
|
|
702
|
+
done
|
|
703
|
+
|
|
704
|
+
if [[ "$found" == "false" ]]; then
|
|
705
|
+
error "Invalid value for $field_path: '$actual_value' (allowed: $allowed_values)"
|
|
706
|
+
has_errors=true
|
|
707
|
+
else
|
|
708
|
+
debug "Enum field $field_path='$actual_value' is valid"
|
|
709
|
+
fi
|
|
710
|
+
done
|
|
711
|
+
|
|
712
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
713
|
+
return 1
|
|
714
|
+
fi
|
|
715
|
+
|
|
716
|
+
success "All enum fields have valid values"
|
|
717
|
+
return 0
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
validate_content_terms() {
|
|
721
|
+
local output_file="$1"
|
|
722
|
+
local content
|
|
723
|
+
content=$(cat "$output_file")
|
|
724
|
+
|
|
725
|
+
local has_errors=false
|
|
726
|
+
|
|
727
|
+
if [[ ${#MUST_CONTAIN_TERMS[@]} -gt 0 ]]; then
|
|
728
|
+
local missing_terms=()
|
|
729
|
+
for term in "${MUST_CONTAIN_TERMS[@]}"; do
|
|
730
|
+
if ! grep -qi "$term" <<< "$content"; then
|
|
731
|
+
missing_terms+=("$term")
|
|
732
|
+
fi
|
|
733
|
+
done
|
|
734
|
+
|
|
735
|
+
if [[ ${#missing_terms[@]} -gt 0 ]]; then
|
|
736
|
+
error "Output missing required terms: ${missing_terms[*]}"
|
|
737
|
+
has_errors=true
|
|
738
|
+
else
|
|
739
|
+
success "All required terms found (WCAG, accessibility)"
|
|
740
|
+
fi
|
|
741
|
+
fi
|
|
742
|
+
|
|
743
|
+
if [[ ${#MUST_NOT_CONTAIN_TERMS[@]} -gt 0 ]]; then
|
|
744
|
+
local found_forbidden=()
|
|
745
|
+
for term in "${MUST_NOT_CONTAIN_TERMS[@]}"; do
|
|
746
|
+
if grep -qi "$term" <<< "$content"; then
|
|
747
|
+
found_forbidden+=("$term")
|
|
748
|
+
fi
|
|
749
|
+
done
|
|
750
|
+
|
|
751
|
+
if [[ ${#found_forbidden[@]} -gt 0 ]]; then
|
|
752
|
+
error "Output contains forbidden terms: ${found_forbidden[*]}"
|
|
753
|
+
has_errors=true
|
|
754
|
+
else
|
|
755
|
+
success "No forbidden terms found"
|
|
756
|
+
fi
|
|
757
|
+
fi
|
|
758
|
+
|
|
759
|
+
if [[ "$has_errors" == "true" ]]; then
|
|
760
|
+
return 1
|
|
761
|
+
fi
|
|
762
|
+
|
|
763
|
+
return 0
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
# =============================================================================
|
|
767
|
+
# Main Validation Flow
|
|
768
|
+
# =============================================================================
|
|
769
|
+
|
|
770
|
+
main() {
|
|
771
|
+
if [[ -z "$OUTPUT_FILE" ]]; then
|
|
772
|
+
error "No output file specified"
|
|
773
|
+
echo "Usage: $0 <output-file> [options]"
|
|
774
|
+
echo "Use --help for more information"
|
|
775
|
+
exit 1
|
|
776
|
+
fi
|
|
777
|
+
|
|
778
|
+
if [[ ! -f "$OUTPUT_FILE" ]]; then
|
|
779
|
+
error "Output file not found: $OUTPUT_FILE"
|
|
780
|
+
exit 1
|
|
781
|
+
fi
|
|
782
|
+
|
|
783
|
+
if [[ "$JSON_ONLY" != "true" ]]; then
|
|
784
|
+
echo "=============================================="
|
|
785
|
+
info "Validating $SKILL_NAME Output"
|
|
786
|
+
echo "=============================================="
|
|
787
|
+
echo ""
|
|
788
|
+
echo " Skill: $SKILL_NAME v$SKILL_VERSION"
|
|
789
|
+
echo " File: $OUTPUT_FILE"
|
|
790
|
+
echo " Schema: ${SCHEMA_PATH:-none}"
|
|
791
|
+
echo ""
|
|
792
|
+
fi
|
|
793
|
+
|
|
794
|
+
# Track validation status
|
|
795
|
+
local tool_status="passed"
|
|
796
|
+
local json_status="passed"
|
|
797
|
+
local schema_status="passed"
|
|
798
|
+
local fields_status="passed"
|
|
799
|
+
local enums_status="passed"
|
|
800
|
+
local content_status="passed"
|
|
801
|
+
local specific_status="passed"
|
|
802
|
+
|
|
803
|
+
local error_count=0
|
|
804
|
+
local warning_count=0
|
|
805
|
+
|
|
806
|
+
# Step 1: Check Required Tools
|
|
807
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 1: Tool Availability ---"
|
|
808
|
+
|
|
809
|
+
if ! validate_tools; then
|
|
810
|
+
tool_status="failed"
|
|
811
|
+
((error_count++)) || true
|
|
812
|
+
[[ "$JSON_ONLY" != "true" ]] && error "Validation cannot proceed without required tools"
|
|
813
|
+
exit $EXIT_SKIP
|
|
814
|
+
fi
|
|
815
|
+
|
|
816
|
+
[[ "$JSON_ONLY" != "true" ]] && success "Tool check passed" && echo ""
|
|
817
|
+
|
|
818
|
+
# Step 2: Validate JSON Syntax
|
|
819
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 2: JSON Syntax ---"
|
|
820
|
+
|
|
821
|
+
if ! validate_json "$OUTPUT_FILE"; then
|
|
822
|
+
json_status="failed"
|
|
823
|
+
((error_count++)) || true
|
|
824
|
+
[[ "$JSON_ONLY" != "true" ]] && error "File is not valid JSON - cannot proceed"
|
|
825
|
+
exit $EXIT_FAIL
|
|
826
|
+
fi
|
|
827
|
+
|
|
828
|
+
[[ "$JSON_ONLY" != "true" ]] && success "JSON syntax valid" && echo ""
|
|
829
|
+
|
|
830
|
+
# Step 3: Validate Against Schema
|
|
831
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 3: Schema Validation ---"
|
|
832
|
+
|
|
833
|
+
local schema_exit_code
|
|
834
|
+
validate_schema "$OUTPUT_FILE" && schema_exit_code=0 || schema_exit_code=$?
|
|
835
|
+
|
|
836
|
+
case $schema_exit_code in
|
|
837
|
+
0) [[ "$JSON_ONLY" != "true" ]] && echo "" ;;
|
|
838
|
+
1) schema_status="failed"; ((error_count++)) || true; [[ "$JSON_ONLY" != "true" ]] && echo "" ;;
|
|
839
|
+
2) schema_status="skipped"; ((warning_count++)) || true; [[ "$JSON_ONLY" != "true" ]] && echo "" ;;
|
|
840
|
+
esac
|
|
841
|
+
|
|
842
|
+
# Step 4: Validate Required Fields
|
|
843
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 4: Required Fields ---"
|
|
844
|
+
|
|
845
|
+
if ! validate_required_fields "$OUTPUT_FILE"; then
|
|
846
|
+
fields_status="failed"
|
|
847
|
+
((error_count++)) || true
|
|
848
|
+
fi
|
|
849
|
+
|
|
850
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
851
|
+
|
|
852
|
+
# Step 5: Validate Enum Values
|
|
853
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 5: Enum Validation ---"
|
|
854
|
+
|
|
855
|
+
if ! validate_enum_fields "$OUTPUT_FILE"; then
|
|
856
|
+
enums_status="failed"
|
|
857
|
+
((error_count++)) || true
|
|
858
|
+
fi
|
|
859
|
+
|
|
860
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
861
|
+
|
|
862
|
+
# Step 6: Validate Content Terms
|
|
863
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 6: Content Terms ---"
|
|
864
|
+
|
|
865
|
+
if ! validate_content_terms "$OUTPUT_FILE"; then
|
|
866
|
+
content_status="failed"
|
|
867
|
+
((error_count++)) || true
|
|
868
|
+
fi
|
|
869
|
+
|
|
870
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
871
|
+
|
|
872
|
+
# Step 7: Skill-Specific Validation (WCAG/POUR)
|
|
873
|
+
[[ "$JSON_ONLY" != "true" ]] && echo "--- Step 7: WCAG/Accessibility Validation ---"
|
|
874
|
+
|
|
875
|
+
if ! validate_skill_specific "$OUTPUT_FILE"; then
|
|
876
|
+
specific_status="failed"
|
|
877
|
+
((error_count++)) || true
|
|
878
|
+
fi
|
|
879
|
+
|
|
880
|
+
[[ "$JSON_ONLY" != "true" ]] && echo ""
|
|
881
|
+
|
|
882
|
+
# Determine Overall Status
|
|
883
|
+
local overall_status="passed"
|
|
884
|
+
local content_overall="passed"
|
|
885
|
+
|
|
886
|
+
if [[ "$fields_status" == "failed" ]] || \
|
|
887
|
+
[[ "$enums_status" == "failed" ]] || \
|
|
888
|
+
[[ "$content_status" == "failed" ]] || \
|
|
889
|
+
[[ "$specific_status" == "failed" ]]; then
|
|
890
|
+
content_overall="failed"
|
|
891
|
+
fi
|
|
892
|
+
|
|
893
|
+
if [[ "$json_status" == "failed" ]] || \
|
|
894
|
+
[[ "$schema_status" == "failed" ]] || \
|
|
895
|
+
[[ "$content_overall" == "failed" ]]; then
|
|
896
|
+
overall_status="failed"
|
|
897
|
+
elif [[ "$schema_status" == "skipped" ]]; then
|
|
898
|
+
overall_status="partial"
|
|
899
|
+
fi
|
|
900
|
+
|
|
901
|
+
# Output Results
|
|
902
|
+
if [[ "$JSON_ONLY" == "true" ]]; then
|
|
903
|
+
output_validation_report "$SKILL_NAME" "$schema_status" "$content_overall" "$tool_status"
|
|
904
|
+
else
|
|
905
|
+
echo "=============================================="
|
|
906
|
+
echo "Validation Summary for $SKILL_NAME"
|
|
907
|
+
echo "=============================================="
|
|
908
|
+
echo ""
|
|
909
|
+
echo " Tools: $tool_status"
|
|
910
|
+
echo " JSON Syntax: $json_status"
|
|
911
|
+
echo " Schema: $schema_status"
|
|
912
|
+
echo " Fields: $fields_status"
|
|
913
|
+
echo " Enums: $enums_status"
|
|
914
|
+
echo " Content: $content_status"
|
|
915
|
+
echo " WCAG/A11y: $specific_status"
|
|
916
|
+
echo ""
|
|
917
|
+
echo " ------------------------------"
|
|
918
|
+
echo " Overall: $overall_status"
|
|
919
|
+
echo " Errors: $error_count"
|
|
920
|
+
echo " Warnings: $warning_count"
|
|
921
|
+
echo "=============================================="
|
|
922
|
+
echo ""
|
|
923
|
+
fi
|
|
924
|
+
|
|
925
|
+
# Exit with appropriate code
|
|
926
|
+
case "$overall_status" in
|
|
927
|
+
"passed")
|
|
928
|
+
[[ "$JSON_ONLY" != "true" ]] && success "Validation PASSED"
|
|
929
|
+
exit $EXIT_PASS
|
|
930
|
+
;;
|
|
931
|
+
"partial")
|
|
932
|
+
[[ "$JSON_ONLY" != "true" ]] && warn "Validation PARTIAL (some checks skipped)"
|
|
933
|
+
exit $EXIT_PASS
|
|
934
|
+
;;
|
|
935
|
+
"failed")
|
|
936
|
+
[[ "$JSON_ONLY" != "true" ]] && error "Validation FAILED"
|
|
937
|
+
exit $EXIT_FAIL
|
|
938
|
+
;;
|
|
939
|
+
esac
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
# Run main function
|
|
943
|
+
main
|