agentic-qe 3.7.9 → 3.7.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/.validation/README.md +111 -111
- package/.claude/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
- package/.claude/skills/.validation/examples/performance-testing-output.example.json +252 -252
- package/.claude/skills/.validation/examples/security-testing-output.example.json +413 -413
- package/.claude/skills/.validation/examples/testability-scoring-output.example.json +350 -350
- package/.claude/skills/.validation/schemas/skill-eval.schema.json +462 -462
- package/.claude/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
- package/.claude/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
- package/.claude/skills/.validation/schemas/skill-output.template.json +610 -610
- package/.claude/skills/.validation/skill-validation-mcp-integration.md +250 -250
- package/.claude/skills/.validation/templates/eval.template.yaml +366 -366
- package/.claude/skills/.validation/templates/schemas/output.json +145 -145
- package/.claude/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
- package/.claude/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
- package/.claude/skills/.validation/test-data/invalid-output.json +5 -5
- package/.claude/skills/.validation/test-data/minimal-output.json +9 -9
- package/.claude/skills/.validation/test-data/sample-output.json +73 -73
- package/.claude/skills/a11y-ally/evals/a11y-ally.yaml +376 -376
- package/.claude/skills/a11y-ally/schemas/output.json +549 -549
- package/.claude/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
- package/.claude/skills/accessibility-testing/schemas/output.json +776 -776
- package/.claude/skills/accessibility-testing/test-data/sample-output.json +191 -191
- package/.claude/skills/agentic-quality-engineering/schemas/output.json +577 -577
- package/.claude/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
- package/.claude/skills/api-testing-patterns/schemas/output.json +845 -845
- package/.claude/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
- package/.claude/skills/brutal-honesty-review/schemas/output.json +291 -291
- package/.claude/skills/bug-reporting-excellence/schemas/output.json +288 -288
- package/.claude/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
- package/.claude/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
- package/.claude/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -157
- package/.claude/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -542
- package/.claude/skills/code-review-quality/schemas/output.json +264 -264
- package/.claude/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
- package/.claude/skills/compatibility-testing/schemas/output.json +551 -551
- package/.claude/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
- package/.claude/skills/compliance-testing/schemas/output.json +845 -845
- package/.claude/skills/consultancy-practices/schemas/output.json +282 -282
- package/.claude/skills/contract-testing/evals/contract-testing.yaml +748 -748
- package/.claude/skills/contract-testing/schemas/output.json +638 -638
- package/.claude/skills/database-testing/evals/database-testing.yaml +968 -968
- package/.claude/skills/database-testing/schemas/output.json +1446 -1446
- package/.claude/skills/debug-loop/SKILL.md +61 -61
- package/.claude/skills/enterprise-integration-testing/SKILL.md +735 -735
- package/.claude/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -158
- package/.claude/skills/enterprise-integration-testing/schemas/output.json +74 -74
- package/.claude/skills/enterprise-integration-testing/scripts/validate-config.json +25 -25
- package/.claude/skills/iterative-loop/SKILL.md +371 -371
- package/.claude/skills/localization-testing/evals/localization-testing.yaml +544 -544
- package/.claude/skills/localization-testing/schemas/output.json +325 -325
- package/.claude/skills/middleware-testing-patterns/SKILL.md +798 -798
- package/.claude/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -153
- package/.claude/skills/middleware-testing-patterns/schemas/output.json +58 -58
- package/.claude/skills/middleware-testing-patterns/scripts/validate-config.json +25 -25
- package/.claude/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
- package/.claude/skills/mobile-testing/schemas/output.json +318 -318
- package/.claude/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
- package/.claude/skills/mutation-testing/schemas/output.json +707 -707
- package/.claude/skills/mutation-testing/test-data/sample-output.json +295 -295
- package/.claude/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -450
- package/.claude/skills/n8n-expression-testing/schemas/output.json +369 -369
- package/.claude/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -522
- package/.claude/skills/n8n-integration-testing-patterns/schemas/output.json +291 -291
- package/.claude/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -493
- package/.claude/skills/n8n-security-testing/schemas/output.json +293 -293
- package/.claude/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -500
- package/.claude/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -295
- package/.claude/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -497
- package/.claude/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -254
- package/.claude/skills/observability-testing-patterns/SKILL.md +930 -930
- package/.claude/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -157
- package/.claude/skills/observability-testing-patterns/schemas/output.json +58 -58
- package/.claude/skills/observability-testing-patterns/scripts/validate-config.json +25 -25
- package/.claude/skills/pentest-validation/SKILL.md +268 -268
- package/.claude/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
- package/.claude/skills/pentest-validation/schemas/output.json +281 -281
- package/.claude/skills/performance-analysis/evals/performance-analysis.yaml +144 -144
- package/.claude/skills/performance-analysis/schemas/output.json +588 -588
- package/.claude/skills/performance-testing/evals/performance-testing.yaml +772 -772
- package/.claude/skills/performance-testing/schemas/output.json +1184 -1184
- package/.claude/skills/pr-review/SKILL.md +61 -61
- package/.claude/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
- package/.claude/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -211
- package/.claude/skills/qcsd-cicd-swarm/schemas/output.json +86 -86
- package/.claude/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -30
- package/.claude/skills/qcsd-development-swarm/SKILL.md +2154 -2154
- package/.claude/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -162
- package/.claude/skills/qcsd-development-swarm/schemas/output.json +72 -72
- package/.claude/skills/qcsd-development-swarm/scripts/validate-config.json +25 -25
- package/.claude/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -138
- package/.claude/skills/qcsd-ideation-swarm/schemas/output.json +568 -568
- package/.claude/skills/qcsd-production-swarm/SKILL.md +2781 -2781
- package/.claude/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -246
- package/.claude/skills/qcsd-production-swarm/schemas/output.json +505 -505
- package/.claude/skills/qcsd-production-swarm/scripts/validate-config.json +25 -25
- package/.claude/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
- package/.claude/skills/qe-chaos-resilience/schemas/output.json +314 -314
- package/.claude/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
- package/.claude/skills/qe-code-intelligence/schemas/output.json +315 -315
- package/.claude/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
- package/.claude/skills/qe-contract-testing/schemas/output.json +295 -295
- package/.claude/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
- package/.claude/skills/qe-coverage-analysis/schemas/output.json +286 -286
- package/.claude/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
- package/.claude/skills/qe-defect-intelligence/schemas/output.json +283 -283
- package/.claude/skills/qe-iterative-loop/schemas/output.json +264 -264
- package/.claude/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
- package/.claude/skills/qe-learning-optimization/schemas/output.json +288 -288
- package/.claude/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
- package/.claude/skills/qe-quality-assessment/schemas/output.json +550 -550
- package/.claude/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
- package/.claude/skills/qe-requirements-validation/schemas/output.json +587 -587
- package/.claude/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
- package/.claude/skills/qe-security-compliance/schemas/output.json +498 -498
- package/.claude/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
- package/.claude/skills/qe-test-execution/schemas/output.json +529 -529
- package/.claude/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
- package/.claude/skills/qe-test-generation/schemas/output.json +439 -439
- package/.claude/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
- package/.claude/skills/qe-visual-accessibility/schemas/output.json +491 -491
- package/.claude/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
- package/.claude/skills/quality-metrics/schemas/output.json +403 -403
- package/.claude/skills/refactoring-patterns/schemas/output.json +475 -475
- package/.claude/skills/regression-testing/evals/regression-testing.yaml +504 -504
- package/.claude/skills/regression-testing/schemas/output.json +311 -311
- package/.claude/skills/release/SKILL.md +347 -347
- package/.claude/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
- package/.claude/skills/risk-based-testing/schemas/output.json +480 -480
- package/.claude/skills/security-testing/evals/security-testing.yaml +789 -789
- package/.claude/skills/security-testing/schemas/output.json +879 -879
- package/.claude/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
- package/.claude/skills/security-visual-testing/schemas/output.json +486 -486
- package/.claude/skills/sfdipot-product-factors/SKILL.md +239 -239
- package/.claude/skills/sherlock-review/schemas/output.json +297 -297
- package/.claude/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
- package/.claude/skills/shift-left-testing/schemas/output.json +459 -459
- package/.claude/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
- package/.claude/skills/shift-right-testing/schemas/output.json +418 -418
- package/.claude/skills/skills-manifest.json +1 -1
- package/.claude/skills/tdd-london-chicago/schemas/output.json +444 -444
- package/.claude/skills/technical-writing/schemas/output.json +268 -268
- package/.claude/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
- package/.claude/skills/test-automation-strategy/schemas/output.json +444 -444
- package/.claude/skills/test-data-management/evals/test-data-management.yaml +504 -504
- package/.claude/skills/test-data-management/schemas/output.json +284 -284
- package/.claude/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
- package/.claude/skills/test-design-techniques/schemas/output.json +295 -295
- package/.claude/skills/test-environment-management/schemas/output.json +310 -310
- package/.claude/skills/test-idea-rewriting/SKILL.md +229 -229
- package/.claude/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
- package/.claude/skills/test-reporting-analytics/schemas/output.json +329 -329
- package/.claude/skills/testability-scoring/evals/testability-scoring.yaml +814 -814
- package/.claude/skills/testability-scoring/resources/templates/config.template.js +84 -84
- package/.claude/skills/testability-scoring/schemas/output.json +606 -606
- package/.claude/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
- package/.claude/skills/verification-quality/evals/verification-quality.yaml +150 -150
- package/.claude/skills/verification-quality/schemas/output.json +432 -432
- package/.claude/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
- package/.claude/skills/visual-testing-advanced/schemas/output.json +294 -294
- package/.claude/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -165
- package/.claude/skills/wms-testing-patterns/schemas/output.json +150 -150
- package/.claude/skills/wms-testing-patterns/scripts/validate-config.json +51 -51
- package/CHANGELOG.md +36 -0
- package/README.md +171 -901
- package/assets/agents/v3/helpers/quality-criteria/evidence-classification.md +116 -116
- package/assets/agents/v3/helpers/quality-criteria/htsm-categories.md +139 -139
- package/assets/governance/constitution.md +202 -202
- package/assets/governance/shards/chaos-resilience.shard.md +221 -221
- package/assets/governance/shards/code-intelligence.shard.md +178 -178
- package/assets/governance/shards/contract-testing.shard.md +206 -206
- package/assets/governance/shards/coverage-analysis.shard.md +146 -146
- package/assets/governance/shards/defect-intelligence.shard.md +182 -182
- package/assets/governance/shards/learning-optimization.shard.md +248 -248
- package/assets/governance/shards/quality-assessment.shard.md +165 -165
- package/assets/governance/shards/requirements-validation.shard.md +177 -177
- package/assets/governance/shards/security-compliance.shard.md +196 -196
- package/assets/governance/shards/test-execution.shard.md +156 -156
- package/assets/governance/shards/test-generation.shard.md +128 -128
- package/assets/governance/shards/visual-accessibility.shard.md +209 -209
- package/assets/hooks/cross-phase-memory.yaml +253 -253
- package/assets/patterns/adr-051-booster-patterns.json +78 -78
- package/assets/patterns/adr-051-embedding-patterns.json +147 -147
- package/assets/patterns/adr-051-integration-summary.json +62 -62
- package/assets/patterns/adr-051-reasoning-patterns.json +166 -166
- package/assets/patterns/adr-051-router-patterns.json +113 -113
- package/assets/patterns/index.json +136 -136
- package/assets/skills/.validation/README.md +111 -111
- package/assets/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
- package/assets/skills/.validation/examples/performance-testing-output.example.json +252 -252
- package/assets/skills/.validation/examples/security-testing-output.example.json +413 -413
- package/assets/skills/.validation/examples/testability-scoring-output.example.json +350 -350
- package/assets/skills/.validation/schemas/skill-eval.schema.json +462 -462
- package/assets/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
- package/assets/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
- package/assets/skills/.validation/schemas/skill-output.template.json +610 -610
- package/assets/skills/.validation/skill-validation-mcp-integration.md +250 -250
- package/assets/skills/.validation/templates/eval.template.yaml +366 -366
- package/assets/skills/.validation/templates/schemas/output.json +145 -145
- package/assets/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
- package/assets/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
- package/assets/skills/.validation/test-data/invalid-output.json +5 -5
- package/assets/skills/.validation/test-data/minimal-output.json +9 -9
- package/assets/skills/.validation/test-data/sample-output.json +73 -73
- package/assets/skills/a11y-ally/evals/a11y-ally.yaml +376 -376
- package/assets/skills/a11y-ally/schemas/output.json +549 -549
- package/assets/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
- package/assets/skills/accessibility-testing/schemas/output.json +776 -776
- package/assets/skills/accessibility-testing/test-data/sample-output.json +191 -191
- package/assets/skills/agentic-quality-engineering/schemas/output.json +577 -577
- package/assets/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
- package/assets/skills/api-testing-patterns/schemas/output.json +845 -845
- package/assets/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
- package/assets/skills/brutal-honesty-review/schemas/output.json +291 -291
- package/assets/skills/bug-reporting-excellence/schemas/output.json +288 -288
- package/assets/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
- package/assets/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
- package/assets/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -157
- package/assets/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -542
- package/assets/skills/code-review-quality/schemas/output.json +264 -264
- package/assets/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
- package/assets/skills/compatibility-testing/schemas/output.json +551 -551
- package/assets/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
- package/assets/skills/compliance-testing/schemas/output.json +845 -845
- package/assets/skills/consultancy-practices/schemas/output.json +282 -282
- package/assets/skills/contract-testing/evals/contract-testing.yaml +748 -748
- package/assets/skills/contract-testing/schemas/output.json +638 -638
- package/assets/skills/database-testing/evals/database-testing.yaml +968 -968
- package/assets/skills/database-testing/schemas/output.json +1446 -1446
- package/assets/skills/debug-loop/SKILL.md +61 -61
- package/assets/skills/enterprise-integration-testing/SKILL.md +735 -735
- package/assets/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -158
- package/assets/skills/enterprise-integration-testing/schemas/output.json +74 -74
- package/assets/skills/enterprise-integration-testing/scripts/validate-config.json +25 -25
- package/assets/skills/localization-testing/evals/localization-testing.yaml +544 -544
- package/assets/skills/localization-testing/schemas/output.json +325 -325
- package/assets/skills/middleware-testing-patterns/SKILL.md +798 -798
- package/assets/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -153
- package/assets/skills/middleware-testing-patterns/schemas/output.json +58 -58
- package/assets/skills/middleware-testing-patterns/scripts/validate-config.json +25 -25
- package/assets/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
- package/assets/skills/mobile-testing/schemas/output.json +318 -318
- package/assets/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
- package/assets/skills/mutation-testing/schemas/output.json +707 -707
- package/assets/skills/mutation-testing/test-data/sample-output.json +295 -295
- package/assets/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -450
- package/assets/skills/n8n-expression-testing/schemas/output.json +369 -369
- package/assets/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -522
- package/assets/skills/n8n-integration-testing-patterns/schemas/output.json +291 -291
- package/assets/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -493
- package/assets/skills/n8n-security-testing/schemas/output.json +293 -293
- package/assets/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -500
- package/assets/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -295
- package/assets/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -497
- package/assets/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -254
- package/assets/skills/observability-testing-patterns/SKILL.md +930 -930
- package/assets/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -157
- package/assets/skills/observability-testing-patterns/schemas/output.json +58 -58
- package/assets/skills/observability-testing-patterns/scripts/validate-config.json +25 -25
- package/assets/skills/pentest-validation/SKILL.md +268 -268
- package/assets/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
- package/assets/skills/pentest-validation/schemas/output.json +281 -281
- package/assets/skills/performance-testing/evals/performance-testing.yaml +772 -772
- package/assets/skills/performance-testing/schemas/output.json +1184 -1184
- package/assets/skills/pr-review/SKILL.md +61 -61
- package/assets/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
- package/assets/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -211
- package/assets/skills/qcsd-cicd-swarm/schemas/output.json +86 -86
- package/assets/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -30
- package/assets/skills/qcsd-development-swarm/SKILL.md +2154 -2154
- package/assets/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -162
- package/assets/skills/qcsd-development-swarm/schemas/output.json +72 -72
- package/assets/skills/qcsd-development-swarm/scripts/validate-config.json +25 -25
- package/assets/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -138
- package/assets/skills/qcsd-ideation-swarm/schemas/output.json +568 -568
- package/assets/skills/qcsd-production-swarm/SKILL.md +2781 -2781
- package/assets/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -246
- package/assets/skills/qcsd-production-swarm/schemas/output.json +505 -505
- package/assets/skills/qcsd-production-swarm/scripts/validate-config.json +25 -25
- package/assets/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
- package/assets/skills/qe-chaos-resilience/schemas/output.json +314 -314
- package/assets/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
- package/assets/skills/qe-code-intelligence/schemas/output.json +315 -315
- package/assets/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
- package/assets/skills/qe-contract-testing/schemas/output.json +295 -295
- package/assets/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
- package/assets/skills/qe-coverage-analysis/schemas/output.json +286 -286
- package/assets/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
- package/assets/skills/qe-defect-intelligence/schemas/output.json +283 -283
- package/assets/skills/qe-iterative-loop/schemas/output.json +264 -264
- package/assets/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
- package/assets/skills/qe-learning-optimization/schemas/output.json +288 -288
- package/assets/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
- package/assets/skills/qe-quality-assessment/schemas/output.json +550 -550
- package/assets/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
- package/assets/skills/qe-requirements-validation/schemas/output.json +587 -587
- package/assets/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
- package/assets/skills/qe-security-compliance/schemas/output.json +498 -498
- package/assets/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
- package/assets/skills/qe-test-execution/schemas/output.json +529 -529
- package/assets/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
- package/assets/skills/qe-test-generation/schemas/output.json +439 -439
- package/assets/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
- package/assets/skills/qe-visual-accessibility/schemas/output.json +491 -491
- package/assets/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
- package/assets/skills/quality-metrics/schemas/output.json +403 -403
- package/assets/skills/refactoring-patterns/schemas/output.json +475 -475
- package/assets/skills/regression-testing/evals/regression-testing.yaml +504 -504
- package/assets/skills/regression-testing/schemas/output.json +311 -311
- package/assets/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
- package/assets/skills/risk-based-testing/schemas/output.json +480 -480
- package/assets/skills/security-testing/evals/security-testing.yaml +789 -789
- package/assets/skills/security-testing/schemas/output.json +879 -879
- package/assets/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
- package/assets/skills/security-visual-testing/schemas/output.json +486 -486
- package/assets/skills/sfdipot-product-factors/SKILL.md +239 -239
- package/assets/skills/sherlock-review/schemas/output.json +297 -297
- package/assets/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
- package/assets/skills/shift-left-testing/schemas/output.json +459 -459
- package/assets/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
- package/assets/skills/shift-right-testing/schemas/output.json +418 -418
- package/assets/skills/tdd-london-chicago/schemas/output.json +444 -444
- package/assets/skills/technical-writing/schemas/output.json +268 -268
- package/assets/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
- package/assets/skills/test-automation-strategy/schemas/output.json +444 -444
- package/assets/skills/test-data-management/evals/test-data-management.yaml +504 -504
- package/assets/skills/test-data-management/schemas/output.json +284 -284
- package/assets/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
- package/assets/skills/test-design-techniques/schemas/output.json +295 -295
- package/assets/skills/test-environment-management/schemas/output.json +310 -310
- package/assets/skills/test-idea-rewriting/SKILL.md +229 -229
- package/assets/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
- package/assets/skills/test-reporting-analytics/schemas/output.json +329 -329
- package/assets/skills/testability-scoring/evals/testability-scoring.yaml +814 -814
- package/assets/skills/testability-scoring/resources/templates/config.template.js +84 -84
- package/assets/skills/testability-scoring/schemas/output.json +606 -606
- package/assets/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
- package/assets/skills/verification-quality/evals/verification-quality.yaml +150 -150
- package/assets/skills/verification-quality/schemas/output.json +432 -432
- package/assets/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
- package/assets/skills/visual-testing-advanced/schemas/output.json +294 -294
- package/assets/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -165
- package/assets/skills/wms-testing-patterns/schemas/output.json +150 -150
- package/assets/skills/wms-testing-patterns/scripts/validate-config.json +51 -51
- package/assets/templates/validation-summary.json +56 -56
- package/dist/agents/claim-verifier/verifiers/output-verifier.d.ts +1 -0
- package/dist/agents/claim-verifier/verifiers/output-verifier.d.ts.map +1 -1
- package/dist/agents/claim-verifier/verifiers/output-verifier.js +16 -3
- package/dist/agents/claim-verifier/verifiers/output-verifier.js.map +1 -1
- package/dist/cli/bundle.js +488 -75
- package/dist/cli/commands/mcp.d.ts.map +1 -1
- package/dist/cli/commands/mcp.js +11 -9
- package/dist/cli/commands/mcp.js.map +1 -1
- package/dist/governance/adversarial-defense-integration.d.ts +29 -0
- package/dist/governance/adversarial-defense-integration.d.ts.map +1 -1
- package/dist/governance/adversarial-defense-integration.js +85 -1
- package/dist/governance/adversarial-defense-integration.js.map +1 -1
- package/dist/governance/continue-gate-integration.d.ts +7 -5
- package/dist/governance/continue-gate-integration.d.ts.map +1 -1
- package/dist/governance/continue-gate-integration.js +72 -15
- package/dist/governance/continue-gate-integration.js.map +1 -1
- package/dist/governance/deterministic-gateway-integration.d.ts +5 -0
- package/dist/governance/deterministic-gateway-integration.d.ts.map +1 -1
- package/dist/governance/deterministic-gateway-integration.js +35 -1
- package/dist/governance/deterministic-gateway-integration.js.map +1 -1
- package/dist/governance/evolution-pipeline-integration.d.ts +5 -0
- package/dist/governance/evolution-pipeline-integration.d.ts.map +1 -1
- package/dist/governance/evolution-pipeline-integration.js +57 -1
- package/dist/governance/evolution-pipeline-integration.js.map +1 -1
- package/dist/governance/memory-write-gate-integration.d.ts +6 -3
- package/dist/governance/memory-write-gate-integration.d.ts.map +1 -1
- package/dist/governance/memory-write-gate-integration.js +48 -10
- package/dist/governance/memory-write-gate-integration.js.map +1 -1
- package/dist/governance/proof-envelope-integration.d.ts +1 -0
- package/dist/governance/proof-envelope-integration.d.ts.map +1 -1
- package/dist/governance/proof-envelope-integration.js +37 -0
- package/dist/governance/proof-envelope-integration.js.map +1 -1
- package/dist/governance/shard-retriever-integration.d.ts +10 -0
- package/dist/governance/shard-retriever-integration.d.ts.map +1 -1
- package/dist/governance/shard-retriever-integration.js +50 -1
- package/dist/governance/shard-retriever-integration.js.map +1 -1
- package/dist/governance/trust-accumulator-integration.d.ts +5 -0
- package/dist/governance/trust-accumulator-integration.d.ts.map +1 -1
- package/dist/governance/trust-accumulator-integration.js +35 -2
- package/dist/governance/trust-accumulator-integration.js.map +1 -1
- package/dist/init/phases/10-workers.js +2 -2
- package/dist/init/phases/10-workers.js.map +1 -1
- package/dist/mcp/bundle.js +478 -62
- package/dist/mcp/protocol-server.js +2 -2
- package/dist/mcp/tool-registry.d.ts.map +1 -1
- package/dist/mcp/tool-registry.js +1 -0
- package/dist/mcp/tool-registry.js.map +1 -1
- package/dist/shared/language-detector.js +1 -1
- package/dist/shared/language-detector.js.map +1 -1
- package/dist/shared/parsers/multi-language-parser.d.ts.map +1 -1
- package/dist/shared/parsers/multi-language-parser.js +55 -16
- package/dist/shared/parsers/multi-language-parser.js.map +1 -1
- package/dist/shared/sql-safety.d.ts.map +1 -1
- package/dist/shared/sql-safety.js +7 -1
- package/dist/shared/sql-safety.js.map +1 -1
- package/package.json +6 -2
- package/scripts/prepare-assets.sh +8 -0
|
@@ -1,136 +1,136 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "ADR-051 Pattern Memory Store",
|
|
3
|
-
"version": "1.0",
|
|
4
|
-
"description": "Persistent cross-session learning patterns for Agent Booster, Model Router, ONNX Embeddings, and ReasoningBank",
|
|
5
|
-
"created": "2026-01-20",
|
|
6
|
-
"namespace_index": {
|
|
7
|
-
"adr-051": {
|
|
8
|
-
"booster-patterns": {
|
|
9
|
-
"file": "adr-051-booster-patterns.json",
|
|
10
|
-
"patterns": [
|
|
11
|
-
"booster-transform-eligibility",
|
|
12
|
-
"booster-batch-optimization",
|
|
13
|
-
"booster-wasm-fallback"
|
|
14
|
-
],
|
|
15
|
-
"description": "Agent Booster WASM transforms and fallback strategies"
|
|
16
|
-
},
|
|
17
|
-
"router-patterns": {
|
|
18
|
-
"file": "adr-051-router-patterns.json",
|
|
19
|
-
"patterns": [
|
|
20
|
-
"router-5tier-complexity",
|
|
21
|
-
"router-budget-enforcement",
|
|
22
|
-
"router-booster-integration"
|
|
23
|
-
],
|
|
24
|
-
"description": "Model Router complexity scoring and budget enforcement"
|
|
25
|
-
},
|
|
26
|
-
"embedding-patterns": {
|
|
27
|
-
"file": "adr-051-embedding-patterns.json",
|
|
28
|
-
"patterns": [
|
|
29
|
-
"embeddings-local-generation",
|
|
30
|
-
"embeddings-lru-cache",
|
|
31
|
-
"embeddings-hyperbolic-space",
|
|
32
|
-
"embeddings-similarity-metrics"
|
|
33
|
-
],
|
|
34
|
-
"description": "ONNX embeddings generation, caching, and similarity computation"
|
|
35
|
-
},
|
|
36
|
-
"reasoning-patterns": {
|
|
37
|
-
"file": "adr-051-reasoning-patterns.json",
|
|
38
|
-
"patterns": [
|
|
39
|
-
"reasoning-trajectory-tracking",
|
|
40
|
-
"reasoning-pattern-quality-gates",
|
|
41
|
-
"reasoning-experience-replay",
|
|
42
|
-
"reasoning-cross-agent-sharing"
|
|
43
|
-
],
|
|
44
|
-
"description": "ReasoningBank trajectory learning and pattern consolidation"
|
|
45
|
-
},
|
|
46
|
-
"summary": {
|
|
47
|
-
"file": "adr-051-integration-summary.json",
|
|
48
|
-
"description": "Complete ADR-051 integration architecture overview"
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
},
|
|
52
|
-
"pattern_statistics": {
|
|
53
|
-
"total_patterns": 15,
|
|
54
|
-
"namespaces": 5,
|
|
55
|
-
"avg_success_rate": 0.915,
|
|
56
|
-
"implementation_status": "Production Ready"
|
|
57
|
-
},
|
|
58
|
-
"access_patterns": [
|
|
59
|
-
{
|
|
60
|
-
"pattern_type": "Simple Lookup",
|
|
61
|
-
"operation": "Load specific pattern by key",
|
|
62
|
-
"time_complexity": "O(1)",
|
|
63
|
-
"use_case": "Quick eligibility checks, cache lookup"
|
|
64
|
-
},
|
|
65
|
-
{
|
|
66
|
-
"pattern_type": "Namespace Query",
|
|
67
|
-
"operation": "Load all patterns in namespace",
|
|
68
|
-
"time_complexity": "O(n) where n=patterns per namespace",
|
|
69
|
-
"use_case": "Multi-pattern decision making, agent coordination"
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
"pattern_type": "Semantic Search",
|
|
73
|
-
"operation": "Find patterns by similarity",
|
|
74
|
-
"time_complexity": "O(n) with embeddings",
|
|
75
|
-
"use_case": "Pattern discovery, cross-session learning"
|
|
76
|
-
}
|
|
77
|
-
],
|
|
78
|
-
"session_initialization": {
|
|
79
|
-
"steps": [
|
|
80
|
-
"Load index.json to discover available patterns",
|
|
81
|
-
"Initialize embedding cache from embeddings-patterns",
|
|
82
|
-
"Load router configuration from router-patterns",
|
|
83
|
-
"Initialize booster eligibility rules from booster-patterns",
|
|
84
|
-
"Setup ReasoningBank trajectory database from reasoning-patterns",
|
|
85
|
-
"Configure quality gates and thresholds",
|
|
86
|
-
"Register pattern sharing channels"
|
|
87
|
-
],
|
|
88
|
-
"initialization_time": "<100ms"
|
|
89
|
-
},
|
|
90
|
-
"cross_session_learning": {
|
|
91
|
-
"enabled": true,
|
|
92
|
-
"persistence_layer": "JSON files in .agentic-qe/patterns/",
|
|
93
|
-
"synchronization": "File-based with timestamps",
|
|
94
|
-
"conflict_resolution": "Latest timestamp wins",
|
|
95
|
-
"archival_strategy": "Keep 30-day history in patterns-archive/",
|
|
96
|
-
"memory_coordination": {
|
|
97
|
-
"agent_communication": "Via shared pattern namespace",
|
|
98
|
-
"knowledge_sharing": "Gossip protocol simulation via file watching",
|
|
99
|
-
"learning_integration": "Apply new patterns on each session start"
|
|
100
|
-
}
|
|
101
|
-
},
|
|
102
|
-
"usage_examples": {
|
|
103
|
-
"example_1_booster_check": {
|
|
104
|
-
"query": "Is 'var-to-const' transform eligible for Agent Booster?",
|
|
105
|
-
"lookup": "patterns['adr-051/booster-patterns']['booster-transform-eligibility'].eligibility_criteria.simple_transforms",
|
|
106
|
-
"result": "YES - found in eligibility list"
|
|
107
|
-
},
|
|
108
|
-
"example_2_router_selection": {
|
|
109
|
-
"query": "Route a 5-file refactoring task",
|
|
110
|
-
"lookup": "patterns['adr-051/router-patterns']['router-5tier-complexity'].scoring_factors",
|
|
111
|
-
"result": "Complexity score 6 → Route to Sonnet tier"
|
|
112
|
-
},
|
|
113
|
-
"example_3_embedding_cache": {
|
|
114
|
-
"query": "Check embedding cache for similarity query",
|
|
115
|
-
"lookup": "patterns['adr-051/embedding-patterns']['embeddings-lru-cache'].cache_key_strategy",
|
|
116
|
-
"result": "Check SHA256(text) in cache, 70% expected hit rate"
|
|
117
|
-
},
|
|
118
|
-
"example_4_pattern_quality": {
|
|
119
|
-
"query": "Is new trajectory eligible for ReasoningBank?",
|
|
120
|
-
"lookup": "patterns['adr-051/reasoning-patterns']['reasoning-pattern-quality-gates'].gate_thresholds",
|
|
121
|
-
"result": "Check composite score >= 0.80 for silver tier"
|
|
122
|
-
}
|
|
123
|
-
},
|
|
124
|
-
"maintenance": {
|
|
125
|
-
"review_frequency": "Monthly",
|
|
126
|
-
"update_triggers": [
|
|
127
|
-
"New successful pattern discovered",
|
|
128
|
-
"Pattern success rate drops below threshold",
|
|
129
|
-
"New component integration requires patterns",
|
|
130
|
-
"Performance optimization identified"
|
|
131
|
-
],
|
|
132
|
-
"archival_directory": ".agentic-qe/patterns-archive/",
|
|
133
|
-
"rollback_capability": "Restore from archive by date"
|
|
134
|
-
},
|
|
135
|
-
"notes": "This index enables fast pattern discovery and multi-session learning without requiring external MCP tools or CLI. Patterns are loaded on session initialization and used throughout agent coordination."
|
|
136
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "ADR-051 Pattern Memory Store",
|
|
3
|
+
"version": "1.0",
|
|
4
|
+
"description": "Persistent cross-session learning patterns for Agent Booster, Model Router, ONNX Embeddings, and ReasoningBank",
|
|
5
|
+
"created": "2026-01-20",
|
|
6
|
+
"namespace_index": {
|
|
7
|
+
"adr-051": {
|
|
8
|
+
"booster-patterns": {
|
|
9
|
+
"file": "adr-051-booster-patterns.json",
|
|
10
|
+
"patterns": [
|
|
11
|
+
"booster-transform-eligibility",
|
|
12
|
+
"booster-batch-optimization",
|
|
13
|
+
"booster-wasm-fallback"
|
|
14
|
+
],
|
|
15
|
+
"description": "Agent Booster WASM transforms and fallback strategies"
|
|
16
|
+
},
|
|
17
|
+
"router-patterns": {
|
|
18
|
+
"file": "adr-051-router-patterns.json",
|
|
19
|
+
"patterns": [
|
|
20
|
+
"router-5tier-complexity",
|
|
21
|
+
"router-budget-enforcement",
|
|
22
|
+
"router-booster-integration"
|
|
23
|
+
],
|
|
24
|
+
"description": "Model Router complexity scoring and budget enforcement"
|
|
25
|
+
},
|
|
26
|
+
"embedding-patterns": {
|
|
27
|
+
"file": "adr-051-embedding-patterns.json",
|
|
28
|
+
"patterns": [
|
|
29
|
+
"embeddings-local-generation",
|
|
30
|
+
"embeddings-lru-cache",
|
|
31
|
+
"embeddings-hyperbolic-space",
|
|
32
|
+
"embeddings-similarity-metrics"
|
|
33
|
+
],
|
|
34
|
+
"description": "ONNX embeddings generation, caching, and similarity computation"
|
|
35
|
+
},
|
|
36
|
+
"reasoning-patterns": {
|
|
37
|
+
"file": "adr-051-reasoning-patterns.json",
|
|
38
|
+
"patterns": [
|
|
39
|
+
"reasoning-trajectory-tracking",
|
|
40
|
+
"reasoning-pattern-quality-gates",
|
|
41
|
+
"reasoning-experience-replay",
|
|
42
|
+
"reasoning-cross-agent-sharing"
|
|
43
|
+
],
|
|
44
|
+
"description": "ReasoningBank trajectory learning and pattern consolidation"
|
|
45
|
+
},
|
|
46
|
+
"summary": {
|
|
47
|
+
"file": "adr-051-integration-summary.json",
|
|
48
|
+
"description": "Complete ADR-051 integration architecture overview"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
"pattern_statistics": {
|
|
53
|
+
"total_patterns": 15,
|
|
54
|
+
"namespaces": 5,
|
|
55
|
+
"avg_success_rate": 0.915,
|
|
56
|
+
"implementation_status": "Production Ready"
|
|
57
|
+
},
|
|
58
|
+
"access_patterns": [
|
|
59
|
+
{
|
|
60
|
+
"pattern_type": "Simple Lookup",
|
|
61
|
+
"operation": "Load specific pattern by key",
|
|
62
|
+
"time_complexity": "O(1)",
|
|
63
|
+
"use_case": "Quick eligibility checks, cache lookup"
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"pattern_type": "Namespace Query",
|
|
67
|
+
"operation": "Load all patterns in namespace",
|
|
68
|
+
"time_complexity": "O(n) where n=patterns per namespace",
|
|
69
|
+
"use_case": "Multi-pattern decision making, agent coordination"
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"pattern_type": "Semantic Search",
|
|
73
|
+
"operation": "Find patterns by similarity",
|
|
74
|
+
"time_complexity": "O(n) with embeddings",
|
|
75
|
+
"use_case": "Pattern discovery, cross-session learning"
|
|
76
|
+
}
|
|
77
|
+
],
|
|
78
|
+
"session_initialization": {
|
|
79
|
+
"steps": [
|
|
80
|
+
"Load index.json to discover available patterns",
|
|
81
|
+
"Initialize embedding cache from embeddings-patterns",
|
|
82
|
+
"Load router configuration from router-patterns",
|
|
83
|
+
"Initialize booster eligibility rules from booster-patterns",
|
|
84
|
+
"Setup ReasoningBank trajectory database from reasoning-patterns",
|
|
85
|
+
"Configure quality gates and thresholds",
|
|
86
|
+
"Register pattern sharing channels"
|
|
87
|
+
],
|
|
88
|
+
"initialization_time": "<100ms"
|
|
89
|
+
},
|
|
90
|
+
"cross_session_learning": {
|
|
91
|
+
"enabled": true,
|
|
92
|
+
"persistence_layer": "JSON files in .agentic-qe/patterns/",
|
|
93
|
+
"synchronization": "File-based with timestamps",
|
|
94
|
+
"conflict_resolution": "Latest timestamp wins",
|
|
95
|
+
"archival_strategy": "Keep 30-day history in patterns-archive/",
|
|
96
|
+
"memory_coordination": {
|
|
97
|
+
"agent_communication": "Via shared pattern namespace",
|
|
98
|
+
"knowledge_sharing": "Gossip protocol simulation via file watching",
|
|
99
|
+
"learning_integration": "Apply new patterns on each session start"
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
"usage_examples": {
|
|
103
|
+
"example_1_booster_check": {
|
|
104
|
+
"query": "Is 'var-to-const' transform eligible for Agent Booster?",
|
|
105
|
+
"lookup": "patterns['adr-051/booster-patterns']['booster-transform-eligibility'].eligibility_criteria.simple_transforms",
|
|
106
|
+
"result": "YES - found in eligibility list"
|
|
107
|
+
},
|
|
108
|
+
"example_2_router_selection": {
|
|
109
|
+
"query": "Route a 5-file refactoring task",
|
|
110
|
+
"lookup": "patterns['adr-051/router-patterns']['router-5tier-complexity'].scoring_factors",
|
|
111
|
+
"result": "Complexity score 6 → Route to Sonnet tier"
|
|
112
|
+
},
|
|
113
|
+
"example_3_embedding_cache": {
|
|
114
|
+
"query": "Check embedding cache for similarity query",
|
|
115
|
+
"lookup": "patterns['adr-051/embedding-patterns']['embeddings-lru-cache'].cache_key_strategy",
|
|
116
|
+
"result": "Check SHA256(text) in cache, 70% expected hit rate"
|
|
117
|
+
},
|
|
118
|
+
"example_4_pattern_quality": {
|
|
119
|
+
"query": "Is new trajectory eligible for ReasoningBank?",
|
|
120
|
+
"lookup": "patterns['adr-051/reasoning-patterns']['reasoning-pattern-quality-gates'].gate_thresholds",
|
|
121
|
+
"result": "Check composite score >= 0.80 for silver tier"
|
|
122
|
+
}
|
|
123
|
+
},
|
|
124
|
+
"maintenance": {
|
|
125
|
+
"review_frequency": "Monthly",
|
|
126
|
+
"update_triggers": [
|
|
127
|
+
"New successful pattern discovered",
|
|
128
|
+
"Pattern success rate drops below threshold",
|
|
129
|
+
"New component integration requires patterns",
|
|
130
|
+
"Performance optimization identified"
|
|
131
|
+
],
|
|
132
|
+
"archival_directory": ".agentic-qe/patterns-archive/",
|
|
133
|
+
"rollback_capability": "Restore from archive by date"
|
|
134
|
+
},
|
|
135
|
+
"notes": "This index enables fast pattern discovery and multi-session learning without requiring external MCP tools or CLI. Patterns are loaded on session initialization and used throughout agent coordination."
|
|
136
|
+
}
|
|
@@ -1,111 +1,111 @@
|
|
|
1
|
-
# Skill Validation Infrastructure
|
|
2
|
-
|
|
3
|
-
**Version**: 1.0.0
|
|
4
|
-
**ADR**: [ADR-056: Deterministic Skill Validation System](../../v3/implementation/adrs/ADR-056-skill-validation-system.md)
|
|
5
|
-
|
|
6
|
-
This directory contains the validation infrastructure for AQE skills - ensuring skill outputs are deterministic and trustworthy.
|
|
7
|
-
|
|
8
|
-
## Directory Structure
|
|
9
|
-
|
|
10
|
-
```
|
|
11
|
-
.validation/
|
|
12
|
-
├── schemas/ # JSON Schema definitions
|
|
13
|
-
│ ├── skill-frontmatter.schema.json # SKILL.md frontmatter validation
|
|
14
|
-
│ ├── skill-output.template.json # Base output schema template
|
|
15
|
-
│ ├── skill-output-meta.schema.json # Meta-schema for skill schemas
|
|
16
|
-
│ └── skill-eval.schema.json # Evaluation suite schema
|
|
17
|
-
├── templates/ # Templates for skill authors
|
|
18
|
-
│ ├── validate.template.sh # Bash validator template
|
|
19
|
-
│ ├── validator-lib.cjs # Shared validation utilities
|
|
20
|
-
│ ├── eval.template.yaml # Evaluation suite template
|
|
21
|
-
│ ├── security-testing-eval.template.yaml # Domain-specific example
|
|
22
|
-
│ └── skill-frontmatter.example.yaml # Frontmatter example
|
|
23
|
-
├── examples/ # Example skill outputs
|
|
24
|
-
│ ├── security-testing-output.example.json
|
|
25
|
-
│ └── testability-scoring-output.example.json
|
|
26
|
-
├── test-data/ # Test data for validation
|
|
27
|
-
│ ├── minimal-output.json
|
|
28
|
-
│ ├── sample-output.json
|
|
29
|
-
│ └── invalid-output.json
|
|
30
|
-
├── skill-validation-mcp-integration.md # MCP integration spec
|
|
31
|
-
└── README.md # This file
|
|
32
|
-
```
|
|
33
|
-
|
|
34
|
-
## 4-Layer Validation Architecture
|
|
35
|
-
|
|
36
|
-
| Layer | Purpose | Files |
|
|
37
|
-
|-------|---------|-------|
|
|
38
|
-
| **L0** | Intent (SKILL.md) | Declarative instructions |
|
|
39
|
-
| **L1** | Schema | `schemas/*.schema.json` |
|
|
40
|
-
| **L2** | Validator | `templates/validate.template.sh` |
|
|
41
|
-
| **L3** | Eval Suite | `templates/eval.template.yaml` |
|
|
42
|
-
|
|
43
|
-
## Trust Tiers
|
|
44
|
-
|
|
45
|
-
| Tier | Name | Requirements |
|
|
46
|
-
|------|------|--------------|
|
|
47
|
-
| 0 | advisory | SKILL.md only |
|
|
48
|
-
| 1 | structured | SKILL.md + JSON Schema |
|
|
49
|
-
| 2 | validated | SKILL.md + Schema + Validator |
|
|
50
|
-
| 3 | verified | All above + Eval Suite |
|
|
51
|
-
| 4 | certified | All above + CI Green + MCP Learning |
|
|
52
|
-
|
|
53
|
-
## Usage
|
|
54
|
-
|
|
55
|
-
### Validate a Skill Output
|
|
56
|
-
|
|
57
|
-
```bash
|
|
58
|
-
# Using the template directly
|
|
59
|
-
.validation/templates/validate.template.sh output.json
|
|
60
|
-
|
|
61
|
-
# With options
|
|
62
|
-
.validation/templates/validate.template.sh --verbose output.json
|
|
63
|
-
.validation/templates/validate.template.sh --json output.json # JSON output
|
|
64
|
-
.validation/templates/validate.template.sh --self-test # Self-test mode
|
|
65
|
-
```
|
|
66
|
-
|
|
67
|
-
### Create a Skill Validator
|
|
68
|
-
|
|
69
|
-
1. Copy `templates/validate.template.sh` to your skill's directory
|
|
70
|
-
2. Customize the configuration section at the top
|
|
71
|
-
3. Add skill-specific validation in `run_skill_specific_validation()`
|
|
72
|
-
4. Update the output schema path
|
|
73
|
-
|
|
74
|
-
### Create an Eval Suite
|
|
75
|
-
|
|
76
|
-
1. Copy `templates/eval.template.yaml` to your skill's directory
|
|
77
|
-
2. Add test cases with inputs and expected outputs
|
|
78
|
-
3. Define passing criteria (pass rate, required patterns)
|
|
79
|
-
|
|
80
|
-
## MCP Integration
|
|
81
|
-
|
|
82
|
-
Skills MUST use AQE MCP tools for shared learning. See `skill-validation-mcp-integration.md` for details.
|
|
83
|
-
|
|
84
|
-
Key MCP calls:
|
|
85
|
-
- `memory_store` - Store validation patterns
|
|
86
|
-
- `memory_query` - Query learned patterns
|
|
87
|
-
- `test_outcome_track` - Track outcomes
|
|
88
|
-
- `memory_share` - Share learning with fleet
|
|
89
|
-
- `quality_assess` - Update quality scores
|
|
90
|
-
|
|
91
|
-
## For Skill Authors
|
|
92
|
-
|
|
93
|
-
When creating a new skill:
|
|
94
|
-
|
|
95
|
-
1. **Start with SKILL.md** (Trust Tier 0)
|
|
96
|
-
2. **Add output schema** in `your-skill/schemas/output.schema.json` (Trust Tier 1)
|
|
97
|
-
3. **Add validator script** in `your-skill/validate-skill.cjs` (Trust Tier 2)
|
|
98
|
-
4. **Add eval suite** in `your-skill/eval.yaml` (Trust Tier 3)
|
|
99
|
-
5. **Enable CI + MCP learning** (Trust Tier 4)
|
|
100
|
-
|
|
101
|
-
## Validation Exit Codes
|
|
102
|
-
|
|
103
|
-
| Code | Meaning |
|
|
104
|
-
|------|---------|
|
|
105
|
-
| 0 | Validation passed |
|
|
106
|
-
| 1 | Validation failed |
|
|
107
|
-
| 2 | Validation skipped (missing tools) |
|
|
108
|
-
|
|
109
|
-
---
|
|
110
|
-
|
|
111
|
-
*Part of AQE v3 Skill Validation System*
|
|
1
|
+
# Skill Validation Infrastructure
|
|
2
|
+
|
|
3
|
+
**Version**: 1.0.0
|
|
4
|
+
**ADR**: [ADR-056: Deterministic Skill Validation System](../../v3/implementation/adrs/ADR-056-skill-validation-system.md)
|
|
5
|
+
|
|
6
|
+
This directory contains the validation infrastructure for AQE skills - ensuring skill outputs are deterministic and trustworthy.
|
|
7
|
+
|
|
8
|
+
## Directory Structure
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
.validation/
|
|
12
|
+
├── schemas/ # JSON Schema definitions
|
|
13
|
+
│ ├── skill-frontmatter.schema.json # SKILL.md frontmatter validation
|
|
14
|
+
│ ├── skill-output.template.json # Base output schema template
|
|
15
|
+
│ ├── skill-output-meta.schema.json # Meta-schema for skill schemas
|
|
16
|
+
│ └── skill-eval.schema.json # Evaluation suite schema
|
|
17
|
+
├── templates/ # Templates for skill authors
|
|
18
|
+
│ ├── validate.template.sh # Bash validator template
|
|
19
|
+
│ ├── validator-lib.cjs # Shared validation utilities
|
|
20
|
+
│ ├── eval.template.yaml # Evaluation suite template
|
|
21
|
+
│ ├── security-testing-eval.template.yaml # Domain-specific example
|
|
22
|
+
│ └── skill-frontmatter.example.yaml # Frontmatter example
|
|
23
|
+
├── examples/ # Example skill outputs
|
|
24
|
+
│ ├── security-testing-output.example.json
|
|
25
|
+
│ └── testability-scoring-output.example.json
|
|
26
|
+
├── test-data/ # Test data for validation
|
|
27
|
+
│ ├── minimal-output.json
|
|
28
|
+
│ ├── sample-output.json
|
|
29
|
+
│ └── invalid-output.json
|
|
30
|
+
├── skill-validation-mcp-integration.md # MCP integration spec
|
|
31
|
+
└── README.md # This file
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## 4-Layer Validation Architecture
|
|
35
|
+
|
|
36
|
+
| Layer | Purpose | Files |
|
|
37
|
+
|-------|---------|-------|
|
|
38
|
+
| **L0** | Intent (SKILL.md) | Declarative instructions |
|
|
39
|
+
| **L1** | Schema | `schemas/*.schema.json` |
|
|
40
|
+
| **L2** | Validator | `templates/validate.template.sh` |
|
|
41
|
+
| **L3** | Eval Suite | `templates/eval.template.yaml` |
|
|
42
|
+
|
|
43
|
+
## Trust Tiers
|
|
44
|
+
|
|
45
|
+
| Tier | Name | Requirements |
|
|
46
|
+
|------|------|--------------|
|
|
47
|
+
| 0 | advisory | SKILL.md only |
|
|
48
|
+
| 1 | structured | SKILL.md + JSON Schema |
|
|
49
|
+
| 2 | validated | SKILL.md + Schema + Validator |
|
|
50
|
+
| 3 | verified | All above + Eval Suite |
|
|
51
|
+
| 4 | certified | All above + CI Green + MCP Learning |
|
|
52
|
+
|
|
53
|
+
## Usage
|
|
54
|
+
|
|
55
|
+
### Validate a Skill Output
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
# Using the template directly
|
|
59
|
+
.validation/templates/validate.template.sh output.json
|
|
60
|
+
|
|
61
|
+
# With options
|
|
62
|
+
.validation/templates/validate.template.sh --verbose output.json
|
|
63
|
+
.validation/templates/validate.template.sh --json output.json # JSON output
|
|
64
|
+
.validation/templates/validate.template.sh --self-test # Self-test mode
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### Create a Skill Validator
|
|
68
|
+
|
|
69
|
+
1. Copy `templates/validate.template.sh` to your skill's directory
|
|
70
|
+
2. Customize the configuration section at the top
|
|
71
|
+
3. Add skill-specific validation in `run_skill_specific_validation()`
|
|
72
|
+
4. Update the output schema path
|
|
73
|
+
|
|
74
|
+
### Create an Eval Suite
|
|
75
|
+
|
|
76
|
+
1. Copy `templates/eval.template.yaml` to your skill's directory
|
|
77
|
+
2. Add test cases with inputs and expected outputs
|
|
78
|
+
3. Define passing criteria (pass rate, required patterns)
|
|
79
|
+
|
|
80
|
+
## MCP Integration
|
|
81
|
+
|
|
82
|
+
Skills MUST use AQE MCP tools for shared learning. See `skill-validation-mcp-integration.md` for details.
|
|
83
|
+
|
|
84
|
+
Key MCP calls:
|
|
85
|
+
- `memory_store` - Store validation patterns
|
|
86
|
+
- `memory_query` - Query learned patterns
|
|
87
|
+
- `test_outcome_track` - Track outcomes
|
|
88
|
+
- `memory_share` - Share learning with fleet
|
|
89
|
+
- `quality_assess` - Update quality scores
|
|
90
|
+
|
|
91
|
+
## For Skill Authors
|
|
92
|
+
|
|
93
|
+
When creating a new skill:
|
|
94
|
+
|
|
95
|
+
1. **Start with SKILL.md** (Trust Tier 0)
|
|
96
|
+
2. **Add output schema** in `your-skill/schemas/output.schema.json` (Trust Tier 1)
|
|
97
|
+
3. **Add validator script** in `your-skill/validate-skill.cjs` (Trust Tier 2)
|
|
98
|
+
4. **Add eval suite** in `your-skill/eval.yaml` (Trust Tier 3)
|
|
99
|
+
5. **Enable CI + MCP learning** (Trust Tier 4)
|
|
100
|
+
|
|
101
|
+
## Validation Exit Codes
|
|
102
|
+
|
|
103
|
+
| Code | Meaning |
|
|
104
|
+
|------|---------|
|
|
105
|
+
| 0 | Validation passed |
|
|
106
|
+
| 1 | Validation failed |
|
|
107
|
+
| 2 | Validation skipped (missing tools) |
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
*Part of AQE v3 Skill Validation System*
|