agentic-qe 3.7.9 → 3.7.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/.validation/README.md +111 -111
- package/.claude/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
- package/.claude/skills/.validation/examples/performance-testing-output.example.json +252 -252
- package/.claude/skills/.validation/examples/security-testing-output.example.json +413 -413
- package/.claude/skills/.validation/examples/testability-scoring-output.example.json +350 -350
- package/.claude/skills/.validation/schemas/skill-eval.schema.json +462 -462
- package/.claude/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
- package/.claude/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
- package/.claude/skills/.validation/schemas/skill-output.template.json +610 -610
- package/.claude/skills/.validation/skill-validation-mcp-integration.md +250 -250
- package/.claude/skills/.validation/templates/eval.template.yaml +366 -366
- package/.claude/skills/.validation/templates/schemas/output.json +145 -145
- package/.claude/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
- package/.claude/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
- package/.claude/skills/.validation/test-data/invalid-output.json +5 -5
- package/.claude/skills/.validation/test-data/minimal-output.json +9 -9
- package/.claude/skills/.validation/test-data/sample-output.json +73 -73
- package/.claude/skills/a11y-ally/evals/a11y-ally.yaml +376 -376
- package/.claude/skills/a11y-ally/schemas/output.json +549 -549
- package/.claude/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
- package/.claude/skills/accessibility-testing/schemas/output.json +776 -776
- package/.claude/skills/accessibility-testing/test-data/sample-output.json +191 -191
- package/.claude/skills/agentic-quality-engineering/schemas/output.json +577 -577
- package/.claude/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
- package/.claude/skills/api-testing-patterns/schemas/output.json +845 -845
- package/.claude/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
- package/.claude/skills/brutal-honesty-review/schemas/output.json +291 -291
- package/.claude/skills/bug-reporting-excellence/schemas/output.json +288 -288
- package/.claude/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
- package/.claude/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
- package/.claude/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -157
- package/.claude/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -542
- package/.claude/skills/code-review-quality/schemas/output.json +264 -264
- package/.claude/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
- package/.claude/skills/compatibility-testing/schemas/output.json +551 -551
- package/.claude/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
- package/.claude/skills/compliance-testing/schemas/output.json +845 -845
- package/.claude/skills/consultancy-practices/schemas/output.json +282 -282
- package/.claude/skills/contract-testing/evals/contract-testing.yaml +748 -748
- package/.claude/skills/contract-testing/schemas/output.json +638 -638
- package/.claude/skills/database-testing/evals/database-testing.yaml +968 -968
- package/.claude/skills/database-testing/schemas/output.json +1446 -1446
- package/.claude/skills/debug-loop/SKILL.md +61 -61
- package/.claude/skills/enterprise-integration-testing/SKILL.md +735 -735
- package/.claude/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -158
- package/.claude/skills/enterprise-integration-testing/schemas/output.json +74 -74
- package/.claude/skills/enterprise-integration-testing/scripts/validate-config.json +25 -25
- package/.claude/skills/iterative-loop/SKILL.md +371 -371
- package/.claude/skills/localization-testing/evals/localization-testing.yaml +544 -544
- package/.claude/skills/localization-testing/schemas/output.json +325 -325
- package/.claude/skills/middleware-testing-patterns/SKILL.md +798 -798
- package/.claude/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -153
- package/.claude/skills/middleware-testing-patterns/schemas/output.json +58 -58
- package/.claude/skills/middleware-testing-patterns/scripts/validate-config.json +25 -25
- package/.claude/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
- package/.claude/skills/mobile-testing/schemas/output.json +318 -318
- package/.claude/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
- package/.claude/skills/mutation-testing/schemas/output.json +707 -707
- package/.claude/skills/mutation-testing/test-data/sample-output.json +295 -295
- package/.claude/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -450
- package/.claude/skills/n8n-expression-testing/schemas/output.json +369 -369
- package/.claude/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -522
- package/.claude/skills/n8n-integration-testing-patterns/schemas/output.json +291 -291
- package/.claude/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -493
- package/.claude/skills/n8n-security-testing/schemas/output.json +293 -293
- package/.claude/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -500
- package/.claude/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -295
- package/.claude/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -497
- package/.claude/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -254
- package/.claude/skills/observability-testing-patterns/SKILL.md +930 -930
- package/.claude/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -157
- package/.claude/skills/observability-testing-patterns/schemas/output.json +58 -58
- package/.claude/skills/observability-testing-patterns/scripts/validate-config.json +25 -25
- package/.claude/skills/pentest-validation/SKILL.md +268 -268
- package/.claude/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
- package/.claude/skills/pentest-validation/schemas/output.json +281 -281
- package/.claude/skills/performance-analysis/evals/performance-analysis.yaml +144 -144
- package/.claude/skills/performance-analysis/schemas/output.json +588 -588
- package/.claude/skills/performance-testing/evals/performance-testing.yaml +772 -772
- package/.claude/skills/performance-testing/schemas/output.json +1184 -1184
- package/.claude/skills/pr-review/SKILL.md +61 -61
- package/.claude/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
- package/.claude/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -211
- package/.claude/skills/qcsd-cicd-swarm/schemas/output.json +86 -86
- package/.claude/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -30
- package/.claude/skills/qcsd-development-swarm/SKILL.md +2154 -2154
- package/.claude/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -162
- package/.claude/skills/qcsd-development-swarm/schemas/output.json +72 -72
- package/.claude/skills/qcsd-development-swarm/scripts/validate-config.json +25 -25
- package/.claude/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -138
- package/.claude/skills/qcsd-ideation-swarm/schemas/output.json +568 -568
- package/.claude/skills/qcsd-production-swarm/SKILL.md +2781 -2781
- package/.claude/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -246
- package/.claude/skills/qcsd-production-swarm/schemas/output.json +505 -505
- package/.claude/skills/qcsd-production-swarm/scripts/validate-config.json +25 -25
- package/.claude/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
- package/.claude/skills/qe-chaos-resilience/schemas/output.json +314 -314
- package/.claude/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
- package/.claude/skills/qe-code-intelligence/schemas/output.json +315 -315
- package/.claude/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
- package/.claude/skills/qe-contract-testing/schemas/output.json +295 -295
- package/.claude/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
- package/.claude/skills/qe-coverage-analysis/schemas/output.json +286 -286
- package/.claude/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
- package/.claude/skills/qe-defect-intelligence/schemas/output.json +283 -283
- package/.claude/skills/qe-iterative-loop/schemas/output.json +264 -264
- package/.claude/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
- package/.claude/skills/qe-learning-optimization/schemas/output.json +288 -288
- package/.claude/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
- package/.claude/skills/qe-quality-assessment/schemas/output.json +550 -550
- package/.claude/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
- package/.claude/skills/qe-requirements-validation/schemas/output.json +587 -587
- package/.claude/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
- package/.claude/skills/qe-security-compliance/schemas/output.json +498 -498
- package/.claude/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
- package/.claude/skills/qe-test-execution/schemas/output.json +529 -529
- package/.claude/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
- package/.claude/skills/qe-test-generation/schemas/output.json +439 -439
- package/.claude/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
- package/.claude/skills/qe-visual-accessibility/schemas/output.json +491 -491
- package/.claude/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
- package/.claude/skills/quality-metrics/schemas/output.json +403 -403
- package/.claude/skills/refactoring-patterns/schemas/output.json +475 -475
- package/.claude/skills/regression-testing/evals/regression-testing.yaml +504 -504
- package/.claude/skills/regression-testing/schemas/output.json +311 -311
- package/.claude/skills/release/SKILL.md +347 -347
- package/.claude/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
- package/.claude/skills/risk-based-testing/schemas/output.json +480 -480
- package/.claude/skills/security-testing/evals/security-testing.yaml +789 -789
- package/.claude/skills/security-testing/schemas/output.json +879 -879
- package/.claude/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
- package/.claude/skills/security-visual-testing/schemas/output.json +486 -486
- package/.claude/skills/sfdipot-product-factors/SKILL.md +239 -239
- package/.claude/skills/sherlock-review/schemas/output.json +297 -297
- package/.claude/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
- package/.claude/skills/shift-left-testing/schemas/output.json +459 -459
- package/.claude/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
- package/.claude/skills/shift-right-testing/schemas/output.json +418 -418
- package/.claude/skills/skills-manifest.json +1 -1
- package/.claude/skills/tdd-london-chicago/schemas/output.json +444 -444
- package/.claude/skills/technical-writing/schemas/output.json +268 -268
- package/.claude/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
- package/.claude/skills/test-automation-strategy/schemas/output.json +444 -444
- package/.claude/skills/test-data-management/evals/test-data-management.yaml +504 -504
- package/.claude/skills/test-data-management/schemas/output.json +284 -284
- package/.claude/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
- package/.claude/skills/test-design-techniques/schemas/output.json +295 -295
- package/.claude/skills/test-environment-management/schemas/output.json +310 -310
- package/.claude/skills/test-idea-rewriting/SKILL.md +229 -229
- package/.claude/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
- package/.claude/skills/test-reporting-analytics/schemas/output.json +329 -329
- package/.claude/skills/testability-scoring/evals/testability-scoring.yaml +814 -814
- package/.claude/skills/testability-scoring/resources/templates/config.template.js +84 -84
- package/.claude/skills/testability-scoring/schemas/output.json +606 -606
- package/.claude/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
- package/.claude/skills/verification-quality/evals/verification-quality.yaml +150 -150
- package/.claude/skills/verification-quality/schemas/output.json +432 -432
- package/.claude/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
- package/.claude/skills/visual-testing-advanced/schemas/output.json +294 -294
- package/.claude/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -165
- package/.claude/skills/wms-testing-patterns/schemas/output.json +150 -150
- package/.claude/skills/wms-testing-patterns/scripts/validate-config.json +51 -51
- package/CHANGELOG.md +36 -0
- package/README.md +171 -901
- package/assets/agents/v3/helpers/quality-criteria/evidence-classification.md +116 -116
- package/assets/agents/v3/helpers/quality-criteria/htsm-categories.md +139 -139
- package/assets/governance/constitution.md +202 -202
- package/assets/governance/shards/chaos-resilience.shard.md +221 -221
- package/assets/governance/shards/code-intelligence.shard.md +178 -178
- package/assets/governance/shards/contract-testing.shard.md +206 -206
- package/assets/governance/shards/coverage-analysis.shard.md +146 -146
- package/assets/governance/shards/defect-intelligence.shard.md +182 -182
- package/assets/governance/shards/learning-optimization.shard.md +248 -248
- package/assets/governance/shards/quality-assessment.shard.md +165 -165
- package/assets/governance/shards/requirements-validation.shard.md +177 -177
- package/assets/governance/shards/security-compliance.shard.md +196 -196
- package/assets/governance/shards/test-execution.shard.md +156 -156
- package/assets/governance/shards/test-generation.shard.md +128 -128
- package/assets/governance/shards/visual-accessibility.shard.md +209 -209
- package/assets/hooks/cross-phase-memory.yaml +253 -253
- package/assets/patterns/adr-051-booster-patterns.json +78 -78
- package/assets/patterns/adr-051-embedding-patterns.json +147 -147
- package/assets/patterns/adr-051-integration-summary.json +62 -62
- package/assets/patterns/adr-051-reasoning-patterns.json +166 -166
- package/assets/patterns/adr-051-router-patterns.json +113 -113
- package/assets/patterns/index.json +136 -136
- package/assets/skills/.validation/README.md +111 -111
- package/assets/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
- package/assets/skills/.validation/examples/performance-testing-output.example.json +252 -252
- package/assets/skills/.validation/examples/security-testing-output.example.json +413 -413
- package/assets/skills/.validation/examples/testability-scoring-output.example.json +350 -350
- package/assets/skills/.validation/schemas/skill-eval.schema.json +462 -462
- package/assets/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
- package/assets/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
- package/assets/skills/.validation/schemas/skill-output.template.json +610 -610
- package/assets/skills/.validation/skill-validation-mcp-integration.md +250 -250
- package/assets/skills/.validation/templates/eval.template.yaml +366 -366
- package/assets/skills/.validation/templates/schemas/output.json +145 -145
- package/assets/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
- package/assets/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
- package/assets/skills/.validation/test-data/invalid-output.json +5 -5
- package/assets/skills/.validation/test-data/minimal-output.json +9 -9
- package/assets/skills/.validation/test-data/sample-output.json +73 -73
- package/assets/skills/a11y-ally/evals/a11y-ally.yaml +376 -376
- package/assets/skills/a11y-ally/schemas/output.json +549 -549
- package/assets/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
- package/assets/skills/accessibility-testing/schemas/output.json +776 -776
- package/assets/skills/accessibility-testing/test-data/sample-output.json +191 -191
- package/assets/skills/agentic-quality-engineering/schemas/output.json +577 -577
- package/assets/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
- package/assets/skills/api-testing-patterns/schemas/output.json +845 -845
- package/assets/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
- package/assets/skills/brutal-honesty-review/schemas/output.json +291 -291
- package/assets/skills/bug-reporting-excellence/schemas/output.json +288 -288
- package/assets/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
- package/assets/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
- package/assets/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -157
- package/assets/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -542
- package/assets/skills/code-review-quality/schemas/output.json +264 -264
- package/assets/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
- package/assets/skills/compatibility-testing/schemas/output.json +551 -551
- package/assets/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
- package/assets/skills/compliance-testing/schemas/output.json +845 -845
- package/assets/skills/consultancy-practices/schemas/output.json +282 -282
- package/assets/skills/contract-testing/evals/contract-testing.yaml +748 -748
- package/assets/skills/contract-testing/schemas/output.json +638 -638
- package/assets/skills/database-testing/evals/database-testing.yaml +968 -968
- package/assets/skills/database-testing/schemas/output.json +1446 -1446
- package/assets/skills/debug-loop/SKILL.md +61 -61
- package/assets/skills/enterprise-integration-testing/SKILL.md +735 -735
- package/assets/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -158
- package/assets/skills/enterprise-integration-testing/schemas/output.json +74 -74
- package/assets/skills/enterprise-integration-testing/scripts/validate-config.json +25 -25
- package/assets/skills/localization-testing/evals/localization-testing.yaml +544 -544
- package/assets/skills/localization-testing/schemas/output.json +325 -325
- package/assets/skills/middleware-testing-patterns/SKILL.md +798 -798
- package/assets/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -153
- package/assets/skills/middleware-testing-patterns/schemas/output.json +58 -58
- package/assets/skills/middleware-testing-patterns/scripts/validate-config.json +25 -25
- package/assets/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
- package/assets/skills/mobile-testing/schemas/output.json +318 -318
- package/assets/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
- package/assets/skills/mutation-testing/schemas/output.json +707 -707
- package/assets/skills/mutation-testing/test-data/sample-output.json +295 -295
- package/assets/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -450
- package/assets/skills/n8n-expression-testing/schemas/output.json +369 -369
- package/assets/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -522
- package/assets/skills/n8n-integration-testing-patterns/schemas/output.json +291 -291
- package/assets/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -493
- package/assets/skills/n8n-security-testing/schemas/output.json +293 -293
- package/assets/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -500
- package/assets/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -295
- package/assets/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -497
- package/assets/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -254
- package/assets/skills/observability-testing-patterns/SKILL.md +930 -930
- package/assets/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -157
- package/assets/skills/observability-testing-patterns/schemas/output.json +58 -58
- package/assets/skills/observability-testing-patterns/scripts/validate-config.json +25 -25
- package/assets/skills/pentest-validation/SKILL.md +268 -268
- package/assets/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
- package/assets/skills/pentest-validation/schemas/output.json +281 -281
- package/assets/skills/performance-testing/evals/performance-testing.yaml +772 -772
- package/assets/skills/performance-testing/schemas/output.json +1184 -1184
- package/assets/skills/pr-review/SKILL.md +61 -61
- package/assets/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
- package/assets/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -211
- package/assets/skills/qcsd-cicd-swarm/schemas/output.json +86 -86
- package/assets/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -30
- package/assets/skills/qcsd-development-swarm/SKILL.md +2154 -2154
- package/assets/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -162
- package/assets/skills/qcsd-development-swarm/schemas/output.json +72 -72
- package/assets/skills/qcsd-development-swarm/scripts/validate-config.json +25 -25
- package/assets/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -138
- package/assets/skills/qcsd-ideation-swarm/schemas/output.json +568 -568
- package/assets/skills/qcsd-production-swarm/SKILL.md +2781 -2781
- package/assets/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -246
- package/assets/skills/qcsd-production-swarm/schemas/output.json +505 -505
- package/assets/skills/qcsd-production-swarm/scripts/validate-config.json +25 -25
- package/assets/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
- package/assets/skills/qe-chaos-resilience/schemas/output.json +314 -314
- package/assets/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
- package/assets/skills/qe-code-intelligence/schemas/output.json +315 -315
- package/assets/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
- package/assets/skills/qe-contract-testing/schemas/output.json +295 -295
- package/assets/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
- package/assets/skills/qe-coverage-analysis/schemas/output.json +286 -286
- package/assets/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
- package/assets/skills/qe-defect-intelligence/schemas/output.json +283 -283
- package/assets/skills/qe-iterative-loop/schemas/output.json +264 -264
- package/assets/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
- package/assets/skills/qe-learning-optimization/schemas/output.json +288 -288
- package/assets/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
- package/assets/skills/qe-quality-assessment/schemas/output.json +550 -550
- package/assets/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
- package/assets/skills/qe-requirements-validation/schemas/output.json +587 -587
- package/assets/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
- package/assets/skills/qe-security-compliance/schemas/output.json +498 -498
- package/assets/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
- package/assets/skills/qe-test-execution/schemas/output.json +529 -529
- package/assets/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
- package/assets/skills/qe-test-generation/schemas/output.json +439 -439
- package/assets/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
- package/assets/skills/qe-visual-accessibility/schemas/output.json +491 -491
- package/assets/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
- package/assets/skills/quality-metrics/schemas/output.json +403 -403
- package/assets/skills/refactoring-patterns/schemas/output.json +475 -475
- package/assets/skills/regression-testing/evals/regression-testing.yaml +504 -504
- package/assets/skills/regression-testing/schemas/output.json +311 -311
- package/assets/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
- package/assets/skills/risk-based-testing/schemas/output.json +480 -480
- package/assets/skills/security-testing/evals/security-testing.yaml +789 -789
- package/assets/skills/security-testing/schemas/output.json +879 -879
- package/assets/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
- package/assets/skills/security-visual-testing/schemas/output.json +486 -486
- package/assets/skills/sfdipot-product-factors/SKILL.md +239 -239
- package/assets/skills/sherlock-review/schemas/output.json +297 -297
- package/assets/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
- package/assets/skills/shift-left-testing/schemas/output.json +459 -459
- package/assets/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
- package/assets/skills/shift-right-testing/schemas/output.json +418 -418
- package/assets/skills/tdd-london-chicago/schemas/output.json +444 -444
- package/assets/skills/technical-writing/schemas/output.json +268 -268
- package/assets/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
- package/assets/skills/test-automation-strategy/schemas/output.json +444 -444
- package/assets/skills/test-data-management/evals/test-data-management.yaml +504 -504
- package/assets/skills/test-data-management/schemas/output.json +284 -284
- package/assets/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
- package/assets/skills/test-design-techniques/schemas/output.json +295 -295
- package/assets/skills/test-environment-management/schemas/output.json +310 -310
- package/assets/skills/test-idea-rewriting/SKILL.md +229 -229
- package/assets/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
- package/assets/skills/test-reporting-analytics/schemas/output.json +329 -329
- package/assets/skills/testability-scoring/evals/testability-scoring.yaml +814 -814
- package/assets/skills/testability-scoring/resources/templates/config.template.js +84 -84
- package/assets/skills/testability-scoring/schemas/output.json +606 -606
- package/assets/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
- package/assets/skills/verification-quality/evals/verification-quality.yaml +150 -150
- package/assets/skills/verification-quality/schemas/output.json +432 -432
- package/assets/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
- package/assets/skills/visual-testing-advanced/schemas/output.json +294 -294
- package/assets/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -165
- package/assets/skills/wms-testing-patterns/schemas/output.json +150 -150
- package/assets/skills/wms-testing-patterns/scripts/validate-config.json +51 -51
- package/assets/templates/validation-summary.json +56 -56
- package/dist/agents/claim-verifier/verifiers/output-verifier.d.ts +1 -0
- package/dist/agents/claim-verifier/verifiers/output-verifier.d.ts.map +1 -1
- package/dist/agents/claim-verifier/verifiers/output-verifier.js +16 -3
- package/dist/agents/claim-verifier/verifiers/output-verifier.js.map +1 -1
- package/dist/cli/bundle.js +488 -75
- package/dist/cli/commands/mcp.d.ts.map +1 -1
- package/dist/cli/commands/mcp.js +11 -9
- package/dist/cli/commands/mcp.js.map +1 -1
- package/dist/governance/adversarial-defense-integration.d.ts +29 -0
- package/dist/governance/adversarial-defense-integration.d.ts.map +1 -1
- package/dist/governance/adversarial-defense-integration.js +85 -1
- package/dist/governance/adversarial-defense-integration.js.map +1 -1
- package/dist/governance/continue-gate-integration.d.ts +7 -5
- package/dist/governance/continue-gate-integration.d.ts.map +1 -1
- package/dist/governance/continue-gate-integration.js +72 -15
- package/dist/governance/continue-gate-integration.js.map +1 -1
- package/dist/governance/deterministic-gateway-integration.d.ts +5 -0
- package/dist/governance/deterministic-gateway-integration.d.ts.map +1 -1
- package/dist/governance/deterministic-gateway-integration.js +35 -1
- package/dist/governance/deterministic-gateway-integration.js.map +1 -1
- package/dist/governance/evolution-pipeline-integration.d.ts +5 -0
- package/dist/governance/evolution-pipeline-integration.d.ts.map +1 -1
- package/dist/governance/evolution-pipeline-integration.js +57 -1
- package/dist/governance/evolution-pipeline-integration.js.map +1 -1
- package/dist/governance/memory-write-gate-integration.d.ts +6 -3
- package/dist/governance/memory-write-gate-integration.d.ts.map +1 -1
- package/dist/governance/memory-write-gate-integration.js +48 -10
- package/dist/governance/memory-write-gate-integration.js.map +1 -1
- package/dist/governance/proof-envelope-integration.d.ts +1 -0
- package/dist/governance/proof-envelope-integration.d.ts.map +1 -1
- package/dist/governance/proof-envelope-integration.js +37 -0
- package/dist/governance/proof-envelope-integration.js.map +1 -1
- package/dist/governance/shard-retriever-integration.d.ts +10 -0
- package/dist/governance/shard-retriever-integration.d.ts.map +1 -1
- package/dist/governance/shard-retriever-integration.js +50 -1
- package/dist/governance/shard-retriever-integration.js.map +1 -1
- package/dist/governance/trust-accumulator-integration.d.ts +5 -0
- package/dist/governance/trust-accumulator-integration.d.ts.map +1 -1
- package/dist/governance/trust-accumulator-integration.js +35 -2
- package/dist/governance/trust-accumulator-integration.js.map +1 -1
- package/dist/init/phases/10-workers.js +2 -2
- package/dist/init/phases/10-workers.js.map +1 -1
- package/dist/mcp/bundle.js +478 -62
- package/dist/mcp/protocol-server.js +2 -2
- package/dist/mcp/tool-registry.d.ts.map +1 -1
- package/dist/mcp/tool-registry.js +1 -0
- package/dist/mcp/tool-registry.js.map +1 -1
- package/dist/shared/language-detector.js +1 -1
- package/dist/shared/language-detector.js.map +1 -1
- package/dist/shared/parsers/multi-language-parser.d.ts.map +1 -1
- package/dist/shared/parsers/multi-language-parser.js +55 -16
- package/dist/shared/parsers/multi-language-parser.js.map +1 -1
- package/dist/shared/sql-safety.d.ts.map +1 -1
- package/dist/shared/sql-safety.js +7 -1
- package/dist/shared/sql-safety.js.map +1 -1
- package/package.json +6 -2
- package/scripts/prepare-assets.sh +8 -0
|
@@ -1,652 +1,652 @@
|
|
|
1
|
-
# =============================================================================
|
|
2
|
-
# Mutation Testing Skill Evaluation Test Suite v1.0.0
|
|
3
|
-
# Path: .claude/skills/mutation-testing/evals/mutation-testing.yaml
|
|
4
|
-
# =============================================================================
|
|
5
|
-
#
|
|
6
|
-
# This evaluation suite validates mutation testing skill behavior through:
|
|
7
|
-
# 1. Input/expected-output test cases for mutation operators
|
|
8
|
-
# 2. Multi-model consistency testing
|
|
9
|
-
# 3. Semantic validation of mutation analysis quality
|
|
10
|
-
# 4. AQE MCP integration for shared learning
|
|
11
|
-
#
|
|
12
|
-
# Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
|
|
13
|
-
# =============================================================================
|
|
14
|
-
|
|
15
|
-
skill: mutation-testing
|
|
16
|
-
version: 1.0.0
|
|
17
|
-
description: >
|
|
18
|
-
Comprehensive evaluation test suite for the mutation-testing skill.
|
|
19
|
-
Tests mutation score calculation, operator detection, surviving mutant
|
|
20
|
-
analysis, and test improvement recommendations across multiple models
|
|
21
|
-
to ensure consistent, high-quality mutation analysis output.
|
|
22
|
-
|
|
23
|
-
# =============================================================================
|
|
24
|
-
# Multi-Model Configuration
|
|
25
|
-
# =============================================================================
|
|
26
|
-
|
|
27
|
-
models_to_test:
|
|
28
|
-
- claude-3.5-sonnet # Primary model (high accuracy expected)
|
|
29
|
-
- claude-3-haiku # Fast model (ensure minimum quality)
|
|
30
|
-
- gpt-4o # Cross-vendor validation
|
|
31
|
-
|
|
32
|
-
# =============================================================================
|
|
33
|
-
# MCP Integration Configuration
|
|
34
|
-
# =============================================================================
|
|
35
|
-
|
|
36
|
-
mcp_integration:
|
|
37
|
-
enabled: true
|
|
38
|
-
namespace: skill-validation/mutation-testing
|
|
39
|
-
|
|
40
|
-
query_patterns: true
|
|
41
|
-
track_outcomes: true
|
|
42
|
-
store_patterns: true
|
|
43
|
-
share_learning: true
|
|
44
|
-
update_quality_gate: true
|
|
45
|
-
|
|
46
|
-
target_agents:
|
|
47
|
-
- qe-learning-coordinator
|
|
48
|
-
- qe-queen-coordinator
|
|
49
|
-
- qe-mutation-tester
|
|
50
|
-
|
|
51
|
-
# =============================================================================
|
|
52
|
-
# ReasoningBank Learning Configuration
|
|
53
|
-
# =============================================================================
|
|
54
|
-
|
|
55
|
-
learning:
|
|
56
|
-
store_success_patterns: true
|
|
57
|
-
store_failure_patterns: true
|
|
58
|
-
pattern_ttl_days: 90
|
|
59
|
-
min_confidence_to_store: 0.7
|
|
60
|
-
cross_model_comparison: true
|
|
61
|
-
|
|
62
|
-
# =============================================================================
|
|
63
|
-
# Result Format Configuration
|
|
64
|
-
# =============================================================================
|
|
65
|
-
|
|
66
|
-
result_format:
|
|
67
|
-
json_output: true
|
|
68
|
-
markdown_report: true
|
|
69
|
-
include_raw_output: false
|
|
70
|
-
include_timing: true
|
|
71
|
-
include_token_usage: true
|
|
72
|
-
|
|
73
|
-
# =============================================================================
|
|
74
|
-
# Environment Setup
|
|
75
|
-
# =============================================================================
|
|
76
|
-
|
|
77
|
-
setup:
|
|
78
|
-
required_tools:
|
|
79
|
-
- jq
|
|
80
|
-
environment_variables:
|
|
81
|
-
MUTATION_TIMEOUT_MS: "30000"
|
|
82
|
-
fixtures:
|
|
83
|
-
- name: simple_arithmetic_code
|
|
84
|
-
content: |
|
|
85
|
-
function calculateTotal(price, quantity) {
|
|
86
|
-
return price * quantity;
|
|
87
|
-
}
|
|
88
|
-
- name: conditional_code
|
|
89
|
-
content: |
|
|
90
|
-
function isAdult(age) {
|
|
91
|
-
return age >= 18;
|
|
92
|
-
}
|
|
93
|
-
- name: complex_code
|
|
94
|
-
content: |
|
|
95
|
-
function processOrder(order) {
|
|
96
|
-
if (order.quantity <= 0) {
|
|
97
|
-
throw new Error('Invalid quantity');
|
|
98
|
-
}
|
|
99
|
-
let discount = 0;
|
|
100
|
-
if (order.quantity >= 10 && order.isPremium) {
|
|
101
|
-
discount = 0.1;
|
|
102
|
-
}
|
|
103
|
-
return order.price * order.quantity * (1 - discount);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
# =============================================================================
|
|
107
|
-
# Test Cases
|
|
108
|
-
# =============================================================================
|
|
109
|
-
|
|
110
|
-
test_cases:
|
|
111
|
-
# -------------------------------------------------------------------------
|
|
112
|
-
# Basic Functionality Tests
|
|
113
|
-
# -------------------------------------------------------------------------
|
|
114
|
-
|
|
115
|
-
- id: tc001_basic_mutation_analysis
|
|
116
|
-
description: "Skill analyzes simple code and identifies potential mutations"
|
|
117
|
-
category: basic
|
|
118
|
-
priority: critical
|
|
119
|
-
|
|
120
|
-
input:
|
|
121
|
-
code: |
|
|
122
|
-
function add(a, b) {
|
|
123
|
-
return a + b;
|
|
124
|
-
}
|
|
125
|
-
context:
|
|
126
|
-
language: javascript
|
|
127
|
-
prompt: |
|
|
128
|
-
Analyze this code for mutation testing. Identify what mutations
|
|
129
|
-
could be applied and what tests would be needed to kill them.
|
|
130
|
-
|
|
131
|
-
expected_output:
|
|
132
|
-
must_contain:
|
|
133
|
-
- "mutation"
|
|
134
|
-
- "arithmetic"
|
|
135
|
-
- "+"
|
|
136
|
-
must_not_contain:
|
|
137
|
-
- "unable to analyze"
|
|
138
|
-
- "error"
|
|
139
|
-
finding_count:
|
|
140
|
-
min: 1
|
|
141
|
-
max: 10
|
|
142
|
-
|
|
143
|
-
validation:
|
|
144
|
-
schema_check: true
|
|
145
|
-
keyword_match_threshold: 0.8
|
|
146
|
-
reasoning_quality_min: 0.7
|
|
147
|
-
|
|
148
|
-
- id: tc002_mutation_score_calculation
|
|
149
|
-
description: "Skill correctly calculates mutation score from kill/survive data"
|
|
150
|
-
category: basic
|
|
151
|
-
priority: critical
|
|
152
|
-
|
|
153
|
-
input:
|
|
154
|
-
prompt: |
|
|
155
|
-
Given these mutation testing results:
|
|
156
|
-
- Total mutants: 100
|
|
157
|
-
- Killed: 85
|
|
158
|
-
- Survived: 12
|
|
159
|
-
- Timeout: 2
|
|
160
|
-
- No Coverage: 1
|
|
161
|
-
|
|
162
|
-
Calculate the mutation score and assess the test suite quality.
|
|
163
|
-
|
|
164
|
-
expected_output:
|
|
165
|
-
must_contain:
|
|
166
|
-
- "85"
|
|
167
|
-
- "mutation score"
|
|
168
|
-
- "quality"
|
|
169
|
-
must_match_regex:
|
|
170
|
-
- "8[0-9](\\.\\d+)?%" # Score around 85%
|
|
171
|
-
|
|
172
|
-
validation:
|
|
173
|
-
schema_check: true
|
|
174
|
-
keyword_match_threshold: 0.9
|
|
175
|
-
|
|
176
|
-
# -------------------------------------------------------------------------
|
|
177
|
-
# Arithmetic Operator Tests (AOR)
|
|
178
|
-
# -------------------------------------------------------------------------
|
|
179
|
-
|
|
180
|
-
- id: tc003_arithmetic_operator_mutation
|
|
181
|
-
description: "Skill identifies arithmetic operator mutations (+, -, *, /)"
|
|
182
|
-
category: operators
|
|
183
|
-
priority: high
|
|
184
|
-
|
|
185
|
-
input:
|
|
186
|
-
code: |
|
|
187
|
-
function calculateTotal(price, quantity, tax) {
|
|
188
|
-
const subtotal = price * quantity;
|
|
189
|
-
const taxAmount = subtotal * tax;
|
|
190
|
-
return subtotal + taxAmount;
|
|
191
|
-
}
|
|
192
|
-
context:
|
|
193
|
-
language: javascript
|
|
194
|
-
prompt: |
|
|
195
|
-
Perform mutation analysis focusing on arithmetic operators.
|
|
196
|
-
Identify all AOR (Arithmetic Operator Replacement) mutations.
|
|
197
|
-
|
|
198
|
-
expected_output:
|
|
199
|
-
must_contain:
|
|
200
|
-
- "arithmetic"
|
|
201
|
-
- "*"
|
|
202
|
-
- "+"
|
|
203
|
-
- "mutant"
|
|
204
|
-
must_not_contain:
|
|
205
|
-
- "no mutations"
|
|
206
|
-
|
|
207
|
-
validation:
|
|
208
|
-
schema_check: true
|
|
209
|
-
keyword_match_threshold: 0.8
|
|
210
|
-
|
|
211
|
-
# -------------------------------------------------------------------------
|
|
212
|
-
# Relational Operator Tests (ROR)
|
|
213
|
-
# -------------------------------------------------------------------------
|
|
214
|
-
|
|
215
|
-
- id: tc004_relational_operator_mutation
|
|
216
|
-
description: "Skill identifies relational operator mutations (>, <, >=, <=, ==, !=)"
|
|
217
|
-
category: operators
|
|
218
|
-
priority: high
|
|
219
|
-
|
|
220
|
-
input:
|
|
221
|
-
code: |
|
|
222
|
-
function isEligible(age, income) {
|
|
223
|
-
if (age >= 18 && income > 30000) {
|
|
224
|
-
return true;
|
|
225
|
-
}
|
|
226
|
-
if (age < 65 && income <= 100000) {
|
|
227
|
-
return true;
|
|
228
|
-
}
|
|
229
|
-
return false;
|
|
230
|
-
}
|
|
231
|
-
context:
|
|
232
|
-
language: javascript
|
|
233
|
-
prompt: |
|
|
234
|
-
Analyze this code for relational operator mutations (ROR).
|
|
235
|
-
Identify boundary conditions that need testing.
|
|
236
|
-
|
|
237
|
-
expected_output:
|
|
238
|
-
must_contain:
|
|
239
|
-
- "relational"
|
|
240
|
-
- ">="
|
|
241
|
-
- "boundary"
|
|
242
|
-
- "18"
|
|
243
|
-
must_not_contain:
|
|
244
|
-
- "unable"
|
|
245
|
-
|
|
246
|
-
validation:
|
|
247
|
-
schema_check: true
|
|
248
|
-
keyword_match_threshold: 0.8
|
|
249
|
-
reasoning_quality_min: 0.7
|
|
250
|
-
|
|
251
|
-
- id: tc005_boundary_mutation_detection
|
|
252
|
-
description: "Skill detects boundary value mutations (>= to >, etc.)"
|
|
253
|
-
category: operators
|
|
254
|
-
priority: critical
|
|
255
|
-
|
|
256
|
-
input:
|
|
257
|
-
code: |
|
|
258
|
-
function getDiscount(quantity) {
|
|
259
|
-
if (quantity >= 10) {
|
|
260
|
-
return 0.1; // 10% discount
|
|
261
|
-
}
|
|
262
|
-
return 0;
|
|
263
|
-
}
|
|
264
|
-
context:
|
|
265
|
-
language: javascript
|
|
266
|
-
prompt: |
|
|
267
|
-
This code has a boundary condition at quantity=10.
|
|
268
|
-
What mutation would test if the boundary is correctly tested?
|
|
269
|
-
What test case would kill that mutant?
|
|
270
|
-
|
|
271
|
-
expected_output:
|
|
272
|
-
must_contain:
|
|
273
|
-
- "10"
|
|
274
|
-
- "boundary"
|
|
275
|
-
- ">="
|
|
276
|
-
- ">"
|
|
277
|
-
must_not_contain:
|
|
278
|
-
- "no boundary"
|
|
279
|
-
|
|
280
|
-
validation:
|
|
281
|
-
schema_check: true
|
|
282
|
-
reasoning_quality_min: 0.8
|
|
283
|
-
|
|
284
|
-
# -------------------------------------------------------------------------
|
|
285
|
-
# Logical Operator Tests (LCR/LOD)
|
|
286
|
-
# -------------------------------------------------------------------------
|
|
287
|
-
|
|
288
|
-
- id: tc006_logical_operator_mutation
|
|
289
|
-
description: "Skill identifies logical operator mutations (&&, ||, !)"
|
|
290
|
-
category: operators
|
|
291
|
-
priority: high
|
|
292
|
-
|
|
293
|
-
input:
|
|
294
|
-
code: |
|
|
295
|
-
function canAccess(user) {
|
|
296
|
-
return user.isActive && (user.role === 'admin' || user.hasPermission);
|
|
297
|
-
}
|
|
298
|
-
context:
|
|
299
|
-
language: javascript
|
|
300
|
-
prompt: |
|
|
301
|
-
Analyze this code for logical connector replacements (LCR).
|
|
302
|
-
What happens if && is changed to || or vice versa?
|
|
303
|
-
|
|
304
|
-
expected_output:
|
|
305
|
-
must_contain:
|
|
306
|
-
- "logical"
|
|
307
|
-
- "&&"
|
|
308
|
-
- "||"
|
|
309
|
-
must_not_contain:
|
|
310
|
-
- "no logical"
|
|
311
|
-
|
|
312
|
-
validation:
|
|
313
|
-
schema_check: true
|
|
314
|
-
keyword_match_threshold: 0.8
|
|
315
|
-
|
|
316
|
-
# -------------------------------------------------------------------------
|
|
317
|
-
# Conditional Operator Tests (COR)
|
|
318
|
-
# -------------------------------------------------------------------------
|
|
319
|
-
|
|
320
|
-
- id: tc007_conditional_mutation
|
|
321
|
-
description: "Skill identifies conditional/decision mutations"
|
|
322
|
-
category: operators
|
|
323
|
-
priority: high
|
|
324
|
-
|
|
325
|
-
input:
|
|
326
|
-
code: |
|
|
327
|
-
function processPayment(payment) {
|
|
328
|
-
if (payment.amount > 0) {
|
|
329
|
-
if (payment.verified) {
|
|
330
|
-
return 'approved';
|
|
331
|
-
}
|
|
332
|
-
return 'pending';
|
|
333
|
-
}
|
|
334
|
-
return 'rejected';
|
|
335
|
-
}
|
|
336
|
-
context:
|
|
337
|
-
language: javascript
|
|
338
|
-
prompt: |
|
|
339
|
-
Analyze conditional mutations for this payment processing code.
|
|
340
|
-
Consider mutations like replacing conditions with true/false.
|
|
341
|
-
|
|
342
|
-
expected_output:
|
|
343
|
-
must_contain:
|
|
344
|
-
- "conditional"
|
|
345
|
-
- "if"
|
|
346
|
-
- "true"
|
|
347
|
-
- "false"
|
|
348
|
-
|
|
349
|
-
validation:
|
|
350
|
-
schema_check: true
|
|
351
|
-
keyword_match_threshold: 0.7
|
|
352
|
-
|
|
353
|
-
# -------------------------------------------------------------------------
|
|
354
|
-
# Return Value Tests (RVR)
|
|
355
|
-
# -------------------------------------------------------------------------
|
|
356
|
-
|
|
357
|
-
- id: tc008_return_value_mutation
|
|
358
|
-
description: "Skill identifies return value mutations"
|
|
359
|
-
category: operators
|
|
360
|
-
priority: medium
|
|
361
|
-
|
|
362
|
-
input:
|
|
363
|
-
code: |
|
|
364
|
-
function getStatus(code) {
|
|
365
|
-
if (code === 200) return 'success';
|
|
366
|
-
if (code === 404) return 'not found';
|
|
367
|
-
if (code >= 500) return 'error';
|
|
368
|
-
return 'unknown';
|
|
369
|
-
}
|
|
370
|
-
context:
|
|
371
|
-
language: javascript
|
|
372
|
-
prompt: |
|
|
373
|
-
Analyze return value mutations for this status code handler.
|
|
374
|
-
What mutations could be applied to the return statements?
|
|
375
|
-
|
|
376
|
-
expected_output:
|
|
377
|
-
must_contain:
|
|
378
|
-
- "return"
|
|
379
|
-
- "mutation"
|
|
380
|
-
- "success"
|
|
381
|
-
- "error"
|
|
382
|
-
|
|
383
|
-
validation:
|
|
384
|
-
schema_check: true
|
|
385
|
-
keyword_match_threshold: 0.7
|
|
386
|
-
|
|
387
|
-
# -------------------------------------------------------------------------
|
|
388
|
-
# Surviving Mutant Analysis Tests
|
|
389
|
-
# -------------------------------------------------------------------------
|
|
390
|
-
|
|
391
|
-
- id: tc009_surviving_mutant_analysis
|
|
392
|
-
description: "Skill analyzes surviving mutants and suggests test improvements"
|
|
393
|
-
category: analysis
|
|
394
|
-
priority: critical
|
|
395
|
-
|
|
396
|
-
input:
|
|
397
|
-
prompt: |
|
|
398
|
-
The following mutant survived:
|
|
399
|
-
- File: src/validator.ts
|
|
400
|
-
- Line: 45
|
|
401
|
-
- Original: if (age >= 18)
|
|
402
|
-
- Mutated: if (age > 18)
|
|
403
|
-
- Tests that cover this line: ['should validate adult', 'should validate minor']
|
|
404
|
-
|
|
405
|
-
Why did this mutant survive and what test would kill it?
|
|
406
|
-
|
|
407
|
-
expected_output:
|
|
408
|
-
must_contain:
|
|
409
|
-
- "boundary"
|
|
410
|
-
- "18"
|
|
411
|
-
- "test"
|
|
412
|
-
- "exactly"
|
|
413
|
-
must_not_contain:
|
|
414
|
-
- "cannot determine"
|
|
415
|
-
|
|
416
|
-
validation:
|
|
417
|
-
schema_check: true
|
|
418
|
-
reasoning_quality_min: 0.8
|
|
419
|
-
|
|
420
|
-
- id: tc010_weak_test_identification
|
|
421
|
-
description: "Skill identifies weak tests based on surviving mutants"
|
|
422
|
-
category: analysis
|
|
423
|
-
priority: high
|
|
424
|
-
|
|
425
|
-
input:
|
|
426
|
-
prompt: |
|
|
427
|
-
Mutation testing results for auth.test.js:
|
|
428
|
-
- 50 mutants generated in auth.js
|
|
429
|
-
- 35 killed by auth.test.js
|
|
430
|
-
- 15 survived
|
|
431
|
-
|
|
432
|
-
Surviving mutant operators:
|
|
433
|
-
- 8 relational (ROR)
|
|
434
|
-
- 5 boundary (BOR)
|
|
435
|
-
- 2 logical (LCR)
|
|
436
|
-
|
|
437
|
-
Analyze what makes auth.test.js weak and how to improve it.
|
|
438
|
-
|
|
439
|
-
expected_output:
|
|
440
|
-
must_contain:
|
|
441
|
-
- "weak"
|
|
442
|
-
- "boundary"
|
|
443
|
-
- "relational"
|
|
444
|
-
- "improve"
|
|
445
|
-
recommendation_count:
|
|
446
|
-
min: 1
|
|
447
|
-
|
|
448
|
-
validation:
|
|
449
|
-
schema_check: true
|
|
450
|
-
reasoning_quality_min: 0.7
|
|
451
|
-
|
|
452
|
-
# -------------------------------------------------------------------------
|
|
453
|
-
# Edge Cases
|
|
454
|
-
# -------------------------------------------------------------------------
|
|
455
|
-
|
|
456
|
-
- id: tc011_empty_code_handling
|
|
457
|
-
description: "Skill handles empty or minimal code gracefully"
|
|
458
|
-
category: edge_cases
|
|
459
|
-
priority: medium
|
|
460
|
-
|
|
461
|
-
input:
|
|
462
|
-
code: |
|
|
463
|
-
// Empty function
|
|
464
|
-
function noop() {}
|
|
465
|
-
context:
|
|
466
|
-
language: javascript
|
|
467
|
-
prompt: Analyze this code for mutation testing.
|
|
468
|
-
|
|
469
|
-
expected_output:
|
|
470
|
-
must_contain:
|
|
471
|
-
- "no mutation"
|
|
472
|
-
must_not_contain:
|
|
473
|
-
- "error"
|
|
474
|
-
- "crash"
|
|
475
|
-
|
|
476
|
-
validation:
|
|
477
|
-
schema_check: true
|
|
478
|
-
allow_partial: true
|
|
479
|
-
|
|
480
|
-
- id: tc012_complex_nested_conditions
|
|
481
|
-
description: "Skill handles complex nested conditions"
|
|
482
|
-
category: edge_cases
|
|
483
|
-
priority: medium
|
|
484
|
-
|
|
485
|
-
input:
|
|
486
|
-
code: |
|
|
487
|
-
function complexValidation(data) {
|
|
488
|
-
if (data && data.type === 'A') {
|
|
489
|
-
if (data.value > 0 && data.value <= 100) {
|
|
490
|
-
if (data.status === 'active' || data.override) {
|
|
491
|
-
return data.priority >= 1 && data.priority <= 5;
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
|
-
return false;
|
|
496
|
-
}
|
|
497
|
-
context:
|
|
498
|
-
language: javascript
|
|
499
|
-
prompt: |
|
|
500
|
-
Analyze all possible mutations in this complex nested validation.
|
|
501
|
-
Prioritize by impact.
|
|
502
|
-
|
|
503
|
-
expected_output:
|
|
504
|
-
must_contain:
|
|
505
|
-
- "nested"
|
|
506
|
-
- "condition"
|
|
507
|
-
- "mutation"
|
|
508
|
-
finding_count:
|
|
509
|
-
min: 5
|
|
510
|
-
|
|
511
|
-
validation:
|
|
512
|
-
schema_check: true
|
|
513
|
-
keyword_match_threshold: 0.7
|
|
514
|
-
|
|
515
|
-
# -------------------------------------------------------------------------
|
|
516
|
-
# Multi-Language Support
|
|
517
|
-
# -------------------------------------------------------------------------
|
|
518
|
-
|
|
519
|
-
- id: tc013_python_mutation_analysis
|
|
520
|
-
description: "Skill correctly analyzes Python code mutations"
|
|
521
|
-
category: language_support
|
|
522
|
-
priority: medium
|
|
523
|
-
|
|
524
|
-
input:
|
|
525
|
-
code: |
|
|
526
|
-
def calculate_price(base_price, discount_percent, tax_rate):
|
|
527
|
-
discount = base_price * (discount_percent / 100)
|
|
528
|
-
subtotal = base_price - discount
|
|
529
|
-
tax = subtotal * tax_rate
|
|
530
|
-
return subtotal + tax
|
|
531
|
-
context:
|
|
532
|
-
language: python
|
|
533
|
-
prompt: Analyze this Python code for mutation testing.
|
|
534
|
-
|
|
535
|
-
expected_output:
|
|
536
|
-
must_contain:
|
|
537
|
-
- "mutation"
|
|
538
|
-
- "arithmetic"
|
|
539
|
-
- "python"
|
|
540
|
-
|
|
541
|
-
validation:
|
|
542
|
-
schema_check: true
|
|
543
|
-
|
|
544
|
-
- id: tc014_typescript_mutation_analysis
|
|
545
|
-
description: "Skill correctly analyzes TypeScript code mutations"
|
|
546
|
-
category: language_support
|
|
547
|
-
priority: medium
|
|
548
|
-
|
|
549
|
-
input:
|
|
550
|
-
code: |
|
|
551
|
-
interface Order {
|
|
552
|
-
quantity: number;
|
|
553
|
-
price: number;
|
|
554
|
-
isPremium: boolean;
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
function calculateDiscount(order: Order): number {
|
|
558
|
-
if (order.quantity >= 10 && order.isPremium) {
|
|
559
|
-
return order.price * 0.15;
|
|
560
|
-
}
|
|
561
|
-
if (order.quantity >= 5) {
|
|
562
|
-
return order.price * 0.05;
|
|
563
|
-
}
|
|
564
|
-
return 0;
|
|
565
|
-
}
|
|
566
|
-
context:
|
|
567
|
-
language: typescript
|
|
568
|
-
prompt: Analyze this TypeScript code for mutation testing.
|
|
569
|
-
|
|
570
|
-
expected_output:
|
|
571
|
-
must_contain:
|
|
572
|
-
- "mutation"
|
|
573
|
-
- "typescript"
|
|
574
|
-
- "boundary"
|
|
575
|
-
|
|
576
|
-
validation:
|
|
577
|
-
schema_check: true
|
|
578
|
-
|
|
579
|
-
# -------------------------------------------------------------------------
|
|
580
|
-
# Integration with Coverage
|
|
581
|
-
# -------------------------------------------------------------------------
|
|
582
|
-
|
|
583
|
-
- id: tc015_coverage_mutation_correlation
|
|
584
|
-
description: "Skill correlates coverage with mutation score"
|
|
585
|
-
category: integration
|
|
586
|
-
priority: high
|
|
587
|
-
|
|
588
|
-
input:
|
|
589
|
-
prompt: |
|
|
590
|
-
Coverage report shows:
|
|
591
|
-
- Line coverage: 95%
|
|
592
|
-
- Branch coverage: 88%
|
|
593
|
-
|
|
594
|
-
Mutation testing shows:
|
|
595
|
-
- Mutation score: 65%
|
|
596
|
-
|
|
597
|
-
Analyze the gap between coverage and mutation score.
|
|
598
|
-
Why can high coverage coexist with low mutation score?
|
|
599
|
-
|
|
600
|
-
expected_output:
|
|
601
|
-
must_contain:
|
|
602
|
-
- "coverage"
|
|
603
|
-
- "mutation"
|
|
604
|
-
- "assertion"
|
|
605
|
-
- "quality"
|
|
606
|
-
must_not_contain:
|
|
607
|
-
- "coverage equals"
|
|
608
|
-
|
|
609
|
-
validation:
|
|
610
|
-
schema_check: true
|
|
611
|
-
reasoning_quality_min: 0.8
|
|
612
|
-
grading_rubric:
|
|
613
|
-
completeness: 0.4
|
|
614
|
-
accuracy: 0.4
|
|
615
|
-
actionability: 0.2
|
|
616
|
-
|
|
617
|
-
# =============================================================================
|
|
618
|
-
# Success Criteria
|
|
619
|
-
# =============================================================================
|
|
620
|
-
|
|
621
|
-
success_criteria:
|
|
622
|
-
# Minimum 90% of tests must pass
|
|
623
|
-
pass_rate: 0.9
|
|
624
|
-
|
|
625
|
-
# All critical tests must pass
|
|
626
|
-
critical_pass_rate: 1.0
|
|
627
|
-
|
|
628
|
-
# Minimum reasoning quality
|
|
629
|
-
avg_reasoning_quality: 0.7
|
|
630
|
-
|
|
631
|
-
# Maximum 5 minutes for full suite
|
|
632
|
-
max_execution_time_ms: 300000
|
|
633
|
-
|
|
634
|
-
# Maximum 15% variance between models
|
|
635
|
-
cross_model_variance: 0.15
|
|
636
|
-
|
|
637
|
-
# =============================================================================
|
|
638
|
-
# Metadata
|
|
639
|
-
# =============================================================================
|
|
640
|
-
|
|
641
|
-
metadata:
|
|
642
|
-
author: "@agentic-qe"
|
|
643
|
-
created: "2026-02-02"
|
|
644
|
-
last_updated: "2026-02-02"
|
|
645
|
-
coverage_target: >
|
|
646
|
-
Comprehensive mutation operator coverage (AOR, ROR, LCR, COR, RVR),
|
|
647
|
-
surviving mutant analysis, weak test identification, multi-language
|
|
648
|
-
support, and coverage correlation analysis.
|
|
649
|
-
related_skills:
|
|
650
|
-
- test-design-techniques
|
|
651
|
-
- coverage-analysis
|
|
652
|
-
- tdd-london-chicago
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# Mutation Testing Skill Evaluation Test Suite v1.0.0
|
|
3
|
+
# Path: .claude/skills/mutation-testing/evals/mutation-testing.yaml
|
|
4
|
+
# =============================================================================
|
|
5
|
+
#
|
|
6
|
+
# This evaluation suite validates mutation testing skill behavior through:
|
|
7
|
+
# 1. Input/expected-output test cases for mutation operators
|
|
8
|
+
# 2. Multi-model consistency testing
|
|
9
|
+
# 3. Semantic validation of mutation analysis quality
|
|
10
|
+
# 4. AQE MCP integration for shared learning
|
|
11
|
+
#
|
|
12
|
+
# Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
|
|
13
|
+
# =============================================================================
|
|
14
|
+
|
|
15
|
+
skill: mutation-testing
|
|
16
|
+
version: 1.0.0
|
|
17
|
+
description: >
|
|
18
|
+
Comprehensive evaluation test suite for the mutation-testing skill.
|
|
19
|
+
Tests mutation score calculation, operator detection, surviving mutant
|
|
20
|
+
analysis, and test improvement recommendations across multiple models
|
|
21
|
+
to ensure consistent, high-quality mutation analysis output.
|
|
22
|
+
|
|
23
|
+
# =============================================================================
|
|
24
|
+
# Multi-Model Configuration
|
|
25
|
+
# =============================================================================
|
|
26
|
+
|
|
27
|
+
models_to_test:
|
|
28
|
+
- claude-3.5-sonnet # Primary model (high accuracy expected)
|
|
29
|
+
- claude-3-haiku # Fast model (ensure minimum quality)
|
|
30
|
+
- gpt-4o # Cross-vendor validation
|
|
31
|
+
|
|
32
|
+
# =============================================================================
|
|
33
|
+
# MCP Integration Configuration
|
|
34
|
+
# =============================================================================
|
|
35
|
+
|
|
36
|
+
mcp_integration:
|
|
37
|
+
enabled: true
|
|
38
|
+
namespace: skill-validation/mutation-testing
|
|
39
|
+
|
|
40
|
+
query_patterns: true
|
|
41
|
+
track_outcomes: true
|
|
42
|
+
store_patterns: true
|
|
43
|
+
share_learning: true
|
|
44
|
+
update_quality_gate: true
|
|
45
|
+
|
|
46
|
+
target_agents:
|
|
47
|
+
- qe-learning-coordinator
|
|
48
|
+
- qe-queen-coordinator
|
|
49
|
+
- qe-mutation-tester
|
|
50
|
+
|
|
51
|
+
# =============================================================================
|
|
52
|
+
# ReasoningBank Learning Configuration
|
|
53
|
+
# =============================================================================
|
|
54
|
+
|
|
55
|
+
learning:
|
|
56
|
+
store_success_patterns: true
|
|
57
|
+
store_failure_patterns: true
|
|
58
|
+
pattern_ttl_days: 90
|
|
59
|
+
min_confidence_to_store: 0.7
|
|
60
|
+
cross_model_comparison: true
|
|
61
|
+
|
|
62
|
+
# =============================================================================
|
|
63
|
+
# Result Format Configuration
|
|
64
|
+
# =============================================================================
|
|
65
|
+
|
|
66
|
+
result_format:
|
|
67
|
+
json_output: true
|
|
68
|
+
markdown_report: true
|
|
69
|
+
include_raw_output: false
|
|
70
|
+
include_timing: true
|
|
71
|
+
include_token_usage: true
|
|
72
|
+
|
|
73
|
+
# =============================================================================
|
|
74
|
+
# Environment Setup
|
|
75
|
+
# =============================================================================
|
|
76
|
+
|
|
77
|
+
setup:
|
|
78
|
+
required_tools:
|
|
79
|
+
- jq
|
|
80
|
+
environment_variables:
|
|
81
|
+
MUTATION_TIMEOUT_MS: "30000"
|
|
82
|
+
fixtures:
|
|
83
|
+
- name: simple_arithmetic_code
|
|
84
|
+
content: |
|
|
85
|
+
function calculateTotal(price, quantity) {
|
|
86
|
+
return price * quantity;
|
|
87
|
+
}
|
|
88
|
+
- name: conditional_code
|
|
89
|
+
content: |
|
|
90
|
+
function isAdult(age) {
|
|
91
|
+
return age >= 18;
|
|
92
|
+
}
|
|
93
|
+
- name: complex_code
|
|
94
|
+
content: |
|
|
95
|
+
function processOrder(order) {
|
|
96
|
+
if (order.quantity <= 0) {
|
|
97
|
+
throw new Error('Invalid quantity');
|
|
98
|
+
}
|
|
99
|
+
let discount = 0;
|
|
100
|
+
if (order.quantity >= 10 && order.isPremium) {
|
|
101
|
+
discount = 0.1;
|
|
102
|
+
}
|
|
103
|
+
return order.price * order.quantity * (1 - discount);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
# =============================================================================
|
|
107
|
+
# Test Cases
|
|
108
|
+
# =============================================================================
|
|
109
|
+
|
|
110
|
+
test_cases:
|
|
111
|
+
# -------------------------------------------------------------------------
|
|
112
|
+
# Basic Functionality Tests
|
|
113
|
+
# -------------------------------------------------------------------------
|
|
114
|
+
|
|
115
|
+
- id: tc001_basic_mutation_analysis
|
|
116
|
+
description: "Skill analyzes simple code and identifies potential mutations"
|
|
117
|
+
category: basic
|
|
118
|
+
priority: critical
|
|
119
|
+
|
|
120
|
+
input:
|
|
121
|
+
code: |
|
|
122
|
+
function add(a, b) {
|
|
123
|
+
return a + b;
|
|
124
|
+
}
|
|
125
|
+
context:
|
|
126
|
+
language: javascript
|
|
127
|
+
prompt: |
|
|
128
|
+
Analyze this code for mutation testing. Identify what mutations
|
|
129
|
+
could be applied and what tests would be needed to kill them.
|
|
130
|
+
|
|
131
|
+
expected_output:
|
|
132
|
+
must_contain:
|
|
133
|
+
- "mutation"
|
|
134
|
+
- "arithmetic"
|
|
135
|
+
- "+"
|
|
136
|
+
must_not_contain:
|
|
137
|
+
- "unable to analyze"
|
|
138
|
+
- "error"
|
|
139
|
+
finding_count:
|
|
140
|
+
min: 1
|
|
141
|
+
max: 10
|
|
142
|
+
|
|
143
|
+
validation:
|
|
144
|
+
schema_check: true
|
|
145
|
+
keyword_match_threshold: 0.8
|
|
146
|
+
reasoning_quality_min: 0.7
|
|
147
|
+
|
|
148
|
+
- id: tc002_mutation_score_calculation
|
|
149
|
+
description: "Skill correctly calculates mutation score from kill/survive data"
|
|
150
|
+
category: basic
|
|
151
|
+
priority: critical
|
|
152
|
+
|
|
153
|
+
input:
|
|
154
|
+
prompt: |
|
|
155
|
+
Given these mutation testing results:
|
|
156
|
+
- Total mutants: 100
|
|
157
|
+
- Killed: 85
|
|
158
|
+
- Survived: 12
|
|
159
|
+
- Timeout: 2
|
|
160
|
+
- No Coverage: 1
|
|
161
|
+
|
|
162
|
+
Calculate the mutation score and assess the test suite quality.
|
|
163
|
+
|
|
164
|
+
expected_output:
|
|
165
|
+
must_contain:
|
|
166
|
+
- "85"
|
|
167
|
+
- "mutation score"
|
|
168
|
+
- "quality"
|
|
169
|
+
must_match_regex:
|
|
170
|
+
- "8[0-9](\\.\\d+)?%" # Score around 85%
|
|
171
|
+
|
|
172
|
+
validation:
|
|
173
|
+
schema_check: true
|
|
174
|
+
keyword_match_threshold: 0.9
|
|
175
|
+
|
|
176
|
+
# -------------------------------------------------------------------------
|
|
177
|
+
# Arithmetic Operator Tests (AOR)
|
|
178
|
+
# -------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
- id: tc003_arithmetic_operator_mutation
|
|
181
|
+
description: "Skill identifies arithmetic operator mutations (+, -, *, /)"
|
|
182
|
+
category: operators
|
|
183
|
+
priority: high
|
|
184
|
+
|
|
185
|
+
input:
|
|
186
|
+
code: |
|
|
187
|
+
function calculateTotal(price, quantity, tax) {
|
|
188
|
+
const subtotal = price * quantity;
|
|
189
|
+
const taxAmount = subtotal * tax;
|
|
190
|
+
return subtotal + taxAmount;
|
|
191
|
+
}
|
|
192
|
+
context:
|
|
193
|
+
language: javascript
|
|
194
|
+
prompt: |
|
|
195
|
+
Perform mutation analysis focusing on arithmetic operators.
|
|
196
|
+
Identify all AOR (Arithmetic Operator Replacement) mutations.
|
|
197
|
+
|
|
198
|
+
expected_output:
|
|
199
|
+
must_contain:
|
|
200
|
+
- "arithmetic"
|
|
201
|
+
- "*"
|
|
202
|
+
- "+"
|
|
203
|
+
- "mutant"
|
|
204
|
+
must_not_contain:
|
|
205
|
+
- "no mutations"
|
|
206
|
+
|
|
207
|
+
validation:
|
|
208
|
+
schema_check: true
|
|
209
|
+
keyword_match_threshold: 0.8
|
|
210
|
+
|
|
211
|
+
# -------------------------------------------------------------------------
|
|
212
|
+
# Relational Operator Tests (ROR)
|
|
213
|
+
# -------------------------------------------------------------------------
|
|
214
|
+
|
|
215
|
+
- id: tc004_relational_operator_mutation
|
|
216
|
+
description: "Skill identifies relational operator mutations (>, <, >=, <=, ==, !=)"
|
|
217
|
+
category: operators
|
|
218
|
+
priority: high
|
|
219
|
+
|
|
220
|
+
input:
|
|
221
|
+
code: |
|
|
222
|
+
function isEligible(age, income) {
|
|
223
|
+
if (age >= 18 && income > 30000) {
|
|
224
|
+
return true;
|
|
225
|
+
}
|
|
226
|
+
if (age < 65 && income <= 100000) {
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
return false;
|
|
230
|
+
}
|
|
231
|
+
context:
|
|
232
|
+
language: javascript
|
|
233
|
+
prompt: |
|
|
234
|
+
Analyze this code for relational operator mutations (ROR).
|
|
235
|
+
Identify boundary conditions that need testing.
|
|
236
|
+
|
|
237
|
+
expected_output:
|
|
238
|
+
must_contain:
|
|
239
|
+
- "relational"
|
|
240
|
+
- ">="
|
|
241
|
+
- "boundary"
|
|
242
|
+
- "18"
|
|
243
|
+
must_not_contain:
|
|
244
|
+
- "unable"
|
|
245
|
+
|
|
246
|
+
validation:
|
|
247
|
+
schema_check: true
|
|
248
|
+
keyword_match_threshold: 0.8
|
|
249
|
+
reasoning_quality_min: 0.7
|
|
250
|
+
|
|
251
|
+
- id: tc005_boundary_mutation_detection
|
|
252
|
+
description: "Skill detects boundary value mutations (>= to >, etc.)"
|
|
253
|
+
category: operators
|
|
254
|
+
priority: critical
|
|
255
|
+
|
|
256
|
+
input:
|
|
257
|
+
code: |
|
|
258
|
+
function getDiscount(quantity) {
|
|
259
|
+
if (quantity >= 10) {
|
|
260
|
+
return 0.1; // 10% discount
|
|
261
|
+
}
|
|
262
|
+
return 0;
|
|
263
|
+
}
|
|
264
|
+
context:
|
|
265
|
+
language: javascript
|
|
266
|
+
prompt: |
|
|
267
|
+
This code has a boundary condition at quantity=10.
|
|
268
|
+
What mutation would test if the boundary is correctly tested?
|
|
269
|
+
What test case would kill that mutant?
|
|
270
|
+
|
|
271
|
+
expected_output:
|
|
272
|
+
must_contain:
|
|
273
|
+
- "10"
|
|
274
|
+
- "boundary"
|
|
275
|
+
- ">="
|
|
276
|
+
- ">"
|
|
277
|
+
must_not_contain:
|
|
278
|
+
- "no boundary"
|
|
279
|
+
|
|
280
|
+
validation:
|
|
281
|
+
schema_check: true
|
|
282
|
+
reasoning_quality_min: 0.8
|
|
283
|
+
|
|
284
|
+
# -------------------------------------------------------------------------
|
|
285
|
+
# Logical Operator Tests (LCR/LOD)
|
|
286
|
+
# -------------------------------------------------------------------------
|
|
287
|
+
|
|
288
|
+
- id: tc006_logical_operator_mutation
|
|
289
|
+
description: "Skill identifies logical operator mutations (&&, ||, !)"
|
|
290
|
+
category: operators
|
|
291
|
+
priority: high
|
|
292
|
+
|
|
293
|
+
input:
|
|
294
|
+
code: |
|
|
295
|
+
function canAccess(user) {
|
|
296
|
+
return user.isActive && (user.role === 'admin' || user.hasPermission);
|
|
297
|
+
}
|
|
298
|
+
context:
|
|
299
|
+
language: javascript
|
|
300
|
+
prompt: |
|
|
301
|
+
Analyze this code for logical connector replacements (LCR).
|
|
302
|
+
What happens if && is changed to || or vice versa?
|
|
303
|
+
|
|
304
|
+
expected_output:
|
|
305
|
+
must_contain:
|
|
306
|
+
- "logical"
|
|
307
|
+
- "&&"
|
|
308
|
+
- "||"
|
|
309
|
+
must_not_contain:
|
|
310
|
+
- "no logical"
|
|
311
|
+
|
|
312
|
+
validation:
|
|
313
|
+
schema_check: true
|
|
314
|
+
keyword_match_threshold: 0.8
|
|
315
|
+
|
|
316
|
+
# -------------------------------------------------------------------------
|
|
317
|
+
# Conditional Operator Tests (COR)
|
|
318
|
+
# -------------------------------------------------------------------------
|
|
319
|
+
|
|
320
|
+
- id: tc007_conditional_mutation
|
|
321
|
+
description: "Skill identifies conditional/decision mutations"
|
|
322
|
+
category: operators
|
|
323
|
+
priority: high
|
|
324
|
+
|
|
325
|
+
input:
|
|
326
|
+
code: |
|
|
327
|
+
function processPayment(payment) {
|
|
328
|
+
if (payment.amount > 0) {
|
|
329
|
+
if (payment.verified) {
|
|
330
|
+
return 'approved';
|
|
331
|
+
}
|
|
332
|
+
return 'pending';
|
|
333
|
+
}
|
|
334
|
+
return 'rejected';
|
|
335
|
+
}
|
|
336
|
+
context:
|
|
337
|
+
language: javascript
|
|
338
|
+
prompt: |
|
|
339
|
+
Analyze conditional mutations for this payment processing code.
|
|
340
|
+
Consider mutations like replacing conditions with true/false.
|
|
341
|
+
|
|
342
|
+
expected_output:
|
|
343
|
+
must_contain:
|
|
344
|
+
- "conditional"
|
|
345
|
+
- "if"
|
|
346
|
+
- "true"
|
|
347
|
+
- "false"
|
|
348
|
+
|
|
349
|
+
validation:
|
|
350
|
+
schema_check: true
|
|
351
|
+
keyword_match_threshold: 0.7
|
|
352
|
+
|
|
353
|
+
# -------------------------------------------------------------------------
|
|
354
|
+
# Return Value Tests (RVR)
|
|
355
|
+
# -------------------------------------------------------------------------
|
|
356
|
+
|
|
357
|
+
- id: tc008_return_value_mutation
|
|
358
|
+
description: "Skill identifies return value mutations"
|
|
359
|
+
category: operators
|
|
360
|
+
priority: medium
|
|
361
|
+
|
|
362
|
+
input:
|
|
363
|
+
code: |
|
|
364
|
+
function getStatus(code) {
|
|
365
|
+
if (code === 200) return 'success';
|
|
366
|
+
if (code === 404) return 'not found';
|
|
367
|
+
if (code >= 500) return 'error';
|
|
368
|
+
return 'unknown';
|
|
369
|
+
}
|
|
370
|
+
context:
|
|
371
|
+
language: javascript
|
|
372
|
+
prompt: |
|
|
373
|
+
Analyze return value mutations for this status code handler.
|
|
374
|
+
What mutations could be applied to the return statements?
|
|
375
|
+
|
|
376
|
+
expected_output:
|
|
377
|
+
must_contain:
|
|
378
|
+
- "return"
|
|
379
|
+
- "mutation"
|
|
380
|
+
- "success"
|
|
381
|
+
- "error"
|
|
382
|
+
|
|
383
|
+
validation:
|
|
384
|
+
schema_check: true
|
|
385
|
+
keyword_match_threshold: 0.7
|
|
386
|
+
|
|
387
|
+
# -------------------------------------------------------------------------
|
|
388
|
+
# Surviving Mutant Analysis Tests
|
|
389
|
+
# -------------------------------------------------------------------------
|
|
390
|
+
|
|
391
|
+
- id: tc009_surviving_mutant_analysis
|
|
392
|
+
description: "Skill analyzes surviving mutants and suggests test improvements"
|
|
393
|
+
category: analysis
|
|
394
|
+
priority: critical
|
|
395
|
+
|
|
396
|
+
input:
|
|
397
|
+
prompt: |
|
|
398
|
+
The following mutant survived:
|
|
399
|
+
- File: src/validator.ts
|
|
400
|
+
- Line: 45
|
|
401
|
+
- Original: if (age >= 18)
|
|
402
|
+
- Mutated: if (age > 18)
|
|
403
|
+
- Tests that cover this line: ['should validate adult', 'should validate minor']
|
|
404
|
+
|
|
405
|
+
Why did this mutant survive and what test would kill it?
|
|
406
|
+
|
|
407
|
+
expected_output:
|
|
408
|
+
must_contain:
|
|
409
|
+
- "boundary"
|
|
410
|
+
- "18"
|
|
411
|
+
- "test"
|
|
412
|
+
- "exactly"
|
|
413
|
+
must_not_contain:
|
|
414
|
+
- "cannot determine"
|
|
415
|
+
|
|
416
|
+
validation:
|
|
417
|
+
schema_check: true
|
|
418
|
+
reasoning_quality_min: 0.8
|
|
419
|
+
|
|
420
|
+
- id: tc010_weak_test_identification
|
|
421
|
+
description: "Skill identifies weak tests based on surviving mutants"
|
|
422
|
+
category: analysis
|
|
423
|
+
priority: high
|
|
424
|
+
|
|
425
|
+
input:
|
|
426
|
+
prompt: |
|
|
427
|
+
Mutation testing results for auth.test.js:
|
|
428
|
+
- 50 mutants generated in auth.js
|
|
429
|
+
- 35 killed by auth.test.js
|
|
430
|
+
- 15 survived
|
|
431
|
+
|
|
432
|
+
Surviving mutant operators:
|
|
433
|
+
- 8 relational (ROR)
|
|
434
|
+
- 5 boundary (BOR)
|
|
435
|
+
- 2 logical (LCR)
|
|
436
|
+
|
|
437
|
+
Analyze what makes auth.test.js weak and how to improve it.
|
|
438
|
+
|
|
439
|
+
expected_output:
|
|
440
|
+
must_contain:
|
|
441
|
+
- "weak"
|
|
442
|
+
- "boundary"
|
|
443
|
+
- "relational"
|
|
444
|
+
- "improve"
|
|
445
|
+
recommendation_count:
|
|
446
|
+
min: 1
|
|
447
|
+
|
|
448
|
+
validation:
|
|
449
|
+
schema_check: true
|
|
450
|
+
reasoning_quality_min: 0.7
|
|
451
|
+
|
|
452
|
+
# -------------------------------------------------------------------------
|
|
453
|
+
# Edge Cases
|
|
454
|
+
# -------------------------------------------------------------------------
|
|
455
|
+
|
|
456
|
+
- id: tc011_empty_code_handling
|
|
457
|
+
description: "Skill handles empty or minimal code gracefully"
|
|
458
|
+
category: edge_cases
|
|
459
|
+
priority: medium
|
|
460
|
+
|
|
461
|
+
input:
|
|
462
|
+
code: |
|
|
463
|
+
// Empty function
|
|
464
|
+
function noop() {}
|
|
465
|
+
context:
|
|
466
|
+
language: javascript
|
|
467
|
+
prompt: Analyze this code for mutation testing.
|
|
468
|
+
|
|
469
|
+
expected_output:
|
|
470
|
+
must_contain:
|
|
471
|
+
- "no mutation"
|
|
472
|
+
must_not_contain:
|
|
473
|
+
- "error"
|
|
474
|
+
- "crash"
|
|
475
|
+
|
|
476
|
+
validation:
|
|
477
|
+
schema_check: true
|
|
478
|
+
allow_partial: true
|
|
479
|
+
|
|
480
|
+
- id: tc012_complex_nested_conditions
|
|
481
|
+
description: "Skill handles complex nested conditions"
|
|
482
|
+
category: edge_cases
|
|
483
|
+
priority: medium
|
|
484
|
+
|
|
485
|
+
input:
|
|
486
|
+
code: |
|
|
487
|
+
function complexValidation(data) {
|
|
488
|
+
if (data && data.type === 'A') {
|
|
489
|
+
if (data.value > 0 && data.value <= 100) {
|
|
490
|
+
if (data.status === 'active' || data.override) {
|
|
491
|
+
return data.priority >= 1 && data.priority <= 5;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
return false;
|
|
496
|
+
}
|
|
497
|
+
context:
|
|
498
|
+
language: javascript
|
|
499
|
+
prompt: |
|
|
500
|
+
Analyze all possible mutations in this complex nested validation.
|
|
501
|
+
Prioritize by impact.
|
|
502
|
+
|
|
503
|
+
expected_output:
|
|
504
|
+
must_contain:
|
|
505
|
+
- "nested"
|
|
506
|
+
- "condition"
|
|
507
|
+
- "mutation"
|
|
508
|
+
finding_count:
|
|
509
|
+
min: 5
|
|
510
|
+
|
|
511
|
+
validation:
|
|
512
|
+
schema_check: true
|
|
513
|
+
keyword_match_threshold: 0.7
|
|
514
|
+
|
|
515
|
+
# -------------------------------------------------------------------------
|
|
516
|
+
# Multi-Language Support
|
|
517
|
+
# -------------------------------------------------------------------------
|
|
518
|
+
|
|
519
|
+
- id: tc013_python_mutation_analysis
|
|
520
|
+
description: "Skill correctly analyzes Python code mutations"
|
|
521
|
+
category: language_support
|
|
522
|
+
priority: medium
|
|
523
|
+
|
|
524
|
+
input:
|
|
525
|
+
code: |
|
|
526
|
+
def calculate_price(base_price, discount_percent, tax_rate):
|
|
527
|
+
discount = base_price * (discount_percent / 100)
|
|
528
|
+
subtotal = base_price - discount
|
|
529
|
+
tax = subtotal * tax_rate
|
|
530
|
+
return subtotal + tax
|
|
531
|
+
context:
|
|
532
|
+
language: python
|
|
533
|
+
prompt: Analyze this Python code for mutation testing.
|
|
534
|
+
|
|
535
|
+
expected_output:
|
|
536
|
+
must_contain:
|
|
537
|
+
- "mutation"
|
|
538
|
+
- "arithmetic"
|
|
539
|
+
- "python"
|
|
540
|
+
|
|
541
|
+
validation:
|
|
542
|
+
schema_check: true
|
|
543
|
+
|
|
544
|
+
- id: tc014_typescript_mutation_analysis
|
|
545
|
+
description: "Skill correctly analyzes TypeScript code mutations"
|
|
546
|
+
category: language_support
|
|
547
|
+
priority: medium
|
|
548
|
+
|
|
549
|
+
input:
|
|
550
|
+
code: |
|
|
551
|
+
interface Order {
|
|
552
|
+
quantity: number;
|
|
553
|
+
price: number;
|
|
554
|
+
isPremium: boolean;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function calculateDiscount(order: Order): number {
|
|
558
|
+
if (order.quantity >= 10 && order.isPremium) {
|
|
559
|
+
return order.price * 0.15;
|
|
560
|
+
}
|
|
561
|
+
if (order.quantity >= 5) {
|
|
562
|
+
return order.price * 0.05;
|
|
563
|
+
}
|
|
564
|
+
return 0;
|
|
565
|
+
}
|
|
566
|
+
context:
|
|
567
|
+
language: typescript
|
|
568
|
+
prompt: Analyze this TypeScript code for mutation testing.
|
|
569
|
+
|
|
570
|
+
expected_output:
|
|
571
|
+
must_contain:
|
|
572
|
+
- "mutation"
|
|
573
|
+
- "typescript"
|
|
574
|
+
- "boundary"
|
|
575
|
+
|
|
576
|
+
validation:
|
|
577
|
+
schema_check: true
|
|
578
|
+
|
|
579
|
+
# -------------------------------------------------------------------------
|
|
580
|
+
# Integration with Coverage
|
|
581
|
+
# -------------------------------------------------------------------------
|
|
582
|
+
|
|
583
|
+
- id: tc015_coverage_mutation_correlation
|
|
584
|
+
description: "Skill correlates coverage with mutation score"
|
|
585
|
+
category: integration
|
|
586
|
+
priority: high
|
|
587
|
+
|
|
588
|
+
input:
|
|
589
|
+
prompt: |
|
|
590
|
+
Coverage report shows:
|
|
591
|
+
- Line coverage: 95%
|
|
592
|
+
- Branch coverage: 88%
|
|
593
|
+
|
|
594
|
+
Mutation testing shows:
|
|
595
|
+
- Mutation score: 65%
|
|
596
|
+
|
|
597
|
+
Analyze the gap between coverage and mutation score.
|
|
598
|
+
Why can high coverage coexist with low mutation score?
|
|
599
|
+
|
|
600
|
+
expected_output:
|
|
601
|
+
must_contain:
|
|
602
|
+
- "coverage"
|
|
603
|
+
- "mutation"
|
|
604
|
+
- "assertion"
|
|
605
|
+
- "quality"
|
|
606
|
+
must_not_contain:
|
|
607
|
+
- "coverage equals"
|
|
608
|
+
|
|
609
|
+
validation:
|
|
610
|
+
schema_check: true
|
|
611
|
+
reasoning_quality_min: 0.8
|
|
612
|
+
grading_rubric:
|
|
613
|
+
completeness: 0.4
|
|
614
|
+
accuracy: 0.4
|
|
615
|
+
actionability: 0.2
|
|
616
|
+
|
|
617
|
+
# =============================================================================
|
|
618
|
+
# Success Criteria
|
|
619
|
+
# =============================================================================
|
|
620
|
+
|
|
621
|
+
success_criteria:
|
|
622
|
+
# Minimum 90% of tests must pass
|
|
623
|
+
pass_rate: 0.9
|
|
624
|
+
|
|
625
|
+
# All critical tests must pass
|
|
626
|
+
critical_pass_rate: 1.0
|
|
627
|
+
|
|
628
|
+
# Minimum reasoning quality
|
|
629
|
+
avg_reasoning_quality: 0.7
|
|
630
|
+
|
|
631
|
+
# Maximum 5 minutes for full suite
|
|
632
|
+
max_execution_time_ms: 300000
|
|
633
|
+
|
|
634
|
+
# Maximum 15% variance between models
|
|
635
|
+
cross_model_variance: 0.15
|
|
636
|
+
|
|
637
|
+
# =============================================================================
|
|
638
|
+
# Metadata
|
|
639
|
+
# =============================================================================
|
|
640
|
+
|
|
641
|
+
metadata:
|
|
642
|
+
author: "@agentic-qe"
|
|
643
|
+
created: "2026-02-02"
|
|
644
|
+
last_updated: "2026-02-02"
|
|
645
|
+
coverage_target: >
|
|
646
|
+
Comprehensive mutation operator coverage (AOR, ROR, LCR, COR, RVR),
|
|
647
|
+
surviving mutant analysis, weak test identification, multi-language
|
|
648
|
+
support, and coverage correlation analysis.
|
|
649
|
+
related_skills:
|
|
650
|
+
- test-design-techniques
|
|
651
|
+
- coverage-analysis
|
|
652
|
+
- tdd-london-chicago
|