agentic-qe 3.7.8 → 3.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (569) hide show
  1. package/.claude/skills/.validation/README.md +111 -111
  2. package/.claude/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
  3. package/.claude/skills/.validation/examples/performance-testing-output.example.json +252 -252
  4. package/.claude/skills/.validation/examples/security-testing-output.example.json +413 -413
  5. package/.claude/skills/.validation/examples/testability-scoring-output.example.json +350 -350
  6. package/.claude/skills/.validation/schemas/skill-eval.schema.json +462 -462
  7. package/.claude/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
  8. package/.claude/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
  9. package/.claude/skills/.validation/schemas/skill-output.template.json +610 -610
  10. package/.claude/skills/.validation/skill-validation-mcp-integration.md +250 -250
  11. package/.claude/skills/.validation/templates/eval.template.yaml +366 -366
  12. package/.claude/skills/.validation/templates/schemas/output.json +145 -145
  13. package/.claude/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
  14. package/.claude/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
  15. package/.claude/skills/.validation/test-data/invalid-output.json +5 -5
  16. package/.claude/skills/.validation/test-data/minimal-output.json +9 -9
  17. package/.claude/skills/.validation/test-data/sample-output.json +73 -73
  18. package/.claude/skills/a11y-ally/evals/a11y-ally.yaml +376 -376
  19. package/.claude/skills/a11y-ally/schemas/output.json +549 -549
  20. package/.claude/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
  21. package/.claude/skills/accessibility-testing/schemas/output.json +776 -776
  22. package/.claude/skills/accessibility-testing/test-data/sample-output.json +191 -191
  23. package/.claude/skills/agentic-quality-engineering/schemas/output.json +577 -577
  24. package/.claude/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
  25. package/.claude/skills/api-testing-patterns/schemas/output.json +845 -845
  26. package/.claude/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
  27. package/.claude/skills/brutal-honesty-review/schemas/output.json +291 -291
  28. package/.claude/skills/bug-reporting-excellence/schemas/output.json +288 -288
  29. package/.claude/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
  30. package/.claude/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
  31. package/.claude/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -157
  32. package/.claude/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -542
  33. package/.claude/skills/code-review-quality/schemas/output.json +264 -264
  34. package/.claude/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
  35. package/.claude/skills/compatibility-testing/schemas/output.json +551 -551
  36. package/.claude/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
  37. package/.claude/skills/compliance-testing/schemas/output.json +845 -845
  38. package/.claude/skills/consultancy-practices/schemas/output.json +282 -282
  39. package/.claude/skills/contract-testing/evals/contract-testing.yaml +748 -748
  40. package/.claude/skills/contract-testing/schemas/output.json +638 -638
  41. package/.claude/skills/database-testing/evals/database-testing.yaml +968 -968
  42. package/.claude/skills/database-testing/schemas/output.json +1446 -1446
  43. package/.claude/skills/debug-loop/SKILL.md +61 -61
  44. package/.claude/skills/enterprise-integration-testing/SKILL.md +735 -735
  45. package/.claude/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -0
  46. package/.claude/skills/enterprise-integration-testing/schemas/output.json +74 -0
  47. package/.claude/skills/enterprise-integration-testing/scripts/validate-config.json +25 -0
  48. package/.claude/skills/iterative-loop/SKILL.md +371 -371
  49. package/.claude/skills/localization-testing/evals/localization-testing.yaml +544 -544
  50. package/.claude/skills/localization-testing/schemas/output.json +325 -325
  51. package/.claude/skills/middleware-testing-patterns/SKILL.md +798 -798
  52. package/.claude/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -0
  53. package/.claude/skills/middleware-testing-patterns/schemas/output.json +58 -0
  54. package/.claude/skills/middleware-testing-patterns/scripts/validate-config.json +25 -0
  55. package/.claude/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
  56. package/.claude/skills/mobile-testing/schemas/output.json +318 -318
  57. package/.claude/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
  58. package/.claude/skills/mutation-testing/schemas/output.json +707 -707
  59. package/.claude/skills/mutation-testing/test-data/sample-output.json +295 -295
  60. package/.claude/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -450
  61. package/.claude/skills/n8n-expression-testing/schemas/output.json +369 -369
  62. package/.claude/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -522
  63. package/.claude/skills/n8n-integration-testing-patterns/schemas/output.json +291 -291
  64. package/.claude/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -493
  65. package/.claude/skills/n8n-security-testing/schemas/output.json +293 -293
  66. package/.claude/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -500
  67. package/.claude/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -295
  68. package/.claude/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -497
  69. package/.claude/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -254
  70. package/.claude/skills/observability-testing-patterns/SKILL.md +930 -930
  71. package/.claude/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -0
  72. package/.claude/skills/observability-testing-patterns/schemas/output.json +58 -0
  73. package/.claude/skills/observability-testing-patterns/scripts/validate-config.json +25 -0
  74. package/.claude/skills/pentest-validation/SKILL.md +268 -268
  75. package/.claude/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
  76. package/.claude/skills/pentest-validation/schemas/output.json +281 -281
  77. package/.claude/skills/performance-analysis/evals/performance-analysis.yaml +144 -144
  78. package/.claude/skills/performance-analysis/schemas/output.json +588 -588
  79. package/.claude/skills/performance-testing/evals/performance-testing.yaml +772 -772
  80. package/.claude/skills/performance-testing/schemas/output.json +1184 -1184
  81. package/.claude/skills/pr-review/SKILL.md +61 -61
  82. package/.claude/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
  83. package/.claude/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -0
  84. package/.claude/skills/qcsd-cicd-swarm/schemas/output.json +86 -0
  85. package/.claude/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -0
  86. package/.claude/skills/qcsd-development-swarm/SKILL.md +2154 -2154
  87. package/.claude/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -0
  88. package/.claude/skills/qcsd-development-swarm/schemas/output.json +72 -0
  89. package/.claude/skills/qcsd-development-swarm/scripts/validate-config.json +25 -0
  90. package/.claude/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -138
  91. package/.claude/skills/qcsd-ideation-swarm/schemas/output.json +568 -568
  92. package/.claude/skills/qcsd-production-swarm/SKILL.md +2781 -2781
  93. package/.claude/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -246
  94. package/.claude/skills/qcsd-production-swarm/schemas/output.json +505 -505
  95. package/.claude/skills/qcsd-production-swarm/scripts/validate-config.json +25 -25
  96. package/.claude/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
  97. package/.claude/skills/qe-chaos-resilience/schemas/output.json +314 -314
  98. package/.claude/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
  99. package/.claude/skills/qe-code-intelligence/schemas/output.json +315 -315
  100. package/.claude/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
  101. package/.claude/skills/qe-contract-testing/schemas/output.json +295 -295
  102. package/.claude/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
  103. package/.claude/skills/qe-coverage-analysis/schemas/output.json +286 -286
  104. package/.claude/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
  105. package/.claude/skills/qe-defect-intelligence/schemas/output.json +283 -283
  106. package/.claude/skills/qe-iterative-loop/schemas/output.json +264 -264
  107. package/.claude/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
  108. package/.claude/skills/qe-learning-optimization/schemas/output.json +288 -288
  109. package/.claude/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
  110. package/.claude/skills/qe-quality-assessment/schemas/output.json +550 -550
  111. package/.claude/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
  112. package/.claude/skills/qe-requirements-validation/schemas/output.json +587 -587
  113. package/.claude/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
  114. package/.claude/skills/qe-security-compliance/schemas/output.json +498 -498
  115. package/.claude/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
  116. package/.claude/skills/qe-test-execution/schemas/output.json +529 -529
  117. package/.claude/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
  118. package/.claude/skills/qe-test-generation/schemas/output.json +439 -439
  119. package/.claude/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
  120. package/.claude/skills/qe-visual-accessibility/schemas/output.json +491 -491
  121. package/.claude/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
  122. package/.claude/skills/quality-metrics/schemas/output.json +403 -403
  123. package/.claude/skills/refactoring-patterns/schemas/output.json +475 -475
  124. package/.claude/skills/regression-testing/evals/regression-testing.yaml +504 -504
  125. package/.claude/skills/regression-testing/schemas/output.json +311 -311
  126. package/.claude/skills/release/SKILL.md +347 -347
  127. package/.claude/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
  128. package/.claude/skills/risk-based-testing/schemas/output.json +480 -480
  129. package/.claude/skills/security-testing/evals/security-testing.yaml +789 -789
  130. package/.claude/skills/security-testing/schemas/output.json +879 -879
  131. package/.claude/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
  132. package/.claude/skills/security-visual-testing/schemas/output.json +486 -486
  133. package/.claude/skills/sfdipot-product-factors/SKILL.md +239 -239
  134. package/.claude/skills/sherlock-review/schemas/output.json +297 -297
  135. package/.claude/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
  136. package/.claude/skills/shift-left-testing/schemas/output.json +459 -459
  137. package/.claude/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
  138. package/.claude/skills/shift-right-testing/schemas/output.json +418 -418
  139. package/.claude/skills/skills-manifest.json +1 -1
  140. package/.claude/skills/tdd-london-chicago/schemas/output.json +444 -444
  141. package/.claude/skills/technical-writing/schemas/output.json +268 -268
  142. package/.claude/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
  143. package/.claude/skills/test-automation-strategy/schemas/output.json +444 -444
  144. package/.claude/skills/test-data-management/evals/test-data-management.yaml +504 -504
  145. package/.claude/skills/test-data-management/schemas/output.json +284 -284
  146. package/.claude/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
  147. package/.claude/skills/test-design-techniques/schemas/output.json +295 -295
  148. package/.claude/skills/test-environment-management/schemas/output.json +310 -310
  149. package/.claude/skills/test-idea-rewriting/SKILL.md +229 -229
  150. package/.claude/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
  151. package/.claude/skills/test-reporting-analytics/schemas/output.json +329 -329
  152. package/.claude/skills/testability-scoring/evals/testability-scoring.yaml +814 -814
  153. package/.claude/skills/testability-scoring/resources/templates/config.template.js +84 -84
  154. package/.claude/skills/testability-scoring/schemas/output.json +606 -606
  155. package/.claude/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
  156. package/.claude/skills/trust-tier-manifest.json +78 -7
  157. package/.claude/skills/verification-quality/evals/verification-quality.yaml +150 -150
  158. package/.claude/skills/verification-quality/schemas/output.json +432 -432
  159. package/.claude/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
  160. package/.claude/skills/visual-testing-advanced/schemas/output.json +294 -294
  161. package/.claude/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -165
  162. package/.claude/skills/wms-testing-patterns/schemas/output.json +150 -150
  163. package/.claude/skills/wms-testing-patterns/scripts/validate-config.json +51 -51
  164. package/CHANGELOG.md +34 -0
  165. package/README.md +169 -900
  166. package/assets/agents/v3/helpers/quality-criteria/evidence-classification.md +116 -116
  167. package/assets/agents/v3/helpers/quality-criteria/htsm-categories.md +139 -139
  168. package/assets/governance/constitution.md +202 -202
  169. package/assets/governance/shards/chaos-resilience.shard.md +221 -221
  170. package/assets/governance/shards/code-intelligence.shard.md +178 -178
  171. package/assets/governance/shards/contract-testing.shard.md +206 -206
  172. package/assets/governance/shards/coverage-analysis.shard.md +146 -146
  173. package/assets/governance/shards/defect-intelligence.shard.md +182 -182
  174. package/assets/governance/shards/learning-optimization.shard.md +248 -248
  175. package/assets/governance/shards/quality-assessment.shard.md +165 -165
  176. package/assets/governance/shards/requirements-validation.shard.md +177 -177
  177. package/assets/governance/shards/security-compliance.shard.md +196 -196
  178. package/assets/governance/shards/test-execution.shard.md +156 -156
  179. package/assets/governance/shards/test-generation.shard.md +128 -128
  180. package/assets/governance/shards/visual-accessibility.shard.md +209 -209
  181. package/assets/hooks/cross-phase-memory.yaml +253 -253
  182. package/assets/patterns/adr-051-booster-patterns.json +78 -78
  183. package/assets/patterns/adr-051-embedding-patterns.json +147 -147
  184. package/assets/patterns/adr-051-integration-summary.json +62 -62
  185. package/assets/patterns/adr-051-reasoning-patterns.json +166 -166
  186. package/assets/patterns/adr-051-router-patterns.json +113 -113
  187. package/assets/patterns/index.json +136 -136
  188. package/assets/skills/.validation/README.md +111 -111
  189. package/assets/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
  190. package/assets/skills/.validation/examples/performance-testing-output.example.json +252 -252
  191. package/assets/skills/.validation/examples/security-testing-output.example.json +413 -413
  192. package/assets/skills/.validation/examples/testability-scoring-output.example.json +350 -350
  193. package/assets/skills/.validation/schemas/skill-eval.schema.json +462 -462
  194. package/assets/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
  195. package/assets/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
  196. package/assets/skills/.validation/schemas/skill-output.template.json +610 -610
  197. package/assets/skills/.validation/skill-validation-mcp-integration.md +250 -250
  198. package/assets/skills/.validation/templates/eval.template.yaml +366 -366
  199. package/assets/skills/.validation/templates/schemas/output.json +145 -145
  200. package/assets/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
  201. package/assets/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
  202. package/assets/skills/.validation/test-data/invalid-output.json +5 -5
  203. package/assets/skills/.validation/test-data/minimal-output.json +9 -9
  204. package/assets/skills/.validation/test-data/sample-output.json +73 -73
  205. package/assets/skills/a11y-ally/SKILL.md +1664 -1658
  206. package/assets/skills/a11y-ally/evals/a11y-ally.yaml +376 -0
  207. package/assets/skills/a11y-ally/schemas/output.json +549 -0
  208. package/assets/skills/a11y-ally/scripts/validate-config.json +42 -0
  209. package/assets/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
  210. package/assets/skills/accessibility-testing/schemas/output.json +776 -776
  211. package/assets/skills/accessibility-testing/test-data/sample-output.json +191 -191
  212. package/assets/skills/agentic-quality-engineering/schemas/output.json +577 -577
  213. package/assets/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
  214. package/assets/skills/api-testing-patterns/schemas/output.json +845 -845
  215. package/assets/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
  216. package/assets/skills/brutal-honesty-review/SKILL.md +5 -0
  217. package/assets/skills/brutal-honesty-review/schemas/output.json +291 -0
  218. package/assets/skills/brutal-honesty-review/scripts/validate-config.json +34 -0
  219. package/assets/skills/bug-reporting-excellence/schemas/output.json +288 -288
  220. package/assets/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
  221. package/assets/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
  222. package/assets/skills/cicd-pipeline-qe-orchestrator/README.md +1 -1
  223. package/assets/skills/cicd-pipeline-qe-orchestrator/SKILL.md +6 -0
  224. package/assets/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -0
  225. package/assets/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -0
  226. package/assets/skills/cicd-pipeline-qe-orchestrator/scripts/validate-config.json +42 -0
  227. package/assets/skills/code-review-quality/schemas/output.json +264 -264
  228. package/assets/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
  229. package/assets/skills/compatibility-testing/schemas/output.json +551 -551
  230. package/assets/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
  231. package/assets/skills/compliance-testing/schemas/output.json +845 -845
  232. package/assets/skills/consultancy-practices/schemas/output.json +282 -282
  233. package/assets/skills/contract-testing/evals/contract-testing.yaml +748 -748
  234. package/assets/skills/contract-testing/schemas/output.json +638 -638
  235. package/assets/skills/database-testing/evals/database-testing.yaml +968 -968
  236. package/assets/skills/database-testing/schemas/output.json +1446 -1446
  237. package/assets/skills/debug-loop/SKILL.md +61 -61
  238. package/assets/skills/enterprise-integration-testing/SKILL.md +735 -735
  239. package/assets/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -0
  240. package/assets/skills/enterprise-integration-testing/schemas/output.json +74 -0
  241. package/assets/skills/enterprise-integration-testing/scripts/validate-config.json +25 -0
  242. package/assets/skills/localization-testing/evals/localization-testing.yaml +544 -544
  243. package/assets/skills/localization-testing/schemas/output.json +325 -325
  244. package/assets/skills/middleware-testing-patterns/SKILL.md +798 -798
  245. package/assets/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -0
  246. package/assets/skills/middleware-testing-patterns/schemas/output.json +58 -0
  247. package/assets/skills/middleware-testing-patterns/scripts/validate-config.json +25 -0
  248. package/assets/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
  249. package/assets/skills/mobile-testing/schemas/output.json +318 -318
  250. package/assets/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
  251. package/assets/skills/mutation-testing/schemas/output.json +707 -707
  252. package/assets/skills/mutation-testing/test-data/sample-output.json +295 -295
  253. package/assets/skills/n8n-expression-testing/SKILL.md +6 -0
  254. package/assets/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -0
  255. package/assets/skills/n8n-expression-testing/schemas/output.json +369 -0
  256. package/assets/skills/n8n-expression-testing/scripts/validate-config.json +39 -0
  257. package/assets/skills/n8n-integration-testing-patterns/SKILL.md +6 -0
  258. package/assets/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -0
  259. package/assets/skills/n8n-integration-testing-patterns/schemas/output.json +291 -0
  260. package/assets/skills/n8n-integration-testing-patterns/scripts/validate-config.json +34 -0
  261. package/assets/skills/n8n-security-testing/SKILL.md +6 -0
  262. package/assets/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -0
  263. package/assets/skills/n8n-security-testing/schemas/output.json +293 -0
  264. package/assets/skills/n8n-security-testing/scripts/validate-config.json +34 -0
  265. package/assets/skills/n8n-trigger-testing-strategies/SKILL.md +6 -0
  266. package/assets/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -0
  267. package/assets/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -0
  268. package/assets/skills/n8n-trigger-testing-strategies/scripts/validate-config.json +34 -0
  269. package/assets/skills/n8n-workflow-testing-fundamentals/SKILL.md +6 -0
  270. package/assets/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -0
  271. package/assets/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -0
  272. package/assets/skills/n8n-workflow-testing-fundamentals/scripts/validate-config.json +35 -0
  273. package/assets/skills/observability-testing-patterns/SKILL.md +930 -930
  274. package/assets/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -0
  275. package/assets/skills/observability-testing-patterns/schemas/output.json +58 -0
  276. package/assets/skills/observability-testing-patterns/scripts/validate-config.json +25 -0
  277. package/assets/skills/pentest-validation/SKILL.md +268 -268
  278. package/assets/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
  279. package/assets/skills/pentest-validation/schemas/output.json +281 -281
  280. package/assets/skills/pentest-validation/scripts/validate-config.json +12 -0
  281. package/assets/skills/performance-testing/evals/performance-testing.yaml +772 -772
  282. package/assets/skills/performance-testing/schemas/output.json +1184 -1184
  283. package/assets/skills/pr-review/SKILL.md +61 -61
  284. package/assets/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
  285. package/assets/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -0
  286. package/assets/skills/qcsd-cicd-swarm/schemas/output.json +86 -0
  287. package/assets/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -0
  288. package/assets/skills/qcsd-development-swarm/SKILL.md +2154 -2154
  289. package/assets/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -0
  290. package/assets/skills/qcsd-development-swarm/schemas/output.json +72 -0
  291. package/assets/skills/qcsd-development-swarm/scripts/validate-config.json +25 -0
  292. package/assets/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -0
  293. package/assets/skills/qcsd-ideation-swarm/schemas/output.json +568 -0
  294. package/assets/skills/qcsd-ideation-swarm/scripts/validate-config.json +25 -0
  295. package/assets/skills/qcsd-production-swarm/SKILL.md +2781 -0
  296. package/assets/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -0
  297. package/assets/skills/qcsd-production-swarm/schemas/output.json +505 -0
  298. package/assets/skills/qcsd-production-swarm/scripts/validate-config.json +25 -0
  299. package/assets/skills/qcsd-refinement-swarm/evals/qcsd-refinement-swarm.yaml +139 -0
  300. package/assets/skills/qcsd-refinement-swarm/schemas/output.json +811 -0
  301. package/assets/skills/qcsd-refinement-swarm/scripts/validate-config.json +25 -0
  302. package/assets/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
  303. package/assets/skills/qe-chaos-resilience/schemas/output.json +314 -314
  304. package/assets/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
  305. package/assets/skills/qe-code-intelligence/schemas/output.json +315 -315
  306. package/assets/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
  307. package/assets/skills/qe-contract-testing/schemas/output.json +295 -295
  308. package/assets/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
  309. package/assets/skills/qe-coverage-analysis/schemas/output.json +286 -286
  310. package/assets/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
  311. package/assets/skills/qe-defect-intelligence/schemas/output.json +283 -283
  312. package/assets/skills/qe-iterative-loop/schemas/output.json +264 -264
  313. package/assets/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
  314. package/assets/skills/qe-learning-optimization/schemas/output.json +288 -288
  315. package/assets/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
  316. package/assets/skills/qe-quality-assessment/schemas/output.json +550 -550
  317. package/assets/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
  318. package/assets/skills/qe-requirements-validation/schemas/output.json +587 -587
  319. package/assets/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
  320. package/assets/skills/qe-security-compliance/schemas/output.json +498 -498
  321. package/assets/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
  322. package/assets/skills/qe-test-execution/schemas/output.json +529 -529
  323. package/assets/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
  324. package/assets/skills/qe-test-generation/schemas/output.json +439 -439
  325. package/assets/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
  326. package/assets/skills/qe-visual-accessibility/schemas/output.json +491 -491
  327. package/assets/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
  328. package/assets/skills/quality-metrics/schemas/output.json +403 -403
  329. package/assets/skills/refactoring-patterns/schemas/output.json +475 -475
  330. package/assets/skills/regression-testing/evals/regression-testing.yaml +504 -504
  331. package/assets/skills/regression-testing/schemas/output.json +311 -311
  332. package/assets/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
  333. package/assets/skills/risk-based-testing/schemas/output.json +480 -480
  334. package/assets/skills/security-testing/evals/security-testing.yaml +789 -789
  335. package/assets/skills/security-testing/schemas/output.json +879 -879
  336. package/assets/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
  337. package/assets/skills/security-visual-testing/schemas/output.json +486 -486
  338. package/assets/skills/security-visual-testing/scripts/validate-config.json +45 -0
  339. package/assets/skills/sfdipot-product-factors/SKILL.md +239 -239
  340. package/assets/skills/sherlock-review/SKILL.md +5 -0
  341. package/assets/skills/sherlock-review/schemas/output.json +297 -0
  342. package/assets/skills/sherlock-review/scripts/validate-config.json +35 -0
  343. package/assets/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
  344. package/assets/skills/shift-left-testing/schemas/output.json +459 -459
  345. package/assets/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
  346. package/assets/skills/shift-right-testing/schemas/output.json +418 -418
  347. package/assets/skills/tdd-london-chicago/schemas/output.json +444 -444
  348. package/assets/skills/technical-writing/schemas/output.json +268 -268
  349. package/assets/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
  350. package/assets/skills/test-automation-strategy/schemas/output.json +444 -444
  351. package/assets/skills/test-data-management/evals/test-data-management.yaml +504 -504
  352. package/assets/skills/test-data-management/schemas/output.json +284 -284
  353. package/assets/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
  354. package/assets/skills/test-design-techniques/schemas/output.json +295 -295
  355. package/assets/skills/test-environment-management/schemas/output.json +310 -310
  356. package/assets/skills/test-idea-rewriting/SKILL.md +229 -229
  357. package/assets/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
  358. package/assets/skills/test-reporting-analytics/schemas/output.json +329 -329
  359. package/assets/skills/testability-scoring/SKILL.md +5 -0
  360. package/assets/skills/testability-scoring/evals/testability-scoring.yaml +814 -0
  361. package/assets/skills/testability-scoring/resources/templates/config.template.js +84 -84
  362. package/assets/skills/testability-scoring/schemas/output.json +606 -0
  363. package/assets/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
  364. package/assets/skills/testability-scoring/scripts/validate-config.json +42 -0
  365. package/assets/skills/trust-tier-manifest.json +2404 -0
  366. package/assets/skills/verification-quality/evals/verification-quality.yaml +150 -150
  367. package/assets/skills/verification-quality/schemas/output.json +432 -432
  368. package/assets/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
  369. package/assets/skills/visual-testing-advanced/schemas/output.json +294 -294
  370. package/assets/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -0
  371. package/assets/skills/wms-testing-patterns/schemas/output.json +150 -0
  372. package/assets/skills/wms-testing-patterns/scripts/validate-config.json +51 -0
  373. package/assets/templates/validation-summary.json +56 -56
  374. package/dist/benchmarks/performance-benchmarks.js +1 -1
  375. package/dist/cli/bundle.js +9158 -2288
  376. package/dist/cli/commands/hooks.d.ts.map +1 -1
  377. package/dist/cli/commands/hooks.js +92 -0
  378. package/dist/cli/commands/hooks.js.map +1 -1
  379. package/dist/cli/commands/mcp.d.ts.map +1 -1
  380. package/dist/cli/commands/mcp.js +11 -9
  381. package/dist/cli/commands/mcp.js.map +1 -1
  382. package/dist/cli/commands/migrate.js +2 -2
  383. package/dist/coordination/constants.d.ts +1 -1
  384. package/dist/coordination/constants.js +1 -1
  385. package/dist/coordination/handlers/coverage-handlers.js +1 -1
  386. package/dist/coordination/handlers/coverage-handlers.js.map +1 -1
  387. package/dist/domains/code-intelligence/services/semantic-analyzer.d.ts +1 -1
  388. package/dist/domains/code-intelligence/services/semantic-analyzer.d.ts.map +1 -1
  389. package/dist/domains/code-intelligence/services/semantic-analyzer.js +1 -1
  390. package/dist/domains/code-intelligence/services/semantic-analyzer.js.map +1 -1
  391. package/dist/domains/coverage-analysis/coordinator.js +1 -1
  392. package/dist/domains/coverage-analysis/services/coverage-analyzer.js +1 -1
  393. package/dist/domains/coverage-analysis/services/coverage-embedder.d.ts +1 -1
  394. package/dist/domains/coverage-analysis/services/coverage-embedder.js +1 -1
  395. package/dist/domains/coverage-analysis/services/gap-detector.js +1 -1
  396. package/dist/domains/coverage-analysis/services/ghost-coverage-analyzer.js +1 -1
  397. package/dist/domains/coverage-analysis/services/hnsw-index.d.ts +2 -2
  398. package/dist/domains/coverage-analysis/services/hnsw-index.js +3 -3
  399. package/dist/domains/coverage-analysis/services/sublinear-analyzer.d.ts +1 -1
  400. package/dist/domains/coverage-analysis/services/sublinear-analyzer.js +1 -1
  401. package/dist/domains/test-execution/services/test-prioritizer.js +1 -1
  402. package/dist/domains/test-generation/context/rust-context-builder.d.ts +31 -0
  403. package/dist/domains/test-generation/context/rust-context-builder.d.ts.map +1 -0
  404. package/dist/domains/test-generation/context/rust-context-builder.js +27 -0
  405. package/dist/domains/test-generation/context/rust-context-builder.js.map +1 -0
  406. package/dist/domains/test-generation/coordinator.js +3 -3
  407. package/dist/domains/test-generation/coordinator.js.map +1 -1
  408. package/dist/domains/test-generation/detectors/mobile-detector.d.ts +41 -0
  409. package/dist/domains/test-generation/detectors/mobile-detector.d.ts.map +1 -0
  410. package/dist/domains/test-generation/detectors/mobile-detector.js +111 -0
  411. package/dist/domains/test-generation/detectors/mobile-detector.js.map +1 -0
  412. package/dist/domains/test-generation/detectors/spring-detector.d.ts +22 -0
  413. package/dist/domains/test-generation/detectors/spring-detector.d.ts.map +1 -0
  414. package/dist/domains/test-generation/detectors/spring-detector.js +37 -0
  415. package/dist/domains/test-generation/detectors/spring-detector.js.map +1 -0
  416. package/dist/domains/test-generation/factories/test-generator-factory.d.ts +2 -1
  417. package/dist/domains/test-generation/factories/test-generator-factory.d.ts.map +1 -1
  418. package/dist/domains/test-generation/factories/test-generator-factory.js +33 -13
  419. package/dist/domains/test-generation/factories/test-generator-factory.js.map +1 -1
  420. package/dist/domains/test-generation/generators/flutter-test-generator.d.ts +107 -0
  421. package/dist/domains/test-generation/generators/flutter-test-generator.d.ts.map +1 -0
  422. package/dist/domains/test-generation/generators/flutter-test-generator.js +590 -0
  423. package/dist/domains/test-generation/generators/flutter-test-generator.js.map +1 -0
  424. package/dist/domains/test-generation/generators/go-test-generator.d.ts +139 -0
  425. package/dist/domains/test-generation/generators/go-test-generator.d.ts.map +1 -0
  426. package/dist/domains/test-generation/generators/go-test-generator.js +654 -0
  427. package/dist/domains/test-generation/generators/go-test-generator.js.map +1 -0
  428. package/dist/domains/test-generation/generators/index.d.ts +8 -0
  429. package/dist/domains/test-generation/generators/index.d.ts.map +1 -1
  430. package/dist/domains/test-generation/generators/index.js +8 -0
  431. package/dist/domains/test-generation/generators/index.js.map +1 -1
  432. package/dist/domains/test-generation/generators/jest-rn-generator.d.ts +95 -0
  433. package/dist/domains/test-generation/generators/jest-rn-generator.d.ts.map +1 -0
  434. package/dist/domains/test-generation/generators/jest-rn-generator.js +591 -0
  435. package/dist/domains/test-generation/generators/jest-rn-generator.js.map +1 -0
  436. package/dist/domains/test-generation/generators/junit5-generator.d.ts +107 -0
  437. package/dist/domains/test-generation/generators/junit5-generator.d.ts.map +1 -0
  438. package/dist/domains/test-generation/generators/junit5-generator.js +588 -0
  439. package/dist/domains/test-generation/generators/junit5-generator.js.map +1 -0
  440. package/dist/domains/test-generation/generators/kotlin-junit-generator.d.ts +109 -0
  441. package/dist/domains/test-generation/generators/kotlin-junit-generator.d.ts.map +1 -0
  442. package/dist/domains/test-generation/generators/kotlin-junit-generator.js +588 -0
  443. package/dist/domains/test-generation/generators/kotlin-junit-generator.js.map +1 -0
  444. package/dist/domains/test-generation/generators/pytest-generator.d.ts +8 -1
  445. package/dist/domains/test-generation/generators/pytest-generator.d.ts.map +1 -1
  446. package/dist/domains/test-generation/generators/pytest-generator.js +57 -0
  447. package/dist/domains/test-generation/generators/pytest-generator.js.map +1 -1
  448. package/dist/domains/test-generation/generators/rust-test-generator.d.ts +80 -0
  449. package/dist/domains/test-generation/generators/rust-test-generator.d.ts.map +1 -0
  450. package/dist/domains/test-generation/generators/rust-test-generator.js +442 -0
  451. package/dist/domains/test-generation/generators/rust-test-generator.js.map +1 -0
  452. package/dist/domains/test-generation/generators/swift-testing-generator.d.ts +97 -0
  453. package/dist/domains/test-generation/generators/swift-testing-generator.d.ts.map +1 -0
  454. package/dist/domains/test-generation/generators/swift-testing-generator.js +482 -0
  455. package/dist/domains/test-generation/generators/swift-testing-generator.js.map +1 -0
  456. package/dist/domains/test-generation/generators/xunit-generator.d.ts +110 -0
  457. package/dist/domains/test-generation/generators/xunit-generator.d.ts.map +1 -0
  458. package/dist/domains/test-generation/generators/xunit-generator.js +611 -0
  459. package/dist/domains/test-generation/generators/xunit-generator.js.map +1 -0
  460. package/dist/domains/test-generation/interfaces.d.ts +11 -2
  461. package/dist/domains/test-generation/interfaces.d.ts.map +1 -1
  462. package/dist/domains/test-generation/prompts/language-prompts.d.ts +29 -0
  463. package/dist/domains/test-generation/prompts/language-prompts.d.ts.map +1 -0
  464. package/dist/domains/test-generation/prompts/language-prompts.js +135 -0
  465. package/dist/domains/test-generation/prompts/language-prompts.js.map +1 -0
  466. package/dist/domains/test-generation/services/compilation-validator.d.ts +43 -0
  467. package/dist/domains/test-generation/services/compilation-validator.d.ts.map +1 -0
  468. package/dist/domains/test-generation/services/compilation-validator.js +134 -0
  469. package/dist/domains/test-generation/services/compilation-validator.js.map +1 -0
  470. package/dist/domains/test-generation/services/index.d.ts +2 -1
  471. package/dist/domains/test-generation/services/index.d.ts.map +1 -1
  472. package/dist/domains/test-generation/services/index.js +3 -1
  473. package/dist/domains/test-generation/services/index.js.map +1 -1
  474. package/dist/domains/test-generation/services/test-file-resolver.d.ts +32 -0
  475. package/dist/domains/test-generation/services/test-file-resolver.d.ts.map +1 -0
  476. package/dist/domains/test-generation/services/test-file-resolver.js +159 -0
  477. package/dist/domains/test-generation/services/test-file-resolver.js.map +1 -0
  478. package/dist/domains/test-generation/services/test-generator.d.ts +10 -0
  479. package/dist/domains/test-generation/services/test-generator.d.ts.map +1 -1
  480. package/dist/domains/test-generation/services/test-generator.js +87 -10
  481. package/dist/domains/test-generation/services/test-generator.js.map +1 -1
  482. package/dist/governance/feature-flags.js +2 -2
  483. package/dist/governance/feature-flags.js.map +1 -1
  484. package/dist/governance/shard-embeddings.js +1 -1
  485. package/dist/init/init-wizard-hooks.d.ts.map +1 -1
  486. package/dist/init/init-wizard-hooks.js +0 -1
  487. package/dist/init/init-wizard-hooks.js.map +1 -1
  488. package/dist/init/phases/07-hooks.d.ts.map +1 -1
  489. package/dist/init/phases/07-hooks.js +0 -2
  490. package/dist/init/phases/07-hooks.js.map +1 -1
  491. package/dist/init/phases/08-mcp.d.ts +8 -4
  492. package/dist/init/phases/08-mcp.d.ts.map +1 -1
  493. package/dist/init/phases/08-mcp.js +13 -31
  494. package/dist/init/phases/08-mcp.js.map +1 -1
  495. package/dist/init/phases/10-workers.js +4 -4
  496. package/dist/init/phases/10-workers.js.map +1 -1
  497. package/dist/init/settings-merge.d.ts.map +1 -1
  498. package/dist/init/settings-merge.js +0 -2
  499. package/dist/init/settings-merge.js.map +1 -1
  500. package/dist/init/token-bootstrap.js +1 -1
  501. package/dist/init/token-bootstrap.js.map +1 -1
  502. package/dist/integrations/rl-suite/algorithms/decision-transformer.js +1 -1
  503. package/dist/kernel/constants.d.ts +2 -2
  504. package/dist/kernel/constants.js +2 -2
  505. package/dist/kernel/hnsw-adapter.js +1 -1
  506. package/dist/kernel/progressive-hnsw-backend.d.ts +2 -2
  507. package/dist/kernel/progressive-hnsw-backend.js +2 -2
  508. package/dist/learning/dream/concept-graph.d.ts +1 -1
  509. package/dist/learning/dream/concept-graph.js +1 -1
  510. package/dist/learning/dream/dream-engine.d.ts +1 -1
  511. package/dist/learning/dream/dream-engine.js +1 -1
  512. package/dist/learning/dream/index.d.ts +1 -1
  513. package/dist/learning/dream/index.js +1 -1
  514. package/dist/learning/dream/types.d.ts +1 -1
  515. package/dist/learning/dream/types.d.ts.map +1 -1
  516. package/dist/learning/dream/types.js +1 -1
  517. package/dist/learning/dream/types.js.map +1 -1
  518. package/dist/learning/token-tracker.js +1 -1
  519. package/dist/learning/token-tracker.js.map +1 -1
  520. package/dist/mcp/bundle.js +7538 -893
  521. package/dist/routing/qe-agent-registry.js +4 -4
  522. package/dist/routing/qe-agent-registry.js.map +1 -1
  523. package/dist/routing/types.d.ts +5 -8
  524. package/dist/routing/types.d.ts.map +1 -1
  525. package/dist/routing/types.js.map +1 -1
  526. package/dist/shared/embeddings/embedding-cache.js +2 -2
  527. package/dist/shared/embeddings/index.d.ts +2 -2
  528. package/dist/shared/embeddings/index.js +2 -2
  529. package/dist/shared/embeddings/nomic-embedder.d.ts +4 -4
  530. package/dist/shared/embeddings/nomic-embedder.js +2 -2
  531. package/dist/shared/embeddings/ollama-client.d.ts +1 -1
  532. package/dist/shared/embeddings/ollama-client.js +2 -2
  533. package/dist/shared/embeddings/ollama-client.js.map +1 -1
  534. package/dist/shared/embeddings/types.d.ts +2 -2
  535. package/dist/shared/embeddings/types.js +2 -2
  536. package/dist/shared/language-detector.d.ts +46 -0
  537. package/dist/shared/language-detector.d.ts.map +1 -0
  538. package/dist/shared/language-detector.js +183 -0
  539. package/dist/shared/language-detector.js.map +1 -0
  540. package/dist/shared/llm/providers/ollama.js +1 -1
  541. package/dist/shared/metrics/code-metrics.d.ts.map +1 -1
  542. package/dist/shared/metrics/code-metrics.js +24 -1
  543. package/dist/shared/metrics/code-metrics.js.map +1 -1
  544. package/dist/shared/parsers/index.d.ts +2 -0
  545. package/dist/shared/parsers/index.d.ts.map +1 -1
  546. package/dist/shared/parsers/index.js +2 -0
  547. package/dist/shared/parsers/index.js.map +1 -1
  548. package/dist/shared/parsers/interfaces.d.ts +81 -0
  549. package/dist/shared/parsers/interfaces.d.ts.map +1 -0
  550. package/dist/shared/parsers/interfaces.js +6 -0
  551. package/dist/shared/parsers/interfaces.js.map +1 -0
  552. package/dist/shared/parsers/multi-language-parser.d.ts +144 -0
  553. package/dist/shared/parsers/multi-language-parser.d.ts.map +1 -0
  554. package/dist/shared/parsers/multi-language-parser.js +1271 -0
  555. package/dist/shared/parsers/multi-language-parser.js.map +1 -0
  556. package/dist/shared/parsers/rust-ownership-analyzer.d.ts +45 -0
  557. package/dist/shared/parsers/rust-ownership-analyzer.d.ts.map +1 -0
  558. package/dist/shared/parsers/rust-ownership-analyzer.js +52 -0
  559. package/dist/shared/parsers/rust-ownership-analyzer.js.map +1 -0
  560. package/dist/shared/parsers/typescript-parser.d.ts +16 -0
  561. package/dist/shared/parsers/typescript-parser.d.ts.map +1 -1
  562. package/dist/shared/parsers/typescript-parser.js +85 -0
  563. package/dist/shared/parsers/typescript-parser.js.map +1 -1
  564. package/dist/shared/types/test-frameworks.d.ts +25 -0
  565. package/dist/shared/types/test-frameworks.d.ts.map +1 -0
  566. package/dist/shared/types/test-frameworks.js +111 -0
  567. package/dist/shared/types/test-frameworks.js.map +1 -0
  568. package/package.json +1 -1
  569. package/scripts/prepare-assets.sh +16 -2
@@ -1,968 +1,968 @@
1
- # =============================================================================
2
- # AQE Skill Evaluation Test Suite: Database Testing v1.0.0
3
- # =============================================================================
4
- #
5
- # Comprehensive evaluation suite for the database-testing skill per ADR-056.
6
- # Tests schema validation, data integrity, migration testing, transaction ACID
7
- # properties, and query performance analysis across multiple database types.
8
- #
9
- # Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
10
- # Validator: .claude/skills/database-testing/scripts/validate-config.json
11
- #
12
- # Coverage:
13
- # - Schema validation (tables, columns, constraints, indexes)
14
- # - Data integrity (unique, foreign key, check constraints)
15
- # - Migration testing (forward, rollback, data preservation)
16
- # - Transaction testing (ACID properties, isolation levels)
17
- # - Query performance (slow queries, missing indexes, N+1)
18
- # - Multi-database support (PostgreSQL, MySQL, MongoDB, SQLite)
19
- #
20
- # =============================================================================
21
-
22
- skill: database-testing
23
- version: 1.0.0
24
- description: >
25
- Comprehensive evaluation suite for the database-testing skill.
26
- Tests database schema validation, data integrity verification,
27
- migration testing, transaction ACID compliance, and query performance
28
- analysis. Supports PostgreSQL, MySQL, MongoDB, and SQLite.
29
- Integrates with ReasoningBank for continuous improvement.
30
-
31
- # =============================================================================
32
- # Multi-Model Configuration
33
- # =============================================================================
34
-
35
- models_to_test:
36
- - claude-sonnet-4 # Primary model (high accuracy expected)
37
- - claude-3-haiku # Fast model (minimum quality threshold)
38
- - gpt-4o # Cross-vendor validation
39
-
40
- # =============================================================================
41
- # MCP Integration Configuration
42
- # =============================================================================
43
-
44
- mcp_integration:
45
- enabled: true
46
- namespace: skill-validation
47
-
48
- # Query existing database patterns before running evals
49
- query_patterns: true
50
-
51
- # Track each test outcome for learning feedback loop
52
- track_outcomes: true
53
-
54
- # Store successful patterns after evals complete
55
- store_patterns: true
56
-
57
- # Share learning with fleet coordinator agents
58
- share_learning: true
59
-
60
- # Update quality gate with validation metrics
61
- update_quality_gate: true
62
-
63
- # Target agents for learning distribution
64
- target_agents:
65
- - qe-learning-coordinator
66
- - qe-queen-coordinator
67
- - qe-test-data-architect
68
- - qe-test-executor
69
- - qe-performance-tester
70
-
71
- # =============================================================================
72
- # ReasoningBank Learning Configuration
73
- # =============================================================================
74
-
75
- learning:
76
- store_success_patterns: true
77
- store_failure_patterns: true
78
- pattern_ttl_days: 90
79
- min_confidence_to_store: 0.7
80
- cross_model_comparison: true
81
-
82
- # =============================================================================
83
- # Result Format Configuration
84
- # =============================================================================
85
-
86
- result_format:
87
- json_output: true
88
- markdown_report: true
89
- include_raw_output: false
90
- include_timing: true
91
- include_token_usage: true
92
-
93
- # =============================================================================
94
- # Environment Setup
95
- # =============================================================================
96
-
97
- setup:
98
- required_tools:
99
- - jq # JSON parsing (required)
100
- - node # ORM validation (optional but recommended)
101
-
102
- environment_variables:
103
- DATABASE_TEST_DEPTH: "comprehensive"
104
- MIGRATION_TEST_ROLLBACK: "true"
105
- ACID_TEST_ENABLED: "true"
106
-
107
- fixtures:
108
- - name: postgresql_schema_test
109
- path: fixtures/postgresql-schema.sql
110
- content: |
111
- -- PostgreSQL test schema with intentional issues
112
- CREATE TABLE users (
113
- id SERIAL PRIMARY KEY,
114
- email VARCHAR(255), -- Missing UNIQUE constraint
115
- password VARCHAR(255), -- Should be NOT NULL
116
- created_at TIMESTAMP DEFAULT NOW()
117
- );
118
-
119
- CREATE TABLE orders (
120
- id SERIAL PRIMARY KEY,
121
- user_id INTEGER, -- Missing FOREIGN KEY
122
- total DECIMAL(10,2),
123
- status VARCHAR(50)
124
- );
125
-
126
- -- Missing index on frequently queried column
127
- CREATE TABLE products (
128
- id SERIAL PRIMARY KEY,
129
- name VARCHAR(255),
130
- category VARCHAR(100),
131
- price DECIMAL(10,2)
132
- );
133
-
134
- - name: migration_test_file
135
- path: fixtures/migration-add-age.js
136
- content: |
137
- exports.up = async function(knex) {
138
- await knex.schema.alterTable('users', (table) => {
139
- table.integer('age').nullable();
140
- });
141
- };
142
-
143
- exports.down = async function(knex) {
144
- await knex.schema.alterTable('users', (table) => {
145
- table.dropColumn('age');
146
- });
147
- };
148
-
149
- # =============================================================================
150
- # TEST CASES
151
- # =============================================================================
152
-
153
- test_cases:
154
- # ---------------------------------------------------------------------------
155
- # CATEGORY: Schema Validation
156
- # ---------------------------------------------------------------------------
157
-
158
- - id: tc001_missing_unique_constraint
159
- description: "Detect missing UNIQUE constraint on email column"
160
- category: schema
161
- priority: critical
162
-
163
- input:
164
- code: |
165
- -- User table without unique email constraint
166
- CREATE TABLE users (
167
- id SERIAL PRIMARY KEY,
168
- email VARCHAR(255),
169
- name VARCHAR(100),
170
- created_at TIMESTAMP DEFAULT NOW()
171
- );
172
-
173
- -- Application code that assumes email uniqueness
174
- INSERT INTO users (email, name) VALUES ('john@example.com', 'John');
175
- INSERT INTO users (email, name) VALUES ('john@example.com', 'Johnny'); -- Duplicate allowed!
176
- context:
177
- database: postgresql
178
- framework: knex
179
- environment: production
180
-
181
- expected_output:
182
- must_contain:
183
- - "unique"
184
- - "constraint"
185
- - "email"
186
- - "duplicate"
187
- must_not_contain:
188
- - "no issues"
189
- - "valid schema"
190
- must_match_regex:
191
- - "DB-\\d{3}"
192
- severity_classification: high
193
- finding_count:
194
- min: 1
195
- max: 5
196
-
197
- validation:
198
- schema_check: true
199
- keyword_match_threshold: 0.8
200
- reasoning_quality_min: 0.7
201
- grading_rubric:
202
- completeness: 0.3
203
- accuracy: 0.5
204
- actionability: 0.2
205
-
206
- timeout_ms: 30000
207
-
208
- - id: tc002_missing_foreign_key
209
- description: "Detect missing foreign key constraint causing orphaned records"
210
- category: schema
211
- priority: critical
212
-
213
- input:
214
- code: |
215
- CREATE TABLE users (
216
- id SERIAL PRIMARY KEY,
217
- email VARCHAR(255) UNIQUE NOT NULL
218
- );
219
-
220
- CREATE TABLE orders (
221
- id SERIAL PRIMARY KEY,
222
- user_id INTEGER, -- No FK constraint!
223
- total DECIMAL(10,2)
224
- );
225
-
226
- -- This allows orphaned orders
227
- INSERT INTO orders (user_id, total) VALUES (999, 100.00);
228
- DELETE FROM users WHERE id = 1; -- Orphans all orders for user 1
229
- context:
230
- database: postgresql
231
- framework: prisma
232
-
233
- expected_output:
234
- must_contain:
235
- - "foreign key"
236
- - "referential integrity"
237
- - "orphaned"
238
- must_match_regex:
239
- - "DB-\\d{3}"
240
- severity_classification: critical
241
- finding_count:
242
- min: 1
243
-
244
- validation:
245
- schema_check: true
246
- keyword_match_threshold: 0.8
247
-
248
- - id: tc003_missing_index_on_foreign_key
249
- description: "Detect missing index on foreign key column affecting performance"
250
- category: schema
251
- priority: high
252
-
253
- input:
254
- code: |
255
- CREATE TABLE orders (
256
- id SERIAL PRIMARY KEY,
257
- user_id INTEGER REFERENCES users(id),
258
- product_id INTEGER REFERENCES products(id),
259
- quantity INTEGER,
260
- created_at TIMESTAMP
261
- );
262
-
263
- -- Frequently run queries without indexes
264
- SELECT * FROM orders WHERE user_id = 123;
265
- SELECT COUNT(*) FROM orders WHERE product_id = 456;
266
- context:
267
- database: postgresql
268
- framework: typeorm
269
-
270
- expected_output:
271
- must_contain:
272
- - "index"
273
- - "foreign key"
274
- - "performance"
275
- - "user_id"
276
- must_match_regex:
277
- - "CREATE INDEX"
278
- severity_classification: medium
279
-
280
- validation:
281
- schema_check: true
282
- keyword_match_threshold: 0.7
283
-
284
- # ---------------------------------------------------------------------------
285
- # CATEGORY: Data Integrity
286
- # ---------------------------------------------------------------------------
287
-
288
- - id: tc004_check_constraint_violation
289
- description: "Detect check constraint violations on status column"
290
- category: integrity
291
- priority: high
292
-
293
- input:
294
- code: |
295
- CREATE TABLE orders (
296
- id SERIAL PRIMARY KEY,
297
- status VARCHAR(20) CHECK (status IN ('pending', 'processing', 'shipped', 'delivered')),
298
- total DECIMAL(10,2) CHECK (total >= 0)
299
- );
300
-
301
- -- Test constraint enforcement
302
- INSERT INTO orders (status, total) VALUES ('invalid_status', 100.00);
303
- INSERT INTO orders (status, total) VALUES ('pending', -50.00);
304
- context:
305
- database: postgresql
306
- test_type: integrity
307
-
308
- expected_output:
309
- must_contain:
310
- - "check constraint"
311
- - "violation"
312
- - "status"
313
- must_not_contain:
314
- - "passed"
315
- - "valid"
316
- severity_classification: high
317
-
318
- validation:
319
- schema_check: true
320
- keyword_match_threshold: 0.8
321
-
322
- - id: tc005_null_constraint_test
323
- description: "Detect NOT NULL constraint violations"
324
- category: integrity
325
- priority: high
326
-
327
- input:
328
- code: |
329
- const user = new User();
330
- user.email = 'test@example.com';
331
- // Missing required field: password
332
- await user.save();
333
-
334
- // Schema:
335
- // email VARCHAR(255) NOT NULL
336
- // password VARCHAR(255) NOT NULL
337
- // name VARCHAR(100) -- nullable
338
- context:
339
- database: mysql
340
- framework: sequelize
341
- test_type: integrity
342
-
343
- expected_output:
344
- must_contain:
345
- - "NOT NULL"
346
- - "constraint"
347
- - "password"
348
- - "required"
349
- severity_classification: high
350
- finding_count:
351
- min: 1
352
-
353
- validation:
354
- schema_check: true
355
- keyword_match_threshold: 0.8
356
-
357
- # ---------------------------------------------------------------------------
358
- # CATEGORY: Migration Testing
359
- # ---------------------------------------------------------------------------
360
-
361
- - id: tc006_migration_forward_test
362
- description: "Test migration applies successfully forward"
363
- category: migration
364
- priority: critical
365
-
366
- input:
367
- code: |
368
- // Migration: add-user-roles.js
369
- exports.up = async function(knex) {
370
- await knex.schema.createTable('roles', (table) => {
371
- table.increments('id');
372
- table.string('name').unique().notNullable();
373
- });
374
-
375
- await knex.schema.alterTable('users', (table) => {
376
- table.integer('role_id').references('id').inTable('roles');
377
- });
378
- };
379
-
380
- exports.down = async function(knex) {
381
- await knex.schema.alterTable('users', (table) => {
382
- table.dropColumn('role_id');
383
- });
384
- await knex.schema.dropTable('roles');
385
- };
386
- context:
387
- database: postgresql
388
- framework: knex
389
- test_type: migration
390
-
391
- expected_output:
392
- must_contain:
393
- - "migration"
394
- - "forward"
395
- - "roles"
396
- - "users"
397
- must_not_contain:
398
- - "failed"
399
- - "error"
400
- must_match_regex:
401
- - "up|forward|apply"
402
-
403
- validation:
404
- schema_check: true
405
- keyword_match_threshold: 0.7
406
-
407
- - id: tc007_migration_rollback_test
408
- description: "Test migration rollback works correctly"
409
- category: migration
410
- priority: critical
411
-
412
- input:
413
- code: |
414
- // Test migration rollback
415
- await migrate.latest(); // Apply all migrations
416
- await migrate.rollback(); // Rollback last batch
417
-
418
- // Verify tables are removed
419
- const tables = await knex.raw("SELECT table_name FROM information_schema.tables");
420
- expect(tables).not.toContain('new_feature_table');
421
- context:
422
- database: postgresql
423
- framework: knex
424
- test_type: migration
425
-
426
- expected_output:
427
- must_contain:
428
- - "rollback"
429
- - "down"
430
- - "reverse"
431
- must_not_contain:
432
- - "irreversible"
433
- - "failed"
434
- severity_classification: medium
435
-
436
- validation:
437
- schema_check: true
438
- keyword_match_threshold: 0.7
439
-
440
- - id: tc008_migration_data_preservation
441
- description: "Test migration preserves existing data"
442
- category: migration
443
- priority: critical
444
-
445
- input:
446
- code: |
447
- // Existing data before migration
448
- await db.users.create({ email: 'existing@example.com', name: 'Existing User' });
449
-
450
- // Migration adds new column
451
- exports.up = async function(knex) {
452
- await knex.schema.alterTable('users', (table) => {
453
- table.string('phone').nullable();
454
- });
455
- };
456
-
457
- // After migration, verify existing data is preserved
458
- const user = await db.users.findOne({ email: 'existing@example.com' });
459
- expect(user.name).toBe('Existing User'); // Data should be intact
460
- expect(user.phone).toBeNull(); // New column should be null
461
- context:
462
- database: postgresql
463
- framework: knex
464
- test_type: migration
465
-
466
- expected_output:
467
- must_contain:
468
- - "data preservation"
469
- - "existing"
470
- - "intact"
471
- - "backward compatible"
472
- must_not_contain:
473
- - "data loss"
474
- - "corrupted"
475
- severity_classification: critical
476
-
477
- validation:
478
- schema_check: true
479
- keyword_match_threshold: 0.7
480
-
481
- # ---------------------------------------------------------------------------
482
- # CATEGORY: Transaction Testing (ACID)
483
- # ---------------------------------------------------------------------------
484
-
485
- - id: tc009_atomicity_test
486
- description: "Test transaction atomicity - all or nothing"
487
- category: transaction
488
- priority: critical
489
-
490
- input:
491
- code: |
492
- test('transaction rolls back on error', async () => {
493
- const initialCount = await db.users.count();
494
-
495
- try {
496
- await db.transaction(async (trx) => {
497
- await trx('users').insert({ email: 'user1@example.com' });
498
- await trx('users').insert({ email: 'user2@example.com' });
499
- throw new Error('Intentional rollback');
500
- });
501
- } catch (error) { /* Expected */ }
502
-
503
- // Count should be unchanged
504
- expect(await db.users.count()).toBe(initialCount);
505
- });
506
- context:
507
- database: postgresql
508
- framework: knex
509
- test_type: transaction
510
-
511
- expected_output:
512
- must_contain:
513
- - "atomicity"
514
- - "rollback"
515
- - "transaction"
516
- - "all or nothing"
517
- must_not_contain:
518
- - "partial commit"
519
- must_match_regex:
520
- - "ACID|atomic"
521
- severity_classification: critical
522
-
523
- validation:
524
- schema_check: true
525
- keyword_match_threshold: 0.8
526
- reasoning_quality_min: 0.8
527
-
528
- - id: tc010_isolation_test
529
- description: "Test transaction isolation - concurrent access"
530
- category: transaction
531
- priority: critical
532
-
533
- input:
534
- code: |
535
- test('concurrent transactions isolated', async () => {
536
- const user = await db.users.create({ balance: 100 });
537
-
538
- // Two concurrent withdrawals - race condition test
539
- const results = await Promise.all([
540
- db.transaction(async (trx) => {
541
- const current = await trx('users').where({ id: user.id }).first();
542
- if (current.balance >= 50) {
543
- await trx('users').where({ id: user.id }).update({
544
- balance: current.balance - 50
545
- });
546
- return 'success';
547
- }
548
- return 'insufficient';
549
- }),
550
- db.transaction(async (trx) => {
551
- const current = await trx('users').where({ id: user.id }).first();
552
- if (current.balance >= 50) {
553
- await trx('users').where({ id: user.id }).update({
554
- balance: current.balance - 50
555
- });
556
- return 'success';
557
- }
558
- return 'insufficient';
559
- })
560
- ]);
561
-
562
- const final = await db.users.findOne({ id: user.id });
563
- // With proper isolation, one should fail or balance should be 0
564
- expect(final.balance).toBeGreaterThanOrEqual(0);
565
- });
566
- context:
567
- database: postgresql
568
- framework: knex
569
- test_type: transaction
570
- isolation_level: REPEATABLE_READ
571
-
572
- expected_output:
573
- must_contain:
574
- - "isolation"
575
- - "concurrent"
576
- - "race condition"
577
- - "lock"
578
- must_match_regex:
579
- - "READ_COMMITTED|REPEATABLE_READ|SERIALIZABLE"
580
- severity_classification: critical
581
-
582
- validation:
583
- schema_check: true
584
- keyword_match_threshold: 0.7
585
-
586
- - id: tc011_deadlock_detection
587
- description: "Test deadlock detection and resolution"
588
- category: transaction
589
- priority: high
590
-
591
- input:
592
- code: |
593
- test('deadlock is detected and handled', async () => {
594
- // Create potential deadlock scenario
595
- const tx1 = db.transaction(async (trx) => {
596
- await trx('accounts').where({ id: 1 }).update({ balance: 100 });
597
- await delay(100);
598
- await trx('accounts').where({ id: 2 }).update({ balance: 200 });
599
- });
600
-
601
- const tx2 = db.transaction(async (trx) => {
602
- await trx('accounts').where({ id: 2 }).update({ balance: 150 });
603
- await delay(100);
604
- await trx('accounts').where({ id: 1 }).update({ balance: 250 });
605
- });
606
-
607
- // One should fail with deadlock, other should succeed
608
- const results = await Promise.allSettled([tx1, tx2]);
609
- const failures = results.filter(r => r.status === 'rejected');
610
- expect(failures.some(f => f.reason.message.includes('deadlock'))).toBe(true);
611
- });
612
- context:
613
- database: postgresql
614
- framework: knex
615
- test_type: transaction
616
-
617
- expected_output:
618
- must_contain:
619
- - "deadlock"
620
- - "detection"
621
- - "retry"
622
- - "conflict"
623
- severity_classification: high
624
-
625
- validation:
626
- schema_check: true
627
- keyword_match_threshold: 0.7
628
-
629
- # ---------------------------------------------------------------------------
630
- # CATEGORY: Performance Testing
631
- # ---------------------------------------------------------------------------
632
-
633
- - id: tc012_slow_query_detection
634
- description: "Detect slow queries without proper indexing"
635
- category: performance
636
- priority: high
637
-
638
- input:
639
- code: |
640
- -- Slow query: full table scan on large table
641
- SELECT * FROM orders
642
- WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31'
643
- AND status = 'pending';
644
-
645
- -- Table has 10 million rows, no index on created_at or status
646
- EXPLAIN ANALYZE SELECT * FROM orders
647
- WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31';
648
- -- Output: Seq Scan on orders (cost=0.00..250000.00 rows=5000000)
649
- context:
650
- database: postgresql
651
- test_type: performance
652
- table_size: 10000000
653
-
654
- expected_output:
655
- must_contain:
656
- - "slow query"
657
- - "full table scan"
658
- - "index"
659
- - "Seq Scan"
660
- must_match_regex:
661
- - "CREATE INDEX"
662
- severity_classification: high
663
- finding_count:
664
- min: 1
665
-
666
- validation:
667
- schema_check: true
668
- keyword_match_threshold: 0.8
669
-
670
- - id: tc013_n_plus_one_detection
671
- description: "Detect N+1 query problem"
672
- category: performance
673
- priority: high
674
-
675
- input:
676
- code: |
677
- // N+1 query problem
678
- const users = await User.findAll(); // 1 query
679
-
680
- for (const user of users) {
681
- const orders = await Order.findAll({
682
- where: { userId: user.id }
683
- }); // N queries!
684
- user.orderCount = orders.length;
685
- }
686
-
687
- // Should be:
688
- // const users = await User.findAll({
689
- // include: [{ model: Order }]
690
- // });
691
- context:
692
- database: postgresql
693
- framework: sequelize
694
- test_type: performance
695
-
696
- expected_output:
697
- must_contain:
698
- - "N+1"
699
- - "query"
700
- - "include"
701
- - "eager loading"
702
- - "join"
703
- must_not_contain:
704
- - "efficient"
705
- - "optimal"
706
- severity_classification: high
707
-
708
- validation:
709
- schema_check: true
710
- keyword_match_threshold: 0.8
711
-
712
- - id: tc014_connection_pool_exhaustion
713
- description: "Detect connection pool exhaustion risk"
714
- category: performance
715
- priority: critical
716
-
717
- input:
718
- code: |
719
- // Connection leak - connections not released
720
- async function getUserData(userId) {
721
- const connection = await pool.getConnection();
722
- const user = await connection.query('SELECT * FROM users WHERE id = ?', [userId]);
723
- // Missing: connection.release()
724
- return user;
725
- }
726
-
727
- // Called in a loop without releasing connections
728
- for (let i = 0; i < 1000; i++) {
729
- await getUserData(i);
730
- }
731
- context:
732
- database: mysql
733
- framework: mysql2
734
- test_type: performance
735
- pool_size: 10
736
-
737
- expected_output:
738
- must_contain:
739
- - "connection"
740
- - "pool"
741
- - "leak"
742
- - "release"
743
- - "exhaustion"
744
- severity_classification: critical
745
-
746
- validation:
747
- schema_check: true
748
- keyword_match_threshold: 0.8
749
-
750
- # ---------------------------------------------------------------------------
751
- # CATEGORY: Multi-Database Support
752
- # ---------------------------------------------------------------------------
753
-
754
- - id: tc015_mongodb_schema_validation
755
- description: "Test MongoDB schema validation"
756
- category: schema
757
- priority: high
758
-
759
- input:
760
- code: |
761
- // MongoDB collection without schema validation
762
- db.createCollection("users");
763
-
764
- // Can insert inconsistent documents
765
- db.users.insertOne({ email: "user@example.com", age: 25 });
766
- db.users.insertOne({ email: 123, age: "twenty" }); // Wrong types!
767
- db.users.insertOne({ name: "John" }); // Missing email
768
-
769
- // Should have JSON Schema validation:
770
- db.createCollection("users", {
771
- validator: {
772
- $jsonSchema: {
773
- bsonType: "object",
774
- required: ["email"],
775
- properties: {
776
- email: { bsonType: "string" },
777
- age: { bsonType: "int" }
778
- }
779
- }
780
- }
781
- });
782
- context:
783
- database: mongodb
784
- test_type: schema
785
-
786
- expected_output:
787
- must_contain:
788
- - "schema validation"
789
- - "MongoDB"
790
- - "jsonSchema"
791
- - "bsonType"
792
- severity_classification: high
793
-
794
- validation:
795
- schema_check: true
796
- keyword_match_threshold: 0.7
797
-
798
- - id: tc016_sqlite_foreign_key_enforcement
799
- description: "Test SQLite foreign key enforcement"
800
- category: integrity
801
- priority: high
802
-
803
- input:
804
- code: |
805
- -- SQLite: Foreign keys are OFF by default!
806
- CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);
807
- CREATE TABLE orders (
808
- id INTEGER PRIMARY KEY,
809
- user_id INTEGER REFERENCES users(id)
810
- );
811
-
812
- -- This succeeds even though user 999 doesn't exist
813
- INSERT INTO orders (user_id) VALUES (999);
814
-
815
- -- Need to enable: PRAGMA foreign_keys = ON;
816
- context:
817
- database: sqlite
818
- test_type: integrity
819
-
820
- expected_output:
821
- must_contain:
822
- - "foreign_keys"
823
- - "PRAGMA"
824
- - "SQLite"
825
- - "enforcement"
826
- must_match_regex:
827
- - "PRAGMA foreign_keys\\s*=\\s*ON"
828
- severity_classification: high
829
-
830
- validation:
831
- schema_check: true
832
- keyword_match_threshold: 0.8
833
-
834
- # ---------------------------------------------------------------------------
835
- # CATEGORY: Negative Tests (Secure/Valid Code)
836
- # ---------------------------------------------------------------------------
837
-
838
- - id: tc017_valid_schema_no_false_positives
839
- description: "Verify well-designed schema is NOT flagged as problematic"
840
- category: negative
841
- priority: critical
842
-
843
- input:
844
- code: |
845
- CREATE TABLE users (
846
- id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
847
- email VARCHAR(255) UNIQUE NOT NULL,
848
- password_hash VARCHAR(255) NOT NULL,
849
- created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
850
- updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
851
- );
852
-
853
- CREATE TABLE orders (
854
- id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
855
- user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
856
- total DECIMAL(10,2) NOT NULL CHECK (total >= 0),
857
- status VARCHAR(20) NOT NULL DEFAULT 'pending'
858
- CHECK (status IN ('pending', 'processing', 'shipped', 'delivered', 'cancelled')),
859
- created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
860
- );
861
-
862
- CREATE INDEX idx_orders_user_id ON orders(user_id);
863
- CREATE INDEX idx_orders_status ON orders(status);
864
- CREATE INDEX idx_orders_created_at ON orders(created_at);
865
- context:
866
- database: postgresql
867
- framework: prisma
868
- environment: production
869
-
870
- expected_output:
871
- must_contain:
872
- - "valid"
873
- - "well-designed"
874
- - "proper constraints"
875
- must_not_contain:
876
- - "missing constraint"
877
- - "critical"
878
- - "high severity"
879
- - "orphaned"
880
- finding_count:
881
- max: 2 # Allow informational findings only
882
-
883
- validation:
884
- schema_check: true
885
- keyword_match_threshold: 0.6
886
- allow_partial: true
887
-
888
- - id: tc018_proper_migration_pattern
889
- description: "Verify proper migration pattern is recognized"
890
- category: negative
891
- priority: high
892
-
893
- input:
894
- code: |
895
- // Proper reversible migration with data preservation
896
- exports.up = async function(knex) {
897
- // Check if column exists before adding
898
- const hasColumn = await knex.schema.hasColumn('users', 'phone');
899
- if (!hasColumn) {
900
- await knex.schema.alterTable('users', (table) => {
901
- table.string('phone').nullable();
902
- });
903
- }
904
- };
905
-
906
- exports.down = async function(knex) {
907
- const hasColumn = await knex.schema.hasColumn('users', 'phone');
908
- if (hasColumn) {
909
- await knex.schema.alterTable('users', (table) => {
910
- table.dropColumn('phone');
911
- });
912
- }
913
- };
914
- context:
915
- database: postgresql
916
- framework: knex
917
- test_type: migration
918
-
919
- expected_output:
920
- must_contain:
921
- - "reversible"
922
- - "idempotent"
923
- - "safe"
924
- must_not_contain:
925
- - "irreversible"
926
- - "data loss"
927
- - "critical"
928
- severity_classification: info
929
-
930
- validation:
931
- schema_check: true
932
- allow_partial: true
933
-
934
- # =============================================================================
935
- # SUCCESS CRITERIA
936
- # =============================================================================
937
-
938
- success_criteria:
939
- # Overall pass rate (90% of tests must pass)
940
- pass_rate: 0.9
941
-
942
- # Critical tests must ALL pass (100%)
943
- critical_pass_rate: 1.0
944
-
945
- # Average reasoning quality score
946
- avg_reasoning_quality: 0.75
947
-
948
- # Maximum suite execution time (5 minutes)
949
- max_execution_time_ms: 300000
950
-
951
- # Maximum variance between model results (15%)
952
- cross_model_variance: 0.15
953
-
954
- # =============================================================================
955
- # METADATA
956
- # =============================================================================
957
-
958
- metadata:
959
- author: "qe-test-data-architect"
960
- created: "2026-02-02"
961
- last_updated: "2026-02-02"
962
- coverage_target: >
963
- Database testing coverage: Schema validation (missing constraints, indexes),
964
- Data integrity (unique, FK, check, NOT NULL), Migration testing (forward,
965
- rollback, data preservation), Transaction testing (ACID: atomicity, isolation,
966
- deadlock), Performance (slow queries, N+1, connection pools). Multi-database
967
- support: PostgreSQL, MySQL, MongoDB, SQLite. 18 test cases with 90% pass rate
968
- requirement and 100% critical pass rate for ACID and migration tests.
1
+ # =============================================================================
2
+ # AQE Skill Evaluation Test Suite: Database Testing v1.0.0
3
+ # =============================================================================
4
+ #
5
+ # Comprehensive evaluation suite for the database-testing skill per ADR-056.
6
+ # Tests schema validation, data integrity, migration testing, transaction ACID
7
+ # properties, and query performance analysis across multiple database types.
8
+ #
9
+ # Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
10
+ # Validator: .claude/skills/database-testing/scripts/validate-config.json
11
+ #
12
+ # Coverage:
13
+ # - Schema validation (tables, columns, constraints, indexes)
14
+ # - Data integrity (unique, foreign key, check constraints)
15
+ # - Migration testing (forward, rollback, data preservation)
16
+ # - Transaction testing (ACID properties, isolation levels)
17
+ # - Query performance (slow queries, missing indexes, N+1)
18
+ # - Multi-database support (PostgreSQL, MySQL, MongoDB, SQLite)
19
+ #
20
+ # =============================================================================
21
+
22
+ skill: database-testing
23
+ version: 1.0.0
24
+ description: >
25
+ Comprehensive evaluation suite for the database-testing skill.
26
+ Tests database schema validation, data integrity verification,
27
+ migration testing, transaction ACID compliance, and query performance
28
+ analysis. Supports PostgreSQL, MySQL, MongoDB, and SQLite.
29
+ Integrates with ReasoningBank for continuous improvement.
30
+
31
+ # =============================================================================
32
+ # Multi-Model Configuration
33
+ # =============================================================================
34
+
35
+ models_to_test:
36
+ - claude-sonnet-4 # Primary model (high accuracy expected)
37
+ - claude-3-haiku # Fast model (minimum quality threshold)
38
+ - gpt-4o # Cross-vendor validation
39
+
40
+ # =============================================================================
41
+ # MCP Integration Configuration
42
+ # =============================================================================
43
+
44
+ mcp_integration:
45
+ enabled: true
46
+ namespace: skill-validation
47
+
48
+ # Query existing database patterns before running evals
49
+ query_patterns: true
50
+
51
+ # Track each test outcome for learning feedback loop
52
+ track_outcomes: true
53
+
54
+ # Store successful patterns after evals complete
55
+ store_patterns: true
56
+
57
+ # Share learning with fleet coordinator agents
58
+ share_learning: true
59
+
60
+ # Update quality gate with validation metrics
61
+ update_quality_gate: true
62
+
63
+ # Target agents for learning distribution
64
+ target_agents:
65
+ - qe-learning-coordinator
66
+ - qe-queen-coordinator
67
+ - qe-test-data-architect
68
+ - qe-test-executor
69
+ - qe-performance-tester
70
+
71
+ # =============================================================================
72
+ # ReasoningBank Learning Configuration
73
+ # =============================================================================
74
+
75
+ learning:
76
+ store_success_patterns: true
77
+ store_failure_patterns: true
78
+ pattern_ttl_days: 90
79
+ min_confidence_to_store: 0.7
80
+ cross_model_comparison: true
81
+
82
+ # =============================================================================
83
+ # Result Format Configuration
84
+ # =============================================================================
85
+
86
+ result_format:
87
+ json_output: true
88
+ markdown_report: true
89
+ include_raw_output: false
90
+ include_timing: true
91
+ include_token_usage: true
92
+
93
+ # =============================================================================
94
+ # Environment Setup
95
+ # =============================================================================
96
+
97
+ setup:
98
+ required_tools:
99
+ - jq # JSON parsing (required)
100
+ - node # ORM validation (optional but recommended)
101
+
102
+ environment_variables:
103
+ DATABASE_TEST_DEPTH: "comprehensive"
104
+ MIGRATION_TEST_ROLLBACK: "true"
105
+ ACID_TEST_ENABLED: "true"
106
+
107
+ fixtures:
108
+ - name: postgresql_schema_test
109
+ path: fixtures/postgresql-schema.sql
110
+ content: |
111
+ -- PostgreSQL test schema with intentional issues
112
+ CREATE TABLE users (
113
+ id SERIAL PRIMARY KEY,
114
+ email VARCHAR(255), -- Missing UNIQUE constraint
115
+ password VARCHAR(255), -- Should be NOT NULL
116
+ created_at TIMESTAMP DEFAULT NOW()
117
+ );
118
+
119
+ CREATE TABLE orders (
120
+ id SERIAL PRIMARY KEY,
121
+ user_id INTEGER, -- Missing FOREIGN KEY
122
+ total DECIMAL(10,2),
123
+ status VARCHAR(50)
124
+ );
125
+
126
+ -- Missing index on frequently queried column
127
+ CREATE TABLE products (
128
+ id SERIAL PRIMARY KEY,
129
+ name VARCHAR(255),
130
+ category VARCHAR(100),
131
+ price DECIMAL(10,2)
132
+ );
133
+
134
+ - name: migration_test_file
135
+ path: fixtures/migration-add-age.js
136
+ content: |
137
+ exports.up = async function(knex) {
138
+ await knex.schema.alterTable('users', (table) => {
139
+ table.integer('age').nullable();
140
+ });
141
+ };
142
+
143
+ exports.down = async function(knex) {
144
+ await knex.schema.alterTable('users', (table) => {
145
+ table.dropColumn('age');
146
+ });
147
+ };
148
+
149
+ # =============================================================================
150
+ # TEST CASES
151
+ # =============================================================================
152
+
153
+ test_cases:
154
+ # ---------------------------------------------------------------------------
155
+ # CATEGORY: Schema Validation
156
+ # ---------------------------------------------------------------------------
157
+
158
+ - id: tc001_missing_unique_constraint
159
+ description: "Detect missing UNIQUE constraint on email column"
160
+ category: schema
161
+ priority: critical
162
+
163
+ input:
164
+ code: |
165
+ -- User table without unique email constraint
166
+ CREATE TABLE users (
167
+ id SERIAL PRIMARY KEY,
168
+ email VARCHAR(255),
169
+ name VARCHAR(100),
170
+ created_at TIMESTAMP DEFAULT NOW()
171
+ );
172
+
173
+ -- Application code that assumes email uniqueness
174
+ INSERT INTO users (email, name) VALUES ('john@example.com', 'John');
175
+ INSERT INTO users (email, name) VALUES ('john@example.com', 'Johnny'); -- Duplicate allowed!
176
+ context:
177
+ database: postgresql
178
+ framework: knex
179
+ environment: production
180
+
181
+ expected_output:
182
+ must_contain:
183
+ - "unique"
184
+ - "constraint"
185
+ - "email"
186
+ - "duplicate"
187
+ must_not_contain:
188
+ - "no issues"
189
+ - "valid schema"
190
+ must_match_regex:
191
+ - "DB-\\d{3}"
192
+ severity_classification: high
193
+ finding_count:
194
+ min: 1
195
+ max: 5
196
+
197
+ validation:
198
+ schema_check: true
199
+ keyword_match_threshold: 0.8
200
+ reasoning_quality_min: 0.7
201
+ grading_rubric:
202
+ completeness: 0.3
203
+ accuracy: 0.5
204
+ actionability: 0.2
205
+
206
+ timeout_ms: 30000
207
+
208
+ - id: tc002_missing_foreign_key
209
+ description: "Detect missing foreign key constraint causing orphaned records"
210
+ category: schema
211
+ priority: critical
212
+
213
+ input:
214
+ code: |
215
+ CREATE TABLE users (
216
+ id SERIAL PRIMARY KEY,
217
+ email VARCHAR(255) UNIQUE NOT NULL
218
+ );
219
+
220
+ CREATE TABLE orders (
221
+ id SERIAL PRIMARY KEY,
222
+ user_id INTEGER, -- No FK constraint!
223
+ total DECIMAL(10,2)
224
+ );
225
+
226
+ -- This allows orphaned orders
227
+ INSERT INTO orders (user_id, total) VALUES (999, 100.00);
228
+ DELETE FROM users WHERE id = 1; -- Orphans all orders for user 1
229
+ context:
230
+ database: postgresql
231
+ framework: prisma
232
+
233
+ expected_output:
234
+ must_contain:
235
+ - "foreign key"
236
+ - "referential integrity"
237
+ - "orphaned"
238
+ must_match_regex:
239
+ - "DB-\\d{3}"
240
+ severity_classification: critical
241
+ finding_count:
242
+ min: 1
243
+
244
+ validation:
245
+ schema_check: true
246
+ keyword_match_threshold: 0.8
247
+
248
+ - id: tc003_missing_index_on_foreign_key
249
+ description: "Detect missing index on foreign key column affecting performance"
250
+ category: schema
251
+ priority: high
252
+
253
+ input:
254
+ code: |
255
+ CREATE TABLE orders (
256
+ id SERIAL PRIMARY KEY,
257
+ user_id INTEGER REFERENCES users(id),
258
+ product_id INTEGER REFERENCES products(id),
259
+ quantity INTEGER,
260
+ created_at TIMESTAMP
261
+ );
262
+
263
+ -- Frequently run queries without indexes
264
+ SELECT * FROM orders WHERE user_id = 123;
265
+ SELECT COUNT(*) FROM orders WHERE product_id = 456;
266
+ context:
267
+ database: postgresql
268
+ framework: typeorm
269
+
270
+ expected_output:
271
+ must_contain:
272
+ - "index"
273
+ - "foreign key"
274
+ - "performance"
275
+ - "user_id"
276
+ must_match_regex:
277
+ - "CREATE INDEX"
278
+ severity_classification: medium
279
+
280
+ validation:
281
+ schema_check: true
282
+ keyword_match_threshold: 0.7
283
+
284
+ # ---------------------------------------------------------------------------
285
+ # CATEGORY: Data Integrity
286
+ # ---------------------------------------------------------------------------
287
+
288
+ - id: tc004_check_constraint_violation
289
+ description: "Detect check constraint violations on status column"
290
+ category: integrity
291
+ priority: high
292
+
293
+ input:
294
+ code: |
295
+ CREATE TABLE orders (
296
+ id SERIAL PRIMARY KEY,
297
+ status VARCHAR(20) CHECK (status IN ('pending', 'processing', 'shipped', 'delivered')),
298
+ total DECIMAL(10,2) CHECK (total >= 0)
299
+ );
300
+
301
+ -- Test constraint enforcement
302
+ INSERT INTO orders (status, total) VALUES ('invalid_status', 100.00);
303
+ INSERT INTO orders (status, total) VALUES ('pending', -50.00);
304
+ context:
305
+ database: postgresql
306
+ test_type: integrity
307
+
308
+ expected_output:
309
+ must_contain:
310
+ - "check constraint"
311
+ - "violation"
312
+ - "status"
313
+ must_not_contain:
314
+ - "passed"
315
+ - "valid"
316
+ severity_classification: high
317
+
318
+ validation:
319
+ schema_check: true
320
+ keyword_match_threshold: 0.8
321
+
322
+ - id: tc005_null_constraint_test
323
+ description: "Detect NOT NULL constraint violations"
324
+ category: integrity
325
+ priority: high
326
+
327
+ input:
328
+ code: |
329
+ const user = new User();
330
+ user.email = 'test@example.com';
331
+ // Missing required field: password
332
+ await user.save();
333
+
334
+ // Schema:
335
+ // email VARCHAR(255) NOT NULL
336
+ // password VARCHAR(255) NOT NULL
337
+ // name VARCHAR(100) -- nullable
338
+ context:
339
+ database: mysql
340
+ framework: sequelize
341
+ test_type: integrity
342
+
343
+ expected_output:
344
+ must_contain:
345
+ - "NOT NULL"
346
+ - "constraint"
347
+ - "password"
348
+ - "required"
349
+ severity_classification: high
350
+ finding_count:
351
+ min: 1
352
+
353
+ validation:
354
+ schema_check: true
355
+ keyword_match_threshold: 0.8
356
+
357
+ # ---------------------------------------------------------------------------
358
+ # CATEGORY: Migration Testing
359
+ # ---------------------------------------------------------------------------
360
+
361
+ - id: tc006_migration_forward_test
362
+ description: "Test migration applies successfully forward"
363
+ category: migration
364
+ priority: critical
365
+
366
+ input:
367
+ code: |
368
+ // Migration: add-user-roles.js
369
+ exports.up = async function(knex) {
370
+ await knex.schema.createTable('roles', (table) => {
371
+ table.increments('id');
372
+ table.string('name').unique().notNullable();
373
+ });
374
+
375
+ await knex.schema.alterTable('users', (table) => {
376
+ table.integer('role_id').references('id').inTable('roles');
377
+ });
378
+ };
379
+
380
+ exports.down = async function(knex) {
381
+ await knex.schema.alterTable('users', (table) => {
382
+ table.dropColumn('role_id');
383
+ });
384
+ await knex.schema.dropTable('roles');
385
+ };
386
+ context:
387
+ database: postgresql
388
+ framework: knex
389
+ test_type: migration
390
+
391
+ expected_output:
392
+ must_contain:
393
+ - "migration"
394
+ - "forward"
395
+ - "roles"
396
+ - "users"
397
+ must_not_contain:
398
+ - "failed"
399
+ - "error"
400
+ must_match_regex:
401
+ - "up|forward|apply"
402
+
403
+ validation:
404
+ schema_check: true
405
+ keyword_match_threshold: 0.7
406
+
407
+ - id: tc007_migration_rollback_test
408
+ description: "Test migration rollback works correctly"
409
+ category: migration
410
+ priority: critical
411
+
412
+ input:
413
+ code: |
414
+ // Test migration rollback
415
+ await migrate.latest(); // Apply all migrations
416
+ await migrate.rollback(); // Rollback last batch
417
+
418
+ // Verify tables are removed
419
+ const tables = await knex.raw("SELECT table_name FROM information_schema.tables");
420
+ expect(tables).not.toContain('new_feature_table');
421
+ context:
422
+ database: postgresql
423
+ framework: knex
424
+ test_type: migration
425
+
426
+ expected_output:
427
+ must_contain:
428
+ - "rollback"
429
+ - "down"
430
+ - "reverse"
431
+ must_not_contain:
432
+ - "irreversible"
433
+ - "failed"
434
+ severity_classification: medium
435
+
436
+ validation:
437
+ schema_check: true
438
+ keyword_match_threshold: 0.7
439
+
440
+ - id: tc008_migration_data_preservation
441
+ description: "Test migration preserves existing data"
442
+ category: migration
443
+ priority: critical
444
+
445
+ input:
446
+ code: |
447
+ // Existing data before migration
448
+ await db.users.create({ email: 'existing@example.com', name: 'Existing User' });
449
+
450
+ // Migration adds new column
451
+ exports.up = async function(knex) {
452
+ await knex.schema.alterTable('users', (table) => {
453
+ table.string('phone').nullable();
454
+ });
455
+ };
456
+
457
+ // After migration, verify existing data is preserved
458
+ const user = await db.users.findOne({ email: 'existing@example.com' });
459
+ expect(user.name).toBe('Existing User'); // Data should be intact
460
+ expect(user.phone).toBeNull(); // New column should be null
461
+ context:
462
+ database: postgresql
463
+ framework: knex
464
+ test_type: migration
465
+
466
+ expected_output:
467
+ must_contain:
468
+ - "data preservation"
469
+ - "existing"
470
+ - "intact"
471
+ - "backward compatible"
472
+ must_not_contain:
473
+ - "data loss"
474
+ - "corrupted"
475
+ severity_classification: critical
476
+
477
+ validation:
478
+ schema_check: true
479
+ keyword_match_threshold: 0.7
480
+
481
+ # ---------------------------------------------------------------------------
482
+ # CATEGORY: Transaction Testing (ACID)
483
+ # ---------------------------------------------------------------------------
484
+
485
+ - id: tc009_atomicity_test
486
+ description: "Test transaction atomicity - all or nothing"
487
+ category: transaction
488
+ priority: critical
489
+
490
+ input:
491
+ code: |
492
+ test('transaction rolls back on error', async () => {
493
+ const initialCount = await db.users.count();
494
+
495
+ try {
496
+ await db.transaction(async (trx) => {
497
+ await trx('users').insert({ email: 'user1@example.com' });
498
+ await trx('users').insert({ email: 'user2@example.com' });
499
+ throw new Error('Intentional rollback');
500
+ });
501
+ } catch (error) { /* Expected */ }
502
+
503
+ // Count should be unchanged
504
+ expect(await db.users.count()).toBe(initialCount);
505
+ });
506
+ context:
507
+ database: postgresql
508
+ framework: knex
509
+ test_type: transaction
510
+
511
+ expected_output:
512
+ must_contain:
513
+ - "atomicity"
514
+ - "rollback"
515
+ - "transaction"
516
+ - "all or nothing"
517
+ must_not_contain:
518
+ - "partial commit"
519
+ must_match_regex:
520
+ - "ACID|atomic"
521
+ severity_classification: critical
522
+
523
+ validation:
524
+ schema_check: true
525
+ keyword_match_threshold: 0.8
526
+ reasoning_quality_min: 0.8
527
+
528
+ - id: tc010_isolation_test
529
+ description: "Test transaction isolation - concurrent access"
530
+ category: transaction
531
+ priority: critical
532
+
533
+ input:
534
+ code: |
535
+ test('concurrent transactions isolated', async () => {
536
+ const user = await db.users.create({ balance: 100 });
537
+
538
+ // Two concurrent withdrawals - race condition test
539
+ const results = await Promise.all([
540
+ db.transaction(async (trx) => {
541
+ const current = await trx('users').where({ id: user.id }).first();
542
+ if (current.balance >= 50) {
543
+ await trx('users').where({ id: user.id }).update({
544
+ balance: current.balance - 50
545
+ });
546
+ return 'success';
547
+ }
548
+ return 'insufficient';
549
+ }),
550
+ db.transaction(async (trx) => {
551
+ const current = await trx('users').where({ id: user.id }).first();
552
+ if (current.balance >= 50) {
553
+ await trx('users').where({ id: user.id }).update({
554
+ balance: current.balance - 50
555
+ });
556
+ return 'success';
557
+ }
558
+ return 'insufficient';
559
+ })
560
+ ]);
561
+
562
+ const final = await db.users.findOne({ id: user.id });
563
+ // With proper isolation, one should fail or balance should be 0
564
+ expect(final.balance).toBeGreaterThanOrEqual(0);
565
+ });
566
+ context:
567
+ database: postgresql
568
+ framework: knex
569
+ test_type: transaction
570
+ isolation_level: REPEATABLE_READ
571
+
572
+ expected_output:
573
+ must_contain:
574
+ - "isolation"
575
+ - "concurrent"
576
+ - "race condition"
577
+ - "lock"
578
+ must_match_regex:
579
+ - "READ_COMMITTED|REPEATABLE_READ|SERIALIZABLE"
580
+ severity_classification: critical
581
+
582
+ validation:
583
+ schema_check: true
584
+ keyword_match_threshold: 0.7
585
+
586
+ - id: tc011_deadlock_detection
587
+ description: "Test deadlock detection and resolution"
588
+ category: transaction
589
+ priority: high
590
+
591
+ input:
592
+ code: |
593
+ test('deadlock is detected and handled', async () => {
594
+ // Create potential deadlock scenario
595
+ const tx1 = db.transaction(async (trx) => {
596
+ await trx('accounts').where({ id: 1 }).update({ balance: 100 });
597
+ await delay(100);
598
+ await trx('accounts').where({ id: 2 }).update({ balance: 200 });
599
+ });
600
+
601
+ const tx2 = db.transaction(async (trx) => {
602
+ await trx('accounts').where({ id: 2 }).update({ balance: 150 });
603
+ await delay(100);
604
+ await trx('accounts').where({ id: 1 }).update({ balance: 250 });
605
+ });
606
+
607
+ // One should fail with deadlock, other should succeed
608
+ const results = await Promise.allSettled([tx1, tx2]);
609
+ const failures = results.filter(r => r.status === 'rejected');
610
+ expect(failures.some(f => f.reason.message.includes('deadlock'))).toBe(true);
611
+ });
612
+ context:
613
+ database: postgresql
614
+ framework: knex
615
+ test_type: transaction
616
+
617
+ expected_output:
618
+ must_contain:
619
+ - "deadlock"
620
+ - "detection"
621
+ - "retry"
622
+ - "conflict"
623
+ severity_classification: high
624
+
625
+ validation:
626
+ schema_check: true
627
+ keyword_match_threshold: 0.7
628
+
629
+ # ---------------------------------------------------------------------------
630
+ # CATEGORY: Performance Testing
631
+ # ---------------------------------------------------------------------------
632
+
633
+ - id: tc012_slow_query_detection
634
+ description: "Detect slow queries without proper indexing"
635
+ category: performance
636
+ priority: high
637
+
638
+ input:
639
+ code: |
640
+ -- Slow query: full table scan on large table
641
+ SELECT * FROM orders
642
+ WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31'
643
+ AND status = 'pending';
644
+
645
+ -- Table has 10 million rows, no index on created_at or status
646
+ EXPLAIN ANALYZE SELECT * FROM orders
647
+ WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31';
648
+ -- Output: Seq Scan on orders (cost=0.00..250000.00 rows=5000000)
649
+ context:
650
+ database: postgresql
651
+ test_type: performance
652
+ table_size: 10000000
653
+
654
+ expected_output:
655
+ must_contain:
656
+ - "slow query"
657
+ - "full table scan"
658
+ - "index"
659
+ - "Seq Scan"
660
+ must_match_regex:
661
+ - "CREATE INDEX"
662
+ severity_classification: high
663
+ finding_count:
664
+ min: 1
665
+
666
+ validation:
667
+ schema_check: true
668
+ keyword_match_threshold: 0.8
669
+
670
+ - id: tc013_n_plus_one_detection
671
+ description: "Detect N+1 query problem"
672
+ category: performance
673
+ priority: high
674
+
675
+ input:
676
+ code: |
677
+ // N+1 query problem
678
+ const users = await User.findAll(); // 1 query
679
+
680
+ for (const user of users) {
681
+ const orders = await Order.findAll({
682
+ where: { userId: user.id }
683
+ }); // N queries!
684
+ user.orderCount = orders.length;
685
+ }
686
+
687
+ // Should be:
688
+ // const users = await User.findAll({
689
+ // include: [{ model: Order }]
690
+ // });
691
+ context:
692
+ database: postgresql
693
+ framework: sequelize
694
+ test_type: performance
695
+
696
+ expected_output:
697
+ must_contain:
698
+ - "N+1"
699
+ - "query"
700
+ - "include"
701
+ - "eager loading"
702
+ - "join"
703
+ must_not_contain:
704
+ - "efficient"
705
+ - "optimal"
706
+ severity_classification: high
707
+
708
+ validation:
709
+ schema_check: true
710
+ keyword_match_threshold: 0.8
711
+
712
+ - id: tc014_connection_pool_exhaustion
713
+ description: "Detect connection pool exhaustion risk"
714
+ category: performance
715
+ priority: critical
716
+
717
+ input:
718
+ code: |
719
+ // Connection leak - connections not released
720
+ async function getUserData(userId) {
721
+ const connection = await pool.getConnection();
722
+ const user = await connection.query('SELECT * FROM users WHERE id = ?', [userId]);
723
+ // Missing: connection.release()
724
+ return user;
725
+ }
726
+
727
+ // Called in a loop without releasing connections
728
+ for (let i = 0; i < 1000; i++) {
729
+ await getUserData(i);
730
+ }
731
+ context:
732
+ database: mysql
733
+ framework: mysql2
734
+ test_type: performance
735
+ pool_size: 10
736
+
737
+ expected_output:
738
+ must_contain:
739
+ - "connection"
740
+ - "pool"
741
+ - "leak"
742
+ - "release"
743
+ - "exhaustion"
744
+ severity_classification: critical
745
+
746
+ validation:
747
+ schema_check: true
748
+ keyword_match_threshold: 0.8
749
+
750
+ # ---------------------------------------------------------------------------
751
+ # CATEGORY: Multi-Database Support
752
+ # ---------------------------------------------------------------------------
753
+
754
+ - id: tc015_mongodb_schema_validation
755
+ description: "Test MongoDB schema validation"
756
+ category: schema
757
+ priority: high
758
+
759
+ input:
760
+ code: |
761
+ // MongoDB collection without schema validation
762
+ db.createCollection("users");
763
+
764
+ // Can insert inconsistent documents
765
+ db.users.insertOne({ email: "user@example.com", age: 25 });
766
+ db.users.insertOne({ email: 123, age: "twenty" }); // Wrong types!
767
+ db.users.insertOne({ name: "John" }); // Missing email
768
+
769
+ // Should have JSON Schema validation:
770
+ db.createCollection("users", {
771
+ validator: {
772
+ $jsonSchema: {
773
+ bsonType: "object",
774
+ required: ["email"],
775
+ properties: {
776
+ email: { bsonType: "string" },
777
+ age: { bsonType: "int" }
778
+ }
779
+ }
780
+ }
781
+ });
782
+ context:
783
+ database: mongodb
784
+ test_type: schema
785
+
786
+ expected_output:
787
+ must_contain:
788
+ - "schema validation"
789
+ - "MongoDB"
790
+ - "jsonSchema"
791
+ - "bsonType"
792
+ severity_classification: high
793
+
794
+ validation:
795
+ schema_check: true
796
+ keyword_match_threshold: 0.7
797
+
798
+ - id: tc016_sqlite_foreign_key_enforcement
799
+ description: "Test SQLite foreign key enforcement"
800
+ category: integrity
801
+ priority: high
802
+
803
+ input:
804
+ code: |
805
+ -- SQLite: Foreign keys are OFF by default!
806
+ CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);
807
+ CREATE TABLE orders (
808
+ id INTEGER PRIMARY KEY,
809
+ user_id INTEGER REFERENCES users(id)
810
+ );
811
+
812
+ -- This succeeds even though user 999 doesn't exist
813
+ INSERT INTO orders (user_id) VALUES (999);
814
+
815
+ -- Need to enable: PRAGMA foreign_keys = ON;
816
+ context:
817
+ database: sqlite
818
+ test_type: integrity
819
+
820
+ expected_output:
821
+ must_contain:
822
+ - "foreign_keys"
823
+ - "PRAGMA"
824
+ - "SQLite"
825
+ - "enforcement"
826
+ must_match_regex:
827
+ - "PRAGMA foreign_keys\\s*=\\s*ON"
828
+ severity_classification: high
829
+
830
+ validation:
831
+ schema_check: true
832
+ keyword_match_threshold: 0.8
833
+
834
+ # ---------------------------------------------------------------------------
835
+ # CATEGORY: Negative Tests (Secure/Valid Code)
836
+ # ---------------------------------------------------------------------------
837
+
838
+ - id: tc017_valid_schema_no_false_positives
839
+ description: "Verify well-designed schema is NOT flagged as problematic"
840
+ category: negative
841
+ priority: critical
842
+
843
+ input:
844
+ code: |
845
+ CREATE TABLE users (
846
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
847
+ email VARCHAR(255) UNIQUE NOT NULL,
848
+ password_hash VARCHAR(255) NOT NULL,
849
+ created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
850
+ updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
851
+ );
852
+
853
+ CREATE TABLE orders (
854
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
855
+ user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
856
+ total DECIMAL(10,2) NOT NULL CHECK (total >= 0),
857
+ status VARCHAR(20) NOT NULL DEFAULT 'pending'
858
+ CHECK (status IN ('pending', 'processing', 'shipped', 'delivered', 'cancelled')),
859
+ created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
860
+ );
861
+
862
+ CREATE INDEX idx_orders_user_id ON orders(user_id);
863
+ CREATE INDEX idx_orders_status ON orders(status);
864
+ CREATE INDEX idx_orders_created_at ON orders(created_at);
865
+ context:
866
+ database: postgresql
867
+ framework: prisma
868
+ environment: production
869
+
870
+ expected_output:
871
+ must_contain:
872
+ - "valid"
873
+ - "well-designed"
874
+ - "proper constraints"
875
+ must_not_contain:
876
+ - "missing constraint"
877
+ - "critical"
878
+ - "high severity"
879
+ - "orphaned"
880
+ finding_count:
881
+ max: 2 # Allow informational findings only
882
+
883
+ validation:
884
+ schema_check: true
885
+ keyword_match_threshold: 0.6
886
+ allow_partial: true
887
+
888
+ - id: tc018_proper_migration_pattern
889
+ description: "Verify proper migration pattern is recognized"
890
+ category: negative
891
+ priority: high
892
+
893
+ input:
894
+ code: |
895
+ // Proper reversible migration with data preservation
896
+ exports.up = async function(knex) {
897
+ // Check if column exists before adding
898
+ const hasColumn = await knex.schema.hasColumn('users', 'phone');
899
+ if (!hasColumn) {
900
+ await knex.schema.alterTable('users', (table) => {
901
+ table.string('phone').nullable();
902
+ });
903
+ }
904
+ };
905
+
906
+ exports.down = async function(knex) {
907
+ const hasColumn = await knex.schema.hasColumn('users', 'phone');
908
+ if (hasColumn) {
909
+ await knex.schema.alterTable('users', (table) => {
910
+ table.dropColumn('phone');
911
+ });
912
+ }
913
+ };
914
+ context:
915
+ database: postgresql
916
+ framework: knex
917
+ test_type: migration
918
+
919
+ expected_output:
920
+ must_contain:
921
+ - "reversible"
922
+ - "idempotent"
923
+ - "safe"
924
+ must_not_contain:
925
+ - "irreversible"
926
+ - "data loss"
927
+ - "critical"
928
+ severity_classification: info
929
+
930
+ validation:
931
+ schema_check: true
932
+ allow_partial: true
933
+
934
+ # =============================================================================
935
+ # SUCCESS CRITERIA
936
+ # =============================================================================
937
+
938
+ success_criteria:
939
+ # Overall pass rate (90% of tests must pass)
940
+ pass_rate: 0.9
941
+
942
+ # Critical tests must ALL pass (100%)
943
+ critical_pass_rate: 1.0
944
+
945
+ # Average reasoning quality score
946
+ avg_reasoning_quality: 0.75
947
+
948
+ # Maximum suite execution time (5 minutes)
949
+ max_execution_time_ms: 300000
950
+
951
+ # Maximum variance between model results (15%)
952
+ cross_model_variance: 0.15
953
+
954
+ # =============================================================================
955
+ # METADATA
956
+ # =============================================================================
957
+
958
+ metadata:
959
+ author: "qe-test-data-architect"
960
+ created: "2026-02-02"
961
+ last_updated: "2026-02-02"
962
+ coverage_target: >
963
+ Database testing coverage: Schema validation (missing constraints, indexes),
964
+ Data integrity (unique, FK, check, NOT NULL), Migration testing (forward,
965
+ rollback, data preservation), Transaction testing (ACID: atomicity, isolation,
966
+ deadlock), Performance (slow queries, N+1, connection pools). Multi-database
967
+ support: PostgreSQL, MySQL, MongoDB, SQLite. 18 test cases with 90% pass rate
968
+ requirement and 100% critical pass rate for ACID and migration tests.