agentic-qe 3.7.9 → 3.7.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (401) hide show
  1. package/.claude/skills/.validation/README.md +111 -111
  2. package/.claude/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
  3. package/.claude/skills/.validation/examples/performance-testing-output.example.json +252 -252
  4. package/.claude/skills/.validation/examples/security-testing-output.example.json +413 -413
  5. package/.claude/skills/.validation/examples/testability-scoring-output.example.json +350 -350
  6. package/.claude/skills/.validation/schemas/skill-eval.schema.json +462 -462
  7. package/.claude/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
  8. package/.claude/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
  9. package/.claude/skills/.validation/schemas/skill-output.template.json +610 -610
  10. package/.claude/skills/.validation/skill-validation-mcp-integration.md +250 -250
  11. package/.claude/skills/.validation/templates/eval.template.yaml +366 -366
  12. package/.claude/skills/.validation/templates/schemas/output.json +145 -145
  13. package/.claude/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
  14. package/.claude/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
  15. package/.claude/skills/.validation/test-data/invalid-output.json +5 -5
  16. package/.claude/skills/.validation/test-data/minimal-output.json +9 -9
  17. package/.claude/skills/.validation/test-data/sample-output.json +73 -73
  18. package/.claude/skills/a11y-ally/evals/a11y-ally.yaml +376 -376
  19. package/.claude/skills/a11y-ally/schemas/output.json +549 -549
  20. package/.claude/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
  21. package/.claude/skills/accessibility-testing/schemas/output.json +776 -776
  22. package/.claude/skills/accessibility-testing/test-data/sample-output.json +191 -191
  23. package/.claude/skills/agentic-quality-engineering/schemas/output.json +577 -577
  24. package/.claude/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
  25. package/.claude/skills/api-testing-patterns/schemas/output.json +845 -845
  26. package/.claude/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
  27. package/.claude/skills/brutal-honesty-review/schemas/output.json +291 -291
  28. package/.claude/skills/bug-reporting-excellence/schemas/output.json +288 -288
  29. package/.claude/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
  30. package/.claude/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
  31. package/.claude/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -157
  32. package/.claude/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -542
  33. package/.claude/skills/code-review-quality/schemas/output.json +264 -264
  34. package/.claude/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
  35. package/.claude/skills/compatibility-testing/schemas/output.json +551 -551
  36. package/.claude/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
  37. package/.claude/skills/compliance-testing/schemas/output.json +845 -845
  38. package/.claude/skills/consultancy-practices/schemas/output.json +282 -282
  39. package/.claude/skills/contract-testing/evals/contract-testing.yaml +748 -748
  40. package/.claude/skills/contract-testing/schemas/output.json +638 -638
  41. package/.claude/skills/database-testing/evals/database-testing.yaml +968 -968
  42. package/.claude/skills/database-testing/schemas/output.json +1446 -1446
  43. package/.claude/skills/debug-loop/SKILL.md +61 -61
  44. package/.claude/skills/enterprise-integration-testing/SKILL.md +735 -735
  45. package/.claude/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -158
  46. package/.claude/skills/enterprise-integration-testing/schemas/output.json +74 -74
  47. package/.claude/skills/enterprise-integration-testing/scripts/validate-config.json +25 -25
  48. package/.claude/skills/iterative-loop/SKILL.md +371 -371
  49. package/.claude/skills/localization-testing/evals/localization-testing.yaml +544 -544
  50. package/.claude/skills/localization-testing/schemas/output.json +325 -325
  51. package/.claude/skills/middleware-testing-patterns/SKILL.md +798 -798
  52. package/.claude/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -153
  53. package/.claude/skills/middleware-testing-patterns/schemas/output.json +58 -58
  54. package/.claude/skills/middleware-testing-patterns/scripts/validate-config.json +25 -25
  55. package/.claude/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
  56. package/.claude/skills/mobile-testing/schemas/output.json +318 -318
  57. package/.claude/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
  58. package/.claude/skills/mutation-testing/schemas/output.json +707 -707
  59. package/.claude/skills/mutation-testing/test-data/sample-output.json +295 -295
  60. package/.claude/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -450
  61. package/.claude/skills/n8n-expression-testing/schemas/output.json +369 -369
  62. package/.claude/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -522
  63. package/.claude/skills/n8n-integration-testing-patterns/schemas/output.json +291 -291
  64. package/.claude/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -493
  65. package/.claude/skills/n8n-security-testing/schemas/output.json +293 -293
  66. package/.claude/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -500
  67. package/.claude/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -295
  68. package/.claude/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -497
  69. package/.claude/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -254
  70. package/.claude/skills/observability-testing-patterns/SKILL.md +930 -930
  71. package/.claude/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -157
  72. package/.claude/skills/observability-testing-patterns/schemas/output.json +58 -58
  73. package/.claude/skills/observability-testing-patterns/scripts/validate-config.json +25 -25
  74. package/.claude/skills/pentest-validation/SKILL.md +268 -268
  75. package/.claude/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
  76. package/.claude/skills/pentest-validation/schemas/output.json +281 -281
  77. package/.claude/skills/performance-analysis/evals/performance-analysis.yaml +144 -144
  78. package/.claude/skills/performance-analysis/schemas/output.json +588 -588
  79. package/.claude/skills/performance-testing/evals/performance-testing.yaml +772 -772
  80. package/.claude/skills/performance-testing/schemas/output.json +1184 -1184
  81. package/.claude/skills/pr-review/SKILL.md +61 -61
  82. package/.claude/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
  83. package/.claude/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -211
  84. package/.claude/skills/qcsd-cicd-swarm/schemas/output.json +86 -86
  85. package/.claude/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -30
  86. package/.claude/skills/qcsd-development-swarm/SKILL.md +2154 -2154
  87. package/.claude/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -162
  88. package/.claude/skills/qcsd-development-swarm/schemas/output.json +72 -72
  89. package/.claude/skills/qcsd-development-swarm/scripts/validate-config.json +25 -25
  90. package/.claude/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -138
  91. package/.claude/skills/qcsd-ideation-swarm/schemas/output.json +568 -568
  92. package/.claude/skills/qcsd-production-swarm/SKILL.md +2781 -2781
  93. package/.claude/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -246
  94. package/.claude/skills/qcsd-production-swarm/schemas/output.json +505 -505
  95. package/.claude/skills/qcsd-production-swarm/scripts/validate-config.json +25 -25
  96. package/.claude/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
  97. package/.claude/skills/qe-chaos-resilience/schemas/output.json +314 -314
  98. package/.claude/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
  99. package/.claude/skills/qe-code-intelligence/schemas/output.json +315 -315
  100. package/.claude/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
  101. package/.claude/skills/qe-contract-testing/schemas/output.json +295 -295
  102. package/.claude/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
  103. package/.claude/skills/qe-coverage-analysis/schemas/output.json +286 -286
  104. package/.claude/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
  105. package/.claude/skills/qe-defect-intelligence/schemas/output.json +283 -283
  106. package/.claude/skills/qe-iterative-loop/schemas/output.json +264 -264
  107. package/.claude/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
  108. package/.claude/skills/qe-learning-optimization/schemas/output.json +288 -288
  109. package/.claude/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
  110. package/.claude/skills/qe-quality-assessment/schemas/output.json +550 -550
  111. package/.claude/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
  112. package/.claude/skills/qe-requirements-validation/schemas/output.json +587 -587
  113. package/.claude/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
  114. package/.claude/skills/qe-security-compliance/schemas/output.json +498 -498
  115. package/.claude/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
  116. package/.claude/skills/qe-test-execution/schemas/output.json +529 -529
  117. package/.claude/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
  118. package/.claude/skills/qe-test-generation/schemas/output.json +439 -439
  119. package/.claude/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
  120. package/.claude/skills/qe-visual-accessibility/schemas/output.json +491 -491
  121. package/.claude/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
  122. package/.claude/skills/quality-metrics/schemas/output.json +403 -403
  123. package/.claude/skills/refactoring-patterns/schemas/output.json +475 -475
  124. package/.claude/skills/regression-testing/evals/regression-testing.yaml +504 -504
  125. package/.claude/skills/regression-testing/schemas/output.json +311 -311
  126. package/.claude/skills/release/SKILL.md +347 -347
  127. package/.claude/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
  128. package/.claude/skills/risk-based-testing/schemas/output.json +480 -480
  129. package/.claude/skills/security-testing/evals/security-testing.yaml +789 -789
  130. package/.claude/skills/security-testing/schemas/output.json +879 -879
  131. package/.claude/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
  132. package/.claude/skills/security-visual-testing/schemas/output.json +486 -486
  133. package/.claude/skills/sfdipot-product-factors/SKILL.md +239 -239
  134. package/.claude/skills/sherlock-review/schemas/output.json +297 -297
  135. package/.claude/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
  136. package/.claude/skills/shift-left-testing/schemas/output.json +459 -459
  137. package/.claude/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
  138. package/.claude/skills/shift-right-testing/schemas/output.json +418 -418
  139. package/.claude/skills/skills-manifest.json +1 -1
  140. package/.claude/skills/tdd-london-chicago/schemas/output.json +444 -444
  141. package/.claude/skills/technical-writing/schemas/output.json +268 -268
  142. package/.claude/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
  143. package/.claude/skills/test-automation-strategy/schemas/output.json +444 -444
  144. package/.claude/skills/test-data-management/evals/test-data-management.yaml +504 -504
  145. package/.claude/skills/test-data-management/schemas/output.json +284 -284
  146. package/.claude/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
  147. package/.claude/skills/test-design-techniques/schemas/output.json +295 -295
  148. package/.claude/skills/test-environment-management/schemas/output.json +310 -310
  149. package/.claude/skills/test-idea-rewriting/SKILL.md +229 -229
  150. package/.claude/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
  151. package/.claude/skills/test-reporting-analytics/schemas/output.json +329 -329
  152. package/.claude/skills/testability-scoring/evals/testability-scoring.yaml +814 -814
  153. package/.claude/skills/testability-scoring/resources/templates/config.template.js +84 -84
  154. package/.claude/skills/testability-scoring/schemas/output.json +606 -606
  155. package/.claude/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
  156. package/.claude/skills/verification-quality/evals/verification-quality.yaml +150 -150
  157. package/.claude/skills/verification-quality/schemas/output.json +432 -432
  158. package/.claude/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
  159. package/.claude/skills/visual-testing-advanced/schemas/output.json +294 -294
  160. package/.claude/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -165
  161. package/.claude/skills/wms-testing-patterns/schemas/output.json +150 -150
  162. package/.claude/skills/wms-testing-patterns/scripts/validate-config.json +51 -51
  163. package/CHANGELOG.md +36 -0
  164. package/README.md +171 -901
  165. package/assets/agents/v3/helpers/quality-criteria/evidence-classification.md +116 -116
  166. package/assets/agents/v3/helpers/quality-criteria/htsm-categories.md +139 -139
  167. package/assets/governance/constitution.md +202 -202
  168. package/assets/governance/shards/chaos-resilience.shard.md +221 -221
  169. package/assets/governance/shards/code-intelligence.shard.md +178 -178
  170. package/assets/governance/shards/contract-testing.shard.md +206 -206
  171. package/assets/governance/shards/coverage-analysis.shard.md +146 -146
  172. package/assets/governance/shards/defect-intelligence.shard.md +182 -182
  173. package/assets/governance/shards/learning-optimization.shard.md +248 -248
  174. package/assets/governance/shards/quality-assessment.shard.md +165 -165
  175. package/assets/governance/shards/requirements-validation.shard.md +177 -177
  176. package/assets/governance/shards/security-compliance.shard.md +196 -196
  177. package/assets/governance/shards/test-execution.shard.md +156 -156
  178. package/assets/governance/shards/test-generation.shard.md +128 -128
  179. package/assets/governance/shards/visual-accessibility.shard.md +209 -209
  180. package/assets/hooks/cross-phase-memory.yaml +253 -253
  181. package/assets/patterns/adr-051-booster-patterns.json +78 -78
  182. package/assets/patterns/adr-051-embedding-patterns.json +147 -147
  183. package/assets/patterns/adr-051-integration-summary.json +62 -62
  184. package/assets/patterns/adr-051-reasoning-patterns.json +166 -166
  185. package/assets/patterns/adr-051-router-patterns.json +113 -113
  186. package/assets/patterns/index.json +136 -136
  187. package/assets/skills/.validation/README.md +111 -111
  188. package/assets/skills/.validation/examples/chaos-engineering-output.example.json +530 -530
  189. package/assets/skills/.validation/examples/performance-testing-output.example.json +252 -252
  190. package/assets/skills/.validation/examples/security-testing-output.example.json +413 -413
  191. package/assets/skills/.validation/examples/testability-scoring-output.example.json +350 -350
  192. package/assets/skills/.validation/schemas/skill-eval.schema.json +462 -462
  193. package/assets/skills/.validation/schemas/skill-frontmatter.schema.json +341 -341
  194. package/assets/skills/.validation/schemas/skill-output-meta.schema.json +199 -199
  195. package/assets/skills/.validation/schemas/skill-output.template.json +610 -610
  196. package/assets/skills/.validation/skill-validation-mcp-integration.md +250 -250
  197. package/assets/skills/.validation/templates/eval.template.yaml +366 -366
  198. package/assets/skills/.validation/templates/schemas/output.json +145 -145
  199. package/assets/skills/.validation/templates/security-testing-eval.template.yaml +725 -725
  200. package/assets/skills/.validation/templates/skill-frontmatter.example.yaml +225 -225
  201. package/assets/skills/.validation/test-data/invalid-output.json +5 -5
  202. package/assets/skills/.validation/test-data/minimal-output.json +9 -9
  203. package/assets/skills/.validation/test-data/sample-output.json +73 -73
  204. package/assets/skills/a11y-ally/evals/a11y-ally.yaml +376 -376
  205. package/assets/skills/a11y-ally/schemas/output.json +549 -549
  206. package/assets/skills/accessibility-testing/evals/accessibility-testing.yaml +719 -719
  207. package/assets/skills/accessibility-testing/schemas/output.json +776 -776
  208. package/assets/skills/accessibility-testing/test-data/sample-output.json +191 -191
  209. package/assets/skills/agentic-quality-engineering/schemas/output.json +577 -577
  210. package/assets/skills/api-testing-patterns/evals/api-testing-patterns.yaml +696 -696
  211. package/assets/skills/api-testing-patterns/schemas/output.json +845 -845
  212. package/assets/skills/aqe-v2-v3-migration/schemas/output.json +513 -513
  213. package/assets/skills/brutal-honesty-review/schemas/output.json +291 -291
  214. package/assets/skills/bug-reporting-excellence/schemas/output.json +288 -288
  215. package/assets/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml +761 -761
  216. package/assets/skills/chaos-engineering-resilience/schemas/output.json +1205 -1205
  217. package/assets/skills/cicd-pipeline-qe-orchestrator/evals/cicd-pipeline-qe-orchestrator.yaml +157 -157
  218. package/assets/skills/cicd-pipeline-qe-orchestrator/schemas/output.json +542 -542
  219. package/assets/skills/code-review-quality/schemas/output.json +264 -264
  220. package/assets/skills/compatibility-testing/evals/compatibility-testing.yaml +410 -410
  221. package/assets/skills/compatibility-testing/schemas/output.json +551 -551
  222. package/assets/skills/compliance-testing/evals/compliance-testing.yaml +1107 -1107
  223. package/assets/skills/compliance-testing/schemas/output.json +845 -845
  224. package/assets/skills/consultancy-practices/schemas/output.json +282 -282
  225. package/assets/skills/contract-testing/evals/contract-testing.yaml +748 -748
  226. package/assets/skills/contract-testing/schemas/output.json +638 -638
  227. package/assets/skills/database-testing/evals/database-testing.yaml +968 -968
  228. package/assets/skills/database-testing/schemas/output.json +1446 -1446
  229. package/assets/skills/debug-loop/SKILL.md +61 -61
  230. package/assets/skills/enterprise-integration-testing/SKILL.md +735 -735
  231. package/assets/skills/enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -158
  232. package/assets/skills/enterprise-integration-testing/schemas/output.json +74 -74
  233. package/assets/skills/enterprise-integration-testing/scripts/validate-config.json +25 -25
  234. package/assets/skills/localization-testing/evals/localization-testing.yaml +544 -544
  235. package/assets/skills/localization-testing/schemas/output.json +325 -325
  236. package/assets/skills/middleware-testing-patterns/SKILL.md +798 -798
  237. package/assets/skills/middleware-testing-patterns/evals/middleware-testing-patterns.yaml +153 -153
  238. package/assets/skills/middleware-testing-patterns/schemas/output.json +58 -58
  239. package/assets/skills/middleware-testing-patterns/scripts/validate-config.json +25 -25
  240. package/assets/skills/mobile-testing/evals/mobile-testing.yaml +537 -537
  241. package/assets/skills/mobile-testing/schemas/output.json +318 -318
  242. package/assets/skills/mutation-testing/evals/mutation-testing.yaml +652 -652
  243. package/assets/skills/mutation-testing/schemas/output.json +707 -707
  244. package/assets/skills/mutation-testing/test-data/sample-output.json +295 -295
  245. package/assets/skills/n8n-expression-testing/evals/n8n-expression-testing.yaml +450 -450
  246. package/assets/skills/n8n-expression-testing/schemas/output.json +369 -369
  247. package/assets/skills/n8n-integration-testing-patterns/evals/n8n-integration-testing-patterns.yaml +522 -522
  248. package/assets/skills/n8n-integration-testing-patterns/schemas/output.json +291 -291
  249. package/assets/skills/n8n-security-testing/evals/n8n-security-testing.yaml +493 -493
  250. package/assets/skills/n8n-security-testing/schemas/output.json +293 -293
  251. package/assets/skills/n8n-trigger-testing-strategies/evals/n8n-trigger-testing-strategies.yaml +500 -500
  252. package/assets/skills/n8n-trigger-testing-strategies/schemas/output.json +295 -295
  253. package/assets/skills/n8n-workflow-testing-fundamentals/evals/n8n-workflow-testing-fundamentals.yaml +497 -497
  254. package/assets/skills/n8n-workflow-testing-fundamentals/schemas/output.json +254 -254
  255. package/assets/skills/observability-testing-patterns/SKILL.md +930 -930
  256. package/assets/skills/observability-testing-patterns/evals/observability-testing-patterns.yaml +157 -157
  257. package/assets/skills/observability-testing-patterns/schemas/output.json +58 -58
  258. package/assets/skills/observability-testing-patterns/scripts/validate-config.json +25 -25
  259. package/assets/skills/pentest-validation/SKILL.md +268 -268
  260. package/assets/skills/pentest-validation/evals/pentest-validation.yaml +708 -708
  261. package/assets/skills/pentest-validation/schemas/output.json +281 -281
  262. package/assets/skills/performance-testing/evals/performance-testing.yaml +772 -772
  263. package/assets/skills/performance-testing/schemas/output.json +1184 -1184
  264. package/assets/skills/pr-review/SKILL.md +61 -61
  265. package/assets/skills/qcsd-cicd-swarm/SKILL.md +2206 -2206
  266. package/assets/skills/qcsd-cicd-swarm/evals/qcsd-cicd-swarm.yaml +211 -211
  267. package/assets/skills/qcsd-cicd-swarm/schemas/output.json +86 -86
  268. package/assets/skills/qcsd-cicd-swarm/scripts/validate-config.json +30 -30
  269. package/assets/skills/qcsd-development-swarm/SKILL.md +2154 -2154
  270. package/assets/skills/qcsd-development-swarm/evals/qcsd-development-swarm.yaml +162 -162
  271. package/assets/skills/qcsd-development-swarm/schemas/output.json +72 -72
  272. package/assets/skills/qcsd-development-swarm/scripts/validate-config.json +25 -25
  273. package/assets/skills/qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -138
  274. package/assets/skills/qcsd-ideation-swarm/schemas/output.json +568 -568
  275. package/assets/skills/qcsd-production-swarm/SKILL.md +2781 -2781
  276. package/assets/skills/qcsd-production-swarm/evals/qcsd-production-swarm.yaml +246 -246
  277. package/assets/skills/qcsd-production-swarm/schemas/output.json +505 -505
  278. package/assets/skills/qcsd-production-swarm/scripts/validate-config.json +25 -25
  279. package/assets/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml +443 -443
  280. package/assets/skills/qe-chaos-resilience/schemas/output.json +314 -314
  281. package/assets/skills/qe-code-intelligence/evals/qe-code-intelligence.yaml +459 -459
  282. package/assets/skills/qe-code-intelligence/schemas/output.json +315 -315
  283. package/assets/skills/qe-contract-testing/evals/qe-contract-testing.yaml +513 -513
  284. package/assets/skills/qe-contract-testing/schemas/output.json +295 -295
  285. package/assets/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -494
  286. package/assets/skills/qe-coverage-analysis/schemas/output.json +286 -286
  287. package/assets/skills/qe-defect-intelligence/evals/qe-defect-intelligence.yaml +511 -511
  288. package/assets/skills/qe-defect-intelligence/schemas/output.json +283 -283
  289. package/assets/skills/qe-iterative-loop/schemas/output.json +264 -264
  290. package/assets/skills/qe-learning-optimization/evals/qe-learning-optimization.yaml +144 -144
  291. package/assets/skills/qe-learning-optimization/schemas/output.json +288 -288
  292. package/assets/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml +506 -506
  293. package/assets/skills/qe-quality-assessment/schemas/output.json +550 -550
  294. package/assets/skills/qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -598
  295. package/assets/skills/qe-requirements-validation/schemas/output.json +587 -587
  296. package/assets/skills/qe-security-compliance/evals/qe-security-compliance.yaml +595 -595
  297. package/assets/skills/qe-security-compliance/schemas/output.json +498 -498
  298. package/assets/skills/qe-test-execution/evals/qe-test-execution.yaml +607 -607
  299. package/assets/skills/qe-test-execution/schemas/output.json +529 -529
  300. package/assets/skills/qe-test-generation/evals/qe-test-generation.yaml +148 -148
  301. package/assets/skills/qe-test-generation/schemas/output.json +439 -439
  302. package/assets/skills/qe-visual-accessibility/evals/qe-visual-accessibility.yaml +142 -142
  303. package/assets/skills/qe-visual-accessibility/schemas/output.json +491 -491
  304. package/assets/skills/quality-metrics/evals/quality-metrics.yaml +494 -494
  305. package/assets/skills/quality-metrics/schemas/output.json +403 -403
  306. package/assets/skills/refactoring-patterns/schemas/output.json +475 -475
  307. package/assets/skills/regression-testing/evals/regression-testing.yaml +504 -504
  308. package/assets/skills/regression-testing/schemas/output.json +311 -311
  309. package/assets/skills/risk-based-testing/evals/risk-based-testing.yaml +141 -141
  310. package/assets/skills/risk-based-testing/schemas/output.json +480 -480
  311. package/assets/skills/security-testing/evals/security-testing.yaml +789 -789
  312. package/assets/skills/security-testing/schemas/output.json +879 -879
  313. package/assets/skills/security-visual-testing/evals/security-visual-testing.yaml +163 -163
  314. package/assets/skills/security-visual-testing/schemas/output.json +486 -486
  315. package/assets/skills/sfdipot-product-factors/SKILL.md +239 -239
  316. package/assets/skills/sherlock-review/schemas/output.json +297 -297
  317. package/assets/skills/shift-left-testing/evals/shift-left-testing.yaml +145 -145
  318. package/assets/skills/shift-left-testing/schemas/output.json +459 -459
  319. package/assets/skills/shift-right-testing/evals/shift-right-testing.yaml +147 -147
  320. package/assets/skills/shift-right-testing/schemas/output.json +418 -418
  321. package/assets/skills/tdd-london-chicago/schemas/output.json +444 -444
  322. package/assets/skills/technical-writing/schemas/output.json +268 -268
  323. package/assets/skills/test-automation-strategy/evals/test-automation-strategy.yaml +148 -148
  324. package/assets/skills/test-automation-strategy/schemas/output.json +444 -444
  325. package/assets/skills/test-data-management/evals/test-data-management.yaml +504 -504
  326. package/assets/skills/test-data-management/schemas/output.json +284 -284
  327. package/assets/skills/test-design-techniques/evals/test-design-techniques.yaml +142 -142
  328. package/assets/skills/test-design-techniques/schemas/output.json +295 -295
  329. package/assets/skills/test-environment-management/schemas/output.json +310 -310
  330. package/assets/skills/test-idea-rewriting/SKILL.md +229 -229
  331. package/assets/skills/test-reporting-analytics/evals/test-reporting-analytics.yaml +155 -155
  332. package/assets/skills/test-reporting-analytics/schemas/output.json +329 -329
  333. package/assets/skills/testability-scoring/evals/testability-scoring.yaml +814 -814
  334. package/assets/skills/testability-scoring/resources/templates/config.template.js +84 -84
  335. package/assets/skills/testability-scoring/schemas/output.json +606 -606
  336. package/assets/skills/testability-scoring/scripts/generate-html-report.js +1007 -1007
  337. package/assets/skills/verification-quality/evals/verification-quality.yaml +150 -150
  338. package/assets/skills/verification-quality/schemas/output.json +432 -432
  339. package/assets/skills/visual-testing-advanced/evals/visual-testing-advanced.yaml +154 -154
  340. package/assets/skills/visual-testing-advanced/schemas/output.json +294 -294
  341. package/assets/skills/wms-testing-patterns/evals/wms-testing-patterns.yaml +165 -165
  342. package/assets/skills/wms-testing-patterns/schemas/output.json +150 -150
  343. package/assets/skills/wms-testing-patterns/scripts/validate-config.json +51 -51
  344. package/assets/templates/validation-summary.json +56 -56
  345. package/dist/agents/claim-verifier/verifiers/output-verifier.d.ts +1 -0
  346. package/dist/agents/claim-verifier/verifiers/output-verifier.d.ts.map +1 -1
  347. package/dist/agents/claim-verifier/verifiers/output-verifier.js +16 -3
  348. package/dist/agents/claim-verifier/verifiers/output-verifier.js.map +1 -1
  349. package/dist/cli/bundle.js +488 -75
  350. package/dist/cli/commands/mcp.d.ts.map +1 -1
  351. package/dist/cli/commands/mcp.js +11 -9
  352. package/dist/cli/commands/mcp.js.map +1 -1
  353. package/dist/governance/adversarial-defense-integration.d.ts +29 -0
  354. package/dist/governance/adversarial-defense-integration.d.ts.map +1 -1
  355. package/dist/governance/adversarial-defense-integration.js +85 -1
  356. package/dist/governance/adversarial-defense-integration.js.map +1 -1
  357. package/dist/governance/continue-gate-integration.d.ts +7 -5
  358. package/dist/governance/continue-gate-integration.d.ts.map +1 -1
  359. package/dist/governance/continue-gate-integration.js +72 -15
  360. package/dist/governance/continue-gate-integration.js.map +1 -1
  361. package/dist/governance/deterministic-gateway-integration.d.ts +5 -0
  362. package/dist/governance/deterministic-gateway-integration.d.ts.map +1 -1
  363. package/dist/governance/deterministic-gateway-integration.js +35 -1
  364. package/dist/governance/deterministic-gateway-integration.js.map +1 -1
  365. package/dist/governance/evolution-pipeline-integration.d.ts +5 -0
  366. package/dist/governance/evolution-pipeline-integration.d.ts.map +1 -1
  367. package/dist/governance/evolution-pipeline-integration.js +57 -1
  368. package/dist/governance/evolution-pipeline-integration.js.map +1 -1
  369. package/dist/governance/memory-write-gate-integration.d.ts +6 -3
  370. package/dist/governance/memory-write-gate-integration.d.ts.map +1 -1
  371. package/dist/governance/memory-write-gate-integration.js +48 -10
  372. package/dist/governance/memory-write-gate-integration.js.map +1 -1
  373. package/dist/governance/proof-envelope-integration.d.ts +1 -0
  374. package/dist/governance/proof-envelope-integration.d.ts.map +1 -1
  375. package/dist/governance/proof-envelope-integration.js +37 -0
  376. package/dist/governance/proof-envelope-integration.js.map +1 -1
  377. package/dist/governance/shard-retriever-integration.d.ts +10 -0
  378. package/dist/governance/shard-retriever-integration.d.ts.map +1 -1
  379. package/dist/governance/shard-retriever-integration.js +50 -1
  380. package/dist/governance/shard-retriever-integration.js.map +1 -1
  381. package/dist/governance/trust-accumulator-integration.d.ts +5 -0
  382. package/dist/governance/trust-accumulator-integration.d.ts.map +1 -1
  383. package/dist/governance/trust-accumulator-integration.js +35 -2
  384. package/dist/governance/trust-accumulator-integration.js.map +1 -1
  385. package/dist/init/phases/10-workers.js +2 -2
  386. package/dist/init/phases/10-workers.js.map +1 -1
  387. package/dist/mcp/bundle.js +478 -62
  388. package/dist/mcp/protocol-server.js +2 -2
  389. package/dist/mcp/tool-registry.d.ts.map +1 -1
  390. package/dist/mcp/tool-registry.js +1 -0
  391. package/dist/mcp/tool-registry.js.map +1 -1
  392. package/dist/shared/language-detector.js +1 -1
  393. package/dist/shared/language-detector.js.map +1 -1
  394. package/dist/shared/parsers/multi-language-parser.d.ts.map +1 -1
  395. package/dist/shared/parsers/multi-language-parser.js +55 -16
  396. package/dist/shared/parsers/multi-language-parser.js.map +1 -1
  397. package/dist/shared/sql-safety.d.ts.map +1 -1
  398. package/dist/shared/sql-safety.js +7 -1
  399. package/dist/shared/sql-safety.js.map +1 -1
  400. package/package.json +6 -2
  401. package/scripts/prepare-assets.sh +8 -0
@@ -1,968 +1,968 @@
1
- # =============================================================================
2
- # AQE Skill Evaluation Test Suite: Database Testing v1.0.0
3
- # =============================================================================
4
- #
5
- # Comprehensive evaluation suite for the database-testing skill per ADR-056.
6
- # Tests schema validation, data integrity, migration testing, transaction ACID
7
- # properties, and query performance analysis across multiple database types.
8
- #
9
- # Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
10
- # Validator: .claude/skills/database-testing/scripts/validate-config.json
11
- #
12
- # Coverage:
13
- # - Schema validation (tables, columns, constraints, indexes)
14
- # - Data integrity (unique, foreign key, check constraints)
15
- # - Migration testing (forward, rollback, data preservation)
16
- # - Transaction testing (ACID properties, isolation levels)
17
- # - Query performance (slow queries, missing indexes, N+1)
18
- # - Multi-database support (PostgreSQL, MySQL, MongoDB, SQLite)
19
- #
20
- # =============================================================================
21
-
22
- skill: database-testing
23
- version: 1.0.0
24
- description: >
25
- Comprehensive evaluation suite for the database-testing skill.
26
- Tests database schema validation, data integrity verification,
27
- migration testing, transaction ACID compliance, and query performance
28
- analysis. Supports PostgreSQL, MySQL, MongoDB, and SQLite.
29
- Integrates with ReasoningBank for continuous improvement.
30
-
31
- # =============================================================================
32
- # Multi-Model Configuration
33
- # =============================================================================
34
-
35
- models_to_test:
36
- - claude-sonnet-4 # Primary model (high accuracy expected)
37
- - claude-3-haiku # Fast model (minimum quality threshold)
38
- - gpt-4o # Cross-vendor validation
39
-
40
- # =============================================================================
41
- # MCP Integration Configuration
42
- # =============================================================================
43
-
44
- mcp_integration:
45
- enabled: true
46
- namespace: skill-validation
47
-
48
- # Query existing database patterns before running evals
49
- query_patterns: true
50
-
51
- # Track each test outcome for learning feedback loop
52
- track_outcomes: true
53
-
54
- # Store successful patterns after evals complete
55
- store_patterns: true
56
-
57
- # Share learning with fleet coordinator agents
58
- share_learning: true
59
-
60
- # Update quality gate with validation metrics
61
- update_quality_gate: true
62
-
63
- # Target agents for learning distribution
64
- target_agents:
65
- - qe-learning-coordinator
66
- - qe-queen-coordinator
67
- - qe-test-data-architect
68
- - qe-test-executor
69
- - qe-performance-tester
70
-
71
- # =============================================================================
72
- # ReasoningBank Learning Configuration
73
- # =============================================================================
74
-
75
- learning:
76
- store_success_patterns: true
77
- store_failure_patterns: true
78
- pattern_ttl_days: 90
79
- min_confidence_to_store: 0.7
80
- cross_model_comparison: true
81
-
82
- # =============================================================================
83
- # Result Format Configuration
84
- # =============================================================================
85
-
86
- result_format:
87
- json_output: true
88
- markdown_report: true
89
- include_raw_output: false
90
- include_timing: true
91
- include_token_usage: true
92
-
93
- # =============================================================================
94
- # Environment Setup
95
- # =============================================================================
96
-
97
- setup:
98
- required_tools:
99
- - jq # JSON parsing (required)
100
- - node # ORM validation (optional but recommended)
101
-
102
- environment_variables:
103
- DATABASE_TEST_DEPTH: "comprehensive"
104
- MIGRATION_TEST_ROLLBACK: "true"
105
- ACID_TEST_ENABLED: "true"
106
-
107
- fixtures:
108
- - name: postgresql_schema_test
109
- path: fixtures/postgresql-schema.sql
110
- content: |
111
- -- PostgreSQL test schema with intentional issues
112
- CREATE TABLE users (
113
- id SERIAL PRIMARY KEY,
114
- email VARCHAR(255), -- Missing UNIQUE constraint
115
- password VARCHAR(255), -- Should be NOT NULL
116
- created_at TIMESTAMP DEFAULT NOW()
117
- );
118
-
119
- CREATE TABLE orders (
120
- id SERIAL PRIMARY KEY,
121
- user_id INTEGER, -- Missing FOREIGN KEY
122
- total DECIMAL(10,2),
123
- status VARCHAR(50)
124
- );
125
-
126
- -- Missing index on frequently queried column
127
- CREATE TABLE products (
128
- id SERIAL PRIMARY KEY,
129
- name VARCHAR(255),
130
- category VARCHAR(100),
131
- price DECIMAL(10,2)
132
- );
133
-
134
- - name: migration_test_file
135
- path: fixtures/migration-add-age.js
136
- content: |
137
- exports.up = async function(knex) {
138
- await knex.schema.alterTable('users', (table) => {
139
- table.integer('age').nullable();
140
- });
141
- };
142
-
143
- exports.down = async function(knex) {
144
- await knex.schema.alterTable('users', (table) => {
145
- table.dropColumn('age');
146
- });
147
- };
148
-
149
- # =============================================================================
150
- # TEST CASES
151
- # =============================================================================
152
-
153
- test_cases:
154
- # ---------------------------------------------------------------------------
155
- # CATEGORY: Schema Validation
156
- # ---------------------------------------------------------------------------
157
-
158
- - id: tc001_missing_unique_constraint
159
- description: "Detect missing UNIQUE constraint on email column"
160
- category: schema
161
- priority: critical
162
-
163
- input:
164
- code: |
165
- -- User table without unique email constraint
166
- CREATE TABLE users (
167
- id SERIAL PRIMARY KEY,
168
- email VARCHAR(255),
169
- name VARCHAR(100),
170
- created_at TIMESTAMP DEFAULT NOW()
171
- );
172
-
173
- -- Application code that assumes email uniqueness
174
- INSERT INTO users (email, name) VALUES ('john@example.com', 'John');
175
- INSERT INTO users (email, name) VALUES ('john@example.com', 'Johnny'); -- Duplicate allowed!
176
- context:
177
- database: postgresql
178
- framework: knex
179
- environment: production
180
-
181
- expected_output:
182
- must_contain:
183
- - "unique"
184
- - "constraint"
185
- - "email"
186
- - "duplicate"
187
- must_not_contain:
188
- - "no issues"
189
- - "valid schema"
190
- must_match_regex:
191
- - "DB-\\d{3}"
192
- severity_classification: high
193
- finding_count:
194
- min: 1
195
- max: 5
196
-
197
- validation:
198
- schema_check: true
199
- keyword_match_threshold: 0.8
200
- reasoning_quality_min: 0.7
201
- grading_rubric:
202
- completeness: 0.3
203
- accuracy: 0.5
204
- actionability: 0.2
205
-
206
- timeout_ms: 30000
207
-
208
- - id: tc002_missing_foreign_key
209
- description: "Detect missing foreign key constraint causing orphaned records"
210
- category: schema
211
- priority: critical
212
-
213
- input:
214
- code: |
215
- CREATE TABLE users (
216
- id SERIAL PRIMARY KEY,
217
- email VARCHAR(255) UNIQUE NOT NULL
218
- );
219
-
220
- CREATE TABLE orders (
221
- id SERIAL PRIMARY KEY,
222
- user_id INTEGER, -- No FK constraint!
223
- total DECIMAL(10,2)
224
- );
225
-
226
- -- This allows orphaned orders
227
- INSERT INTO orders (user_id, total) VALUES (999, 100.00);
228
- DELETE FROM users WHERE id = 1; -- Orphans all orders for user 1
229
- context:
230
- database: postgresql
231
- framework: prisma
232
-
233
- expected_output:
234
- must_contain:
235
- - "foreign key"
236
- - "referential integrity"
237
- - "orphaned"
238
- must_match_regex:
239
- - "DB-\\d{3}"
240
- severity_classification: critical
241
- finding_count:
242
- min: 1
243
-
244
- validation:
245
- schema_check: true
246
- keyword_match_threshold: 0.8
247
-
248
- - id: tc003_missing_index_on_foreign_key
249
- description: "Detect missing index on foreign key column affecting performance"
250
- category: schema
251
- priority: high
252
-
253
- input:
254
- code: |
255
- CREATE TABLE orders (
256
- id SERIAL PRIMARY KEY,
257
- user_id INTEGER REFERENCES users(id),
258
- product_id INTEGER REFERENCES products(id),
259
- quantity INTEGER,
260
- created_at TIMESTAMP
261
- );
262
-
263
- -- Frequently run queries without indexes
264
- SELECT * FROM orders WHERE user_id = 123;
265
- SELECT COUNT(*) FROM orders WHERE product_id = 456;
266
- context:
267
- database: postgresql
268
- framework: typeorm
269
-
270
- expected_output:
271
- must_contain:
272
- - "index"
273
- - "foreign key"
274
- - "performance"
275
- - "user_id"
276
- must_match_regex:
277
- - "CREATE INDEX"
278
- severity_classification: medium
279
-
280
- validation:
281
- schema_check: true
282
- keyword_match_threshold: 0.7
283
-
284
- # ---------------------------------------------------------------------------
285
- # CATEGORY: Data Integrity
286
- # ---------------------------------------------------------------------------
287
-
288
- - id: tc004_check_constraint_violation
289
- description: "Detect check constraint violations on status column"
290
- category: integrity
291
- priority: high
292
-
293
- input:
294
- code: |
295
- CREATE TABLE orders (
296
- id SERIAL PRIMARY KEY,
297
- status VARCHAR(20) CHECK (status IN ('pending', 'processing', 'shipped', 'delivered')),
298
- total DECIMAL(10,2) CHECK (total >= 0)
299
- );
300
-
301
- -- Test constraint enforcement
302
- INSERT INTO orders (status, total) VALUES ('invalid_status', 100.00);
303
- INSERT INTO orders (status, total) VALUES ('pending', -50.00);
304
- context:
305
- database: postgresql
306
- test_type: integrity
307
-
308
- expected_output:
309
- must_contain:
310
- - "check constraint"
311
- - "violation"
312
- - "status"
313
- must_not_contain:
314
- - "passed"
315
- - "valid"
316
- severity_classification: high
317
-
318
- validation:
319
- schema_check: true
320
- keyword_match_threshold: 0.8
321
-
322
- - id: tc005_null_constraint_test
323
- description: "Detect NOT NULL constraint violations"
324
- category: integrity
325
- priority: high
326
-
327
- input:
328
- code: |
329
- const user = new User();
330
- user.email = 'test@example.com';
331
- // Missing required field: password
332
- await user.save();
333
-
334
- // Schema:
335
- // email VARCHAR(255) NOT NULL
336
- // password VARCHAR(255) NOT NULL
337
- // name VARCHAR(100) -- nullable
338
- context:
339
- database: mysql
340
- framework: sequelize
341
- test_type: integrity
342
-
343
- expected_output:
344
- must_contain:
345
- - "NOT NULL"
346
- - "constraint"
347
- - "password"
348
- - "required"
349
- severity_classification: high
350
- finding_count:
351
- min: 1
352
-
353
- validation:
354
- schema_check: true
355
- keyword_match_threshold: 0.8
356
-
357
- # ---------------------------------------------------------------------------
358
- # CATEGORY: Migration Testing
359
- # ---------------------------------------------------------------------------
360
-
361
- - id: tc006_migration_forward_test
362
- description: "Test migration applies successfully forward"
363
- category: migration
364
- priority: critical
365
-
366
- input:
367
- code: |
368
- // Migration: add-user-roles.js
369
- exports.up = async function(knex) {
370
- await knex.schema.createTable('roles', (table) => {
371
- table.increments('id');
372
- table.string('name').unique().notNullable();
373
- });
374
-
375
- await knex.schema.alterTable('users', (table) => {
376
- table.integer('role_id').references('id').inTable('roles');
377
- });
378
- };
379
-
380
- exports.down = async function(knex) {
381
- await knex.schema.alterTable('users', (table) => {
382
- table.dropColumn('role_id');
383
- });
384
- await knex.schema.dropTable('roles');
385
- };
386
- context:
387
- database: postgresql
388
- framework: knex
389
- test_type: migration
390
-
391
- expected_output:
392
- must_contain:
393
- - "migration"
394
- - "forward"
395
- - "roles"
396
- - "users"
397
- must_not_contain:
398
- - "failed"
399
- - "error"
400
- must_match_regex:
401
- - "up|forward|apply"
402
-
403
- validation:
404
- schema_check: true
405
- keyword_match_threshold: 0.7
406
-
407
- - id: tc007_migration_rollback_test
408
- description: "Test migration rollback works correctly"
409
- category: migration
410
- priority: critical
411
-
412
- input:
413
- code: |
414
- // Test migration rollback
415
- await migrate.latest(); // Apply all migrations
416
- await migrate.rollback(); // Rollback last batch
417
-
418
- // Verify tables are removed
419
- const tables = await knex.raw("SELECT table_name FROM information_schema.tables");
420
- expect(tables).not.toContain('new_feature_table');
421
- context:
422
- database: postgresql
423
- framework: knex
424
- test_type: migration
425
-
426
- expected_output:
427
- must_contain:
428
- - "rollback"
429
- - "down"
430
- - "reverse"
431
- must_not_contain:
432
- - "irreversible"
433
- - "failed"
434
- severity_classification: medium
435
-
436
- validation:
437
- schema_check: true
438
- keyword_match_threshold: 0.7
439
-
440
- - id: tc008_migration_data_preservation
441
- description: "Test migration preserves existing data"
442
- category: migration
443
- priority: critical
444
-
445
- input:
446
- code: |
447
- // Existing data before migration
448
- await db.users.create({ email: 'existing@example.com', name: 'Existing User' });
449
-
450
- // Migration adds new column
451
- exports.up = async function(knex) {
452
- await knex.schema.alterTable('users', (table) => {
453
- table.string('phone').nullable();
454
- });
455
- };
456
-
457
- // After migration, verify existing data is preserved
458
- const user = await db.users.findOne({ email: 'existing@example.com' });
459
- expect(user.name).toBe('Existing User'); // Data should be intact
460
- expect(user.phone).toBeNull(); // New column should be null
461
- context:
462
- database: postgresql
463
- framework: knex
464
- test_type: migration
465
-
466
- expected_output:
467
- must_contain:
468
- - "data preservation"
469
- - "existing"
470
- - "intact"
471
- - "backward compatible"
472
- must_not_contain:
473
- - "data loss"
474
- - "corrupted"
475
- severity_classification: critical
476
-
477
- validation:
478
- schema_check: true
479
- keyword_match_threshold: 0.7
480
-
481
- # ---------------------------------------------------------------------------
482
- # CATEGORY: Transaction Testing (ACID)
483
- # ---------------------------------------------------------------------------
484
-
485
- - id: tc009_atomicity_test
486
- description: "Test transaction atomicity - all or nothing"
487
- category: transaction
488
- priority: critical
489
-
490
- input:
491
- code: |
492
- test('transaction rolls back on error', async () => {
493
- const initialCount = await db.users.count();
494
-
495
- try {
496
- await db.transaction(async (trx) => {
497
- await trx('users').insert({ email: 'user1@example.com' });
498
- await trx('users').insert({ email: 'user2@example.com' });
499
- throw new Error('Intentional rollback');
500
- });
501
- } catch (error) { /* Expected */ }
502
-
503
- // Count should be unchanged
504
- expect(await db.users.count()).toBe(initialCount);
505
- });
506
- context:
507
- database: postgresql
508
- framework: knex
509
- test_type: transaction
510
-
511
- expected_output:
512
- must_contain:
513
- - "atomicity"
514
- - "rollback"
515
- - "transaction"
516
- - "all or nothing"
517
- must_not_contain:
518
- - "partial commit"
519
- must_match_regex:
520
- - "ACID|atomic"
521
- severity_classification: critical
522
-
523
- validation:
524
- schema_check: true
525
- keyword_match_threshold: 0.8
526
- reasoning_quality_min: 0.8
527
-
528
- - id: tc010_isolation_test
529
- description: "Test transaction isolation - concurrent access"
530
- category: transaction
531
- priority: critical
532
-
533
- input:
534
- code: |
535
- test('concurrent transactions isolated', async () => {
536
- const user = await db.users.create({ balance: 100 });
537
-
538
- // Two concurrent withdrawals - race condition test
539
- const results = await Promise.all([
540
- db.transaction(async (trx) => {
541
- const current = await trx('users').where({ id: user.id }).first();
542
- if (current.balance >= 50) {
543
- await trx('users').where({ id: user.id }).update({
544
- balance: current.balance - 50
545
- });
546
- return 'success';
547
- }
548
- return 'insufficient';
549
- }),
550
- db.transaction(async (trx) => {
551
- const current = await trx('users').where({ id: user.id }).first();
552
- if (current.balance >= 50) {
553
- await trx('users').where({ id: user.id }).update({
554
- balance: current.balance - 50
555
- });
556
- return 'success';
557
- }
558
- return 'insufficient';
559
- })
560
- ]);
561
-
562
- const final = await db.users.findOne({ id: user.id });
563
- // With proper isolation, one should fail or balance should be 0
564
- expect(final.balance).toBeGreaterThanOrEqual(0);
565
- });
566
- context:
567
- database: postgresql
568
- framework: knex
569
- test_type: transaction
570
- isolation_level: REPEATABLE_READ
571
-
572
- expected_output:
573
- must_contain:
574
- - "isolation"
575
- - "concurrent"
576
- - "race condition"
577
- - "lock"
578
- must_match_regex:
579
- - "READ_COMMITTED|REPEATABLE_READ|SERIALIZABLE"
580
- severity_classification: critical
581
-
582
- validation:
583
- schema_check: true
584
- keyword_match_threshold: 0.7
585
-
586
- - id: tc011_deadlock_detection
587
- description: "Test deadlock detection and resolution"
588
- category: transaction
589
- priority: high
590
-
591
- input:
592
- code: |
593
- test('deadlock is detected and handled', async () => {
594
- // Create potential deadlock scenario
595
- const tx1 = db.transaction(async (trx) => {
596
- await trx('accounts').where({ id: 1 }).update({ balance: 100 });
597
- await delay(100);
598
- await trx('accounts').where({ id: 2 }).update({ balance: 200 });
599
- });
600
-
601
- const tx2 = db.transaction(async (trx) => {
602
- await trx('accounts').where({ id: 2 }).update({ balance: 150 });
603
- await delay(100);
604
- await trx('accounts').where({ id: 1 }).update({ balance: 250 });
605
- });
606
-
607
- // One should fail with deadlock, other should succeed
608
- const results = await Promise.allSettled([tx1, tx2]);
609
- const failures = results.filter(r => r.status === 'rejected');
610
- expect(failures.some(f => f.reason.message.includes('deadlock'))).toBe(true);
611
- });
612
- context:
613
- database: postgresql
614
- framework: knex
615
- test_type: transaction
616
-
617
- expected_output:
618
- must_contain:
619
- - "deadlock"
620
- - "detection"
621
- - "retry"
622
- - "conflict"
623
- severity_classification: high
624
-
625
- validation:
626
- schema_check: true
627
- keyword_match_threshold: 0.7
628
-
629
- # ---------------------------------------------------------------------------
630
- # CATEGORY: Performance Testing
631
- # ---------------------------------------------------------------------------
632
-
633
- - id: tc012_slow_query_detection
634
- description: "Detect slow queries without proper indexing"
635
- category: performance
636
- priority: high
637
-
638
- input:
639
- code: |
640
- -- Slow query: full table scan on large table
641
- SELECT * FROM orders
642
- WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31'
643
- AND status = 'pending';
644
-
645
- -- Table has 10 million rows, no index on created_at or status
646
- EXPLAIN ANALYZE SELECT * FROM orders
647
- WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31';
648
- -- Output: Seq Scan on orders (cost=0.00..250000.00 rows=5000000)
649
- context:
650
- database: postgresql
651
- test_type: performance
652
- table_size: 10000000
653
-
654
- expected_output:
655
- must_contain:
656
- - "slow query"
657
- - "full table scan"
658
- - "index"
659
- - "Seq Scan"
660
- must_match_regex:
661
- - "CREATE INDEX"
662
- severity_classification: high
663
- finding_count:
664
- min: 1
665
-
666
- validation:
667
- schema_check: true
668
- keyword_match_threshold: 0.8
669
-
670
- - id: tc013_n_plus_one_detection
671
- description: "Detect N+1 query problem"
672
- category: performance
673
- priority: high
674
-
675
- input:
676
- code: |
677
- // N+1 query problem
678
- const users = await User.findAll(); // 1 query
679
-
680
- for (const user of users) {
681
- const orders = await Order.findAll({
682
- where: { userId: user.id }
683
- }); // N queries!
684
- user.orderCount = orders.length;
685
- }
686
-
687
- // Should be:
688
- // const users = await User.findAll({
689
- // include: [{ model: Order }]
690
- // });
691
- context:
692
- database: postgresql
693
- framework: sequelize
694
- test_type: performance
695
-
696
- expected_output:
697
- must_contain:
698
- - "N+1"
699
- - "query"
700
- - "include"
701
- - "eager loading"
702
- - "join"
703
- must_not_contain:
704
- - "efficient"
705
- - "optimal"
706
- severity_classification: high
707
-
708
- validation:
709
- schema_check: true
710
- keyword_match_threshold: 0.8
711
-
712
- - id: tc014_connection_pool_exhaustion
713
- description: "Detect connection pool exhaustion risk"
714
- category: performance
715
- priority: critical
716
-
717
- input:
718
- code: |
719
- // Connection leak - connections not released
720
- async function getUserData(userId) {
721
- const connection = await pool.getConnection();
722
- const user = await connection.query('SELECT * FROM users WHERE id = ?', [userId]);
723
- // Missing: connection.release()
724
- return user;
725
- }
726
-
727
- // Called in a loop without releasing connections
728
- for (let i = 0; i < 1000; i++) {
729
- await getUserData(i);
730
- }
731
- context:
732
- database: mysql
733
- framework: mysql2
734
- test_type: performance
735
- pool_size: 10
736
-
737
- expected_output:
738
- must_contain:
739
- - "connection"
740
- - "pool"
741
- - "leak"
742
- - "release"
743
- - "exhaustion"
744
- severity_classification: critical
745
-
746
- validation:
747
- schema_check: true
748
- keyword_match_threshold: 0.8
749
-
750
- # ---------------------------------------------------------------------------
751
- # CATEGORY: Multi-Database Support
752
- # ---------------------------------------------------------------------------
753
-
754
- - id: tc015_mongodb_schema_validation
755
- description: "Test MongoDB schema validation"
756
- category: schema
757
- priority: high
758
-
759
- input:
760
- code: |
761
- // MongoDB collection without schema validation
762
- db.createCollection("users");
763
-
764
- // Can insert inconsistent documents
765
- db.users.insertOne({ email: "user@example.com", age: 25 });
766
- db.users.insertOne({ email: 123, age: "twenty" }); // Wrong types!
767
- db.users.insertOne({ name: "John" }); // Missing email
768
-
769
- // Should have JSON Schema validation:
770
- db.createCollection("users", {
771
- validator: {
772
- $jsonSchema: {
773
- bsonType: "object",
774
- required: ["email"],
775
- properties: {
776
- email: { bsonType: "string" },
777
- age: { bsonType: "int" }
778
- }
779
- }
780
- }
781
- });
782
- context:
783
- database: mongodb
784
- test_type: schema
785
-
786
- expected_output:
787
- must_contain:
788
- - "schema validation"
789
- - "MongoDB"
790
- - "jsonSchema"
791
- - "bsonType"
792
- severity_classification: high
793
-
794
- validation:
795
- schema_check: true
796
- keyword_match_threshold: 0.7
797
-
798
- - id: tc016_sqlite_foreign_key_enforcement
799
- description: "Test SQLite foreign key enforcement"
800
- category: integrity
801
- priority: high
802
-
803
- input:
804
- code: |
805
- -- SQLite: Foreign keys are OFF by default!
806
- CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);
807
- CREATE TABLE orders (
808
- id INTEGER PRIMARY KEY,
809
- user_id INTEGER REFERENCES users(id)
810
- );
811
-
812
- -- This succeeds even though user 999 doesn't exist
813
- INSERT INTO orders (user_id) VALUES (999);
814
-
815
- -- Need to enable: PRAGMA foreign_keys = ON;
816
- context:
817
- database: sqlite
818
- test_type: integrity
819
-
820
- expected_output:
821
- must_contain:
822
- - "foreign_keys"
823
- - "PRAGMA"
824
- - "SQLite"
825
- - "enforcement"
826
- must_match_regex:
827
- - "PRAGMA foreign_keys\\s*=\\s*ON"
828
- severity_classification: high
829
-
830
- validation:
831
- schema_check: true
832
- keyword_match_threshold: 0.8
833
-
834
- # ---------------------------------------------------------------------------
835
- # CATEGORY: Negative Tests (Secure/Valid Code)
836
- # ---------------------------------------------------------------------------
837
-
838
- - id: tc017_valid_schema_no_false_positives
839
- description: "Verify well-designed schema is NOT flagged as problematic"
840
- category: negative
841
- priority: critical
842
-
843
- input:
844
- code: |
845
- CREATE TABLE users (
846
- id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
847
- email VARCHAR(255) UNIQUE NOT NULL,
848
- password_hash VARCHAR(255) NOT NULL,
849
- created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
850
- updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
851
- );
852
-
853
- CREATE TABLE orders (
854
- id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
855
- user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
856
- total DECIMAL(10,2) NOT NULL CHECK (total >= 0),
857
- status VARCHAR(20) NOT NULL DEFAULT 'pending'
858
- CHECK (status IN ('pending', 'processing', 'shipped', 'delivered', 'cancelled')),
859
- created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
860
- );
861
-
862
- CREATE INDEX idx_orders_user_id ON orders(user_id);
863
- CREATE INDEX idx_orders_status ON orders(status);
864
- CREATE INDEX idx_orders_created_at ON orders(created_at);
865
- context:
866
- database: postgresql
867
- framework: prisma
868
- environment: production
869
-
870
- expected_output:
871
- must_contain:
872
- - "valid"
873
- - "well-designed"
874
- - "proper constraints"
875
- must_not_contain:
876
- - "missing constraint"
877
- - "critical"
878
- - "high severity"
879
- - "orphaned"
880
- finding_count:
881
- max: 2 # Allow informational findings only
882
-
883
- validation:
884
- schema_check: true
885
- keyword_match_threshold: 0.6
886
- allow_partial: true
887
-
888
- - id: tc018_proper_migration_pattern
889
- description: "Verify proper migration pattern is recognized"
890
- category: negative
891
- priority: high
892
-
893
- input:
894
- code: |
895
- // Proper reversible migration with data preservation
896
- exports.up = async function(knex) {
897
- // Check if column exists before adding
898
- const hasColumn = await knex.schema.hasColumn('users', 'phone');
899
- if (!hasColumn) {
900
- await knex.schema.alterTable('users', (table) => {
901
- table.string('phone').nullable();
902
- });
903
- }
904
- };
905
-
906
- exports.down = async function(knex) {
907
- const hasColumn = await knex.schema.hasColumn('users', 'phone');
908
- if (hasColumn) {
909
- await knex.schema.alterTable('users', (table) => {
910
- table.dropColumn('phone');
911
- });
912
- }
913
- };
914
- context:
915
- database: postgresql
916
- framework: knex
917
- test_type: migration
918
-
919
- expected_output:
920
- must_contain:
921
- - "reversible"
922
- - "idempotent"
923
- - "safe"
924
- must_not_contain:
925
- - "irreversible"
926
- - "data loss"
927
- - "critical"
928
- severity_classification: info
929
-
930
- validation:
931
- schema_check: true
932
- allow_partial: true
933
-
934
- # =============================================================================
935
- # SUCCESS CRITERIA
936
- # =============================================================================
937
-
938
- success_criteria:
939
- # Overall pass rate (90% of tests must pass)
940
- pass_rate: 0.9
941
-
942
- # Critical tests must ALL pass (100%)
943
- critical_pass_rate: 1.0
944
-
945
- # Average reasoning quality score
946
- avg_reasoning_quality: 0.75
947
-
948
- # Maximum suite execution time (5 minutes)
949
- max_execution_time_ms: 300000
950
-
951
- # Maximum variance between model results (15%)
952
- cross_model_variance: 0.15
953
-
954
- # =============================================================================
955
- # METADATA
956
- # =============================================================================
957
-
958
- metadata:
959
- author: "qe-test-data-architect"
960
- created: "2026-02-02"
961
- last_updated: "2026-02-02"
962
- coverage_target: >
963
- Database testing coverage: Schema validation (missing constraints, indexes),
964
- Data integrity (unique, FK, check, NOT NULL), Migration testing (forward,
965
- rollback, data preservation), Transaction testing (ACID: atomicity, isolation,
966
- deadlock), Performance (slow queries, N+1, connection pools). Multi-database
967
- support: PostgreSQL, MySQL, MongoDB, SQLite. 18 test cases with 90% pass rate
968
- requirement and 100% critical pass rate for ACID and migration tests.
1
+ # =============================================================================
2
+ # AQE Skill Evaluation Test Suite: Database Testing v1.0.0
3
+ # =============================================================================
4
+ #
5
+ # Comprehensive evaluation suite for the database-testing skill per ADR-056.
6
+ # Tests schema validation, data integrity, migration testing, transaction ACID
7
+ # properties, and query performance analysis across multiple database types.
8
+ #
9
+ # Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
10
+ # Validator: .claude/skills/database-testing/scripts/validate-config.json
11
+ #
12
+ # Coverage:
13
+ # - Schema validation (tables, columns, constraints, indexes)
14
+ # - Data integrity (unique, foreign key, check constraints)
15
+ # - Migration testing (forward, rollback, data preservation)
16
+ # - Transaction testing (ACID properties, isolation levels)
17
+ # - Query performance (slow queries, missing indexes, N+1)
18
+ # - Multi-database support (PostgreSQL, MySQL, MongoDB, SQLite)
19
+ #
20
+ # =============================================================================
21
+
22
+ skill: database-testing
23
+ version: 1.0.0
24
+ description: >
25
+ Comprehensive evaluation suite for the database-testing skill.
26
+ Tests database schema validation, data integrity verification,
27
+ migration testing, transaction ACID compliance, and query performance
28
+ analysis. Supports PostgreSQL, MySQL, MongoDB, and SQLite.
29
+ Integrates with ReasoningBank for continuous improvement.
30
+
31
+ # =============================================================================
32
+ # Multi-Model Configuration
33
+ # =============================================================================
34
+
35
+ models_to_test:
36
+ - claude-sonnet-4 # Primary model (high accuracy expected)
37
+ - claude-3-haiku # Fast model (minimum quality threshold)
38
+ - gpt-4o # Cross-vendor validation
39
+
40
+ # =============================================================================
41
+ # MCP Integration Configuration
42
+ # =============================================================================
43
+
44
+ mcp_integration:
45
+ enabled: true
46
+ namespace: skill-validation
47
+
48
+ # Query existing database patterns before running evals
49
+ query_patterns: true
50
+
51
+ # Track each test outcome for learning feedback loop
52
+ track_outcomes: true
53
+
54
+ # Store successful patterns after evals complete
55
+ store_patterns: true
56
+
57
+ # Share learning with fleet coordinator agents
58
+ share_learning: true
59
+
60
+ # Update quality gate with validation metrics
61
+ update_quality_gate: true
62
+
63
+ # Target agents for learning distribution
64
+ target_agents:
65
+ - qe-learning-coordinator
66
+ - qe-queen-coordinator
67
+ - qe-test-data-architect
68
+ - qe-test-executor
69
+ - qe-performance-tester
70
+
71
+ # =============================================================================
72
+ # ReasoningBank Learning Configuration
73
+ # =============================================================================
74
+
75
+ learning:
76
+ store_success_patterns: true
77
+ store_failure_patterns: true
78
+ pattern_ttl_days: 90
79
+ min_confidence_to_store: 0.7
80
+ cross_model_comparison: true
81
+
82
+ # =============================================================================
83
+ # Result Format Configuration
84
+ # =============================================================================
85
+
86
+ result_format:
87
+ json_output: true
88
+ markdown_report: true
89
+ include_raw_output: false
90
+ include_timing: true
91
+ include_token_usage: true
92
+
93
+ # =============================================================================
94
+ # Environment Setup
95
+ # =============================================================================
96
+
97
+ setup:
98
+ required_tools:
99
+ - jq # JSON parsing (required)
100
+ - node # ORM validation (optional but recommended)
101
+
102
+ environment_variables:
103
+ DATABASE_TEST_DEPTH: "comprehensive"
104
+ MIGRATION_TEST_ROLLBACK: "true"
105
+ ACID_TEST_ENABLED: "true"
106
+
107
+ fixtures:
108
+ - name: postgresql_schema_test
109
+ path: fixtures/postgresql-schema.sql
110
+ content: |
111
+ -- PostgreSQL test schema with intentional issues
112
+ CREATE TABLE users (
113
+ id SERIAL PRIMARY KEY,
114
+ email VARCHAR(255), -- Missing UNIQUE constraint
115
+ password VARCHAR(255), -- Should be NOT NULL
116
+ created_at TIMESTAMP DEFAULT NOW()
117
+ );
118
+
119
+ CREATE TABLE orders (
120
+ id SERIAL PRIMARY KEY,
121
+ user_id INTEGER, -- Missing FOREIGN KEY
122
+ total DECIMAL(10,2),
123
+ status VARCHAR(50)
124
+ );
125
+
126
+ -- Missing index on frequently queried column
127
+ CREATE TABLE products (
128
+ id SERIAL PRIMARY KEY,
129
+ name VARCHAR(255),
130
+ category VARCHAR(100),
131
+ price DECIMAL(10,2)
132
+ );
133
+
134
+ - name: migration_test_file
135
+ path: fixtures/migration-add-age.js
136
+ content: |
137
+ exports.up = async function(knex) {
138
+ await knex.schema.alterTable('users', (table) => {
139
+ table.integer('age').nullable();
140
+ });
141
+ };
142
+
143
+ exports.down = async function(knex) {
144
+ await knex.schema.alterTable('users', (table) => {
145
+ table.dropColumn('age');
146
+ });
147
+ };
148
+
149
+ # =============================================================================
150
+ # TEST CASES
151
+ # =============================================================================
152
+
153
+ test_cases:
154
+ # ---------------------------------------------------------------------------
155
+ # CATEGORY: Schema Validation
156
+ # ---------------------------------------------------------------------------
157
+
158
+ - id: tc001_missing_unique_constraint
159
+ description: "Detect missing UNIQUE constraint on email column"
160
+ category: schema
161
+ priority: critical
162
+
163
+ input:
164
+ code: |
165
+ -- User table without unique email constraint
166
+ CREATE TABLE users (
167
+ id SERIAL PRIMARY KEY,
168
+ email VARCHAR(255),
169
+ name VARCHAR(100),
170
+ created_at TIMESTAMP DEFAULT NOW()
171
+ );
172
+
173
+ -- Application code that assumes email uniqueness
174
+ INSERT INTO users (email, name) VALUES ('john@example.com', 'John');
175
+ INSERT INTO users (email, name) VALUES ('john@example.com', 'Johnny'); -- Duplicate allowed!
176
+ context:
177
+ database: postgresql
178
+ framework: knex
179
+ environment: production
180
+
181
+ expected_output:
182
+ must_contain:
183
+ - "unique"
184
+ - "constraint"
185
+ - "email"
186
+ - "duplicate"
187
+ must_not_contain:
188
+ - "no issues"
189
+ - "valid schema"
190
+ must_match_regex:
191
+ - "DB-\\d{3}"
192
+ severity_classification: high
193
+ finding_count:
194
+ min: 1
195
+ max: 5
196
+
197
+ validation:
198
+ schema_check: true
199
+ keyword_match_threshold: 0.8
200
+ reasoning_quality_min: 0.7
201
+ grading_rubric:
202
+ completeness: 0.3
203
+ accuracy: 0.5
204
+ actionability: 0.2
205
+
206
+ timeout_ms: 30000
207
+
208
+ - id: tc002_missing_foreign_key
209
+ description: "Detect missing foreign key constraint causing orphaned records"
210
+ category: schema
211
+ priority: critical
212
+
213
+ input:
214
+ code: |
215
+ CREATE TABLE users (
216
+ id SERIAL PRIMARY KEY,
217
+ email VARCHAR(255) UNIQUE NOT NULL
218
+ );
219
+
220
+ CREATE TABLE orders (
221
+ id SERIAL PRIMARY KEY,
222
+ user_id INTEGER, -- No FK constraint!
223
+ total DECIMAL(10,2)
224
+ );
225
+
226
+ -- This allows orphaned orders
227
+ INSERT INTO orders (user_id, total) VALUES (999, 100.00);
228
+ DELETE FROM users WHERE id = 1; -- Orphans all orders for user 1
229
+ context:
230
+ database: postgresql
231
+ framework: prisma
232
+
233
+ expected_output:
234
+ must_contain:
235
+ - "foreign key"
236
+ - "referential integrity"
237
+ - "orphaned"
238
+ must_match_regex:
239
+ - "DB-\\d{3}"
240
+ severity_classification: critical
241
+ finding_count:
242
+ min: 1
243
+
244
+ validation:
245
+ schema_check: true
246
+ keyword_match_threshold: 0.8
247
+
248
+ - id: tc003_missing_index_on_foreign_key
249
+ description: "Detect missing index on foreign key column affecting performance"
250
+ category: schema
251
+ priority: high
252
+
253
+ input:
254
+ code: |
255
+ CREATE TABLE orders (
256
+ id SERIAL PRIMARY KEY,
257
+ user_id INTEGER REFERENCES users(id),
258
+ product_id INTEGER REFERENCES products(id),
259
+ quantity INTEGER,
260
+ created_at TIMESTAMP
261
+ );
262
+
263
+ -- Frequently run queries without indexes
264
+ SELECT * FROM orders WHERE user_id = 123;
265
+ SELECT COUNT(*) FROM orders WHERE product_id = 456;
266
+ context:
267
+ database: postgresql
268
+ framework: typeorm
269
+
270
+ expected_output:
271
+ must_contain:
272
+ - "index"
273
+ - "foreign key"
274
+ - "performance"
275
+ - "user_id"
276
+ must_match_regex:
277
+ - "CREATE INDEX"
278
+ severity_classification: medium
279
+
280
+ validation:
281
+ schema_check: true
282
+ keyword_match_threshold: 0.7
283
+
284
+ # ---------------------------------------------------------------------------
285
+ # CATEGORY: Data Integrity
286
+ # ---------------------------------------------------------------------------
287
+
288
+ - id: tc004_check_constraint_violation
289
+ description: "Detect check constraint violations on status column"
290
+ category: integrity
291
+ priority: high
292
+
293
+ input:
294
+ code: |
295
+ CREATE TABLE orders (
296
+ id SERIAL PRIMARY KEY,
297
+ status VARCHAR(20) CHECK (status IN ('pending', 'processing', 'shipped', 'delivered')),
298
+ total DECIMAL(10,2) CHECK (total >= 0)
299
+ );
300
+
301
+ -- Test constraint enforcement
302
+ INSERT INTO orders (status, total) VALUES ('invalid_status', 100.00);
303
+ INSERT INTO orders (status, total) VALUES ('pending', -50.00);
304
+ context:
305
+ database: postgresql
306
+ test_type: integrity
307
+
308
+ expected_output:
309
+ must_contain:
310
+ - "check constraint"
311
+ - "violation"
312
+ - "status"
313
+ must_not_contain:
314
+ - "passed"
315
+ - "valid"
316
+ severity_classification: high
317
+
318
+ validation:
319
+ schema_check: true
320
+ keyword_match_threshold: 0.8
321
+
322
+ - id: tc005_null_constraint_test
323
+ description: "Detect NOT NULL constraint violations"
324
+ category: integrity
325
+ priority: high
326
+
327
+ input:
328
+ code: |
329
+ const user = new User();
330
+ user.email = 'test@example.com';
331
+ // Missing required field: password
332
+ await user.save();
333
+
334
+ // Schema:
335
+ // email VARCHAR(255) NOT NULL
336
+ // password VARCHAR(255) NOT NULL
337
+ // name VARCHAR(100) -- nullable
338
+ context:
339
+ database: mysql
340
+ framework: sequelize
341
+ test_type: integrity
342
+
343
+ expected_output:
344
+ must_contain:
345
+ - "NOT NULL"
346
+ - "constraint"
347
+ - "password"
348
+ - "required"
349
+ severity_classification: high
350
+ finding_count:
351
+ min: 1
352
+
353
+ validation:
354
+ schema_check: true
355
+ keyword_match_threshold: 0.8
356
+
357
+ # ---------------------------------------------------------------------------
358
+ # CATEGORY: Migration Testing
359
+ # ---------------------------------------------------------------------------
360
+
361
+ - id: tc006_migration_forward_test
362
+ description: "Test migration applies successfully forward"
363
+ category: migration
364
+ priority: critical
365
+
366
+ input:
367
+ code: |
368
+ // Migration: add-user-roles.js
369
+ exports.up = async function(knex) {
370
+ await knex.schema.createTable('roles', (table) => {
371
+ table.increments('id');
372
+ table.string('name').unique().notNullable();
373
+ });
374
+
375
+ await knex.schema.alterTable('users', (table) => {
376
+ table.integer('role_id').references('id').inTable('roles');
377
+ });
378
+ };
379
+
380
+ exports.down = async function(knex) {
381
+ await knex.schema.alterTable('users', (table) => {
382
+ table.dropColumn('role_id');
383
+ });
384
+ await knex.schema.dropTable('roles');
385
+ };
386
+ context:
387
+ database: postgresql
388
+ framework: knex
389
+ test_type: migration
390
+
391
+ expected_output:
392
+ must_contain:
393
+ - "migration"
394
+ - "forward"
395
+ - "roles"
396
+ - "users"
397
+ must_not_contain:
398
+ - "failed"
399
+ - "error"
400
+ must_match_regex:
401
+ - "up|forward|apply"
402
+
403
+ validation:
404
+ schema_check: true
405
+ keyword_match_threshold: 0.7
406
+
407
+ - id: tc007_migration_rollback_test
408
+ description: "Test migration rollback works correctly"
409
+ category: migration
410
+ priority: critical
411
+
412
+ input:
413
+ code: |
414
+ // Test migration rollback
415
+ await migrate.latest(); // Apply all migrations
416
+ await migrate.rollback(); // Rollback last batch
417
+
418
+ // Verify tables are removed
419
+ const tables = await knex.raw("SELECT table_name FROM information_schema.tables");
420
+ expect(tables).not.toContain('new_feature_table');
421
+ context:
422
+ database: postgresql
423
+ framework: knex
424
+ test_type: migration
425
+
426
+ expected_output:
427
+ must_contain:
428
+ - "rollback"
429
+ - "down"
430
+ - "reverse"
431
+ must_not_contain:
432
+ - "irreversible"
433
+ - "failed"
434
+ severity_classification: medium
435
+
436
+ validation:
437
+ schema_check: true
438
+ keyword_match_threshold: 0.7
439
+
440
+ - id: tc008_migration_data_preservation
441
+ description: "Test migration preserves existing data"
442
+ category: migration
443
+ priority: critical
444
+
445
+ input:
446
+ code: |
447
+ // Existing data before migration
448
+ await db.users.create({ email: 'existing@example.com', name: 'Existing User' });
449
+
450
+ // Migration adds new column
451
+ exports.up = async function(knex) {
452
+ await knex.schema.alterTable('users', (table) => {
453
+ table.string('phone').nullable();
454
+ });
455
+ };
456
+
457
+ // After migration, verify existing data is preserved
458
+ const user = await db.users.findOne({ email: 'existing@example.com' });
459
+ expect(user.name).toBe('Existing User'); // Data should be intact
460
+ expect(user.phone).toBeNull(); // New column should be null
461
+ context:
462
+ database: postgresql
463
+ framework: knex
464
+ test_type: migration
465
+
466
+ expected_output:
467
+ must_contain:
468
+ - "data preservation"
469
+ - "existing"
470
+ - "intact"
471
+ - "backward compatible"
472
+ must_not_contain:
473
+ - "data loss"
474
+ - "corrupted"
475
+ severity_classification: critical
476
+
477
+ validation:
478
+ schema_check: true
479
+ keyword_match_threshold: 0.7
480
+
481
+ # ---------------------------------------------------------------------------
482
+ # CATEGORY: Transaction Testing (ACID)
483
+ # ---------------------------------------------------------------------------
484
+
485
+ - id: tc009_atomicity_test
486
+ description: "Test transaction atomicity - all or nothing"
487
+ category: transaction
488
+ priority: critical
489
+
490
+ input:
491
+ code: |
492
+ test('transaction rolls back on error', async () => {
493
+ const initialCount = await db.users.count();
494
+
495
+ try {
496
+ await db.transaction(async (trx) => {
497
+ await trx('users').insert({ email: 'user1@example.com' });
498
+ await trx('users').insert({ email: 'user2@example.com' });
499
+ throw new Error('Intentional rollback');
500
+ });
501
+ } catch (error) { /* Expected */ }
502
+
503
+ // Count should be unchanged
504
+ expect(await db.users.count()).toBe(initialCount);
505
+ });
506
+ context:
507
+ database: postgresql
508
+ framework: knex
509
+ test_type: transaction
510
+
511
+ expected_output:
512
+ must_contain:
513
+ - "atomicity"
514
+ - "rollback"
515
+ - "transaction"
516
+ - "all or nothing"
517
+ must_not_contain:
518
+ - "partial commit"
519
+ must_match_regex:
520
+ - "ACID|atomic"
521
+ severity_classification: critical
522
+
523
+ validation:
524
+ schema_check: true
525
+ keyword_match_threshold: 0.8
526
+ reasoning_quality_min: 0.8
527
+
528
+ - id: tc010_isolation_test
529
+ description: "Test transaction isolation - concurrent access"
530
+ category: transaction
531
+ priority: critical
532
+
533
+ input:
534
+ code: |
535
+ test('concurrent transactions isolated', async () => {
536
+ const user = await db.users.create({ balance: 100 });
537
+
538
+ // Two concurrent withdrawals - race condition test
539
+ const results = await Promise.all([
540
+ db.transaction(async (trx) => {
541
+ const current = await trx('users').where({ id: user.id }).first();
542
+ if (current.balance >= 50) {
543
+ await trx('users').where({ id: user.id }).update({
544
+ balance: current.balance - 50
545
+ });
546
+ return 'success';
547
+ }
548
+ return 'insufficient';
549
+ }),
550
+ db.transaction(async (trx) => {
551
+ const current = await trx('users').where({ id: user.id }).first();
552
+ if (current.balance >= 50) {
553
+ await trx('users').where({ id: user.id }).update({
554
+ balance: current.balance - 50
555
+ });
556
+ return 'success';
557
+ }
558
+ return 'insufficient';
559
+ })
560
+ ]);
561
+
562
+ const final = await db.users.findOne({ id: user.id });
563
+ // With proper isolation, one should fail or balance should be 0
564
+ expect(final.balance).toBeGreaterThanOrEqual(0);
565
+ });
566
+ context:
567
+ database: postgresql
568
+ framework: knex
569
+ test_type: transaction
570
+ isolation_level: REPEATABLE_READ
571
+
572
+ expected_output:
573
+ must_contain:
574
+ - "isolation"
575
+ - "concurrent"
576
+ - "race condition"
577
+ - "lock"
578
+ must_match_regex:
579
+ - "READ_COMMITTED|REPEATABLE_READ|SERIALIZABLE"
580
+ severity_classification: critical
581
+
582
+ validation:
583
+ schema_check: true
584
+ keyword_match_threshold: 0.7
585
+
586
+ - id: tc011_deadlock_detection
587
+ description: "Test deadlock detection and resolution"
588
+ category: transaction
589
+ priority: high
590
+
591
+ input:
592
+ code: |
593
+ test('deadlock is detected and handled', async () => {
594
+ // Create potential deadlock scenario
595
+ const tx1 = db.transaction(async (trx) => {
596
+ await trx('accounts').where({ id: 1 }).update({ balance: 100 });
597
+ await delay(100);
598
+ await trx('accounts').where({ id: 2 }).update({ balance: 200 });
599
+ });
600
+
601
+ const tx2 = db.transaction(async (trx) => {
602
+ await trx('accounts').where({ id: 2 }).update({ balance: 150 });
603
+ await delay(100);
604
+ await trx('accounts').where({ id: 1 }).update({ balance: 250 });
605
+ });
606
+
607
+ // One should fail with deadlock, other should succeed
608
+ const results = await Promise.allSettled([tx1, tx2]);
609
+ const failures = results.filter(r => r.status === 'rejected');
610
+ expect(failures.some(f => f.reason.message.includes('deadlock'))).toBe(true);
611
+ });
612
+ context:
613
+ database: postgresql
614
+ framework: knex
615
+ test_type: transaction
616
+
617
+ expected_output:
618
+ must_contain:
619
+ - "deadlock"
620
+ - "detection"
621
+ - "retry"
622
+ - "conflict"
623
+ severity_classification: high
624
+
625
+ validation:
626
+ schema_check: true
627
+ keyword_match_threshold: 0.7
628
+
629
+ # ---------------------------------------------------------------------------
630
+ # CATEGORY: Performance Testing
631
+ # ---------------------------------------------------------------------------
632
+
633
+ - id: tc012_slow_query_detection
634
+ description: "Detect slow queries without proper indexing"
635
+ category: performance
636
+ priority: high
637
+
638
+ input:
639
+ code: |
640
+ -- Slow query: full table scan on large table
641
+ SELECT * FROM orders
642
+ WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31'
643
+ AND status = 'pending';
644
+
645
+ -- Table has 10 million rows, no index on created_at or status
646
+ EXPLAIN ANALYZE SELECT * FROM orders
647
+ WHERE created_at BETWEEN '2024-01-01' AND '2024-12-31';
648
+ -- Output: Seq Scan on orders (cost=0.00..250000.00 rows=5000000)
649
+ context:
650
+ database: postgresql
651
+ test_type: performance
652
+ table_size: 10000000
653
+
654
+ expected_output:
655
+ must_contain:
656
+ - "slow query"
657
+ - "full table scan"
658
+ - "index"
659
+ - "Seq Scan"
660
+ must_match_regex:
661
+ - "CREATE INDEX"
662
+ severity_classification: high
663
+ finding_count:
664
+ min: 1
665
+
666
+ validation:
667
+ schema_check: true
668
+ keyword_match_threshold: 0.8
669
+
670
+ - id: tc013_n_plus_one_detection
671
+ description: "Detect N+1 query problem"
672
+ category: performance
673
+ priority: high
674
+
675
+ input:
676
+ code: |
677
+ // N+1 query problem
678
+ const users = await User.findAll(); // 1 query
679
+
680
+ for (const user of users) {
681
+ const orders = await Order.findAll({
682
+ where: { userId: user.id }
683
+ }); // N queries!
684
+ user.orderCount = orders.length;
685
+ }
686
+
687
+ // Should be:
688
+ // const users = await User.findAll({
689
+ // include: [{ model: Order }]
690
+ // });
691
+ context:
692
+ database: postgresql
693
+ framework: sequelize
694
+ test_type: performance
695
+
696
+ expected_output:
697
+ must_contain:
698
+ - "N+1"
699
+ - "query"
700
+ - "include"
701
+ - "eager loading"
702
+ - "join"
703
+ must_not_contain:
704
+ - "efficient"
705
+ - "optimal"
706
+ severity_classification: high
707
+
708
+ validation:
709
+ schema_check: true
710
+ keyword_match_threshold: 0.8
711
+
712
+ - id: tc014_connection_pool_exhaustion
713
+ description: "Detect connection pool exhaustion risk"
714
+ category: performance
715
+ priority: critical
716
+
717
+ input:
718
+ code: |
719
+ // Connection leak - connections not released
720
+ async function getUserData(userId) {
721
+ const connection = await pool.getConnection();
722
+ const user = await connection.query('SELECT * FROM users WHERE id = ?', [userId]);
723
+ // Missing: connection.release()
724
+ return user;
725
+ }
726
+
727
+ // Called in a loop without releasing connections
728
+ for (let i = 0; i < 1000; i++) {
729
+ await getUserData(i);
730
+ }
731
+ context:
732
+ database: mysql
733
+ framework: mysql2
734
+ test_type: performance
735
+ pool_size: 10
736
+
737
+ expected_output:
738
+ must_contain:
739
+ - "connection"
740
+ - "pool"
741
+ - "leak"
742
+ - "release"
743
+ - "exhaustion"
744
+ severity_classification: critical
745
+
746
+ validation:
747
+ schema_check: true
748
+ keyword_match_threshold: 0.8
749
+
750
+ # ---------------------------------------------------------------------------
751
+ # CATEGORY: Multi-Database Support
752
+ # ---------------------------------------------------------------------------
753
+
754
+ - id: tc015_mongodb_schema_validation
755
+ description: "Test MongoDB schema validation"
756
+ category: schema
757
+ priority: high
758
+
759
+ input:
760
+ code: |
761
+ // MongoDB collection without schema validation
762
+ db.createCollection("users");
763
+
764
+ // Can insert inconsistent documents
765
+ db.users.insertOne({ email: "user@example.com", age: 25 });
766
+ db.users.insertOne({ email: 123, age: "twenty" }); // Wrong types!
767
+ db.users.insertOne({ name: "John" }); // Missing email
768
+
769
+ // Should have JSON Schema validation:
770
+ db.createCollection("users", {
771
+ validator: {
772
+ $jsonSchema: {
773
+ bsonType: "object",
774
+ required: ["email"],
775
+ properties: {
776
+ email: { bsonType: "string" },
777
+ age: { bsonType: "int" }
778
+ }
779
+ }
780
+ }
781
+ });
782
+ context:
783
+ database: mongodb
784
+ test_type: schema
785
+
786
+ expected_output:
787
+ must_contain:
788
+ - "schema validation"
789
+ - "MongoDB"
790
+ - "jsonSchema"
791
+ - "bsonType"
792
+ severity_classification: high
793
+
794
+ validation:
795
+ schema_check: true
796
+ keyword_match_threshold: 0.7
797
+
798
+ - id: tc016_sqlite_foreign_key_enforcement
799
+ description: "Test SQLite foreign key enforcement"
800
+ category: integrity
801
+ priority: high
802
+
803
+ input:
804
+ code: |
805
+ -- SQLite: Foreign keys are OFF by default!
806
+ CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);
807
+ CREATE TABLE orders (
808
+ id INTEGER PRIMARY KEY,
809
+ user_id INTEGER REFERENCES users(id)
810
+ );
811
+
812
+ -- This succeeds even though user 999 doesn't exist
813
+ INSERT INTO orders (user_id) VALUES (999);
814
+
815
+ -- Need to enable: PRAGMA foreign_keys = ON;
816
+ context:
817
+ database: sqlite
818
+ test_type: integrity
819
+
820
+ expected_output:
821
+ must_contain:
822
+ - "foreign_keys"
823
+ - "PRAGMA"
824
+ - "SQLite"
825
+ - "enforcement"
826
+ must_match_regex:
827
+ - "PRAGMA foreign_keys\\s*=\\s*ON"
828
+ severity_classification: high
829
+
830
+ validation:
831
+ schema_check: true
832
+ keyword_match_threshold: 0.8
833
+
834
+ # ---------------------------------------------------------------------------
835
+ # CATEGORY: Negative Tests (Secure/Valid Code)
836
+ # ---------------------------------------------------------------------------
837
+
838
+ - id: tc017_valid_schema_no_false_positives
839
+ description: "Verify well-designed schema is NOT flagged as problematic"
840
+ category: negative
841
+ priority: critical
842
+
843
+ input:
844
+ code: |
845
+ CREATE TABLE users (
846
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
847
+ email VARCHAR(255) UNIQUE NOT NULL,
848
+ password_hash VARCHAR(255) NOT NULL,
849
+ created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
850
+ updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
851
+ );
852
+
853
+ CREATE TABLE orders (
854
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
855
+ user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
856
+ total DECIMAL(10,2) NOT NULL CHECK (total >= 0),
857
+ status VARCHAR(20) NOT NULL DEFAULT 'pending'
858
+ CHECK (status IN ('pending', 'processing', 'shipped', 'delivered', 'cancelled')),
859
+ created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
860
+ );
861
+
862
+ CREATE INDEX idx_orders_user_id ON orders(user_id);
863
+ CREATE INDEX idx_orders_status ON orders(status);
864
+ CREATE INDEX idx_orders_created_at ON orders(created_at);
865
+ context:
866
+ database: postgresql
867
+ framework: prisma
868
+ environment: production
869
+
870
+ expected_output:
871
+ must_contain:
872
+ - "valid"
873
+ - "well-designed"
874
+ - "proper constraints"
875
+ must_not_contain:
876
+ - "missing constraint"
877
+ - "critical"
878
+ - "high severity"
879
+ - "orphaned"
880
+ finding_count:
881
+ max: 2 # Allow informational findings only
882
+
883
+ validation:
884
+ schema_check: true
885
+ keyword_match_threshold: 0.6
886
+ allow_partial: true
887
+
888
+ - id: tc018_proper_migration_pattern
889
+ description: "Verify proper migration pattern is recognized"
890
+ category: negative
891
+ priority: high
892
+
893
+ input:
894
+ code: |
895
+ // Proper reversible migration with data preservation
896
+ exports.up = async function(knex) {
897
+ // Check if column exists before adding
898
+ const hasColumn = await knex.schema.hasColumn('users', 'phone');
899
+ if (!hasColumn) {
900
+ await knex.schema.alterTable('users', (table) => {
901
+ table.string('phone').nullable();
902
+ });
903
+ }
904
+ };
905
+
906
+ exports.down = async function(knex) {
907
+ const hasColumn = await knex.schema.hasColumn('users', 'phone');
908
+ if (hasColumn) {
909
+ await knex.schema.alterTable('users', (table) => {
910
+ table.dropColumn('phone');
911
+ });
912
+ }
913
+ };
914
+ context:
915
+ database: postgresql
916
+ framework: knex
917
+ test_type: migration
918
+
919
+ expected_output:
920
+ must_contain:
921
+ - "reversible"
922
+ - "idempotent"
923
+ - "safe"
924
+ must_not_contain:
925
+ - "irreversible"
926
+ - "data loss"
927
+ - "critical"
928
+ severity_classification: info
929
+
930
+ validation:
931
+ schema_check: true
932
+ allow_partial: true
933
+
934
+ # =============================================================================
935
+ # SUCCESS CRITERIA
936
+ # =============================================================================
937
+
938
+ success_criteria:
939
+ # Overall pass rate (90% of tests must pass)
940
+ pass_rate: 0.9
941
+
942
+ # Critical tests must ALL pass (100%)
943
+ critical_pass_rate: 1.0
944
+
945
+ # Average reasoning quality score
946
+ avg_reasoning_quality: 0.75
947
+
948
+ # Maximum suite execution time (5 minutes)
949
+ max_execution_time_ms: 300000
950
+
951
+ # Maximum variance between model results (15%)
952
+ cross_model_variance: 0.15
953
+
954
+ # =============================================================================
955
+ # METADATA
956
+ # =============================================================================
957
+
958
+ metadata:
959
+ author: "qe-test-data-architect"
960
+ created: "2026-02-02"
961
+ last_updated: "2026-02-02"
962
+ coverage_target: >
963
+ Database testing coverage: Schema validation (missing constraints, indexes),
964
+ Data integrity (unique, FK, check, NOT NULL), Migration testing (forward,
965
+ rollback, data preservation), Transaction testing (ACID: atomicity, isolation,
966
+ deadlock), Performance (slow queries, N+1, connection pools). Multi-database
967
+ support: PostgreSQL, MySQL, MongoDB, SQLite. 18 test cases with 90% pass rate
968
+ requirement and 100% critical pass rate for ACID and migration tests.