@dzhechkov/keysarium 1.7.15 → 1.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +214 -54
- package/README.md +29 -6
- package/README.ru.md +17 -0
- package/package.json +3 -3
- package/sbom.json +455 -55
- package/src/cli.js +32 -2
- package/src/commands/doctor.js +25 -1
- package/src/commands/init.js +56 -12
- package/src/commands/update.js +30 -3
- package/src/utils.js +7 -0
- package/templates/.agents/skills/casarium/SKILL.md +29 -0
- package/templates/.claude/agents/code-goal-planner.md +447 -0
- package/templates/.claude/commands/casarium.md +3 -1
- package/templates/.claude/commands/cjm-prototype.md +2 -2
- package/templates/.claude/commands/research.md +4 -1
- package/templates/.claude/rules/checkpoint-protocol.md +2 -1
- package/templates/.claude/rules/research-quality.md +11 -6
- package/templates/.claude/shards/phase-2-research.shard.md +12 -8
- package/templates/.claude/shards/phase-25-cjm.shard.md +5 -3
- package/templates/.claude/skills/feature-adr/SKILL.md +17 -15
- package/templates/.claude/skills/feature-adr/modules/03-adr.md +1 -1
- package/templates/.claude/skills/feature-adr/modules/03.5-ideation-swarm.md +1 -1
- package/templates/.claude/skills/feature-adr/modules/06-implementation-plan.md +2 -2
- package/templates/.claude/skills/feature-adr/modules/08-qe.md +1 -1
- package/templates/.claude/skills/feature-adr/modules/09-fleet-qe.md +12 -12
- package/templates/.claude/skills/feature-adr/references/agentic-qe/LICENSE.agentic-qe +21 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/README.md +20 -16
- package/templates/.claude/skills/feature-adr/references/agentic-qe/UPSTREAM-MANIFEST.json +246 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-brutal-honesty-review/README.md +218 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/{brutal-honesty-review.md → qe-ref-brutal-honesty-review/REFERENCE.md} +14 -1
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-brutal-honesty-review/resources/assessment-rubrics.md +295 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-brutal-honesty-review/resources/review-template.md +102 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-brutal-honesty-review/schemas/output.json +291 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-brutal-honesty-review/scripts/assess-code.sh +179 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-brutal-honesty-review/scripts/assess-tests.sh +223 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-brutal-honesty-review/scripts/validate-config.json +34 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-enterprise-integration-testing/REFERENCE.md +758 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-enterprise-integration-testing/evals/enterprise-integration-testing.yaml +158 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-enterprise-integration-testing/schemas/output.json +74 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-enterprise-integration-testing/scripts/validate-config.json +25 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/REFERENCE.md +226 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/evals/qcsd-ideation-swarm.yaml +138 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/schemas/output.json +568 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/scripts/validate-config.json +25 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/01-flag-detection.md +57 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/02-core-agents.md +29 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/03-batch1-results.md +15 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/04-conditional-agents.md +23 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/05-decision-synthesis.md +29 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/06-report-generation.md +18 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/07-learning-persistence.md +29 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qcsd-ideation-swarm/steps/08-final-output.md +18 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/{qe-coverage-analysis.md → qe-ref-qe-coverage-analysis/REFERENCE.md} +42 -3
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qe-coverage-analysis/evals/qe-coverage-analysis.yaml +494 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qe-coverage-analysis/schemas/output.json +286 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qe-coverage-analysis/scripts/validate-config.json +42 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/{qe-requirements-validation.md → qe-ref-qe-requirements-validation/REFERENCE.md} +13 -3
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qe-requirements-validation/evals/qe-requirements-validation.yaml +598 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qe-requirements-validation/schemas/output.json +587 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-qe-requirements-validation/scripts/validate-config.json +37 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/{regression-testing.md → qe-ref-regression-testing/REFERENCE.md} +30 -49
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-regression-testing/evals/regression-testing.yaml +504 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-regression-testing/schemas/output.json +311 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-regression-testing/scripts/validate-config.json +50 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/{risk-based-testing.md → qe-ref-risk-based-testing/REFERENCE.md} +23 -52
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-risk-based-testing/evals/risk-based-testing.yaml +141 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-risk-based-testing/schemas/output.json +480 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-risk-based-testing/scripts/validate-config.json +25 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/{shift-left-testing.md → qe-ref-shift-left-testing/REFERENCE.md} +33 -88
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-shift-left-testing/evals/shift-left-testing.yaml +145 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-shift-left-testing/schemas/output.json +459 -0
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qe-ref-shift-left-testing/scripts/validate-config.json +25 -0
- package/templates/CLAUDE.md +3 -1
- package/templates/docs/feature-adr/04-admin-guide.md +1 -1
- package/templates/docs/improvement-directions.md +3 -3
- package/templates/docs/keysarium/02-admin-guide.md +2 -2
- package/templates/docs/keysarium/03-user-guide.md +3 -3
- package/templates/docs/keysarium/05-architecture.md +9 -8
- package/templates/docs/keysarium/06-user-flows.md +4 -4
- package/templates/lib/platform-templates/copilot.md +1 -1
- package/templates/lib/platform-templates/opencode.md +1 -1
- package/templates/.claude/skills/feature-adr/references/agentic-qe/enterprise-integration-testing.md +0 -111
- package/templates/.claude/skills/feature-adr/references/agentic-qe/qcsd-ideation-swarm.md +0 -126
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Step 4: Spawn Conditional Agents (Parallel Batch 2)
|
|
2
|
+
|
|
3
|
+
## Prerequisites
|
|
4
|
+
- Step 3 completed, flags from Step 1
|
|
5
|
+
|
|
6
|
+
## Instructions
|
|
7
|
+
|
|
8
|
+
| Flag | Agent | Output File |
|
|
9
|
+
|------|-------|-------------|
|
|
10
|
+
| HAS_UI | qe-accessibility-auditor | `05-accessibility-audit.md` |
|
|
11
|
+
| HAS_SECURITY | qe-security-auditor | `06-security-audit.md` |
|
|
12
|
+
| HAS_UX | qe-qx-partner | `07-ux-assessment.md` |
|
|
13
|
+
| HAS_MIDDLEWARE | qe-middleware-validator | `08-middleware-validation.md` |
|
|
14
|
+
| HAS_SAP_INTEGRATION | qe-sap-rfc-tester | `09-sap-assessment.md` |
|
|
15
|
+
| HAS_AUTHORIZATION | qe-sod-analyzer | `10-sod-analysis.md` |
|
|
16
|
+
|
|
17
|
+
If ALL flags FALSE, skip to Step 5.
|
|
18
|
+
|
|
19
|
+
## Success Criteria
|
|
20
|
+
- [ ] All flagged agents completed
|
|
21
|
+
|
|
22
|
+
## Navigation
|
|
23
|
+
- On success: proceed to Step 5 by reading `steps/05-decision-synthesis.md`
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Step 5: Synthesize Results & Determine Recommendation
|
|
2
|
+
|
|
3
|
+
## Prerequisites
|
|
4
|
+
- Core and conditional results available
|
|
5
|
+
|
|
6
|
+
## Instructions
|
|
7
|
+
|
|
8
|
+
### Apply GO/CONDITIONAL/NO-GO logic EXACTLY
|
|
9
|
+
|
|
10
|
+
**NO-GO conditions (ANY triggers NO-GO):**
|
|
11
|
+
- Testability score < 30%
|
|
12
|
+
- Critical requirements gaps with no mitigation
|
|
13
|
+
- Security risks rated Critical with no mitigation path
|
|
14
|
+
- No measurable acceptance criteria
|
|
15
|
+
|
|
16
|
+
**GO conditions (ALL required):**
|
|
17
|
+
- Testability score >= 70%
|
|
18
|
+
- All HTSM categories assessed
|
|
19
|
+
- Risk assessment complete with mitigations
|
|
20
|
+
- Requirements validation passed
|
|
21
|
+
- No unresolved critical issues
|
|
22
|
+
|
|
23
|
+
**Default: CONDITIONAL**
|
|
24
|
+
|
|
25
|
+
## Success Criteria
|
|
26
|
+
- [ ] FINAL RECOMMENDATION: GO / CONDITIONAL / NO-GO
|
|
27
|
+
|
|
28
|
+
## Navigation
|
|
29
|
+
- On success: proceed to Step 6 by reading `steps/06-report-generation.md`
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Step 6: Generate Ideation Report
|
|
2
|
+
|
|
3
|
+
## Prerequisites
|
|
4
|
+
- Step 5 completed
|
|
5
|
+
|
|
6
|
+
## Instructions
|
|
7
|
+
|
|
8
|
+
Generate the Ideation Report (HTML if html_output flag is set, otherwise Markdown).
|
|
9
|
+
Save to `${OUTPUT_FOLDER}/01-executive-summary.md` (or .html).
|
|
10
|
+
|
|
11
|
+
Include: Executive Summary, Quality Criteria (HTSM), Risk Assessment, Requirements
|
|
12
|
+
Validation, conditional sections, Recommended Actions, Follow-up Recommendations.
|
|
13
|
+
|
|
14
|
+
## Success Criteria
|
|
15
|
+
- [ ] Complete report saved
|
|
16
|
+
|
|
17
|
+
## Navigation
|
|
18
|
+
- On success: proceed to Step 7 by reading `steps/07-learning-persistence.md`
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Step 7: Store Learnings & Persist State
|
|
2
|
+
|
|
3
|
+
## Prerequisites
|
|
4
|
+
- Step 6 completed
|
|
5
|
+
|
|
6
|
+
## Instructions
|
|
7
|
+
|
|
8
|
+
### ALWAYS RUN.
|
|
9
|
+
|
|
10
|
+
Store ideation findings:
|
|
11
|
+
```bash
|
|
12
|
+
aqe memory store \
|
|
13
|
+
--key "qcsd-ideation-${epicId}-${Date.now()}" \
|
|
14
|
+
--namespace "qcsd-ideation" \
|
|
15
|
+
--value '{...}' \
|
|
16
|
+
--json
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Save to `${OUTPUT_FOLDER}/11-learning-persistence.json`.
|
|
20
|
+
|
|
21
|
+
### Follow-up Recommendations
|
|
22
|
+
|
|
23
|
+
If HAS_VIDEO is TRUE, output a prominent box recommending `/a11y-ally` for video caption generation.
|
|
24
|
+
|
|
25
|
+
## Success Criteria
|
|
26
|
+
- [ ] Persistence completed
|
|
27
|
+
|
|
28
|
+
## Navigation
|
|
29
|
+
- On success: proceed to Step 8 by reading `steps/08-final-output.md`
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Step 8: Final Output & Completion Summary
|
|
2
|
+
|
|
3
|
+
## Prerequisites
|
|
4
|
+
- All steps completed
|
|
5
|
+
|
|
6
|
+
## Instructions
|
|
7
|
+
|
|
8
|
+
Display completion summary with: GO / CONDITIONAL / NO-GO recommendation,
|
|
9
|
+
testability score, quality criteria count, risk count, requirements validation result,
|
|
10
|
+
and all deliverables.
|
|
11
|
+
|
|
12
|
+
If HAS_VIDEO was TRUE, display prominent follow-up recommendation for /a11y-ally.
|
|
13
|
+
|
|
14
|
+
## Success Criteria
|
|
15
|
+
- [ ] Summary displayed
|
|
16
|
+
|
|
17
|
+
## Navigation
|
|
18
|
+
- Terminal step.
|
|
@@ -1,14 +1,25 @@
|
|
|
1
1
|
---
|
|
2
|
-
name: "
|
|
3
|
-
description: "
|
|
2
|
+
name: "qe-coverage-analysis"
|
|
3
|
+
description: "Analyzes test coverage data (Istanbul, c8, lcov) to identify uncovered lines, branches, and functions with risk-weighted gap detection. Use when analyzing coverage reports, identifying coverage gaps, comparing coverage between branches, or prioritizing which untested code to cover first."
|
|
4
4
|
trust_tier: 3
|
|
5
5
|
validation:
|
|
6
6
|
schema_path: schemas/output.json
|
|
7
7
|
validator_path: scripts/validate-config.json
|
|
8
8
|
eval_path: evals/qe-coverage-analysis.yaml
|
|
9
|
-
|
|
10
9
|
---
|
|
11
10
|
|
|
11
|
+
<!-- Vendored from https://github.com/proffesor-for-testing/agentic-qe at 0a6de058575cbe292afe220df105cc74120f7674 (MIT, Copyright (c) 2025 Agentic QE Contributors), 2026-10-23. Bundled references do not install aqe, MCP tools, pattern memory, or fleet runtime. See ../LICENSE.agentic-qe and ../README.md. -->
|
|
12
|
+
|
|
13
|
+
## Keysarium execution adapter
|
|
14
|
+
|
|
15
|
+
This pinned upstream body is methodology inside the bundled feature-ADR pipeline. In Reference Mode, the feature-ADR module that loaded this file owns orchestration and checkpoints. Use the reasoning and output shape below with the host's available tools. Do not treat upstream `aqe` CLI/MCP calls, named agent roles, browser tools, or example absolute paths as installed capabilities. Name a missing capability or evidence in the checkpoint instead of reporting that an upstream step ran. The schemas and eval files are bundled for explicit validation; bundling them does not run those checks automatically.
|
|
16
|
+
|
|
17
|
+
Relative paths such as `schemas/output.json`, `evals/...` and `steps/...` resolve from this skill directory under `references/agentic-qe/`. When Direct Mode has a separately installed `aqe` runtime, follow the feature-ADR module's direct-mode branch and verify that each required runtime capability is actually available.
|
|
18
|
+
|
|
19
|
+
The upstream `run-history.json` is execution history, not a reference contract, and is intentionally omitted. Do not run the upstream example that reads or writes it unless an installed aqe runtime supplies its own history.
|
|
20
|
+
|
|
21
|
+
<!-- BEGIN AGENTIC-QE UPSTREAM BODY -->
|
|
22
|
+
|
|
12
23
|
# QE Coverage Analysis
|
|
13
24
|
|
|
14
25
|
## Purpose
|
|
@@ -186,8 +197,36 @@ quality_gates:
|
|
|
186
197
|
- alert_on_decline: 3 # consecutive PRs
|
|
187
198
|
```
|
|
188
199
|
|
|
200
|
+
## Run History
|
|
201
|
+
|
|
202
|
+
After each coverage analysis, append results to `run-history.json` in this skill directory:
|
|
203
|
+
```bash
|
|
204
|
+
# Read current history, append new entry, write back
|
|
205
|
+
node -e "
|
|
206
|
+
const fs = require('fs');
|
|
207
|
+
const h = JSON.parse(fs.readFileSync('.claude/skills/qe-coverage-analysis/run-history.json'));
|
|
208
|
+
h.runs.push({date: new Date().toISOString().split('T')[0], statements_pct: STATEMENTS, branches_pct: BRANCHES, gaps_found: GAPS});
|
|
209
|
+
fs.writeFileSync('.claude/skills/qe-coverage-analysis/run-history.json', JSON.stringify(h, null, 2));
|
|
210
|
+
"
|
|
211
|
+
```
|
|
212
|
+
Read `run-history.json` before each run to detect trends (e.g., "coverage dropped 3 consecutive times").
|
|
213
|
+
|
|
214
|
+
## Skill Composition
|
|
215
|
+
|
|
216
|
+
- **Coverage dropped?** → Use `/coverage-drop-investigator` to trace the cause
|
|
217
|
+
- **Need more tests** → Use `/qe-test-generation` to fill gaps
|
|
218
|
+
- **Validate quality** → Use `/mutation-testing` to ensure coverage means quality
|
|
219
|
+
- **Ship decision** → Feed into `/qe-quality-assessment` for deployment readiness
|
|
220
|
+
|
|
221
|
+
## Gotchas
|
|
222
|
+
|
|
223
|
+
- High line coverage does NOT mean good tests — 100% coverage with 0% assertions is common agent output. Use mutation testing to verify
|
|
224
|
+
- coverage-analysis domain has 86% success rate — 14% of runs fail on initialization. Always verify results and have fallback plan (e.g. manual coverage tools)
|
|
225
|
+
- Self-learning pipeline may silently stop learning (statusline frozen for days) — only human inspection catches this
|
|
226
|
+
|
|
189
227
|
## Coordination
|
|
190
228
|
|
|
191
229
|
**Primary Agents**: qe-coverage-specialist, qe-coverage-analyzer, qe-gap-detector
|
|
192
230
|
**Coordinator**: qe-coverage-coordinator
|
|
193
231
|
**Related Skills**: qe-test-generation, qe-quality-assessment
|
|
232
|
+
<!-- END AGENTIC-QE UPSTREAM BODY -->
|
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# AQE Skill Evaluation Test Suite: QE Coverage Analysis v1.0.0
|
|
3
|
+
# =============================================================================
|
|
4
|
+
#
|
|
5
|
+
# Comprehensive evaluation suite for the qe-coverage-analysis skill.
|
|
6
|
+
# Tests O(log n) sublinear gap detection, risk-weighted analysis, differential
|
|
7
|
+
# coverage, test prioritization, and coverage trend analysis.
|
|
8
|
+
#
|
|
9
|
+
# Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
|
|
10
|
+
# Validator: .claude/skills/qe-coverage-analysis/scripts/validate-config.json
|
|
11
|
+
#
|
|
12
|
+
# Coverage:
|
|
13
|
+
# - Sublinear gap detection (O(log n) complexity)
|
|
14
|
+
# - Risk-weighted coverage scoring
|
|
15
|
+
# - Differential coverage (branch diffs)
|
|
16
|
+
# - Test prioritization by impact
|
|
17
|
+
# - Coverage regression detection
|
|
18
|
+
# - Quality gate enforcement
|
|
19
|
+
#
|
|
20
|
+
# =============================================================================
|
|
21
|
+
|
|
22
|
+
skill: qe-coverage-analysis
|
|
23
|
+
version: 1.0.0
|
|
24
|
+
description: >
|
|
25
|
+
Comprehensive evaluation suite for the qe-coverage-analysis skill.
|
|
26
|
+
Tests O(log n) sublinear coverage gap detection, risk-weighted analysis,
|
|
27
|
+
differential coverage scoring, intelligent test prioritization, and coverage
|
|
28
|
+
trend analysis with quality gate enforcement.
|
|
29
|
+
|
|
30
|
+
# =============================================================================
|
|
31
|
+
# Multi-Model Configuration
|
|
32
|
+
# =============================================================================
|
|
33
|
+
|
|
34
|
+
models_to_test:
|
|
35
|
+
- claude-sonnet-4-6 # Primary (high accuracy expected)
|
|
36
|
+
- claude-haiku-4-5 # Fast model (minimum quality floor)
|
|
37
|
+
|
|
38
|
+
# =============================================================================
|
|
39
|
+
# MCP Integration Configuration
|
|
40
|
+
# =============================================================================
|
|
41
|
+
|
|
42
|
+
mcp_integration:
|
|
43
|
+
enabled: true
|
|
44
|
+
namespace: skill-validation
|
|
45
|
+
|
|
46
|
+
query_patterns: true
|
|
47
|
+
track_outcomes: true
|
|
48
|
+
store_patterns: true
|
|
49
|
+
share_learning: true
|
|
50
|
+
update_quality_gate: true
|
|
51
|
+
|
|
52
|
+
target_agents:
|
|
53
|
+
- qe-learning-coordinator
|
|
54
|
+
- qe-queen-coordinator
|
|
55
|
+
- qe-coverage-specialist
|
|
56
|
+
- qe-gap-detector
|
|
57
|
+
|
|
58
|
+
# =============================================================================
|
|
59
|
+
# ReasoningBank Learning Configuration
|
|
60
|
+
# =============================================================================
|
|
61
|
+
|
|
62
|
+
learning:
|
|
63
|
+
store_success_patterns: true
|
|
64
|
+
store_failure_patterns: true
|
|
65
|
+
pattern_ttl_days: 90
|
|
66
|
+
min_confidence_to_store: 0.7
|
|
67
|
+
cross_model_comparison: true
|
|
68
|
+
|
|
69
|
+
# =============================================================================
|
|
70
|
+
# Result Format Configuration
|
|
71
|
+
# =============================================================================
|
|
72
|
+
|
|
73
|
+
result_format:
|
|
74
|
+
json_output: true
|
|
75
|
+
markdown_report: true
|
|
76
|
+
include_raw_output: false
|
|
77
|
+
include_timing: true
|
|
78
|
+
include_token_usage: true
|
|
79
|
+
|
|
80
|
+
# =============================================================================
|
|
81
|
+
# Environment Setup
|
|
82
|
+
# =============================================================================
|
|
83
|
+
|
|
84
|
+
setup:
|
|
85
|
+
required_tools:
|
|
86
|
+
- jq
|
|
87
|
+
environment_variables:
|
|
88
|
+
COVERAGE_ALGORITHM: "sublinear"
|
|
89
|
+
RISK_WEIGHTING: "enabled"
|
|
90
|
+
MIN_COVERAGE_THRESHOLD: "80"
|
|
91
|
+
fixtures: []
|
|
92
|
+
|
|
93
|
+
# =============================================================================
|
|
94
|
+
# TEST CASES
|
|
95
|
+
# =============================================================================
|
|
96
|
+
|
|
97
|
+
test_cases:
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
# CATEGORY: Sublinear Gap Detection
|
|
100
|
+
# ---------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
- id: tc001_sublinear_gap_detection
|
|
103
|
+
description: "Perform O(log n) gap detection on large codebase"
|
|
104
|
+
category: gap_detection
|
|
105
|
+
priority: critical
|
|
106
|
+
|
|
107
|
+
input:
|
|
108
|
+
prompt: |
|
|
109
|
+
Analyze coverage for a large codebase (1000+ files) using O(log n) algorithm:
|
|
110
|
+
1. Use sampling-based analysis with 95% confidence
|
|
111
|
+
2. Identify coverage gaps efficiently
|
|
112
|
+
3. Report uncovered critical paths
|
|
113
|
+
4. Suggest tests for maximum impact
|
|
114
|
+
|
|
115
|
+
How would you achieve sublinear complexity?
|
|
116
|
+
context:
|
|
117
|
+
source: "src/**/*.ts"
|
|
118
|
+
algorithm: "sublinear"
|
|
119
|
+
confidence: 0.95
|
|
120
|
+
max_samples: 1000
|
|
121
|
+
|
|
122
|
+
expected_output:
|
|
123
|
+
must_contain:
|
|
124
|
+
- "O(log n)"
|
|
125
|
+
- "gap"
|
|
126
|
+
- "uncovered"
|
|
127
|
+
- "critical"
|
|
128
|
+
- "sampling"
|
|
129
|
+
must_not_contain:
|
|
130
|
+
- "linear"
|
|
131
|
+
- "exhaustive"
|
|
132
|
+
severity_classification: critical
|
|
133
|
+
finding_count:
|
|
134
|
+
min: 1
|
|
135
|
+
|
|
136
|
+
validation:
|
|
137
|
+
schema_check: true
|
|
138
|
+
keyword_match_threshold: 0.8
|
|
139
|
+
reasoning_quality_min: 0.75
|
|
140
|
+
|
|
141
|
+
- id: tc002_coverage_gap_prioritization
|
|
142
|
+
description: "Identify and prioritize coverage gaps by impact"
|
|
143
|
+
category: gap_detection
|
|
144
|
+
priority: critical
|
|
145
|
+
|
|
146
|
+
input:
|
|
147
|
+
prompt: |
|
|
148
|
+
After detecting coverage gaps, prioritize by:
|
|
149
|
+
1. Business criticality (payment, auth, data)
|
|
150
|
+
2. Code complexity (cyclomatic > 10)
|
|
151
|
+
3. Recent bug frequency (bugs in last 90 days)
|
|
152
|
+
4. Change frequency (modified recently)
|
|
153
|
+
|
|
154
|
+
Which gap should be tested first and why?
|
|
155
|
+
context:
|
|
156
|
+
gap_prioritization: true
|
|
157
|
+
business_impact: true
|
|
158
|
+
change_analysis: true
|
|
159
|
+
|
|
160
|
+
expected_output:
|
|
161
|
+
must_contain:
|
|
162
|
+
- "priority"
|
|
163
|
+
- "critical"
|
|
164
|
+
- "complexity"
|
|
165
|
+
- "bug"
|
|
166
|
+
- "impact"
|
|
167
|
+
must_not_contain:
|
|
168
|
+
- "random"
|
|
169
|
+
- "arbitrary"
|
|
170
|
+
severity_classification: critical
|
|
171
|
+
|
|
172
|
+
validation:
|
|
173
|
+
schema_check: true
|
|
174
|
+
keyword_match_threshold: 0.8
|
|
175
|
+
|
|
176
|
+
# ---------------------------------------------------------------------------
|
|
177
|
+
# CATEGORY: Risk-Weighted Coverage
|
|
178
|
+
# ---------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
- id: tc003_risk_weighted_scoring
|
|
181
|
+
description: "Calculate risk-weighted coverage scores"
|
|
182
|
+
category: risk_analysis
|
|
183
|
+
priority: critical
|
|
184
|
+
|
|
185
|
+
input:
|
|
186
|
+
prompt: |
|
|
187
|
+
Calculate risk-weighted coverage for a module using factors:
|
|
188
|
+
1. Complexity weight: 0.3 (cyclomatic > 10)
|
|
189
|
+
2. Change frequency weight: 0.25 (modified in 90d)
|
|
190
|
+
3. Bug history weight: 0.25 (bugs in 180d)
|
|
191
|
+
4. Criticality weight: 0.2 (business-critical tag)
|
|
192
|
+
|
|
193
|
+
For each uncovered section, calculate risk score 0-1.
|
|
194
|
+
How would you identify high-risk uncovered code?
|
|
195
|
+
context:
|
|
196
|
+
complexity_weight: 0.3
|
|
197
|
+
change_frequency_weight: 0.25
|
|
198
|
+
bug_history_weight: 0.25
|
|
199
|
+
criticality_weight: 0.2
|
|
200
|
+
|
|
201
|
+
expected_output:
|
|
202
|
+
must_contain:
|
|
203
|
+
- "risk"
|
|
204
|
+
- "weight"
|
|
205
|
+
- "score"
|
|
206
|
+
- "complexity"
|
|
207
|
+
- "bug"
|
|
208
|
+
- "critical"
|
|
209
|
+
must_not_contain:
|
|
210
|
+
- "simple"
|
|
211
|
+
- "low priority"
|
|
212
|
+
severity_classification: critical
|
|
213
|
+
|
|
214
|
+
validation:
|
|
215
|
+
schema_check: true
|
|
216
|
+
keyword_match_threshold: 0.8
|
|
217
|
+
reasoning_quality_min: 0.75
|
|
218
|
+
|
|
219
|
+
- id: tc004_high_risk_uncovered_identification
|
|
220
|
+
description: "Find high-risk code with no test coverage"
|
|
221
|
+
category: risk_analysis
|
|
222
|
+
priority: critical
|
|
223
|
+
|
|
224
|
+
input:
|
|
225
|
+
prompt: |
|
|
226
|
+
Identify high-risk uncovered code:
|
|
227
|
+
1. Payment processing logic - 0% coverage, complexity 15
|
|
228
|
+
2. Authentication middleware - 0% coverage, 10 bugs in 90d
|
|
229
|
+
3. Error handling - 50% coverage
|
|
230
|
+
4. Logging - 0% coverage, low complexity
|
|
231
|
+
|
|
232
|
+
Which requires urgent attention? Why?
|
|
233
|
+
context:
|
|
234
|
+
risk_threshold: 0.7
|
|
235
|
+
focus_uncovered: true
|
|
236
|
+
|
|
237
|
+
expected_output:
|
|
238
|
+
must_contain:
|
|
239
|
+
- "high-risk"
|
|
240
|
+
- "payment"
|
|
241
|
+
- "authentication"
|
|
242
|
+
- "urgent"
|
|
243
|
+
- "coverage"
|
|
244
|
+
finding_count:
|
|
245
|
+
min: 1
|
|
246
|
+
|
|
247
|
+
validation:
|
|
248
|
+
schema_check: true
|
|
249
|
+
keyword_match_threshold: 0.75
|
|
250
|
+
|
|
251
|
+
# ---------------------------------------------------------------------------
|
|
252
|
+
# CATEGORY: Differential Coverage
|
|
253
|
+
# ---------------------------------------------------------------------------
|
|
254
|
+
|
|
255
|
+
- id: tc005_differential_coverage_analysis
|
|
256
|
+
description: "Compare coverage between branches with quality gates"
|
|
257
|
+
category: differential
|
|
258
|
+
priority: critical
|
|
259
|
+
|
|
260
|
+
input:
|
|
261
|
+
prompt: |
|
|
262
|
+
Compare coverage between main and feature-branch:
|
|
263
|
+
1. New code coverage: must be >= 85%
|
|
264
|
+
2. Modified code coverage: must not decrease
|
|
265
|
+
3. Deleted code: ignore coverage
|
|
266
|
+
4. Overall coverage: must not regress > 2%
|
|
267
|
+
|
|
268
|
+
What happens if new code has 75% coverage?
|
|
269
|
+
context:
|
|
270
|
+
base_branch: "main"
|
|
271
|
+
head_branch: "feature-branch"
|
|
272
|
+
new_code_threshold: 0.85
|
|
273
|
+
modified_code_requirement: "maintain"
|
|
274
|
+
|
|
275
|
+
expected_output:
|
|
276
|
+
must_contain:
|
|
277
|
+
- "differential"
|
|
278
|
+
- "new code"
|
|
279
|
+
- "coverage"
|
|
280
|
+
- "regression"
|
|
281
|
+
- "quality gate"
|
|
282
|
+
must_not_contain:
|
|
283
|
+
- "pass"
|
|
284
|
+
- "acceptable"
|
|
285
|
+
severity_classification: critical
|
|
286
|
+
|
|
287
|
+
validation:
|
|
288
|
+
schema_check: true
|
|
289
|
+
keyword_match_threshold: 0.8
|
|
290
|
+
reasoning_quality_min: 0.75
|
|
291
|
+
|
|
292
|
+
- id: tc006_coverage_regression_detection
|
|
293
|
+
description: "Detect coverage regressions between releases"
|
|
294
|
+
category: differential
|
|
295
|
+
priority: high
|
|
296
|
+
|
|
297
|
+
input:
|
|
298
|
+
prompt: |
|
|
299
|
+
Detect coverage regressions:
|
|
300
|
+
- v1.0: 85% statement coverage
|
|
301
|
+
- v2.0: 80% statement coverage (5% regression)
|
|
302
|
+
|
|
303
|
+
How would you alert on:
|
|
304
|
+
1. Individual file regression > 10%
|
|
305
|
+
2. Overall regression > 2%
|
|
306
|
+
3. Critical module regression > 1%
|
|
307
|
+
|
|
308
|
+
Should this block merge?
|
|
309
|
+
context:
|
|
310
|
+
regression_detection: true
|
|
311
|
+
block_on_regression: true
|
|
312
|
+
|
|
313
|
+
expected_output:
|
|
314
|
+
must_contain:
|
|
315
|
+
- "regression"
|
|
316
|
+
- "detect"
|
|
317
|
+
- "block"
|
|
318
|
+
- "merge"
|
|
319
|
+
- "threshold"
|
|
320
|
+
severity_classification: high
|
|
321
|
+
|
|
322
|
+
validation:
|
|
323
|
+
schema_check: true
|
|
324
|
+
keyword_match_threshold: 0.75
|
|
325
|
+
|
|
326
|
+
# ---------------------------------------------------------------------------
|
|
327
|
+
# CATEGORY: Test Prioritization
|
|
328
|
+
# ---------------------------------------------------------------------------
|
|
329
|
+
|
|
330
|
+
- id: tc007_test_prioritization_strategy
|
|
331
|
+
description: "Prioritize tests to write based on coverage impact"
|
|
332
|
+
category: prioritization
|
|
333
|
+
priority: high
|
|
334
|
+
|
|
335
|
+
input:
|
|
336
|
+
prompt: |
|
|
337
|
+
For these uncovered code sections, estimate test writing impact:
|
|
338
|
+
1. UserService.validateEmail() - 15 lines, 1 bug fix needed
|
|
339
|
+
2. PaymentProcessor.process() - 50 lines, critical path
|
|
340
|
+
3. ErrorHandler.retry() - 20 lines, improved recently
|
|
341
|
+
|
|
342
|
+
Which should you test first?
|
|
343
|
+
How would you estimate test writing effort vs benefit?
|
|
344
|
+
context:
|
|
345
|
+
impact_estimation: true
|
|
346
|
+
effort_assessment: true
|
|
347
|
+
|
|
348
|
+
expected_output:
|
|
349
|
+
must_contain:
|
|
350
|
+
- "prioritize"
|
|
351
|
+
- "impact"
|
|
352
|
+
- "effort"
|
|
353
|
+
- "critical"
|
|
354
|
+
- "benefit"
|
|
355
|
+
finding_count:
|
|
356
|
+
min: 1
|
|
357
|
+
|
|
358
|
+
validation:
|
|
359
|
+
schema_check: true
|
|
360
|
+
keyword_match_threshold: 0.75
|
|
361
|
+
|
|
362
|
+
# ---------------------------------------------------------------------------
|
|
363
|
+
# CATEGORY: Coverage Thresholds & Gates
|
|
364
|
+
# ---------------------------------------------------------------------------
|
|
365
|
+
|
|
366
|
+
- id: tc008_quality_gate_enforcement
|
|
367
|
+
description: "Enforce coverage quality gates in CI/CD"
|
|
368
|
+
category: quality_gates
|
|
369
|
+
priority: critical
|
|
370
|
+
|
|
371
|
+
input:
|
|
372
|
+
prompt: |
|
|
373
|
+
Define quality gates:
|
|
374
|
+
- Global: statements >= 80%, branches >= 75%, functions >= 85%
|
|
375
|
+
- New code: statements >= 85%, branches >= 80%
|
|
376
|
+
- Critical paths: >= 90%
|
|
377
|
+
|
|
378
|
+
If statements = 79%, should merge be blocked?
|
|
379
|
+
How would you make this configurable per project?
|
|
380
|
+
context:
|
|
381
|
+
block_on_fail: true
|
|
382
|
+
gates_per_module: true
|
|
383
|
+
|
|
384
|
+
expected_output:
|
|
385
|
+
must_contain:
|
|
386
|
+
- "gate"
|
|
387
|
+
- "threshold"
|
|
388
|
+
- "block"
|
|
389
|
+
- "critical"
|
|
390
|
+
- "enforce"
|
|
391
|
+
must_not_contain:
|
|
392
|
+
- "optional"
|
|
393
|
+
- "warning"
|
|
394
|
+
severity_classification: critical
|
|
395
|
+
|
|
396
|
+
validation:
|
|
397
|
+
schema_check: true
|
|
398
|
+
keyword_match_threshold: 0.8
|
|
399
|
+
|
|
400
|
+
- id: tc009_coverage_trend_analysis
|
|
401
|
+
description: "Track coverage trends over time"
|
|
402
|
+
category: quality_gates
|
|
403
|
+
priority: high
|
|
404
|
+
|
|
405
|
+
input:
|
|
406
|
+
prompt: |
|
|
407
|
+
Analyze coverage trend:
|
|
408
|
+
- Week 1: 75%
|
|
409
|
+
- Week 2: 76%
|
|
410
|
+
- Week 3: 75% (regression)
|
|
411
|
+
- Week 4: 73% (2 week decline)
|
|
412
|
+
|
|
413
|
+
How would you detect:
|
|
414
|
+
1. 3 consecutive regressions
|
|
415
|
+
2. Significant decline (> 3% in 2 weeks)
|
|
416
|
+
3. Stagnation (not improving)
|
|
417
|
+
context:
|
|
418
|
+
trend_window: "4 weeks"
|
|
419
|
+
regression_alert: true
|
|
420
|
+
|
|
421
|
+
expected_output:
|
|
422
|
+
must_contain:
|
|
423
|
+
- "trend"
|
|
424
|
+
- "regression"
|
|
425
|
+
- "decline"
|
|
426
|
+
- "alert"
|
|
427
|
+
finding_count:
|
|
428
|
+
min: 1
|
|
429
|
+
|
|
430
|
+
validation:
|
|
431
|
+
schema_check: true
|
|
432
|
+
keyword_match_threshold: 0.75
|
|
433
|
+
|
|
434
|
+
# ---------------------------------------------------------------------------
|
|
435
|
+
# CATEGORY: Negative Tests
|
|
436
|
+
# ---------------------------------------------------------------------------
|
|
437
|
+
|
|
438
|
+
- id: tc010_coverage_improvement_recommendations
|
|
439
|
+
description: "Provide actionable recommendations to improve coverage"
|
|
440
|
+
category: negative
|
|
441
|
+
priority: high
|
|
442
|
+
|
|
443
|
+
input:
|
|
444
|
+
prompt: |
|
|
445
|
+
For code with 60% coverage (target 85%), recommend:
|
|
446
|
+
1. Which modules to focus on (ROI analysis)
|
|
447
|
+
2. How many tests needed (estimate)
|
|
448
|
+
3. Expected coverage improvement
|
|
449
|
+
4. Time to complete estimate
|
|
450
|
+
5. Priority ranking
|
|
451
|
+
|
|
452
|
+
How would you help teams decide where to focus?
|
|
453
|
+
context:
|
|
454
|
+
actionable_recommendations: true
|
|
455
|
+
roi_focused: true
|
|
456
|
+
|
|
457
|
+
expected_output:
|
|
458
|
+
must_contain:
|
|
459
|
+
- "recommend"
|
|
460
|
+
- "focus"
|
|
461
|
+
- "priority"
|
|
462
|
+
- "estimate"
|
|
463
|
+
- "improve"
|
|
464
|
+
finding_count:
|
|
465
|
+
min: 1
|
|
466
|
+
|
|
467
|
+
validation:
|
|
468
|
+
schema_check: true
|
|
469
|
+
allow_partial: true
|
|
470
|
+
|
|
471
|
+
# =============================================================================
|
|
472
|
+
# SUCCESS CRITERIA
|
|
473
|
+
# =============================================================================
|
|
474
|
+
|
|
475
|
+
success_criteria:
|
|
476
|
+
pass_rate: 0.8
|
|
477
|
+
critical_pass_rate: 1.0
|
|
478
|
+
avg_reasoning_quality: 0.75
|
|
479
|
+
max_execution_time_ms: 300000
|
|
480
|
+
cross_model_variance: 0.15
|
|
481
|
+
|
|
482
|
+
# =============================================================================
|
|
483
|
+
# METADATA
|
|
484
|
+
# =============================================================================
|
|
485
|
+
|
|
486
|
+
metadata:
|
|
487
|
+
author: "qe-coverage-specialist"
|
|
488
|
+
created: "2026-02-02"
|
|
489
|
+
last_updated: "2026-02-02"
|
|
490
|
+
coverage_target: >
|
|
491
|
+
O(log n) sublinear gap detection with sampling, risk-weighted coverage
|
|
492
|
+
scoring with multi-factor analysis, differential coverage with quality gates,
|
|
493
|
+
test prioritization by impact, regression detection, trend analysis,
|
|
494
|
+
and comprehensive improvement recommendations.
|