bmad-method-test-architecture-enterprise 1.2.2 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -12
- package/docs/how-to/workflows/setup-ci.md +3 -1
- package/docs/how-to/workflows/setup-test-framework.md +29 -6
- package/docs/reference/configuration.md +97 -0
- package/docs/reference/knowledge-base.md +15 -6
- package/package.json +1 -1
- package/release_notes.md +6 -4
- package/src/agents/tea.agent.yaml +2 -2
- package/src/module.yaml +78 -5
- package/src/testarch/knowledge/adr-quality-readiness-checklist.md +9 -9
- package/src/testarch/tea-index.csv +36 -36
- package/src/workflows/testarch/atdd/atdd-checklist-template.md +2 -0
- package/src/workflows/testarch/atdd/steps-c/step-01-preflight-and-context.md +65 -12
- package/src/workflows/testarch/atdd/steps-c/step-02-generation-mode.md +5 -0
- package/src/workflows/testarch/atdd/steps-c/step-03-test-strategy.md +10 -1
- package/src/workflows/testarch/atdd/steps-c/step-05-validate-and-complete.md +13 -2
- package/src/workflows/testarch/automate/steps-c/step-01-preflight-and-context.md +46 -2
- package/src/workflows/testarch/automate/steps-c/step-02-identify-targets.md +12 -0
- package/src/workflows/testarch/automate/steps-c/step-03-generate-tests.md +110 -31
- package/src/workflows/testarch/automate/steps-c/step-03b-subprocess-backend.md +246 -0
- package/src/workflows/testarch/automate/steps-c/step-03c-aggregate.md +90 -38
- package/src/workflows/testarch/automate/steps-c/step-04-validate-and-summarize.md +13 -2
- package/src/workflows/testarch/ci/azure-pipelines-template.yaml +155 -0
- package/src/workflows/testarch/ci/checklist.md +48 -7
- package/src/workflows/testarch/ci/github-actions-template.yaml +22 -10
- package/src/workflows/testarch/ci/gitlab-ci-template.yaml +21 -12
- package/src/workflows/testarch/ci/harness-pipeline-template.yaml +159 -0
- package/src/workflows/testarch/ci/jenkins-pipeline-template.groovy +129 -0
- package/src/workflows/testarch/ci/steps-c/step-01-preflight.md +58 -17
- package/src/workflows/testarch/ci/steps-c/step-02-generate-pipeline.md +21 -10
- package/src/workflows/testarch/ci/steps-c/step-03-configure-quality-gates.md +5 -0
- package/src/workflows/testarch/ci/workflow.yaml +5 -3
- package/src/workflows/testarch/framework/checklist.md +11 -10
- package/src/workflows/testarch/framework/steps-c/step-01-preflight.md +34 -2
- package/src/workflows/testarch/framework/steps-c/step-02-select-framework.md +20 -1
- package/src/workflows/testarch/framework/steps-c/step-03-scaffold-framework.md +56 -5
- package/src/workflows/testarch/framework/steps-c/step-04-docs-and-scripts.md +16 -4
- package/src/workflows/testarch/nfr-assess/nfr-report-template.md +3 -1
- package/src/workflows/testarch/nfr-assess/steps-c/step-01-load-context.md +12 -0
- package/src/workflows/testarch/nfr-assess/steps-c/step-05-generate-report.md +14 -3
- package/src/workflows/testarch/test-design/checklist.md +20 -9
- package/src/workflows/testarch/test-design/instructions.md +3 -3
- package/src/workflows/testarch/test-design/steps-c/step-02-load-context.md +34 -0
- package/src/workflows/testarch/test-design/steps-c/step-05-generate-output.md +29 -2
- package/src/workflows/testarch/test-design/test-design-architecture-template.md +16 -14
- package/src/workflows/testarch/test-design/test-design-handoff-template.md +70 -0
- package/src/workflows/testarch/test-design/test-design-qa-template.md +11 -9
- package/src/workflows/testarch/test-design/workflow.yaml +8 -1
- package/src/workflows/testarch/test-review/steps-c/step-01-load-context.md +34 -1
- package/src/workflows/testarch/test-review/steps-c/step-04-generate-report.md +14 -3
- package/src/workflows/testarch/test-review/test-review-template.md +4 -2
- package/src/workflows/testarch/test-review/workflow.yaml +1 -0
- package/src/workflows/testarch/trace/trace-template.md +7 -5
- package/test/test-installation-components.js +1 -1
- package/test/test-knowledge-base.js +10 -1
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: 'step-03b-subprocess-backend'
|
|
3
|
+
description: 'Subprocess: Generate backend tests only (unit, integration, contract)'
|
|
4
|
+
subprocess: true
|
|
5
|
+
outputFile: '/tmp/tea-automate-backend-tests-{{timestamp}}.json'
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# Subprocess 3B-backend: Generate Backend Tests
|
|
9
|
+
|
|
10
|
+
## SUBPROCESS CONTEXT
|
|
11
|
+
|
|
12
|
+
This is an **isolated subprocess** running in parallel with API test generation (and optionally E2E test generation for fullstack projects).
|
|
13
|
+
|
|
14
|
+
**What you have from parent workflow:**
|
|
15
|
+
|
|
16
|
+
- Target features/services identified in Step 2
|
|
17
|
+
- Knowledge fragments loaded: test-levels-framework, test-priorities-matrix, data-factories
|
|
18
|
+
- Config: test framework, detected stack type
|
|
19
|
+
- Coverage plan: which services/modules need backend testing
|
|
20
|
+
|
|
21
|
+
**Your task:** Generate backend tests ONLY (unit, integration, contract - not API endpoint tests, not E2E).
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## MANDATORY EXECUTION RULES
|
|
26
|
+
|
|
27
|
+
- Read this entire subprocess file before acting
|
|
28
|
+
- Generate backend tests ONLY (unit, integration, contract)
|
|
29
|
+
- Output structured JSON to temp file using the subprocess output schema contract
|
|
30
|
+
- Follow knowledge fragment patterns
|
|
31
|
+
- Do NOT generate API endpoint tests (that's subprocess 3A)
|
|
32
|
+
- Do NOT generate E2E tests (that's subprocess 3B)
|
|
33
|
+
- Do NOT run tests (that's step 4)
|
|
34
|
+
- Do NOT generate fixtures yet (that's step 3C aggregation)
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## SUBPROCESS TASK
|
|
39
|
+
|
|
40
|
+
### 1. Identify Test Targets
|
|
41
|
+
|
|
42
|
+
From the coverage plan (Step 2 output), identify:
|
|
43
|
+
|
|
44
|
+
- Which services/modules need unit test coverage
|
|
45
|
+
- Which integrations need integration test coverage (database, message queues, external services)
|
|
46
|
+
- Which service contracts need contract test coverage (Pact, schema validation)
|
|
47
|
+
- Business logic functions requiring edge case coverage
|
|
48
|
+
|
|
49
|
+
### 2. Detect Framework & Language
|
|
50
|
+
|
|
51
|
+
From `config.test_framework` and project manifests, determine:
|
|
52
|
+
|
|
53
|
+
- **Python (pytest)**: Use `pytest` conventions, `conftest.py` fixtures, `@pytest.mark` decorators
|
|
54
|
+
- **Java/Kotlin (JUnit)**: Use JUnit 5 annotations (`@Test`, `@BeforeEach`, `@Nested`), Mockito for mocking
|
|
55
|
+
- **Go (go test)**: Use `*_test.go` files, `testing.T`, table-driven tests, `testify` assertions
|
|
56
|
+
- **C#/.NET (xUnit)**: Use `[Fact]`, `[Theory]`, `[InlineData]`, `Moq` for mocking
|
|
57
|
+
- **Ruby (RSpec)**: Use `describe`/`context`/`it` blocks, `let`/`before` helpers, `FactoryBot`
|
|
58
|
+
|
|
59
|
+
### 3. Generate Unit Tests
|
|
60
|
+
|
|
61
|
+
For each module/service, create test files following language-idiomatic patterns:
|
|
62
|
+
|
|
63
|
+
**Python (pytest) example:**
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
import pytest
|
|
67
|
+
from unittest.mock import MagicMock, patch
|
|
68
|
+
from myapp.services.user_service import UserService
|
|
69
|
+
|
|
70
|
+
class TestUserService:
|
|
71
|
+
"""[P0] Unit tests for UserService"""
|
|
72
|
+
|
|
73
|
+
def test_create_user_with_valid_data(self, user_factory):
|
|
74
|
+
"""Should create user when data is valid"""
|
|
75
|
+
user_data = user_factory.build()
|
|
76
|
+
result = UserService.create(user_data)
|
|
77
|
+
assert result.email == user_data["email"]
|
|
78
|
+
|
|
79
|
+
def test_create_user_rejects_duplicate_email(self, user_factory):
|
|
80
|
+
"""[P1] Should reject duplicate email"""
|
|
81
|
+
user_data = user_factory.build(email="existing@test.com")
|
|
82
|
+
with pytest.raises(DuplicateEmailError):
|
|
83
|
+
UserService.create(user_data)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
**Go (go test) example:**
|
|
87
|
+
|
|
88
|
+
```go
|
|
89
|
+
func TestUserService_Create(t *testing.T) {
|
|
90
|
+
tests := []struct {
|
|
91
|
+
name string
|
|
92
|
+
input CreateUserInput
|
|
93
|
+
wantErr bool
|
|
94
|
+
}{
|
|
95
|
+
{"valid user", validInput(), false},
|
|
96
|
+
{"duplicate email", duplicateInput(), true},
|
|
97
|
+
}
|
|
98
|
+
for _, tt := range tests {
|
|
99
|
+
t.Run(tt.name, func(t *testing.T) {
|
|
100
|
+
svc := NewUserService(mockRepo)
|
|
101
|
+
_, err := svc.Create(tt.input)
|
|
102
|
+
if (err != nil) != tt.wantErr {
|
|
103
|
+
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
|
|
104
|
+
}
|
|
105
|
+
})
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**Requirements:**
|
|
111
|
+
|
|
112
|
+
- Follow the detected framework's idiomatic test patterns
|
|
113
|
+
- Include priority tags [P0], [P1], [P2], [P3] in test descriptions
|
|
114
|
+
- Use proper mocking for external dependencies (database, APIs, message queues)
|
|
115
|
+
- Test both happy path and error cases
|
|
116
|
+
- Use proper typing/type hints where applicable
|
|
117
|
+
- No hard-coded test data; use factories or builders
|
|
118
|
+
|
|
119
|
+
### 4. Generate Integration Tests
|
|
120
|
+
|
|
121
|
+
For service integrations, create integration test files:
|
|
122
|
+
|
|
123
|
+
- Database integration tests (with test database or in-memory alternatives)
|
|
124
|
+
- Message queue consumer/producer tests
|
|
125
|
+
- Cache integration tests
|
|
126
|
+
- External service integration tests (with mocked HTTP clients)
|
|
127
|
+
|
|
128
|
+
### 5. Generate Contract Tests (if applicable)
|
|
129
|
+
|
|
130
|
+
If the project uses microservices or has defined API contracts:
|
|
131
|
+
|
|
132
|
+
- Pact consumer/provider tests
|
|
133
|
+
- Schema validation tests (JSON Schema, Protobuf)
|
|
134
|
+
- OpenAPI spec compliance tests
|
|
135
|
+
|
|
136
|
+
### 6. Track Fixture Needs
|
|
137
|
+
|
|
138
|
+
Identify fixtures/helpers needed for backend tests:
|
|
139
|
+
|
|
140
|
+
- Database fixtures (seed data, cleanup)
|
|
141
|
+
- Factory functions (test data builders)
|
|
142
|
+
- Mock services (HTTP mocks, message queue mocks)
|
|
143
|
+
- Configuration fixtures (test environment config)
|
|
144
|
+
|
|
145
|
+
**Do NOT create fixtures yet** - just track what's needed for aggregation step.
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
## OUTPUT FORMAT
|
|
150
|
+
|
|
151
|
+
Write JSON to temp file: `/tmp/tea-automate-backend-tests-{{timestamp}}.json`
|
|
152
|
+
|
|
153
|
+
```json
|
|
154
|
+
{
|
|
155
|
+
"subprocessType": "backend",
|
|
156
|
+
"testsGenerated": [
|
|
157
|
+
{
|
|
158
|
+
"file": "tests/unit/test_user_service.py",
|
|
159
|
+
"content": "[full test file content]",
|
|
160
|
+
"description": "Unit tests for UserService",
|
|
161
|
+
"priority_coverage": {
|
|
162
|
+
"P0": 3,
|
|
163
|
+
"P1": 2,
|
|
164
|
+
"P2": 1,
|
|
165
|
+
"P3": 0
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
"file": "tests/integration/test_user_repository.py",
|
|
170
|
+
"content": "[full test file content]",
|
|
171
|
+
"description": "Integration tests for user database operations",
|
|
172
|
+
"priority_coverage": {
|
|
173
|
+
"P0": 1,
|
|
174
|
+
"P1": 2,
|
|
175
|
+
"P2": 1,
|
|
176
|
+
"P3": 0
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
],
|
|
180
|
+
"coverageSummary": {
|
|
181
|
+
"totalTests": 15,
|
|
182
|
+
"testLevels": ["unit", "integration", "contract"],
|
|
183
|
+
"fixtureNeeds": ["databaseFixture", "userFactory", "mockHttpClient"]
|
|
184
|
+
},
|
|
185
|
+
"status": "complete",
|
|
186
|
+
"success": true,
|
|
187
|
+
"subprocess": "backend-tests",
|
|
188
|
+
"knowledge_fragments_used": ["test-levels-framework", "test-priorities-matrix", "data-factories"],
|
|
189
|
+
"summary": "Generated 15 backend test cases (10 unit, 4 integration, 1 contract)"
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
**On Error:**
|
|
194
|
+
|
|
195
|
+
```json
|
|
196
|
+
{
|
|
197
|
+
"subprocessType": "backend",
|
|
198
|
+
"testsGenerated": [],
|
|
199
|
+
"coverageSummary": {
|
|
200
|
+
"totalTests": 0,
|
|
201
|
+
"testLevels": [],
|
|
202
|
+
"fixtureNeeds": []
|
|
203
|
+
},
|
|
204
|
+
"status": "partial",
|
|
205
|
+
"success": false,
|
|
206
|
+
"subprocess": "backend-tests",
|
|
207
|
+
"error": "Error message describing what went wrong",
|
|
208
|
+
"partial_output": {
|
|
209
|
+
/* any tests generated before error */
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
---
|
|
215
|
+
|
|
216
|
+
## EXIT CONDITION
|
|
217
|
+
|
|
218
|
+
Subprocess completes when:
|
|
219
|
+
|
|
220
|
+
- All identified modules have backend test files generated
|
|
221
|
+
- All tests follow language-idiomatic patterns
|
|
222
|
+
- JSON output written to temp file using the subprocess output schema contract
|
|
223
|
+
- Fixture needs tracked
|
|
224
|
+
|
|
225
|
+
**Subprocess terminates here.** Parent workflow will read output and proceed to aggregation.
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## SUBPROCESS SUCCESS METRICS
|
|
230
|
+
|
|
231
|
+
### SUCCESS:
|
|
232
|
+
|
|
233
|
+
- All backend tests generated following idiomatic patterns
|
|
234
|
+
- JSON output valid and complete, matches subprocess output schema contract
|
|
235
|
+
- No E2E or browser tests included (out of scope)
|
|
236
|
+
- Proper mocking used for external dependencies
|
|
237
|
+
- Priority tags assigned to all test cases
|
|
238
|
+
|
|
239
|
+
### FAILURE:
|
|
240
|
+
|
|
241
|
+
- Generated tests other than backend tests (unit/integration/contract)
|
|
242
|
+
- Did not follow language-idiomatic patterns
|
|
243
|
+
- Invalid or missing JSON output
|
|
244
|
+
- Output schema does not match the contract
|
|
245
|
+
- Ran tests (not subprocess responsibility)
|
|
246
|
+
- Used real external services instead of mocks
|
|
@@ -9,7 +9,7 @@ nextStepFile: './step-04-validate-and-summarize.md'
|
|
|
9
9
|
|
|
10
10
|
## STEP GOAL
|
|
11
11
|
|
|
12
|
-
Read outputs from parallel subprocesses (API + E2E test generation), aggregate results, and create supporting infrastructure (fixtures, helpers).
|
|
12
|
+
Read outputs from parallel subprocesses (API + E2E and/or Backend test generation based on `{detected_stack}`), aggregate results, and create supporting infrastructure (fixtures, helpers).
|
|
13
13
|
|
|
14
14
|
---
|
|
15
15
|
|
|
@@ -46,25 +46,39 @@ Read outputs from parallel subprocesses (API + E2E test generation), aggregate r
|
|
|
46
46
|
|
|
47
47
|
### 1. Read Subprocess Outputs
|
|
48
48
|
|
|
49
|
-
**Read API test subprocess output:**
|
|
49
|
+
**Read API test subprocess output (always):**
|
|
50
50
|
|
|
51
51
|
```javascript
|
|
52
52
|
const apiTestsPath = '/tmp/tea-automate-api-tests-{{timestamp}}.json';
|
|
53
53
|
const apiTestsOutput = JSON.parse(fs.readFileSync(apiTestsPath, 'utf8'));
|
|
54
54
|
```
|
|
55
55
|
|
|
56
|
-
**Read E2E test subprocess output:**
|
|
56
|
+
**Read E2E test subprocess output (if {detected_stack} is `frontend` or `fullstack`):**
|
|
57
57
|
|
|
58
58
|
```javascript
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
let e2eTestsOutput = null;
|
|
60
|
+
if (detected_stack === 'frontend' || detected_stack === 'fullstack') {
|
|
61
|
+
const e2eTestsPath = '/tmp/tea-automate-e2e-tests-{{timestamp}}.json';
|
|
62
|
+
e2eTestsOutput = JSON.parse(fs.readFileSync(e2eTestsPath, 'utf8'));
|
|
63
|
+
}
|
|
61
64
|
```
|
|
62
65
|
|
|
63
|
-
**
|
|
66
|
+
**Read Backend test subprocess output (if {detected_stack} is `backend` or `fullstack`):**
|
|
67
|
+
|
|
68
|
+
```javascript
|
|
69
|
+
let backendTestsOutput = null;
|
|
70
|
+
if (detected_stack === 'backend' || detected_stack === 'fullstack') {
|
|
71
|
+
const backendTestsPath = '/tmp/tea-automate-backend-tests-{{timestamp}}.json';
|
|
72
|
+
backendTestsOutput = JSON.parse(fs.readFileSync(backendTestsPath, 'utf8'));
|
|
73
|
+
}
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
**Verify all launched subprocesses succeeded:**
|
|
64
77
|
|
|
65
78
|
- Check `apiTestsOutput.success === true`
|
|
66
|
-
-
|
|
67
|
-
- If
|
|
79
|
+
- If E2E was launched: check `e2eTestsOutput.success === true`
|
|
80
|
+
- If Backend was launched: check `backendTestsOutput.success === true`
|
|
81
|
+
- If any failed, report error and stop (don't proceed)
|
|
68
82
|
|
|
69
83
|
---
|
|
70
84
|
|
|
@@ -79,23 +93,40 @@ apiTestsOutput.tests.forEach((test) => {
|
|
|
79
93
|
});
|
|
80
94
|
```
|
|
81
95
|
|
|
82
|
-
**Write E2E test files:**
|
|
96
|
+
**Write E2E test files (if {detected_stack} is `frontend` or `fullstack`):**
|
|
83
97
|
|
|
84
98
|
```javascript
|
|
85
|
-
e2eTestsOutput
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
});
|
|
99
|
+
if (e2eTestsOutput) {
|
|
100
|
+
e2eTestsOutput.tests.forEach((test) => {
|
|
101
|
+
fs.writeFileSync(test.file, test.content, 'utf8');
|
|
102
|
+
console.log(`✅ Created: ${test.file}`);
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
**Write Backend test files (if {detected_stack} is `backend` or `fullstack`):**
|
|
108
|
+
|
|
109
|
+
```javascript
|
|
110
|
+
if (backendTestsOutput) {
|
|
111
|
+
backendTestsOutput.testsGenerated.forEach((test) => {
|
|
112
|
+
fs.writeFileSync(test.file, test.content, 'utf8');
|
|
113
|
+
console.log(`✅ Created: ${test.file}`);
|
|
114
|
+
});
|
|
115
|
+
}
|
|
89
116
|
```
|
|
90
117
|
|
|
91
118
|
---
|
|
92
119
|
|
|
93
120
|
### 3. Aggregate Fixture Needs
|
|
94
121
|
|
|
95
|
-
**Collect all fixture needs from
|
|
122
|
+
**Collect all fixture needs from all launched subprocesses:**
|
|
96
123
|
|
|
97
124
|
```javascript
|
|
98
|
-
const allFixtureNeeds = [
|
|
125
|
+
const allFixtureNeeds = [
|
|
126
|
+
...apiTestsOutput.fixture_needs,
|
|
127
|
+
...(e2eTestsOutput ? e2eTestsOutput.fixture_needs : []),
|
|
128
|
+
...(backendTestsOutput ? backendTestsOutput.coverageSummary?.fixtureNeeds || [] : []),
|
|
129
|
+
];
|
|
99
130
|
|
|
100
131
|
// Remove duplicates
|
|
101
132
|
const uniqueFixtures = [...new Set(allFixtureNeeds)];
|
|
@@ -190,28 +221,47 @@ export const waitForApiResponse = async (page: Page, urlPattern: string) => {
|
|
|
190
221
|
|
|
191
222
|
### 5. Calculate Summary Statistics
|
|
192
223
|
|
|
193
|
-
**Aggregate test counts:**
|
|
224
|
+
**Aggregate test counts (based on `{detected_stack}`):**
|
|
194
225
|
|
|
195
226
|
```javascript
|
|
227
|
+
const e2eCount = e2eTestsOutput ? e2eTestsOutput.test_count : 0;
|
|
228
|
+
const backendCount = backendTestsOutput ? (backendTestsOutput.coverageSummary?.totalTests ?? 0) : 0;
|
|
229
|
+
|
|
196
230
|
const summary = {
|
|
197
|
-
|
|
231
|
+
detected_stack: '{detected_stack}',
|
|
232
|
+
total_tests: apiTestsOutput.test_count + e2eCount + backendCount,
|
|
198
233
|
api_tests: apiTestsOutput.test_count,
|
|
199
|
-
e2e_tests:
|
|
234
|
+
e2e_tests: e2eCount,
|
|
235
|
+
backend_tests: backendCount,
|
|
200
236
|
fixtures_created: uniqueFixtures.length,
|
|
201
237
|
api_test_files: apiTestsOutput.tests.length,
|
|
202
|
-
e2e_test_files: e2eTestsOutput.tests.length,
|
|
238
|
+
e2e_test_files: e2eTestsOutput ? e2eTestsOutput.tests.length : 0,
|
|
239
|
+
backend_test_files: backendTestsOutput ? backendTestsOutput.testsGenerated.length : 0,
|
|
203
240
|
priority_coverage: {
|
|
204
|
-
P0:
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
241
|
+
P0:
|
|
242
|
+
(apiTestsOutput.priority_coverage?.P0 ?? 0) +
|
|
243
|
+
(e2eTestsOutput?.priority_coverage?.P0 ?? 0) +
|
|
244
|
+
(backendTestsOutput?.testsGenerated?.reduce((sum, t) => sum + (t.priority_coverage?.P0 ?? 0), 0) ?? 0),
|
|
245
|
+
P1:
|
|
246
|
+
(apiTestsOutput.priority_coverage?.P1 ?? 0) +
|
|
247
|
+
(e2eTestsOutput?.priority_coverage?.P1 ?? 0) +
|
|
248
|
+
(backendTestsOutput?.testsGenerated?.reduce((sum, t) => sum + (t.priority_coverage?.P1 ?? 0), 0) ?? 0),
|
|
249
|
+
P2:
|
|
250
|
+
(apiTestsOutput.priority_coverage?.P2 ?? 0) +
|
|
251
|
+
(e2eTestsOutput?.priority_coverage?.P2 ?? 0) +
|
|
252
|
+
(backendTestsOutput?.testsGenerated?.reduce((sum, t) => sum + (t.priority_coverage?.P2 ?? 0), 0) ?? 0),
|
|
253
|
+
P3:
|
|
254
|
+
(apiTestsOutput.priority_coverage?.P3 ?? 0) +
|
|
255
|
+
(e2eTestsOutput?.priority_coverage?.P3 ?? 0) +
|
|
256
|
+
(backendTestsOutput?.testsGenerated?.reduce((sum, t) => sum + (t.priority_coverage?.P3 ?? 0), 0) ?? 0),
|
|
208
257
|
},
|
|
209
258
|
knowledge_fragments_used: [
|
|
210
259
|
...apiTestsOutput.knowledge_fragments_used,
|
|
211
|
-
...e2eTestsOutput.knowledge_fragments_used
|
|
260
|
+
...(e2eTestsOutput ? e2eTestsOutput.knowledge_fragments_used : []),
|
|
261
|
+
...(backendTestsOutput ? backendTestsOutput.knowledge_fragments_used || [] : []),
|
|
212
262
|
],
|
|
213
|
-
subprocess_execution:
|
|
214
|
-
performance_gain: '~
|
|
263
|
+
subprocess_execution: `PARALLEL (based on ${detected_stack})`,
|
|
264
|
+
performance_gain: '~40-70% faster than sequential',
|
|
215
265
|
};
|
|
216
266
|
```
|
|
217
267
|
|
|
@@ -230,7 +280,8 @@ fs.writeFileSync('/tmp/tea-automate-summary-{{timestamp}}.json', JSON.stringify(
|
|
|
230
280
|
|
|
231
281
|
```javascript
|
|
232
282
|
fs.unlinkSync(apiTestsPath);
|
|
233
|
-
fs.unlinkSync(
|
|
283
|
+
if (e2eTestsOutput) fs.unlinkSync('/tmp/tea-automate-e2e-tests-{{timestamp}}.json');
|
|
284
|
+
if (backendTestsOutput) fs.unlinkSync('/tmp/tea-automate-backend-tests-{{timestamp}}.json');
|
|
234
285
|
console.log('✅ Subprocess temp files cleaned up');
|
|
235
286
|
```
|
|
236
287
|
|
|
@@ -244,9 +295,11 @@ Display to user:
|
|
|
244
295
|
✅ Test Generation Complete (Parallel Execution)
|
|
245
296
|
|
|
246
297
|
📊 Summary:
|
|
298
|
+
- Stack Type: {detected_stack}
|
|
247
299
|
- Total Tests: {total_tests}
|
|
248
300
|
- API Tests: {api_tests} ({api_test_files} files)
|
|
249
|
-
- E2E Tests: {e2e_tests} ({e2e_test_files} files)
|
|
301
|
+
- E2E Tests: {e2e_tests} ({e2e_test_files} files) [if frontend/fullstack]
|
|
302
|
+
- Backend Tests: {backend_tests} ({backend_test_files} files) [if backend/fullstack]
|
|
250
303
|
- Fixtures Created: {fixtures_created}
|
|
251
304
|
- Priority Coverage:
|
|
252
305
|
- P0 (Critical): {P0} tests
|
|
@@ -254,15 +307,14 @@ Display to user:
|
|
|
254
307
|
- P2 (Medium): {P2} tests
|
|
255
308
|
- P3 (Low): {P3} tests
|
|
256
309
|
|
|
257
|
-
🚀 Performance: Parallel execution ~
|
|
310
|
+
🚀 Performance: Parallel execution ~40-70% faster than sequential
|
|
258
311
|
|
|
259
312
|
📂 Generated Files:
|
|
260
|
-
- tests/api/[feature].spec.ts
|
|
261
|
-
- tests/e2e/[feature].spec.ts
|
|
262
|
-
- tests/
|
|
263
|
-
- tests/
|
|
264
|
-
- tests/fixtures/
|
|
265
|
-
- tests/fixtures/helpers.ts
|
|
313
|
+
- tests/api/[feature].spec.ts [always]
|
|
314
|
+
- tests/e2e/[feature].spec.ts [if frontend/fullstack]
|
|
315
|
+
- tests/unit/[feature].test.* [if backend/fullstack]
|
|
316
|
+
- tests/integration/[feature].test.* [if backend/fullstack]
|
|
317
|
+
- tests/fixtures/ or tests/support/ [shared infrastructure]
|
|
266
318
|
|
|
267
319
|
✅ Ready for validation (Step 4)
|
|
268
320
|
```
|
|
@@ -273,7 +325,7 @@ Display to user:
|
|
|
273
325
|
|
|
274
326
|
Proceed to Step 4 when:
|
|
275
327
|
|
|
276
|
-
- ✅ All test files written to disk (API + E2E)
|
|
328
|
+
- ✅ All test files written to disk (API + E2E and/or Backend, based on `{detected_stack}`)
|
|
277
329
|
- ✅ All fixtures and helpers created
|
|
278
330
|
- ✅ Summary statistics calculated and saved
|
|
279
331
|
- ✅ Output displayed to user
|
|
@@ -310,14 +362,14 @@ Load next step: `{nextStepFile}`
|
|
|
310
362
|
|
|
311
363
|
### ✅ SUCCESS:
|
|
312
364
|
|
|
313
|
-
-
|
|
365
|
+
- All launched subprocesses succeeded (based on `{detected_stack}`)
|
|
314
366
|
- All test files written to disk
|
|
315
367
|
- Fixtures generated based on subprocess needs
|
|
316
368
|
- Summary complete and accurate
|
|
317
369
|
|
|
318
370
|
### ❌ SYSTEM FAILURE:
|
|
319
371
|
|
|
320
|
-
- One or
|
|
372
|
+
- One or more subprocesses failed
|
|
321
373
|
- Test files not written to disk
|
|
322
374
|
- Fixtures missing or incomplete
|
|
323
375
|
- Summary missing or inaccurate
|
|
@@ -50,7 +50,18 @@ Fix gaps before proceeding.
|
|
|
50
50
|
|
|
51
51
|
---
|
|
52
52
|
|
|
53
|
-
## 2.
|
|
53
|
+
## 2. Polish Output
|
|
54
|
+
|
|
55
|
+
Before finalizing, review the complete output document for quality:
|
|
56
|
+
|
|
57
|
+
1. **Remove duplication**: Progressive-append workflow may have created repeated sections — consolidate
|
|
58
|
+
2. **Verify consistency**: Ensure terminology, risk scores, and references are consistent throughout
|
|
59
|
+
3. **Check completeness**: All template sections should be populated or explicitly marked N/A
|
|
60
|
+
4. **Format cleanup**: Ensure markdown formatting is clean (tables aligned, headers consistent, no orphaned references)
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## 3. Summary Output
|
|
54
65
|
|
|
55
66
|
Write `{outputFile}` including:
|
|
56
67
|
|
|
@@ -61,7 +72,7 @@ Write `{outputFile}` including:
|
|
|
61
72
|
|
|
62
73
|
---
|
|
63
74
|
|
|
64
|
-
##
|
|
75
|
+
## 4. Save Progress
|
|
65
76
|
|
|
66
77
|
**Save this step's accumulated work to `{outputFile}`.**
|
|
67
78
|
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# Azure DevOps CI/CD Pipeline for Test Execution
|
|
2
|
+
# Generated by BMad TEA Agent - Test Architect Module
|
|
3
|
+
# Optimized for: Parallel Sharding, Burn-In Loop
|
|
4
|
+
# Stack: {test_stack_type} | Framework: {test_framework}
|
|
5
|
+
#
|
|
6
|
+
# Variables to customize per project:
|
|
7
|
+
# INSTALL_CMD - dependency install command (e.g., npm ci, pnpm install --frozen-lockfile)
|
|
8
|
+
# TEST_CMD - main test command (e.g., npm run test:e2e, npm test, npx vitest)
|
|
9
|
+
# LINT_CMD - lint command (e.g., npm run lint)
|
|
10
|
+
# BROWSER_INSTALL - browser install command (frontend/fullstack only; omit for backend)
|
|
11
|
+
# DEFAULT_NODE_VERSION - Node.js version (read from .nvmrc or default to 24)
|
|
12
|
+
|
|
13
|
+
trigger:
|
|
14
|
+
branches:
|
|
15
|
+
include:
|
|
16
|
+
- main
|
|
17
|
+
- develop
|
|
18
|
+
|
|
19
|
+
pr:
|
|
20
|
+
branches:
|
|
21
|
+
include:
|
|
22
|
+
- main
|
|
23
|
+
- develop
|
|
24
|
+
|
|
25
|
+
variables:
|
|
26
|
+
DEFAULT_NODE_VERSION: "24"
|
|
27
|
+
npm_config_cache: $(Pipeline.Workspace)/.npm
|
|
28
|
+
# Set TEST_STACK_TYPE to 'backend' to skip Playwright browser installs
|
|
29
|
+
TEST_STACK_TYPE: "" # Values: frontend, backend, fullstack (leave empty for auto)
|
|
30
|
+
|
|
31
|
+
stages:
|
|
32
|
+
# Lint stage - Code quality checks
|
|
33
|
+
- stage: Lint
|
|
34
|
+
displayName: "Lint"
|
|
35
|
+
jobs:
|
|
36
|
+
- job: LintJob
|
|
37
|
+
displayName: "Code Quality"
|
|
38
|
+
pool:
|
|
39
|
+
vmImage: "ubuntu-latest"
|
|
40
|
+
timeoutInMinutes: 5
|
|
41
|
+
steps:
|
|
42
|
+
- task: NodeTool@0
|
|
43
|
+
inputs:
|
|
44
|
+
versionSpec: $(DEFAULT_NODE_VERSION)
|
|
45
|
+
displayName: "Setup Node.js"
|
|
46
|
+
|
|
47
|
+
- task: Cache@2
|
|
48
|
+
inputs:
|
|
49
|
+
key: 'npm | "$(Agent.OS)" | package-lock.json'
|
|
50
|
+
restoreKeys: 'npm | "$(Agent.OS)"'
|
|
51
|
+
path: $(npm_config_cache)
|
|
52
|
+
displayName: "Cache npm"
|
|
53
|
+
|
|
54
|
+
- script: npm ci
|
|
55
|
+
displayName: "Install dependencies" # Replace with INSTALL_CMD
|
|
56
|
+
|
|
57
|
+
- script: npm run lint
|
|
58
|
+
displayName: "Run linter" # Replace with LINT_CMD
|
|
59
|
+
|
|
60
|
+
# Test stage - Parallel execution with sharding
|
|
61
|
+
- stage: Test
|
|
62
|
+
displayName: "Test"
|
|
63
|
+
dependsOn: Lint
|
|
64
|
+
jobs:
|
|
65
|
+
- job: TestShard
|
|
66
|
+
displayName: "Test Shard"
|
|
67
|
+
pool:
|
|
68
|
+
vmImage: "ubuntu-latest"
|
|
69
|
+
timeoutInMinutes: 30
|
|
70
|
+
strategy:
|
|
71
|
+
matrix:
|
|
72
|
+
Shard1:
|
|
73
|
+
SHARD_INDEX: 1
|
|
74
|
+
Shard2:
|
|
75
|
+
SHARD_INDEX: 2
|
|
76
|
+
Shard3:
|
|
77
|
+
SHARD_INDEX: 3
|
|
78
|
+
Shard4:
|
|
79
|
+
SHARD_INDEX: 4
|
|
80
|
+
steps:
|
|
81
|
+
- task: NodeTool@0
|
|
82
|
+
inputs:
|
|
83
|
+
versionSpec: $(DEFAULT_NODE_VERSION)
|
|
84
|
+
displayName: "Setup Node.js"
|
|
85
|
+
|
|
86
|
+
- task: Cache@2
|
|
87
|
+
inputs:
|
|
88
|
+
key: 'npm | "$(Agent.OS)" | package-lock.json'
|
|
89
|
+
restoreKeys: 'npm | "$(Agent.OS)"'
|
|
90
|
+
path: $(npm_config_cache)
|
|
91
|
+
displayName: "Cache npm"
|
|
92
|
+
|
|
93
|
+
- script: npm ci
|
|
94
|
+
displayName: "Install dependencies" # Replace with INSTALL_CMD
|
|
95
|
+
|
|
96
|
+
# Frontend/Fullstack only — skipped for backend-only stacks
|
|
97
|
+
- script: npx playwright install --with-deps chromium
|
|
98
|
+
condition: ne(variables['TEST_STACK_TYPE'], 'backend')
|
|
99
|
+
displayName: "Install Playwright browsers" # Replace with BROWSER_INSTALL
|
|
100
|
+
|
|
101
|
+
- script: npm run test:e2e -- --shard=$(SHARD_INDEX)/4
|
|
102
|
+
displayName: "Run tests (shard $(SHARD_INDEX)/4)" # Replace with TEST_CMD + shard args
|
|
103
|
+
|
|
104
|
+
- task: PublishTestResults@2
|
|
105
|
+
condition: always()
|
|
106
|
+
inputs:
|
|
107
|
+
testResultsFormat: "JUnit"
|
|
108
|
+
testResultsFiles: "test-results/**/*.xml"
|
|
109
|
+
mergeTestResults: true
|
|
110
|
+
displayName: "Publish test results"
|
|
111
|
+
|
|
112
|
+
- publish: test-results/
|
|
113
|
+
artifact: test-results-$(SHARD_INDEX)
|
|
114
|
+
condition: failed()
|
|
115
|
+
displayName: "Upload failure artifacts"
|
|
116
|
+
|
|
117
|
+
# Burn-in stage - Flaky test detection
|
|
118
|
+
# Note: Burn-in targets UI flakiness. For backend-only stacks, remove this stage entirely.
|
|
119
|
+
- stage: BurnIn
|
|
120
|
+
displayName: "Burn-In (Flaky Detection)"
|
|
121
|
+
dependsOn: Test
|
|
122
|
+
condition: and(succeeded(), or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.CronSchedule.DisplayName'], 'Weekly burn-in')))
|
|
123
|
+
jobs:
|
|
124
|
+
- job: BurnInJob
|
|
125
|
+
displayName: "Burn-In Loop"
|
|
126
|
+
pool:
|
|
127
|
+
vmImage: "ubuntu-latest"
|
|
128
|
+
timeoutInMinutes: 60
|
|
129
|
+
steps:
|
|
130
|
+
- task: NodeTool@0
|
|
131
|
+
inputs:
|
|
132
|
+
versionSpec: $(DEFAULT_NODE_VERSION)
|
|
133
|
+
displayName: "Setup Node.js"
|
|
134
|
+
|
|
135
|
+
- script: npm ci
|
|
136
|
+
displayName: "Install dependencies" # Replace with INSTALL_CMD
|
|
137
|
+
|
|
138
|
+
# Frontend/Fullstack only — skipped for backend-only stacks
|
|
139
|
+
- script: npx playwright install --with-deps chromium
|
|
140
|
+
condition: ne(variables['TEST_STACK_TYPE'], 'backend')
|
|
141
|
+
displayName: "Install Playwright browsers" # Replace with BROWSER_INSTALL
|
|
142
|
+
|
|
143
|
+
- script: |
|
|
144
|
+
echo "Starting burn-in loop - detecting flaky tests"
|
|
145
|
+
for i in $(seq 1 10); do
|
|
146
|
+
echo "Burn-in iteration $i/10"
|
|
147
|
+
npm run test:e2e || exit 1
|
|
148
|
+
done
|
|
149
|
+
echo "Burn-in complete - no flaky tests detected"
|
|
150
|
+
displayName: "Run burn-in loop (10 iterations)" # Replace npm run test:e2e with TEST_CMD
|
|
151
|
+
|
|
152
|
+
- publish: test-results/
|
|
153
|
+
artifact: burn-in-failures
|
|
154
|
+
condition: failed()
|
|
155
|
+
displayName: "Upload burn-in failure artifacts"
|