bmad-method-test-architecture-enterprise 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/release_notes.md +7 -7
- package/src/testarch/knowledge/ci-burn-in.md +1 -1
- package/src/testarch/knowledge/contract-testing.md +66 -2
- package/src/testarch/knowledge/error-handling.md +1 -1
- package/src/testarch/knowledge/feature-flags.md +1 -1
- package/src/testarch/knowledge/fixture-architecture.md +1 -1
- package/src/testarch/knowledge/network-recorder.md +5 -5
- package/src/testarch/knowledge/overview.md +1 -1
- package/src/testarch/knowledge/pact-consumer-di.md +4 -4
- package/src/testarch/knowledge/pact-consumer-framework-setup.md +7 -3
- package/src/testarch/knowledge/pactjs-utils-consumer-helpers.md +2 -0
- package/src/testarch/knowledge/playwright-config.md +1 -1
- package/src/testarch/knowledge/risk-governance.md +1 -1
- package/src/testarch/knowledge/selective-testing.md +1 -1
- package/src/testarch/knowledge/visual-debugging.md +1 -1
- package/src/workflows/testarch/atdd/steps-c/step-04-generate-tests.md +3 -0
- package/src/workflows/testarch/atdd/steps-c/step-04a-subagent-api-failing.md +72 -1
- package/src/workflows/testarch/automate/checklist.md +29 -0
- package/src/workflows/testarch/automate/steps-c/step-02-identify-targets.md +24 -0
- package/src/workflows/testarch/automate/steps-c/step-03-generate-tests.md +2 -0
- package/src/workflows/testarch/automate/steps-c/step-03a-subagent-api.md +70 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package.json",
|
|
3
3
|
"name": "bmad-method-test-architecture-enterprise",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.6.0",
|
|
5
5
|
"description": "Master Test Architect for quality strategy, test automation, and release gates",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"bmad",
|
package/release_notes.md
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
## 🚀 What's New in v1.
|
|
1
|
+
## 🚀 What's New in v1.6.0
|
|
2
2
|
|
|
3
3
|
### ✨ New Features
|
|
4
|
-
- feat:
|
|
5
|
-
|
|
6
|
-
### 🐛 Bug Fixes
|
|
7
|
-
- fix: pr comments
|
|
4
|
+
- feat: refine pactjs automate
|
|
8
5
|
|
|
9
6
|
### 📦 Other Changes
|
|
10
|
-
-
|
|
7
|
+
- pactjs knowledge base refinements
|
|
8
|
+
- addressed PR comments
|
|
9
|
+
- addressed augment comments
|
|
10
|
+
- Merge pull request #53 from bmad-code-org/feat/refine-pactjs-automate
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
## 📦 Installation
|
|
@@ -17,4 +17,4 @@ npx bmad-method install
|
|
|
17
17
|
# Select "Test Architect" from module menu
|
|
18
18
|
```
|
|
19
19
|
|
|
20
|
-
**Full Changelog**: https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/compare/v1.5.
|
|
20
|
+
**Full Changelog**: https://github.com/bmad-code-org/bmad-method-test-architecture-enterprise/compare/v1.5.3...v1.6.0
|
|
@@ -714,4 +714,4 @@ Before deploying your CI pipeline, verify:
|
|
|
714
714
|
- Related fragments: `selective-testing.md`, `playwright-config.md`, `test-quality.md`
|
|
715
715
|
- CI tools: GitHub Actions, GitLab CI, CircleCI, Jenkins
|
|
716
716
|
|
|
717
|
-
_Source: Murat CI/CD strategy blog, Playwright/Cypress workflow examples,
|
|
717
|
+
_Source: Murat CI/CD strategy blog, Playwright/Cypress workflow examples, enterprise production pipelines_
|
|
@@ -130,7 +130,7 @@ describe('User API Contract', () => {
|
|
|
130
130
|
'Content-Type': 'application/json',
|
|
131
131
|
Accept: 'application/json',
|
|
132
132
|
},
|
|
133
|
-
body:
|
|
133
|
+
body: newUser,
|
|
134
134
|
})
|
|
135
135
|
.willRespondWith({
|
|
136
136
|
status: 201,
|
|
@@ -176,7 +176,7 @@ describe('User API Contract', () => {
|
|
|
176
176
|
**Key Points**:
|
|
177
177
|
|
|
178
178
|
- **Consumer-driven**: Frontend defines expectations, not backend
|
|
179
|
-
- **Matchers**: `like`, `string`, `integer` for flexible matching
|
|
179
|
+
- **Matchers (Postel's Law)**: Use `like`, `string`, `integer` matchers in `willRespondWith` (responses) for flexible matching. Do NOT use `like()` on request bodies in `withRequest` — the consumer controls what it sends, so request bodies should use exact values. This follows Postel's Law: be strict in what you send (requests), be lenient in what you accept (responses).
|
|
180
180
|
- **Provider states**: given() sets up test preconditions
|
|
181
181
|
- **Isolation**: No real backend needed, runs fast
|
|
182
182
|
- **Pact generation**: Automatically creates JSON pact files
|
|
@@ -944,6 +944,67 @@ jobs:
|
|
|
944
944
|
|
|
945
945
|
---
|
|
946
946
|
|
|
947
|
+
## Provider Scrutiny Protocol
|
|
948
|
+
|
|
949
|
+
When generating consumer contract tests, the agent **MUST** analyze provider source code — or the provider's OpenAPI/Swagger spec — before writing any Pact interaction. Generating contracts from consumer-side assumptions alone leads to mismatches that only surface during provider verification — wrong response shapes, wrong status codes, wrong field names, wrong types, missing required fields, and wrong enum values.
|
|
950
|
+
|
|
951
|
+
**Source priority**: Provider source code is the most authoritative reference. When an OpenAPI/Swagger spec exists (`openapi.yaml`, `openapi.json`, `swagger.json`), use it as a complementary or alternative source — it documents the provider's contract explicitly and can be faster to parse than tracing through handler code. When both exist, cross-reference them; if they disagree, the source code wins.
|
|
952
|
+
|
|
953
|
+
### Provider Endpoint Comment
|
|
954
|
+
|
|
955
|
+
Every Pact interaction MUST include a provider endpoint comment immediately above the `.given()` call:
|
|
956
|
+
|
|
957
|
+
```typescript
|
|
958
|
+
// Provider endpoint: server/src/routes/userRouteHandlers.ts -> GET /api/v2/users/:userId
|
|
959
|
+
await provider.given('user with id 1 exists').uponReceiving('a request for user 1');
|
|
960
|
+
```
|
|
961
|
+
|
|
962
|
+
**Format**: `// Provider endpoint: <relative-path-to-handler> -> <METHOD> <route-pattern>`
|
|
963
|
+
|
|
964
|
+
If the provider source is not accessible, use: `// Provider endpoint: TODO — provider source not accessible, verify manually`
|
|
965
|
+
|
|
966
|
+
### Seven-Point Scrutiny Checklist
|
|
967
|
+
|
|
968
|
+
Before generating each Pact interaction, read the provider route handler and/or OpenAPI spec and verify:
|
|
969
|
+
|
|
970
|
+
| # | Check | What to Read (source code / OpenAPI spec) | Common Mismatch |
|
|
971
|
+
| --- | --------------------- | ----------------------------------------------------------------- | ------------------------------------------------------------- |
|
|
972
|
+
| 1 | **Response shape** | Handler's `res.json()` calls / OpenAPI `responses.content.schema` | Nested object vs flat; array wrapper vs direct |
|
|
973
|
+
| 2 | **Status codes** | Handler's `res.status()` calls / OpenAPI `responses` keys | 200 vs 201 for creation; 204 vs 200 for delete |
|
|
974
|
+
| 3 | **Field names** | Response type/DTO definitions / OpenAPI `schema.properties` | `transaction_id` vs `transactionId`; `fraud_score` vs `score` |
|
|
975
|
+
| 4 | **Enum values** | Validation schemas, constants / OpenAPI `schema.enum` | `"active"` vs `"ACTIVE"`; `"pending"` vs `"in_progress"` |
|
|
976
|
+
| 5 | **Required fields** | Request validation (Joi, Zod) / OpenAPI `schema.required` | Missing required header; optional field assumed required |
|
|
977
|
+
| 6 | **Data types** | TypeScript types, DB models / OpenAPI `schema.type` + `format` | `string` ID vs `number` ID; ISO date vs Unix timestamp |
|
|
978
|
+
| 7 | **Nested structures** | Response builder, serializer / OpenAPI `$ref` + `allOf`/`oneOf` | `{ data: { items: [] } }` vs `{ items: [] }` |
|
|
979
|
+
|
|
980
|
+
### Scrutiny Evidence Block
|
|
981
|
+
|
|
982
|
+
Document what was found from provider source and/or OpenAPI spec as a block comment in the test file:
|
|
983
|
+
|
|
984
|
+
```typescript
|
|
985
|
+
/*
|
|
986
|
+
* Provider Scrutiny Evidence:
|
|
987
|
+
* - Handler: server/src/routes/userRouteHandlers.ts:45
|
|
988
|
+
* - OpenAPI: server/openapi.yaml paths./api/v2/users/{userId}.get (if available)
|
|
989
|
+
* - Response type: UserResponseDto (server/src/types/user.ts:12)
|
|
990
|
+
* - Status: 200 (line 52), 404 (line 48)
|
|
991
|
+
* - Fields: { id: number, name: string, email: string, role: "user" | "admin", createdAt: string }
|
|
992
|
+
* - Required request headers: Authorization (Bearer token)
|
|
993
|
+
* - Validation: Zod schema at server/src/validation/user.ts:8
|
|
994
|
+
*/
|
|
995
|
+
```
|
|
996
|
+
|
|
997
|
+
### Graceful Degradation
|
|
998
|
+
|
|
999
|
+
When provider source code is not accessible (different repo, no access, closed source):
|
|
1000
|
+
|
|
1001
|
+
1. **OpenAPI/Swagger spec available**: Use the spec as the source of truth for response shapes, status codes, and field names
|
|
1002
|
+
2. **Pact Broker has existing contracts**: Use `pact_mcp` tools to fetch existing provider states and verified interactions as reference
|
|
1003
|
+
3. **Neither available**: Generate contracts from consumer-side types but use the TODO form of the mandatory comment: `// Provider endpoint: TODO — provider source not accessible, verify manually` and add a `provider_scrutiny: "pending"` field to the output JSON
|
|
1004
|
+
4. **Never silently guess**: If you cannot verify, document what you assumed and why
|
|
1005
|
+
|
|
1006
|
+
---
|
|
1007
|
+
|
|
947
1008
|
## Contract Testing Checklist
|
|
948
1009
|
|
|
949
1010
|
Before implementing contract testing, verify:
|
|
@@ -956,6 +1017,9 @@ Before implementing contract testing, verify:
|
|
|
956
1017
|
- [ ] **Webhooks configured**: Consumer changes trigger provider verification
|
|
957
1018
|
- [ ] **Retention policy**: Old pacts archived (keep 30 days, all production tags)
|
|
958
1019
|
- [ ] **Resilience tested**: Timeouts, retries, error codes in contracts
|
|
1020
|
+
- [ ] **Provider endpoint comments**: Every Pact interaction has `// Provider endpoint:` comment
|
|
1021
|
+
- [ ] **Provider scrutiny completed**: Seven-point checklist verified for each interaction
|
|
1022
|
+
- [ ] **Scrutiny evidence documented**: Block comment with handler, types, status codes, and fields
|
|
959
1023
|
|
|
960
1024
|
## Integration Points
|
|
961
1025
|
|
|
@@ -722,4 +722,4 @@ Before shipping error handling code, verify:
|
|
|
722
722
|
- Related fragments: `network-first.md`, `test-quality.md`, `contract-testing.md`
|
|
723
723
|
- Monitoring tools: Sentry, Datadog, LogRocket
|
|
724
724
|
|
|
725
|
-
_Source: Murat error-handling patterns, Pact resilience guidance,
|
|
725
|
+
_Source: Murat error-handling patterns, Pact resilience guidance, enterprise production error handling_
|
|
@@ -747,4 +747,4 @@ Before merging flag-related code, verify:
|
|
|
747
747
|
- Related fragments: `test-quality.md`, `selective-testing.md`
|
|
748
748
|
- Flag services: LaunchDarkly, Split.io, Unleash, custom implementations
|
|
749
749
|
|
|
750
|
-
_Source: LaunchDarkly strategy blog, Murat test architecture notes,
|
|
750
|
+
_Source: LaunchDarkly strategy blog, Murat test architecture notes, enterprise feature flag governance_
|
|
@@ -398,4 +398,4 @@ When deciding whether to create a fixture, follow these rules:
|
|
|
398
398
|
- **1 use** → Keep inline (avoid premature abstraction)
|
|
399
399
|
- **Complex logic** → Factory function pattern (dynamic data generation)
|
|
400
400
|
|
|
401
|
-
_Source: Murat Testing Philosophy (lines 74-122),
|
|
401
|
+
_Source: Murat Testing Philosophy (lines 74-122), enterprise production patterns, Playwright fixture docs._
|
|
@@ -304,13 +304,13 @@ await networkRecorder.setup(context, {
|
|
|
304
304
|
playback: {
|
|
305
305
|
urlMapping: {
|
|
306
306
|
hostMapping: {
|
|
307
|
-
'localhost:3000': 'admin.
|
|
308
|
-
'admin-staging.
|
|
309
|
-
'admin.
|
|
307
|
+
'localhost:3000': 'admin.example.com',
|
|
308
|
+
'admin-staging.example.com': 'admin.example.com',
|
|
309
|
+
'admin.example.com': 'admin.example.com',
|
|
310
310
|
},
|
|
311
311
|
patterns: [
|
|
312
|
-
{ match: /admin-\d+\.
|
|
313
|
-
{ match: /admin-staging-pr-\w+-\d\.
|
|
312
|
+
{ match: /admin-\d+\.example\.com/, replace: 'admin.example.com' },
|
|
313
|
+
{ match: /admin-staging-pr-\w+-\d\.example\.com/, replace: 'admin.example.com' },
|
|
314
314
|
],
|
|
315
315
|
},
|
|
316
316
|
},
|
|
@@ -15,7 +15,7 @@ Writing Playwright utilities from scratch for every project leads to:
|
|
|
15
15
|
|
|
16
16
|
`@seontechnologies/playwright-utils` provides:
|
|
17
17
|
|
|
18
|
-
- **Production-tested
|
|
18
|
+
- **Production-tested**: Used in enterprise production environments
|
|
19
19
|
- **Functional-first design**: Core logic as pure functions, fixtures for convenience
|
|
20
20
|
- **Composable fixtures**: Use `mergeTests` to combine utilities
|
|
21
21
|
- **TypeScript support**: Full type safety with generic types
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
Inject the Pact mock server URL into consumer code via an optional `baseUrl` field on the API context type instead of using raw `fetch()` inside `executeTest()`. This ensures contract tests exercise the real consumer HTTP client — including retry logic, header assembly, timeout configuration, error handling, and metrics — rather than testing Pact itself.
|
|
6
6
|
|
|
7
|
-
The base URL is typically a module-level constant evaluated at import time (`export const API_BASE_URL = env.
|
|
7
|
+
The base URL is typically a module-level constant evaluated at import time (`export const API_BASE_URL = env.API_BASE_URL`), but `mockServer.url` is only available at runtime inside `executeTest()`. Dependency injection solves this timing mismatch cleanly: add one optional field to the context type, use nullish coalescing in the HTTP client factory, and inject the mock server URL in tests.
|
|
8
8
|
|
|
9
9
|
## Rationale
|
|
10
10
|
|
|
@@ -124,7 +124,7 @@ export function createTestContext(mockServerUrl: string): ApiContext {
|
|
|
124
124
|
|
|
125
125
|
```typescript
|
|
126
126
|
.executeTest(async (mockServer: V3MockServer) => {
|
|
127
|
-
const api =
|
|
127
|
+
const api = createApiClient(createTestContext(mockServer.url));
|
|
128
128
|
const result = await api.getFilterFields();
|
|
129
129
|
expect(result).toEqual(
|
|
130
130
|
expect.arrayContaining([
|
|
@@ -277,7 +277,7 @@ expect(response.status).toBe(200);
|
|
|
277
277
|
|
|
278
278
|
```typescript
|
|
279
279
|
// GOOD: Exercises real client, validates parsed return value
|
|
280
|
-
const api =
|
|
280
|
+
const api = createApiClient(createTestContext(mockServer.url));
|
|
281
281
|
const result = await api.searchTransactions(request);
|
|
282
282
|
expect(result.transactions).toBeDefined();
|
|
283
283
|
```
|
|
@@ -307,4 +307,4 @@ Used in workflows:
|
|
|
307
307
|
|
|
308
308
|
## Source
|
|
309
309
|
|
|
310
|
-
Pattern derived from
|
|
310
|
+
Pattern derived from my-consumer-app Pact consumer test refactor (March 2026). Implements dependency injection for testability as described in Pact.js best practices.
|
|
@@ -418,15 +418,15 @@ describe('Movies API Consumer Contract', () => {
|
|
|
418
418
|
)
|
|
419
419
|
.willRespondWith(
|
|
420
420
|
200,
|
|
421
|
-
|
|
422
|
-
|
|
421
|
+
setJsonBody(
|
|
422
|
+
like({
|
|
423
423
|
id: integer(1),
|
|
424
424
|
name: string('The Matrix'),
|
|
425
425
|
year: integer(1999),
|
|
426
426
|
rating: like(8.7),
|
|
427
427
|
director: string('Wachowskis'),
|
|
428
428
|
}),
|
|
429
|
-
|
|
429
|
+
),
|
|
430
430
|
)
|
|
431
431
|
.executeTest(async (mockServer: V3MockServer) => {
|
|
432
432
|
// Inject mock server URL into the REAL consumer code
|
|
@@ -463,6 +463,7 @@ describe('Movies API Consumer Contract', () => {
|
|
|
463
463
|
- Consumer code MUST expose a URL injection mechanism: `setApiUrl()`, env var override, or constructor parameter
|
|
464
464
|
- If the consumer code doesn't support URL injection, add it — this is a design prerequisite for CDC testing
|
|
465
465
|
- Use PactV4 `addInteraction()` builder (not PactV3 fluent API with `withRequest({...})` object)
|
|
466
|
+
- **Interaction naming convention**: Use the pattern `"a request to <action> <resource> [<condition>]"` for `uponReceiving()`. Examples: `"a request to get a movie by ID"`, `"a request to delete a non-existing movie"`, `"a request to create a movie that already exists"`. These names appear in Pact Broker UI and verification logs — keep them descriptive and unique within the consumer-provider pair.
|
|
466
467
|
- Use `setJsonContent` for request/response builder callbacks with query/header/body concerns; use `setJsonBody` for body-only response callbacks
|
|
467
468
|
- Provider state factory functions (`movieExists`) return `ProviderStateInput` objects
|
|
468
469
|
- `createProviderState` converts to `[stateName, stateParams]` tuple for `.given()`
|
|
@@ -616,7 +617,10 @@ Before presenting the consumer CDC framework to the user, verify:
|
|
|
616
617
|
- [ ] `.github/actions/detect-breaking-change/action.yml` exists
|
|
617
618
|
- [ ] Consumer tests use `.pacttest.ts` extension
|
|
618
619
|
- [ ] Consumer tests use PactV4 `addInteraction()` builder
|
|
620
|
+
- [ ] `uponReceiving()` names follow `"a request to <action> <resource> [<condition>]"` pattern and are unique within the consumer-provider pair
|
|
619
621
|
- [ ] Interaction callbacks use `setJsonContent` for query/header/body and `setJsonBody` for body-only responses
|
|
622
|
+
- [ ] Request bodies use exact values (no `like()` wrapper) — Postel's Law: be strict in what you send
|
|
623
|
+
- [ ] `like()`, `eachLike()`, `string()`, `integer()` matchers are only used in `willRespondWith` (responses), not in `withRequest` (requests) — matchers check type/shape, not exact values
|
|
620
624
|
- [ ] Consumer tests call REAL consumer code (actual API client functions), NOT raw `fetch()`
|
|
621
625
|
- [ ] Consumer code exposes URL injection mechanism (`setApiUrl()`, env var, or constructor param)
|
|
622
626
|
- [ ] Local consumer-helpers shim present if pactjs-utils not installed
|
|
@@ -196,6 +196,8 @@ await pact
|
|
|
196
196
|
- **Message pacts**: Works identically with `MessageConsumerPact` — same `.given()` API
|
|
197
197
|
- **Builder reuse**: `setJsonContent` works for both `.withRequest(...)` and `.willRespondWith(...)` callbacks (query is ignored on response builders)
|
|
198
198
|
- **Body shorthand**: `setJsonBody` keeps body-only responses concise and readable
|
|
199
|
+
- **Matchers check type, not value**: `string('My movie')` means "any string", `integer(1)` means "any integer". The example values are arbitrary — the provider can return different values and verification still passes as long as the type matches. Use matchers only in `.willRespondWith()` (responses), never in `.withRequest()` (requests) — Postel's Law applies.
|
|
200
|
+
- **Reuse test values across files**: Interactions are uniquely identified by `uponReceiving` + `.given()`, not by placeholder values. Two test files can both use `testId: 100` without conflicting. On the provider side, shared values simplify state handlers — idempotent handlers (check if exists, create if not) only need to ensure one record exists. Use different values only when testing different states of the same entity type (e.g., `movieExists(100)` for happy paths vs. `movieNotFound(999)` for error paths).
|
|
199
201
|
|
|
200
202
|
## Related Fragments
|
|
201
203
|
|
|
@@ -727,4 +727,4 @@ jobs:
|
|
|
727
727
|
- [ ] Projects defined for cross-browser/device testing (if needed)
|
|
728
728
|
- [ ] CI uploads artifacts on failure with 30-day retention
|
|
729
729
|
|
|
730
|
-
_Source: Playwright book repo,
|
|
730
|
+
_Source: Playwright book repo, enterprise configuration example, Murat testing philosophy (lines 216-271)._
|
|
@@ -612,4 +612,4 @@ Before deploying to production, ensure:
|
|
|
612
612
|
- **Related fragments**: `probability-impact.md` (scoring definitions), `test-priorities-matrix.md` (P0-P3 classification), `nfr-criteria.md` (non-functional risks)
|
|
613
613
|
- **Tools**: Risk tracking dashboards (Jira, Linear), gate automation (CI/CD), traceability reports (Markdown, Confluence)
|
|
614
614
|
|
|
615
|
-
_Source: Murat risk governance notes, gate schema guidance,
|
|
615
|
+
_Source: Murat risk governance notes, gate schema guidance, enterprise production gate workflows, ISO 31000 risk management standards_
|
|
@@ -728,5 +728,5 @@ Before implementing selective testing, verify:
|
|
|
728
728
|
- Related fragments: `ci-burn-in.md`, `test-priorities-matrix.md`, `test-quality.md`
|
|
729
729
|
- Selection tools: Playwright --grep, Cypress @cypress/grep, git diff
|
|
730
730
|
|
|
731
|
-
_Source: 32+ selective testing strategies blog, Murat testing philosophy,
|
|
731
|
+
_Source: 32+ selective testing strategies blog, Murat testing philosophy, enterprise CI optimization_
|
|
732
732
|
```
|
|
@@ -521,4 +521,4 @@ Before deploying tests to CI, ensure:
|
|
|
521
521
|
- **Related fragments**: `playwright-config.md` (artifact configuration), `ci-burn-in.md` (CI artifact upload), `test-quality.md` (debugging best practices)
|
|
522
522
|
- **Tools**: Playwright Trace Viewer, Cypress Debug UI, axe-core, HAR files
|
|
523
523
|
|
|
524
|
-
_Source: Playwright official docs, Murat testing philosophy (visual debugging manifesto),
|
|
524
|
+
_Source: Playwright official docs, Murat testing philosophy (visual debugging manifesto), enterprise production debugging patterns_
|
|
@@ -71,9 +71,12 @@ const subagentContext = {
|
|
|
71
71
|
config: {
|
|
72
72
|
test_framework: config.test_framework,
|
|
73
73
|
use_playwright_utils: config.tea_use_playwright_utils,
|
|
74
|
+
use_pactjs_utils: config.tea_use_pactjs_utils,
|
|
75
|
+
pact_mcp: config.tea_pact_mcp, // "mcp" | "none"
|
|
74
76
|
browser_automation: config.tea_browser_automation,
|
|
75
77
|
execution_mode: config.tea_execution_mode || 'auto', // "auto" | "subagent" | "agent-team" | "sequential"
|
|
76
78
|
capability_probe: parseBooleanFlag(config.tea_capability_probe, true), // supports booleans and "false"/"true" strings
|
|
79
|
+
provider_endpoint_map: /* from Step 1/3 context, if use_pactjs_utils enabled */,
|
|
77
80
|
},
|
|
78
81
|
timestamp: timestamp
|
|
79
82
|
};
|
|
@@ -16,7 +16,8 @@ This is an **isolated subagent** running in parallel with E2E failing test gener
|
|
|
16
16
|
- Story acceptance criteria from Step 1
|
|
17
17
|
- Test strategy and scenarios from Step 3
|
|
18
18
|
- Knowledge fragments loaded: api-request, data-factories, api-testing-patterns
|
|
19
|
-
- Config: test framework, Playwright Utils enabled/disabled
|
|
19
|
+
- Config: test framework, Playwright Utils enabled/disabled, Pact.js Utils enabled/disabled (`use_pactjs_utils`), Pact MCP mode (`pact_mcp`)
|
|
20
|
+
- Provider Endpoint Map (if `use_pactjs_utils` enabled and provider source accessible)
|
|
20
21
|
|
|
21
22
|
**Your task:** Generate API tests that will FAIL because the feature is not implemented yet (TDD RED PHASE).
|
|
22
23
|
|
|
@@ -113,6 +114,72 @@ test.describe('[Story Name] API Tests (ATDD)', () => {
|
|
|
113
114
|
- ✅ Use data factories for test data (from data-factories fragment)
|
|
114
115
|
- ✅ Include priority tags [P0], [P1], [P2], [P3]
|
|
115
116
|
|
|
117
|
+
### 1.5 Provider Source Scrutiny for CDC in TDD Red Phase (If `use_pactjs_utils` Enabled)
|
|
118
|
+
|
|
119
|
+
When generating Pact consumer contract tests in the ATDD red phase, provider scrutiny applies with TDD-specific rules. Apply the **Seven-Point Scrutiny Checklist** from `contract-testing.md` (Response shape, Status codes, Field names, Enum values, Required fields, Data types, Nested structures) for both existing and new endpoints.
|
|
120
|
+
|
|
121
|
+
**If provider endpoint already exists** (extending an existing API):
|
|
122
|
+
|
|
123
|
+
- READ the provider route handler, types, and validation schemas
|
|
124
|
+
- Verify all seven scrutiny points against the provider source: Response shape, Status codes, Field names, Enum values, Required fields, Data types, Nested structures
|
|
125
|
+
- Add `// Provider endpoint:` comment and scrutiny evidence block documenting findings for each point
|
|
126
|
+
- Wrap the entire test function in `test.skip()` (so the whole test including `executeTest` is skipped), not just the callback
|
|
127
|
+
|
|
128
|
+
**If provider endpoint is new** (TDD — endpoint not implemented yet):
|
|
129
|
+
|
|
130
|
+
- Use acceptance criteria as the source of truth for expected behavior
|
|
131
|
+
- Acceptance criteria should specify all seven scrutiny points where possible (status codes, field names, types, etc.) — note any gaps as assumptions in the evidence block
|
|
132
|
+
- Add `// Provider endpoint: TODO — new endpoint, not yet implemented`
|
|
133
|
+
- Document expected behavior from acceptance criteria in scrutiny evidence block
|
|
134
|
+
- Wrap the entire test function in `test.skip()` and use realistic expectations from the story
|
|
135
|
+
|
|
136
|
+
**Graceful degradation when provider source is inaccessible:**
|
|
137
|
+
|
|
138
|
+
1. **OpenAPI/Swagger spec available**: Use the spec as the source of truth for response shapes, status codes, and field names
|
|
139
|
+
2. **Pact Broker available** (when `pact_mcp` is `"mcp"`): Use SmartBear MCP tools to fetch existing provider states and verified interactions as reference
|
|
140
|
+
3. **Neither available**: For new endpoints, use acceptance criteria; for existing endpoints, use consumer-side types. Mark with `// Provider endpoint: TODO — provider source not accessible, verify manually` and set `provider_scrutiny: "pending"` in output JSON
|
|
141
|
+
4. **Never silently guess**: Document all assumptions in the scrutiny evidence block
|
|
142
|
+
|
|
143
|
+
**Provider endpoint comments are MANDATORY** even in red-phase tests — they document the intent.
|
|
144
|
+
|
|
145
|
+
**Example: Red-phase Pact test with provider scrutiny:**
|
|
146
|
+
|
|
147
|
+
```typescript
|
|
148
|
+
// Provider endpoint: TODO — new endpoint, not yet implemented
|
|
149
|
+
/*
|
|
150
|
+
* Provider Scrutiny Evidence:
|
|
151
|
+
* - Handler: NEW — not yet implemented (TDD red phase)
|
|
152
|
+
* - Expected from acceptance criteria:
|
|
153
|
+
* - Endpoint: POST /api/v2/users/register
|
|
154
|
+
* - Status: 201 for success, 400 for duplicate email, 422 for validation error
|
|
155
|
+
* - Response: { id: number, email: string, createdAt: string }
|
|
156
|
+
*/
|
|
157
|
+
test.skip('[P0] should generate consumer contract for user registration', async () => {
|
|
158
|
+
await provider
|
|
159
|
+
.given('no users exist')
|
|
160
|
+
.uponReceiving('a request to register a new user')
|
|
161
|
+
.withRequest({
|
|
162
|
+
method: 'POST',
|
|
163
|
+
path: '/api/v2/users/register',
|
|
164
|
+
headers: { 'Content-Type': 'application/json' },
|
|
165
|
+
body: { email: 'newuser@example.com', password: 'SecurePass123!' },
|
|
166
|
+
})
|
|
167
|
+
.willRespondWith({
|
|
168
|
+
status: 201,
|
|
169
|
+
headers: { 'Content-Type': 'application/json' },
|
|
170
|
+
body: like({
|
|
171
|
+
id: integer(1),
|
|
172
|
+
email: string('newuser@example.com'),
|
|
173
|
+
createdAt: string('2025-01-15T10:00:00Z'),
|
|
174
|
+
}),
|
|
175
|
+
})
|
|
176
|
+
.executeTest(async (mockServer) => {
|
|
177
|
+
const result = await registerUser({ email: 'newuser@example.com', password: 'SecurePass123!' }, { baseUrl: mockServer.url });
|
|
178
|
+
expect(result.id).toEqual(expect.any(Number));
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
```
|
|
182
|
+
|
|
116
183
|
**Why test.skip():**
|
|
117
184
|
|
|
118
185
|
- Tests are written correctly for EXPECTED behavior
|
|
@@ -163,6 +230,7 @@ Write JSON to temp file: `/tmp/tea-atdd-api-tests-{{timestamp}}.json`
|
|
|
163
230
|
"knowledge_fragments_used": ["api-request", "data-factories", "api-testing-patterns"],
|
|
164
231
|
"test_count": 3,
|
|
165
232
|
"tdd_phase": "RED",
|
|
233
|
+
"provider_scrutiny": "completed",
|
|
166
234
|
"summary": "Generated 3 FAILING API tests for user registration story"
|
|
167
235
|
}
|
|
168
236
|
```
|
|
@@ -205,6 +273,8 @@ Subagent completes when:
|
|
|
205
273
|
- JSON output valid and complete
|
|
206
274
|
- No E2E/component/unit tests included (out of scope)
|
|
207
275
|
- Tests follow knowledge fragment patterns
|
|
276
|
+
- Every Pact interaction has `// Provider endpoint:` comment (if CDC enabled)
|
|
277
|
+
- Provider scrutiny completed or TODO markers added for new endpoints (if CDC enabled)
|
|
208
278
|
|
|
209
279
|
### ❌ FAILURE:
|
|
210
280
|
|
|
@@ -213,3 +283,4 @@ Subagent completes when:
|
|
|
213
283
|
- Placeholder assertions (expect(true).toBe(true))
|
|
214
284
|
- Did not follow knowledge fragment patterns
|
|
215
285
|
- Invalid or missing JSON output
|
|
286
|
+
- Pact interactions missing provider endpoint comments (if CDC enabled)
|
|
@@ -191,6 +191,33 @@ Before starting this workflow, verify:
|
|
|
191
191
|
- [ ] Error cases tested (400, 401, 403, 404, 500)
|
|
192
192
|
- [ ] JWT token format validated (if auth tests)
|
|
193
193
|
|
|
194
|
+
### Consumer Contract Tests / CDC (If `use_pactjs_utils` Enabled)
|
|
195
|
+
|
|
196
|
+
**Provider Endpoint Comments:**
|
|
197
|
+
|
|
198
|
+
- [ ] Every Pact interaction has `// Provider endpoint:` comment
|
|
199
|
+
- [ ] Comment includes exact file path to provider route handler, OR uses the TODO form when provider is inaccessible
|
|
200
|
+
- [ ] Comment follows format: `// Provider endpoint: <path> -> <METHOD> <route>` or `// Provider endpoint: TODO — provider source not accessible, verify manually`
|
|
201
|
+
|
|
202
|
+
**Provider Source Scrutiny:**
|
|
203
|
+
|
|
204
|
+
- [ ] Provider route handlers and/or OpenAPI spec read before generating each interaction
|
|
205
|
+
- [ ] Status codes verified against provider source (e.g., 201 not assumed 200)
|
|
206
|
+
- [ ] Field names cross-referenced with provider type/DTO definitions
|
|
207
|
+
- [ ] Data types verified (string ID vs number ID, date formats)
|
|
208
|
+
- [ ] Enum/union values extracted from provider validation schemas
|
|
209
|
+
- [ ] Required request fields and headers checked against provider validation
|
|
210
|
+
- [ ] Nested response structures match provider's actual response construction
|
|
211
|
+
- [ ] Scrutiny evidence documented as block comment in each test file
|
|
212
|
+
|
|
213
|
+
**CDC Quality Gates:**
|
|
214
|
+
|
|
215
|
+
- [ ] Postel's Law enforced: exact values in `withRequest`, matchers in `willRespondWith`
|
|
216
|
+
- [ ] Response matchers (`like`, `eachLike`, `string`, `integer`) used only in `willRespondWith`
|
|
217
|
+
- [ ] Provider state names are consistent with provider's state handler naming
|
|
218
|
+
- [ ] DI pattern used for consumer function imports (actual consumer code, not raw `fetch()`)
|
|
219
|
+
- [ ] One logical endpoint per Pact interaction (no multi-endpoint interactions)
|
|
220
|
+
|
|
194
221
|
### Component Tests (If Applicable)
|
|
195
222
|
|
|
196
223
|
- [ ] Component test files created in `tests/component/`
|
|
@@ -467,6 +494,8 @@ All of the following must be true before marking this workflow as complete:
|
|
|
467
494
|
- [ ] **Output file formatted correctly**
|
|
468
495
|
- [ ] **Knowledge base references applied** and documented (including healing fragments if used)
|
|
469
496
|
- [ ] **No test quality issues** (flaky patterns, race conditions, hardcoded data, page objects)
|
|
497
|
+
- [ ] **Provider scrutiny completed or gracefully degraded** for all CDC interactions — each interaction either has scrutiny evidence or a TODO marker (if `use_pactjs_utils` enabled)
|
|
498
|
+
- [ ] **Provider endpoint comments present** on every Pact interaction (if `use_pactjs_utils` enabled)
|
|
470
499
|
|
|
471
500
|
---
|
|
472
501
|
|
|
@@ -77,6 +77,30 @@ Use CLI to explore the application and identify testable pages/flows:
|
|
|
77
77
|
|
|
78
78
|
---
|
|
79
79
|
|
|
80
|
+
**If `use_pactjs_utils` is enabled — Provider Endpoint Mapping (all stacks):**
|
|
81
|
+
|
|
82
|
+
When consumer-driven contract tests will be generated, build a Provider Endpoint Map during target identification. This applies to all `{detected_stack}` values — frontend, backend, and fullstack consumers all need provider scrutiny.
|
|
83
|
+
|
|
84
|
+
1. **Locate provider source and/or OpenAPI spec**: Scan workspace for provider project (from config, monorepo structure, or adjacent repositories). Also check for OpenAPI/Swagger spec files (`openapi.yaml`, `openapi.json`, `swagger.json`) — these document the provider's contract explicitly and can supplement or replace handler code analysis.
|
|
85
|
+
2. **Map each consumer endpoint** to its provider counterpart:
|
|
86
|
+
- Provider file path (route handler)
|
|
87
|
+
- Route pattern (METHOD + path)
|
|
88
|
+
- Validation schema location (Joi, Zod, class-validator) or OpenAPI request schema
|
|
89
|
+
- Response type/DTO definition location or OpenAPI response schema
|
|
90
|
+
- OpenAPI spec path (if available, e.g., `server/openapi.yaml`)
|
|
91
|
+
3. **Output as "Provider Endpoint Map" table** in the coverage plan:
|
|
92
|
+
|
|
93
|
+
```markdown
|
|
94
|
+
| Consumer Endpoint | Provider File | Route | Validation Schema | Response Type | OpenAPI Spec |
|
|
95
|
+
| --------------------- | --------------------------------- | ------------------------- | ----------------------------------- | --------------- | ------------------------------------------------- |
|
|
96
|
+
| GET /api/v2/users/:id | server/src/routes/userHandlers.ts | GET /api/v2/users/:userId | server/src/validation/user.ts | UserResponseDto | server/openapi.yaml#/paths/~1api~1v2~1users~1{id} |
|
|
97
|
+
| POST /api/v2/users | server/src/routes/userHandlers.ts | POST /api/v2/users | server/src/validation/createUser.ts | UserResponseDto | server/openapi.yaml#/paths/~1api~1v2~1users |
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
4. **If provider source not accessible**: Mark entries with `TODO — provider source not accessible` and note in coverage plan that provider scrutiny will use graceful degradation (see `contract-testing.md` Provider Scrutiny Protocol)
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
80
104
|
## 2. Choose Test Levels
|
|
81
105
|
|
|
82
106
|
Use `test-levels-framework.md` to select:
|
|
@@ -74,6 +74,7 @@ const subagentContext = {
|
|
|
74
74
|
detected_stack: '{detected_stack}', // "frontend" | "backend" | "fullstack"
|
|
75
75
|
execution_mode: config.tea_execution_mode || 'auto', // "auto" | "subagent" | "agent-team" | "sequential"
|
|
76
76
|
capability_probe: parseBooleanFlag(config.tea_capability_probe, true), // supports booleans and "false"/"true" strings
|
|
77
|
+
provider_endpoint_map: /* from Step 2 coverage plan, if use_pactjs_utils enabled */,
|
|
77
78
|
},
|
|
78
79
|
timestamp: timestamp
|
|
79
80
|
};
|
|
@@ -190,6 +191,7 @@ When `use_pactjs_utils` is enabled, the API test generation subagent (step-03a)
|
|
|
190
191
|
- **Provider verification tests**: Using `buildVerifierOptions` for one-call verifier setup
|
|
191
192
|
- **Message contract tests**: Using `buildMessageVerifierOptions` if async/Kafka patterns detected
|
|
192
193
|
- **Helper files**: Request filter setup with `createRequestFilter`, shared state constants
|
|
194
|
+
- **Provider scrutiny**: Subagent reads provider route handlers, types, and validation schemas before generating each interaction (see `contract-testing.md` Provider Scrutiny Protocol)
|
|
193
195
|
|
|
194
196
|
When `pact_mcp` is `"mcp"`, the subagent can use SmartBear MCP tools to fetch existing provider states and generate tests informed by broker data.
|
|
195
197
|
|
|
@@ -97,6 +97,69 @@ test.describe('[Feature] API Tests', () => {
|
|
|
97
97
|
- ✅ Generate request filter helpers in `pact/http/helpers/` using `createRequestFilter({ tokenGenerator: () => string })`
|
|
98
98
|
- ✅ Generate shared state constants in `pact/http/helpers/states.ts`
|
|
99
99
|
- ✅ If async/message patterns detected, generate message consumer tests in `pact/message/` using `buildMessageVerifierOptions`
|
|
100
|
+
- ✅ **Provider endpoint comment MANDATORY** on every Pact interaction: `// Provider endpoint: <path> -> <METHOD> <route>`
|
|
101
|
+
- ⚠️ **Postel's Law for matchers**: Use `like()`, `eachLike()`, `string()`, `integer()` matchers ONLY in `willRespondWith` (responses). Request bodies in `withRequest` MUST use exact values — never wrap request bodies in `like()`. The consumer controls what it sends, so contracts should be strict about request shape.
|
|
102
|
+
|
|
103
|
+
### 1.5 Provider Source Scrutiny (CDC Only)
|
|
104
|
+
|
|
105
|
+
**CRITICAL**: Before generating ANY Pact consumer interaction, perform provider source scrutiny per the **Seven-Point Scrutiny Checklist** defined in `contract-testing.md`. Do NOT generate response matchers from consumer-side types alone — this is the #1 cause of contract verification failures.
|
|
106
|
+
|
|
107
|
+
The seven points to verify for each interaction:
|
|
108
|
+
|
|
109
|
+
1. Response shape
|
|
110
|
+
2. Status codes
|
|
111
|
+
3. Field names
|
|
112
|
+
4. Enum values
|
|
113
|
+
5. Required fields
|
|
114
|
+
6. Data types
|
|
115
|
+
7. Nested structures
|
|
116
|
+
|
|
117
|
+
**Source priority**: Provider source code is most authoritative. When an OpenAPI/Swagger spec exists (`openapi.yaml`, `openapi.json`, `swagger.json`), use it as a complementary or alternative source — it documents the provider's contract explicitly and can be faster to parse than tracing through handler code. When both exist, cross-reference them; if they disagree, the source code wins. Document the discrepancy in the scrutiny evidence block (e.g., `OpenAPI shows 200 but handler returns 201; using handler behavior`) and flag it in the output JSON `summary` so it is discoverable by downstream consumers or audits.
|
|
118
|
+
|
|
119
|
+
**Scrutiny Sequence** (for each endpoint in the coverage plan):
|
|
120
|
+
|
|
121
|
+
1. **READ provider route handler and/or OpenAPI spec**: Find the handler file from `subagentContext.config.provider_endpoint_map` or by scanning the provider codebase. Also check for OpenAPI/Swagger spec files. Extract:
|
|
122
|
+
- Exact status codes returned (`res.status(201)` / OpenAPI `responses` keys)
|
|
123
|
+
- Response construction (`res.json({ data: ... })` / OpenAPI `schema`)
|
|
124
|
+
- Error handling paths (what status codes for what conditions)
|
|
125
|
+
|
|
126
|
+
2. **READ provider type/model/DTO definitions**: Find the response type referenced by the handler or OpenAPI `$ref` schemas. Extract:
|
|
127
|
+
- Exact field names (`transaction_id` not `transactionId`)
|
|
128
|
+
- Field types (`string` ID vs `number` ID / OpenAPI `type` + `format`)
|
|
129
|
+
- Optional vs required fields (OpenAPI `required` array)
|
|
130
|
+
- Nested object structures (OpenAPI `$ref`, `allOf`, `oneOf`)
|
|
131
|
+
|
|
132
|
+
3. **READ provider validation schemas**: Find Joi/Zod/class-validator schemas or OpenAPI request body `schema.required`. Extract:
|
|
133
|
+
- Required request fields and headers
|
|
134
|
+
- Enum/union type allowed values (`"active" | "inactive"` / OpenAPI `enum`)
|
|
135
|
+
- Request body constraints
|
|
136
|
+
|
|
137
|
+
4. **Cross-reference findings** against consumer expectations:
|
|
138
|
+
- Does the consumer expect the same field names the provider sends?
|
|
139
|
+
- Does the consumer expect the same status codes the provider returns?
|
|
140
|
+
- Does the consumer expect the same nesting the provider produces?
|
|
141
|
+
|
|
142
|
+
5. **Document scrutiny evidence** as a block comment in the generated test:
|
|
143
|
+
|
|
144
|
+
```typescript
|
|
145
|
+
/*
|
|
146
|
+
* Provider Scrutiny Evidence:
|
|
147
|
+
* - Handler: server/src/routes/userHandlers.ts:45
|
|
148
|
+
* - OpenAPI: server/openapi.yaml paths./api/v2/users/{userId}.get (if available)
|
|
149
|
+
* - Response type: UserResponseDto (server/src/types/user.ts:12)
|
|
150
|
+
* - Status: 201 for creation (line 52), 400 for validation error (line 48)
|
|
151
|
+
* - Fields: { id: number, name: string, email: string, role: "user" | "admin" }
|
|
152
|
+
* - Required request headers: Authorization (Bearer token)
|
|
153
|
+
*/
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
6. **Graceful degradation** when provider source is not accessible (follows the canonical four-step protocol from `contract-testing.md`):
|
|
157
|
+
1. **OpenAPI/Swagger spec available**: Use the spec as the source of truth for response shapes, status codes, and field names
|
|
158
|
+
2. **Pact Broker available** (when `pact_mcp` is `"mcp"` in `subagentContext.config`): Use SmartBear MCP tools to fetch existing provider states and verified interactions as reference
|
|
159
|
+
3. **Neither available**: Generate from consumer types but use the TODO form of the mandatory comment: `// Provider endpoint: TODO — provider source not accessible, verify manually`. Set `provider_scrutiny: "pending"` in output JSON
|
|
160
|
+
4. **Never silently guess**: Document all assumptions in the scrutiny evidence block
|
|
161
|
+
|
|
162
|
+
> ⚠️ **Anti-pattern**: Generating response matchers from consumer-side types alone. This produces contracts that reflect what the consumer _wishes_ the provider returns, not what it _actually_ returns. Always read provider source or OpenAPI spec first.
|
|
100
163
|
|
|
101
164
|
### 3. Track Fixture Needs
|
|
102
165
|
|
|
@@ -144,6 +207,8 @@ Write JSON to temp file: `/tmp/tea-automate-api-tests-{{timestamp}}.json`
|
|
|
144
207
|
],
|
|
145
208
|
"fixture_needs": ["authToken", "userDataFactory", "productDataFactory"],
|
|
146
209
|
"knowledge_fragments_used": ["api-request", "data-factories", "api-testing-patterns"],
|
|
210
|
+
"provider_scrutiny": "completed",
|
|
211
|
+
"provider_files_read": ["server/src/routes/authHandlers.ts", "server/src/routes/checkoutHandlers.ts", "server/src/types/auth.ts"],
|
|
147
212
|
"test_count": 12,
|
|
148
213
|
"summary": "Generated 12 API test cases covering 3 features"
|
|
149
214
|
}
|
|
@@ -184,6 +249,9 @@ Subagent completes when:
|
|
|
184
249
|
- All API tests generated following patterns
|
|
185
250
|
- JSON output valid and complete
|
|
186
251
|
- No E2E/component/unit tests included (out of scope)
|
|
252
|
+
- Every Pact interaction has `// Provider endpoint:` comment (if CDC enabled)
|
|
253
|
+
- Provider source scrutiny completed or gracefully degraded with TODO markers (if CDC enabled)
|
|
254
|
+
- Scrutiny evidence documented as block comments in test files (if CDC enabled)
|
|
187
255
|
|
|
188
256
|
### ❌ FAILURE:
|
|
189
257
|
|
|
@@ -191,3 +259,5 @@ Subagent completes when:
|
|
|
191
259
|
- Did not follow knowledge fragment patterns
|
|
192
260
|
- Invalid or missing JSON output
|
|
193
261
|
- Ran tests (not subagent responsibility)
|
|
262
|
+
- Pact interactions missing provider endpoint comments (if CDC enabled)
|
|
263
|
+
- Response matchers generated from consumer-side types without provider scrutiny (if CDC enabled)
|