@freshworks/shiftleft-tools 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +351 -0
- package/bin/shiftleft.js +95 -0
- package/package.json +57 -0
- package/src/commands/doctor.js +208 -0
- package/src/commands/init-postman.js +298 -0
- package/src/commands/init-rules.js +78 -0
- package/src/commands/link.js +172 -0
- package/src/commands/protect.js +61 -0
- package/src/commands/run-tests.js +182 -0
- package/src/commands/setup-pipeline.js +209 -0
- package/src/commands/update.js +203 -0
- package/src/index.js +4 -0
- package/src/utils/copy-tree.js +98 -0
- package/src/utils/gitignore.js +26 -0
- package/src/utils/logger.js +9 -0
- package/src/utils/manifest.js +145 -0
- package/src/utils/stack.js +80 -0
- package/src/utils/template.js +135 -0
- package/templates/AGENTS.md +109 -0
- package/templates/CLAUDE.md +3 -0
- package/templates/jenkins/Jenkinsfile-java.groovy +432 -0
- package/templates/jenkins/Jenkinsfile-node.groovy +450 -0
- package/templates/postman/.husky/pre-commit +19 -0
- package/templates/postman/.prettierrc.json +5 -0
- package/templates/postman/README.md.ejs +147 -0
- package/templates/postman/collections/01-core.json.ejs +91 -0
- package/templates/postman/config/local.json.ejs +12 -0
- package/templates/postman/config/staging.json.ejs +26 -0
- package/templates/postman/environments/local.postman_environment.json.ejs +31 -0
- package/templates/postman/environments/staging.postman_environment.json.ejs +31 -0
- package/templates/postman/gitignore +16 -0
- package/templates/postman/npmrc +31 -0
- package/templates/postman/package.json.ejs +66 -0
- package/templates/postman/run-all-shim.sh +16 -0
- package/templates/postman/scripts/auth/generate-jwt.sh +113 -0
- package/templates/postman/scripts/auth/get-issuer-secret.sh +140 -0
- package/templates/postman/scripts/infra/start-mocks.sh +138 -0
- package/templates/postman/scripts/infra/stop-mocks.sh +43 -0
- package/templates/postman/scripts/lib/api_coverage.py +1122 -0
- package/templates/postman/scripts/lib/cleanup-reports.sh +101 -0
- package/templates/postman/scripts/lib/cleanup-stryker.sh +44 -0
- package/templates/postman/scripts/lib/report_combined.py +527 -0
- package/templates/postman/scripts/lib/report_consolidated.py +363 -0
- package/templates/postman/scripts/lib/report_generator.py +121 -0
- package/templates/postman/scripts/lib/report_migration.py +156 -0
- package/templates/postman/scripts/lib/report_mutation.py +110 -0
- package/templates/postman/scripts/lib/report_unit.py +353 -0
- package/templates/postman/scripts/lib/report_utils.py +973 -0
- package/templates/postman/scripts/report-generators/generate-consolidated-report.sh +445 -0
- package/templates/postman/scripts/report-generators/java-api-coverage-matrix.sh +257 -0
- package/templates/postman/scripts/report-generators/mutation-report.sh +672 -0
- package/templates/postman/scripts/report-generators/node-api-coverage-matrix.sh +167 -0
- package/templates/postman/scripts/report-generators/stage-report-artifacts.sh +27 -0
- package/templates/postman/scripts/run-all.sh +452 -0
- package/templates/postman/scripts/runners/run-mutation-tests.sh +113 -0
- package/templates/postman/scripts/runners/run-tests-local.sh +936 -0
- package/templates/postman/scripts/runners/run-tests-staging.sh +741 -0
- package/templates/postman-node/README.md.ejs +26 -0
- package/templates/postman-node/collections/crud/01-bootstrap.json.ejs +34 -0
- package/templates/postman-node/config/local.json.ejs +46 -0
- package/templates/postman-node/config/staging.json.ejs +31 -0
- package/templates/postman-node/local.test.env.ejs +3 -0
- package/templates/postman-node/mocks/external.js +14 -0
- package/templates/postman-node/package.json.ejs +39 -0
- package/templates/postman-node/requirements.txt +1 -0
- package/templates/postman-node/scripts/database/cleanup-mysql.sh +12 -0
- package/templates/postman-node/scripts/database/run-migrations.js +29 -0
- package/templates/postman-node/scripts/database/start-mysql.sh +34 -0
- package/templates/postman-node/scripts/database/wait-for-mysql.sh +36 -0
- package/templates/postman-node/scripts/lib/api_coverage_node.py +1137 -0
- package/templates/postman-node/scripts/lib/fetch-jwt.sh +86 -0
- package/templates/postman-node/scripts/lib/run-newman.sh +104 -0
- package/templates/postman-node/scripts/lib/setup-database.sh +55 -0
- package/templates/postman-node/scripts/lib/start-app.sh +48 -0
- package/templates/postman-node/scripts/lib/utils.sh +114 -0
- package/templates/postman-node/scripts/report-generators/stage-report-artifacts.sh +26 -0
- package/templates/postman-node/scripts/run-all.sh +303 -0
- package/templates/postman-node/scripts/runners/run-tests.sh +123 -0
- package/templates/postman-node/scripts/setup-mocks.js.ejs +29 -0
- package/templates/postman-node/stryker.config.js.ejs +51 -0
- package/templates/rules/local-test-setup.mdc +420 -0
- package/templates/rules/testing-node.mdc +66 -0
- package/templates/rules/testing.mdc +248 -0
- package/templates/skills/_shared/postman-standards.md +380 -0
- package/templates/skills/enhance-test-pipeline/SKILL-java.md +483 -0
- package/templates/skills/enhance-test-pipeline/SKILL-node.md +431 -0
- package/templates/skills/enhance-test-pipeline/SKILL.md +9 -0
- package/templates/skills/review-test-suite/SKILL-java.md +137 -0
- package/templates/skills/review-test-suite/SKILL-node.md +78 -0
- package/templates/skills/review-test-suite/SKILL.md +9 -0
- package/templates/skills/run-test-suite/SKILL-java.md +186 -0
- package/templates/skills/run-test-suite/SKILL-node.md +191 -0
- package/templates/skills/run-test-suite/SKILL.md +9 -0
- package/templates/skills/setup-api-tests/SKILL-java.md +1094 -0
- package/templates/skills/setup-api-tests/SKILL-node.md +141 -0
- package/templates/skills/setup-api-tests/SKILL.md +9 -0
- package/templates/skills/setup-mutation-tests/SKILL-java.md +303 -0
- package/templates/skills/setup-mutation-tests/SKILL-node.md +408 -0
- package/templates/skills/setup-mutation-tests/SKILL.md +9 -0
- package/templates/skills/setup-test-pipeline/SKILL-java.md +454 -0
- package/templates/skills/setup-test-pipeline/SKILL-node.md +318 -0
- package/templates/skills/setup-test-pipeline/SKILL.md +9 -0
- package/templates/skills/write-api-tests/SKILL-java.md +115 -0
- package/templates/skills/write-api-tests/SKILL-node.md +83 -0
- package/templates/skills/write-api-tests/SKILL.md +9 -0
- package/templates/stryker.config.js +50 -0
|
@@ -0,0 +1,1094 @@
|
|
|
1
|
+
# Setup API Tests Skill — Java Spring Boot
|
|
2
|
+
|
|
3
|
+
Set up Postman/Newman integration test infrastructure for a Java Spring Boot service.
|
|
4
|
+
|
|
5
|
+
## When to Use
|
|
6
|
+
|
|
7
|
+
Invoke this skill when the user says:
|
|
8
|
+
- "setup postman", "add postman to this project"
|
|
9
|
+
- "setup integration tests", "add newman"
|
|
10
|
+
- "create postman folder", "initialize postman"
|
|
11
|
+
- "add API testing infrastructure"
|
|
12
|
+
- "setup test suite", "setup tests"
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
# CRITICAL: Explore-First Methodology
|
|
17
|
+
|
|
18
|
+
**NEVER assume conventions. ALWAYS verify by reading actual source files.**
|
|
19
|
+
|
|
20
|
+
## The Pattern (MANDATORY for ALL work)
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
1. EXPLORE → Read the actual source files that define the contract
|
|
24
|
+
2. TEST → Run/call something once to verify your understanding
|
|
25
|
+
3. DOCUMENT → State what you found before proceeding
|
|
26
|
+
4. IMPLEMENT → Only now write the code
|
|
27
|
+
5. VERIFY → Run it and confirm it works
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Why This Matters
|
|
31
|
+
|
|
32
|
+
Every project has unique conventions:
|
|
33
|
+
- JSON field naming (snake_case vs camelCase)
|
|
34
|
+
- Enum values (VISIBLE vs public, ACTIVE vs active)
|
|
35
|
+
- Request/response wrapper structures
|
|
36
|
+
- Pagination field names (page vs current_page)
|
|
37
|
+
- Shell compatibility (bash 3.2 vs 4+)
|
|
38
|
+
- Classpath requirements (test-scoped dependencies)
|
|
39
|
+
|
|
40
|
+
**You cannot guess these. You must read the actual code.**
|
|
41
|
+
|
|
42
|
+
## Before Writing ANY Code
|
|
43
|
+
|
|
44
|
+
| Task | MUST DO First |
|
|
45
|
+
|------|---------------|
|
|
46
|
+
| Writing Postman assertions | Read the Response DTO class, call endpoint with curl |
|
|
47
|
+
| Writing request bodies | Read the Request DTO class, check validation annotations |
|
|
48
|
+
| Using enum values | Read the actual Enum class definition |
|
|
49
|
+
| Modifying shell scripts | Run `bash --version`, check for reserved variable names |
|
|
50
|
+
| Modifying properties | Read existing properties, check parent config inheritance |
|
|
51
|
+
| Writing field names | Check Jackson naming strategy AND actual DTO fields |
|
|
52
|
+
|
|
53
|
+
## After Initial Setup
|
|
54
|
+
|
|
55
|
+
**Run ONE test first and observe actual output:**
|
|
56
|
+
|
|
57
|
+
1. Start the app with postman profile
|
|
58
|
+
2. Call one real endpoint with `curl` and see actual JSON response
|
|
59
|
+
3. Compare actual field names to your assertions
|
|
60
|
+
4. Fix any mismatches before writing more tests
|
|
61
|
+
|
|
62
|
+
**Do NOT write all tests assuming they'll work. Verify incrementally.**
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
# Phase 1: Project Analysis (REQUIRED)
|
|
67
|
+
|
|
68
|
+
**Before generating ANY files, you MUST analyze the project thoroughly.**
|
|
69
|
+
|
|
70
|
+
## 1.1 Check for Existing Integration Tests
|
|
71
|
+
|
|
72
|
+
Search for existing test infrastructure:
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
Search for: *IntegrationTest.java, *IT.java, @SpringBootTest
|
|
76
|
+
Check for: application-test.properties, application-integration.properties
|
|
77
|
+
Check if: postman/ folder already exists
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Decision Point: ASK THE USER (MANDATORY)
|
|
81
|
+
|
|
82
|
+
**If existing Java integration tests are found:**
|
|
83
|
+
|
|
84
|
+
> **I found existing integration tests in your project:**
|
|
85
|
+
> - Found X `*IntegrationTest.java` files
|
|
86
|
+
> - Test configuration: [list files found]
|
|
87
|
+
> - Database: [H2 / Testcontainers / etc.]
|
|
88
|
+
>
|
|
89
|
+
> **Options:**
|
|
90
|
+
> 1. **Keep current setup** - I can help improve your existing tests
|
|
91
|
+
> 2. **Add Postman tests alongside** - Adds API coverage reporting and contract testing
|
|
92
|
+
>
|
|
93
|
+
> **Which would you prefer?**
|
|
94
|
+
|
|
95
|
+
**STOP AND WAIT for user response. Do NOT proceed without explicit choice.**
|
|
96
|
+
|
|
97
|
+
If user chooses Option 1: Help improve existing tests, do not proceed with Postman setup.
|
|
98
|
+
If user chooses Option 2: Continue with Phase 1.2.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## 1.2 Check JSON Serialization Strategy
|
|
103
|
+
|
|
104
|
+
**CRITICAL: This determines field names in ALL test assertions.**
|
|
105
|
+
|
|
106
|
+
Search for Jackson configuration:
|
|
107
|
+
|
|
108
|
+
1. In `application.properties` or `application.yml`:
|
|
109
|
+
```
|
|
110
|
+
spring.jackson.property-naming-strategy=SNAKE_CASE
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
2. In `@Configuration` classes, look for custom `ObjectMapper` beans
|
|
114
|
+
|
|
115
|
+
3. **If SNAKE_CASE is configured:**
|
|
116
|
+
- All API responses use `snake_case` field names
|
|
117
|
+
- Use: `external_id`, `created_at`, `subscription_type`, `total_items`
|
|
118
|
+
|
|
119
|
+
4. **If not configured (default LOWER_CAMEL_CASE):**
|
|
120
|
+
- All API responses use `camelCase` field names
|
|
121
|
+
- Use: `externalId`, `createdAt`, `subscriptionType`, `totalItems`
|
|
122
|
+
|
|
123
|
+
**Record finding:**
|
|
124
|
+
```
|
|
125
|
+
JSON_NAMING = SNAKE_CASE | CAMEL_CASE
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## 1.3 Check Database Setup
|
|
131
|
+
|
|
132
|
+
### In pom.xml:
|
|
133
|
+
|
|
134
|
+
Check H2 dependency scope:
|
|
135
|
+
```xml
|
|
136
|
+
<dependency>
|
|
137
|
+
<groupId>com.h2database</groupId>
|
|
138
|
+
<artifactId>h2</artifactId>
|
|
139
|
+
<scope>test</scope> <!-- or runtime -->
|
|
140
|
+
</dependency>
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
**Record finding:**
|
|
144
|
+
```
|
|
145
|
+
H2_SCOPE = test | runtime | compile | not_present
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
**IMPORTANT:** If `H2_SCOPE = test`, the app cannot be started with `java -jar`.
|
|
149
|
+
Must use `mvn spring-boot:run` OR `java -cp` with test classpath.
|
|
150
|
+
|
|
151
|
+
### In application.properties:
|
|
152
|
+
|
|
153
|
+
Check Flyway configuration:
|
|
154
|
+
```
|
|
155
|
+
spring.flyway.user=???
|
|
156
|
+
spring.flyway.password=???
|
|
157
|
+
spring.datasource.username=???
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
**Record finding:**
|
|
161
|
+
```
|
|
162
|
+
FLYWAY_USER = sa | root | <other>
|
|
163
|
+
DATASOURCE_USER = sa | root | <other>
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
**IMPORTANT:** If `spring.flyway.user` is set to anything other than `sa`,
|
|
167
|
+
you MUST override it in `application-postman.properties` to use `sa` for H2.
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## 1.4 Check Server Configuration
|
|
172
|
+
|
|
173
|
+
In `application.properties`:
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
server.port=???
|
|
177
|
+
server.servlet.context-path=???
|
|
178
|
+
management.server.port=???
|
|
179
|
+
management.endpoints.web.exposure.include=???
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
**Record findings:**
|
|
183
|
+
```
|
|
184
|
+
APP_PORT = 8080 | 9090 | <other>
|
|
185
|
+
CONTEXT_PATH = /api | /marketplace/api | <other>
|
|
186
|
+
MANAGEMENT_PORT = same as APP_PORT | different (e.g., 9091)
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**IMPORTANT:** If `MANAGEMENT_PORT` is different from `APP_PORT`:
|
|
190
|
+
- Health checks on management port may fail due to Spring Security
|
|
191
|
+
- Use main app port for health checks instead
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## 1.5 Check Existing Test Profiles
|
|
196
|
+
|
|
197
|
+
Look for existing test property files:
|
|
198
|
+
- `src/test/resources/application-*.properties`
|
|
199
|
+
- `src/main/resources/application-test.properties`
|
|
200
|
+
- `src/main/resources/application-integration.properties`
|
|
201
|
+
|
|
202
|
+
**If found:** REUSE their configuration patterns for consistency.
|
|
203
|
+
|
|
204
|
+
**Record finding:**
|
|
205
|
+
```
|
|
206
|
+
EXISTING_TEST_PROFILE = test | integration | none
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
---
|
|
210
|
+
|
|
211
|
+
## 1.6 Check Response Structure
|
|
212
|
+
|
|
213
|
+
**Before writing assertions, understand the actual API response format:**
|
|
214
|
+
|
|
215
|
+
1. Read DTO classes (e.g., `*Response.java`, `*DTO.java`)
|
|
216
|
+
2. Check pagination wrapper classes for field names:
|
|
217
|
+
- `totalItems` vs `total` vs `count`
|
|
218
|
+
- `data` vs `items` vs `listings` vs `results`
|
|
219
|
+
3. Note any nested structures
|
|
220
|
+
|
|
221
|
+
---
|
|
222
|
+
|
|
223
|
+
## 1.7 Record All Findings
|
|
224
|
+
|
|
225
|
+
Document before proceeding:
|
|
226
|
+
|
|
227
|
+
```
|
|
228
|
+
JSON_NAMING = SNAKE_CASE | CAMEL_CASE
|
|
229
|
+
H2_SCOPE = test | runtime
|
|
230
|
+
FLYWAY_USER = sa | root | <other>
|
|
231
|
+
APP_PORT = <port>
|
|
232
|
+
CONTEXT_PATH = <path>
|
|
233
|
+
MANAGEMENT_PORT = <port> (same | different)
|
|
234
|
+
EXISTING_TEST_PROFILE = <name> | none
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
# Phase 2: Generate Infrastructure
|
|
240
|
+
|
|
241
|
+
## 2.1 Run shiftleft CLI
|
|
242
|
+
|
|
243
|
+
```bash
|
|
244
|
+
shiftleft init-postman
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
This creates:
|
|
248
|
+
- `postman/` folder with all scripts
|
|
249
|
+
- `runners/run-tests-local.sh`, `run-all.sh`, `report-generators/mutation-report.sh`
|
|
250
|
+
- `report-generators/java-api-coverage-matrix.sh`, `lib/api_coverage.py`
|
|
251
|
+
- `start-mocks.sh`, `stop-mocks.sh`
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
## 2.2 Create application-postman.properties
|
|
256
|
+
|
|
257
|
+
**IMPORTANT:** Copy from `EXISTING_TEST_PROFILE` if available, then modify.
|
|
258
|
+
|
|
259
|
+
Create `src/main/resources/application-postman.properties`:
|
|
260
|
+
|
|
261
|
+
```properties
|
|
262
|
+
# =============================================================================
|
|
263
|
+
# POSTMAN API TESTS CONFIGURATION
|
|
264
|
+
# =============================================================================
|
|
265
|
+
|
|
266
|
+
spring.application.name=SERVICE_NAME
|
|
267
|
+
|
|
268
|
+
# Server - use values from Phase 1.4
|
|
269
|
+
server.port=APP_PORT
|
|
270
|
+
server.servlet.context-path=CONTEXT_PATH
|
|
271
|
+
|
|
272
|
+
# =============================================================================
|
|
273
|
+
# H2 Database Configuration
|
|
274
|
+
# =============================================================================
|
|
275
|
+
spring.datasource.url=jdbc:h2:mem:testdb;MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE
|
|
276
|
+
spring.datasource.driver-class-name=org.h2.Driver
|
|
277
|
+
spring.datasource.username=sa
|
|
278
|
+
spring.datasource.password=
|
|
279
|
+
|
|
280
|
+
spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
|
|
281
|
+
spring.jpa.hibernate.ddl-auto=none
|
|
282
|
+
|
|
283
|
+
# H2 JSON compatibility
|
|
284
|
+
spring.jpa.properties.hibernate.type.preferred_jdbc_type_for_json=VARCHAR
|
|
285
|
+
|
|
286
|
+
# =============================================================================
|
|
287
|
+
# Flyway Configuration - ALWAYS OVERRIDE CREDENTIALS FOR H2
|
|
288
|
+
# =============================================================================
|
|
289
|
+
spring.flyway.enabled=true
|
|
290
|
+
spring.flyway.locations=classpath:db/migration,classpath:db/test
|
|
291
|
+
spring.flyway.baseline-on-migrate=true
|
|
292
|
+
spring.flyway.url=${spring.datasource.url}
|
|
293
|
+
spring.flyway.user=sa
|
|
294
|
+
spring.flyway.password=
|
|
295
|
+
|
|
296
|
+
# =============================================================================
|
|
297
|
+
# Disable Features Not Needed for Local Tests
|
|
298
|
+
# =============================================================================
|
|
299
|
+
spring.cache.type=none
|
|
300
|
+
|
|
301
|
+
# =============================================================================
|
|
302
|
+
# External Service Configuration
|
|
303
|
+
# =============================================================================
|
|
304
|
+
# Point Feign clients to WireMock (update based on YOUR project's clients)
|
|
305
|
+
# Example: freshid.api.base-url=http://localhost:8089
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
**CRITICAL:** The `spring.flyway.user=sa` and `spring.flyway.password=` lines
|
|
309
|
+
MUST be included to prevent inheriting MySQL credentials from main config.
|
|
310
|
+
|
|
311
|
+
---
|
|
312
|
+
|
|
313
|
+
## 2.3 Customize run-tests-local.sh Based on H2_SCOPE
|
|
314
|
+
|
|
315
|
+
Edit `postman/scripts/runners/run-tests-local.sh`:
|
|
316
|
+
|
|
317
|
+
### If H2_SCOPE = test (most common):
|
|
318
|
+
|
|
319
|
+
The app cannot be started with `java -jar`. Use one of:
|
|
320
|
+
|
|
321
|
+
**Option A: Use mvn spring-boot:run**
|
|
322
|
+
```bash
|
|
323
|
+
mvn spring-boot:run -Dspring-boot.run.profiles=postman &
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
**Option B: Use java -cp with test classpath**
|
|
327
|
+
```bash
|
|
328
|
+
# Build classpath including test dependencies
|
|
329
|
+
TEST_CLASSPATH=$(mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=/dev/stdout -q)
|
|
330
|
+
CLASSES_DIR="$PROJECT_ROOT/target/classes:$PROJECT_ROOT/target/test-classes"
|
|
331
|
+
|
|
332
|
+
java -cp "$CLASSES_DIR:$TEST_CLASSPATH" \
|
|
333
|
+
com.example.YourApplication \
|
|
334
|
+
--spring.profiles.active=postman \
|
|
335
|
+
--server.port=$APP_PORT &
|
|
336
|
+
```
|
|
337
|
+
|
|
338
|
+
### If H2_SCOPE = runtime:
|
|
339
|
+
|
|
340
|
+
Standard approach works:
|
|
341
|
+
```bash
|
|
342
|
+
mvn clean package -DskipTests -q
|
|
343
|
+
java -jar target/*.jar --spring.profiles.active=postman &
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
---
|
|
347
|
+
|
|
348
|
+
## 2.4 Fix Health Check Logic
|
|
349
|
+
|
|
350
|
+
Edit the health check section in `run-tests-local.sh`:
|
|
351
|
+
|
|
352
|
+
### If MANAGEMENT_PORT = APP_PORT:
|
|
353
|
+
|
|
354
|
+
```bash
|
|
355
|
+
HEALTH_URL="http://localhost:$APP_PORT/actuator/health"
|
|
356
|
+
until curl -s "$HEALTH_URL" | grep -q "UP"; do
|
|
357
|
+
sleep 2
|
|
358
|
+
done
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
### If MANAGEMENT_PORT != APP_PORT (may have security issues):
|
|
362
|
+
|
|
363
|
+
```bash
|
|
364
|
+
# Use main app port - management port may have Spring Security issues
|
|
365
|
+
HEALTH_URL="http://localhost:$APP_PORT$CONTEXT_PATH"
|
|
366
|
+
|
|
367
|
+
MAX_WAIT=120
|
|
368
|
+
COUNTER=0
|
|
369
|
+
while [ $COUNTER -lt $MAX_WAIT ]; do
|
|
370
|
+
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTH_URL" 2>/dev/null || echo "000")
|
|
371
|
+
|
|
372
|
+
case "$HTTP_CODE" in
|
|
373
|
+
2[0-9][0-9]|3[0-9][0-9]|4[0-9][0-9])
|
|
374
|
+
echo "Application ready (HTTP $HTTP_CODE)"
|
|
375
|
+
break
|
|
376
|
+
;;
|
|
377
|
+
esac
|
|
378
|
+
|
|
379
|
+
sleep 2
|
|
380
|
+
COUNTER=$((COUNTER + 2))
|
|
381
|
+
done
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
---
|
|
385
|
+
|
|
386
|
+
# Phase 3: Create Postman Collections
|
|
387
|
+
|
|
388
|
+
## 3.1 MANDATORY: Read Before Writing
|
|
389
|
+
|
|
390
|
+
**Before writing ANY assertion, you MUST read the actual source files.**
|
|
391
|
+
|
|
392
|
+
### For Response Assertions:
|
|
393
|
+
|
|
394
|
+
1. **Find and read the Response DTO class:**
|
|
395
|
+
```
|
|
396
|
+
Search for: *Response.java, *DTO.java, *Details.java
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
2. **Check the actual field names in the DTO:**
|
|
400
|
+
- Look for `@JsonProperty` annotations (override field names)
|
|
401
|
+
- Check if fields are wrapped in nested objects
|
|
402
|
+
- Note pagination fields exactly as defined
|
|
403
|
+
|
|
404
|
+
3. **Check enum values:**
|
|
405
|
+
```
|
|
406
|
+
Search for: enum *Status, enum *Type, enum *Visibility
|
|
407
|
+
Read the actual enum values - don't assume "public/private" or "active/inactive"
|
|
408
|
+
```
|
|
409
|
+
|
|
410
|
+
4. **Verify with curl BEFORE writing tests:**
|
|
411
|
+
```bash
|
|
412
|
+
curl -s http://localhost:PORT/CONTEXT_PATH/v1/endpoint | jq .
|
|
413
|
+
```
|
|
414
|
+
Use the ACTUAL field names from this response.
|
|
415
|
+
|
|
416
|
+
### For Request Bodies:
|
|
417
|
+
|
|
418
|
+
1. **Find and read the Request DTO class:**
|
|
419
|
+
```
|
|
420
|
+
Search for: *Request.java, *CreateRequest.java, *UpdateRequest.java
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
2. **Check validation annotations:**
|
|
424
|
+
- `@NotNull` - field is required
|
|
425
|
+
- `@Size(min=X, max=Y)` - length constraints
|
|
426
|
+
- `@Pattern` - format requirements
|
|
427
|
+
|
|
428
|
+
3. **Use exact field names from the Request DTO, not guesses.**
|
|
429
|
+
|
|
430
|
+
### Common Mistakes to Avoid:
|
|
431
|
+
|
|
432
|
+
| Mistake | Reality |
|
|
433
|
+
|---------|---------|
|
|
434
|
+
| Assuming `name` field | Might be `displayName` or `display_name` |
|
|
435
|
+
| Assuming `public/private` | Might be `VISIBLE/HIDDEN` enum values |
|
|
436
|
+
| Assuming `page/size` | Might be `current_page/page_size` |
|
|
437
|
+
| Assuming direct array response | Might be wrapped: `{ "items": [...] }` |
|
|
438
|
+
| Assuming `totalItems` | Might be `total_items`, `total`, `count`, or `size` |
|
|
439
|
+
|
|
440
|
+
## 3.2 Use Correct JSON Field Names
|
|
441
|
+
|
|
442
|
+
**Based on JSON_NAMING from Phase 1.2 AND actual DTO inspection:**
|
|
443
|
+
|
|
444
|
+
### If JSON_NAMING = SNAKE_CASE:
|
|
445
|
+
|
|
446
|
+
```javascript
|
|
447
|
+
pm.test("Response has correct fields", function () {
|
|
448
|
+
const json = pm.response.json();
|
|
449
|
+
pm.expect(json.external_id).to.exist;
|
|
450
|
+
pm.expect(json.created_at).to.exist;
|
|
451
|
+
pm.expect(json.subscription_type).to.eql("FREE");
|
|
452
|
+
pm.expect(json.total_items).to.be.a("number");
|
|
453
|
+
pm.expect(json.display_name).to.eql("Test App");
|
|
454
|
+
});
|
|
455
|
+
```
|
|
456
|
+
|
|
457
|
+
### If JSON_NAMING = CAMEL_CASE:
|
|
458
|
+
|
|
459
|
+
```javascript
|
|
460
|
+
pm.test("Response has correct fields", function () {
|
|
461
|
+
const json = pm.response.json();
|
|
462
|
+
pm.expect(json.externalId).to.exist;
|
|
463
|
+
pm.expect(json.createdAt).to.exist;
|
|
464
|
+
pm.expect(json.subscriptionType).to.eql("FREE");
|
|
465
|
+
pm.expect(json.totalItems).to.be.a("number");
|
|
466
|
+
pm.expect(json.displayName).to.eql("Test App");
|
|
467
|
+
});
|
|
468
|
+
```
|
|
469
|
+
|
|
470
|
+
## 3.3 Use Variables for All Ports
|
|
471
|
+
|
|
472
|
+
In collection JSON, never hardcode ports:
|
|
473
|
+
|
|
474
|
+
```json
|
|
475
|
+
{
|
|
476
|
+
"url": {
|
|
477
|
+
"raw": "{{base_url}}/v1/listings",
|
|
478
|
+
"host": ["{{base_url}}"],
|
|
479
|
+
"path": ["v1", "listings"]
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
```
|
|
483
|
+
|
|
484
|
+
Environment should have:
|
|
485
|
+
```json
|
|
486
|
+
{
|
|
487
|
+
"key": "base_url",
|
|
488
|
+
"value": "http://localhost:APP_PORT/CONTEXT_PATH"
|
|
489
|
+
}
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
---
|
|
493
|
+
|
|
494
|
+
# Phase 4: H2 Migration Compatibility
|
|
495
|
+
|
|
496
|
+
## 4.1 Check Existing Migrations
|
|
497
|
+
|
|
498
|
+
Read migrations in `src/main/resources/db/migration/` and identify incompatible syntax:
|
|
499
|
+
|
|
500
|
+
| MySQL Syntax | H2 Alternative |
|
|
501
|
+
|-------------|----------------|
|
|
502
|
+
| `DATETIME` | `TIMESTAMP` |
|
|
503
|
+
| `TINYINT(1)` | `BOOLEAN` |
|
|
504
|
+
| `ENGINE=InnoDB` | Remove |
|
|
505
|
+
| `CHARSET=utf8mb4` | Remove |
|
|
506
|
+
| `ON UPDATE CURRENT_TIMESTAMP` | Remove |
|
|
507
|
+
| `JSON` | `CLOB` or `VARCHAR(4000)` |
|
|
508
|
+
| `ENUM('A', 'B')` | `VARCHAR(50)` |
|
|
509
|
+
|
|
510
|
+
## 4.2 Create H2-Specific Migrations (if needed)
|
|
511
|
+
|
|
512
|
+
If incompatible syntax found, create override migrations in `src/main/resources/db/test/`:
|
|
513
|
+
|
|
514
|
+
The Flyway config in `application-postman.properties` includes both locations:
|
|
515
|
+
```properties
|
|
516
|
+
spring.flyway.locations=classpath:db/migration,classpath:db/test
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
---
|
|
520
|
+
|
|
521
|
+
# Phase 5: Add PIT Mutation Testing Plugin
|
|
522
|
+
|
|
523
|
+
**Before running mutation tests, the PIT plugin must be configured in `pom.xml`.**
|
|
524
|
+
|
|
525
|
+
## 5.1 Check if PIT Plugin Exists
|
|
526
|
+
|
|
527
|
+
Search `pom.xml` for `pitest-maven`. If not found, add it.
|
|
528
|
+
|
|
529
|
+
## 5.2 Add PIT Plugin to pom.xml
|
|
530
|
+
|
|
531
|
+
Add inside `<build><plugins>`:
|
|
532
|
+
|
|
533
|
+
```xml
|
|
534
|
+
<!-- PIT Mutation Testing Plugin -->
|
|
535
|
+
<plugin>
|
|
536
|
+
<groupId>org.pitest</groupId>
|
|
537
|
+
<artifactId>pitest-maven</artifactId>
|
|
538
|
+
<version>1.15.3</version>
|
|
539
|
+
<dependencies>
|
|
540
|
+
<dependency>
|
|
541
|
+
<groupId>org.pitest</groupId>
|
|
542
|
+
<artifactId>pitest-junit5-plugin</artifactId>
|
|
543
|
+
<version>1.2.1</version>
|
|
544
|
+
</dependency>
|
|
545
|
+
</dependencies>
|
|
546
|
+
<configuration>
|
|
547
|
+
<!-- Target classes to mutate - UPDATE THESE TO MATCH YOUR PROJECT -->
|
|
548
|
+
<targetClasses>
|
|
549
|
+
<param>com.yourcompany.yourproject.service.*</param>
|
|
550
|
+
<param>com.yourcompany.yourproject.mapper.*</param>
|
|
551
|
+
<param>com.yourcompany.yourproject.converter.*</param>
|
|
552
|
+
<param>com.yourcompany.yourproject.filter.*</param>
|
|
553
|
+
<param>com.yourcompany.yourproject.utils.*</param>
|
|
554
|
+
<param>com.yourcompany.yourproject.validator.*</param>
|
|
555
|
+
</targetClasses>
|
|
556
|
+
<!-- Target test classes - UPDATE THESE TO MATCH YOUR PROJECT -->
|
|
557
|
+
<targetTests>
|
|
558
|
+
<param>com.yourcompany.yourproject.service.*Test</param>
|
|
559
|
+
<param>com.yourcompany.yourproject.mapper.*Test</param>
|
|
560
|
+
<param>com.yourcompany.yourproject.utils.*Test</param>
|
|
561
|
+
</targetTests>
|
|
562
|
+
<!-- Exclude controller tests (too slow, use Postman for API tests) -->
|
|
563
|
+
<excludedTestClasses>
|
|
564
|
+
<param>com.yourcompany.yourproject.controller.*</param>
|
|
565
|
+
</excludedTestClasses>
|
|
566
|
+
<!-- Thresholds -->
|
|
567
|
+
<mutationThreshold>60</mutationThreshold>
|
|
568
|
+
<coverageThreshold>70</coverageThreshold>
|
|
569
|
+
<!-- Use STRONGER mutators for better test quality -->
|
|
570
|
+
<mutators>
|
|
571
|
+
<mutator>STRONGER</mutator>
|
|
572
|
+
</mutators>
|
|
573
|
+
<!-- Exclude classes that don't need mutation testing -->
|
|
574
|
+
<excludedClasses>
|
|
575
|
+
<param>*DTO</param>
|
|
576
|
+
<param>*Entity</param>
|
|
577
|
+
<param>*Config</param>
|
|
578
|
+
<param>*Exception</param>
|
|
579
|
+
<param>*Application</param>
|
|
580
|
+
<param>*MapperImpl</param>
|
|
581
|
+
<param>*Constants</param>
|
|
582
|
+
</excludedClasses>
|
|
583
|
+
<!-- Exclude generated methods -->
|
|
584
|
+
<excludedMethods>
|
|
585
|
+
<param>hashCode</param>
|
|
586
|
+
<param>equals</param>
|
|
587
|
+
<param>toString</param>
|
|
588
|
+
</excludedMethods>
|
|
589
|
+
<outputFormats>
|
|
590
|
+
<outputFormat>HTML</outputFormat>
|
|
591
|
+
<outputFormat>XML</outputFormat>
|
|
592
|
+
</outputFormats>
|
|
593
|
+
<timestampedReports>false</timestampedReports>
|
|
594
|
+
<threads>4</threads>
|
|
595
|
+
<timeoutConstant>8000</timeoutConstant>
|
|
596
|
+
</configuration>
|
|
597
|
+
</plugin>
|
|
598
|
+
```
|
|
599
|
+
|
|
600
|
+
## 5.3 Customize PIT Configuration
|
|
601
|
+
|
|
602
|
+
**IMPORTANT:** Update `targetClasses` and `targetTests` based on YOUR project's package structure.
|
|
603
|
+
|
|
604
|
+
1. Read your project's package structure from `src/main/java/`
|
|
605
|
+
2. Replace `com.yourcompany.yourproject` with actual package path
|
|
606
|
+
3. Include only business logic packages (service, mapper, utils, validator)
|
|
607
|
+
4. Exclude controllers, DTOs, entities, configs
|
|
608
|
+
|
|
609
|
+
---
|
|
610
|
+
|
|
611
|
+
# Phase 6: Run Tests and Generate Reports (MANDATORY)
|
|
612
|
+
|
|
613
|
+
**After setup is complete, you MUST run tests and generate reports.**
|
|
614
|
+
|
|
615
|
+
## 6.1 Run the Complete Test Suite
|
|
616
|
+
|
|
617
|
+
Execute these commands IN ORDER:
|
|
618
|
+
|
|
619
|
+
```bash
|
|
620
|
+
# 1. Install dependencies
|
|
621
|
+
cd postman && npm ci --ignore-scripts
|
|
622
|
+
|
|
623
|
+
# 2. Run local tests (this starts app, runs Newman tests, stops app)
|
|
624
|
+
cd scripts && ./runners/run-tests-local.sh
|
|
625
|
+
```
|
|
626
|
+
|
|
627
|
+
**Wait for run-tests-local.sh to complete, then continue:**
|
|
628
|
+
|
|
629
|
+
```bash
|
|
630
|
+
# 3. Generate API coverage report
|
|
631
|
+
./report-generators/java-api-coverage-matrix.sh
|
|
632
|
+
|
|
633
|
+
# 4. Run mutation tests (unit test quality)
|
|
634
|
+
./report-generators/mutation-report.sh
|
|
635
|
+
```
|
|
636
|
+
|
|
637
|
+
## 6.2 Review and Report Results
|
|
638
|
+
|
|
639
|
+
**After tests complete, you MUST:**
|
|
640
|
+
|
|
641
|
+
1. **Check test results** - Report pass/fail count from Newman output
|
|
642
|
+
2. **Check API coverage** - Run `./report-generators/java-api-coverage-matrix.sh` and report coverage percentage
|
|
643
|
+
3. **Identify gaps** - List any endpoints without test coverage
|
|
644
|
+
4. **Report to user** - Summarize results:
|
|
645
|
+
|
|
646
|
+
> **Test Results:**
|
|
647
|
+
> - Postman tests: X passed, Y failed
|
|
648
|
+
> - API coverage: Z% of endpoints have 2xx tests
|
|
649
|
+
> - Endpoints needing tests: [list any gaps]
|
|
650
|
+
>
|
|
651
|
+
> **Next steps:**
|
|
652
|
+
> - [Fix any failing tests]
|
|
653
|
+
> - [Add tests for uncovered endpoints]
|
|
654
|
+
|
|
655
|
+
## 6.3 Verification Checklist
|
|
656
|
+
|
|
657
|
+
Before considering setup COMPLETE, verify:
|
|
658
|
+
|
|
659
|
+
- [ ] `application-postman.properties` exists with H2 config
|
|
660
|
+
- [ ] `spring.flyway.user=sa` is explicitly set
|
|
661
|
+
- [ ] H2 dependency exists in `pom.xml`
|
|
662
|
+
- [ ] App starts successfully with postman profile
|
|
663
|
+
- [ ] Flyway migrations complete without errors
|
|
664
|
+
- [ ] Health check correctly waits for app readiness
|
|
665
|
+
- [ ] `./runners/run-tests-local.sh` completes successfully
|
|
666
|
+
- [ ] `./report-generators/java-api-coverage-matrix.sh` runs and shows coverage
|
|
667
|
+
- [ ] All Postman assertions use correct field naming
|
|
668
|
+
- [ ] All ports use variables, not hardcoded values
|
|
669
|
+
|
|
670
|
+
---
|
|
671
|
+
|
|
672
|
+
# Troubleshooting
|
|
673
|
+
|
|
674
|
+
| Issue | Cause | Fix |
|
|
675
|
+
|-------|-------|-----|
|
|
676
|
+
| "Cannot load driver class: org.h2.Driver" | H2 is test-scoped, not in runtime classpath | Use `mvn spring-boot:run` or `java -cp` with test dependencies |
|
|
677
|
+
| "Wrong user name or password" from Flyway | Flyway inheriting MySQL credentials | Add explicit `spring.flyway.user=sa` in postman profile |
|
|
678
|
+
| "Failed to find servlet [dispatcherServletRegistration]" | Spring Security on separate management port | Use main app port for health checks |
|
|
679
|
+
| Assertions fail: "expected 'externalId', got 'external_id'" | Jackson configured with SNAKE_CASE | Update all assertions to use snake_case field names |
|
|
680
|
+
| "connect ECONNREFUSED" during tests | Health check passed before app ready | Use HTTP code pattern matching (2xx/3xx/4xx) |
|
|
681
|
+
| Health check always passes immediately | curl returns multi-char codes like "000000" | Use case statement with proper patterns |
|
|
682
|
+
| `shiftleft: command not found` | CLI not installed | Run `cd <dev-tools-path> && npm link` |
|
|
683
|
+
|
|
684
|
+
## Script Compatibility Issues
|
|
685
|
+
|
|
686
|
+
| Issue | Cause | Fix |
|
|
687
|
+
|-------|-------|-----|
|
|
688
|
+
| `declare -A: command not found` | macOS default bash is 3.2, needs 4+ | Use `/opt/homebrew/bin/bash` or `/usr/local/bin/bash` |
|
|
689
|
+
| `grep: command not found` inside script | Script overwrote PATH variable | Never use `PATH` as variable name; use `ENDPOINT_PATH` |
|
|
690
|
+
| `tr: command not found` | Same as above - PATH overwritten | Rename any variable called PATH, HOME, USER, etc. |
|
|
691
|
+
|
|
692
|
+
## Assertion Mismatch Issues
|
|
693
|
+
|
|
694
|
+
| Issue | Cause | Fix |
|
|
695
|
+
|-------|-------|-----|
|
|
696
|
+
| Expected `externalId`, got `external_id` | Jackson uses SNAKE_CASE | Read actual response or check `spring.jackson.property-naming-strategy` |
|
|
697
|
+
| Expected `public`, got `VISIBLE` | Wrong enum value assumed | Read the actual Enum class definition |
|
|
698
|
+
| Expected array, got object | Response is wrapped | Read DTO to see wrapper structure like `{ "items": [...] }` |
|
|
699
|
+
| Expected `page`, got `current_page` | Pagination fields differ | Read PaginationDetails or PageResponse DTO |
|
|
700
|
+
| Expected `name`, got `display_name` | Field name differs | Read actual DTO class fields |
|
|
701
|
+
|
|
702
|
+
## Classpath Issues
|
|
703
|
+
|
|
704
|
+
| Issue | Cause | Fix |
|
|
705
|
+
|-------|-------|-----|
|
|
706
|
+
| `ClassNotFoundException: org.h2.Driver` | H2 is test-scoped | Use `mvn spring-boot:run` or `java -cp` with test classpath |
|
|
707
|
+
| `ClassNotFoundException: *MapperImpl` | MapStruct not on classpath | Include `target/generated-sources/annotations` in classpath |
|
|
708
|
+
| Bean creation failures | Missing test resources | Include `src/test/resources` in classpath |
|
|
709
|
+
| `NoSuchBeanDefinitionException` | Profile not active | Ensure `--spring.profiles.active=postman` is passed |
|
|
710
|
+
|
|
711
|
+
---
|
|
712
|
+
|
|
713
|
+
# Quick Reference
|
|
714
|
+
|
|
715
|
+
## Required Configuration Overrides
|
|
716
|
+
|
|
717
|
+
Always include in `application-postman.properties`:
|
|
718
|
+
|
|
719
|
+
```properties
|
|
720
|
+
# Datasource for H2
|
|
721
|
+
spring.datasource.url=jdbc:h2:mem:testdb;MODE=MySQL;...
|
|
722
|
+
spring.datasource.username=sa
|
|
723
|
+
spring.datasource.password=
|
|
724
|
+
|
|
725
|
+
# CRITICAL: Override Flyway credentials
|
|
726
|
+
spring.flyway.url=${spring.datasource.url}
|
|
727
|
+
spring.flyway.user=sa
|
|
728
|
+
spring.flyway.password=
|
|
729
|
+
|
|
730
|
+
# H2 JSON compatibility
|
|
731
|
+
spring.jpa.properties.hibernate.type.preferred_jdbc_type_for_json=VARCHAR
|
|
732
|
+
```
|
|
733
|
+
|
|
734
|
+
## App Startup Based on H2 Scope
|
|
735
|
+
|
|
736
|
+
| H2 Scope | Startup Command |
|
|
737
|
+
|----------|-----------------|
|
|
738
|
+
| `test` | `mvn spring-boot:run -Dspring-boot.run.profiles=postman` |
|
|
739
|
+
| `runtime` | `java -jar target/*.jar --spring.profiles.active=postman` |
|
|
740
|
+
|
|
741
|
+
## JSON Field Naming
|
|
742
|
+
|
|
743
|
+
| Strategy | Example Fields |
|
|
744
|
+
|----------|----------------|
|
|
745
|
+
| SNAKE_CASE | `external_id`, `created_at`, `total_items` |
|
|
746
|
+
| CAMEL_CASE | `externalId`, `createdAt`, `totalItems` |
|
|
747
|
+
|
|
748
|
+
---
|
|
749
|
+
|
|
750
|
+
# Phase 7: Collection Lifecycle Scaffolding
|
|
751
|
+
|
|
752
|
+
A well-structured test suite uses a **numbered lifecycle** — not one large monolithic collection. The runner must execute them in order so later collections can reference data created by earlier ones.
|
|
753
|
+
|
|
754
|
+
## 7.1 Required Collection Structure
|
|
755
|
+
|
|
756
|
+
```
|
|
757
|
+
postman/collections/
|
|
758
|
+
01-bootstrap.json ← Creates all prerequisite test data, sets env vars
|
|
759
|
+
02-<concern>.json ← e.g. happy path CRUD
|
|
760
|
+
03-<concern>.json ← e.g. validation / negative tests
|
|
761
|
+
04-<concern>.json ← e.g. auth / 401 tests
|
|
762
|
+
...
|
|
763
|
+
99-cleanup.json ← Deletes everything created in bootstrap, always runs last
|
|
764
|
+
```
|
|
765
|
+
|
|
766
|
+
**Rules:**
|
|
767
|
+
- Bootstrap (01) always runs first — creates resources and exports IDs to env vars
|
|
768
|
+
- Cleanup (99) always runs last — even if mid-collections fail
|
|
769
|
+
- Mid-collections are grouped by concern, not by endpoint
|
|
770
|
+
- Each collection is small and focused (under ~20 requests each)
|
|
771
|
+
|
|
772
|
+
## 7.2 Bootstrap Collection Template
|
|
773
|
+
|
|
774
|
+
The bootstrap collection should:
|
|
775
|
+
1. Create the minimum set of entities needed for all subsequent tests
|
|
776
|
+
2. Store each created ID in an environment variable via `pm.environment.set()`
|
|
777
|
+
3. Never assert business logic — just create data and export IDs
|
|
778
|
+
|
|
779
|
+
```javascript
|
|
780
|
+
// Pre-request script (bootstrap): authenticate once
|
|
781
|
+
pm.environment.set("auth_token", pm.variables.get("test_token"));
|
|
782
|
+
|
|
783
|
+
// Test script (after create entity):
|
|
784
|
+
pm.test("Bootstrap: entity created", function () {
|
|
785
|
+
pm.response.to.have.status(201);
|
|
786
|
+
const json = pm.response.json();
|
|
787
|
+
pm.environment.set("entity_id", json.id);
|
|
788
|
+
});
|
|
789
|
+
```
|
|
790
|
+
|
|
791
|
+
## 7.3 Environment Variable Chaining
|
|
792
|
+
|
|
793
|
+
The runner script must export the working environment after each collection so subsequent collections inherit variables set by earlier ones. Check `run-tests-local.sh` (or `run-tests-staging.sh`) for this pattern:
|
|
794
|
+
|
|
795
|
+
```bash
|
|
796
|
+
# Each collection: export env after run, pass it to next
|
|
797
|
+
newman run collections/01-bootstrap.json \
|
|
798
|
+
--environment working-env.json \
|
|
799
|
+
--export-environment working-env.json
|
|
800
|
+
|
|
801
|
+
newman run collections/02-happy-path.json \
|
|
802
|
+
--environment working-env.json \
|
|
803
|
+
--export-environment working-env.json
|
|
804
|
+
|
|
805
|
+
# Cleanup always runs, even if tests failed
|
|
806
|
+
newman run collections/99-cleanup.json \
|
|
807
|
+
--environment working-env.json
|
|
808
|
+
```
|
|
809
|
+
|
|
810
|
+
## 7.4 Cleanup Collection
|
|
811
|
+
|
|
812
|
+
The cleanup collection must:
|
|
813
|
+
- Delete every entity created in bootstrap (use stored IDs from env vars)
|
|
814
|
+
- Run unconditionally — the runner must NOT skip it on test failure
|
|
815
|
+
- Assert 200/204 on each delete (silent failures leave orphaned test data)
|
|
816
|
+
|
|
817
|
+
```javascript
|
|
818
|
+
// Test script (cleanup item):
|
|
819
|
+
pm.test("Cleanup: entity deleted", function () {
|
|
820
|
+
pm.response.to.have.status(204);
|
|
821
|
+
pm.environment.unset("entity_id");
|
|
822
|
+
});
|
|
823
|
+
```
|
|
824
|
+
|
|
825
|
+
---
|
|
826
|
+
|
|
827
|
+
# Phase 8: CI Pipeline Integration
|
|
828
|
+
|
|
829
|
+
**After local tests pass, add the integration test stage to your CI pipeline.**
|
|
830
|
+
|
|
831
|
+
## 8.1 Jenkinsfile Integration Test Stage
|
|
832
|
+
|
|
833
|
+
Add a shift-left integration stage to your `Jenkinsfile`. This stage:
|
|
834
|
+
- Runs only when triggered by a PR comment starting with `shiftleft` (or on merge to main)
|
|
835
|
+
- Sets up an isolated environment (isoforge) before tests
|
|
836
|
+
- Tears down isolation even on failure
|
|
837
|
+
- Posts the result back as a GitHub commit status
|
|
838
|
+
|
|
839
|
+
```groovy
|
|
840
|
+
stage('Shift Left Integration Test') {
|
|
841
|
+
when {
|
|
842
|
+
anyOf {
|
|
843
|
+
// Run when PR comment starts with "shiftleft"
|
|
844
|
+
expression {
|
|
845
|
+
return env.CHANGE_AUTHOR_DISPLAY_NAME != null &&
|
|
846
|
+
env.ghprbCommentBody?.startsWith("shiftleft")
|
|
847
|
+
}
|
|
848
|
+
// Always run on main branch
|
|
849
|
+
branch 'main'
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
steps {
|
|
853
|
+
script {
|
|
854
|
+
try {
|
|
855
|
+
// 1. Set up isolation environment (isoforge or equivalent)
|
|
856
|
+
// Replace with your isolation setup command
|
|
857
|
+
sh """
|
|
858
|
+
isoforge create --service ${SERVICE_NAME} --ref ${GIT_COMMIT}
|
|
859
|
+
"""
|
|
860
|
+
|
|
861
|
+
// 2. Run the integration test suite
|
|
862
|
+
sh """
|
|
863
|
+
cd postman
|
|
864
|
+
npm ci --ignore-scripts
|
|
865
|
+
cd scripts
|
|
866
|
+
./runners/run-tests-staging.sh
|
|
867
|
+
"""
|
|
868
|
+
} finally {
|
|
869
|
+
// 3. Teardown always runs (even on failure)
|
|
870
|
+
sh """
|
|
871
|
+
isoforge destroy --service ${SERVICE_NAME} || true
|
|
872
|
+
"""
|
|
873
|
+
|
|
874
|
+
// 4. Publish per-collection HTML reports
|
|
875
|
+
publishHTML(target: [
|
|
876
|
+
allowMissing: true,
|
|
877
|
+
alwaysLinkToLastBuild: true,
|
|
878
|
+
keepAll: true,
|
|
879
|
+
reportDir: 'postman/reports',
|
|
880
|
+
reportFiles: 'consolidated-*.html',
|
|
881
|
+
reportName: 'Integration Test Report'
|
|
882
|
+
])
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
post {
|
|
887
|
+
always {
|
|
888
|
+
// Post commit status back to the PR
|
|
889
|
+
step([
|
|
890
|
+
$class: 'GitHubCommitStatusSetter',
|
|
891
|
+
contextSource: [$class: 'ManuallyEnteredCommitContextSource', context: 'integration-tests'],
|
|
892
|
+
statusResultSource: [$class: 'ConditionalStatusResultSource', results: [
|
|
893
|
+
[$class: 'AnyBuildResult', message: 'Integration tests', state: 'SUCCESS']
|
|
894
|
+
]]
|
|
895
|
+
])
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
```
|
|
900
|
+
|
|
901
|
+
## 8.2 Combined Quality Report Stage
|
|
902
|
+
|
|
903
|
+
Add a stage that aggregates unit test results, coverage, mutation score, and API test results into one HTML report:
|
|
904
|
+
|
|
905
|
+
```groovy
|
|
906
|
+
stage('Quality Report') {
|
|
907
|
+
steps {
|
|
908
|
+
script {
|
|
909
|
+
sh """
|
|
910
|
+
cd postman/scripts
|
|
911
|
+
./run-all.sh --skip-app-start
|
|
912
|
+
"""
|
|
913
|
+
}
|
|
914
|
+
publishHTML(target: [
|
|
915
|
+
allowMissing: false,
|
|
916
|
+
alwaysLinkToLastBuild: true,
|
|
917
|
+
keepAll: true,
|
|
918
|
+
reportDir: 'postman/reports',
|
|
919
|
+
reportFiles: 'quality-report-*.html',
|
|
920
|
+
reportName: 'Quality Report'
|
|
921
|
+
])
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
```
|
|
925
|
+
|
|
926
|
+
---
|
|
927
|
+
|
|
928
|
+
# Phase 9: GitHub Actions Workflows
|
|
929
|
+
|
|
930
|
+
**Two GitHub Actions workflows should be added. If your repo already has them, verify they cover these behaviours.**
|
|
931
|
+
|
|
932
|
+
## 9.1 Postman Format Validation Workflow
|
|
933
|
+
|
|
934
|
+
Triggers when any collection or environment file changes on a PR. Catches malformed JSON before CI fails.
|
|
935
|
+
|
|
936
|
+
Create `.github/workflows/postman-format-check.yml`:
|
|
937
|
+
|
|
938
|
+
```yaml
|
|
939
|
+
name: Postman Format Check
|
|
940
|
+
|
|
941
|
+
on:
|
|
942
|
+
pull_request:
|
|
943
|
+
paths:
|
|
944
|
+
- 'postman/**'
|
|
945
|
+
|
|
946
|
+
jobs:
|
|
947
|
+
format-check:
|
|
948
|
+
runs-on: ubuntu-latest
|
|
949
|
+
steps:
|
|
950
|
+
- uses: actions/checkout@v4
|
|
951
|
+
|
|
952
|
+
- name: Set up Node.js
|
|
953
|
+
uses: actions/setup-node@v4
|
|
954
|
+
with:
|
|
955
|
+
node-version: '20'
|
|
956
|
+
|
|
957
|
+
- name: Install dependencies
|
|
958
|
+
working-directory: postman
|
|
959
|
+
run: npm ci --ignore-scripts
|
|
960
|
+
|
|
961
|
+
- name: Check collection JSON format
|
|
962
|
+
working-directory: postman
|
|
963
|
+
run: npm run format:check
|
|
964
|
+
|
|
965
|
+
- name: Validate collection schema
|
|
966
|
+
working-directory: postman
|
|
967
|
+
run: |
|
|
968
|
+
# Check that all collection files have required Postman schema fields
|
|
969
|
+
for f in collections/*.json; do
|
|
970
|
+
jq -e '.info.schema' "$f" > /dev/null || \
|
|
971
|
+
(echo "FAIL: $f missing .info.schema" && exit 1)
|
|
972
|
+
done
|
|
973
|
+
echo "All collections have valid schema"
|
|
974
|
+
```
|
|
975
|
+
|
|
976
|
+
Add to `postman/package.json`:
|
|
977
|
+
|
|
978
|
+
```json
|
|
979
|
+
{
|
|
980
|
+
"scripts": {
|
|
981
|
+
"format:check": "prettier --check 'collections/**/*.json' 'environments/**/*.json'",
|
|
982
|
+
"format:fix": "prettier --write 'collections/**/*.json' 'environments/**/*.json'"
|
|
983
|
+
},
|
|
984
|
+
"devDependencies": {
|
|
985
|
+
"prettier": "^3.0.0"
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
```
|
|
989
|
+
|
|
990
|
+
## 9.2 Repository Authorization Workflow
|
|
991
|
+
|
|
992
|
+
Prevents the pipeline from running if the repo is forked to an unauthorised account.
|
|
993
|
+
|
|
994
|
+
Create `.github/workflows/security_verification.yml`:
|
|
995
|
+
|
|
996
|
+
```yaml
|
|
997
|
+
name: Security Verification
|
|
998
|
+
|
|
999
|
+
on:
|
|
1000
|
+
push:
|
|
1001
|
+
pull_request:
|
|
1002
|
+
|
|
1003
|
+
jobs:
|
|
1004
|
+
verify-repo-authorization:
|
|
1005
|
+
runs-on: ubuntu-latest
|
|
1006
|
+
steps:
|
|
1007
|
+
- name: Verify repository owner
|
|
1008
|
+
run: |
|
|
1009
|
+
OWNER="${{ github.repository_owner }}"
|
|
1010
|
+
ALLOWED_ORGS="YOUR_ORG_NAME" # Replace with your organisation(s)
|
|
1011
|
+
|
|
1012
|
+
if echo "$ALLOWED_ORGS" | grep -qw "$OWNER"; then
|
|
1013
|
+
echo "Repository owner '$OWNER' is authorized."
|
|
1014
|
+
else
|
|
1015
|
+
echo "ERROR: Repository owner '$OWNER' is not in the authorized list."
|
|
1016
|
+
echo "This workflow only runs in authorized organizations."
|
|
1017
|
+
exit 1
|
|
1018
|
+
fi
|
|
1019
|
+
```
|
|
1020
|
+
|
|
1021
|
+
**IMPORTANT:** Replace `YOUR_ORG_NAME` with your actual GitHub organisation name(s).
|
|
1022
|
+
|
|
1023
|
+
---
|
|
1024
|
+
|
|
1025
|
+
# Phase 10: Isoforge / Isolation Environment Setup
|
|
1026
|
+
|
|
1027
|
+
**If your service depends on other microservices, tests must run against an isolated stack — not shared staging data.**
|
|
1028
|
+
|
|
1029
|
+
## 10.1 What Isolation Provides
|
|
1030
|
+
|
|
1031
|
+
An isolation environment:
|
|
1032
|
+
- Routes test traffic to a dedicated instance of your service (and its dependencies)
|
|
1033
|
+
- Prevents tests from reading or writing data that other developers or users are touching
|
|
1034
|
+
- Allows teardown to clean up all test data automatically
|
|
1035
|
+
|
|
1036
|
+
## 10.2 Check What Isolation Mechanism Is Available
|
|
1037
|
+
|
|
1038
|
+
| Mechanism | When to use |
|
|
1039
|
+
|-----------|-------------|
|
|
1040
|
+
| isoforge | Freshworks-internal isolation platform — preferred if available |
|
|
1041
|
+
| docker-compose | Good for local development, harder to maintain in CI |
|
|
1042
|
+
| Testcontainers (Java) | In-process, no network isolation, good for unit/integration |
|
|
1043
|
+
| Feature flags | Route by header/flag — least isolation, easiest to set up |
|
|
1044
|
+
|
|
1045
|
+
## 10.3 isoforge Integration Pattern
|
|
1046
|
+
|
|
1047
|
+
If isoforge is available in your organisation:
|
|
1048
|
+
|
|
1049
|
+
```bash
|
|
1050
|
+
# In run-tests-staging.sh: set isolation identifier before running Newman
|
|
1051
|
+
ISO_ID=$(isoforge create --service ${SERVICE_NAME} --branch ${BRANCH_NAME})
|
|
1052
|
+
export ISO_HEADER_VALUE="$ISO_ID"
|
|
1053
|
+
|
|
1054
|
+
# Pass the isolation header to Newman via environment variable
|
|
1055
|
+
newman run collections/01-bootstrap.json \
|
|
1056
|
+
--environment working-env.json \
|
|
1057
|
+
--env-var "iso_header=${ISO_HEADER_VALUE}" \
|
|
1058
|
+
--export-environment working-env.json
|
|
1059
|
+
|
|
1060
|
+
# In each Postman request, add the isolation header:
|
|
1061
|
+
# Header: X-Isoforge-Id: {{iso_header}}
|
|
1062
|
+
```
|
|
1063
|
+
|
|
1064
|
+
Teardown (always run, even on failure):
|
|
1065
|
+
|
|
1066
|
+
```bash
|
|
1067
|
+
isoforge destroy --service ${SERVICE_NAME} || true
|
|
1068
|
+
```
|
|
1069
|
+
|
|
1070
|
+
## 10.4 Postman Request Header for Isolation
|
|
1071
|
+
|
|
1072
|
+
In each request that needs isolation routing, add:
|
|
1073
|
+
|
|
1074
|
+
```json
|
|
1075
|
+
{
|
|
1076
|
+
"key": "X-Isoforge-Id",
|
|
1077
|
+
"value": "{{iso_header}}",
|
|
1078
|
+
"description": "Routes request to isolated service instance"
|
|
1079
|
+
}
|
|
1080
|
+
```
|
|
1081
|
+
|
|
1082
|
+
Or add it as a pre-request script in the collection-level scripts to apply to all requests automatically.
|
|
1083
|
+
|
|
1084
|
+
---
|
|
1085
|
+
|
|
1086
|
+
## Validation
|
|
1087
|
+
|
|
1088
|
+
After completing setup, run the validation skill to confirm all checks pass:
|
|
1089
|
+
|
|
1090
|
+
```
|
|
1091
|
+
/review-test-suite
|
|
1092
|
+
```
|
|
1093
|
+
|
|
1094
|
+
This checks all 20 quality gates: unit tests, coverage, mutation testing, collection lifecycle, secret security, CI pipeline, GitHub Actions, and developer experience.
|