@freshworks/shiftleft-tools 1.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +351 -0
  2. package/bin/shiftleft.js +95 -0
  3. package/package.json +57 -0
  4. package/src/commands/doctor.js +208 -0
  5. package/src/commands/init-postman.js +298 -0
  6. package/src/commands/init-rules.js +78 -0
  7. package/src/commands/link.js +172 -0
  8. package/src/commands/protect.js +61 -0
  9. package/src/commands/run-tests.js +182 -0
  10. package/src/commands/setup-pipeline.js +209 -0
  11. package/src/commands/update.js +203 -0
  12. package/src/index.js +4 -0
  13. package/src/utils/copy-tree.js +98 -0
  14. package/src/utils/gitignore.js +26 -0
  15. package/src/utils/logger.js +9 -0
  16. package/src/utils/manifest.js +145 -0
  17. package/src/utils/stack.js +80 -0
  18. package/src/utils/template.js +135 -0
  19. package/templates/AGENTS.md +109 -0
  20. package/templates/CLAUDE.md +3 -0
  21. package/templates/jenkins/Jenkinsfile-java.groovy +432 -0
  22. package/templates/jenkins/Jenkinsfile-node.groovy +450 -0
  23. package/templates/postman/.husky/pre-commit +19 -0
  24. package/templates/postman/.prettierrc.json +5 -0
  25. package/templates/postman/README.md.ejs +147 -0
  26. package/templates/postman/collections/01-core.json.ejs +91 -0
  27. package/templates/postman/config/local.json.ejs +12 -0
  28. package/templates/postman/config/staging.json.ejs +26 -0
  29. package/templates/postman/environments/local.postman_environment.json.ejs +31 -0
  30. package/templates/postman/environments/staging.postman_environment.json.ejs +31 -0
  31. package/templates/postman/gitignore +16 -0
  32. package/templates/postman/npmrc +31 -0
  33. package/templates/postman/package.json.ejs +66 -0
  34. package/templates/postman/run-all-shim.sh +16 -0
  35. package/templates/postman/scripts/auth/generate-jwt.sh +113 -0
  36. package/templates/postman/scripts/auth/get-issuer-secret.sh +140 -0
  37. package/templates/postman/scripts/infra/start-mocks.sh +138 -0
  38. package/templates/postman/scripts/infra/stop-mocks.sh +43 -0
  39. package/templates/postman/scripts/lib/api_coverage.py +1122 -0
  40. package/templates/postman/scripts/lib/cleanup-reports.sh +101 -0
  41. package/templates/postman/scripts/lib/cleanup-stryker.sh +44 -0
  42. package/templates/postman/scripts/lib/report_combined.py +527 -0
  43. package/templates/postman/scripts/lib/report_consolidated.py +363 -0
  44. package/templates/postman/scripts/lib/report_generator.py +121 -0
  45. package/templates/postman/scripts/lib/report_migration.py +156 -0
  46. package/templates/postman/scripts/lib/report_mutation.py +110 -0
  47. package/templates/postman/scripts/lib/report_unit.py +353 -0
  48. package/templates/postman/scripts/lib/report_utils.py +973 -0
  49. package/templates/postman/scripts/report-generators/generate-consolidated-report.sh +445 -0
  50. package/templates/postman/scripts/report-generators/java-api-coverage-matrix.sh +257 -0
  51. package/templates/postman/scripts/report-generators/mutation-report.sh +672 -0
  52. package/templates/postman/scripts/report-generators/node-api-coverage-matrix.sh +167 -0
  53. package/templates/postman/scripts/report-generators/stage-report-artifacts.sh +27 -0
  54. package/templates/postman/scripts/run-all.sh +452 -0
  55. package/templates/postman/scripts/runners/run-mutation-tests.sh +113 -0
  56. package/templates/postman/scripts/runners/run-tests-local.sh +936 -0
  57. package/templates/postman/scripts/runners/run-tests-staging.sh +741 -0
  58. package/templates/postman-node/README.md.ejs +26 -0
  59. package/templates/postman-node/collections/crud/01-bootstrap.json.ejs +34 -0
  60. package/templates/postman-node/config/local.json.ejs +46 -0
  61. package/templates/postman-node/config/staging.json.ejs +31 -0
  62. package/templates/postman-node/local.test.env.ejs +3 -0
  63. package/templates/postman-node/mocks/external.js +14 -0
  64. package/templates/postman-node/package.json.ejs +39 -0
  65. package/templates/postman-node/requirements.txt +1 -0
  66. package/templates/postman-node/scripts/database/cleanup-mysql.sh +12 -0
  67. package/templates/postman-node/scripts/database/run-migrations.js +29 -0
  68. package/templates/postman-node/scripts/database/start-mysql.sh +34 -0
  69. package/templates/postman-node/scripts/database/wait-for-mysql.sh +36 -0
  70. package/templates/postman-node/scripts/lib/api_coverage_node.py +1137 -0
  71. package/templates/postman-node/scripts/lib/fetch-jwt.sh +86 -0
  72. package/templates/postman-node/scripts/lib/run-newman.sh +104 -0
  73. package/templates/postman-node/scripts/lib/setup-database.sh +55 -0
  74. package/templates/postman-node/scripts/lib/start-app.sh +48 -0
  75. package/templates/postman-node/scripts/lib/utils.sh +114 -0
  76. package/templates/postman-node/scripts/report-generators/stage-report-artifacts.sh +26 -0
  77. package/templates/postman-node/scripts/run-all.sh +303 -0
  78. package/templates/postman-node/scripts/runners/run-tests.sh +123 -0
  79. package/templates/postman-node/scripts/setup-mocks.js.ejs +29 -0
  80. package/templates/postman-node/stryker.config.js.ejs +51 -0
  81. package/templates/rules/local-test-setup.mdc +420 -0
  82. package/templates/rules/testing-node.mdc +66 -0
  83. package/templates/rules/testing.mdc +248 -0
  84. package/templates/skills/_shared/postman-standards.md +380 -0
  85. package/templates/skills/enhance-test-pipeline/SKILL-java.md +483 -0
  86. package/templates/skills/enhance-test-pipeline/SKILL-node.md +431 -0
  87. package/templates/skills/enhance-test-pipeline/SKILL.md +9 -0
  88. package/templates/skills/review-test-suite/SKILL-java.md +137 -0
  89. package/templates/skills/review-test-suite/SKILL-node.md +78 -0
  90. package/templates/skills/review-test-suite/SKILL.md +9 -0
  91. package/templates/skills/run-test-suite/SKILL-java.md +186 -0
  92. package/templates/skills/run-test-suite/SKILL-node.md +191 -0
  93. package/templates/skills/run-test-suite/SKILL.md +9 -0
  94. package/templates/skills/setup-api-tests/SKILL-java.md +1094 -0
  95. package/templates/skills/setup-api-tests/SKILL-node.md +141 -0
  96. package/templates/skills/setup-api-tests/SKILL.md +9 -0
  97. package/templates/skills/setup-mutation-tests/SKILL-java.md +303 -0
  98. package/templates/skills/setup-mutation-tests/SKILL-node.md +408 -0
  99. package/templates/skills/setup-mutation-tests/SKILL.md +9 -0
  100. package/templates/skills/setup-test-pipeline/SKILL-java.md +454 -0
  101. package/templates/skills/setup-test-pipeline/SKILL-node.md +318 -0
  102. package/templates/skills/setup-test-pipeline/SKILL.md +9 -0
  103. package/templates/skills/write-api-tests/SKILL-java.md +115 -0
  104. package/templates/skills/write-api-tests/SKILL-node.md +83 -0
  105. package/templates/skills/write-api-tests/SKILL.md +9 -0
  106. package/templates/stryker.config.js +50 -0
@@ -0,0 +1,1122 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ API Coverage Matrix Generator.
4
+
5
+ Analyzes Java Spring Boot controllers and Postman collections to produce
6
+ an HTML coverage report showing which endpoints have test coverage.
7
+
8
+ Usage:
9
+ python3 api_coverage.py --controller-dir DIR --postman-dir DIR --html-output FILE
10
+ """
11
+
12
+ import argparse
13
+ import os
14
+ import re
15
+ import json
16
+ import sys
17
+ from collections import defaultdict
18
+ from datetime import datetime
19
+ from html import escape as esc
20
+
21
+ try:
22
+ from report_utils import (
23
+ get_css, get_html_template,
24
+ build_summary_card, build_coverage_item, build_gaps_section,
25
+ build_collapsible_table, build_endpoint_row
26
+ )
27
+ SHARED_CSS = get_css()
28
+ USE_HELPERS = True
29
+ except ImportError:
30
+ SHARED_CSS = None
31
+ USE_HELPERS = False
32
+
33
+
34
+ def parse_args():
35
+ parser = argparse.ArgumentParser(description='API Coverage Matrix Generator')
36
+ parser.add_argument('--controller-dir', required=True, help='Path to Java controller directory')
37
+ parser.add_argument('--postman-dir', required=True, help='Path to Postman directory')
38
+ parser.add_argument('--html-output', required=True, help='Path for HTML output file')
39
+ return parser.parse_args()
40
+
41
+
42
+ args = parse_args()
43
+ CONTROLLER_DIR = args.controller_dir
44
+ POSTMAN_DIR = args.postman_dir
45
+ HTML_OUTPUT_FILE = args.html_output
46
+
47
+ endpoints = []
48
+ tested = []
49
+ errors = []
50
+
51
+ print(f"Scanning {CONTROLLER_DIR}", flush=True)
52
+ print(f"Scanning {POSTMAN_DIR}", flush=True)
53
+
54
+ def safe_read_file(filepath, max_size_mb=50):
55
+ """Safely read a file with size limits."""
56
+ try:
57
+ file_size = os.path.getsize(filepath)
58
+ if file_size > max_size_mb * 1024 * 1024:
59
+ errors.append(f"Skipped {filepath}: file too large")
60
+ return None
61
+ with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
62
+ return f.read()
63
+ except Exception as e:
64
+ errors.append(f"Error reading {filepath}: {str(e)}")
65
+ return None
66
+
67
+ def join_concatenated_strings(text):
68
+ """Join Java string concatenations like '"part1" + "part2"' into single strings."""
69
+ result = text
70
+ result = re.sub(r'"\s*\n\s*\+\s*"', '', result) # NOSONAR - input is trusted local Java source files
71
+ result = re.sub(r'"\s*\+\s*\n\s*"', '', result) # NOSONAR
72
+ result = re.sub(r'"\s*\+\s*"', '', result)
73
+ return result
74
+
75
+ def resolve_constants(content, controller_dir):
76
+ """Resolve Java constant references (e.g., PathConstants.V1) to their string values."""
77
+ constants = {}
78
+ constants_dir = os.path.dirname(controller_dir)
79
+
80
+ for root, dirs, files in os.walk(constants_dir):
81
+ dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['target', 'build']]
82
+ for f in files:
83
+ if f.endswith('.java'):
84
+ fpath = os.path.join(root, f)
85
+ try:
86
+ fcontent = safe_read_file(fpath)
87
+ if fcontent is None:
88
+ continue
89
+ class_match = re.search(r'(?:public\s+)?class\s+(\w+)', fcontent)
90
+ if not class_match:
91
+ continue
92
+ class_name = class_match.group(1)
93
+ for m in re.finditer(
94
+ r'(?:public\s+)?static\s+final\s+String\s+(\w+)\s*=\s*"([^"]*)"',
95
+ fcontent
96
+ ):
97
+ constants[f"{class_name}.{m.group(1)}"] = m.group(2)
98
+ except Exception:
99
+ continue
100
+
101
+ if not constants:
102
+ return content
103
+
104
+ def replace_constant(match):
105
+ ref = match.group(0)
106
+ return f'"{constants[ref]}"' if ref in constants else match.group(0)
107
+
108
+ pattern = '|'.join(re.escape(k) for k in sorted(constants.keys(), key=len, reverse=True))
109
+ resolved = re.sub(pattern, replace_constant, content)
110
+ return resolved
111
+
112
+
113
+ def extract_endpoints_from_file(filepath):
114
+ """Extract API endpoints from a Java controller file."""
115
+ content = safe_read_file(filepath)
116
+ if content is None:
117
+ return
118
+
119
+ content = resolve_constants(content, CONTROLLER_DIR)
120
+ content = join_concatenated_strings(content)
121
+ controller_name = os.path.basename(filepath).replace('.java', '')
122
+
123
+ class_path_match = re.search(r'@RequestMapping\s*\(\s*(?:path\s*=\s*)?["\']([^"\']+)["\']', content)
124
+ class_path = class_path_match.group(1) if class_path_match else ''
125
+
126
+ method_blocks = re.split(r'(?=@(?:Get|Post|Put|Delete|Patch)Mapping)', content)
127
+
128
+ for block in method_blocks:
129
+ if len(endpoints) >= 1000:
130
+ return
131
+
132
+ if not re.match(r'@(?:Get|Post|Put|Delete|Patch)Mapping', block.strip()):
133
+ continue
134
+
135
+ mapping_patterns = [
136
+ r'@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*value\s*=\s*\{?\s*["\']([^"\']+)["\']',
137
+ r'@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*\{?\s*["\']([^"\']+)["\']',
138
+ r'@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*["\']([^"\']+)["\']',
139
+ r'@(Get|Post|Put|Delete|Patch)Mapping\(["\']([^"\']+)["\']',
140
+ ]
141
+
142
+ http_method = None
143
+ path = None
144
+
145
+ for pattern in mapping_patterns:
146
+ match = re.search(pattern, block)
147
+ if match:
148
+ http_method = match.group(1).upper()
149
+ path = match.group(2)
150
+ break
151
+
152
+ if not http_method:
153
+ first_line = block.strip().split('\n')[0].strip()
154
+ no_path_match = re.match( # NOSONAR - input is trusted local Java source files
155
+ r'@(Get|Post|Put|Delete|Patch)Mapping\s*(?:\([^"\']*\))?\s*$',
156
+ first_line
157
+ )
158
+ if not no_path_match:
159
+ no_path_match = re.match(
160
+ r'@(Get|Post|Put|Delete|Patch)Mapping\s*$',
161
+ first_line
162
+ )
163
+ if no_path_match:
164
+ http_method = no_path_match.group(1).upper()
165
+ path = ''
166
+
167
+ if not http_method:
168
+ continue
169
+
170
+ if class_path and path:
171
+ full_path = f"{class_path.rstrip('/')}/{path.lstrip('/')}".replace('//', '/')
172
+ elif class_path:
173
+ full_path = class_path
174
+ else:
175
+ full_path = path
176
+
177
+ full_path = re.sub(r'\.json$', '', full_path)
178
+ full_path = full_path.rstrip('/')
179
+ if not full_path.startswith('/'):
180
+ full_path = '/' + full_path
181
+
182
+ param_block = block
183
+ defined_params = set()
184
+ include_params = set()
185
+
186
+ named_param_pattern = r'@RequestParam\s*\(\s*(?:(?:name|value)\s*=\s*)?["\']([^"\']+)["\'][^)]*\)' # NOSONAR
187
+ unnamed_param_pattern = r'@RequestParam\s*(?:\([^"\']*\))?\s*\w+\s+(\w+)' # NOSONAR
188
+
189
+ named_params = set()
190
+ for pm in re.finditer(named_param_pattern, param_block):
191
+ param_name = pm.group(1)
192
+ if param_name:
193
+ named_params.add(param_name)
194
+ defined_params.add(param_name.lower().replace('_', ''))
195
+ if 'include' in param_name.lower():
196
+ include_params.add(param_name)
197
+
198
+ for pm in re.finditer(unnamed_param_pattern, param_block):
199
+ var_name = pm.group(1)
200
+ if var_name:
201
+ match_start = pm.start()
202
+ preceding = param_block[:match_start]
203
+ last_annotation = preceding.rfind('@RequestParam')
204
+ if last_annotation >= 0:
205
+ annotation_text = param_block[last_annotation:match_start]
206
+ if 'name' not in annotation_text and '"\'' not in annotation_text[:50]:
207
+ defined_params.add(var_name.lower())
208
+ if 'include' in var_name.lower():
209
+ include_params.add(var_name)
210
+
211
+ endpoint = {
212
+ 'method': http_method,
213
+ 'path': full_path,
214
+ 'controller': controller_name,
215
+ 'defined_params': defined_params,
216
+ 'has_include_param': len(include_params) > 0
217
+ }
218
+
219
+ is_duplicate = False
220
+ for existing in endpoints:
221
+ if existing['method'] == endpoint['method'] and existing['path'] == endpoint['path']:
222
+ existing['defined_params'].update(endpoint['defined_params'])
223
+ if endpoint['has_include_param']:
224
+ existing['has_include_param'] = True
225
+ is_duplicate = True
226
+ break
227
+
228
+ if not is_duplicate:
229
+ endpoints.append(endpoint)
230
+
231
+ # Process controller files
232
+ try:
233
+ for root, dirs, files in os.walk(CONTROLLER_DIR):
234
+ dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['target', 'build', 'node_modules']]
235
+
236
+ for file in files:
237
+ if file.endswith('.java'):
238
+ filepath = os.path.join(root, file)
239
+ try:
240
+ extract_endpoints_from_file(filepath)
241
+ except Exception as e:
242
+ errors.append(f"Error processing {file}: {str(e)}")
243
+ except Exception as e:
244
+ print(f"ERROR: Failed to scan controller directory: {e}", flush=True)
245
+ sys.exit(1)
246
+
247
+ def extract_tests_from_item(item, collection_name, depth=0):
248
+ """Extract test information from Postman collection items."""
249
+ if depth > 20:
250
+ return []
251
+
252
+ results = []
253
+
254
+ if isinstance(item, list):
255
+ for i in item:
256
+ results.extend(extract_tests_from_item(i, collection_name, depth + 1))
257
+ elif isinstance(item, dict):
258
+ if 'request' in item:
259
+ try:
260
+ request = item['request']
261
+ if not isinstance(request, dict):
262
+ return results
263
+
264
+ method = request.get('method', 'GET')
265
+
266
+ url = request.get('url', {})
267
+ query_params = set()
268
+ query_param_values = defaultdict(set)
269
+
270
+ if isinstance(url, str):
271
+ url_path = url
272
+ if '?' in url_path:
273
+ url_path, query_string = url_path.split('?', 1)
274
+ for param in query_string.split('&'):
275
+ if '=' in param:
276
+ key, val = param.split('=', 1)
277
+ query_params.add(key)
278
+ query_param_values[key].add(val)
279
+ elif isinstance(url, dict):
280
+ path_parts = url.get('path', [])
281
+ if isinstance(path_parts, list):
282
+ url_path = '/' + '/'.join(str(p) for p in path_parts) if path_parts else ''
283
+ else:
284
+ url_path = str(path_parts)
285
+
286
+ for qp in url.get('query', []) or []:
287
+ if isinstance(qp, dict):
288
+ key = qp.get('key', '')
289
+ val = qp.get('value', '')
290
+ if key:
291
+ query_params.add(key)
292
+ if val:
293
+ query_param_values[key].add(val)
294
+ else:
295
+ url_path = ''
296
+
297
+ if '?' in url_path:
298
+ url_path, extra_qs = url_path.split('?', 1)
299
+ for param in extra_qs.split('&'):
300
+ if '=' in param:
301
+ key, val = param.split('=', 1)
302
+ query_params.add(key)
303
+ query_param_values[key].add(val)
304
+
305
+ url_path = re.sub(r'\.json$', '', url_path)
306
+ url_path = re.sub(r'\{\{[^}]+\}\}', '{VAR}', url_path)
307
+ url_path = re.sub(r':[^/]+', '{VAR}', url_path)
308
+
309
+ status_codes = set()
310
+ body_assertions = 0
311
+ has_oneof = False
312
+ has_skip = False
313
+ skip_reason = None
314
+ skip_env = None
315
+
316
+ # Check pre-request scripts for skips
317
+ for event in item.get('event', []) or []:
318
+ if isinstance(event, dict) and event.get('listen') == 'prerequest':
319
+ script = event.get('script', {})
320
+ if isinstance(script, dict):
321
+ exec_lines = script.get('exec', [])
322
+ if isinstance(exec_lines, list):
323
+ prereq_text = '\n'.join(str(line) for line in exec_lines)
324
+ else:
325
+ prereq_text = str(exec_lines)
326
+
327
+ # Detect pm.execution.skipRequest()
328
+ if 'pm.execution.skipRequest()' in prereq_text:
329
+ has_skip = True
330
+
331
+ # Extract skip reason - multiple formats supported:
332
+ # Format 1: console.log('[SKIP] Reason: ... | Env: ...')
333
+ # Format 2: console.log('[SKIP] ...') (reason only)
334
+ # Format 3: // SKIP: reason
335
+
336
+ # Try format with Reason and Env (flexible spacing/separators)
337
+ skip_match = re.search(
338
+ r"\[SKIP\].*?Reason:\s*([^|,;]+)(?:[|,;]\s*Env:\s*(\S+))?",
339
+ prereq_text, re.IGNORECASE
340
+ )
341
+ if skip_match:
342
+ skip_reason = skip_match.group(1).strip()
343
+ if skip_match.group(2):
344
+ skip_env = skip_match.group(2).strip().lower()
345
+
346
+ # Try simple [SKIP] message format
347
+ if not skip_reason:
348
+ simple_match = re.search(r"\[SKIP\]\s*(.+?)(?:['\"]|$)", prereq_text)
349
+ if simple_match and 'Reason:' not in simple_match.group(1):
350
+ skip_reason = simple_match.group(1).strip()
351
+
352
+ # Try comment-based skip reason
353
+ if not skip_reason:
354
+ comment_match = re.search(r"//\s*SKIP[:\s]+(.+)", prereq_text)
355
+ if comment_match:
356
+ skip_reason = comment_match.group(1).strip()
357
+
358
+ # Auto-detect environment from the if condition if not specified
359
+ if not skip_env:
360
+ # Check for environment condition patterns
361
+ env_patterns = [
362
+ # pm.environment.get('environment') === 'local'
363
+ (r"pm\.environment\.get\(['\"]environment['\"]\)\s*===\s*['\"](\w+)['\"]", lambda m: m.group(1)),
364
+ # pm.environment.get('environment') !== 'local' -> skip in non-local
365
+ (r"pm\.environment\.get\(['\"]environment['\"]\)\s*!==\s*['\"]local['\"]", lambda m: 'staging'),
366
+ (r"pm\.environment\.get\(['\"]environment['\"]\)\s*!==\s*['\"]staging['\"]", lambda m: 'local'),
367
+ # pm.variables.get('env') patterns
368
+ (r"pm\.variables\.get\(['\"]env['\"]\)\s*===\s*['\"](\w+)['\"]", lambda m: m.group(1)),
369
+ ]
370
+ for pattern, extractor in env_patterns:
371
+ env_match = re.search(pattern, prereq_text)
372
+ if env_match:
373
+ skip_env = extractor(env_match).lower()
374
+ break
375
+
376
+ for event in item.get('event', []) or []:
377
+ if isinstance(event, dict) and event.get('listen') == 'test':
378
+ script = event.get('script', {})
379
+ if isinstance(script, dict):
380
+ exec_lines = script.get('exec', [])
381
+ if isinstance(exec_lines, list):
382
+ script_text = '\n'.join(str(line) for line in exec_lines)
383
+ else:
384
+ script_text = str(exec_lines)
385
+
386
+ matches = re.findall(r'to\.have\.status\((\d+)\)', script_text)
387
+ for m in matches:
388
+ status_codes.add(str(m))
389
+
390
+ matches_equal = re.findall(r'pm\.response\.code\)\.to\.equal\((\d+)\)', script_text)
391
+ for m in matches_equal:
392
+ status_codes.add(str(m))
393
+
394
+ matches_eql = re.findall(r'pm\.response\.code\)\.to\.eql\((\d+)\)', script_text)
395
+ for m in matches_eql:
396
+ status_codes.add(str(m))
397
+
398
+ # Detect weak assertions: oneOf, to.include with array for status codes
399
+ if 'oneOf' in script_text:
400
+ has_oneof = True
401
+ # Detect pm.expect([...]).to.include(pm.response.code) pattern - weak status code assertion
402
+ if re.search(r'pm\.expect\s*\(\s*\[.*?\]\s*\)\.to\.include\s*\(\s*pm\.response\.code', script_text):
403
+ has_oneof = True
404
+ # Detect pm.expect(pm.response.code).to.be.oneOf([...]) pattern
405
+ if re.search(r'pm\.response\.code\s*\)\s*\.to\.be\.oneOf', script_text):
406
+ has_oneof = True
407
+
408
+ # Extract status codes from oneOf/include arrays (weak but still counts for coverage)
409
+ for oneof_match in re.finditer(r'\.to\.be\.oneOf\s*\(\s*\[([^\]]+)\]', script_text):
410
+ for code in re.findall(r'\d{3}', oneof_match.group(1)):
411
+ status_codes.add(str(code))
412
+ for include_match in re.finditer(r'pm\.expect\s*\(\s*\[([\d,\s]+)\]\s*\)\.to\.include', script_text):
413
+ for code in re.findall(r'\d{3}', include_match.group(1)):
414
+ status_codes.add(str(code))
415
+
416
+ # Check for 5xx status code assertions (bad practice)
417
+ has_5xx = any(str(code).startswith('5') for code in status_codes)
418
+
419
+ body_patterns = [
420
+ r'pm\.expect\s*\(\s*jsonData\.',
421
+ r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.be\.an\s*\(',
422
+ r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.have\.property\s*\(',
423
+ r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.not\.be\.null',
424
+ r'pm\.expect\s*\(\s*response\.',
425
+ r'\.to\.have\.property\(',
426
+ r'\.to\.include\(',
427
+ r'\.to\.eql\(',
428
+ ]
429
+ for p in body_patterns:
430
+ body_assertions += len(re.findall(p, script_text))
431
+
432
+ has_request_body = False
433
+ request_body_hash = None
434
+ request_body_fields = set()
435
+ body = request.get('body', {})
436
+ if isinstance(body, dict):
437
+ raw_body = body.get('raw', '')
438
+ if raw_body and isinstance(raw_body, str) and raw_body.strip():
439
+ has_request_body = True
440
+ request_body_hash = hash(raw_body.strip())
441
+ # Try JSON parsing first, fallback to regex for Postman template variables
442
+ try:
443
+ parsed = json.loads(raw_body)
444
+ if isinstance(parsed, dict):
445
+ request_body_fields = set(parsed.keys())
446
+ except (json.JSONDecodeError, ValueError):
447
+ # Use regex to extract field names when JSON parsing fails
448
+ # This handles Postman template variables like {{variable}}
449
+ field_pattern = r'"([^"]+)"\s*:'
450
+ request_body_fields = set(re.findall(field_pattern, raw_body))
451
+
452
+ if url_path:
453
+ results.append({
454
+ 'method': method,
455
+ 'path': url_path,
456
+ 'name': item.get('name', ''),
457
+ 'status_codes': list(status_codes) if status_codes else [],
458
+ 'collection': collection_name,
459
+ 'body_assertions': body_assertions,
460
+ 'has_request_body': has_request_body,
461
+ 'request_body_hash': request_body_hash,
462
+ 'request_body_fields': request_body_fields,
463
+ 'query_params': query_params,
464
+ 'query_param_values': dict(query_param_values),
465
+ 'has_oneof': has_oneof,
466
+ 'has_5xx': any(str(code).startswith('5') for code in status_codes),
467
+ 'has_skip': has_skip,
468
+ 'skip_reason': skip_reason,
469
+ 'skip_env': skip_env
470
+ })
471
+ except Exception as e:
472
+ errors.append(f"Error parsing request in {collection_name}: {str(e)}")
473
+
474
+ if 'item' in item:
475
+ results.extend(extract_tests_from_item(item['item'], collection_name, depth + 1))
476
+
477
+ return results
478
+
479
+ def find_collection_files(base_dir):
480
+ """Find all Postman collection JSON files in the given directory."""
481
+ collection_files = []
482
+ seen_files = set() # Track seen file paths to avoid duplicates
483
+
484
+ search_dirs = [base_dir]
485
+
486
+ for subdir in ['collections', 'api-tests', 'tests']:
487
+ subpath = os.path.join(base_dir, subdir)
488
+ if os.path.isdir(subpath):
489
+ search_dirs.append(subpath)
490
+
491
+ seen_dirs = set()
492
+
493
+ for search_dir in search_dirs:
494
+ if search_dir in seen_dirs:
495
+ continue
496
+ seen_dirs.add(search_dir)
497
+
498
+ try:
499
+ for root, dirs, files in os.walk(search_dir):
500
+ # Skip directories that definitely don't contain Postman collections
501
+ dirs[:] = [d for d in dirs if d not in [
502
+ 'node_modules', 'environments', 'reports', 'config',
503
+ 'scripts', '.git', 'data', 'schemas'
504
+ ]]
505
+
506
+ for file in files:
507
+ if file.endswith('.json'):
508
+ # Only skip files that are DEFINITELY not Postman collections
509
+ # Use exact matches or specific patterns to avoid false positives
510
+ # The Postman structure validation later will handle edge cases
511
+ skip_exact = [
512
+ 'package.json', 'package-lock.json',
513
+ 'tsconfig.json', 'jsconfig.json',
514
+ '.eslintrc.json', '.prettierrc.json',
515
+ 'newman-reporter-config.json'
516
+ ]
517
+
518
+ # Skip Postman environment files
519
+ # Common patterns: *.environment.json, *_environment.json,
520
+ # *.postman_environment.json (Postman export format)
521
+ is_environment_file = (
522
+ file.lower().endswith('.environment.json') or
523
+ file.lower().endswith('_environment.json') or
524
+ file.lower().endswith('.postman_environment.json') or
525
+ file.lower() == 'environment.json'
526
+ )
527
+
528
+ if file.lower() in skip_exact:
529
+ continue
530
+ if is_environment_file:
531
+ continue
532
+
533
+ # Deduplicate by absolute path
534
+ full_path = os.path.abspath(os.path.join(root, file))
535
+ if full_path not in seen_files:
536
+ seen_files.add(full_path)
537
+ collection_files.append(full_path)
538
+ except Exception as e:
539
+ errors.append(f"Error scanning {search_dir}: {str(e)}")
540
+
541
+ return collection_files
542
+
543
+ try:
544
+ collection_files = find_collection_files(POSTMAN_DIR)
545
+ print(f"Found {len(collection_files)} collection file(s)", flush=True)
546
+
547
+ for filepath in collection_files:
548
+ file = os.path.basename(filepath)
549
+
550
+ file_size = os.path.getsize(filepath)
551
+ if file_size > 50 * 1024 * 1024:
552
+ errors.append(f"Skipped {file}: collection too large")
553
+ continue
554
+
555
+ try:
556
+ content = safe_read_file(filepath)
557
+ if content:
558
+ collection = json.loads(content)
559
+ if not isinstance(collection, dict) or ('item' not in collection and 'info' not in collection):
560
+ continue
561
+ collection_name = file.replace('.json', '')
562
+ tests = extract_tests_from_item(collection.get('item', []), collection_name)
563
+ tested.extend(tests)
564
+ except json.JSONDecodeError as e:
565
+ errors.append(f"Invalid JSON in {file}: {str(e)}")
566
+ except Exception as e:
567
+ errors.append(f"Error processing {file}: {str(e)}")
568
+ except Exception as e:
569
+ print(f"ERROR: Failed to scan Postman directory: {e}", flush=True)
570
+ sys.exit(1)
571
+
572
+ # Sort and deduplicate endpoints
573
+ seen = set()
574
+ unique_endpoints = []
575
+ for ep in endpoints:
576
+ key = (ep['method'], ep['path'])
577
+ if key not in seen:
578
+ seen.add(key)
579
+ unique_endpoints.append(ep)
580
+ endpoints = sorted(unique_endpoints, key=lambda x: (x['path'], x['method']))
581
+
582
+ # Aggregate stats per endpoint
583
+ endpoint_stats = defaultdict(lambda: {
584
+ 'test_count': 0,
585
+ 'status_codes': set(),
586
+ 'body_assertions': 0,
587
+ 'request_body_variations': set(),
588
+ 'has_body_test': False,
589
+ 'query_params': set(),
590
+ 'query_param_values': defaultdict(set),
591
+ 'request_body_fields': set(),
592
+ 'has_oneof': False,
593
+ 'oneof_count': 0,
594
+ 'has_5xx': False,
595
+ 'count_5xx': 0,
596
+ 'skipped_tests': []
597
+ })
598
+
599
+ # Track all skipped tests separately for reporting
600
+ all_skipped_tests = []
601
+
602
+ def normalize_path(path):
603
+ """Normalize path for comparison."""
604
+ if not path:
605
+ return ''
606
+ normalized = re.sub(r'^/?(api/)?v\d+/', '/', path)
607
+ normalized = re.sub(r'\{\{[^}]+\}\}', '{VAR}', normalized)
608
+ normalized = re.sub(r'\{[^}]+\}', '{VAR}', normalized)
609
+ normalized = re.sub(r':[^/]+', '{VAR}', normalized)
610
+ normalized = re.sub(r'/(\d+)(?=/|$)', '/{VAR}', normalized)
611
+ normalized = re.sub(r'\.json$', '', normalized)
612
+ return normalized.lower().replace('{var}', '{VAR}')
613
+
614
+ def paths_match(controller_path, test_path):
615
+ """Check if a test path matches a controller path."""
616
+ norm_controller = normalize_path(controller_path)
617
+ norm_test = normalize_path(test_path)
618
+
619
+ if norm_controller == norm_test:
620
+ return True
621
+
622
+ try:
623
+ pattern = re.escape(norm_controller).replace(r'\{VAR\}', '[^/]+')
624
+ if re.fullmatch(pattern, norm_test):
625
+ return True
626
+ except re.error:
627
+ pass
628
+
629
+ return False
630
+
631
+ for test in tested:
632
+ test_path = test['path']
633
+ test_method = test['method']
634
+
635
+ # Track skipped tests globally
636
+ if test.get('has_skip', False):
637
+ skip_info = {
638
+ 'name': test.get('name', 'Unknown'),
639
+ 'collection': test.get('collection', ''),
640
+ 'method': test_method,
641
+ 'path': test_path,
642
+ 'reason': test.get('skip_reason'),
643
+ 'env': test.get('skip_env')
644
+ }
645
+ all_skipped_tests.append(skip_info)
646
+
647
+ matched = False
648
+ for ep in endpoints:
649
+ if test_method == ep['method'] and paths_match(ep['path'], test_path):
650
+ key = f"{ep['method']} {ep['path']}"
651
+ endpoint_stats[key]['test_count'] += 1
652
+ endpoint_stats[key]['status_codes'].update(test['status_codes'])
653
+ endpoint_stats[key]['body_assertions'] += test['body_assertions']
654
+ if test['body_assertions'] > 0:
655
+ endpoint_stats[key]['has_body_test'] = True
656
+ if test['request_body_hash']:
657
+ endpoint_stats[key]['request_body_variations'].add(test['request_body_hash'])
658
+ endpoint_stats[key]['query_params'].update(test.get('query_params', set()))
659
+ for param, values in test.get('query_param_values', {}).items():
660
+ endpoint_stats[key]['query_param_values'][param].update(values)
661
+ endpoint_stats[key]['request_body_fields'].update(test.get('request_body_fields', set()))
662
+ if test.get('has_oneof', False):
663
+ endpoint_stats[key]['has_oneof'] = True
664
+ endpoint_stats[key]['oneof_count'] += 1
665
+ if test.get('has_5xx', False):
666
+ endpoint_stats[key]['has_5xx'] = True
667
+ endpoint_stats[key]['count_5xx'] += 1
668
+ if test.get('has_skip', False):
669
+ endpoint_stats[key]['skipped_tests'].append({
670
+ 'name': test.get('name', ''),
671
+ 'reason': test.get('skip_reason'),
672
+ 'env': test.get('skip_env')
673
+ })
674
+ matched = True
675
+ break
676
+
677
+ if not matched:
678
+ key = f"{test_method} {test_path}"
679
+ endpoint_stats[key]['test_count'] += 1
680
+ endpoint_stats[key]['status_codes'].update(test['status_codes'])
681
+ endpoint_stats[key]['body_assertions'] += test['body_assertions']
682
+ if test['body_assertions'] > 0:
683
+ endpoint_stats[key]['has_body_test'] = True
684
+ if test['request_body_hash']:
685
+ endpoint_stats[key]['request_body_variations'].add(test['request_body_hash'])
686
+ endpoint_stats[key]['query_params'].update(test.get('query_params', set()))
687
+ for param, values in test.get('query_param_values', {}).items():
688
+ endpoint_stats[key]['query_param_values'][param].update(values)
689
+ endpoint_stats[key]['request_body_fields'].update(test.get('request_body_fields', set()))
690
+ if test.get('has_oneof', False):
691
+ endpoint_stats[key]['has_oneof'] = True
692
+ endpoint_stats[key]['oneof_count'] += 1
693
+ if test.get('has_5xx', False):
694
+ endpoint_stats[key]['has_5xx'] = True
695
+ endpoint_stats[key]['count_5xx'] += 1
696
+ if test.get('has_skip', False):
697
+ endpoint_stats[key]['skipped_tests'].append({
698
+ 'name': test.get('name', ''),
699
+ 'reason': test.get('skip_reason'),
700
+ 'env': test.get('skip_env')
701
+ })
702
+
703
+ # Calculate metrics
704
+ total_tests = sum(s['test_count'] for s in endpoint_stats.values())
705
+ total_body = sum(s['body_assertions'] for s in endpoint_stats.values())
706
+ tested_ep = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('test_count', 0) > 0)
707
+ well_tested = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('test_count', 0) >= 3)
708
+ with_body = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_body_test', False))
709
+ not_tested = len(endpoints) - tested_ep
710
+
711
+ # Success path tested (has at least one 2xx status code)
712
+ def has_success_code(status_codes):
713
+ """Check if any status code is a 2xx success code."""
714
+ return any(str(code).startswith('2') for code in status_codes)
715
+
716
+ def has_401_code(status_codes):
717
+ """Check if any status code is 401 (unauthorized)."""
718
+ return '401' in [str(code) for code in status_codes]
719
+
720
+ success_tested = sum(1 for ep in endpoints if has_success_code(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('status_codes', set())))
721
+ error_only = tested_ep - success_tested
722
+
723
+ # 401 coverage (endpoints with auth tests)
724
+ auth_tested = sum(1 for ep in endpoints if has_401_code(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('status_codes', set())))
725
+ auth_pct = (auth_tested / len(endpoints) * 100) if endpoints else 0
726
+
727
+ # oneOf (weak assertion) metrics
728
+ oneof_endpoints = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_oneof', False))
729
+ total_oneof_tests = sum(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('oneof_count', 0) for ep in endpoints)
730
+
731
+ # 5xx assertion metrics (bad practice)
732
+ endpoints_with_5xx = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_5xx', False))
733
+ total_5xx_tests = sum(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('count_5xx', 0) for ep in endpoints)
734
+
735
+ # Skip metrics
736
+ total_skipped_tests = len(all_skipped_tests)
737
+ skips_with_reason = sum(1 for s in all_skipped_tests if s.get('reason'))
738
+ skips_without_reason = total_skipped_tests - skips_with_reason
739
+ skips_by_env = defaultdict(list)
740
+ for skip in all_skipped_tests:
741
+ env = skip.get('env', 'unknown')
742
+ skips_by_env[env].append(skip)
743
+
744
+ # Query params metrics
745
+ total_defined_params = 0
746
+ total_tested_params = 0
747
+ endpoints_with_params = []
748
+ for ep in endpoints:
749
+ defined_params = ep.get('defined_params', set())
750
+ if defined_params:
751
+ total_defined_params += len(defined_params)
752
+ key = f"{ep['method']} {ep['path']}"
753
+ stats = endpoint_stats.get(key, {})
754
+ tested_params = stats.get('query_params', set())
755
+ tested_normalized = set(p.lower().replace('_', '') for p in tested_params)
756
+ covered = len(defined_params.intersection(tested_normalized))
757
+ total_tested_params += covered
758
+
759
+ param_values = stats.get('query_param_values', {})
760
+ if tested_params:
761
+ endpoints_with_params.append({
762
+ 'method': ep['method'],
763
+ 'path': ep['path'],
764
+ 'params': tested_params,
765
+ 'param_values': param_values,
766
+ 'total_values': sum(len(v) for v in param_values.values())
767
+ })
768
+
769
+ # Include params metrics
770
+ # Define expected include values per endpoint type
771
+ # - List endpoints: configs, oauth_configs_list, secure_iparams
772
+ # - Configuration endpoints: secure_iparams, oauth_iparams, extension_type
773
+ INCLUDE_VALUES_LIST = {'configs', 'oauth_configs_list', 'secure_iparams'}
774
+ INCLUDE_VALUES_CONFIG = {'secure_iparams', 'oauth_iparams', 'extension_type'}
775
+
776
+ endpoints_with_include = []
777
+ total_expected_include_values = 0
778
+ total_tested_include_values = 0
779
+
780
+ for ep in endpoints:
781
+ if ep.get('has_include_param', False):
782
+ key = f"{ep['method']} {ep['path']}"
783
+ stats = endpoint_stats.get(key, {})
784
+ tested_params = stats.get('query_params', set())
785
+ param_values = stats.get('query_param_values', {})
786
+
787
+ include_tested = any('include' in p.lower() for p in tested_params)
788
+ include_values = set()
789
+ include_variations = 0
790
+
791
+ for param, values in param_values.items():
792
+ if 'include' in param.lower():
793
+ for v in values:
794
+ for single_val in v.split(','):
795
+ include_values.add(single_val.strip())
796
+ include_variations = len(values)
797
+
798
+ # Determine expected values based on endpoint type
799
+ if 'configurations' in ep['path'].lower():
800
+ expected_values = INCLUDE_VALUES_CONFIG
801
+ else:
802
+ expected_values = INCLUDE_VALUES_LIST
803
+
804
+ tested_expected = include_values & expected_values
805
+ missing_values = expected_values - include_values
806
+
807
+ total_expected_include_values += len(expected_values)
808
+ total_tested_include_values += len(tested_expected)
809
+
810
+ endpoints_with_include.append({
811
+ 'method': ep['method'],
812
+ 'path': ep['path'],
813
+ 'tested': include_tested,
814
+ 'values': include_values,
815
+ 'expected_values': expected_values,
816
+ 'tested_expected': tested_expected,
817
+ 'missing_values': missing_values,
818
+ 'variations': include_variations
819
+ })
820
+
821
+ # Request body metrics
822
+ endpoints_with_body = []
823
+ for ep in endpoints:
824
+ if ep['method'] in ['POST', 'PUT', 'PATCH']:
825
+ key = f"{ep['method']} {ep['path']}"
826
+ stats = endpoint_stats.get(key, {})
827
+ fields = stats.get('request_body_fields', set())
828
+ variations = len(stats.get('request_body_variations', set()))
829
+
830
+ if fields or variations > 0:
831
+ endpoints_with_body.append({
832
+ 'method': ep['method'],
833
+ 'path': ep['path'],
834
+ 'fields': fields,
835
+ 'variations': variations
836
+ })
837
+
838
+ # Calculate percentages
839
+ tested_pct = (tested_ep / len(endpoints) * 100) if endpoints else 0
840
+ well_tested_pct = (well_tested / len(endpoints) * 100) if endpoints else 0
841
+ body_pct = (with_body / len(endpoints) * 100) if endpoints else 0
842
+ qp_pct = (total_tested_params / total_defined_params * 100) if total_defined_params > 0 else 0
843
+ success_pct = (success_tested / len(endpoints) * 100) if endpoints else 0
844
+
845
+ # Include param coverage - now based on individual values tested vs expected
846
+ include_tested_count = sum(1 for ep in endpoints_with_include if ep['tested'])
847
+ include_total_count = len(endpoints_with_include)
848
+ include_pct = (total_tested_include_values / total_expected_include_values * 100) if total_expected_include_values > 0 else 100
849
+ include_endpoints_pct = (include_tested_count / include_total_count * 100) if include_total_count > 0 else 100
850
+
851
+ # Categorize endpoints for gaps analysis
852
+ gaps_no_tests = []
853
+ gaps_no_2xx = []
854
+ gaps_weak_assertions = []
855
+ gaps_5xx_assertions = []
856
+
857
+ for ep in endpoints:
858
+ key = f"{ep['method']} {ep['path']}"
859
+ stats = endpoint_stats.get(key, {'test_count': 0, 'status_codes': set()})
860
+
861
+ if stats['test_count'] == 0:
862
+ gaps_no_tests.append(ep)
863
+ elif not has_success_code(stats.get('status_codes', set())):
864
+ gaps_no_2xx.append(ep)
865
+
866
+ if stats.get('has_oneof', False):
867
+ gaps_weak_assertions.append({**ep, 'oneof_count': stats.get('oneof_count', 0)})
868
+
869
+ if stats.get('has_5xx', False):
870
+ gaps_5xx_assertions.append({**ep, 'count_5xx': stats.get('count_5xx', 0)})
871
+
872
+ def get_status_class(pct):
873
+ if pct >= 80: return 'success'
874
+ if pct >= 50: return 'warning'
875
+ return 'error'
876
+
877
+ def get_health_status(success_pct, well_tested_pct, error_only, oneof_count):
878
+ """Determine overall API test health."""
879
+ if success_pct >= 100 and well_tested_pct >= 80 and error_only == 0 and oneof_count == 0:
880
+ return ('excellent', '🟢', 'Excellent', 'All endpoints have 2xx tests with strong assertions')
881
+ elif success_pct >= 90 and well_tested_pct >= 60 and error_only <= 2:
882
+ return ('good', '🟡', 'Good', 'Most endpoints well tested, minor gaps exist')
883
+ elif success_pct >= 70 and well_tested_pct >= 40:
884
+ return ('fair', '🟠', 'Fair', 'Moderate coverage, needs improvement')
885
+ else:
886
+ return ('poor', '🔴', 'Needs Work', 'Significant testing gaps exist')
887
+
888
+ health_class, health_icon, health_label, health_desc = get_health_status(success_pct, well_tested_pct, error_only, total_oneof_tests)
889
+
890
+ # Build HTML content sections
891
+ def card(val, lbl, sub=None, status=None):
892
+ """Build a summary card."""
893
+ cls = f'summary-card{" " + esc(str(status)) if status else ""}'
894
+ sub_html = f'<div class="sub-value">{esc(str(sub))}</div>' if sub else ''
895
+ return f'<div class="{cls}"><div class="value">{esc(str(val))}</div><div class="label">{esc(str(lbl))}</div>{sub_html}</div>'
896
+
897
+ def coverage_item(name, pct, details):
898
+ """Build a coverage metric item."""
899
+ status = 'success' if pct >= 80 else 'warning' if pct >= 50 else 'error'
900
+ safe_pct = min(float(pct), 100)
901
+ return f'''<div class="coverage-item">
902
+ <div class="header"><span class="metric-name">{esc(str(name))}</span><span class="metric-value" style="color: var(--color-{status});">{safe_pct:.1f}%</span></div>
903
+ <div class="progress-bar"><div class="progress-fill {status}" style="width: {safe_pct:.1f}%;"></div></div>
904
+ <div class="details">{esc(str(details))}</div>
905
+ </div>'''
906
+
907
+ def gaps_section(title, items, warning=False, count_key=None):
908
+ """Build a gaps section."""
909
+ if not items: return ''
910
+ cls = 'gaps-section warning' if warning else 'gaps-section'
911
+ def gap_line(ep):
912
+ method = esc(str(ep["method"]))
913
+ path = esc(str(ep["path"]))
914
+ count_suffix = f' ({esc(str(ep.get(count_key, 0)))} tests)' if count_key else ''
915
+ return f'<div class="gap-item"><span class="method method-{method}">{method}</span> {path}{count_suffix}</div>'
916
+ items_html = ''.join(gap_line(ep) for ep in items)
917
+ return f'<div class="{cls}"><div class="title">{esc(str(title))}</div><div class="gap-list">{items_html}</div></div>'
918
+
919
+ def collapsible(title, count, headers, rows):
920
+ """Build a collapsible table section."""
921
+ if not rows: return ''
922
+ h = ''.join(f'<th style="{hdr.get("style","")}">{hdr["label"]}</th>' for hdr in headers)
923
+ r = ''.join(rows)
924
+ return f'''<details><summary>{title}<span class="count" style="background: var(--color-border); padding: 0.125rem 0.5rem; border-radius: 10px; font-size: 0.75rem; font-weight: 500;">{count}</span></summary>
925
+ <div class="content"><table><thead><tr>{h}</tr></thead><tbody>{r}</tbody></table></div></details>'''
926
+
927
+ # Compact summary - single row with key metrics
928
+ def metric(label, value, detail=None, status=None):
929
+ """Inline metric span."""
930
+ color = f'var(--color-{status})' if status else 'var(--color-primary)'
931
+ det = f' <span style="color:var(--color-muted);font-size:0.75rem;">({detail})</span>' if detail else ''
932
+ return f'<span style="margin-right:1.5rem;"><strong style="color:{color};">{value}</strong> {label}{det}</span>'
933
+
934
+ issues = []
935
+ if not_tested > 0: issues.append(f'{not_tested} untested')
936
+ if total_5xx_tests > 0: issues.append(f'{total_5xx_tests} 5xx')
937
+ if total_oneof_tests > 0: issues.append(f'{total_oneof_tests} weak')
938
+ if skips_without_reason > 0: issues.append(f'{skips_without_reason} undoc')
939
+ issues_str = ', '.join(issues) if issues else None
940
+
941
+ summary_html = f'''<div class="section">
942
+ <div style="display:flex;align-items:center;justify-content:space-between;flex-wrap:wrap;gap:1rem;padding:1rem 1.25rem;background:var(--color-card);border:1px solid var(--color-border);border-radius:var(--radius-sm);box-shadow:var(--shadow-sm);">
943
+ <div style="display:flex;align-items:center;gap:0.75rem;">
944
+ <span style="font-size:1.5rem;">{health_icon}</span>
945
+ <span style="font-weight:600;color:var(--color-heading);">{health_label}</span>
946
+ </div>
947
+ <div style="display:flex;flex-wrap:wrap;align-items:center;">
948
+ {metric('endpoints', len(endpoints), f'{total_tests} tests')}
949
+ {metric('2xx', f'{success_pct:.0f}%', f'{success_tested}/{len(endpoints)}', 'success' if success_pct >= 100 else 'warning' if success_pct >= 80 else 'error')}
950
+ {metric('well tested', f'{well_tested_pct:.0f}%', f'{well_tested}/{len(endpoints)}', 'success' if well_tested_pct >= 100 else 'warning' if well_tested_pct >= 80 else 'error')}
951
+ {metric('body validated', f'{body_pct:.0f}%', f'{with_body}/{len(endpoints)}', 'success' if body_pct >= 100 else 'warning' if body_pct >= 80 else 'error')}
952
+ {metric('query params', f'{qp_pct:.0f}%', f'{total_tested_params}/{total_defined_params}', 'success' if qp_pct >= 100 else 'warning' if qp_pct >= 80 else 'error')}
953
+ {f'<span style="color:var(--color-error);font-weight:500;">⚠ {issues_str}</span>' if issues_str else ''}
954
+ </div>
955
+ </div>
956
+ </div>'''
957
+
958
+ # No separate coverage breakdown needed - all in summary
959
+ coverage_html = ''
960
+
961
+ # Gaps analysis
962
+ gaps_html = ''
963
+ if gaps_no_tests or gaps_no_2xx or gaps_weak_assertions or gaps_5xx_assertions:
964
+ gaps_html = '<div class="section"><div class="section-title">⚠️ Gaps Analysis</div>'
965
+ gaps_html += gaps_section(f'❌ No Tests ({len(gaps_no_tests)} endpoints)', gaps_no_tests)
966
+ gaps_html += gaps_section(f'❌ No 2xx Tests ({len(gaps_no_2xx)} endpoints)', gaps_no_2xx)
967
+ gaps_html += gaps_section(f'⚠️ Weak Assertions (oneOf) - {len(gaps_weak_assertions)} endpoints', gaps_weak_assertions, warning=True, count_key='oneof_count')
968
+ gaps_html += gaps_section(f'🔴 5xx Assertions (bad practice) - {len(gaps_5xx_assertions)} endpoints', gaps_5xx_assertions, count_key='count_5xx')
969
+ gaps_html += '</div>'
970
+
971
+ # Endpoint details table
972
+ def endpoint_row(ep, stats):
973
+ tc = stats.get('test_count', 0)
974
+ codes = stats.get('status_codes', set())
975
+ codes_str = ', '.join(sorted(codes, key=lambda x: int(x) if x.isdigit() else 0)) if codes else '—'
976
+ body = stats.get('body_assertions', 0) or '—'
977
+ has_2xx = any(str(c).startswith('2') for c in codes)
978
+ success_icon = '✅' if has_2xx else ('❌' if tc > 0 else '—')
979
+ defined = ep.get('defined_params', set())
980
+ tested = stats.get('query_params', set())
981
+ qp_str = '—' if not defined else f'{len(defined.intersection(set(p.lower().replace("_","") for p in tested)))}/{len(defined)}' if tested else f'0/{len(defined)}'
982
+ oneof = stats.get('oneof_count', 0)
983
+ weak = f'⚠️ {oneof}' if oneof > 0 else '—'
984
+ icon, style = ('❌', 'background:#fef2f2;') if tc == 0 else ('⚠️', '') if tc < 3 else ('✅', '')
985
+ return f'<tr style="{style}"><td class="status-icon">{icon}</td><td><span class="method method-{ep["method"]}">{ep["method"]}</span></td><td class="left"><span class="path">{ep["path"]}</span></td><td>{tc}</td><td class="status-icon">{success_icon}</td><td>{codes_str}</td><td>{body}</td><td>{qp_str}</td><td>{weak}</td></tr>'
986
+
987
+ endpoint_rows = [endpoint_row(ep, endpoint_stats.get(f"{ep['method']} {ep['path']}", {})) for ep in endpoints]
988
+ details_html = f'''<div class="section"><div class="section-title">Endpoint Details<span class="count">{len(endpoints)} endpoints</span></div>
989
+ <table><thead><tr><th style="width:40px;">Status</th><th style="width:70px;">Method</th><th style="text-align:left;">Endpoint</th><th style="width:50px;">Tests</th><th style="width:50px;">2xx</th><th style="width:120px;">Status Codes Asserted</th><th style="width:80px;">Response Assertions</th><th style="width:80px;">Query Params Coverage</th><th style="width:80px;">Weak Assertions</th></tr></thead>
990
+ <tbody>{''.join(endpoint_rows)}</tbody></table></div>'''
991
+
992
+ # Collapsible detail sections
993
+ qp_rows = [f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td><code>{", ".join(sorted(ep["params"]))}</code></td><td>{ep["total_values"]}</td></tr>' for ep in endpoints_with_params]
994
+ qp_html = collapsible('Query Parameter Details', f'{len(endpoints_with_params)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Parameters Tested'},{'label':'Value Variations'}], qp_rows)
995
+
996
+ body_rows = [f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td>{len(ep["fields"])} fields</td><td>{ep["variations"]}</td></tr>' for ep in endpoints_with_body]
997
+ body_html = collapsible('Request Body Coverage', f'{len(endpoints_with_body)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Fields Tested'},{'label':'Variations'}], body_rows)
998
+
999
+ def include_row(ep):
1000
+ tc, ec = len(ep['tested_expected']), len(ep['expected_values'])
1001
+ pct = (tc / ec * 100) if ec > 0 else 0
1002
+ col = 'var(--color-success)' if pct >= 100 else 'var(--color-warning)' if pct >= 50 else 'var(--color-error)'
1003
+ exp = ', '.join(sorted(ep['expected_values'])) or 'None'
1004
+ vals = ', '.join(sorted(ep['values'])) or 'None'
1005
+ miss = ', '.join(sorted(ep['missing_values'])) or '—'
1006
+ miss_col = 'var(--color-error)' if ep['missing_values'] else 'var(--color-success)'
1007
+ return f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td style="color:{col};">{tc}/{ec} ({pct:.0f}%)</td><td><code>{exp}</code></td><td><code>{vals}</code></td><td style="color:{miss_col};"><code>{miss}</code></td></tr>'
1008
+
1009
+ include_rows = [include_row(ep) for ep in endpoints_with_include]
1010
+ include_html = collapsible('Include Parameter Details', f'{len(endpoints_with_include)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Coverage'},{'label':'Expected Values'},{'label':'Tested Values'},{'label':'Missing'}], include_rows)
1011
+
1012
+ skip_rows = [f'<tr><td class="left"><span class="path">{s["collection"]}: {s["name"]}</span></td><td>{s.get("env","unknown").upper()}</td><td class="left" style="color:{"var(--color-error)" if not s.get("reason") else "inherit"};">{s.get("reason") or "Missing reason"}</td></tr>' for s in all_skipped_tests]
1013
+ skip_html = collapsible('Environment-Conditional Skips', f'{total_skipped_tests} tests', [{'label':'Test','style':'text-align:left;'},{'label':'Skipped In'},{'label':'Reason','style':'text-align:left;'}], skip_rows) if total_skipped_tests > 0 else ''
1014
+
1015
+ # Assemble full HTML
1016
+ content = summary_html + coverage_html + gaps_html + details_html + qp_html + body_html + include_html + skip_html
1017
+
1018
+ if USE_HELPERS:
1019
+ html = get_html_template(
1020
+ title='API Test Coverage Report',
1021
+ logo='DP Apps',
1022
+ content=content,
1023
+ health_badge={'class': health_class, 'icon': health_icon, 'label': health_label},
1024
+ footer_text=f'API Test Coverage Report • Generated by API Coverage Matrix Generator • {total_tests} test cases analyzed'
1025
+ )
1026
+ else:
1027
+ # Fallback: manual HTML construction
1028
+ html = f'''<!DOCTYPE html>
1029
+ <html lang="en">
1030
+ <head>
1031
+ <meta charset="UTF-8">
1032
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
1033
+ <title>API Coverage Report | DP Apps</title>
1034
+ <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
1035
+ <style>{SHARED_CSS if SHARED_CSS else "/* Error: shared CSS library not loaded */"}</style>
1036
+ </head>
1037
+ <body>
1038
+ <div class="container">
1039
+ <div class="header">
1040
+ <div class="header-left">
1041
+ <div class="logo">DP Apps</div>
1042
+ <h1>API Test Coverage Report</h1>
1043
+ <p class="timestamp">Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</p>
1044
+ </div>
1045
+ <div class="health-badge {health_class}">{health_icon} {health_label}</div>
1046
+ </div>
1047
+ {content}
1048
+ <footer>API Test Coverage Report • Generated by API Coverage Matrix Generator • {total_tests} test cases analyzed</footer>
1049
+ </div>
1050
+ </body>
1051
+ </html>'''
1052
+
1053
+ # Write HTML
1054
+ with open(HTML_OUTPUT_FILE, 'w', encoding='utf-8') as f:
1055
+ f.write(html)
1056
+
1057
+ # Also generate JSON summary for consolidated reports
1058
+ json_output_file = HTML_OUTPUT_FILE.replace('.html', '.json')
1059
+
1060
+ # Helper to check if endpoint has 2xx coverage
1061
+ def ep_has_2xx(ep):
1062
+ key = f"{ep['method']} {ep['path']}"
1063
+ stats = endpoint_stats.get(key, {})
1064
+ return has_success_code(stats.get('status_codes', set()))
1065
+
1066
+ # Calculate by-method stats
1067
+ method_stats = {}
1068
+ gaps = []
1069
+ for ep in endpoints:
1070
+ m = ep['method']
1071
+ if m not in method_stats:
1072
+ method_stats[m] = {'tested': 0, 'total': 0}
1073
+ method_stats[m]['total'] += 1
1074
+ if ep_has_2xx(ep):
1075
+ method_stats[m]['tested'] += 1
1076
+ else:
1077
+ gaps.append({
1078
+ 'method': ep['method'],
1079
+ 'path': ep['path'],
1080
+ 'controller': ep['controller']
1081
+ })
1082
+
1083
+ coverage_summary = {
1084
+ 'coverage_percent': round(success_pct, 1),
1085
+ 'tested': success_tested,
1086
+ 'total': len(endpoints),
1087
+ 'well_tested_percent': round(well_tested_pct, 1),
1088
+ 'query_params_percent': round(qp_pct, 1),
1089
+ 'gaps': gaps[:20],
1090
+ 'by_method': method_stats,
1091
+ 'full_report': os.path.basename(HTML_OUTPUT_FILE)
1092
+ }
1093
+
1094
+ with open(json_output_file, 'w', encoding='utf-8') as f:
1095
+ json.dump(coverage_summary, f, indent=2)
1096
+
1097
+ print(f"\nEndpoints in Code: {len(endpoints)}", flush=True)
1098
+ print(f"Total Test Cases: {total_tests}", flush=True)
1099
+ print(f"Success Path (2xx) Tested: {success_pct:.1f}% ({success_tested}/{len(endpoints)})", flush=True)
1100
+ print(f"Well Tested (≥3 tests): {well_tested_pct:.1f}% ({well_tested}/{len(endpoints)})", flush=True)
1101
+ print(f"Response Body Validated: {body_pct:.1f}% ({with_body}/{len(endpoints)})", flush=True)
1102
+ print(f"Query Params Tested: {qp_pct:.1f}% ({total_tested_params}/{total_defined_params})", flush=True)
1103
+ if include_total_count > 0:
1104
+ print(f"Include Params Tested: {include_pct:.1f}% ({total_tested_include_values}/{total_expected_include_values} values across {include_total_count} endpoints)", flush=True)
1105
+ if error_only > 0:
1106
+ print(f"⚠️ Error-Only Tests (no 2xx): {error_only} endpoints", flush=True)
1107
+ if total_oneof_tests > 0:
1108
+ print(f"⚠️ oneOf Tests (weak assertions): {total_oneof_tests} tests in {oneof_endpoints} endpoints", flush=True)
1109
+
1110
+ if total_skipped_tests > 0:
1111
+ print(f"⏭️ Environment-conditional skips: {total_skipped_tests} tests", flush=True)
1112
+ for env, skips in sorted(skips_by_env.items()):
1113
+ print(f" - {env}: {len(skips)} tests", flush=True)
1114
+ if skips_without_reason > 0:
1115
+ print(f"❌ Undocumented skips: {skips_without_reason} tests (add [SKIP] Reason: ... | Env: ...)", flush=True)
1116
+
1117
+ if errors:
1118
+ print(f"\nWarnings ({len(errors)}):", flush=True)
1119
+ for err in errors[:5]:
1120
+ print(f" - {err}", flush=True)
1121
+ if len(errors) > 5:
1122
+ print(f" ... and {len(errors) - 5} more", flush=True)